@loucompanion/forge-bridge 0.1.3-dev.f66395c3ac21 → 0.1.4-dev.bfa11ebe98c6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -1
- package/dist/bridge/bridge-chat.types.d.ts +111 -0
- package/dist/bridge/bridge-chat.types.js +401 -0
- package/dist/bridge/bridge.base.d.ts +9 -1
- package/dist/bridge/bridge.service.d.ts +129 -3
- package/dist/bridge/bridge.service.js +2426 -70
- package/dist/bridge/bridge.types.d.ts +218 -2
- package/dist/bridge/dispatch-result-outbox.d.ts +25 -0
- package/dist/bridge/dispatch-result-outbox.js +62 -0
- package/dist/bridge/dispatch-result-worker.d.ts +2 -0
- package/dist/bridge/dispatch-result-worker.js +122 -0
- package/dist/bridge/internals.d.ts +2 -0
- package/dist/bridge/internals.js +1 -0
- package/dist/bridge/readiness-result-outbox.d.ts +21 -0
- package/dist/bridge/readiness-result-outbox.js +54 -0
- package/dist/bridge/readiness-result-worker.d.ts +2 -0
- package/dist/bridge/readiness-result-worker.js +121 -0
- package/dist/bridge/runner-host-capabilities.d.ts +15 -0
- package/dist/bridge/runner-host-capabilities.js +93 -0
- package/dist/cleanup/cleanup.service.d.ts +14 -2
- package/dist/cleanup/cleanup.service.js +496 -3
- package/dist/cleanup/cleanup.types.d.ts +10 -0
- package/dist/cleanup/index.d.ts +1 -1
- package/dist/cleanup/index.js +1 -1
- package/dist/cleanup/internals.d.ts +2 -2
- package/dist/cleanup/internals.js +1 -1
- package/dist/cli/cli.help.d.ts +2 -0
- package/dist/cli/cli.help.js +217 -0
- package/dist/cli/cli.service.d.ts +55 -7
- package/dist/cli/cli.service.js +429 -219
- package/dist/cli/local-config.d.ts +7 -0
- package/dist/cli/local-config.js +51 -3
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +36 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +341 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +18 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +140 -0
- package/dist/runtime/attempt-path-operation.d.ts +21 -0
- package/dist/runtime/attempt-path-operation.js +94 -0
- package/dist/runtime/attempt-private-directory.d.ts +6 -0
- package/dist/runtime/attempt-private-directory.js +54 -0
- package/dist/runtime/attempt-process-supervisor-registry.d.ts +20 -0
- package/dist/runtime/attempt-process-supervisor-registry.js +64 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +97 -0
- package/dist/runtime/attempt-process-supervisor.js +397 -0
- package/dist/runtime/canonical-source-file.d.ts +3 -0
- package/dist/runtime/canonical-source-file.js +54 -0
- package/dist/runtime/docker-engine-probe.d.ts +90 -0
- package/dist/runtime/docker-engine-probe.js +333 -0
- package/dist/runtime/generated-output-reset.d.ts +20 -0
- package/dist/runtime/generated-output-reset.js +157 -0
- package/dist/runtime/host-database-capability-boundary.d.ts +16 -0
- package/dist/runtime/host-database-capability-boundary.js +62 -0
- package/dist/runtime/host-diagnostics.d.ts +11 -0
- package/dist/runtime/host-diagnostics.js +191 -0
- package/dist/runtime/host-execution-root.d.ts +4 -0
- package/dist/runtime/host-execution-root.js +39 -0
- package/dist/runtime/host-listener-inspector.d.ts +51 -0
- package/dist/runtime/host-listener-inspector.js +385 -0
- package/dist/runtime/host-port-broker.d.ts +76 -0
- package/dist/runtime/host-port-broker.js +333 -0
- package/dist/runtime/host-runtime-contract.d.ts +58 -0
- package/dist/runtime/host-runtime-contract.js +754 -0
- package/dist/runtime/host-runtime-lease.d.ts +215 -0
- package/dist/runtime/host-runtime-lease.js +2449 -0
- package/dist/runtime/host-runtime-promotion.d.ts +32 -0
- package/dist/runtime/host-runtime-promotion.js +206 -0
- package/dist/runtime/memory-capacity.d.ts +70 -0
- package/dist/runtime/memory-capacity.js +123 -0
- package/dist/runtime/native-test-postgres-fallback.d.ts +25 -0
- package/dist/runtime/native-test-postgres-fallback.js +91 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +54 -0
- package/dist/runtime/node-host-runtime-adapter.js +285 -0
- package/dist/runtime/pinned-directory-process.d.ts +9 -0
- package/dist/runtime/pinned-directory-process.js +61 -0
- package/dist/runtime/pinned-source-directory.d.ts +17 -0
- package/dist/runtime/pinned-source-directory.js +94 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +21 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +268 -0
- package/dist/runtime/prepared-runtime-identity.d.ts +6 -0
- package/dist/runtime/prepared-runtime-identity.js +218 -0
- package/dist/runtime/prepared-runtime-readiness.d.ts +30 -0
- package/dist/runtime/prepared-runtime-readiness.js +285 -0
- package/dist/runtime/runtime-isolation-lease.d.ts +67 -0
- package/dist/runtime/runtime-isolation-lease.js +161 -0
- package/dist/runtime/workspace-resource-estimator.d.ts +17 -0
- package/dist/runtime/workspace-resource-estimator.js +174 -0
- package/dist/sandbox/command-runner.d.ts +4 -0
- package/dist/sandbox/command-runner.js +47 -0
- package/dist/sandbox/contract.d.ts +5 -0
- package/dist/sandbox/contract.js +276 -0
- package/dist/sandbox/event-budget.d.ts +11 -0
- package/dist/sandbox/event-budget.js +27 -0
- package/dist/sandbox/image-policy.d.ts +35 -0
- package/dist/sandbox/image-policy.js +247 -0
- package/dist/sandbox/index.d.ts +5 -0
- package/dist/sandbox/index.js +5 -0
- package/dist/sandbox/lease-watchdog.d.ts +18 -0
- package/dist/sandbox/lease-watchdog.js +67 -0
- package/dist/sandbox/preflight.d.ts +84 -0
- package/dist/sandbox/preflight.js +521 -0
- package/dist/sandbox/process.d.ts +20 -0
- package/dist/sandbox/process.js +209 -0
- package/dist/sandbox/provider-auth/codex.d.ts +26 -0
- package/dist/sandbox/provider-auth/codex.js +212 -0
- package/dist/sandbox/repository-lock.d.ts +6 -0
- package/dist/sandbox/repository-lock.js +128 -0
- package/dist/sandbox/runtime.d.ts +302 -0
- package/dist/sandbox/runtime.js +2367 -0
- package/dist/sandbox/sandbox.types.d.ts +93 -0
- package/dist/sandbox/sandbox.types.js +1 -0
- package/dist/sandbox/smoke.d.ts +76 -0
- package/dist/sandbox/smoke.js +380 -0
- package/dist/sandbox/worker-supervisor.d.ts +13 -0
- package/dist/sandbox/worker-supervisor.js +36 -0
- package/dist/sandbox/worker.d.ts +1 -0
- package/dist/sandbox/worker.js +225 -0
- package/dist/sandbox/workspace.service.d.ts +13 -0
- package/dist/sandbox/workspace.service.js +74 -0
- package/dist/session/managed-attempt-tools.d.ts +28 -0
- package/dist/session/managed-attempt-tools.js +1011 -0
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +105 -0
- package/dist/session/managed-build-receipt.d.ts +21 -0
- package/dist/session/managed-build-receipt.js +337 -0
- package/dist/session/managed-verification-timeouts.d.ts +8 -0
- package/dist/session/managed-verification-timeouts.js +17 -0
- package/dist/session/phase-completion.artifacts.d.ts +27 -0
- package/dist/session/phase-completion.artifacts.js +829 -0
- package/dist/session/phase-completion.d.ts +53 -42
- package/dist/session/phase-completion.helpers.d.ts +117 -0
- package/dist/session/phase-completion.helpers.js +707 -0
- package/dist/session/phase-completion.js +125 -3007
- package/dist/session/phase-completion.prompts.d.ts +14 -0
- package/dist/session/phase-completion.prompts.js +233 -0
- package/dist/session/phase-completion.runtime.d.ts +49 -0
- package/dist/session/phase-completion.runtime.js +833 -0
- package/dist/session/phase-completion.schemas.d.ts +3 -0
- package/dist/session/phase-completion.schemas.js +859 -0
- package/dist/session/phase-completion.validation.d.ts +38 -0
- package/dist/session/phase-completion.validation.js +918 -0
- package/dist/session/pre-publication-validation.d.ts +75 -0
- package/dist/session/pre-publication-validation.js +657 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.d.ts +72 -0
- package/dist/session/provider-cli/bridge-cli.conformance-fixtures.js +462 -0
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +16 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +110 -0
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +95 -0
- package/dist/session/provider-cli/provider-cli.artifacts.js +812 -0
- package/dist/session/provider-cli/provider-cli.base.d.ts +1 -0
- package/dist/session/provider-cli/provider-cli.deterministic-payloads.d.ts +12 -0
- package/dist/session/provider-cli/provider-cli.deterministic-payloads.js +279 -0
- package/dist/session/provider-cli/provider-cli.deterministic.d.ts +14 -0
- package/dist/session/provider-cli/provider-cli.deterministic.js +407 -0
- package/dist/session/provider-cli/provider-cli.process.d.ts +99 -0
- package/dist/session/provider-cli/provider-cli.process.js +874 -0
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +52 -0
- package/dist/session/provider-cli/provider-cli.prompts.js +715 -0
- package/dist/session/provider-cli/provider-cli.qa-sidecars.d.ts +11 -0
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +66 -0
- package/dist/session/provider-cli/provider-cli.repair-epoch.d.ts +6 -0
- package/dist/session/provider-cli/provider-cli.repair-epoch.js +11 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +16 -0
- package/dist/session/provider-cli/provider-cli.runtime.js +78 -0
- package/dist/session/provider-cli/provider-cli.service.d.ts +47 -53
- package/dist/session/provider-cli/provider-cli.service.js +1257 -2497
- package/dist/session/provider-cli/provider-cli.types.d.ts +8 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +32 -0
- package/dist/session/provider-cli/provider-output-state.js +817 -0
- package/dist/session/provider-cli/provider-usage.d.ts +7 -0
- package/dist/session/provider-cli/provider-usage.js +28 -5
- package/dist/session/session-redaction.d.ts +3 -0
- package/dist/session/session-redaction.js +448 -0
- package/dist/session/session.base.d.ts +6 -1
- package/dist/session/session.service.d.ts +8 -2
- package/dist/session/session.service.js +586 -59
- package/dist/session/session.types.d.ts +324 -3
- package/dist/session/session.types.js +939 -10
- package/dist/session/unreferenced-exports.d.ts +12 -0
- package/dist/session/unreferenced-exports.js +59 -0
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/git/git.js +2 -1
- package/dist/shared/path/path.d.ts +1 -0
- package/dist/shared/path/path.js +21 -1
- package/dist/shared/process/dotnet-toolchain.d.ts +17 -0
- package/dist/shared/process/dotnet-toolchain.js +307 -0
- package/dist/shared/process/host-process-recovery.d.ts +12 -0
- package/dist/shared/process/host-process-recovery.js +78 -0
- package/dist/shared/process/host-process-supervisor.d.ts +59 -0
- package/dist/shared/process/host-process-supervisor.js +330 -0
- package/dist/shared/process/process.d.ts +3 -0
- package/dist/shared/process/process.js +13 -7
- package/dist/shared/process/windows-launch.d.ts +8 -0
- package/dist/shared/process/windows-launch.js +41 -0
- package/dist/shared/provider-capabilities.d.ts +4 -0
- package/dist/shared/provider-capabilities.js +88 -0
- package/dist/shared/redaction/redaction.d.ts +1 -0
- package/dist/shared/redaction/redaction.js +47 -4
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.d.ts +3 -2
- package/dist/worktree/artifact-injector.js +82 -10
- package/dist/worktree/dependency-warmer.d.ts +53 -1
- package/dist/worktree/dependency-warmer.js +619 -11
- package/dist/worktree/index.d.ts +2 -1
- package/dist/worktree/index.js +1 -0
- package/dist/worktree/prior-phase-artifact-hydrator.d.ts +2 -0
- package/dist/worktree/prior-phase-artifact-hydrator.js +136 -0
- package/dist/worktree/story-qa-reference-hydrator.d.ts +4 -0
- package/dist/worktree/story-qa-reference-hydrator.js +253 -0
- package/dist/worktree/worktree-paths.d.ts +4 -0
- package/dist/worktree/worktree-paths.js +30 -0
- package/dist/worktree/worktree.base.d.ts +3 -0
- package/dist/worktree/worktree.service.d.ts +16 -3
- package/dist/worktree/worktree.service.js +334 -54
- package/dist/worktree/worktree.types.d.ts +9 -1
- package/package.json +12 -7
- package/runtime/windows-job-supervisor.ps1 +722 -0
- package/runtime/windows-listener-owner.ps1 +33 -0
|
@@ -1,47 +1,270 @@
|
|
|
1
|
-
import { HubConnectionBuilder, LogLevel } from "@microsoft/signalr";
|
|
1
|
+
import { HubConnectionBuilder, HttpTransportType, LogLevel } from "@microsoft/signalr";
|
|
2
2
|
import fs from "node:fs";
|
|
3
3
|
import path from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import os from "node:os";
|
|
4
6
|
import { defaultForgeBridgeConfig } from "../config.js";
|
|
5
|
-
import {
|
|
7
|
+
import { bridgeChatCapabilityFor, supportsDraftImageAttachmentsFor } from "../shared/provider-capabilities.js";
|
|
8
|
+
import { redactText } from "../shared/redaction/redaction.js";
|
|
6
9
|
import { BridgeSessionService } from "../session/session.service.js";
|
|
7
|
-
import {
|
|
10
|
+
import { ProcessBackedProviderCliService, RoutingProviderCliService, StreamingProviderProcessRunner, } from "../session/provider-cli/provider-cli.service.js";
|
|
11
|
+
import { codexSandboxModeForPlatform, formatProviderCliCommandForCheckpoint } from "../session/provider-cli/provider-cli.prompts.js";
|
|
12
|
+
import { clip, createProviderOutputState } from "../session/provider-cli/provider-output-state.js";
|
|
13
|
+
import { redactRunnerDispatchEvent, redactRunOnceResultPacket } from "../session/session-redaction.js";
|
|
14
|
+
import { PrePublicationValidator } from "../session/pre-publication-validation.js";
|
|
15
|
+
import { HostRuntimeLeaseManager, } from "../runtime/host-runtime-lease.js";
|
|
16
|
+
import { assertHostDatabaseCapabilityBoundary, establishHostDatabaseCapabilityBoundary, } from "../runtime/host-database-capability-boundary.js";
|
|
17
|
+
import { defaultHostExecutionRoot } from "../runtime/host-execution-root.js";
|
|
18
|
+
import { buildMemoryAdmissionDiagnostics } from "../runtime/memory-capacity.js";
|
|
19
|
+
import { DockerEngineProbe } from "../runtime/docker-engine-probe.js";
|
|
20
|
+
import { readNativeTestPostgresFallback } from "../runtime/native-test-postgres-fallback.js";
|
|
21
|
+
import { estimateWorkspaceBytesFromRepo } from "../runtime/workspace-resource-estimator.js";
|
|
22
|
+
import { GitWorktreeService } from "../worktree/worktree.service.js";
|
|
23
|
+
import { RepositoryCoordinatorLock } from "../sandbox/repository-lock.js";
|
|
24
|
+
import { AttemptCoordinatedProviderProcessRunner, AttemptProcessSupervisorRegistry, } from "../runtime/attempt-process-supervisor-registry.js";
|
|
25
|
+
import { failureFactsForCode, failureFactsFromError, failureResult, parseRunOnceDispatchPacket } from "../session/session.types.js";
|
|
8
26
|
import { forgeBridgeVersion } from "../version.js";
|
|
27
|
+
import { canRunDockerMaintenance, DockerPreflight } from "../sandbox/preflight.js";
|
|
28
|
+
import { validateSandboxContract } from "../sandbox/contract.js";
|
|
29
|
+
import { runSandboxProcess } from "../sandbox/process.js";
|
|
30
|
+
import { DockerSandboxRuntime, selectSandboxConfigRefs } from "../sandbox/runtime.js";
|
|
31
|
+
import { codexSandboxProviderAuthAdapter, sandboxProviderAuthAdapterFor, sandboxProviderAuthUnavailableMessage } from "../sandbox/provider-auth/codex.js";
|
|
32
|
+
import { assertBridgeChatCallbackPayload, bridgeChatMaxProviderThreadIdChars, bridgeChatProtocolVersion, isTerminalChatTurnCallbackKind, parseChatTurnDispatchPacket, } from "./bridge-chat.types.js";
|
|
33
|
+
import { formatBytes } from "../shared/format/format-bytes.js";
|
|
34
|
+
import { emptyRunnerHostCapabilitySnapshot, ProcessRunnerHostCapabilityProbe, } from "./runner-host-capabilities.js";
|
|
35
|
+
import { FileReadinessResultOutbox } from "./readiness-result-outbox.js";
|
|
36
|
+
import { FileDispatchResultOutbox } from "./dispatch-result-outbox.js";
|
|
37
|
+
/**
|
|
38
|
+
* Protocol-v2 events are acknowledged one at a time before runOnce returns.
|
|
39
|
+
* Sending the complete transcript again in DispatchResult made long QA and CI
|
|
40
|
+
* sessions exceed SignalR's receive limit, disconnecting the bridge after the
|
|
41
|
+
* provider had already completed. The API combines its acknowledged streamed
|
|
42
|
+
* events with the terminal result, so the callback only needs the outcome and
|
|
43
|
+
* artifacts. Protocol v1 retains the transcript for legacy servers.
|
|
44
|
+
*/
|
|
45
|
+
export function compactTerminalResult(dispatch, result) {
|
|
46
|
+
return dispatch.schemaVersion >= 2 && result.events.length > 0
|
|
47
|
+
? { ...result, events: [] }
|
|
48
|
+
: result;
|
|
49
|
+
}
|
|
9
50
|
export class ForgeBridgeService {
|
|
10
51
|
config;
|
|
11
|
-
sessions;
|
|
12
52
|
connections;
|
|
53
|
+
hostCapabilityProbe;
|
|
54
|
+
chatProcessRunner;
|
|
13
55
|
connection = null;
|
|
14
56
|
heartbeatTimer = null;
|
|
57
|
+
heartbeatInFlight = false;
|
|
58
|
+
consecutiveHeartbeatFailures = 0;
|
|
59
|
+
reconnectingFromHeartbeatFailure = false;
|
|
60
|
+
leaseWatchdogTimer = null;
|
|
61
|
+
sandboxMaintenanceTimer = null;
|
|
62
|
+
sandboxMaintenanceRunning = false;
|
|
15
63
|
closed = null;
|
|
64
|
+
transportRecovery = null;
|
|
65
|
+
transportRecoveryGeneration = 0;
|
|
16
66
|
stopped = false;
|
|
17
67
|
activeSessions = new Map();
|
|
18
|
-
|
|
68
|
+
reservations = new Map();
|
|
69
|
+
activeReadinessProbes = new Map();
|
|
70
|
+
readinessResultOutbox;
|
|
71
|
+
readinessResultFlush = null;
|
|
72
|
+
dispatchResultOutbox;
|
|
73
|
+
dispatchResultFlush = null;
|
|
74
|
+
dispatchResultGeneration = 0;
|
|
75
|
+
dispatchResultFlushedGeneration = -1;
|
|
76
|
+
dispatchResultHandoffs = new Map();
|
|
77
|
+
preflight;
|
|
78
|
+
dockerEngineProbe;
|
|
79
|
+
sandboxRuntime;
|
|
80
|
+
sandboxResourceResolver;
|
|
81
|
+
sessions;
|
|
82
|
+
enforceHostDatabaseCapabilityBoundary;
|
|
83
|
+
hostCapabilities = emptyRunnerHostCapabilitySnapshot();
|
|
84
|
+
// Reaping deletes the stale ledger, so its stopped-process proof is held here
|
|
85
|
+
// until a Hello carrying it is accepted. A bridge restart forgets it, which
|
|
86
|
+
// leaves those attempts on Forge's conservative unknown-outcome path.
|
|
87
|
+
recoveredHostAttempts = new Map();
|
|
88
|
+
hostCapabilitiesDetected = false;
|
|
89
|
+
constructor(config, sessions = undefined, connections = new SignalRBridgeConnectionFactory(), preflight, sandboxRuntime, sandboxResourceResolver, dockerEngineProbe, hostCapabilityProbe = new ProcessRunnerHostCapabilityProbe(), readinessResultOutbox, chatProcessRunner = new StreamingProviderProcessRunner(), dispatchResultOutbox) {
|
|
19
90
|
this.config = config;
|
|
20
|
-
this.sessions = sessions;
|
|
21
91
|
this.connections = connections;
|
|
92
|
+
this.hostCapabilityProbe = hostCapabilityProbe;
|
|
93
|
+
this.chatProcessRunner = chatProcessRunner;
|
|
94
|
+
this.enforceHostDatabaseCapabilityBoundary = sessions === undefined;
|
|
95
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
96
|
+
establishHostDatabaseCapabilityBoundary();
|
|
97
|
+
}
|
|
98
|
+
this.preflight = preflight ?? new DockerPreflight({
|
|
99
|
+
dockerExecutable: config.dockerExecutable,
|
|
100
|
+
dockerContext: config.dockerContext,
|
|
101
|
+
stateRoot: config.sandboxStateRoot,
|
|
102
|
+
runnerInstanceId: config.runnerInstanceId,
|
|
103
|
+
maxConcurrentSandboxes: config.maxConcurrentSandboxes,
|
|
104
|
+
diskHighWatermarkPercent: config.diskHighWatermarkPercent,
|
|
105
|
+
diskLowWatermarkPercent: config.diskLowWatermarkPercent,
|
|
106
|
+
minimumWorkspaceBytes: config.minimumWorkspaceBytes,
|
|
107
|
+
hostMemoryReserveBytes: config.sandboxHostMemoryReserveBytes,
|
|
108
|
+
defaultWorkerMemoryLimitBytes: config.defaultSandboxWorkerMemoryLimitBytes,
|
|
109
|
+
defaultServiceMemoryLimitBytes: config.defaultSandboxServiceMemoryLimitBytes,
|
|
110
|
+
runtimeOverheadBytes: config.sandboxRuntimeOverheadBytes,
|
|
111
|
+
requireRootless: config.requireRootlessDocker
|
|
112
|
+
});
|
|
113
|
+
this.dockerEngineProbe = dockerEngineProbe ?? new DockerEngineProbe({
|
|
114
|
+
dockerExecutable: config.dockerExecutable,
|
|
115
|
+
dockerContext: config.dockerContext,
|
|
116
|
+
recovery: {
|
|
117
|
+
enabled: config.dockerDesktopRecoveryEnabled,
|
|
118
|
+
failureThreshold: config.dockerRecoveryFailureThreshold,
|
|
119
|
+
cooldownMs: config.dockerRecoveryCooldownMs,
|
|
120
|
+
startupTimeoutMs: config.dockerRecoveryStartupTimeoutMs,
|
|
121
|
+
canRecover: () => this.activeSessions.size === 0 && this.reservations.size === 0,
|
|
122
|
+
},
|
|
123
|
+
onStatus: logBridgeStatus,
|
|
124
|
+
});
|
|
125
|
+
this.sessions = sessions ?? createDefaultSessionService(config.sandboxStateRoot, config.sandboxExecutionRoot, this.dockerEngineProbe);
|
|
126
|
+
this.sandboxRuntime = sandboxRuntime ?? new DockerSandboxRuntime(config, (request, signal) => this.redeemPublishGrant(request, signal));
|
|
127
|
+
this.sandboxResourceResolver = sandboxResourceResolver
|
|
128
|
+
?? (dispatch => resolveSandboxResourcesFromRepo(dispatch, this.config.sandboxRuntimeOverheadBytes));
|
|
129
|
+
this.readinessResultOutbox = readinessResultOutbox ?? new FileReadinessResultOutbox(config.sandboxStateRoot);
|
|
130
|
+
this.dispatchResultOutbox = dispatchResultOutbox ?? new FileDispatchResultOutbox(config.sandboxStateRoot);
|
|
22
131
|
}
|
|
23
132
|
async connectOnce() {
|
|
24
133
|
await this.stopConnection();
|
|
25
134
|
this.stopped = false;
|
|
135
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
136
|
+
assertHostDatabaseCapabilityBoundary();
|
|
137
|
+
}
|
|
138
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
139
|
+
runnerId: this.config.runnerId,
|
|
140
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
141
|
+
});
|
|
142
|
+
this.rememberRecoveredHostAttempts(hostReap);
|
|
143
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
144
|
+
logBridgeStatus(`Managed-host recovery reaped ${hostReap.reaped} stale attempt(s), protected ${hostReap.reusedPidsProtected} reused PID(s), and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
145
|
+
}
|
|
146
|
+
if (this.config.dockerEnabled) {
|
|
147
|
+
const preflight = await this.preflight.refresh();
|
|
148
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
149
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow });
|
|
150
|
+
await this.preflight.refresh();
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
await this.dockerEngineProbe.refresh(true);
|
|
154
|
+
if (!this.hostCapabilitiesDetected) {
|
|
155
|
+
this.hostCapabilities = await this.hostCapabilityProbe.detect(this.config);
|
|
156
|
+
this.hostCapabilitiesDetected = true;
|
|
157
|
+
}
|
|
26
158
|
const connection = this.connections.create(this.config);
|
|
27
159
|
this.connection = connection;
|
|
160
|
+
this.transportRecovery = null;
|
|
28
161
|
this.closed = new Promise((resolve) => {
|
|
29
|
-
connection.onClose(() => {
|
|
162
|
+
connection.onClose((error) => {
|
|
30
163
|
this.stopHeartbeat();
|
|
164
|
+
this.completeTransportRecovery(error ?? new Error("bridge connection closed"));
|
|
31
165
|
this.abortActiveSessions("bridge connection closed");
|
|
166
|
+
this.abortActiveReadinessProbes("bridge connection closed");
|
|
32
167
|
resolve();
|
|
33
168
|
});
|
|
34
169
|
});
|
|
170
|
+
connection.onReconnecting?.((error) => {
|
|
171
|
+
if (connection !== this.connection || this.stopped)
|
|
172
|
+
return;
|
|
173
|
+
this.stopHeartbeat();
|
|
174
|
+
this.beginTransportRecovery();
|
|
175
|
+
logBridgeStatus(`Forge connection interrupted${error ? `: ${errorMessage(error)}` : ""}. Reconnecting in place; active dispatches remain supervised...`);
|
|
176
|
+
});
|
|
177
|
+
connection.onReconnected?.(async () => {
|
|
178
|
+
if (connection !== this.connection || this.stopped)
|
|
179
|
+
return;
|
|
180
|
+
try {
|
|
181
|
+
await this.registerConnection(connection);
|
|
182
|
+
this.completeTransportRecovery();
|
|
183
|
+
this.startHeartbeat();
|
|
184
|
+
await this.flushDispatchResults(true).catch(error => {
|
|
185
|
+
logBridgeStatus(`Dispatch result replay deferred after reconnect: ${errorMessage(error)}`);
|
|
186
|
+
});
|
|
187
|
+
await this.flushReadinessResults().catch(error => {
|
|
188
|
+
logBridgeStatus(`Readiness result replay deferred after reconnect: ${errorMessage(error)}`);
|
|
189
|
+
});
|
|
190
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
191
|
+
logBridgeStatus("Forge connection restored; active dispatches remain attached to this runner process.");
|
|
192
|
+
}
|
|
193
|
+
catch (error) {
|
|
194
|
+
this.completeTransportRecovery(error);
|
|
195
|
+
logBridgeStatus(`Forge reconnect registration failed: ${errorMessage(error)}. Closing the connection for a clean retry...`);
|
|
196
|
+
await connection.stop().catch(() => undefined);
|
|
197
|
+
}
|
|
198
|
+
});
|
|
35
199
|
connection.onDispatch((dispatch) => this.handleDispatch(dispatch));
|
|
200
|
+
connection.onChatTurn?.((dispatch) => this.handleChatTurn(dispatch));
|
|
201
|
+
connection.onOffer?.((dispatch) => this.handleOffer(dispatch));
|
|
202
|
+
connection.onStart?.((dispatch) => this.handleStart(dispatch));
|
|
203
|
+
connection.onLeaseUpdate?.((request) => this.handleLeaseUpdate(request));
|
|
36
204
|
connection.onCancel((request) => this.handleCancel(request));
|
|
205
|
+
connection.onReap?.((request) => this.handleReap(request));
|
|
206
|
+
connection.onReadinessProbe?.((request) => this.handleReadinessProbe(request));
|
|
37
207
|
await connection.start();
|
|
38
|
-
await this.
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
minServerVersion: this.config.minServerVersion ?? null,
|
|
42
|
-
capabilities: this.capabilities()
|
|
208
|
+
await this.registerConnection(connection);
|
|
209
|
+
await this.flushDispatchResults(true).catch(error => {
|
|
210
|
+
logBridgeStatus(`Dispatch result replay deferred: ${errorMessage(error)}`);
|
|
43
211
|
});
|
|
212
|
+
await this.flushReadinessResults().catch(error => {
|
|
213
|
+
logBridgeStatus(`Readiness result replay deferred: ${errorMessage(error)}`);
|
|
214
|
+
});
|
|
215
|
+
await this.flushCleanupReports();
|
|
44
216
|
this.startHeartbeat();
|
|
217
|
+
this.startLeaseWatchdog();
|
|
218
|
+
this.startSandboxMaintenance();
|
|
219
|
+
}
|
|
220
|
+
rememberRecoveredHostAttempts(report) {
|
|
221
|
+
for (const attempt of report?.recoveredAttempts ?? []) {
|
|
222
|
+
if (this.recoveredHostAttempts.size >= maxRecoveredHostAttempts)
|
|
223
|
+
break;
|
|
224
|
+
this.recoveredHostAttempts.set(attempt.dispatchAttemptId, attempt);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
async recoveredHostAttemptsForHello() {
|
|
228
|
+
if (this.recoveredHostAttempts.size === 0)
|
|
229
|
+
return [];
|
|
230
|
+
// A durable terminal result outranks the stopped-process proof: it replays
|
|
231
|
+
// after Hello and Forge accepts the provider's real outcome instead.
|
|
232
|
+
const pendingResults = new Set((await this.dispatchResultOutbox.pending())
|
|
233
|
+
.map(entry => entry.payload.dispatchAttemptId)
|
|
234
|
+
.filter((id) => typeof id === "string"));
|
|
235
|
+
return [...this.recoveredHostAttempts.values()]
|
|
236
|
+
.filter(attempt => !pendingResults.has(attempt.dispatchAttemptId));
|
|
237
|
+
}
|
|
238
|
+
async registerConnection(connection) {
|
|
239
|
+
let storageRecoveryAttempt = 0;
|
|
240
|
+
while (true) {
|
|
241
|
+
try {
|
|
242
|
+
const recoveredHostAttempts = await this.recoveredHostAttemptsForHello();
|
|
243
|
+
const ack = await connection.invoke("Hello", {
|
|
244
|
+
version: this.config.version,
|
|
245
|
+
protocolVersion: this.config.protocolVersion,
|
|
246
|
+
minServerVersion: this.config.minServerVersion ?? null,
|
|
247
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
248
|
+
capabilities: this.capabilities(),
|
|
249
|
+
recoveredHostAttempts,
|
|
250
|
+
});
|
|
251
|
+
if (!ack.accepted) {
|
|
252
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? "Hello callback was rejected.");
|
|
253
|
+
}
|
|
254
|
+
this.recoveredHostAttempts.clear();
|
|
255
|
+
logBridgeStatus(`Registered runner ${this.config.runnerId}, instance ${this.config.runnerInstanceId}, build ${this.config.version}, protocol ${this.config.protocolVersion}.`);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
catch (error) {
|
|
259
|
+
if (!isProviderStorageUnavailable(error) || storageRecoveryAttempt >= this.config.reconnectBackoffMs.length) {
|
|
260
|
+
throw error;
|
|
261
|
+
}
|
|
262
|
+
const backoff = this.config.reconnectBackoffMs[storageRecoveryAttempt] ?? 5000;
|
|
263
|
+
storageRecoveryAttempt += 1;
|
|
264
|
+
logBridgeStatus(`Hello reached Forge while storage was recovering. Retrying registration ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
265
|
+
await delay(backoff);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
45
268
|
}
|
|
46
269
|
async run(signal) {
|
|
47
270
|
let attempt = 0;
|
|
@@ -72,6 +295,7 @@ export class ForgeBridgeService {
|
|
|
72
295
|
finally {
|
|
73
296
|
if (signal?.aborted) {
|
|
74
297
|
this.abortActiveSessions(String(signal.reason ?? "bridge stopping"));
|
|
298
|
+
this.abortActiveReadinessProbes(String(signal.reason ?? "bridge stopping"));
|
|
75
299
|
}
|
|
76
300
|
await this.stopConnection();
|
|
77
301
|
}
|
|
@@ -79,31 +303,407 @@ export class ForgeBridgeService {
|
|
|
79
303
|
async stop() {
|
|
80
304
|
this.stopped = true;
|
|
81
305
|
this.abortActiveSessions("bridge stopping");
|
|
306
|
+
this.abortActiveReadinessProbes("bridge stopping");
|
|
82
307
|
await this.stopConnection();
|
|
308
|
+
await this.preflight.release();
|
|
309
|
+
}
|
|
310
|
+
async handleChatTurn(rawDispatch) {
|
|
311
|
+
const packet = this.parseChatTurnOrReport(rawDispatch);
|
|
312
|
+
if (!packet) {
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
if (packet.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
316
|
+
logBridgeStatus(`Rejected Bridge Chat turn ${packet.turnId}: stale runner instance.`);
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
const capabilityFailure = this.requiredChatCapabilityFailure(packet);
|
|
320
|
+
if (capabilityFailure) {
|
|
321
|
+
await this.sendStandaloneChatTerminal(packet, "failed", {
|
|
322
|
+
code: "bridge_chat.runner_unavailable",
|
|
323
|
+
message: capabilityFailure,
|
|
324
|
+
}).catch(() => undefined);
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
const existing = this.activeSessions.get(packet.leaseId);
|
|
328
|
+
if (existing) {
|
|
329
|
+
if (existing.kind === "chat"
|
|
330
|
+
&& existing.fencingToken === packet.fencingToken
|
|
331
|
+
&& existing.chatRunId === packet.chatRunId
|
|
332
|
+
&& existing.conversationId === packet.conversationId
|
|
333
|
+
&& existing.turnId === packet.turnId) {
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
await this.sendStandaloneChatTerminal(packet, "failed", {
|
|
337
|
+
code: "bridge_chat.stale_fencing_token",
|
|
338
|
+
message: "Bridge Chat turn fence conflicts with an active bridge session.",
|
|
339
|
+
}).catch(() => undefined);
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
342
|
+
const controller = new AbortController();
|
|
343
|
+
const active = {
|
|
344
|
+
leaseId: packet.leaseId,
|
|
345
|
+
kind: "chat",
|
|
346
|
+
fencingToken: packet.fencingToken,
|
|
347
|
+
dispatchAttemptId: null,
|
|
348
|
+
runnerInstanceId: packet.runnerInstanceId,
|
|
349
|
+
leaseExpiresAt: chatTurnLeaseExpiresAt(packet),
|
|
350
|
+
sandboxed: false,
|
|
351
|
+
resources: zeroSandboxResources(),
|
|
352
|
+
controller,
|
|
353
|
+
chatRunId: packet.chatRunId,
|
|
354
|
+
conversationId: packet.conversationId,
|
|
355
|
+
turnId: packet.turnId,
|
|
356
|
+
providerThreadId: packet.providerThreadId ?? null,
|
|
357
|
+
chatSequence: 0,
|
|
358
|
+
terminalCallbackAttempted: false,
|
|
359
|
+
};
|
|
360
|
+
this.activeSessions.set(packet.leaseId, active);
|
|
361
|
+
let workspacePath = null;
|
|
362
|
+
let cleanupAttempted = false;
|
|
363
|
+
let cleanupOutcome = null;
|
|
364
|
+
let cleanupReported = false;
|
|
365
|
+
const cleanup = async () => {
|
|
366
|
+
if (!workspacePath || cleanupAttempted)
|
|
367
|
+
return null;
|
|
368
|
+
cleanupAttempted = true;
|
|
369
|
+
cleanupOutcome = await cleanupChatTurnWorkspace(workspacePath, chatRedactionSecrets(packet));
|
|
370
|
+
return cleanupOutcome;
|
|
371
|
+
};
|
|
372
|
+
const reportCleanup = async (outcome) => {
|
|
373
|
+
if (!outcome || cleanupReported)
|
|
374
|
+
return;
|
|
375
|
+
cleanupReported = true;
|
|
376
|
+
await this.sendChatCallback(active, "output_delta", outcome).catch(() => undefined);
|
|
377
|
+
};
|
|
378
|
+
try {
|
|
379
|
+
workspacePath = await this.createChatTurnWorkspace(packet);
|
|
380
|
+
await materializeChatTurnMetadata(packet, workspacePath);
|
|
381
|
+
const run = await this.runChatProviderProcess(packet, active, workspacePath);
|
|
382
|
+
cleanupOutcome = await cleanup();
|
|
383
|
+
await reportCleanup(cleanupOutcome);
|
|
384
|
+
if (controller.signal.aborted) {
|
|
385
|
+
await this.sendChatAbortTerminal(packet, active, cleanupOutcome);
|
|
386
|
+
return;
|
|
387
|
+
}
|
|
388
|
+
if (cleanupOutcome?.status === "failed") {
|
|
389
|
+
throw new Error(`chat_turn_cleanup_failed: ${cleanupOutcome.message}`);
|
|
390
|
+
}
|
|
391
|
+
if (run.result.exitCode !== 0) {
|
|
392
|
+
throw chatProviderExitFailure(run.command, run.result, chatRedactionSecrets(packet));
|
|
393
|
+
}
|
|
394
|
+
await this.sendChatCallback(active, "output_completed", {
|
|
395
|
+
code: "output_completed",
|
|
396
|
+
text: clip(redactText(run.output.finalMessage ?? "", chatRedactionSecrets(packet)), maxChatTerminalTextChars),
|
|
397
|
+
provider: packet.providerKind,
|
|
398
|
+
model: packet.model,
|
|
399
|
+
cleanup: cleanupOutcome,
|
|
400
|
+
});
|
|
401
|
+
}
|
|
402
|
+
catch (error) {
|
|
403
|
+
cleanupOutcome = cleanupOutcome ?? await cleanup();
|
|
404
|
+
if (active.terminalCallbackAttempted) {
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
await reportCleanup(cleanupOutcome);
|
|
408
|
+
const payload = controller.signal.aborted
|
|
409
|
+
? chatAbortPayload(controller.signal.reason, cleanupOutcome, packet)
|
|
410
|
+
: {
|
|
411
|
+
code: "provider_failed",
|
|
412
|
+
message: clip(redactText(errorMessage(error), chatRedactionSecrets(packet)), 2000),
|
|
413
|
+
cleanup: cleanupOutcome,
|
|
414
|
+
};
|
|
415
|
+
const kind = controller.signal.aborted ? chatAbortKind(controller.signal.reason) : "failed";
|
|
416
|
+
await this.sendChatCallback(active, kind, payload).catch(callbackError => {
|
|
417
|
+
logBridgeStatus(`Bridge Chat terminal callback for turn ${packet.turnId} was not accepted: ${errorMessage(callbackError)}`);
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
finally {
|
|
421
|
+
await this.releaseActiveSessionAndRefreshCapacity(packet.leaseId);
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
async createChatTurnWorkspace(packet) {
|
|
425
|
+
const root = path.join(this.config.sandboxStateRoot, "chat-turns");
|
|
426
|
+
await fs.promises.mkdir(root, { recursive: true });
|
|
427
|
+
return await fs.promises.mkdtemp(path.join(root, `${safePathComponent(packet.turnId)}-`));
|
|
428
|
+
}
|
|
429
|
+
async runChatProviderProcess(packet, active, cwd) {
|
|
430
|
+
const output = createProviderOutputState(packet.providerKind);
|
|
431
|
+
const plan = buildChatProviderCommand(packet, cwd);
|
|
432
|
+
const redactionSecrets = chatRedactionSecrets(packet);
|
|
433
|
+
let started = false;
|
|
434
|
+
const result = await this.chatProcessRunner.run({
|
|
435
|
+
command: plan.command.command,
|
|
436
|
+
args: plan.command.args,
|
|
437
|
+
cwd,
|
|
438
|
+
env: {
|
|
439
|
+
FORGE_BRIDGE_CHAT_RUN_ID: packet.chatRunId,
|
|
440
|
+
FORGE_BRIDGE_CHAT_TURN_ID: packet.turnId,
|
|
441
|
+
},
|
|
442
|
+
stdin: plan.stdin,
|
|
443
|
+
}, {
|
|
444
|
+
onStarted: async () => {
|
|
445
|
+
started = true;
|
|
446
|
+
await this.sendChatCallback(active, "turn_started", {
|
|
447
|
+
status: "running",
|
|
448
|
+
provider: packet.providerKind,
|
|
449
|
+
model: packet.model,
|
|
450
|
+
command: redactText(formatProviderCliCommandForCheckpoint(plan.command), redactionSecrets),
|
|
451
|
+
});
|
|
452
|
+
},
|
|
453
|
+
onStdoutLine: async (line) => {
|
|
454
|
+
for (const event of output.consume(line)) {
|
|
455
|
+
await this.sendChatProviderEvent(packet, active, event, output.providerThreadId);
|
|
456
|
+
}
|
|
457
|
+
},
|
|
458
|
+
}, active.controller.signal);
|
|
459
|
+
if (started && !active.controller.signal.aborted) {
|
|
460
|
+
await this.sendChatCallback(active, "output_delta", {
|
|
461
|
+
eventType: "provider_cli_completed",
|
|
462
|
+
role: "runner",
|
|
463
|
+
kind: "checkpoint",
|
|
464
|
+
content: "Bridge Chat provider process exited.",
|
|
465
|
+
}).catch(() => undefined);
|
|
466
|
+
}
|
|
467
|
+
return { command: plan.command, output, result };
|
|
468
|
+
}
|
|
469
|
+
async sendChatProviderEvent(packet, active, event, providerThreadId) {
|
|
470
|
+
if (providerThreadId) {
|
|
471
|
+
active.providerThreadId = providerThreadId;
|
|
472
|
+
}
|
|
473
|
+
else if (event.eventType === "provider_thread_started") {
|
|
474
|
+
active.providerThreadId = providerThreadIdFromEvent(event) ?? active.providerThreadId;
|
|
475
|
+
}
|
|
476
|
+
if (event.eventType === "provider_tool_heartbeat") {
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
const redactionSecrets = chatRedactionSecrets(packet);
|
|
480
|
+
const payload = {
|
|
481
|
+
role: event.role,
|
|
482
|
+
kind: event.kind,
|
|
483
|
+
eventType: event.eventType ?? "provider_event",
|
|
484
|
+
content: clip(redactText(event.content, redactionSecrets), maxChatEventContentChars),
|
|
485
|
+
data: event.data ? clip(redactText(event.data, redactionSecrets), maxChatEventContentChars) : undefined,
|
|
486
|
+
};
|
|
487
|
+
await this.sendChatCallback(active, "output_delta", payload);
|
|
488
|
+
}
|
|
489
|
+
async sendChatAbortTerminal(packet, active, cleanupOutcome) {
|
|
490
|
+
await this.sendChatCallback(active, chatAbortKind(active.controller.signal.reason), chatAbortPayload(active.controller.signal.reason, cleanupOutcome, packet));
|
|
491
|
+
}
|
|
492
|
+
async sendStandaloneChatTerminal(packet, kind, payload) {
|
|
493
|
+
const active = {
|
|
494
|
+
leaseId: packet.leaseId,
|
|
495
|
+
kind: "chat",
|
|
496
|
+
fencingToken: packet.fencingToken,
|
|
497
|
+
dispatchAttemptId: null,
|
|
498
|
+
runnerInstanceId: packet.runnerInstanceId,
|
|
499
|
+
leaseExpiresAt: new Date(Date.now() + defaultChatTurnLeaseMs).toISOString(),
|
|
500
|
+
sandboxed: false,
|
|
501
|
+
resources: zeroSandboxResources(),
|
|
502
|
+
controller: new AbortController(),
|
|
503
|
+
chatRunId: packet.chatRunId,
|
|
504
|
+
conversationId: packet.conversationId,
|
|
505
|
+
turnId: packet.turnId,
|
|
506
|
+
providerThreadId: packet.providerThreadId ?? null,
|
|
507
|
+
chatSequence: 0,
|
|
508
|
+
terminalCallbackAttempted: false,
|
|
509
|
+
};
|
|
510
|
+
await this.sendChatCallback(active, kind, payload);
|
|
511
|
+
}
|
|
512
|
+
async sendChatCallback(active, kind, payload = null) {
|
|
513
|
+
if (active.kind !== "chat" || !active.chatRunId || !active.conversationId || !active.turnId) {
|
|
514
|
+
throw new Error("Bridge Chat callback requires an active chat turn.");
|
|
515
|
+
}
|
|
516
|
+
const terminal = isTerminalChatTurnCallbackKind(kind);
|
|
517
|
+
if (terminal && active.terminalCallbackAttempted) {
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
520
|
+
assertBridgeChatCallbackPayload(payload);
|
|
521
|
+
const sequence = (active.chatSequence ?? 0) + 1;
|
|
522
|
+
const envelope = {
|
|
523
|
+
callbackId: randomUUID(),
|
|
524
|
+
chatRunId: active.chatRunId,
|
|
525
|
+
conversationId: active.conversationId,
|
|
526
|
+
turnId: active.turnId,
|
|
527
|
+
leaseId: active.leaseId,
|
|
528
|
+
fencingToken: active.fencingToken,
|
|
529
|
+
runnerId: this.config.runnerId,
|
|
530
|
+
runnerInstanceId: active.runnerInstanceId ?? this.config.runnerInstanceId,
|
|
531
|
+
sequence,
|
|
532
|
+
eventType: kind,
|
|
533
|
+
redactionStatus: "redacted",
|
|
534
|
+
providerThreadId: active.providerThreadId ?? null,
|
|
535
|
+
payload,
|
|
536
|
+
occurredAt: new Date().toISOString(),
|
|
537
|
+
protocolVersion: bridgeChatProtocolVersion,
|
|
538
|
+
};
|
|
539
|
+
if (terminal) {
|
|
540
|
+
active.terminalCallbackAttempted = true;
|
|
541
|
+
}
|
|
542
|
+
await this.invokeChatCallbackWithRetry(envelope);
|
|
543
|
+
active.chatSequence = sequence;
|
|
544
|
+
}
|
|
545
|
+
async invokeChatCallbackWithRetry(envelope) {
|
|
546
|
+
const maxAttempts = 3;
|
|
547
|
+
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
548
|
+
try {
|
|
549
|
+
await this.invokeAck("ChatTurnCallback", envelope);
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
catch (error) {
|
|
553
|
+
if (error instanceof BridgeCallbackRejectedError || attempt === maxAttempts - 1) {
|
|
554
|
+
throw error;
|
|
555
|
+
}
|
|
556
|
+
await delay(200 * (attempt + 1));
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
requiredChatCapabilityFailure(packet) {
|
|
561
|
+
const capability = this.bridgeChatCapability();
|
|
562
|
+
if (capability.availability === "unavailable" || !capability.supportsTextStreaming) {
|
|
563
|
+
return capability.unavailableReason ?? "Bridge Chat text capability is unavailable.";
|
|
564
|
+
}
|
|
565
|
+
if (capability.supportedProviders.length > 0
|
|
566
|
+
&& !containsIgnoreCase(capability.supportedProviders, packet.providerKind)) {
|
|
567
|
+
return `Bridge Chat provider '${packet.providerKind}' is not advertised by this bridge.`;
|
|
568
|
+
}
|
|
569
|
+
if (capability.supportedModels.length > 0
|
|
570
|
+
&& !containsIgnoreCase(capability.supportedModels, packet.model)) {
|
|
571
|
+
return `Bridge Chat model '${packet.model}' is not advertised by this bridge.`;
|
|
572
|
+
}
|
|
573
|
+
if (packet.requestedCapabilities.includes("cancel") && !capability.supportsCancel) {
|
|
574
|
+
return "Bridge Chat cancellation is not advertised by this bridge.";
|
|
575
|
+
}
|
|
576
|
+
if (packet.requestedCapabilities.includes("resume") && !capability.supportsResume) {
|
|
577
|
+
return "Bridge Chat resume is not advertised by this bridge.";
|
|
578
|
+
}
|
|
579
|
+
if (packet.requestedCapabilities.includes("thread_continuity")
|
|
580
|
+
&& !capability.supportsProviderThreadContinuity) {
|
|
581
|
+
return "Bridge Chat provider thread continuity is not advertised by this bridge.";
|
|
582
|
+
}
|
|
583
|
+
if (packet.requestedCapabilities.includes("attachment_handoffs") && !capability.supportsAttachments) {
|
|
584
|
+
return "Bridge Chat attachment handoffs are not advertised by this bridge.";
|
|
585
|
+
}
|
|
586
|
+
if (packet.requestedCapabilities.includes("action_broker_grants") && !capability.supportsActionBrokerGrants) {
|
|
587
|
+
return "Bridge Chat action-broker grants are not advertised by this bridge.";
|
|
588
|
+
}
|
|
589
|
+
return null;
|
|
590
|
+
}
|
|
591
|
+
bridgeChatCapability() {
|
|
592
|
+
return bridgeChatCapabilityFor(this.config.cliProviders, this.config.providerStatuses, this.config.bridgeChat);
|
|
83
593
|
}
|
|
84
|
-
async handleDispatch(rawDispatch) {
|
|
85
|
-
const dispatch = this.parseDispatchOrReport(rawDispatch);
|
|
594
|
+
async handleDispatch(rawDispatch, reservationHandoff) {
|
|
595
|
+
const dispatch = await this.parseDispatchOrReport(rawDispatch);
|
|
86
596
|
if (!dispatch) {
|
|
597
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
if (this.enforceHostDatabaseCapabilityBoundary) {
|
|
601
|
+
try {
|
|
602
|
+
assertHostDatabaseCapabilityBoundary();
|
|
603
|
+
}
|
|
604
|
+
catch (error) {
|
|
605
|
+
const message = errorMessage(error);
|
|
606
|
+
await this.sendResult(dispatch, failureResult(message, [{
|
|
607
|
+
role: "runner",
|
|
608
|
+
kind: "error",
|
|
609
|
+
content: message,
|
|
610
|
+
eventType: "host_database_capability_forbidden",
|
|
611
|
+
}], failureFactsFromError(error, { source: "bridge" }))).catch(() => undefined);
|
|
612
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
let capabilityFailure;
|
|
617
|
+
try {
|
|
618
|
+
capabilityFailure = await this.requiredCapabilityFailure(dispatch);
|
|
619
|
+
}
|
|
620
|
+
catch (error) {
|
|
621
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
622
|
+
throw error;
|
|
623
|
+
}
|
|
624
|
+
if (capabilityFailure) {
|
|
625
|
+
await this.sendResult(dispatch, failureResult(capabilityFailure, [{
|
|
626
|
+
role: "runner",
|
|
627
|
+
kind: "error",
|
|
628
|
+
content: capabilityFailure,
|
|
629
|
+
eventType: "docker_engine_unavailable",
|
|
630
|
+
}], failureFactsForCode("docker_engine_unavailable"))).catch(() => undefined);
|
|
631
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
87
632
|
return;
|
|
88
633
|
}
|
|
89
634
|
const leaseId = dispatch.lease.id;
|
|
90
635
|
const fencingToken = dispatch.lease.fencingToken;
|
|
91
636
|
const redactionSecrets = exactSecrets(dispatch, this.config);
|
|
92
|
-
if (this.
|
|
93
|
-
await this.sendResult(
|
|
637
|
+
if (reservationHandoff && !this.reservedDispatchHandoffIsCurrent(reservationHandoff, dispatch)) {
|
|
638
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
94
639
|
return;
|
|
95
640
|
}
|
|
96
|
-
const
|
|
97
|
-
|
|
641
|
+
const existing = this.activeSessions.get(leaseId);
|
|
642
|
+
if (existing) {
|
|
643
|
+
if (existing.fencingToken === fencingToken) {
|
|
644
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
645
|
+
return;
|
|
646
|
+
}
|
|
647
|
+
await this.sendResult(dispatch, failureResult("Dispatch fence conflicts with an active bridge session.", [], failureFactsForCode("bridge.dispatch_fence_conflict"))).catch(() => undefined);
|
|
648
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
649
|
+
return;
|
|
650
|
+
}
|
|
651
|
+
const controller = reservationHandoff?.controller ?? new AbortController();
|
|
652
|
+
this.activeSessions.set(leaseId, {
|
|
653
|
+
leaseId,
|
|
654
|
+
kind: "dispatch",
|
|
655
|
+
fencingToken,
|
|
656
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
657
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
658
|
+
leaseExpiresAt: dispatch.lease.expiresAt,
|
|
659
|
+
sandboxed: dispatch.sandbox?.executionMode === "docker",
|
|
660
|
+
resources: dispatch.sandbox?.executionMode === "docker"
|
|
661
|
+
? dispatch.sandbox.requestedResources
|
|
662
|
+
: zeroSandboxResources(),
|
|
663
|
+
controller
|
|
664
|
+
});
|
|
665
|
+
if (reservationHandoff) {
|
|
666
|
+
// Promotion is synchronous with active-session registration. A qualified
|
|
667
|
+
// pre-start Cancel therefore sees either the cancellable reservation or
|
|
668
|
+
// the fully active owner; there is no cancellation-blind handoff gap.
|
|
669
|
+
this.releaseReservedDispatchHandoff(reservationHandoff);
|
|
670
|
+
}
|
|
671
|
+
// Server-requested runtime readiness is background maintenance. A probe may
|
|
672
|
+
// have passed admission during the narrow handoff between prepared-runtime
|
|
673
|
+
// verification and the foreground attempt's final admission. Preempt it as
|
|
674
|
+
// soon as real agent work arrives so the probe cannot consume the memory
|
|
675
|
+
// headroom that the foreground attempt is waiting to acquire.
|
|
676
|
+
this.deferActiveReadinessProbes();
|
|
98
677
|
try {
|
|
99
678
|
const normalized = this.normalizeDispatch(dispatch);
|
|
100
|
-
|
|
679
|
+
if (normalized.sandbox) {
|
|
680
|
+
const started = {
|
|
681
|
+
dispatchAttemptId: normalized.sandbox.dispatchAttemptId,
|
|
682
|
+
leaseId,
|
|
683
|
+
fencingToken,
|
|
684
|
+
runnerInstanceId: normalized.sandbox.runnerInstanceId
|
|
685
|
+
};
|
|
686
|
+
await this.invokeAck("DispatchStarted", started);
|
|
687
|
+
}
|
|
688
|
+
const sink = {
|
|
101
689
|
emit: async (event) => {
|
|
690
|
+
if (event.eventType === "provider_cli_started") {
|
|
691
|
+
const active = this.activeSessions.get(leaseId);
|
|
692
|
+
if (active)
|
|
693
|
+
active.resources = { ...active.resources, buildSlots: 0 };
|
|
694
|
+
}
|
|
102
695
|
try {
|
|
103
696
|
await this.invokeAck("DispatchEvent", {
|
|
104
697
|
leaseId,
|
|
105
698
|
fencingToken,
|
|
106
|
-
|
|
699
|
+
...(dispatch.sandbox ? {
|
|
700
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
701
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId
|
|
702
|
+
} : {}),
|
|
703
|
+
event: redactRunnerDispatchEvent({ ...event, eventId: event.eventId ?? randomUUID() }, redactionSecrets)
|
|
704
|
+
}, {
|
|
705
|
+
replayCallbackTimeout: true,
|
|
706
|
+
signal: controller.signal,
|
|
107
707
|
});
|
|
108
708
|
}
|
|
109
709
|
catch (error) {
|
|
@@ -114,20 +714,386 @@ export class ForgeBridgeService {
|
|
|
114
714
|
throw error;
|
|
115
715
|
}
|
|
116
716
|
}
|
|
117
|
-
}
|
|
118
|
-
|
|
717
|
+
};
|
|
718
|
+
const result = normalized.sandbox?.executionMode === "docker"
|
|
719
|
+
? await this.sandboxRuntime.runOnce(normalized, sink, controller.signal)
|
|
720
|
+
: await this.sessions.runOnce(normalized, sink, controller.signal);
|
|
721
|
+
if (normalized.sandbox?.executionMode === "docker") {
|
|
722
|
+
await this.flushCleanupReports().catch(() => undefined);
|
|
723
|
+
}
|
|
724
|
+
const redactedResult = redactRunOnceResultPacket(result, redactionSecrets);
|
|
725
|
+
await this.sendResult(dispatch, compactTerminalResult(dispatch, redactedResult));
|
|
119
726
|
}
|
|
120
727
|
catch (error) {
|
|
121
728
|
const summary = controller.signal.aborted
|
|
122
729
|
? `Dispatch cancelled: ${String(controller.signal.reason ?? "cancelled")}`
|
|
123
730
|
: redactText(error instanceof Error ? error.message : String(error), redactionSecrets);
|
|
124
|
-
|
|
731
|
+
const failureFacts = failureFactsFromError(error, {
|
|
732
|
+
cancelled: controller.signal.aborted,
|
|
733
|
+
source: "bridge",
|
|
734
|
+
});
|
|
735
|
+
const failureEventType = failureFacts.category === "unknown" ? null : failureFacts.code;
|
|
736
|
+
const failureEvents = failureEventType ? [{
|
|
737
|
+
role: "runner",
|
|
738
|
+
kind: "error",
|
|
739
|
+
content: summary,
|
|
740
|
+
eventType: failureEventType,
|
|
741
|
+
}] : [];
|
|
742
|
+
await this.sendResult(dispatch, failureResult(summary, failureEvents, failureFacts)).catch(() => undefined);
|
|
125
743
|
}
|
|
126
744
|
finally {
|
|
127
|
-
this.
|
|
745
|
+
await this.releaseActiveSessionAndRefreshCapacity(leaseId);
|
|
128
746
|
}
|
|
129
747
|
}
|
|
130
|
-
|
|
748
|
+
reservedDispatchHandoffIsCurrent(handoff, dispatch) {
|
|
749
|
+
const sandbox = dispatch.sandbox;
|
|
750
|
+
const reservation = handoff.reservation;
|
|
751
|
+
return sandbox !== null
|
|
752
|
+
&& sandbox !== undefined
|
|
753
|
+
&& sandbox.dispatchAttemptId === handoff.dispatchAttemptId
|
|
754
|
+
&& this.reservations.get(handoff.dispatchAttemptId) === reservation
|
|
755
|
+
&& reservation.startClaim === handoff
|
|
756
|
+
&& !handoff.controller.signal.aborted
|
|
757
|
+
&& reservation.leaseId === dispatch.lease.id
|
|
758
|
+
&& reservation.fencingToken === dispatch.lease.fencingToken
|
|
759
|
+
&& reservation.runnerInstanceId === sandbox.runnerInstanceId
|
|
760
|
+
&& Date.parse(reservation.reservedUntil) > Date.now();
|
|
761
|
+
}
|
|
762
|
+
releaseReservedDispatchHandoff(handoff) {
|
|
763
|
+
if (!handoff
|
|
764
|
+
|| this.reservations.get(handoff.dispatchAttemptId) !== handoff.reservation
|
|
765
|
+
|| handoff.reservation.startClaim !== handoff)
|
|
766
|
+
return;
|
|
767
|
+
this.reservations.delete(handoff.dispatchAttemptId);
|
|
768
|
+
}
|
|
769
|
+
async releaseActiveSessionAndRefreshCapacity(leaseId, expectedDispatch) {
|
|
770
|
+
if (expectedDispatch) {
|
|
771
|
+
const active = this.activeSessions.get(leaseId);
|
|
772
|
+
// Admission failures may arrive after a newer attempt starts on the same
|
|
773
|
+
// lease. A terminal callback can release only the dispatch it identifies.
|
|
774
|
+
if (!active || active.kind !== "dispatch"
|
|
775
|
+
|| active.fencingToken !== expectedDispatch.lease.fencingToken
|
|
776
|
+
|| active.dispatchAttemptId !== (expectedDispatch.sandbox?.dispatchAttemptId ?? null)
|
|
777
|
+
|| active.runnerInstanceId !== (expectedDispatch.sandbox?.runnerInstanceId ?? null))
|
|
778
|
+
return;
|
|
779
|
+
}
|
|
780
|
+
if (!this.activeSessions.delete(leaseId))
|
|
781
|
+
return;
|
|
782
|
+
// Publish released host/runtime capacity before the terminal result can
|
|
783
|
+
// enqueue the next pipeline phase. Otherwise the server may reject that
|
|
784
|
+
// handoff from the previous periodic heartbeat's active reservation.
|
|
785
|
+
// Publish the updated reservation counts from cached samples only. Probes
|
|
786
|
+
// belong to periodic/admission heartbeats and must not outlive this bounded
|
|
787
|
+
// terminal handoff. Storage recovery must not block durable result delivery.
|
|
788
|
+
const controller = new AbortController();
|
|
789
|
+
let timer;
|
|
790
|
+
try {
|
|
791
|
+
await Promise.race([
|
|
792
|
+
this.sendHeartbeatSnapshot(controller.signal).catch(() => undefined),
|
|
793
|
+
new Promise(resolve => {
|
|
794
|
+
timer = setTimeout(() => {
|
|
795
|
+
controller.abort("terminal capacity refresh deadline");
|
|
796
|
+
resolve();
|
|
797
|
+
}, Math.min(5000, positiveFiniteNumber(this.config.callbackAckTimeoutMs ?? 30_000, 30_000)));
|
|
798
|
+
}),
|
|
799
|
+
]);
|
|
800
|
+
}
|
|
801
|
+
finally {
|
|
802
|
+
if (timer)
|
|
803
|
+
clearTimeout(timer);
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
async handleOffer(rawDispatch) {
|
|
807
|
+
const envelope = parseOfferEnvelope(rawDispatch);
|
|
808
|
+
if (!envelope) {
|
|
809
|
+
return;
|
|
810
|
+
}
|
|
811
|
+
const dispatch = await this.parseDispatchOrReport(envelope.dispatch);
|
|
812
|
+
if (!dispatch?.sandbox
|
|
813
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
814
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
815
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
816
|
+
return;
|
|
817
|
+
const sandbox = dispatch.sandbox;
|
|
818
|
+
this.expireReservations(Date.now());
|
|
819
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
820
|
+
if (existing) {
|
|
821
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
822
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
823
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
824
|
+
await this.sendOfferDecision(dispatch, sameFence, sameFence ? null : "stale_fence", sameFence ? null : "Offer fence conflicts with the existing reservation.", sameFence ? existing.reservedUntil : null, sameFence ? existing.resources : null);
|
|
825
|
+
return;
|
|
826
|
+
}
|
|
827
|
+
if (sandbox.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
828
|
+
await this.sendOfferDecision(dispatch, false, "stale_runner_instance", "Offer targets a different runner process instance.");
|
|
829
|
+
return;
|
|
830
|
+
}
|
|
831
|
+
if (Date.parse(envelope.expiresAt) <= Date.now()) {
|
|
832
|
+
await this.sendOfferDecision(dispatch, false, "lease_expired", "Offer lease is already expired.");
|
|
833
|
+
return;
|
|
834
|
+
}
|
|
835
|
+
const preflight = sandbox.executionMode === "docker" && this.config.dockerEnabled
|
|
836
|
+
? await this.preflight.run()
|
|
837
|
+
: this.preflight.current();
|
|
838
|
+
if (sandbox.executionMode === "docker") {
|
|
839
|
+
if (!this.config.dockerEnabled || !preflight?.healthy) {
|
|
840
|
+
await this.sendOfferDecision(dispatch, false, preflight?.unavailableCode ?? "docker_unavailable", preflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
841
|
+
return;
|
|
842
|
+
}
|
|
843
|
+
const networkFailure = sandboxNetworkPolicyFailure(dispatch, this.config);
|
|
844
|
+
if (networkFailure) {
|
|
845
|
+
await this.sendOfferDecision(dispatch, false, "policy_mismatch", networkFailure);
|
|
846
|
+
return;
|
|
847
|
+
}
|
|
848
|
+
const providerNetworkFailure = sandboxProviderNetworkFailure(dispatch, this.config);
|
|
849
|
+
if (providerNetworkFailure) {
|
|
850
|
+
await this.sendOfferDecision(dispatch, false, "provider_network_policy_denied", providerNetworkFailure);
|
|
851
|
+
return;
|
|
852
|
+
}
|
|
853
|
+
const authFailure = sandboxAuthFailure(dispatch, this.config);
|
|
854
|
+
if (authFailure) {
|
|
855
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_auth_unavailable", authFailure);
|
|
856
|
+
return;
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
let requestedResources = sandbox.requestedResources;
|
|
860
|
+
let sandboxConfigCommitSha = sandbox.sandboxConfigCommitSha ?? null;
|
|
861
|
+
const normalizedDispatch = this.normalizeDispatch(dispatch);
|
|
862
|
+
if (sandbox.executionMode === "docker") {
|
|
863
|
+
try {
|
|
864
|
+
const profile = await this.sandboxResourceResolver(normalizedDispatch);
|
|
865
|
+
requestedResources = profile.resources;
|
|
866
|
+
sandboxConfigCommitSha = profile.sandboxConfigCommitSha;
|
|
867
|
+
}
|
|
868
|
+
catch (error) {
|
|
869
|
+
await this.sendOfferDecision(dispatch, false, "sandbox_capacity_profile_unavailable", error instanceof Error ? error.message : "The immutable sandbox contract could not be profiled for admission.");
|
|
870
|
+
return;
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
else if (normalizedDispatch.repo.serverLocalPath) {
|
|
874
|
+
// Host execution uses the same project-derived disk reservation as Docker.
|
|
875
|
+
// The API supplies a placement-time estimate; the runner can safely raise it
|
|
876
|
+
// from its authoritative checkout before admitting the attempt.
|
|
877
|
+
const measuredWorkspaceBytes = await estimateWorkspaceBytesFromRepo(normalizedDispatch.repo.serverLocalPath, { role: normalizedDispatch.execution.role }).catch(() => null);
|
|
878
|
+
if (measuredWorkspaceBytes !== null) {
|
|
879
|
+
requestedResources = {
|
|
880
|
+
...requestedResources,
|
|
881
|
+
workspaceBytes: Math.max(requestedResources.workspaceBytes, measuredWorkspaceBytes),
|
|
882
|
+
};
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
// Resource profiling can await filesystem and Docker work. Admission needs
|
|
886
|
+
// a dynamic host/daemon sample that began after profiling completed, so an
|
|
887
|
+
// older in-flight heartbeat inspection cannot supply the commit decision.
|
|
888
|
+
const admissionPreflight = sandbox.executionMode === "docker"
|
|
889
|
+
? await this.preflight.refreshCapacityForAdmission()
|
|
890
|
+
: preflight;
|
|
891
|
+
// Resource profiling and preflight may await slow filesystem or Docker I/O.
|
|
892
|
+
// Re-read every mutable capacity input only after that work, then decide and
|
|
893
|
+
// commit synchronously. JavaScript cannot interleave another Offer handler
|
|
894
|
+
// inside this non-async method, so the ledger check and insertion are one
|
|
895
|
+
// atomic event-loop turn without holding a lock across external I/O.
|
|
896
|
+
const decision = this.commitOfferReservation(dispatch, requestedResources, admissionPreflight);
|
|
897
|
+
await this.sendOfferDecision(dispatch, decision.accepted, decision.code, decision.message, decision.reservedUntil, decision.resources, sandboxConfigCommitSha);
|
|
898
|
+
}
|
|
899
|
+
commitOfferReservation(dispatch, requestedResources, preparedPreflight) {
|
|
900
|
+
const sandbox = dispatch.sandbox;
|
|
901
|
+
const now = Date.now();
|
|
902
|
+
this.expireReservations(now);
|
|
903
|
+
const existing = this.reservations.get(sandbox.dispatchAttemptId);
|
|
904
|
+
if (existing) {
|
|
905
|
+
const sameFence = existing.leaseId === dispatch.lease.id
|
|
906
|
+
&& existing.fencingToken === dispatch.lease.fencingToken
|
|
907
|
+
&& existing.runnerInstanceId === sandbox.runnerInstanceId;
|
|
908
|
+
return {
|
|
909
|
+
accepted: sameFence,
|
|
910
|
+
code: sameFence ? null : "stale_fence",
|
|
911
|
+
message: sameFence ? null : "Offer fence conflicts with the existing reservation.",
|
|
912
|
+
reservedUntil: sameFence ? existing.reservedUntil : null,
|
|
913
|
+
resources: sameFence ? existing.resources : null,
|
|
914
|
+
};
|
|
915
|
+
}
|
|
916
|
+
if (Date.parse(dispatch.lease.expiresAt) <= now) {
|
|
917
|
+
return rejectedOfferReservation("lease_expired", "Offer lease is already expired.");
|
|
918
|
+
}
|
|
919
|
+
if (sandbox.executionMode === "docker") {
|
|
920
|
+
// A concurrent heartbeat/preflight refresh may have produced a newer
|
|
921
|
+
// conservative dual-domain sample while this offer was being profiled.
|
|
922
|
+
const admissionPreflight = this.preflight.current() ?? preparedPreflight;
|
|
923
|
+
if (!this.config.dockerEnabled || !admissionPreflight?.healthy) {
|
|
924
|
+
return rejectedOfferReservation(admissionPreflight?.unavailableCode ?? "docker_unavailable", admissionPreflight?.unavailableReason ?? "Docker sandbox preflight is unavailable.");
|
|
925
|
+
}
|
|
926
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
927
|
+
const usedResources = this.usedSandboxResources(runtimeState.degraded);
|
|
928
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
929
|
+
const capacity = this.sandboxResourceCapacity(admissionPreflight, activeResources);
|
|
930
|
+
if (!canReserveSandboxResources(usedResources, requestedResources, capacity)) {
|
|
931
|
+
return rejectedOfferReservation("capacity_unavailable", sandboxCapacityFailureMessage(usedResources, requestedResources, capacity, admissionPreflight, this.config));
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
const reservedUntil = new Date(Math.min(Date.parse(dispatch.lease.expiresAt), now + 30_000)).toISOString();
|
|
935
|
+
const reservation = {
|
|
936
|
+
leaseId: dispatch.lease.id,
|
|
937
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
938
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
939
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
940
|
+
reservedUntil,
|
|
941
|
+
sandboxed: sandbox.executionMode === "docker",
|
|
942
|
+
resources: requestedResources,
|
|
943
|
+
};
|
|
944
|
+
this.reservations.set(sandbox.dispatchAttemptId, reservation);
|
|
945
|
+
return {
|
|
946
|
+
accepted: true,
|
|
947
|
+
code: null,
|
|
948
|
+
message: null,
|
|
949
|
+
reservedUntil,
|
|
950
|
+
resources: requestedResources,
|
|
951
|
+
};
|
|
952
|
+
}
|
|
953
|
+
async handleStart(rawDispatch) {
|
|
954
|
+
const envelope = parseStartEnvelope(rawDispatch);
|
|
955
|
+
if (!envelope) {
|
|
956
|
+
return;
|
|
957
|
+
}
|
|
958
|
+
let dispatch = await this.parseDispatchOrReport(envelope.dispatch);
|
|
959
|
+
if (!dispatch?.sandbox
|
|
960
|
+
|| dispatch.sandbox.dispatchAttemptId !== envelope.dispatchAttemptId
|
|
961
|
+
|| dispatch.lease.id !== envelope.leaseId
|
|
962
|
+
|| dispatch.lease.fencingToken !== envelope.fencingToken)
|
|
963
|
+
return;
|
|
964
|
+
const active = this.activeSessions.get(dispatch.lease.id);
|
|
965
|
+
if (active?.kind === "dispatch"
|
|
966
|
+
&& active.fencingToken === dispatch.lease.fencingToken
|
|
967
|
+
&& active.dispatchAttemptId === dispatch.sandbox.dispatchAttemptId
|
|
968
|
+
&& active.runnerInstanceId === dispatch.sandbox.runnerInstanceId) {
|
|
969
|
+
return;
|
|
970
|
+
}
|
|
971
|
+
const reservation = this.reservations.get(dispatch.sandbox.dispatchAttemptId);
|
|
972
|
+
if (!reservation
|
|
973
|
+
|| reservation.leaseId !== dispatch.lease.id
|
|
974
|
+
|| reservation.fencingToken !== dispatch.lease.fencingToken
|
|
975
|
+
|| reservation.runnerInstanceId !== dispatch.sandbox.runnerInstanceId
|
|
976
|
+
|| Date.parse(reservation.reservedUntil) <= Date.now()) {
|
|
977
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
978
|
+
return;
|
|
979
|
+
}
|
|
980
|
+
// Claim the exact reservation before redemption or any other post-validation
|
|
981
|
+
// await. A duplicate Start for the same reservation is an idempotent replay:
|
|
982
|
+
// it must neither redeem a one-time grant nor disturb the original owner.
|
|
983
|
+
if (reservation.startClaim)
|
|
984
|
+
return;
|
|
985
|
+
const startClaim = {
|
|
986
|
+
dispatchAttemptId: envelope.dispatchAttemptId,
|
|
987
|
+
reservation,
|
|
988
|
+
controller: new AbortController(),
|
|
989
|
+
};
|
|
990
|
+
reservation.startClaim = startClaim;
|
|
991
|
+
if (dispatch.apiKeySecretGrant) {
|
|
992
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
993
|
+
await this.sendResult(dispatch, failureResult("Start rejected because protocol-v2 packets may not contain a plaintext provider grant.", [], failureFactsForCode("contract_plaintext_secret_rejected"))).catch(() => undefined);
|
|
994
|
+
return;
|
|
995
|
+
}
|
|
996
|
+
if (dispatch.apiKeySecretGrantHandle) {
|
|
997
|
+
let response;
|
|
998
|
+
try {
|
|
999
|
+
response = await this.redeemSecretGrant(dispatch);
|
|
1000
|
+
}
|
|
1001
|
+
catch (error) {
|
|
1002
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1003
|
+
throw error;
|
|
1004
|
+
}
|
|
1005
|
+
// Redemption may yield while a qualified Cancel releases this reservation.
|
|
1006
|
+
// Never launch from it or remove a successor reservation after that wait.
|
|
1007
|
+
if (!this.reservedDispatchHandoffIsCurrent(startClaim, dispatch)) {
|
|
1008
|
+
await this.sendResult(dispatch, failureResult("Start rejected because its reservation is missing or stale.", [], failureFactsForCode("bridge.reservation_stale"))).catch(() => undefined);
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
const grant = response.grant;
|
|
1012
|
+
if (!response.accepted
|
|
1013
|
+
|| !grant
|
|
1014
|
+
|| grant.grantId !== dispatch.apiKeySecretGrantHandle.grantId
|
|
1015
|
+
|| grant.provider !== dispatch.apiKeySecretGrantHandle.provider
|
|
1016
|
+
|| Date.parse(grant.expiresAt) <= Date.now()) {
|
|
1017
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1018
|
+
await this.sendResult(dispatch, failureResult(response.message ?? "Secret grant redemption was rejected.", [], failureFactsForCode("bridge.api_key_secret_grant_denied"))).catch(() => undefined);
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
dispatch = { ...dispatch, apiKeySecretGrant: grant, apiKeySecretGrantHandle: null };
|
|
1022
|
+
}
|
|
1023
|
+
else if (dispatch.execution.requiresApiKeySecretGrant) {
|
|
1024
|
+
this.releaseReservedDispatchHandoff(startClaim);
|
|
1025
|
+
await this.sendResult(dispatch, failureResult("Start rejected because the provider secret grant handle is missing.", [], failureFactsForCode("bridge.api_key_secret_grant_missing"))).catch(() => undefined);
|
|
1026
|
+
return;
|
|
1027
|
+
}
|
|
1028
|
+
dispatch = {
|
|
1029
|
+
...dispatch,
|
|
1030
|
+
sandbox: { ...dispatch.sandbox, requestedResources: reservation.resources },
|
|
1031
|
+
};
|
|
1032
|
+
await this.handleDispatch(dispatch, startClaim);
|
|
1033
|
+
}
|
|
1034
|
+
async redeemSecretGrant(dispatch) {
|
|
1035
|
+
const sandbox = dispatch.sandbox;
|
|
1036
|
+
const handle = dispatch.apiKeySecretGrantHandle;
|
|
1037
|
+
const connection = this.connection;
|
|
1038
|
+
if (!connection)
|
|
1039
|
+
return { accepted: false, code: "bridge.disconnected", message: "Bridge is disconnected.", grant: null };
|
|
1040
|
+
return await connection.invoke("RedeemSecretGrant", {
|
|
1041
|
+
dispatchAttemptId: sandbox.dispatchAttemptId,
|
|
1042
|
+
leaseId: dispatch.lease.id,
|
|
1043
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
1044
|
+
runnerInstanceId: sandbox.runnerInstanceId,
|
|
1045
|
+
grantId: handle.grantId
|
|
1046
|
+
});
|
|
1047
|
+
}
|
|
1048
|
+
async redeemPublishGrant(request, signal) {
|
|
1049
|
+
if (signal?.aborted)
|
|
1050
|
+
throw new Error(String(signal.reason ?? "publication cancelled"));
|
|
1051
|
+
const connection = this.connection;
|
|
1052
|
+
if (!connection)
|
|
1053
|
+
throw new Error("sandbox_output_publish_forbidden: bridge is disconnected.");
|
|
1054
|
+
const response = await connection.invoke("RedeemPublishGrant", request);
|
|
1055
|
+
if (!response.accepted || !response.credential) {
|
|
1056
|
+
throw new Error(`sandbox_output_publish_forbidden: ${response.message ?? response.code ?? "publication grant redemption was rejected"}`);
|
|
1057
|
+
}
|
|
1058
|
+
return response.credential;
|
|
1059
|
+
}
|
|
1060
|
+
async handleLeaseUpdate(request) {
|
|
1061
|
+
if (request.runnerInstanceId !== this.config.runnerInstanceId) {
|
|
1062
|
+
return;
|
|
1063
|
+
}
|
|
1064
|
+
const reservation = this.reservations.get(request.dispatchAttemptId);
|
|
1065
|
+
if (reservation && reservation.leaseId === request.leaseId && reservation.fencingToken === request.fencingToken) {
|
|
1066
|
+
reservation.expiresAt = request.expiresAt;
|
|
1067
|
+
}
|
|
1068
|
+
const active = this.activeSessions.get(request.leaseId);
|
|
1069
|
+
if (active && active.fencingToken === request.fencingToken && active.dispatchAttemptId === request.dispatchAttemptId) {
|
|
1070
|
+
active.leaseExpiresAt = request.expiresAt;
|
|
1071
|
+
if (active.sandboxed) {
|
|
1072
|
+
await this.sandboxRuntime.updateLease(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
1073
|
+
}
|
|
1074
|
+
else {
|
|
1075
|
+
await this.sessions.updateLease?.(request.dispatchAttemptId, request.fencingToken, request.expiresAt);
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
async sendOfferDecision(dispatch, accepted, code = null, message = null, reservedUntil = null, reservedResources = null, sandboxConfigCommitSha = null) {
|
|
1080
|
+
if (!dispatch.sandbox)
|
|
1081
|
+
return;
|
|
1082
|
+
const payload = {
|
|
1083
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
1084
|
+
leaseId: dispatch.lease.id,
|
|
1085
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
1086
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId,
|
|
1087
|
+
accepted,
|
|
1088
|
+
code,
|
|
1089
|
+
message,
|
|
1090
|
+
reservedUntil: accepted ? reservedUntil ?? dispatch.lease.expiresAt : null,
|
|
1091
|
+
reservedResources: accepted ? reservedResources : null,
|
|
1092
|
+
sandboxConfigCommitSha: accepted ? sandboxConfigCommitSha : null,
|
|
1093
|
+
};
|
|
1094
|
+
await this.invokeAck("OfferDecision", payload);
|
|
1095
|
+
}
|
|
1096
|
+
async parseDispatchOrReport(rawDispatch) {
|
|
131
1097
|
try {
|
|
132
1098
|
return parseRunOnceDispatchPacket(rawDispatch);
|
|
133
1099
|
}
|
|
@@ -137,32 +1103,253 @@ export class ForgeBridgeService {
|
|
|
137
1103
|
return null;
|
|
138
1104
|
}
|
|
139
1105
|
const summary = invalidDispatchSummary(error);
|
|
140
|
-
|
|
1106
|
+
await this.sendLegacyResult(identifiers.leaseId, identifiers.fencingToken, failureResult(summary, [
|
|
141
1107
|
{
|
|
142
1108
|
role: "runner",
|
|
143
1109
|
kind: "error",
|
|
144
1110
|
content: summary,
|
|
145
1111
|
eventType: "invalid_dispatch"
|
|
146
1112
|
}
|
|
147
|
-
])).catch(() => undefined);
|
|
1113
|
+
], failureFactsForCode("invalid_dispatch"))).catch(() => undefined);
|
|
1114
|
+
return null;
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
parseChatTurnOrReport(rawDispatch) {
|
|
1118
|
+
try {
|
|
1119
|
+
return parseChatTurnDispatchPacket(rawDispatch, this.bridgeChatCapability());
|
|
1120
|
+
}
|
|
1121
|
+
catch (error) {
|
|
1122
|
+
logBridgeStatus(invalidChatTurnSummary(error));
|
|
148
1123
|
return null;
|
|
149
1124
|
}
|
|
150
1125
|
}
|
|
151
1126
|
async handleCancel(request) {
|
|
1127
|
+
if (request.dispatchAttemptId !== undefined && request.dispatchAttemptId !== null) {
|
|
1128
|
+
// Qualified cancellation is reservation-only. In particular, a delayed
|
|
1129
|
+
// pre-start Cancel must not abort an active session if Start won the race.
|
|
1130
|
+
const reservation = this.reservations.get(request.dispatchAttemptId);
|
|
1131
|
+
if (reservation
|
|
1132
|
+
&& request.runnerId === this.config.runnerId
|
|
1133
|
+
&& request.runnerInstanceId === this.config.runnerInstanceId
|
|
1134
|
+
&& reservation.runnerInstanceId === request.runnerInstanceId
|
|
1135
|
+
&& reservation.leaseId === request.leaseId
|
|
1136
|
+
&& reservation.fencingToken === request.fencingToken) {
|
|
1137
|
+
reservation.startClaim?.controller.abort(request.reason ?? "cancelled by server");
|
|
1138
|
+
this.reservations.delete(request.dispatchAttemptId);
|
|
1139
|
+
}
|
|
1140
|
+
return;
|
|
1141
|
+
}
|
|
152
1142
|
const active = this.activeSessions.get(request.leaseId);
|
|
153
1143
|
if (!active || active.fencingToken !== request.fencingToken) {
|
|
154
1144
|
return;
|
|
155
1145
|
}
|
|
1146
|
+
if (request.runnerId && request.runnerId !== this.config.runnerId) {
|
|
1147
|
+
return;
|
|
1148
|
+
}
|
|
1149
|
+
if (request.runnerInstanceId && request.runnerInstanceId !== (active.runnerInstanceId ?? this.config.runnerInstanceId)) {
|
|
1150
|
+
return;
|
|
1151
|
+
}
|
|
1152
|
+
if (active.kind === "chat" && !this.chatCancelMatchesActiveTurn(active, request)) {
|
|
1153
|
+
return;
|
|
1154
|
+
}
|
|
156
1155
|
active.controller.abort(request.reason ?? "cancelled by server");
|
|
157
1156
|
}
|
|
1157
|
+
chatCancelMatchesActiveTurn(active, request) {
|
|
1158
|
+
if (request.protocolVersion && request.protocolVersion !== bridgeChatProtocolVersion) {
|
|
1159
|
+
return false;
|
|
1160
|
+
}
|
|
1161
|
+
try {
|
|
1162
|
+
assertBridgeChatCallbackPayload(request.payload, "cancel payload");
|
|
1163
|
+
}
|
|
1164
|
+
catch {
|
|
1165
|
+
return false;
|
|
1166
|
+
}
|
|
1167
|
+
if (request.chatRunId && request.chatRunId !== active.chatRunId)
|
|
1168
|
+
return false;
|
|
1169
|
+
if (request.conversationId && request.conversationId !== active.conversationId)
|
|
1170
|
+
return false;
|
|
1171
|
+
if (request.turnId && request.turnId !== active.turnId)
|
|
1172
|
+
return false;
|
|
1173
|
+
if (request.providerThreadId && active.providerThreadId && request.providerThreadId !== active.providerThreadId) {
|
|
1174
|
+
return false;
|
|
1175
|
+
}
|
|
1176
|
+
return true;
|
|
1177
|
+
}
|
|
1178
|
+
async handleReap(request) {
|
|
1179
|
+
if (request.runnerId !== this.config.runnerId
|
|
1180
|
+
|| request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
1181
|
+
return;
|
|
1182
|
+
await this.runSandboxMaintenance();
|
|
1183
|
+
await this.sendHeartbeat();
|
|
1184
|
+
}
|
|
1185
|
+
async handleReadinessProbe(rawRequest) {
|
|
1186
|
+
const request = parseReadinessProbe(rawRequest);
|
|
1187
|
+
if (!request || request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
1188
|
+
return;
|
|
1189
|
+
const existing = this.activeReadinessProbes.get(request.requestId);
|
|
1190
|
+
if (existing)
|
|
1191
|
+
return;
|
|
1192
|
+
const controller = new AbortController();
|
|
1193
|
+
this.activeReadinessProbes.set(request.requestId, controller);
|
|
1194
|
+
const deadlineMs = Date.parse(request.expiresAt) - Date.now();
|
|
1195
|
+
const timer = setTimeout(() => controller.abort("readiness probe expired"), Math.max(0, deadlineMs));
|
|
1196
|
+
timer.unref?.();
|
|
1197
|
+
let result;
|
|
1198
|
+
try {
|
|
1199
|
+
const accepted = {
|
|
1200
|
+
schemaVersion: 1,
|
|
1201
|
+
requestId: request.requestId,
|
|
1202
|
+
fencingToken: request.fencingToken,
|
|
1203
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1204
|
+
acceptedAt: new Date().toISOString(),
|
|
1205
|
+
};
|
|
1206
|
+
await this.invokeAck("ReadinessAccepted", accepted);
|
|
1207
|
+
if (this.activeSessions.size > 0) {
|
|
1208
|
+
result = deferredReadinessResult(request, this.config.runnerInstanceId);
|
|
1209
|
+
}
|
|
1210
|
+
else if (!this.sessions.probeHostRuntime) {
|
|
1211
|
+
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
1212
|
+
}
|
|
1213
|
+
else {
|
|
1214
|
+
const probe = await this.sessions.probeHostRuntime(this.normalizeReadinessProbe(request), controller.signal);
|
|
1215
|
+
result = readinessDeferralReason(controller.signal)
|
|
1216
|
+
? deferredReadinessResult(request, this.config.runnerInstanceId)
|
|
1217
|
+
: {
|
|
1218
|
+
schemaVersion: 1,
|
|
1219
|
+
requestId: request.requestId,
|
|
1220
|
+
fencingToken: request.fencingToken,
|
|
1221
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1222
|
+
...probe,
|
|
1223
|
+
};
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
catch (error) {
|
|
1227
|
+
const deferredReason = readinessDeferralReason(controller.signal);
|
|
1228
|
+
result = {
|
|
1229
|
+
schemaVersion: 1,
|
|
1230
|
+
requestId: request.requestId,
|
|
1231
|
+
fencingToken: request.fencingToken,
|
|
1232
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1233
|
+
status: "failed",
|
|
1234
|
+
reasonCode: deferredReason ? "host_runtime_probe_deferred" : "host_runtime_probe_failed",
|
|
1235
|
+
reasonMessage: (deferredReason ?? errorMessage(error)).slice(0, 1000),
|
|
1236
|
+
checkedAt: new Date().toISOString(),
|
|
1237
|
+
};
|
|
1238
|
+
}
|
|
1239
|
+
finally {
|
|
1240
|
+
clearTimeout(timer);
|
|
1241
|
+
this.activeReadinessProbes.delete(request.requestId);
|
|
1242
|
+
}
|
|
1243
|
+
try {
|
|
1244
|
+
await this.readinessResultOutbox.persist(result, request.expiresAt);
|
|
1245
|
+
}
|
|
1246
|
+
catch (error) {
|
|
1247
|
+
if (errorMessage(error).includes("runtime_readiness_result_conflict:")) {
|
|
1248
|
+
logBridgeStatus(`Readiness result ${request.requestId} rejected conflicting reuse of a durable request identity.`);
|
|
1249
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
1250
|
+
return;
|
|
1251
|
+
}
|
|
1252
|
+
logBridgeStatus(`Readiness result ${request.requestId} could not be persisted; attempting direct delivery: ${errorMessage(error)}`);
|
|
1253
|
+
await this.invokeAck("ReadinessResult", result).catch(deliveryError => {
|
|
1254
|
+
logBridgeStatus(`Readiness result ${request.requestId} could not be persisted or delivered: ${errorMessage(deliveryError)}`);
|
|
1255
|
+
});
|
|
1256
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
1257
|
+
return;
|
|
1258
|
+
}
|
|
1259
|
+
try {
|
|
1260
|
+
await this.flushReadinessResults();
|
|
1261
|
+
}
|
|
1262
|
+
catch (error) {
|
|
1263
|
+
logBridgeStatus(`Readiness result ${request.requestId} remains queued for replay: ${errorMessage(error)}`);
|
|
1264
|
+
}
|
|
1265
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
1266
|
+
}
|
|
1267
|
+
deferActiveReadinessProbes() {
|
|
1268
|
+
for (const controller of this.activeReadinessProbes.values()) {
|
|
1269
|
+
if (!controller.signal.aborted) {
|
|
1270
|
+
controller.abort("host_runtime_probe_deferred: foreground agent work took priority over background runtime readiness.");
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1274
|
+
async flushReadinessResults() {
|
|
1275
|
+
if (this.readinessResultFlush)
|
|
1276
|
+
return await this.readinessResultFlush;
|
|
1277
|
+
const flush = this.flushReadinessResultsCore();
|
|
1278
|
+
this.readinessResultFlush = flush;
|
|
1279
|
+
try {
|
|
1280
|
+
await flush;
|
|
1281
|
+
}
|
|
1282
|
+
finally {
|
|
1283
|
+
if (this.readinessResultFlush === flush)
|
|
1284
|
+
this.readinessResultFlush = null;
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
async flushReadinessResultsCore() {
|
|
1288
|
+
for (const pending of await this.readinessResultOutbox.pending()) {
|
|
1289
|
+
if (Date.parse(pending.expiresAt) <= Date.now()) {
|
|
1290
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1291
|
+
continue;
|
|
1292
|
+
}
|
|
1293
|
+
try {
|
|
1294
|
+
await this.invokeAck("ReadinessResult", pending.result);
|
|
1295
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1296
|
+
}
|
|
1297
|
+
catch (error) {
|
|
1298
|
+
if (isReadinessNoLongerPending(error)) {
|
|
1299
|
+
await this.readinessResultOutbox.acknowledge(pending.result.requestId);
|
|
1300
|
+
continue;
|
|
1301
|
+
}
|
|
1302
|
+
throw error;
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
158
1306
|
abortActiveSessions(reason) {
|
|
159
1307
|
for (const active of this.activeSessions.values()) {
|
|
160
1308
|
active.controller.abort(reason);
|
|
161
1309
|
}
|
|
162
1310
|
this.activeSessions.clear();
|
|
163
1311
|
}
|
|
1312
|
+
abortActiveReadinessProbes(reason) {
|
|
1313
|
+
for (const controller of this.activeReadinessProbes.values())
|
|
1314
|
+
controller.abort(reason);
|
|
1315
|
+
this.activeReadinessProbes.clear();
|
|
1316
|
+
}
|
|
1317
|
+
normalizeReadinessProbe(request) {
|
|
1318
|
+
const mapping = findRepoMapping({
|
|
1319
|
+
id: request.repoId,
|
|
1320
|
+
name: request.repoName,
|
|
1321
|
+
remoteUrl: request.remoteUrl,
|
|
1322
|
+
}, repoMappingsFor(this.config));
|
|
1323
|
+
const configuredPath = mapping?.localPath?.trim();
|
|
1324
|
+
if (!configuredPath) {
|
|
1325
|
+
throw new Error("bridge_repo_mapping_missing: readiness requires a matching local repository mapping.");
|
|
1326
|
+
}
|
|
1327
|
+
// Local configuration, not a path supplied by the server, grants checkout
|
|
1328
|
+
// authority. Equality also excludes a different child/sibling of this repo.
|
|
1329
|
+
const suppliedPath = request.runnerRepoLocalPath.trim();
|
|
1330
|
+
if (!path.isAbsolute(suppliedPath)
|
|
1331
|
+
|| (process.platform === "win32" && path.parse(suppliedPath).root.length === 1)) {
|
|
1332
|
+
throw new Error("bridge_repo_mapping_mismatch: readiness requires a fully qualified checkout path.");
|
|
1333
|
+
}
|
|
1334
|
+
const localPath = path.resolve(configuredPath);
|
|
1335
|
+
const requestedPath = path.resolve(suppliedPath);
|
|
1336
|
+
const pathsMatch = process.platform === "win32"
|
|
1337
|
+
? localPath.toLowerCase() === requestedPath.toLowerCase()
|
|
1338
|
+
: localPath === requestedPath;
|
|
1339
|
+
if (!pathsMatch) {
|
|
1340
|
+
throw new Error("bridge_repo_mapping_mismatch: readiness checkout differs from the locally configured repository.");
|
|
1341
|
+
}
|
|
1342
|
+
return {
|
|
1343
|
+
...request,
|
|
1344
|
+
runnerId: this.config.runnerId,
|
|
1345
|
+
runnerRepoLocalPath: localPath,
|
|
1346
|
+
cloneOnDemand: false,
|
|
1347
|
+
repoRoot: path.dirname(localPath),
|
|
1348
|
+
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
1349
|
+
};
|
|
1350
|
+
}
|
|
164
1351
|
normalizeDispatch(dispatch) {
|
|
165
|
-
const mapping = findRepoMapping(dispatch, repoMappingsFor(this.config));
|
|
1352
|
+
const mapping = findRepoMapping(dispatch.repo, repoMappingsFor(this.config));
|
|
166
1353
|
const localPath = mapping?.localPath?.trim();
|
|
167
1354
|
const repoRoot = localPath ? path.dirname(path.resolve(localPath)) : this.config.defaultRepoRoot;
|
|
168
1355
|
const qaLaunchTarget = normalizeQaAppLaunchTarget(mapping?.qaAppLaunchTarget) ?? dispatch.qaBrowser?.launchTarget ?? null;
|
|
@@ -185,47 +1372,290 @@ export class ForgeBridgeService {
|
|
|
185
1372
|
: dispatch.qaBrowser
|
|
186
1373
|
};
|
|
187
1374
|
}
|
|
188
|
-
async sendResult(
|
|
1375
|
+
async sendResult(dispatch, result) {
|
|
189
1376
|
const payload = {
|
|
190
|
-
leaseId,
|
|
191
|
-
fencingToken,
|
|
192
|
-
|
|
1377
|
+
leaseId: dispatch.lease.id,
|
|
1378
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
1379
|
+
dispatchAttemptId: dispatch.sandbox?.dispatchAttemptId ?? null,
|
|
1380
|
+
runnerInstanceId: dispatch.sandbox?.runnerInstanceId ?? null,
|
|
1381
|
+
// Every terminal writer, including early admission failures and exception
|
|
1382
|
+
// paths, crosses the same redaction boundary before outbox persistence.
|
|
1383
|
+
result: redactRunOnceResultPacket(result, exactSecrets(dispatch, this.config))
|
|
193
1384
|
};
|
|
1385
|
+
await this.sendResultPayload(payload, () => this.releaseActiveSessionAndRefreshCapacity(dispatch.lease.id, dispatch));
|
|
1386
|
+
}
|
|
1387
|
+
async sendLegacyResult(leaseId, fencingToken, result) {
|
|
1388
|
+
await this.sendResultPayload({ leaseId, fencingToken, result });
|
|
1389
|
+
}
|
|
1390
|
+
async sendResultPayload(payload, beforeDelivery) {
|
|
1391
|
+
// Register before persistence: replay can observe a record as soon as the
|
|
1392
|
+
// durable write becomes visible, even before persist() has returned.
|
|
1393
|
+
let releaseHandoff;
|
|
1394
|
+
const priorHandoff = this.dispatchResultHandoffs.get(payload.leaseId);
|
|
1395
|
+
const handoff = new Promise(resolve => { releaseHandoff = resolve; });
|
|
1396
|
+
this.dispatchResultHandoffs.set(payload.leaseId, handoff);
|
|
1397
|
+
let persisted = false;
|
|
1398
|
+
try {
|
|
1399
|
+
try {
|
|
1400
|
+
await this.dispatchResultOutbox.persist(payload);
|
|
1401
|
+
persisted = true;
|
|
1402
|
+
this.dispatchResultGeneration += 1;
|
|
1403
|
+
}
|
|
1404
|
+
catch (error) {
|
|
1405
|
+
logBridgeStatus(`Dispatch result ${payload.leaseId} could not be persisted; attempting direct delivery: ${errorMessage(error)}`);
|
|
1406
|
+
}
|
|
1407
|
+
await priorHandoff;
|
|
1408
|
+
await beforeDelivery?.();
|
|
1409
|
+
}
|
|
1410
|
+
finally {
|
|
1411
|
+
releaseHandoff();
|
|
1412
|
+
if (this.dispatchResultHandoffs.get(payload.leaseId) === handoff)
|
|
1413
|
+
this.dispatchResultHandoffs.delete(payload.leaseId);
|
|
1414
|
+
}
|
|
1415
|
+
if (!persisted) {
|
|
1416
|
+
await this.sendResultPayloadDirect(payload);
|
|
1417
|
+
return;
|
|
1418
|
+
}
|
|
1419
|
+
await this.flushDispatchResults().catch(error => {
|
|
1420
|
+
logBridgeStatus(`Dispatch result ${payload.leaseId} remains queued for replay: ${errorMessage(error)}`);
|
|
1421
|
+
});
|
|
1422
|
+
}
|
|
1423
|
+
async sendResultPayloadDirect(payload) {
|
|
194
1424
|
try {
|
|
195
1425
|
await this.invokeAck("DispatchResult", payload);
|
|
196
1426
|
}
|
|
197
|
-
catch (error) {
|
|
198
|
-
if (isDispatchNoLongerPending(error))
|
|
199
|
-
return;
|
|
1427
|
+
catch (error) {
|
|
1428
|
+
if (isDispatchNoLongerPending(error))
|
|
1429
|
+
return;
|
|
1430
|
+
throw error;
|
|
1431
|
+
}
|
|
1432
|
+
}
|
|
1433
|
+
async flushDispatchResults(force = false) {
|
|
1434
|
+
if (!force && this.dispatchResultGeneration === this.dispatchResultFlushedGeneration)
|
|
1435
|
+
return;
|
|
1436
|
+
if (this.dispatchResultFlush) {
|
|
1437
|
+
await this.dispatchResultFlush;
|
|
1438
|
+
if (!force && this.dispatchResultGeneration === this.dispatchResultFlushedGeneration)
|
|
1439
|
+
return;
|
|
1440
|
+
}
|
|
1441
|
+
const flush = this.flushDispatchResultsCore();
|
|
1442
|
+
this.dispatchResultFlush = flush;
|
|
1443
|
+
try {
|
|
1444
|
+
await flush;
|
|
1445
|
+
}
|
|
1446
|
+
finally {
|
|
1447
|
+
if (this.dispatchResultFlush === flush)
|
|
1448
|
+
this.dispatchResultFlush = null;
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
async flushDispatchResultsCore() {
|
|
1452
|
+
const startedGeneration = this.dispatchResultGeneration;
|
|
1453
|
+
while (true) {
|
|
1454
|
+
const records = await this.dispatchResultOutbox.pending();
|
|
1455
|
+
for (const pending of records) {
|
|
1456
|
+
// Periodic heartbeats and reconnect replay share the same release
|
|
1457
|
+
// barrier as the terminal writer. Restart replay has no live handoff.
|
|
1458
|
+
while (this.dispatchResultHandoffs.has(pending.payload.leaseId)) {
|
|
1459
|
+
await this.dispatchResultHandoffs.get(pending.payload.leaseId);
|
|
1460
|
+
}
|
|
1461
|
+
try {
|
|
1462
|
+
await this.sendResultPayloadDirect(pending.payload);
|
|
1463
|
+
await this.dispatchResultOutbox.acknowledge(pending.payload.leaseId);
|
|
1464
|
+
}
|
|
1465
|
+
catch (error) {
|
|
1466
|
+
if (isDispatchNoLongerPending(error)) {
|
|
1467
|
+
await this.dispatchResultOutbox.acknowledge(pending.payload.leaseId);
|
|
1468
|
+
continue;
|
|
1469
|
+
}
|
|
1470
|
+
throw error;
|
|
1471
|
+
}
|
|
200
1472
|
}
|
|
201
|
-
|
|
1473
|
+
if (records.length < 3)
|
|
1474
|
+
break;
|
|
1475
|
+
}
|
|
1476
|
+
if (this.dispatchResultGeneration === startedGeneration) {
|
|
1477
|
+
this.dispatchResultFlushedGeneration = startedGeneration;
|
|
202
1478
|
}
|
|
203
1479
|
}
|
|
204
1480
|
startHeartbeat() {
|
|
205
1481
|
this.stopHeartbeat();
|
|
1482
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
1483
|
+
this.reconnectingFromHeartbeatFailure = false;
|
|
206
1484
|
const intervalMs = positiveFiniteNumber(this.config.heartbeatIntervalMs, 30_000);
|
|
207
1485
|
this.heartbeatTimer = setInterval(() => {
|
|
208
|
-
this.
|
|
1486
|
+
void this.heartbeatTick();
|
|
209
1487
|
}, intervalMs);
|
|
210
1488
|
this.heartbeatTimer.unref?.();
|
|
211
1489
|
}
|
|
1490
|
+
async heartbeatTick() {
|
|
1491
|
+
if (this.heartbeatInFlight || this.reconnectingFromHeartbeatFailure)
|
|
1492
|
+
return;
|
|
1493
|
+
const connection = this.connection;
|
|
1494
|
+
if (!connection)
|
|
1495
|
+
return;
|
|
1496
|
+
this.heartbeatInFlight = true;
|
|
1497
|
+
try {
|
|
1498
|
+
await this.flushDispatchResults();
|
|
1499
|
+
await this.flushReadinessResults();
|
|
1500
|
+
await this.sendHeartbeat();
|
|
1501
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
1502
|
+
}
|
|
1503
|
+
catch (error) {
|
|
1504
|
+
if (connection !== this.connection || this.stopped)
|
|
1505
|
+
return;
|
|
1506
|
+
this.consecutiveHeartbeatFailures += 1;
|
|
1507
|
+
logBridgeStatus(`Heartbeat ${this.consecutiveHeartbeatFailures} failed: ${errorMessage(error)}.`);
|
|
1508
|
+
if (this.consecutiveHeartbeatFailures < 2)
|
|
1509
|
+
return;
|
|
1510
|
+
this.reconnectingFromHeartbeatFailure = true;
|
|
1511
|
+
this.stopHeartbeat();
|
|
1512
|
+
logBridgeStatus("Heartbeat failed twice. Closing the stale connection so the Bridge can reconnect...");
|
|
1513
|
+
await connection.stop().catch(() => undefined);
|
|
1514
|
+
}
|
|
1515
|
+
finally {
|
|
1516
|
+
this.heartbeatInFlight = false;
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1519
|
+
startLeaseWatchdog() {
|
|
1520
|
+
this.stopLeaseWatchdog();
|
|
1521
|
+
this.leaseWatchdogTimer = setInterval(() => this.expireLeases(), 1000);
|
|
1522
|
+
this.leaseWatchdogTimer.unref?.();
|
|
1523
|
+
}
|
|
1524
|
+
stopLeaseWatchdog() {
|
|
1525
|
+
if (this.leaseWatchdogTimer) {
|
|
1526
|
+
clearInterval(this.leaseWatchdogTimer);
|
|
1527
|
+
this.leaseWatchdogTimer = null;
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1530
|
+
expireLeases() {
|
|
1531
|
+
const now = Date.now();
|
|
1532
|
+
this.expireReservations(now);
|
|
1533
|
+
for (const active of this.activeSessions.values()) {
|
|
1534
|
+
if (Date.parse(active.leaseExpiresAt) <= now && !active.controller.signal.aborted) {
|
|
1535
|
+
active.controller.abort("dispatch lease expired");
|
|
1536
|
+
}
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
expireReservations(now) {
|
|
1540
|
+
for (const [attemptId, reservation] of this.reservations) {
|
|
1541
|
+
if (Date.parse(reservation.expiresAt) <= now || Date.parse(reservation.reservedUntil) <= now) {
|
|
1542
|
+
this.reservations.delete(attemptId);
|
|
1543
|
+
}
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
212
1546
|
stopHeartbeat() {
|
|
213
1547
|
if (this.heartbeatTimer) {
|
|
214
1548
|
clearInterval(this.heartbeatTimer);
|
|
215
1549
|
this.heartbeatTimer = null;
|
|
216
1550
|
}
|
|
217
1551
|
}
|
|
1552
|
+
startSandboxMaintenance() {
|
|
1553
|
+
this.stopSandboxMaintenance();
|
|
1554
|
+
this.sandboxMaintenanceTimer = setInterval(() => {
|
|
1555
|
+
this.runSandboxMaintenance().catch(() => undefined);
|
|
1556
|
+
}, 60_000);
|
|
1557
|
+
this.sandboxMaintenanceTimer.unref?.();
|
|
1558
|
+
}
|
|
1559
|
+
stopSandboxMaintenance() {
|
|
1560
|
+
if (this.sandboxMaintenanceTimer) {
|
|
1561
|
+
clearInterval(this.sandboxMaintenanceTimer);
|
|
1562
|
+
this.sandboxMaintenanceTimer = null;
|
|
1563
|
+
}
|
|
1564
|
+
}
|
|
1565
|
+
async runSandboxMaintenance() {
|
|
1566
|
+
if (this.sandboxMaintenanceRunning)
|
|
1567
|
+
return;
|
|
1568
|
+
this.sandboxMaintenanceRunning = true;
|
|
1569
|
+
try {
|
|
1570
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
1571
|
+
runnerId: this.config.runnerId,
|
|
1572
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1573
|
+
});
|
|
1574
|
+
this.rememberRecoveredHostAttempts(hostReap);
|
|
1575
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
1576
|
+
logBridgeStatus(`Managed-host maintenance reaped ${hostReap.reaped} stale attempt(s) and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
1577
|
+
}
|
|
1578
|
+
if (this.config.dockerEnabled) {
|
|
1579
|
+
const preflight = await this.preflight.refresh();
|
|
1580
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
1581
|
+
const attemptSnapshot = await this.readAttemptSnapshot();
|
|
1582
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow, attemptSnapshot });
|
|
1583
|
+
await this.flushCleanupReports();
|
|
1584
|
+
}
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
finally {
|
|
1588
|
+
this.sandboxMaintenanceRunning = false;
|
|
1589
|
+
}
|
|
1590
|
+
}
|
|
1591
|
+
async readAttemptSnapshot() {
|
|
1592
|
+
const connection = this.connection;
|
|
1593
|
+
if (!connection)
|
|
1594
|
+
throw new Error("Bridge is not connected.");
|
|
1595
|
+
const response = await connection.invoke("AttemptSnapshot", {
|
|
1596
|
+
runnerInstanceId: this.config.runnerInstanceId
|
|
1597
|
+
});
|
|
1598
|
+
if (!response.accepted) {
|
|
1599
|
+
throw new BridgeCallbackRejectedError(response.code ?? null, response.message ?? response.code ?? "Attempt snapshot was rejected.");
|
|
1600
|
+
}
|
|
1601
|
+
return response.attempts ?? [];
|
|
1602
|
+
}
|
|
1603
|
+
async flushCleanupReports() {
|
|
1604
|
+
if (!this.sandboxRuntime.pendingCleanupReports || !this.sandboxRuntime.acknowledgeCleanupReport)
|
|
1605
|
+
return;
|
|
1606
|
+
for (const report of await this.sandboxRuntime.pendingCleanupReports()) {
|
|
1607
|
+
await this.invokeAck("CleanupReport", {
|
|
1608
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1609
|
+
report
|
|
1610
|
+
});
|
|
1611
|
+
await this.sandboxRuntime.acknowledgeCleanupReport(report.reportId);
|
|
1612
|
+
}
|
|
1613
|
+
}
|
|
218
1614
|
async sendHeartbeat() {
|
|
1615
|
+
// Heartbeats are the recovery path for capacity-blocked work. Refresh the
|
|
1616
|
+
// dynamic sample before advertising it, while reusing the static Docker
|
|
1617
|
+
// capability snapshot established at startup or by an offer.
|
|
1618
|
+
if (this.config.dockerEnabled)
|
|
1619
|
+
await this.preflight.refreshCapacity();
|
|
1620
|
+
// A due probe can start a Docker Desktop recovery that runs for minutes.
|
|
1621
|
+
// Wait only as long as a normal probe takes, then advertise the last
|
|
1622
|
+
// snapshot so the server doesn't mark the runner offline meanwhile.
|
|
1623
|
+
const probeWait = new AbortController();
|
|
1624
|
+
await Promise.race([
|
|
1625
|
+
this.dockerEngineProbe.refresh().catch(() => undefined),
|
|
1626
|
+
delay(positiveFiniteNumber(this.config.heartbeatDockerProbeWaitMs ?? 15_000, 15_000), probeWait.signal),
|
|
1627
|
+
]);
|
|
1628
|
+
probeWait.abort();
|
|
1629
|
+
await this.sendHeartbeatSnapshot();
|
|
1630
|
+
}
|
|
1631
|
+
async sendHeartbeatSnapshot(signal) {
|
|
219
1632
|
await this.invokeAck("Heartbeat", {
|
|
220
1633
|
status: "online",
|
|
221
1634
|
capabilities: this.capabilities(),
|
|
222
1635
|
version: this.config.version,
|
|
223
1636
|
protocolVersion: this.config.protocolVersion
|
|
224
|
-
});
|
|
1637
|
+
}, { signal });
|
|
225
1638
|
}
|
|
226
1639
|
capabilities() {
|
|
1640
|
+
const preflight = this.preflight.current();
|
|
1641
|
+
const dockerEngine = this.dockerEngineProbe.current();
|
|
1642
|
+
const runtimeState = this.sandboxRuntime.snapshot();
|
|
1643
|
+
const reserved = [...this.reservations.values()].filter(reservation => reservation.sandboxed).length;
|
|
1644
|
+
const used = this.usedSandboxResources(runtimeState.degraded);
|
|
1645
|
+
const activeResources = this.activeSandboxResources(runtimeState.degraded);
|
|
1646
|
+
const capacity = this.sandboxResourceCapacity(preflight, activeResources);
|
|
1647
|
+
const memoryAdmission = preflight ? buildMemoryAdmissionDiagnostics({
|
|
1648
|
+
sampledAt: preflight.sampledAt,
|
|
1649
|
+
totalMemoryBytes: preflight.totalMemoryBytes,
|
|
1650
|
+
rawAvailableMemoryBytes: preflight.rawAvailableMemoryBytes,
|
|
1651
|
+
stabilizedAvailableMemoryBytes: preflight.availableMemoryBytes,
|
|
1652
|
+
controlPlaneMemoryBytes: process.memoryUsage().rss,
|
|
1653
|
+
reserveMemoryBytes: preflight.hostReserveBytes,
|
|
1654
|
+
committedMemoryBytes: used.memoryBytes,
|
|
1655
|
+
pendingCommitmentMemoryBytes: Math.max(0, used.memoryBytes - activeResources.memoryBytes),
|
|
1656
|
+
}) : null;
|
|
227
1657
|
return {
|
|
228
|
-
schemaVersion:
|
|
1658
|
+
schemaVersion: 6,
|
|
229
1659
|
activeSessionCount: this.activeSessions.size,
|
|
230
1660
|
cliProviders: this.config.cliProviders,
|
|
231
1661
|
providerStatuses: this.config.providerStatuses,
|
|
@@ -233,32 +1663,208 @@ export class ForgeBridgeService {
|
|
|
233
1663
|
repoMappings: repoMappingsFor(this.config),
|
|
234
1664
|
os: this.config.os,
|
|
235
1665
|
sandboxProfiles: this.config.sandboxProfiles,
|
|
1666
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
1667
|
+
trustDomainId: this.config.trustDomainId,
|
|
1668
|
+
executionFacilities: dockerEngine?.available ? ["docker_engine"] : [],
|
|
1669
|
+
dockerEngine,
|
|
1670
|
+
toolchains: {
|
|
1671
|
+
...this.hostCapabilities.toolchains,
|
|
1672
|
+
docker: {
|
|
1673
|
+
available: dockerEngine?.available === true,
|
|
1674
|
+
executable: this.config.dockerExecutable,
|
|
1675
|
+
version: dockerEngine?.runtimeVersion ?? null,
|
|
1676
|
+
},
|
|
1677
|
+
},
|
|
1678
|
+
playwrightProjects: this.hostCapabilities.playwrightProjects,
|
|
1679
|
+
browserLaunch: this.hostCapabilities.browserLaunch,
|
|
1680
|
+
providerConnectivity: this.hostCapabilities.providerConnectivity,
|
|
1681
|
+
maxConcurrentSandboxes: capacity.sandboxSlots,
|
|
1682
|
+
reservedSandboxCount: reserved,
|
|
1683
|
+
activeSandboxCount: runtimeState.active,
|
|
1684
|
+
startingSandboxCount: runtimeState.starting,
|
|
1685
|
+
degradedReservationCount: runtimeState.degraded,
|
|
1686
|
+
sandboxRuntimes: preflight?.healthy ? ["docker"] : [],
|
|
1687
|
+
sandboxRuntimeVersion: preflight?.runtimeVersion ?? null,
|
|
1688
|
+
sandboxFeatures: preflight?.features ?? null,
|
|
1689
|
+
allowsOpenPublicSandbox: this.config.allowOpenPublicSandbox,
|
|
1690
|
+
sandboxPublicationModes: [this.config.sandboxPublicationMode ?? "shared_filesystem"],
|
|
1691
|
+
sandboxAuthModes: sandboxAuthModesFor(this.config),
|
|
1692
|
+
availableMemoryBytes: memoryAdmission?.availableForAdmissionBytes ?? null,
|
|
1693
|
+
availableCpuMillis: preflight ? Math.max(0, capacity.cpuMillis - used.cpuMillis) : null,
|
|
1694
|
+
availableWorkspaceBytes: preflight ? Math.max(0, capacity.workspaceBytes - used.workspaceBytes) : null,
|
|
1695
|
+
diskLow: preflight?.diskLow ?? null,
|
|
1696
|
+
preflightGeneration: preflight?.generation ?? 0,
|
|
1697
|
+
sandboxUnavailableCode: this.config.dockerEnabled ? preflight?.unavailableCode ?? null : "docker_disabled",
|
|
1698
|
+
sandboxUnavailableReason: this.config.dockerEnabled ? preflight?.unavailableReason ?? null : "Docker sandboxing is disabled by runner configuration.",
|
|
1699
|
+
sandboxMemoryCapacity: preflight ? {
|
|
1700
|
+
defaultWorkerMemoryLimitBytes: this.config.defaultSandboxWorkerMemoryLimitBytes,
|
|
1701
|
+
defaultServiceMemoryLimitBytes: this.config.defaultSandboxServiceMemoryLimitBytes,
|
|
1702
|
+
runtimeOverheadBytes: this.config.sandboxRuntimeOverheadBytes,
|
|
1703
|
+
hostReserveBytes: preflight.hostReserveBytes,
|
|
1704
|
+
activeReservationBytes: used.memoryBytes,
|
|
1705
|
+
daemonTotalBytes: preflight.totalMemoryBytes,
|
|
1706
|
+
daemonAvailableBytes: preflight.rawDaemonAvailableMemoryBytes ?? 0,
|
|
1707
|
+
hostAvailableBytes: preflight.rawHostAvailableMemoryBytes,
|
|
1708
|
+
conservativeAvailableBytes: preflight.rawAvailableMemoryBytes,
|
|
1709
|
+
availableForAdmissionBytes: memoryAdmission.availableForAdmissionBytes,
|
|
1710
|
+
memoryAdmission: memoryAdmission,
|
|
1711
|
+
} : null,
|
|
1712
|
+
hostRuntime: this.sessions.runtimeCapacity?.() ?? null,
|
|
236
1713
|
supportsApiSecretGrant: this.config.supportsApiSecretGrant,
|
|
237
1714
|
supportsManagedWorktrees: this.config.supportsManagedWorktrees,
|
|
238
1715
|
supportsManagedForgeArtifacts: this.config.supportsManagedForgeArtifacts,
|
|
239
1716
|
supportsRunnerAuthoredPhaseArtifacts: this.config.supportsRunnerAuthoredPhaseArtifacts,
|
|
240
|
-
supportsDraftRunOnce: this.config.supportsDraftRunOnce
|
|
1717
|
+
supportsDraftRunOnce: this.config.supportsDraftRunOnce,
|
|
1718
|
+
supportsDraftAttachments: this.config.supportsDraftAttachments,
|
|
1719
|
+
supportsDraftImageAttachments: this.config.supportsDraftImageAttachments,
|
|
1720
|
+
bridgeChat: bridgeChatCapabilityFor(this.config.cliProviders, this.config.providerStatuses, this.config.bridgeChat)
|
|
241
1721
|
};
|
|
242
1722
|
}
|
|
243
|
-
async
|
|
244
|
-
const
|
|
245
|
-
if (!
|
|
246
|
-
|
|
1723
|
+
async requiredCapabilityFailure(dispatch) {
|
|
1724
|
+
const requiredCapabilities = dispatch.qaScope?.required_capabilities ?? [];
|
|
1725
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
1726
|
+
return null;
|
|
1727
|
+
const executionMode = dispatch.sandbox?.executionMode;
|
|
1728
|
+
if (executionMode === "docker") {
|
|
1729
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, null);
|
|
1730
|
+
}
|
|
1731
|
+
return dockerEngineRequirementFailure(requiredCapabilities, executionMode, await this.dockerEngineProbe.refresh(true));
|
|
1732
|
+
}
|
|
1733
|
+
usedSandboxResources(degradedReservations) {
|
|
1734
|
+
const resources = [
|
|
1735
|
+
...[...this.reservations.values()]
|
|
1736
|
+
.filter(reservation => reservation.sandboxed)
|
|
1737
|
+
.map(reservation => reservation.resources),
|
|
1738
|
+
...[...this.activeSessions.values()]
|
|
1739
|
+
.filter(session => session.sandboxed)
|
|
1740
|
+
.map(session => session.resources),
|
|
1741
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
1742
|
+
];
|
|
1743
|
+
return sumSandboxResources(resources);
|
|
1744
|
+
}
|
|
1745
|
+
activeSandboxResources(degradedReservations) {
|
|
1746
|
+
return sumSandboxResources([
|
|
1747
|
+
...[...this.activeSessions.values()]
|
|
1748
|
+
.filter(session => session.sandboxed)
|
|
1749
|
+
.map(session => session.resources),
|
|
1750
|
+
...Array.from({ length: degradedReservations }, () => defaultSandboxResources())
|
|
1751
|
+
]);
|
|
1752
|
+
}
|
|
1753
|
+
sandboxResourceCapacity(preflight, activeResources) {
|
|
1754
|
+
if (!preflight)
|
|
1755
|
+
return zeroSandboxResources();
|
|
1756
|
+
return {
|
|
1757
|
+
// Host free memory and disk already reflect active containers. Add their
|
|
1758
|
+
// commitments back to form a stable total capacity, then subtract them
|
|
1759
|
+
// exactly once in usedSandboxResources.
|
|
1760
|
+
sandboxSlots: Math.min(this.config.maxConcurrentSandboxes, preflight.maxConcurrentSandboxes + activeResources.sandboxSlots),
|
|
1761
|
+
cpuMillis: preflight.availableCpuMillis,
|
|
1762
|
+
workerMemoryLimitBytes: 0,
|
|
1763
|
+
serviceMemoryLimitBytes: 0,
|
|
1764
|
+
runtimeOverheadBytes: 0,
|
|
1765
|
+
memoryBytes: sandboxMemoryCommitmentCapacity(preflight, activeResources.memoryBytes),
|
|
1766
|
+
workspaceBytes: Math.max(0, preflight.availableWorkspaceBytes
|
|
1767
|
+
- this.config.minimumWorkspaceBytes
|
|
1768
|
+
+ activeResources.workspaceBytes),
|
|
1769
|
+
pids: this.config.sandboxPidCapacity,
|
|
1770
|
+
buildSlots: this.config.maxConcurrentSandboxBuilds
|
|
1771
|
+
};
|
|
1772
|
+
}
|
|
1773
|
+
async invokeAck(methodName, payload, options = {}) {
|
|
1774
|
+
let storageRecoveryAttempt = 0;
|
|
1775
|
+
let callbackTimeoutAttempt = 0;
|
|
1776
|
+
while (true) {
|
|
1777
|
+
if (options.signal?.aborted) {
|
|
1778
|
+
throw new Error(String(options.signal.reason ?? `${methodName} callback was cancelled.`));
|
|
1779
|
+
}
|
|
1780
|
+
const connection = this.connection;
|
|
1781
|
+
if (!connection) {
|
|
1782
|
+
throw new Error("Bridge is not connected.");
|
|
1783
|
+
}
|
|
1784
|
+
await this.waitForTransportRecovery(connection, this.transportRecovery, options.signal);
|
|
1785
|
+
const recoveryGeneration = this.transportRecoveryGeneration;
|
|
1786
|
+
try {
|
|
1787
|
+
const ack = await waitForCallbackAck(connection.invoke(methodName, payload), methodName, positiveFiniteNumber(this.config.callbackAckTimeoutMs ?? 30_000, 30_000), options.signal);
|
|
1788
|
+
if (!ack.accepted) {
|
|
1789
|
+
throw new BridgeCallbackRejectedError(ack.code ?? null, ack.message ?? ack.code ?? `${methodName} callback was rejected.`);
|
|
1790
|
+
}
|
|
1791
|
+
return ack;
|
|
1792
|
+
}
|
|
1793
|
+
catch (error) {
|
|
1794
|
+
if (options.signal?.aborted)
|
|
1795
|
+
throw error;
|
|
1796
|
+
if (connection !== this.connection || this.stopped)
|
|
1797
|
+
throw error;
|
|
1798
|
+
const recovery = this.transportRecovery;
|
|
1799
|
+
if (recovery && recovery.generation > recoveryGeneration) {
|
|
1800
|
+
await this.waitForTransportRecovery(connection, recovery, options.signal);
|
|
1801
|
+
continue;
|
|
1802
|
+
}
|
|
1803
|
+
if (isProviderStorageUnavailable(error)) {
|
|
1804
|
+
const backoff = this.config.reconnectBackoffMs[Math.min(storageRecoveryAttempt, this.config.reconnectBackoffMs.length - 1)] ?? 5000;
|
|
1805
|
+
storageRecoveryAttempt += 1;
|
|
1806
|
+
logBridgeStatus(`${methodName} reached Forge while storage was recovering. Retrying callback ${storageRecoveryAttempt} in ${formatDelay(backoff)}...`);
|
|
1807
|
+
await delay(backoff, options.signal);
|
|
1808
|
+
continue;
|
|
1809
|
+
}
|
|
1810
|
+
if (options.replayCallbackTimeout && isBridgeCallbackTimeout(error)) {
|
|
1811
|
+
const backoff = this.config.reconnectBackoffMs[Math.min(callbackTimeoutAttempt, this.config.reconnectBackoffMs.length - 1)] ?? 5000;
|
|
1812
|
+
callbackTimeoutAttempt += 1;
|
|
1813
|
+
logBridgeStatus(`${methodName} acknowledgement timed out. Replaying the idempotent callback ${callbackTimeoutAttempt} in ${formatDelay(backoff)}...`);
|
|
1814
|
+
await delay(backoff, options.signal);
|
|
1815
|
+
continue;
|
|
1816
|
+
}
|
|
1817
|
+
throw error;
|
|
1818
|
+
}
|
|
247
1819
|
}
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
1820
|
+
}
|
|
1821
|
+
beginTransportRecovery() {
|
|
1822
|
+
if (this.transportRecovery && !this.transportRecovery.completed)
|
|
1823
|
+
return;
|
|
1824
|
+
let resolve = () => undefined;
|
|
1825
|
+
const promise = new Promise((complete) => { resolve = complete; });
|
|
1826
|
+
this.transportRecoveryGeneration += 1;
|
|
1827
|
+
this.transportRecovery = {
|
|
1828
|
+
generation: this.transportRecoveryGeneration,
|
|
1829
|
+
promise,
|
|
1830
|
+
resolve,
|
|
1831
|
+
completed: false,
|
|
1832
|
+
error: null,
|
|
1833
|
+
};
|
|
1834
|
+
}
|
|
1835
|
+
completeTransportRecovery(error = null) {
|
|
1836
|
+
const recovery = this.transportRecovery;
|
|
1837
|
+
if (!recovery || recovery.completed)
|
|
1838
|
+
return;
|
|
1839
|
+
recovery.completed = true;
|
|
1840
|
+
recovery.error = error;
|
|
1841
|
+
recovery.resolve();
|
|
1842
|
+
}
|
|
1843
|
+
async waitForTransportRecovery(connection, recovery, signal) {
|
|
1844
|
+
if (!recovery)
|
|
1845
|
+
return;
|
|
1846
|
+
if (recovery.completed) {
|
|
1847
|
+
if (recovery.error)
|
|
1848
|
+
throw recovery.error;
|
|
1849
|
+
return;
|
|
251
1850
|
}
|
|
252
|
-
|
|
1851
|
+
await waitForPromiseOrAbort(recovery.promise, signal);
|
|
1852
|
+
signal?.throwIfAborted();
|
|
1853
|
+
if (connection !== this.connection)
|
|
1854
|
+
throw new Error("Bridge connection changed while recovering.");
|
|
1855
|
+
if (recovery.error)
|
|
1856
|
+
throw recovery.error;
|
|
253
1857
|
}
|
|
254
1858
|
async waitForClose(signal) {
|
|
255
1859
|
if (!this.closed) {
|
|
256
1860
|
return;
|
|
257
1861
|
}
|
|
258
|
-
await
|
|
1862
|
+
await waitForPromiseOrAbort(this.closed, signal);
|
|
259
1863
|
}
|
|
260
1864
|
async stopConnection() {
|
|
261
1865
|
this.stopHeartbeat();
|
|
1866
|
+
this.stopLeaseWatchdog();
|
|
1867
|
+
this.stopSandboxMaintenance();
|
|
262
1868
|
if (this.connection) {
|
|
263
1869
|
await this.connection.stop().catch(() => undefined);
|
|
264
1870
|
this.connection = null;
|
|
@@ -279,6 +1885,24 @@ function isDispatchNoLongerPending(error) {
|
|
|
279
1885
|
}
|
|
280
1886
|
return error instanceof Error && error.message === "Dispatch is no longer pending.";
|
|
281
1887
|
}
|
|
1888
|
+
function isReadinessNoLongerPending(error) {
|
|
1889
|
+
return error instanceof BridgeCallbackRejectedError
|
|
1890
|
+
&& (error.code === "bridge.stale_readiness_probe" || error.code === "bridge.stale_runner_instance");
|
|
1891
|
+
}
|
|
1892
|
+
function isProviderStorageUnavailable(error) {
|
|
1893
|
+
const message = errorMessage(error).toLowerCase();
|
|
1894
|
+
return message.includes("provider_storage_unavailable")
|
|
1895
|
+
|| (message.includes("storage is recovering") && message.includes("503"));
|
|
1896
|
+
}
|
|
1897
|
+
function isBridgeCallbackTimeout(error) {
|
|
1898
|
+
return errorMessage(error).startsWith("bridge_callback_timeout:");
|
|
1899
|
+
}
|
|
1900
|
+
function createDefaultSessionService(sandboxStateRoot, sandboxExecutionRoot, dockerEngineProbe) {
|
|
1901
|
+
const rawProcessRunner = new StreamingProviderProcessRunner();
|
|
1902
|
+
const processSupervisors = new AttemptProcessSupervisorRegistry();
|
|
1903
|
+
const coordinatedProcessRunner = new AttemptCoordinatedProviderProcessRunner(processSupervisors, rawProcessRunner);
|
|
1904
|
+
return new BridgeSessionService(new GitWorktreeService(undefined, new RepositoryCoordinatorLock(sandboxStateRoot)), new RoutingProviderCliService(new ProcessBackedProviderCliService(coordinatedProcessRunner, undefined, undefined, undefined, new PrePublicationValidator(undefined, dockerEngineProbe, readNativeTestPostgresFallback))), new HostRuntimeLeaseManager(sandboxStateRoot, undefined, undefined, processSupervisors, rawProcessRunner, undefined, undefined, undefined, undefined, undefined, sandboxExecutionRoot));
|
|
1905
|
+
}
|
|
282
1906
|
export class SignalRBridgeConnectionFactory {
|
|
283
1907
|
create(config) {
|
|
284
1908
|
return new SignalRBridgeConnection(config);
|
|
@@ -293,10 +1917,20 @@ export class SignalRBridgeConnection {
|
|
|
293
1917
|
url.searchParams.set("runnerId", config.runnerId);
|
|
294
1918
|
this.connection = new HubConnectionBuilder()
|
|
295
1919
|
.withUrl(url.toString(), {
|
|
296
|
-
|
|
1920
|
+
// Query-string access tokens are intentionally rejected by the API. The bridge is
|
|
1921
|
+
// a Node CLI, so use long polling where Authorization headers are carried on each
|
|
1922
|
+
// HTTP request instead of relying on browser/WebSocket token fallbacks.
|
|
1923
|
+
transport: HttpTransportType.LongPolling,
|
|
1924
|
+
headers: {
|
|
1925
|
+
Authorization: `Bearer ${config.deviceToken}`
|
|
1926
|
+
}
|
|
297
1927
|
})
|
|
1928
|
+
.withAutomaticReconnect([...config.reconnectBackoffMs])
|
|
298
1929
|
.configureLogging(LogLevel.Warning)
|
|
299
1930
|
.build();
|
|
1931
|
+
this.connection.on("Superseded", () => {
|
|
1932
|
+
void this.connection.stop().catch(() => undefined);
|
|
1933
|
+
});
|
|
300
1934
|
}
|
|
301
1935
|
start() {
|
|
302
1936
|
return this.connection.start();
|
|
@@ -307,16 +1941,54 @@ export class SignalRBridgeConnection {
|
|
|
307
1941
|
onClose(handler) {
|
|
308
1942
|
this.connection.onclose(handler);
|
|
309
1943
|
}
|
|
1944
|
+
onReconnecting(handler) {
|
|
1945
|
+
this.connection.onreconnecting(handler);
|
|
1946
|
+
}
|
|
1947
|
+
onReconnected(handler) {
|
|
1948
|
+
this.connection.onreconnected((connectionId) => {
|
|
1949
|
+
void Promise.resolve(handler(connectionId)).catch(() => undefined);
|
|
1950
|
+
});
|
|
1951
|
+
}
|
|
310
1952
|
onDispatch(handler) {
|
|
311
1953
|
this.connection.on("Dispatch", (raw) => {
|
|
312
1954
|
void handler(raw).catch(() => undefined);
|
|
313
1955
|
});
|
|
314
1956
|
}
|
|
1957
|
+
onChatTurn(handler) {
|
|
1958
|
+
this.connection.on("ChatTurn", (raw) => {
|
|
1959
|
+
void handler(raw).catch(() => undefined);
|
|
1960
|
+
});
|
|
1961
|
+
}
|
|
1962
|
+
onOffer(handler) {
|
|
1963
|
+
this.connection.on("Offer", (raw) => {
|
|
1964
|
+
void handler(raw).catch(() => undefined);
|
|
1965
|
+
});
|
|
1966
|
+
}
|
|
1967
|
+
onStart(handler) {
|
|
1968
|
+
this.connection.on("Start", (raw) => {
|
|
1969
|
+
void handler(raw).catch(() => undefined);
|
|
1970
|
+
});
|
|
1971
|
+
}
|
|
1972
|
+
onLeaseUpdate(handler) {
|
|
1973
|
+
this.connection.on("LeaseUpdate", (request) => {
|
|
1974
|
+
void handler(request).catch(() => undefined);
|
|
1975
|
+
});
|
|
1976
|
+
}
|
|
315
1977
|
onCancel(handler) {
|
|
316
1978
|
this.connection.on("Cancel", (request) => {
|
|
317
1979
|
void handler(request).catch(() => undefined);
|
|
318
1980
|
});
|
|
319
1981
|
}
|
|
1982
|
+
onReap(handler) {
|
|
1983
|
+
this.connection.on("Reap", (request) => {
|
|
1984
|
+
void handler(request).catch(() => undefined);
|
|
1985
|
+
});
|
|
1986
|
+
}
|
|
1987
|
+
onReadinessProbe(handler) {
|
|
1988
|
+
this.connection.on("ReadinessProbe", (request) => {
|
|
1989
|
+
void handler(request).catch(() => undefined);
|
|
1990
|
+
});
|
|
1991
|
+
}
|
|
320
1992
|
invoke(methodName, payload) {
|
|
321
1993
|
return this.connection.invoke(methodName, payload);
|
|
322
1994
|
}
|
|
@@ -324,9 +1996,42 @@ export class SignalRBridgeConnection {
|
|
|
324
1996
|
export function defaultBridgeServiceConfig(overrides = {}) {
|
|
325
1997
|
const defaults = defaultForgeBridgeConfig();
|
|
326
1998
|
const cliProviders = splitCsv(process.env.FORGE_CLI_PROVIDERS ?? "codex");
|
|
1999
|
+
const configuredSandboxHostMemoryReserveBytes = optionalPositiveFiniteNumber(process.env.FORGE_SANDBOX_HOST_MEMORY_RESERVE_BYTES);
|
|
2000
|
+
const sandboxPublicationMode = process.env.FORGE_SANDBOX_PUBLICATION_MODE === "bridge_push" ? "bridge_push" : "shared_filesystem";
|
|
327
2001
|
const config = {
|
|
328
2002
|
serverUrl: process.env.FORGE_URL ?? "http://localhost:5208",
|
|
329
2003
|
runnerId: process.env.FORGE_RUNNER_ID ?? "",
|
|
2004
|
+
runnerInstanceId: process.env.FORGE_RUNNER_INSTANCE_ID ?? randomUUID(),
|
|
2005
|
+
trustDomainId: process.env.FORGE_TRUST_DOMAIN_ID ?? "default",
|
|
2006
|
+
maxConcurrentSandboxes: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOXES ?? 3), 3),
|
|
2007
|
+
maxConcurrentSandboxBuilds: positiveFiniteNumber(Number(process.env.FORGE_MAX_CONCURRENT_SANDBOX_BUILDS ?? 1), 1),
|
|
2008
|
+
sandboxPidCapacity: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_PID_CAPACITY ?? 2048), 2048),
|
|
2009
|
+
dockerEnabled: process.env.FORGE_DOCKER_SANDBOX !== "false",
|
|
2010
|
+
allowOpenPublicSandbox: process.env.FORGE_ALLOW_OPEN_PUBLIC_SANDBOX === "true",
|
|
2011
|
+
sandboxPublicationMode,
|
|
2012
|
+
dockerExecutable: process.env.FORGE_DOCKER_EXECUTABLE ?? "docker",
|
|
2013
|
+
dockerContext: process.env.FORGE_DOCKER_CONTEXT ?? "",
|
|
2014
|
+
dockerDesktopRecoveryEnabled: process.env.FORGE_DOCKER_DESKTOP_AUTO_RECOVERY === "true",
|
|
2015
|
+
dockerRecoveryFailureThreshold: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_FAILURE_THRESHOLD ?? 3), 3),
|
|
2016
|
+
dockerRecoveryCooldownMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_COOLDOWN_MS ?? 10 * 60_000), 10 * 60_000),
|
|
2017
|
+
dockerRecoveryStartupTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_DOCKER_RECOVERY_STARTUP_TIMEOUT_MS ?? 2 * 60_000), 2 * 60_000),
|
|
2018
|
+
sandboxStateRoot: process.env.FORGE_SANDBOX_STATE_ROOT ?? path.join(os.homedir(), ".forge", "bridge", "sandboxes"),
|
|
2019
|
+
sandboxExecutionRoot: defaultHostExecutionRoot(),
|
|
2020
|
+
requireRootlessDocker: process.env.FORGE_REQUIRE_ROOTLESS_DOCKER === "true" || process.platform === "linux",
|
|
2021
|
+
diskHighWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_HIGH_WATERMARK_PERCENT ?? 85), 85),
|
|
2022
|
+
diskLowWatermarkPercent: positiveFiniteNumber(Number(process.env.FORGE_DISK_LOW_WATERMARK_PERCENT ?? 75), 75),
|
|
2023
|
+
minimumWorkspaceBytes: positiveFiniteNumber(Number(process.env.FORGE_MINIMUM_WORKSPACE_BYTES ?? 10 * 1024 ** 3), 10 * 1024 ** 3),
|
|
2024
|
+
// Undefined means preflight derives the bounded reserve from the actual
|
|
2025
|
+
// Docker/host capacity domain after sampling it.
|
|
2026
|
+
sandboxHostMemoryReserveBytes: configuredSandboxHostMemoryReserveBytes,
|
|
2027
|
+
defaultSandboxWorkerMemoryLimitBytes: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_WORKER_MEMORY_LIMIT_BYTES ?? 2 * 1024 ** 3), 2 * 1024 ** 3),
|
|
2028
|
+
defaultSandboxServiceMemoryLimitBytes: nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_DEFAULT_SERVICE_MEMORY_LIMIT_BYTES ?? 0), 0),
|
|
2029
|
+
sandboxRuntimeOverheadBytes: Math.max(128 * 1024 ** 2, nonNegativeFiniteNumber(Number(process.env.FORGE_SANDBOX_RUNTIME_OVERHEAD_BYTES ?? 128 * 1024 ** 2), 128 * 1024 ** 2)),
|
|
2030
|
+
sandboxMemorySampleIntervalMs: Math.min(30_000, Math.max(250, positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_MEMORY_SAMPLE_INTERVAL_MS ?? 1000), 1000))),
|
|
2031
|
+
imageRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_COUNT ?? 3), 3),
|
|
2032
|
+
imageRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_IMAGE_RETENTION_TTL_MS ?? 7 * 24 * 60 * 60 * 1000), 7 * 24 * 60 * 60 * 1000),
|
|
2033
|
+
auditRetentionCount: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_COUNT ?? 200), 200),
|
|
2034
|
+
auditRetentionTtlMs: positiveFiniteNumber(Number(process.env.FORGE_SANDBOX_AUDIT_RETENTION_TTL_MS ?? 30 * 24 * 60 * 60 * 1000), 30 * 24 * 60 * 60 * 1000),
|
|
330
2035
|
deviceToken: process.env.FORGE_DEVICE_TOKEN ?? "",
|
|
331
2036
|
deviceName: process.env.FORGE_DEVICE_NAME ?? "",
|
|
332
2037
|
defaultRepoRoot: process.env.FORGE_REPO_ROOT ?? defaults.defaultRepoRoot,
|
|
@@ -342,17 +2047,25 @@ export function defaultBridgeServiceConfig(overrides = {}) {
|
|
|
342
2047
|
supportsManagedForgeArtifacts: true,
|
|
343
2048
|
supportsRunnerAuthoredPhaseArtifacts: true,
|
|
344
2049
|
supportsDraftRunOnce: true,
|
|
2050
|
+
supportsDraftAttachments: true,
|
|
2051
|
+
supportsDraftImageAttachments: supportsDraftImageAttachmentsFor(cliProviders),
|
|
2052
|
+
bridgeChat: bridgeChatCapabilityFor(cliProviders, providerStatusesFor(cliProviders)),
|
|
345
2053
|
version: process.env.FORGE_BRIDGE_VERSION ?? forgeBridgeVersion,
|
|
346
|
-
protocolVersion:
|
|
2054
|
+
protocolVersion: 4,
|
|
347
2055
|
minServerVersion: null,
|
|
348
2056
|
heartbeatIntervalMs: positiveFiniteNumber(Number(process.env.FORGE_HEARTBEAT_INTERVAL_MS ?? 30_000), 30_000),
|
|
2057
|
+
callbackAckTimeoutMs: positiveFiniteNumber(Number(process.env.FORGE_CALLBACK_ACK_TIMEOUT_MS ?? 30_000), 30_000),
|
|
349
2058
|
reconnectBackoffMs: [1000, 2000, 5000, 10000, 30000],
|
|
350
2059
|
...overrides
|
|
351
2060
|
};
|
|
352
2061
|
return {
|
|
353
2062
|
...config,
|
|
2063
|
+
providerStatuses: overrides.providerStatuses ?? providerStatusesFor(config.cliProviders),
|
|
354
2064
|
sandboxProfiles: config.sandboxProfiles ?? defaultSandboxProfiles(config.cliProviders),
|
|
355
|
-
|
|
2065
|
+
supportsDraftImageAttachments: (overrides.supportsDraftImageAttachments ?? true) && supportsDraftImageAttachmentsFor(config.cliProviders),
|
|
2066
|
+
bridgeChat: overrides.bridgeChat ?? bridgeChatCapabilityFor(config.cliProviders, overrides.providerStatuses ?? providerStatusesFor(config.cliProviders), config.bridgeChat),
|
|
2067
|
+
heartbeatIntervalMs: positiveFiniteNumber(config.heartbeatIntervalMs, 30_000),
|
|
2068
|
+
callbackAckTimeoutMs: positiveFiniteNumber(config.callbackAckTimeoutMs ?? 30_000, 30_000),
|
|
356
2069
|
};
|
|
357
2070
|
}
|
|
358
2071
|
export function resolveBridgeRunnerHubUrl(serverUrl) {
|
|
@@ -367,9 +2080,42 @@ function normalizeServerUrl(serverUrl) {
|
|
|
367
2080
|
function positiveFiniteNumber(value, fallback) {
|
|
368
2081
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
369
2082
|
}
|
|
2083
|
+
function optionalPositiveFiniteNumber(value) {
|
|
2084
|
+
if (value === undefined || value.trim().length === 0)
|
|
2085
|
+
return undefined;
|
|
2086
|
+
const parsed = Number(value);
|
|
2087
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined;
|
|
2088
|
+
}
|
|
2089
|
+
function nonNegativeFiniteNumber(value, fallback) {
|
|
2090
|
+
return Number.isFinite(value) && value >= 0 ? value : fallback;
|
|
2091
|
+
}
|
|
370
2092
|
function providerStatusesFor(providers) {
|
|
371
2093
|
return Object.fromEntries(providers.map((provider) => [provider, { signedIn: true, version: "", models: configuredModelsFor(provider) }]));
|
|
372
2094
|
}
|
|
2095
|
+
function sandboxAuthModesFor(config) {
|
|
2096
|
+
return Object.fromEntries(config.cliProviders.map((provider) => {
|
|
2097
|
+
const modes = ["api_key"];
|
|
2098
|
+
if (sandboxBridgeCliAuthAvailable(config, provider)) {
|
|
2099
|
+
modes.push("bridge_cli");
|
|
2100
|
+
}
|
|
2101
|
+
return [provider, modes];
|
|
2102
|
+
}));
|
|
2103
|
+
}
|
|
2104
|
+
function sandboxBridgeCliAuthAvailable(config, provider) {
|
|
2105
|
+
if (provider.trim().toLowerCase() !== "codex") {
|
|
2106
|
+
return false;
|
|
2107
|
+
}
|
|
2108
|
+
const status = providerStatus(config, provider);
|
|
2109
|
+
if (status?.signedIn === false) {
|
|
2110
|
+
return false;
|
|
2111
|
+
}
|
|
2112
|
+
return codexSandboxProviderAuthAdapter.doctor().available;
|
|
2113
|
+
}
|
|
2114
|
+
function providerStatus(config, provider) {
|
|
2115
|
+
const normalized = provider.trim().toLowerCase();
|
|
2116
|
+
return Object.entries(config.providerStatuses)
|
|
2117
|
+
.find(([key]) => key.trim().toLowerCase() === normalized)?.[1];
|
|
2118
|
+
}
|
|
373
2119
|
function defaultSandboxProfiles(providers, platform = process.platform) {
|
|
374
2120
|
if (!providers.some((provider) => provider.toLowerCase() === "codex")) {
|
|
375
2121
|
return ["none"];
|
|
@@ -444,15 +2190,32 @@ function mappingKeys(mapping) {
|
|
|
444
2190
|
mapping.localPath ? `path:${path.resolve(mapping.localPath).toLowerCase()}` : null
|
|
445
2191
|
].filter((key) => Boolean(key));
|
|
446
2192
|
}
|
|
447
|
-
function findRepoMapping(
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
2193
|
+
function findRepoMapping(repo, mappings) {
|
|
2194
|
+
const usable = mappings.filter(mapping => Boolean(mapping.localPath?.trim()));
|
|
2195
|
+
const selectUnique = (candidates, identity) => {
|
|
2196
|
+
if (candidates.length > 1) {
|
|
2197
|
+
throw new Error(`bridge_repo_mapping_ambiguous: multiple local checkouts match ${identity}.`);
|
|
451
2198
|
}
|
|
452
|
-
return
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
2199
|
+
return candidates[0];
|
|
2200
|
+
};
|
|
2201
|
+
// Repository identity is authoritative. A same-name mapping for a different
|
|
2202
|
+
// repository must never win merely because it appears first in configuration.
|
|
2203
|
+
const byId = usable.filter(mapping => equalsIgnoreCase(mapping.repoId, repo.id));
|
|
2204
|
+
if (byId.length > 0)
|
|
2205
|
+
return selectUnique(byId, `repo id '${repo.id}'`);
|
|
2206
|
+
const byRemote = repo.remoteUrl?.trim()
|
|
2207
|
+
? usable.filter(mapping => !mapping.repoId?.trim()
|
|
2208
|
+
&& Boolean(mapping.remoteUrl?.trim())
|
|
2209
|
+
&& equalsIgnoreCase(mapping.remoteUrl, repo.remoteUrl))
|
|
2210
|
+
: [];
|
|
2211
|
+
if (byRemote.length > 0)
|
|
2212
|
+
return selectUnique(byRemote, `remote '${repo.remoteUrl}'`);
|
|
2213
|
+
const byName = usable.filter(mapping => !mapping.repoId?.trim()
|
|
2214
|
+
&& !mapping.remoteUrl?.trim()
|
|
2215
|
+
&& equalsIgnoreCase(mapping.name, repo.name));
|
|
2216
|
+
return byName.length > 0
|
|
2217
|
+
? selectUnique(byName, `repo name '${repo.name}'`)
|
|
2218
|
+
: undefined;
|
|
456
2219
|
}
|
|
457
2220
|
function normalizeQaAppLaunchTarget(target) {
|
|
458
2221
|
if (!target) {
|
|
@@ -464,10 +2227,11 @@ function normalizeQaAppLaunchTarget(target) {
|
|
|
464
2227
|
command: cleanString(target.command),
|
|
465
2228
|
working_directory: cleanString(target.working_directory),
|
|
466
2229
|
ready_url: cleanString(target.ready_url),
|
|
2230
|
+
managed_runtime_port: cleanString(target.managed_runtime_port),
|
|
467
2231
|
routes,
|
|
468
2232
|
notes: cleanString(target.notes)
|
|
469
2233
|
};
|
|
470
|
-
return normalized.url || normalized.command || normalized.ready_url ||
|
|
2234
|
+
return normalized.url || normalized.command || normalized.ready_url || normalized.managed_runtime_port
|
|
471
2235
|
? normalized
|
|
472
2236
|
: null;
|
|
473
2237
|
}
|
|
@@ -500,6 +2264,10 @@ function invalidDispatchSummary(error) {
|
|
|
500
2264
|
const detail = error instanceof Error ? error.message : String(error);
|
|
501
2265
|
return `Invalid bridge dispatch packet: ${detail}`;
|
|
502
2266
|
}
|
|
2267
|
+
function invalidChatTurnSummary(error) {
|
|
2268
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
2269
|
+
return `Invalid Bridge Chat turn packet: ${detail}`;
|
|
2270
|
+
}
|
|
503
2271
|
function errorMessage(error) {
|
|
504
2272
|
return error instanceof Error ? error.message : String(error);
|
|
505
2273
|
}
|
|
@@ -509,33 +2277,621 @@ function formatDelay(ms) {
|
|
|
509
2277
|
}
|
|
510
2278
|
return `${ms}ms`;
|
|
511
2279
|
}
|
|
2280
|
+
async function waitForCallbackAck(callback, methodName, timeoutMs, signal) {
|
|
2281
|
+
let timeout = null;
|
|
2282
|
+
let rejectAbort = null;
|
|
2283
|
+
const expired = new Promise((_resolve, reject) => {
|
|
2284
|
+
timeout = setTimeout(() => {
|
|
2285
|
+
reject(new Error(`bridge_callback_timeout: ${methodName} callback was not acknowledged within ${formatDelay(timeoutMs)}.`));
|
|
2286
|
+
}, timeoutMs);
|
|
2287
|
+
});
|
|
2288
|
+
const aborted = new Promise((_resolve, reject) => {
|
|
2289
|
+
rejectAbort = reject;
|
|
2290
|
+
});
|
|
2291
|
+
const onAbort = () => rejectAbort?.(new Error(String(signal?.reason ?? `${methodName} callback was cancelled.`)));
|
|
2292
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2293
|
+
if (signal?.aborted)
|
|
2294
|
+
onAbort();
|
|
2295
|
+
try {
|
|
2296
|
+
return await Promise.race(signal ? [callback, expired, aborted] : [callback, expired]);
|
|
2297
|
+
}
|
|
2298
|
+
finally {
|
|
2299
|
+
if (timeout)
|
|
2300
|
+
clearTimeout(timeout);
|
|
2301
|
+
signal?.removeEventListener("abort", onAbort);
|
|
2302
|
+
}
|
|
2303
|
+
}
|
|
512
2304
|
function logBridgeStatus(message) {
|
|
513
2305
|
process.stderr.write(`[${new Date().toISOString()}] ${message}\n`);
|
|
514
2306
|
}
|
|
515
|
-
function delay(ms, signal) {
|
|
2307
|
+
export function delay(ms, signal) {
|
|
516
2308
|
if (signal?.aborted) {
|
|
517
2309
|
return Promise.resolve();
|
|
518
2310
|
}
|
|
519
2311
|
return new Promise((resolve) => {
|
|
520
|
-
const
|
|
521
|
-
signal?.addEventListener("abort", () => {
|
|
2312
|
+
const complete = () => {
|
|
522
2313
|
clearTimeout(timeout);
|
|
2314
|
+
signal?.removeEventListener("abort", complete);
|
|
523
2315
|
resolve();
|
|
524
|
-
}
|
|
2316
|
+
};
|
|
2317
|
+
const timeout = setTimeout(complete, ms);
|
|
2318
|
+
signal?.addEventListener("abort", complete, { once: true });
|
|
2319
|
+
if (signal?.aborted)
|
|
2320
|
+
complete();
|
|
525
2321
|
});
|
|
526
2322
|
}
|
|
527
|
-
function
|
|
2323
|
+
export async function waitForPromiseOrAbort(promise, signal) {
|
|
528
2324
|
if (!signal) {
|
|
529
|
-
|
|
2325
|
+
await promise;
|
|
2326
|
+
return;
|
|
530
2327
|
}
|
|
531
2328
|
if (signal.aborted) {
|
|
532
|
-
return
|
|
2329
|
+
return;
|
|
2330
|
+
}
|
|
2331
|
+
let resolveAbort;
|
|
2332
|
+
const abort = new Promise(resolve => { resolveAbort = resolve; });
|
|
2333
|
+
signal.addEventListener("abort", resolveAbort, { once: true });
|
|
2334
|
+
try {
|
|
2335
|
+
await Promise.race([promise, abort]);
|
|
2336
|
+
}
|
|
2337
|
+
finally {
|
|
2338
|
+
signal.removeEventListener("abort", resolveAbort);
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
// Matches the API Hello bound on settled recovered host attempts.
|
|
2342
|
+
const maxRecoveredHostAttempts = 128;
|
|
2343
|
+
const defaultChatTurnLeaseMs = 60 * 60 * 1000;
|
|
2344
|
+
const maxChatEventContentChars = 4000;
|
|
2345
|
+
const maxChatTerminalTextChars = 120_000;
|
|
2346
|
+
const maxChatExactSecretChars = 8 * 1024;
|
|
2347
|
+
const chatTurnMetadataFileName = "forge-chat-metadata.json";
|
|
2348
|
+
async function materializeChatTurnMetadata(packet, workspacePath) {
|
|
2349
|
+
if (packet.attachmentHandoffs.length === 0 && packet.actionBrokerGrants.length === 0) {
|
|
2350
|
+
return;
|
|
2351
|
+
}
|
|
2352
|
+
await fs.promises.writeFile(path.join(workspacePath, chatTurnMetadataFileName), `${JSON.stringify({
|
|
2353
|
+
schemaVersion: 1,
|
|
2354
|
+
attachmentHandoffs: packet.attachmentHandoffs.map((handoff) => ({
|
|
2355
|
+
turnId: handoff.turnId,
|
|
2356
|
+
attachmentId: handoff.attachmentId,
|
|
2357
|
+
tempFileToken: handoff.tempFileToken,
|
|
2358
|
+
mimeType: handoff.mimeType,
|
|
2359
|
+
sizeBytes: handoff.sizeBytes,
|
|
2360
|
+
sha256: handoff.sha256,
|
|
2361
|
+
fileNameDisplay: handoff.fileNameDisplay,
|
|
2362
|
+
expiresAt: handoff.expiresAt,
|
|
2363
|
+
status: handoff.status,
|
|
2364
|
+
})),
|
|
2365
|
+
actionBrokerGrants: packet.actionBrokerGrants.map((grant) => ({
|
|
2366
|
+
grantId: grant.grantId,
|
|
2367
|
+
turnId: grant.turnId,
|
|
2368
|
+
leaseId: grant.leaseId,
|
|
2369
|
+
fencingToken: grant.fencingToken,
|
|
2370
|
+
allowedScopes: grant.allowedScopes,
|
|
2371
|
+
expiresAt: grant.expiresAt,
|
|
2372
|
+
redactionStatus: grant.redactionStatus,
|
|
2373
|
+
nonceHeldByBridge: true,
|
|
2374
|
+
})),
|
|
2375
|
+
}, null, 2)}\n`, "utf8");
|
|
2376
|
+
}
|
|
2377
|
+
function chatTurnLeaseExpiresAt(packet, nowMs = Date.now()) {
|
|
2378
|
+
const defaultExpiresAtMs = nowMs + defaultChatTurnLeaseMs;
|
|
2379
|
+
const metadataExpiresAtMs = [
|
|
2380
|
+
...packet.attachmentHandoffs.map(handoff => Date.parse(handoff.expiresAt)),
|
|
2381
|
+
...packet.actionBrokerGrants.map(grant => Date.parse(grant.expiresAt)),
|
|
2382
|
+
].filter(Number.isFinite);
|
|
2383
|
+
return new Date(Math.min(defaultExpiresAtMs, ...metadataExpiresAtMs)).toISOString();
|
|
2384
|
+
}
|
|
2385
|
+
async function cleanupChatTurnWorkspace(workspacePath, redactionSecrets) {
|
|
2386
|
+
try {
|
|
2387
|
+
await fs.promises.rm(workspacePath, { recursive: true, force: true, maxRetries: 3 });
|
|
2388
|
+
return {
|
|
2389
|
+
status: "completed",
|
|
2390
|
+
eventType: "chat_turn_cleanup_completed",
|
|
2391
|
+
role: "runner",
|
|
2392
|
+
kind: "checkpoint",
|
|
2393
|
+
content: "Bridge Chat workspace cleanup completed.",
|
|
2394
|
+
};
|
|
2395
|
+
}
|
|
2396
|
+
catch (error) {
|
|
2397
|
+
return {
|
|
2398
|
+
status: "failed",
|
|
2399
|
+
eventType: "chat_turn_cleanup_failed",
|
|
2400
|
+
role: "runner",
|
|
2401
|
+
kind: "error",
|
|
2402
|
+
content: "Bridge Chat workspace cleanup failed.",
|
|
2403
|
+
message: clip(redactText(errorMessage(error), redactionSecrets), 1000),
|
|
2404
|
+
};
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
function buildChatProviderCommand(packet, cwd) {
|
|
2408
|
+
const provider = packet.providerKind.trim().toLowerCase();
|
|
2409
|
+
const model = packet.model.trim();
|
|
2410
|
+
const prompt = buildChatPrompt(packet);
|
|
2411
|
+
const resumeThreadId = packet.providerThreadId?.trim() || null;
|
|
2412
|
+
if (provider === "codex") {
|
|
2413
|
+
const args = ["-s", codexSandboxModeForPlatform(process.platform, "host"), "-a", "never", "exec"];
|
|
2414
|
+
if (resumeThreadId) {
|
|
2415
|
+
args.push("--cd", cwd, "resume");
|
|
2416
|
+
}
|
|
2417
|
+
args.push("--model", model, "--json");
|
|
2418
|
+
if (!resumeThreadId) {
|
|
2419
|
+
args.push("--cd", cwd);
|
|
2420
|
+
}
|
|
2421
|
+
args.push("--skip-git-repo-check");
|
|
2422
|
+
if (resumeThreadId) {
|
|
2423
|
+
args.push(resumeThreadId);
|
|
2424
|
+
}
|
|
2425
|
+
args.push("-");
|
|
2426
|
+
return { command: { command: "codex", args }, stdin: prompt };
|
|
2427
|
+
}
|
|
2428
|
+
if (provider === "cursor") {
|
|
2429
|
+
const args = ["--print", "--output-format", "stream-json", "--model", model, "--workspace", cwd];
|
|
2430
|
+
if (resumeThreadId) {
|
|
2431
|
+
args.push("--resume", resumeThreadId);
|
|
2432
|
+
}
|
|
2433
|
+
args.push("--yolo", "--approve-mcps", "--trust", prompt);
|
|
2434
|
+
return { command: { command: "cursor-agent", args } };
|
|
2435
|
+
}
|
|
2436
|
+
if (provider === "claude") {
|
|
2437
|
+
const args = ["-p", "--output-format", "stream-json", "--verbose", "--model", model, "--permission-mode", "bypassPermissions"];
|
|
2438
|
+
if (resumeThreadId) {
|
|
2439
|
+
args.push("--resume", resumeThreadId);
|
|
2440
|
+
}
|
|
2441
|
+
return { command: { command: "claude", args }, stdin: prompt };
|
|
2442
|
+
}
|
|
2443
|
+
return {
|
|
2444
|
+
command: {
|
|
2445
|
+
command: provider,
|
|
2446
|
+
args: ["--model", model, prompt],
|
|
2447
|
+
},
|
|
2448
|
+
};
|
|
2449
|
+
}
|
|
2450
|
+
function buildChatPrompt(packet) {
|
|
2451
|
+
const messages = packet.messages
|
|
2452
|
+
.map((message, index) => {
|
|
2453
|
+
const role = message.role.trim() || `message-${index + 1}`;
|
|
2454
|
+
return `[${role}]\n${message.content}`;
|
|
2455
|
+
})
|
|
2456
|
+
.join("\n\n");
|
|
2457
|
+
const contextRefs = packet.contextRefs.length === 0
|
|
2458
|
+
? ""
|
|
2459
|
+
: `\n\n# Context References\n\n${packet.contextRefs.map((ref, index) => {
|
|
2460
|
+
const title = ref.title?.trim() ? `\nTitle: ${ref.title.trim()}` : "";
|
|
2461
|
+
const summary = ref.summary?.trim() ? `\nSummary: ${ref.summary.trim()}` : "";
|
|
2462
|
+
return `${index + 1}. ${ref.kind}:${ref.refId}${title}${summary}`;
|
|
2463
|
+
}).join("\n\n")}`;
|
|
2464
|
+
const continuation = packet.providerThreadId
|
|
2465
|
+
? "\n\nContinue the existing provider thread identified by the bridge; do not restart the conversation context."
|
|
2466
|
+
: "";
|
|
2467
|
+
const boundedMetadata = buildChatBoundedMetadataPrompt(packet);
|
|
2468
|
+
return `You are handling a Forge Bridge Chat turn.
|
|
2469
|
+
|
|
2470
|
+
Conversation ID: ${packet.conversationId}
|
|
2471
|
+
Turn ID: ${packet.turnId}
|
|
2472
|
+
Provider: ${packet.providerKind}
|
|
2473
|
+
Model: ${packet.model}${continuation}${boundedMetadata}
|
|
2474
|
+
|
|
2475
|
+
# Conversation
|
|
2476
|
+
|
|
2477
|
+
${messages}${contextRefs}
|
|
2478
|
+
`;
|
|
2479
|
+
}
|
|
2480
|
+
function buildChatBoundedMetadataPrompt(packet) {
|
|
2481
|
+
if (packet.attachmentHandoffs.length === 0 && packet.actionBrokerGrants.length === 0) {
|
|
2482
|
+
return "";
|
|
2483
|
+
}
|
|
2484
|
+
const sections = [
|
|
2485
|
+
`\n\n# Bounded Handoff Metadata\n\nMetadata file: ${chatTurnMetadataFileName}\n\nThe metadata file is turn-scoped and cleaned up by the bridge when the turn ends. It contains opaque handles only; attachment content, raw paths, and grant nonces are not included.`,
|
|
2486
|
+
];
|
|
2487
|
+
if (packet.attachmentHandoffs.length > 0) {
|
|
2488
|
+
sections.push(`Attachment handoffs:\n${packet.attachmentHandoffs.map((handoff, index) => [
|
|
2489
|
+
`${index + 1}. turnId: ${handoff.turnId}`,
|
|
2490
|
+
` attachmentId: ${handoff.attachmentId}`,
|
|
2491
|
+
` tempFileToken: ${handoff.tempFileToken}`,
|
|
2492
|
+
` mimeType: ${handoff.mimeType}`,
|
|
2493
|
+
` sizeBytes: ${handoff.sizeBytes}`,
|
|
2494
|
+
` sha256: ${handoff.sha256}`,
|
|
2495
|
+
` fileNameDisplay: ${handoff.fileNameDisplay}`,
|
|
2496
|
+
` expiresAt: ${handoff.expiresAt}`,
|
|
2497
|
+
` status: ${handoff.status}`,
|
|
2498
|
+
].join("\n")).join("\n\n")}`);
|
|
2499
|
+
}
|
|
2500
|
+
if (packet.actionBrokerGrants.length > 0) {
|
|
2501
|
+
sections.push(`Action-broker grants:\n${packet.actionBrokerGrants.map((grant, index) => [
|
|
2502
|
+
`${index + 1}. grantId: ${grant.grantId}`,
|
|
2503
|
+
` turnId: ${grant.turnId}`,
|
|
2504
|
+
` leaseId: ${grant.leaseId}`,
|
|
2505
|
+
` fencingToken: ${grant.fencingToken}`,
|
|
2506
|
+
` allowedScopes: ${grant.allowedScopes.join(", ")}`,
|
|
2507
|
+
` expiresAt: ${grant.expiresAt}`,
|
|
2508
|
+
` redactionStatus: ${grant.redactionStatus}`,
|
|
2509
|
+
" nonce: held by bridge",
|
|
2510
|
+
].join("\n")).join("\n\n")}`);
|
|
2511
|
+
}
|
|
2512
|
+
return sections.join("\n\n");
|
|
2513
|
+
}
|
|
2514
|
+
function chatProviderExitFailure(command, result, redactionSecrets) {
|
|
2515
|
+
const commandText = redactText(formatProviderCliCommandForCheckpoint(command), redactionSecrets);
|
|
2516
|
+
const stderr = clip(redactText(result.stderr.trim(), redactionSecrets), 1000);
|
|
2517
|
+
const stdout = clip(redactText(result.stdout.trim(), redactionSecrets), 1000);
|
|
2518
|
+
const diagnostics = [
|
|
2519
|
+
stderr ? `stderr: ${stderr}` : null,
|
|
2520
|
+
stdout ? `stdout: ${stdout}` : null,
|
|
2521
|
+
].filter(Boolean).join(" ");
|
|
2522
|
+
return new Error(`provider_failed: ${commandText} exited with code ${result.exitCode}.${diagnostics ? ` ${diagnostics}` : ""}`);
|
|
2523
|
+
}
|
|
2524
|
+
function chatAbortKind(reason) {
|
|
2525
|
+
const message = typeof reason === "string" ? reason.toLowerCase() : "";
|
|
2526
|
+
return message.includes("expired") ? "expired" : "cancelled";
|
|
2527
|
+
}
|
|
2528
|
+
function chatAbortPayload(reason, cleanupOutcome, packet) {
|
|
2529
|
+
const kind = chatAbortKind(reason);
|
|
2530
|
+
const message = typeof reason === "string" && reason.trim()
|
|
2531
|
+
? reason.trim()
|
|
2532
|
+
: (kind === "expired" ? "Bridge Chat turn expired." : "Bridge Chat turn cancelled.");
|
|
2533
|
+
return {
|
|
2534
|
+
code: kind === "expired" ? "chat_turn_expired" : "chat_turn_cancelled",
|
|
2535
|
+
message: clip(redactText(message, packet ? chatRedactionSecrets(packet) : []), 1000),
|
|
2536
|
+
cleanup: cleanupOutcome,
|
|
2537
|
+
};
|
|
2538
|
+
}
|
|
2539
|
+
function providerThreadIdFromEvent(event) {
|
|
2540
|
+
if (!event.data) {
|
|
2541
|
+
return null;
|
|
2542
|
+
}
|
|
2543
|
+
try {
|
|
2544
|
+
const data = JSON.parse(event.data);
|
|
2545
|
+
for (const key of ["threadId", "thread_id", "sessionId", "session_id", "providerThreadId"]) {
|
|
2546
|
+
const value = data[key];
|
|
2547
|
+
if (typeof value === "string") {
|
|
2548
|
+
const clean = value.trim();
|
|
2549
|
+
if (clean.length > 0 && clean.length <= bridgeChatMaxProviderThreadIdChars) {
|
|
2550
|
+
return clean;
|
|
2551
|
+
}
|
|
2552
|
+
}
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
catch {
|
|
2556
|
+
return null;
|
|
2557
|
+
}
|
|
2558
|
+
return null;
|
|
2559
|
+
}
|
|
2560
|
+
function chatRedactionSecrets(packet) {
|
|
2561
|
+
return [
|
|
2562
|
+
packet.chatRunId,
|
|
2563
|
+
packet.conversationId,
|
|
2564
|
+
packet.turnId,
|
|
2565
|
+
packet.leaseId,
|
|
2566
|
+
packet.fencingToken,
|
|
2567
|
+
packet.runnerInstanceId,
|
|
2568
|
+
packet.providerThreadId,
|
|
2569
|
+
...packet.attachmentHandoffs.flatMap((handoff) => [
|
|
2570
|
+
handoff.attachmentId,
|
|
2571
|
+
handoff.tempFileToken,
|
|
2572
|
+
]),
|
|
2573
|
+
...packet.actionBrokerGrants.flatMap((grant) => [
|
|
2574
|
+
grant.grantId,
|
|
2575
|
+
grant.nonce,
|
|
2576
|
+
]),
|
|
2577
|
+
...packet.messages.flatMap((message) => boundedChatSecretFragments(message.content)),
|
|
2578
|
+
...packet.contextRefs.flatMap((ref) => [
|
|
2579
|
+
...boundedChatSecretFragments(ref.title ?? ""),
|
|
2580
|
+
...boundedChatSecretFragments(ref.summary ?? ""),
|
|
2581
|
+
]),
|
|
2582
|
+
].filter((value) => typeof value === "string" && value.length > 0);
|
|
2583
|
+
}
|
|
2584
|
+
function boundedChatSecretFragments(value) {
|
|
2585
|
+
const clean = value.trim();
|
|
2586
|
+
if (!clean) {
|
|
2587
|
+
return [];
|
|
2588
|
+
}
|
|
2589
|
+
if (clean.length <= maxChatExactSecretChars) {
|
|
2590
|
+
return [clean];
|
|
2591
|
+
}
|
|
2592
|
+
const half = Math.floor(maxChatExactSecretChars / 2);
|
|
2593
|
+
return [clean.slice(0, half), clean.slice(clean.length - half)];
|
|
2594
|
+
}
|
|
2595
|
+
function safePathComponent(value) {
|
|
2596
|
+
const clean = value.trim().replace(/[^a-zA-Z0-9_.-]+/gu, "-").replace(/^-+|-+$/gu, "");
|
|
2597
|
+
return clean || "chat-turn";
|
|
2598
|
+
}
|
|
2599
|
+
function containsIgnoreCase(values, value) {
|
|
2600
|
+
const normalized = value.trim().toLowerCase();
|
|
2601
|
+
return values.some((candidate) => candidate.trim().toLowerCase() === normalized);
|
|
2602
|
+
}
|
|
2603
|
+
export function canReapManagedSandboxes(preflight) {
|
|
2604
|
+
return canRunDockerMaintenance(preflight);
|
|
2605
|
+
}
|
|
2606
|
+
function sandboxNetworkPolicyFailure(dispatch, config) {
|
|
2607
|
+
const sandbox = dispatch.sandbox;
|
|
2608
|
+
if (!sandbox)
|
|
2609
|
+
return "Protocol-v2 sandbox context is missing.";
|
|
2610
|
+
if (sandbox.networkPolicy.mode === "open_public"
|
|
2611
|
+
&& (sandbox.executionPolicy === "sandbox_required" || !config.allowOpenPublicSandbox)) {
|
|
2612
|
+
return "open_public sandbox networking is not permitted by this runner policy.";
|
|
2613
|
+
}
|
|
2614
|
+
if (sandbox.networkPolicy.mode === "public_allowlist"
|
|
2615
|
+
&& sandbox.networkPolicy.allowedHosts.length > 0
|
|
2616
|
+
&& process.env.FORGE_SANDBOX_EGRESS_ENFORCED !== "true") {
|
|
2617
|
+
return "public_allowlist requires an operator-enforced Forge egress proxy or firewall.";
|
|
2618
|
+
}
|
|
2619
|
+
return null;
|
|
2620
|
+
}
|
|
2621
|
+
export function dockerEngineRequirementFailure(requiredCapabilities, executionMode, capability) {
|
|
2622
|
+
if (!requiredCapabilities.includes("docker_engine"))
|
|
2623
|
+
return null;
|
|
2624
|
+
if (executionMode === "docker") {
|
|
2625
|
+
return "docker_engine_unavailable: docker_engine is not exposed inside Forge agent sandboxes; use a managed host runner with Docker-daemon access.";
|
|
2626
|
+
}
|
|
2627
|
+
return capability?.available
|
|
2628
|
+
? null
|
|
2629
|
+
: `docker_engine_unavailable: ${capability?.unavailableReason ?? "Docker daemon is unreachable from the bridge process."}`;
|
|
2630
|
+
}
|
|
2631
|
+
function parseReadinessProbe(value) {
|
|
2632
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
2633
|
+
return null;
|
|
2634
|
+
const request = value;
|
|
2635
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
2636
|
+
const bounded = (candidate, maximum) => typeof candidate === "string" && candidate.trim().length > 0 && candidate.length <= maximum;
|
|
2637
|
+
if (request.schemaVersion !== 1
|
|
2638
|
+
|| !uuid.test(request.requestId ?? "")
|
|
2639
|
+
|| !uuid.test(request.fencingToken ?? "")
|
|
2640
|
+
|| !uuid.test(request.userId ?? "")
|
|
2641
|
+
|| !uuid.test(request.projectId ?? "")
|
|
2642
|
+
|| !uuid.test(request.repoId ?? "")
|
|
2643
|
+
|| !uuid.test(request.runnerInstanceId ?? "")
|
|
2644
|
+
|| !Number.isInteger(request.attempt) || (request.attempt ?? 0) < 1
|
|
2645
|
+
|| !bounded(request.repoName, 160)
|
|
2646
|
+
|| typeof request.remoteUrl !== "string" || request.remoteUrl.length > 2048
|
|
2647
|
+
|| !bounded(request.defaultBranch, 512)
|
|
2648
|
+
|| typeof request.cloneOnDemand !== "boolean"
|
|
2649
|
+
|| !bounded(request.runnerRepoLocalPath, 4096)
|
|
2650
|
+
|| !bounded(request.revision, 512)
|
|
2651
|
+
|| !Number.isFinite(Date.parse(request.expiresAt ?? ""))
|
|
2652
|
+
|| Date.parse(request.expiresAt ?? "") <= Date.now()) {
|
|
2653
|
+
return null;
|
|
2654
|
+
}
|
|
2655
|
+
return request;
|
|
2656
|
+
}
|
|
2657
|
+
function deferredReadinessResult(request, runnerInstanceId) {
|
|
2658
|
+
return {
|
|
2659
|
+
schemaVersion: 1,
|
|
2660
|
+
requestId: request.requestId,
|
|
2661
|
+
fencingToken: request.fencingToken,
|
|
2662
|
+
runnerInstanceId,
|
|
2663
|
+
status: "failed",
|
|
2664
|
+
reasonCode: "host_runtime_probe_deferred",
|
|
2665
|
+
reasonMessage: "Foreground agent work is active; Forge deferred background runtime readiness so it cannot consume the agent's host capacity.",
|
|
2666
|
+
checkedAt: new Date().toISOString(),
|
|
2667
|
+
};
|
|
2668
|
+
}
|
|
2669
|
+
function readinessDeferralReason(signal) {
|
|
2670
|
+
if (!signal.aborted)
|
|
2671
|
+
return null;
|
|
2672
|
+
const reason = errorMessage(signal.reason);
|
|
2673
|
+
return reason.startsWith("host_runtime_probe_deferred:") ? reason : null;
|
|
2674
|
+
}
|
|
2675
|
+
function sandboxProviderNetworkFailure(dispatch, config) {
|
|
2676
|
+
const sandbox = dispatch.sandbox;
|
|
2677
|
+
if (!sandbox || sandbox.networkPolicy.mode !== "offline")
|
|
2678
|
+
return null;
|
|
2679
|
+
const provider = dispatch.execution.provider.trim().toLowerCase();
|
|
2680
|
+
const isNetworkProvider = config.cliProviders.some(candidate => candidate.trim().toLowerCase() === provider);
|
|
2681
|
+
if (!isNetworkProvider)
|
|
2682
|
+
return null;
|
|
2683
|
+
return `Provider ${dispatch.execution.provider} requires outbound network access, but this Docker attempt uses the offline network policy.`;
|
|
2684
|
+
}
|
|
2685
|
+
function sandboxAuthFailure(dispatch, config) {
|
|
2686
|
+
const unsupported = sandboxProviderAuthUnavailableMessage(dispatch.execution.provider, dispatch.execution.authKind);
|
|
2687
|
+
if (unsupported) {
|
|
2688
|
+
return unsupported;
|
|
2689
|
+
}
|
|
2690
|
+
if (dispatch.execution.authKind.trim().toLowerCase() !== "bridge_cli") {
|
|
2691
|
+
return null;
|
|
2692
|
+
}
|
|
2693
|
+
const status = providerStatus(config, dispatch.execution.provider);
|
|
2694
|
+
if (status?.signedIn === false) {
|
|
2695
|
+
return `${dispatch.execution.provider} Local CLI is not signed in on this runner.`;
|
|
2696
|
+
}
|
|
2697
|
+
const adapter = sandboxProviderAuthAdapterFor(dispatch.execution.provider, dispatch.execution.authKind);
|
|
2698
|
+
if (!adapter) {
|
|
2699
|
+
return `${dispatch.execution.provider} Local CLI auth is not portable into Docker yet.`;
|
|
2700
|
+
}
|
|
2701
|
+
const doctor = adapter.doctor();
|
|
2702
|
+
return doctor.available ? null : doctor.message;
|
|
2703
|
+
}
|
|
2704
|
+
function rejectedOfferReservation(code, message) {
|
|
2705
|
+
return {
|
|
2706
|
+
accepted: false,
|
|
2707
|
+
code,
|
|
2708
|
+
message,
|
|
2709
|
+
reservedUntil: null,
|
|
2710
|
+
resources: null,
|
|
2711
|
+
};
|
|
2712
|
+
}
|
|
2713
|
+
export function canReserveSandboxResources(used, requested, capacity) {
|
|
2714
|
+
return used.sandboxSlots + requested.sandboxSlots <= capacity.sandboxSlots
|
|
2715
|
+
&& used.cpuMillis + requested.cpuMillis <= capacity.cpuMillis
|
|
2716
|
+
&& used.memoryBytes + requested.memoryBytes <= capacity.memoryBytes
|
|
2717
|
+
&& used.workspaceBytes + requested.workspaceBytes <= capacity.workspaceBytes
|
|
2718
|
+
&& used.pids + requested.pids <= capacity.pids
|
|
2719
|
+
&& used.buildSlots + requested.buildSlots <= capacity.buildSlots;
|
|
2720
|
+
}
|
|
2721
|
+
export function sandboxMemoryCommitmentCapacity(preflight, activeMemoryBytes) {
|
|
2722
|
+
const ambientCapacity = preflight.availableMemoryBytes
|
|
2723
|
+
- preflight.hostReserveBytes
|
|
2724
|
+
+ activeMemoryBytes;
|
|
2725
|
+
const commitmentCapacity = preflight.totalMemoryBytes
|
|
2726
|
+
- preflight.hostReserveBytes;
|
|
2727
|
+
return Math.max(0, Math.min(ambientCapacity, commitmentCapacity));
|
|
2728
|
+
}
|
|
2729
|
+
export function sandboxCapacityFailureMessage(used, requested, capacity, preflight, config) {
|
|
2730
|
+
const availableSlots = Math.max(0, capacity.sandboxSlots - used.sandboxSlots);
|
|
2731
|
+
if (requested.sandboxSlots > availableSlots) {
|
|
2732
|
+
return `Sandbox slot capacity is insufficient: requested ${requested.sandboxSlots}, available ${availableSlots}, ${used.sandboxSlots} already reserved.`;
|
|
2733
|
+
}
|
|
2734
|
+
const availableMemoryBytes = Math.max(0, capacity.memoryBytes - used.memoryBytes);
|
|
2735
|
+
if (requested.memoryBytes > availableMemoryBytes) {
|
|
2736
|
+
return [
|
|
2737
|
+
`Sandbox memory capacity is insufficient: requested ${formatBytes(requested.memoryBytes)} total`,
|
|
2738
|
+
`(worker ${formatBytes(requested.workerMemoryLimitBytes)}, services ${formatBytes(requested.serviceMemoryLimitBytes)}, runtime overhead ${formatBytes(requested.runtimeOverheadBytes)}),`,
|
|
2739
|
+
`but ${formatBytes(availableMemoryBytes)} is available for admission.`,
|
|
2740
|
+
`Docker reports ${formatBytes(preflight?.availableMemoryBytes ?? 0)} available of ${formatBytes(preflight?.totalMemoryBytes ?? 0)};`,
|
|
2741
|
+
`Forge reserves ${formatBytes(preflight?.hostReserveBytes ?? config.sandboxHostMemoryReserveBytes ?? 0)} for the host and tracks ${formatBytes(used.memoryBytes)} in active or reserved attempts.`,
|
|
2742
|
+
].join(" ");
|
|
2743
|
+
}
|
|
2744
|
+
const availableCpuMillis = Math.max(0, capacity.cpuMillis - used.cpuMillis);
|
|
2745
|
+
if (requested.cpuMillis > availableCpuMillis) {
|
|
2746
|
+
return `Sandbox CPU capacity is insufficient: requested ${requested.cpuMillis} millis, available ${availableCpuMillis} millis, ${used.cpuMillis} millis already reserved.`;
|
|
533
2747
|
}
|
|
534
|
-
|
|
2748
|
+
const availableWorkspaceBytes = Math.max(0, capacity.workspaceBytes - used.workspaceBytes);
|
|
2749
|
+
if (requested.workspaceBytes > availableWorkspaceBytes) {
|
|
2750
|
+
return `Sandbox workspace capacity is insufficient: requested ${formatBytes(requested.workspaceBytes)}, available ${formatBytes(availableWorkspaceBytes)}, ${formatBytes(used.workspaceBytes)} already reserved.`;
|
|
2751
|
+
}
|
|
2752
|
+
const availablePids = Math.max(0, capacity.pids - used.pids);
|
|
2753
|
+
if (requested.pids > availablePids) {
|
|
2754
|
+
return `Sandbox PID capacity is insufficient: requested ${requested.pids}, available ${availablePids}, ${used.pids} already reserved.`;
|
|
2755
|
+
}
|
|
2756
|
+
const availableBuildSlots = Math.max(0, capacity.buildSlots - used.buildSlots);
|
|
2757
|
+
return `Sandbox build capacity is insufficient: requested ${requested.buildSlots}, available ${availableBuildSlots}, ${used.buildSlots} already reserved.`;
|
|
2758
|
+
}
|
|
2759
|
+
function sumSandboxResources(resources) {
|
|
2760
|
+
return resources.reduce((total, resource) => ({
|
|
2761
|
+
sandboxSlots: total.sandboxSlots + resource.sandboxSlots,
|
|
2762
|
+
cpuMillis: total.cpuMillis + resource.cpuMillis,
|
|
2763
|
+
workerMemoryLimitBytes: total.workerMemoryLimitBytes + resource.workerMemoryLimitBytes,
|
|
2764
|
+
serviceMemoryLimitBytes: total.serviceMemoryLimitBytes + resource.serviceMemoryLimitBytes,
|
|
2765
|
+
runtimeOverheadBytes: total.runtimeOverheadBytes + resource.runtimeOverheadBytes,
|
|
2766
|
+
memoryBytes: total.memoryBytes + resource.memoryBytes,
|
|
2767
|
+
workspaceBytes: total.workspaceBytes + resource.workspaceBytes,
|
|
2768
|
+
pids: total.pids + resource.pids,
|
|
2769
|
+
buildSlots: total.buildSlots + resource.buildSlots
|
|
2770
|
+
}), zeroSandboxResources());
|
|
2771
|
+
}
|
|
2772
|
+
function zeroSandboxResources() {
|
|
2773
|
+
return {
|
|
2774
|
+
sandboxSlots: 0,
|
|
2775
|
+
cpuMillis: 0,
|
|
2776
|
+
workerMemoryLimitBytes: 0,
|
|
2777
|
+
serviceMemoryLimitBytes: 0,
|
|
2778
|
+
runtimeOverheadBytes: 0,
|
|
2779
|
+
memoryBytes: 0,
|
|
2780
|
+
workspaceBytes: 0,
|
|
2781
|
+
pids: 0,
|
|
2782
|
+
buildSlots: 0,
|
|
2783
|
+
};
|
|
2784
|
+
}
|
|
2785
|
+
function defaultSandboxResources() {
|
|
2786
|
+
return {
|
|
2787
|
+
sandboxSlots: 1,
|
|
2788
|
+
cpuMillis: 2000,
|
|
2789
|
+
workerMemoryLimitBytes: 2 * 1024 ** 3,
|
|
2790
|
+
serviceMemoryLimitBytes: 0,
|
|
2791
|
+
runtimeOverheadBytes: 128 * 1024 ** 2,
|
|
2792
|
+
memoryBytes: 2 * 1024 ** 3 + 128 * 1024 ** 2,
|
|
2793
|
+
workspaceBytes: 10 * 1024 ** 3,
|
|
2794
|
+
pids: 512,
|
|
2795
|
+
buildSlots: 0
|
|
2796
|
+
};
|
|
2797
|
+
}
|
|
2798
|
+
export function resolveDynamicSandboxResources(requested, serviceCount, networkMode, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
2799
|
+
const serviceMemoryLimitBytes = Math.max(0, serviceCount) * 1024 ** 3;
|
|
2800
|
+
const runtimeOverheadBytes = Math.max(128 * 1024 ** 2, baseRuntimeOverheadBytes)
|
|
2801
|
+
+ (networkMode === "public_allowlist" ? 256 * 1024 ** 2 : 0);
|
|
2802
|
+
return {
|
|
2803
|
+
...requested,
|
|
2804
|
+
serviceMemoryLimitBytes,
|
|
2805
|
+
runtimeOverheadBytes,
|
|
2806
|
+
memoryBytes: requested.workerMemoryLimitBytes + serviceMemoryLimitBytes + runtimeOverheadBytes,
|
|
2807
|
+
};
|
|
2808
|
+
}
|
|
2809
|
+
export async function resolveSandboxResourcesFromRepo(dispatch, baseRuntimeOverheadBytes = 128 * 1024 ** 2) {
|
|
2810
|
+
const repoPath = dispatch.repo.serverLocalPath;
|
|
2811
|
+
const sandbox = dispatch.sandbox;
|
|
2812
|
+
if (!repoPath || !sandbox) {
|
|
2813
|
+
throw new Error("Sandbox admission needs a mapped repository.");
|
|
2814
|
+
}
|
|
2815
|
+
const selection = selectSandboxConfigRefs(sandbox.executionPolicy, sandbox.sandboxConfigCommitSha, dispatch.repo.defaultBranch);
|
|
2816
|
+
const configCommitSha = await resolveFirstGitCommit(repoPath, selection.refs);
|
|
2817
|
+
if (!configCommitSha) {
|
|
2818
|
+
throw new Error("Sandbox admission could not resolve the immutable sandbox config commit.");
|
|
2819
|
+
}
|
|
2820
|
+
const contractResult = await runSandboxProcess({
|
|
2821
|
+
command: "git",
|
|
2822
|
+
args: ["-C", repoPath, "show", `${configCommitSha}:.forge/forge.sandbox.yml`],
|
|
2823
|
+
timeoutMs: 30_000,
|
|
2824
|
+
maxOutputBytes: 1024 * 1024,
|
|
2825
|
+
});
|
|
2826
|
+
if (contractResult.exitCode !== 0) {
|
|
2827
|
+
throw new Error("The approved immutable sandbox contract is unavailable in the mapped repository.");
|
|
2828
|
+
}
|
|
2829
|
+
const contract = validateSandboxContract(contractResult.stdout);
|
|
2830
|
+
const workspaceBytes = await estimateWorkspaceBytesFromRepo(repoPath, {
|
|
2831
|
+
role: dispatch.execution.role,
|
|
2832
|
+
serviceCount: contract.contract.services.length,
|
|
2833
|
+
browser: contract.contract.browser,
|
|
2834
|
+
});
|
|
2835
|
+
return {
|
|
2836
|
+
resources: resolveDynamicSandboxResources({
|
|
2837
|
+
...sandbox.requestedResources,
|
|
2838
|
+
workspaceBytes: Math.max(sandbox.requestedResources.workspaceBytes, workspaceBytes),
|
|
2839
|
+
}, contract.contract.services.length, sandbox.networkPolicy.mode, baseRuntimeOverheadBytes),
|
|
2840
|
+
sandboxConfigCommitSha: configCommitSha,
|
|
2841
|
+
};
|
|
2842
|
+
}
|
|
2843
|
+
async function resolveFirstGitCommit(repoPath, refs) {
|
|
2844
|
+
for (const ref of refs) {
|
|
2845
|
+
const result = await runSandboxProcess({
|
|
2846
|
+
command: "git",
|
|
2847
|
+
args: ["-C", repoPath, "rev-parse", "--verify", `${ref}^{commit}`],
|
|
2848
|
+
timeoutMs: 30_000,
|
|
2849
|
+
maxOutputBytes: 1024 * 1024,
|
|
2850
|
+
});
|
|
2851
|
+
const commitSha = result.stdout.trim().toLowerCase();
|
|
2852
|
+
if (result.exitCode === 0 && /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/u.test(commitSha)) {
|
|
2853
|
+
return commitSha;
|
|
2854
|
+
}
|
|
2855
|
+
}
|
|
2856
|
+
return null;
|
|
535
2857
|
}
|
|
2858
|
+
function parseOfferEnvelope(value) {
|
|
2859
|
+
return parseDispatchEnvelope(value);
|
|
2860
|
+
}
|
|
2861
|
+
function parseStartEnvelope(value) {
|
|
2862
|
+
return parseDispatchEnvelope(value);
|
|
2863
|
+
}
|
|
2864
|
+
function parseDispatchEnvelope(value) {
|
|
2865
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
2866
|
+
return null;
|
|
2867
|
+
const envelope = value;
|
|
2868
|
+
if (envelope.schemaVersion !== 1
|
|
2869
|
+
|| typeof envelope.dispatchAttemptId !== "string"
|
|
2870
|
+
|| typeof envelope.leaseId !== "string"
|
|
2871
|
+
|| typeof envelope.fencingToken !== "string"
|
|
2872
|
+
|| typeof envelope.runnerInstanceId !== "string"
|
|
2873
|
+
|| typeof envelope.expiresAt !== "string"
|
|
2874
|
+
|| !envelope.dispatch)
|
|
2875
|
+
return null;
|
|
2876
|
+
return {
|
|
2877
|
+
dispatchAttemptId: envelope.dispatchAttemptId,
|
|
2878
|
+
leaseId: envelope.leaseId,
|
|
2879
|
+
fencingToken: envelope.fencingToken,
|
|
2880
|
+
runnerInstanceId: envelope.runnerInstanceId,
|
|
2881
|
+
expiresAt: envelope.expiresAt,
|
|
2882
|
+
dispatch: envelope.dispatch
|
|
2883
|
+
};
|
|
2884
|
+
}
|
|
2885
|
+
const maxAttachmentExactSecretLength = 8 * 1024;
|
|
536
2886
|
function exactSecrets(dispatch, config) {
|
|
537
2887
|
return [
|
|
538
2888
|
dispatch.apiKeySecretGrant?.secret,
|
|
539
|
-
config.deviceToken
|
|
2889
|
+
config.deviceToken,
|
|
2890
|
+
...boundedAttachmentExactSecrets(dispatch)
|
|
540
2891
|
].filter((secret) => typeof secret === "string" && secret.length > 0);
|
|
541
2892
|
}
|
|
2893
|
+
function boundedAttachmentExactSecrets(dispatch) {
|
|
2894
|
+
return (dispatch.draft?.attachments ?? [])
|
|
2895
|
+
.flatMap((attachment) => [attachment.textContent, attachment.contentBase64])
|
|
2896
|
+
.filter((secret) => typeof secret === "string" && secret.length > 0 && secret.length <= maxAttachmentExactSecretLength);
|
|
2897
|
+
}
|