@loucompanion/forge-bridge 0.1.3-dev.8777774c67bc → 0.1.3-dev.88e93f53db5b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -0
- package/dist/bridge/bridge.base.d.ts +4 -1
- package/dist/bridge/bridge.service.d.ts +50 -3
- package/dist/bridge/bridge.service.js +730 -87
- package/dist/bridge/bridge.types.d.ts +63 -1
- package/dist/cleanup/cleanup.service.d.ts +14 -2
- package/dist/cleanup/cleanup.service.js +496 -3
- package/dist/cleanup/cleanup.types.d.ts +10 -0
- package/dist/cleanup/index.d.ts +1 -1
- package/dist/cleanup/index.js +1 -1
- package/dist/cleanup/internals.d.ts +2 -2
- package/dist/cleanup/internals.js +1 -1
- package/dist/cli/cli.help.js +37 -3
- package/dist/cli/cli.service.d.ts +30 -2
- package/dist/cli/cli.service.js +191 -27
- package/dist/cli/local-config.d.ts +4 -0
- package/dist/cli/local-config.js +41 -1
- package/dist/runtime/aspnet-runtime-prebuild.d.ts +36 -0
- package/dist/runtime/aspnet-runtime-prebuild.js +340 -0
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +17 -0
- package/dist/runtime/attempt-node-runtime-launcher.js +105 -0
- package/dist/runtime/attempt-process-supervisor-registry.d.ts +20 -0
- package/dist/runtime/attempt-process-supervisor-registry.js +64 -0
- package/dist/runtime/attempt-process-supervisor.d.ts +97 -0
- package/dist/runtime/attempt-process-supervisor.js +384 -0
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +103 -0
- package/dist/runtime/host-listener-inspector.d.ts +46 -0
- package/dist/runtime/host-listener-inspector.js +349 -0
- package/dist/runtime/host-port-broker.d.ts +76 -0
- package/dist/runtime/host-port-broker.js +333 -0
- package/dist/runtime/host-runtime-contract.d.ts +47 -0
- package/dist/runtime/host-runtime-contract.js +700 -0
- package/dist/runtime/host-runtime-lease.d.ts +194 -0
- package/dist/runtime/host-runtime-lease.js +2138 -0
- package/dist/runtime/host-runtime-promotion.d.ts +23 -0
- package/dist/runtime/host-runtime-promotion.js +159 -0
- package/dist/runtime/memory-capacity.d.ts +68 -0
- package/dist/runtime/memory-capacity.js +121 -0
- package/dist/runtime/node-host-runtime-adapter.d.ts +54 -0
- package/dist/runtime/node-host-runtime-adapter.js +284 -0
- package/dist/runtime/playwright-host-runtime-adapter.d.ts +21 -0
- package/dist/runtime/playwright-host-runtime-adapter.js +225 -0
- package/dist/runtime/runtime-isolation-lease.d.ts +67 -0
- package/dist/runtime/runtime-isolation-lease.js +158 -0
- package/dist/runtime/workspace-resource-estimator.d.ts +17 -0
- package/dist/runtime/workspace-resource-estimator.js +174 -0
- package/dist/sandbox/image-policy.d.ts +3 -1
- package/dist/sandbox/image-policy.js +4 -2
- package/dist/sandbox/preflight.d.ts +49 -1
- package/dist/sandbox/preflight.js +313 -31
- package/dist/sandbox/process.js +125 -40
- package/dist/sandbox/repository-lock.js +41 -3
- package/dist/sandbox/runtime.d.ts +85 -4
- package/dist/sandbox/runtime.js +806 -126
- package/dist/sandbox/sandbox.types.d.ts +10 -0
- package/dist/sandbox/smoke.d.ts +76 -0
- package/dist/sandbox/smoke.js +380 -0
- package/dist/sandbox/worker-supervisor.js +1 -1
- package/dist/sandbox/worker.js +2 -5
- package/dist/sandbox/workspace.service.d.ts +1 -0
- package/dist/sandbox/workspace.service.js +28 -5
- package/dist/session/managed-attempt-tools.d.ts +18 -0
- package/dist/session/managed-attempt-tools.js +854 -0
- package/dist/session/managed-build-coordinator.d.ts +5 -0
- package/dist/session/managed-build-coordinator.js +105 -0
- package/dist/session/managed-build-receipt.d.ts +21 -0
- package/dist/session/managed-build-receipt.js +335 -0
- package/dist/session/phase-completion.artifacts.d.ts +17 -0
- package/dist/session/phase-completion.artifacts.js +233 -24
- package/dist/session/phase-completion.d.ts +11 -2
- package/dist/session/phase-completion.helpers.d.ts +5 -0
- package/dist/session/phase-completion.helpers.js +51 -10
- package/dist/session/phase-completion.js +12 -3
- package/dist/session/phase-completion.prompts.d.ts +3 -3
- package/dist/session/phase-completion.prompts.js +21 -6
- package/dist/session/phase-completion.runtime.d.ts +13 -1
- package/dist/session/phase-completion.runtime.js +189 -8
- package/dist/session/phase-completion.schemas.js +108 -3
- package/dist/session/phase-completion.validation.d.ts +4 -1
- package/dist/session/phase-completion.validation.js +151 -21
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +14 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +105 -0
- package/dist/session/provider-cli/provider-cli.artifacts.d.ts +3 -2
- package/dist/session/provider-cli/provider-cli.artifacts.js +65 -7
- package/dist/session/provider-cli/provider-cli.deterministic.js +47 -1
- package/dist/session/provider-cli/provider-cli.process.d.ts +23 -0
- package/dist/session/provider-cli/provider-cli.process.js +420 -32
- package/dist/session/provider-cli/provider-cli.prompts.d.ts +4 -1
- package/dist/session/provider-cli/provider-cli.prompts.js +99 -19
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +1 -0
- package/dist/session/provider-cli/provider-cli.runtime.d.ts +1 -1
- package/dist/session/provider-cli/provider-cli.runtime.js +6 -3
- package/dist/session/provider-cli/provider-cli.service.d.ts +20 -2
- package/dist/session/provider-cli/provider-cli.service.js +464 -102
- package/dist/session/provider-cli/provider-cli.types.d.ts +5 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.js +91 -16
- package/dist/session/provider-cli/provider-usage.d.ts +7 -0
- package/dist/session/provider-cli/provider-usage.js +28 -5
- package/dist/session/session-redaction.d.ts +3 -0
- package/dist/session/session-redaction.js +38 -0
- package/dist/session/session.base.d.ts +6 -1
- package/dist/session/session.service.d.ts +8 -2
- package/dist/session/session.service.js +522 -41
- package/dist/session/session.types.d.ts +70 -0
- package/dist/session/session.types.js +76 -4
- package/dist/shared/format/format-bytes.d.ts +1 -0
- package/dist/shared/format/format-bytes.js +9 -0
- package/dist/shared/path/path.d.ts +1 -0
- package/dist/shared/path/path.js +21 -1
- package/dist/shared/process/host-process-recovery.d.ts +12 -0
- package/dist/shared/process/host-process-recovery.js +78 -0
- package/dist/shared/process/host-process-supervisor.d.ts +59 -0
- package/dist/shared/process/host-process-supervisor.js +330 -0
- package/dist/shared/process/process.d.ts +1 -0
- package/dist/shared/process/process.js +1 -0
- package/dist/shared/redaction/redaction.js +12 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.d.ts +3 -2
- package/dist/worktree/artifact-injector.js +43 -4
- package/dist/worktree/dependency-warmer.d.ts +28 -1
- package/dist/worktree/dependency-warmer.js +308 -9
- package/dist/worktree/index.d.ts +2 -1
- package/dist/worktree/index.js +1 -0
- package/dist/worktree/prior-phase-artifact-hydrator.d.ts +2 -0
- package/dist/worktree/prior-phase-artifact-hydrator.js +136 -0
- package/dist/worktree/worktree-paths.d.ts +4 -0
- package/dist/worktree/worktree-paths.js +30 -0
- package/dist/worktree/worktree.base.d.ts +3 -0
- package/dist/worktree/worktree.service.d.ts +9 -2
- package/dist/worktree/worktree.service.js +217 -48
- package/dist/worktree/worktree.types.d.ts +5 -1
- package/package.json +5 -3
- package/runtime/windows-job-supervisor.ps1 +444 -0
- package/runtime/windows-listener-owner.ps1 +33 -0
package/dist/sandbox/runtime.js
CHANGED
|
@@ -10,8 +10,10 @@ import { evaluateSandboxImagePolicy, hasSandboxImagePolicyRecord, sandboxImagePo
|
|
|
10
10
|
import { SandboxEventBudget } from "./event-budget.js";
|
|
11
11
|
import { RepositoryCoordinatorLock } from "./repository-lock.js";
|
|
12
12
|
import { redactText } from "../shared/redaction/redaction.js";
|
|
13
|
+
import { createRuntimeIsolationLease, renewRuntimeIsolationLease, transitionRuntimeIsolationLease, validRuntimeIsolationLease, withRuntimeIsolationResources, } from "../runtime/runtime-isolation-lease.js";
|
|
13
14
|
import { sandboxProviderAuthAdapterFor, sandboxProviderAuthUnavailableMessage } from "./provider-auth/codex.js";
|
|
14
15
|
const managedLabel = "dev.forge.managed=true";
|
|
16
|
+
const bridgeRuntimeDigestLabel = "dev.forge.bridge-runtime-digest";
|
|
15
17
|
export class DockerSandboxRuntime {
|
|
16
18
|
config;
|
|
17
19
|
publicationGrantRedeemer;
|
|
@@ -31,12 +33,40 @@ export class DockerSandboxRuntime {
|
|
|
31
33
|
const controller = new AbortController();
|
|
32
34
|
const abort = () => controller.abort(signal?.reason ?? "sandbox cancelled");
|
|
33
35
|
signal?.addEventListener("abort", abort, { once: true });
|
|
36
|
+
const runtimeLease = createRuntimeIsolationLease({
|
|
37
|
+
leaseId: dispatch.sandbox.executionLeaseId,
|
|
38
|
+
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
39
|
+
executionLeaseId: dispatch.sandbox.executionLeaseId,
|
|
40
|
+
fencingToken: dispatch.lease.fencingToken,
|
|
41
|
+
runnerId: dispatch.execution.runnerId ?? "",
|
|
42
|
+
runnerInstanceId: dispatch.sandbox.runnerInstanceId,
|
|
43
|
+
worktreeKey: dispatch.lease.worktreeKey,
|
|
44
|
+
worktreePath: "/workspace",
|
|
45
|
+
mode: "docker",
|
|
46
|
+
expiresAt: dispatch.lease.expiresAt,
|
|
47
|
+
resources: {
|
|
48
|
+
memoryReservationBytes: dispatch.sandbox.requestedResources.memoryBytes,
|
|
49
|
+
cpuMillis: dispatch.sandbox.requestedResources.cpuMillis,
|
|
50
|
+
pids: dispatch.sandbox.requestedResources.pids,
|
|
51
|
+
ports: [],
|
|
52
|
+
paths: [
|
|
53
|
+
{ path: "/workspace", role: "worktree" },
|
|
54
|
+
{ path: "/tmp", role: "temporary" },
|
|
55
|
+
{ path: "/control/output/logs", role: "logs" },
|
|
56
|
+
{ path: "/tmp/forge-browser-profile", role: "browser_profile" },
|
|
57
|
+
],
|
|
58
|
+
processSupervisorId: null,
|
|
59
|
+
dockerResources: [],
|
|
60
|
+
singletonKeys: [],
|
|
61
|
+
},
|
|
62
|
+
});
|
|
34
63
|
const active = {
|
|
35
64
|
dispatchAttemptId: dispatch.sandbox.dispatchAttemptId,
|
|
36
65
|
fencingToken: dispatch.lease.fencingToken,
|
|
37
66
|
expiresAt: dispatch.lease.expiresAt,
|
|
38
67
|
controller,
|
|
39
68
|
phase: "starting",
|
|
69
|
+
runtimeLease,
|
|
40
70
|
ledger: null,
|
|
41
71
|
leaseControlKey: randomBytes(32).toString("base64url")
|
|
42
72
|
};
|
|
@@ -60,6 +90,7 @@ export class DockerSandboxRuntime {
|
|
|
60
90
|
const git = new HostGit(this.config, repoPath);
|
|
61
91
|
const prepared = await git.prepare(dispatch, attemptRoot, runtimeSignal);
|
|
62
92
|
const validated = validateSandboxContract(prepared.contractSource);
|
|
93
|
+
validateSandboxMemoryReservation(dispatch, validated);
|
|
63
94
|
validateDockerfilePolicy(prepared.dockerfile, dispatch.sandbox.executionPolicy === "sandbox_required");
|
|
64
95
|
const upstreamImages = dockerfileUpstreamImages(prepared.dockerfile);
|
|
65
96
|
const securityOptions = resolveSandboxSecurityOptions(dispatch.sandbox.executionPolicy);
|
|
@@ -78,20 +109,22 @@ export class DockerSandboxRuntime {
|
|
|
78
109
|
imageDigest
|
|
79
110
|
}));
|
|
80
111
|
const resources = resourceNames(identity.attemptHex, validated.contract.caches.filter(cache => cache.mode === "per_attempt").map(cache => cache.name));
|
|
81
|
-
const labels =
|
|
82
|
-
const networkArgs = networkCreateArgs(runtimeDispatch);
|
|
112
|
+
const labels = sandboxResourceLabels(runtimeDispatch, imageDigest, validated.digest);
|
|
113
|
+
const networkArgs = networkCreateArgs(runtimeDispatch, process.env, this.config.allowOpenPublicSandbox);
|
|
114
|
+
active.runtimeLease = withRuntimeIsolationResources(transitionRuntimeIsolationLease(active.runtimeLease, "starting"), {
|
|
115
|
+
...active.runtimeLease.resources,
|
|
116
|
+
dockerResources: plannedDockerLeaseResources(resources, runtimeDispatch.sandbox.networkPolicy.mode === "public_allowlist", Boolean(providerAuthAdapter)),
|
|
117
|
+
});
|
|
83
118
|
active.ledger = {
|
|
84
|
-
schemaVersion:
|
|
85
|
-
|
|
86
|
-
fencingToken: active.fencingToken,
|
|
87
|
-
runnerInstanceId: this.config.runnerInstanceId,
|
|
88
|
-
expiresAt: active.expiresAt,
|
|
119
|
+
schemaVersion: 3,
|
|
120
|
+
runtimeLease: active.runtimeLease,
|
|
89
121
|
resources,
|
|
90
122
|
labels,
|
|
91
123
|
includeCredentials: Boolean(providerAuthAdapter)
|
|
92
124
|
};
|
|
93
125
|
await this.writeLedger(active.ledger);
|
|
94
126
|
let teardownComplete = false;
|
|
127
|
+
let memoryTelemetryEvent = null;
|
|
95
128
|
try {
|
|
96
129
|
const internalNetworkId = dockerObjectId((await this.docker(["network", "create", ...networkArgs, ...labelArgs(labels), resources.network], 30_000, runtimeSignal)).stdout, "network");
|
|
97
130
|
await this.docker(["volume", "create", ...labelArgs(labels), resources.workspaceVolume], 30_000, runtimeSignal);
|
|
@@ -108,32 +141,47 @@ export class DockerSandboxRuntime {
|
|
|
108
141
|
egressNetworkId = dockerObjectId((await this.docker(["network", "create", ...labelArgs(labels), resources.egressNetwork], 30_000, runtimeSignal)).stdout, "network");
|
|
109
142
|
await this.docker(["volume", "create", ...labelArgs(labels), resources.egressConfigVolume], 30_000, runtimeSignal);
|
|
110
143
|
}
|
|
111
|
-
await this.initializeVolumes(builtImage.helperImageDigest, resources, Boolean(providerAuthAdapter), runtimeSignal);
|
|
112
|
-
await this.initializeProviderCredentials(builtImage.helperImageDigest, runtimeDispatch, resources, runtimeSignal);
|
|
144
|
+
await this.initializeVolumes(builtImage.helperImageDigest, resources, labels, Boolean(providerAuthAdapter), runtimeSignal);
|
|
145
|
+
await this.initializeProviderCredentials(builtImage.helperImageDigest, runtimeDispatch, resources, labels, runtimeSignal);
|
|
113
146
|
await sink.emit(event("sandbox_cache_policy", "Runtime caches are builder-managed or isolated to this attempt.", {
|
|
114
147
|
builderManaged: validated.contract.caches.filter(cache => cache.mode === "readonly").map(cache => cache.name),
|
|
115
148
|
perAttempt: Object.keys(resources.cacheVolumes)
|
|
116
149
|
}));
|
|
117
|
-
await this.initializeServiceSecrets(builtImage.helperImageDigest, validated, resources, runtimeSignal);
|
|
118
|
-
await this.initializeEgressPolicy(builtImage.helperImageDigest, runtimeDispatch, resources, runtimeSignal);
|
|
150
|
+
await this.initializeServiceSecrets(builtImage.helperImageDigest, validated, resources, labels, runtimeSignal);
|
|
151
|
+
await this.initializeEgressPolicy(builtImage.helperImageDigest, runtimeDispatch, resources, labels, runtimeSignal);
|
|
119
152
|
const egressProxy = await this.startEgressProxy(runtimeDispatch, resources, labels, securityOptions, sink, runtimeSignal);
|
|
120
153
|
const serviceContainers = await this.startServices(validated, runtimeDispatch.sandbox.executionPolicy, resources, labels, securityOptions, sink, runtimeSignal);
|
|
121
154
|
const createArgs = workerCreateArgs(runtimeDispatch, validated, resources, labels, imageDigest, securityOptions);
|
|
122
155
|
const createdContainer = await this.docker(createArgs, 60_000, runtimeSignal);
|
|
123
156
|
const containerId = dockerObjectId(createdContainer.stdout, "container");
|
|
124
157
|
await this.docker(["cp", prepared.bundlePath, `${containerId}:/control/input/repo.bundle`], 60_000, runtimeSignal);
|
|
158
|
+
const runtimeContainers = [
|
|
159
|
+
{ id: containerId, name: resources.container, role: "agent" },
|
|
160
|
+
...(egressProxy ? [egressProxy] : []),
|
|
161
|
+
...serviceContainers
|
|
162
|
+
];
|
|
163
|
+
active.runtimeLease = withRuntimeIsolationResources(transitionRuntimeIsolationLease(active.runtimeLease, "active"), {
|
|
164
|
+
...active.runtimeLease.resources,
|
|
165
|
+
dockerResources: activeDockerLeaseResources(runtimeContainers, [
|
|
166
|
+
{ id: internalNetworkId, name: resources.network, role: "internal" },
|
|
167
|
+
...(egressNetworkId ? [{ id: egressNetworkId, name: resources.egressNetwork, role: "egress" }] : []),
|
|
168
|
+
], sandboxRuntimeVolumes(resources, runtimeDispatch.sandbox.networkPolicy.mode === "public_allowlist", Boolean(providerAuthAdapter))),
|
|
169
|
+
});
|
|
170
|
+
active.ledger = { ...active.ledger, runtimeLease: active.runtimeLease };
|
|
171
|
+
await this.writeLedger(active.ledger);
|
|
125
172
|
await sink.emit(event("sandbox_started", `Docker sandbox ${identity.attemptHex} started.`, {
|
|
126
|
-
agentContainer:
|
|
173
|
+
agentContainer: runtimeContainers[0],
|
|
127
174
|
sidecarContainers: [...(egressProxy ? [egressProxy] : []), ...serviceContainers],
|
|
128
175
|
networks: [
|
|
129
176
|
{ id: internalNetworkId, name: resources.network, role: "internal" },
|
|
130
177
|
...(egressNetworkId ? [{ id: egressNetworkId, name: resources.egressNetwork, role: "egress" }] : [])
|
|
131
178
|
],
|
|
132
179
|
volumes: sandboxRuntimeVolumes(resources, runtimeDispatch.sandbox.networkPolicy.mode === "public_allowlist", Boolean(providerAuthAdapter)),
|
|
133
|
-
publishedPorts: []
|
|
180
|
+
publishedPorts: [],
|
|
181
|
+
runtimeLease: active.runtimeLease,
|
|
134
182
|
}));
|
|
135
183
|
const eventBudget = new SandboxEventBudget();
|
|
136
|
-
const
|
|
184
|
+
const startPromise = runSandboxProcess({
|
|
137
185
|
command: this.config.dockerExecutable,
|
|
138
186
|
args: this.dockerArgs(["start", "--attach", "--interactive", containerId]),
|
|
139
187
|
input: JSON.stringify({
|
|
@@ -156,6 +204,18 @@ export class DockerSandboxRuntime {
|
|
|
156
204
|
}
|
|
157
205
|
}
|
|
158
206
|
});
|
|
207
|
+
const memorySampler = this.startPeakMemorySampler(runtimeContainers);
|
|
208
|
+
const start = await startPromise.finally(async () => {
|
|
209
|
+
const telemetry = await memorySampler.stop();
|
|
210
|
+
if (!telemetry)
|
|
211
|
+
return;
|
|
212
|
+
memoryTelemetryEvent = event("sandbox_peak_memory_observed", `Observed sandbox peak memory ${formatMemoryBytes(telemetry.peakMemoryBytes)} across ${telemetry.containers.length} container${telemetry.containers.length === 1 ? "" : "s"} from ${telemetry.sampleCount} sample${telemetry.sampleCount === 1 ? "" : "s"}.`, { ...telemetry });
|
|
213
|
+
if (active.ledger) {
|
|
214
|
+
active.ledger = { ...active.ledger, peakMemory: telemetry };
|
|
215
|
+
await this.writeLedger(active.ledger).catch(() => undefined);
|
|
216
|
+
}
|
|
217
|
+
await sink.emit(memoryTelemetryEvent).catch(() => undefined);
|
|
218
|
+
});
|
|
159
219
|
if (start.truncated)
|
|
160
220
|
throw new Error("sandbox_log_limit: sandbox output exceeded the 8 MiB supervisor capture budget.");
|
|
161
221
|
const containerState = await this.inspectContainerState(containerId);
|
|
@@ -176,7 +236,16 @@ export class DockerSandboxRuntime {
|
|
|
176
236
|
workerExitCode: start.exitCode
|
|
177
237
|
}));
|
|
178
238
|
await this.docker(["cp", `${containerId}:/control/output/.`, path.join(attemptRoot, "output")], 60_000);
|
|
179
|
-
|
|
239
|
+
let output;
|
|
240
|
+
try {
|
|
241
|
+
output = await validateOutput(attemptRoot, runtimeDispatch);
|
|
242
|
+
}
|
|
243
|
+
catch (error) {
|
|
244
|
+
if (start.exitCode !== 0 && isMissingSandboxManifest(error)) {
|
|
245
|
+
throw new Error(sandboxWorkerFailureMessage(start.exitCode, start.stderr, containerState.error), { cause: error });
|
|
246
|
+
}
|
|
247
|
+
throw error;
|
|
248
|
+
}
|
|
180
249
|
await git.importResult(runtimeDispatch, output, () => assertSandboxImportCurrent(active, this.active.get(active.dispatchAttemptId), controller.signal));
|
|
181
250
|
if (runtimeDispatch.sandbox.publicationMode === "bridge_push") {
|
|
182
251
|
const handle = runtimeDispatch.sandbox.publishGrantHandle;
|
|
@@ -207,36 +276,49 @@ export class DockerSandboxRuntime {
|
|
|
207
276
|
...result.events.flatMap(resultEvent => [resultEvent.eventType, resultEvent.content, resultEvent.data ?? ""])
|
|
208
277
|
].join("\n"));
|
|
209
278
|
if (resourceFailure) {
|
|
210
|
-
return {
|
|
279
|
+
return appendRunOnceEvent({
|
|
211
280
|
...result,
|
|
212
281
|
summary: resourceFailure.message,
|
|
213
282
|
events: [...result.events, event(resourceFailure.code, resourceFailure.message, { phase: "agent" })]
|
|
214
|
-
};
|
|
283
|
+
}, memoryTelemetryEvent);
|
|
215
284
|
}
|
|
216
285
|
}
|
|
217
286
|
await sink.emit(event("sandbox_output_imported", "Validated and imported sandbox output.", {
|
|
218
287
|
resultCommitSha: output.manifest.resultCommitSha,
|
|
219
288
|
}));
|
|
220
|
-
return result;
|
|
289
|
+
return appendRunOnceEvent(result, memoryTelemetryEvent);
|
|
221
290
|
}
|
|
222
291
|
catch (error) {
|
|
223
292
|
const message = error instanceof Error ? error.message : String(error);
|
|
224
293
|
const classified = classifySandboxRuntimeFailure(message);
|
|
225
|
-
return failureResult(classified.message, [event(classified.code, classified.message, { phase: classified.phase })]);
|
|
294
|
+
return appendRunOnceEvent(failureResult(classified.message, [event(classified.code, classified.message, { phase: classified.phase })]), memoryTelemetryEvent);
|
|
226
295
|
}
|
|
227
296
|
finally {
|
|
228
297
|
try {
|
|
229
298
|
try {
|
|
230
|
-
|
|
299
|
+
active.runtimeLease = transitionForCleanup(active.runtimeLease, "releasing");
|
|
300
|
+
if (active.ledger) {
|
|
301
|
+
active.ledger = { ...active.ledger, runtimeLease: active.runtimeLease };
|
|
302
|
+
await this.writeLedger(active.ledger);
|
|
303
|
+
}
|
|
304
|
+
await sink.emit(event("sandbox_teardown_started", `Removing Docker resources for attempt ${active.dispatchAttemptId}.`, {
|
|
305
|
+
runtimeLease: active.runtimeLease,
|
|
306
|
+
})).catch(() => undefined);
|
|
231
307
|
const cleanupReport = await this.teardown(resources, labels, "teardown", Boolean(providerAuthAdapter));
|
|
232
308
|
await this.persistCleanupReport(cleanupReport);
|
|
233
309
|
teardownComplete = true;
|
|
310
|
+
active.runtimeLease = transitionRuntimeIsolationLease(active.runtimeLease, "released");
|
|
234
311
|
this.degraded.delete(active.dispatchAttemptId);
|
|
235
312
|
await this.removeLedger(active.dispatchAttemptId);
|
|
236
313
|
await sink.emit(event("sandbox_teardown_completed", `Removed Docker resources for attempt ${active.dispatchAttemptId}.`, { ...cleanupReport })).catch(() => undefined);
|
|
237
314
|
}
|
|
238
315
|
catch (error) {
|
|
239
316
|
this.degraded.add(active.dispatchAttemptId);
|
|
317
|
+
active.runtimeLease = transitionForCleanup(active.runtimeLease, "degraded");
|
|
318
|
+
if (active.ledger) {
|
|
319
|
+
active.ledger = { ...active.ledger, runtimeLease: active.runtimeLease };
|
|
320
|
+
await this.writeLedger(active.ledger).catch(() => undefined);
|
|
321
|
+
}
|
|
240
322
|
const cleanupReport = error instanceof SandboxTeardownError ? error.report : emptyCleanupReport("teardown");
|
|
241
323
|
await this.persistCleanupReport(cleanupReport);
|
|
242
324
|
await sink.emit(event("teardown_failed", `Docker resources for attempt ${active.dispatchAttemptId} require reaping.`, { ...cleanupReport })).catch(() => undefined);
|
|
@@ -255,14 +337,16 @@ export class DockerSandboxRuntime {
|
|
|
255
337
|
}
|
|
256
338
|
}
|
|
257
339
|
async updateLease(dispatchAttemptId, fencingToken, expiresAt) {
|
|
258
|
-
if (!Number.isFinite(Date.parse(expiresAt)))
|
|
259
|
-
return;
|
|
260
340
|
const active = this.active.get(dispatchAttemptId);
|
|
261
341
|
if (!active || active.fencingToken !== fencingToken)
|
|
262
342
|
return;
|
|
343
|
+
const renewed = renewRuntimeIsolationLease(active.runtimeLease, fencingToken, expiresAt);
|
|
344
|
+
if (!renewed)
|
|
345
|
+
return;
|
|
263
346
|
active.expiresAt = expiresAt;
|
|
347
|
+
active.runtimeLease = renewed;
|
|
264
348
|
if (active.ledger) {
|
|
265
|
-
active.ledger = { ...active.ledger,
|
|
349
|
+
active.ledger = { ...active.ledger, runtimeLease: renewed };
|
|
266
350
|
await this.writeLedger(active.ledger);
|
|
267
351
|
await this.pushLeaseUpdate(active).catch(() => undefined);
|
|
268
352
|
}
|
|
@@ -303,29 +387,34 @@ export class DockerSandboxRuntime {
|
|
|
303
387
|
for (const ledger of await this.readLedgers()) {
|
|
304
388
|
if (ledger.labels["dev.forge.runner-id"] !== this.config.runnerId)
|
|
305
389
|
continue;
|
|
306
|
-
const
|
|
390
|
+
const lease = ledger.runtimeLease;
|
|
391
|
+
const current = this.active.get(lease.dispatchAttemptId);
|
|
307
392
|
if (current
|
|
308
|
-
&& current.fencingToken ===
|
|
309
|
-
&&
|
|
310
|
-
liveAttempts.add(
|
|
393
|
+
&& current.fencingToken === lease.fencingToken
|
|
394
|
+
&& lease.runnerInstanceId === this.config.runnerInstanceId) {
|
|
395
|
+
liveAttempts.add(lease.dispatchAttemptId);
|
|
311
396
|
continue;
|
|
312
397
|
}
|
|
313
|
-
const serverAttempt = snapshotByAttempt.get(
|
|
398
|
+
const serverAttempt = snapshotByAttempt.get(lease.dispatchAttemptId);
|
|
314
399
|
if (!shouldReapSandboxLedger(ledger, this.config.runnerInstanceId, serverAttempt, options.attemptSnapshot !== undefined, now)) {
|
|
315
|
-
liveAttempts.add(
|
|
400
|
+
liveAttempts.add(lease.dispatchAttemptId);
|
|
316
401
|
continue;
|
|
317
402
|
}
|
|
318
403
|
try {
|
|
404
|
+
const releasing = transitionForCleanup(lease, "releasing");
|
|
405
|
+
await this.writeLedger({ ...ledger, runtimeLease: releasing });
|
|
319
406
|
const report = await this.teardown(ledger.resources, ledger.labels, "reaper", ledger.includeCredentials !== false);
|
|
320
407
|
await this.persistCleanupReport(report);
|
|
321
|
-
await this.removeLedger(
|
|
322
|
-
this.degraded.delete(
|
|
323
|
-
await fs.rm(path.join(this.config.sandboxStateRoot, "attempts",
|
|
408
|
+
await this.removeLedger(lease.dispatchAttemptId);
|
|
409
|
+
this.degraded.delete(lease.dispatchAttemptId);
|
|
410
|
+
await fs.rm(path.join(this.config.sandboxStateRoot, "attempts", lease.dispatchAttemptId.replace(/-/g, "").toLowerCase()), { recursive: true, force: true }).catch(() => undefined);
|
|
324
411
|
}
|
|
325
412
|
catch (error) {
|
|
326
413
|
if (error instanceof SandboxTeardownError)
|
|
327
414
|
await this.persistCleanupReport(error.report);
|
|
328
|
-
|
|
415
|
+
const degraded = transitionForCleanup(lease, "degraded");
|
|
416
|
+
await this.writeLedger({ ...ledger, runtimeLease: degraded }).catch(() => undefined);
|
|
417
|
+
this.degraded.add(lease.dispatchAttemptId);
|
|
329
418
|
}
|
|
330
419
|
}
|
|
331
420
|
for (const kind of ["container", "network", "volume"]) {
|
|
@@ -337,9 +426,14 @@ export class DockerSandboxRuntime {
|
|
|
337
426
|
}
|
|
338
427
|
}
|
|
339
428
|
await this.purgeRestrictedAuditArtifacts();
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
429
|
+
await this.withBuildPermit(async () => {
|
|
430
|
+
if (options.diskLow) {
|
|
431
|
+
await this.collectExpiredImages();
|
|
432
|
+
}
|
|
433
|
+
if (this.active.size === 0 && this.builds.size === 0) {
|
|
434
|
+
await this.stopBuildxBuilder().catch(() => undefined);
|
|
435
|
+
}
|
|
436
|
+
});
|
|
343
437
|
}
|
|
344
438
|
async pendingCleanupReports() {
|
|
345
439
|
const root = path.join(this.config.sandboxStateRoot, "cleanup-reports");
|
|
@@ -399,7 +493,7 @@ export class DockerSandboxRuntime {
|
|
|
399
493
|
async writeLedger(ledger) {
|
|
400
494
|
const root = path.join(this.config.sandboxStateRoot, "ledger");
|
|
401
495
|
await fs.mkdir(root, { recursive: true, mode: 0o700 });
|
|
402
|
-
const target = this.ledgerPath(ledger.dispatchAttemptId);
|
|
496
|
+
const target = this.ledgerPath(ledger.runtimeLease.dispatchAttemptId);
|
|
403
497
|
const temporary = `${target}.${process.pid}.${Date.now()}.tmp`;
|
|
404
498
|
const handle = await fs.open(temporary, "wx", 0o600);
|
|
405
499
|
try {
|
|
@@ -422,8 +516,8 @@ export class DockerSandboxRuntime {
|
|
|
422
516
|
const stat = await fs.lstat(target);
|
|
423
517
|
if (stat.isSymbolicLink() || stat.size > 64 * 1024)
|
|
424
518
|
continue;
|
|
425
|
-
const ledger = JSON.parse(await fs.readFile(target, "utf8"));
|
|
426
|
-
if (
|
|
519
|
+
const ledger = normalizeSandboxLedger(JSON.parse(await fs.readFile(target, "utf8")));
|
|
520
|
+
if (ledger)
|
|
427
521
|
result.push(ledger);
|
|
428
522
|
}
|
|
429
523
|
return result;
|
|
@@ -446,17 +540,20 @@ export class DockerSandboxRuntime {
|
|
|
446
540
|
if (!/^\d+\.\d+\.\d+(?:-[a-z0-9.-]+)?$/i.test(codexVersion))
|
|
447
541
|
throw new Error("sandbox_build_failed: invalid pinned Codex package version.");
|
|
448
542
|
const policyConfigurationDigest = await sandboxImagePolicyConfigurationDigest();
|
|
449
|
-
const
|
|
543
|
+
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
544
|
+
const bridgeRuntimeDigest = await bridgeRuntimeContentDigest(packageRoot);
|
|
545
|
+
const key = sandboxImageCacheKey({
|
|
450
546
|
contextDigest,
|
|
451
|
-
contract.digest,
|
|
452
|
-
process.platform,
|
|
453
|
-
process.arch,
|
|
454
|
-
this.config.version,
|
|
547
|
+
contractDigest: contract.digest,
|
|
548
|
+
platform: process.platform,
|
|
549
|
+
architecture: process.arch,
|
|
550
|
+
bridgeVersion: this.config.version,
|
|
551
|
+
bridgeRuntimeDigest,
|
|
455
552
|
toolingImage,
|
|
456
553
|
codexVersion,
|
|
457
554
|
policy,
|
|
458
555
|
policyConfigurationDigest
|
|
459
|
-
|
|
556
|
+
});
|
|
460
557
|
const existing = this.builds.get(key);
|
|
461
558
|
if (existing) {
|
|
462
559
|
await sink.emit(event("sandbox_image_cache_miss", `Sandbox image ${key.slice(0, 12)} is being built by another dispatch.`, { cacheKey: key, singleflight: true }));
|
|
@@ -468,7 +565,16 @@ export class DockerSandboxRuntime {
|
|
|
468
565
|
}));
|
|
469
566
|
return built;
|
|
470
567
|
}
|
|
471
|
-
const promise = this.withBuildPermit(() =>
|
|
568
|
+
const promise = this.withBuildPermit(async () => {
|
|
569
|
+
try {
|
|
570
|
+
return await this.buildImageCore(contextRoot, contextDigest, contract.digest, key, project, packageRoot, bridgeRuntimeDigest, toolingImage, upstreamImages, codexVersion, policy, sink, signal);
|
|
571
|
+
}
|
|
572
|
+
finally {
|
|
573
|
+
await this.stopBuildxBuilder().catch(async (error) => {
|
|
574
|
+
await sink.emit(event("sandbox_builder_release_failed", `The dedicated BuildKit builder could not be stopped: ${error instanceof Error ? error.message : String(error)}.`)).catch(() => undefined);
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
}, signal);
|
|
472
578
|
this.builds.set(key, promise);
|
|
473
579
|
try {
|
|
474
580
|
return await promise;
|
|
@@ -492,16 +598,16 @@ export class DockerSandboxRuntime {
|
|
|
492
598
|
release();
|
|
493
599
|
}
|
|
494
600
|
}
|
|
495
|
-
async buildImageCore(contextRoot, contextDigest, contractDigest, key, project, toolingImage, upstreamImages, codexVersion, policy, sink, signal) {
|
|
601
|
+
async buildImageCore(contextRoot, contextDigest, contractDigest, key, project, packageRoot, bridgeRuntimeDigest, toolingImage, upstreamImages, codexVersion, policy, sink, signal) {
|
|
496
602
|
const projectTag = `forge/project:${key}`;
|
|
497
603
|
const helperTag = `forge/tooling:${key}`;
|
|
498
604
|
const overlayTag = `forge/agent:${key}`;
|
|
499
605
|
const [cachedHelper, cachedOverlay] = await Promise.all([
|
|
500
|
-
this.
|
|
501
|
-
this.
|
|
606
|
+
this.inspectImageMetadata(helperTag),
|
|
607
|
+
this.inspectImageMetadata(overlayTag)
|
|
502
608
|
]);
|
|
503
|
-
const cachedHelperDigest = cachedHelper?.
|
|
504
|
-
const cachedOverlayDigest = cachedOverlay?.
|
|
609
|
+
const cachedHelperDigest = cachedHelper?.id ?? "";
|
|
610
|
+
const cachedOverlayDigest = cachedOverlay?.id ?? "";
|
|
505
611
|
const cacheRecordPath = path.join(this.config.sandboxStateRoot, "image-cache", `${key}.json`);
|
|
506
612
|
const cacheRecord = await this.readImageCacheRecord(cacheRecordPath);
|
|
507
613
|
const policyRecordValid = cacheRecord?.policyDigest
|
|
@@ -511,6 +617,7 @@ export class DockerSandboxRuntime {
|
|
|
511
617
|
&& /^sha256:[0-9a-f]{64}$/i.test(cachedOverlayDigest)
|
|
512
618
|
&& cacheRecord?.helperImageDigest === cachedHelperDigest
|
|
513
619
|
&& cacheRecord.imageDigest === cachedOverlayDigest
|
|
620
|
+
&& bridgeRuntimeCacheMatches(cacheRecord, bridgeRuntimeDigest, [cachedHelper?.labels, cachedOverlay?.labels])
|
|
514
621
|
&& policyRecordValid) {
|
|
515
622
|
await this.writeImageCacheRecord({
|
|
516
623
|
...cacheRecord,
|
|
@@ -530,29 +637,44 @@ export class DockerSandboxRuntime {
|
|
|
530
637
|
await sink.emit(event("sandbox_build_started", `Building sandbox image ${key.slice(0, 12)}.`, { cacheKey: key }));
|
|
531
638
|
const builder = await this.ensureBuildxBuilder(signal);
|
|
532
639
|
const metadataRoot = path.join(this.config.sandboxStateRoot, "attestations", key, "buildkit");
|
|
533
|
-
await fs.mkdir(metadataRoot, { recursive: true, mode: 0o700 });
|
|
534
|
-
const cached = await this.docker(["image", "inspect", projectTag, "--format", "{{.Id}}"], 15_000).catch(() => null);
|
|
535
|
-
let projectImage = cached?.stdout.trim();
|
|
536
|
-
if (!projectImage) {
|
|
537
|
-
await this.runBuildStage("project", key, buildxBuildArgs(builder, path.join(metadataRoot, "project.json"), ["--pull=false", ...imageLabelArgs(project, key, "project"), "--tag", projectTag, "--file", path.join(contextRoot, ".forge", "Dockerfile"), contextRoot]), sink, signal);
|
|
538
|
-
projectImage = (await this.docker(["image", "inspect", projectTag, "--format", "{{.Id}}"], 15_000)).stdout.trim();
|
|
539
|
-
}
|
|
540
|
-
if (!/^sha256:[0-9a-f]{64}$/i.test(projectImage))
|
|
541
|
-
throw new Error("sandbox_build_failed: project image did not resolve to an immutable digest.");
|
|
542
640
|
const overlayRoot = path.join(this.config.sandboxStateRoot, "build", `overlay-${key}`);
|
|
543
|
-
await
|
|
544
|
-
await fs.mkdir(
|
|
545
|
-
const
|
|
546
|
-
|
|
547
|
-
await
|
|
548
|
-
|
|
549
|
-
|
|
641
|
+
await prepareOverlayBuildRoot(overlayRoot);
|
|
642
|
+
await fs.mkdir(metadataRoot, { recursive: true, mode: 0o700 });
|
|
643
|
+
const projectOciArchive = path.join(overlayRoot, "project.oci.tar");
|
|
644
|
+
const projectOciRoot = path.join(overlayRoot, "project-oci");
|
|
645
|
+
await this.runBuildStage("project", key, buildxBuildArgs(builder, path.join(metadataRoot, "project.json"), [
|
|
646
|
+
"--pull=false",
|
|
647
|
+
...imageLabelArgs(project, key, "project", bridgeRuntimeDigest),
|
|
648
|
+
"--tag", projectTag,
|
|
649
|
+
"--output", `type=oci,dest=${projectOciArchive}`,
|
|
650
|
+
"--file", path.join(contextRoot, ".forge", "Dockerfile"),
|
|
651
|
+
contextRoot
|
|
652
|
+
], process.env, { load: false }), sink, signal);
|
|
653
|
+
await extractOciLayoutArchive(projectOciArchive, projectOciRoot, signal);
|
|
654
|
+
const projectImage = await readOciLayoutImageDigest(projectOciRoot);
|
|
655
|
+
const projectContextName = "forge_project";
|
|
656
|
+
const stagedBridgeRoot = path.join(overlayRoot, "bridge");
|
|
657
|
+
await fs.cp(path.join(packageRoot, "dist"), path.join(stagedBridgeRoot, "dist"), { recursive: true });
|
|
658
|
+
const installMode = await copyBridgePackageManifests(packageRoot, stagedBridgeRoot);
|
|
659
|
+
const stagedBridgeRuntimeDigest = await bridgeRuntimeContentDigest(stagedBridgeRoot);
|
|
660
|
+
if (stagedBridgeRuntimeDigest !== bridgeRuntimeDigest) {
|
|
661
|
+
throw new Error("sandbox_build_failed: bridge runtime payload changed while preparing the sandbox image; retry the dispatch.");
|
|
662
|
+
}
|
|
663
|
+
const dockerfile = agentOverlayDockerfile(toolingImage, projectContextName, installMode, codexVersion);
|
|
550
664
|
await fs.writeFile(path.join(overlayRoot, "Dockerfile"), dockerfile);
|
|
551
|
-
await
|
|
665
|
+
await fs.writeFile(path.join(overlayRoot, ".dockerignore"), agentOverlayDockerignore());
|
|
666
|
+
await this.runBuildStage("tooling", key, buildxBuildArgs(builder, path.join(metadataRoot, "tooling.json"), ["--pull=false", "--target", "tooling", ...imageLabelArgs(project, key, "tooling", bridgeRuntimeDigest), "--tag", helperTag, overlayRoot]), sink, signal);
|
|
552
667
|
const helperImageDigest = (await this.docker(["image", "inspect", helperTag, "--format", "{{.Id}}"], 15_000)).stdout.trim();
|
|
553
668
|
if (!/^sha256:[0-9a-f]{64}$/i.test(helperImageDigest))
|
|
554
669
|
throw new Error("sandbox_build_failed: helper image did not resolve to an immutable digest.");
|
|
555
|
-
|
|
670
|
+
const projectContextRef = ociLayoutContextRef(path.relative(overlayRoot, projectOciRoot));
|
|
671
|
+
await this.runBuildStage("agent", key, buildxBuildArgs(builder, path.join(metadataRoot, "agent.json"), [
|
|
672
|
+
"--pull=false",
|
|
673
|
+
"--build-context", `${projectContextName}=${projectContextRef}`,
|
|
674
|
+
...imageLabelArgs(project, key, "agent", bridgeRuntimeDigest),
|
|
675
|
+
"--tag", overlayTag,
|
|
676
|
+
"."
|
|
677
|
+
]), sink, signal, { cwd: overlayRoot });
|
|
556
678
|
const digest = (await this.docker(["image", "inspect", overlayTag, "--format", "{{.Id}}"], 15_000)).stdout.trim();
|
|
557
679
|
if (!/^sha256:[0-9a-f]{64}$/i.test(digest))
|
|
558
680
|
throw new Error("sandbox_build_failed: overlay image did not resolve to an immutable digest.");
|
|
@@ -568,9 +690,11 @@ export class DockerSandboxRuntime {
|
|
|
568
690
|
upstreamImages,
|
|
569
691
|
codexVersion,
|
|
570
692
|
images: { project: projectImage, tooling: helperImageDigest, agent: digest },
|
|
693
|
+
imageScanTargets: { project: ociLayoutScanTarget(projectOciRoot) },
|
|
571
694
|
signal
|
|
572
695
|
});
|
|
573
696
|
await sink.emit(event(imagePolicy.status === "passed" ? "sandbox_image_policy_passed" : "sandbox_image_policy_warning", imagePolicy.status === "passed" ? `Image policy passed for ${digest}.` : `Image attestations are incomplete for adoption-mode image ${digest}.`, { policyDigest: imagePolicy.policyDigest }));
|
|
697
|
+
await removeExtractedOciLayout(projectOciRoot);
|
|
574
698
|
await this.writeImageCacheRecord({
|
|
575
699
|
schemaVersion: 1,
|
|
576
700
|
key,
|
|
@@ -582,6 +706,7 @@ export class DockerSandboxRuntime {
|
|
|
582
706
|
overlayTag,
|
|
583
707
|
imageDigest: digest,
|
|
584
708
|
helperImageDigest,
|
|
709
|
+
bridgeRuntimeDigest,
|
|
585
710
|
policyDigest: imagePolicy.policyDigest
|
|
586
711
|
});
|
|
587
712
|
await sink.emit(event("sandbox_build_completed", `Sandbox image ready at ${digest}.`, {
|
|
@@ -590,10 +715,11 @@ export class DockerSandboxRuntime {
|
|
|
590
715
|
}));
|
|
591
716
|
return { imageDigest: digest, helperImageDigest };
|
|
592
717
|
}
|
|
593
|
-
async runBuildStage(stage, key, args, sink, signal) {
|
|
718
|
+
async runBuildStage(stage, key, args, sink, signal, options = {}) {
|
|
594
719
|
const result = await runSandboxProcess({
|
|
595
720
|
command: this.config.dockerExecutable,
|
|
596
721
|
args: this.dockerArgs(args),
|
|
722
|
+
cwd: options.cwd,
|
|
597
723
|
timeoutMs: 30 * 60_000,
|
|
598
724
|
maxOutputBytes: 2 * 1024 * 1024,
|
|
599
725
|
terminateOnOutputLimit: true,
|
|
@@ -635,21 +761,35 @@ export class DockerSandboxRuntime {
|
|
|
635
761
|
await this.docker(["buildx", "create", "--name", name, "--driver", "docker-container"], 60_000, signal);
|
|
636
762
|
}
|
|
637
763
|
await this.docker(["buildx", "inspect", "--bootstrap", name], 2 * 60_000, signal, 1024 * 1024);
|
|
638
|
-
const
|
|
764
|
+
const policy = buildResourcePolicy();
|
|
639
765
|
const container = dedicatedBuildxBuilderContainerName(name);
|
|
640
|
-
await this.docker([
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
766
|
+
await this.docker([
|
|
767
|
+
"container", "update",
|
|
768
|
+
"--pids-limit", String(policy.builderPidsLimit),
|
|
769
|
+
"--memory", String(policy.builderMemoryBytes),
|
|
770
|
+
"--memory-swap", String(policy.builderMemoryBytes),
|
|
771
|
+
container
|
|
772
|
+
], 30_000, signal);
|
|
773
|
+
const applied = await this.docker(["container", "inspect", container, "--format", "{{json .HostConfig}}"], 30_000, signal);
|
|
774
|
+
const hostConfig = JSON.parse(applied.stdout);
|
|
775
|
+
if (Number(hostConfig.PidsLimit) !== policy.builderPidsLimit
|
|
776
|
+
|| Number(hostConfig.Memory) !== policy.builderMemoryBytes
|
|
777
|
+
|| Number(hostConfig.MemorySwap) !== policy.builderMemoryBytes) {
|
|
778
|
+
throw new Error("sandbox_build_policy_failed: dedicated BuildKit builder resource limits were not applied.");
|
|
644
779
|
}
|
|
645
780
|
return name;
|
|
646
781
|
}
|
|
647
|
-
async
|
|
782
|
+
async stopBuildxBuilder() {
|
|
783
|
+
const name = dedicatedBuildxBuilderName(this.config.runnerId, this.config.dockerContext);
|
|
784
|
+
const container = dedicatedBuildxBuilderContainerName(name);
|
|
785
|
+
await this.docker(["container", "stop", "--time", "10", container], 30_000);
|
|
786
|
+
}
|
|
787
|
+
async initializeVolumes(helperImage, resources, labels, includeCredentials, signal) {
|
|
648
788
|
const cacheMounts = Object.entries(resources.cacheVolumes).flatMap(([name, volume]) => ["--volume", `${volume}:/cache/${name}`]);
|
|
649
|
-
const credentialSetup = includeCredentials ? "f.
|
|
650
|
-
const script = `const f=require('fs');f.mkdirSync('/control/input',{recursive:true});f.mkdirSync('/control/output',{recursive:true});for(const p of
|
|
789
|
+
const credentialSetup = includeCredentials ? "f.chmodSync('/credentials',0o700);" : "";
|
|
790
|
+
const script = `const f=require('fs'),owned=['/control','/control/input','/control/output','/workspace'];f.mkdirSync('/control/input',{recursive:true});f.mkdirSync('/control/output',{recursive:true});for(const p of owned)f.chmodSync(p,0o700);for(const p of owned.toReversed())f.chownSync(p,10001,10001);${credentialSetup}for(const p of f.readdirSync('/cache').map(n=>'/cache/'+n)){f.chmodSync(p,0o700);f.chownSync(p,10001,10001)}`;
|
|
651
791
|
const credentialMount = includeCredentials ? ["--volume", `${resources.credentialsVolume}:/credentials`] : [];
|
|
652
|
-
await this.docker(["run", "--rm", "--network", "none", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--pids-limit", "64", "--memory", "128m", "--memory-swap", "128m", "--cpus", "0.25", "--log-driver", "none", "--user", "0", "--entrypoint", "node", "--volume", `${resources.workspaceVolume}:/workspace`, "--volume", `${resources.controlVolume}:/control`, ...credentialMount, ...cacheMounts, "--tmpfs", "/cache:rw,nosuid,nodev,noexec,size=1m", helperImage, "-e", script], 60_000, signal);
|
|
792
|
+
await this.docker(["run", "--rm", ...labelArgs(labels), "--network", "none", "--read-only", "--cap-drop", "ALL", "--cap-add", "CHOWN", "--security-opt", "no-new-privileges", "--pids-limit", "64", "--memory", "128m", "--memory-swap", "128m", "--cpus", "0.25", "--log-driver", "none", "--user", "0", "--entrypoint", "node", "--volume", `${resources.workspaceVolume}:/workspace`, "--volume", `${resources.controlVolume}:/control`, ...credentialMount, ...cacheMounts, "--tmpfs", "/cache:rw,nosuid,nodev,noexec,size=1m", helperImage, "-e", script], 60_000, signal);
|
|
653
793
|
}
|
|
654
794
|
async inspectContainerState(containerId) {
|
|
655
795
|
const result = await this.docker(["container", "inspect", containerId, "--format", "{{json .State}}"], 15_000, undefined, 256 * 1024);
|
|
@@ -660,14 +800,14 @@ export class DockerSandboxRuntime {
|
|
|
660
800
|
error: typeof value.Error === "string" && value.Error.trim() ? value.Error.trim().slice(0, 1000) : null
|
|
661
801
|
};
|
|
662
802
|
}
|
|
663
|
-
async initializeServiceSecrets(helperImage, contract, resources, signal) {
|
|
803
|
+
async initializeServiceSecrets(helperImage, contract, resources, labels, signal) {
|
|
664
804
|
const script = "let s='';process.stdin.setEncoding('utf8');process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>require('fs').writeFileSync('/secrets/postgres-password',s,{mode:0o444}))";
|
|
665
805
|
for (const service of contract.contract.services) {
|
|
666
806
|
const secretName = service.type === "postgres" ? "postgres-password" : service.type === "redis" ? "redis-password" : "browser-token";
|
|
667
807
|
const writeScript = script.replace("postgres-password", secretName);
|
|
668
808
|
await requireSandboxProcess({
|
|
669
809
|
command: this.config.dockerExecutable,
|
|
670
|
-
args: this.dockerArgs(["run", "--rm", "--interactive", "--network", "none", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--pids-limit", "64", "--memory", "128m", "--memory-swap", "128m", "--cpus", "0.25", "--log-driver", "none", "--user", "0", "--entrypoint", "node", "--volume", `${resources.secretsVolume}:/secrets`, helperImage, "-e", writeScript]),
|
|
810
|
+
args: this.dockerArgs(["run", "--rm", "--interactive", ...labelArgs(labels), "--network", "none", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--pids-limit", "64", "--memory", "128m", "--memory-swap", "128m", "--cpus", "0.25", "--log-driver", "none", "--user", "0", "--entrypoint", "node", "--volume", `${resources.secretsVolume}:/secrets`, helperImage, "-e", writeScript]),
|
|
671
811
|
input: randomBytes(32).toString("base64url"),
|
|
672
812
|
timeoutMs: 60_000,
|
|
673
813
|
maxOutputBytes: 64 * 1024,
|
|
@@ -675,28 +815,28 @@ export class DockerSandboxRuntime {
|
|
|
675
815
|
});
|
|
676
816
|
}
|
|
677
817
|
}
|
|
678
|
-
async initializeProviderCredentials(helperImage, dispatch, resources, signal) {
|
|
818
|
+
async initializeProviderCredentials(helperImage, dispatch, resources, labels, signal) {
|
|
679
819
|
const providerAuth = await sandboxProviderAuthForDispatch(dispatch);
|
|
680
820
|
if (!providerAuth) {
|
|
681
821
|
return;
|
|
682
822
|
}
|
|
683
|
-
const script = "let s='';process.stdin.setEncoding('utf8');process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>{const f=require('fs'),p=require('path'),root='/credentials';for(const e of f.readdirSync(root))f.rmSync(p.join(root,e),{recursive:true,force:true});const packet=JSON.parse(s);for(const file of packet.files){if(!/^[a-z0-9_.-]+$/i.test(file.name))throw new Error('invalid credential file');const target=p.resolve(root,file.name);if(!target.startsWith(root+'/'))throw new Error('invalid credential file');const body=Buffer.from(file.contentBase64,'base64');f.writeFileSync(target,body,{mode:0o600});f.chownSync(target,10001,10001)}f.
|
|
823
|
+
const script = "let s='';process.stdin.setEncoding('utf8');process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>{const f=require('fs'),p=require('path'),root='/credentials';for(const e of f.readdirSync(root))f.rmSync(p.join(root,e),{recursive:true,force:true});const packet=JSON.parse(s);for(const file of packet.files){if(!/^[a-z0-9_.-]+$/i.test(file.name))throw new Error('invalid credential file');const target=p.resolve(root,file.name);if(!target.startsWith(root+'/'))throw new Error('invalid credential file');const body=Buffer.from(file.contentBase64,'base64');f.writeFileSync(target,body,{mode:0o600});f.chmodSync(target,0o600);f.chownSync(target,10001,10001)}f.chmodSync(root,0o700);f.chownSync(root,10001,10001)})";
|
|
684
824
|
await requireSandboxProcess({
|
|
685
825
|
command: this.config.dockerExecutable,
|
|
686
|
-
args: this.dockerArgs(["run", "--rm", "--interactive", "--network", "none", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--pids-limit", "64", "--memory", "128m", "--memory-swap", "128m", "--cpus", "0.25", "--log-driver", "none", "--user", "0", "--entrypoint", "node", "--volume", `${resources.credentialsVolume}:/credentials`, helperImage, "-e", script]),
|
|
826
|
+
args: this.dockerArgs(["run", "--rm", "--interactive", ...labelArgs(labels), "--network", "none", "--read-only", "--cap-drop", "ALL", "--cap-add", "CHOWN", "--security-opt", "no-new-privileges", "--pids-limit", "64", "--memory", "128m", "--memory-swap", "128m", "--cpus", "0.25", "--log-driver", "none", "--user", "0", "--entrypoint", "node", "--volume", `${resources.credentialsVolume}:/credentials`, helperImage, "-e", script]),
|
|
687
827
|
input: JSON.stringify({ files: providerAuth.files }),
|
|
688
828
|
timeoutMs: 60_000,
|
|
689
829
|
maxOutputBytes: 64 * 1024,
|
|
690
830
|
signal
|
|
691
831
|
});
|
|
692
832
|
}
|
|
693
|
-
async initializeEgressPolicy(helperImage, dispatch, resources, signal) {
|
|
833
|
+
async initializeEgressPolicy(helperImage, dispatch, resources, labels, signal) {
|
|
694
834
|
if (dispatch.sandbox.networkPolicy.mode !== "public_allowlist")
|
|
695
835
|
return;
|
|
696
836
|
const script = "let s='';process.stdin.setEncoding('utf8');process.stdin.on('data',c=>s+=c);process.stdin.on('end',()=>require('fs').writeFileSync('/policy/forge.conf',s,{mode:0o444}))";
|
|
697
837
|
await requireSandboxProcess({
|
|
698
838
|
command: this.config.dockerExecutable,
|
|
699
|
-
args: this.dockerArgs(["run", "--rm", "--interactive", "--network", "none", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--pids-limit", "64", "--memory", "128m", "--memory-swap", "128m", "--cpus", "0.25", "--log-driver", "none", "--user", "0", "--entrypoint", "node", "--volume", `${resources.egressConfigVolume}:/policy`, helperImage, "-e", script]),
|
|
839
|
+
args: this.dockerArgs(["run", "--rm", "--interactive", ...labelArgs(labels), "--network", "none", "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--pids-limit", "64", "--memory", "128m", "--memory-swap", "128m", "--cpus", "0.25", "--log-driver", "none", "--user", "0", "--entrypoint", "node", "--volume", `${resources.egressConfigVolume}:/policy`, helperImage, "-e", script]),
|
|
700
840
|
input: egressProxyConfig(dispatch.sandbox.networkPolicy.allowedHosts),
|
|
701
841
|
timeoutMs: 60_000,
|
|
702
842
|
maxOutputBytes: 64 * 1024,
|
|
@@ -919,6 +1059,51 @@ export class DockerSandboxRuntime {
|
|
|
919
1059
|
}
|
|
920
1060
|
}
|
|
921
1061
|
}
|
|
1062
|
+
async inspectImageMetadata(tag) {
|
|
1063
|
+
const result = await this.docker(["image", "inspect", tag], 15_000, undefined, 1024 * 1024).catch(() => null);
|
|
1064
|
+
if (!result)
|
|
1065
|
+
return null;
|
|
1066
|
+
const document = JSON.parse(result.stdout);
|
|
1067
|
+
const image = document[0];
|
|
1068
|
+
if (!image || typeof image.Id !== "string" || !/^sha256:[0-9a-f]{64}$/i.test(image.Id))
|
|
1069
|
+
return null;
|
|
1070
|
+
const labels = image.Config?.Labels;
|
|
1071
|
+
return {
|
|
1072
|
+
id: image.Id,
|
|
1073
|
+
labels: labels && typeof labels === "object" && !Array.isArray(labels)
|
|
1074
|
+
? Object.fromEntries(Object.entries(labels).filter((entry) => typeof entry[1] === "string"))
|
|
1075
|
+
: {}
|
|
1076
|
+
};
|
|
1077
|
+
}
|
|
1078
|
+
startPeakMemorySampler(containers) {
|
|
1079
|
+
const intervalMs = this.config.sandboxMemorySampleIntervalMs;
|
|
1080
|
+
const controller = new AbortController();
|
|
1081
|
+
const accumulator = createSandboxMemoryAccumulator(intervalMs);
|
|
1082
|
+
const done = (async () => {
|
|
1083
|
+
while (!controller.signal.aborted) {
|
|
1084
|
+
try {
|
|
1085
|
+
const result = await this.docker(["stats", "--no-stream", "--format", "{{json .}}", ...containers.map(container => container.id)], Math.max(5_000, intervalMs * 3), controller.signal, 128 * 1024);
|
|
1086
|
+
const sample = parseDockerStatsMemorySample(result.stdout);
|
|
1087
|
+
if (sample.length === 0)
|
|
1088
|
+
accumulator.failedSampleCount += 1;
|
|
1089
|
+
else
|
|
1090
|
+
recordSandboxMemorySample(accumulator, containers, sample, new Date().toISOString());
|
|
1091
|
+
}
|
|
1092
|
+
catch {
|
|
1093
|
+
if (!controller.signal.aborted)
|
|
1094
|
+
accumulator.failedSampleCount += 1;
|
|
1095
|
+
}
|
|
1096
|
+
await waitForMemorySampleInterval(intervalMs, controller.signal);
|
|
1097
|
+
}
|
|
1098
|
+
})();
|
|
1099
|
+
return {
|
|
1100
|
+
stop: async () => {
|
|
1101
|
+
controller.abort("sandbox memory sampling complete");
|
|
1102
|
+
await done.catch(() => undefined);
|
|
1103
|
+
return finalizeSandboxMemoryTelemetry(accumulator);
|
|
1104
|
+
}
|
|
1105
|
+
};
|
|
1106
|
+
}
|
|
922
1107
|
docker(args, timeoutMs, signal, maxOutputBytes = 1024 * 1024) {
|
|
923
1108
|
return requireSandboxProcess({ command: this.config.dockerExecutable, args: this.dockerArgs(args), timeoutMs, maxOutputBytes, signal });
|
|
924
1109
|
}
|
|
@@ -939,9 +1124,7 @@ class HostGit {
|
|
|
939
1124
|
async prepareLocked(dispatch, attemptRoot) {
|
|
940
1125
|
const baseBranch = dispatch.cycleRun.baseBranch || dispatch.cycleRun.requestedBase || dispatch.repo.defaultBranch;
|
|
941
1126
|
await this.git(["fetch", "--prune", "origin"], 5 * 60_000).catch(() => null);
|
|
942
|
-
const
|
|
943
|
-
const baseCandidates = [worktreeRef, `refs/remotes/origin/${baseBranch}`, `refs/heads/${baseBranch}`];
|
|
944
|
-
const inputCommitSha = await this.resolveFirst(baseCandidates);
|
|
1127
|
+
const inputCommitSha = await this.resolveFirst([...selectSandboxInputRefs(dispatch.lease.worktreeKey, baseBranch)]);
|
|
945
1128
|
const configSelection = selectSandboxConfigRefs(dispatch.sandbox.executionPolicy, dispatch.sandbox.sandboxConfigCommitSha, dispatch.repo.defaultBranch);
|
|
946
1129
|
const configCommitSha = await this.resolveFirst([...configSelection.refs]);
|
|
947
1130
|
const approvedConfigSha = configSelection.approvedSha;
|
|
@@ -1048,7 +1231,8 @@ class HostGit {
|
|
|
1048
1231
|
const listing = (await this.git(["worktree", "list", "--porcelain"], 30_000)).stdout;
|
|
1049
1232
|
const checkedOut = parseWorktreeListing(listing).find(entry => entry.branch === branchRef);
|
|
1050
1233
|
if (!checkedOut) {
|
|
1051
|
-
await this.git(["
|
|
1234
|
+
const current = (await this.git(["for-each-ref", "--format=%(objectname)", branchRef], 30_000)).stdout.trim();
|
|
1235
|
+
await this.git(["update-ref", branchRef, imported, managedBranchUpdateExpectedValue(current, expected, imported)], 30_000);
|
|
1052
1236
|
return;
|
|
1053
1237
|
}
|
|
1054
1238
|
const head = (await requireSandboxProcess({ command: "git", args: ["-C", checkedOut.path, "rev-parse", "HEAD"], timeoutMs: 30_000 })).stdout.trim();
|
|
@@ -1069,6 +1253,16 @@ class HostGit {
|
|
|
1069
1253
|
}
|
|
1070
1254
|
git(args, timeoutMs, signal, env) { return requireSandboxProcess({ command: "git", args: ["-C", this.repoPath, ...args], timeoutMs, maxOutputBytes: 4 * 1024 * 1024, signal, env }); }
|
|
1071
1255
|
}
|
|
1256
|
+
export function managedBranchUpdateExpectedValue(current, expectedInput, imported) {
|
|
1257
|
+
const normalized = current.trim();
|
|
1258
|
+
if (!normalized) {
|
|
1259
|
+
return "0".repeat(imported.length);
|
|
1260
|
+
}
|
|
1261
|
+
if (normalized !== expectedInput) {
|
|
1262
|
+
throw new Error("sandbox_output_conflict: managed branch changed after sandbox input was captured.");
|
|
1263
|
+
}
|
|
1264
|
+
return normalized;
|
|
1265
|
+
}
|
|
1072
1266
|
export function publicationPushArgs(branch, resultCommitSha, inputCommitSha, remoteCommitSha, remote = "origin") {
|
|
1073
1267
|
const isCommit = (value) => /^[0-9a-f]{40,64}$/iu.test(value);
|
|
1074
1268
|
if (!branch || branch.length > 240 || !isCommit(resultCommitSha) || !isCommit(inputCommitSha) || (remoteCommitSha && !isCommit(remoteCommitSha))) {
|
|
@@ -1188,11 +1382,18 @@ export function selectSandboxConfigRefs(executionPolicy, configuredSha, defaultB
|
|
|
1188
1382
|
}
|
|
1189
1383
|
return approvedSha
|
|
1190
1384
|
? { refs: [approvedSha], approvedSha }
|
|
1191
|
-
: { refs: [`refs/
|
|
1385
|
+
: { refs: [`refs/heads/${defaultBranch}`, `refs/remotes/origin/${defaultBranch}`], approvedSha: null };
|
|
1386
|
+
}
|
|
1387
|
+
export function selectSandboxInputRefs(worktreeKey, baseBranch) {
|
|
1388
|
+
return [
|
|
1389
|
+
`refs/heads/${worktreeKey}`,
|
|
1390
|
+
`refs/heads/${baseBranch}`,
|
|
1391
|
+
`refs/remotes/origin/${baseBranch}`
|
|
1392
|
+
];
|
|
1192
1393
|
}
|
|
1193
1394
|
export function sandboxTimeLimits(environment = process.env) {
|
|
1194
1395
|
const hardWallSeconds = boundedInteger(environment.FORGE_SANDBOX_HARD_WALL_SECONDS, 8 * 60 * 60, 60, 8 * 60 * 60);
|
|
1195
|
-
const idleSeconds = boundedInteger(environment.FORGE_SANDBOX_IDLE_SECONDS,
|
|
1396
|
+
const idleSeconds = boundedInteger(environment.FORGE_SANDBOX_IDLE_SECONDS, 10 * 60, 60, hardWallSeconds);
|
|
1196
1397
|
return { hardWallMs: hardWallSeconds * 1000, idleMs: idleSeconds * 1000 };
|
|
1197
1398
|
}
|
|
1198
1399
|
function boundedInteger(value, fallback, minimum, maximum) {
|
|
@@ -1220,7 +1421,12 @@ function parseWorktreeListing(value) {
|
|
|
1220
1421
|
async function validateOutput(attemptRoot, dispatch) {
|
|
1221
1422
|
const outputRoot = path.join(attemptRoot, "output");
|
|
1222
1423
|
const manifestPath = path.join(outputRoot, "manifest.json");
|
|
1223
|
-
const stat = await fs.lstat(manifestPath)
|
|
1424
|
+
const stat = await fs.lstat(manifestPath).catch(error => {
|
|
1425
|
+
if (isNodeErrorWithCode(error, "ENOENT")) {
|
|
1426
|
+
throw new Error("sandbox_output_missing: sandbox worker did not produce manifest.json.");
|
|
1427
|
+
}
|
|
1428
|
+
throw error;
|
|
1429
|
+
});
|
|
1224
1430
|
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 1024 * 1024)
|
|
1225
1431
|
throw new Error("sandbox_output_invalid: manifest must be a bounded regular file.");
|
|
1226
1432
|
const manifest = JSON.parse(await fs.readFile(manifestPath, "utf8"));
|
|
@@ -1254,6 +1460,22 @@ async function validateManifestFile(root, file, max) {
|
|
|
1254
1460
|
throw new Error("sandbox_output_invalid: output hash mismatch.");
|
|
1255
1461
|
return target;
|
|
1256
1462
|
}
|
|
1463
|
+
function isMissingSandboxManifest(error) {
|
|
1464
|
+
return error instanceof Error && error.message.startsWith("sandbox_output_missing:");
|
|
1465
|
+
}
|
|
1466
|
+
function isNodeErrorWithCode(error, code) {
|
|
1467
|
+
return error instanceof Error && "code" in error && error.code === code;
|
|
1468
|
+
}
|
|
1469
|
+
export function sandboxWorkerFailureMessage(exitCode, stderr, containerError) {
|
|
1470
|
+
const diagnostic = [containerError, stderr]
|
|
1471
|
+
.filter((value) => Boolean(value?.trim()))
|
|
1472
|
+
.join(" ")
|
|
1473
|
+
.replace(/[\r\n\t]+/gu, " ")
|
|
1474
|
+
.trim()
|
|
1475
|
+
.slice(0, 1000);
|
|
1476
|
+
const suffix = diagnostic ? ` Worker diagnostic: ${diagnostic}` : " No worker diagnostic was captured.";
|
|
1477
|
+
return `sandbox_worker_failed: Sandbox worker exited with code ${exitCode} before producing its output manifest.${suffix}`;
|
|
1478
|
+
}
|
|
1257
1479
|
function identityFor(dispatch) {
|
|
1258
1480
|
const value = dispatch.sandbox?.dispatchAttemptId.replace(/-/g, "").toLowerCase() ?? "";
|
|
1259
1481
|
if (!/^[0-9a-f]{32}$/.test(value))
|
|
@@ -1321,6 +1543,145 @@ function hasControlCharacters(value) {
|
|
|
1321
1543
|
return code < 32 || code === 127;
|
|
1322
1544
|
});
|
|
1323
1545
|
}
|
|
1546
|
+
function createSandboxMemoryAccumulator(sampleIntervalMs) {
|
|
1547
|
+
return {
|
|
1548
|
+
sampleIntervalMs,
|
|
1549
|
+
sampleCount: 0,
|
|
1550
|
+
failedSampleCount: 0,
|
|
1551
|
+
peakMemoryBytes: 0,
|
|
1552
|
+
firstSampledAt: null,
|
|
1553
|
+
lastSampledAt: null,
|
|
1554
|
+
containerPeaks: new Map()
|
|
1555
|
+
};
|
|
1556
|
+
}
|
|
1557
|
+
export function parseDockerMemoryBytes(value) {
|
|
1558
|
+
const match = /^([0-9]+(?:\.[0-9]+)?)\s*([kmgt]?i?b)$/iu.exec(value.trim());
|
|
1559
|
+
if (!match)
|
|
1560
|
+
return null;
|
|
1561
|
+
const amount = Number(match[1]);
|
|
1562
|
+
const multipliers = {
|
|
1563
|
+
b: 1,
|
|
1564
|
+
kb: 1000,
|
|
1565
|
+
mb: 1000 ** 2,
|
|
1566
|
+
gb: 1000 ** 3,
|
|
1567
|
+
tb: 1000 ** 4,
|
|
1568
|
+
kib: 1024,
|
|
1569
|
+
mib: 1024 ** 2,
|
|
1570
|
+
gib: 1024 ** 3,
|
|
1571
|
+
tib: 1024 ** 4
|
|
1572
|
+
};
|
|
1573
|
+
const multiplier = multipliers[match[2].toLowerCase()];
|
|
1574
|
+
const bytes = amount * multiplier;
|
|
1575
|
+
return Number.isFinite(bytes) && bytes >= 0 && bytes <= Number.MAX_SAFE_INTEGER
|
|
1576
|
+
? Math.round(bytes)
|
|
1577
|
+
: null;
|
|
1578
|
+
}
|
|
1579
|
+
export function parseDockerStatsMemorySample(output) {
|
|
1580
|
+
const samples = [];
|
|
1581
|
+
for (const line of output.split(/\r?\n/u).map(value => value.trim()).filter(Boolean)) {
|
|
1582
|
+
try {
|
|
1583
|
+
const value = JSON.parse(line);
|
|
1584
|
+
const containerId = typeof value.ID === "string" ? value.ID.trim().toLowerCase() : "";
|
|
1585
|
+
const name = typeof value.Name === "string" ? value.Name.trim()
|
|
1586
|
+
: typeof value.Container === "string" ? value.Container.trim()
|
|
1587
|
+
: "";
|
|
1588
|
+
const usage = typeof value.MemUsage === "string" ? value.MemUsage.split("/", 1)[0]?.trim() ?? "" : "";
|
|
1589
|
+
const memoryBytes = parseDockerMemoryBytes(usage);
|
|
1590
|
+
if ((!containerId && !name) || memoryBytes == null)
|
|
1591
|
+
continue;
|
|
1592
|
+
samples.push({ containerId, name, memoryBytes });
|
|
1593
|
+
}
|
|
1594
|
+
catch {
|
|
1595
|
+
// A malformed line is ignored; the sampler records the round as failed only if no valid rows remain.
|
|
1596
|
+
}
|
|
1597
|
+
}
|
|
1598
|
+
return samples;
|
|
1599
|
+
}
|
|
1600
|
+
function recordSandboxMemorySample(accumulator, containers, sample, sampledAt) {
|
|
1601
|
+
let aggregateMemoryBytes = 0;
|
|
1602
|
+
let matched = 0;
|
|
1603
|
+
for (const container of containers) {
|
|
1604
|
+
const row = sample.find(candidate => candidate.name === container.name
|
|
1605
|
+
|| (candidate.containerId.length > 0 && (container.id.startsWith(candidate.containerId) || candidate.containerId.startsWith(container.id))));
|
|
1606
|
+
if (!row)
|
|
1607
|
+
continue;
|
|
1608
|
+
matched += 1;
|
|
1609
|
+
aggregateMemoryBytes += row.memoryBytes;
|
|
1610
|
+
const previous = accumulator.containerPeaks.get(container.id);
|
|
1611
|
+
if (!previous || row.memoryBytes > previous.peakMemoryBytes) {
|
|
1612
|
+
accumulator.containerPeaks.set(container.id, {
|
|
1613
|
+
containerId: container.id,
|
|
1614
|
+
name: container.name,
|
|
1615
|
+
role: container.role,
|
|
1616
|
+
peakMemoryBytes: row.memoryBytes
|
|
1617
|
+
});
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
if (matched === 0) {
|
|
1621
|
+
accumulator.failedSampleCount += 1;
|
|
1622
|
+
return;
|
|
1623
|
+
}
|
|
1624
|
+
accumulator.sampleCount += 1;
|
|
1625
|
+
accumulator.peakMemoryBytes = Math.max(accumulator.peakMemoryBytes, aggregateMemoryBytes);
|
|
1626
|
+
accumulator.firstSampledAt ??= sampledAt;
|
|
1627
|
+
accumulator.lastSampledAt = sampledAt;
|
|
1628
|
+
}
|
|
1629
|
+
function finalizeSandboxMemoryTelemetry(accumulator) {
|
|
1630
|
+
if (accumulator.sampleCount === 0 || !accumulator.firstSampledAt || !accumulator.lastSampledAt)
|
|
1631
|
+
return null;
|
|
1632
|
+
return {
|
|
1633
|
+
schemaVersion: 1,
|
|
1634
|
+
source: "docker_stats_sampled",
|
|
1635
|
+
sampleIntervalMs: accumulator.sampleIntervalMs,
|
|
1636
|
+
sampleCount: accumulator.sampleCount,
|
|
1637
|
+
failedSampleCount: accumulator.failedSampleCount,
|
|
1638
|
+
peakMemoryBytes: accumulator.peakMemoryBytes,
|
|
1639
|
+
firstSampledAt: accumulator.firstSampledAt,
|
|
1640
|
+
lastSampledAt: accumulator.lastSampledAt,
|
|
1641
|
+
containers: [...accumulator.containerPeaks.values()].sort((left, right) => left.role.localeCompare(right.role) || left.name.localeCompare(right.name))
|
|
1642
|
+
};
|
|
1643
|
+
}
|
|
1644
|
+
function validSandboxPeakMemoryTelemetry(value) {
|
|
1645
|
+
return value?.schemaVersion === 1
|
|
1646
|
+
&& value.source === "docker_stats_sampled"
|
|
1647
|
+
&& Number.isInteger(value.sampleIntervalMs) && value.sampleIntervalMs >= 250 && value.sampleIntervalMs <= 30_000
|
|
1648
|
+
&& Number.isInteger(value.sampleCount) && value.sampleCount > 0
|
|
1649
|
+
&& Number.isInteger(value.failedSampleCount) && value.failedSampleCount >= 0
|
|
1650
|
+
&& Number.isSafeInteger(value.peakMemoryBytes) && value.peakMemoryBytes >= 0
|
|
1651
|
+
&& typeof value.firstSampledAt === "string" && Number.isFinite(Date.parse(value.firstSampledAt))
|
|
1652
|
+
&& typeof value.lastSampledAt === "string" && Number.isFinite(Date.parse(value.lastSampledAt))
|
|
1653
|
+
&& Array.isArray(value.containers) && value.containers.length <= 32
|
|
1654
|
+
&& value.containers.every(container => typeof container.containerId === "string" && /^[0-9a-f]{12,64}$/iu.test(container.containerId)
|
|
1655
|
+
&& typeof container.name === "string" && container.name.length > 0 && container.name.length <= 255 && !hasControlCharacters(container.name)
|
|
1656
|
+
&& typeof container.role === "string" && container.role.length > 0 && container.role.length <= 255 && !hasControlCharacters(container.role)
|
|
1657
|
+
&& Number.isSafeInteger(container.peakMemoryBytes) && container.peakMemoryBytes >= 0);
|
|
1658
|
+
}
|
|
1659
|
+
function waitForMemorySampleInterval(intervalMs, signal) {
|
|
1660
|
+
if (signal.aborted)
|
|
1661
|
+
return Promise.resolve();
|
|
1662
|
+
return new Promise(resolve => {
|
|
1663
|
+
const complete = () => {
|
|
1664
|
+
clearTimeout(timeout);
|
|
1665
|
+
signal.removeEventListener("abort", complete);
|
|
1666
|
+
resolve();
|
|
1667
|
+
};
|
|
1668
|
+
const timeout = setTimeout(complete, intervalMs);
|
|
1669
|
+
timeout.unref?.();
|
|
1670
|
+
signal.addEventListener("abort", complete, { once: true });
|
|
1671
|
+
});
|
|
1672
|
+
}
|
|
1673
|
+
function formatMemoryBytes(value) {
|
|
1674
|
+
if (value >= 1024 ** 3)
|
|
1675
|
+
return `${(value / 1024 ** 3).toFixed(2)} GiB`;
|
|
1676
|
+
if (value >= 1024 ** 2)
|
|
1677
|
+
return `${(value / 1024 ** 2).toFixed(1)} MiB`;
|
|
1678
|
+
if (value >= 1024)
|
|
1679
|
+
return `${(value / 1024).toFixed(1)} KiB`;
|
|
1680
|
+
return `${value} B`;
|
|
1681
|
+
}
|
|
1682
|
+
function appendRunOnceEvent(result, dispatchEvent) {
|
|
1683
|
+
return dispatchEvent ? { ...result, events: [...result.events, dispatchEvent] } : result;
|
|
1684
|
+
}
|
|
1324
1685
|
export function assertSandboxImportCurrent(expected, current, signal, nowMs = Date.now()) {
|
|
1325
1686
|
if (signal.aborted
|
|
1326
1687
|
|| !current
|
|
@@ -1330,16 +1691,89 @@ export function assertSandboxImportCurrent(expected, current, signal, nowMs = Da
|
|
|
1330
1691
|
throw new Error("sandbox_output_stale: output cannot move the managed ref because its attempt, fence, or lease is no longer current.");
|
|
1331
1692
|
}
|
|
1332
1693
|
}
|
|
1333
|
-
function
|
|
1334
|
-
return value
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1694
|
+
export function validSandboxLedger(value) {
|
|
1695
|
+
return normalizeSandboxLedger(value) !== null;
|
|
1696
|
+
}
|
|
1697
|
+
export function normalizeSandboxLedger(value) {
|
|
1698
|
+
if (validCurrentSandboxLedger(value))
|
|
1699
|
+
return value;
|
|
1700
|
+
if (!validLegacySandboxLedger(value))
|
|
1701
|
+
return null;
|
|
1702
|
+
// Schema v2 did not retain the reservation timestamp. Use the earlier of
|
|
1703
|
+
// its expiry and migration time so an old-instance ledger can transition
|
|
1704
|
+
// immediately during startup reaping, even when its lease has not expired.
|
|
1705
|
+
const createdAt = new Date(Math.min(Date.parse(value.expiresAt), Date.now())).toISOString();
|
|
1706
|
+
let runtimeLease = createRuntimeIsolationLease({
|
|
1707
|
+
leaseId: value.resourceLeaseId,
|
|
1708
|
+
dispatchAttemptId: value.dispatchAttemptId,
|
|
1709
|
+
executionLeaseId: value.resourceLeaseId,
|
|
1710
|
+
fencingToken: value.fencingToken,
|
|
1711
|
+
runnerId: value.labels["dev.forge.runner-id"],
|
|
1712
|
+
runnerInstanceId: value.runnerInstanceId,
|
|
1713
|
+
worktreeKey: `legacy/${value.dispatchAttemptId}`,
|
|
1714
|
+
worktreePath: "/workspace",
|
|
1715
|
+
mode: "docker",
|
|
1716
|
+
createdAt,
|
|
1717
|
+
expiresAt: value.expiresAt,
|
|
1718
|
+
resources: {
|
|
1719
|
+
memoryReservationBytes: 0,
|
|
1720
|
+
cpuMillis: 0,
|
|
1721
|
+
pids: 0,
|
|
1722
|
+
ports: [],
|
|
1723
|
+
paths: [{ path: "/workspace", role: "worktree" }],
|
|
1724
|
+
processSupervisorId: null,
|
|
1725
|
+
dockerResources: plannedDockerLeaseResources(value.resources, true, value.includeCredentials !== false),
|
|
1726
|
+
singletonKeys: [],
|
|
1727
|
+
},
|
|
1728
|
+
});
|
|
1729
|
+
runtimeLease = transitionRuntimeIsolationLease(runtimeLease, "starting", createdAt);
|
|
1730
|
+
runtimeLease = transitionRuntimeIsolationLease(runtimeLease, "active", createdAt);
|
|
1731
|
+
return {
|
|
1732
|
+
schemaVersion: 3,
|
|
1733
|
+
runtimeLease,
|
|
1734
|
+
resources: value.resources,
|
|
1735
|
+
labels: value.labels,
|
|
1736
|
+
includeCredentials: value.includeCredentials,
|
|
1737
|
+
peakMemory: value.peakMemory,
|
|
1738
|
+
};
|
|
1739
|
+
}
|
|
1740
|
+
function validCurrentSandboxLedger(value) {
|
|
1741
|
+
if (typeof value !== "object" || value === null)
|
|
1742
|
+
return false;
|
|
1743
|
+
const ledger = value;
|
|
1744
|
+
const lease = ledger.runtimeLease;
|
|
1745
|
+
return ledger.schemaVersion === 3
|
|
1746
|
+
&& validRuntimeIsolationLease(lease)
|
|
1747
|
+
&& lease.mode === "docker"
|
|
1748
|
+
&& typeof ledger.resources?.network === "string"
|
|
1749
|
+
&& ledger.labels?.["dev.forge.managed"] === "true"
|
|
1750
|
+
&& ledger.labels?.["dev.forge.runner-id"] === lease.runnerId
|
|
1751
|
+
&& ledger.labels?.["dev.forge.resource-lease-id"] === lease.leaseId
|
|
1752
|
+
&& ledger.labels?.["dev.forge.execution-lease-id"] === lease.executionLeaseId
|
|
1753
|
+
&& ledger.labels?.["dev.forge.dispatch-attempt-id"] === lease.dispatchAttemptId
|
|
1754
|
+
&& ledger.labels?.["dev.forge.fencing-token"] === lease.fencingToken
|
|
1755
|
+
&& ledger.labels?.["dev.forge.runner-instance-id"] === lease.runnerInstanceId
|
|
1756
|
+
&& (ledger.peakMemory == null || validSandboxPeakMemoryTelemetry(ledger.peakMemory));
|
|
1757
|
+
}
|
|
1758
|
+
function validLegacySandboxLedger(value) {
|
|
1759
|
+
if (typeof value !== "object" || value === null)
|
|
1760
|
+
return false;
|
|
1761
|
+
const ledger = value;
|
|
1762
|
+
return ledger.schemaVersion === 2
|
|
1763
|
+
&& isFullUuid(ledger.resourceLeaseId)
|
|
1764
|
+
&& isFullUuid(ledger.dispatchAttemptId)
|
|
1765
|
+
&& isFullUuid(ledger.fencingToken)
|
|
1766
|
+
&& isFullUuid(ledger.runnerInstanceId)
|
|
1767
|
+
&& Number.isFinite(Date.parse(ledger.expiresAt))
|
|
1768
|
+
&& typeof ledger.resources?.network === "string"
|
|
1769
|
+
&& ledger.labels?.["dev.forge.managed"] === "true"
|
|
1770
|
+
&& isFullUuid(ledger.labels?.["dev.forge.runner-id"])
|
|
1771
|
+
&& ledger.labels?.["dev.forge.resource-lease-id"] === ledger.resourceLeaseId
|
|
1772
|
+
&& ledger.labels?.["dev.forge.execution-lease-id"] === ledger.resourceLeaseId
|
|
1773
|
+
&& ledger.labels?.["dev.forge.dispatch-attempt-id"] === ledger.dispatchAttemptId
|
|
1774
|
+
&& ledger.labels?.["dev.forge.fencing-token"] === ledger.fencingToken
|
|
1775
|
+
&& ledger.labels?.["dev.forge.runner-instance-id"] === ledger.runnerInstanceId
|
|
1776
|
+
&& (ledger.peakMemory == null || validSandboxPeakMemoryTelemetry(ledger.peakMemory));
|
|
1343
1777
|
}
|
|
1344
1778
|
export function resourceNames(attemptHex, perAttemptCaches = []) { const prefix = `forge-${attemptHex}`; return { prefix, container: `forge-agent-${attemptHex}`, network: prefix, egressNetwork: `${prefix}-egress`, workspaceVolume: `${prefix}-workspace`, controlVolume: `${prefix}-control`, secretsVolume: `${prefix}-secrets`, credentialsVolume: `${prefix}-credentials`, egressConfigVolume: `${prefix}-egress-policy`, cacheVolumes: Object.fromEntries(perAttemptCaches.map(name => [name, `${prefix}-cache-${name}`])) }; }
|
|
1345
1779
|
export function dockerObjectId(output, kind) {
|
|
@@ -1364,9 +1798,41 @@ export function sandboxRuntimeCleanupVolumeNames(resources, includeCredentials =
|
|
|
1364
1798
|
.map(volume => volume.name)
|
|
1365
1799
|
.filter((name) => typeof name === "string" && name.length > 0);
|
|
1366
1800
|
}
|
|
1367
|
-
function
|
|
1801
|
+
function plannedDockerLeaseResources(resources, includeEgressPolicy, includeCredentials) {
|
|
1802
|
+
return [
|
|
1803
|
+
{ type: "container", name: resources.container, role: "agent" },
|
|
1804
|
+
{ type: "network", name: resources.network, role: "internal" },
|
|
1805
|
+
...(includeEgressPolicy ? [{ type: "network", name: resources.egressNetwork, role: "egress" }] : []),
|
|
1806
|
+
...sandboxRuntimeVolumes(resources, includeEgressPolicy, includeCredentials)
|
|
1807
|
+
.map(volume => ({ type: "volume", name: volume.name, role: volume.role })),
|
|
1808
|
+
];
|
|
1809
|
+
}
|
|
1810
|
+
function activeDockerLeaseResources(containers, networks, volumes) {
|
|
1811
|
+
return [
|
|
1812
|
+
...containers.map(resource => ({ type: "container", ...resource })),
|
|
1813
|
+
...networks.map(resource => ({ type: "network", ...resource })),
|
|
1814
|
+
...volumes.map(resource => ({ type: "volume", ...resource })),
|
|
1815
|
+
];
|
|
1816
|
+
}
|
|
1817
|
+
function transitionForCleanup(lease, state) {
|
|
1818
|
+
if (lease.state === state)
|
|
1819
|
+
return lease;
|
|
1820
|
+
return transitionRuntimeIsolationLease(lease, state);
|
|
1821
|
+
}
|
|
1822
|
+
export function sandboxResourceLabels(dispatch, image, contract) {
|
|
1823
|
+
const resourceLeaseId = dispatch.sandbox.executionLeaseId;
|
|
1824
|
+
const ownershipIds = [
|
|
1825
|
+
["resource lease", resourceLeaseId],
|
|
1826
|
+
["dispatch attempt", dispatch.sandbox.dispatchAttemptId],
|
|
1827
|
+
["runner", dispatch.execution.runnerId],
|
|
1828
|
+
["runner instance", dispatch.sandbox.runnerInstanceId],
|
|
1829
|
+
["fencing token", dispatch.lease.fencingToken]
|
|
1830
|
+
];
|
|
1831
|
+
const invalid = ownershipIds.find(([, value]) => !isFullUuid(value));
|
|
1832
|
+
if (invalid)
|
|
1833
|
+
throw new Error(`sandbox_identity_invalid: ${invalid[0]} id must be a full UUID.`);
|
|
1368
1834
|
return {
|
|
1369
|
-
"dev.forge.managed": "true", "dev.forge.session-id": dispatch.session.id, "dev.forge.
|
|
1835
|
+
"dev.forge.managed": "true", "dev.forge.session-id": dispatch.session.id, "dev.forge.resource-lease-id": resourceLeaseId, "dev.forge.execution-lease-id": resourceLeaseId,
|
|
1370
1836
|
"dev.forge.dispatch-attempt-id": dispatch.sandbox.dispatchAttemptId, "dev.forge.fencing-token": dispatch.lease.fencingToken,
|
|
1371
1837
|
"dev.forge.runner-id": dispatch.execution.runnerId ?? "", "dev.forge.runner-instance-id": dispatch.sandbox.runnerInstanceId,
|
|
1372
1838
|
"dev.forge.project": dispatch.repo.name.slice(0, 120), "dev.forge.image-digest": image, "dev.forge.contract-digest": contract,
|
|
@@ -1377,31 +1843,47 @@ function labelArgs(labels) { return Object.entries(labels).flatMap(([key, value]
|
|
|
1377
1843
|
function labelsMatch(actual, expected) { return Object.entries(expected).every(([key, value]) => actual[key] === value); }
|
|
1378
1844
|
export function mayReapManagedResource(labels, runnerId, liveAttempts) {
|
|
1379
1845
|
const attemptId = labels["dev.forge.dispatch-attempt-id"];
|
|
1846
|
+
const resourceLeaseId = labels["dev.forge.resource-lease-id"];
|
|
1380
1847
|
return labels["dev.forge.managed"] === "true"
|
|
1381
1848
|
&& labels["dev.forge.runner-id"] === runnerId
|
|
1382
|
-
&&
|
|
1849
|
+
&& isFullUuid(resourceLeaseId)
|
|
1850
|
+
&& labels["dev.forge.execution-lease-id"] === resourceLeaseId
|
|
1851
|
+
&& isFullUuid(attemptId)
|
|
1852
|
+
&& isFullUuid(labels["dev.forge.runner-instance-id"])
|
|
1853
|
+
&& isFullUuid(labels["dev.forge.fencing-token"])
|
|
1383
1854
|
&& !liveAttempts.has(attemptId);
|
|
1384
1855
|
}
|
|
1856
|
+
function isFullUuid(value) {
|
|
1857
|
+
return typeof value === "string" && /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/iu.test(value);
|
|
1858
|
+
}
|
|
1385
1859
|
export function shouldReapSandboxLedger(ledger, currentRunnerInstanceId, serverAttempt, snapshotAvailable, nowMs = Date.now()) {
|
|
1386
|
-
|
|
1860
|
+
const lease = "runtimeLease" in ledger ? ledger.runtimeLease : ledger;
|
|
1861
|
+
if (lease.runnerInstanceId !== currentRunnerInstanceId)
|
|
1387
1862
|
return true;
|
|
1388
1863
|
if (!snapshotAvailable)
|
|
1389
|
-
return Date.parse(
|
|
1864
|
+
return Date.parse(lease.expiresAt) <= nowMs;
|
|
1390
1865
|
return !serverAttempt
|
|
1391
|
-
|| serverAttempt.dispatchAttemptId !==
|
|
1392
|
-
|| serverAttempt.fencingToken !==
|
|
1393
|
-
|| serverAttempt.runnerInstanceId !==
|
|
1866
|
+
|| serverAttempt.dispatchAttemptId !== lease.dispatchAttemptId
|
|
1867
|
+
|| serverAttempt.fencingToken !== lease.fencingToken
|
|
1868
|
+
|| serverAttempt.runnerInstanceId !== lease.runnerInstanceId
|
|
1394
1869
|
|| Date.parse(serverAttempt.expiresAt) <= nowMs;
|
|
1395
1870
|
}
|
|
1396
1871
|
function dockerLabelValue(value) { return value.replace(/[\r\n\0]/gu, " ").trim().slice(0, 120) || "unknown"; }
|
|
1397
|
-
function imageLabelArgs(project, key, kind) {
|
|
1872
|
+
function imageLabelArgs(project, key, kind, bridgeRuntimeDigest) {
|
|
1398
1873
|
return [
|
|
1399
1874
|
"--label", managedLabel,
|
|
1400
1875
|
"--label", `dev.forge.project=${dockerLabelValue(project)}`,
|
|
1401
1876
|
"--label", `dev.forge.image-key=${key}`,
|
|
1402
|
-
"--label", `dev.forge.image-kind=${kind}
|
|
1877
|
+
"--label", `dev.forge.image-kind=${kind}`,
|
|
1878
|
+
"--label", `${bridgeRuntimeDigestLabel}=${bridgeRuntimeDigest}`
|
|
1403
1879
|
];
|
|
1404
1880
|
}
|
|
1881
|
+
export function bridgeRuntimeCacheMatches(record, expectedDigest, imageLabels) {
|
|
1882
|
+
return /^[0-9a-f]{64}$/i.test(expectedDigest)
|
|
1883
|
+
&& record?.bridgeRuntimeDigest === expectedDigest
|
|
1884
|
+
&& imageLabels.length > 0
|
|
1885
|
+
&& imageLabels.every(labels => labels?.[bridgeRuntimeDigestLabel] === expectedDigest);
|
|
1886
|
+
}
|
|
1405
1887
|
function validImageCacheRecord(value) {
|
|
1406
1888
|
return value?.schemaVersion === 1
|
|
1407
1889
|
&& /^[0-9a-f]{64}$/i.test(value.key)
|
|
@@ -1411,25 +1893,213 @@ function validImageCacheRecord(value) {
|
|
|
1411
1893
|
&& [value.projectTag, value.helperTag, value.overlayTag].every(tag => /^forge\/(?:project|tooling|agent):[0-9a-f]{64}$/i.test(tag))
|
|
1412
1894
|
&& /^sha256:[0-9a-f]{64}$/i.test(value.imageDigest)
|
|
1413
1895
|
&& /^sha256:[0-9a-f]{64}$/i.test(value.helperImageDigest)
|
|
1896
|
+
&& (value.bridgeRuntimeDigest === undefined || /^[0-9a-f]{64}$/i.test(value.bridgeRuntimeDigest))
|
|
1414
1897
|
&& /^[0-9a-f]{64}$/i.test(value.policyDigest);
|
|
1415
1898
|
}
|
|
1416
|
-
export function
|
|
1899
|
+
export async function prepareOverlayBuildRoot(overlayRoot) {
|
|
1900
|
+
const buildRoot = path.dirname(overlayRoot);
|
|
1901
|
+
await fs.mkdir(buildRoot, { recursive: true, mode: 0o700 });
|
|
1902
|
+
await fs.chmod(buildRoot, 0o700);
|
|
1903
|
+
await fs.rm(overlayRoot, { recursive: true, force: true });
|
|
1904
|
+
await fs.mkdir(overlayRoot, { recursive: true, mode: 0o700 });
|
|
1905
|
+
await fs.chmod(overlayRoot, 0o700);
|
|
1906
|
+
const bridgeRoot = path.join(overlayRoot, "bridge");
|
|
1907
|
+
await fs.mkdir(bridgeRoot, { recursive: true, mode: 0o700 });
|
|
1908
|
+
await fs.chmod(bridgeRoot, 0o700);
|
|
1909
|
+
}
|
|
1910
|
+
export async function extractOciLayoutArchive(archivePath, destination, signal) {
|
|
1911
|
+
await fs.rm(destination, { recursive: true, force: true });
|
|
1912
|
+
await fs.mkdir(destination, { recursive: true, mode: 0o700 });
|
|
1913
|
+
await fs.chmod(destination, 0o700);
|
|
1914
|
+
const result = await runSandboxProcess({
|
|
1915
|
+
command: process.platform === "win32" ? "tar.exe" : "tar",
|
|
1916
|
+
args: ["-xf", archivePath, "-C", destination],
|
|
1917
|
+
timeoutMs: 5 * 60_000,
|
|
1918
|
+
maxOutputBytes: 1024 * 1024,
|
|
1919
|
+
signal
|
|
1920
|
+
});
|
|
1921
|
+
if (result.exitCode !== 0 || result.truncated) {
|
|
1922
|
+
throw new Error(`sandbox_build_failed: project OCI layout extraction failed with exit code ${result.exitCode}; ${formatOciExtractionFailure(result)}. Archive retained at ${archivePath}.`);
|
|
1923
|
+
}
|
|
1924
|
+
await fs.rm(archivePath, { force: true }).catch(() => undefined);
|
|
1925
|
+
}
|
|
1926
|
+
export function ociLayoutContextRef(layoutPath) {
|
|
1927
|
+
if (/^[A-Za-z]:[\\/]/u.test(layoutPath)) {
|
|
1928
|
+
throw new Error("sandbox_build_failed: Windows absolute OCI layout paths are not BuildKit-safe; use a relative path from the build working directory.");
|
|
1929
|
+
}
|
|
1930
|
+
const normalized = layoutPath.replace(/\\/gu, "/").replace(/^\.\//u, "");
|
|
1931
|
+
// BuildKit's oci-layout resolver expects a raw filesystem path here; it does not percent-decode
|
|
1932
|
+
// URI path escapes such as %20 before opening index.json. On Windows, absolute drive paths
|
|
1933
|
+
// such as D:/state/project-oci are parsed as OCI layout ref/tag syntax, so callers should
|
|
1934
|
+
// run buildx from the layout parent and pass a relative path (for example project-oci).
|
|
1935
|
+
return `oci-layout://${normalized}`;
|
|
1936
|
+
}
|
|
1937
|
+
export function formatOciExtractionFailure(result) {
|
|
1938
|
+
const output = result.stderr.trim() || result.stdout.trim() || "no tar output";
|
|
1939
|
+
return `tar output: ${output.slice(0, 2048)}${result.truncated ? " (output truncated)" : ""}`;
|
|
1940
|
+
}
|
|
1941
|
+
export async function readOciLayoutImageDigest(layoutRoot) {
|
|
1942
|
+
const indexPath = path.join(layoutRoot, "index.json");
|
|
1943
|
+
const index = JSON.parse(await fs.readFile(indexPath, "utf8"));
|
|
1944
|
+
const manifest = index.manifests?.find(entry => typeof entry.digest === "string"
|
|
1945
|
+
&& /^sha256:[0-9a-f]{64}$/iu.test(entry.digest)
|
|
1946
|
+
&& entry.annotations?.["vnd.docker.reference.type"] !== "attestation-manifest");
|
|
1947
|
+
if (!manifest || typeof manifest.digest !== "string") {
|
|
1948
|
+
throw new Error("sandbox_build_failed: project OCI layout did not expose an immutable image digest.");
|
|
1949
|
+
}
|
|
1950
|
+
return manifest.digest;
|
|
1951
|
+
}
|
|
1952
|
+
export async function removeExtractedOciLayout(layoutRoot) {
|
|
1953
|
+
await fs.rm(layoutRoot, { recursive: true, force: true });
|
|
1954
|
+
}
|
|
1955
|
+
export function ociLayoutScanTarget(layoutPath) {
|
|
1956
|
+
const absolute = /^[A-Za-z]:[\\/]/u.test(layoutPath) ? layoutPath : path.resolve(layoutPath);
|
|
1957
|
+
return `oci-dir:${absolute.replace(/\\/gu, "/")}`;
|
|
1958
|
+
}
|
|
1959
|
+
export function agentOverlayDockerignore() {
|
|
1960
|
+
return "project.oci.tar\nproject-oci\nproject-oci/**\n";
|
|
1961
|
+
}
|
|
1962
|
+
export function sandboxImageCacheKey(input) {
|
|
1963
|
+
return createHash("sha256").update([
|
|
1964
|
+
input.contextDigest,
|
|
1965
|
+
input.contractDigest,
|
|
1966
|
+
input.platform,
|
|
1967
|
+
input.architecture,
|
|
1968
|
+
input.bridgeVersion,
|
|
1969
|
+
input.bridgeRuntimeDigest,
|
|
1970
|
+
input.toolingImage,
|
|
1971
|
+
input.codexVersion,
|
|
1972
|
+
input.policy,
|
|
1973
|
+
input.policyConfigurationDigest
|
|
1974
|
+
].join("\0")).digest("hex");
|
|
1975
|
+
}
|
|
1976
|
+
export async function bridgeRuntimeContentDigest(packageRoot) {
|
|
1977
|
+
const root = path.resolve(packageRoot);
|
|
1978
|
+
const packageManifest = await requireBridgeRuntimeFile(root, "package.json");
|
|
1979
|
+
const lockManifest = await optionalBridgeRuntimeFile(root, "package-lock.json");
|
|
1980
|
+
const distRoot = path.join(root, "dist");
|
|
1981
|
+
const distStat = await fs.lstat(distRoot).catch(() => null);
|
|
1982
|
+
if (!distStat?.isDirectory() || distStat.isSymbolicLink()) {
|
|
1983
|
+
throw new Error("sandbox_build_failed: bridge runtime dist must be a real directory.");
|
|
1984
|
+
}
|
|
1985
|
+
const distFiles = await collectBridgeRuntimeFiles(root, "dist");
|
|
1986
|
+
if (distFiles.length === 0) {
|
|
1987
|
+
throw new Error("sandbox_build_failed: bridge runtime dist is empty.");
|
|
1988
|
+
}
|
|
1989
|
+
const files = [packageManifest, ...(lockManifest ? [lockManifest] : []), ...distFiles]
|
|
1990
|
+
.sort((left, right) => compareBridgeRuntimePaths(left.relativePath, right.relativePath));
|
|
1991
|
+
const digest = createHash("sha256");
|
|
1992
|
+
digest.update("forge-bridge-runtime-v1\0");
|
|
1993
|
+
digest.update(`${files.length}\0`);
|
|
1994
|
+
for (const file of files) {
|
|
1995
|
+
const contents = await fs.readFile(file.absolutePath);
|
|
1996
|
+
digest.update(`${Buffer.byteLength(file.relativePath, "utf8")}:${file.relativePath}\0`);
|
|
1997
|
+
digest.update(`${contents.length}:`);
|
|
1998
|
+
digest.update(contents);
|
|
1999
|
+
digest.update("\0");
|
|
2000
|
+
}
|
|
2001
|
+
return digest.digest("hex");
|
|
2002
|
+
}
|
|
2003
|
+
async function requireBridgeRuntimeFile(root, relativePath) {
|
|
2004
|
+
const file = await optionalBridgeRuntimeFile(root, relativePath);
|
|
2005
|
+
if (!file)
|
|
2006
|
+
throw new Error(`sandbox_build_failed: bridge runtime ${relativePath} is missing.`);
|
|
2007
|
+
return file;
|
|
2008
|
+
}
|
|
2009
|
+
async function optionalBridgeRuntimeFile(root, relativePath) {
|
|
2010
|
+
const absolutePath = path.join(root, relativePath);
|
|
2011
|
+
const stat = await fs.lstat(absolutePath).catch(() => null);
|
|
2012
|
+
if (!stat)
|
|
2013
|
+
return null;
|
|
2014
|
+
if (!stat.isFile() || stat.isSymbolicLink()) {
|
|
2015
|
+
throw new Error(`sandbox_build_failed: bridge runtime ${relativePath} must be a regular file.`);
|
|
2016
|
+
}
|
|
2017
|
+
return { absolutePath, relativePath: relativePath.replace(/\\/gu, "/") };
|
|
2018
|
+
}
|
|
2019
|
+
async function collectBridgeRuntimeFiles(root, relativeDirectory) {
|
|
2020
|
+
const absoluteDirectory = path.join(root, relativeDirectory);
|
|
2021
|
+
const entries = (await fs.readdir(absoluteDirectory, { withFileTypes: true }))
|
|
2022
|
+
.sort((left, right) => compareBridgeRuntimePaths(left.name, right.name));
|
|
2023
|
+
const files = [];
|
|
2024
|
+
for (const entry of entries) {
|
|
2025
|
+
const relativePath = path.join(relativeDirectory, entry.name);
|
|
2026
|
+
if (entry.isSymbolicLink()) {
|
|
2027
|
+
throw new Error(`sandbox_build_failed: bridge runtime ${relativePath.replace(/\\/gu, "/")} cannot be a symbolic link.`);
|
|
2028
|
+
}
|
|
2029
|
+
if (entry.isDirectory()) {
|
|
2030
|
+
files.push(...await collectBridgeRuntimeFiles(root, relativePath));
|
|
2031
|
+
continue;
|
|
2032
|
+
}
|
|
2033
|
+
if (!entry.isFile()) {
|
|
2034
|
+
throw new Error(`sandbox_build_failed: bridge runtime ${relativePath.replace(/\\/gu, "/")} must be a regular file.`);
|
|
2035
|
+
}
|
|
2036
|
+
files.push({
|
|
2037
|
+
absolutePath: path.join(root, relativePath),
|
|
2038
|
+
relativePath: relativePath.replace(/\\/gu, "/")
|
|
2039
|
+
});
|
|
2040
|
+
}
|
|
2041
|
+
return files;
|
|
2042
|
+
}
|
|
2043
|
+
function compareBridgeRuntimePaths(left, right) {
|
|
2044
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
2045
|
+
}
|
|
2046
|
+
export function agentOverlayDockerfile(toolingImage, projectContextName, installMode, codexVersion) {
|
|
2047
|
+
return `FROM ${toolingImage} AS tooling
|
|
2048
|
+
WORKDIR /opt/forge-bridge
|
|
2049
|
+
COPY bridge/package*.json ./
|
|
2050
|
+
RUN ${bridgePackageInstallCommand(installMode, codexVersion)}
|
|
2051
|
+
COPY bridge/dist ./dist
|
|
2052
|
+
FROM ${projectContextName}
|
|
2053
|
+
USER 0
|
|
2054
|
+
COPY --from=tooling /usr/local/bin/node /usr/local/bin/node
|
|
2055
|
+
COPY --from=tooling /usr/local/lib/node_modules/@openai/codex /usr/local/lib/node_modules/@openai/codex
|
|
2056
|
+
COPY --from=tooling /opt/forge-bridge /opt/forge-bridge
|
|
2057
|
+
RUN ln -sfn ../lib/node_modules/@openai/codex/bin/codex.js /usr/local/bin/codex \
|
|
2058
|
+
&& (getent group 10001 || groupadd -g 10001 forge) \
|
|
2059
|
+
&& (getent passwd 10001 || useradd -u 10001 -g 10001 -d /home/forge -m forge)
|
|
2060
|
+
ENV HOME=/home/forge TMPDIR=/tmp
|
|
2061
|
+
ENTRYPOINT []
|
|
2062
|
+
CMD []
|
|
2063
|
+
USER 10001:10001
|
|
2064
|
+
`;
|
|
2065
|
+
}
|
|
2066
|
+
export async function copyBridgePackageManifests(packageRoot, bridgeRoot) {
|
|
2067
|
+
await fs.copyFile(path.join(packageRoot, "package.json"), path.join(bridgeRoot, "package.json"));
|
|
2068
|
+
return await copyOptionalBridgeManifest(packageRoot, bridgeRoot, "package-lock.json") ? "ci" : "install";
|
|
2069
|
+
}
|
|
2070
|
+
export function bridgePackageInstallCommand(mode, codexVersion) {
|
|
2071
|
+
const bridgeInstall = mode === "ci"
|
|
2072
|
+
? "npm ci --omit=dev --ignore-scripts"
|
|
2073
|
+
: "npm install --omit=dev --ignore-scripts";
|
|
2074
|
+
return `${bridgeInstall} && npm install --global @openai/codex@${codexVersion} && npm cache clean --force`;
|
|
2075
|
+
}
|
|
2076
|
+
async function copyOptionalBridgeManifest(packageRoot, bridgeRoot, fileName) {
|
|
2077
|
+
try {
|
|
2078
|
+
await fs.copyFile(path.join(packageRoot, fileName), path.join(bridgeRoot, fileName));
|
|
2079
|
+
return true;
|
|
2080
|
+
}
|
|
2081
|
+
catch (error) {
|
|
2082
|
+
if (error.code === "ENOENT")
|
|
2083
|
+
return false;
|
|
2084
|
+
throw error;
|
|
2085
|
+
}
|
|
2086
|
+
}
|
|
2087
|
+
export function buildxBuildArgs(builder, metadataFile, tail, environment = process.env, options = {}) {
|
|
1417
2088
|
if (!/^forge-[0-9a-f]{20}$/u.test(builder))
|
|
1418
2089
|
throw new Error("sandbox_build_failed: invalid dedicated builder name.");
|
|
1419
2090
|
const resources = buildResourcePolicy(environment);
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
"--
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
...tail
|
|
1426
|
-
];
|
|
2091
|
+
const args = ["buildx", "build", "--builder", builder];
|
|
2092
|
+
if (options.load !== false)
|
|
2093
|
+
args.push("--load");
|
|
2094
|
+
args.push("--ulimit", `nofile=${resources.nofile}:${resources.nofile}`, "--shm-size", resources.sharedMemory, "--provenance=mode=max", "--sbom=true", "--metadata-file", metadataFile, ...tail);
|
|
2095
|
+
return args;
|
|
1427
2096
|
}
|
|
1428
2097
|
export function buildResourcePolicy(environment = process.env) {
|
|
1429
2098
|
return {
|
|
1430
2099
|
nofile: boundedInteger(environment.FORGE_SANDBOX_BUILD_NOFILE, 4_096, 256, 65_536),
|
|
1431
2100
|
sharedMemory: "256m",
|
|
1432
|
-
builderPidsLimit: boundedInteger(environment.FORGE_SANDBOX_BUILDER_PIDS_LIMIT, 512, 128, 4_096)
|
|
2101
|
+
builderPidsLimit: boundedInteger(environment.FORGE_SANDBOX_BUILDER_PIDS_LIMIT, 512, 128, 4_096),
|
|
2102
|
+
builderMemoryBytes: boundedInteger(environment.FORGE_SANDBOX_BUILDER_MEMORY_BYTES, 3 * 1024 ** 3, 512 * 1024 ** 2, 16 * 1024 ** 3)
|
|
1433
2103
|
};
|
|
1434
2104
|
}
|
|
1435
2105
|
export function dedicatedBuildxBuilderName(runnerId, dockerContext) {
|
|
@@ -1469,7 +2139,17 @@ async function sandboxProviderAuthForDispatch(dispatch) {
|
|
|
1469
2139
|
}
|
|
1470
2140
|
export function workerCreateArgs(dispatch, contract, resources, labels, image, securityOptions = resolveSandboxSecurityOptions(dispatch.sandbox.executionPolicy)) {
|
|
1471
2141
|
const r = dispatch.sandbox.requestedResources;
|
|
1472
|
-
return ["create", "--name", resources.container, ...labelArgs(labels), "--network", resources.network, "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", ...securityOptions, "--pids-limit", String(r.pids), "--memory", String(r.
|
|
2142
|
+
return ["create", "--interactive", "--name", resources.container, ...labelArgs(labels), "--network", resources.network, "--read-only", "--cap-drop", "ALL", "--security-opt", "no-new-privileges", ...securityOptions, "--pids-limit", String(r.pids), "--memory", String(r.workerMemoryLimitBytes), "--memory-swap", String(r.workerMemoryLimitBytes), "--cpus", String(r.cpuMillis / 1000), "--ulimit", "nofile=4096:4096", "--ulimit", "core=0", "--tmpfs", "/tmp:rw,nosuid,nodev,size=1g,uid=10001,gid=10001", "--tmpfs", "/run/forge:rw,nosuid,nodev,noexec,size=32m,uid=10001,gid=10001", "--user", "10001:10001", "--workdir", contract.contract.workingDirectory, "--init", "--stop-timeout", "10", "--entrypoint", "node", "--log-driver", "none", "--mount", `type=volume,src=${resources.workspaceVolume},dst=/workspace,volume-nocopy`, "--volume", `${resources.controlVolume}:/control`, "--volume", `${resources.secretsVolume}:/run/forge-secrets:ro`, ...providerAuthVolumeArgs(dispatch, resources), ...cacheVolumeArgs(contract, resources), "--env", `FORGE_SESSION_ID=${dispatch.session.id}`, "--env", `FORGE_DISPATCH_ATTEMPT_ID=${dispatch.sandbox.dispatchAttemptId}`, "--env", `FORGE_RESOURCE_LEASE_ID=${dispatch.sandbox.executionLeaseId}`, "--env", `FORGE_EXECUTION_LEASE_ID=${dispatch.sandbox.executionLeaseId}`, "--env", `FORGE_FENCING_TOKEN=${dispatch.lease.fencingToken}`, "--env", "FORGE_RUNTIME_MODE=docker", "--env", `FORGE_SANDBOX_ID=${dispatch.sandbox.dispatchAttemptId}`, "--env", "FORGE_WORKSPACE=/workspace", "--env", "FORGE_WORKTREE_PATH=/workspace", "--env", "FORGE_TMPDIR=/tmp", "--env", "FORGE_LOG_DIR=/control/output/logs", "--env", "FORGE_BROWSER_PROFILE_DIR=/tmp/forge-browser-profile", "--env", `FORGE_INPUT_COMMIT_SHA=${dispatch.sandbox.inputCommitSha}`, "--env", "FORGE_INPUT_BUNDLE=/control/input/repo.bundle", ...proxyEnvironmentArgs(dispatch, contract, resources), "--env", "HOME=/home/forge", "--env", "TMPDIR=/tmp", image, "/opt/forge-bridge/dist/sandbox/worker.js"];
|
|
2143
|
+
}
|
|
2144
|
+
export function validateSandboxMemoryReservation(dispatch, contract) {
|
|
2145
|
+
const requested = dispatch.sandbox.requestedResources;
|
|
2146
|
+
const requiredServiceMemoryBytes = contract.contract.services.length * 1024 ** 3;
|
|
2147
|
+
const requiredRuntimeOverheadBytes = 128 * 1024 ** 2
|
|
2148
|
+
+ (dispatch.sandbox.networkPolicy.mode === "public_allowlist" ? 256 * 1024 ** 2 : 0);
|
|
2149
|
+
if (requested.serviceMemoryLimitBytes < requiredServiceMemoryBytes
|
|
2150
|
+
|| requested.runtimeOverheadBytes < requiredRuntimeOverheadBytes) {
|
|
2151
|
+
throw new Error(`sandbox_capacity_profile_mismatch: reserved ${requested.serviceMemoryLimitBytes} service bytes and ${requested.runtimeOverheadBytes} runtime overhead bytes; the contract requires at least ${requiredServiceMemoryBytes} service bytes and ${requiredRuntimeOverheadBytes} runtime overhead bytes.`);
|
|
2152
|
+
}
|
|
1473
2153
|
}
|
|
1474
2154
|
function providerAuthVolumeArgs(dispatch, resources) {
|
|
1475
2155
|
const adapter = sandboxProviderAuthAdapterFor(dispatch.execution.provider, dispatch.execution.authKind);
|
|
@@ -1485,7 +2165,7 @@ export function cacheVolumeArgs(contract, resources) {
|
|
|
1485
2165
|
};
|
|
1486
2166
|
return contract.contract.caches
|
|
1487
2167
|
.filter(cache => cache.mode === "per_attempt")
|
|
1488
|
-
.flatMap(cache => ["--
|
|
2168
|
+
.flatMap(cache => ["--mount", `type=volume,src=${resources.cacheVolumes[cache.name]},dst=${targets[cache.name]},volume-nocopy`]);
|
|
1489
2169
|
}
|
|
1490
2170
|
export function proxyEnvironmentArgs(dispatch, contract, resources) {
|
|
1491
2171
|
if (dispatch.sandbox.networkPolicy.mode !== "public_allowlist")
|
|
@@ -1554,10 +2234,10 @@ export function resolveSandboxSecurityOptions(executionPolicy, environment = pro
|
|
|
1554
2234
|
...(lsm ? ["--security-opt", lsm] : [])
|
|
1555
2235
|
];
|
|
1556
2236
|
}
|
|
1557
|
-
export function networkCreateArgs(dispatch, environment = process.env) {
|
|
2237
|
+
export function networkCreateArgs(dispatch, environment = process.env, allowOpenPublicSandbox = environment.FORGE_ALLOW_OPEN_PUBLIC_SANDBOX === "true") {
|
|
1558
2238
|
const policy = dispatch.sandbox.networkPolicy;
|
|
1559
2239
|
if (policy.mode === "open_public") {
|
|
1560
|
-
if (dispatch.sandbox.executionPolicy === "sandbox_required" ||
|
|
2240
|
+
if (dispatch.sandbox.executionPolicy === "sandbox_required" || !allowOpenPublicSandbox) {
|
|
1561
2241
|
throw new Error("policy_mismatch: open_public sandbox networking is not permitted by this runner policy.");
|
|
1562
2242
|
}
|
|
1563
2243
|
return [];
|