@wardby/cli 0.2.1 → 0.4.0

package/dist/providers/jobs/docker.d.ts

@@ -1,4 +1,14 @@
+import { type CollectExclusions } from "../../coding/collect-exclude.js";
+import type { CodingProvider } from "../../coding/provider.js";
+import { type ServiceStateReporter } from "./service-state.js";
 import type { JobHandle, JobResult, JobSpec, JobStatus, WorkspaceJobLauncher } from "./types.js";
+/** How long one service image pull may take before the run fails as service_unready. */
+export declare const SERVICE_IMAGE_PULL_TIMEOUT_MS = 300000;
+/**
+ * The default bound on a run's whole service start-up (every service's create, start, inspection and
+ * readiness probing; image pulls excluded), matching the Kubernetes launcher's pod-start bound.
+ */
+export declare const DEFAULT_SERVICE_READY_TIMEOUT_MS = 120000;
 export declare const SAFE_WORKER_DIAGNOSTIC: RegExp;
 export interface WorkerDiagnostic {
     diagnostic: string;
@@ -14,6 +24,8 @@ export declare function parseWorkerDiagnosticLine(line: string): WorkerDiagnosti
 export interface DockerCommandOptions {
     env?: Record<string, string>;
     maxOutputBytes?: number;
+    /** Kill the Docker client and fail after this long (service readiness probes and image pulls). */
+    timeoutMs?: number;
 }
 export interface DockerCommandResult {
     stdout: string;
@@ -58,7 +70,7 @@ export declare class NodeDockerCommandRunner implements DockerCommandRunner {
 export interface DockerArtifactTransfer {
     seedDirectory(sourceDirectory: string, container: string, destination: string): Promise<void>;
     seedInput(sourceFile: string, container: string): Promise<void>;
-    materializeDirectory(container: string, source: string, destination: string, maxBytes: number): Promise<void>;
+    materializeDirectory(container: string, source: string, destination: string, maxBytes: number, exclusions: CollectExclusions): Promise<void>;
 }
 export declare function dockerTransferEnvironment(path: string): NodeJS.ProcessEnv;
 /** Streams tar archives into the unprivileged keeper; no host bind mounts are used. */
@@ -68,7 +80,7 @@ export declare class NodeDockerArtifactTransfer implements DockerArtifactTransfe
     constructor(dockerBinary?: string, path?: string);
     seedDirectory(sourceDirectory: string, container: string, destination: string): Promise<void>;
     seedInput(sourceFile: string, container: string): Promise<void>;
-    materializeDirectory(container: string, source: string, destination: string, maxBytes: number): Promise<void>;
+    materializeDirectory(container: string, source: string, destination: string, maxBytes: number, exclusions: CollectExclusions): Promise<void>;
 }
 export declare function validateMaterializedWorkspace(root: string, maxBytes: number): Promise<void>;
 export interface DockerJobLauncherOptions {
@@ -81,6 +93,15 @@ export interface DockerJobLauncherOptions {
     docker?: DockerCommandRunner;
     transfer?: DockerArtifactTransfer;
     now?: () => number;
+    /** Waits between service readiness probes; tests replace it. */
+    sleep?: (milliseconds: number) => Promise<void>;
+    /**
+     * The bound on a run's whole service start-up, image pulls excluded (default
+     * DEFAULT_SERVICE_READY_TIMEOUT_MS). Start-up never runs past the run's own deadline either.
+     */
+    serviceReadyTimeoutMs?: number;
+    /** Display-only observer of each service's start-up state (the viewer); its errors are ignored. */
+    onServiceState?: ServiceStateReporter;
     /** Optional observer invoked after provisioning fails but before resources are cleaned up. */
     onProvisionFailure?: (context: {
         runId: string;
@@ -94,10 +115,14 @@ export declare class DockerJobLauncher implements WorkspaceJobLauncher {
     private readonly docker;
     private readonly transfer;
     private readonly now;
+    private readonly sleep;
+    private readonly serviceReadyTimeoutMs;
     private readonly locks;
     private readonly timers;
     private readonly ready;
     constructor(options: DockerJobLauncherOptions);
+    /** Starts JobSpec.services (a network keeper plus one hardened container per service) for Codex and Claude Code runs. */
+    supportsServicesFor(provider: CodingProvider): boolean;
     launch(spec: JobSpec): Promise<JobHandle>;
     status(handle: JobHandle): Promise<JobStatus>;
     collect(handle: JobHandle): Promise<JobResult>;
@@ -107,8 +132,32 @@ export declare class DockerJobLauncher implements WorkspaceJobLauncher {
     private initialize;
     private launchLocked;
     private provision;
+    /**
+     * A run with services (docs/coding-services.md). The network keeper joins the run's internal
+     * network and owns the namespace; each service joins it and must pass its catalog readiness
+     * command before the next one starts, as Kubernetes starts native sidecars. The whole start-up is
+     * bounded (ServiceStartBudget): running out fails as coding_service_unready for the service being
+     * waited on. Returns the keeper's container ID, which Docker may report as the services' and
+     * worker's network mode.
+     */
+    private startServices;
+    /**
+     * A local copy of the digest-pinned image is used as is; otherwise it is pulled, bounded by its
+     * own limit and the run's deadline. Pull time does not count toward the start-up budget.
+     */
+    private ensureServiceImage;
+    /**
+     * The catalog readiness command, like a Kubernetes startup probe: every periodSeconds, each
+     * attempt bounded by timeoutSeconds, failing after failureThreshold consecutive failures. A
+     * service that exited or vanished fails at once, and so does one still unready when the start-up
+     * budget runs out (each probe and wait is clipped to what is left of it). Probe output is never
+     * logged or kept, so these errors carry no cause. Returns the number of probes made; records
+     * why it failed in `failure` for the display-only state report.
+     */
+    private waitForServiceReady;
     private refresh;
     private stopRecord;
+    private stopContainer;
     private sweepExpired;
     private cleanupResources;
     private removeContainerIfAttested;
@@ -118,6 +167,8 @@ export declare class DockerJobLauncher implements WorkspaceJobLauncher {
     private readResultArtifact;
     private readWorkerFailureDiagnostic;
     private keeperName;
+    /** For a run with services: the attested network keeper's ID, which the worker's network mode may name. */
+    private attestedNetworkKeeperId;
     private assertHost;
     private createAndAssert;
     private waitForStorage;

package/dist/providers/jobs/docker.js

@@ -1,4 +1,6 @@
 import { createHash } from "node:crypto";
+import { normalizeCollectExclusions } from "../../coding/collect-exclude.js";
+import { pruneCollectExcluded } from "./collect-prune.js";
 import { spawn } from "node:child_process";
 import { lstat, mkdir, mkdtemp, open, opendir, readdir, readFile, readlink, realpath, rename, rm, unlink, } from "node:fs/promises";
 import { dirname, isAbsolute, join, resolve, sep } from "node:path";
@@ -6,13 +8,25 @@ import { logger } from "../../core/logger.js";
 import { ensurePrivateDirectory } from "../../core/private-directory.js";
 import { readBoundedRegularFile } from "../../coding-worker/artifact.js";
 import { parseCodingAgentOutputJson, redactAndTruncate, MAX_CODING_ARTIFACT_BYTES, MAX_CODING_OUTPUT_ISSUES, SAFE_CODING_OUTPUT_ISSUE, } from "../../coding/protocol.js";
-import { assertDockerHostSupportsIsolation, assertClaudeAgentContainerInspection, assertClaudeToolRunnerContainerInspection, assertIsolationNetworkInspection, assertKeeperContainerInspection, assertProxyContainerInspection, assertStorageVolumeInspection, assertWorkerContainerInspection, buildDockerIsolationPlan, } from "./docker-isolation.js";
+import { assertDockerHostSupportsIsolation, assertClaudeAgentContainerInspection, assertClaudeToolRunnerContainerInspection, assertIsolationNetworkInspection, assertKeeperContainerInspection, assertNetworkKeeperContainerInspection, assertProxyContainerInspection, assertStorageVolumeInspection, assertWorkerContainerInspection, buildDockerIsolationPlan, CLAUDE_TOOL_SETUP_ENV, dockerSupportsServicesFor, } from "./docker-isolation.js";
+import { serviceUnreadyError } from "../../coding/services/wording.js";
+import { assertServiceContainerInspection, buildDockerServicePlan, dockerServiceContainerNames, } from "./docker-services.js";
+import { claudeToolSetup } from "./claude-tool-setup.js";
+import { reportServiceState, } from "./service-state.js";
 import { replaceDirectoryFromStaging } from "./workspace-swap.js";
 const BACKEND = "docker";
 const STATE_SCHEMA_VERSION = 1;
 const MAX_DOCKER_OUTPUT_BYTES = 1024 * 1024;
 const MAX_DOCKER_SEED_DIAGNOSTIC_BYTES = 4 * 1024;
 const MAX_WORKSPACE_ENTRIES = 100_000;
+/** How long one service image pull may take before the run fails as service_unready. */
+export const SERVICE_IMAGE_PULL_TIMEOUT_MS = 300_000;
+/**
+ * The default bound on a run's whole service start-up (every service's create, start, inspection and
+ * readiness probing; image pulls excluded), matching the Kubernetes launcher's pod-start bound.
+ */
+export const DEFAULT_SERVICE_READY_TIMEOUT_MS = 120_000;
+const MAX_SERVICE_PROBE_OUTPUT_BYTES = 64 * 1024;
 const TERMINAL_PHASES = new Set(["succeeded", "failed", "stopped", "lost", "removed"]);
 export const SAFE_WORKER_DIAGNOSTIC = /^(?:worker_[a-z_]+|coding_[a-z_]+|wardby_[a-z_]+)$/;
 /**
@@ -81,7 +95,7 @@ export class NodeDockerCommandRunner {
         if (args.some((argument) => argument.includes("\0")))
             throw new Error("docker_argument_invalid");
         const extraEnv = options.env ?? {};
-        if (Object.keys(extraEnv).some((key) => key !== "WARDBY_RUN_CAPABILITY"))
+        if (Object.keys(extraEnv).some((key) => key !== "WARDBY_RUN_CAPABILITY" && key !== CLAUDE_TOOL_SETUP_ENV))
             throw new Error("docker_environment_invalid");
         const env = {
             PATH: this.path,
@@ -100,6 +114,13 @@ export class NodeDockerCommandRunner {
                 rejectPromise(new DockerCommandError(null));
                 return;
             }
+            let timedOut = false;
+            const timer = options.timeoutMs === undefined
+                ? undefined
+                : setTimeout(() => {
+                    timedOut = true;
+                    child.kill("SIGKILL");
+                }, options.timeoutMs);
             const stdout = [];
             const stderr = [];
             let bytes = 0;
@@ -120,6 +141,11 @@ export class NodeDockerCommandRunner {
                 spawnFailed = true;
             });
             child.once("close", (code) => {
+                if (timer)
+                    clearTimeout(timer);
+                if (timedOut) {
+                    return rejectPromise(new DockerCommandError(code, false, false, { subcommand: args[0], stderr: "timed out" }));
+                }
                 if (exceeded)
                     return rejectPromise(new DockerCommandError(code, false, true));
                 if (spawnFailed)
@@ -256,7 +282,7 @@ export class NodeDockerArtifactTransfer {
         });
         await docker.run(["container", "cp", sourceFile, `${container}:/run/wardby/storage/input/input.json`]);
     }
-    async materializeDirectory(container, source, destination, maxBytes) {
+    async materializeDirectory(container, source, destination, maxBytes, exclusions) {
         await replaceDirectoryFromStaging(destination, maxBytes, "docker_workspace_destination_invalid", async (staging) => {
             await new NodeDockerCommandRunner({ dockerBinary: this.dockerBinary, homeDir: "/tmp", path: this.path }).run([
                 "container",
@@ -264,6 +290,8 @@ export class NodeDockerArtifactTransfer {
                 `${container}:${source}/.`,
                 staging,
             ]);
+            // Before replaceDirectoryFromStaging validates: excluded folders never count.
+            await pruneCollectExcluded(staging, exclusions);
         });
     }
 }
@@ -380,6 +408,42 @@ function resourceLabels(record) {
 function labelsMatch(actual, expected) {
     return Object.entries(expected).every(([key, value]) => actual?.[key] === value);
 }
+/**
+ * A run's service start-up budget: `budgetMs` from the start, extended by the time image pulls took,
+ * and never past the run's deadline.
+ */
+class ServiceStartBudget {
+    now;
+    budgetMs;
+    deadlineAt;
+    startedAt;
+    excludedMs = 0;
+    constructor(now, budgetMs, deadlineAt) {
+        this.now = now;
+        this.budgetMs = budgetMs;
+        this.deadlineAt = deadlineAt;
+        this.startedAt = now();
+    }
+    /** What is left of the budget, capped by the run's deadline; zero once either runs out. */
+    remainingMs() {
+        const now = this.now();
+        return Math.max(0, Math.min(this.startedAt + this.excludedMs + this.budgetMs - now, this.deadlineAt - now));
+    }
+    /** What is left before the run's deadline: the only bound (besides its own) on an image pull. */
+    untilDeadlineMs() {
+        return Math.max(0, this.deadlineAt - this.now());
+    }
+    exclude(milliseconds) {
+        this.excludedMs += Math.max(0, milliseconds);
+    }
+}
+/** `coding_service_unready:<name>` (the executor's service_unready), keeping the Docker failure as the operator's cause. */
+function serviceUnready(name, cause) {
+    const error = serviceUnreadyError(name);
+    if (cause !== undefined)
+        error.cause = cause;
+    return error;
+}
 function capabilityFromInspection(container) {
     return container.Config?.Env?.find((value) => value.startsWith("WARDBY_RUN_CAPABILITY="))?.slice("WARDBY_RUN_CAPABILITY=".length);
 }
@@ -417,6 +481,8 @@ export class DockerJobLauncher {
     docker;
     transfer;
     now;
+    sleep;
+    serviceReadyTimeoutMs;
     locks = new Map();
     timers = new Map();
     ready;
@@ -429,8 +495,17 @@ export class DockerJobLauncher {
         this.docker = options.docker ?? new NodeDockerCommandRunner({ homeDir: join(this.stateRoot, ".home") });
         this.transfer = options.transfer ?? new NodeDockerArtifactTransfer();
         this.now = options.now ?? Date.now;
+        this.sleep =
+            options.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)));
+        this.serviceReadyTimeoutMs = options.serviceReadyTimeoutMs ?? DEFAULT_SERVICE_READY_TIMEOUT_MS;
+        if (!Number.isFinite(this.serviceReadyTimeoutMs) || this.serviceReadyTimeoutMs <= 0)
+            throw new Error("docker_launcher_service_timeout_invalid");
         this.ready = this.initialize();
     }
+    /** Starts JobSpec.services (a network keeper plus one hardened container per service) for Codex and Claude Code runs. */
+    supportsServicesFor(provider) {
+        return dockerSupportsServicesFor(provider);
+    }
     async launch(spec) {
         await this.ready;
         return this.withRun(spec.runId, async () => this.launchLocked(spec));
@@ -493,7 +568,9 @@ export class DockerJobLauncher {
             if (resolve(destination) !== expected || !expected.startsWith(`${this.workspaceRoot}${sep}`)) {
                 throw new Error("docker_workspace_destination_invalid");
             }
-            await this.transfer.materializeDirectory(this.keeperName(current), "/run/wardby/storage/workspace", expected, current.spec.limits.diskMb * 1024 * 1024);
+            await this.transfer.materializeDirectory(this.keeperName(current), "/run/wardby/storage/workspace", expected, current.spec.limits.diskMb * 1024 * 1024, 
+            // Names are always the current built-in list; only the paths come from the spec.
+            normalizeCollectExclusions(current.spec.collectExclude?.paths));
         });
     }
     async stop(handle, _reason) {
@@ -557,6 +634,7 @@ export class DockerJobLauncher {
     }
     async launchLocked(spec) {
         const plan = buildDockerIsolationPlan(spec, this.options.proxyContainer);
+        const services = buildDockerServicePlan(spec);
         const specHash = stableSpecHash(spec);
         const existing = await this.readRecord(spec.runId);
         if (existing) {
@@ -581,7 +659,7 @@ export class DockerJobLauncher {
                 await this.refresh(existing);
                 return clone(existing.handle);
             }
-            return this.provision(existing, plan, capability);
+            return this.provision(existing, plan, services, capability);
         }
         await this.validateInputArtifact(spec.inputArtifact);
         const capability = ensureCapability(await this.options.resolveCapability(spec.runId));
@@ -600,7 +678,7 @@ export class DockerJobLauncher {
         // This durable record is the idempotency fence before any Docker side effect.
         await this.writeRecord(record);
         try {
-            return await this.provision(record, plan, capability);
+            return await this.provision(record, plan, services, capability);
         }
         catch (error) {
             await this.options
@@ -613,7 +691,7 @@ export class DockerJobLauncher {
             throw error;
         }
     }
-    async provision(record, plan, capability) {
+    async provision(record, plan, services, capability) {
         await this.assertHost();
         const labels = resourceLabels(record);
         await this.createAndAssert(appendLabels(plan.networkCreateArgs, labels), ["network", "inspect", plan.names.network], (value) => {
@@ -640,13 +718,21 @@ export class DockerJobLauncher {
         await this.run(plan.proxyNetworkConnectArgs);
         const proxy = await this.inspect(["container", "inspect", this.options.proxyContainer]);
         assertProxyContainerInspection(proxy, record.runId);
+        // Built before anything starts, so a setup past the tool runner's bounds fails the launch early.
+        const toolSetup = record.spec.provider === "claude-code" ? claudeToolSetup(record.spec, capability) : undefined;
+        // Services first (a run with services only): the Codex worker and Claude Code's tool runner join
+        // the network keeper's namespace, so it must exist, and every service must pass readiness before
+        // anything that runs the run's commands starts.
+        const networkKeeperId = plan.networkKeeperCreateArgs
+            ? await this.startServices(record, plan.networkKeeperCreateArgs, plan.names.networkKeeperContainer, services, labels)
+            : undefined;
         if (plan.toolCreateArgs) {
             await this.createAndAssert(appendLabels(plan.toolCreateArgs, labels), ["container", "inspect", plan.names.toolContainer], (value) => {
-                assertClaudeToolRunnerContainerInspection(value, record.spec);
+                assertClaudeToolRunnerContainerInspection(value, record.spec, toolSetup, networkKeeperId);
                 if (!labelsMatch(value.Config?.Labels, labels)) {
                     throw new Error("docker_resource_attestation_failed");
                 }
-            });
+            }, { env: { WARDBY_TOOL_SETUP: toolSetup } });
             await this.startContainer(plan.names.toolContainer);
             await this.waitForToolRunner(plan.names.toolContainer);
         }
@@ -657,7 +743,7 @@ export class DockerJobLauncher {
                 assertClaudeAgentContainerInspection(inspected, record.spec, capability);
             }
             else {
-                assertWorkerContainerInspection(inspected, record.spec, capability);
+                assertWorkerContainerInspection(inspected, record.spec, capability, networkKeeperId);
             }
             if (!labelsMatch(inspected.Config?.Labels, labels))
                 throw new Error("docker_resource_attestation_failed");
@@ -668,6 +754,165 @@ export class DockerJobLauncher {
         this.scheduleDeadline(record);
         return clone(record.handle);
     }
+    /**
+     * A run with services (docs/coding-services.md). The network keeper joins the run's internal
+     * network and owns the namespace; each service joins it and must pass its catalog readiness
+     * command before the next one starts, as Kubernetes starts native sidecars. The whole start-up is
+     * bounded (ServiceStartBudget): running out fails as coding_service_unready for the service being
+     * waited on. Returns the keeper's container ID, which Docker may report as the services' and
+     * worker's network mode.
+     */
+    async startServices(record, keeperArgs, keeperName, services, labels) {
+        const budget = new ServiceStartBudget(this.now, this.serviceReadyTimeoutMs, record.deadlineAt);
+        const keeper = (await this.createAndAssert(appendLabels(keeperArgs, labels), ["container", "inspect", keeperName], (value) => {
+            const inspected = value;
+            assertNetworkKeeperContainerInspection(inspected, record.spec);
+            if (!labelsMatch(inspected.Config?.Labels, labels))
+                throw new Error("docker_resource_attestation_failed");
+        }));
+        const keeperId = keeper.Id;
+        if (!keeperId)
+            throw new Error("docker_resource_attestation_failed");
+        await this.startContainer(keeperName);
+        const report = (name, update) => reportServiceState(this.options.onServiceState, { runId: record.runId, name, ...update });
+        for (const planned of services)
+            await report(planned.service.name, { state: "pending" });
+        for (const planned of services) {
+            const name = planned.service.name;
+            const failure = { reason: "image_unavailable" };
+            let attempts;
+            try {
+                if (budget.remainingMs() <= 0)
+                    throw serviceUnready(name);
+                await this.ensureServiceImage(planned, budget);
+                if (budget.remainingMs() <= 0)
+                    throw serviceUnready(name);
+                failure.reason = "start_failed";
+                let createFailure;
+                try {
+                    await this.createAndAssert(appendLabels(planned.createArgs, labels), ["container", "inspect", planned.container], (value) => {
+                        const inspected = value;
+                        assertServiceContainerInspection(inspected, record.spec, planned.service, keeperId);
+                        if (!labelsMatch(inspected.Config?.Labels, labels))
+                            throw new Error("docker_resource_attestation_failed");
+                    }, undefined, (error) => {
+                        createFailure = error;
+                    });
+                }
+                catch (error) {
+                    // createAndAssert tolerates a failed create (a crash retry may find it made), so a service
+                    // Docker refused to create shows up as a failed inspection; the create's own failure is the
+                    // useful cause, not that generic "not found". Attestation failures rethrow as is.
+                    if (error instanceof DockerCommandError)
+                        throw serviceUnready(name, createFailure ?? error);
+                    throw error;
+                }
+                if (budget.remainingMs() <= 0)
+                    throw serviceUnready(name);
+                try {
+                    await this.startContainer(planned.container);
+                }
+                catch (error) {
+                    throw serviceUnready(name, error);
+                }
+                failure.reason = "probe_failed";
+                await report(name, { state: "probing", attempts: 0 });
+                attempts = await this.waitForServiceReady(planned, budget, failure, (made) => report(name, { state: "probing", attempts: made }));
+            }
+            catch (error) {
+                const reason = failure.reason !== "exited" && budget.remainingMs() <= 0 ? "timed_out" : failure.reason;
+                await report(name, {
+                    state: "failed",
+                    reason,
+                    ...(failure.attempts === undefined ? {} : { attempts: failure.attempts }),
+                });
+                throw error;
+            }
+            await report(name, { state: "ready", attempts });
+        }
+        return keeperId;
+    }
+    /**
+     * A local copy of the digest-pinned image is used as is; otherwise it is pulled, bounded by its
+     * own limit and the run's deadline. Pull time does not count toward the start-up budget.
+     */
+    async ensureServiceImage(planned, budget) {
+        try {
+            await this.run(["image", "inspect", "--format", "{{.Id}}", planned.service.image]);
+            return;
+        }
+        catch {
+            // Missing (or unreadable): pull it by its digest below.
+        }
+        const untilDeadline = budget.untilDeadlineMs();
+        if (untilDeadline <= 0)
+            throw serviceUnready(planned.service.name);
+        const startedAt = this.now();
+        try {
+            await this.run(["image", "pull", "--quiet", planned.service.image], {
+                timeoutMs: Math.min(SERVICE_IMAGE_PULL_TIMEOUT_MS, untilDeadline),
+            });
+        }
+        catch (error) {
+            throw serviceUnready(planned.service.name, error);
+        }
+        finally {
+            budget.exclude(this.now() - startedAt);
+        }
+    }
+    /**
+     * The catalog readiness command, like a Kubernetes startup probe: every periodSeconds, each
+     * attempt bounded by timeoutSeconds, failing after failureThreshold consecutive failures. A
+     * service that exited or vanished fails at once, and so does one still unready when the start-up
+     * budget runs out (each probe and wait is clipped to what is left of it). Probe output is never
+     * logged or kept, so these errors carry no cause. Returns the number of probes made; records
+     * why it failed in `failure` for the display-only state report.
+     */
+    async waitForServiceReady(planned, budget, failure, onProbeFailed) {
+        const { command, periodSeconds, timeoutSeconds, failureThreshold } = planned.service.readiness;
+        const name = planned.service.name;
+        let failures = 0;
+        for (;;) {
+            if (budget.remainingMs() <= 0)
+                throw serviceUnready(name);
+            let state;
+            try {
+                state = await this.inspect(["container", "inspect", planned.container]);
+            }
+            catch (error) {
+                if (error instanceof DockerCommandError) {
+                    failure.reason = "exited";
+                    throw serviceUnready(name);
+                }
+                throw error;
+            }
+            if (state.State?.Running !== true) {
+                failure.reason = "exited";
+                throw serviceUnready(name);
+            }
+            const remaining = budget.remainingMs();
+            if (remaining <= 0)
+                throw serviceUnready(name);
+            try {
+                await this.run(["container", "exec", planned.container, ...command], {
+                    timeoutMs: Math.min(timeoutSeconds * 1_000, remaining),
+                    maxOutputBytes: MAX_SERVICE_PROBE_OUTPUT_BYTES,
+                });
+                return failures + 1;
+            }
+            catch {
+                failures += 1;
+                failure.attempts = failures;
+                if (failures >= failureThreshold)
+                    throw serviceUnready(name);
+            }
+            await onProbeFailed(failures);
+            const wait = Math.min(periodSeconds * 1_000, budget.remainingMs());
+            if (wait <= 0)
+                throw serviceUnready(name);
+            await this.sleep(wait);
+        }
+    }
     async refresh(record) {
         if (isTerminal(record))
             return;
@@ -701,7 +946,7 @@ export class DockerJobLauncher {
                 "inspect",
                 buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.toolContainer,
             ]);
-            assertClaudeToolRunnerContainerInspection(tool, record.spec);
+            assertClaudeToolRunnerContainerInspection(tool, record.spec, claudeToolSetup(record.spec, capability), await this.attestedNetworkKeeperId(record));
             if (tool.State?.Running !== true || tool.State?.Status !== "running") {
                 record.phase = "failed";
                 record.result = { exitCode: 1, reason: "failed", diagnostic: "worker_tool_runner_failed" };
@@ -711,7 +956,7 @@ export class DockerJobLauncher {
             }
         }
         else {
-            assertWorkerContainerInspection(inspection, record.spec, capability);
+            assertWorkerContainerInspection(inspection, record.spec, capability, await this.attestedNetworkKeeperId(record));
         }
         const phase = dockerState(inspection);
         if (!phase || phase === "provisioning" || phase === "active")
@@ -724,28 +969,27 @@ export class DockerJobLauncher {
     async stopRecord(record, timedOut) {
         if (isTerminal(record))
             return;
-        try {
-            await this.run(["container", "stop", "--time", "10", record.handle.id]);
-        }
-        catch (error) {
-            if (!(error instanceof DockerCommandError && error.notFound))
-                throw new Error("docker_stop_failed", { cause: error });
-        }
+        await this.stopContainer(record.handle.id);
         if (record.spec.provider === "claude-code") {
-            const tool = buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.toolContainer;
-            try {
-                await this.run(["container", "stop", "--time", "10", tool]);
-            }
-            catch (error) {
-                if (!(error instanceof DockerCommandError && error.notFound))
-                    throw new Error("docker_stop_failed", { cause: error });
-            }
+            await this.stopContainer(buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.toolContainer);
         }
+        // Services stop with the run; they (and the network keeper) are removed with its other resources.
+        for (const service of dockerServiceContainerNames(record.spec))
+            await this.stopContainer(service);
         record.phase = timedOut ? "failed" : "stopped";
         record.result = timedOut ? { exitCode: 124, reason: "timed_out" } : resultFor("stopped");
         await this.writeRecord(record);
         this.clearDeadline(record);
     }
+    async stopContainer(name) {
+        try {
+            await this.run(["container", "stop", "--time", "10", name]);
+        }
+        catch (error) {
+            if (!(error instanceof DockerCommandError && error.notFound))
+                throw new Error("docker_stop_failed", { cause: error });
+        }
+    }
     async sweepExpired(record) {
         // The DB verifier ran immediately before this call; never sweep by label alone.
         try {
@@ -765,17 +1009,26 @@ export class DockerJobLauncher {
         await this.removeContainerIfAttested(plan.names.workerContainer, labels);
         if (plan.toolCreateArgs)
             await this.removeContainerIfAttested(plan.names.toolContainer, labels);
+        // Services, then the network keeper: nothing (the Codex worker, Claude's tool runner, a service)
+        // may still share its namespace when it goes.
+        // --volumes: an image VOLUME the attestation rejected must not outlive its container.
+        for (const service of dockerServiceContainerNames(record.spec)) {
+            await this.removeContainerIfAttested(service, labels, true);
+        }
+        if (plan.networkKeeperCreateArgs) {
+            await this.removeContainerIfAttested(plan.names.networkKeeperContainer, labels);
+        }
         await this.runIgnoreMissing(["network", "disconnect", plan.names.network, this.options.proxyContainer]);
         await this.removeContainerIfAttested(plan.names.keeperContainer, labels);
         await this.removeNetworkIfAttested(plan.names.network, labels);
         await this.removeVolumeIfAttested(plan.names.storageVolume, labels);
     }
-    async removeContainerIfAttested(name, labels) {
+    async removeContainerIfAttested(name, labels, anonymousVolumes = false) {
         try {
             const inspected = await this.inspect(["container", "inspect", name]);
             if (!labelsMatch(inspected.Config?.Labels, labels))
                 throw new Error("docker_resource_attestation_failed");
-            await this.runIgnoreMissing(["container", "rm", "--force", name]);
+            await this.runIgnoreMissing(["container", "rm", "--force", ...(anonymousVolumes ? ["--volumes"] : []), name]);
         }
         catch (error) {
             if (!(error instanceof DockerCommandError && error.notFound))
@@ -852,18 +1105,40 @@ export class DockerJobLauncher {
     keeperName(record) {
         return buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.keeperContainer;
     }
+    /** For a run with services: the attested network keeper's ID, which the worker's network mode may name. */
+    async attestedNetworkKeeperId(record) {
+        if (record.spec.services === undefined)
+            return undefined;
+        const name = buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.networkKeeperContainer;
+        let keeper;
+        try {
+            keeper = await this.inspect(["container", "inspect", name]);
+        }
+        catch (error) {
+            throw new Error("docker_status_unavailable", { cause: error });
+        }
+        assertNetworkKeeperContainerInspection(keeper, record.spec);
+        if (!labelsMatch(keeper.Config?.Labels, resourceLabels(record)) || !keeper.Id) {
+            throw new Error("docker_resource_attestation_failed");
+        }
+        return keeper.Id;
+    }
     async assertHost() {
         const info = await this.inspect(["info", "--format", "{{json .}}"]);
         assertDockerHostSupportsIsolation(info);
     }
-    async createAndAssert(createArgs, inspectArgs, assert, options) {
+    async createAndAssert(createArgs, inspectArgs, assert, options, onCreateFailure) {
         try {
             await this.run(createArgs, options);
         }
-        catch {
+        catch (error) {
             // A retry may find a resource created before the previous process crashed.
+            if (error instanceof DockerCommandError)
+                onCreateFailure?.(error);
         }
-        assert(await this.inspect(inspectArgs));
+        const inspected = await this.inspect(inspectArgs);
+        assert(inspected);
+        return inspected;
     }
     async waitForStorage(keeper) {
         // The keeper emits this fixed marker only after creating every storage

package/dist/providers/jobs/fake-kubernetes-api.d.ts

@@ -8,6 +8,7 @@ export interface FakeExecCall {
     command: string[];
     stdin?: Readable;
     stdout?: Writable;
+    timeoutMs: number;
 }
 type Kind = "configmap" | "secret" | "pod" | "networkpolicy" | "service" | "endpoints";
 type Obj = {