@wardby/cli 0.2.0 → 0.3.0

package/dist/providers/jobs/docker.js

@@ -1,18 +1,58 @@
 import { createHash } from "node:crypto";
+import { normalizeCollectExclusions } from "../../coding/collect-exclude.js";
+import { pruneCollectExcluded } from "./collect-prune.js";
 import { spawn } from "node:child_process";
 import { lstat, mkdir, mkdtemp, open, opendir, readdir, readFile, readlink, realpath, rename, rm, unlink, } from "node:fs/promises";
 import { dirname, isAbsolute, join, resolve, sep } from "node:path";
 import { logger } from "../../core/logger.js";
-import { parseCodingAgentOutputJson, redactAndTruncate, MAX_CODING_ARTIFACT_BYTES } from "../../coding/protocol.js";
-import { assertDockerHostSupportsIsolation, assertClaudeAgentContainerInspection, assertClaudeToolRunnerContainerInspection, assertIsolationNetworkInspection, assertKeeperContainerInspection, assertProxyContainerInspection, assertStorageVolumeInspection, assertWorkerContainerInspection, buildDockerIsolationPlan, } from "./docker-isolation.js";
+import { ensurePrivateDirectory } from "../../core/private-directory.js";
+import { readBoundedRegularFile } from "../../coding-worker/artifact.js";
+import { parseCodingAgentOutputJson, redactAndTruncate, MAX_CODING_ARTIFACT_BYTES, MAX_CODING_OUTPUT_ISSUES, SAFE_CODING_OUTPUT_ISSUE, } from "../../coding/protocol.js";
+import { assertDockerHostSupportsIsolation, assertClaudeAgentContainerInspection, assertClaudeToolRunnerContainerInspection, assertIsolationNetworkInspection, assertKeeperContainerInspection, assertNetworkKeeperContainerInspection, assertProxyContainerInspection, assertStorageVolumeInspection, assertWorkerContainerInspection, buildDockerIsolationPlan, CLAUDE_TOOL_SETUP_ENV, dockerSupportsServicesFor, } from "./docker-isolation.js";
+import { serviceUnreadyError } from "../../coding/services/wording.js";
+import { assertServiceContainerInspection, buildDockerServicePlan, dockerServiceContainerNames, } from "./docker-services.js";
+import { claudeToolSetup } from "./claude-tool-setup.js";
 import { replaceDirectoryFromStaging } from "./workspace-swap.js";
 const BACKEND = "docker";
 const STATE_SCHEMA_VERSION = 1;
 const MAX_DOCKER_OUTPUT_BYTES = 1024 * 1024;
 const MAX_DOCKER_SEED_DIAGNOSTIC_BYTES = 4 * 1024;
 const MAX_WORKSPACE_ENTRIES = 100_000;
+/** How long one service image pull may take before the run fails as service_unready. */
+export const SERVICE_IMAGE_PULL_TIMEOUT_MS = 300_000;
+/**
+ * The default bound on a run's whole service start-up (every service's create, start, inspection and
+ * readiness probing; image pulls excluded), matching the Kubernetes launcher's pod-start bound.
+ */
+export const DEFAULT_SERVICE_READY_TIMEOUT_MS = 120_000;
+const MAX_SERVICE_PROBE_OUTPUT_BYTES = 64 * 1024;
 const TERMINAL_PHASES = new Set(["succeeded", "failed", "stopped", "lost", "removed"]);
 export const SAFE_WORKER_DIAGNOSTIC = /^(?:worker_[a-z_]+|coding_[a-z_]+|wardby_[a-z_]+)$/;
+/**
+ * Worker output is untrusted: recognizes only the fixed `{"error": code}` line,
+ * plus an optional `issues` list whose every entry must pass the shared pattern.
+ * A list with any entry that does not is dropped whole rather than filtered, so
+ * a malformed worker cannot smuggle a partial value through.
+ */
+export function parseWorkerDiagnosticLine(line) {
+    let value;
+    try {
+        value = JSON.parse(line);
+    }
+    catch {
+        return undefined;
+    }
+    if (typeof value?.error !== "string" || !SAFE_WORKER_DIAGNOSTIC.test(value.error))
+        return undefined;
+    const issues = value.issues;
+    const safeIssues = Array.isArray(issues) &&
+        issues.length > 0 &&
+        issues.length <= MAX_CODING_OUTPUT_ISSUES &&
+        issues.every((issue) => typeof issue === "string" && SAFE_CODING_OUTPUT_ISSUE.test(issue))
+        ? issues
+        : undefined;
+    return { diagnostic: value.error, ...(safeIssues ? { diagnosticIssues: safeIssues } : {}) };
+}
 const dockerLog = logger.child({ module: "docker-jobs" });
 /**
  * Docker failures never include daemon output in their public message -- that
@@ -54,7 +94,7 @@ export class NodeDockerCommandRunner {
         if (args.some((argument) => argument.includes("\0")))
             throw new Error("docker_argument_invalid");
         const extraEnv = options.env ?? {};
-        if (Object.keys(extraEnv).some((key) => key !== "WARDBY_RUN_CAPABILITY"))
+        if (Object.keys(extraEnv).some((key) => key !== "WARDBY_RUN_CAPABILITY" && key !== CLAUDE_TOOL_SETUP_ENV))
             throw new Error("docker_environment_invalid");
         const env = {
             PATH: this.path,
@@ -73,6 +113,13 @@ export class NodeDockerCommandRunner {
                 rejectPromise(new DockerCommandError(null));
                 return;
             }
+            let timedOut = false;
+            const timer = options.timeoutMs === undefined
+                ? undefined
+                : setTimeout(() => {
+                    timedOut = true;
+                    child.kill("SIGKILL");
+                }, options.timeoutMs);
             const stdout = [];
             const stderr = [];
             let bytes = 0;
@@ -93,6 +140,11 @@ export class NodeDockerCommandRunner {
                 spawnFailed = true;
             });
             child.once("close", (code) => {
+                if (timer)
+                    clearTimeout(timer);
+                if (timedOut) {
+                    return rejectPromise(new DockerCommandError(code, false, false, { subcommand: args[0], stderr: "timed out" }));
+                }
                 if (exceeded)
                     return rejectPromise(new DockerCommandError(code, false, true));
                 if (spawnFailed)
@@ -229,7 +281,7 @@ export class NodeDockerArtifactTransfer {
         });
         await docker.run(["container", "cp", sourceFile, `${container}:/run/wardby/storage/input/input.json`]);
     }
-    async materializeDirectory(container, source, destination, maxBytes) {
+    async materializeDirectory(container, source, destination, maxBytes, exclusions) {
         await replaceDirectoryFromStaging(destination, maxBytes, "docker_workspace_destination_invalid", async (staging) => {
             await new NodeDockerCommandRunner({ dockerBinary: this.dockerBinary, homeDir: "/tmp", path: this.path }).run([
                 "container",
@@ -237,6 +289,8 @@ export class NodeDockerArtifactTransfer {
                 `${container}:${source}/.`,
                 staging,
             ]);
+            // Before replaceDirectoryFromStaging validates: excluded folders never count.
+            await pruneCollectExcluded(staging, exclusions);
         });
     }
 }
@@ -353,6 +407,42 @@ function resourceLabels(record) {
 function labelsMatch(actual, expected) {
     return Object.entries(expected).every(([key, value]) => actual?.[key] === value);
 }
+/**
+ * A run's service start-up budget: `budgetMs` from the start, extended by the time image pulls took,
+ * and never past the run's deadline.
+ */
+class ServiceStartBudget {
+    now;
+    budgetMs;
+    deadlineAt;
+    startedAt;
+    excludedMs = 0;
+    constructor(now, budgetMs, deadlineAt) {
+        this.now = now;
+        this.budgetMs = budgetMs;
+        this.deadlineAt = deadlineAt;
+        this.startedAt = now();
+    }
+    /** What is left of the budget, capped by the run's deadline; zero once either runs out. */
+    remainingMs() {
+        const now = this.now();
+        return Math.max(0, Math.min(this.startedAt + this.excludedMs + this.budgetMs - now, this.deadlineAt - now));
+    }
+    /** What is left before the run's deadline: the only bound (besides its own) on an image pull. */
+    untilDeadlineMs() {
+        return Math.max(0, this.deadlineAt - this.now());
+    }
+    exclude(milliseconds) {
+        this.excludedMs += Math.max(0, milliseconds);
+    }
+}
+/** `coding_service_unready:<name>` (the executor's service_unready), keeping the Docker failure as the operator's cause. */
+function serviceUnready(name, cause) {
+    const error = serviceUnreadyError(name);
+    if (cause !== undefined)
+        error.cause = cause;
+    return error;
+}
 function capabilityFromInspection(container) {
     return container.Config?.Env?.find((value) => value.startsWith("WARDBY_RUN_CAPABILITY="))?.slice("WARDBY_RUN_CAPABILITY=".length);
 }
@@ -390,6 +480,8 @@ export class DockerJobLauncher {
     docker;
     transfer;
     now;
+    sleep;
+    serviceReadyTimeoutMs;
     locks = new Map();
     timers = new Map();
     ready;
@@ -402,8 +494,17 @@ export class DockerJobLauncher {
         this.docker = options.docker ?? new NodeDockerCommandRunner({ homeDir: join(this.stateRoot, ".home") });
         this.transfer = options.transfer ?? new NodeDockerArtifactTransfer();
         this.now = options.now ?? Date.now;
+        this.sleep =
+            options.sleep ?? ((milliseconds) => new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds)));
+        this.serviceReadyTimeoutMs = options.serviceReadyTimeoutMs ?? DEFAULT_SERVICE_READY_TIMEOUT_MS;
+        if (!Number.isFinite(this.serviceReadyTimeoutMs) || this.serviceReadyTimeoutMs <= 0)
+            throw new Error("docker_launcher_service_timeout_invalid");
         this.ready = this.initialize();
     }
+    /** Starts JobSpec.services (a network keeper plus one hardened container per service) for Codex and Claude Code runs. */
+    supportsServicesFor(provider) {
+        return dockerSupportsServicesFor(provider);
+    }
     async launch(spec) {
         await this.ready;
         return this.withRun(spec.runId, async () => this.launchLocked(spec));
@@ -440,7 +541,7 @@ export class DockerJobLauncher {
             if (current.phase === "failed" && !current.result.diagnostic) {
                 const diagnostic = await this.readWorkerFailureDiagnostic(current);
                 if (diagnostic) {
-                    current.result = { ...current.result, diagnostic };
+                    current.result = { ...current.result, ...diagnostic };
                     await this.writeRecord(current);
                 }
             }
@@ -466,7 +567,9 @@ export class DockerJobLauncher {
             if (resolve(destination) !== expected || !expected.startsWith(`${this.workspaceRoot}${sep}`)) {
                 throw new Error("docker_workspace_destination_invalid");
             }
-            await this.transfer.materializeDirectory(this.keeperName(current), "/run/wardby/storage/workspace", expected, current.spec.limits.diskMb * 1024 * 1024);
+            await this.transfer.materializeDirectory(this.keeperName(current), "/run/wardby/storage/workspace", expected, current.spec.limits.diskMb * 1024 * 1024, 
+            // Names are always the current built-in list; only the paths come from the spec.
+            normalizeCollectExclusions(current.spec.collectExclude?.paths));
         });
     }
     async stop(handle, _reason) {
@@ -500,7 +603,7 @@ export class DockerJobLauncher {
         });
     }
     async initialize() {
-        await mkdir(this.stateRoot, { recursive: true, mode: 0o700 });
+        await ensurePrivateDirectory(this.stateRoot);
         await mkdir(join(this.stateRoot, ".home"), { recursive: true, mode: 0o700 });
         await realpath(this.stateRoot);
         const records = await this.allRecords();
@@ -530,6 +633,7 @@ export class DockerJobLauncher {
     }
     async launchLocked(spec) {
         const plan = buildDockerIsolationPlan(spec, this.options.proxyContainer);
+        const services = buildDockerServicePlan(spec);
         const specHash = stableSpecHash(spec);
         const existing = await this.readRecord(spec.runId);
         if (existing) {
@@ -554,7 +658,7 @@ export class DockerJobLauncher {
                 await this.refresh(existing);
                 return clone(existing.handle);
             }
-            return this.provision(existing, plan, capability);
+            return this.provision(existing, plan, services, capability);
         }
         await this.validateInputArtifact(spec.inputArtifact);
         const capability = ensureCapability(await this.options.resolveCapability(spec.runId));
@@ -573,7 +677,7 @@ export class DockerJobLauncher {
         // This durable record is the idempotency fence before any Docker side effect.
         await this.writeRecord(record);
         try {
-            return await this.provision(record, plan, capability);
+            return await this.provision(record, plan, services, capability);
         }
         catch (error) {
             await this.options
@@ -586,7 +690,7 @@ export class DockerJobLauncher {
             throw error;
         }
     }
-    async provision(record, plan, capability) {
+    async provision(record, plan, services, capability) {
         await this.assertHost();
         const labels = resourceLabels(record);
         await this.createAndAssert(appendLabels(plan.networkCreateArgs, labels), ["network", "inspect", plan.names.network], (value) => {
@@ -613,13 +717,21 @@ export class DockerJobLauncher {
         await this.run(plan.proxyNetworkConnectArgs);
         const proxy = await this.inspect(["container", "inspect", this.options.proxyContainer]);
         assertProxyContainerInspection(proxy, record.runId);
+        // Built before anything starts, so a setup past the tool runner's bounds fails the launch early.
+        const toolSetup = record.spec.provider === "claude-code" ? claudeToolSetup(record.spec, capability) : undefined;
+        // Services first (a run with services only): the Codex worker and Claude Code's tool runner join
+        // the network keeper's namespace, so it must exist, and every service must pass readiness before
+        // anything that runs the run's commands starts.
+        const networkKeeperId = plan.networkKeeperCreateArgs
+            ? await this.startServices(record, plan.networkKeeperCreateArgs, plan.names.networkKeeperContainer, services, labels)
+            : undefined;
         if (plan.toolCreateArgs) {
             await this.createAndAssert(appendLabels(plan.toolCreateArgs, labels), ["container", "inspect", plan.names.toolContainer], (value) => {
-                assertClaudeToolRunnerContainerInspection(value, record.spec);
+                assertClaudeToolRunnerContainerInspection(value, record.spec, toolSetup, networkKeeperId);
                 if (!labelsMatch(value.Config?.Labels, labels)) {
                     throw new Error("docker_resource_attestation_failed");
                 }
-            });
+            }, { env: { WARDBY_TOOL_SETUP: toolSetup } });
             await this.startContainer(plan.names.toolContainer);
             await this.waitForToolRunner(plan.names.toolContainer);
         }
@@ -630,7 +742,7 @@ export class DockerJobLauncher {
                 assertClaudeAgentContainerInspection(inspected, record.spec, capability);
             }
             else {
-                assertWorkerContainerInspection(inspected, record.spec, capability);
+                assertWorkerContainerInspection(inspected, record.spec, capability, networkKeeperId);
             }
             if (!labelsMatch(inspected.Config?.Labels, labels))
                 throw new Error("docker_resource_attestation_failed");
@@ -641,6 +753,138 @@ export class DockerJobLauncher {
         this.scheduleDeadline(record);
         return clone(record.handle);
     }
+    /**
+     * A run with services (docs/coding-services.md). The network keeper joins the run's internal
+     * network and owns the namespace; each service joins it and must pass its catalog readiness
+     * command before the next one starts, as Kubernetes starts native sidecars. The whole start-up is
+     * bounded (ServiceStartBudget): running out fails as coding_service_unready for the service being
+     * waited on. Returns the keeper's container ID, which Docker may report as the services' and
+     * worker's network mode.
+     */
+    async startServices(record, keeperArgs, keeperName, services, labels) {
+        const budget = new ServiceStartBudget(this.now, this.serviceReadyTimeoutMs, record.deadlineAt);
+        const keeper = (await this.createAndAssert(appendLabels(keeperArgs, labels), ["container", "inspect", keeperName], (value) => {
+            const inspected = value;
+            assertNetworkKeeperContainerInspection(inspected, record.spec);
+            if (!labelsMatch(inspected.Config?.Labels, labels))
+                throw new Error("docker_resource_attestation_failed");
+        }));
+        const keeperId = keeper.Id;
+        if (!keeperId)
+            throw new Error("docker_resource_attestation_failed");
+        await this.startContainer(keeperName);
+        for (const planned of services) {
+            const name = planned.service.name;
+            if (budget.remainingMs() <= 0)
+                throw serviceUnready(name);
+            await this.ensureServiceImage(planned, budget);
+            if (budget.remainingMs() <= 0)
+                throw serviceUnready(name);
+            let createFailure;
+            try {
+                await this.createAndAssert(appendLabels(planned.createArgs, labels), ["container", "inspect", planned.container], (value) => {
+                    const inspected = value;
+                    assertServiceContainerInspection(inspected, record.spec, planned.service, keeperId);
+                    if (!labelsMatch(inspected.Config?.Labels, labels))
+                        throw new Error("docker_resource_attestation_failed");
+                }, undefined, (error) => {
+                    createFailure = error;
+                });
+            }
+            catch (error) {
+                // createAndAssert tolerates a failed create (a crash retry may find it made), so a service
+                // Docker refused to create shows up as a failed inspection; the create's own failure is the
+                // useful cause, not that generic "not found". Attestation failures rethrow as is.
+                if (error instanceof DockerCommandError)
+                    throw serviceUnready(name, createFailure ?? error);
+                throw error;
+            }
+            if (budget.remainingMs() <= 0)
+                throw serviceUnready(name);
+            try {
+                await this.startContainer(planned.container);
+            }
+            catch (error) {
+                throw serviceUnready(name, error);
+            }
+            await this.waitForServiceReady(planned, budget);
+        }
+        return keeperId;
+    }
+    /**
+     * A local copy of the digest-pinned image is used as is; otherwise it is pulled, bounded by its
+     * own limit and the run's deadline. Pull time does not count toward the start-up budget.
+     */
+    async ensureServiceImage(planned, budget) {
+        try {
+            await this.run(["image", "inspect", "--format", "{{.Id}}", planned.service.image]);
+            return;
+        }
+        catch {
+            // Missing (or unreadable): pull it by its digest below.
+        }
+        const untilDeadline = budget.untilDeadlineMs();
+        if (untilDeadline <= 0)
+            throw serviceUnready(planned.service.name);
+        const startedAt = this.now();
+        try {
+            await this.run(["image", "pull", "--quiet", planned.service.image], {
+                timeoutMs: Math.min(SERVICE_IMAGE_PULL_TIMEOUT_MS, untilDeadline),
+            });
+        }
+        catch (error) {
+            throw serviceUnready(planned.service.name, error);
+        }
+        finally {
+            budget.exclude(this.now() - startedAt);
+        }
+    }
+    /**
+     * The catalog readiness command, like a Kubernetes startup probe: every periodSeconds, each
+     * attempt bounded by timeoutSeconds, failing after failureThreshold consecutive failures. A
+     * service that exited or vanished fails at once, and so does one still unready when the start-up
+     * budget runs out (each probe and wait is clipped to what is left of it). Probe output is never
+     * logged or kept, so these errors carry no cause.
+     */
+    async waitForServiceReady(planned, budget) {
+        const { command, periodSeconds, timeoutSeconds, failureThreshold } = planned.service.readiness;
+        const name = planned.service.name;
+        let failures = 0;
+        for (;;) {
+            if (budget.remainingMs() <= 0)
+                throw serviceUnready(name);
+            let state;
+            try {
+                state = await this.inspect(["container", "inspect", planned.container]);
+            }
+            catch (error) {
+                if (error instanceof DockerCommandError)
+                    throw serviceUnready(name);
+                throw error;
+            }
+            if (state.State?.Running !== true)
+                throw serviceUnready(name);
+            const remaining = budget.remainingMs();
+            if (remaining <= 0)
+                throw serviceUnready(name);
+            try {
+                await this.run(["container", "exec", planned.container, ...command], {
+                    timeoutMs: Math.min(timeoutSeconds * 1_000, remaining),
+                    maxOutputBytes: MAX_SERVICE_PROBE_OUTPUT_BYTES,
+                });
+                return;
+            }
+            catch {
+                failures += 1;
+                if (failures >= failureThreshold)
+                    throw serviceUnready(name);
+            }
+            const wait = Math.min(periodSeconds * 1_000, budget.remainingMs());
+            if (wait <= 0)
+                throw serviceUnready(name);
+            await this.sleep(wait);
+        }
+    }
     async refresh(record) {
         if (isTerminal(record))
             return;
@@ -674,7 +918,7 @@ export class DockerJobLauncher {
                 "inspect",
                 buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.toolContainer,
             ]);
-            assertClaudeToolRunnerContainerInspection(tool, record.spec);
+            assertClaudeToolRunnerContainerInspection(tool, record.spec, claudeToolSetup(record.spec, capability), await this.attestedNetworkKeeperId(record));
             if (tool.State?.Running !== true || tool.State?.Status !== "running") {
                 record.phase = "failed";
                 record.result = { exitCode: 1, reason: "failed", diagnostic: "worker_tool_runner_failed" };
@@ -684,7 +928,7 @@ export class DockerJobLauncher {
             }
         }
         else {
-            assertWorkerContainerInspection(inspection, record.spec, capability);
+            assertWorkerContainerInspection(inspection, record.spec, capability, await this.attestedNetworkKeeperId(record));
         }
         const phase = dockerState(inspection);
         if (!phase || phase === "provisioning" || phase === "active")
@@ -697,28 +941,27 @@ export class DockerJobLauncher {
     async stopRecord(record, timedOut) {
         if (isTerminal(record))
             return;
-        try {
-            await this.run(["container", "stop", "--time", "10", record.handle.id]);
-        }
-        catch (error) {
-            if (!(error instanceof DockerCommandError && error.notFound))
-                throw new Error("docker_stop_failed", { cause: error });
-        }
+        await this.stopContainer(record.handle.id);
         if (record.spec.provider === "claude-code") {
-            const tool = buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.toolContainer;
-            try {
-                await this.run(["container", "stop", "--time", "10", tool]);
-            }
-            catch (error) {
-                if (!(error instanceof DockerCommandError && error.notFound))
-                    throw new Error("docker_stop_failed", { cause: error });
-            }
+            await this.stopContainer(buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.toolContainer);
         }
+        // Services stop with the run; they (and the network keeper) are removed with its other resources.
+        for (const service of dockerServiceContainerNames(record.spec))
+            await this.stopContainer(service);
         record.phase = timedOut ? "failed" : "stopped";
         record.result = timedOut ? { exitCode: 124, reason: "timed_out" } : resultFor("stopped");
         await this.writeRecord(record);
         this.clearDeadline(record);
     }
+    async stopContainer(name) {
+        try {
+            await this.run(["container", "stop", "--time", "10", name]);
+        }
+        catch (error) {
+            if (!(error instanceof DockerCommandError && error.notFound))
+                throw new Error("docker_stop_failed", { cause: error });
+        }
+    }
     async sweepExpired(record) {
         // The DB verifier ran immediately before this call; never sweep by label alone.
         try {
@@ -738,17 +981,26 @@ export class DockerJobLauncher {
         await this.removeContainerIfAttested(plan.names.workerContainer, labels);
         if (plan.toolCreateArgs)
             await this.removeContainerIfAttested(plan.names.toolContainer, labels);
+        // Services, then the network keeper: nothing (the Codex worker, Claude's tool runner, a service)
+        // may still share its namespace when it goes.
+        // --volumes: an image VOLUME the attestation rejected must not outlive its container.
+        for (const service of dockerServiceContainerNames(record.spec)) {
+            await this.removeContainerIfAttested(service, labels, true);
+        }
+        if (plan.networkKeeperCreateArgs) {
+            await this.removeContainerIfAttested(plan.names.networkKeeperContainer, labels);
+        }
         await this.runIgnoreMissing(["network", "disconnect", plan.names.network, this.options.proxyContainer]);
         await this.removeContainerIfAttested(plan.names.keeperContainer, labels);
         await this.removeNetworkIfAttested(plan.names.network, labels);
         await this.removeVolumeIfAttested(plan.names.storageVolume, labels);
     }
-    async removeContainerIfAttested(name, labels) {
+    async removeContainerIfAttested(name, labels, anonymousVolumes = false) {
         try {
             const inspected = await this.inspect(["container", "inspect", name]);
             if (!labelsMatch(inspected.Config?.Labels, labels))
                 throw new Error("docker_resource_attestation_failed");
-            await this.runIgnoreMissing(["container", "rm", "--force", name]);
+            await this.runIgnoreMissing(["container", "rm", "--force", ...(anonymousVolumes ? ["--volumes"] : []), name]);
         }
         catch (error) {
             if (!(error instanceof DockerCommandError && error.notFound))
@@ -792,13 +1044,7 @@ export class DockerJobLauncher {
         const target = join(staging, "result.json");
         try {
             await this.run(["container", "cp", `${this.keeperName(record)}:/run/wardby/storage/output/result.json`, target]);
-            const metadata = await lstat(target);
-            if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > MAX_CODING_ARTIFACT_BYTES) {
-                throw new Error("docker_result_artifact_invalid");
-            }
-            const raw = await readFile(target, "utf8");
-            if (Buffer.byteLength(raw, "utf8") > MAX_CODING_ARTIFACT_BYTES)
-                throw new Error("docker_result_artifact_invalid");
+            const raw = await readBoundedRegularFile(target, MAX_CODING_ARTIFACT_BYTES, "docker_result_artifact_invalid");
             const output = parseCodingAgentOutputJson(raw);
             if (output.runId !== record.runId)
                 throw new Error("docker_result_run_mismatch");
@@ -818,14 +1064,9 @@ export class DockerJobLauncher {
             const logs = await this.run(["container", "logs", "--tail", "8", record.handle.id]);
             const raw = `${logs.stdout}\n${logs.stderr}`;
             for (const line of raw.split("\n").reverse()) {
-                try {
-                    const value = JSON.parse(line);
-                    if (typeof value.error === "string" && SAFE_WORKER_DIAGNOSTIC.test(value.error))
-                        return value.error;
-                }
-                catch {
-                    // Worker output is untrusted; only parse one fixed JSON shape.
-                }
+                const diagnostic = parseWorkerDiagnosticLine(line);
+                if (diagnostic)
+                    return diagnostic;
             }
         }
         catch {
@@ -836,18 +1077,40 @@ export class DockerJobLauncher {
     keeperName(record) {
         return buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.keeperContainer;
     }
+    /** For a run with services: the attested network keeper's ID, which the worker's network mode may name. */
+    async attestedNetworkKeeperId(record) {
+        if (record.spec.services === undefined)
+            return undefined;
+        const name = buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.networkKeeperContainer;
+        let keeper;
+        try {
+            keeper = await this.inspect(["container", "inspect", name]);
+        }
+        catch (error) {
+            throw new Error("docker_status_unavailable", { cause: error });
+        }
+        assertNetworkKeeperContainerInspection(keeper, record.spec);
+        if (!labelsMatch(keeper.Config?.Labels, resourceLabels(record)) || !keeper.Id) {
+            throw new Error("docker_resource_attestation_failed");
+        }
+        return keeper.Id;
+    }
     async assertHost() {
         const info = await this.inspect(["info", "--format", "{{json .}}"]);
         assertDockerHostSupportsIsolation(info);
     }
-    async createAndAssert(createArgs, inspectArgs, assert, options) {
+    async createAndAssert(createArgs, inspectArgs, assert, options, onCreateFailure) {
         try {
             await this.run(createArgs, options);
         }
-        catch {
+        catch (error) {
             // A retry may find a resource created before the previous process crashed.
+            if (error instanceof DockerCommandError)
+                onCreateFailure?.(error);
         }
-        assert(await this.inspect(inspectArgs));
+        const inspected = await this.inspect(inspectArgs);
+        assert(inspected);
+        return inspected;
     }
     async waitForStorage(keeper) {
         // The keeper emits this fixed marker only after creating every storage

package/dist/providers/jobs/fake-kubernetes-api.d.ts

@@ -8,6 +8,7 @@ export interface FakeExecCall {
     command: string[];
     stdin?: Readable;
     stdout?: Writable;
+    timeoutMs: number;
 }
 type Kind = "configmap" | "secret" | "pod" | "networkpolicy" | "service" | "endpoints";
 type Obj = {

package/dist/providers/jobs/fake-kubernetes-api.js

@@ -122,8 +122,9 @@ export class FakeKubernetesApi {
     }
     async exec(namespace, pod, container, command, options) {
         // Like the real API: exec into a missing pod or an unknown container fails instead of succeeding
-        // silently, so no test can pass on behaviour a cluster would reject. (The adapter's timeoutMs is
-        // not modelled: tests drive slow execs through onExec directly.)
+        // silently, so no test can pass on behaviour a cluster would reject. (The adapter does not
+        // actually enforce timeoutMs itself: tests drive slow execs through onExec directly. The value
+        // is still recorded on the call, so a test can assert which timeout the caller configured.)
         const target = this.read("pod", namespace, pod);
         if (!target)
             throw new KubernetesNotFoundError(this.key("pod", namespace, pod));
@@ -131,7 +132,15 @@ export class FakeKubernetesApi {
         if (!containers.some((entry) => entry.name === container)) {
             throw new KubernetesNotFoundError(`${this.key("pod", namespace, pod)}/${container}`);
         }
-        const call = { namespace, pod, container, command, stdin: options.stdin, stdout: options.stdout };
+        const call = {
+            namespace,
+            pod,
+            container,
+            command,
+            stdin: options.stdin,
+            stdout: options.stdout,
+            timeoutMs: options.timeoutMs,
+        };
         this.execCalls.push(call);
         return this.onExec(call);
     }

package/dist/providers/jobs/kubernetes-isolation.d.ts

@@ -1,13 +1,26 @@
 import type { V1NetworkPolicy, V1Pod, V1Secret } from "@kubernetes/client-node";
+import { type ResolvedCodingService } from "../../coding/services/catalog.js";
 import type { JobSpec } from "./types.js";
 import { type KubernetesPlatform } from "./kubernetes-platform.js";
 export declare const KUBERNETES_ISOLATION_ERROR = "kubernetes_isolation_unsupported";
-export declare const KUBERNETES_PROVIDER_UNSUPPORTED = "kubernetes_provider_unsupported";
 /** Extra seconds past timeoutSec before Kubernetes kills the pod (keeper included). */
 export declare const POD_DEADLINE_GRACE_SECONDS = 300;
 export declare const KEEPER_CONTAINER = "keeper";
 export declare const STORAGE_INIT_CONTAINER = "storage-init";
 export declare const WORKER_CONTAINER = "worker";
+/** Claude's credential-free command runner: a native sidecar in the run pod. */
+export declare const TOOL_RUNNER_CONTAINER = "tool-runner";
+/** The run Secret's key holding the tool runner's setup (claude-tool-setup.ts). */
+export declare const TOOL_SETUP_SECRET_KEY = "tool-setup";
+/** Where the tool runner listens and the agent's relay connects (claude-tool-runner/main.mjs). */
+export declare const TOOL_SOCKET_PATH = "/run/wardby/tool/runner.sock";
+/** Every service sidecar's container name starts with this (service-postgres, service-redis, ...). */
+export declare const SERVICE_CONTAINER_PREFIX = "service-";
+/** Disk-backed scratch for each of a service's writablePaths (a socket directory, /tmp). */
+export declare const SERVICE_SCRATCH_MIB = 64;
+export declare function serviceContainerName(name: string): string;
+/** What a service sidecar reserves of the pod's ephemeral storage: its data volume plus its scratch volumes. */
+export declare function serviceEphemeralStorageMib(service: ResolvedCodingService): number;
 export declare const STORAGE_ROOT = "/run/wardby/storage";
 export declare const KEEPER_SEEDED_MARKER = "/run/wardby/storage/input/.seeded";
 export declare const PROXY_POD_LABEL: {
@@ -53,12 +66,14 @@ export interface RunPodOptions {
     namespace: string;
     proxyIp: string;
     runtimeClassName?: string;
+    /** Emitted only when set, so an unconfigured launcher submits exactly the pod it always has. */
+    priorityClassName?: string;
     /** Which platform's admission rules the emitted resources must already satisfy. Default: "generic". */
     platform?: KubernetesPlatform;
 }
 export declare function buildRunPod(spec: JobSpec, options: RunPodOptions): V1Pod;
 export declare function buildRunNetworkPolicy(spec: JobSpec, namespace: string): V1NetworkPolicy;
-export declare function buildCapabilitySecret(spec: JobSpec, namespace: string, capability: string): V1Secret;
+export declare function buildCapabilitySecret(spec: JobSpec, namespace: string, capability: string, toolSetup?: string): V1Secret;
 /**
  * A copy of `pod` with ONLY the API server's own defaulting and aliasing undone
  * — the same `normalizeSpec` pass `assertRunPodMatches` runs, and nothing else.