@wardby/cli 0.1.0

package/dist/providers/jobs/docker.js

@@ -0,0 +1,1027 @@
+import { createHash } from "node:crypto";
+import { spawn } from "node:child_process";
+import { lstat, mkdir, mkdtemp, open, opendir, readdir, readFile, readlink, realpath, rename, rm, unlink, } from "node:fs/promises";
+import { dirname, isAbsolute, join, resolve, sep } from "node:path";
+import { logger } from "../../core/logger.js";
+import { parseCodingAgentOutputJson, redactAndTruncate, MAX_CODING_ARTIFACT_BYTES } from "../../coding/protocol.js";
+import { assertDockerHostSupportsIsolation, assertClaudeAgentContainerInspection, assertClaudeToolRunnerContainerInspection, assertIsolationNetworkInspection, assertKeeperContainerInspection, assertProxyContainerInspection, assertStorageVolumeInspection, assertWorkerContainerInspection, buildDockerIsolationPlan, } from "./docker-isolation.js";
+import { replaceDirectoryFromStaging } from "./workspace-swap.js";
+const BACKEND = "docker";
+const STATE_SCHEMA_VERSION = 1;
+const MAX_DOCKER_OUTPUT_BYTES = 1024 * 1024;
+const MAX_DOCKER_SEED_DIAGNOSTIC_BYTES = 4 * 1024;
+const MAX_WORKSPACE_ENTRIES = 100_000;
+const TERMINAL_PHASES = new Set(["succeeded", "failed", "stopped", "lost", "removed"]);
+export const SAFE_WORKER_DIAGNOSTIC = /^(?:worker_[a-z_]+|coding_[a-z_]+|wardby_[a-z_]+)$/;
+const dockerLog = logger.child({ module: "docker-jobs" });
+/**
+ * Docker failures never include daemon output in their public message -- that
+ * message reaches the agent owner. The operator still needs the reason, so the
+ * failing subcommand and a redacted, bounded slice of stderr ride along as the
+ * `cause`, which only the control-plane log walks (see describeFailure in
+ * ../executor/container.ts). Without it a provisioning failure is just
+ * `docker_command_failed:1`, which cost a live smoke two blind runs.
+ */
+export class DockerCommandError extends Error {
+    exitCode;
+    notFound;
+    outputExceeded;
+    constructor(exitCode, notFound = false, outputExceeded = false, detail) {
+        super(outputExceeded ? "docker_output_limit" : `docker_command_failed:${exitCode ?? "spawn"}`, {
+            cause: detail === undefined
+                ? undefined
+                : new Error(`docker ${detail.subcommand ?? "?"}: ${redactAndTruncate(detail.stderr?.trim() || "(no stderr)", 2_048)}`),
+        });
+        this.exitCode = exitCode;
+        this.notFound = notFound;
+        this.outputExceeded = outputExceeded;
+    }
+}
+export function isMissingDockerResource(stderr) {
+    return /no such (?:container|network|volume|object)|network .+ not found|is not connected to network/i.test(stderr);
+}
+/** Runs Docker with argument arrays and an explicit, minimal environment. */
+export class NodeDockerCommandRunner {
+    options;
+    dockerBinary;
+    path;
+    constructor(options) {
+        this.options = options;
+        this.dockerBinary = options.dockerBinary ?? "docker";
+        this.path = options.path ?? "/usr/local/bin:/usr/bin:/bin";
+    }
+    async run(args, options = {}) {
+        if (args.some((argument) => argument.includes("\0")))
+            throw new Error("docker_argument_invalid");
+        const extraEnv = options.env ?? {};
+        if (Object.keys(extraEnv).some((key) => key !== "WARDBY_RUN_CAPABILITY"))
+            throw new Error("docker_environment_invalid");
+        const env = {
+            PATH: this.path,
+            HOME: this.options.homeDir,
+            LANG: "C",
+            LC_ALL: "C",
+            ...(this.options.dockerHost ? { DOCKER_HOST: this.options.dockerHost } : {}),
+            ...(this.options.dockerContext ? { DOCKER_CONTEXT: this.options.dockerContext } : {}),
+            ...extraEnv,
+        };
+        const maxBytes = options.maxOutputBytes ?? MAX_DOCKER_OUTPUT_BYTES;
+        return new Promise((resolvePromise, rejectPromise) => {
+            const child = spawn(this.dockerBinary, [...args], { env, shell: false, stdio: ["ignore", "pipe", "pipe"] });
+            if (!child.stdout || !child.stderr) {
+                child.kill("SIGKILL");
+                rejectPromise(new DockerCommandError(null));
+                return;
+            }
+            const stdout = [];
+            const stderr = [];
+            let bytes = 0;
+            let exceeded = false;
+            let spawnFailed = false;
+            const capture = (chunks) => (chunk) => {
+                bytes += chunk.length;
+                if (bytes > maxBytes) {
+                    exceeded = true;
+                    child.kill("SIGKILL");
+                    return;
+                }
+                chunks.push(chunk);
+            };
+            child.stdout.on("data", capture(stdout));
+            child.stderr.on("data", capture(stderr));
+            child.once("error", () => {
+                spawnFailed = true;
+            });
+            child.once("close", (code) => {
+                if (exceeded)
+                    return rejectPromise(new DockerCommandError(code, false, true));
+                if (spawnFailed)
+                    return rejectPromise(new DockerCommandError(null));
+                const result = {
+                    stdout: Buffer.concat(stdout).toString("utf8"),
+                    stderr: Buffer.concat(stderr).toString("utf8"),
+                };
+                if (code !== 0) {
+                    return rejectPromise(new DockerCommandError(code, isMissingDockerResource(result.stderr), false, {
+                        subcommand: args[0],
+                        stderr: result.stderr,
+                    }));
+                }
+                resolvePromise(result);
+            });
+        });
+    }
+}
+export function dockerTransferEnvironment(path) {
+    // Docker CLI needs a home directory for its isolated client configuration.
+    return { PATH: path, HOME: "/tmp", LANG: "C", LC_ALL: "C", COPYFILE_DISABLE: "1" };
+}
+function dockerSeedFailure(stage, exitCode, stderr) {
+    const output = Buffer.concat(stderr).toString("utf8");
+    let diagnostic = exitCode === null ? "spawn" : `exit_${exitCode}`;
+    if (/permission denied/i.test(output))
+        diagnostic = "permission_denied";
+    else if (/executable file not found|command not found/i.test(output))
+        diagnostic = "command_missing";
+    else if (/no such file or directory/i.test(output))
+        diagnostic = "path_missing";
+    // CI enables this only for the synthetic Vitest fixture, never for production runs.
+    const debugOutput = process.env.NODE_ENV === "test" && process.env.WARDBY_DOCKER_SEED_DEBUG === "1";
+    const details = debugOutput ? `:stderr=${JSON.stringify(output.trim() || "<empty>")}` : "";
+    return new Error(`docker_seed_failed:${stage}:${diagnostic}${details}`);
+}
+/** Streams tar archives into the unprivileged keeper; no host bind mounts are used. */
+export class NodeDockerArtifactTransfer {
+    dockerBinary;
+    path;
+    constructor(dockerBinary = "docker", path = "/usr/local/bin:/usr/bin:/bin") {
+        this.dockerBinary = dockerBinary;
+        this.path = path;
+    }
+    async seedDirectory(sourceDirectory, container, destination) {
+        const metadata = await lstat(sourceDirectory);
+        if (!metadata.isDirectory() || metadata.isSymbolicLink())
+            throw new Error("docker_seed_directory_invalid");
+        await new Promise((resolvePromise, rejectPromise) => {
+            const archive = spawn("tar", ["-C", sourceDirectory, "-cf", "-", "."], {
+                env: dockerTransferEnvironment(this.path),
+                shell: false,
+                stdio: ["ignore", "pipe", "pipe"],
+            });
+            const extract = spawn(this.dockerBinary, [
+                "container",
+                "exec",
+                "--interactive",
+                "--user",
+                "10001:10001",
+                "--workdir",
+                "/",
+                container,
+                "tar",
+                "-C",
+                destination,
+                "--no-same-owner",
+                "--no-same-permissions",
+                "-xf",
+                "-",
+            ], { env: dockerTransferEnvironment(this.path), shell: false, stdio: ["pipe", "ignore", "pipe"] });
+            let failed = false;
+            let archiveExit;
+            let extractExit;
+            const archiveStderr = [];
+            const extractStderr = [];
+            const capture = (chunks) => {
+                let capturedBytes = 0;
+                return (chunk) => {
+                    const remaining = MAX_DOCKER_SEED_DIAGNOSTIC_BYTES - capturedBytes;
+                    if (remaining <= 0)
+                        return;
+                    const captured = chunk.subarray(0, remaining);
+                    chunks.push(captured);
+                    capturedBytes += captured.length;
+                };
+            };
+            const fail = (error) => {
+                if (failed)
+                    return;
+                failed = true;
+                archive.kill("SIGKILL");
+                extract.kill("SIGKILL");
+                rejectPromise(error);
+            };
+            const finish = () => {
+                if (failed || archiveExit === undefined || extractExit === undefined)
+                    return;
+                if (archiveExit !== 0)
+                    return fail(dockerSeedFailure("archive", archiveExit, archiveStderr));
+                if (extractExit !== 0)
+                    return fail(dockerSeedFailure("extract", extractExit, extractStderr));
+                resolvePromise();
+            };
+            archive.once("error", () => fail(dockerSeedFailure("archive", null, archiveStderr)));
+            extract.once("error", () => fail(dockerSeedFailure("extract", null, extractStderr)));
+            if (!archive.stdout || !archive.stderr || !extract.stdin || !extract.stderr)
+                return fail(dockerSeedFailure("extract", null, extractStderr));
+            archive.stdout.pipe(extract.stdin);
+            archive.stderr.on("data", capture(archiveStderr));
+            extract.stderr.on("data", capture(extractStderr));
+            archive.stdout.once("error", () => fail(dockerSeedFailure("archive", null, archiveStderr)));
+            extract.stdin.once("error", () => fail(dockerSeedFailure("extract", null, extractStderr)));
+            archive.once("close", (code) => {
+                archiveExit = code;
+                finish();
+            });
+            extract.once("close", (code) => {
+                extractExit = code;
+                finish();
+            });
+        });
+    }
+    async seedInput(sourceFile, container) {
+        const metadata = await lstat(sourceFile);
+        if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > MAX_CODING_ARTIFACT_BYTES) {
+            throw new Error("docker_input_artifact_invalid");
+        }
+        const docker = new NodeDockerCommandRunner({
+            dockerBinary: this.dockerBinary,
+            homeDir: "/tmp",
+            path: this.path,
+        });
+        await docker.run(["container", "cp", sourceFile, `${container}:/run/wardby/storage/input/input.json`]);
+    }
+    async materializeDirectory(container, source, destination, maxBytes) {
+        await replaceDirectoryFromStaging(destination, maxBytes, "docker_workspace_destination_invalid", async (staging) => {
+            await new NodeDockerCommandRunner({ dockerBinary: this.dockerBinary, homeDir: "/tmp", path: this.path }).run([
+                "container",
+                "cp",
+                `${container}:${source}/.`,
+                staging,
+            ]);
+        });
+    }
+}
+export async function validateMaterializedWorkspace(root, maxBytes) {
+    if (!Number.isSafeInteger(maxBytes) || maxBytes <= 0)
+        throw new Error("docker_workspace_limit_invalid");
+    let entries = 0;
+    let bytes = 0;
+    const visit = async (directory) => {
+        const handle = await opendir(directory);
+        for await (const entry of handle) {
+            entries += 1;
+            if (entries > MAX_WORKSPACE_ENTRIES)
+                throw new Error("docker_workspace_entry_limit");
+            if (entry.name.toLowerCase() === ".git")
+                throw new Error("docker_workspace_nested_repository");
+            const fullPath = resolve(directory, entry.name);
+            const metadata = await lstat(fullPath);
+            if (metadata.isSymbolicLink()) {
+                const target = await readlink(fullPath);
+                const resolvedTarget = resolve(directory, target);
+                if (isAbsolute(target) || (resolvedTarget !== root && !resolvedTarget.startsWith(`${root}${sep}`))) {
+                    throw new Error("docker_workspace_symlink_escape");
+                }
+            }
+            else if (metadata.isDirectory()) {
+                await visit(fullPath);
+            }
+            else if (metadata.isFile()) {
+                bytes += metadata.size;
+                if (bytes > maxBytes)
+                    throw new Error("docker_workspace_size_limit");
+            }
+            else {
+                throw new Error("docker_workspace_special_file");
+            }
+        }
+    };
+    await visit(root);
+}
+function hash(value) {
+    return createHash("sha256").update(value).digest("hex");
+}
+function stableSpecHash(spec) {
+    return hash(JSON.stringify({
+        ...spec,
+        labels: Object.fromEntries(Object.entries(spec.labels).sort(([left], [right]) => left.localeCompare(right))),
+    }));
+}
+function clone(value) {
+    return structuredClone(value);
+}
+function isTerminal(record) {
+    return TERMINAL_PHASES.has(record.phase);
+}
+function statusFor(record) {
+    switch (record.phase) {
+        case "provisioning":
+            return { state: "pending" };
+        case "active":
+            return { state: "running" };
+        case "succeeded":
+            return { state: "succeeded" };
+        case "failed":
+            return { state: "failed", ...(record.result?.reason === "timed_out" ? { reason: "timed_out" } : {}) };
+        case "stopped":
+            return { state: "stopped" };
+        case "lost":
+            return { state: "lost" };
+        case "removed":
+            throw new Error("job_removed");
+    }
+}
+function resultFor(phase) {
+    switch (phase) {
+        case "succeeded":
+            return { exitCode: 0, reason: "completed" };
+        case "failed":
+            return { exitCode: 1, reason: "failed" };
+        case "stopped":
+            return { exitCode: 143, reason: "stopped" };
+        case "lost":
+            return { exitCode: 1, reason: "lost" };
+    }
+}
+function ensureCapability(value) {
+    if (!/^rrp_[A-Za-z0-9_-]{16,512}$/.test(value))
+        throw new Error("docker_capability_invalid");
+    return value;
+}
+function appendLabels(args, labels) {
+    const entrypointIndex = args.indexOf("--entrypoint");
+    const imageIndex = entrypointIndex >= 0 ? entrypointIndex + 2 : args.length - 1;
+    if (!args[imageIndex])
+        throw new Error("docker_plan_invalid");
+    const trusted = Object.entries(labels).flatMap(([key, value]) => ["--label", `${key}=${value}`]);
+    return [...args.slice(0, imageIndex), ...trusted, ...args.slice(imageIndex)];
+}
+function stateFileName(runId) {
+    return `${hash(runId)}.json`;
+}
+function resourceLabels(record) {
+    return {
+        "io.wardby.managed": "true",
+        "io.wardby.component": "coding-worker",
+        "io.wardby.run-sha256": hash(record.runId),
+        "io.wardby.run-id": record.runId,
+        "io.wardby.job-id": record.jobId,
+        "io.wardby.spec-sha256": record.specHash,
+        "io.wardby.created-at": new Date(record.createdAt).toISOString(),
+        "io.wardby.schema": String(STATE_SCHEMA_VERSION),
+    };
+}
+function labelsMatch(actual, expected) {
+    return Object.entries(expected).every(([key, value]) => actual?.[key] === value);
+}
+function capabilityFromInspection(container) {
+    return container.Config?.Env?.find((value) => value.startsWith("WARDBY_RUN_CAPABILITY="))?.slice("WARDBY_RUN_CAPABILITY=".length);
+}
+function dockerState(inspection) {
+    const state = inspection.State;
+    if (!state)
+        return undefined;
+    if (state.Running || state.Status === "restarting")
+        return "active";
+    if (state.Status === "created")
+        return "provisioning";
+    if (state.Status === "exited") {
+        if (state.OOMKilled)
+            return "failed";
+        return state.ExitCode === 0 ? "succeeded" : "failed";
+    }
+    return undefined;
+}
+function terminalResult(phase, inspection) {
+    if (phase === "succeeded")
+        return { exitCode: 0, reason: "completed" };
+    if (phase === "failed")
+        return {
+            exitCode: inspection?.State?.OOMKilled ? 137 : Math.max(1, inspection?.State?.ExitCode ?? 1),
+            reason: "failed",
+        };
+    if (phase === "stopped" || phase === "lost")
+        return resultFor(phase);
+    return undefined;
+}
+export class DockerJobLauncher {
+    options;
+    stateRoot;
+    workspaceRoot;
+    docker;
+    transfer;
+    now;
+    locks = new Map();
+    timers = new Map();
+    ready;
+    constructor(options) {
+        this.options = options;
+        this.stateRoot = resolve(options.stateRoot);
+        this.workspaceRoot = resolve(options.workspaceRoot);
+        if (this.stateRoot === resolve("/") || this.workspaceRoot === resolve("/"))
+            throw new Error("docker_launcher_path_invalid");
+        this.docker = options.docker ?? new NodeDockerCommandRunner({ homeDir: join(this.stateRoot, ".home") });
+        this.transfer = options.transfer ?? new NodeDockerArtifactTransfer();
+        this.now = options.now ?? Date.now;
+        this.ready = this.initialize();
+    }
+    async launch(spec) {
+        await this.ready;
+        return this.withRun(spec.runId, async () => this.launchLocked(spec));
+    }
+    async status(handle) {
+        await this.ready;
+        const record = await this.findRecord(handle);
+        if (!record)
+            throw new Error("job_not_found");
+        return this.withRun(record.runId, async () => {
+            const current = await this.readRecord(record.runId);
+            if (!current)
+                throw new Error("job_not_found");
+            if (current.phase === "removed")
+                throw new Error("job_removed");
+            await this.refresh(current);
+            return clone(statusFor(current));
+        });
+    }
+    async collect(handle) {
+        await this.ready;
+        const record = await this.findRecord(handle);
+        if (!record)
+            throw new Error("job_not_found");
+        return this.withRun(record.runId, async () => {
+            const current = await this.requireRecord(record.runId);
+            if (current.phase === "removed")
+                throw new Error("job_removed");
+            await this.refresh(current);
+            if (!isTerminal(current))
+                throw new Error("job_not_terminal");
+            if (!current.result)
+                current.result = resultFor(current.phase);
+            if (current.phase === "failed" && !current.result.diagnostic) {
+                const diagnostic = await this.readWorkerFailureDiagnostic(current);
+                if (diagnostic) {
+                    current.result = { ...current.result, diagnostic };
+                    await this.writeRecord(current);
+                }
+            }
+            if (current.phase === "succeeded" && !current.result.resultArtifact) {
+                const artifact = await this.readResultArtifact(current);
+                current.result = { ...current.result, resultArtifact: artifact };
+                await this.writeRecord(current);
+            }
+            return clone(current.result);
+        });
+    }
+    async materializeWorkspace(handle, destination) {
+        await this.ready;
+        const record = await this.findRecord(handle);
+        if (!record)
+            throw new Error("job_not_found");
+        await this.withRun(record.runId, async () => {
+            const current = await this.requireRecord(record.runId);
+            await this.refresh(current);
+            if (current.phase !== "succeeded")
+                throw new Error("job_not_succeeded");
+            const expected = resolve(this.workspaceRoot, current.runId, "workspace");
+            if (resolve(destination) !== expected || !expected.startsWith(`${this.workspaceRoot}${sep}`)) {
+                throw new Error("docker_workspace_destination_invalid");
+            }
+            await this.transfer.materializeDirectory(this.keeperName(current), "/run/wardby/storage/workspace", expected, current.spec.limits.diskMb * 1024 * 1024);
+        });
+    }
+    async stop(handle, _reason) {
+        await this.ready;
+        const record = await this.findRecord(handle);
+        if (!record)
+            return;
+        await this.withRun(record.runId, async () => {
+            const current = await this.readRecord(record.runId);
+            if (!current || isTerminal(current))
+                return;
+            await this.stopRecord(current, false);
+        });
+    }
+    async remove(handle) {
+        await this.ready;
+        const record = await this.findRecord(handle);
+        if (!record)
+            return;
+        await this.withRun(record.runId, async () => {
+            const current = await this.requireRecord(record.runId);
+            if (current.phase === "removed")
+                return;
+            await this.refresh(current);
+            if (!isTerminal(current))
+                throw new Error("job_not_terminal");
+            await this.cleanupResources(current);
+            current.phase = "removed";
+            await this.writeRecord(current);
+            this.clearDeadline(current);
+        });
+    }
+    async initialize() {
+        await mkdir(this.stateRoot, { recursive: true, mode: 0o700 });
+        await mkdir(join(this.stateRoot, ".home"), { recursive: true, mode: 0o700 });
+        await realpath(this.stateRoot);
+        const records = await this.allRecords();
+        for (const record of records) {
+            if (record.phase === "removed")
+                continue;
+            try {
+                if (record.deadlineAt <= this.now()) {
+                    const active = await this.options.isRunActive(record.runId, clone(record.handle));
+                    if (active)
+                        await this.withRun(record.runId, async () => this.stopRecord(record, true));
+                    else
+                        await this.withRun(record.runId, async () => this.sweepExpired(record));
+                }
+                else if (!isTerminal(record)) {
+                    this.scheduleDeadline(record);
+                }
+            }
+            catch (error) {
+                // One record's startup recovery must never abort the whole launcher:
+                // a stale container/network another process already reaped, a
+                // transient daemon hiccup, or an unrecognized Docker error string
+                // should surface as a logged recovery failure, not crash the server.
+                dockerLog.warn({ err: error, runId: record.runId }, "docker_startup_recovery_failed");
+            }
+        }
+    }
+    async launchLocked(spec) {
+        const plan = buildDockerIsolationPlan(spec, this.options.proxyContainer);
+        const specHash = stableSpecHash(spec);
+        const existing = await this.readRecord(spec.runId);
+        if (existing) {
+            if (existing.specHash !== specHash)
+                throw new Error("job_spec_conflict");
+            if (existing.phase === "removed" || isTerminal(existing))
+                return clone(existing.handle);
+            if (existing.phase === "provisioning") {
+                // Provisioning can create some resources before a host crash. Never
+                // guess which side effects completed or launch another agent pair.
+                await this.cleanupResources(existing).catch(() => undefined);
+                existing.phase = "lost";
+                existing.result = resultFor("lost");
+                await this.writeRecord(existing);
+                this.clearDeadline(existing);
+                return clone(existing.handle);
+            }
+            const capability = ensureCapability(await this.options.resolveCapability(spec.runId));
+            if (hash(capability) !== existing.capabilityHash)
+                throw new Error("docker_capability_changed");
+            if (existing.phase === "active") {
+                await this.refresh(existing);
+                return clone(existing.handle);
+            }
+            return this.provision(existing, plan, capability);
+        }
+        await this.validateInputArtifact(spec.inputArtifact);
+        const capability = ensureCapability(await this.options.resolveCapability(spec.runId));
+        const record = {
+            schemaVersion: STATE_SCHEMA_VERSION,
+            runId: spec.runId,
+            spec: clone(spec),
+            specHash,
+            handle: { backend: BACKEND, id: plan.names.workerContainer },
+            jobId: `job-${hash(`${spec.runId}:${specHash}`).slice(0, 24)}`,
+            createdAt: this.now(),
+            deadlineAt: this.now() + plan.deadlineMs,
+            capabilityHash: hash(capability),
+            phase: "provisioning",
+        };
+        // This durable record is the idempotency fence before any Docker side effect.
+        await this.writeRecord(record);
+        try {
+            return await this.provision(record, plan, capability);
+        }
+        catch (error) {
+            await this.options
+                .onProvisionFailure?.({ runId: record.runId, keeperContainer: plan.names.keeperContainer })
+                .catch(() => undefined);
+            await this.cleanupResources(record).catch(() => undefined);
+            record.phase = "lost";
+            record.result = resultFor("lost");
+            await this.writeRecord(record).catch(() => undefined);
+            throw error;
+        }
+    }
+    async provision(record, plan, capability) {
+        await this.assertHost();
+        const labels = resourceLabels(record);
+        await this.createAndAssert(appendLabels(plan.networkCreateArgs, labels), ["network", "inspect", plan.names.network], (value) => {
+            const inspected = value;
+            assertIsolationNetworkInspection(inspected, record.runId);
+            if (!labelsMatch(inspected.Labels, labels))
+                throw new Error("docker_resource_attestation_failed");
+        });
+        await this.createAndAssert(appendLabels(plan.storageVolumeCreateArgs, labels), ["volume", "inspect", plan.names.storageVolume], (value) => {
+            const inspected = value;
+            assertStorageVolumeInspection(inspected, record.spec);
+            if (!labelsMatch(inspected.Labels, labels))
+                throw new Error("docker_resource_attestation_failed");
+        });
+        await this.createAndAssert(appendLabels(plan.keeperCreateArgs, labels), ["container", "inspect", plan.names.keeperContainer], (value) => {
+            const inspected = value;
+            assertKeeperContainerInspection(inspected, record.spec);
+            if (!labelsMatch(inspected.Config?.Labels, labels))
+                throw new Error("docker_resource_attestation_failed");
+        });
+        await this.startContainer(plan.names.keeperContainer);
+        await this.waitForStorage(plan.names.keeperContainer);
+        await this.seedRun(record, plan.names.keeperContainer);
+        await this.run(plan.proxyNetworkConnectArgs);
+        const proxy = await this.inspect(["container", "inspect", this.options.proxyContainer]);
+        assertProxyContainerInspection(proxy, record.runId);
+        if (plan.toolCreateArgs) {
+            await this.createAndAssert(appendLabels(plan.toolCreateArgs, labels), ["container", "inspect", plan.names.toolContainer], (value) => {
+                assertClaudeToolRunnerContainerInspection(value, record.spec);
+                if (!labelsMatch(value.Config?.Labels, labels)) {
+                    throw new Error("docker_resource_attestation_failed");
+                }
+            });
+            await this.startContainer(plan.names.toolContainer);
+            await this.waitForToolRunner(plan.names.toolContainer);
+        }
+        const workerArgs = appendLabels(plan.workerCreateArgs, labels);
+        await this.createAndAssert(workerArgs, ["container", "inspect", plan.names.workerContainer], (value) => {
+            const inspected = value;
+            if (record.spec.provider === "claude-code") {
+                assertClaudeAgentContainerInspection(inspected, record.spec, capability);
+            }
+            else {
+                assertWorkerContainerInspection(inspected, record.spec, capability);
+            }
+            if (!labelsMatch(inspected.Config?.Labels, labels))
+                throw new Error("docker_resource_attestation_failed");
+        }, { env: { WARDBY_RUN_CAPABILITY: capability } });
+        await this.startContainer(plan.names.workerContainer);
+        record.phase = "active";
+        await this.writeRecord(record);
+        this.scheduleDeadline(record);
+        return clone(record.handle);
+    }
+    async refresh(record) {
+        if (isTerminal(record))
+            return;
+        if (record.deadlineAt <= this.now()) {
+            await this.stopRecord(record, true);
+            return;
+        }
+        let inspection;
+        try {
+            inspection = await this.inspect(["container", "inspect", record.handle.id]);
+        }
+        catch (error) {
+            if (error instanceof DockerCommandError && error.notFound) {
+                record.phase = "lost";
+                record.result = resultFor("lost");
+                await this.writeRecord(record);
+                this.clearDeadline(record);
+                return;
+            }
+            throw new Error("docker_status_unavailable", { cause: error });
+        }
+        if (!labelsMatch(inspection.Config?.Labels, resourceLabels(record)))
+            throw new Error("docker_resource_attestation_failed");
+        const capability = capabilityFromInspection(inspection);
+        if (!capability || hash(capability) !== record.capabilityHash)
+            throw new Error("docker_resource_attestation_failed");
+        if (record.spec.provider === "claude-code") {
+            assertClaudeAgentContainerInspection(inspection, record.spec, capability);
+            const tool = await this.inspect([
+                "container",
+                "inspect",
+                buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.toolContainer,
+            ]);
+            assertClaudeToolRunnerContainerInspection(tool, record.spec);
+            if (tool.State?.Running !== true || tool.State?.Status !== "running") {
+                record.phase = "failed";
+                record.result = { exitCode: 1, reason: "failed", diagnostic: "worker_tool_runner_failed" };
+                await this.writeRecord(record);
+                this.clearDeadline(record);
+                return;
+            }
+        }
+        else {
+            assertWorkerContainerInspection(inspection, record.spec, capability);
+        }
+        const phase = dockerState(inspection);
+        if (!phase || phase === "provisioning" || phase === "active")
+            return;
+        record.phase = phase;
+        record.result = terminalResult(phase, inspection);
+        await this.writeRecord(record);
+        this.clearDeadline(record);
+    }
+    async stopRecord(record, timedOut) {
+        if (isTerminal(record))
+            return;
+        try {
+            await this.run(["container", "stop", "--time", "10", record.handle.id]);
+        }
+        catch (error) {
+            if (!(error instanceof DockerCommandError && error.notFound))
+                throw new Error("docker_stop_failed", { cause: error });
+        }
+        if (record.spec.provider === "claude-code") {
+            const tool = buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.toolContainer;
+            try {
+                await this.run(["container", "stop", "--time", "10", tool]);
+            }
+            catch (error) {
+                if (!(error instanceof DockerCommandError && error.notFound))
+                    throw new Error("docker_stop_failed", { cause: error });
+            }
+        }
+        record.phase = timedOut ? "failed" : "stopped";
+        record.result = timedOut ? { exitCode: 124, reason: "timed_out" } : resultFor("stopped");
+        await this.writeRecord(record);
+        this.clearDeadline(record);
+    }
+    async sweepExpired(record) {
+        // The DB verifier ran immediately before this call; never sweep by label alone.
+        try {
+            await this.cleanupResources(record);
+        }
+        catch (error) {
+            if (!(error instanceof DockerCommandError && error.notFound))
+                throw error;
+        }
+        record.phase = "removed";
+        await this.writeRecord(record);
+        this.clearDeadline(record);
+    }
+    async cleanupResources(record) {
+        const plan = buildDockerIsolationPlan(record.spec, this.options.proxyContainer);
+        const labels = resourceLabels(record);
+        await this.removeContainerIfAttested(plan.names.workerContainer, labels);
+        if (plan.toolCreateArgs)
+            await this.removeContainerIfAttested(plan.names.toolContainer, labels);
+        await this.runIgnoreMissing(["network", "disconnect", plan.names.network, this.options.proxyContainer]);
+        await this.removeContainerIfAttested(plan.names.keeperContainer, labels);
+        await this.removeNetworkIfAttested(plan.names.network, labels);
+        await this.removeVolumeIfAttested(plan.names.storageVolume, labels);
+    }
+    async removeContainerIfAttested(name, labels) {
+        try {
+            const inspected = await this.inspect(["container", "inspect", name]);
+            if (!labelsMatch(inspected.Config?.Labels, labels))
+                throw new Error("docker_resource_attestation_failed");
+            await this.runIgnoreMissing(["container", "rm", "--force", name]);
+        }
+        catch (error) {
+            if (!(error instanceof DockerCommandError && error.notFound))
+                throw error;
+        }
+    }
+    async removeNetworkIfAttested(name, labels) {
+        try {
+            const inspected = await this.inspect(["network", "inspect", name]);
+            if (!labelsMatch(inspected.Labels, labels))
+                throw new Error("docker_resource_attestation_failed");
+            await this.runIgnoreMissing(["network", "rm", name]);
+        }
+        catch (error) {
+            if (!(error instanceof DockerCommandError && error.notFound))
+                throw error;
+        }
+    }
+    async removeVolumeIfAttested(name, labels) {
+        try {
+            const inspected = await this.inspect(["volume", "inspect", name]);
+            if (!labelsMatch(inspected.Labels, labels))
+                throw new Error("docker_resource_attestation_failed");
+            await this.runIgnoreMissing(["volume", "rm", name]);
+        }
+        catch (error) {
+            if (!(error instanceof DockerCommandError && error.notFound))
+                throw error;
+        }
+    }
+    async seedRun(record, keeper) {
+        const runRoot = resolve(this.workspaceRoot, record.runId);
+        if (!runRoot.startsWith(`${this.workspaceRoot}/`))
+            throw new Error("docker_workspace_path_invalid");
+        await this.transfer.seedDirectory(resolve(runRoot, "workspace"), keeper, "/run/wardby/storage/workspace");
+        await this.transfer.seedDirectory(resolve(runRoot, "git"), keeper, "/run/wardby/storage/git");
+        await this.transfer.seedInput(record.spec.inputArtifact, keeper);
+    }
+    async readResultArtifact(record) {
+        const staging = await mkdtemp(join(this.stateRoot, ".wardby-result-"));
+        const target = join(staging, "result.json");
+        try {
+            await this.run(["container", "cp", `${this.keeperName(record)}:/run/wardby/storage/output/result.json`, target]);
+            const metadata = await lstat(target);
+            if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > MAX_CODING_ARTIFACT_BYTES) {
+                throw new Error("docker_result_artifact_invalid");
+            }
+            const raw = await readFile(target, "utf8");
+            if (Buffer.byteLength(raw, "utf8") > MAX_CODING_ARTIFACT_BYTES)
+                throw new Error("docker_result_artifact_invalid");
+            const output = parseCodingAgentOutputJson(raw);
+            if (output.runId !== record.runId)
+                throw new Error("docker_result_run_mismatch");
+            return JSON.stringify(output);
+        }
+        catch (error) {
+            if (error instanceof Error && error.message === "docker_result_run_mismatch")
+                throw error;
+            throw new Error("docker_result_artifact_invalid", { cause: error });
+        }
+        finally {
+            await rm(staging, { recursive: true, force: true });
+        }
+    }
+    async readWorkerFailureDiagnostic(record) {
+        try {
+            const logs = await this.run(["container", "logs", "--tail", "8", record.handle.id]);
+            const raw = `${logs.stdout}\n${logs.stderr}`;
+            for (const line of raw.split("\n").reverse()) {
+                try {
+                    const value = JSON.parse(line);
+                    if (typeof value.error === "string" && SAFE_WORKER_DIAGNOSTIC.test(value.error))
+                        return value.error;
+                }
+                catch {
+                    // Worker output is untrusted; only parse one fixed JSON shape.
+                }
+            }
+        }
+        catch {
+            // Diagnostics are optional and must never affect terminal cleanup.
+        }
+        return undefined;
+    }
+    keeperName(record) {
+        return buildDockerIsolationPlan(record.spec, this.options.proxyContainer).names.keeperContainer;
+    }
+    async assertHost() {
+        const info = await this.inspect(["info", "--format", "{{json .}}"]);
+        assertDockerHostSupportsIsolation(info);
+    }
+    async createAndAssert(createArgs, inspectArgs, assert, options) {
+        try {
+            await this.run(createArgs, options);
+        }
+        catch {
+            // A retry may find a resource created before the previous process crashed.
+        }
+        assert(await this.inspect(inspectArgs));
+    }
+    async waitForStorage(keeper) {
+        // The keeper emits this fixed marker only after creating every storage
+        // directory. Reading logs avoids consuming one of its 16 PID slots.
+        for (let attempt = 0; attempt < 100; attempt += 1) {
+            try {
+                const logs = await this.run(["container", "logs", "--tail", "8", keeper]);
+                if (`${logs.stdout}\n${logs.stderr}`.split("\n").includes("wardby_storage_ready"))
+                    return;
+            }
+            catch {
+                // A just-started container can reject log reads before its log driver initializes.
+            }
+            await new Promise((resolvePromise) => setTimeout(resolvePromise, 50));
+        }
+        throw new Error("docker_storage_not_ready");
+    }
+    async waitForToolRunner(tool) {
+        // The runner prints this fixed marker only after binding its Unix socket.
+        // Reading logs avoids adding a process to its tightly bounded PID cgroup.
+        for (let attempt = 0; attempt < 100; attempt += 1) {
+            try {
+                const logs = await this.run(["container", "logs", "--tail", "8", tool]);
+                if (`${logs.stdout}\n${logs.stderr}`.split("\n").includes("wardby_tool_runner_ready"))
+                    return;
+            }
+            catch {
+                // A just-started container can reject log reads before its log driver initializes.
+            }
+            await new Promise((resolvePromise) => setTimeout(resolvePromise, 50));
+        }
+        throw new Error("docker_tool_runner_not_ready");
+    }
+    async validateInputArtifact(path) {
+        const metadata = await lstat(path);
+        if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > MAX_CODING_ARTIFACT_BYTES) {
+            throw new Error("docker_input_artifact_invalid");
+        }
+    }
+    async run(args, options) {
+        return this.docker.run(args, options);
+    }
+    async runIgnoreMissing(args) {
+        try {
+            await this.run(args);
+        }
+        catch (error) {
+            if (!(error instanceof DockerCommandError && error.notFound))
+                throw error;
+        }
+    }
+    async startContainer(name) {
+        try {
+            await this.run(["container", "start", name]);
+        }
+        catch (error) {
+            if (!(error instanceof DockerCommandError))
+                throw error;
+            const inspection = await this.inspect(["container", "inspect", name]);
+            if (inspection.State?.Running !== true)
+                throw new Error("docker_start_failed", { cause: error });
+        }
+    }
+    async inspect(args) {
+        const output = (await this.run(args, { maxOutputBytes: MAX_DOCKER_OUTPUT_BYTES })).stdout;
+        let decoded;
+        try {
+            decoded = JSON.parse(output);
+        }
+        catch {
+            throw new Error("docker_inspection_invalid");
+        }
+        return (Array.isArray(decoded) ? decoded[0] : decoded);
+    }
+    async findRecord(handle) {
+        if (handle.backend !== BACKEND)
+            throw new Error("job_backend_mismatch");
+        return (await this.allRecords()).find((record) => record.handle.id === handle.id);
+    }
+    async requireRecord(runId) {
+        const record = await this.readRecord(runId);
+        if (!record)
+            throw new Error("job_not_found");
+        return record;
+    }
+    async readRecord(runId) {
+        try {
+            const decoded = JSON.parse(await readFile(join(this.stateRoot, stateFileName(runId)), "utf8"));
+            if (decoded.schemaVersion !== STATE_SCHEMA_VERSION ||
+                decoded.runId !== runId ||
+                decoded.handle?.backend !== BACKEND ||
+                !decoded.handle.id ||
+                !decoded.specHash ||
+                !decoded.capabilityHash) {
+                throw new Error("docker_state_invalid");
+            }
+            return decoded;
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return undefined;
+            throw error;
+        }
+    }
+    async allRecords() {
+        const entries = await readdir(this.stateRoot, { withFileTypes: true });
+        if (entries.length > 10_000)
+            throw new Error("docker_state_limit");
+        const records = await Promise.all(entries
+            .filter((entry) => entry.isFile() && /^[a-f0-9]{64}\.json$/.test(entry.name))
+            .map(async (entry) => JSON.parse(await readFile(join(this.stateRoot, entry.name), "utf8"))));
+        if (records.some((record) => record.schemaVersion !== STATE_SCHEMA_VERSION || stateFileName(record.runId) !== hash(record.runId) + ".json")) {
+            throw new Error("docker_state_invalid");
+        }
+        return records;
+    }
+    async writeRecord(record) {
+        const path = join(this.stateRoot, stateFileName(record.runId));
+        const temporary = `${path}.${process.pid}.${Date.now()}.tmp`;
+        const file = await open(temporary, "wx", 0o600);
+        try {
+            await file.writeFile(`${JSON.stringify(record)}\n`);
+            await file.sync();
+            await file.close();
+            await rename(temporary, path);
+            const directory = await open(dirname(path), "r");
+            try {
+                await directory.sync();
+            }
+            finally {
+                await directory.close();
+            }
+        }
+        catch (error) {
+            await file.close().catch(() => undefined);
+            await unlink(temporary).catch(() => undefined);
+            throw error;
+        }
+    }
+    async withRun(runId, action) {
+        const previous = this.locks.get(runId) ?? Promise.resolve();
+        let release = () => undefined;
+        const next = new Promise((resolvePromise) => {
+            release = resolvePromise;
+        });
+        const queued = previous.then(() => next);
+        this.locks.set(runId, queued);
+        await previous;
+        try {
+            return await action();
+        }
+        finally {
+            release();
+            if (this.locks.get(runId) === queued)
+                this.locks.delete(runId);
+        }
+    }
+    scheduleDeadline(record) {
+        this.clearDeadline(record);
+        const delay = Math.max(0, record.deadlineAt - this.now());
+        const timer = setTimeout(() => {
+            void this.withRun(record.runId, async () => {
+                const current = await this.readRecord(record.runId);
+                if (current && !isTerminal(current) && current.deadlineAt <= this.now())
+                    await this.stopRecord(current, true);
+            }).catch(() => undefined);
+        }, delay);
+        timer.unref();
+        this.timers.set(record.runId, timer);
+    }
+    clearDeadline(record) {
+        const timer = this.timers.get(record.runId);
+        if (timer)
+            clearTimeout(timer);
+        this.timers.delete(record.runId);
+    }
+}