@redwoodjs/agent-ci 0.1.0

package/dist/runner/local-job.js

@@ -0,0 +1,691 @@
+import Docker from "dockerode";
+import path from "path";
+import fs from "fs";
+import { execSync } from "child_process";
+import { createInterface } from "readline";
+import { config } from "../config.js";
+import { createLogContext } from "../output/logger.js";
+import { getWorkingDirectory } from "../output/working-directory.js";
+import { debugRunner, debugBoot } from "../output/debug.js";
+import { startServiceContainers, cleanupServiceContainers, } from "../docker/service-containers.js";
+import { killRunnerContainers } from "../docker/shutdown.js";
+import { startEphemeralDtu } from "dtu-github-actions/ephemeral";
+import { tailLogFile } from "../output/reporter.js";
+import { writeJobMetadata } from "./metadata.js";
+import { computeFakeSha, writeGitShim } from "./git-shim.js";
+import { prepareWorkspace } from "./workspace.js";
+import { createRunDirectories } from "./directory-setup.js";
+import { buildContainerEnv, buildContainerBinds, buildContainerCmd, resolveDtuHost, resolveDockerApiUrl, } from "../docker/container-config.js";
+import { buildJobResult, sanitizeStepName } from "./result-builder.js";
+import { wrapJobSteps } from "./step-wrapper.js";
+import { syncWorkspaceForRetry } from "./sync.js";
+// ─── Docker setup ─────────────────────────────────────────────────────────────
+const dockerHost = process.env.DOCKER_HOST || "unix:///var/run/docker.sock";
+const dockerConfig = dockerHost.startsWith("unix://")
+    ? { socketPath: dockerHost.replace("unix://", "") }
+    : { host: dockerHost, protocol: "ssh" };
+const docker = new Docker(dockerConfig);
+const IMAGE = "ghcr.io/actions/actions-runner:latest";
+// ─── Pre-baked runner credentials ─────────────────────────────────────────────
+// The GitHub Actions runner normally requires `config.sh` (a .NET binary) to
+// generate .runner, .credentials and .credentials_rsaparams before run.sh can
+// start.  Each invocation cold-starts .NET 6, costing ~3-5s — and we were
+// running it twice (remove + register).
+//
+// Since the DTU mock accepts any credential values, we write these files
+// directly with deterministic content, saving ~5-10s per container start.
+function writeRunnerCredentials(runnerDir, runnerName, serverUrl) {
+    // .runner — tells run.sh who it is and where to connect
+    const dotRunner = {
+        agentId: 1,
+        agentName: runnerName,
+        poolId: 1,
+        poolName: "Default",
+        serverUrl: new URL(serverUrl).origin,
+        gitHubUrl: serverUrl,
+        workFolder: "_work",
+        ephemeral: true,
+    };
+    fs.writeFileSync(path.join(runnerDir, ".runner"), JSON.stringify(dotRunner, null, 2));
+    // .credentials — OAuth scheme that run.sh reads to authenticate with the DTU
+    const dotCredentials = {
+        scheme: "OAuth",
+        data: {
+            clientId: "00000000-0000-0000-0000-000000000000",
+            authorizationUrl: `${serverUrl}/_apis/oauth2/token`,
+            oAuthEndpointUrl: `${serverUrl}/_apis/oauth2/token`,
+            requireFipsCryptography: "False",
+        },
+    };
+    fs.writeFileSync(path.join(runnerDir, ".credentials"), JSON.stringify(dotCredentials, null, 2));
+    // .credentials_rsaparams — RSA key the runner uses for token signing.
+    // Format: RSAParametersSerializable JSON (ISerializable with lowercase keys
+    // matching the RSAParametersSerializable constructor). The DTU mock never
+    // validates signatures, so we use a static pre-generated RSA 2048-bit key.
+    const dotRsaParams = {
+        d: "CQpCI+sO2GD1N/JsHHI9zEhMlu5Fcc8mU4O2bO6iscOsagFjvEnTesJgydC/Go1HuOBlx+GT9EG2h7+juS0z2o5n8Mvt5BBxlK+tqoDOs8VfQ9CSUl3hqYRPeNdBfnA1w8ovLW0wqfPO08FWTLI0urYsnwjZ5BQrBM+D7zYeA0aCsKdo75bKmaEKnmqrtIEhb7hE45XQa32Yt0RPCPi8QcQAY2HLHbdWdZYDj6k/UuDvz9H/xlDzwYq6Yikk2RSMArFzaufxCGS9tBZNEACDPYgnZnEMXRcvsnZ9FYbq81KOSifCmq7Yocq+j3rY5zJCD+PIDY9QJwPxB4PGasRKAQ==",
+        dp: "A0sY1oOz1+3uUMiy+I5xGuHGHOrEQPYspd1xGClBYYsa/Za0UDWS7V0Tn1cbRWfWtNe5vTpxcvwQd6UZBwrtHF6R2zyXFhE++PLPhCe0tH4C5FY9i9jUw9Vo8t44i/s5JUHU2B1mEptXFUA0GcVrLKS8toZSgqELSS2Q/YLRxoE=",
+        dq: "GrLC9dPJ5n3VYw51ghCH7tybUN9/Oe4T8d9v4dLQ34RQEWHwRd4g3U3zkvuhpXFPloUTMmkxS7MF5pS1evrtzkay4QUTDv+28s0xRuAsw5qNTzuFygg8t93MvpvTVZ2TNApW6C7NFvkL9NbxAnU8+I61/3ow7i6a7oYJJ0hWAxE=",
+        exponent: "AQAB",
+        inverseQ: "8DVz9FSvEdt5W4B9OjgakZHwGfnhn2VLDUxrsR5ilC5tPC/IgA8C2xEfKQM1t+K/N3pAYHBYQ6EPgtW4kquBS/Sy102xbRI7GSCnUbRtTpWYPOaCn6EaxBNzwWzbp5vCbCGvFqlSu4+OBYRVe+iCj+gAnkmT/TKPhHHbTjJHvw==",
+        modulus: "x0eoW2DD7xsW5YiorMN8pNHVvZk4ED1SHlA/bmVnRz5FjEDnQloMn0nBgIUHxoNArksknrp/FOVJv5sJHJTiRZkOp+ZmH7d3W3gmw63IxK2C5pV+6xfav9jR2+Wt/6FMYMgG2utBdF95oif1f2XREFovHoXkWms2l0CPLLHVPO44Hh9EEmBmjOeMJEZkulHJ44z9y8e+GZ2nYqO0ZiRWQcRObZ0vlRaGg6PPOl4ltay0BfNksMB3NDtlhkdVkAEFQxEaZZDK9NtkvNljXCioP3TyTAbqNUGsYCA5D+IHGZT9An99J9vUqTFP6TKjqUvy9WNiIzaUksCySA0a4SVBkQ==",
+        p: "8fgAdmWy+sTzAN19fYkWMQqeC7t1BCQMo5z5knfVLg8TtwP9ZGqDtoe+r0bGv3UgVsvvDdP/QwRvRVP+5G9l999Y6b4VbSdUbrfPfOgjpPDmRTQzHDve5jh5xBENQoRXYm7PMgHGmjwuFsE/tKtSGTrvt2Z3qcYAo0IOqLLhYmE=",
+        q: "0tXx4+P7gUWePf92UJLkzhNBClvdnmDbIt52Lui7YCARczbN/asCDJxcMy6Bh3qmIx/bNuOUrfzHkYZHfnRw8AGEK80qmiLLPI6jrUBOGRajmzemGQx0W8FWalEQfGdNIv9R2nsegDRoMq255Zo/qX60xQ6abpp0c6UNhVYSjTE=",
+    };
+    fs.writeFileSync(path.join(runnerDir, ".credentials_rsaparams"), JSON.stringify(dotRsaParams));
+}
+// ─── Main ─────────────────────────────────────────────────────────────────────
+export async function executeLocalJob(job, options) {
+    const pauseOnFailure = options?.pauseOnFailure ?? false;
+    const startTime = Date.now();
+    const store = options?.store;
+    // ── Pre-flight: verify Docker is reachable ────────────────────────────────
+    try {
+        await docker.ping();
+    }
+    catch (err) {
+        const isSocket = err?.code === "ECONNREFUSED" || err?.code === "ENOENT";
+        const hint = isSocket
+            ? "Docker does not appear to be running."
+            : `Docker is not reachable: ${err?.message || err}`;
+        throw new Error(`${hint}\n` +
+            "\n" +
+            "  To fix this:\n" +
+            "    1. Start your Docker runtime (OrbStack, Docker Desktop, etc.)\n" +
+            "    2. Wait for the engine to be ready\n" +
+            "    3. Re-run the workflow\n");
+    }
+    // ── Prepare directories ───────────────────────────────────────────────────
+    const { name: containerName, runDir, logDir, debugLogPath, } = createLogContext("agent-ci", job.runnerName);
+    // Register the job in the store so the render loop can show the boot spinner
+    store?.addJob(job.workflowPath ?? "", job.taskId ?? "job", containerName, {
+        logDir,
+        debugLogPath,
+    });
+    store?.updateJob(containerName, {
+        status: "booting",
+        startedAt: new Date().toISOString(),
+    });
+    const bootStart = Date.now();
+    const bt = (label, since) => {
+        debugBoot(`${containerName} ${label}: ${Date.now() - since}ms`);
+        return Date.now();
+    };
+    // Start an ephemeral in-process DTU for this job run so each job gets its
+    // own isolated DTU instance on a random port — eliminating port conflicts.
+    let t0 = Date.now();
+    const dtuCacheDir = path.resolve(getWorkingDirectory(), "cache", "dtu");
+    const ephemeralDtu = await startEphemeralDtu(dtuCacheDir).catch(() => null);
+    const dtuUrl = ephemeralDtu?.url ?? config.GITHUB_API_URL;
+    t0 = bt("dtu-start", t0);
+    await fetch(`${dtuUrl}/_dtu/start-runner`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+            runnerName: containerName,
+            logDir,
+            timelineDir: logDir,
+            // Package manager stores are bind-mounted into the container, so there's
+            // no need for the runner to tar/gzip them. Tell the DTU to return a
+            // synthetic hit for any cache key matching these patterns — skipping the
+            // 60s+ tar entirely.
+            virtualCachePatterns: ["pnpm", "npm", "yarn", "bun"],
+        }),
+    }).catch(() => {
+        /* non-fatal */
+    });
+    t0 = bt("dtu-register", t0);
+    // Write metadata if available (to help the UI map logs to workflows)
+    writeJobMetadata({ logDir, containerName, job });
+    // Open debug stream to capture raw container output
+    const debugStream = fs.createWriteStream(debugLogPath);
+    // ── Create run directories ────────────────────────────────────────────────
+    const dirs = createRunDirectories({
+        runDir,
+        githubRepo: job.githubRepo,
+        workflowPath: job.workflowPath,
+    });
+    // Signal handler: ensure cleanup runs even when killed.
+    const signalCleanup = () => {
+        killRunnerContainers(containerName);
+        for (const d of [dirs.containerWorkDir, dirs.shimsDir, dirs.signalsDir, dirs.diagDir]) {
+            try {
+                fs.rmSync(d, { recursive: true, force: true });
+            }
+            catch { }
+        }
+        process.exit(1);
+    };
+    process.once("SIGINT", signalCleanup);
+    process.once("SIGTERM", signalCleanup);
+    try {
+        // 1. Seed the job to Local DTU
+        const [githubOwner, githubRepoName] = (job.githubRepo || "").split("/");
+        const overriddenRepository = job.githubRepo
+            ? {
+                full_name: job.githubRepo,
+                name: githubRepoName,
+                owner: { login: githubOwner },
+                default_branch: job.repository?.default_branch || "main",
+            }
+            : job.repository;
+        const seededSteps = pauseOnFailure ? wrapJobSteps(job.steps ?? [], true) : job.steps;
+        t0 = Date.now();
+        const seedResponse = await fetch(`${dtuUrl}/_dtu/seed`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+                id: job.githubJobId || "1",
+                name: "job",
+                status: "queued",
+                localPath: dirs.workspaceDir,
+                ...job,
+                steps: seededSteps,
+                repository: overriddenRepository,
+            }),
+        });
+        if (!seedResponse.ok) {
+            throw new Error(`Failed to seed DTU: ${seedResponse.status} ${seedResponse.statusText}`);
+        }
+        t0 = bt("dtu-seed", t0);
+        // 2. Registration token (mock for local)
+        const registrationToken = "mock_local_token";
+        // 4. Write git shim BEFORE container start so the entrypoint can install it
+        // immediately. On Linux, prepareWorkspace (rsync) is slow enough that the
+        // container entrypoint would race ahead and find an empty shims dir.
+        const fakeSha = computeFakeSha(job.headSha);
+        writeGitShim(dirs.shimsDir, fakeSha);
+        // Prepare workspace files in parallel with container setup
+        const workspacePrepStart = Date.now();
+        const workspacePrepPromise = (async () => {
+            try {
+                prepareWorkspace({
+                    workflowPath: job.workflowPath,
+                    headSha: job.headSha,
+                    githubRepo: job.githubRepo,
+                    workspaceDir: dirs.workspaceDir,
+                });
+            }
+            catch (err) {
+                debugRunner(`Failed to prepare workspace: ${err}. Using host fallback.`);
+            }
+            try {
+                execSync(`chmod -R 777 "${dirs.containerWorkDir}" "${dirs.diagDir}"`, { stdio: "pipe" });
+            }
+            catch {
+                // Non-fatal: entrypoint has a fallback
+            }
+            bt("workspace-prep", workspacePrepStart);
+        })();
+        // 6. Spawn container
+        const dtuPort = new URL(dtuUrl).port || "80";
+        const dtuHost = resolveDtuHost();
+        const dockerApiUrl = resolveDockerApiUrl(dtuUrl, dtuHost);
+        const githubRepo = job.githubRepo || config.GITHUB_REPO;
+        const repoUrl = `${dockerApiUrl}/${githubRepo}`;
+        debugRunner(`Spawning container ${containerName}...`);
+        // Pre-cleanup: remove any stale container with the same name
+        try {
+            const stale = docker.getContainer(containerName);
+            await stale.remove({ force: true });
+        }
+        catch {
+            // Ignore - container doesn't exist
+        }
+        // ── Service containers ────────────────────────────────────────────────────
+        let serviceCtx;
+        if (job.services && job.services.length > 0) {
+            const svcStart = Date.now();
+            debugRunner(`Starting ${job.services.length} service container(s)...`);
+            serviceCtx = await startServiceContainers(docker, job.services, containerName, (line) => debugRunner(line));
+            bt("service-containers", svcStart);
+        }
+        const svcPortForwardSnippet = serviceCtx?.portForwards.length
+            ? serviceCtx.portForwards.join(" \n") + " \nsleep 0.3 && "
+            : "";
+        // ── Direct container injection ─────────────────────────────────────────────
+        const hostWorkDir = dirs.containerWorkDir;
+        const hostRunnerSeedDir = path.resolve(getWorkingDirectory(), "runner");
+        const hostRunnerDir = path.resolve(runDir, "runner");
+        const useDirectContainer = !!job.container;
+        const containerImage = useDirectContainer ? job.container.image : IMAGE;
+        if (useDirectContainer) {
+            await fs.promises.mkdir(hostRunnerSeedDir, { recursive: true });
+            const markerFile = path.join(hostRunnerSeedDir, ".seeded");
+            try {
+                await fs.promises.access(markerFile);
+            }
+            catch {
+                debugRunner(`Extracting runner binary to host (one-time)...`);
+                const tmpName = `agent-ci-seed-runner-${Date.now()}`;
+                const seedContainer = await docker.createContainer({
+                    Image: IMAGE,
+                    name: tmpName,
+                    Cmd: ["true"],
+                });
+                const { execSync } = await import("node:child_process");
+                execSync(`docker cp ${tmpName}:/home/runner/. "${hostRunnerSeedDir}/"`, { stdio: "pipe" });
+                await seedContainer.remove();
+                const configShPath = path.join(hostRunnerSeedDir, "config.sh");
+                let configSh = await fs.promises.readFile(configShPath, "utf8");
+                configSh = configSh.replace(/# Check dotnet Core.*?^fi$/ms, "# Dependency checks removed for container injection");
+                await fs.promises.writeFile(configShPath, configSh);
+                await fs.promises.writeFile(markerFile, new Date().toISOString());
+                debugRunner(`Runner extracted.`);
+            }
+            for (const staleFile of [".runner", ".credentials", ".credentials_rsaparams"]) {
+                try {
+                    fs.rmSync(path.join(hostRunnerSeedDir, staleFile));
+                }
+                catch {
+                    /* not present */
+                }
+            }
+            execSync(`cp -a "${hostRunnerSeedDir}" "${hostRunnerDir}"`, { stdio: "pipe" });
+            const resolvedUrl = `${dockerApiUrl}/${githubRepo}`;
+            writeRunnerCredentials(hostRunnerDir, containerName, resolvedUrl);
+        }
+        if (useDirectContainer) {
+            debugRunner(`Pulling ${containerImage}...`);
+            await new Promise((resolve, reject) => {
+                docker.pull(containerImage, (err, stream) => {
+                    if (err) {
+                        return reject(err);
+                    }
+                    docker.modem.followProgress(stream, (err) => {
+                        if (err) {
+                            return reject(err);
+                        }
+                        resolve();
+                    });
+                });
+            });
+        }
+        const containerEnv = buildContainerEnv({
+            containerName,
+            registrationToken,
+            repoUrl,
+            dockerApiUrl,
+            githubRepo,
+            headSha: job.headSha,
+            dtuHost,
+            useDirectContainer,
+        });
+        const containerBinds = buildContainerBinds({
+            hostWorkDir,
+            shimsDir: dirs.shimsDir,
+            signalsDir: pauseOnFailure ? dirs.signalsDir : undefined,
+            diagDir: dirs.diagDir,
+            toolCacheDir: dirs.toolCacheDir,
+            pnpmStoreDir: dirs.pnpmStoreDir,
+            npmCacheDir: dirs.npmCacheDir,
+            bunCacheDir: dirs.bunCacheDir,
+            playwrightCacheDir: dirs.playwrightCacheDir,
+            warmModulesDir: dirs.warmModulesDir,
+            hostRunnerDir,
+            useDirectContainer,
+        });
+        const containerCmd = buildContainerCmd({
+            svcPortForwardSnippet,
+            dtuPort,
+            dtuHost,
+            useDirectContainer,
+            containerName,
+        });
+        t0 = Date.now();
+        const container = await docker.createContainer({
+            Image: containerImage,
+            name: containerName,
+            Env: containerEnv,
+            ...(useDirectContainer ? { Entrypoint: ["bash"] } : {}),
+            Cmd: containerCmd,
+            HostConfig: {
+                Binds: containerBinds,
+                AutoRemove: false,
+                Ulimits: [{ Name: "nofile", Soft: 65536, Hard: 65536 }],
+                ...(serviceCtx ? { NetworkMode: serviceCtx.networkName } : {}),
+            },
+            Tty: true,
+        });
+        t0 = bt("container-create", t0);
+        await workspacePrepPromise;
+        t0 = Date.now();
+        await container.start();
+        bt("container-start", t0);
+        // 7. Stream logs ───────────────────────────────────────────────────────────
+        const rawStream = (await container.logs({
+            follow: true,
+            stdout: true,
+            stderr: true,
+        }));
+        let tailDone = false;
+        let lastFailedStep = null;
+        let isPaused = false;
+        let pausedStepName = null;
+        let pausedAtMs = null;
+        let lastSeenAttempt = 0;
+        let isBooting = true;
+        let stdinListening = false;
+        const timelinePath = path.join(logDir, "timeline.json");
+        const pausedSignalPath = path.join(dirs.signalsDir, "paused");
+        const signalsRunDir = path.dirname(dirs.signalsDir);
+        // Listen for Enter key to trigger retry when paused
+        const setupStdinRetry = () => {
+            if (stdinListening || !process.stdin.isTTY) {
+                return;
+            }
+            stdinListening = true;
+            process.stdin.setRawMode(true);
+            process.stdin.resume();
+            process.stdin.on("data", (key) => {
+                if (key[0] === 3) {
+                    process.stdin.setRawMode(false);
+                    process.exit(130);
+                }
+                if (key[0] === 13 && isPaused) {
+                    syncWorkspaceForRetry(signalsRunDir);
+                    fs.writeFileSync(path.join(dirs.signalsDir, "retry"), "");
+                }
+            });
+        };
+        const cleanupStdin = () => {
+            if (stdinListening && process.stdin.isTTY) {
+                process.stdin.setRawMode(false);
+                process.stdin.pause();
+                process.stdin.removeAllListeners("data");
+                stdinListening = false;
+            }
+        };
+        // ── Timeline → store updater ──────────────────────────────────────────────
+        // Reads timeline.json and the paused signal, then updates the RunStateStore.
+        // The render loop in cli.ts reads the store and calls renderRunState().
+        const updateStoreFromTimeline = () => {
+            try {
+                // ── Pause-on-failure: check for paused signal ───────────────────────
+                if (pauseOnFailure && fs.existsSync(pausedSignalPath)) {
+                    const content = fs.readFileSync(pausedSignalPath, "utf-8").trim();
+                    const lines = content.split("\n");
+                    pausedStepName = lines[0] || null;
+                    const attempt = parseInt(lines[1] || "1", 10);
+                    if (attempt !== lastSeenAttempt) {
+                        lastSeenAttempt = attempt;
+                        isPaused = true;
+                        pausedAtMs = Date.now();
+                        setupStdinRetry();
+                        // Read last output lines from the failed step's log
+                        let tailLines = [];
+                        if (pausedStepName) {
+                            const stepsDir = path.join(logDir, "steps");
+                            const sanitized = sanitizeStepName(pausedStepName);
+                            const byName = path.join(stepsDir, `${sanitized}.log`);
+                            tailLines = tailLogFile(byName, 20);
+                            if (tailLines.length === 0 && fs.existsSync(stepsDir)) {
+                                let newest = "";
+                                let newestMtime = 0;
+                                for (const f of fs.readdirSync(stepsDir)) {
+                                    if (!f.endsWith(".log")) {
+                                        continue;
+                                    }
+                                    const mt = fs.statSync(path.join(stepsDir, f)).mtimeMs;
+                                    if (mt > newestMtime) {
+                                        newestMtime = mt;
+                                        newest = f;
+                                    }
+                                }
+                                if (newest) {
+                                    tailLines = tailLogFile(path.join(stepsDir, newest), 20);
+                                }
+                            }
+                        }
+                        store?.updateJob(containerName, {
+                            status: "paused",
+                            pausedAtStep: pausedStepName || undefined,
+                            pausedAtMs: new Date(pausedAtMs).toISOString(),
+                            attempt: lastSeenAttempt,
+                            lastOutputLines: tailLines,
+                        });
+                    }
+                }
+                else if (isPaused && !fs.existsSync(pausedSignalPath)) {
+                    // Pause signal removed — job is retrying
+                    isPaused = false;
+                    pausedAtMs = null;
+                    store?.updateJob(containerName, { status: "running", pausedAtMs: undefined });
+                }
+                if (!fs.existsSync(timelinePath)) {
+                    return;
+                }
+                const records = JSON.parse(fs.readFileSync(timelinePath, "utf-8"));
+                const steps = records
+                    .filter((r) => r.type === "Task" && r.name)
+                    .sort((a, b) => (a.order ?? 0) - (b.order ?? 0));
+                if (steps.length === 0) {
+                    return;
+                }
+                // ── Transition from booting to running on first timeline entry ────────
+                if (isBooting) {
+                    isBooting = false;
+                    bt("total", bootStart);
+                    store?.updateJob(containerName, {
+                        status: isPaused ? "paused" : "running",
+                        bootDurationMs: Date.now() - bootStart,
+                    });
+                }
+                // ── Build StepState[] from timeline records ───────────────────────────
+                const seenNames = new Set();
+                let hasPostSteps = false;
+                let completeJobRecord = null;
+                const preCountNames = new Set();
+                for (const r of steps) {
+                    if (!preCountNames.has(r.name)) {
+                        preCountNames.add(r.name);
+                    }
+                    else {
+                        hasPostSteps = true;
+                    }
+                }
+                const hasCompleteJob = preCountNames.has("Complete job");
+                // Total = unique names (minus "Complete job") + "Post Setup" (if any) + "Complete job"
+                const totalSteps = preCountNames.size -
+                    (hasCompleteJob ? 1 : 0) +
+                    (hasPostSteps ? 1 : 0) +
+                    (hasCompleteJob ? 1 : 0);
+                const padW = String(totalSteps).length;
+                let stepIdx = 0;
+                const newSteps = [];
+                for (const r of steps) {
+                    if (seenNames.has(r.name)) {
+                        continue;
+                    }
+                    seenNames.add(r.name);
+                    if (r.name === "Complete job") {
+                        completeJobRecord = r;
+                        continue;
+                    }
+                    stepIdx++;
+                    const durationMs = r.startTime && r.finishTime
+                        ? new Date(r.finishTime).getTime() - new Date(r.startTime).getTime()
+                        : undefined;
+                    let status;
+                    if (!r.result && r.state !== "completed") {
+                        if (r.startTime) {
+                            status = isPaused && pausedStepName === r.name ? "paused" : "running";
+                        }
+                        else {
+                            status = "pending";
+                        }
+                    }
+                    else {
+                        const result = (r.result || "").toLowerCase();
+                        if (result === "failed") {
+                            lastFailedStep = r.name;
+                            status = "failed";
+                        }
+                        else if (result === "skipped") {
+                            status = "skipped";
+                        }
+                        else {
+                            status = "completed";
+                        }
+                    }
+                    newSteps.push({
+                        name: r.name,
+                        index: stepIdx,
+                        status,
+                        startedAt: r.startTime,
+                        completedAt: r.finishTime,
+                        durationMs,
+                    });
+                    void padW; // used for totalSteps calculation above
+                }
+                const jobFinished = !!completeJobRecord?.result;
+                if (hasPostSteps && jobFinished) {
+                    stepIdx++;
+                    newSteps.push({ name: "Post Setup", index: stepIdx, status: "completed" });
+                }
+                if (completeJobRecord && jobFinished) {
+                    stepIdx++;
+                    const durationMs = completeJobRecord.startTime && completeJobRecord.finishTime
+                        ? new Date(completeJobRecord.finishTime).getTime() -
+                            new Date(completeJobRecord.startTime).getTime()
+                        : undefined;
+                    newSteps.push({
+                        name: "Complete job",
+                        index: stepIdx,
+                        status: "completed",
+                        startedAt: completeJobRecord.startTime,
+                        completedAt: completeJobRecord.finishTime,
+                        durationMs,
+                    });
+                }
+                // Compute total duration from timeline step times
+                let totalDurationMs;
+                if (jobFinished) {
+                    const allTimes = steps
+                        .filter((r) => r.startTime && r.finishTime)
+                        .map((r) => ({
+                        start: new Date(r.startTime).getTime(),
+                        end: new Date(r.finishTime).getTime(),
+                    }));
+                    if (allTimes.length > 0) {
+                        const earliest = Math.min(...allTimes.map((t) => t.start));
+                        const latest = Math.max(...allTimes.map((t) => t.end));
+                        const ms = latest - earliest;
+                        if (!isNaN(ms) && ms >= 0) {
+                            totalDurationMs = ms;
+                        }
+                    }
+                }
+                store?.updateJob(containerName, {
+                    steps: newSteps,
+                    ...(jobFinished
+                        ? {
+                            status: lastFailedStep ? "failed" : "completed",
+                            failedStep: lastFailedStep || undefined,
+                            durationMs: totalDurationMs,
+                        }
+                        : {}),
+                });
+            }
+            catch {
+                // Best-effort
+            }
+        };
+        const pollPromise = (async () => {
+            while (!tailDone) {
+                updateStoreFromTimeline();
+                await new Promise((r) => setTimeout(r, 100));
+            }
+            // Final update
+            updateStoreFromTimeline();
+        })();
+        // Start waiting for container exit in parallel with log streaming.
+        const containerWaitPromise = container.wait();
+        await new Promise((resolve) => {
+            const rl = createInterface({ input: rawStream, crlfDelay: Infinity });
+            rl.on("line", (line) => {
+                debugStream.write(line + "\n");
+            });
+            rl.on("close", () => {
+                resolve();
+            });
+            containerWaitPromise
+                .then(() => {
+                rawStream.destroy?.();
+            })
+                .catch(() => { });
+        });
+        tailDone = true;
+        cleanupStdin();
+        await pollPromise;
+        // 8. Wait for completion
+        const CONTAINER_EXIT_TIMEOUT_MS = 30000;
+        let waitResult;
+        try {
+            waitResult = await Promise.race([
+                containerWaitPromise,
+                new Promise((_, reject) => setTimeout(() => reject(new Error("Container exit timeout")), CONTAINER_EXIT_TIMEOUT_MS)),
+            ]);
+        }
+        catch {
+            debugRunner(`Runner did not exit within ${CONTAINER_EXIT_TIMEOUT_MS / 1000}s, force-stopping container…`);
+            try {
+                await container.stop({ t: 5 });
+            }
+            catch {
+                /* already stopped */
+            }
+            waitResult = await container.wait();
+        }
+        const containerExitCode = waitResult.StatusCode;
+        const jobSucceeded = lastFailedStep === null && containerExitCode === 0;
+        // Update store with final exit code on failure
+        if (!jobSucceeded) {
+            store?.updateJob(containerName, {
+                failedExitCode: containerExitCode !== 0 ? containerExitCode : undefined,
+            });
+        }
+        await new Promise((resolve) => debugStream.end(resolve));
+        // Cleanup
+        try {
+            await container.remove({ force: true });
+        }
+        catch {
+            /* already removed */
+        }
+        if (serviceCtx) {
+            await cleanupServiceContainers(docker, serviceCtx, (line) => debugRunner(line));
+        }
+        if (fs.existsSync(dirs.shimsDir)) {
+            fs.rmSync(dirs.shimsDir, { recursive: true, force: true });
+        }
+        if (!pauseOnFailure && fs.existsSync(dirs.signalsDir)) {
+            fs.rmSync(dirs.signalsDir, { recursive: true, force: true });
+        }
+        if (fs.existsSync(dirs.diagDir)) {
+            fs.rmSync(dirs.diagDir, { recursive: true, force: true });
+        }
+        if (fs.existsSync(hostRunnerDir)) {
+            fs.rmSync(hostRunnerDir, { recursive: true, force: true });
+        }
+        if (jobSucceeded && fs.existsSync(dirs.containerWorkDir)) {
+            fs.rmSync(dirs.containerWorkDir, { recursive: true, force: true });
+        }
+        await ephemeralDtu?.close().catch(() => { });
+        return buildJobResult({
+            containerName,
+            job,
+            startTime,
+            jobSucceeded,
+            lastFailedStep,
+            containerExitCode,
+            timelinePath,
+            logDir,
+            debugLogPath,
+        });
+    }
+    finally {
+        process.removeListener("SIGINT", signalCleanup);
+        process.removeListener("SIGTERM", signalCleanup);
+    }
+}