@moldesk/core 0.0.4 → 0.0.6

package/dist/cloud/run.test.js

@@ -0,0 +1,597 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { parseManifest } from "@moldesk/registry";
+import { getMoldeskPaths, } from "@moldesk/runtime";
+import { MoldeskError } from "../errors.js";
+import { buildQuote } from "./quote.js";
+import { planCloudRun, runCloudModel } from "./run.js";
+import { CloudJournal } from "./state.js";
+import { recoverCloudRun } from "./manage.js";
+const DIGEST = "a".repeat(64);
+const homes = [];
+function manifest() {
+    return parseManifest({
+        schemaVersion: 1,
+        name: "fake-model",
+        displayName: "Fake",
+        modelVersion: "1.0.0",
+        adapterVersion: "0.1.0",
+        description: "t",
+        category: "other",
+        status: "planned",
+        runtimes: [{ kind: "python", python: "3.11", installer: "uv", platforms: ["linux-x64"], requirements: [{ name: "torch" }] }],
+        hardware: { platforms: ["linux-x64"] },
+        input: { formats: [".txt"], required: true },
+        outputs: [{ id: "result", glob: "*.out", required: true }],
+        cloud: {
+            schemaVersion: 1,
+            platform: "linux-x64",
+            images: { gpu: `registry.example/w/gpu@sha256:${DIGEST}`, cpu: `registry.example/w/cpu@sha256:${DIGEST}` },
+            minVramGb: 24,
+            minRamGb: 32,
+            ephemeralDiskGb: 40,
+            volumeGb: 40,
+            assetLayoutRevision: "1",
+            cpuAssetPreparation: true,
+        },
+    }, "x/manifest.yaml");
+}
+function adapter() {
+    return {
+        modelName: "fake-model",
+        params: [],
+        async validateInput() { },
+        async installPlan() {
+            return { steps: [] };
+        },
+        async command() {
+            throw new Error("the client must never build a command for a remote run");
+        },
+        async collectOutputs(context) {
+            const files = fs.existsSync(context.outputDir) ? fs.readdirSync(context.outputDir).filter((name) => name.endsWith(".out")) : [];
+            if (files.length === 0)
+                throw new MoldeskError({ code: "MISSING_REQUIRED_OUTPUT", message: "No .out file was produced.", remediation: "x" });
+            return files.map((name) => ({ id: "result", path: path.join(context.outputDir, name) }));
+        },
+        async verifyInstallation() {
+            return { passed: true };
+        },
+    };
+}
+const catalog = () => ({
+    retrievedAt: "2026-10-13T12:00:00.000Z",
+    gpus: [{ id: "a40", name: "A40", memoryGb: 48, priceUsdPerHour: 0.6, availability: "LOW", dataCenters: [{ id: "DC-1", availability: "LOW" }], cudaVersions: [] }],
+    cpus: [{ id: "cpu3c", name: "c", priceUsdPerVcpuHour: 0.03, ramGbPerVcpu: 2, minVcpu: 2, maxVcpu: 32 }],
+    dataCenters: [{ id: "DC-1", region: "EU", networkVolumeTypes: ["STANDARD"] }],
+});
+class FakeProvider {
+    name = "runpod";
+    storageRates;
+    pods = new Map();
+    nextId = 1;
+    failTerminate = false;
+    terminated = [];
+    async validateCredentials() {
+        return { ok: true, checkedAt: "t" };
+    }
+    async getCatalog() {
+        return { ...catalog(), ...(this.storageRates ? { storageRates: this.storageRates } : {}) };
+    }
+    async createPod(request) {
+        const pod = { id: `pod${this.nextId++}`, name: request.name, status: "RUNNING", dataCenterId: "DC-1", costUsdPerHour: 0.6, createdAt: null };
+        this.pods.set(pod.id, pod);
+        return pod;
+    }
+    async getPod(id) {
+        return this.pods.get(id) ?? null;
+    }
+    async listPods() {
+        return [...this.pods.values()];
+    }
+    async terminatePod(id) {
+        if (this.failTerminate)
+            throw new Error("provider down");
+        this.terminated.push(id);
+        const existed = this.pods.delete(id);
+        return { id, alreadyAbsent: !existed };
+    }
+    async createVolume() {
+        throw new Error("unused");
+    }
+    async getVolume() {
+        return null;
+    }
+    async listVolumes() {
+        return [];
+    }
+    async deleteVolume() {
+        throw new Error("unused");
+    }
+}
+class FakeBackend {
+    provider;
+    script;
+    events;
+    starts = 0;
+    observes = 0;
+    collects = 0;
+    cancelled = false;
+    startError;
+    /** Pods are created BEFORE start() can fail, to test cleanup after a half-finished provision. */
+    createThenFail = false;
+    collectFailures = 0;
+    files = { "answer.out": "result-bytes" };
+    constructor(provider, script, events) {
+        this.provider = provider;
+        this.script = script;
+        this.events = events;
+    }
+    async start(job, context) {
+        this.starts += 1;
+        this.events.push("backend.start");
+        const intent = context.ledger.beginCreate("gpu");
+        const pod = await this.provider.createPod({ name: intent.name });
+        context.ledger.created(intent.intentId, pod.id);
+        if (this.startError)
+            throw this.startError;
+        return { version: 1, runId: job.runId, podId: pod.id, endpoint: "https://worker.example" };
+    }
+    reconnect(runId, podId) {
+        return { version: 1, runId, podId, endpoint: "https://worker.example" };
+    }
+    async observe(_handle, _after) {
+        const snapshot = this.script[Math.min(this.observes, this.script.length - 1)];
+        this.observes += 1;
+        return snapshot;
+    }
+    async cancel() {
+        this.cancelled = true;
+    }
+    hangCollect = false;
+    async collect(_handle, dir, options) {
+        this.collects += 1;
+        if (this.hangCollect) {
+            await new Promise((_resolve, reject) => options?.signal?.addEventListener("abort", () => reject(new MoldeskError({ code: "RUN_CANCELLED", message: "Cancelled during download.", remediation: "None." }))));
+        }
+        if (this.collectFailures > 0) {
+            this.collectFailures -= 1;
+            throw new Error("connection reset");
+        }
+        fs.mkdirSync(dir, { recursive: true });
+        for (const [name, text] of Object.entries(this.files))
+            fs.writeFileSync(path.join(dir, name), text);
+        return { version: 1, files: Object.keys(this.files).map((name) => ({ path: name, sizeBytes: 1, sha256: "x" })) };
+    }
+    async release(handle, context) {
+        this.events.push("backend.release");
+        void handle;
+        void context;
+    }
+}
+const runningSnap = (extra = {}) => ({ stage: "running", state: "running", logs: { cursor: 1, stdout: "", stderr: "" }, ...extra });
+const doneSnap = {
+    stage: "running",
+    state: "succeeded",
+    exitCode: 0,
+    logs: { cursor: 2, stdout: "predicting\n", stderr: "" },
+    runtime: { pythonVersion: "3.11.9", dependencyFreeze: ["torch==2.2.1"], gpu: { name: "A40", memoryMb: 49152 }, command: { executable: "/job/x/runtime/.venv/bin/fake", args: ["--in", "/job/x/input/in.txt"], cwd: "/job/x" }, installationFingerprint: "python-" + "f".repeat(16) },
+};
+function sandbox(script = [runningSnap(), doneSnap], overrides = {}) {
+    const home = fs.mkdtempSync(path.join(os.tmpdir(), "moldesk cloud run "));
+    homes.push(home);
+    const paths = getMoldeskPaths({ MOLDESK_HOME: home });
+    const inputPath = path.join(home, "in.txt");
+    fs.writeFileSync(inputPath, "job");
+    const provider = new FakeProvider();
+    const events = [];
+    const backend = new FakeBackend(provider, script, events);
+    const clock = { ms: Date.parse("2026-10-13T12:00:00.000Z") };
+    const deps = {
+        provider,
+        paths,
+        backend,
+        accountScope: "acct",
+        resolveModel: () => ({ manifest: manifest(), adapter: adapter() }),
+        now: () => new Date(clock.ms),
+        sleep: async (ms) => {
+            clock.ms += ms;
+        },
+        pollIntervalMs: 1000,
+        recoveryWindowSeconds: 30,
+        ...overrides,
+    };
+    const quoteOptions = { cachePolicy: "ephemeral", maxBudgetUsd: 5, maxLifetimeSeconds: 3600 };
+    const optionsFor = async (extra = {}) => {
+        const plan = await planCloudRun("fake-model", inputPath, { cloud: quoteOptions }, deps);
+        return { cloud: { ...quoteOptions, quote: plan.quote, approval: { quoteId: plan.quote.id, approvedAt: new Date(clock.ms).toISOString(), acknowledged: plan.quote.warnings.map((w) => w.id) } }, ...extra };
+    };
+    return { paths, inputPath, provider, backend, events, clock, deps, optionsFor };
+}
+afterEach(() => {
+    for (const home of homes.splice(0))
+        fs.rmSync(home, { recursive: true, force: true });
+});
+describe("runCloudModel", () => {
+    it("runs end to end: verified outputs, pod gone before the report, risks acknowledged, journal closed out", async () => {
+        const box = sandbox();
+        const logs = [];
+        const options = await box.optionsFor({ onLog: (_stream, chunk) => logs.push(chunk) });
+        const result = await runCloudModel("fake-model", box.inputPath, options, box.deps);
+        expect(result.status).toBe("succeeded");
+        expect(result.cleanupPending).toBeUndefined();
+        expect(box.events).toEqual(["backend.start", "backend.release"]);
+        expect(result.record.compute.acknowledgedRisks).toEqual(["lifetime-risk"]);
+        expect(box.provider.pods.size).toBe(0);
+        expect(logs.join("")).toContain("predicting");
+        expect(result.record.outputs).toHaveLength(1);
+        expect(result.record.outputs[0].sha256).toMatch(/^[0-9a-f]{64}$/);
+        expect(result.record.runtime).toBe("python");
+        expect(result.record.installationFingerprint).toBe("python-" + "f".repeat(16));
+        expect(result.record.compute).toMatchObject({
+            provider: "runpod",
+            cachePolicy: "ephemeral",
+            gpuId: "a40",
+            datacenterId: "DC-1",
+            cleanup: "confirmed",
+            hardware: { name: "A40" },
+            pythonVersion: "3.11.9",
+            dependencyFreeze: ["torch==2.2.1"],
+        });
+        expect(result.record.compute.resources).toEqual([{ role: "gpu", providerId: "pod1" }]);
+        expect(result.record.compute.remoteCommand.cwd).toBe("/job/x"); // POSIX worker path
+        expect(result.record.command.cwd).toContain("runs"); // local run directory is unchanged
+        const onDisk = JSON.parse(fs.readFileSync(path.join(result.record.command.cwd, "run.json"), "utf8"));
+        expect(onDisk.status).toBe("succeeded");
+        const journal = CloudJournal.open(box.paths);
+        expect(journal.getJob(result.record.id)).toMatchObject({ stage: "done", status: "succeeded", transferState: "verified" });
+        expect(journal.listResources({ states: ["create_pending", "active", "delete_pending"] })).toHaveLength(0);
+        journal.close();
+    });
+    it("rejects mismatched or expired approvals before any run directory, job or resource exists", async () => {
+        const box = sandbox();
+        const options = await box.optionsFor();
+        await expect(runCloudModel("fake-model", box.inputPath, { ...options, cloud: { ...options.cloud, approval: { quoteId: "other", approvedAt: new Date(box.clock.ms).toISOString(), acknowledged: ["lifetime-risk"] } } }, box.deps)).rejects.toMatchObject({ code: "CLOUD_APPROVAL_MISMATCH" });
+        box.clock.ms += 16 * 60 * 1000;
+        await expect(runCloudModel("fake-model", box.inputPath, options, box.deps)).rejects.toMatchObject({ code: "CLOUD_APPROVAL_EXPIRED" });
+        expect(fs.existsSync(box.paths.runs)).toBe(false);
+        expect(box.backend.starts).toBe(0);
+        expect(box.events).toEqual([]);
+    });
+    it("requires a new quote when the input file changed since it was quoted", async () => {
+        const box = sandbox();
+        const options = await box.optionsFor();
+        fs.writeFileSync(box.inputPath, "different job");
+        await expect(runCloudModel("fake-model", box.inputPath, options, box.deps)).rejects.toMatchObject({ code: "CLOUD_REQUOTE_REQUIRED" });
+        expect(box.events).toEqual([]);
+    });
+    it("rejects a run whose approval did not acknowledge the shutdown risk, before anything exists", async () => {
+        const box = sandbox();
+        const options = await box.optionsFor();
+        await expect(runCloudModel("fake-model", box.inputPath, { ...options, cloud: { ...options.cloud, approval: { ...options.cloud.approval, acknowledged: [] } } }, box.deps)).rejects.toMatchObject({ code: "CLOUD_ACKNOWLEDGMENT_REQUIRED" });
+        expect(fs.existsSync(box.paths.runs)).toBe(false);
+        expect(box.backend.starts).toBe(0);
+        expect(box.provider.nextId).toBe(1);
+    });
+    it("fails on a nonzero worker exit and still terminates the machine", async () => {
+        const box = sandbox([runningSnap(), { stage: "running", state: "failed", exitCode: 1, logs: { cursor: 2, stdout: "", stderr: "boom\n" } }]);
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps);
+        expect(result.status).toBe("failed");
+        expect(result.record.error).toEqual({ code: "RUN_EXECUTION_FAILED", message: "fake-model exited with code 1." });
+        expect(box.provider.pods.size).toBe(0);
+        expect(result.record.outputs).toHaveLength(0);
+    });
+    it("treats a missing required output as failure even when the worker reported success", async () => {
+        const box = sandbox();
+        box.backend.files = {};
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps);
+        expect(result.status).toBe("failed");
+        expect(result.record.error?.code).toBe("MISSING_REQUIRED_OUTPUT");
+        expect(box.provider.pods.size).toBe(0);
+    });
+    it("retries a failed download inside a bounded window without rerunning inference", async () => {
+        const box = sandbox();
+        box.backend.collectFailures = 2;
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps);
+        expect(result.status).toBe("succeeded");
+        expect(box.backend.collects).toBe(3);
+        expect(box.backend.starts).toBe(1);
+        expect(box.backend.observes).toBe(2);
+    });
+    it("reports a transfer failure after the window, preserves the failure and still cleans up", async () => {
+        const box = sandbox();
+        box.backend.collectFailures = 1000;
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps);
+        expect(result.status).toBe("failed");
+        expect(result.record.error?.code).toBe("CLOUD_TRANSFER_FAILED");
+        expect(box.backend.collects).toBeLessThan(20);
+        expect(box.backend.starts).toBe(1);
+        expect(box.provider.pods.size).toBe(0);
+        const journal = CloudJournal.open(box.paths);
+        expect(journal.getJob(result.record.id).transferState).toBe("failed");
+        journal.close();
+    });
+    it("cancels on abort, marks the run cancelled and terminates the machine", async () => {
+        const controller = new AbortController();
+        const box = sandbox([runningSnap({ logs: { cursor: 1, stdout: "working\n", stderr: "" } })]);
+        const options = await box.optionsFor({ signal: controller.signal, onLog: () => controller.abort() });
+        const result = await runCloudModel("fake-model", box.inputPath, options, box.deps);
+        expect(result.status).toBe("cancelled");
+        expect(result.record.error?.code).toBe("RUN_CANCELLED");
+        expect(box.backend.cancelled).toBe(true);
+        expect(box.provider.pods.size).toBe(0);
+    });
+    it("cancels a download that is in flight, marks the run cancelled and still terminates the machine", async () => {
+        const controller = new AbortController();
+        const box = sandbox();
+        box.backend.hangCollect = true;
+        const options = await box.optionsFor({ signal: controller.signal, onProgress: (event) => event.stage === "collecting" && setTimeout(() => controller.abort(), 10) });
+        const result = await runCloudModel("fake-model", box.inputPath, options, box.deps);
+        expect(result.status).toBe("cancelled");
+        expect(result.record.error?.code).toBe("RUN_CANCELLED");
+        expect(box.provider.pods.size).toBe(0);
+        expect(box.backend.starts).toBe(1);
+    });
+    it("stops at the approved lifetime and reports CLOUD_DEADLINE_EXCEEDED", async () => {
+        const box = sandbox([runningSnap()], { pollIntervalMs: 600_000 });
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps);
+        expect(result.status).toBe("failed");
+        expect(result.record.error?.code).toBe("CLOUD_DEADLINE_EXCEEDED");
+        expect(box.backend.cancelled).toBe(true);
+        expect(box.provider.pods.size).toBe(0);
+    });
+    it("fails fast when the worker becomes unreachable and still cleans up", async () => {
+        const box = sandbox();
+        box.backend.observe = async () => {
+            throw new Error("ECONNRESET");
+        };
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps);
+        expect(result.record.error?.code).toBe("CLOUD_WORKER_UNREACHABLE");
+        expect(box.provider.pods.size).toBe(0);
+    });
+    it("cleans up a machine created before start() failed", async () => {
+        const box = sandbox();
+        box.backend.startError = new Error("bootstrap failed");
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps);
+        expect(result.status).toBe("failed");
+        expect(result.record.error?.message).toBe("bootstrap failed");
+        expect(box.provider.pods.size).toBe(0);
+        expect(box.provider.terminated).toEqual(["pod1"]);
+    });
+    it("surfaces CLOUD_CLEANUP_PENDING without discarding a valid result", async () => {
+        const box = sandbox();
+        box.provider.failTerminate = true;
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), { ...box.deps, cleanupAttempts: 2 });
+        expect(result.status).toBe("succeeded");
+        expect(result.record.outputs).toHaveLength(1);
+        expect(result.cleanupPending).toEqual({ code: "CLOUD_CLEANUP_PENDING", resources: [{ role: "gpu", providerId: "pod1" }] });
+        expect(result.record.compute.cleanup).toBe("pending");
+        const journal = CloudJournal.open(box.paths);
+        expect(journal.getJob(result.record.id)).toMatchObject({ status: "cleanup_pending", error: { code: "CLOUD_CLEANUP_PENDING" } });
+        expect(journal.listOpenJobs()).toHaveLength(1);
+        journal.close();
+    });
+});
+describe("approval revalidation against the live catalog", () => {
+    it("rejects before allocating anything when an unknown storage rate has become known since approval", async () => {
+        const box = sandbox();
+        const options = await box.optionsFor(); // approved while disk rates were unknown
+        box.provider.storageRates = { containerDiskUsdPerGbMonth: 0.1 };
+        await expect(runCloudModel("fake-model", box.inputPath, options, box.deps)).rejects.toMatchObject({ code: "CLOUD_REQUOTE_REQUIRED" });
+        expect(fs.existsSync(box.paths.runs)).toBe(false);
+        expect(box.events).toEqual([]);
+        expect(box.backend.starts).toBe(0);
+    });
+    it("proceeds when the fresh quote has the same storage coverage", async () => {
+        const box = sandbox();
+        box.provider.storageRates = { containerDiskUsdPerGbMonth: 0.1 };
+        const options = await box.optionsFor();
+        const result = await runCloudModel("fake-model", box.inputPath, options, box.deps);
+        expect(result.status).toBe("succeeded");
+    });
+});
+describe("incomplete worker logs", () => {
+    const snap = (stdout, state, cursor, lost) => ({
+        stage: "running",
+        state,
+        ...(state === "succeeded" ? { exitCode: 0 } : {}),
+        logs: { cursor, stdout, stderr: "", ...(lost ? { lostEntries: lost, evictedBytes: lost * 100 } : {}) },
+    });
+    it("marks the local logs, warns on the run and records incompleteness, without failing a valid result", async () => {
+        const box = sandbox([snap("tail-1\n", "running", 5, 4), snap("tail-2\n", "succeeded", 7, 2)]);
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps);
+        expect(result.status).toBe("succeeded");
+        expect(result.record.compute.logs).toEqual({ complete: false, lostChunks: 6 });
+        const warnings = result.record.warnings.filter((w) => w.code === "CLOUD_LOGS_INCOMPLETE");
+        expect(warnings).toHaveLength(1);
+        expect(warnings[0].message).toMatch(/6 worker log chunk\(s\) were dropped/);
+        const stdout = fs.readFileSync(result.record.logs.stdout, "utf8");
+        expect(stdout).toMatch(/\[moldesk: 4 worker log chunk\(s\) were dropped.*incomplete here\]/);
+        expect(stdout).toContain("tail-1");
+        expect(stdout).toContain("tail-2");
+        expect(fs.readFileSync(result.record.logs.stderr, "utf8")).toMatch(/\[moldesk: 2 worker log chunk/);
+        const onDisk = JSON.parse(fs.readFileSync(path.join(result.record.command.cwd, "run.json"), "utf8"));
+        expect(onDisk.compute.logs).toEqual({ complete: false, lostChunks: 6 });
+        expect(onDisk.warnings.map((w) => w.code)).toContain("CLOUD_LOGS_INCOMPLETE");
+    });
+    it("leaves complete logs unmarked", async () => {
+        const box = sandbox([snap("a\n", "running", 1), snap("b\n", "succeeded", 2)]);
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps);
+        expect(result.record.compute.logs).toBeUndefined();
+        expect((result.record.warnings ?? []).some((w) => w.code === "CLOUD_LOGS_INCOMPLETE")).toBe(false);
+        expect(fs.readFileSync(result.record.logs.stdout, "utf8")).toBe("a\nb\n");
+    });
+    it("recovery reports logs lost while the client was away", async () => {
+        const box = sandbox([snap("a\n", "running", 1), snap("b\n", "succeeded", 2)]);
+        box.provider.failTerminate = true;
+        box.backend.collectFailures = 1000;
+        const failed = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), { ...box.deps, cleanupAttempts: 1 });
+        box.backend.script.splice(0, box.backend.script.length, snap("later\n", "succeeded", 12, 9));
+        box.backend.observes = 0;
+        box.provider.failTerminate = false;
+        box.backend.collectFailures = 0;
+        const recovered = await recoverCloudRun(failed.record.id, { provider: box.provider, paths: box.paths, backend: box.backend, resolveModel: box.deps.resolveModel, sleep: box.deps.sleep, cleanupAttempts: 1 });
+        expect(recovered.result?.record.compute?.logs).toEqual({ complete: false, lostChunks: 9 });
+        expect(recovered.result?.record.warnings?.some((w) => w.code === "CLOUD_LOGS_INCOMPLETE")).toBe(true);
+        expect(fs.readFileSync(failed.record.logs.stdout, "utf8")).toContain("9 worker log chunk(s) were dropped");
+    });
+});
+describe("log persistence", () => {
+    const logSnap = (stdout, stderr, state, cursor) => ({ stage: "running", state, ...(state === "succeeded" ? { exitCode: 0 } : state === "failed" ? { exitCode: 1 } : {}), logs: { cursor, stdout, stderr } });
+    it("saves worker stdout/stderr to the files run.json references, without any terminal tee, and they outlive cleanup", async () => {
+        const box = sandbox([logSnap("step 1\n", "warn\n", "running", 1), logSnap("step 2\n", "", "succeeded", 2)]);
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps); // no onLog
+        expect(box.provider.pods.size).toBe(0);
+        const onDisk = JSON.parse(fs.readFileSync(path.join(result.record.command.cwd, "run.json"), "utf8"));
+        expect(fs.readFileSync(onDisk.logs.stdout, "utf8")).toBe("step 1\nstep 2\n");
+        expect(fs.readFileSync(onDisk.logs.stderr, "utf8")).toBe("warn\n");
+        expect(onDisk.compute.workerLogCursor).toBe(2);
+        expect(path.dirname(onDisk.logs.stdout)).toBe(result.record.command.cwd);
+    });
+    it("keeps the logs of a failed run, including the tail drained before cleanup", async () => {
+        const box = sandbox([logSnap("", "", "running", 0), logSnap("", "Traceback...\n", "failed", 1)]);
+        const result = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps);
+        expect(result.status).toBe("failed");
+        expect(fs.readFileSync(result.record.logs.stderr, "utf8")).toBe("Traceback...\n");
+    });
+    it("drains the remaining tail when the run is cancelled mid-way", async () => {
+        const controller = new AbortController();
+        const box = sandbox([logSnap("early\n", "", "running", 1)]);
+        const options = await box.optionsFor({ signal: controller.signal, onLog: () => controller.abort() });
+        const result = await runCloudModel("fake-model", box.inputPath, options, box.deps);
+        expect(result.status).toBe("cancelled");
+        expect(fs.readFileSync(result.record.logs.stdout, "utf8")).toContain("early");
+    });
+    it("recovery appends only the log tail written since the saved cursor", async () => {
+        const box = sandbox([logSnap("a\n", "", "running", 1), logSnap("b\n", "", "succeeded", 2)]);
+        box.provider.failTerminate = true;
+        box.backend.collectFailures = 1000;
+        const failed = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), { ...box.deps, cleanupAttempts: 1 });
+        expect(fs.readFileSync(failed.record.logs.stdout, "utf8")).toBe("a\nb\n");
+        // The worker later produced one more line.
+        box.backend.script.splice(0, box.backend.script.length, logSnap("c\n", "", "succeeded", 3));
+        box.backend.observes = 0;
+        box.provider.failTerminate = false;
+        box.backend.collectFailures = 0;
+        await recoverCloudRun(failed.record.id, { provider: box.provider, paths: box.paths, backend: box.backend, resolveModel: box.deps.resolveModel, sleep: box.deps.sleep, cleanupAttempts: 1 });
+        expect(fs.readFileSync(failed.record.logs.stdout, "utf8")).toBe("a\nb\nc\n");
+    });
+});
+describe("preflight reconciliation", () => {
+    it("refuses new paid work while an orphaned or owed resource exists, before allocating anything", async () => {
+        const box = sandbox();
+        const options = await box.optionsFor();
+        const journal = CloudJournal.open(box.paths);
+        const prefix = journal.clientId.replace(/-/g, "").slice(0, 8);
+        journal.close();
+        box.provider.pods.set("podOrphan", { id: "podOrphan", name: `moldesk-${prefix}-deadbeef-gpu`, status: "RUNNING", dataCenterId: null, costUsdPerHour: 0.5, createdAt: null });
+        await expect(runCloudModel("fake-model", box.inputPath, options, box.deps)).rejects.toMatchObject({ code: "CLOUD_UNRESOLVED_RESOURCES" });
+        expect(fs.existsSync(box.paths.runs)).toBe(false);
+        expect(box.events).toEqual([]);
+    });
+    it("refuses when a previous run still owes cleanup", async () => {
+        const box = sandbox();
+        box.provider.failTerminate = true;
+        const first = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), { ...box.deps, cleanupAttempts: 1 });
+        expect(first.cleanupPending).toBeDefined();
+        await expect(runCloudModel("fake-model", box.inputPath, await box.optionsFor(), box.deps)).rejects.toMatchObject({ code: "CLOUD_UNRESOLVED_RESOURCES" });
+        expect(box.backend.starts).toBe(1);
+    });
+    it("does not start blind when the provider cannot be listed", async () => {
+        const box = sandbox();
+        const options = await box.optionsFor();
+        box.provider.listPods = async () => {
+            throw new Error("offline");
+        };
+        await expect(runCloudModel("fake-model", box.inputPath, options, box.deps)).rejects.toMatchObject({ code: "CLOUD_UNAVAILABLE" });
+        expect(fs.existsSync(box.paths.runs)).toBe(false);
+    });
+});
+describe("planCloudRun", () => {
+    it("rejects models without a cloud block and jobs with companion inputs", async () => {
+        const box = sandbox();
+        const plain = parseManifest({ ...JSON.parse(JSON.stringify(manifest())), cloud: undefined }, "x/manifest.yaml");
+        await expect(planCloudRun("fake-model", box.inputPath, { cloud: { cachePolicy: "ephemeral", maxBudgetUsd: 5, maxLifetimeSeconds: 3600 } }, { ...box.deps, resolveModel: () => ({ manifest: plain, adapter: adapter() }) })).rejects.toMatchObject({ code: "CLOUD_MODEL_UNSUPPORTED" });
+        const withCompanions = { ...adapter(), resolveCompanionInputs: async () => [{ id: "protein", sourcePath: "p.pdb" }] };
+        await expect(planCloudRun("fake-model", box.inputPath, { cloud: { cachePolicy: "ephemeral", maxBudgetUsd: 5, maxLifetimeSeconds: 3600 } }, { ...box.deps, resolveModel: () => ({ manifest: manifest(), adapter: withCompanions }) })).rejects.toMatchObject({ code: "CLOUD_COMPANION_INPUTS_UNSUPPORTED" });
+    });
+    it("produces the same quote that buildQuote would, without allocating anything", async () => {
+        const box = sandbox();
+        const plan = await planCloudRun("fake-model", box.inputPath, { cloud: { cachePolicy: "ephemeral", maxBudgetUsd: 5, maxLifetimeSeconds: 3600 } }, box.deps);
+        expect(plan.quote.id).toBe(buildQuote({ provider: "runpod", manifest: plan.manifest, cloud: plan.cloud, catalog: catalog(), cachePolicy: "ephemeral", maxBudgetUsd: 5, maxLifetimeSeconds: 3600, inputs: [{ role: "job", sha256: plan.input.sha256, sizeBytes: 3 }] }).id);
+        expect(fs.existsSync(box.paths.runs)).toBe(false);
+    });
+});
+describe("recoverCloudRun", () => {
+    const recoverDeps = (box) => ({ provider: box.provider, paths: box.paths, backend: box.backend, resolveModel: box.deps.resolveModel, sleep: box.deps.sleep, cleanupAttempts: 1 });
+    it("finishes owed cleanup after the provider recovers and never restarts inference", async () => {
+        const box = sandbox();
+        box.provider.failTerminate = true;
+        const first = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), { ...box.deps, cleanupAttempts: 1 });
+        expect(first.cleanupPending).toBeDefined();
+        box.provider.failTerminate = false;
+        const recovered = await recoverCloudRun(first.record.id, recoverDeps(box));
+        expect(recovered.cleanup.confirmed).toBe(true);
+        expect(box.provider.pods.size).toBe(0);
+        expect(box.backend.starts).toBe(1);
+        const journal = CloudJournal.open(box.paths);
+        expect(journal.getJob(first.record.id)).toMatchObject({ status: "succeeded" });
+        expect(journal.listOpenJobs()).toHaveLength(0);
+        journal.close();
+        const onDisk = JSON.parse(fs.readFileSync(path.join(first.record.command.cwd, "run.json"), "utf8"));
+        expect(onDisk.compute.cleanup).toBe("confirmed");
+    });
+    it("downloads results left on a still-alive pod after a transfer failure, then cleans up", async () => {
+        const box = sandbox();
+        box.provider.failTerminate = true;
+        box.backend.collectFailures = 1000;
+        const failed = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), { ...box.deps, cleanupAttempts: 1 });
+        expect(failed.record.error?.code).toBe("CLOUD_TRANSFER_FAILED");
+        expect(box.provider.pods.size).toBe(1);
+        box.provider.failTerminate = false;
+        box.backend.collectFailures = 0;
+        const recovered = await recoverCloudRun(failed.record.id, recoverDeps(box));
+        expect(recovered.collected).toBe(true);
+        expect(recovered.status).toBe("succeeded");
+        expect(recovered.result?.record.outputs).toHaveLength(1);
+        expect(box.provider.pods.size).toBe(0);
+        expect(box.backend.starts).toBe(1);
+        const onDisk = JSON.parse(fs.readFileSync(path.join(failed.record.command.cwd, "run.json"), "utf8"));
+        expect(onDisk.status).toBe("succeeded");
+        expect(onDisk.error).toBeUndefined();
+    });
+    it("marks an interrupted run failed and terminates its pod", async () => {
+        const box = sandbox();
+        box.provider.failTerminate = true;
+        const first = await runCloudModel("fake-model", box.inputPath, await box.optionsFor(), { ...box.deps, cleanupAttempts: 1 });
+        // Rewrite history as if the client died mid-run.
+        const file = path.join(first.record.command.cwd, "run.json");
+        const record = JSON.parse(fs.readFileSync(file, "utf8"));
+        record.status = "running";
+        delete record.finishedAt;
+        delete record.error;
+        record.outputs = [];
+        fs.writeFileSync(file, JSON.stringify(record));
+        fs.rmSync(path.join(first.record.command.cwd, "output"), { recursive: true, force: true });
+        box.backend.script.splice(0, box.backend.script.length, runningSnap());
+        box.backend.observes = 0;
+        box.provider.failTerminate = false;
+        // The worker is still running: recovery must not silently kill an approved job.
+        const waiting = await recoverCloudRun(first.record.id, recoverDeps(box));
+        expect(waiting.status).toBe("running");
+        expect(waiting.cleanup.confirmed).toBe(false);
+        expect(box.provider.pods.size).toBe(1);
+        const forced = await recoverCloudRun(first.record.id, recoverDeps(box), { terminateRunning: true });
+        expect(forced.cleanup.confirmed).toBe(true);
+        expect(forced.status).toBe("failed");
+        expect(box.provider.pods.size).toBe(0);
+        const after = JSON.parse(fs.readFileSync(file, "utf8"));
+        expect(after.error.code).toBe("CLOUD_RUN_INTERRUPTED");
+        expect(after.finishedAt).toBeDefined();
+    });
+    it("rejects unknown runs", async () => {
+        const box = sandbox();
+        await expect(recoverCloudRun("2026-01-01T00-00-00-000Z-00000000-0000-0000-0000-000000000000", recoverDeps(box))).rejects.toMatchObject({ code: "CLOUD_RUN_UNKNOWN" });
+    });
+});