@moldesk/core 0.0.5 → 0.0.6

package/dist/cloud/backend.test.js

@@ -0,0 +1,339 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { MIN_WORKER_READY_TIMEOUT_MS, RunpodExecutionBackend, WorkerClient, } from "@moldesk/runtime";
+import { cacheLedgerFor } from "./run.js";
+import { CloudJournal } from "./state.js";
+import { createWorkerServer } from "./worker/server.js";
+import { parseManifest } from "@moldesk/registry";
+const DIGEST = "a".repeat(64);
+const dirs = [];
+const fleets = [];
+const manifest = parseManifest({
+    schemaVersion: 1,
+    name: "m",
+    displayName: "M",
+    modelVersion: "1.0.0",
+    adapterVersion: "0.1.0",
+    description: "t",
+    category: "other",
+    status: "planned",
+    source: { repository: "https://example.com/r", revision: "b".repeat(40) },
+    runtimes: [{ kind: "python", python: "3.11", installer: "uv", platforms: ["linux-x64"], requirements: [{ name: "torch" }] }],
+    hardware: { platforms: ["linux-x64"] },
+    input: { formats: [".txt"], required: true },
+    outputs: [{ id: "o", glob: "*.out", required: true }],
+    cloud: { schemaVersion: 1, platform: "linux-x64", images: { gpu: `r.example/g@sha256:${DIGEST}`, cpu: `r.example/c@sha256:${DIGEST}` }, minVramGb: 24, minRamGb: 32, ephemeralDiskGb: 40, volumeGb: 40, assetLayoutRevision: "1", cpuAssetPreparation: true },
+}, "x/manifest.yaml");
+class Fleet {
+    name = "runpod";
+    log = [];
+    pods = new Map();
+    volumes = new Map();
+    nextId = 1;
+    gpuAvailable = true;
+    /** Make the next volume claim `cacheReady` (a warm cache) or not. */
+    podStatus = "RUNNING";
+    terminateFails = false;
+    /** Set to make GPU workers claim the cache is missing once. */
+    forgetCache = false;
+    async validateCredentials() {
+        return { ok: true, checkedAt: "t" };
+    }
+    async getCatalog() {
+        return {
+            retrievedAt: "t",
+            gpus: [{ id: "a40", name: "A40", memoryGb: 48, priceUsdPerHour: 0.6, availability: this.gpuAvailable ? "LOW" : "NONE", dataCenters: [{ id: "DC-1", availability: this.gpuAvailable ? "LOW" : "NONE" }], cudaVersions: [] }],
+            cpus: [],
+            dataCenters: [{ id: "DC-1", region: "EU", networkVolumeTypes: ["STANDARD"] }],
+        };
+    }
+    async createVolume(request) {
+        const volume = { id: `vol${this.nextId++}`, name: request.name, sizeGb: request.sizeGb, dataCenterId: request.dataCenterId, type: "STANDARD", cacheReady: false };
+        this.volumes.set(volume.id, volume);
+        this.log.push(`createVolume ${volume.id}`);
+        return volume;
+    }
+    async createPod(request) {
+        if (request.gpu && !this.gpuAvailable)
+            throw Object.assign(new Error("no capacity"), { code: "CLOUD_CAPACITY_UNAVAILABLE" });
+        const id = `pod${this.nextId++}`;
+        const role = request.gpu ? "gpu" : "cpu";
+        this.log.push(`createPod ${role} ${id}`);
+        const volumeId = request.volume?.id;
+        const pipeline = async ({ job, outputDir, log, setStage }) => {
+            const volume = volumeId ? this.volumes.get(volumeId) : undefined;
+            if (job.mode === "prepare-assets") {
+                setStage("preparing_assets");
+                volume.cacheReady = true;
+                return { exitCode: 0, runtime: { dependencyFreeze: [], cache: { checksumManifestSha256: "c".repeat(64), sizeBytes: 123, fileCount: 2, reused: false } } };
+            }
+            setStage("preparing_assets");
+            if (volume && (!volume.cacheReady || this.forgetCache)) {
+                this.forgetCache = false;
+                volume.cacheReady = false;
+                throw Object.assign(new Error("no cache"), { code: "CLOUD_CACHE_MISSING" });
+            }
+            setStage("running");
+            log("stdout", "predicting\n");
+            fs.writeFileSync(path.join(outputDir, "a.out"), "result");
+            return { exitCode: 0, runtime: { dependencyFreeze: ["torch==2"] } };
+        };
+        const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "moldesk fleet "));
+        dirs.push(rootDir);
+        const worker = createWorkerServer({ token: request.env.MOLDESK_WORKER_TOKEN, rootDir, pipeline });
+        const port = await worker.listen(0, "127.0.0.1");
+        const pod = { id, name: request.name, status: this.podStatus, dataCenterId: request.dataCenterId, costUsdPerHour: 0.6, createdAt: null };
+        this.pods.set(id, { pod, worker, endpoint: `http://127.0.0.1:${port}`, request });
+        return pod;
+    }
+    async getPod(id) {
+        return this.pods.get(id)?.pod ?? null;
+    }
+    async listPods() {
+        return [...this.pods.values()].map((entry) => entry.pod);
+    }
+    async terminatePod(id) {
+        if (this.terminateFails)
+            throw new Error("provider down");
+        this.log.push(`terminate ${id}`);
+        const entry = this.pods.get(id);
+        if (entry)
+            await entry.worker.close();
+        this.pods.delete(id);
+        return { id, alreadyAbsent: !entry };
+    }
+    async getVolume(id) {
+        return this.volumes.get(id) ?? null;
+    }
+    async listVolumes() {
+        return [...this.volumes.values()];
+    }
+    async deleteVolume(id) {
+        this.log.push(`deleteVolume ${id}`);
+        this.volumes.delete(id);
+        return { id, alreadyAbsent: false };
+    }
+}
+function memoryTokens() {
+    const data = new Map();
+    return { data, async set(id, token) { data.set(id, token); }, async get(id) { return data.get(id) ?? null; }, async delete(id) { data.delete(id); } };
+}
+function setup(overrides = {}) {
+    const fleet = new Fleet();
+    fleets.push(fleet);
+    const tokens = memoryTokens();
+    const home = fs.mkdtempSync(path.join(os.tmpdir(), "moldesk backend "));
+    dirs.push(home);
+    const journal = CloudJournal.openFile(path.join(home, "j.sqlite"));
+    const backend = new RunpodExecutionBackend({
+        provider: fleet,
+        tokenStore: tokens,
+        ...(overrides.bundle === false ? {} : { workerBundle: { url: "https://example.com/worker.tgz", sha256: "d".repeat(64) } }),
+        endpointFor: (id) => fleet.pods.get(id).endpoint,
+        createClient: (endpoint, token) => new WorkerClient({ endpoint, token, allowInsecureLoopback: true, sleep: async () => undefined }),
+        sleep: async () => undefined,
+        terminationWaitMs: 100,
+        pollIntervalMs: 1,
+    });
+    const runId = (n) => `2026-10-13T11-55-45-123Z-0a1b2c3d-1111-2222-3333-44445555666${n}`;
+    const inputPath = path.join(home, "in.txt");
+    fs.writeFileSync(inputPath, "job");
+    const job = (n, patch = {}) => ({
+        version: 1,
+        runId: runId(n),
+        model: "m",
+        modelVersion: "1.0.0",
+        manifestDigest: "md",
+        cachePolicy: "ephemeral",
+        gpuId: "a40",
+        datacenterId: "DC-1",
+        images: manifest.cloud.images,
+        containerDiskGb: 40,
+        volumeGb: 40,
+        assetLayoutRevision: "1",
+        cpuAssetPreparation: true,
+        cacheKey: "k".repeat(64),
+        params: {},
+        inputs: [{ role: "job", relativePath: "in.txt", sha256: "3f7b3f21d1d8f2fe8f9f3a3dc5f0c0ca7c3fbb5e0cf0c7f9ad7f3b4d5d9c3d3c", sizeBytes: 3, sourcePath: inputPath }],
+        deadlineAt: "2099-01-01T00:00:00.000Z",
+        ...patch,
+    });
+    // Real sha so the worker's upload verification passes.
+    const realSha = require("node:crypto").createHash("sha256").update("job").digest("hex");
+    const ctx = (n, persistent) => ({
+        ledger: {
+            beginCreate: (role) => {
+                const intent = journal.recordResourceIntent({ ...(role === "volume" ? {} : { runId: runId(n) }), role: role, deadlineAt: "2099-01-01T00:00:00.000Z" });
+                return { intentId: intent.intentId, name: intent.name };
+            },
+            created: (intentId, providerId) => void journal.recordResourceCreated(intentId, providerId),
+            beginDelete: (intentId) => void journal.markResourceDeletePending(intentId),
+            deleted: (intentId) => void journal.markResourceDeleted(intentId),
+        },
+        ...(persistent ? { cache: cacheLedgerFor(journal, { accountScope: "acct", runId: runId(n), manifest, cloud: manifest.cloud, cacheKey: "k".repeat(64), manifestDigest: "md" }) } : {}),
+    });
+    const withSha = (j) => ({ ...j, inputs: j.inputs.map((file) => ({ ...file, sha256: realSha })) });
+    return { fleet, tokens, journal, backend, job: (n, patch = {}) => withSha(job(n, patch)), ctx, runId };
+}
+async function finish(backend, handle) {
+    for (let i = 0; i < 400; i += 1) {
+        const snapshot = await backend.observe(handle);
+        if (snapshot.state !== "running")
+            return snapshot;
+        await new Promise((resolve) => setTimeout(resolve, 5));
+    }
+    throw new Error("did not finish");
+}
+afterEach(async () => {
+    for (const fleet of fleets.splice(0))
+        for (const entry of fleet.pods.values())
+            await entry.worker.close();
+    for (const dir of dirs.splice(0))
+        fs.rmSync(dir, { recursive: true, force: true });
+});
+describe("RunpodExecutionBackend (ephemeral)", () => {
+    it("provisions a GPU pod, uploads, runs, collects and terminates with confirmation", async () => {
+        const box = setup();
+        const handle = await box.backend.start(box.job(1), box.ctx(1, false));
+        expect(box.fleet.pods.get(handle.podId).request).toMatchObject({
+            image: manifest.cloud.images.gpu,
+            gpu: { id: "a40", count: 1 },
+            ports: ["8000/http"],
+            entrypoint: ["/bin/bash", "-c"],
+        });
+        expect(box.fleet.pods.get(handle.podId).request.env).toMatchObject({ MOLDESK_WORKER_DEADLINE_AT: "2099-01-01T00:00:00.000Z", MOLDESK_WORKER_BUNDLE_SHA256: "d".repeat(64) });
+        expect(box.tokens.data.get(handle.runId)).toMatch(/^[0-9a-f]{64}$/);
+        expect((await finish(box.backend, handle)).state).toBe("succeeded");
+        const out = fs.mkdtempSync(path.join(os.tmpdir(), "moldesk out "));
+        dirs.push(out);
+        await box.backend.collect(handle, out);
+        expect(fs.readFileSync(path.join(out, "a.out"), "utf8")).toBe("result");
+        await box.backend.release(handle, box.ctx(1, false));
+        expect(box.fleet.pods.size).toBe(0);
+        expect(box.tokens.data.size).toBe(0);
+        expect(box.journal.listResources({ states: ["create_pending", "active", "delete_pending"] })).toHaveLength(0);
+    });
+    it("refuses to start without a configured worker bundle and creates nothing", async () => {
+        const box = setup({ bundle: false });
+        await expect(box.backend.start(box.job(1), box.ctx(1, false))).rejects.toMatchObject({ code: "CLOUD_WORKER_BUNDLE_UNCONFIGURED" });
+        expect(box.fleet.log).toEqual([]);
+    });
+    it("stops waiting and surfaces a boot failure when the pod enters ERROR, leaving it releasable", async () => {
+        const box = setup();
+        box.fleet.podStatus = "ERROR";
+        // Make the worker unreachable so readiness polling consults the pod status.
+        const original = box.fleet.createPod.bind(box.fleet);
+        box.fleet.createPod = async (request) => {
+            const pod = await original(request);
+            await box.fleet.pods.get(pod.id).worker.close();
+            return pod;
+        };
+        await expect(box.backend.start(box.job(1), box.ctx(1, false))).rejects.toMatchObject({ code: "CLOUD_WORKER_BOOT_FAILED" });
+        await box.backend.release({ runId: box.job(1).runId }, box.ctx(1, false));
+        expect(box.fleet.pods.size).toBe(0);
+    });
+    it("does not mark a pod deleted when the provider cannot terminate it", async () => {
+        const box = setup();
+        const handle = await box.backend.start(box.job(1), box.ctx(1, false));
+        box.fleet.terminateFails = true;
+        await box.backend.release(handle, box.ctx(1, false));
+        expect(box.fleet.pods.size).toBe(1);
+        expect(box.journal.listResources({ states: ["delete_pending"] })).toHaveLength(1);
+    });
+    it("clamps the readiness timeout to at least five minutes", () => {
+        expect(MIN_WORKER_READY_TIMEOUT_MS).toBe(300_000);
+    });
+});
+describe("RunpodExecutionBackend (persistent)", () => {
+    const persistent = { cachePolicy: "persistent" };
+    it("cold: prepares on CPU, terminates the CPU pod before any GPU exists, retains the volume, marks the cache READY", async () => {
+        const box = setup();
+        const handle = await box.backend.start(box.job(1, persistent), box.ctx(1, true));
+        expect(box.fleet.log).toEqual(["createVolume vol1", "createPod cpu pod2", "terminate pod2", expect.stringMatching(/^createPod gpu pod3$/)]);
+        expect(handle.volumeId).toBe("vol1");
+        expect((await finish(box.backend, handle)).state).toBe("succeeded");
+        await box.backend.release(handle, box.ctx(1, true));
+        expect(box.fleet.pods.size).toBe(0);
+        expect(box.fleet.volumes.has("vol1")).toBe(true); // volume survives ordinary cleanup
+        expect(box.fleet.log).not.toContain("deleteVolume vol1");
+        const cache = box.journal.listCaches()[0];
+        expect(cache).toMatchObject({ state: "READY", volumeId: "vol1", sizeBytes: 123, checksumManifestDigest: "c".repeat(64), leaseOwner: null });
+    });
+    it("warm: a second run reuses the volume with zero CPU pods and no new volume", async () => {
+        const box = setup();
+        const first = await box.backend.start(box.job(1, persistent), box.ctx(1, true));
+        await finish(box.backend, first);
+        await box.backend.release(first, box.ctx(1, true));
+        box.fleet.log.length = 0;
+        const second = await box.backend.start(box.job(2, persistent), box.ctx(2, true));
+        expect(box.fleet.log).toEqual([expect.stringMatching(/^createPod gpu /)]);
+        expect((await finish(box.backend, second)).state).toBe("succeeded");
+        await box.backend.release(second, box.ctx(2, true));
+        expect(box.fleet.volumes.size).toBe(1);
+    });
+    it("warm but the volume lost its cache: the GPU pod is dropped, the CPU rebuilds it, then a fresh GPU runs", async () => {
+        const box = setup();
+        const first = await box.backend.start(box.job(1, persistent), box.ctx(1, true));
+        await finish(box.backend, first);
+        await box.backend.release(first, box.ctx(1, true));
+        box.fleet.log.length = 0;
+        box.fleet.forgetCache = true;
+        const second = await box.backend.start(box.job(2, persistent), box.ctx(2, true));
+        expect(box.fleet.log.map((line) => line.split(" ").slice(0, 2).join(" "))).toEqual(["createPod gpu", "terminate pod4", "createPod cpu", "terminate pod5", "createPod gpu"].map((line, index) => (index === 1 || index === 3 ? line.split(" ").slice(0, 2).join(" ") : line)));
+        expect((await finish(box.backend, second)).state).toBe("succeeded");
+        expect(box.journal.listCaches()[0].state).toBe("READY");
+        await box.backend.release(second, box.ctx(2, true));
+        expect(box.fleet.pods.size).toBe(0);
+    });
+    it("loses GPU capacity after preparation: the CPU pod is gone, the volume and READY cache are retained, with guidance", async () => {
+        const box = setup();
+        const originalCatalog = box.fleet.getCatalog.bind(box.fleet);
+        box.fleet.getCatalog = async () => {
+            const catalog = await originalCatalog();
+            return box.fleet.log.some((line) => line.startsWith("terminate")) ? { ...catalog, gpus: catalog.gpus.map((gpu) => ({ ...gpu, availability: "NONE", dataCenters: [] })) } : catalog;
+        };
+        await expect(box.backend.start(box.job(1, persistent), box.ctx(1, true))).rejects.toMatchObject({ code: "CLOUD_CAPACITY_UNAVAILABLE" });
+        expect(box.fleet.pods.size).toBe(0);
+        expect(box.fleet.volumes.has("vol1")).toBe(true);
+        expect(box.journal.listCaches()[0].state).toBe("READY");
+        expect(box.fleet.log.filter((line) => line.startsWith("createPod gpu"))).toHaveLength(0);
+    });
+    it("refuses to prepare while another process holds the cache lease", async () => {
+        const box = setup();
+        const context = box.ctx(1, true);
+        context.cache.begin({ volumeId: "volX", datacenterId: "DC-1" });
+        expect(box.journal.acquireCacheLease({ accountScope: "acct", datacenterId: "DC-1", volumeId: "volX", cacheKey: "k".repeat(64) }, "someone-else", 600)).toBe(true);
+        const job = box.job(1, { ...persistent, reuseVolumeId: "volX" });
+        await expect(box.backend.start(job, context)).rejects.toMatchObject({ code: "CLOUD_CACHE_LOCKED" });
+        expect(box.fleet.pods.size).toBe(0);
+    });
+    it("terminates the CPU pod even when preparation fails and leaves the cache FAILED, never READY", async () => {
+        const box = setup();
+        const originalCreate = box.fleet.createPod.bind(box.fleet);
+        box.fleet.createPod = async (request) => {
+            const pod = await originalCreate(request);
+            if (!request.gpu) {
+                const entry = box.fleet.pods.get(pod.id);
+                await entry.worker.close();
+                const volumeId = request.volume.id;
+                const worker = createWorkerServer({
+                    token: request.env.MOLDESK_WORKER_TOKEN,
+                    rootDir: fs.mkdtempSync(path.join(os.tmpdir(), "moldesk fail ")),
+                    pipeline: async () => {
+                        void volumeId;
+                        throw Object.assign(new Error("disk full"), { code: "CLOUD_ASSET_PREPARATION_FAILED" });
+                    },
+                });
+                const port = await worker.listen(0, "127.0.0.1");
+                box.fleet.pods.set(pod.id, { ...entry, worker, endpoint: `http://127.0.0.1:${port}` });
+            }
+            return pod;
+        };
+        await expect(box.backend.start(box.job(1, persistent), box.ctx(1, true))).rejects.toMatchObject({ code: "CLOUD_ASSET_PREPARATION_FAILED" });
+        expect(box.fleet.pods.size).toBe(0);
+        expect(box.journal.listCaches()[0].state).toBe("FAILED");
+        expect(box.fleet.log.filter((line) => line.startsWith("createPod gpu"))).toHaveLength(0);
+    });
+});

package/dist/cloud/deps.d.ts

@@ -0,0 +1,15 @@
+import { CredentialStore, type MoldeskPaths, type TokenStore, type WorkerBundle } from "@moldesk/runtime";
+import type { CloudRunDeps } from "./run.js";
+export interface RunpodDepsOptions {
+    paths?: MoldeskPaths;
+    store?: CredentialStore;
+    tokenStore?: TokenStore;
+    workerBundle?: WorkerBundle;
+    env?: NodeJS.ProcessEnv;
+}
+/**
+ * Production wiring for Runpod: key from the credential store (read per request), account scope from
+ * a keyed credential fingerprint and per-job tokens in the keyring. There is no independent
+ * lifetime scheduler: shutdown is enforced by this client while connected and by the in-Pod watchdog.
+ */
+export declare function createRunpodCloudDeps(options?: RunpodDepsOptions): Promise<CloudRunDeps>;

package/dist/cloud/deps.js

@@ -0,0 +1,30 @@
+import { CredentialStore, RunpodExecutionBackend, RunpodProvider, createKeyringTokenStore, credentialFingerprint, getMoldeskPaths, } from "@moldesk/runtime";
+import { CloudJournal } from "./state.js";
+import { resolveWorkerBundle } from "./worker-release.js";
+/**
+ * Production wiring for Runpod: key from the credential store (read per request), account scope from
+ * a keyed credential fingerprint and per-job tokens in the keyring. There is no independent
+ * lifetime scheduler: shutdown is enforced by this client while connected and by the in-Pod watchdog.
+ */
+export async function createRunpodCloudDeps(options = {}) {
+    const paths = options.paths ?? getMoldeskPaths();
+    const store = options.store ?? new CredentialStore();
+    const credential = await store.require("runpod");
+    const journal = CloudJournal.open(paths);
+    const accountScope = (() => {
+        try {
+            return credentialFingerprint(credential.key, journal.clientId);
+        }
+        finally {
+            journal.close();
+        }
+    })();
+    const provider = new RunpodProvider({ getApiKey: async () => (await store.require("runpod")).key });
+    const bundle = options.workerBundle ?? resolveWorkerBundle(options.env);
+    const backend = new RunpodExecutionBackend({
+        provider,
+        tokenStore: options.tokenStore ?? createKeyringTokenStore(),
+        ...(bundle ? { workerBundle: bundle } : {}),
+    });
+    return { provider, paths, backend, accountScope };
+}

package/dist/cloud/manage.d.ts

@@ -0,0 +1,57 @@
+import { type ExecutionBackend } from "@moldesk/runtime";
+import type { RunModelResult, RunRecord } from "../run.js";
+import { type CleanupOutcome, type CloudRunDeps } from "./run.js";
+import { type CloudCacheRecord } from "./state.js";
+export type RecoverDeps = Pick<CloudRunDeps, "provider" | "paths" | "resolveModel" | "sleep" | "cleanupAttempts"> & {
+    backend?: ExecutionBackend;
+};
+export interface RecoverOptions {
+    /** Terminate even if the worker still reports the job as running (stops further rental; loses unfinished results). */
+    terminateRunning?: boolean;
+}
+export interface RecoverResult {
+    runId: string;
+    /** Results were downloaded and verified during recovery. */
+    collected: boolean;
+    cleanup: CleanupOutcome;
+    /** Present when a previously failed run became a verified success. */
+    result?: RunModelResult;
+    status: RunRecord["status"];
+}
+/**
+ * Resumes cleanup (and, when possible, result transfer) for a cloud run after a crash, cancellation
+ * or transfer failure. It never starts inference again: the only paid actions it can take are
+ * downloading from a Pod that already exists and terminating Pods this installation owns.
+ */
+export declare function recoverCloudRun(runId: string, deps: RecoverDeps, options?: RecoverOptions): Promise<RecoverResult>;
+export interface CloudCacheListing {
+    cache: CloudCacheRecord;
+    /** Whether the provider still has the volume (a deleted volume means the cache is gone). */
+    volumeExists: boolean | undefined;
+    volumeSizeGb?: number;
+}
+export declare function listCloudCaches(deps: Pick<CloudRunDeps, "provider" | "paths" | "accountScope">): Promise<CloudCacheListing[]>;
+export interface CacheDeletionPlan {
+    model: string;
+    /** Exactly what will be removed. Nothing else is touched. */
+    volumes: Array<{
+        volumeId: string;
+        datacenterId: string;
+        sizeGb?: number;
+        cacheKeys: string[];
+    }>;
+    /** Reasons deletion cannot proceed right now. */
+    blockers: string[];
+}
+export declare function planCloudCacheDeletion(model: string, deps: Pick<CloudRunDeps, "provider" | "paths" | "accountScope">): Promise<CacheDeletionPlan>;
+export interface CacheDeletionResult {
+    deleted: Array<{
+        volumeId: string;
+    }>;
+    failed: Array<{
+        volumeId: string;
+        reason: string;
+    }>;
+}
+/** Deletes exactly the volumes in a plan the caller already showed and had confirmed. Journal ownership is required. */
+export declare function executeCloudCacheDeletion(plan: CacheDeletionPlan, deps: Pick<CloudRunDeps, "provider" | "paths" | "accountScope">): Promise<CacheDeletionResult>;

package/dist/cloud/manage.js

@@ -0,0 +1,210 @@
+import fs from "node:fs";
+import path from "node:path";
+import { getMoldeskPaths, runDir } from "@moldesk/runtime";
+import { MoldeskError } from "../errors.js";
+import { collectRunOutputs, finalizeRun, writeRunRecord } from "../run-lifecycle.js";
+import { confirmRunResourcesGone, defaultResolveModelForRecovery, recordLostLogs } from "./run.js";
+import { CloudJournal } from "./state.js";
+function failure(code, message, remediation, details) {
+    return new MoldeskError({ code, message, remediation, details });
+}
+function readRunRecord(paths, runId) {
+    const runDirPath = runDir(runId, { MOLDESK_HOME: paths.home });
+    const file = path.join(runDirPath, "run.json");
+    if (!fs.existsSync(file))
+        throw failure("CLOUD_RUN_UNKNOWN", `No run record for ${runId}.`, "Check the run id with `moldesk cloud status`.");
+    return { record: JSON.parse(fs.readFileSync(file, "utf8")), runDirPath };
+}
+/**
+ * Resumes cleanup (and, when possible, result transfer) for a cloud run after a crash, cancellation
+ * or transfer failure. It never starts inference again: the only paid actions it can take are
+ * downloading from a Pod that already exists and terminating Pods this installation owns.
+ */
+export async function recoverCloudRun(runId, deps, options = {}) {
+    const paths = deps.paths ?? getMoldeskPaths();
+    const journal = CloudJournal.open(paths);
+    try {
+        const job = journal.getJob(runId);
+        if (!job)
+            throw failure("CLOUD_RUN_UNKNOWN", `No cloud job is recorded for ${runId}.`, "Only runs started with --compute runpod can be recovered.");
+        const { record, runDirPath } = readRunRecord(paths, runId);
+        let collected = false;
+        let result;
+        // --- Try to fetch results that were left on a still-running Pod. ---
+        const livePod = journal.listResources({ runId, states: ["active", "delete_pending"] }).find((resource) => resource.role === "gpu" && resource.providerId);
+        let stillRunning = false;
+        if (record.status !== "succeeded" && livePod?.providerId && deps.backend?.reconnect) {
+            const handle = deps.backend.reconnect(runId, livePod.providerId, job.volumeId ?? undefined);
+            try {
+                const snapshot = await deps.backend.observe(handle, record.compute?.workerLogCursor ?? 0);
+                // Keep the local logs complete: append whatever the worker produced since the last saved cursor.
+                if (record.compute)
+                    recordLostLogs(snapshot, { stdoutPath: record.logs.stdout, stderrPath: record.logs.stderr, compute: record.compute, record });
+                if (snapshot.logs.stdout)
+                    fs.appendFileSync(record.logs.stdout, snapshot.logs.stdout);
+                if (snapshot.logs.stderr)
+                    fs.appendFileSync(record.logs.stderr, snapshot.logs.stderr);
+                if (record.compute)
+                    record.compute.workerLogCursor = snapshot.logs.cursor;
+                if (snapshot.state === "succeeded") {
+                    const outputDir = path.join(runDirPath, "output");
+                    await deps.backend.collect(handle, outputDir);
+                    const resolved = (deps.resolveModel ?? defaultResolveModelForRecovery)(record.model);
+                    const input = record.input[0];
+                    const context = {
+                        manifestName: record.model,
+                        inputPath: input.storedPath,
+                        outputDir,
+                        modelDir: path.join(runDirPath, "remote-model"),
+                        assetsDir: path.join(runDirPath, "remote-assets"),
+                        params: record.parameters.effective,
+                        runtimeExecutable: "",
+                        platform: "linux-x64",
+                    };
+                    record.outputs = await collectRunOutputs(resolved.adapter, context);
+                    record.status = "succeeded";
+                    delete record.error;
+                    journal.updateJob(runId, { transferState: "verified" });
+                    collected = true;
+                    result = { status: "succeeded", record };
+                }
+                else if (snapshot.state === "running") {
+                    stillRunning = true;
+                }
+            }
+            catch {
+                // Still unreachable or incomplete: fall through to cleanup; the failure record stays as is.
+            }
+        }
+        // A job that is genuinely still running keeps its (already approved) rental unless told otherwise.
+        if (stillRunning && !options.terminateRunning) {
+            return { runId, collected: false, cleanup: { confirmed: false, pending: [{ role: "gpu", providerId: livePod.providerId }] }, status: record.status };
+        }
+        // A run interrupted before it finished must not stay "running" forever.
+        if (record.status === "preparing" || record.status === "running") {
+            const finishedAt = new Date().toISOString();
+            record.status = "failed";
+            record.error = { code: "CLOUD_RUN_INTERRUPTED", message: "The run was interrupted before it finished; recovery cleaned up its cloud resources." };
+            record.finishedAt = finishedAt;
+            record.durationMs = Date.parse(finishedAt) - Date.parse(record.startedAt);
+        }
+        // --- Cleanup: terminate and verify only what the journal says this installation owns. ---
+        const cleanup = await confirmRunResourcesGone(runId, deps, journal);
+        if (record.compute)
+            record.compute.cleanup = cleanup.confirmed ? "confirmed" : "pending";
+        journal.updateJob(runId, {
+            stage: "done",
+            status: cleanup.confirmed ? (record.status === "succeeded" ? "succeeded" : record.status === "cancelled" ? "cancelled" : "failed") : "cleanup_pending",
+            error: cleanup.confirmed ? record.error ?? null : { code: "CLOUD_CLEANUP_PENDING", message: "Paid resources could not be confirmed deleted." },
+        });
+        if (collected && result) {
+            const resolved = (deps.resolveModel ?? defaultResolveModelForRecovery)(record.model);
+            const input = record.input[0];
+            result = await finalizeRun({
+                adapter: resolved.adapter,
+                context: {
+                    manifestName: record.model,
+                    inputPath: input.storedPath,
+                    outputDir: path.join(runDirPath, "output"),
+                    modelDir: path.join(runDirPath, "remote-model"),
+                    assetsDir: path.join(runDirPath, "remote-assets"),
+                    params: record.parameters.effective,
+                    runtimeExecutable: "",
+                    platform: "linux-x64",
+                },
+                runDirPath,
+                record,
+                startedAt: record.startedAt,
+            });
+        }
+        else {
+            writeRunRecord(runDirPath, record);
+        }
+        return { runId, collected, cleanup, ...(result ? { result } : {}), status: record.status };
+    }
+    finally {
+        journal.close();
+    }
+}
+export async function listCloudCaches(deps) {
+    const journal = CloudJournal.open(deps.paths ?? getMoldeskPaths());
+    try {
+        const volumes = await deps.provider.listVolumes().catch(() => undefined);
+        const byId = new Map((volumes ?? []).map((volume) => [volume.id, volume]));
+        return journal
+            .listCaches()
+            .filter((cache) => !deps.accountScope || cache.accountScope === deps.accountScope)
+            .map((cache) => ({
+            cache,
+            volumeExists: volumes ? byId.has(cache.volumeId) : undefined,
+            ...(byId.get(cache.volumeId) ? { volumeSizeGb: byId.get(cache.volumeId).sizeGb } : {}),
+        }));
+    }
+    finally {
+        journal.close();
+    }
+}
+export async function planCloudCacheDeletion(model, deps) {
+    const journal = CloudJournal.open(deps.paths ?? getMoldeskPaths());
+    try {
+        const caches = journal.listCaches().filter((cache) => cache.model === model && (!deps.accountScope || cache.accountScope === deps.accountScope));
+        const volumes = new Map();
+        const volumeList = await deps.provider.listVolumes();
+        for (const cache of caches) {
+            const entry = volumes.get(cache.volumeId) ?? { volumeId: cache.volumeId, datacenterId: cache.datacenterId, sizeGb: volumeList.find((volume) => volume.id === cache.volumeId)?.sizeGb, cacheKeys: [] };
+            entry.cacheKeys.push(cache.cacheKey);
+            volumes.set(cache.volumeId, entry);
+        }
+        const blockers = [];
+        for (const job of journal.listOpenJobs()) {
+            if (job.volumeId && volumes.has(job.volumeId))
+                blockers.push(`Run ${job.runId} is still using volume ${job.volumeId} (${job.stage}, ${job.status}).`);
+        }
+        // A volume other caches also live on (another model) must not be deleted from under them.
+        for (const volumeId of volumes.keys()) {
+            const sharedBy = journal.listCaches().filter((cache) => cache.volumeId === volumeId && cache.model !== model);
+            if (sharedBy.length > 0)
+                blockers.push(`Volume ${volumeId} also holds caches for: ${[...new Set(sharedBy.map((cache) => cache.model))].join(", ")}.`);
+        }
+        return { model, volumes: [...volumes.values()], blockers };
+    }
+    finally {
+        journal.close();
+    }
+}
+/** Deletes exactly the volumes in a plan the caller already showed and had confirmed. Journal ownership is required. */
+export async function executeCloudCacheDeletion(plan, deps) {
+    if (plan.blockers.length > 0)
+        throw failure("CLOUD_CACHE_IN_USE", plan.blockers.join(" "), "Wait for active runs to finish, or delete the other caches first.", { blockers: plan.blockers });
+    const journal = CloudJournal.open(deps.paths ?? getMoldeskPaths());
+    const result = { deleted: [], failed: [] };
+    try {
+        for (const volume of plan.volumes) {
+            const owner = journal.findOwnedByProviderId(volume.volumeId);
+            if (!owner || owner.role !== "volume") {
+                result.failed.push({ volumeId: volume.volumeId, reason: "This installation did not create that volume, so it will not delete it." });
+                continue;
+            }
+            try {
+                journal.markResourceDeletePending(owner.intentId);
+                await deps.provider.deleteVolume(volume.volumeId);
+                if ((await deps.provider.getVolume(volume.volumeId)) !== null) {
+                    result.failed.push({ volumeId: volume.volumeId, reason: "The provider still lists the volume after deletion." });
+                    continue;
+                }
+                journal.markResourceDeleted(owner.intentId);
+                for (const key of volume.cacheKeys) {
+                    journal.deleteCache({ accountScope: deps.accountScope ?? journal.listCaches().find((cache) => cache.volumeId === volume.volumeId)?.accountScope ?? "", datacenterId: volume.datacenterId, volumeId: volume.volumeId, cacheKey: key });
+                }
+                result.deleted.push({ volumeId: volume.volumeId });
+            }
+            catch (error) {
+                result.failed.push({ volumeId: volume.volumeId, reason: error instanceof Error ? error.message : String(error) });
+            }
+        }
+        return result;
+    }
+    finally {
+        journal.close();
+    }
+}

package/dist/cloud/manage.test.d.ts

@@ -0,0 +1 @@
+export {};