@moldesk/runtime 0.0.4 → 0.0.6

package/dist/cloud/runpod-backend.js

@@ -0,0 +1,272 @@
+import { randomBytes } from "node:crypto";
+import { MoldeskError } from "../providers.js";
+import { createKeyringBackend } from "./credentials.js";
+import { EXECUTION_CONTRACT_VERSION, WORKER_PORT, toWireJob, } from "./execution.js";
+import { WorkerClient } from "./worker-client.js";
+export const MIN_WORKER_READY_TIMEOUT_MS = 5 * 60_000;
+export const VOLUME_MOUNT = "/workspace";
+const TOKEN_SERVICE = "moldesk-worker";
+export function createKeyringTokenStore(backend = createKeyringBackend()) {
+    return {
+        set: (runId, token) => backend.set(TOKEN_SERVICE, runId, token),
+        get: (runId) => backend.get(TOKEN_SERVICE, runId),
+        delete: async (runId) => void (await backend.delete(TOKEN_SERVICE, runId)),
+    };
+}
+/**
+ * Container start command: fetch the pinned bundle, verify its checksum, and run it. The image must
+ * provide Node 20+, curl, tar and sha256sum. Nothing here comes from the user's input.
+ */
+export const WORKER_BOOTSTRAP_SCRIPT = [
+    "set -euo pipefail",
+    'command -v node >/dev/null || { echo "node is required in the worker image" >&2; exit 127; }',
+    "mkdir -p /opt/moldesk && cd /opt/moldesk",
+    'curl -fsSL "$MOLDESK_WORKER_BUNDLE_URL" -o worker.tgz',
+    'echo "$MOLDESK_WORKER_BUNDLE_SHA256  worker.tgz" | sha256sum -c -',
+    "tar -xzf worker.tgz",
+    "export MOLDESK_MODELS_DIR=/opt/moldesk/worker/models",
+    "exec node worker/main.js",
+].join("\n");
+function backendError(code, message, remediation, details, cause) {
+    return new MoldeskError({ code, message, remediation, details, cause });
+}
+export class RunpodExecutionBackend {
+    options;
+    clients = new Map();
+    tracked = new Map();
+    /** Intents already confirmed deleted, so release() stays idempotent. */
+    deletedIntents = new Set();
+    sleep;
+    readyTimeoutMs;
+    constructor(options) {
+        this.options = options;
+        this.sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+        this.readyTimeoutMs = Math.max(options.readyTimeoutMs ?? 10 * 60_000, MIN_WORKER_READY_TIMEOUT_MS);
+    }
+    endpoint(podId) {
+        return (this.options.endpointFor ?? ((id) => `https://${id}-${WORKER_PORT}.proxy.runpod.net`))(podId);
+    }
+    client(endpoint, token) {
+        return (this.options.createClient ?? ((url, secret) => new WorkerClient({ endpoint: url, token: secret })))(endpoint, token);
+    }
+    async clientFor(handle) {
+        const cached = this.clients.get(handle.runId);
+        if (cached)
+            return cached;
+        const token = await this.options.tokenStore.get(handle.runId);
+        if (!token)
+            throw backendError("CLOUD_WORKER_TOKEN_MISSING", "The worker token for this run is not in the credential store.", "Results can no longer be fetched through this client; the Pod will be terminated.");
+        const created = this.client(handle.endpoint, token);
+        this.clients.set(handle.runId, created);
+        return created;
+    }
+    async start(job, context) {
+        if (!this.options.workerBundle) {
+            throw backendError("CLOUD_WORKER_BUNDLE_UNCONFIGURED", "No worker release bundle is configured.", "Cloud execution needs a published, checksummed worker bundle; none is configured in this build.");
+        }
+        if (job.cachePolicy === "ephemeral")
+            return (await this.startGpu(job, context, undefined)).handle;
+        return this.startPersistent(job, context);
+    }
+    async startPersistent(job, context) {
+        const cache = context.cache;
+        if (!cache)
+            throw backendError("CLOUD_CACHE_LEDGER_MISSING", "Persistent runs need a cache journal.", "This is an internal error.");
+        const known = cache.lookup();
+        let volumeId = job.reuseVolumeId ?? known?.volumeId;
+        let warm = known !== undefined;
+        if (!volumeId) {
+            const intent = context.ledger.beginCreate("volume");
+            const volume = await this.options.provider.createVolume({ name: intent.name, sizeGb: job.volumeGb, dataCenterId: job.datacenterId });
+            context.ledger.created(intent.intentId, volume.id);
+            volumeId = volume.id;
+            cache.begin({ volumeId, datacenterId: job.datacenterId });
+            warm = false;
+        }
+        if (warm) {
+            // Warm: the GPU Pod verifies the volume's cache itself; local READY alone is never trusted.
+            context.onProgress?.({ stage: "provisioning", message: "Reusing retained volume" });
+            const attempt = await this.startGpu(job, context, volumeId, { watchCacheFailure: true });
+            if (attempt.handle)
+                return attempt.handle;
+            cache.failed();
+            context.onProgress?.({ stage: "preparing_assets", message: "Cache missing or invalid; rebuilding on a CPU machine" });
+        }
+        await this.prepareOnCpu(job, context, volumeId, cache);
+        const handle = await this.startGpu(job, context, volumeId);
+        return handle.handle;
+    }
+    /** Cold path: CPU Pod prepares the volume, is terminated and confirmed gone, then capacity is rechecked. */
+    async prepareOnCpu(job, context, volumeId, cache) {
+        if (!cache.acquireLease(3600)) {
+            throw backendError("CLOUD_CACHE_LOCKED", "Another process in this installation is preparing this cache.", "Wait for it to finish.");
+        }
+        let cpuPodId;
+        let cpuIntent;
+        try {
+            cache.begin({ volumeId, datacenterId: job.datacenterId });
+            context.onProgress?.({ stage: "provisioning", message: "Starting CPU machine to prepare model assets" });
+            const token = randomBytes(32).toString("hex");
+            const runId = `${job.runId}-prep`;
+            await this.options.tokenStore.set(runId, token);
+            const pod = await this.createWorkerPod(job, context, { role: "cpu", volumeId, token });
+            cpuPodId = pod.pod.id;
+            cpuIntent = pod.intentId;
+            const client = this.client(this.endpoint(pod.pod.id), token);
+            await client.waitUntilReady({ timeoutMs: this.readyTimeoutMs, intervalMs: this.options.pollIntervalMs, signal: context.signal, abortIf: () => this.podFatal(pod.pod.id) });
+            await client.start({ ...toWireJob({ ...job, inputs: [] }, { mode: "prepare-assets", volumeMount: VOLUME_MOUNT }), runId });
+            const snapshot = await this.waitForTerminal(client, runId, context);
+            if (snapshot.state !== "succeeded" || !snapshot.runtime?.cache) {
+                cache.failed();
+                throw backendError(snapshot.error?.code ?? "CLOUD_ASSET_PREPARATION_FAILED", snapshot.error?.message ?? "Asset preparation failed on the CPU machine.", "The volume cache is not marked ready; retry.");
+            }
+            cache.verifying();
+            cache.ready({ sizeBytes: snapshot.runtime.cache.sizeBytes, checksumManifestDigest: snapshot.runtime.cache.checksumManifestSha256 });
+            await this.options.tokenStore.delete(runId).catch(() => undefined);
+        }
+        catch (error) {
+            cache.failed();
+            throw error;
+        }
+        finally {
+            // Terminate and confirm the CPU Pod before any GPU is requested, whatever happened.
+            if (cpuPodId && cpuIntent)
+                await this.terminateAndConfirm(cpuPodId, cpuIntent, context);
+            cache.releaseLease();
+        }
+        // No capacity is reserved by the quote: re-check before renting the GPU.
+        const catalog = await this.options.provider.getCatalog();
+        const gpu = catalog.gpus.find((candidate) => candidate.id === job.gpuId);
+        const available = gpu && gpu.availability !== "NONE" && gpu.dataCenters.some((dc) => dc.id === job.datacenterId && dc.availability !== "NONE");
+        if (!available) {
+            throw backendError("CLOUD_CAPACITY_UNAVAILABLE", `${job.gpuId} no longer has capacity in ${job.datacenterId}. The prepared volume is retained.`, "Retry later; the cache is ready and the next run will skip preparation. Delete the volume with `moldesk cloud cache delete` if you no longer want it.");
+        }
+    }
+    async createWorkerPod(job, context, params) {
+        const bundle = this.options.workerBundle;
+        const intent = context.ledger.beginCreate(params.role);
+        const tracked = this.tracked.get(job.runId) ?? [];
+        const entry = { intentId: intent.intentId };
+        tracked.push(entry);
+        this.tracked.set(job.runId, tracked);
+        const pod = await this.options.provider.createPod({
+            name: intent.name,
+            image: params.role === "gpu" ? job.images.gpu : job.images.cpu,
+            containerDiskGb: job.containerDiskGb,
+            dataCenterId: job.datacenterId,
+            ...(params.role === "gpu" ? { gpu: { id: job.gpuId, count: 1 } } : { cpu: { id: "cpu3c", vcpuCount: 2 } }),
+            ...(params.volumeId ? { volume: { id: params.volumeId, mountPath: VOLUME_MOUNT } } : {}),
+            ports: [`${WORKER_PORT}/http`],
+            env: {
+                MOLDESK_WORKER_TOKEN: params.token,
+                MOLDESK_WORKER_DEADLINE_AT: job.deadlineAt,
+                MOLDESK_WORKER_PORT: String(WORKER_PORT),
+                MOLDESK_WORKER_ROOT: "/job",
+                MOLDESK_WORKER_BUNDLE_URL: bundle.url,
+                MOLDESK_WORKER_BUNDLE_SHA256: bundle.sha256,
+            },
+            entrypoint: ["/bin/bash", "-c"],
+            cmd: [WORKER_BOOTSTRAP_SCRIPT],
+        });
+        context.ledger.created(intent.intentId, pod.id);
+        entry.podId = pod.id;
+        return { pod, intentId: intent.intentId };
+    }
+    /** Returns a reason string when the Pod can never serve (so booting should stop waiting). */
+    async podFatal(podId) {
+        const pod = await this.options.provider.getPod(podId).catch(() => undefined);
+        if (pod === null)
+            return "The Pod disappeared before the worker started.";
+        if (pod && (pod.status === "ERROR" || pod.status === "EXITED" || pod.status === "TERMINATED"))
+            return `The Pod entered ${pod.status} before the worker started.`;
+        return undefined;
+    }
+    async waitForTerminal(client, runId, context) {
+        let after = 0;
+        for (;;) {
+            if (context.signal?.aborted) {
+                await client.cancel(runId).catch(() => undefined);
+                throw backendError("RUN_CANCELLED", "Cancelled.", "None.");
+            }
+            const snapshot = await client.status(runId, after);
+            after = snapshot.logs.cursor;
+            if (snapshot.state === "succeeded" || snapshot.state === "failed" || snapshot.state === "cancelled")
+                return snapshot;
+            await this.sleep(this.options.pollIntervalMs ?? 5000);
+        }
+    }
+    /** Starts a GPU worker. With `watchCacheFailure`, a cache-missing/invalid failure returns no handle so the caller can rebuild. */
+    async startGpu(job, context, volumeId, options = {}) {
+        const token = randomBytes(32).toString("hex");
+        await this.options.tokenStore.set(job.runId, token);
+        context.onProgress?.({ stage: "provisioning" });
+        const { pod, intentId } = await this.createWorkerPod(job, context, { role: "gpu", volumeId, token });
+        const endpoint = this.endpoint(pod.id);
+        const client = this.client(endpoint, token);
+        this.clients.set(job.runId, client);
+        context.onProgress?.({ stage: "booting" });
+        await client.waitUntilReady({ timeoutMs: this.readyTimeoutMs, intervalMs: this.options.pollIntervalMs, signal: context.signal, abortIf: () => this.podFatal(pod.id) });
+        context.onProgress?.({ stage: "uploading" });
+        for (const file of job.inputs)
+            await client.uploadInput(job.runId, file);
+        await client.start(toWireJob(job, { mode: "execute", ...(volumeId ? { volumeMount: VOLUME_MOUNT } : {}) }));
+        if (options.watchCacheFailure) {
+            // The worker verifies the volume cache first; a bad cache fails before anything expensive.
+            for (let i = 0; i < 120; i += 1) {
+                const snapshot = await client.status(job.runId);
+                if (snapshot.state === "failed" && (snapshot.error?.code === "CLOUD_CACHE_MISSING" || snapshot.error?.code === "CLOUD_CACHE_INVALID")) {
+                    await this.terminateAndConfirm(pod.id, intentId, context);
+                    this.clients.delete(job.runId);
+                    return {};
+                }
+                if (snapshot.state !== "running" || snapshot.stage !== "preparing_assets")
+                    break;
+                await this.sleep(this.options.pollIntervalMs ?? 2000);
+            }
+        }
+        return { handle: { version: EXECUTION_CONTRACT_VERSION, runId: job.runId, podId: pod.id, ...(volumeId ? { volumeId } : {}), endpoint } };
+    }
+    /** Terminate through the provider, then require the provider to show the Pod gone before marking it deleted. */
+    async terminateAndConfirm(podId, intentId, context) {
+        context.ledger.beginDelete(intentId);
+        try {
+            await this.options.provider.terminatePod(podId);
+        }
+        catch {
+            return false;
+        }
+        const deadline = (this.options.now ?? Date.now)() + (this.options.terminationWaitMs ?? 60_000);
+        for (;;) {
+            const pod = await this.options.provider.getPod(podId).catch(() => undefined);
+            if (pod === null || pod?.status === "TERMINATED") {
+                context.ledger.deleted(intentId);
+                this.deletedIntents.add(intentId);
+                return true;
+            }
+            if ((this.options.now ?? Date.now)() >= deadline)
+                return false;
+            await this.sleep(2000);
+        }
+    }
+    reconnect(runId, podId, volumeId) {
+        return { version: EXECUTION_CONTRACT_VERSION, runId, podId, ...(volumeId ? { volumeId } : {}), endpoint: this.endpoint(podId) };
+    }
+    async observe(handle, after = 0) {
+        return (await this.clientFor(handle)).status(handle.runId, after);
+    }
+    async cancel(handle) {
+        await (await this.clientFor(handle)).cancel(handle.runId);
+    }
+    async collect(handle, localOutputDir, options = {}) {
+        return (await this.clientFor(handle)).collect(handle.runId, localOutputDir, options);
+    }
+    async release(handle, context) {
+        for (const entry of this.tracked.get(handle.runId) ?? []) {
+            if (!entry.podId || this.deletedIntents.has(entry.intentId))
+                continue;
+            await this.terminateAndConfirm(entry.podId, entry.intentId, context);
+        }
+        this.clients.delete(handle.runId);
+        await this.options.tokenStore.delete(handle.runId).catch(() => undefined);
+    }
+}

package/dist/cloud/runpod.d.ts

@@ -0,0 +1,41 @@
+import type { CloudAccountCheck, CloudCatalog, CloudDeleteResult, CloudPod, CloudProvider, CloudVolume, CreatePodRequest, CreateVolumeRequest } from "./types.js";
+export declare const RUNPOD_API_BASE = "https://api.runpod.io/v2";
+/** Runpod's edge rejects default HTTP-client User-Agents (observed: Cloudflare 403, error code 1010). */
+export declare const RUNPOD_USER_AGENT = "moldesk-runpod/1";
+export interface RunpodProviderOptions {
+    /** Resolved per request so the key never lives in a long-lived field or log. */
+    getApiKey: () => Promise<string>;
+    fetch?: typeof fetch;
+    sleep?: (ms: number) => Promise<void>;
+    now?: () => Date;
+    baseUrl?: string;
+    requestTimeoutMs?: number;
+}
+export declare class RunpodProvider implements CloudProvider {
+    private readonly options;
+    readonly name = "runpod";
+    private readonly fetchFn;
+    private readonly sleep;
+    private readonly now;
+    private readonly baseUrl;
+    private readonly timeoutMs;
+    constructor(options: RunpodProviderOptions);
+    validateCredentials(): Promise<CloudAccountCheck>;
+    getCatalog(): Promise<CloudCatalog>;
+    createPod(request: CreatePodRequest): Promise<CloudPod>;
+    getPod(id: string): Promise<CloudPod | null>;
+    listPods(): Promise<CloudPod[]>;
+    terminatePod(id: string): Promise<CloudDeleteResult>;
+    createVolume(request: CreateVolumeRequest): Promise<CloudVolume>;
+    getVolume(id: string): Promise<CloudVolume | null>;
+    listVolumes(): Promise<CloudVolume[]>;
+    deleteVolume(id: string): Promise<CloudDeleteResult>;
+    /**
+     * One HTTP call with status-to-error mapping. Only `retry` calls (reads and idempotent
+     * terminate/delete) are retried, on 429/5xx/network failure. Creates are never retried; a
+     * failure that may have taken effect is surfaced as CLOUD_AMBIGUOUS_CREATE.
+     */
+    private request;
+    private send;
+    private mapStatus;
+}

package/dist/cloud/runpod.js

@@ -0,0 +1,276 @@
+import { MoldeskError } from "../providers.js";
+export const RUNPOD_API_BASE = "https://api.runpod.io/v2";
+/** Runpod's edge rejects default HTTP-client User-Agents (observed: Cloudflare 403, error code 1010). */
+export const RUNPOD_USER_AGENT = "moldesk-runpod/1";
+const POD_STATUSES = ["PROVISIONING", "STARTING", "RUNNING", "EXITED", "ERROR", "TERMINATED"];
+const IDEMPOTENT_RETRIES = 3;
+const MAX_BODY_CHARS = 500;
+function cloudError(code, message, remediation, details, cause) {
+    return new MoldeskError({ code, message, remediation, details, cause });
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function asString(value, field) {
+    if (typeof value !== "string")
+        throw cloudError("CLOUD_BAD_RESPONSE", `Runpod response is missing "${field}".`, "Retry; if it persists the Runpod API may have changed.");
+    return value;
+}
+function asNumber(value, field) {
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+        throw cloudError("CLOUD_BAD_RESPONSE", `Runpod response has an invalid "${field}".`, "Retry; if it persists the Runpod API may have changed.");
+    }
+    return value;
+}
+function asArray(value, field) {
+    if (!Array.isArray(value))
+        throw cloudError("CLOUD_BAD_RESPONSE", `Runpod response is missing list "${field}".`, "Retry; if it persists the Runpod API may have changed.");
+    return value;
+}
+function asRecord(value, field) {
+    if (!isRecord(value))
+        throw cloudError("CLOUD_BAD_RESPONSE", `Runpod response has an invalid "${field}".`, "Retry; if it persists the Runpod API may have changed.");
+    return value;
+}
+function parsePod(value) {
+    const pod = asRecord(value, "pod");
+    const status = asString(pod.status, "pod.status");
+    if (!POD_STATUSES.includes(status)) {
+        throw cloudError("CLOUD_BAD_RESPONSE", `Runpod returned an unknown pod status "${status}".`, "Update MoleculeDesk or retry.");
+    }
+    return {
+        id: asString(pod.id, "pod.id"),
+        name: typeof pod.name === "string" ? pod.name : "",
+        status,
+        dataCenterId: typeof pod.dataCenterId === "string" ? pod.dataCenterId : null,
+        costUsdPerHour: typeof pod.cost === "number" ? pod.cost : null,
+        createdAt: typeof pod.createdAt === "string" ? pod.createdAt : null,
+    };
+}
+function parseVolume(value) {
+    const volume = asRecord(value, "volume");
+    return {
+        id: asString(volume.id, "volume.id"),
+        name: typeof volume.name === "string" ? volume.name : "",
+        sizeGb: asNumber(volume.size, "volume.size"),
+        dataCenterId: asString(volume.dataCenter, "volume.dataCenter"),
+        type: typeof volume.type === "string" ? volume.type : null,
+    };
+}
+export class RunpodProvider {
+    options;
+    name = "runpod";
+    fetchFn;
+    sleep;
+    now;
+    baseUrl;
+    timeoutMs;
+    constructor(options) {
+        this.options = options;
+        this.fetchFn = options.fetch ?? fetch;
+        this.sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+        this.now = options.now ?? (() => new Date());
+        this.baseUrl = options.baseUrl ?? RUNPOD_API_BASE;
+        this.timeoutMs = options.requestTimeoutMs ?? 30_000;
+    }
+    async validateCredentials() {
+        // Smallest read-only call; a restricted key may lack Pod list access (403), which is reported as such.
+        await this.request("GET", "/pods", undefined, { retry: true });
+        return { ok: true, checkedAt: this.now().toISOString() };
+    }
+    async getCatalog() {
+        const [gpus, cpus, dataCenters] = await Promise.all([
+            this.request("GET", "/catalog/gpus?include=AVAILABILITY&product=POD", undefined, { retry: true }),
+            this.request("GET", "/catalog/cpus", undefined, { retry: true }),
+            this.request("GET", "/catalog/datacenters", undefined, { retry: true }),
+        ]);
+        return {
+            retrievedAt: this.now().toISOString(),
+            gpus: asArray(asRecord(gpus.body, "gpus").gpus, "gpus").map((raw) => {
+                const gpu = asRecord(raw, "gpu");
+                const price = asRecord(gpu.price, "gpu.price");
+                const availability = typeof gpu.availability === "string" ? gpu.availability : "UNKNOWN";
+                return {
+                    id: asString(gpu.id, "gpu.id"),
+                    name: typeof gpu.name === "string" ? gpu.name : asString(gpu.id, "gpu.id"),
+                    memoryGb: asNumber(gpu.memory, "gpu.memory"),
+                    priceUsdPerHour: asNumber(price.secure, "gpu.price.secure"),
+                    availability: (["NONE", "LOW", "MEDIUM", "HIGH"].includes(availability) ? availability : "UNKNOWN"),
+                    dataCenters: Array.isArray(gpu.dataCenters)
+                        ? gpu.dataCenters.filter(isRecord).map((dc) => ({ id: asString(dc.id, "gpu.dataCenters.id"), availability: String(dc.availability ?? "UNKNOWN") }))
+                        : [],
+                    cudaVersions: Array.isArray(gpu.cudaVersions)
+                        ? gpu.cudaVersions.filter(isRecord).filter((v) => v.available === true).map((v) => String(v.version))
+                        : [],
+                };
+            }),
+            cpus: asArray(asRecord(cpus.body, "cpus").cpus, "cpus").map((raw) => {
+                const cpu = asRecord(raw, "cpu");
+                const vcpu = asRecord(cpu.vcpu, "cpu.vcpu");
+                return {
+                    id: asString(cpu.id, "cpu.id"),
+                    name: typeof cpu.name === "string" ? cpu.name : asString(cpu.id, "cpu.id"),
+                    priceUsdPerVcpuHour: asNumber(asRecord(cpu.price, "cpu.price").securePerVcpu, "cpu.price.securePerVcpu"),
+                    ramGbPerVcpu: asNumber(cpu.ramGbPerVcpu, "cpu.ramGbPerVcpu"),
+                    minVcpu: asNumber(vcpu.min, "cpu.vcpu.min"),
+                    maxVcpu: asNumber(vcpu.max, "cpu.vcpu.max"),
+                };
+            }),
+            dataCenters: asArray(asRecord(dataCenters.body, "datacenters").dataCenters, "dataCenters").map((raw) => {
+                const dc = asRecord(raw, "dataCenter");
+                return {
+                    id: asString(dc.id, "dataCenter.id"),
+                    region: typeof dc.region === "string" ? dc.region : "",
+                    networkVolumeTypes: Array.isArray(dc.networkVolumeTypes) ? dc.networkVolumeTypes.map(String) : [],
+                };
+            }),
+        };
+    }
+    async createPod(request) {
+        if ((request.gpu ? 1 : 0) + (request.cpu ? 1 : 0) !== 1) {
+            throw cloudError("CLOUD_REQUEST_REJECTED", "A pod needs exactly one of gpu or cpu.", "Fix the pod request.");
+        }
+        const body = {
+            name: request.name,
+            image: request.image,
+            disk: request.containerDiskGb,
+            dataCenterIds: [request.dataCenterId],
+            ...(request.gpu ? { gpu: { id: request.gpu.id, count: request.gpu.count } } : {}),
+            ...(request.cpu ? { cpu: { id: request.cpu.id, vcpuCount: request.cpu.vcpuCount } } : {}),
+            ...(request.volume ? { mounts: { network: [{ volumeId: request.volume.id, path: request.volume.mountPath }] } } : {}),
+            ...(request.ports ? { ports: request.ports } : {}),
+            ...(request.env ? { env: request.env } : {}),
+            // `args` as an {entrypoint, cmd} object is the form verified live against the v2 API.
+            ...(request.entrypoint || request.cmd
+                ? { args: JSON.stringify({ ...(request.entrypoint ? { entrypoint: request.entrypoint } : {}), ...(request.cmd ? { cmd: request.cmd } : {}) }) }
+                : {}),
+        };
+        const result = await this.request("POST", "/pods", body, { retry: false, ambiguousOnFailure: true });
+        return parsePod(result.body);
+    }
+    async getPod(id) {
+        const result = await this.request("GET", `/pods/${encodeURIComponent(id)}`, undefined, { retry: true, allow404: true });
+        return result.status === 404 ? null : parsePod(result.body);
+    }
+    async listPods() {
+        const result = await this.request("GET", "/pods", undefined, { retry: true });
+        return asArray(asRecord(result.body, "pods").pods, "pods").map(parsePod);
+    }
+    async terminatePod(id) {
+        const result = await this.request("POST", `/pods/${encodeURIComponent(id)}/action`, { action: "terminate" }, { retry: true, allow404: true });
+        return { id, alreadyAbsent: result.status === 404 };
+    }
+    async createVolume(request) {
+        const body = { name: request.name, size: request.sizeGb, dataCenter: request.dataCenterId };
+        const result = await this.request("POST", "/network-volumes", body, { retry: false, ambiguousOnFailure: true });
+        return parseVolume(result.body);
+    }
+    async getVolume(id) {
+        const result = await this.request("GET", `/network-volumes/${encodeURIComponent(id)}`, undefined, { retry: true, allow404: true });
+        return result.status === 404 ? null : parseVolume(result.body);
+    }
+    async listVolumes() {
+        const result = await this.request("GET", "/network-volumes", undefined, { retry: true });
+        return asArray(asRecord(result.body, "volumes").networkVolumes, "networkVolumes").map(parseVolume);
+    }
+    async deleteVolume(id) {
+        const result = await this.request("DELETE", `/network-volumes/${encodeURIComponent(id)}`, undefined, { retry: true, allow404: true });
+        return { id, alreadyAbsent: result.status === 404 };
+    }
+    /**
+     * One HTTP call with status-to-error mapping. Only `retry` calls (reads and idempotent
+     * terminate/delete) are retried, on 429/5xx/network failure. Creates are never retried; a
+     * failure that may have taken effect is surfaced as CLOUD_AMBIGUOUS_CREATE.
+     */
+    async request(method, path, body, flags) {
+        const attempts = flags.retry ? IDEMPOTENT_RETRIES : 1;
+        let lastError;
+        for (let attempt = 1; attempt <= attempts; attempt += 1) {
+            let result;
+            try {
+                result = await this.send(method, path, body);
+            }
+            catch (error) {
+                lastError = flags.ambiguousOnFailure
+                    ? cloudError("CLOUD_AMBIGUOUS_CREATE", "The Runpod create request did not complete; the resource may exist.", "List owned resources and match by name before retrying.", { path }, error)
+                    : cloudError("CLOUD_UNAVAILABLE", "Could not reach Runpod.", "Check your connection and retry.", { path }, error);
+                if (attempt < attempts) {
+                    await this.sleep(500 * 2 ** (attempt - 1));
+                    continue;
+                }
+                throw lastError;
+            }
+            if (result.status >= 200 && result.status < 300)
+                return result;
+            if (result.status === 404 && flags.allow404)
+                return result;
+            const retryable = result.status === 429 || result.status >= 500;
+            if (retryable && attempt < attempts) {
+                await this.sleep(result.retryAfterSeconds !== undefined ? result.retryAfterSeconds * 1000 : 500 * 2 ** (attempt - 1));
+                continue;
+            }
+            throw this.mapStatus(result, path, flags.ambiguousOnFailure === true);
+        }
+        throw lastError;
+    }
+    async send(method, path, body) {
+        const key = await this.options.getApiKey();
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.timeoutMs);
+        try {
+            const response = await this.fetchFn(`${this.baseUrl}${path}`, {
+                method,
+                headers: {
+                    Authorization: `Bearer ${key}`,
+                    "Content-Type": "application/json",
+                    "User-Agent": RUNPOD_USER_AGENT,
+                },
+                ...(body !== undefined ? { body: JSON.stringify(body) } : {}),
+                signal: controller.signal,
+            });
+            const text = await response.text();
+            let parsed;
+            if (text) {
+                try {
+                    parsed = JSON.parse(text);
+                }
+                catch {
+                    parsed = text.slice(0, MAX_BODY_CHARS);
+                }
+            }
+            const retryAfter = Number(response.headers.get("retry-after"));
+            return {
+                status: response.status,
+                body: parsed,
+                ...(Number.isFinite(retryAfter) && retryAfter >= 0 && response.headers.get("retry-after") ? { retryAfterSeconds: retryAfter } : {}),
+            };
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    mapStatus(result, path, isCreate) {
+        const detail = isRecord(result.body) && typeof result.body.detail === "string" ? result.body.detail.slice(0, MAX_BODY_CHARS) : undefined;
+        const details = { status: result.status, path, ...(detail ? { detail } : {}) };
+        switch (result.status) {
+            case 401:
+                return cloudError("CLOUD_AUTH_FAILED", "Runpod rejected the API key.", "Run `moldesk cloud connect runpod` with a valid key.", details);
+            case 402:
+                return cloudError("CLOUD_PAYMENT_REQUIRED", "Runpod reports insufficient credit.", "Add credit to the Runpod account.", details);
+            case 403:
+                return cloudError("CLOUD_FORBIDDEN", "The Runpod API key lacks permission for this action.", "Use a key with Pod and network volume permissions.", details);
+            case 429:
+                return cloudError("CLOUD_RATE_LIMITED", "Runpod rate limit reached.", "Wait and retry.", { ...details, retryAfterSeconds: result.retryAfterSeconds });
+            default:
+                if (result.status === 400 && detail && /no longer any instances|no instances available|not enough free/i.test(detail)) {
+                    return cloudError("CLOUD_CAPACITY_UNAVAILABLE", "Runpod has no capacity for the requested resources.", "Pick another GPU or datacenter, or retry later. Capacity is never reserved by a quote.", details);
+                }
+                if (result.status >= 500) {
+                    return isCreate
+                        ? cloudError("CLOUD_AMBIGUOUS_CREATE", "Runpod failed while creating a resource; it may exist.", "List owned resources and match by name before retrying.", details)
+                        : cloudError("CLOUD_UNAVAILABLE", "Runpod is temporarily unavailable.", "Retry shortly.", details);
+                }
+                return cloudError("CLOUD_REQUEST_REJECTED", `Runpod rejected the request (${result.status}).`, "Check the request parameters.", details);
+        }
+    }
+}

package/dist/cloud/runpod.test.d.ts

@@ -0,0 +1 @@
+export {};