@moldesk/core 0.0.5 → 0.0.6

package/dist/cloud/worker/pipeline.js

@@ -0,0 +1,160 @@
+import fs from "node:fs";
+import path from "node:path";
+import { getAdapter } from "@moldesk/adapters";
+import { effectiveRuntimeSource, listAvailableModels, selectPythonRuntime } from "@moldesk/registry";
+import { PythonRuntimeProvider, cacheAsset, getMoldeskPaths, getSystemReport, materializeAsset, preparePythonEnvironment, } from "@moldesk/runtime";
+import { MoldeskError } from "../../errors.js";
+import { digest } from "../../hash.js";
+import { acquireVolumeLock, cacheRoot, publishCache, verifyCache } from "./cache.js";
+const REMOTE_PLATFORM = "linux-x64";
+function defaultResolve(name) {
+    const manifest = listAvailableModels().find((entry) => entry.name === name);
+    const adapter = getAdapter(name);
+    return manifest && adapter ? { manifest, adapter } : undefined;
+}
+function fail(code, message, remediation) {
+    return new MoldeskError({ code, message, remediation });
+}
+/**
+ * The real worker pipeline: prepare assets, install the Linux runtime, verify (CUDA included), build
+ * the adapter's own command with POSIX paths, and execute. It reuses the same installer, adapter and
+ * execution code as local runs and never accepts a command or manifest from the client; the client's
+ * job only selects a bundled model and must match that manifest's digest exactly.
+ */
+export function createManifestPipeline(options = {}) {
+    const steps = {
+        cacheAsset,
+        materializeAsset,
+        preparePythonEnvironment,
+        getSystemReport,
+        runtime: new PythonRuntimeProvider(),
+        ...options.steps,
+    };
+    const resolve = options.resolveModel ?? defaultResolve;
+    return async (context) => {
+        const { job, jobDir, inputDir, outputDir, signal } = context;
+        const resolved = resolve(job.model);
+        if (!resolved)
+            throw fail("WORKER_MODEL_UNKNOWN", `This worker does not bundle ${job.model}.`, "Use a worker release that matches the client.");
+        const { manifest, adapter } = resolved;
+        if (digest(manifest) !== job.manifestDigest || manifest.modelVersion !== job.modelVersion) {
+            throw fail("WORKER_MANIFEST_MISMATCH", "The worker's bundled manifest differs from the one the job was approved against.", "Use a worker release that matches the client version.");
+        }
+        if (manifest.cloud === undefined)
+            throw fail("WORKER_MODEL_NOT_CLOUD", `${job.model} is not cloud-eligible.`, "Use a model with a cloud block.");
+        const persistentRoot = job.volumeMount ? cacheRoot(job.volumeMount) : undefined;
+        // Raw download objects live on the volume for persistent runs so a rebuild never re-downloads.
+        const paths = getMoldeskPaths({ MOLDESK_HOME: persistentRoot ? path.join(persistentRoot, "home") : path.join(jobDir, "home") });
+        const prepareAssetsInto = async (assetsTarget) => {
+            fs.mkdirSync(assetsTarget, { recursive: true });
+            for (const asset of manifest.assets ?? []) {
+                if (signal.aborted)
+                    throw fail("RUN_CANCELLED", "Cancelled.", "None.");
+                const object = await steps.cacheAsset(asset, paths, {
+                    onProgress: (progress) => context.log("stderr", `[assets] ${progress.message ?? progress.step}\n`),
+                });
+                await steps.materializeAsset(asset, object, assetsTarget);
+            }
+        };
+        // --- CPU stage of a persistent run: assets only. No Python, no CUDA, no verification probe. ---
+        if (job.mode === "prepare-assets") {
+            if (!persistentRoot)
+                throw fail("WORKER_BAD_JOB", "prepare-assets needs a persistent volume.", "Retry with a persistent cache policy.");
+            context.setStage("preparing_assets");
+            const params = { root: persistentRoot, key: job.cacheKey, manifestDigest: job.manifestDigest, layoutRevision: job.assetLayoutRevision };
+            const existing = await verifyCache(params);
+            if (existing.ok) {
+                return { exitCode: 0, runtime: { dependencyFreeze: [], cache: { checksumManifestSha256: existing.metadata.checksumManifestSha256, sizeBytes: existing.metadata.sizeBytes, fileCount: existing.metadata.files.length, reused: true } } };
+            }
+            const lock = acquireVolumeLock({ root: persistentRoot, key: job.cacheKey, owner: job.runId, ttlSeconds: 3600, reclaimStale: job.reclaimStaleLock });
+            try {
+                const staging = path.join(persistentRoot, `${job.cacheKey}.staging-${job.runId}`);
+                fs.rmSync(staging, { recursive: true, force: true });
+                await prepareAssetsInto(path.join(staging, "assets"));
+                const metadata = await publishCache({ ...params, stagingDir: staging });
+                return { exitCode: 0, runtime: { dependencyFreeze: [], cache: { checksumManifestSha256: metadata.checksumManifestSha256, sizeBytes: metadata.sizeBytes, fileCount: metadata.files.length, reused: false } } };
+            }
+            finally {
+                lock.release();
+            }
+        }
+        // --- Assets for an execute job: ephemeral prepares them here; persistent verifies the volume's cache. ---
+        let assetsDir = path.join(jobDir, "assets");
+        const modelDir = path.join(jobDir, "runtime");
+        context.setStage("preparing_assets");
+        if (persistentRoot) {
+            const verified = await verifyCache({ root: persistentRoot, key: job.cacheKey, manifestDigest: job.manifestDigest, layoutRevision: job.assetLayoutRevision });
+            if (!verified.ok)
+                throw Object.assign(fail(verified.code, verified.message, "The cache will be rebuilt on a CPU machine."), {});
+            assetsDir = verified.assetsDir;
+        }
+        else {
+            await prepareAssetsInto(assetsDir);
+        }
+        // --- Linux runtime in an ephemeral environment. ---
+        context.setStage("installing_runtime");
+        const runtime = selectPythonRuntime(manifest.runtimes, REMOTE_PLATFORM);
+        const source = effectiveRuntimeSource(manifest, runtime);
+        if (!source)
+            throw fail("SOURCE_PROVENANCE_MISSING", `${job.model} has no pinned source.`, "Fix the manifest.");
+        const prepared = await steps.preparePythonEnvironment({
+            targetDir: modelDir,
+            repository: source.repository,
+            revision: source.revision,
+            runtime,
+            paths,
+            onProgress: (progress) => context.log("stderr", `[install] ${progress.message ?? progress.step}\n`),
+        });
+        // --- GPU/CUDA verification happens here, after install and before any inference. ---
+        context.setStage("verifying_runtime");
+        const verification = await adapter.verifyInstallation({ manifestName: manifest.name, modelDir, assetsDir, platform: REMOTE_PLATFORM });
+        if (!verification.passed) {
+            throw fail("INSTALL_VERIFICATION_FAILED", `Verification failed for ${manifest.name}: ${verification.output ?? "unknown reason"}.`, "The worker image or GPU is not compatible with this model.");
+        }
+        // --- Build the command with the adapter, from worker-local POSIX paths. ---
+        const primary = job.inputs.find((input) => input.role === "job");
+        if (!primary)
+            throw fail("WORKER_BAD_JOB", "The job has no primary input.", "Retry with a valid job.");
+        const runContext = {
+            manifestName: manifest.name,
+            inputPath: path.join(inputDir, ...primary.relativePath.split("/")),
+            outputDir,
+            modelDir,
+            assetsDir,
+            params: job.params,
+            runtimeExecutable: prepared.executable,
+            platform: REMOTE_PLATFORM,
+        };
+        const command = await adapter.command(runContext);
+        context.setStage("running");
+        const execution = await steps.runtime.execute({
+            executable: command.executable,
+            args: command.args,
+            cwd: jobDir,
+            env: command.env,
+            signal,
+            onStdout: (chunk) => context.log("stdout", chunk),
+            onStderr: (chunk) => context.log("stderr", chunk),
+        });
+        const system = await steps.getSystemReport().catch(() => undefined);
+        const gpu = system?.nvidia.gpus[0];
+        const report = {
+            pythonVersion: prepared.pythonVersion,
+            dependencyFreeze: prepared.lock,
+            sourceRevision: source.revision,
+            ...(gpu
+                ? {
+                    gpu: {
+                        name: gpu.name,
+                        ...(gpu.totalVramBytes !== undefined ? { memoryMb: Math.round(gpu.totalVramBytes / 1024 ** 2) } : {}),
+                        ...(system?.nvidia.driverVersion ? { driverVersion: system.nvidia.driverVersion } : {}),
+                        ...(system?.nvidia.driverCudaVersion ? { cudaVersion: system.nvidia.driverCudaVersion } : {}),
+                    },
+                }
+                : {}),
+            command: { executable: command.executable, args: command.args, cwd: jobDir },
+            installationFingerprint: `python-${digest({ lock: prepared.lock, python: prepared.pythonVersion, revision: source.revision, platform: REMOTE_PLATFORM }).slice(0, 32)}`,
+        };
+        return { exitCode: execution.cancelled ? 130 : execution.exitCode, runtime: report };
+    };
+}

package/dist/cloud/worker/pipeline.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/cloud/worker/pipeline.test.js

@@ -0,0 +1,298 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import { parseManifest } from "@moldesk/registry";
+import { digest } from "../../hash.js";
+import { createManifestPipeline } from "./pipeline.js";
+import { startDeadlineWatchdog } from "./watchdog.js";
+const DIGEST = "a".repeat(64);
+const dirs = [];
+afterEach(() => {
+    for (const dir of dirs.splice(0))
+        fs.rmSync(dir, { recursive: true, force: true });
+});
+function manifest() {
+    return parseManifest({
+        schemaVersion: 1,
+        name: "fake-model",
+        displayName: "Fake",
+        modelVersion: "1.0.0",
+        adapterVersion: "0.1.0",
+        description: "t",
+        category: "other",
+        status: "planned",
+        source: { repository: "https://example.com/r", revision: "b".repeat(40) },
+        runtimes: [{ kind: "python", python: "3.11", installer: "uv", platforms: ["linux-x64"], requirements: [{ name: "torch" }] }],
+        hardware: { platforms: ["linux-x64"] },
+        assets: [{ id: "w", url: "https://example.com/w.bin", sha256: "c".repeat(64), target: "w.bin", sizeBytes: 3 }],
+        input: { formats: [".txt"], required: true },
+        outputs: [{ id: "o", glob: "*.out", required: true }],
+        cloud: {
+            schemaVersion: 1,
+            platform: "linux-x64",
+            images: { gpu: `r.example/g@sha256:${DIGEST}`, cpu: `r.example/c@sha256:${DIGEST}` },
+            minVramGb: 24,
+            minRamGb: 32,
+            ephemeralDiskGb: 40,
+            volumeGb: 40,
+            assetLayoutRevision: "1",
+            cpuAssetPreparation: true,
+        },
+    }, "x/manifest.yaml");
+}
+function setup(overrides = {}) {
+    const jobDir = fs.mkdtempSync(path.join(os.tmpdir(), "moldesk pipeline "));
+    dirs.push(jobDir);
+    const m = manifest();
+    const calls = [];
+    let seenContext;
+    const adapter = {
+        modelName: "fake-model",
+        params: [],
+        async validateInput() { },
+        async installPlan() {
+            return { steps: [] };
+        },
+        async command(context) {
+            calls.push("command");
+            seenContext = context;
+            return { executable: "/job/x/runtime/.venv/bin/fake", args: ["--in", context.inputPath, "--out", context.outputDir], env: { A: "1" } };
+        },
+        async collectOutputs() {
+            return [];
+        },
+        async verifyInstallation() {
+            calls.push("verify");
+            return { passed: overrides.verify ?? true, output: "cuda probe failed" };
+        },
+    };
+    const steps = {
+        async cacheAsset() {
+            calls.push("cacheAsset");
+            return "/obj/w";
+        },
+        async materializeAsset() {
+            calls.push("materializeAsset");
+            return "/assets/w.bin";
+        },
+        async preparePythonEnvironment(request) {
+            calls.push("prepareEnv");
+            expect(request.runtime.platforms).toEqual(["linux-x64"]);
+            return { executable: path.join(request.targetDir, ".venv", "bin", "python"), lock: ["torch==2.2.1"], pythonVersion: "3.11.9", postInstall: [] };
+        },
+        async getSystemReport() {
+            return { nvidia: { available: true, gpus: [{ name: "A40", totalVramBytes: 48 * 1024 ** 3 }], driverVersion: "550", driverCudaVersion: "12.4" } };
+        },
+        runtime: {
+            kind: "python",
+            async inspect() {
+                return { kind: "python", available: true };
+            },
+            async prepare() {
+                throw new Error("unused");
+            },
+            async execute(request) {
+                calls.push("execute");
+                request.onStdout?.("model output\n");
+                return { exitCode: overrides.exitCode ?? 0, stdoutPath: "", stderrPath: "" };
+            },
+            async remove() { },
+        },
+    };
+    const stages = [];
+    const logs = [];
+    const job = (patch = {}) => ({
+        version: 1,
+        runId: "run-1",
+        model: "fake-model",
+        modelVersion: "1.0.0",
+        manifestDigest: digest(m),
+        cachePolicy: "ephemeral",
+        gpuId: "g",
+        datacenterId: "dc",
+        images: m.cloud.images,
+        containerDiskGb: 40,
+        volumeGb: 40,
+        assetLayoutRevision: "1",
+        cpuAssetPreparation: true,
+        params: { seed: 1 },
+        inputs: [{ role: "job", relativePath: "in.txt", sha256: "d".repeat(64), sizeBytes: 3 }],
+        deadlineAt: "2099-01-01T00:00:00.000Z",
+        cacheKey: "k".repeat(64),
+        mode: "execute",
+        ...patch,
+    });
+    const run = (patch = {}, signal = new AbortController().signal) => createManifestPipeline({ resolveModel: (name) => (name === "fake-model" ? { manifest: m, adapter } : undefined), steps })({
+        job: job(patch),
+        jobDir,
+        inputDir: path.join(jobDir, "input"),
+        outputDir: path.join(jobDir, "output"),
+        signal,
+        setStage: (stage) => void stages.push(stage),
+        log: (_stream, text) => void logs.push(text),
+    });
+    return { run, calls, stages, logs, jobDir, context: () => seenContext };
+}
+describe("manifest pipeline", () => {
+    it("runs assets, runtime, verification and execution in order and reports what ran", async () => {
+        const box = setup();
+        const result = await box.run();
+        expect(box.calls).toEqual(["cacheAsset", "materializeAsset", "prepareEnv", "verify", "command", "execute"]);
+        expect(box.stages).toEqual(["preparing_assets", "installing_runtime", "verifying_runtime", "running"]);
+        expect(result.exitCode).toBe(0);
+        expect(result.runtime).toMatchObject({
+            pythonVersion: "3.11.9",
+            dependencyFreeze: ["torch==2.2.1"],
+            sourceRevision: "b".repeat(40),
+            gpu: { name: "A40", memoryMb: 49152, driverVersion: "550", cudaVersion: "12.4" },
+            command: { executable: "/job/x/runtime/.venv/bin/fake", cwd: box.jobDir },
+        });
+        expect(result.runtime.installationFingerprint).toMatch(/^python-[0-9a-f]{32}$/);
+        expect(box.logs.join("")).toContain("model output");
+    });
+    it("builds the adapter command from worker-local paths and the approved params only", async () => {
+        const box = setup();
+        await box.run();
+        const context = box.context();
+        expect(context.platform).toBe("linux-x64");
+        expect(context.params).toEqual({ seed: 1 });
+        expect(context.inputPath).toBe(path.join(box.jobDir, "input", "in.txt"));
+        expect(context.outputDir).toBe(path.join(box.jobDir, "output"));
+        expect(context.runtimeExecutable).toBe(path.join(box.jobDir, "runtime", ".venv", "bin", "python"));
+    });
+    it("refuses a job approved against a different manifest, an unknown model or a non-cloud model", async () => {
+        await expect(setup().run({ manifestDigest: "0".repeat(64) })).rejects.toMatchObject({ code: "WORKER_MANIFEST_MISMATCH" });
+        await expect(setup().run({ model: "other" })).rejects.toMatchObject({ code: "WORKER_MODEL_UNKNOWN" });
+    });
+    it("stops at failed verification before any inference", async () => {
+        const box = setup({ verify: false });
+        await expect(box.run()).rejects.toMatchObject({ code: "INSTALL_VERIFICATION_FAILED" });
+        expect(box.calls).not.toContain("execute");
+        expect(box.calls).not.toContain("command");
+    });
+    it("returns the model's nonzero exit code unchanged", async () => {
+        expect((await setup({ exitCode: 3 }).run()).exitCode).toBe(3);
+    });
+    it("does not start when already cancelled", async () => {
+        const controller = new AbortController();
+        controller.abort();
+        const box = setup();
+        await expect(box.run({}, controller.signal)).rejects.toMatchObject({ code: "RUN_CANCELLED" });
+        expect(box.calls).toEqual([]);
+    });
+});
+describe("persistent cache modes", () => {
+    const key = "k".repeat(64);
+    function volume() {
+        const mount = fs.mkdtempSync(path.join(os.tmpdir(), "moldesk volume "));
+        dirs.push(mount);
+        return mount;
+    }
+    const persistent = (mount, mode) => ({ cachePolicy: "persistent", mode, volumeMount: mount, cacheKey: key });
+    it("prepare-assets prepares and publishes the cache without installing Python or probing CUDA", async () => {
+        const mount = volume();
+        const box = setup();
+        const result = await box.run(persistent(mount, "prepare-assets"));
+        expect(box.calls).toEqual(["cacheAsset", "materializeAsset"]);
+        expect(box.stages).toEqual(["preparing_assets"]);
+        expect(result.runtime?.cache).toMatchObject({ reused: false, fileCount: 0 });
+        expect(fs.existsSync(path.join(mount, "moldesk-cache", key, "ready.json"))).toBe(true);
+        expect(fs.readdirSync(path.join(mount, "moldesk-cache")).filter((n) => n.includes("staging") || n.endsWith(".lock"))).toEqual([]);
+    });
+    it("a second prepare-assets on a valid cache downloads nothing", async () => {
+        const mount = volume();
+        const first = setup();
+        await first.run(persistent(mount, "prepare-assets"));
+        const second = setup();
+        const result = await second.run(persistent(mount, "prepare-assets"));
+        expect(second.calls).toEqual([]);
+        expect(result.runtime?.cache?.reused).toBe(true);
+    });
+    it("an execute job on a warm volume skips asset preparation and uses the verified assets directory", async () => {
+        const mount = volume();
+        // Real materialize so the cache has content to hash.
+        const prepare = setup();
+        await prepare.run(persistent(mount, "prepare-assets"));
+        const box = setup();
+        await box.run(persistent(mount, "execute"));
+        expect(box.calls).toEqual(["prepareEnv", "verify", "command", "execute"]);
+        expect(box.context().assetsDir).toBe(path.join(mount, "moldesk-cache", key, "assets"));
+    });
+    it("an execute job fails fast, before installing anything, when the volume has no valid cache", async () => {
+        const mount = volume();
+        const missing = setup();
+        await expect(missing.run(persistent(mount, "execute"))).rejects.toMatchObject({ code: "CLOUD_CACHE_MISSING" });
+        expect(missing.calls).toEqual([]);
+        const prepared = setup();
+        await prepared.run(persistent(mount, "prepare-assets"));
+        fs.writeFileSync(path.join(mount, "moldesk-cache", key, "assets", "injected.bin"), "x");
+        const damaged = setup();
+        await expect(damaged.run(persistent(mount, "execute"))).rejects.toMatchObject({ code: "CLOUD_CACHE_INVALID" });
+        expect(damaged.calls).toEqual([]);
+    });
+    it("prepare-assets without a volume is a bad job", async () => {
+        await expect(setup().run({ mode: "prepare-assets" })).rejects.toMatchObject({ code: "WORKER_BAD_JOB" });
+    });
+});
+describe("deadline watchdog", () => {
+    function fakeEnv(responses) {
+        const calls = [];
+        const timers = [];
+        const logs = [];
+        const fetchFn = (async (url, init) => {
+            calls.push({ url, auth: new Headers(init.headers).get("authorization"), body: JSON.parse(String(init.body)) });
+            const next = responses.shift() ?? 204;
+            if (next instanceof Error)
+                throw next;
+            return new Response(null, { status: next });
+        });
+        return { calls, timers, logs, fetchFn };
+    }
+    it("terminates its own pod at the deadline with the pod-scoped key", async () => {
+        const env = fakeEnv([204]);
+        const dog = startDeadlineWatchdog({
+            deadlineAt: "2026-10-13T13:00:00.000Z",
+            podId: "pod9",
+            apiKey: "pod-scoped-key",
+            fetch: env.fetchFn,
+            now: () => Date.parse("2026-10-13T12:00:00.000Z"),
+            setTimer: (run, ms) => void env.timers.push({ ms, run }),
+            log: (m) => env.logs.push(m),
+        });
+        expect(env.timers[0].ms).toBe(3_600_000);
+        env.timers[0].run();
+        await dog.done;
+        expect(env.calls).toEqual([{ url: "https://api.runpod.io/v2/pods/pod9/action", auth: "Bearer pod-scoped-key", body: { action: "terminate" } }]);
+        expect(env.logs.join(" ")).not.toContain("pod-scoped-key");
+    });
+    it("fires immediately when the deadline already passed and retries until accepted", async () => {
+        const env = fakeEnv([503, new Error("network"), 404]);
+        const dog = startDeadlineWatchdog({
+            deadlineAt: "2026-10-13T11:00:00.000Z",
+            podId: "pod9",
+            apiKey: "k",
+            fetch: env.fetchFn,
+            now: () => Date.parse("2026-10-13T12:00:00.000Z"),
+            retryMs: 30_000,
+            setTimer: (run, ms) => void env.timers.push({ ms, run }),
+        });
+        expect(env.timers[0].ms).toBe(0);
+        env.timers[0].run();
+        await new Promise((resolve) => setTimeout(resolve, 5));
+        expect(env.timers[1].ms).toBe(30_000);
+        env.timers[1].run();
+        await new Promise((resolve) => setTimeout(resolve, 5));
+        env.timers[2].run();
+        await dog.done;
+        expect(env.calls).toHaveLength(3);
+    });
+    it("does nothing once stopped", async () => {
+        const env = fakeEnv([204]);
+        const dog = startDeadlineWatchdog({ deadlineAt: "2000-01-01T00:00:00.000Z", podId: "p", apiKey: "k", fetch: env.fetchFn, now: () => 0, setTimer: (run, ms) => void env.timers.push({ ms, run }) });
+        dog.stop();
+        env.timers[0].run();
+        await new Promise((resolve) => setTimeout(resolve, 5));
+        expect(env.calls).toHaveLength(0);
+    });
+});

package/dist/cloud/worker/server.d.ts

@@ -0,0 +1,38 @@
+import http from "node:http";
+import { type JobStage, type RemoteRuntimeReport, type WireJob } from "@moldesk/runtime";
+export declare const WORKER_VERSION = "1";
+export declare const DEFAULT_MAX_LOG_BYTES: number;
+export interface PipelineContext {
+    job: WireJob;
+    jobDir: string;
+    inputDir: string;
+    outputDir: string;
+    signal: AbortSignal;
+    setStage(stage: JobStage): void;
+    log(stream: "stdout" | "stderr", text: string): void;
+}
+export interface PipelineResult {
+    exitCode: number;
+    runtime?: RemoteRuntimeReport;
+}
+/** The real work (assets, runtime, command, execution). Injected so the HTTP surface is testable on its own. */
+export type WorkerPipeline = (context: PipelineContext) => Promise<PipelineResult>;
+export interface WorkerServerOptions {
+    /** Per-job bearer token; every route, including health, requires it. */
+    token: string;
+    /** Jobs live under `<rootDir>/<runId>/{input,output}`. */
+    rootDir: string;
+    pipeline: WorkerPipeline;
+    /** Extra secrets to scrub from logs (the token itself is always scrubbed). */
+    secrets?: string[];
+    /** In-memory log tail limit; older chunks are evicted and reported as lost to clients that missed them. */
+    maxLogBytes?: number;
+}
+export interface RunningWorker {
+    server: http.Server;
+    listen(port: number, host?: string): Promise<number>;
+    close(): Promise<void>;
+    /** Terminal state reached; used by tests and the idle watchdog. */
+    isFinished(): boolean;
+}
+export declare function createWorkerServer(options: WorkerServerOptions): RunningWorker;