@nodetool-ai/compute 0.8.1 → 0.8.2

package/dist/providers/verda-api.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"verda-api.js","sourceRoot":"","sources":["../../src/providers/verda-api.ts"],"names":[],"mappings":"AAAA,8EAA8E;AAC9E,EAAE;AACF,2EAA2E;AAC3E,gDAAgD;AAChD,EAAE;AACF,+EAA+E;AAC/E,6DAA6D;AAC7D,gFAAgF;AAChF,6EAA6E;AAC7E,eAAe;AACf,8EAA8E;AAC9E,+EAA+E;AAC/E,4EAA4E;AAC5E,8BAA8B;AAC9B,EAAE;AACF,gFAAgF;AAChF,4DAA4D;AAE5D;+EAC+E;AAC/E,MAAM,kBAAkB,GAAG,uBAAuB,CAAC;AAEnD,kFAAkF;AAClF,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAErC,yDAAyD;AACzD,MAAM,WAAW,GAAG,CAAC,CAAC;AAEtB,mEAAmE;AACnE,MAAM,sBAAsB,GAAG,KAAK,CAAC;AAgBrC;;;;GAIG;AACH,MAAM,OAAO,aAAc,SAAQ,KAAK;IAG3B,MAAM;IACN,IAAI;IAHf,YACE,OAAe,EACN,MAAc,EACd,IAAY;QAErB,KAAK,CAAC,OAAO,CAAC,CAAC;sBAHN,MAAM;oBACN,IAAI;QAGb,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;IAC9B,CAAC;IAED,yEAAyE;IACzE,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC;IAC7B,CAAC;IAED,gFAAgF;IAChF,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC;IACpD,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,EAAU,EAAE,KAAa;IACpD,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CACb,gBAAgB,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,yBAAyB;YAClE,6CAA6C,CAChD,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,0EAA0E;AAC1E,SAAS,YAAY,CAAC,MAAqB;IACzC,IAAI,CAAC,MAAM;QAAE,OAAO,sBAAsB,CAAC;IAC3C,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAC/B,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QAC7C,OAAO,OAAO,GAAG,KAAK,CAAC;IACzB,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,sBAAsB,CAAC;AAChC,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE,CAC1C,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEpD;;;;GAIG;AACH,MAAM,OAAO,cAAc;IAON,WAAW;IACX,OAAO;IAPlB,WAAW,GAAkB,IAAI,CAAC;IAClC,SAAS,GAAG,CAAC,CAAC;IACtB,wEAAwE;IAChE,OAAO,GAA2B,IAAI,CAAC;IAE/C,YACmB,WAA6B,EAC7B,OAAO,GAAW,kBAAkB;2BADpC,WAAW;uBACX,OAAO;QAExB,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,YAAY,EAAE,CAAC;YACvD,MAAM,IAAI,KAAK,CACb,6DAA6D,CAC9D,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO,CACX,MAAmB,EACnB,IAAY,EACZ,IAA8B;QAE9B,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,KAAK,IAAI,OAAO,GAAG,CAAC,GAAI,OAAO,EAAE,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YAE5D,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACzC,oEAAoE;gBACpE,QAAQ,GAAG,IAAI,CAAC;gBAChB,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,SAAS;YACX,CAAC;YAED,IACE,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC;gBACpD,OAAO,GAAG,WAAW,EACrB,CAAC;gBACD,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;gBAC/D,SAAS;YACX,CAAC;YAED,OAAO,IAAI,CAAC,MAAM,CAAI,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,UAAU;QACR,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;IACrB,CAAC;IAED,2EAA2E;IAEnE,KAAK,CAAC,IAAI,CAChB,MAAmB,EACnB,IAAY,EACZ,KAAa,EACb,IAA8B;QAE9B,MAAM,IAAI,GAAgB;YACxB,MAAM;YACN,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,KAAK,EAAE;gBAChC,cAAc,EAAE,kBAAkB;gBAClC,MAAM,EAAE,kBAAkB;aAC3B;YACD,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;SACpC,CAAC;QACF,IAAI,IAAI,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YAC7B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;IAC/C,CAAC;IAEO,KAAK,CAAC,MAAM,CAClB,QAAkB,EAClB,MAAmB,EACnB,IAAY;QAEZ,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,uEAAuE;YACvE,mEAAmE;YACnE,MAAM,IAAI,aAAa,CACrB,SAAS,MAAM,IAAI,IAAI,YAAY,QAAQ,CAAC,MAAM,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAC5E,QAAQ,CAAC,MAAM,EACf,IAAI,CACL,CAAC;QACJ,CAAC;QACD,4EAA4E;QAC5E,IAAI,MAAM,GAAY,IAAI,CAAC;QAC3B,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;YAAC,MAAM,CAAC;gBACP,sEAAsE;gBACtE,MAAM,GAAG,IAAI,CAAC;YAChB,CAAC;QACH,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,MAAW,EAAE,CAAC;IACxD,CAAC;IAED,yEAAyE;IACjE,KAAK;QACX,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,GAAG,qBAAqB,EAAE,CAAC;YAC5E,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC3C,CAAC;QACD,2EAA2E;QAC3E,kEAAkE;QAClE,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YAC3C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,kBAAkB,EAAE;YAC9D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,EAAE,kBAAkB,EAAE;YAC3E,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,UAAU,EAAE,oBAAoB;gBAChC,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,QAAQ;gBACpC,aAAa,EAAE,IAAI,CAAC,WAAW,CAAC,YAAY;aAC7C,CAAC;YACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,mEAAmE;YACnE,oCAAoC;YACpC,MAAM,IAAI,aAAa,CACrB,gCAAgC,QAAQ,CAAC,MAAM,WAAW;gBACxD,0CAA0C,EAC5C,QAAQ,CAAC,MAAM,EACf,kBAAkB,CACnB,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAGnC,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC,YAAY,CAAC;QACtC,yEAAyE;QACzE,MAAM,GAAG,GAAG,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC;QAC1E,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,KAAK,CAAC;QAC1C,OAAO,KAAK,CAAC,YAAY,CAAC;IAC5B,CAAC;CACF"}

package/dist/providers/verda.d.ts

@@ -0,0 +1,75 @@
+import { type ProviderInstance, type ProvisionResult, type WorkerProvider, type WorkerSpec, type WorkerStatus } from "./types.js";
+import { VerdaApiClient, type VerdaCredentials } from "./verda-api.js";
+export declare class VerdaProvider implements WorkerProvider {
+    private readonly api;
+    constructor(credentials: VerdaCredentials, api?: VerdaApiClient);
+    provision(spec: WorkerSpec): Promise<ProvisionResult>;
+    status(ref: string): Promise<WorkerStatus>;
+    /**
+     * Pause: release the GPU compute, retain the OS volume and its model cache.
+     *
+     * This DELETES the Verda instance. A `shutdown` would keep billing the full
+     * instance rate (Verda removed hibernate for the same reason), so it is not
+     * the operation the cost guard needs.
+     */
+    stop(ref: string): Promise<void>;
+    /**
+     * Resume: create a NEW machine that boots from the retained OS volume.
+     *
+     * Returns a new provider ref (the instance id changed). Verda can refuse for
+     * lack of capacity, which surfaces as a distinct error rather than a retry.
+     */
+    resume(ref: string): Promise<ProvisionResult>;
+    /**
+     * Destroy the worker AND its OS volume — the real teardown that ends every
+     * charge. The volume is deleted PERMANENTLY: a trashed volume still occupies
+     * the storage quota and can be restored (at a charge for the deleted
+     * interval), which is not what "terminate" promises. The model cache is gone.
+     */
+    terminate(ref: string): Promise<void>;
+    list(): Promise<ProviderInstance[]>;
+    private resultFor;
+    /**
+     * Pick the guest OS image. An explicit `osImage` is validated against the
+     * live catalog (a typo would otherwise fail deep inside provisioning); with
+     * none set, prefer an image that ships CUDA and Docker, which the bootstrap
+     * script needs.
+     */
+    private resolveOsImage;
+    /**
+     * Resolve the required `location_code` against LIVE availability — a catalog
+     * entry is not a reservation, so this is re-read immediately before each create.
+     * An explicit region is verified rather than trusted.
+     */
+    private resolveLocation;
+    /** Register the spec's public key, reusing an identical existing key. */
+    private ensureSshKey;
+    /**
+     * Create the startup script that turns a bare VM into a NodeTool worker.
+     *
+     * The script is idempotent: Verda runs it on every boot, including after a
+     * resume, and it must not fail on a machine that already has the container.
+     */
+    private createStartupScript;
+    /** `POST /v1/instances`, returning the accepted instance id. */
+    private createInstance;
+    /**
+     * Delete an instance together with every volume attached to it, permanently.
+     *
+     * The volume ids are read back from the instance and NAMED explicitly: an
+     * empty `volume_ids` retains them all, and omitting the field deletes only
+     * the OS volume while detaching the rest — both leave billable storage
+     * behind. Returns false when the instance no longer exists.
+     */
+    private destroyWithVolumes;
+    /**
+     * `PUT /v1/instances` delete. `volumeIds: []` retains every attached volume;
+     * a non-empty list deletes exactly those.
+     */
+    private deleteInstance;
+    /** `GET /v1/instances/{id}`, or null when it no longer exists. */
+    private getInstance;
+    /** Poll until the instance reports `running`, or fail on a terminal state. */
+    private waitForReady;
+}
+//# sourceMappingURL=verda.d.ts.map

package/dist/providers/verda.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"verda.d.ts","sourceRoot":"","sources":["../../src/providers/verda.ts"],"names":[],"mappings":"AA2BA,OAAO,EAIL,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,UAAU,EACf,KAAK,YAAY,EAClB,MAAM,YAAY,CAAC;AACpB,OAAO,EAEL,cAAc,EAEd,KAAK,gBAAgB,EACtB,MAAM,gBAAgB,CAAC;AAiJxB,qBAAa,aAAc,YAAW,cAAc;IAClD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAiB;IAErC,YAAY,WAAW,EAAE,gBAAgB,EAAE,GAAG,CAAC,EAAE,cAAc,EAE9D;IAEK,SAAS,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,eAAe,CAAC,CA2E1D;IAEK,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAa/C;IAED;;;;;;OAMG;IACG,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAerC;IAED;;;;;OAKG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAgDlD;IAED;;;;;OAKG;IACG,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB1C;IAEK,IAAI,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAUxC;IAID,OAAO,CAAC,SAAS;IASjB;;;;;OAKG;YACW,cAAc;IAwC5B;;;;OAIG;YACW,eAAe;IAkC7B,yEAAyE;YAC3D,YAAY;IAkB1B;;;;;OAKG;YACW,mBAAmB;IA0CjC,gEAAgE;YAClD,cAAc;IAmB5B;;;;;;;OAOG;YACW,kBAAkB;IAqBhC;;;OAGG;YACW,cAAc;IA6B5B,kEAAkE;YACpD,WAAW;IAoBzB,8EAA8E;YAChE,YAAY;CA2B3B"}

package/dist/providers/verda.js

@@ -0,0 +1,560 @@
+// Verda-backed `WorkerProvider` (https://api.verda.com/v1).
+//
+// Verda rents VIRTUAL MACHINES, not containers. The NodeTool worker image is
+// something the guest RUNS, so provisioning is two-layered: pick a CUDA+Docker
+// OS image for the machine, then hand Verda a startup script that runs the
+// worker image on it. `WorkerSpec.osImage` names the former, `WorkerSpec.image`
+// the latter.
+//
+// Lifecycle deliberately does NOT mirror RunPod/Vast, because Verda's does not:
+//
+//   * Verda bills a SHUT-DOWN instance at the full rate, and removed its
+//     hibernate action for exactly that reason. Mapping `stop` to `shutdown`
+//     would make the idle reaper stop workers while saving nothing, so `stop`
+//     instead DELETES the instance while retaining its OS volume — the
+//     "release compute, keep disks" operation. The model cache survives; the
+//     GPU bill ends.
+//   * `resume` therefore CREATES A NEW MACHINE from the retained OS volume. It
+//     returns a new `providerRef`, which the manager persists.
+//   * `terminate` deletes compute AND permanently deletes the OS volume, so no
+//     billable storage is left behind in the 96-hour trash window.
+//
+// Because `stop` destroys the instance record, everything `resume` needs to
+// rebuild the machine is encoded in the provider ref (see `encodeRef`).
+//
+// Credentials are injected by the `WorkerManager`; this provider never reads
+// `process.env`.
+import { DEFAULT_VOLUME_GB, WORKER_HF_HOME, WORKER_VOLUME_MOUNT, } from "./types.js";
+import { assertSafeId, VerdaApiClient, VerdaApiError, } from "./verda-api.js";
+/** Port the worker serves on, published from the container to the VM. */
+const WORKER_PORT = 7777;
+/** Pay-as-you-go only. Long-term contracts are prepaid and are never chosen
+ * on the user's behalf; spot instances can be discontinued mid-workflow. */
+const CONTRACT = "PAY_AS_YOU_GO";
+/** Name of the container the startup script runs, used for idempotent reruns. */
+const CONTAINER_NAME = "nodetool-worker";
+/**
+ * Encode a create plan as an opaque provider handle.
+ *
+ * `provision` and `list` both build this from the SAME `GET` representation, so
+ * a live instance encodes to the identical string in both paths and the
+ * manager's orphan reconcile matches it by equality.
+ */
+function encodeRef(ref) {
+    const ordered = {
+        i: ref.i,
+        v: ref.v,
+        t: ref.t,
+        l: ref.l,
+        s: ref.s,
+        k: ref.k,
+        h: ref.h,
+    };
+    return `verda:${Buffer.from(JSON.stringify(ordered)).toString("base64url")}`;
+}
+/** Decode a provider handle produced by `encodeRef`. */
+function decodeRef(ref) {
+    if (!ref.startsWith("verda:")) {
+        throw new Error(`Not a Verda worker handle: ${JSON.stringify(ref)}`);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(Buffer.from(ref.slice("verda:".length), "base64url").toString("utf8"));
+    }
+    catch {
+        throw new Error(`Corrupt Verda worker handle: ${JSON.stringify(ref)}`);
+    }
+    const plan = parsed;
+    if (typeof plan?.v !== "string" || typeof plan?.t !== "string") {
+        throw new Error(`Incomplete Verda worker handle: ${JSON.stringify(ref)}`);
+    }
+    return plan;
+}
+/** Build the handle for an instance as the API currently reports it. */
+function refFromInstance(instance) {
+    return encodeRef({
+        i: instance.id ?? "",
+        v: instance.os_volume_id ?? "",
+        t: instance.instance_type ?? "",
+        l: instance.location ?? "",
+        s: instance.startup_script_id || undefined,
+        k: instance.ssh_key_ids?.length ? instance.ssh_key_ids : undefined,
+        h: instance.hostname || undefined,
+    });
+}
+/**
+ * Map Verda's instance status onto the shared `WorkerStatus`.
+ *
+ * Unknown states map to `error`, not to a hopeful `provisioning`: a state this
+ * code does not recognise must not read as progress to the poll loop.
+ */
+function mapInstanceStatus(status) {
+    switch (status) {
+        case "running":
+            return "running";
+        case "provisioning":
+        case "ordered":
+        case "new":
+        case "validating":
+        case "deploying":
+            return "provisioning";
+        case "offline":
+            // Shut down but still allocated — and still billed. Not our paused state.
+            return "stopped";
+        case "deleting":
+            return "stopping";
+        case "discontinued":
+        case "notfound":
+            return "terminated";
+        default:
+            // error, no_capacity, installation_failed, unknown, and anything new.
+            return "error";
+    }
+}
+/** Single-quote a value for safe interpolation into the bootstrap script. */
+function shellQuote(value) {
+    return `'${value.replace(/'/g, `'\\''`)}'`;
+}
+export class VerdaProvider {
+    api;
+    constructor(credentials, api) {
+        this.api = api ?? new VerdaApiClient(credentials);
+    }
+    async provision(spec) {
+        // The startup script publishes 7777 with Docker, and Verda's own guide
+        // warns that a UFW rule does NOT block a Docker-published port. The worker
+        // is therefore reachable from the internet the moment it boots, so refuse
+        // to launch one that accepts unauthenticated requests.
+        if (!spec.token) {
+            throw new Error("Verda workers publish port 7777 on a public IP, so a bearer token is " +
+                "required. Use a profile with token_policy 'generate' or 'fixed'.");
+        }
+        const instanceType = spec.gpu?.trim();
+        if (!instanceType) {
+            throw new Error("Verda requires an instance type (e.g. \"1H100.80S.30V\"). Set the " +
+                "profile's gpu field to a type from GET /v1/instance-types.");
+        }
+        const [osImage, locationCode] = await Promise.all([
+            this.resolveOsImage(spec.osImage),
+            this.resolveLocation(instanceType, spec.region),
+        ]);
+        const sshKeyIds = spec.sshPublicKey
+            ? [await this.ensureSshKey(spec.name, spec.sshPublicKey)]
+            : [];
+        const scriptId = await this.createStartupScript(spec);
+        const hostname = sanitizeHostname(spec.name);
+        const create = {
+            instance_type: instanceType,
+            image: osImage,
+            location_code: locationCode,
+            hostname,
+            description: `NodeTool worker (${spec.name})`,
+            contract: CONTRACT,
+            startup_script_id: scriptId,
+            os_volume: {
+                name: `${hostname}-os`,
+                size: spec.disk ?? DEFAULT_VOLUME_GB,
+            },
+        };
+        if (sshKeyIds.length > 0) {
+            create.ssh_key_ids = sshKeyIds;
+        }
+        // `POST /v1/instances` answers 202 (accepted) — the machine does not exist
+        // yet. Persist nothing until the id is known, then treat every later
+        // failure as owning a billing resource that must be torn down.
+        const instanceId = await this.createInstance(create);
+        try {
+            const instance = await this.waitForReady(instanceId);
+            if (!instance.ip) {
+                throw new Error(`Verda instance ${instanceId} became running without an IP address`);
+            }
+            return {
+                providerRef: refFromInstance(instance),
+                wsUrl: `ws://${instance.ip}:${WORKER_PORT}`,
+                token: spec.token,
+                status: "running",
+                costUsd: instance.price_per_hour,
+            };
+        }
+        catch (err) {
+            // Destroy compute AND the just-created OS volume: nothing here is worth
+            // retaining, and an orphan would bill until a reconcile scan found it.
+            // The volumes must be NAMED — an empty list retains every one of them.
+            await this.destroyWithVolumes(instanceId).catch(() => {
+                // Best-effort teardown; the provision failure below is the real error.
+            });
+            throw err;
+        }
+    }
+    async status(ref) {
+        const plan = decodeRef(ref);
+        if (!plan.i) {
+            // Compute was released by `stop`; only the retained volume remains.
+            return "stopped";
+        }
+        const instance = await this.getInstance(plan.i);
+        if (!instance) {
+            // The machine is gone but the OS volume was retained — this is the state
+            // `stop` leaves behind, and it is resumable.
+            return "stopped";
+        }
+        return mapInstanceStatus(instance.status);
+    }
+    /**
+     * Pause: release the GPU compute, retain the OS volume and its model cache.
+     *
+     * This DELETES the Verda instance. A `shutdown` would keep billing the full
+     * instance rate (Verda removed hibernate for the same reason), so it is not
+     * the operation the cost guard needs.
+     */
+    async stop(ref) {
+        const plan = decodeRef(ref);
+        if (!plan.i) {
+            return;
+        }
+        if (!plan.v) {
+            throw new Error("Refusing to pause a Verda worker with no recorded OS volume: " +
+                "releasing its compute would destroy the machine with nothing to " +
+                "resume from. Terminate it instead.");
+        }
+        // `volume_ids: []` is the documented "retain ALL attached volumes" form.
+        // Omitting the field deletes the OS volume instead, so it is always sent.
+        await this.deleteInstance(plan.i, { volumeIds: [], permanent: false });
+    }
+    /**
+     * Resume: create a NEW machine that boots from the retained OS volume.
+     *
+     * Returns a new provider ref (the instance id changed). Verda can refuse for
+     * lack of capacity, which surfaces as a distinct error rather than a retry.
+     */
+    async resume(ref) {
+        const plan = decodeRef(ref);
+        if (plan.i) {
+            const existing = await this.getInstance(plan.i);
+            if (existing && mapInstanceStatus(existing.status) === "running") {
+                // Already live (a stop that failed after the caller gave up, say).
+                return this.resultFor(existing);
+            }
+        }
+        if (!plan.v) {
+            throw new Error("This Verda worker has no retained OS volume to resume from.");
+        }
+        const create = {
+            instance_type: plan.t,
+            // A customized OS volume id stands in for an image type: the machine
+            // boots the retained disk, model cache and all.
+            image: assertSafeId(plan.v, "volume id"),
+            location_code: plan.l,
+            hostname: plan.h ?? CONTAINER_NAME,
+            description: "NodeTool worker (resumed)",
+            contract: CONTRACT,
+        };
+        if (plan.s)
+            create.startup_script_id = plan.s;
+        if (plan.k?.length)
+            create.ssh_key_ids = plan.k;
+        const instanceId = await this.createInstance(create);
+        try {
+            const instance = await this.waitForReady(instanceId);
+            if (!instance.ip) {
+                throw new Error(`Verda instance ${instanceId} resumed without an IP address`);
+            }
+            return this.resultFor(instance);
+        }
+        catch (err) {
+            // Release the failed machine but KEEP the volume: it is the user's model
+            // cache and predates this resume attempt.
+            await this.deleteInstance(instanceId, {
+                volumeIds: [],
+                permanent: false,
+            }).catch(() => {
+                // Best-effort; surface the resume failure below.
+            });
+            throw err;
+        }
+    }
+    /**
+     * Destroy the worker AND its OS volume — the real teardown that ends every
+     * charge. The volume is deleted PERMANENTLY: a trashed volume still occupies
+     * the storage quota and can be restored (at a charge for the deleted
+     * interval), which is not what "terminate" promises. The model cache is gone.
+     */
+    async terminate(ref) {
+        const plan = decodeRef(ref);
+        if (plan.i) {
+            const destroyed = await this.destroyWithVolumes(plan.i, plan.v);
+            if (destroyed) {
+                return;
+            }
+        }
+        // Compute is already released; the retained volume is what still bills.
+        if (plan.v) {
+            await this.api.request("PUT", "/v1/volumes", {
+                action: "delete",
+                id: assertSafeId(plan.v, "volume id"),
+                is_permanent: true,
+            });
+        }
+    }
+    async list() {
+        const { body } = await this.api.request("GET", "/v1/instances");
+        const instances = Array.isArray(body) ? body : [];
+        return instances.map((instance) => ({
+            providerRef: refFromInstance(instance),
+            status: mapInstanceStatus(instance.status),
+        }));
+    }
+    // --- Internals ----------------------------------------------------------
+    resultFor(instance) {
+        return {
+            providerRef: refFromInstance(instance),
+            wsUrl: `ws://${instance.ip}:${WORKER_PORT}`,
+            status: "running",
+            costUsd: instance.price_per_hour,
+        };
+    }
+    /**
+     * Pick the guest OS image. An explicit `osImage` is validated against the
+     * live catalog (a typo would otherwise fail deep inside provisioning); with
+     * none set, prefer an image that ships CUDA and Docker, which the bootstrap
+     * script needs.
+     */
+    async resolveOsImage(requested) {
+        const { body } = await this.api.request("GET", "/v1/images");
+        const images = Array.isArray(body) ? body : [];
+        const types = images
+            .map((image) => image.image_type)
+            .filter((type) => Boolean(type));
+        if (requested) {
+            if (!types.includes(requested)) {
+                throw new Error(`Verda OS image "${requested}" is not in the catalog. ` +
+                    `Available: ${types.join(", ")}`);
+            }
+            return requested;
+        }
+        const describes = (image) => [image.image_type, image.name, ...(image.details ?? [])]
+            .filter(Boolean)
+            .join(" ")
+            .toLowerCase();
+        const preferred = images.find((image) => {
+            const text = describes(image);
+            return text.includes("cuda") && text.includes("docker");
+        });
+        const fallback = images.find((image) => describes(image).includes("docker"));
+        const chosen = (preferred ?? fallback)?.image_type;
+        if (!chosen) {
+            throw new Error("No Verda OS image with CUDA and Docker found. Set the profile's " +
+                `osImage explicitly. Available: ${types.join(", ")}`);
+        }
+        return chosen;
+    }
+    /**
+     * Resolve the required `location_code` against LIVE availability — a catalog
+     * entry is not a reservation, so this is re-read immediately before each create.
+     * An explicit region is verified rather than trusted.
+     */
+    async resolveLocation(instanceType, requested) {
+        const { body } = await this.api.request("GET", "/v1/instance-availability");
+        const rows = Array.isArray(body) ? body : [];
+        const withType = rows.filter((row) => (row.availabilities ?? []).includes(instanceType));
+        if (requested) {
+            const match = withType.find((row) => row.location_code === requested);
+            if (!match) {
+                throw new Error(`Verda instance type "${instanceType}" is not available in ` +
+                    `"${requested}". Available in: ` +
+                    `${withType.map((r) => r.location_code).join(", ") || "no location"}`);
+            }
+            return requested;
+        }
+        const first = withType[0]?.location_code;
+        if (!first) {
+            throw new Error(`No Verda location currently has capacity for instance type ` +
+                `"${instanceType}".`);
+        }
+        return first;
+    }
+    /** Register the spec's public key, reusing an identical existing key. */
+    async ensureSshKey(name, publicKey) {
+        const key = publicKey.trim();
+        const { body } = await this.api.request("GET", "/v1/sshkeys");
+        const existing = Array.isArray(body)
+            ? body.find((entry) => entry.key?.trim() === key)
+            : undefined;
+        if (existing?.id) {
+            return existing.id;
+        }
+        const created = await this.api.request("POST", "/v1/sshkeys", {
+            name: `nodetool-${sanitizeHostname(name)}`,
+            key,
+        });
+        return extractId(created.body, "SSH key");
+    }
+    /**
+     * Create the startup script that turns a bare VM into a NodeTool worker.
+     *
+     * The script is idempotent: Verda runs it on every boot, including after a
+     * resume, and it must not fail on a machine that already has the container.
+     */
+    async createStartupScript(spec) {
+        const env = { ...(spec.env ?? {}) };
+        if (spec.token) {
+            env.NODETOOL_WORKER_TOKEN = spec.token;
+        }
+        // Cache HF models on the OS volume so they survive a pause/resume.
+        env.HF_HOME = WORKER_HF_HOME;
+        const envFlags = Object.entries(env)
+            .map(([key, value]) => `  -e ${shellQuote(`${key}=${value}`)} \\`)
+            .join("\n");
+        const script = `#!/bin/bash
+# NodeTool worker bootstrap. Rerun-safe: Verda runs it on every boot.
+set -euo pipefail
+
+mkdir -p ${WORKER_HF_HOME}
+
+# Reuse the container across reboots; only pull and create it when absent.
+if docker inspect ${CONTAINER_NAME} >/dev/null 2>&1; then
+  docker start ${CONTAINER_NAME}
+  exit 0
+fi
+
+docker pull ${shellQuote(spec.image)}
+docker run -d \\
+  --name ${CONTAINER_NAME} \\
+  --restart unless-stopped \\
+  --gpus all \\
+  -p ${WORKER_PORT}:${WORKER_PORT} \\
+  -v ${WORKER_VOLUME_MOUNT}:${WORKER_VOLUME_MOUNT} \\
+${envFlags}
+  ${shellQuote(spec.image)}
+`;
+        const created = await this.api.request("POST", "/v1/scripts", {
+            name: `nodetool-${sanitizeHostname(spec.name)}-${Date.now()}`,
+            script,
+        });
+        return extractId(created.body, "startup script");
+    }
+    /** `POST /v1/instances`, returning the accepted instance id. */
+    async createInstance(create) {
+        try {
+            const { body } = await this.api.request("POST", "/v1/instances", create);
+            return extractId(body, "instance");
+        }
+        catch (err) {
+            if (err instanceof VerdaApiError && err.isNoCapacity) {
+                throw new Error(`Verda has no capacity for instance type ` +
+                    `"${String(create.instance_type)}" in "${String(create.location_code)}".`);
+            }
+            throw err;
+        }
+    }
+    /**
+     * Delete an instance together with every volume attached to it, permanently.
+     *
+     * The volume ids are read back from the instance and NAMED explicitly: an
+     * empty `volume_ids` retains them all, and omitting the field deletes only
+     * the OS volume while detaching the rest — both leave billable storage
+     * behind. Returns false when the instance no longer exists.
+     */
+    async destroyWithVolumes(instanceId, extraVolumeId) {
+        const instance = await this.getInstance(instanceId);
+        if (!instance) {
+            return false;
+        }
+        const volumeIds = [
+            ...new Set([
+                instance.os_volume_id,
+                ...(instance.volume_ids ?? []),
+                extraVolumeId,
+            ].filter((id) => Boolean(id))),
+        ];
+        await this.deleteInstance(instanceId, { volumeIds, permanent: true });
+        return true;
+    }
+    /**
+     * `PUT /v1/instances` delete. `volumeIds: []` retains every attached volume;
+     * a non-empty list deletes exactly those.
+     */
+    async deleteInstance(instanceId, opts) {
+        const body = {
+            action: "delete",
+            id: assertSafeId(instanceId, "instance id"),
+            volume_ids: opts.volumeIds.map((id) => assertSafeId(id, "volume id")),
+        };
+        if (opts.permanent && opts.volumeIds.length > 0) {
+            body.delete_permanently = true;
+        }
+        const { status, body: result } = await this.api.request("PUT", "/v1/instances", body);
+        // 204 means the action was already satisfied. 202 accepted it. 207 means a
+        // bulk action partly failed — with a single id, that is a failure.
+        if (status === 207) {
+            const failure = Array.isArray(result)
+                ? result.find((entry) => entry.status === "error")
+                : undefined;
+            throw new Error(`Verda refused to delete instance ${instanceId}: ` +
+                `${failure?.message ?? "unknown error"}`);
+        }
+    }
+    /** `GET /v1/instances/{id}`, or null when it no longer exists. */
+    async getInstance(id) {
+        try {
+            const { body } = await this.api.request("GET", `/v1/instances/${assertSafeId(id, "instance id")}`);
+            // A deleted instance can answer 200 with the `notfound` status rather
+            // than a 404, so both shapes are treated as gone.
+            if (!body || body.status === "notfound") {
+                return null;
+            }
+            return body;
+        }
+        catch (err) {
+            if (err instanceof VerdaApiError && err.status === 404) {
+                return null;
+            }
+            throw err;
+        }
+    }
+    /** Poll until the instance reports `running`, or fail on a terminal state. */
+    async waitForReady(id, opts = {}) {
+        const deadline = Date.now() + (opts.timeoutMs ?? 600_000);
+        const intervalMs = opts.intervalMs ?? 10_000;
+        let last;
+        while (Date.now() < deadline) {
+            const instance = await this.getInstance(id);
+            last = instance?.status;
+            const status = mapInstanceStatus(instance?.status);
+            if (instance && status === "running") {
+                return instance;
+            }
+            if (status === "error" || status === "terminated") {
+                throw new Error(`Verda instance ${id} reached terminal status ` +
+                    `"${instance?.status ?? "deleted"}"`);
+            }
+            await new Promise((resolve) => setTimeout(resolve, intervalMs));
+        }
+        throw new Error(`Verda instance ${id} did not become ready within the timeout ` +
+            `(last status: ${last ?? "unknown"})`);
+    }
+}
+/** Pull a resource id out of a create response, which may be a bare string. */
+function extractId(body, label) {
+    if (typeof body === "string" && body.trim()) {
+        return body.trim().replace(/^"|"$/g, "");
+    }
+    if (body && typeof body === "object") {
+        const id = body.id;
+        if (typeof id === "string" && id) {
+            return id;
+        }
+    }
+    throw new Error(`Verda ${label} creation returned no id`);
+}
+/** Reduce a profile name to a valid hostname label. */
+function sanitizeHostname(name) {
+    const cleaned = name
+        .toLowerCase()
+        .replace(/[^a-z0-9-]/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^-|-$/g, "")
+        .slice(0, 60);
+    return cleaned || "nodetool-worker";
+}
+//# sourceMappingURL=verda.js.map