@infernetprotocol/deploy-providers 0.1.22 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/digitalocean.js +258 -0
- package/src/index.js +96 -13
- package/src/pricing.js +6 -1
- package/src/providers/crusoe.js +278 -0
- package/src/providers/datacrunch.js +301 -0
- package/src/providers/hyperstack.js +277 -0
- package/src/providers/lambda.js +396 -0
- package/src/providers/runpod.js +382 -0
- package/src/providers/tensordock.js +362 -0
- package/src/providers/voltagepark.js +281 -0
- package/src/vast.js +245 -0
- package/src/vllm-bootstrap.js +145 -0
package/src/vast.js
CHANGED
|
@@ -165,3 +165,248 @@ export async function destroyDeployment({ apiKey, deploymentId }) {
|
|
|
165
165
|
await vastFetch({ apiKey, path: `/instances/${deploymentId}/`, method: "DELETE" });
|
|
166
166
|
return { ok: true };
|
|
167
167
|
}
|
|
168
|
+
|
|
169
|
+
// ---- IPIP-0019 DeployProvider class adapter ----
|
|
170
|
+
|
|
171
|
+
import { DeployProvider, NotSupportedError } from "./providers/base.js";
|
|
172
|
+
import { canonicalize, aliasesFor } from "./gpu-normalize.js";
|
|
173
|
+
import * as state from "./state.js";
|
|
174
|
+
import { composeInstallScript, pollVllmHealth } from "./vllm-bootstrap.js";
|
|
175
|
+
|
|
176
|
+
const DEFAULT_INSTALLER =
|
|
177
|
+
process.env.INFERNET_INSTALLER_URL
|
|
178
|
+
?? "https://infernetprotocol.com/install.sh";
|
|
179
|
+
|
|
180
|
+
export class VastProvider extends DeployProvider {
|
|
181
|
+
constructor(config = {}) {
|
|
182
|
+
super({ ...config, providerId: "vast" });
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async _fetch(path, { method = "GET", body } = {}) {
|
|
186
|
+
return vastFetch({ apiKey: this.config.apiKey, path, method, body });
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
async validateAuth() {
|
|
190
|
+
const data = await this._fetch("/users/current/");
|
|
191
|
+
if (!data?.id && !data?.email) throw new Error("Vast.ai: API key invalid");
|
|
192
|
+
return { ok: true, accountInfo: { id: data.id, email: data.email } };
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
async listGpuTypes() {
|
|
196
|
+
// Vast doesn't have a closed GPU catalog — derive from current offers.
|
|
197
|
+
const offers = await searchOffers({ apiKey: this.config.apiKey, rentable: true });
|
|
198
|
+
const seen = new Map();
|
|
199
|
+
for (const o of offers) {
|
|
200
|
+
if (!seen.has(o.gpuName)) {
|
|
201
|
+
seen.set(o.gpuName, { id: o.gpuName, name: o.gpuName, vramGb: null, pricePerHour: o.pricePerHour });
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return [...seen.values()];
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
async listRegions() {
|
|
208
|
+
const offers = await searchOffers({ apiKey: this.config.apiKey, rentable: true });
|
|
209
|
+
const regions = new Set();
|
|
210
|
+
for (const o of offers) if (o.region) regions.add(o.region);
|
|
211
|
+
return [...regions].map((r) => ({ id: r, name: r }));
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
async findOffers(request = {}) {
|
|
215
|
+
const wantCanonical = request.gpu ? canonicalize(request.gpu) : null;
|
|
216
|
+
// Vast's gpu_name filter wants a specific string; we'll loosely
|
|
217
|
+
// match by querying broadly and filtering client-side.
|
|
218
|
+
const raw = await searchOffers({
|
|
219
|
+
apiKey: this.config.apiKey,
|
|
220
|
+
numGpus: request.gpuCount ?? 1,
|
|
221
|
+
maxPrice: request.maxPricePerHour ?? null,
|
|
222
|
+
rentable: true
|
|
223
|
+
});
|
|
224
|
+
const aliases = wantCanonical
|
|
225
|
+
? [wantCanonical, ...aliasesFor(wantCanonical)].map((a) => a.toLowerCase())
|
|
226
|
+
: null;
|
|
227
|
+
|
|
228
|
+
const offers = [];
|
|
229
|
+
for (const o of raw) {
|
|
230
|
+
const nameLower = (o.gpuName ?? "").toLowerCase();
|
|
231
|
+
if (aliases && !aliases.some((a) => nameLower.includes(a))) continue;
|
|
232
|
+
if (request.region && o.region !== request.region) continue;
|
|
233
|
+
offers.push({
|
|
234
|
+
providerId: "vast",
|
|
235
|
+
offerId: String(o.id),
|
|
236
|
+
gpu: { name: o.gpuName, count: o.numGpus, vramGb: 0 },
|
|
237
|
+
cpu: o.cpuName ? { cores: undefined, model: o.cpuName } : undefined,
|
|
238
|
+
ramGb: o.ramGb,
|
|
239
|
+
diskGb: o.diskGb,
|
|
240
|
+
region: o.region,
|
|
241
|
+
pricePerHour: o.pricePerHour,
|
|
242
|
+
deployStyle: "docker",
|
|
243
|
+
available: true,
|
|
244
|
+
raw: o
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
return offers;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
_buildOnstart(envMap, opts = {}) {
|
|
251
|
+
if (opts.engine === "vllm") {
|
|
252
|
+
return composeInstallScript({
|
|
253
|
+
infernetEnv: envMap,
|
|
254
|
+
vllmConfig: opts.vllmConfig ?? {}
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
const exports = Object.entries(envMap)
|
|
258
|
+
.map(([k, v]) => `export ${k}=${JSON.stringify(String(v))}`)
|
|
259
|
+
.join("\n");
|
|
260
|
+
return [
|
|
261
|
+
"#!/bin/bash",
|
|
262
|
+
"set -eux",
|
|
263
|
+
exports,
|
|
264
|
+
`curl -fsSL ${DEFAULT_INSTALLER} | bash`,
|
|
265
|
+
"infernet register || true",
|
|
266
|
+
"infernet start || true"
|
|
267
|
+
].join("\n");
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
async createNode(request = {}) {
|
|
271
|
+
const {
|
|
272
|
+
offerId, name, image = "nvidia/cuda:12.4.1-runtime-ubuntu22.04",
|
|
273
|
+
diskSizeGb = 40, env = {}, controlPlaneUrl = "",
|
|
274
|
+
engine = "ollama", model = null, hourlyPrice = 0, vramGb = 0,
|
|
275
|
+
gpu, gpuCount = 1, region
|
|
276
|
+
} = request;
|
|
277
|
+
|
|
278
|
+
if (!offerId) {
|
|
279
|
+
throw new Error("vast: createNode requires offerId (call findOffers first)");
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
const localId = state.generateNodeId();
|
|
283
|
+
const label = name ?? localId;
|
|
284
|
+
|
|
285
|
+
const onstart = this._buildOnstart({
|
|
286
|
+
INFERNET_NODE_NAME: label,
|
|
287
|
+
INFERNET_CONTROL_PLANE_URL: controlPlaneUrl,
|
|
288
|
+
INFERNET_ENGINE: engine,
|
|
289
|
+
...(model ? { INFERNET_MODEL: model } : {}),
|
|
290
|
+
...env
|
|
291
|
+
}, { engine, vllmConfig: { model, ...(request.vllm ?? {}) } });
|
|
292
|
+
|
|
293
|
+
const result = await createDeployment({
|
|
294
|
+
apiKey: this.config.apiKey,
|
|
295
|
+
offerId, userData: onstart, image, diskGb: diskSizeGb, label
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
const record = {
|
|
299
|
+
id: localId,
|
|
300
|
+
provider: "vast",
|
|
301
|
+
providerNodeId: result.deploymentId,
|
|
302
|
+
name: label,
|
|
303
|
+
gpu: gpu ?? "",
|
|
304
|
+
gpuCount,
|
|
305
|
+
vramGb,
|
|
306
|
+
region,
|
|
307
|
+
hourlyPrice,
|
|
308
|
+
engine,
|
|
309
|
+
model: model ?? undefined,
|
|
310
|
+
status: "creating",
|
|
311
|
+
createdAt: new Date().toISOString(),
|
|
312
|
+
controlPlaneUrl
|
|
313
|
+
};
|
|
314
|
+
await state.saveNode(record);
|
|
315
|
+
return record;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
async waitUntilReady(node, timeoutMs = 5 * 60 * 1000) {
|
|
319
|
+
if (!node?.providerNodeId) throw new Error("vast: waitUntilReady requires node.providerNodeId");
|
|
320
|
+
const deadline = Date.now() + timeoutMs;
|
|
321
|
+
while (Date.now() < deadline) {
|
|
322
|
+
const live = await getDeployment({ apiKey: this.config.apiKey, deploymentId: node.providerNodeId });
|
|
323
|
+
const s = (live?.status ?? "").toLowerCase();
|
|
324
|
+
if (s === "running" && live?.endpoint) {
|
|
325
|
+
const endpointUrl = `http://${live.endpoint}:46337`;
|
|
326
|
+
return state.updateNode(node.id, { status: "running", endpointUrl, ip: live.endpoint });
|
|
327
|
+
}
|
|
328
|
+
await new Promise((r) => setTimeout(r, 5000));
|
|
329
|
+
}
|
|
330
|
+
await state.updateNode(node.id, { status: "error", error_log: "vast instance did not reach RUNNING in time" });
|
|
331
|
+
throw new Error(`Vast.ai instance ${node.providerNodeId} did not reach RUNNING within ${Math.round(timeoutMs / 1000)}s`);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
async bootstrapNode(node, request = {}) {
|
|
335
|
+
if (!node?.endpointUrl && !node?.ip) return { ok: false, reason: "no endpointUrl/ip on node" };
|
|
336
|
+
if ((request.engine ?? node.engine) === "vllm") {
|
|
337
|
+
return pollVllmHealth(node.ip ?? node.endpointUrl, { port: request.vllm?.port ?? 8000 });
|
|
338
|
+
}
|
|
339
|
+
const healthEndpoint = `${node.endpointUrl}/health`;
|
|
340
|
+
const deadline = Date.now() + 5 * 60 * 1000;
|
|
341
|
+
let lastErr = null;
|
|
342
|
+
while (Date.now() < deadline) {
|
|
343
|
+
try {
|
|
344
|
+
const ctrl = new AbortController();
|
|
345
|
+
const t = setTimeout(() => ctrl.abort(), 5000);
|
|
346
|
+
const res = await fetch(healthEndpoint, { signal: ctrl.signal });
|
|
347
|
+
clearTimeout(t);
|
|
348
|
+
if (res.ok) return { ok: true, healthEndpoint };
|
|
349
|
+
lastErr = `HTTP ${res.status}`;
|
|
350
|
+
} catch (err) { lastErr = err?.message ?? String(err); }
|
|
351
|
+
await new Promise((r) => setTimeout(r, 10000));
|
|
352
|
+
}
|
|
353
|
+
return { ok: false, reason: lastErr ?? "bootstrap health-check timed out", healthEndpoint };
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
async getNode(nodeId) {
|
|
357
|
+
const local = await state.loadNode(nodeId);
|
|
358
|
+
if (!local || local.provider !== "vast") {
|
|
359
|
+
throw new Error(`vast: no local record for nodeId=${nodeId}`);
|
|
360
|
+
}
|
|
361
|
+
try {
|
|
362
|
+
const live = await getDeployment({ apiKey: this.config.apiKey, deploymentId: local.providerNodeId });
|
|
363
|
+
if (live?.status) local.liveStatus = live.status;
|
|
364
|
+
} catch { /* best-effort */ }
|
|
365
|
+
return local;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
async listNodes() {
|
|
369
|
+
const all = await state.listNodes();
|
|
370
|
+
return all.filter((n) => n.provider === "vast");
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
async stopNode(nodeId) {
|
|
374
|
+
const local = await state.loadNode(nodeId);
|
|
375
|
+
if (!local || local.provider !== "vast") throw new Error(`vast: no local record for nodeId=${nodeId}`);
|
|
376
|
+
// Vast.ai pauses instances via state=stopped action.
|
|
377
|
+
await this._fetch(`/instances/${local.providerNodeId}/`, { method: "PUT", body: { state: "stopped" } });
|
|
378
|
+
await state.updateNode(nodeId, { status: "stopped" });
|
|
379
|
+
return { ok: true };
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
async startNode(nodeId) {
|
|
383
|
+
const local = await state.loadNode(nodeId);
|
|
384
|
+
if (!local || local.provider !== "vast") throw new Error(`vast: no local record for nodeId=${nodeId}`);
|
|
385
|
+
await this._fetch(`/instances/${local.providerNodeId}/`, { method: "PUT", body: { state: "running" } });
|
|
386
|
+
await state.updateNode(nodeId, { status: "running" });
|
|
387
|
+
return { ok: true };
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
async destroyNode(nodeId) {
|
|
391
|
+
const local = await state.loadNode(nodeId);
|
|
392
|
+
if (!local || local.provider !== "vast") throw new Error(`vast: no local record for nodeId=${nodeId}`);
|
|
393
|
+
await destroyDeployment({ apiKey: this.config.apiKey, deploymentId: local.providerNodeId });
|
|
394
|
+
await state.updateNode(nodeId, { status: "destroyed" });
|
|
395
|
+
return { ok: true };
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
async logs(nodeId, _opts = {}) {
|
|
399
|
+
const local = await state.loadNode(nodeId).catch(() => null);
|
|
400
|
+
const url = local?.providerNodeId
|
|
401
|
+
? `https://cloud.vast.ai/instances/?id=${local.providerNodeId}`
|
|
402
|
+
: "https://cloud.vast.ai/instances/";
|
|
403
|
+
const err = new NotSupportedError("logs", "vast");
|
|
404
|
+
const sshHint = local?.ip
|
|
405
|
+
? `\nSSH and tail the daemon log:\n ssh root@${local.ip} 'tail -200 /var/log/infernet/daemon.log'`
|
|
406
|
+
: "";
|
|
407
|
+
err.message = `vast: container logs are not exposed via the public API.\nView at: ${url}` + sshHint;
|
|
408
|
+
throw err;
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
export default VastProvider;
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared vLLM bootstrap helper — used by every DeployProvider that
|
|
3
|
+
* supports cloud-init / SSH bootstrap (TensorDock, Lambda, Vast,
|
|
4
|
+
* DigitalOcean). RunPod uses a vLLM-ready Docker image instead, so
|
|
5
|
+
* it only consumes the health-check helpers from here.
|
|
6
|
+
*
|
|
7
|
+
* Spec: IPIP-0019 §9.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const DEFAULT_PORT = 8000;
|
|
11
|
+
const DEFAULT_INSTALLER =
|
|
12
|
+
process.env.INFERNET_INSTALLER_URL
|
|
13
|
+
?? "https://infernetprotocol.com/install.sh";
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Compose the `vllm serve …` command line from a config object.
|
|
17
|
+
* Returns a single-line string suitable for an ExecStart= entry.
|
|
18
|
+
*/
|
|
19
|
+
export function composeServeCommand(config = {}) {
|
|
20
|
+
const {
|
|
21
|
+
model, port = DEFAULT_PORT,
|
|
22
|
+
servedModelName, maxModelLen, tensorParallelSize,
|
|
23
|
+
gpuMemoryUtilization, quantization, dtype
|
|
24
|
+
} = config;
|
|
25
|
+
|
|
26
|
+
if (!model) throw new Error("vllm: model is required");
|
|
27
|
+
|
|
28
|
+
const args = [
|
|
29
|
+
`vllm serve ${shellQuote(model)}`,
|
|
30
|
+
`--host 0.0.0.0`,
|
|
31
|
+
`--port ${port}`
|
|
32
|
+
];
|
|
33
|
+
if (servedModelName) args.push(`--served-model-name ${shellQuote(servedModelName)}`);
|
|
34
|
+
if (maxModelLen) args.push(`--max-model-len ${Number(maxModelLen)}`);
|
|
35
|
+
if (tensorParallelSize) args.push(`--tensor-parallel-size ${Number(tensorParallelSize)}`);
|
|
36
|
+
if (gpuMemoryUtilization) args.push(`--gpu-memory-utilization ${Number(gpuMemoryUtilization)}`);
|
|
37
|
+
if (quantization && quantization !== "none") args.push(`--quantization ${shellQuote(quantization)}`);
|
|
38
|
+
if (dtype && dtype !== "auto") args.push(`--dtype ${shellQuote(dtype)}`);
|
|
39
|
+
|
|
40
|
+
return args.join(" ");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Compose the cloud-init shell script that installs vLLM and runs it
|
|
45
|
+
* as a systemd service, then runs the Infernet daemon pointing at the
|
|
46
|
+
* local vLLM endpoint.
|
|
47
|
+
*
|
|
48
|
+
* @param {object} opts
|
|
49
|
+
* @param {object} opts.infernetEnv env vars exported before the installer
|
|
50
|
+
* @param {object} opts.vllmConfig passed to composeServeCommand
|
|
51
|
+
* @returns {string} bash script
|
|
52
|
+
*/
|
|
53
|
+
export function composeInstallScript({ infernetEnv = {}, vllmConfig = {} }) {
|
|
54
|
+
const port = vllmConfig.port ?? DEFAULT_PORT;
|
|
55
|
+
const serveCmd = composeServeCommand(vllmConfig);
|
|
56
|
+
const exports = Object.entries({
|
|
57
|
+
...infernetEnv,
|
|
58
|
+
INFERNET_ENGINE_URL: `http://127.0.0.1:${port}/v1`,
|
|
59
|
+
INFERNET_ENGINE: "vllm"
|
|
60
|
+
})
|
|
61
|
+
.map(([k, v]) => `export ${k}=${shellQuote(String(v))}`)
|
|
62
|
+
.join("\n");
|
|
63
|
+
|
|
64
|
+
return [
|
|
65
|
+
"#!/bin/bash",
|
|
66
|
+
"set -eux",
|
|
67
|
+
"",
|
|
68
|
+
"# --- export config ---",
|
|
69
|
+
exports,
|
|
70
|
+
"",
|
|
71
|
+
"# --- install vLLM in /opt/vllm (isolated venv) ---",
|
|
72
|
+
"apt-get update -y || true",
|
|
73
|
+
"DEBIAN_FRONTEND=noninteractive apt-get install -y python3 python3-pip python3-venv curl ca-certificates",
|
|
74
|
+
"python3 -m venv /opt/vllm",
|
|
75
|
+
"/opt/vllm/bin/pip install --upgrade pip",
|
|
76
|
+
"/opt/vllm/bin/pip install vllm",
|
|
77
|
+
"",
|
|
78
|
+
"# --- write systemd unit so vLLM restarts on reboot ---",
|
|
79
|
+
"cat > /etc/systemd/system/vllm.service <<'UNIT'",
|
|
80
|
+
"[Unit]",
|
|
81
|
+
"Description=vLLM OpenAI-compatible inference server",
|
|
82
|
+
"After=network-online.target",
|
|
83
|
+
"Wants=network-online.target",
|
|
84
|
+
"",
|
|
85
|
+
"[Service]",
|
|
86
|
+
"Type=simple",
|
|
87
|
+
`ExecStart=/opt/vllm/bin/${serveCmd}`,
|
|
88
|
+
"Restart=on-failure",
|
|
89
|
+
"RestartSec=10",
|
|
90
|
+
"",
|
|
91
|
+
"[Install]",
|
|
92
|
+
"WantedBy=multi-user.target",
|
|
93
|
+
"UNIT",
|
|
94
|
+
"",
|
|
95
|
+
"systemctl daemon-reload",
|
|
96
|
+
"systemctl enable --now vllm.service",
|
|
97
|
+
"",
|
|
98
|
+
"# --- install + start the Infernet daemon (talks to local vLLM) ---",
|
|
99
|
+
`curl -fsSL ${DEFAULT_INSTALLER} | bash`,
|
|
100
|
+
"infernet register || true",
|
|
101
|
+
"infernet start || true"
|
|
102
|
+
].join("\n");
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Build the `/v1/models` health-check URL for a node.
|
|
107
|
+
*/
|
|
108
|
+
export function vllmHealthCheckUrl(host, port = DEFAULT_PORT) {
|
|
109
|
+
if (!host) throw new Error("vllm: host required");
|
|
110
|
+
const cleanHost = host.replace(/^https?:\/\//, "").replace(/:\d+$/, "");
|
|
111
|
+
return `http://${cleanHost}:${port}/v1/models`;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Poll the vLLM health endpoint until 200 or timeout.
|
|
116
|
+
* Returns { ok, lastErr, healthEndpoint }.
|
|
117
|
+
*/
|
|
118
|
+
export async function pollVllmHealth(host, {
|
|
119
|
+
port = DEFAULT_PORT,
|
|
120
|
+
timeoutMs = 5 * 60 * 1000,
|
|
121
|
+
intervalMs = 10 * 1000
|
|
122
|
+
} = {}) {
|
|
123
|
+
const healthEndpoint = vllmHealthCheckUrl(host, port);
|
|
124
|
+
const deadline = Date.now() + timeoutMs;
|
|
125
|
+
let lastErr = null;
|
|
126
|
+
while (Date.now() < deadline) {
|
|
127
|
+
try {
|
|
128
|
+
const ctrl = new AbortController();
|
|
129
|
+
const t = setTimeout(() => ctrl.abort(), 5000);
|
|
130
|
+
const res = await fetch(healthEndpoint, { signal: ctrl.signal });
|
|
131
|
+
clearTimeout(t);
|
|
132
|
+
if (res.ok) return { ok: true, healthEndpoint };
|
|
133
|
+
lastErr = `HTTP ${res.status}`;
|
|
134
|
+
} catch (err) {
|
|
135
|
+
lastErr = err?.message ?? String(err);
|
|
136
|
+
}
|
|
137
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
138
|
+
}
|
|
139
|
+
return { ok: false, lastErr, healthEndpoint };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Single-quote a string for safe inclusion in a bash script. */
|
|
143
|
+
function shellQuote(s) {
|
|
144
|
+
return `'${String(s).replace(/'/g, "'\\''")}'`;
|
|
145
|
+
}
|