@infernetprotocol/cli 0.1.34 → 0.1.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/start.js +181 -1
- package/commands/train.js +126 -1
- package/lib/inference/petals-engine.js +75 -0
- package/lib/node-client.js +20 -0
- package/package.json +6 -6
package/commands/start.js
CHANGED
|
@@ -505,6 +505,73 @@ async function runDaemon(args, ctx) {
|
|
|
505
505
|
}
|
|
506
506
|
}
|
|
507
507
|
|
|
508
|
+
/**
|
|
509
|
+
* IPIP-0030: poll the open training market, claim a shard if any
|
|
510
|
+
* fit our hardware, run it via runTrainShard, report back. Single
|
|
511
|
+
* shard per poll cycle — keeps a node from monopolizing a job.
|
|
512
|
+
*/
|
|
513
|
+
let trainingBusy = false;
|
|
514
|
+
async function pollTrainingMarket() {
|
|
515
|
+
if (node.role !== 'provider') return;
|
|
516
|
+
if (trainingBusy) return; // one at a time
|
|
517
|
+
|
|
518
|
+
let listing;
|
|
519
|
+
try {
|
|
520
|
+
listing = await client.listAvailableTrainingShards(3);
|
|
521
|
+
} catch (err) {
|
|
522
|
+
process.stderr.write(`training market poll error: ${err?.message ?? err}\n`);
|
|
523
|
+
return;
|
|
524
|
+
}
|
|
525
|
+
const shards = listing?.shards ?? [];
|
|
526
|
+
if (shards.length === 0) return;
|
|
527
|
+
|
|
528
|
+
// Race-claim the first one. If another node beats us, just exit;
|
|
529
|
+
// we'll retry on the next poll cycle.
|
|
530
|
+
let claimed = null;
|
|
531
|
+
for (const shard of shards) {
|
|
532
|
+
try {
|
|
533
|
+
const res = await client.claimTrainingShard(shard.shard_id);
|
|
534
|
+
claimed = res;
|
|
535
|
+
break;
|
|
536
|
+
} catch (err) {
|
|
537
|
+
if (err?.status === 409) continue; // someone else got it
|
|
538
|
+
process.stderr.write(`training claim error: ${err?.message ?? err}\n`);
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
if (!claimed) return;
|
|
542
|
+
|
|
543
|
+
trainingBusy = true;
|
|
544
|
+
const shardId = claimed.shard_id;
|
|
545
|
+
process.stdout.write(
|
|
546
|
+
`[training] claimed shard ${shardId} of job ${claimed.job_id} ` +
|
|
547
|
+
`(base ${claimed.base_model}, idx ${claimed.shard_index})\n`
|
|
548
|
+
);
|
|
549
|
+
const t0 = Date.now();
|
|
550
|
+
try {
|
|
551
|
+
// Reuse the same daemon-side handler used by node_commands.
|
|
552
|
+
const result = await runTrainShard({
|
|
553
|
+
shard_url: claimed.shard_url,
|
|
554
|
+
upload_url: claimed.upload_url,
|
|
555
|
+
base_model: claimed.base_model,
|
|
556
|
+
...(claimed.config ?? {})
|
|
557
|
+
}, shardId);
|
|
558
|
+
await client.reportTrainingShard(shardId, {
|
|
559
|
+
status: 'completed',
|
|
560
|
+
adapter_url: result?.uploaded ? claimed.upload_url : null,
|
|
561
|
+
metrics: { duration_ms: Date.now() - t0 }
|
|
562
|
+
});
|
|
563
|
+
process.stdout.write(`[training] reported completed shard ${shardId}\n`);
|
|
564
|
+
} catch (err) {
|
|
565
|
+
const msg = err?.message ?? String(err);
|
|
566
|
+
process.stderr.write(`[training] shard ${shardId} failed: ${msg}\n`);
|
|
567
|
+
try {
|
|
568
|
+
await client.reportTrainingShard(shardId, { status: 'failed', error: msg });
|
|
569
|
+
} catch { /* swallow */ }
|
|
570
|
+
} finally {
|
|
571
|
+
trainingBusy = false;
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
|
|
508
575
|
async function runNodeCommand(cmd) {
|
|
509
576
|
const { id, command, args } = cmd;
|
|
510
577
|
const t0 = new Date().toISOString();
|
|
@@ -709,6 +776,81 @@ async function runDaemon(args, ctx) {
|
|
|
709
776
|
};
|
|
710
777
|
}
|
|
711
778
|
|
|
779
|
+
/**
|
|
780
|
+
* IPIP-0030: shard-hosting routes layered on top of /healthz.
|
|
781
|
+
* Returns true if the request matched and was handled.
|
|
782
|
+
*/
|
|
783
|
+
async function handleTrainingHttp(req, res) {
|
|
784
|
+
if (!req.url) return false;
|
|
785
|
+
const url = new URL(req.url, 'http://localhost');
|
|
786
|
+
const pathname = url.pathname;
|
|
787
|
+
|
|
788
|
+
// GET /v1/training/shards/<run_id>/<shard_index>.jsonl
|
|
789
|
+
const shardMatch = pathname.match(/^\/v1\/training\/shards\/([A-Za-z0-9_-]+)\/shard-(\d+)\.jsonl$/);
|
|
790
|
+
if (shardMatch && req.method === 'GET') {
|
|
791
|
+
const fs = await import('node:fs');
|
|
792
|
+
const path = await import('node:path');
|
|
793
|
+
const [, runId, idx] = shardMatch;
|
|
794
|
+
const filePath = path.join(
|
|
795
|
+
process.env.HOME ?? '/tmp',
|
|
796
|
+
'.infernet', 'training-runs', runId, 'shards', `shard-${idx}.jsonl`
|
|
797
|
+
);
|
|
798
|
+
if (!fs.existsSync(filePath)) {
|
|
799
|
+
res.writeHead(404, { 'content-type': 'text/plain' });
|
|
800
|
+
res.end('shard not found\n');
|
|
801
|
+
return true;
|
|
802
|
+
}
|
|
803
|
+
res.writeHead(200, {
|
|
804
|
+
'content-type': 'application/jsonl',
|
|
805
|
+
'content-length': fs.statSync(filePath).size,
|
|
806
|
+
'cache-control': 'no-store'
|
|
807
|
+
});
|
|
808
|
+
fs.createReadStream(filePath).pipe(res);
|
|
809
|
+
return true;
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
// PUT /v1/training/adapters/<run_id>/<shard_index>?token=<t>
|
|
813
|
+
const adapterMatch = pathname.match(/^\/v1\/training\/adapters\/([A-Za-z0-9_-]+)\/(\d+)$/);
|
|
814
|
+
if (adapterMatch && req.method === 'PUT') {
|
|
815
|
+
const fsp = await import('node:fs/promises');
|
|
816
|
+
const fs = await import('node:fs');
|
|
817
|
+
const path = await import('node:path');
|
|
818
|
+
const [, runId, idx] = adapterMatch;
|
|
819
|
+
const runDir = path.join(process.env.HOME ?? '/tmp', '.infernet', 'training-runs', runId);
|
|
820
|
+
const manifestPath = path.join(runDir, 'manifest.json');
|
|
821
|
+
if (!fs.existsSync(manifestPath)) {
|
|
822
|
+
res.writeHead(404, { 'content-type': 'text/plain' });
|
|
823
|
+
res.end('run not found\n');
|
|
824
|
+
return true;
|
|
825
|
+
}
|
|
826
|
+
// Token check: only accept PUTs whose ?token=<t> matches the
|
|
827
|
+
// run's manifest. The submitter mints the token at run-start
|
|
828
|
+
// and shares it with the control plane only via upload_url.
|
|
829
|
+
const manifest = JSON.parse(await fsp.readFile(manifestPath, 'utf8'));
|
|
830
|
+
const presentedToken = url.searchParams.get('token');
|
|
831
|
+
if (!presentedToken || presentedToken !== manifest.upload_token) {
|
|
832
|
+
res.writeHead(403, { 'content-type': 'text/plain' });
|
|
833
|
+
res.end('bad token\n');
|
|
834
|
+
return true;
|
|
835
|
+
}
|
|
836
|
+
const adapterDir = path.join(runDir, 'adapters');
|
|
837
|
+
await fsp.mkdir(adapterDir, { recursive: true });
|
|
838
|
+
const outPath = path.join(adapterDir, `shard-${idx}.adapter.safetensors`);
|
|
839
|
+
await new Promise((resolve, reject) => {
|
|
840
|
+
const ws = fs.createWriteStream(outPath);
|
|
841
|
+
req.pipe(ws);
|
|
842
|
+
ws.on('finish', resolve);
|
|
843
|
+
ws.on('error', reject);
|
|
844
|
+
req.on('error', reject);
|
|
845
|
+
});
|
|
846
|
+
res.writeHead(200, { 'content-type': 'application/json' });
|
|
847
|
+
res.end(JSON.stringify({ ok: true, path: outPath }));
|
|
848
|
+
return true;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
return false;
|
|
852
|
+
}
|
|
853
|
+
|
|
712
854
|
function snapshot() {
|
|
713
855
|
return {
|
|
714
856
|
pid: process.pid,
|
|
@@ -879,7 +1021,7 @@ async function runDaemon(args, ctx) {
|
|
|
879
1021
|
// firewalled). Returns 200 + a JSON daemon snapshot.
|
|
880
1022
|
const healthPort = Number.parseInt(process.env.PORT ?? '', 10) || 8080;
|
|
881
1023
|
try {
|
|
882
|
-
healthServer = http.createServer((req, res) => {
|
|
1024
|
+
healthServer = http.createServer(async (req, res) => {
|
|
883
1025
|
if (req.url === '/healthz' || req.url === '/health' || req.url === '/') {
|
|
884
1026
|
const snap = snapshot();
|
|
885
1027
|
const body = JSON.stringify({
|
|
@@ -901,6 +1043,26 @@ async function runDaemon(args, ctx) {
|
|
|
901
1043
|
res.end(body);
|
|
902
1044
|
return;
|
|
903
1045
|
}
|
|
1046
|
+
|
|
1047
|
+
// IPIP-0030: serve training shards directly from this node's
|
|
1048
|
+
// local filesystem. The submitter's CLI writes shards to
|
|
1049
|
+
// ~/.infernet/training-runs/<run_id>/shards/shard-<i>.jsonl
|
|
1050
|
+
// and posts the URL referencing this same daemon's endpoint.
|
|
1051
|
+
// No third-party storage, no separate ports, no signup —
|
|
1052
|
+
// the daemon's already on a reachable port.
|
|
1053
|
+
//
|
|
1054
|
+
// Routes:
|
|
1055
|
+
// GET /v1/training/shards/<run_id>/<shard_index>.jsonl
|
|
1056
|
+
// PUT /v1/training/adapters/<run_id>/<shard_index>?token=<t>
|
|
1057
|
+
try {
|
|
1058
|
+
const handled = await handleTrainingHttp(req, res);
|
|
1059
|
+
if (handled) return;
|
|
1060
|
+
} catch (err) {
|
|
1061
|
+
res.writeHead(500, { 'content-type': 'text/plain' });
|
|
1062
|
+
res.end(`training http error: ${err?.message ?? err}\n`);
|
|
1063
|
+
return;
|
|
1064
|
+
}
|
|
1065
|
+
|
|
904
1066
|
res.writeHead(404, { 'content-type': 'text/plain' });
|
|
905
1067
|
res.end('not found\n');
|
|
906
1068
|
});
|
|
@@ -977,6 +1139,24 @@ async function runDaemon(args, ctx) {
|
|
|
977
1139
|
}, commandPollMs);
|
|
978
1140
|
if (typeof commandTimer.unref === 'function') commandTimer.unref();
|
|
979
1141
|
|
|
1142
|
+
// Open-market training (IPIP-0030). Daemons that opted in via
|
|
1143
|
+
// `infernet config set engine.acceptTraining true` (or env var
|
|
1144
|
+
// INFERNET_ACCEPT_TRAINING=1) poll for available shards and run them
|
|
1145
|
+
// for pay. Every ~60s — training shards are rare + slow, so polling
|
|
1146
|
+
// hot wastes RPS.
|
|
1147
|
+
const acceptTraining = process.env.INFERNET_ACCEPT_TRAINING === '1'
|
|
1148
|
+
|| config?.engine?.acceptTraining === true;
|
|
1149
|
+
if (acceptTraining) {
|
|
1150
|
+
process.stdout.write('[training] open-market shard claiming enabled\n');
|
|
1151
|
+
const trainingPollMs = 60_000;
|
|
1152
|
+
const trainingTimer = setInterval(() => {
|
|
1153
|
+
pollTrainingMarket().catch((err) =>
|
|
1154
|
+
process.stderr.write(`training poll threw: ${err?.message ?? err}\n`)
|
|
1155
|
+
);
|
|
1156
|
+
}, trainingPollMs);
|
|
1157
|
+
if (typeof trainingTimer.unref === 'function') trainingTimer.unref();
|
|
1158
|
+
}
|
|
1159
|
+
|
|
980
1160
|
// Self-update: check npm registry every 5 minutes. If a newer version is
|
|
981
1161
|
// available, pull the installer, re-generate any missing keys + re-register
|
|
982
1162
|
// specs (idempotent), then re-exec so the new code is running immediately.
|
package/commands/train.js
CHANGED
|
@@ -34,8 +34,14 @@ Usage:
|
|
|
34
34
|
[--num <n>] (default: 20 results)
|
|
35
35
|
[--domains <a,b,c>] (whitelist filter)
|
|
36
36
|
infernet train init [--output <dir>] Scaffold infernet.train.yml
|
|
37
|
-
infernet train run [--config <file>] Run training (local
|
|
37
|
+
infernet train run [--config <file>] Run training (local | P2P | market)
|
|
38
38
|
[--local] Force local execution
|
|
39
|
+
[--open-market] Post to the open network market — any
|
|
40
|
+
opted-in operator can claim shards
|
|
41
|
+
for pay (--budget <usd>, --max-nodes)
|
|
42
|
+
[--shard-base-url <url>] Public prefix where daemons fetch shards
|
|
43
|
+
[--budget <usd>] Total budget; per-shard = budget / max-nodes
|
|
44
|
+
[--max-nodes <n>] Number of shards to split into
|
|
39
45
|
[--output <dir>] Where to write run artifacts
|
|
40
46
|
infernet train status [<run-id>] Show run status
|
|
41
47
|
infernet train list List recent runs
|
|
@@ -329,6 +335,13 @@ async function cmdRun(args) {
|
|
|
329
335
|
|
|
330
336
|
const useLocal = forceLocal || workloadClass === "C1";
|
|
331
337
|
|
|
338
|
+
// Open-market: distribute across the entire network of opted-in
|
|
339
|
+
// operators (--open-market). The submitter pays per shard via the
|
|
340
|
+
// existing payment rails.
|
|
341
|
+
if (args.has("open-market")) {
|
|
342
|
+
return runOpenMarket({ trainConfig, runDir, runId, args });
|
|
343
|
+
}
|
|
344
|
+
|
|
332
345
|
// P2P (federated LoRA) — workload_class C2/C3 or explicit --p2p flag.
|
|
333
346
|
if (!useLocal) {
|
|
334
347
|
return runP2PFederated({ trainConfig, runDir, runId });
|
|
@@ -342,6 +355,118 @@ async function cmdRun(args) {
|
|
|
342
355
|
return runMicrogptLocal({ trainConfig, runDir, runId });
|
|
343
356
|
}
|
|
344
357
|
|
|
358
|
+
/**
|
|
359
|
+
* IPIP-0030: post a training job to the open market. Any operator on
|
|
360
|
+
* the network running with --accept-training claims shards and reports
|
|
361
|
+
* back; we poll until all shards complete, then FedAvg the adapters.
|
|
362
|
+
*
|
|
363
|
+
* Shards are hosted by the SUBMITTER'S OWN DAEMON — no S3, no HF, no
|
|
364
|
+
* IPFS, no third-party storage. The local daemon's HTTP server already
|
|
365
|
+
* has a reachable endpoint registered with the control plane; we just
|
|
366
|
+
* drop shard files in ~/.infernet/training-runs/<run_id>/shards/ and
|
|
367
|
+
* the daemon serves them at /v1/training/shards/<run_id>/shard-<i>.jsonl
|
|
368
|
+
* (handler in commands/start.js handleTrainingHttp).
|
|
369
|
+
*/
|
|
370
|
+
async function runOpenMarket({ trainConfig, runDir, runId, args }) {
|
|
371
|
+
const cfg = await loadConfig().catch(() => ({}));
|
|
372
|
+
const controlPlaneUrl = cfg?.controlPlane?.url ?? process.env.NEXT_PUBLIC_APP_URL ?? "https://infernetprotocol.com";
|
|
373
|
+
const sessionToken = cfg?.session?.token ?? process.env.INFERNET_SESSION_TOKEN;
|
|
374
|
+
if (!sessionToken) {
|
|
375
|
+
process.stderr.write(`error: open-market training needs a logged-in session — run \`infernet login\` first\n`);
|
|
376
|
+
return 1;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// Discover this submitter's own daemon endpoint. The daemon
|
|
380
|
+
// registered an endpoint_url at startup; it's stored locally in
|
|
381
|
+
// config under node.endpoint_url OR can be derived from advertised
|
|
382
|
+
// address + PORT (default 8080 — the same /healthz HTTP server).
|
|
383
|
+
const advertised = cfg?.node?.advertised_endpoint
|
|
384
|
+
?? cfg?.node?.endpoint_url
|
|
385
|
+
?? process.env.INFERNET_DAEMON_ENDPOINT;
|
|
386
|
+
if (!advertised) {
|
|
387
|
+
process.stderr.write(
|
|
388
|
+
`error: can't find this node's public endpoint. Either:\n` +
|
|
389
|
+
` - run \`infernet start\` so the daemon registers an endpoint, OR\n` +
|
|
390
|
+
` - set INFERNET_DAEMON_ENDPOINT=http://<your-public-ip>:8080\n` +
|
|
391
|
+
`\nThe submitter's daemon hosts the shards directly — no S3, no HF.\n` +
|
|
392
|
+
`If your machine is behind NAT, run a tunnel:\n` +
|
|
393
|
+
` cloudflared tunnel --url http://localhost:8080\n` +
|
|
394
|
+
`and set INFERNET_DAEMON_ENDPOINT to the cloudflared URL.\n`
|
|
395
|
+
);
|
|
396
|
+
return 1;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
const numShards = Number.parseInt(args.get("max-nodes") ?? args.get("num-shards") ?? "5", 10);
|
|
400
|
+
const budget = Number.parseFloat(args.get("budget") ?? "0");
|
|
401
|
+
const pricePerShard = numShards > 0 ? budget / numShards : 0;
|
|
402
|
+
|
|
403
|
+
const datasetPath = trainConfig.input?.dataset;
|
|
404
|
+
if (!datasetPath) {
|
|
405
|
+
process.stderr.write(`error: input.dataset is required\n`);
|
|
406
|
+
return 1;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
// Split the local JSONL into N shards under ~/.infernet/training-runs/<run_id>/.
|
|
410
|
+
// The daemon's HTTP server reads from this exact path.
|
|
411
|
+
const { splitJsonl } = await import("../lib/training/p2p-coordinator.js");
|
|
412
|
+
const home = process.env.HOME ?? "/tmp";
|
|
413
|
+
const localRunDir = path.join(home, ".infernet", "training-runs", runId);
|
|
414
|
+
const shardsDir = path.join(localRunDir, "shards");
|
|
415
|
+
process.stdout.write(`Splitting ${datasetPath} into ${numShards} shard(s)…\n`);
|
|
416
|
+
const shards = await splitJsonl({ inputPath: datasetPath, outDir: shardsDir, numShards });
|
|
417
|
+
|
|
418
|
+
// Mint a per-run upload token so operators that complete a shard
|
|
419
|
+
// can PUT their adapter back to the submitter's daemon, and only
|
|
420
|
+
// those who claimed the shard via the control plane have the token.
|
|
421
|
+
const uploadToken = (await import("node:crypto")).randomBytes(16).toString("hex");
|
|
422
|
+
await fs.writeFile(
|
|
423
|
+
path.join(localRunDir, "manifest.json"),
|
|
424
|
+
JSON.stringify({ run_id: runId, num_shards: shards.length, upload_token: uploadToken, created_at: new Date().toISOString() }, null, 2)
|
|
425
|
+
);
|
|
426
|
+
|
|
427
|
+
// Build the canonical URLs the control plane will hand out to claimers.
|
|
428
|
+
const base = advertised.replace(/\/$/, "");
|
|
429
|
+
const datasetBaseUrl = `${base}/v1/training/shards/${runId}`;
|
|
430
|
+
const uploadBaseUrl = `${base}/v1/training/adapters/${runId}?token=${uploadToken}`;
|
|
431
|
+
|
|
432
|
+
process.stdout.write(` shard URLs: ${datasetBaseUrl}/shard-N.jsonl\n`);
|
|
433
|
+
process.stdout.write(` adapter PUT URL: ${uploadBaseUrl.replace(uploadToken, "<token>")}\n`);
|
|
434
|
+
|
|
435
|
+
// Submit the job
|
|
436
|
+
process.stdout.write(`\nPosting job to the open market…\n`);
|
|
437
|
+
const res = await fetch(`${controlPlaneUrl}/api/v1/training/jobs`, {
|
|
438
|
+
method: "POST",
|
|
439
|
+
headers: {
|
|
440
|
+
Authorization: `Bearer ${sessionToken}`,
|
|
441
|
+
"Content-Type": "application/json"
|
|
442
|
+
},
|
|
443
|
+
body: JSON.stringify({
|
|
444
|
+
name: trainConfig.name ?? runId,
|
|
445
|
+
base_model: trainConfig.base_model,
|
|
446
|
+
config: trainConfig,
|
|
447
|
+
dataset_base_url: datasetBaseUrl,
|
|
448
|
+
upload_base_url: uploadBaseUrl,
|
|
449
|
+
num_shards: shards.length,
|
|
450
|
+
min_vram_gb: trainConfig.resources?.min_vram_gb ?? 16,
|
|
451
|
+
price_per_shard_usd: pricePerShard
|
|
452
|
+
})
|
|
453
|
+
});
|
|
454
|
+
if (!res.ok) {
|
|
455
|
+
const body = await res.text();
|
|
456
|
+
process.stderr.write(`submit failed: HTTP ${res.status} ${body.slice(0, 300)}\n`);
|
|
457
|
+
return 1;
|
|
458
|
+
}
|
|
459
|
+
const { data } = await res.json();
|
|
460
|
+
process.stdout.write(`\n✓ job ${data.job.id} posted (${data.shards.length} shards · $${pricePerShard.toFixed(4)}/shard · $${budget.toFixed(2)} budget)\n`);
|
|
461
|
+
process.stdout.write(`Shards hosted by your local daemon at ${base}\n`);
|
|
462
|
+
process.stdout.write(`Adapters land in ${path.join(localRunDir, "adapters")} as operators report\n`);
|
|
463
|
+
process.stdout.write(`\nKeep the daemon running until all shards complete. When done:\n`);
|
|
464
|
+
process.stdout.write(` python3 ${path.join(runDir, "_fedavg.py")} --out ${path.join(runDir, "checkpoint-final")} ${path.join(localRunDir, "adapters")}/*\n`);
|
|
465
|
+
|
|
466
|
+
await fs.writeFile(path.join(runDir, "open-market-job.json"), JSON.stringify(data, null, 2));
|
|
467
|
+
return 0;
|
|
468
|
+
}
|
|
469
|
+
|
|
345
470
|
async function runUnslothLocal({ trainConfig, runDir, runId }) {
|
|
346
471
|
const dataset = trainConfig.input?.dataset;
|
|
347
472
|
if (!dataset) {
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Petals engine adapter — DISTRIBUTED inference across the volunteer
|
|
3
|
+
* swarm (IPIP-0031, follow-up PR).
|
|
4
|
+
*
|
|
5
|
+
* Petals (https://github.com/bigscience-workshop/petals) splits a
|
|
6
|
+
* Llama/Mixtral-style transformer by layer across DHT-discovered nodes.
|
|
7
|
+
* Each node holds a few transformer blocks; activations flow through.
|
|
8
|
+
* Designed for residential bandwidth — exactly the Infernet model.
|
|
9
|
+
*
|
|
10
|
+
* STATUS: scaffolding only. The flow:
|
|
11
|
+
*
|
|
12
|
+
* serve() — operator runs `infernet inference serve --backend petals
|
|
13
|
+
* --model meta-llama/Llama-3.1-70B`. We:
|
|
14
|
+
* 1. spawn `python -m petals.cli.run_server <model>`
|
|
15
|
+
* 2. register the operator as a Petals server in our
|
|
16
|
+
* own registry: row in providers.specs.petals_models
|
|
17
|
+
* 3. heartbeat that the server is alive
|
|
18
|
+
*
|
|
19
|
+
* client() — control plane's chat router, when input_spec.distributed
|
|
20
|
+
* is true, runs a Petals CLIENT (Python subprocess or
|
|
21
|
+
* a thin TS wrapper) that fans out across the swarm.
|
|
22
|
+
* Tokens stream back to the SSE handler exactly like
|
|
23
|
+
* Ollama's stream.
|
|
24
|
+
*
|
|
25
|
+
* The SSE handler in apps/web/app/api/chat/stream/[jobId]/route.js
|
|
26
|
+
* already has the typed events plumbing — when distributed===true,
|
|
27
|
+
* it would call into routePetalsRequest below instead of subscribing
|
|
28
|
+
* to job_events.
|
|
29
|
+
*
|
|
30
|
+
* To-do for the follow-up PR:
|
|
31
|
+
* - bundle a thin Python venv with Petals at infernet setup time
|
|
32
|
+
* - add `infernet inference serve --backend petals` as a top-level command
|
|
33
|
+
* - register petals_models on heartbeat so the dashboard can show
|
|
34
|
+
* "this model is served by 8 nodes via Petals · ~12 tok/s" badges
|
|
35
|
+
* - wire the SSE handler's distributed branch through routePetalsRequest
|
|
36
|
+
* - per-token receipts to operators contributing layers (CPR / IPIP-0007)
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
import { spawn } from "node:child_process";
|
|
40
|
+
|
|
41
|
+
const DEFAULT_DHT_PREFIX = "/petals/v3-public";
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Start a Petals server contributing this node's VRAM to the swarm.
|
|
45
|
+
* Returns a child handle the daemon can monitor + restart.
|
|
46
|
+
*/
|
|
47
|
+
export function startPetalsServer({ model, numBlocks, port = 31330, prefix = DEFAULT_DHT_PREFIX }) {
|
|
48
|
+
const args = ["-m", "petals.cli.run_server", model];
|
|
49
|
+
if (numBlocks) args.push("--num_blocks", String(numBlocks));
|
|
50
|
+
if (port) args.push("--port", String(port));
|
|
51
|
+
if (prefix) args.push("--initial_peers", prefix);
|
|
52
|
+
const child = spawn("python3", args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
53
|
+
return child;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Run a single inference through a Petals client and stream tokens.
|
|
58
|
+
* Returns an async iterable of { type: 'token' | 'done' | 'error', data }.
|
|
59
|
+
*
|
|
60
|
+
* Currently a stub — wires up the subprocess but doesn't translate the
|
|
61
|
+
* Python output into typed events. Follow-up PR: pip install
|
|
62
|
+
* `petals` in our control-plane container and call from a Python
|
|
63
|
+
* worker, OR use a JS-native bridge once one exists.
|
|
64
|
+
*/
|
|
65
|
+
export async function* routePetalsRequest({ model, messages, maxTokens = 256, temperature = 0.7 }) {
|
|
66
|
+
yield {
|
|
67
|
+
type: "error",
|
|
68
|
+
data: {
|
|
69
|
+
message:
|
|
70
|
+
`Distributed inference via Petals is in scaffolding (IPIP-0031). ` +
|
|
71
|
+
`For now, this request needs to fall back to a single-node provider. ` +
|
|
72
|
+
`Track progress at https://github.com/infernetprotocol/infernet-protocol/issues`
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
}
|
package/lib/node-client.js
CHANGED
|
@@ -131,6 +131,26 @@ export function createNodeClient({ url, publicKey, privateKey, role, timeoutMs =
|
|
|
131
131
|
);
|
|
132
132
|
},
|
|
133
133
|
|
|
134
|
+
// ---- Open-market training (IPIP-0030) ----
|
|
135
|
+
|
|
136
|
+
listAvailableTrainingShards(limit = 5) {
|
|
137
|
+
return signedFetch('/api/v1/training/shards/available', { limit });
|
|
138
|
+
},
|
|
139
|
+
|
|
140
|
+
claimTrainingShard(shardId) {
|
|
141
|
+
return signedFetch(
|
|
142
|
+
`/api/v1/training/shards/${encodeURIComponent(shardId)}/claim`,
|
|
143
|
+
{}
|
|
144
|
+
);
|
|
145
|
+
},
|
|
146
|
+
|
|
147
|
+
reportTrainingShard(shardId, payload) {
|
|
148
|
+
return signedFetch(
|
|
149
|
+
`/api/v1/training/shards/${encodeURIComponent(shardId)}/report`,
|
|
150
|
+
payload
|
|
151
|
+
);
|
|
152
|
+
},
|
|
153
|
+
|
|
134
154
|
listPayouts() {
|
|
135
155
|
return signedFetch('/api/v1/node/payouts/list', {});
|
|
136
156
|
},
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.36",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -30,11 +30,11 @@
|
|
|
30
30
|
"ora": "^9.4.0",
|
|
31
31
|
"react": "^17.0.2",
|
|
32
32
|
"react-blessed": "^0.7.2",
|
|
33
|
-
"@infernetprotocol/auth": "0.1.
|
|
34
|
-
"@infernetprotocol/
|
|
35
|
-
"@infernetprotocol/
|
|
36
|
-
"@infernetprotocol/engine": "0.1.
|
|
37
|
-
"@infernetprotocol/gpu": "0.1.
|
|
33
|
+
"@infernetprotocol/auth": "0.1.36",
|
|
34
|
+
"@infernetprotocol/config": "0.1.36",
|
|
35
|
+
"@infernetprotocol/deploy-providers": "0.1.36",
|
|
36
|
+
"@infernetprotocol/engine": "0.1.36",
|
|
37
|
+
"@infernetprotocol/gpu": "0.1.36"
|
|
38
38
|
},
|
|
39
39
|
"repository": {
|
|
40
40
|
"type": "git",
|