@infernetprotocol/cli 0.1.95 → 0.1.96
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/setup.js +7 -2
- package/commands/start.js +22 -5
- package/lib/concurrency.js +105 -0
- package/package.json +9 -9
package/commands/setup.js
CHANGED
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
import { isDaemonAlive } from "../lib/ipc.js";
|
|
36
36
|
import { ensureService, systemUnitPresent } from "./service.js";
|
|
37
37
|
import { createNodeClient } from "../lib/node-client.js";
|
|
38
|
+
import { slotsForVram, totalVramGbOf } from "../lib/concurrency.js";
|
|
38
39
|
|
|
39
40
|
const pExec = promisify(execFile);
|
|
40
41
|
|
|
@@ -201,7 +202,7 @@ async function confirmRun({ label, command, yes }) {
|
|
|
201
202
|
});
|
|
202
203
|
}
|
|
203
204
|
|
|
204
|
-
async function startOllama({ yes }) {
|
|
205
|
+
async function startOllama({ yes, gpus = [] }) {
|
|
205
206
|
// Linux: usually has the ollama systemd unit from the installer.
|
|
206
207
|
// macOS: usually opens the Ollama.app. Fall back to `ollama serve &`.
|
|
207
208
|
// Non-systemd Linux (containers, WSL, distros without systemd as
|
|
@@ -221,8 +222,12 @@ async function startOllama({ yes }) {
|
|
|
221
222
|
// Flash attention + q8_0 KV cache unless the operator set them: halves
|
|
222
223
|
// the KV cache, so long-context models fit the GPU (see install.sh
|
|
223
224
|
// ensure_ollama_memory_env, which does the same for ollama.service).
|
|
225
|
+
// GPUs with 10 GB+ also get parallel slots (lib/concurrency.js), the
|
|
226
|
+
// same number the daemon then runs jobs at once.
|
|
227
|
+
const slots = slotsForVram(totalVramGbOf(gpus));
|
|
224
228
|
const NOHUP = "(OLLAMA_FLASH_ATTENTION=${OLLAMA_FLASH_ATTENTION:-1}"
|
|
225
229
|
+ " OLLAMA_KV_CACHE_TYPE=${OLLAMA_KV_CACHE_TYPE:-q8_0}"
|
|
230
|
+
+ (slots >= 2 ? ` OLLAMA_NUM_PARALLEL=\${OLLAMA_NUM_PARALLEL:-${slots}}` : "")
|
|
226
231
|
+ " nohup ollama serve > /tmp/ollama.log 2>&1 < /dev/null &)";
|
|
227
232
|
let cmd;
|
|
228
233
|
if (isMac) {
|
|
@@ -675,7 +680,7 @@ export default async function setup(args) {
|
|
|
675
680
|
? `Ollama installed (${ollamaState.version}) but not reachable at ${host}`
|
|
676
681
|
: `Ollama installed but not reachable at ${host}`
|
|
677
682
|
);
|
|
678
|
-
const started = await startOllama({ yes });
|
|
683
|
+
const started = await startOllama({ yes, gpus: detectedGpus });
|
|
679
684
|
if (started) {
|
|
680
685
|
// Background `ollama serve &` returns instantly; the
|
|
681
686
|
// daemon needs a few seconds to bind on cold start.
|
package/commands/start.js
CHANGED
|
@@ -59,6 +59,7 @@ import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
|
|
|
59
59
|
import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
|
|
60
60
|
import { startVllmServe, stopVllmServe, vllmInstalled, waitForVllmModel, extractVllmError } from '../lib/vllm.js';
|
|
61
61
|
import { detectGpus, detectHost } from '@infernetprotocol/gpu';
|
|
62
|
+
import { ollamaNumParallel, resolveMaxConcurrentJobs, totalVramGbOf } from '../lib/concurrency.js';
|
|
62
63
|
import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
|
|
63
64
|
import { pullLatestBinary } from './upgrade.js';
|
|
64
65
|
import { PROBE_INTERVAL_MS, runProbeRound } from '../lib/aggregator.js';
|
|
@@ -227,10 +228,18 @@ async function runDaemon(args, ctx) {
|
|
|
227
228
|
const heartbeatMs = Number.parseInt(args.get('heartbeat-interval') ?? '', 10) || DEFAULT_HEARTBEAT_MS;
|
|
228
229
|
const pollMs = Number.parseInt(args.get('poll-interval') ?? '', 10) || DEFAULT_POLL_MS;
|
|
229
230
|
const once = args.has('once');
|
|
230
|
-
//
|
|
231
|
-
//
|
|
232
|
-
//
|
|
233
|
-
|
|
231
|
+
// Jobs run at once: capacity-based by default (lib/concurrency.js). CPU
|
|
232
|
+
// nodes run 1; GPU nodes as many as Ollama has parallel slots
|
|
233
|
+
// (OLLAMA_NUM_PARALLEL, which install.sh sizes from VRAM). Override with
|
|
234
|
+
// INFERNET_MAX_CONCURRENT_JOBS or engine.max_concurrent_jobs.
|
|
235
|
+
let totalVramGb = 0;
|
|
236
|
+
try { totalVramGb = totalVramGbOf(await detectGpus()); } catch { /* CPU-only */ }
|
|
237
|
+
const concurrency = resolveMaxConcurrentJobs({
|
|
238
|
+
config,
|
|
239
|
+
totalVramGb,
|
|
240
|
+
runtimeSlots: totalVramGb > 0 ? await ollamaNumParallel() : null
|
|
241
|
+
});
|
|
242
|
+
const maxConcurrentJobs = concurrency.value;
|
|
234
243
|
|
|
235
244
|
const p2pDisabled = args.has('no-p2p');
|
|
236
245
|
const noAdvertise = args.has('no-advertise') || node.address === null;
|
|
@@ -293,6 +302,7 @@ async function runDaemon(args, ctx) {
|
|
|
293
302
|
process.stdout.write(` config: ${configPath}\n`);
|
|
294
303
|
process.stdout.write(` heartbeat: ${heartbeatMs}ms\n`);
|
|
295
304
|
process.stdout.write(` poll: ${pollMs}ms\n`);
|
|
305
|
+
process.stdout.write(` jobs: up to ${maxConcurrentJobs} at once (${concurrency.source})\n`);
|
|
296
306
|
if (!p2pDisabled) {
|
|
297
307
|
process.stdout.write(` p2p: ${formatEndpoint(advertisedAddress ?? '-', advertisedPort)}${bindPort !== advertisedPort ? ` (binds locally :${bindPort})` : ''}\n`);
|
|
298
308
|
} else {
|
|
@@ -661,7 +671,14 @@ async function runDaemon(args, ctx) {
|
|
|
661
671
|
// and the restarted daemon picks them up.
|
|
662
672
|
if (stats.draining) break;
|
|
663
673
|
if (stats.activeJobIds.size >= maxConcurrentJobs) break;
|
|
664
|
-
|
|
674
|
+
if (maxConcurrentJobs <= 1) {
|
|
675
|
+
await processJob(job);
|
|
676
|
+
} else {
|
|
677
|
+
// Parallel slots: start it and keep filling slots from
|
|
678
|
+
// this batch. processJob marks the job active before its
|
|
679
|
+
// first await, so the cap check above sees it.
|
|
680
|
+
processJob(job).catch((err) => process.stderr.write(`job ${job.id} threw: ${err?.message ?? err}\n`));
|
|
681
|
+
}
|
|
665
682
|
}
|
|
666
683
|
} catch (err) {
|
|
667
684
|
stats.pollsFailed += 1;
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How many jobs a node runs at once (Anthony, 2026-11-18: "a node should be
|
|
3
|
+
* able to handle parallel jobs if capacity exists, make that the default").
|
|
4
|
+
*
|
|
5
|
+
* Until 0.1.96 every node ran one job at a time unless the operator set
|
|
6
|
+
* INFERNET_MAX_CONCURRENT_JOBS, and a second request queued behind the first.
|
|
7
|
+
* Now the default follows capacity:
|
|
8
|
+
*
|
|
9
|
+
* - CPU-only nodes: 1. CPU Ollama shares the same cores between requests,
|
|
10
|
+
* so two at once each run at half speed and both finish later.
|
|
11
|
+
* - GPU nodes: the runtime's parallel slots. When Ollama's
|
|
12
|
+
* OLLAMA_NUM_PARALLEL is known (set on ollama.service, by install.sh or
|
|
13
|
+
* the operator, or in this process's environment) that is the number;
|
|
14
|
+
* otherwise it is sized from total VRAM (slotsForVram), the same table
|
|
15
|
+
* install.sh uses to set OLLAMA_NUM_PARALLEL.
|
|
16
|
+
*
|
|
17
|
+
* Operator override, in order: INFERNET_MAX_CONCURRENT_JOBS, then
|
|
18
|
+
* `engine.max_concurrent_jobs` in the node config.
|
|
19
|
+
*
|
|
20
|
+
* The heartbeat advertises the result as specs.load.concurrency_cap, and the
|
|
21
|
+
* control plane routes a second job to a node only while it has a free slot
|
|
22
|
+
* (apps/web/lib/routing/smart-pick.js).
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { execFile } from 'node:child_process';
|
|
26
|
+
|
|
27
|
+
/** Max slots we size from VRAM alone. */
|
|
28
|
+
export const MAX_AUTO_SLOTS = 4;
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Parallel slots for a GPU with `totalVramGb` of memory. Each Ollama slot
|
|
32
|
+
* gets its own KV cache for the full context (32K, q8_0 with flash
|
|
33
|
+
* attention). The house 9B takes 6.2 GB with one slot, so a 12 GB card has
|
|
34
|
+
* room for a second; an 8 GB card does not.
|
|
35
|
+
* no GPU or < 10 GB -> 1, 10-20 GB -> 2, 20-40 GB -> 3, 40 GB+ -> 4.
|
|
36
|
+
* Keep in sync with ollama_parallel_slots in install.sh.
|
|
37
|
+
*/
|
|
38
|
+
export function slotsForVram(totalVramGb) {
|
|
39
|
+
const v = Number(totalVramGb);
|
|
40
|
+
if (!Number.isFinite(v) || v < 10) return 1;
|
|
41
|
+
if (v < 20) return 2;
|
|
42
|
+
if (v < 40) return 3;
|
|
43
|
+
return MAX_AUTO_SLOTS;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function positiveInt(value) {
|
|
47
|
+
if (value === undefined || value === null || String(value).trim() === '') return null;
|
|
48
|
+
const n = Number.parseInt(String(value), 10);
|
|
49
|
+
return Number.isFinite(n) && n > 0 ? n : null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** OLLAMA_NUM_PARALLEL out of `systemctl show -p Environment --value` output. */
|
|
53
|
+
export function parseOllamaNumParallel(environment) {
|
|
54
|
+
if (typeof environment !== 'string') return null;
|
|
55
|
+
const m = environment.match(/(?:^|\s|")OLLAMA_NUM_PARALLEL=(\d+)/);
|
|
56
|
+
return m ? positiveInt(m[1]) : null;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The OLLAMA_NUM_PARALLEL the local Ollama runs with, or null when unknown.
|
|
61
|
+
* Reads ollama.service (no root needed), then this process's environment
|
|
62
|
+
* (the `ollama serve` fallback that `infernet setup` starts inherits it).
|
|
63
|
+
*/
|
|
64
|
+
export async function ollamaNumParallel({ env = process.env, run = systemctlEnvironment } = {}) {
|
|
65
|
+
const fromUnit = parseOllamaNumParallel(await run().catch(() => null));
|
|
66
|
+
if (fromUnit) return fromUnit;
|
|
67
|
+
return positiveInt(env.OLLAMA_NUM_PARALLEL);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function systemctlEnvironment() {
|
|
71
|
+
if (process.platform !== 'linux') return Promise.resolve(null);
|
|
72
|
+
return new Promise((resolve) => {
|
|
73
|
+
execFile('systemctl', ['show', 'ollama.service', '-p', 'Environment', '--value'], { timeout: 3000 }, (err, stdout) => {
|
|
74
|
+
resolve(err ? null : String(stdout));
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Pure: decide the daemon's concurrency. Returns { value, source }.
|
|
81
|
+
* env process env (INFERNET_MAX_CONCURRENT_JOBS)
|
|
82
|
+
* config node config (engine.max_concurrent_jobs)
|
|
83
|
+
* totalVramGb summed GPU memory; 0/null on CPU-only nodes
|
|
84
|
+
* runtimeSlots the runtime's parallel slots when known (OLLAMA_NUM_PARALLEL)
|
|
85
|
+
*/
|
|
86
|
+
export function resolveMaxConcurrentJobs({ env = process.env, config = {}, totalVramGb = null, runtimeSlots = null } = {}) {
|
|
87
|
+
const fromEnv = positiveInt(env.INFERNET_MAX_CONCURRENT_JOBS);
|
|
88
|
+
if (fromEnv) return { value: fromEnv, source: 'env INFERNET_MAX_CONCURRENT_JOBS' };
|
|
89
|
+
const fromConfig = positiveInt(config?.engine?.max_concurrent_jobs);
|
|
90
|
+
if (fromConfig) return { value: fromConfig, source: 'config engine.max_concurrent_jobs' };
|
|
91
|
+
const vram = Number(totalVramGb);
|
|
92
|
+
if (!Number.isFinite(vram) || vram <= 0) return { value: 1, source: 'cpu-only node' };
|
|
93
|
+
const slots = positiveInt(runtimeSlots);
|
|
94
|
+
if (slots) return { value: slots, source: 'OLLAMA_NUM_PARALLEL' };
|
|
95
|
+
return { value: slotsForVram(vram), source: `${vram.toFixed(0)} GB VRAM` };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Summed VRAM (GB) from @infernetprotocol/gpu detectGpus() output. */
|
|
99
|
+
export function totalVramGbOf(gpus) {
|
|
100
|
+
let total = 0;
|
|
101
|
+
for (const g of Array.isArray(gpus) ? gpus : []) {
|
|
102
|
+
if (Number.isFinite(g?.vram_mb)) total += g.vram_mb / 1024;
|
|
103
|
+
}
|
|
104
|
+
return total;
|
|
105
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.96",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -32,14 +32,14 @@
|
|
|
32
32
|
"ora": "^9.4.1",
|
|
33
33
|
"react": "^17.0.2",
|
|
34
34
|
"react-blessed": "^0.7.2",
|
|
35
|
-
"@infernetprotocol/auth": "0.1.
|
|
36
|
-
"@infernetprotocol/config": "0.1.
|
|
37
|
-
"@infernetprotocol/deploy-providers": "0.1.
|
|
38
|
-
"@infernetprotocol/engine": "0.1.
|
|
39
|
-
"@infernetprotocol/gpu": "0.1.
|
|
40
|
-
"@infernetprotocol/liquid-agent": "0.1.
|
|
41
|
-
"@infernetprotocol/rpc-adapter": "0.1.
|
|
42
|
-
"@infernetprotocol/training": "0.1.
|
|
35
|
+
"@infernetprotocol/auth": "0.1.96",
|
|
36
|
+
"@infernetprotocol/config": "0.1.96",
|
|
37
|
+
"@infernetprotocol/deploy-providers": "0.1.96",
|
|
38
|
+
"@infernetprotocol/engine": "0.1.96",
|
|
39
|
+
"@infernetprotocol/gpu": "0.1.96",
|
|
40
|
+
"@infernetprotocol/liquid-agent": "0.1.96",
|
|
41
|
+
"@infernetprotocol/rpc-adapter": "0.1.96",
|
|
42
|
+
"@infernetprotocol/training": "0.1.96"
|
|
43
43
|
},
|
|
44
44
|
"repository": {
|
|
45
45
|
"type": "git",
|