@infernetprotocol/cli 0.1.95 → 0.1.96

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/commands/setup.js CHANGED
@@ -35,6 +35,7 @@ import {
35
35
  import { isDaemonAlive } from "../lib/ipc.js";
36
36
  import { ensureService, systemUnitPresent } from "./service.js";
37
37
  import { createNodeClient } from "../lib/node-client.js";
38
+ import { slotsForVram, totalVramGbOf } from "../lib/concurrency.js";
38
39
 
39
40
  const pExec = promisify(execFile);
40
41
 
@@ -201,7 +202,7 @@ async function confirmRun({ label, command, yes }) {
201
202
  });
202
203
  }
203
204
 
204
- async function startOllama({ yes }) {
205
+ async function startOllama({ yes, gpus = [] }) {
205
206
  // Linux: usually has the ollama systemd unit from the installer.
206
207
  // macOS: usually opens the Ollama.app. Fall back to `ollama serve &`.
207
208
  // Non-systemd Linux (containers, WSL, distros without systemd as
@@ -221,8 +222,12 @@ async function startOllama({ yes }) {
221
222
  // Flash attention + q8_0 KV cache unless the operator set them: halves
222
223
  // the KV cache, so long-context models fit the GPU (see install.sh
223
224
  // ensure_ollama_memory_env, which does the same for ollama.service).
225
+ // GPUs with 10 GB+ also get parallel slots (lib/concurrency.js), the
226
+ // same number the daemon then runs jobs at once.
227
+ const slots = slotsForVram(totalVramGbOf(gpus));
224
228
  const NOHUP = "(OLLAMA_FLASH_ATTENTION=${OLLAMA_FLASH_ATTENTION:-1}"
225
229
  + " OLLAMA_KV_CACHE_TYPE=${OLLAMA_KV_CACHE_TYPE:-q8_0}"
230
+ + (slots >= 2 ? ` OLLAMA_NUM_PARALLEL=\${OLLAMA_NUM_PARALLEL:-${slots}}` : "")
226
231
  + " nohup ollama serve > /tmp/ollama.log 2>&1 < /dev/null &)";
227
232
  let cmd;
228
233
  if (isMac) {
@@ -675,7 +680,7 @@ export default async function setup(args) {
675
680
  ? `Ollama installed (${ollamaState.version}) but not reachable at ${host}`
676
681
  : `Ollama installed but not reachable at ${host}`
677
682
  );
678
- const started = await startOllama({ yes });
683
+ const started = await startOllama({ yes, gpus: detectedGpus });
679
684
  if (started) {
680
685
  // Background `ollama serve &` returns instantly; the
681
686
  // daemon needs a few seconds to bind on cold start.
package/commands/start.js CHANGED
@@ -59,6 +59,7 @@ import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
59
59
  import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
60
60
  import { startVllmServe, stopVllmServe, vllmInstalled, waitForVllmModel, extractVllmError } from '../lib/vllm.js';
61
61
  import { detectGpus, detectHost } from '@infernetprotocol/gpu';
62
+ import { ollamaNumParallel, resolveMaxConcurrentJobs, totalVramGbOf } from '../lib/concurrency.js';
62
63
  import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
63
64
  import { pullLatestBinary } from './upgrade.js';
64
65
  import { PROBE_INTERVAL_MS, runProbeRound } from '../lib/aggregator.js';
@@ -227,10 +228,18 @@ async function runDaemon(args, ctx) {
227
228
  const heartbeatMs = Number.parseInt(args.get('heartbeat-interval') ?? '', 10) || DEFAULT_HEARTBEAT_MS;
228
229
  const pollMs = Number.parseInt(args.get('poll-interval') ?? '', 10) || DEFAULT_POLL_MS;
229
230
  const once = args.has('once');
230
- // Chat jobs run at once. Ollama on one GPU serves a single request at a
231
- // time unless OLLAMA_NUM_PARALLEL is raised, so extra jobs only queue
232
- // inside it; raise this together with that.
233
- const maxConcurrentJobs = Number.parseInt(process.env.INFERNET_MAX_CONCURRENT_JOBS ?? '', 10) || 1;
231
+ // Jobs run at once: capacity-based by default (lib/concurrency.js). CPU
232
+ // nodes run 1; GPU nodes as many as Ollama has parallel slots
233
+ // (OLLAMA_NUM_PARALLEL, which install.sh sizes from VRAM). Override with
234
+ // INFERNET_MAX_CONCURRENT_JOBS or engine.max_concurrent_jobs.
235
+ let totalVramGb = 0;
236
+ try { totalVramGb = totalVramGbOf(await detectGpus()); } catch { /* CPU-only */ }
237
+ const concurrency = resolveMaxConcurrentJobs({
238
+ config,
239
+ totalVramGb,
240
+ runtimeSlots: totalVramGb > 0 ? await ollamaNumParallel() : null
241
+ });
242
+ const maxConcurrentJobs = concurrency.value;
234
243
 
235
244
  const p2pDisabled = args.has('no-p2p');
236
245
  const noAdvertise = args.has('no-advertise') || node.address === null;
@@ -293,6 +302,7 @@ async function runDaemon(args, ctx) {
293
302
  process.stdout.write(` config: ${configPath}\n`);
294
303
  process.stdout.write(` heartbeat: ${heartbeatMs}ms\n`);
295
304
  process.stdout.write(` poll: ${pollMs}ms\n`);
305
+ process.stdout.write(` jobs: up to ${maxConcurrentJobs} at once (${concurrency.source})\n`);
296
306
  if (!p2pDisabled) {
297
307
  process.stdout.write(` p2p: ${formatEndpoint(advertisedAddress ?? '-', advertisedPort)}${bindPort !== advertisedPort ? ` (binds locally :${bindPort})` : ''}\n`);
298
308
  } else {
@@ -661,7 +671,14 @@ async function runDaemon(args, ctx) {
661
671
  // and the restarted daemon picks them up.
662
672
  if (stats.draining) break;
663
673
  if (stats.activeJobIds.size >= maxConcurrentJobs) break;
664
- await processJob(job);
674
+ if (maxConcurrentJobs <= 1) {
675
+ await processJob(job);
676
+ } else {
677
+ // Parallel slots: start it and keep filling slots from
678
+ // this batch. processJob marks the job active before its
679
+ // first await, so the cap check above sees it.
680
+ processJob(job).catch((err) => process.stderr.write(`job ${job.id} threw: ${err?.message ?? err}\n`));
681
+ }
665
682
  }
666
683
  } catch (err) {
667
684
  stats.pollsFailed += 1;
@@ -0,0 +1,105 @@
1
+ /**
2
+ * How many jobs a node runs at once (Anthony, 2026-11-18: "a node should be
3
+ * able to handle parallel jobs if capacity exists, make that the default").
4
+ *
5
+ * Until 0.1.96 every node ran one job at a time unless the operator set
6
+ * INFERNET_MAX_CONCURRENT_JOBS, and a second request queued behind the first.
7
+ * Now the default follows capacity:
8
+ *
9
+ * - CPU-only nodes: 1. CPU Ollama shares the same cores between requests,
10
+ * so two at once each run at half speed and both finish later.
11
+ * - GPU nodes: the runtime's parallel slots. When Ollama's
12
+ * OLLAMA_NUM_PARALLEL is known (set on ollama.service, by install.sh or
13
+ * the operator, or in this process's environment) that is the number;
14
+ * otherwise it is sized from total VRAM (slotsForVram), the same table
15
+ * install.sh uses to set OLLAMA_NUM_PARALLEL.
16
+ *
17
+ * Operator override, in order: INFERNET_MAX_CONCURRENT_JOBS, then
18
+ * `engine.max_concurrent_jobs` in the node config.
19
+ *
20
+ * The heartbeat advertises the result as specs.load.concurrency_cap, and the
21
+ * control plane routes a second job to a node only while it has a free slot
22
+ * (apps/web/lib/routing/smart-pick.js).
23
+ */
24
+
25
+ import { execFile } from 'node:child_process';
26
+
27
+ /** Max slots we size from VRAM alone. */
28
+ export const MAX_AUTO_SLOTS = 4;
29
+
30
+ /**
31
+ * Parallel slots for a GPU with `totalVramGb` of memory. Each Ollama slot
32
+ * gets its own KV cache for the full context (32K, q8_0 with flash
33
+ * attention). The house 9B takes 6.2 GB with one slot, so a 12 GB card has
34
+ * room for a second; an 8 GB card does not.
35
+ * no GPU or < 10 GB -> 1, 10-20 GB -> 2, 20-40 GB -> 3, 40 GB+ -> 4.
36
+ * Keep in sync with ollama_parallel_slots in install.sh.
37
+ */
38
+ export function slotsForVram(totalVramGb) {
39
+ const v = Number(totalVramGb);
40
+ if (!Number.isFinite(v) || v < 10) return 1;
41
+ if (v < 20) return 2;
42
+ if (v < 40) return 3;
43
+ return MAX_AUTO_SLOTS;
44
+ }
45
+
46
+ function positiveInt(value) {
47
+ if (value === undefined || value === null || String(value).trim() === '') return null;
48
+ const n = Number.parseInt(String(value), 10);
49
+ return Number.isFinite(n) && n > 0 ? n : null;
50
+ }
51
+
52
+ /** OLLAMA_NUM_PARALLEL out of `systemctl show -p Environment --value` output. */
53
+ export function parseOllamaNumParallel(environment) {
54
+ if (typeof environment !== 'string') return null;
55
+ const m = environment.match(/(?:^|\s|")OLLAMA_NUM_PARALLEL=(\d+)/);
56
+ return m ? positiveInt(m[1]) : null;
57
+ }
58
+
59
+ /**
60
+ * The OLLAMA_NUM_PARALLEL the local Ollama runs with, or null when unknown.
61
+ * Reads ollama.service (no root needed), then this process's environment
62
+ * (the `ollama serve` fallback that `infernet setup` starts inherits it).
63
+ */
64
+ export async function ollamaNumParallel({ env = process.env, run = systemctlEnvironment } = {}) {
65
+ const fromUnit = parseOllamaNumParallel(await run().catch(() => null));
66
+ if (fromUnit) return fromUnit;
67
+ return positiveInt(env.OLLAMA_NUM_PARALLEL);
68
+ }
69
+
70
+ function systemctlEnvironment() {
71
+ if (process.platform !== 'linux') return Promise.resolve(null);
72
+ return new Promise((resolve) => {
73
+ execFile('systemctl', ['show', 'ollama.service', '-p', 'Environment', '--value'], { timeout: 3000 }, (err, stdout) => {
74
+ resolve(err ? null : String(stdout));
75
+ });
76
+ });
77
+ }
78
+
79
+ /**
80
+ * Pure: decide the daemon's concurrency. Returns { value, source }.
81
+ * env process env (INFERNET_MAX_CONCURRENT_JOBS)
82
+ * config node config (engine.max_concurrent_jobs)
83
+ * totalVramGb summed GPU memory; 0/null on CPU-only nodes
84
+ * runtimeSlots the runtime's parallel slots when known (OLLAMA_NUM_PARALLEL)
85
+ */
86
+ export function resolveMaxConcurrentJobs({ env = process.env, config = {}, totalVramGb = null, runtimeSlots = null } = {}) {
87
+ const fromEnv = positiveInt(env.INFERNET_MAX_CONCURRENT_JOBS);
88
+ if (fromEnv) return { value: fromEnv, source: 'env INFERNET_MAX_CONCURRENT_JOBS' };
89
+ const fromConfig = positiveInt(config?.engine?.max_concurrent_jobs);
90
+ if (fromConfig) return { value: fromConfig, source: 'config engine.max_concurrent_jobs' };
91
+ const vram = Number(totalVramGb);
92
+ if (!Number.isFinite(vram) || vram <= 0) return { value: 1, source: 'cpu-only node' };
93
+ const slots = positiveInt(runtimeSlots);
94
+ if (slots) return { value: slots, source: 'OLLAMA_NUM_PARALLEL' };
95
+ return { value: slotsForVram(vram), source: `${vram.toFixed(0)} GB VRAM` };
96
+ }
97
+
98
+ /** Summed VRAM (GB) from @infernetprotocol/gpu detectGpus() output. */
99
+ export function totalVramGbOf(gpus) {
100
+ let total = 0;
101
+ for (const g of Array.isArray(gpus) ? gpus : []) {
102
+ if (Number.isFinite(g?.vram_mb)) total += g.vram_mb / 1024;
103
+ }
104
+ return total;
105
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@infernetprotocol/cli",
3
- "version": "0.1.95",
3
+ "version": "0.1.96",
4
4
  "description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -32,14 +32,14 @@
32
32
  "ora": "^9.4.1",
33
33
  "react": "^17.0.2",
34
34
  "react-blessed": "^0.7.2",
35
- "@infernetprotocol/auth": "0.1.95",
36
- "@infernetprotocol/config": "0.1.95",
37
- "@infernetprotocol/deploy-providers": "0.1.95",
38
- "@infernetprotocol/engine": "0.1.95",
39
- "@infernetprotocol/gpu": "0.1.95",
40
- "@infernetprotocol/liquid-agent": "0.1.95",
41
- "@infernetprotocol/rpc-adapter": "0.1.95",
42
- "@infernetprotocol/training": "0.1.95"
35
+ "@infernetprotocol/auth": "0.1.96",
36
+ "@infernetprotocol/config": "0.1.96",
37
+ "@infernetprotocol/deploy-providers": "0.1.96",
38
+ "@infernetprotocol/engine": "0.1.96",
39
+ "@infernetprotocol/gpu": "0.1.96",
40
+ "@infernetprotocol/liquid-agent": "0.1.96",
41
+ "@infernetprotocol/rpc-adapter": "0.1.96",
42
+ "@infernetprotocol/training": "0.1.96"
43
43
  },
44
44
  "repository": {
45
45
  "type": "git",