@infernetprotocol/cli 0.1.95 → 0.1.97

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/commands/setup.js CHANGED
@@ -35,6 +35,7 @@ import {
35
35
  import { isDaemonAlive } from "../lib/ipc.js";
36
36
  import { ensureService, systemUnitPresent } from "./service.js";
37
37
  import { createNodeClient } from "../lib/node-client.js";
38
+ import { slotsForVram, totalVramGbOf } from "../lib/concurrency.js";
38
39
 
39
40
  const pExec = promisify(execFile);
40
41
 
@@ -201,7 +202,7 @@ async function confirmRun({ label, command, yes }) {
201
202
  });
202
203
  }
203
204
 
204
- async function startOllama({ yes }) {
205
+ async function startOllama({ yes, gpus = [] }) {
205
206
  // Linux: usually has the ollama systemd unit from the installer.
206
207
  // macOS: usually opens the Ollama.app. Fall back to `ollama serve &`.
207
208
  // Non-systemd Linux (containers, WSL, distros without systemd as
@@ -221,8 +222,12 @@ async function startOllama({ yes }) {
221
222
  // Flash attention + q8_0 KV cache unless the operator set them: halves
222
223
  // the KV cache, so long-context models fit the GPU (see install.sh
223
224
  // ensure_ollama_memory_env, which does the same for ollama.service).
225
+ // GPUs with 10 GB+ also get parallel slots (lib/concurrency.js), the
226
+ // same number the daemon then runs jobs at once.
227
+ const slots = slotsForVram(totalVramGbOf(gpus));
224
228
  const NOHUP = "(OLLAMA_FLASH_ATTENTION=${OLLAMA_FLASH_ATTENTION:-1}"
225
229
  + " OLLAMA_KV_CACHE_TYPE=${OLLAMA_KV_CACHE_TYPE:-q8_0}"
230
+ + (slots >= 2 ? ` OLLAMA_NUM_PARALLEL=\${OLLAMA_NUM_PARALLEL:-${slots}}` : "")
226
231
  + " nohup ollama serve > /tmp/ollama.log 2>&1 < /dev/null &)";
227
232
  let cmd;
228
233
  if (isMac) {
@@ -675,7 +680,7 @@ export default async function setup(args) {
675
680
  ? `Ollama installed (${ollamaState.version}) but not reachable at ${host}`
676
681
  : `Ollama installed but not reachable at ${host}`
677
682
  );
678
- const started = await startOllama({ yes });
683
+ const started = await startOllama({ yes, gpus: detectedGpus });
679
684
  if (started) {
680
685
  // Background `ollama serve &` returns instantly; the
681
686
  // daemon needs a few seconds to bind on cold start.
package/commands/start.js CHANGED
@@ -59,6 +59,8 @@ import { checkModelFits, formatFitFailure } from '../lib/model-fit.js';
59
59
  import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
60
60
  import { startVllmServe, stopVllmServe, vllmInstalled, waitForVllmModel, extractVllmError } from '../lib/vllm.js';
61
61
  import { detectGpus, detectHost } from '@infernetprotocol/gpu';
62
+ import { ollamaNumParallel, resolveMaxConcurrentJobs, totalVramGbOf } from '../lib/concurrency.js';
63
+ import { requeueTurnedAway, TURN_AWAY_BACKOFF_MS } from '../lib/turn-away.js';
62
64
  import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
63
65
  import { pullLatestBinary } from './upgrade.js';
64
66
  import { PROBE_INTERVAL_MS, runProbeRound } from '../lib/aggregator.js';
@@ -227,10 +229,18 @@ async function runDaemon(args, ctx) {
227
229
  const heartbeatMs = Number.parseInt(args.get('heartbeat-interval') ?? '', 10) || DEFAULT_HEARTBEAT_MS;
228
230
  const pollMs = Number.parseInt(args.get('poll-interval') ?? '', 10) || DEFAULT_POLL_MS;
229
231
  const once = args.has('once');
230
- // Chat jobs run at once. Ollama on one GPU serves a single request at a
231
- // time unless OLLAMA_NUM_PARALLEL is raised, so extra jobs only queue
232
- // inside it; raise this together with that.
233
- const maxConcurrentJobs = Number.parseInt(process.env.INFERNET_MAX_CONCURRENT_JOBS ?? '', 10) || 1;
232
+ // Jobs run at once: capacity-based by default (lib/concurrency.js). CPU
233
+ // nodes run 1; GPU nodes as many as Ollama has parallel slots
234
+ // (OLLAMA_NUM_PARALLEL, which install.sh sizes from VRAM). Override with
235
+ // INFERNET_MAX_CONCURRENT_JOBS or engine.max_concurrent_jobs.
236
+ let totalVramGb = 0;
237
+ try { totalVramGb = totalVramGbOf(await detectGpus()); } catch { /* CPU-only */ }
238
+ const concurrency = resolveMaxConcurrentJobs({
239
+ config,
240
+ totalVramGb,
241
+ runtimeSlots: totalVramGb > 0 ? await ollamaNumParallel() : null
242
+ });
243
+ const maxConcurrentJobs = concurrency.value;
234
244
 
235
245
  const p2pDisabled = args.has('no-p2p');
236
246
  const noAdvertise = args.has('no-advertise') || node.address === null;
@@ -293,6 +303,7 @@ async function runDaemon(args, ctx) {
293
303
  process.stdout.write(` config: ${configPath}\n`);
294
304
  process.stdout.write(` heartbeat: ${heartbeatMs}ms\n`);
295
305
  process.stdout.write(` poll: ${pollMs}ms\n`);
306
+ process.stdout.write(` jobs: up to ${maxConcurrentJobs} at once (${concurrency.source})\n`);
296
307
  if (!p2pDisabled) {
297
308
  process.stdout.write(` p2p: ${formatEndpoint(advertisedAddress ?? '-', advertisedPort)}${bindPort !== advertisedPort ? ` (binds locally :${bindPort})` : ''}\n`);
298
309
  } else {
@@ -570,6 +581,9 @@ async function runDaemon(args, ctx) {
570
581
  if (finishedJobAt.size > 1000) finishedJobAt.delete(finishedJobAt.keys().next().value);
571
582
  }
572
583
 
584
+ // Jobs this daemon turned away (busy), not to be taken again before.
585
+ const turnedAwayUntil = new Map();
586
+
573
587
  async function processJob(job) {
574
588
  const t0 = new Date().toISOString();
575
589
  stats.jobsPicked += 1;
@@ -607,6 +621,15 @@ async function runDaemon(args, ctx) {
607
621
  stats.activeJobIds.delete(job.id);
608
622
  markFinished(job.id);
609
623
  const msg = err?.message ?? String(err);
624
+ // Busy (engine queue full, Ollama restarting): hand the job back
625
+ // to the front of the queue instead of failing it, and leave it
626
+ // alone here for a moment (lib/turn-away.js).
627
+ if (job.type === 'chat' && err?.turnedAway === true && await requeueTurnedAway(client, job.id, msg)) {
628
+ stats.jobsFailed -= 1;
629
+ turnedAwayUntil.set(job.id, Date.now() + TURN_AWAY_BACKOFF_MS);
630
+ process.stdout.write(`[${new Date().toISOString()}] turned away job ${job.id} (${msg}); requeued at the front\n`);
631
+ return;
632
+ }
610
633
  process.stderr.write(`job ${job.id} failed: ${msg}\n`);
611
634
  if (job.type === 'chat') {
612
635
  await failChatJob({ client, jobId: job.id, message: msg });
@@ -653,6 +676,11 @@ async function runDaemon(args, ctx) {
653
676
  // complete, so without this guard the poll loop queues the
654
677
  // same job on every tick while it's in flight.
655
678
  if (stats.activeJobIds.has(job.id)) continue;
679
+ const awayUntil = turnedAwayUntil.get(job.id);
680
+ if (awayUntil !== undefined) {
681
+ if (awayUntil > Date.now()) continue;
682
+ turnedAwayUntil.delete(job.id);
683
+ }
656
684
  if ((finishedJobAt.get(job.id) ?? 0) >= polledAt) continue;
657
685
  // A self-update can start draining while this batch runs
658
686
  // (each job is awaited in turn). Stop picking the rest up, or
@@ -661,7 +689,14 @@ async function runDaemon(args, ctx) {
661
689
  // and the restarted daemon picks them up.
662
690
  if (stats.draining) break;
663
691
  if (stats.activeJobIds.size >= maxConcurrentJobs) break;
664
- await processJob(job);
692
+ if (maxConcurrentJobs <= 1) {
693
+ await processJob(job);
694
+ } else {
695
+ // Parallel slots: start it and keep filling slots from
696
+ // this batch. processJob marks the job active before its
697
+ // first await, so the cap check above sees it.
698
+ processJob(job).catch((err) => process.stderr.write(`job ${job.id} threw: ${err?.message ?? err}\n`));
699
+ }
665
700
  }
666
701
  } catch (err) {
667
702
  stats.pollsFailed += 1;
@@ -1815,6 +1850,9 @@ async function runDaemon(args, ctx) {
1815
1850
  // Release whatever is still running after the drain window.
1816
1851
  for (const jobId of stats.activeJobIds) {
1817
1852
  try {
1853
+ // Not started yet: back to the front of the queue, for the
1854
+ // restarted daemon or a free node (lib/turn-away.js).
1855
+ if (await requeueTurnedAway(client, jobId, 'daemon restarting for self-update')) continue;
1818
1856
  await client.failJob(jobId, 'daemon restarting for self-update');
1819
1857
  } catch { /* best-effort */ }
1820
1858
  }
@@ -13,6 +13,7 @@
13
13
  */
14
14
 
15
15
  import { createEngine, MSG } from "@infernetprotocol/engine";
16
+ import { isTurnAwayError } from "./turn-away.js";
16
17
  import { loadConfig } from "./config.js";
17
18
  import { getConversationKey, decrypt, encrypt } from "./nip44.js";
18
19
  import { getModelKeyPair } from "./model-key.js";
@@ -255,7 +256,14 @@ export async function executeChatJob({ client, job, node }) {
255
256
  }
256
257
  }
257
258
 
258
- const engine = await getEngineForModel(job.model_name);
259
+ let engine;
260
+ try {
261
+ engine = await getEngineForModel(job.model_name);
262
+ } catch (err) {
263
+ // No engine reachable right now (Ollama restarting): busy, not broken.
264
+ if (isTurnAwayError(err?.message)) err.turnedAway = true;
265
+ throw err;
266
+ }
259
267
  const buffer = new EventBuffer(client, job.id, convKey);
260
268
 
261
269
  const t0 = Date.now();
@@ -295,10 +303,20 @@ export async function executeChatJob({ client, job, node }) {
295
303
  finished_at: ev.finished_at ?? new Date().toISOString()
296
304
  });
297
305
  break;
298
- case MSG.ERROR:
299
- await buffer.push("error", { message: ev.message ?? "engine error" });
306
+ case MSG.ERROR: {
307
+ const message = ev.message ?? "engine error";
308
+ if (tokenCount === 0 && isTurnAwayError(message)) {
309
+ // Busy before the first token: no error event, so the
310
+ // caller can requeue the job at the front instead.
311
+ await buffer.flush();
312
+ const err = new Error(message);
313
+ err.turnedAway = true;
314
+ throw err;
315
+ }
316
+ await buffer.push("error", { message });
300
317
  await buffer.flush();
301
- throw new Error(ev.message ?? "engine error");
318
+ throw new Error(message);
319
+ }
302
320
  default:
303
321
  // forward unknown event types verbatim — useful for backend
304
322
  // extensions (logits, tool_call, etc.) once the protocol grows.
@@ -0,0 +1,105 @@
1
+ /**
2
+ * How many jobs a node runs at once (Anthony, 2026-11-18: "a node should be
3
+ * able to handle parallel jobs if capacity exists, make that the default").
4
+ *
5
+ * Until 0.1.96 every node ran one job at a time unless the operator set
6
+ * INFERNET_MAX_CONCURRENT_JOBS, and a second request queued behind the first.
7
+ * Now the default follows capacity:
8
+ *
9
+ * - CPU-only nodes: 1. CPU Ollama shares the same cores between requests,
10
+ * so two at once each run at half speed and both finish later.
11
+ * - GPU nodes: the runtime's parallel slots. When Ollama's
12
+ * OLLAMA_NUM_PARALLEL is known (set on ollama.service, by install.sh or
13
+ * the operator, or in this process's environment) that is the number;
14
+ * otherwise it is sized from total VRAM (slotsForVram), the same table
15
+ * install.sh uses to set OLLAMA_NUM_PARALLEL.
16
+ *
17
+ * Operator override, in order: INFERNET_MAX_CONCURRENT_JOBS, then
18
+ * `engine.max_concurrent_jobs` in the node config.
19
+ *
20
+ * The heartbeat advertises the result as specs.load.concurrency_cap, and the
21
+ * control plane routes a second job to a node only while it has a free slot
22
+ * (apps/web/lib/routing/smart-pick.js).
23
+ */
24
+
25
+ import { execFile } from 'node:child_process';
26
+
27
+ /** Max slots we size from VRAM alone. */
28
+ export const MAX_AUTO_SLOTS = 4;
29
+
30
+ /**
31
+ * Parallel slots for a GPU with `totalVramGb` of memory. Each Ollama slot
32
+ * gets its own KV cache for the full context (32K, q8_0 with flash
33
+ * attention). The house 9B takes 6.2 GB with one slot, so a 12 GB card has
34
+ * room for a second; an 8 GB card does not.
35
+ * no GPU or < 10 GB -> 1, 10-20 GB -> 2, 20-40 GB -> 3, 40 GB+ -> 4.
36
+ * Keep in sync with ollama_parallel_slots in install.sh.
37
+ */
38
+ export function slotsForVram(totalVramGb) {
39
+ const v = Number(totalVramGb);
40
+ if (!Number.isFinite(v) || v < 10) return 1;
41
+ if (v < 20) return 2;
42
+ if (v < 40) return 3;
43
+ return MAX_AUTO_SLOTS;
44
+ }
45
+
46
+ function positiveInt(value) {
47
+ if (value === undefined || value === null || String(value).trim() === '') return null;
48
+ const n = Number.parseInt(String(value), 10);
49
+ return Number.isFinite(n) && n > 0 ? n : null;
50
+ }
51
+
52
+ /** OLLAMA_NUM_PARALLEL out of `systemctl show -p Environment --value` output. */
53
+ export function parseOllamaNumParallel(environment) {
54
+ if (typeof environment !== 'string') return null;
55
+ const m = environment.match(/(?:^|\s|")OLLAMA_NUM_PARALLEL=(\d+)/);
56
+ return m ? positiveInt(m[1]) : null;
57
+ }
58
+
59
+ /**
60
+ * The OLLAMA_NUM_PARALLEL the local Ollama runs with, or null when unknown.
61
+ * Reads ollama.service (no root needed), then this process's environment
62
+ * (the `ollama serve` fallback that `infernet setup` starts inherits it).
63
+ */
64
+ export async function ollamaNumParallel({ env = process.env, run = systemctlEnvironment } = {}) {
65
+ const fromUnit = parseOllamaNumParallel(await run().catch(() => null));
66
+ if (fromUnit) return fromUnit;
67
+ return positiveInt(env.OLLAMA_NUM_PARALLEL);
68
+ }
69
+
70
+ function systemctlEnvironment() {
71
+ if (process.platform !== 'linux') return Promise.resolve(null);
72
+ return new Promise((resolve) => {
73
+ execFile('systemctl', ['show', 'ollama.service', '-p', 'Environment', '--value'], { timeout: 3000 }, (err, stdout) => {
74
+ resolve(err ? null : String(stdout));
75
+ });
76
+ });
77
+ }
78
+
79
+ /**
80
+ * Pure: decide the daemon's concurrency. Returns { value, source }.
81
+ * env process env (INFERNET_MAX_CONCURRENT_JOBS)
82
+ * config node config (engine.max_concurrent_jobs)
83
+ * totalVramGb summed GPU memory; 0/null on CPU-only nodes
84
+ * runtimeSlots the runtime's parallel slots when known (OLLAMA_NUM_PARALLEL)
85
+ */
86
+ export function resolveMaxConcurrentJobs({ env = process.env, config = {}, totalVramGb = null, runtimeSlots = null } = {}) {
87
+ const fromEnv = positiveInt(env.INFERNET_MAX_CONCURRENT_JOBS);
88
+ if (fromEnv) return { value: fromEnv, source: 'env INFERNET_MAX_CONCURRENT_JOBS' };
89
+ const fromConfig = positiveInt(config?.engine?.max_concurrent_jobs);
90
+ if (fromConfig) return { value: fromConfig, source: 'config engine.max_concurrent_jobs' };
91
+ const vram = Number(totalVramGb);
92
+ if (!Number.isFinite(vram) || vram <= 0) return { value: 1, source: 'cpu-only node' };
93
+ const slots = positiveInt(runtimeSlots);
94
+ if (slots) return { value: slots, source: 'OLLAMA_NUM_PARALLEL' };
95
+ return { value: slotsForVram(vram), source: `${vram.toFixed(0)} GB VRAM` };
96
+ }
97
+
98
+ /** Summed VRAM (GB) from @infernetprotocol/gpu detectGpus() output. */
99
+ export function totalVramGbOf(gpus) {
100
+ let total = 0;
101
+ for (const g of Array.isArray(gpus) ? gpus : []) {
102
+ if (Number.isFinite(g?.vram_mb)) total += g.vram_mb / 1024;
103
+ }
104
+ return total;
105
+ }
@@ -97,6 +97,13 @@ export function createNodeClient({ url, publicKey, privateKey, role, timeoutMs =
97
97
  });
98
98
  },
99
99
 
100
+ // Busy: put the job back at the front of the queue (CLI 0.1.97).
101
+ requeueJob(jobId, reason) {
102
+ return signedFetch(`/api/v1/node/jobs/${encodeURIComponent(jobId)}/requeue`, {
103
+ reason: typeof reason === 'string' ? reason.slice(0, 300) : String(reason ?? 'busy')
104
+ });
105
+ },
106
+
100
107
  postJobEvents(jobId, events) {
101
108
  return signedFetch(`/api/v1/node/jobs/${encodeURIComponent(jobId)}/events`, { events });
102
109
  },
@@ -0,0 +1,33 @@
1
+ /**
2
+ * Busy-type failures: the node turns a chat job away instead of failing it,
3
+ * and the control plane puts it back at the front of the queue
4
+ * (POST /api/v1/node/jobs/<id>/requeue, apps/web/lib/routing/requeue.js,
5
+ * which keeps the same pattern). Anthony, 2026-11-18: "if it turns away a job
6
+ * send it back to the front of the queue".
7
+ */
8
+
9
+ const TURN_AWAY_RE = /restarting for self-update|server busy|too many requests|HTTP 429|HTTP 503|queue (?:is )?full|no free slot|no inference engine reachable|ECONNREFUSED|ECONNRESET|socket hang up|fetch failed|timed? ?out/i;
10
+
11
+ export function isTurnAwayError(message) {
12
+ return typeof message === 'string' && TURN_AWAY_RE.test(message);
13
+ }
14
+
15
+ /** How long this daemon leaves a job it turned away before taking it again. */
16
+ export const TURN_AWAY_BACKOFF_MS = 15_000;
17
+
18
+ /**
19
+ * Ask the control plane to requeue `jobId`. Resolves true when it did (the
20
+ * caller must then neither post an error nor fail the job), false when the
21
+ * job must fail as before (answer already streaming, too many turn-aways, or
22
+ * a control plane without the endpoint).
23
+ */
24
+ export async function requeueTurnedAway(client, jobId, reason) {
25
+ if (typeof client?.requeueJob !== 'function') return false;
26
+ try {
27
+ const res = await client.requeueJob(jobId, reason);
28
+ const data = res?.data ?? res;
29
+ return data?.requeued === true;
30
+ } catch {
31
+ return false;
32
+ }
33
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@infernetprotocol/cli",
3
- "version": "0.1.95",
3
+ "version": "0.1.97",
4
4
  "description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -32,14 +32,14 @@
32
32
  "ora": "^9.4.1",
33
33
  "react": "^17.0.2",
34
34
  "react-blessed": "^0.7.2",
35
- "@infernetprotocol/auth": "0.1.95",
36
- "@infernetprotocol/config": "0.1.95",
37
- "@infernetprotocol/deploy-providers": "0.1.95",
38
- "@infernetprotocol/engine": "0.1.95",
39
- "@infernetprotocol/gpu": "0.1.95",
40
- "@infernetprotocol/liquid-agent": "0.1.95",
41
- "@infernetprotocol/rpc-adapter": "0.1.95",
42
- "@infernetprotocol/training": "0.1.95"
35
+ "@infernetprotocol/auth": "0.1.97",
36
+ "@infernetprotocol/config": "0.1.97",
37
+ "@infernetprotocol/deploy-providers": "0.1.97",
38
+ "@infernetprotocol/engine": "0.1.97",
39
+ "@infernetprotocol/gpu": "0.1.97",
40
+ "@infernetprotocol/liquid-agent": "0.1.97",
41
+ "@infernetprotocol/rpc-adapter": "0.1.97",
42
+ "@infernetprotocol/training": "0.1.97"
43
43
  },
44
44
  "repository": {
45
45
  "type": "git",