@infernetprotocol/cli 0.1.96 → 0.1.97

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/commands/start.js CHANGED
@@ -60,6 +60,7 @@ import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
60
60
  import { startVllmServe, stopVllmServe, vllmInstalled, waitForVllmModel, extractVllmError } from '../lib/vllm.js';
61
61
  import { detectGpus, detectHost } from '@infernetprotocol/gpu';
62
62
  import { ollamaNumParallel, resolveMaxConcurrentJobs, totalVramGbOf } from '../lib/concurrency.js';
63
+ import { requeueTurnedAway, TURN_AWAY_BACKOFF_MS } from '../lib/turn-away.js';
63
64
  import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
64
65
  import { pullLatestBinary } from './upgrade.js';
65
66
  import { PROBE_INTERVAL_MS, runProbeRound } from '../lib/aggregator.js';
@@ -580,6 +581,9 @@ async function runDaemon(args, ctx) {
580
581
  if (finishedJobAt.size > 1000) finishedJobAt.delete(finishedJobAt.keys().next().value);
581
582
  }
582
583
 
584
+ // Jobs this daemon turned away (busy), not to be taken again before.
585
+ const turnedAwayUntil = new Map();
586
+
583
587
  async function processJob(job) {
584
588
  const t0 = new Date().toISOString();
585
589
  stats.jobsPicked += 1;
@@ -617,6 +621,15 @@ async function runDaemon(args, ctx) {
617
621
  stats.activeJobIds.delete(job.id);
618
622
  markFinished(job.id);
619
623
  const msg = err?.message ?? String(err);
624
+ // Busy (engine queue full, Ollama restarting): hand the job back
625
+ // to the front of the queue instead of failing it, and leave it
626
+ // alone here for a moment (lib/turn-away.js).
627
+ if (job.type === 'chat' && err?.turnedAway === true && await requeueTurnedAway(client, job.id, msg)) {
628
+ stats.jobsFailed -= 1;
629
+ turnedAwayUntil.set(job.id, Date.now() + TURN_AWAY_BACKOFF_MS);
630
+ process.stdout.write(`[${new Date().toISOString()}] turned away job ${job.id} (${msg}); requeued at the front\n`);
631
+ return;
632
+ }
620
633
  process.stderr.write(`job ${job.id} failed: ${msg}\n`);
621
634
  if (job.type === 'chat') {
622
635
  await failChatJob({ client, jobId: job.id, message: msg });
@@ -663,6 +676,11 @@ async function runDaemon(args, ctx) {
663
676
  // complete, so without this guard the poll loop queues the
664
677
  // same job on every tick while it's in flight.
665
678
  if (stats.activeJobIds.has(job.id)) continue;
679
+ const awayUntil = turnedAwayUntil.get(job.id);
680
+ if (awayUntil !== undefined) {
681
+ if (awayUntil > Date.now()) continue;
682
+ turnedAwayUntil.delete(job.id);
683
+ }
666
684
  if ((finishedJobAt.get(job.id) ?? 0) >= polledAt) continue;
667
685
  // A self-update can start draining while this batch runs
668
686
  // (each job is awaited in turn). Stop picking the rest up, or
@@ -1832,6 +1850,9 @@ async function runDaemon(args, ctx) {
1832
1850
  // Release whatever is still running after the drain window.
1833
1851
  for (const jobId of stats.activeJobIds) {
1834
1852
  try {
1853
+ // Not started yet: back to the front of the queue, for the
1854
+ // restarted daemon or a free node (lib/turn-away.js).
1855
+ if (await requeueTurnedAway(client, jobId, 'daemon restarting for self-update')) continue;
1835
1856
  await client.failJob(jobId, 'daemon restarting for self-update');
1836
1857
  } catch { /* best-effort */ }
1837
1858
  }
@@ -13,6 +13,7 @@
13
13
  */
14
14
 
15
15
  import { createEngine, MSG } from "@infernetprotocol/engine";
16
+ import { isTurnAwayError } from "./turn-away.js";
16
17
  import { loadConfig } from "./config.js";
17
18
  import { getConversationKey, decrypt, encrypt } from "./nip44.js";
18
19
  import { getModelKeyPair } from "./model-key.js";
@@ -255,7 +256,14 @@ export async function executeChatJob({ client, job, node }) {
255
256
  }
256
257
  }
257
258
 
258
- const engine = await getEngineForModel(job.model_name);
259
+ let engine;
260
+ try {
261
+ engine = await getEngineForModel(job.model_name);
262
+ } catch (err) {
263
+ // No engine reachable right now (Ollama restarting): busy, not broken.
264
+ if (isTurnAwayError(err?.message)) err.turnedAway = true;
265
+ throw err;
266
+ }
259
267
  const buffer = new EventBuffer(client, job.id, convKey);
260
268
 
261
269
  const t0 = Date.now();
@@ -295,10 +303,20 @@ export async function executeChatJob({ client, job, node }) {
295
303
  finished_at: ev.finished_at ?? new Date().toISOString()
296
304
  });
297
305
  break;
298
- case MSG.ERROR:
299
- await buffer.push("error", { message: ev.message ?? "engine error" });
306
+ case MSG.ERROR: {
307
+ const message = ev.message ?? "engine error";
308
+ if (tokenCount === 0 && isTurnAwayError(message)) {
309
+ // Busy before the first token: no error event, so the
310
+ // caller can requeue the job at the front instead.
311
+ await buffer.flush();
312
+ const err = new Error(message);
313
+ err.turnedAway = true;
314
+ throw err;
315
+ }
316
+ await buffer.push("error", { message });
300
317
  await buffer.flush();
301
- throw new Error(ev.message ?? "engine error");
318
+ throw new Error(message);
319
+ }
302
320
  default:
303
321
  // forward unknown event types verbatim — useful for backend
304
322
  // extensions (logits, tool_call, etc.) once the protocol grows.
@@ -97,6 +97,13 @@ export function createNodeClient({ url, publicKey, privateKey, role, timeoutMs =
97
97
  });
98
98
  },
99
99
 
100
+ // Busy: put the job back at the front of the queue (CLI 0.1.97).
101
+ requeueJob(jobId, reason) {
102
+ return signedFetch(`/api/v1/node/jobs/${encodeURIComponent(jobId)}/requeue`, {
103
+ reason: typeof reason === 'string' ? reason.slice(0, 300) : String(reason ?? 'busy')
104
+ });
105
+ },
106
+
100
107
  postJobEvents(jobId, events) {
101
108
  return signedFetch(`/api/v1/node/jobs/${encodeURIComponent(jobId)}/events`, { events });
102
109
  },
@@ -0,0 +1,33 @@
1
+ /**
2
+ * Busy-type failures: the node turns a chat job away instead of failing it,
3
+ * and the control plane puts it back at the front of the queue
4
+ * (POST /api/v1/node/jobs/<id>/requeue, apps/web/lib/routing/requeue.js,
5
+ * which keeps the same pattern). Anthony, 2026-11-18: "if it turns away a job
6
+ * send it back to the front of the queue".
7
+ */
8
+
9
+ const TURN_AWAY_RE = /restarting for self-update|server busy|too many requests|HTTP 429|HTTP 503|queue (?:is )?full|no free slot|no inference engine reachable|ECONNREFUSED|ECONNRESET|socket hang up|fetch failed|timed? ?out/i;
10
+
11
+ export function isTurnAwayError(message) {
12
+ return typeof message === 'string' && TURN_AWAY_RE.test(message);
13
+ }
14
+
15
+ /** How long this daemon leaves a job it turned away before taking it again. */
16
+ export const TURN_AWAY_BACKOFF_MS = 15_000;
17
+
18
+ /**
19
+ * Ask the control plane to requeue `jobId`. Resolves true when it did (the
20
+ * caller must then neither post an error nor fail the job), false when the
21
+ * job must fail as before (answer already streaming, too many turn-aways, or
22
+ * a control plane without the endpoint).
23
+ */
24
+ export async function requeueTurnedAway(client, jobId, reason) {
25
+ if (typeof client?.requeueJob !== 'function') return false;
26
+ try {
27
+ const res = await client.requeueJob(jobId, reason);
28
+ const data = res?.data ?? res;
29
+ return data?.requeued === true;
30
+ } catch {
31
+ return false;
32
+ }
33
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@infernetprotocol/cli",
3
- "version": "0.1.96",
3
+ "version": "0.1.97",
4
4
  "description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -32,14 +32,14 @@
32
32
  "ora": "^9.4.1",
33
33
  "react": "^17.0.2",
34
34
  "react-blessed": "^0.7.2",
35
- "@infernetprotocol/auth": "0.1.96",
36
- "@infernetprotocol/config": "0.1.96",
37
- "@infernetprotocol/deploy-providers": "0.1.96",
38
- "@infernetprotocol/engine": "0.1.96",
39
- "@infernetprotocol/gpu": "0.1.96",
40
- "@infernetprotocol/liquid-agent": "0.1.96",
41
- "@infernetprotocol/rpc-adapter": "0.1.96",
42
- "@infernetprotocol/training": "0.1.96"
35
+ "@infernetprotocol/auth": "0.1.97",
36
+ "@infernetprotocol/config": "0.1.97",
37
+ "@infernetprotocol/deploy-providers": "0.1.97",
38
+ "@infernetprotocol/engine": "0.1.97",
39
+ "@infernetprotocol/gpu": "0.1.97",
40
+ "@infernetprotocol/liquid-agent": "0.1.97",
41
+ "@infernetprotocol/rpc-adapter": "0.1.97",
42
+ "@infernetprotocol/training": "0.1.97"
43
43
  },
44
44
  "repository": {
45
45
  "type": "git",