@infernetprotocol/cli 0.1.96 → 0.1.97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/start.js +21 -0
- package/lib/chat-executor.js +22 -4
- package/lib/node-client.js +7 -0
- package/lib/turn-away.js +33 -0
- package/package.json +9 -9
package/commands/start.js
CHANGED
|
@@ -60,6 +60,7 @@ import { downloadHfModel, resolveHfToken } from '../lib/hf-model.js';
|
|
|
60
60
|
import { startVllmServe, stopVllmServe, vllmInstalled, waitForVllmModel, extractVllmError } from '../lib/vllm.js';
|
|
61
61
|
import { detectGpus, detectHost } from '@infernetprotocol/gpu';
|
|
62
62
|
import { ollamaNumParallel, resolveMaxConcurrentJobs, totalVramGbOf } from '../lib/concurrency.js';
|
|
63
|
+
import { requeueTurnedAway, TURN_AWAY_BACKOFF_MS } from '../lib/turn-away.js';
|
|
63
64
|
import { getOrCreateModelKey, getModelPublicKeys } from '../lib/model-key.js';
|
|
64
65
|
import { pullLatestBinary } from './upgrade.js';
|
|
65
66
|
import { PROBE_INTERVAL_MS, runProbeRound } from '../lib/aggregator.js';
|
|
@@ -580,6 +581,9 @@ async function runDaemon(args, ctx) {
|
|
|
580
581
|
if (finishedJobAt.size > 1000) finishedJobAt.delete(finishedJobAt.keys().next().value);
|
|
581
582
|
}
|
|
582
583
|
|
|
584
|
+
// Jobs this daemon turned away (busy), not to be taken again before.
|
|
585
|
+
const turnedAwayUntil = new Map();
|
|
586
|
+
|
|
583
587
|
async function processJob(job) {
|
|
584
588
|
const t0 = new Date().toISOString();
|
|
585
589
|
stats.jobsPicked += 1;
|
|
@@ -617,6 +621,15 @@ async function runDaemon(args, ctx) {
|
|
|
617
621
|
stats.activeJobIds.delete(job.id);
|
|
618
622
|
markFinished(job.id);
|
|
619
623
|
const msg = err?.message ?? String(err);
|
|
624
|
+
// Busy (engine queue full, Ollama restarting): hand the job back
|
|
625
|
+
// to the front of the queue instead of failing it, and leave it
|
|
626
|
+
// alone here for a moment (lib/turn-away.js).
|
|
627
|
+
if (job.type === 'chat' && err?.turnedAway === true && await requeueTurnedAway(client, job.id, msg)) {
|
|
628
|
+
stats.jobsFailed -= 1;
|
|
629
|
+
turnedAwayUntil.set(job.id, Date.now() + TURN_AWAY_BACKOFF_MS);
|
|
630
|
+
process.stdout.write(`[${new Date().toISOString()}] turned away job ${job.id} (${msg}); requeued at the front\n`);
|
|
631
|
+
return;
|
|
632
|
+
}
|
|
620
633
|
process.stderr.write(`job ${job.id} failed: ${msg}\n`);
|
|
621
634
|
if (job.type === 'chat') {
|
|
622
635
|
await failChatJob({ client, jobId: job.id, message: msg });
|
|
@@ -663,6 +676,11 @@ async function runDaemon(args, ctx) {
|
|
|
663
676
|
// complete, so without this guard the poll loop queues the
|
|
664
677
|
// same job on every tick while it's in flight.
|
|
665
678
|
if (stats.activeJobIds.has(job.id)) continue;
|
|
679
|
+
const awayUntil = turnedAwayUntil.get(job.id);
|
|
680
|
+
if (awayUntil !== undefined) {
|
|
681
|
+
if (awayUntil > Date.now()) continue;
|
|
682
|
+
turnedAwayUntil.delete(job.id);
|
|
683
|
+
}
|
|
666
684
|
if ((finishedJobAt.get(job.id) ?? 0) >= polledAt) continue;
|
|
667
685
|
// A self-update can start draining while this batch runs
|
|
668
686
|
// (each job is awaited in turn). Stop picking the rest up, or
|
|
@@ -1832,6 +1850,9 @@ async function runDaemon(args, ctx) {
|
|
|
1832
1850
|
// Release whatever is still running after the drain window.
|
|
1833
1851
|
for (const jobId of stats.activeJobIds) {
|
|
1834
1852
|
try {
|
|
1853
|
+
// Not started yet: back to the front of the queue, for the
|
|
1854
|
+
// restarted daemon or a free node (lib/turn-away.js).
|
|
1855
|
+
if (await requeueTurnedAway(client, jobId, 'daemon restarting for self-update')) continue;
|
|
1835
1856
|
await client.failJob(jobId, 'daemon restarting for self-update');
|
|
1836
1857
|
} catch { /* best-effort */ }
|
|
1837
1858
|
}
|
package/lib/chat-executor.js
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
15
|
import { createEngine, MSG } from "@infernetprotocol/engine";
|
|
16
|
+
import { isTurnAwayError } from "./turn-away.js";
|
|
16
17
|
import { loadConfig } from "./config.js";
|
|
17
18
|
import { getConversationKey, decrypt, encrypt } from "./nip44.js";
|
|
18
19
|
import { getModelKeyPair } from "./model-key.js";
|
|
@@ -255,7 +256,14 @@ export async function executeChatJob({ client, job, node }) {
|
|
|
255
256
|
}
|
|
256
257
|
}
|
|
257
258
|
|
|
258
|
-
|
|
259
|
+
let engine;
|
|
260
|
+
try {
|
|
261
|
+
engine = await getEngineForModel(job.model_name);
|
|
262
|
+
} catch (err) {
|
|
263
|
+
// No engine reachable right now (Ollama restarting): busy, not broken.
|
|
264
|
+
if (isTurnAwayError(err?.message)) err.turnedAway = true;
|
|
265
|
+
throw err;
|
|
266
|
+
}
|
|
259
267
|
const buffer = new EventBuffer(client, job.id, convKey);
|
|
260
268
|
|
|
261
269
|
const t0 = Date.now();
|
|
@@ -295,10 +303,20 @@ export async function executeChatJob({ client, job, node }) {
|
|
|
295
303
|
finished_at: ev.finished_at ?? new Date().toISOString()
|
|
296
304
|
});
|
|
297
305
|
break;
|
|
298
|
-
case MSG.ERROR:
|
|
299
|
-
|
|
306
|
+
case MSG.ERROR: {
|
|
307
|
+
const message = ev.message ?? "engine error";
|
|
308
|
+
if (tokenCount === 0 && isTurnAwayError(message)) {
|
|
309
|
+
// Busy before the first token: no error event, so the
|
|
310
|
+
// caller can requeue the job at the front instead.
|
|
311
|
+
await buffer.flush();
|
|
312
|
+
const err = new Error(message);
|
|
313
|
+
err.turnedAway = true;
|
|
314
|
+
throw err;
|
|
315
|
+
}
|
|
316
|
+
await buffer.push("error", { message });
|
|
300
317
|
await buffer.flush();
|
|
301
|
-
throw new Error(
|
|
318
|
+
throw new Error(message);
|
|
319
|
+
}
|
|
302
320
|
default:
|
|
303
321
|
// forward unknown event types verbatim — useful for backend
|
|
304
322
|
// extensions (logits, tool_call, etc.) once the protocol grows.
|
package/lib/node-client.js
CHANGED
|
@@ -97,6 +97,13 @@ export function createNodeClient({ url, publicKey, privateKey, role, timeoutMs =
|
|
|
97
97
|
});
|
|
98
98
|
},
|
|
99
99
|
|
|
100
|
+
// Busy: put the job back at the front of the queue (CLI 0.1.97).
|
|
101
|
+
requeueJob(jobId, reason) {
|
|
102
|
+
return signedFetch(`/api/v1/node/jobs/${encodeURIComponent(jobId)}/requeue`, {
|
|
103
|
+
reason: typeof reason === 'string' ? reason.slice(0, 300) : String(reason ?? 'busy')
|
|
104
|
+
});
|
|
105
|
+
},
|
|
106
|
+
|
|
100
107
|
postJobEvents(jobId, events) {
|
|
101
108
|
return signedFetch(`/api/v1/node/jobs/${encodeURIComponent(jobId)}/events`, { events });
|
|
102
109
|
},
|
package/lib/turn-away.js
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Busy-type failures: the node turns a chat job away instead of failing it,
|
|
3
|
+
* and the control plane puts it back at the front of the queue
|
|
4
|
+
* (POST /api/v1/node/jobs/<id>/requeue, apps/web/lib/routing/requeue.js,
|
|
5
|
+
* which keeps the same pattern). Anthony, 2026-11-18: "if it turns away a job
|
|
6
|
+
* send it back to the front of the queue".
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
const TURN_AWAY_RE = /restarting for self-update|server busy|too many requests|HTTP 429|HTTP 503|queue (?:is )?full|no free slot|no inference engine reachable|ECONNREFUSED|ECONNRESET|socket hang up|fetch failed|timed? ?out/i;
|
|
10
|
+
|
|
11
|
+
export function isTurnAwayError(message) {
|
|
12
|
+
return typeof message === 'string' && TURN_AWAY_RE.test(message);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** How long this daemon leaves a job it turned away before taking it again. */
|
|
16
|
+
export const TURN_AWAY_BACKOFF_MS = 15_000;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Ask the control plane to requeue `jobId`. Resolves true when it did (the
|
|
20
|
+
* caller must then neither post an error nor fail the job), false when the
|
|
21
|
+
* job must fail as before (answer already streaming, too many turn-aways, or
|
|
22
|
+
* a control plane without the endpoint).
|
|
23
|
+
*/
|
|
24
|
+
export async function requeueTurnedAway(client, jobId, reason) {
|
|
25
|
+
if (typeof client?.requeueJob !== 'function') return false;
|
|
26
|
+
try {
|
|
27
|
+
const res = await client.requeueJob(jobId, reason);
|
|
28
|
+
const data = res?.data ?? res;
|
|
29
|
+
return data?.requeued === true;
|
|
30
|
+
} catch {
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infernetprotocol/cli",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.97",
|
|
4
4
|
"description": "GPU-node CLI / daemon for the Infernet Protocol — register a GPU server with a Supabase control plane and start earning.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -32,14 +32,14 @@
|
|
|
32
32
|
"ora": "^9.4.1",
|
|
33
33
|
"react": "^17.0.2",
|
|
34
34
|
"react-blessed": "^0.7.2",
|
|
35
|
-
"@infernetprotocol/auth": "0.1.
|
|
36
|
-
"@infernetprotocol/config": "0.1.
|
|
37
|
-
"@infernetprotocol/deploy-providers": "0.1.
|
|
38
|
-
"@infernetprotocol/engine": "0.1.
|
|
39
|
-
"@infernetprotocol/gpu": "0.1.
|
|
40
|
-
"@infernetprotocol/liquid-agent": "0.1.
|
|
41
|
-
"@infernetprotocol/rpc-adapter": "0.1.
|
|
42
|
-
"@infernetprotocol/training": "0.1.
|
|
35
|
+
"@infernetprotocol/auth": "0.1.97",
|
|
36
|
+
"@infernetprotocol/config": "0.1.97",
|
|
37
|
+
"@infernetprotocol/deploy-providers": "0.1.97",
|
|
38
|
+
"@infernetprotocol/engine": "0.1.97",
|
|
39
|
+
"@infernetprotocol/gpu": "0.1.97",
|
|
40
|
+
"@infernetprotocol/liquid-agent": "0.1.97",
|
|
41
|
+
"@infernetprotocol/rpc-adapter": "0.1.97",
|
|
42
|
+
"@infernetprotocol/training": "0.1.97"
|
|
43
43
|
},
|
|
44
44
|
"repository": {
|
|
45
45
|
"type": "git",
|