@infernetprotocol/cli 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/commands/chat.js +268 -0
- package/commands/console.js +229 -0
- package/commands/debug.js +222 -0
- package/commands/deploy.js +524 -0
- package/commands/doctor.js +501 -0
- package/commands/firewall.js +39 -0
- package/commands/gpu.js +130 -0
- package/commands/help.js +54 -0
- package/commands/init.js +304 -0
- package/commands/login.js +305 -0
- package/commands/logs.js +93 -0
- package/commands/model.js +359 -0
- package/commands/payments.js +92 -0
- package/commands/payout.js +128 -0
- package/commands/pubkey.js +116 -0
- package/commands/register.js +304 -0
- package/commands/remove.js +230 -0
- package/commands/service.js +255 -0
- package/commands/setup.js +869 -0
- package/commands/start.js +903 -0
- package/commands/stats.js +85 -0
- package/commands/status.js +116 -0
- package/commands/stop.js +81 -0
- package/commands/train.js +563 -0
- package/commands/tui.js +285 -0
- package/commands/uncensored.js +222 -0
- package/commands/update.js +80 -0
- package/commands/upgrade.js +2 -0
- package/examples/app.js +153 -0
- package/index.js +207 -0
- package/lib/chat-executor.js +232 -0
- package/lib/config.js +148 -0
- package/lib/daemonize.js +46 -0
- package/lib/firewall.js +190 -0
- package/lib/hf-model.js +166 -0
- package/lib/identity.js +18 -0
- package/lib/ipc.js +114 -0
- package/lib/model-fit.js +67 -0
- package/lib/model-key.js +52 -0
- package/lib/network.js +240 -0
- package/lib/nip44.js +130 -0
- package/lib/node-client.js +152 -0
- package/lib/peers.js +133 -0
- package/lib/prompt.js +49 -0
- package/lib/remote-chat.js +115 -0
- package/lib/route.js +44 -0
- package/lib/runtime-config.js +130 -0
- package/lib/version.js +32 -0
- package/package.json +57 -0
package/examples/app.js
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Infernet Protocol - Main Application
|
|
3
|
+
* Demonstrates the use of the Supabase-backed database layer.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import db from '@infernetprotocol/db';
|
|
7
|
+
import config from '../lib/runtime-config.js';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Initialize the application
|
|
11
|
+
*/
|
|
12
|
+
async function init() {
|
|
13
|
+
try {
|
|
14
|
+
console.log('Initializing Infernet Protocol application...');
|
|
15
|
+
|
|
16
|
+
await db.connect(config.supabase.url);
|
|
17
|
+
|
|
18
|
+
console.log('Application initialized successfully');
|
|
19
|
+
return true;
|
|
20
|
+
} catch (error) {
|
|
21
|
+
console.error('Failed to initialize application:', error);
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Example: Register a new provider node
|
|
28
|
+
* @param {Object} providerData - Provider data
|
|
29
|
+
* @returns {Promise<Object>} - Created provider record
|
|
30
|
+
*/
|
|
31
|
+
async function registerProvider(providerData) {
|
|
32
|
+
try {
|
|
33
|
+
const Provider = db.model('provider');
|
|
34
|
+
const provider = await Provider.register({
|
|
35
|
+
name: providerData.name || `Provider-${Date.now()}`,
|
|
36
|
+
gpu_model: providerData.gpuModel,
|
|
37
|
+
price: providerData.price || 0
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
console.log('Provider registered:', provider);
|
|
41
|
+
return provider;
|
|
42
|
+
} catch (error) {
|
|
43
|
+
console.error('Failed to register provider:', error);
|
|
44
|
+
throw error;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Example: Submit a new inference job
|
|
50
|
+
* @param {Object} jobData - Job data
|
|
51
|
+
* @returns {Promise<Object>} - Created job record
|
|
52
|
+
*/
|
|
53
|
+
async function submitJob(jobData) {
|
|
54
|
+
try {
|
|
55
|
+
const Job = db.model('job');
|
|
56
|
+
const job = await Job.create({
|
|
57
|
+
title: jobData.title || `Job-${Date.now()}`,
|
|
58
|
+
status: 'pending',
|
|
59
|
+
payment_offer: jobData.paymentOffer,
|
|
60
|
+
model_name: jobData.model,
|
|
61
|
+
client_name: jobData.clientName
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
console.log('Job submitted:', job);
|
|
65
|
+
|
|
66
|
+
if (jobData.isMultiNode) {
|
|
67
|
+
await assignAggregator(job.id);
|
|
68
|
+
} else {
|
|
69
|
+
await assignProvider(job.id);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return job;
|
|
73
|
+
} catch (error) {
|
|
74
|
+
console.error('Failed to submit job:', error);
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Example: Assign an aggregator to a multi-node job
|
|
81
|
+
* @param {string} jobId - Job ID
|
|
82
|
+
* @returns {Promise<Object>} - Updated job record
|
|
83
|
+
*/
|
|
84
|
+
async function assignAggregator(jobId) {
|
|
85
|
+
try {
|
|
86
|
+
const Aggregator = db.model('aggregator');
|
|
87
|
+
const Job = db.model('job');
|
|
88
|
+
|
|
89
|
+
const aggregators = await Aggregator.findAvailable();
|
|
90
|
+
if (aggregators.length === 0) {
|
|
91
|
+
throw new Error('No aggregators available');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const aggregator = aggregators[0];
|
|
95
|
+
const updatedJob = await Job.assignToAggregator(jobId, aggregator.id);
|
|
96
|
+
await Aggregator.updateLoad(aggregator.id, 1);
|
|
97
|
+
|
|
98
|
+
console.log(`Job ${jobId} assigned to aggregator ${aggregator.id}`);
|
|
99
|
+
return updatedJob;
|
|
100
|
+
} catch (error) {
|
|
101
|
+
console.error(`Failed to assign aggregator for job ${jobId}:`, error);
|
|
102
|
+
throw error;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Example: Assign a provider to a single-node job
|
|
108
|
+
* @param {string} jobId - Job ID
|
|
109
|
+
* @returns {Promise<Object>} - Updated job record
|
|
110
|
+
*/
|
|
111
|
+
async function assignProvider(jobId) {
|
|
112
|
+
try {
|
|
113
|
+
const Provider = db.model('provider');
|
|
114
|
+
const Job = db.model('job');
|
|
115
|
+
|
|
116
|
+
const providers = await Provider.findAvailable();
|
|
117
|
+
if (providers.length === 0) {
|
|
118
|
+
throw new Error('No suitable providers available');
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const provider = providers[0];
|
|
122
|
+
const updatedJob = await Job.assignToProvider(jobId, provider.id);
|
|
123
|
+
await Provider.updateStatus(provider.id, 'busy');
|
|
124
|
+
|
|
125
|
+
console.log(`Job ${jobId} assigned to provider ${provider.id}`);
|
|
126
|
+
return updatedJob;
|
|
127
|
+
} catch (error) {
|
|
128
|
+
console.error(`Failed to assign provider for job ${jobId}:`, error);
|
|
129
|
+
throw error;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Example: Cleanup and shutdown
|
|
135
|
+
*/
|
|
136
|
+
async function shutdown() {
|
|
137
|
+
try {
|
|
138
|
+
console.log('Shutting down application...');
|
|
139
|
+
await db.disconnect();
|
|
140
|
+
console.log('Application shutdown complete');
|
|
141
|
+
} catch (error) {
|
|
142
|
+
console.error('Error during shutdown:', error);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export {
|
|
147
|
+
init,
|
|
148
|
+
registerProvider,
|
|
149
|
+
submitJob,
|
|
150
|
+
assignAggregator,
|
|
151
|
+
assignProvider,
|
|
152
|
+
shutdown
|
|
153
|
+
};
|
package/index.js
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* `infernet` — GPU-node CLI / daemon for the Infernet Protocol.
|
|
4
|
+
*
|
|
5
|
+
* This is NOT Next.js code. It is a standalone Node.js ESM binary intended
|
|
6
|
+
* to ship with (or alongside) the control-plane app; GPU operators install
|
|
7
|
+
* it on each rented/owned server to register the node, heartbeat, accept
|
|
8
|
+
* jobs, and report earnings.
|
|
9
|
+
*
|
|
10
|
+
* Nodes authenticate to the control plane using Nostr-signed HTTP requests.
|
|
11
|
+
* No database credentials are stored on the node — the signing keypair IS
|
|
12
|
+
* the credential.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { loadConfig, getConfigPath } from './lib/config.js';
|
|
16
|
+
import { createNodeClientFromConfig } from './lib/node-client.js';
|
|
17
|
+
import { decideRoute } from './lib/route.js';
|
|
18
|
+
import { CURRENT_VERSION, fetchLatestVersion, isNewerVersion } from './lib/version.js';
|
|
19
|
+
|
|
20
|
+
import help, { USAGE } from './commands/help.js';
|
|
21
|
+
import init from './commands/init.js';
|
|
22
|
+
import login from './commands/login.js';
|
|
23
|
+
import register from './commands/register.js';
|
|
24
|
+
import update from './commands/update.js';
|
|
25
|
+
import upgrade from './commands/upgrade.js';
|
|
26
|
+
import remove from './commands/remove.js';
|
|
27
|
+
import start from './commands/start.js';
|
|
28
|
+
import status from './commands/status.js';
|
|
29
|
+
import stop from './commands/stop.js';
|
|
30
|
+
import stats from './commands/stats.js';
|
|
31
|
+
import logs from './commands/logs.js';
|
|
32
|
+
import payout from './commands/payout.js';
|
|
33
|
+
import payments from './commands/payments.js';
|
|
34
|
+
import gpu from './commands/gpu.js';
|
|
35
|
+
import firewall from './commands/firewall.js';
|
|
36
|
+
import chat from './commands/chat.js';
|
|
37
|
+
import setup from './commands/setup.js';
|
|
38
|
+
import model from './commands/model.js';
|
|
39
|
+
import tui from './commands/tui.js';
|
|
40
|
+
import doctor from './commands/doctor.js';
|
|
41
|
+
import service from './commands/service.js';
|
|
42
|
+
import pubkey from './commands/pubkey.js';
|
|
43
|
+
import debug from './commands/debug.js';
|
|
44
|
+
import deploy from './commands/deploy.js';
|
|
45
|
+
import consoleCmd from './commands/console.js';
|
|
46
|
+
import train from './commands/train.js';
|
|
47
|
+
import uncensored from './commands/uncensored.js';
|
|
48
|
+
|
|
49
|
+
function parseArgs(argv) {
|
|
50
|
+
const positional = [];
|
|
51
|
+
const flags = new Map();
|
|
52
|
+
|
|
53
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
54
|
+
const tok = argv[i];
|
|
55
|
+
if (tok === '--') {
|
|
56
|
+
positional.push(...argv.slice(i + 1));
|
|
57
|
+
break;
|
|
58
|
+
}
|
|
59
|
+
if (tok.startsWith('--')) {
|
|
60
|
+
const body = tok.slice(2);
|
|
61
|
+
const eq = body.indexOf('=');
|
|
62
|
+
if (eq >= 0) {
|
|
63
|
+
flags.set(body.slice(0, eq), body.slice(eq + 1));
|
|
64
|
+
} else {
|
|
65
|
+
const next = argv[i + 1];
|
|
66
|
+
if (next !== undefined && !next.startsWith('-')) {
|
|
67
|
+
flags.set(body, next);
|
|
68
|
+
i += 1;
|
|
69
|
+
} else {
|
|
70
|
+
flags.set(body, true);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
} else if (tok.startsWith('-') && tok.length > 1) {
|
|
74
|
+
const body = tok.slice(1);
|
|
75
|
+
if (body === 'h') {
|
|
76
|
+
flags.set('help', true);
|
|
77
|
+
flags.set('h', true);
|
|
78
|
+
} else {
|
|
79
|
+
flags.set(body, true);
|
|
80
|
+
}
|
|
81
|
+
} else {
|
|
82
|
+
positional.push(tok);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return {
|
|
87
|
+
positional,
|
|
88
|
+
flags,
|
|
89
|
+
has(name) { return flags.has(name); },
|
|
90
|
+
get(name) {
|
|
91
|
+
const v = flags.get(name);
|
|
92
|
+
if (v === undefined) return undefined;
|
|
93
|
+
return v === true ? undefined : v;
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function restart(args, ctx) {
|
|
99
|
+
const stopCode = await stop(args, ctx);
|
|
100
|
+
if (stopCode !== 0) return stopCode;
|
|
101
|
+
return start(args, ctx);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const COMMANDS = {
|
|
105
|
+
init, login, register, update, upgrade, remove,
|
|
106
|
+
// aliases
|
|
107
|
+
uninstall: remove,
|
|
108
|
+
start, status, stop, restart, stats, logs,
|
|
109
|
+
payout, payments, gpu, firewall, chat, setup, model, train, uncensored, tui, doctor, service, pubkey, debug, deploy, console: consoleCmd, help
|
|
110
|
+
};
|
|
111
|
+
|
|
112
|
+
// Commands that can run without a loaded config.
|
|
113
|
+
const NO_CONFIG = new Set(['init', 'login', 'help', 'stats', 'logs', 'stop', 'restart', 'gpu', 'firewall', 'chat', 'setup', 'model', 'train', 'uncensored', 'tui', 'doctor', 'service', 'pubkey', 'debug', 'deploy', 'console', 'upgrade', 'update', 'remove', 'uninstall']);
|
|
114
|
+
// Commands that need a config but not a control-plane client (none today
|
|
115
|
+
// — kept as a future escape hatch).
|
|
116
|
+
const NO_CLIENT = new Set();
|
|
117
|
+
|
|
118
|
+
const KNOWN_COMMAND_NAMES = new Set(Object.keys(COMMANDS));
|
|
119
|
+
|
|
120
|
+
async function printVersion() {
|
|
121
|
+
process.stdout.write(`infernet v${CURRENT_VERSION}\n`);
|
|
122
|
+
const latest = await fetchLatestVersion();
|
|
123
|
+
if (!latest) {
|
|
124
|
+
process.stdout.write(` update check failed (offline?)\n`);
|
|
125
|
+
} else if (isNewerVersion(CURRENT_VERSION, latest)) {
|
|
126
|
+
process.stdout.write(` update available: v${latest} → run \`infernet upgrade\`\n`);
|
|
127
|
+
} else {
|
|
128
|
+
process.stdout.write(` up to date\n`);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
async function main() {
|
|
133
|
+
const argv = process.argv.slice(2);
|
|
134
|
+
|
|
135
|
+
// --version / -v / -V (checked before full routing)
|
|
136
|
+
if (argv[0] === '--version' || argv[0] === '-v' || argv[0] === '-V') {
|
|
137
|
+
await printVersion();
|
|
138
|
+
process.exit(0);
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const route = decideRoute(argv, KNOWN_COMMAND_NAMES, { isTTY: !!process.stdin.isTTY });
|
|
142
|
+
const sub = route.command;
|
|
143
|
+
const rest = route.rest;
|
|
144
|
+
|
|
145
|
+
if (sub === 'help') {
|
|
146
|
+
// Bare `infernet` → show version + up-to-date status, then help.
|
|
147
|
+
if (argv.length === 0) {
|
|
148
|
+
await printVersion();
|
|
149
|
+
process.stdout.write('\n');
|
|
150
|
+
}
|
|
151
|
+
await help();
|
|
152
|
+
process.exit(argv.length === 0 ? 0 : 0);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const handler = COMMANDS[sub];
|
|
156
|
+
if (!handler) {
|
|
157
|
+
// Defensive — decideRoute should never return a name we don't have.
|
|
158
|
+
process.stderr.write(`Internal error: no handler for '${sub}'\n\n`);
|
|
159
|
+
process.stderr.write(USAGE);
|
|
160
|
+
process.exit(1);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const args = parseArgs(rest);
|
|
164
|
+
|
|
165
|
+
let ctx = { config: null, client: null, configPath: getConfigPath() };
|
|
166
|
+
|
|
167
|
+
const helpRequested = args.has('help') || args.has('h');
|
|
168
|
+
|
|
169
|
+
if (!NO_CONFIG.has(sub) && !helpRequested) {
|
|
170
|
+
const config = await loadConfig();
|
|
171
|
+
if (!config) {
|
|
172
|
+
process.stderr.write(
|
|
173
|
+
`No infernet config found at ${getConfigPath()}. Run \`infernet init\` first.\n`
|
|
174
|
+
);
|
|
175
|
+
process.exit(1);
|
|
176
|
+
}
|
|
177
|
+
let client = null;
|
|
178
|
+
if (!NO_CLIENT.has(sub)) {
|
|
179
|
+
try {
|
|
180
|
+
client = createNodeClientFromConfig(config);
|
|
181
|
+
} catch (err) {
|
|
182
|
+
process.stderr.write(`${err.message}\n`);
|
|
183
|
+
process.exit(1);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
ctx = { config, client, configPath: getConfigPath() };
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
try {
|
|
190
|
+
const code = await handler(args, ctx);
|
|
191
|
+
process.exit(typeof code === 'number' ? code : 0);
|
|
192
|
+
} catch (err) {
|
|
193
|
+
process.stderr.write(`Error: ${err?.message ?? err}\n`);
|
|
194
|
+
if (process.env.INFERNET_DEBUG) {
|
|
195
|
+
process.stderr.write(String(err?.stack ?? '') + '\n');
|
|
196
|
+
}
|
|
197
|
+
process.exit(1);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
main().catch((err) => {
|
|
202
|
+
process.stderr.write(`Fatal: ${err?.message ?? err}\n`);
|
|
203
|
+
if (process.env.INFERNET_DEBUG) {
|
|
204
|
+
process.stderr.write(String(err?.stack ?? '') + '\n');
|
|
205
|
+
}
|
|
206
|
+
process.exit(1);
|
|
207
|
+
});
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Chat executor for the provider daemon.
|
|
3
|
+
*
|
|
4
|
+
* Pulls tokens from `@infernetprotocol/engine` (Mojo+MAX binary or
|
|
5
|
+
* in-process stub, depending on env) and forwards them to the control
|
|
6
|
+
* plane's `job_events` stream via signed POSTs. The daemon doesn't know
|
|
7
|
+
* which engine backend is loaded — that's `createEngine()`'s problem.
|
|
8
|
+
*
|
|
9
|
+
* Backend selection precedence (set by the operator at daemon start):
|
|
10
|
+
* 1. INFERNET_ENGINE_BACKEND=mojo|stub
|
|
11
|
+
* 2. INFERNET_ENGINE_BIN set → mojo
|
|
12
|
+
* 3. otherwise → stub (canned tokens, daemon still works on a fresh box)
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { createEngine, MSG } from "@infernetprotocol/engine";
|
|
16
|
+
import { loadConfig } from "./config.js";
|
|
17
|
+
import { getConversationKey, decrypt } from "./nip44.js";
|
|
18
|
+
import { getModelKeyPair } from "./model-key.js";
|
|
19
|
+
|
|
20
|
+
// Flush the event buffer when it hits this many tokens OR when we haven't
|
|
21
|
+
// flushed in this many ms. After merging (see mergeTokens), each flush
|
|
22
|
+
// produces at most one token row in job_events → one Supabase Realtime
|
|
23
|
+
// push → one SSE frame, regardless of how many model tokens accumulated.
|
|
24
|
+
const EVENT_BATCH_MAX = 16;
|
|
25
|
+
const EVENT_BATCH_FLUSH_MS = 250;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Merge consecutive token events in a batch into a single token event
|
|
29
|
+
* by concatenating their text. Non-token events preserve order.
|
|
30
|
+
*
|
|
31
|
+
* Before: [meta, token("H"), token("el"), token("lo"), done]
|
|
32
|
+
* After: [meta, token("Hello"), done]
|
|
33
|
+
*
|
|
34
|
+
* This reduces Supabase Realtime row-insert count from n_tokens to
|
|
35
|
+
* n_flushes, cutting streaming latency by ~10-20x (IPIP-0024).
|
|
36
|
+
*/
|
|
37
|
+
function mergeTokens(events) {
|
|
38
|
+
const out = [];
|
|
39
|
+
let tokenText = "";
|
|
40
|
+
for (const ev of events) {
|
|
41
|
+
if (ev.event_type === "token") {
|
|
42
|
+
tokenText += ev.data?.text ?? "";
|
|
43
|
+
} else {
|
|
44
|
+
if (tokenText) {
|
|
45
|
+
out.push({ event_type: "token", data: { text: tokenText } });
|
|
46
|
+
tokenText = "";
|
|
47
|
+
}
|
|
48
|
+
out.push(ev);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (tokenText) {
|
|
52
|
+
out.push({ event_type: "token", data: { text: tokenText } });
|
|
53
|
+
}
|
|
54
|
+
return out;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
class EventBuffer {
|
|
58
|
+
constructor(client, jobId) {
|
|
59
|
+
this.client = client;
|
|
60
|
+
this.jobId = jobId;
|
|
61
|
+
this.events = [];
|
|
62
|
+
this.lastFlush = Date.now();
|
|
63
|
+
}
|
|
64
|
+
async push(event_type, data) {
|
|
65
|
+
this.events.push({ event_type, data });
|
|
66
|
+
if (
|
|
67
|
+
this.events.length >= EVENT_BATCH_MAX ||
|
|
68
|
+
Date.now() - this.lastFlush >= EVENT_BATCH_FLUSH_MS
|
|
69
|
+
) {
|
|
70
|
+
await this.flush();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
async flush() {
|
|
74
|
+
if (this.events.length === 0) return;
|
|
75
|
+
const batch = mergeTokens(this.events);
|
|
76
|
+
this.events = [];
|
|
77
|
+
this.lastFlush = Date.now();
|
|
78
|
+
try {
|
|
79
|
+
await this.client.postJobEvents(this.jobId, batch);
|
|
80
|
+
} catch (err) {
|
|
81
|
+
process.stderr.write(`postJobEvents failed: ${err?.message ?? err}\n`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// One engine per daemon process — model load happens once. Lazy so the CLI
|
|
87
|
+
// doesn't pay the cost (or pull in the Mojo binary) until the first chat
|
|
88
|
+
// job actually arrives.
|
|
89
|
+
//
|
|
90
|
+
// We pull engine.{backend,model,ollamaHost} from the saved config so the
|
|
91
|
+
// daemon defaults to whatever `infernet setup` chose. The Ollama backend
|
|
92
|
+
// uses `defaultModel` when the job doesn't specify one (the playground
|
|
93
|
+
// /chat endpoint sometimes doesn't pass a model name). Without this
|
|
94
|
+
// fallback, every model-unspecified job died with
|
|
95
|
+
// "ollama backend: no model — set INFERNET_ENGINE_MODEL or pass model in the job"
|
|
96
|
+
let enginePromise = null;
|
|
97
|
+
function getEngine() {
|
|
98
|
+
if (!enginePromise) {
|
|
99
|
+
enginePromise = (async () => {
|
|
100
|
+
const config = (await loadConfig()) ?? {};
|
|
101
|
+
const eng = config.engine ?? {};
|
|
102
|
+
const opts = {};
|
|
103
|
+
if (eng.backend) opts.backend = eng.backend;
|
|
104
|
+
if (eng.model) opts.defaultModel = eng.model;
|
|
105
|
+
if (eng.ollamaHost) opts.host = eng.ollamaHost;
|
|
106
|
+
// Optional override for the per-request num_thread cap.
|
|
107
|
+
// Default (50% of cores) is computed inside the Ollama
|
|
108
|
+
// backend; only forward when the operator explicitly set
|
|
109
|
+
// an override in config.
|
|
110
|
+
if (Number.isFinite(eng.ollama_num_thread) && eng.ollama_num_thread > 0) {
|
|
111
|
+
opts.numThread = eng.ollama_num_thread;
|
|
112
|
+
}
|
|
113
|
+
return createEngine(opts);
|
|
114
|
+
})().catch((err) => {
|
|
115
|
+
// Reset so a transient failure doesn't permanently poison the
|
|
116
|
+
// daemon — next job will retry initialization.
|
|
117
|
+
enginePromise = null;
|
|
118
|
+
throw err;
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
return enginePromise;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export async function shutdownEngine() {
|
|
125
|
+
if (!enginePromise) return;
|
|
126
|
+
try {
|
|
127
|
+
const engine = await enginePromise;
|
|
128
|
+
await engine.shutdown();
|
|
129
|
+
} catch {
|
|
130
|
+
// best-effort
|
|
131
|
+
} finally {
|
|
132
|
+
enginePromise = null;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Run the chat executor for one job.
|
|
138
|
+
*
|
|
139
|
+
* @param {{ client: any, job: any, node: any }} ctx
|
|
140
|
+
* @returns {Promise<{ text: string, token_count: number, duration_ms: number }>}
|
|
141
|
+
* Full assistant response + token count + wall-clock duration.
|
|
142
|
+
* Caller (start.js) uses these to maintain a rolling tokens-per-second
|
|
143
|
+
* benchmark advertised via heartbeat → enables speed-aware routing.
|
|
144
|
+
*/
|
|
145
|
+
export async function executeChatJob({ client, job, node }) {
|
|
146
|
+
const input = job?.input_spec ?? {};
|
|
147
|
+
|
|
148
|
+
// IPIP-0027: if the job carries NIP-44 encrypted messages, decrypt them
|
|
149
|
+
// using our node privkey + the consumer's pubkey. Falls back to plaintext
|
|
150
|
+
// messages for legacy (unencrypted) jobs.
|
|
151
|
+
let messages = input.messages ?? [];
|
|
152
|
+
let convKey = null;
|
|
153
|
+
if (input.encrypted_messages && job.client_pubkey) {
|
|
154
|
+
// IPIP-0028: prefer model-specific key; fall back to node key (IPIP-0027).
|
|
155
|
+
const modelKP = job.model_name ? await getModelKeyPair(job.model_name) : null;
|
|
156
|
+
const decryptPrivKey = modelKP?.privateKey ?? node.privateKey ?? null;
|
|
157
|
+
if (!decryptPrivKey) {
|
|
158
|
+
throw new Error("No private key available to decrypt E2E messages");
|
|
159
|
+
}
|
|
160
|
+
try {
|
|
161
|
+
convKey = getConversationKey(decryptPrivKey, job.client_pubkey);
|
|
162
|
+
const plainJson = decrypt(convKey, input.encrypted_messages);
|
|
163
|
+
messages = JSON.parse(plainJson);
|
|
164
|
+
} catch (err) {
|
|
165
|
+
process.stderr.write(`nip44 decrypt failed: ${err?.message ?? err}\n`);
|
|
166
|
+
throw new Error("Failed to decrypt E2E messages — key mismatch or corrupted payload");
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const engine = await getEngine();
|
|
171
|
+
const buffer = new EventBuffer(client, job.id);
|
|
172
|
+
|
|
173
|
+
const t0 = Date.now();
|
|
174
|
+
const generation = engine.generate({
|
|
175
|
+
messages,
|
|
176
|
+
model: job.model_name ?? null,
|
|
177
|
+
max_tokens: input.max_tokens,
|
|
178
|
+
temperature: input.temperature
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
let accumulated = "";
|
|
182
|
+
let tokenCount = 0;
|
|
183
|
+
|
|
184
|
+
for await (const ev of generation.stream) {
|
|
185
|
+
switch (ev.type) {
|
|
186
|
+
case MSG.META:
|
|
187
|
+
await buffer.push("meta", {
|
|
188
|
+
provider_node_id: node.nodeId,
|
|
189
|
+
provider_name: node.name ?? null,
|
|
190
|
+
model: ev.model ?? job.model_name ?? null,
|
|
191
|
+
started_at: ev.started_at ?? new Date().toISOString(),
|
|
192
|
+
engine: engine.kind
|
|
193
|
+
});
|
|
194
|
+
break;
|
|
195
|
+
case MSG.TOKEN:
|
|
196
|
+
accumulated += ev.text ?? "";
|
|
197
|
+
tokenCount += 1;
|
|
198
|
+
await buffer.push("token", { text: ev.text ?? "" });
|
|
199
|
+
break;
|
|
200
|
+
case MSG.DONE:
|
|
201
|
+
if (typeof ev.text === "string" && ev.text.length > accumulated.length) {
|
|
202
|
+
accumulated = ev.text;
|
|
203
|
+
}
|
|
204
|
+
await buffer.push("done", {
|
|
205
|
+
text: accumulated,
|
|
206
|
+
reason: ev.reason ?? "stop",
|
|
207
|
+
finished_at: ev.finished_at ?? new Date().toISOString()
|
|
208
|
+
});
|
|
209
|
+
break;
|
|
210
|
+
case MSG.ERROR:
|
|
211
|
+
await buffer.push("error", { message: ev.message ?? "engine error" });
|
|
212
|
+
await buffer.flush();
|
|
213
|
+
throw new Error(ev.message ?? "engine error");
|
|
214
|
+
default:
|
|
215
|
+
// forward unknown event types verbatim — useful for backend
|
|
216
|
+
// extensions (logits, tool_call, etc.) once the protocol grows.
|
|
217
|
+
await buffer.push(ev.type, ev);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
await buffer.flush();
|
|
222
|
+
const duration_ms = Date.now() - t0;
|
|
223
|
+
return { text: accumulated, token_count: tokenCount, duration_ms };
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
export async function failChatJob({ client, jobId, message }) {
|
|
227
|
+
try {
|
|
228
|
+
await client.postJobEvents(jobId, [{ event_type: "error", data: { message } }]);
|
|
229
|
+
} catch {
|
|
230
|
+
// Non-fatal; the job row still gets marked 'failed' by the caller.
|
|
231
|
+
}
|
|
232
|
+
}
|