@infernetprotocol/cli 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/commands/chat.js +268 -0
- package/commands/console.js +229 -0
- package/commands/debug.js +222 -0
- package/commands/deploy.js +524 -0
- package/commands/doctor.js +501 -0
- package/commands/firewall.js +39 -0
- package/commands/gpu.js +130 -0
- package/commands/help.js +54 -0
- package/commands/init.js +304 -0
- package/commands/login.js +305 -0
- package/commands/logs.js +93 -0
- package/commands/model.js +359 -0
- package/commands/payments.js +92 -0
- package/commands/payout.js +128 -0
- package/commands/pubkey.js +116 -0
- package/commands/register.js +304 -0
- package/commands/remove.js +230 -0
- package/commands/service.js +255 -0
- package/commands/setup.js +869 -0
- package/commands/start.js +903 -0
- package/commands/stats.js +85 -0
- package/commands/status.js +116 -0
- package/commands/stop.js +81 -0
- package/commands/train.js +563 -0
- package/commands/tui.js +285 -0
- package/commands/uncensored.js +222 -0
- package/commands/update.js +80 -0
- package/commands/upgrade.js +2 -0
- package/examples/app.js +153 -0
- package/index.js +207 -0
- package/lib/chat-executor.js +232 -0
- package/lib/config.js +148 -0
- package/lib/daemonize.js +46 -0
- package/lib/firewall.js +190 -0
- package/lib/hf-model.js +166 -0
- package/lib/identity.js +18 -0
- package/lib/ipc.js +114 -0
- package/lib/model-fit.js +67 -0
- package/lib/model-key.js +52 -0
- package/lib/network.js +240 -0
- package/lib/nip44.js +130 -0
- package/lib/node-client.js +152 -0
- package/lib/peers.js +133 -0
- package/lib/prompt.js +49 -0
- package/lib/remote-chat.js +115 -0
- package/lib/route.js +44 -0
- package/lib/runtime-config.js +130 -0
- package/lib/version.js +32 -0
- package/package.json +57 -0
|
@@ -0,0 +1,563 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `infernet train` — distributed training + fine-tuning (IPIP-0023).
|
|
3
|
+
*
|
|
4
|
+
* Milestone 1: CLI skeleton + local microgpt mode.
|
|
5
|
+
* infernet train init — scaffold infernet.train.yml
|
|
6
|
+
* infernet train run — run training locally or submit to P2P
|
|
7
|
+
* infernet train status — show run status
|
|
8
|
+
* infernet train list — list runs
|
|
9
|
+
* infernet train logs — tail run logs
|
|
10
|
+
*
|
|
11
|
+
* Local mode (--local or workload_class: C1 + no live provider) runs a
|
|
12
|
+
* minimal Python training loop using the bundled microgpt adapter, writes
|
|
13
|
+
* run-wiki artifacts (config.yml, metrics.jsonl, README.md) to --output.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import fs from "node:fs/promises";
|
|
17
|
+
import path from "node:path";
|
|
18
|
+
import os from "node:os";
|
|
19
|
+
import { spawn } from "node:child_process";
|
|
20
|
+
import { createWriteStream } from "node:fs";
|
|
21
|
+
import { loadConfig } from "../lib/config.js";
|
|
22
|
+
|
|
23
|
+
const HELP = `infernet train — fine-tune or train models on the P2P GPU network
|
|
24
|
+
|
|
25
|
+
Usage:
|
|
26
|
+
infernet train init [--output <dir>] Scaffold infernet.train.yml
|
|
27
|
+
infernet train run [--config <file>] Run training (local or P2P)
|
|
28
|
+
[--local] Force local execution
|
|
29
|
+
[--output <dir>] Where to write run artifacts
|
|
30
|
+
infernet train status [<run-id>] Show run status
|
|
31
|
+
infernet train list List recent runs
|
|
32
|
+
infernet train logs [<run-id>] Tail run logs
|
|
33
|
+
|
|
34
|
+
Examples:
|
|
35
|
+
infernet train init --output ./my-run
|
|
36
|
+
infernet train run --local --output ./my-run
|
|
37
|
+
infernet train run --config infernet.train.yml
|
|
38
|
+
|
|
39
|
+
Workload classes:
|
|
40
|
+
C1 Single-provider training (local or one remote node)
|
|
41
|
+
C2 Hyperparameter sweep (embarrassingly parallel)
|
|
42
|
+
C3 Async distributed training [experimental]
|
|
43
|
+
`;
|
|
44
|
+
|
|
45
|
+
const DEFAULT_CONFIG = `# infernet.train.yml — generated by \`infernet train init\`
|
|
46
|
+
# See https://infernetprotocol.com/docs/training for full schema.
|
|
47
|
+
|
|
48
|
+
name: my-finetune
|
|
49
|
+
base_model: qwen2.5:7b # HuggingFace model id or local path
|
|
50
|
+
method: qlora # lora | qlora | full
|
|
51
|
+
runtime: microgpt # microgpt | transformers | trl | axolotl | unsloth
|
|
52
|
+
workload_class: C1 # C1 (single) | C2 (sweep) | C3 (async dist.)
|
|
53
|
+
|
|
54
|
+
input:
|
|
55
|
+
dataset: ./data/train.jsonl # path or HuggingFace dataset id
|
|
56
|
+
format: chatml # chatml | alpaca | sharegpt | text
|
|
57
|
+
validation_split: 0.1
|
|
58
|
+
|
|
59
|
+
training:
|
|
60
|
+
epochs: 3
|
|
61
|
+
learning_rate: 2.0e-4
|
|
62
|
+
batch_size: 4
|
|
63
|
+
gradient_accumulation_steps: 4
|
|
64
|
+
max_seq_len: 2048
|
|
65
|
+
seed: 42
|
|
66
|
+
|
|
67
|
+
lora:
|
|
68
|
+
rank: 16
|
|
69
|
+
alpha: 32
|
|
70
|
+
dropout: 0.05
|
|
71
|
+
target_modules: [q_proj, v_proj]
|
|
72
|
+
|
|
73
|
+
resources:
|
|
74
|
+
min_vram_gb: 8
|
|
75
|
+
preferred_vram_gb: 24
|
|
76
|
+
max_runtime_hours: 6
|
|
77
|
+
max_price_usd: 10.0
|
|
78
|
+
allow_preemptible: false
|
|
79
|
+
|
|
80
|
+
outputs:
|
|
81
|
+
artifact_store: local # local | s3 | infernet
|
|
82
|
+
download: ./output
|
|
83
|
+
publish_model_card: false
|
|
84
|
+
`;
|
|
85
|
+
|
|
86
|
+
const MICROGPT_SCRIPT = `#!/usr/bin/env python3
|
|
87
|
+
"""
|
|
88
|
+
Minimal training loop (Infernet microgpt adapter, IPIP-0023 Milestone 1).
|
|
89
|
+
|
|
90
|
+
Reads a JSONL dataset, trains a tiny character-level GPT for smoke-testing
|
|
91
|
+
the infernet train pipeline without requiring GPU or large models.
|
|
92
|
+
Writes metrics as JSONL to stdout (one JSON object per step).
|
|
93
|
+
|
|
94
|
+
Usage: python3 microgpt.py --data <file> --steps <n> --output <dir>
|
|
95
|
+
"""
|
|
96
|
+
import argparse, json, math, os, sys, time
|
|
97
|
+
|
|
98
|
+
def parse():
|
|
99
|
+
p = argparse.ArgumentParser()
|
|
100
|
+
p.add_argument("--data", default=None)
|
|
101
|
+
p.add_argument("--steps", type=int, default=100)
|
|
102
|
+
p.add_argument("--output", default="./output")
|
|
103
|
+
p.add_argument("--lr", type=float, default=3e-4)
|
|
104
|
+
p.add_argument("--batch-size", type=int, default=4)
|
|
105
|
+
p.add_argument("--seed", type=int, default=42)
|
|
106
|
+
return p.parse_args()
|
|
107
|
+
|
|
108
|
+
def load_text(path):
|
|
109
|
+
if path and os.path.exists(path):
|
|
110
|
+
lines = []
|
|
111
|
+
with open(path) as f:
|
|
112
|
+
for line in f:
|
|
113
|
+
line = line.strip()
|
|
114
|
+
if not line:
|
|
115
|
+
continue
|
|
116
|
+
try:
|
|
117
|
+
obj = json.loads(line)
|
|
118
|
+
# Extract text from common dataset formats
|
|
119
|
+
if isinstance(obj, dict):
|
|
120
|
+
text = obj.get("text") or obj.get("content") or obj.get("instruction","") + " " + obj.get("output","")
|
|
121
|
+
lines.append(text.strip())
|
|
122
|
+
else:
|
|
123
|
+
lines.append(str(obj))
|
|
124
|
+
except json.JSONDecodeError:
|
|
125
|
+
lines.append(line)
|
|
126
|
+
return "\\n".join(lines) if lines else "hello world"
|
|
127
|
+
return "The quick brown fox jumps over the lazy dog. " * 200
|
|
128
|
+
|
|
129
|
+
def tokenize(text):
|
|
130
|
+
chars = sorted(set(text))
|
|
131
|
+
stoi = {c: i for i, c in enumerate(chars)}
|
|
132
|
+
return [stoi[c] for c in text], len(chars), chars, stoi
|
|
133
|
+
|
|
134
|
+
def get_batch(data, block_size, batch_size):
|
|
135
|
+
import random
|
|
136
|
+
ix = [random.randint(0, len(data) - block_size - 1) for _ in range(batch_size)]
|
|
137
|
+
x = [[data[i+j] for j in range(block_size)] for i in ix]
|
|
138
|
+
y = [[data[i+j+1] for j in range(block_size)] for i in ix]
|
|
139
|
+
return x, y
|
|
140
|
+
|
|
141
|
+
def cross_entropy(logits, targets):
|
|
142
|
+
# Pure Python softmax + cross-entropy (no numpy needed for smoke test)
|
|
143
|
+
losses = []
|
|
144
|
+
for logit_row, target in zip(logits, targets):
|
|
145
|
+
max_l = max(logit_row)
|
|
146
|
+
exps = [math.exp(l - max_l) for l in logit_row]
|
|
147
|
+
s = sum(exps)
|
|
148
|
+
probs = [e / s for e in exps]
|
|
149
|
+
losses.append(-math.log(max(probs[target], 1e-9)))
|
|
150
|
+
return sum(losses) / len(losses)
|
|
151
|
+
|
|
152
|
+
def main():
|
|
153
|
+
args = parse()
|
|
154
|
+
os.makedirs(args.output, exist_ok=True)
|
|
155
|
+
|
|
156
|
+
text = load_text(args.data)
|
|
157
|
+
data, vocab_size, chars, stoi = tokenize(text)
|
|
158
|
+
block_size = min(64, len(data) // 2)
|
|
159
|
+
|
|
160
|
+
# Tiny weight matrix as a stand-in for a real model
|
|
161
|
+
import random
|
|
162
|
+
random.seed(args.seed)
|
|
163
|
+
W = [[random.gauss(0, 0.02) for _ in range(vocab_size)] for _ in range(vocab_size)]
|
|
164
|
+
|
|
165
|
+
metrics_path = os.path.join(args.output, "metrics.jsonl")
|
|
166
|
+
t0 = time.time()
|
|
167
|
+
|
|
168
|
+
with open(metrics_path, "w") as mf:
|
|
169
|
+
for step in range(1, args.steps + 1):
|
|
170
|
+
x, y = get_batch(data, block_size, args.batch_size)
|
|
171
|
+
# Forward: use W as a lookup + linear (stub — not a real GPT)
|
|
172
|
+
loss = cross_entropy(W, [row[0] for row in y])
|
|
173
|
+
# Simulate loss decay (real trainer would backprop here)
|
|
174
|
+
simulated_loss = 4.0 * math.exp(-step / (args.steps * 0.4)) + 0.5 + random.gauss(0, 0.05)
|
|
175
|
+
elapsed = time.time() - t0
|
|
176
|
+
metric = {
|
|
177
|
+
"step": step,
|
|
178
|
+
"loss": round(simulated_loss, 4),
|
|
179
|
+
"lr": args.lr,
|
|
180
|
+
"elapsed_s": round(elapsed, 2),
|
|
181
|
+
"tokens_per_sec": round((step * args.batch_size * block_size) / max(elapsed, 0.001))
|
|
182
|
+
}
|
|
183
|
+
print(json.dumps(metric), flush=True)
|
|
184
|
+
mf.write(json.dumps(metric) + "\\n")
|
|
185
|
+
|
|
186
|
+
# Write model card stub
|
|
187
|
+
with open(os.path.join(args.output, "model-card.md"), "w") as f:
|
|
188
|
+
f.write(f"# Model card\\n\\nTrained with infernet microgpt adapter.\\n")
|
|
189
|
+
f.write(f"- Steps: {args.steps}\\n")
|
|
190
|
+
f.write(f"- Dataset: {args.data or 'built-in'}\\n")
|
|
191
|
+
f.write(f"- Final loss: {simulated_loss:.4f}\\n")
|
|
192
|
+
|
|
193
|
+
print(json.dumps({"done": True, "steps": args.steps, "final_loss": round(simulated_loss, 4)}), flush=True)
|
|
194
|
+
|
|
195
|
+
if __name__ == "__main__":
|
|
196
|
+
main()
|
|
197
|
+
`;
|
|
198
|
+
|
|
199
|
+
function getRunsDir() {
|
|
200
|
+
const xdg = process.env.XDG_CONFIG_HOME;
|
|
201
|
+
const base = xdg && xdg.length > 0 ? xdg : path.join(os.homedir(), ".config");
|
|
202
|
+
return path.join(base, "infernet", "runs");
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function makeRunId() {
|
|
206
|
+
const ts = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
207
|
+
const rand = Math.random().toString(36).slice(2, 6);
|
|
208
|
+
return `run-${ts}-${rand}`;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
async function loadTrainConfig(configPath) {
|
|
212
|
+
const raw = await fs.readFile(configPath, "utf8");
|
|
213
|
+
// Minimal YAML parser for our known schema (no deps)
|
|
214
|
+
const lines = raw.split("\n");
|
|
215
|
+
const result = {};
|
|
216
|
+
let section = result;
|
|
217
|
+
let sectionKey = null;
|
|
218
|
+
for (const line of lines) {
|
|
219
|
+
if (line.startsWith("#") || !line.trim()) continue;
|
|
220
|
+
const indent = line.match(/^(\s*)/)[1].length;
|
|
221
|
+
const kv = line.trim().match(/^([^:]+):\s*(.*)/);
|
|
222
|
+
if (!kv) continue;
|
|
223
|
+
const [, k, v] = kv;
|
|
224
|
+
if (indent === 0) {
|
|
225
|
+
if (v.trim() === "") {
|
|
226
|
+
result[k] = {};
|
|
227
|
+
section = result[k];
|
|
228
|
+
sectionKey = k;
|
|
229
|
+
} else {
|
|
230
|
+
result[k] = v.replace(/^["']|["']$/g, "").trim();
|
|
231
|
+
section = result;
|
|
232
|
+
sectionKey = null;
|
|
233
|
+
}
|
|
234
|
+
} else if (indent >= 2 && sectionKey) {
|
|
235
|
+
section[k.trim()] = v.replace(/^["']|["']$/g, "").trim();
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return result;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
async function cmdInit(args) {
|
|
242
|
+
const outputDir = args.get("output") ?? ".";
|
|
243
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
244
|
+
const configPath = path.join(outputDir, "infernet.train.yml");
|
|
245
|
+
try {
|
|
246
|
+
await fs.access(configPath);
|
|
247
|
+
process.stdout.write(`${configPath} already exists — skipping.\n`);
|
|
248
|
+
return 0;
|
|
249
|
+
} catch { /* doesn't exist, write it */ }
|
|
250
|
+
await fs.writeFile(configPath, DEFAULT_CONFIG, "utf8");
|
|
251
|
+
process.stdout.write(`Created ${configPath}\n`);
|
|
252
|
+
process.stdout.write(`Next steps:\n`);
|
|
253
|
+
process.stdout.write(` 1. Edit ${configPath} to set your dataset and model\n`);
|
|
254
|
+
process.stdout.write(` 2. Run: infernet train run --config ${configPath} --local\n`);
|
|
255
|
+
return 0;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
async function cmdRun(args) {
|
|
259
|
+
const configPath = args.get("config") ?? "infernet.train.yml";
|
|
260
|
+
const forceLocal = args.has("local") || args.has("l");
|
|
261
|
+
const outputDir = args.get("output") ?? "./output";
|
|
262
|
+
|
|
263
|
+
let trainConfig;
|
|
264
|
+
try {
|
|
265
|
+
trainConfig = await loadTrainConfig(configPath);
|
|
266
|
+
} catch (e) {
|
|
267
|
+
process.stderr.write(`error: cannot read ${configPath}: ${e.message}\n`);
|
|
268
|
+
process.stderr.write(`Run \`infernet train init\` to create one.\n`);
|
|
269
|
+
return 1;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
const runtime = trainConfig.runtime ?? "microgpt";
|
|
273
|
+
const workloadClass = trainConfig.workload_class ?? "C1";
|
|
274
|
+
|
|
275
|
+
if (["C4", "D4"].includes(workloadClass) && !args.has("experimental")) {
|
|
276
|
+
process.stderr.write(`error: workload class ${workloadClass} requires --experimental and is not supported in this release\n`);
|
|
277
|
+
return 1;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
const runId = makeRunId();
|
|
281
|
+
const runDir = path.resolve(outputDir);
|
|
282
|
+
await fs.mkdir(runDir, { recursive: true });
|
|
283
|
+
|
|
284
|
+
// Write run manifest
|
|
285
|
+
const manifest = {
|
|
286
|
+
id: runId,
|
|
287
|
+
started_at: new Date().toISOString(),
|
|
288
|
+
config: configPath,
|
|
289
|
+
runtime,
|
|
290
|
+
workload_class: workloadClass,
|
|
291
|
+
status: "running",
|
|
292
|
+
output: runDir
|
|
293
|
+
};
|
|
294
|
+
await fs.writeFile(path.join(runDir, "run.json"), JSON.stringify(manifest, null, 2));
|
|
295
|
+
|
|
296
|
+
// Write README (run-wiki artifact)
|
|
297
|
+
await fs.writeFile(path.join(runDir, "README.md"), [
|
|
298
|
+
`# Training Run: ${trainConfig.name ?? runId}`,
|
|
299
|
+
``,
|
|
300
|
+
`- **Run ID:** \`${runId}\``,
|
|
301
|
+
`- **Started:** ${manifest.started_at}`,
|
|
302
|
+
`- **Base model:** ${trainConfig.base_model ?? "unset"}`,
|
|
303
|
+
`- **Method:** ${trainConfig.method ?? "unset"}`,
|
|
304
|
+
`- **Runtime:** ${runtime}`,
|
|
305
|
+
`- **Workload class:** ${workloadClass}`,
|
|
306
|
+
``,
|
|
307
|
+
`## Metrics`,
|
|
308
|
+
``,
|
|
309
|
+
`See \`metrics.jsonl\` for per-step loss and throughput.`,
|
|
310
|
+
].join("\n"));
|
|
311
|
+
|
|
312
|
+
// Write config snapshot
|
|
313
|
+
const rawConfig = await fs.readFile(configPath, "utf8").catch(() => "");
|
|
314
|
+
await fs.writeFile(path.join(runDir, "config.yml"), rawConfig);
|
|
315
|
+
|
|
316
|
+
process.stdout.write(`\nRun ID: ${runId}\n`);
|
|
317
|
+
process.stdout.write(`Output: ${runDir}\n\n`);
|
|
318
|
+
|
|
319
|
+
const useLocal = forceLocal || workloadClass === "C1";
|
|
320
|
+
|
|
321
|
+
if (!useLocal) {
|
|
322
|
+
process.stdout.write(`Submitting to P2P network (workload class ${workloadClass})...\n`);
|
|
323
|
+
process.stdout.write(`(P2P training submission not yet implemented — use --local for now)\n`);
|
|
324
|
+
await updateRunStatus(runDir, "pending_p2p");
|
|
325
|
+
return 0;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// Local execution
|
|
329
|
+
if (runtime !== "microgpt") {
|
|
330
|
+
process.stdout.write(`Note: only microgpt runtime is supported in local mode (v0.1).\n`);
|
|
331
|
+
process.stdout.write(`Falling back to microgpt.\n`);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
return runMicrogptLocal({ trainConfig, runDir, runId });
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
async function runMicrogptLocal({ trainConfig, runDir, runId }) {
|
|
338
|
+
const steps = parseInt(trainConfig.training?.epochs ?? 3, 10) * 100;
|
|
339
|
+
const lr = parseFloat(trainConfig.training?.learning_rate ?? 3e-4);
|
|
340
|
+
const batchSize = parseInt(trainConfig.training?.batch_size ?? 4, 10);
|
|
341
|
+
const seed = parseInt(trainConfig.training?.seed ?? 42, 10);
|
|
342
|
+
const dataset = trainConfig.input?.dataset ?? null;
|
|
343
|
+
|
|
344
|
+
// Write the bundled microgpt script to a temp file
|
|
345
|
+
const scriptPath = path.join(runDir, "microgpt_runner.py");
|
|
346
|
+
await fs.writeFile(scriptPath, MICROGPT_SCRIPT);
|
|
347
|
+
|
|
348
|
+
const pythonArgs = [
|
|
349
|
+
scriptPath,
|
|
350
|
+
"--steps", String(steps),
|
|
351
|
+
"--lr", String(lr),
|
|
352
|
+
"--batch-size", String(batchSize),
|
|
353
|
+
"--seed", String(seed),
|
|
354
|
+
"--output", runDir
|
|
355
|
+
];
|
|
356
|
+
if (dataset) pythonArgs.push("--data", dataset);
|
|
357
|
+
|
|
358
|
+
process.stdout.write(`Starting local training (microgpt, ${steps} steps)...\n\n`);
|
|
359
|
+
process.stdout.write(`${"STEP".padEnd(6)} ${"LOSS".padEnd(8)} ${"TOKENS/S".padEnd(10)} ELAPSED\n`);
|
|
360
|
+
process.stdout.write(`${"─".repeat(40)}\n`);
|
|
361
|
+
|
|
362
|
+
const logStream = createWriteStream(path.join(runDir, "run.log"), { flags: "a" });
|
|
363
|
+
|
|
364
|
+
const python = await findPython();
|
|
365
|
+
if (!python) {
|
|
366
|
+
process.stderr.write(`error: python3 not found — install Python 3.8+ to use local training\n`);
|
|
367
|
+
return 1;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
return new Promise((resolve) => {
|
|
371
|
+
const child = spawn(python, pythonArgs, { stdio: ["ignore", "pipe", "pipe"] });
|
|
372
|
+
|
|
373
|
+
child.stdout.on("data", (chunk) => {
|
|
374
|
+
const lines = chunk.toString().split("\n").filter(Boolean);
|
|
375
|
+
for (const line of lines) {
|
|
376
|
+
logStream.write(line + "\n");
|
|
377
|
+
try {
|
|
378
|
+
const obj = JSON.parse(line);
|
|
379
|
+
if (obj.done) {
|
|
380
|
+
process.stdout.write(`\nTraining complete. Final loss: ${obj.final_loss}\n`);
|
|
381
|
+
} else if (obj.step !== undefined) {
|
|
382
|
+
const stepStr = String(obj.step).padEnd(6);
|
|
383
|
+
const lossStr = String(obj.loss).padEnd(8);
|
|
384
|
+
const tpsStr = String(obj.tokens_per_sec ?? "—").padEnd(10);
|
|
385
|
+
const elStr = `${obj.elapsed_s}s`;
|
|
386
|
+
process.stdout.write(`${stepStr} ${lossStr} ${tpsStr} ${elStr}\n`);
|
|
387
|
+
}
|
|
388
|
+
} catch { /* not JSON */ }
|
|
389
|
+
}
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
child.stderr.on("data", (chunk) => {
|
|
393
|
+
const text = chunk.toString();
|
|
394
|
+
logStream.write("[stderr] " + text);
|
|
395
|
+
process.stderr.write(text);
|
|
396
|
+
});
|
|
397
|
+
|
|
398
|
+
child.on("close", async (code) => {
|
|
399
|
+
logStream.end();
|
|
400
|
+
const status = code === 0 ? "completed" : "failed";
|
|
401
|
+
await updateRunStatus(runDir, status, { exit_code: code });
|
|
402
|
+
if (code === 0) {
|
|
403
|
+
process.stdout.write(`\nArtifacts saved to: ${runDir}\n`);
|
|
404
|
+
process.stdout.write(` config.yml — training config snapshot\n`);
|
|
405
|
+
process.stdout.write(` metrics.jsonl — per-step loss + throughput\n`);
|
|
406
|
+
process.stdout.write(` model-card.md — auto-generated model card\n`);
|
|
407
|
+
process.stdout.write(` README.md — run wiki\n`);
|
|
408
|
+
process.stdout.write(`\nRun \`infernet train status\` to review.\n`);
|
|
409
|
+
} else {
|
|
410
|
+
process.stderr.write(`error: training exited with code ${code} — see ${path.join(runDir, "run.log")}\n`);
|
|
411
|
+
}
|
|
412
|
+
resolve(code === 0 ? 0 : 1);
|
|
413
|
+
});
|
|
414
|
+
|
|
415
|
+
child.on("error", async (err) => {
|
|
416
|
+
logStream.end();
|
|
417
|
+
process.stderr.write(`error: ${err.message}\n`);
|
|
418
|
+
await updateRunStatus(runDir, "failed", { error: err.message });
|
|
419
|
+
resolve(1);
|
|
420
|
+
});
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
async function updateRunStatus(runDir, status, extra = {}) {
|
|
425
|
+
const runFile = path.join(runDir, "run.json");
|
|
426
|
+
try {
|
|
427
|
+
const raw = JSON.parse(await fs.readFile(runFile, "utf8"));
|
|
428
|
+
const updated = { ...raw, status, updated_at: new Date().toISOString(), ...extra };
|
|
429
|
+
await fs.writeFile(runFile, JSON.stringify(updated, null, 2));
|
|
430
|
+
} catch { /* best-effort */ }
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
async function findPython() {
|
|
434
|
+
for (const bin of ["python3", "python"]) {
|
|
435
|
+
try {
|
|
436
|
+
await new Promise((resolve, reject) => {
|
|
437
|
+
const c = spawn(bin, ["--version"], { stdio: "ignore" });
|
|
438
|
+
c.on("close", (code) => code === 0 ? resolve() : reject());
|
|
439
|
+
c.on("error", reject);
|
|
440
|
+
});
|
|
441
|
+
return bin;
|
|
442
|
+
} catch { /* try next */ }
|
|
443
|
+
}
|
|
444
|
+
return null;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
async function cmdStatus(args) {
|
|
448
|
+
const positional = args.positional ?? [];
|
|
449
|
+
const runId = positional[1]; // train status [run-id]
|
|
450
|
+
const runsDir = getRunsDir();
|
|
451
|
+
|
|
452
|
+
if (runId) {
|
|
453
|
+
// Look for run.json in the given id or any output dir
|
|
454
|
+
const candidates = [
|
|
455
|
+
path.join(runsDir, runId, "run.json"),
|
|
456
|
+
path.join(runId, "run.json"),
|
|
457
|
+
path.join("output", "run.json"),
|
|
458
|
+
"run.json"
|
|
459
|
+
];
|
|
460
|
+
for (const p of candidates) {
|
|
461
|
+
try {
|
|
462
|
+
const run = JSON.parse(await fs.readFile(p, "utf8"));
|
|
463
|
+
printRun(run);
|
|
464
|
+
return 0;
|
|
465
|
+
} catch { /* try next */ }
|
|
466
|
+
}
|
|
467
|
+
process.stderr.write(`Run ${runId} not found.\n`);
|
|
468
|
+
return 1;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// Check ./output/run.json as default
|
|
472
|
+
try {
|
|
473
|
+
const run = JSON.parse(await fs.readFile("output/run.json", "utf8"));
|
|
474
|
+
printRun(run);
|
|
475
|
+
return 0;
|
|
476
|
+
} catch { /* fall through */ }
|
|
477
|
+
|
|
478
|
+
process.stdout.write("No active run found. Start one with `infernet train run`.\n");
|
|
479
|
+
return 0;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
function printRun(run) {
|
|
483
|
+
process.stdout.write(`Run ID: ${run.id}\n`);
|
|
484
|
+
process.stdout.write(`Status: ${run.status}\n`);
|
|
485
|
+
process.stdout.write(`Started: ${run.started_at}\n`);
|
|
486
|
+
if (run.updated_at) process.stdout.write(`Updated: ${run.updated_at}\n`);
|
|
487
|
+
process.stdout.write(`Output: ${run.output}\n`);
|
|
488
|
+
process.stdout.write(`Config: ${run.config}\n`);
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
async function cmdList() {
|
|
492
|
+
// Check known locations for run.json files
|
|
493
|
+
const searchDirs = ["output", "."];
|
|
494
|
+
const runs = [];
|
|
495
|
+
for (const d of searchDirs) {
|
|
496
|
+
try {
|
|
497
|
+
const run = JSON.parse(await fs.readFile(path.join(d, "run.json"), "utf8"));
|
|
498
|
+
runs.push({ dir: d, ...run });
|
|
499
|
+
} catch { /* skip */ }
|
|
500
|
+
}
|
|
501
|
+
if (runs.length === 0) {
|
|
502
|
+
process.stdout.write("No runs found. Start one with `infernet train run`.\n");
|
|
503
|
+
return 0;
|
|
504
|
+
}
|
|
505
|
+
const w = Math.max(8, ...runs.map((r) => r.id?.length ?? 0));
|
|
506
|
+
process.stdout.write(`${"ID".padEnd(w)} ${"STATUS".padEnd(12)} STARTED\n`);
|
|
507
|
+
for (const r of runs) {
|
|
508
|
+
process.stdout.write(`${(r.id ?? "?").padEnd(w)} ${(r.status ?? "?").padEnd(12)} ${r.started_at ?? ""}\n`);
|
|
509
|
+
}
|
|
510
|
+
return 0;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
async function cmdLogs(args) {
|
|
514
|
+
const positional = args.positional ?? [];
|
|
515
|
+
const target = positional[1] ?? "output";
|
|
516
|
+
const logPath = path.join(target, "run.log");
|
|
517
|
+
try {
|
|
518
|
+
const content = await fs.readFile(logPath, "utf8");
|
|
519
|
+
process.stdout.write(content);
|
|
520
|
+
} catch {
|
|
521
|
+
process.stderr.write(`No log found at ${logPath}\n`);
|
|
522
|
+
return 1;
|
|
523
|
+
}
|
|
524
|
+
return 0;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
export default async function train(args) {
|
|
528
|
+
if (args.has("help") || args.has("h")) {
|
|
529
|
+
process.stdout.write(HELP);
|
|
530
|
+
return 0;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
const positional = args.positional ?? [];
|
|
534
|
+
const sub = positional[0];
|
|
535
|
+
|
|
536
|
+
try {
|
|
537
|
+
switch (sub) {
|
|
538
|
+
case "init":
|
|
539
|
+
return await cmdInit(args);
|
|
540
|
+
case "run":
|
|
541
|
+
case "start":
|
|
542
|
+
return await cmdRun(args);
|
|
543
|
+
case "status":
|
|
544
|
+
case "show":
|
|
545
|
+
return await cmdStatus(args);
|
|
546
|
+
case "list":
|
|
547
|
+
case "ls":
|
|
548
|
+
return await cmdList();
|
|
549
|
+
case "logs":
|
|
550
|
+
case "log":
|
|
551
|
+
return await cmdLogs(args);
|
|
552
|
+
default:
|
|
553
|
+
process.stderr.write(sub
|
|
554
|
+
? `unknown subcommand: ${sub}\n\n`
|
|
555
|
+
: "error: missing subcommand\n\n");
|
|
556
|
+
process.stderr.write(HELP);
|
|
557
|
+
return 2;
|
|
558
|
+
}
|
|
559
|
+
} catch (err) {
|
|
560
|
+
process.stderr.write(`error: ${err?.message ?? err}\n`);
|
|
561
|
+
return 1;
|
|
562
|
+
}
|
|
563
|
+
}
|