badgr-cli 1.0.45 → 1.0.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -2
- package/src/api.js +57 -0
- package/src/badgr.js +21 -0
- package/src/commands/batch.js +612 -0
- package/src/commands/heartbeat.js +38 -0
- package/src/commands/rerun.js +75 -0
- package/src/commands/restart.js +74 -0
- package/src/commands/run.js +3 -19
- package/src/commands/serve.js +26 -4
- package/src/commands/train.js +49 -21
- package/src/store.js +27 -0
- package/src/workloadSpec.js +126 -0
- package/tests/api.test.js +29 -1
- package/tests/batch.test.js +329 -0
- package/tests/heartbeat.test.js +70 -0
- package/tests/rerun.test.js +94 -0
- package/tests/restart.test.js +88 -0
- package/tests/serve-lifecycle.test.js +72 -0
- package/tests/train-lora-dataset.test.js +176 -0
- package/tests/workload-spec.test.js +180 -0
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { requireApiKey } from '../config.js';
|
|
2
|
+
import { findDeployment, addDeployment, addReceipt, generateReceiptId } from '../store.js';
|
|
3
|
+
import { rerunDeployment } from '../api.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* badgr rerun <deployment-id|name>
|
|
7
|
+
*
|
|
8
|
+
* Replays a previous job or endpoint with its exact original spec — same
|
|
9
|
+
* image, command, env vars, GPU, and cost/runtime caps. Works for one-off
|
|
10
|
+
* jobs (unlike `badgr restart`, which is endpoint-only) and never tears
|
|
11
|
+
* down the source deployment. Always creates a new deployment_id.
|
|
12
|
+
*/
|
|
13
|
+
export async function rerunCommand(config, args, chalk) {
|
|
14
|
+
const idOrName = args.find(a => !a.startsWith('--'));
|
|
15
|
+
|
|
16
|
+
if (!idOrName) {
|
|
17
|
+
console.error(chalk.red('Usage: badgr rerun <deployment-id|name>'));
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
requireApiKey(config);
|
|
22
|
+
|
|
23
|
+
const localDep = findDeployment(idOrName);
|
|
24
|
+
const deploymentId = localDep?.id ?? idOrName;
|
|
25
|
+
|
|
26
|
+
process.stdout.write(chalk.dim(` Replaying ${deploymentId}...`));
|
|
27
|
+
|
|
28
|
+
let dep;
|
|
29
|
+
try {
|
|
30
|
+
dep = await rerunDeployment(config, deploymentId);
|
|
31
|
+
} catch (err) {
|
|
32
|
+
process.stdout.write('\n');
|
|
33
|
+
console.error(chalk.red(`\n ✗ Could not rerun deployment: ${err.message}\n`));
|
|
34
|
+
process.exitCode = 1;
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
process.stdout.write('\n');
|
|
39
|
+
|
|
40
|
+
addDeployment({
|
|
41
|
+
id: dep.deployment_id,
|
|
42
|
+
name: dep.name,
|
|
43
|
+
type: dep.workload_type,
|
|
44
|
+
model: dep.model,
|
|
45
|
+
gpu: dep.gpu_type,
|
|
46
|
+
count: dep.gpu_count,
|
|
47
|
+
status: dep.status,
|
|
48
|
+
endpointUrl: dep.endpoint_url || dep.openai_base_url,
|
|
49
|
+
receiptId: dep.receipt_id,
|
|
50
|
+
createdAt: new Date().toISOString(),
|
|
51
|
+
costPerHour: dep.cost_per_hour || 0,
|
|
52
|
+
providerRoute: dep.provider ?? null,
|
|
53
|
+
tier: dep.tier ?? null,
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
const rcptId = dep.receipt_id || generateReceiptId();
|
|
57
|
+
addReceipt({
|
|
58
|
+
receiptId: rcptId,
|
|
59
|
+
action: 'badgr rerun',
|
|
60
|
+
deploymentId: dep.deployment_id,
|
|
61
|
+
gpu: dep.gpu_type,
|
|
62
|
+
status: dep.status,
|
|
63
|
+
rerunOf: deploymentId,
|
|
64
|
+
createdAt: new Date().toISOString(),
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
const endpointUrl = dep.endpoint_url || dep.openai_base_url;
|
|
68
|
+
console.log(chalk.green('\n ✓ Rerun submitted\n'));
|
|
69
|
+
console.log(` ${chalk.bold('Replayed from:')} ${chalk.dim(deploymentId)}`);
|
|
70
|
+
console.log(` ${chalk.bold('New deployment:')} ${chalk.cyan(dep.deployment_id)}`);
|
|
71
|
+
if (endpointUrl) console.log(` ${chalk.bold('Base URL:')} ${chalk.cyan(endpointUrl)}`);
|
|
72
|
+
console.log(` ${chalk.bold('Status:')} badgr status ${dep.deployment_id}`);
|
|
73
|
+
console.log(` ${chalk.bold('Logs:')} badgr logs ${dep.deployment_id}`);
|
|
74
|
+
console.log();
|
|
75
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { requireApiKey } from '../config.js';
|
|
2
|
+
import { findDeployment, removeDeployment, addDeployment, addReceipt, generateReceiptId } from '../store.js';
|
|
3
|
+
import { restartDeployment } from '../api.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* badgr restart <deployment-id|name>
|
|
7
|
+
*
|
|
8
|
+
* Tears down the current pod and relaunches an endpoint with the same
|
|
9
|
+
* config (GPU, model, price/cost/runtime caps, endpoint API key). Returns
|
|
10
|
+
* a *new* deployment_id and endpoint_url — the old pod's IP is gone.
|
|
11
|
+
*/
|
|
12
|
+
export async function restartCommand(config, args, chalk) {
|
|
13
|
+
const idOrName = args.find(a => !a.startsWith('--'));
|
|
14
|
+
|
|
15
|
+
if (!idOrName) {
|
|
16
|
+
console.error(chalk.red('Usage: badgr restart <deployment-id|name>'));
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
requireApiKey(config);
|
|
21
|
+
|
|
22
|
+
const localDep = findDeployment(idOrName);
|
|
23
|
+
const deploymentId = localDep?.id ?? idOrName;
|
|
24
|
+
|
|
25
|
+
process.stdout.write(chalk.dim(` Restarting ${deploymentId}...`));
|
|
26
|
+
|
|
27
|
+
let dep;
|
|
28
|
+
try {
|
|
29
|
+
dep = await restartDeployment(config, deploymentId);
|
|
30
|
+
} catch (err) {
|
|
31
|
+
process.stdout.write('\n');
|
|
32
|
+
console.error(chalk.red(`\n ✗ Could not restart deployment: ${err.message}\n`));
|
|
33
|
+
process.exitCode = 1;
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
process.stdout.write('\n');
|
|
38
|
+
|
|
39
|
+
removeDeployment(idOrName);
|
|
40
|
+
addDeployment({
|
|
41
|
+
id: dep.deployment_id,
|
|
42
|
+
name: dep.name,
|
|
43
|
+
type: dep.workload_type,
|
|
44
|
+
model: dep.model,
|
|
45
|
+
gpu: dep.gpu_type,
|
|
46
|
+
count: dep.gpu_count,
|
|
47
|
+
status: dep.status,
|
|
48
|
+
endpointUrl: dep.endpoint_url || dep.openai_base_url,
|
|
49
|
+
receiptId: dep.receipt_id,
|
|
50
|
+
createdAt: new Date().toISOString(),
|
|
51
|
+
costPerHour: dep.cost_per_hour || 0,
|
|
52
|
+
providerRoute: dep.provider ?? null,
|
|
53
|
+
tier: dep.tier ?? null,
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
const rcptId = dep.receipt_id || generateReceiptId();
|
|
57
|
+
addReceipt({
|
|
58
|
+
receiptId: rcptId,
|
|
59
|
+
action: 'badgr restart',
|
|
60
|
+
deploymentId: dep.deployment_id,
|
|
61
|
+
gpu: dep.gpu_type,
|
|
62
|
+
status: dep.status,
|
|
63
|
+
createdAt: new Date().toISOString(),
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
const endpointUrl = dep.endpoint_url || dep.openai_base_url;
|
|
67
|
+
console.log(chalk.green('\n ✓ Restarted\n'));
|
|
68
|
+
console.log(` ${chalk.bold('New deployment:')} ${chalk.cyan(dep.deployment_id)}`);
|
|
69
|
+
if (endpointUrl) console.log(` ${chalk.bold('New base URL:')} ${chalk.cyan(endpointUrl)}`);
|
|
70
|
+
console.log(chalk.dim(' Old endpoint URL is gone — update any client pointed at it.'));
|
|
71
|
+
console.log(chalk.dim(' Your endpoint API key is unchanged — no need to re-issue it.'));
|
|
72
|
+
console.log(` ${chalk.bold('Status:')} badgr status ${dep.deployment_id}`);
|
|
73
|
+
console.log();
|
|
74
|
+
}
|
package/src/commands/run.js
CHANGED
|
@@ -3,7 +3,7 @@ import path from 'path';
|
|
|
3
3
|
import os from 'os';
|
|
4
4
|
import { createWriteStream } from 'fs';
|
|
5
5
|
import { requireApiKey } from '../config.js';
|
|
6
|
-
import { callApi, terminateDeployment } from '../api.js';
|
|
6
|
+
import { callApi, terminateDeployment, uploadBlob } from '../api.js';
|
|
7
7
|
import { addReceipt, updateReceipt, generateReceiptId } from '../store.js';
|
|
8
8
|
import { normalizeTier, callWithFallback, HIGH_RATE_THRESHOLD } from '../fallback.js';
|
|
9
9
|
import { formatCliError } from '../errors.js';
|
|
@@ -428,32 +428,16 @@ export async function _uploadCodeZip(config, dirPath, chalk) {
|
|
|
428
428
|
process.stdout.write(chalk.dim(` ${sizeMb} MB\n`));
|
|
429
429
|
|
|
430
430
|
process.stdout.write(chalk.dim(' Uploading project...'));
|
|
431
|
-
// POST zip directly to the Badgr backend — no S3, no presigned URLs.
|
|
432
|
-
// Built-in FormData/Blob/fetch (Node >=18) keep the CLI dependency-free for
|
|
433
|
-
// multipart uploads; fetch sets the multipart boundary header automatically.
|
|
434
431
|
const fileData = fs.readFileSync(tmpFile);
|
|
435
|
-
const form = new FormData();
|
|
436
|
-
form.append('file', new Blob([fileData], { type: 'application/zip' }), 'project.zip');
|
|
437
|
-
|
|
438
|
-
const baseUrl = config.baseUrl.replace(/\/v1\/?$/, '');
|
|
439
432
|
let uploadResp;
|
|
440
433
|
try {
|
|
441
|
-
uploadResp = await
|
|
442
|
-
method: 'POST',
|
|
443
|
-
body: form,
|
|
444
|
-
headers: { 'Authorization': `Bearer ${config.apiKey}` },
|
|
445
|
-
});
|
|
434
|
+
uploadResp = await uploadBlob(config, { data: fileData, filename: 'project.zip', contentType: 'application/zip' });
|
|
446
435
|
} finally {
|
|
447
436
|
fs.unlinkSync(tmpFile);
|
|
448
437
|
}
|
|
449
438
|
|
|
450
|
-
if (!uploadResp.ok) {
|
|
451
|
-
const text = await uploadResp.text().catch(() => '');
|
|
452
|
-
throw new Error(`Upload failed: ${uploadResp.status} ${uploadResp.statusText}${text ? ` — ${text}` : ''}`);
|
|
453
|
-
}
|
|
454
|
-
const { code_uri: codeUri } = await uploadResp.json();
|
|
455
439
|
process.stdout.write(chalk.dim(' done\n'));
|
|
456
|
-
return
|
|
440
|
+
return uploadResp.code_uri;
|
|
457
441
|
}
|
|
458
442
|
|
|
459
443
|
export async function runCommand(config, args, chalk) {
|
package/src/commands/serve.js
CHANGED
|
@@ -32,6 +32,7 @@ export function parseServeArgs(args) {
|
|
|
32
32
|
if (args[i] === '--name') { flags.name = args[++i]; i++; continue; }
|
|
33
33
|
if (args[i] === '--no-wait') { flags.noWait = true; i++; continue; }
|
|
34
34
|
if (args[i] === '--max-cost') { flags.maxCost = parseFloat(args[++i]); i++; continue; }
|
|
35
|
+
if (args[i] === '--idle-timeout') { flags.idleTimeout = parseInt(args[++i], 10); i++; continue; }
|
|
35
36
|
if (args[i] === '--health-path') { flags.healthPath = args[++i]; i++; continue; }
|
|
36
37
|
if (args[i] === '--check-nodes') { flags.checkNodes = args[++i]; i++; continue; }
|
|
37
38
|
// All three aliases map to noMarketplaceFallback
|
|
@@ -208,7 +209,7 @@ async function validateComfyNodes(baseUrl, nodeList, chalk) {
|
|
|
208
209
|
// Known badgr serve flags — used to detect broken shell line continuation.
|
|
209
210
|
const _KNOWN_SERVE_FLAGS = new Set([
|
|
210
211
|
'--gpu', '--image', '--task', '--count', '--region', '--tier', '--max-price',
|
|
211
|
-
'--name', '--no-wait', '--max-cost', '--health-path', '--check-nodes',
|
|
212
|
+
'--name', '--no-wait', '--max-cost', '--idle-timeout', '--health-path', '--check-nodes',
|
|
212
213
|
'--no-fallback', '--strict-capacity', '--no-expanded-search', '--env',
|
|
213
214
|
'--persistent', '--yes', '-y', '--runtime', '--hf-repo', '--hf-file',
|
|
214
215
|
]);
|
|
@@ -543,6 +544,7 @@ export async function serveCommand(config, args, chalk) {
|
|
|
543
544
|
tier: tierOverride || effectiveTier,
|
|
544
545
|
...(Object.keys(effectiveEnv).length > 0 ? { env: effectiveEnv } : {}),
|
|
545
546
|
...(flags.maxCost ? { max_cost_usd: flags.maxCost } : {}),
|
|
547
|
+
...(flags.idleTimeout ? { idle_timeout_minutes: flags.idleTimeout } : {}),
|
|
546
548
|
...(flags.healthPath ? { health_path: flags.healthPath } : {}),
|
|
547
549
|
};
|
|
548
550
|
}
|
|
@@ -746,22 +748,34 @@ export async function serveCommand(config, args, chalk) {
|
|
|
746
748
|
else if (dep.model || effectiveModel) console.log(` ${chalk.bold('Model:')} ${dep.model || effectiveModel}`);
|
|
747
749
|
if (customImage) console.log(` ${chalk.bold('Image:')} ${customImage}`);
|
|
748
750
|
if (flags.maxCost) console.log(` ${chalk.bold('Max cost:')} $${flags.maxCost.toFixed(2)}`);
|
|
751
|
+
if (flags.idleTimeout) console.log(` ${chalk.bold('Idle timeout:')} ${flags.idleTimeout}m (auto-stops if idle — see Heartbeat below)`);
|
|
749
752
|
console.log(` ${chalk.bold('Logs:')} badgr logs ${dep.deployment_id}`);
|
|
753
|
+
console.log(` ${chalk.bold('Restart:')} badgr restart ${dep.deployment_id}`);
|
|
750
754
|
console.log(` ${chalk.bold('Stop billing:')} ${chalk.cyan(`badgr down ${dep.deployment_id}`)}`);
|
|
751
755
|
console.log(` ${chalk.bold('Receipt:')} badgr receipts ${rcptId}`);
|
|
752
756
|
console.log();
|
|
753
757
|
console.log(chalk.dim(' Billing continues until you run: ') + chalk.cyan(`badgr down ${dep.deployment_id}`));
|
|
754
758
|
|
|
755
759
|
if (endpointReady && !customImage) {
|
|
756
|
-
|
|
760
|
+
// dep.endpoint_api_key is a per-endpoint key generated for this deployment
|
|
761
|
+
// (vLLM model serves only) — shown exactly once, here. Falls back to the
|
|
762
|
+
// account-wide key (truncated) for serves that don't get one yet
|
|
763
|
+
// (custom images, managed transcribe/image tasks).
|
|
764
|
+
const hasEndpointKey = Boolean(dep.endpoint_api_key);
|
|
765
|
+
const authKey = hasEndpointKey ? dep.endpoint_api_key : `${config.apiKey?.slice(0, 4) || 'sk-...'}...`;
|
|
757
766
|
const sdkModel = isLlamaCpp ? 'default' : (dep.model || effectiveModel);
|
|
767
|
+
|
|
768
|
+
if (hasEndpointKey) {
|
|
769
|
+
console.log(` ${chalk.bold('API key:')} ${chalk.yellow(dep.endpoint_api_key)}`);
|
|
770
|
+
console.log(chalk.dim(' Shown once — copy it now. This key is scoped to this endpoint only.'));
|
|
771
|
+
}
|
|
758
772
|
console.log(` ${chalk.bold('Test with curl:')}`);
|
|
759
773
|
console.log(chalk.dim(` curl ${endpointUrl}/chat/completions \\`));
|
|
760
|
-
console.log(chalk.dim(` -H "Authorization: Bearer ${
|
|
774
|
+
console.log(chalk.dim(` -H "Authorization: Bearer ${authKey}" -H "Content-Type: application/json" \\`));
|
|
761
775
|
console.log(chalk.dim(` -d '{"model":"${sdkModel}","messages":[{"role":"user","content":"Hello"}]}'`));
|
|
762
776
|
console.log(` ${chalk.bold('Use with OpenAI SDK:')}`);
|
|
763
777
|
console.log(chalk.dim(` from openai import OpenAI`));
|
|
764
|
-
console.log(chalk.dim(` client = OpenAI(base_url="${endpointUrl}", api_key="${
|
|
778
|
+
console.log(chalk.dim(` client = OpenAI(base_url="${endpointUrl}", api_key="${authKey}")`));
|
|
765
779
|
if (flags.task === 'transcribe') {
|
|
766
780
|
console.log(chalk.dim(` with open("audio.mp3", "rb") as f:`));
|
|
767
781
|
console.log(chalk.dim(` t = client.audio.transcriptions.create(model="${sdkModel}", file=f, response_format="text")`));
|
|
@@ -774,5 +788,13 @@ export async function serveCommand(config, args, chalk) {
|
|
|
774
788
|
console.log(chalk.dim(` resp = client.chat.completions.create(model="${sdkModel}", messages=[{"role": "user", "content": "Hello"}])`));
|
|
775
789
|
}
|
|
776
790
|
console.log();
|
|
791
|
+
if (flags.idleTimeout) {
|
|
792
|
+
console.log(` ${chalk.bold('Heartbeat (required for --idle-timeout):')}`);
|
|
793
|
+
console.log(chalk.dim(` badgr heartbeat ${dep.deployment_id}`));
|
|
794
|
+
console.log(chalk.dim(' Badgr does not proxy your inference traffic, so call this on each real'));
|
|
795
|
+
console.log(chalk.dim(` request (or wire it into your client) — otherwise this endpoint auto-stops`));
|
|
796
|
+
console.log(chalk.dim(` after ${flags.idleTimeout}m even if it's still reachable.`));
|
|
797
|
+
console.log();
|
|
798
|
+
}
|
|
777
799
|
}
|
|
778
800
|
}
|
package/src/commands/train.js
CHANGED
|
@@ -11,6 +11,7 @@ import { addReceipt, updateReceipt, generateReceiptId } from '../store.js';
|
|
|
11
11
|
import { normalizeTier, callWithFallback } from '../fallback.js';
|
|
12
12
|
import { monitorBatchJob, fmtRuntime } from '../batch.js';
|
|
13
13
|
import { pollJobUntilTerminal, renderJobClosingBlock } from '../progress.js';
|
|
14
|
+
import { uploadBlob } from '../api.js';
|
|
14
15
|
|
|
15
16
|
const MAX_CONFIG_B = 512 * 1024; // 512 KB config limit
|
|
16
17
|
|
|
@@ -116,6 +117,7 @@ export function parseTrainLoraArgs(args) {
|
|
|
116
117
|
if (a === '--max-runtime') { flags.maxRuntime = parseFloat(args[++i]); i++; continue; }
|
|
117
118
|
if (a === '--tier') { flags.tier = args[++i]; i++; continue; }
|
|
118
119
|
if (a === '--gpu-type') { flags.gpuType = args[++i]; i++; continue; }
|
|
120
|
+
if (a === '--resume') { flags.resume = args[++i]; i++; continue; }
|
|
119
121
|
if (a === '--dry-run') { flags.dryRun = true; i++; continue; }
|
|
120
122
|
i++;
|
|
121
123
|
}
|
|
@@ -138,7 +140,8 @@ export async function trainLoraCommand(config, args, chalk) {
|
|
|
138
140
|
console.error(chalk.dim(' Dataset sources:'));
|
|
139
141
|
console.error(chalk.dim(' --dataset ./train.jsonl local file (uploaded first)'));
|
|
140
142
|
console.error(chalk.dim(' --dataset https://... direct URL'));
|
|
141
|
-
console.error(chalk.dim(' --file-id up_abc123 Badgr upload ID
|
|
143
|
+
console.error(chalk.dim(' --file-id up_abc123 Badgr upload ID'));
|
|
144
|
+
console.error(chalk.dim(' --resume <checkpoint-url> continue training from a prior job\'s checkpoint\n'));
|
|
142
145
|
process.exitCode = 1;
|
|
143
146
|
return;
|
|
144
147
|
}
|
|
@@ -164,6 +167,7 @@ export async function trainLoraCommand(config, args, chalk) {
|
|
|
164
167
|
}
|
|
165
168
|
if (flags.maxCost) console.log(` ${chalk.bold('Max cost:')} $${flags.maxCost}`);
|
|
166
169
|
console.log(` ${chalk.bold('Max runtime:')} ${flags.maxRuntime ?? 240}min`);
|
|
170
|
+
if (flags.resume) console.log(` ${chalk.bold('Resume from:')} ${flags.resume}`);
|
|
167
171
|
console.log(chalk.dim('\n Remove --dry-run to submit (local file datasets are uploaded first).\n'));
|
|
168
172
|
return;
|
|
169
173
|
}
|
|
@@ -177,36 +181,58 @@ export async function trainLoraCommand(config, args, chalk) {
|
|
|
177
181
|
} else if (flags.dataset && (flags.dataset.startsWith('http://') || flags.dataset.startsWith('https://') || flags.dataset.startsWith('s3://'))) {
|
|
178
182
|
input.dataset_url = flags.dataset;
|
|
179
183
|
} else if (flags.dataset) {
|
|
180
|
-
// Local file — upload first
|
|
181
|
-
|
|
184
|
+
// Local file — upload first (skipping re-upload if content is unchanged
|
|
185
|
+
// from a prior run, per the local upload cache).
|
|
182
186
|
if (!existsSync(flags.dataset)) {
|
|
183
187
|
console.error(chalk.red(`\n ✗ Dataset file not found: ${flags.dataset}\n`));
|
|
184
188
|
process.exitCode = 1;
|
|
185
189
|
return;
|
|
186
190
|
}
|
|
187
|
-
|
|
191
|
+
const { createHash } = await import('crypto');
|
|
188
192
|
const { readFileSync: readDs } = await import('fs');
|
|
193
|
+
const { getCachedUploadId, setCachedUploadId } = await import('../store.js');
|
|
189
194
|
const fileData = readDs(flags.dataset);
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
195
|
+
|
|
196
|
+
// .jsonl datasets are the common case — catch a malformed file before
|
|
197
|
+
// spending an upload + a GPU on it rather than failing mid-training.
|
|
198
|
+
if (flags.dataset.endsWith('.jsonl')) {
|
|
199
|
+
const lines = fileData.toString('utf8').split('\n').filter(l => l.trim());
|
|
200
|
+
const badLine = lines.findIndex(l => { try { JSON.parse(l); return false; } catch { return true; } });
|
|
201
|
+
if (badLine !== -1) {
|
|
202
|
+
console.error(chalk.red(`\n ✗ Dataset is not valid JSONL — line ${badLine + 1} is not valid JSON.\n`));
|
|
203
|
+
process.exitCode = 1;
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
if (lines.length === 0) {
|
|
207
|
+
console.error(chalk.red(`\n ✗ Dataset file ${flags.dataset} is empty.\n`));
|
|
208
|
+
process.exitCode = 1;
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const contentHash = createHash('sha256').update(fileData).digest('hex');
|
|
214
|
+
const cachedUploadId = getCachedUploadId(contentHash);
|
|
215
|
+
if (cachedUploadId) {
|
|
216
|
+
input.dataset_file_id = cachedUploadId;
|
|
217
|
+
console.log(chalk.dim(`\n Dataset unchanged since last run — reusing upload ${cachedUploadId} (no re-upload)`));
|
|
218
|
+
} else {
|
|
219
|
+
console.log(chalk.dim(`\n Uploading dataset ${flags.dataset}…`));
|
|
220
|
+
let uploadResp;
|
|
221
|
+
try {
|
|
222
|
+
uploadResp = await uploadBlob(config, { data: fileData, filename: flags.dataset.split('/').pop() || 'dataset.jsonl' });
|
|
223
|
+
} catch (err) {
|
|
224
|
+
console.error(chalk.red(`\n ✗ Dataset upload failed: ${err.message}\n`));
|
|
225
|
+
process.exitCode = 1;
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
input.dataset_file_id = uploadResp.upload_id;
|
|
229
|
+
setCachedUploadId(contentHash, uploadResp.upload_id, { file: flags.dataset });
|
|
230
|
+
console.log(chalk.dim(` Uploaded: ${uploadResp.upload_id}`));
|
|
203
231
|
}
|
|
204
|
-
const uploadResp = await uploadRes.json();
|
|
205
|
-
input.dataset_file_id = uploadResp.upload_id;
|
|
206
|
-
console.log(chalk.dim(` Uploaded: ${uploadResp.upload_id}`));
|
|
207
232
|
}
|
|
208
233
|
|
|
209
234
|
if (flags.gpuType) input.gpu_type = flags.gpuType;
|
|
235
|
+
if (flags.resume) input.resume_from_checkpoint_url = flags.resume;
|
|
210
236
|
|
|
211
237
|
const rcptId = generateReceiptId();
|
|
212
238
|
const maxRuntime = flags.maxRuntime ?? 240;
|
|
@@ -215,7 +241,9 @@ export async function trainLoraCommand(config, args, chalk) {
|
|
|
215
241
|
console.log(` ${chalk.bold('Base model:')} ${flags.baseModel}`);
|
|
216
242
|
console.log(` ${chalk.bold('Preset:')} ${input.config_preset}`);
|
|
217
243
|
console.log(` ${chalk.bold('Max cost:')} $${flags.maxCost}`);
|
|
218
|
-
console.log(` ${chalk.bold('Max runtime:')} ${maxRuntime} min
|
|
244
|
+
console.log(` ${chalk.bold('Max runtime:')} ${maxRuntime} min`);
|
|
245
|
+
if (flags.resume) console.log(` ${chalk.bold('Resuming from:')} ${flags.resume}`);
|
|
246
|
+
console.log();
|
|
219
247
|
|
|
220
248
|
let job;
|
|
221
249
|
try {
|
package/src/store.js
CHANGED
|
@@ -106,3 +106,30 @@ export function findReceipt(idOrJobId, storeFile = STORE_FILE) {
|
|
|
106
106
|
const { receipts } = loadStore(storeFile);
|
|
107
107
|
return receipts.find(r => r.receiptId === idOrJobId || r.job_id === idOrJobId) ?? null;
|
|
108
108
|
}
|
|
109
|
+
|
|
110
|
+
// ---------------------------------------------------------------------------
|
|
111
|
+
// Upload cache — content-hash → upload_id, so re-running the same dataset
|
|
112
|
+
// (e.g. `badgr train lora` reruns, iterating on a config) skips re-uploading
|
|
113
|
+
// unchanged files. Persisted to ~/.badgr/upload-cache.json.
|
|
114
|
+
// ---------------------------------------------------------------------------
|
|
115
|
+
export const UPLOAD_CACHE_FILE = join(CONFIG_DIR, 'upload-cache.json');
|
|
116
|
+
|
|
117
|
+
export function loadUploadCache(cacheFile = UPLOAD_CACHE_FILE) {
|
|
118
|
+
if (!existsSync(cacheFile)) return {};
|
|
119
|
+
try {
|
|
120
|
+
return JSON.parse(readFileSync(cacheFile, 'utf8'));
|
|
121
|
+
} catch {
|
|
122
|
+
return {};
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export function getCachedUploadId(contentHash, cacheFile = UPLOAD_CACHE_FILE) {
|
|
127
|
+
return loadUploadCache(cacheFile)[contentHash]?.uploadId ?? null;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export function setCachedUploadId(contentHash, uploadId, meta = {}, cacheFile = UPLOAD_CACHE_FILE) {
|
|
131
|
+
const cache = loadUploadCache(cacheFile);
|
|
132
|
+
cache[contentHash] = { uploadId, cachedAt: new Date().toISOString(), ...meta };
|
|
133
|
+
mkdirSync(dirname(cacheFile), { recursive: true });
|
|
134
|
+
writeFileSync(cacheFile, JSON.stringify(cache, null, 2));
|
|
135
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* workload.yml parsing/validation for `badgr batch`.
|
|
3
|
+
*
|
|
4
|
+
* Deliberately generic — not robotics-specific. Physical-AI is just one
|
|
5
|
+
* workload that happens to use this schema, not a special case of it.
|
|
6
|
+
*
|
|
7
|
+
* Schema:
|
|
8
|
+
* name: <string> required
|
|
9
|
+
* image: <string> required
|
|
10
|
+
* command: [<string>, ...] required
|
|
11
|
+
* inputs: ["<local>:<container abs path>", ...] optional
|
|
12
|
+
* outputs: ["<container abs path>", ...] optional
|
|
13
|
+
* env: {KEY: value} optional
|
|
14
|
+
* max_cost: <number> required
|
|
15
|
+
* max_runtime_minutes: <number> required
|
|
16
|
+
* success_metric: optional
|
|
17
|
+
* file: <container abs path>
|
|
18
|
+
* key: <string>
|
|
19
|
+
* higher_is_better: <bool>
|
|
20
|
+
*/
|
|
21
|
+
import { readFileSync } from 'fs';
|
|
22
|
+
import { dirname, resolve } from 'path';
|
|
23
|
+
import yaml from 'js-yaml';
|
|
24
|
+
|
|
25
|
+
export class WorkloadSpecError extends Error {}
|
|
26
|
+
|
|
27
|
+
function isNonEmptyString(v) {
|
|
28
|
+
return typeof v === 'string' && v.length > 0;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Parse a single "local:containerPath" inputs entry. containerPath must be
|
|
33
|
+
* an absolute path (workload.yml is container-path-oriented for inputs);
|
|
34
|
+
* splitting on the *last* colon so Windows-style local paths with a drive
|
|
35
|
+
* letter don't break parsing.
|
|
36
|
+
*/
|
|
37
|
+
export function parseInputEntry(entry) {
|
|
38
|
+
const idx = entry.lastIndexOf(':');
|
|
39
|
+
if (idx <= 0 || idx === entry.length - 1) {
|
|
40
|
+
throw new WorkloadSpecError(`inputs entry must be "<local path>:<container path>": ${entry}`);
|
|
41
|
+
}
|
|
42
|
+
const localPath = entry.slice(0, idx);
|
|
43
|
+
const containerPath = entry.slice(idx + 1);
|
|
44
|
+
if (!containerPath.startsWith('/')) {
|
|
45
|
+
throw new WorkloadSpecError(`inputs entry's container path must be absolute: ${entry}`);
|
|
46
|
+
}
|
|
47
|
+
return { localPath, containerPath };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function validateWorkloadSpec(raw) {
|
|
51
|
+
const errors = [];
|
|
52
|
+
if (!isNonEmptyString(raw?.name)) errors.push('name is required');
|
|
53
|
+
if (!isNonEmptyString(raw?.image)) errors.push('image is required');
|
|
54
|
+
if (!Array.isArray(raw?.command) || raw.command.length === 0) errors.push('command is required and must be a non-empty list');
|
|
55
|
+
if (raw?.max_cost == null || typeof raw.max_cost !== 'number' || raw.max_cost <= 0) errors.push('max_cost is required and must be a positive number');
|
|
56
|
+
if (raw?.max_runtime_minutes == null || typeof raw.max_runtime_minutes !== 'number' || raw.max_runtime_minutes <= 0) errors.push('max_runtime_minutes is required and must be a positive number');
|
|
57
|
+
|
|
58
|
+
if (raw?.inputs !== undefined) {
|
|
59
|
+
if (!Array.isArray(raw.inputs)) {
|
|
60
|
+
errors.push('inputs must be a list');
|
|
61
|
+
} else {
|
|
62
|
+
for (const entry of raw.inputs) {
|
|
63
|
+
try { parseInputEntry(entry); } catch (err) { errors.push(err.message); }
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (raw?.outputs !== undefined) {
|
|
68
|
+
if (!Array.isArray(raw.outputs) || raw.outputs.some(p => typeof p !== 'string' || !p.startsWith('/'))) {
|
|
69
|
+
errors.push('outputs must be a list of absolute container paths');
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (raw?.success_metric !== undefined) {
|
|
73
|
+
const sm = raw.success_metric;
|
|
74
|
+
if (!sm || !isNonEmptyString(sm.file) || !sm.file.startsWith('/')) errors.push('success_metric.file must be an absolute container path');
|
|
75
|
+
if (!isNonEmptyString(sm?.key)) errors.push('success_metric.key is required when success_metric is set');
|
|
76
|
+
if (sm?.higher_is_better !== undefined && typeof sm.higher_is_better !== 'boolean') errors.push('success_metric.higher_is_better must be a boolean');
|
|
77
|
+
}
|
|
78
|
+
return errors;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Load and validate a workload.yml file. Returns a normalized spec object.
|
|
83
|
+
* Throws WorkloadSpecError with all validation problems on failure.
|
|
84
|
+
*/
|
|
85
|
+
export function parseWorkloadYaml(path) {
|
|
86
|
+
let raw;
|
|
87
|
+
try {
|
|
88
|
+
raw = readFileSync(path, 'utf8');
|
|
89
|
+
} catch (err) {
|
|
90
|
+
throw new WorkloadSpecError(`Could not read ${path}: ${err.message}`);
|
|
91
|
+
}
|
|
92
|
+
let doc;
|
|
93
|
+
try {
|
|
94
|
+
doc = yaml.load(raw);
|
|
95
|
+
} catch (err) {
|
|
96
|
+
throw new WorkloadSpecError(`Invalid YAML in ${path}: ${err.message}`);
|
|
97
|
+
}
|
|
98
|
+
const errors = validateWorkloadSpec(doc);
|
|
99
|
+
if (errors.length > 0) {
|
|
100
|
+
throw new WorkloadSpecError(`Invalid workload.yml:\n${errors.map(e => ` - ${e}`).join('\n')}`);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const baseDir = dirname(resolve(path));
|
|
104
|
+
const inputs = (doc.inputs ?? []).map(entry => {
|
|
105
|
+
const { localPath, containerPath } = parseInputEntry(entry);
|
|
106
|
+
return { localPath: resolve(baseDir, localPath), containerPath };
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
return {
|
|
110
|
+
name: doc.name,
|
|
111
|
+
image: doc.image,
|
|
112
|
+
command: doc.command,
|
|
113
|
+
inputs,
|
|
114
|
+
outputs: doc.outputs ?? [],
|
|
115
|
+
env: doc.env ?? {},
|
|
116
|
+
maxCost: doc.max_cost,
|
|
117
|
+
maxRuntimeMinutes: doc.max_runtime_minutes,
|
|
118
|
+
successMetric: doc.success_metric
|
|
119
|
+
? {
|
|
120
|
+
file: doc.success_metric.file,
|
|
121
|
+
key: doc.success_metric.key,
|
|
122
|
+
higherIsBetter: doc.success_metric.higher_is_better ?? true,
|
|
123
|
+
}
|
|
124
|
+
: null,
|
|
125
|
+
};
|
|
126
|
+
}
|
package/tests/api.test.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
|
2
|
-
import { callApi, listModels, chatCompletion, submitJob, getJobStatus, listReceipts, getReceipt, runJob, serveModel } from '../src/api.js';
|
|
2
|
+
import { callApi, listModels, chatCompletion, submitJob, getJobStatus, listReceipts, getReceipt, runJob, serveModel, uploadBlob } from '../src/api.js';
|
|
3
3
|
|
|
4
4
|
const mockConfig = { apiKey: 'sk-test', baseUrl: 'https://api.test/v1', defaultModel: 'llama-3' };
|
|
5
5
|
|
|
@@ -138,3 +138,31 @@ describe('serveModel', () => {
|
|
|
138
138
|
expect(result.deployment_id).toBe('dep-xyz');
|
|
139
139
|
});
|
|
140
140
|
});
|
|
141
|
+
|
|
142
|
+
describe('uploadBlob', () => {
|
|
143
|
+
it('POSTs multipart form data to /v1/uploads with Authorization, no manual Content-Type', async () => {
|
|
144
|
+
mockFetch({ upload_id: 'up_1', code_uri: 'https://api.test/v1/uploads/up_1/download?token=t' });
|
|
145
|
+
const result = await uploadBlob(mockConfig, { data: Buffer.from('hello'), filename: 'a.txt' });
|
|
146
|
+
const [url, init] = global.fetch.mock.calls[0];
|
|
147
|
+
expect(url).toBe('https://api.test/v1/uploads');
|
|
148
|
+
expect(init.method).toBe('POST');
|
|
149
|
+
expect(init.body).toBeInstanceOf(FormData);
|
|
150
|
+
expect(init.headers.Authorization).toBe('Bearer sk-test');
|
|
151
|
+
expect(init.headers['Content-Type']).toBeUndefined();
|
|
152
|
+
expect(result.upload_id).toBe('up_1');
|
|
153
|
+
expect(result.code_uri).toContain('up_1');
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
it('sets the Blob content type when contentType is given', async () => {
|
|
157
|
+
mockFetch({ upload_id: 'up_2', code_uri: 'https://x/up_2' });
|
|
158
|
+
await uploadBlob(mockConfig, { data: Buffer.from('x'), filename: 'a.tar.gz', contentType: 'application/gzip' });
|
|
159
|
+
const [, init] = global.fetch.mock.calls[0];
|
|
160
|
+
expect(init.body.get('file').type).toBe('application/gzip');
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it('throws a clear error on a non-ok response', async () => {
|
|
164
|
+
mockFetch('boom', false, 500);
|
|
165
|
+
await expect(uploadBlob(mockConfig, { data: Buffer.from('x'), filename: 'a.txt' }))
|
|
166
|
+
.rejects.toThrow(/Upload failed: 500/);
|
|
167
|
+
});
|
|
168
|
+
});
|