badgr-cli 1.0.44 → 1.0.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +178 -240
- package/package.json +1 -1
- package/src/api.js +18 -0
- package/src/badgr.js +18 -0
- package/src/catalog.js +31 -0
- package/src/commands/comfyui.js +70 -56
- package/src/commands/detect.js +58 -0
- package/src/commands/heartbeat.js +38 -0
- package/src/commands/receipts.js +39 -2
- package/src/commands/restart.js +74 -0
- package/src/commands/run.js +78 -3
- package/src/commands/serve.js +142 -9
- package/src/commands/train.js +22 -27
- package/src/detect.js +362 -0
- package/src/progress.js +160 -0
- package/src/store.js +11 -0
- package/tests/detect.test.js +191 -0
- package/tests/heartbeat.test.js +70 -0
- package/tests/job-progress-poll.test.js +136 -0
- package/tests/productized-runners.test.js +7 -0
- package/tests/restart.test.js +88 -0
- package/tests/run-lifecycle.test.js +111 -1
- package/tests/serve-apps.test.js +189 -0
- package/tests/serve-lifecycle.test.js +93 -0
- package/tests/store.test.js +22 -1
- package/tests/template.test.js +4 -4
- package/tests/workload-templates.test.js +22 -0
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import os from 'os';
|
|
4
|
+
import path from 'path';
|
|
5
|
+
import { detectWorkload, workloadTypeLabel } from '../src/detect.js';
|
|
6
|
+
|
|
7
|
+
let tmpRoot;
|
|
8
|
+
|
|
9
|
+
function makeProject(files) {
|
|
10
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'badgr-detect-'));
|
|
11
|
+
for (const [rel, content] of Object.entries(files)) {
|
|
12
|
+
const abs = path.join(dir, rel);
|
|
13
|
+
fs.mkdirSync(path.dirname(abs), { recursive: true });
|
|
14
|
+
fs.writeFileSync(abs, content);
|
|
15
|
+
}
|
|
16
|
+
return dir;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
beforeEach(() => { tmpRoot = []; });
|
|
20
|
+
afterEach(() => {
|
|
21
|
+
for (const dir of tmpRoot) {
|
|
22
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
function project(files) {
|
|
27
|
+
const dir = makeProject(files);
|
|
28
|
+
tmpRoot.push(dir);
|
|
29
|
+
return dir;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
describe('detectWorkload — missing / empty directory', () => {
|
|
33
|
+
it('reports an error without throwing for a missing directory', () => {
|
|
34
|
+
const report = detectWorkload('/nonexistent/badgr-detect-fixture');
|
|
35
|
+
expect(report.error).toBeTruthy();
|
|
36
|
+
expect(report.confidence).toBe('low');
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it('is low confidence with no command for an empty directory', () => {
|
|
40
|
+
const dir = project({});
|
|
41
|
+
const report = detectWorkload(dir);
|
|
42
|
+
expect(report.command).toBeNull();
|
|
43
|
+
expect(report.confidence).toBe('low');
|
|
44
|
+
});
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
describe('detectWorkload — plain Python CUDA script', () => {
|
|
48
|
+
it('finds the entry point and requirements, classifies as custom', () => {
|
|
49
|
+
const dir = project({
|
|
50
|
+
'train.py': `
|
|
51
|
+
import torch
|
|
52
|
+
model = torch.nn.Linear(10, 10).cuda()
|
|
53
|
+
for step in range(100):
|
|
54
|
+
optimizer.step()
|
|
55
|
+
`,
|
|
56
|
+
'requirements.txt': 'torch\nnumpy\n',
|
|
57
|
+
'output/.gitkeep': '',
|
|
58
|
+
});
|
|
59
|
+
const report = detectWorkload(dir);
|
|
60
|
+
expect(report.command).toBe('python train.py');
|
|
61
|
+
expect(report.requirements).toContain('requirements.txt');
|
|
62
|
+
expect(report.outputs).toContain('output/');
|
|
63
|
+
expect(report.workloadType).toBe('training');
|
|
64
|
+
expect(['medium', 'high']).toContain(report.confidence);
|
|
65
|
+
});
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
describe('detectWorkload — vLLM endpoint', () => {
|
|
69
|
+
it('classifies as endpoint and detects the port', () => {
|
|
70
|
+
const dir = project({
|
|
71
|
+
'server.py': `
|
|
72
|
+
from vllm import LLM
|
|
73
|
+
import uvicorn
|
|
74
|
+
app = None
|
|
75
|
+
|
|
76
|
+
if __name__ == "__main__":
|
|
77
|
+
uvicorn.run(app, port=8000)
|
|
78
|
+
`,
|
|
79
|
+
'requirements.txt': 'vllm\nfastapi\nuvicorn\n',
|
|
80
|
+
});
|
|
81
|
+
const report = detectWorkload(dir);
|
|
82
|
+
expect(report.command).toBe('python server.py');
|
|
83
|
+
expect(report.workloadType).toBe('endpoint');
|
|
84
|
+
expect(report.ports).toContain(8000);
|
|
85
|
+
expect(report.healthPath).toBe('/health');
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
describe('detectWorkload — LoRA training repo', () => {
|
|
90
|
+
it('classifies as training and estimates high VRAM for a large model', () => {
|
|
91
|
+
const dir = project({
|
|
92
|
+
'train.py': `
|
|
93
|
+
from peft import LoraConfig
|
|
94
|
+
base_model = "meta-llama/Llama-3.1-70B-Instruct"
|
|
95
|
+
trainer = Trainer(model=model)
|
|
96
|
+
trainer.train()
|
|
97
|
+
`,
|
|
98
|
+
'config.yaml': 'base_model: meta-llama/Llama-3.1-70B-Instruct\nlora_r: 16\n',
|
|
99
|
+
'requirements.txt': 'peft\ntransformers\naccelerate\n',
|
|
100
|
+
'checkpoints/.gitkeep': '',
|
|
101
|
+
});
|
|
102
|
+
const report = detectWorkload(dir);
|
|
103
|
+
expect(report.workloadType).toBe('training');
|
|
104
|
+
expect(report.checkpoints).toContain('checkpoints/');
|
|
105
|
+
expect(report.models.some(m => m.includes('Llama-3.1-70B'))).toBe(true);
|
|
106
|
+
expect(report.vram).toBe('80+ GB');
|
|
107
|
+
});
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
describe('detectWorkload — ComfyUI-style project', () => {
|
|
111
|
+
it('classifies as image_gen from a workflow json without special-casing ComfyUI by name', () => {
|
|
112
|
+
const dir = project({
|
|
113
|
+
'workflow_api.json': JSON.stringify({ '1': { class_type: 'CheckpointLoaderSimple' } }),
|
|
114
|
+
'requirements.txt': 'torch\npillow\n',
|
|
115
|
+
});
|
|
116
|
+
const report = detectWorkload(dir);
|
|
117
|
+
expect(report.workloadType).toBe('image_gen');
|
|
118
|
+
expect(workloadTypeLabel(report.workloadType)).toMatch(/image generation/);
|
|
119
|
+
});
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
describe('detectWorkload — custom Docker image', () => {
|
|
123
|
+
it('detects the Dockerfile and exposed port instead of guessing a base image', () => {
|
|
124
|
+
const dir = project({
|
|
125
|
+
Dockerfile: `
|
|
126
|
+
FROM python:3.11-slim
|
|
127
|
+
COPY . /app
|
|
128
|
+
EXPOSE 7860
|
|
129
|
+
CMD ["python", "app.py"]
|
|
130
|
+
`,
|
|
131
|
+
'app.py': `
|
|
132
|
+
import gradio as gr
|
|
133
|
+
demo = gr.Interface()
|
|
134
|
+
demo.launch(server_port=7860)
|
|
135
|
+
`,
|
|
136
|
+
});
|
|
137
|
+
const report = detectWorkload(dir);
|
|
138
|
+
expect(report.dockerfile).toBe('Dockerfile');
|
|
139
|
+
expect(report.image).toBeNull();
|
|
140
|
+
expect(report.ports).toContain(7860);
|
|
141
|
+
expect(report.workloadType).toBe('endpoint');
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
describe('detectWorkload — RunPod-style serverless handler job', () => {
|
|
146
|
+
it('classifies a handler-based job as an endpoint without naming any provider', () => {
|
|
147
|
+
const dir = project({
|
|
148
|
+
'rp_handler.py': `
|
|
149
|
+
import runpod
|
|
150
|
+
|
|
151
|
+
def handler(event):
|
|
152
|
+
return {"output": "ok"}
|
|
153
|
+
|
|
154
|
+
runpod.serverless.start({"handler": handler})
|
|
155
|
+
`,
|
|
156
|
+
'requirements.txt': 'runpod\n',
|
|
157
|
+
});
|
|
158
|
+
const report = detectWorkload(dir);
|
|
159
|
+
expect(report.command).toBe('python rp_handler.py');
|
|
160
|
+
expect(report.workloadType).toBe('endpoint');
|
|
161
|
+
// No provider name should leak into any user-visible field.
|
|
162
|
+
const serialized = JSON.stringify(report).toLowerCase();
|
|
163
|
+
expect(serialized).not.toContain('runpod.com');
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
describe('detectWorkload — batch image generation over an input directory', () => {
|
|
168
|
+
it('classifies as batch when the script iterates an input directory', () => {
|
|
169
|
+
const dir = project({
|
|
170
|
+
'generate.py': `
|
|
171
|
+
import os
|
|
172
|
+
for fname in os.listdir("input"):
|
|
173
|
+
process(fname)
|
|
174
|
+
`,
|
|
175
|
+
'input/.gitkeep': '',
|
|
176
|
+
'output/.gitkeep': '',
|
|
177
|
+
});
|
|
178
|
+
const report = detectWorkload(dir);
|
|
179
|
+
expect(report.workloadType).toBe('batch');
|
|
180
|
+
expect(report.inputs).toContain('input/');
|
|
181
|
+
expect(report.outputs).toContain('output/');
|
|
182
|
+
});
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
describe('workloadTypeLabel', () => {
|
|
186
|
+
it('never returns a provider-specific label', () => {
|
|
187
|
+
for (const t of ['endpoint', 'training', 'image_gen', 'batch', 'custom', 'unknown']) {
|
|
188
|
+
expect(workloadTypeLabel(t)).not.toMatch(/runpod|vast|hyperstack/i);
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
});
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* badgr heartbeat — resets an endpoint's idle-timeout clock
|
|
3
|
+
*/
|
|
4
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
5
|
+
import { heartbeatCommand } from '../src/commands/heartbeat.js';
|
|
6
|
+
|
|
7
|
+
vi.mock('../src/api.js', () => ({
|
|
8
|
+
heartbeatDeployment: vi.fn(),
|
|
9
|
+
}));
|
|
10
|
+
|
|
11
|
+
vi.mock('../src/store.js', () => ({
|
|
12
|
+
findDeployment: vi.fn(() => null),
|
|
13
|
+
}));
|
|
14
|
+
|
|
15
|
+
vi.mock('../src/config.js', () => ({
|
|
16
|
+
requireApiKey: vi.fn(),
|
|
17
|
+
}));
|
|
18
|
+
|
|
19
|
+
import * as api from '../src/api.js';
|
|
20
|
+
import * as store from '../src/store.js';
|
|
21
|
+
|
|
22
|
+
const config = { apiKey: 'sk-test', baseUrl: 'https://api.test/v1' };
|
|
23
|
+
|
|
24
|
+
const chalk = {
|
|
25
|
+
bold: s => s,
|
|
26
|
+
dim: s => s,
|
|
27
|
+
red: s => s,
|
|
28
|
+
yellow: s => s,
|
|
29
|
+
green: s => s,
|
|
30
|
+
cyan: s => s,
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
beforeEach(() => {
|
|
34
|
+
process.exitCode = undefined;
|
|
35
|
+
vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
36
|
+
vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
37
|
+
vi.resetAllMocks();
|
|
38
|
+
store.findDeployment.mockReturnValue(null);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
afterEach(() => {
|
|
42
|
+
vi.restoreAllMocks();
|
|
43
|
+
process.exitCode = undefined;
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('requires a deployment id', async () => {
|
|
47
|
+
await heartbeatCommand(config, [], chalk);
|
|
48
|
+
expect(api.heartbeatDeployment).not.toHaveBeenCalled();
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it('calls heartbeatDeployment and prints confirmation', async () => {
|
|
52
|
+
api.heartbeatDeployment.mockResolvedValue({ deployment_id: 'dep-abc123', last_activity_at: 1700000000 });
|
|
53
|
+
|
|
54
|
+
const lines = [];
|
|
55
|
+
console.log.mockImplementation((...args) => lines.push(args.join(' ')));
|
|
56
|
+
|
|
57
|
+
await heartbeatCommand(config, ['dep-abc123'], chalk);
|
|
58
|
+
|
|
59
|
+
expect(api.heartbeatDeployment).toHaveBeenCalledWith(config, 'dep-abc123');
|
|
60
|
+
expect(lines.some(l => l.includes('Heartbeat recorded for dep-abc123'))).toBe(true);
|
|
61
|
+
expect(process.exitCode).toBeFalsy();
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it('reports an error and sets exitCode on failure', async () => {
|
|
65
|
+
api.heartbeatDeployment.mockRejectedValue(new Error('deployment not found'));
|
|
66
|
+
|
|
67
|
+
await heartbeatCommand(config, ['dep-missing'], chalk);
|
|
68
|
+
|
|
69
|
+
expect(process.exitCode).toBe(1);
|
|
70
|
+
});
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { pollJobUntilTerminal, renderJobClosingBlock } from '../src/progress.js';
|
|
3
|
+
|
|
4
|
+
const chalk = new Proxy({}, { get: () => (s) => s });
|
|
5
|
+
const config = { apiKey: 'k', baseUrl: 'https://api.test' };
|
|
6
|
+
|
|
7
|
+
function silenceStdout() {
|
|
8
|
+
return vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
describe('pollJobUntilTerminal', () => {
|
|
12
|
+
it('returns immediately once the job reaches a terminal status', async () => {
|
|
13
|
+
const writeSpy = silenceStdout();
|
|
14
|
+
const detail = { job_id: 'job_1', status: 'completed', stage_label: 'Complete', health: 'completed' };
|
|
15
|
+
const callApi = vi.fn().mockResolvedValue(detail);
|
|
16
|
+
|
|
17
|
+
const result = await pollJobUntilTerminal(callApi, config, 'job_1', { chalk, maxMs: 60_000, pollMs: 1 });
|
|
18
|
+
|
|
19
|
+
expect(result.outcome).toBe('completed');
|
|
20
|
+
expect(result.detail).toBe(detail);
|
|
21
|
+
writeSpy.mockRestore();
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
it('surfaces progress fields (stage/current/total) while polling', async () => {
|
|
25
|
+
const writeSpy = silenceStdout();
|
|
26
|
+
const callApi = vi.fn()
|
|
27
|
+
.mockResolvedValueOnce({ job_id: 'job_2', status: 'running', stage_label: 'Generating images', progress_current: 3, progress_total: 10, progress_unit: 'images', health: 'progressing' })
|
|
28
|
+
.mockResolvedValueOnce({ job_id: 'job_2', status: 'completed', stage_label: 'Complete', health: 'completed' });
|
|
29
|
+
|
|
30
|
+
const result = await pollJobUntilTerminal(callApi, config, 'job_2', { chalk, maxMs: 60_000, pollMs: 1 });
|
|
31
|
+
|
|
32
|
+
expect(result.outcome).toBe('completed');
|
|
33
|
+
const output = writeSpy.mock.calls.map(c => c[0]).join('');
|
|
34
|
+
expect(output).toContain('3/10 images');
|
|
35
|
+
writeSpy.mockRestore();
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it('does not swallow repeated poll failures silently — gives up and reports polling_failed', async () => {
|
|
39
|
+
const writeSpy = silenceStdout();
|
|
40
|
+
const callApi = vi.fn().mockRejectedValue(new Error('network down'));
|
|
41
|
+
|
|
42
|
+
const result = await pollJobUntilTerminal(callApi, config, 'job_3', { chalk, maxMs: 60_000, pollMs: 1 });
|
|
43
|
+
|
|
44
|
+
expect(result.outcome).toBe('polling_failed');
|
|
45
|
+
// Must have warned the user at least once instead of looping silently.
|
|
46
|
+
const output = writeSpy.mock.calls.map(c => c[0]).join('');
|
|
47
|
+
expect(output).toMatch(/Lost contact/i);
|
|
48
|
+
writeSpy.mockRestore();
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it('announces a retry-on-a-different-route once, with teardown/billing status', async () => {
|
|
52
|
+
const writeSpy = silenceStdout();
|
|
53
|
+
const callApi = vi.fn()
|
|
54
|
+
.mockResolvedValueOnce({
|
|
55
|
+
job_id: 'job_retry', status: 'running', stage: 'retrying_different_route',
|
|
56
|
+
stage_label: 'Retrying on a different route', health: 'progressing',
|
|
57
|
+
progress_message: 'Retrying on a different route... attempt 2/2',
|
|
58
|
+
teardown_status: 'ok',
|
|
59
|
+
})
|
|
60
|
+
.mockResolvedValueOnce({
|
|
61
|
+
job_id: 'job_retry', status: 'completed', stage_label: 'Complete', health: 'completed',
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
const result = await pollJobUntilTerminal(callApi, config, 'job_retry', { chalk, maxMs: 60_000, pollMs: 1 });
|
|
65
|
+
|
|
66
|
+
expect(result.outcome).toBe('completed');
|
|
67
|
+
const output = writeSpy.mock.calls.map(c => c[0]).join('');
|
|
68
|
+
expect(output).toContain('Retrying on a different route... attempt 2/2');
|
|
69
|
+
expect(output).toContain('Previous attempt teardown: succeeded');
|
|
70
|
+
expect(output).toContain('Billing: stopped');
|
|
71
|
+
writeSpy.mockRestore();
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('recovers from transient poll failures once the API responds again', async () => {
|
|
75
|
+
const writeSpy = silenceStdout();
|
|
76
|
+
const callApi = vi.fn()
|
|
77
|
+
.mockRejectedValueOnce(new Error('timeout'))
|
|
78
|
+
.mockRejectedValueOnce(new Error('timeout'))
|
|
79
|
+
.mockResolvedValueOnce({ job_id: 'job_4', status: 'completed', stage_label: 'Complete', health: 'completed' });
|
|
80
|
+
|
|
81
|
+
const result = await pollJobUntilTerminal(callApi, config, 'job_4', { chalk, maxMs: 60_000, pollMs: 1 });
|
|
82
|
+
|
|
83
|
+
expect(result.outcome).toBe('completed');
|
|
84
|
+
writeSpy.mockRestore();
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
describe('renderJobClosingBlock', () => {
|
|
89
|
+
it('shows teardown/billing/logs/receipt for a completed job', () => {
|
|
90
|
+
const block = renderJobClosingBlock(chalk, {
|
|
91
|
+
job_id: 'job_5', status: 'completed', elapsed_seconds: 252,
|
|
92
|
+
charged_usd: 0.08, teardown_status: 'ok', billing_status: 'stopped',
|
|
93
|
+
}, 'rcpt-abc');
|
|
94
|
+
|
|
95
|
+
expect(block).toContain('Complete');
|
|
96
|
+
expect(block).toContain('Teardown: succeeded');
|
|
97
|
+
expect(block).toContain('Billing: stopped');
|
|
98
|
+
expect(block).toContain('badgr logs job_5');
|
|
99
|
+
expect(block).toContain('badgr receipts rcpt-abc');
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it('shows failure class, stage, and next action for a failed job', () => {
|
|
103
|
+
const block = renderJobClosingBlock(chalk, {
|
|
104
|
+
job_id: 'job_6', status: 'failed', elapsed_seconds: 1112,
|
|
105
|
+
error: { code: 'comfy_health_failed', message: 'ComfyUI did not start within 10 minutes.' },
|
|
106
|
+
failure_class: 'health_check_failed', stage: 'comfy_waiting_for_health',
|
|
107
|
+
teardown_status: 'ok', billing_status: 'stopped',
|
|
108
|
+
next_action: 'Run `badgr logs <job_id>` or retry with a longer max-runtime.',
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
expect(block).toContain('Failed');
|
|
112
|
+
expect(block).toContain('Class: health_check_failed');
|
|
113
|
+
expect(block).toContain('Stage: comfy_waiting_for_health');
|
|
114
|
+
expect(block).toContain('Next:');
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it('labels a runtime/spend cap failure as "Stopped — limit reached", not a crash', () => {
|
|
118
|
+
const block = renderJobClosingBlock(chalk, {
|
|
119
|
+
job_id: 'job_7', status: 'failed', elapsed_seconds: 545,
|
|
120
|
+
error: { code: 'max_runtime_reached', message: 'Job reached max runtime and was stopped. Billing ended.' },
|
|
121
|
+
failure_class: 'runtime_cap_reached',
|
|
122
|
+
teardown_status: 'ok', billing_status: 'stopped',
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
expect(block).toContain('Stopped — limit reached');
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it('shows cancelled block distinctly from failed', () => {
|
|
129
|
+
const block = renderJobClosingBlock(chalk, {
|
|
130
|
+
job_id: 'job_8', status: 'canceled', elapsed_seconds: 164,
|
|
131
|
+
teardown_status: 'ok', billing_status: 'stopped',
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
expect(block).toContain('Cancelled by user');
|
|
135
|
+
});
|
|
136
|
+
});
|
|
@@ -131,6 +131,13 @@ describe('BLESSED_COMFY_WORKFLOWS catalog', () => {
|
|
|
131
131
|
expect(wf.gpu_type).toBe('RTX_4090');
|
|
132
132
|
expect(wf.output_type).toBe('images');
|
|
133
133
|
});
|
|
134
|
+
|
|
135
|
+
it('has flux-basic with gpu_type and output_type', () => {
|
|
136
|
+
const wf = BLESSED_COMFY_WORKFLOWS['flux-basic'];
|
|
137
|
+
expect(wf).toBeDefined();
|
|
138
|
+
expect(wf.gpu_type).toBe('RTX_4090');
|
|
139
|
+
expect(wf.output_type).toBe('images');
|
|
140
|
+
});
|
|
134
141
|
});
|
|
135
142
|
|
|
136
143
|
// ── parseServeArgs alias recognition ─────────────────────────────────────────
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* badgr restart — relaunches an endpoint with the same config and API key
|
|
3
|
+
*/
|
|
4
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
5
|
+
import { restartCommand } from '../src/commands/restart.js';
|
|
6
|
+
|
|
7
|
+
vi.mock('../src/api.js', () => ({
|
|
8
|
+
restartDeployment: vi.fn(),
|
|
9
|
+
}));
|
|
10
|
+
|
|
11
|
+
vi.mock('../src/store.js', () => ({
|
|
12
|
+
findDeployment: vi.fn(() => null),
|
|
13
|
+
removeDeployment: vi.fn(),
|
|
14
|
+
addDeployment: vi.fn(),
|
|
15
|
+
addReceipt: vi.fn(),
|
|
16
|
+
generateReceiptId: vi.fn(() => 'rcpt-restart-001'),
|
|
17
|
+
}));
|
|
18
|
+
|
|
19
|
+
vi.mock('../src/config.js', () => ({
|
|
20
|
+
requireApiKey: vi.fn(),
|
|
21
|
+
}));
|
|
22
|
+
|
|
23
|
+
import * as api from '../src/api.js';
|
|
24
|
+
import * as store from '../src/store.js';
|
|
25
|
+
|
|
26
|
+
const config = { apiKey: 'sk-test', baseUrl: 'https://api.test/v1' };
|
|
27
|
+
|
|
28
|
+
const chalk = {
|
|
29
|
+
bold: s => s,
|
|
30
|
+
dim: s => s,
|
|
31
|
+
red: s => s,
|
|
32
|
+
yellow: s => s,
|
|
33
|
+
green: s => s,
|
|
34
|
+
cyan: s => s,
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
beforeEach(() => {
|
|
38
|
+
process.exitCode = undefined;
|
|
39
|
+
vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
40
|
+
vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
41
|
+
vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
|
|
42
|
+
vi.resetAllMocks();
|
|
43
|
+
store.findDeployment.mockReturnValue(null);
|
|
44
|
+
store.generateReceiptId.mockReturnValue('rcpt-restart-001');
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
afterEach(() => {
|
|
48
|
+
vi.restoreAllMocks();
|
|
49
|
+
process.exitCode = undefined;
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it('requires a deployment id', async () => {
|
|
53
|
+
await restartCommand(config, [], chalk);
|
|
54
|
+
expect(api.restartDeployment).not.toHaveBeenCalled();
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it('calls restartDeployment and prints the new deployment id and URL', async () => {
|
|
58
|
+
api.restartDeployment.mockResolvedValue({
|
|
59
|
+
deployment_id: 'dep-new-002',
|
|
60
|
+
workload_type: 'endpoint',
|
|
61
|
+
model: 'Qwen/Qwen2.5-7B-Instruct',
|
|
62
|
+
gpu_type: 'L40S',
|
|
63
|
+
gpu_count: 1,
|
|
64
|
+
status: 'provisioning',
|
|
65
|
+
endpoint_url: 'https://dep-new-002.aibadgr.com/v1',
|
|
66
|
+
cost_per_hour: 1.2,
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
const lines = [];
|
|
70
|
+
console.log.mockImplementation((...args) => lines.push(args.join(' ')));
|
|
71
|
+
|
|
72
|
+
await restartCommand(config, ['dep-old-001'], chalk);
|
|
73
|
+
|
|
74
|
+
expect(api.restartDeployment).toHaveBeenCalledWith(config, 'dep-old-001');
|
|
75
|
+
expect(store.removeDeployment).toHaveBeenCalledWith('dep-old-001');
|
|
76
|
+
expect(store.addDeployment).toHaveBeenCalledWith(expect.objectContaining({ id: 'dep-new-002' }));
|
|
77
|
+
expect(lines.some(l => l.includes('dep-new-002'))).toBe(true);
|
|
78
|
+
expect(lines.some(l => l.includes('https://dep-new-002.aibadgr.com/v1'))).toBe(true);
|
|
79
|
+
expect(process.exitCode).toBeFalsy();
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('reports an error and sets exitCode on failure', async () => {
|
|
83
|
+
api.restartDeployment.mockRejectedValue(new Error('deployment not found'));
|
|
84
|
+
|
|
85
|
+
await restartCommand(config, ['dep-missing'], chalk);
|
|
86
|
+
|
|
87
|
+
expect(process.exitCode).toBe(1);
|
|
88
|
+
});
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
* - Receipt explains state → status, exitCode, failureType, runtimeSeconds, finalCost
|
|
16
16
|
*/
|
|
17
17
|
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
18
|
-
import { runCommand } from '../src/commands/run.js';
|
|
18
|
+
import { runCommand, parseRunArgs } from '../src/commands/run.js';
|
|
19
19
|
|
|
20
20
|
// ── Module mocks ──────────────────────────────────────────────────────────────
|
|
21
21
|
|
|
@@ -222,6 +222,26 @@ describe('container fails to start', () => {
|
|
|
222
222
|
|
|
223
223
|
expect(process.exitCode).toBe(1);
|
|
224
224
|
});
|
|
225
|
+
|
|
226
|
+
it('prints the backend failure_class/next_action using the same vocabulary as the job API', async () => {
|
|
227
|
+
api.callApi
|
|
228
|
+
.mockResolvedValueOnce(makeDep({ status: 'provisioning' }))
|
|
229
|
+
.mockResolvedValueOnce({
|
|
230
|
+
status: 'failed',
|
|
231
|
+
failure_class: 'container_start_failed',
|
|
232
|
+
next_action: 'Check the image and command, then retry.',
|
|
233
|
+
});
|
|
234
|
+
const logs = [];
|
|
235
|
+
const origError = console.error;
|
|
236
|
+
console.error = (...args) => { logs.push(args.join(' ')); origError(...args); };
|
|
237
|
+
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
238
|
+
await vi.advanceTimersByTimeAsync(4000);
|
|
239
|
+
await p;
|
|
240
|
+
console.error = origError;
|
|
241
|
+
|
|
242
|
+
expect(logs.some(l => l.includes('Class: container_start_failed'))).toBe(true);
|
|
243
|
+
expect(logs.some(l => l.includes('Next: Check the image and command, then retry.'))).toBe(true);
|
|
244
|
+
});
|
|
225
245
|
});
|
|
226
246
|
|
|
227
247
|
// ─────────────────────────────────────────────────────────────────────────────
|
|
@@ -543,3 +563,93 @@ describe('--dry-run', () => {
|
|
|
543
563
|
expect(api.callApi).not.toHaveBeenCalled();
|
|
544
564
|
});
|
|
545
565
|
});
|
|
566
|
+
|
|
567
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
568
|
+
// 13. Progress-loss handling — --output / --checkpoint / --retry-safe / --resume-cmd
|
|
569
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
570
|
+
|
|
571
|
+
describe('--output / --checkpoint / --retry-safe / --resume-cmd', () => {
|
|
572
|
+
it('parseRunArgs parses all four flags', () => {
|
|
573
|
+
const { flags } = parseRunArgs([
|
|
574
|
+
'--output', './outputs', '--checkpoint', './checkpoints', '--retry-safe',
|
|
575
|
+
'--resume-cmd', 'python train.py --resume ./checkpoints/latest',
|
|
576
|
+
'--max-cost', '5', '--', 'python', 'train.py',
|
|
577
|
+
]);
|
|
578
|
+
expect(flags.output).toBe('./outputs');
|
|
579
|
+
expect(flags.checkpoint).toBe('./checkpoints');
|
|
580
|
+
expect(flags.retrySafe).toBe(true);
|
|
581
|
+
expect(flags.resumeCmd).toBe('python train.py --resume ./checkpoints/latest');
|
|
582
|
+
});
|
|
583
|
+
|
|
584
|
+
it('wires --output/--checkpoint/--retry-safe into the job env as BADGR_* vars', async () => {
|
|
585
|
+
api.callApi
|
|
586
|
+
.mockResolvedValueOnce(makeDep())
|
|
587
|
+
.mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
|
|
588
|
+
.mockResolvedValueOnce({ logs: [] });
|
|
589
|
+
const p = runCommand(config, [
|
|
590
|
+
'--output', './outputs', '--checkpoint', './checkpoints', '--retry-safe',
|
|
591
|
+
'--max-cost', '5', '--', 'python', 'train.py',
|
|
592
|
+
], chalk);
|
|
593
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
594
|
+
await p;
|
|
595
|
+
|
|
596
|
+
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
597
|
+
expect(postRunBody.env).toEqual(expect.objectContaining({
|
|
598
|
+
BADGR_OUTPUT_DIR: './outputs',
|
|
599
|
+
BADGR_CHECKPOINT_DIR: './checkpoints',
|
|
600
|
+
BADGR_RETRY_SAFE: '1',
|
|
601
|
+
}));
|
|
602
|
+
});
|
|
603
|
+
|
|
604
|
+
it('explicit --env overrides the BADGR_* convention var for the same key', async () => {
|
|
605
|
+
api.callApi
|
|
606
|
+
.mockResolvedValueOnce(makeDep())
|
|
607
|
+
.mockResolvedValueOnce({ status: 'completed', exit_code: 0 })
|
|
608
|
+
.mockResolvedValueOnce({ logs: [] });
|
|
609
|
+
const p = runCommand(config, [
|
|
610
|
+
'--output', './outputs', '--env', 'BADGR_OUTPUT_DIR=./custom-outputs',
|
|
611
|
+
'--max-cost', '5', '--', 'python', 'train.py',
|
|
612
|
+
], chalk);
|
|
613
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
614
|
+
await p;
|
|
615
|
+
|
|
616
|
+
const postRunBody = api.callApi.mock.calls[0][1].body;
|
|
617
|
+
expect(postRunBody.env.BADGR_OUTPUT_DIR).toBe('./custom-outputs');
|
|
618
|
+
});
|
|
619
|
+
|
|
620
|
+
it('prints the --resume-cmd hint when the job fails', async () => {
|
|
621
|
+
api.callApi
|
|
622
|
+
.mockResolvedValueOnce(makeDep())
|
|
623
|
+
.mockResolvedValueOnce({ status: 'failed', exit_code: 1 })
|
|
624
|
+
.mockResolvedValueOnce({ logs: [] });
|
|
625
|
+
const logs = [];
|
|
626
|
+
const captureChalk = { bold: s => s, dim: s => s, red: s => s, yellow: s => s, green: s => s, cyan: s => s };
|
|
627
|
+
const origLog = console.log;
|
|
628
|
+
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
629
|
+
const p = runCommand(config, [
|
|
630
|
+
'--resume-cmd', 'python train.py --resume ./checkpoints/latest',
|
|
631
|
+
'--max-cost', '5', '--', 'python', 'train.py',
|
|
632
|
+
], captureChalk);
|
|
633
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
634
|
+
await p;
|
|
635
|
+
console.log = origLog;
|
|
636
|
+
|
|
637
|
+
expect(logs.some(l => l.includes('python train.py --resume ./checkpoints/latest'))).toBe(true);
|
|
638
|
+
});
|
|
639
|
+
|
|
640
|
+
it('does not print a Resume line when --resume-cmd was not given', async () => {
|
|
641
|
+
api.callApi
|
|
642
|
+
.mockResolvedValueOnce(makeDep())
|
|
643
|
+
.mockResolvedValueOnce({ status: 'failed', exit_code: 1 })
|
|
644
|
+
.mockResolvedValueOnce({ logs: [] });
|
|
645
|
+
const logs = [];
|
|
646
|
+
const origLog = console.log;
|
|
647
|
+
console.log = (...args) => { logs.push(args.join(' ')); origLog(...args); };
|
|
648
|
+
const p = runCommand(config, ['python', 'train.py', '--max-cost', '5'], chalk);
|
|
649
|
+
await vi.advanceTimersByTimeAsync(5000);
|
|
650
|
+
await p;
|
|
651
|
+
console.log = origLog;
|
|
652
|
+
|
|
653
|
+
expect(logs.some(l => l.includes('Resume:'))).toBe(false);
|
|
654
|
+
});
|
|
655
|
+
});
|