@plaiful/cli 0.16.0 → 0.16.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli.js +5 -0
- package/src/train.js +264 -260
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -446,6 +446,11 @@ async function docsPull(ctx, r = repo(ctx)) {
|
|
|
446
446
|
}
|
|
447
447
|
const where = whereThingsGo(ctx, r);
|
|
448
448
|
lines.push(...where.lines);
|
|
449
|
+
// The web template reads .env.development, which plaiful env writes (PLAI-202): say so when it's missing.
|
|
450
|
+
const needsEnv = fs.existsSync(path.join(r.root, `${ENV_FILE}.example`)) && !fs.existsSync(path.join(r.root, ENV_FILE));
|
|
451
|
+
if (needsEnv) {
|
|
452
|
+
lines.push(`No ${ENV_FILE} yet: run plaiful env to write it from ${r.product}'s dev config${fs.existsSync(path.join(r.root, '.env')) ? ' (the game doesn\'t read .env)' : ''}.`);
|
|
453
|
+
}
|
|
449
454
|
lines.push(copies ? 'Local copies as of now; the live docs are `plaiful docs`.' : 'The platform docs are live: plaiful docs (start with plaiful docs read README).');
|
|
450
455
|
ctx.out(lines.join('\n'), { success: true, dir: r.docsDir, ...result, gitignore: ignore, misplaced_specs: where.misplaced });
|
|
451
456
|
return 0;
|
package/src/train.js
CHANGED
|
@@ -1,260 +1,264 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* `plaiful models train …`: run a training job on a Google Cloud GPU (Vertex AI custom training) instead of the
|
|
5
|
-
* developer's own machine, so the GPU time draws on Plaiful's Google Cloud credits (PlaifulCMS
|
|
6
|
-
* docs/design/model-registry.md §5). Plaiful staff only for now, and it runs on this machine's `gcloud` sign-in: the
|
|
7
|
-
* platform holds no Google credentials.
|
|
8
|
-
*
|
|
9
|
-
* A job: the code folder and the data folders are copied to the project's bucket (data is cached across jobs), Vertex
|
|
10
|
-
* AI runs the entry command on one GPU with the bucket mounted at /gcs/<bucket>, the job writes to {out}, and
|
|
11
|
-
* `train fetch` brings the output home, ready to card and `plaiful models publish`.
|
|
12
|
-
*
|
|
13
|
-
* Settings, from flags, else .plaiful.json "training": { project, region, bucket, image }, else PLAIFUL_GCP_* env.
|
|
14
|
-
*/
|
|
15
|
-
|
|
16
|
-
const fs = require('fs');
|
|
17
|
-
const os = require('os');
|
|
18
|
-
const path = require('path');
|
|
19
|
-
|
|
20
|
-
/** One GPU per job, by name: the machine Vertex AI pairs with it. */
|
|
21
|
-
const GPUS = {
|
|
22
|
-
l4: { machineType: 'g2-standard-8', acceleratorType: 'NVIDIA_L4' },
|
|
23
|
-
t4: { machineType: 'n1-standard-8', acceleratorType: 'NVIDIA_TESLA_T4' },
|
|
24
|
-
a100: { machineType: 'a2-highgpu-1g', acceleratorType: 'NVIDIA_TESLA_A100' },
|
|
25
|
-
};
|
|
26
|
-
// Google's prebuilt PyTorch training image.
|
|
27
|
-
|
|
28
|
-
const
|
|
29
|
-
const
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
function
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
'
|
|
61
|
-
'
|
|
62
|
-
'
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
{ name: '
|
|
83
|
-
{ name: '
|
|
84
|
-
{ name: '
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
if (!s.
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
const
|
|
118
|
-
const
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
fine = ok(
|
|
127
|
-
fine = ok(!!s.
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
lines.push(
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
const
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
if (!
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
const
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
`
|
|
194
|
-
`
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
const
|
|
210
|
-
const
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* `plaiful models train …`: run a training job on a Google Cloud GPU (Vertex AI custom training) instead of the
|
|
5
|
+
* developer's own machine, so the GPU time draws on Plaiful's Google Cloud credits (PlaifulCMS
|
|
6
|
+
* docs/design/model-registry.md §5). Plaiful staff only for now, and it runs on this machine's `gcloud` sign-in: the
|
|
7
|
+
* platform holds no Google credentials.
|
|
8
|
+
*
|
|
9
|
+
* A job: the code folder and the data folders are copied to the project's bucket (data is cached across jobs), Vertex
|
|
10
|
+
* AI runs the entry command on one GPU with the bucket mounted at /gcs/<bucket>, the job writes to {out}, and
|
|
11
|
+
* `train fetch` brings the output home, ready to card and `plaiful models publish`.
|
|
12
|
+
*
|
|
13
|
+
* Settings, from flags, else .plaiful.json "training": { project, region, bucket, image }, else PLAIFUL_GCP_* env.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const fs = require('fs');
|
|
17
|
+
const os = require('os');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
|
|
20
|
+
/** One GPU per job, by name: the machine Vertex AI pairs with it. */
|
|
21
|
+
const GPUS = {
|
|
22
|
+
l4: { machineType: 'g2-standard-8', acceleratorType: 'NVIDIA_L4' },
|
|
23
|
+
t4: { machineType: 'n1-standard-8', acceleratorType: 'NVIDIA_TESLA_T4' },
|
|
24
|
+
a100: { machineType: 'a2-highgpu-1g', acceleratorType: 'NVIDIA_TESLA_A100' },
|
|
25
|
+
};
|
|
26
|
+
// Google's newest prebuilt PyTorch training image. The 2.1 image is deprecated, and Vertex AI failed every job on it
|
|
27
|
+
// with only "Internal error" (2026-11-16).
|
|
28
|
+
const DEFAULT_IMAGE = 'us-docker.pkg.dev/vertex-ai/training/pytorch-gpu.2-4.py310:latest';
|
|
29
|
+
const DEFAULT_REGION = 'us-central1';
|
|
30
|
+
const DEFAULT_HOURS = 12;
|
|
31
|
+
const CODE_EXCLUDE = '(^|/)(\\.git|node_modules|__pycache__|\\.venv|venv)(/|$)';
|
|
32
|
+
|
|
33
|
+
const jobsFile = () => path.join(process.env.PLAIFUL_HOME || path.join(os.homedir(), '.plaiful'), 'train-jobs.json');
|
|
34
|
+
function readJobs() { try { return JSON.parse(fs.readFileSync(jobsFile(), 'utf8')); } catch { return []; } }
|
|
35
|
+
function writeJobs(jobs) { fs.mkdirSync(path.dirname(jobsFile()), { recursive: true }); fs.writeFileSync(jobsFile(), `${JSON.stringify(jobs, null, 2)}\n`); }
|
|
36
|
+
|
|
37
|
+
/** A name gcloud and the bucket accept: lowercase letters, digits and dashes. */
|
|
38
|
+
const slug = (s) => String(s).toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40);
|
|
39
|
+
const stamp = (d = new Date()) => d.toISOString().replace(/[-:]/g, '').replace(/\..*/, '').replace('T', '-').toLowerCase();
|
|
40
|
+
|
|
41
|
+
/** The job's settings → { project, region, bucket, image } (bucket without gs://). */
|
|
42
|
+
function settings(flags, repoConfig) {
|
|
43
|
+
const t = (repoConfig && repoConfig.training) || {};
|
|
44
|
+
const s = {
|
|
45
|
+
project: flags.project || t.project || process.env.PLAIFUL_GCP_PROJECT,
|
|
46
|
+
region: flags.region || t.region || process.env.PLAIFUL_GCP_REGION || DEFAULT_REGION,
|
|
47
|
+
bucket: String(flags.bucket || t.bucket || process.env.PLAIFUL_GCP_BUCKET || '').replace(/^gs:\/\//, '').replace(/\/+$/, ''),
|
|
48
|
+
image: flags.image || t.image || DEFAULT_IMAGE,
|
|
49
|
+
};
|
|
50
|
+
return s;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** The shell script the container runs: copy the code out of the bucket, install, run the entry. */
|
|
54
|
+
function jobScript({ bucket, id, entry }) {
|
|
55
|
+
const root = `/gcs/${bucket}`;
|
|
56
|
+
const cmd = entry.replace(/\{data\}/g, `${root}/data`).replace(/\{out\}/g, `${root}/jobs/${id}/out`);
|
|
57
|
+
return [
|
|
58
|
+
'set -euo pipefail',
|
|
59
|
+
`mkdir -p /tmp/code && cp -r ${root}/jobs/${id}/code/. /tmp/code/`,
|
|
60
|
+
'cd /tmp/code',
|
|
61
|
+
'if [ -f requirements.txt ]; then pip install -q -r requirements.txt; fi',
|
|
62
|
+
'mkdir -p "$PLAIFUL_OUT"',
|
|
63
|
+
'nvidia-smi --query-gpu=name,memory.total --format=csv || true',
|
|
64
|
+
cmd,
|
|
65
|
+
].join('\n');
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** The Vertex AI CustomJobSpec (gcloud ai custom-jobs create --config). */
|
|
69
|
+
function jobSpec({ bucket, id, entry, gpu, machine, image, hours, task }) {
|
|
70
|
+
const g = GPUS[gpu];
|
|
71
|
+
const root = `/gcs/${bucket}`;
|
|
72
|
+
return {
|
|
73
|
+
workerPoolSpecs: [{
|
|
74
|
+
machineSpec: { machineType: machine || g.machineType, acceleratorType: g.acceleratorType, acceleratorCount: 1 },
|
|
75
|
+
replicaCount: 1,
|
|
76
|
+
diskSpec: { bootDiskType: 'pd-ssd', bootDiskSizeGb: 100 },
|
|
77
|
+
containerSpec: {
|
|
78
|
+
imageUri: image,
|
|
79
|
+
command: ['bash', '-c'],
|
|
80
|
+
args: [jobScript({ bucket, id, entry })],
|
|
81
|
+
env: [
|
|
82
|
+
{ name: 'PLAIFUL_JOB', value: id },
|
|
83
|
+
{ name: 'PLAIFUL_TASK', value: task },
|
|
84
|
+
{ name: 'PLAIFUL_DATA', value: `${root}/data` },
|
|
85
|
+
{ name: 'PLAIFUL_OUT', value: `${root}/jobs/${id}/out` },
|
|
86
|
+
],
|
|
87
|
+
},
|
|
88
|
+
}],
|
|
89
|
+
scheduling: { timeout: `${Math.round(hours * 3600)}s` },
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function makeTrain({ UsageError, ApiError, repo, models }) {
|
|
94
|
+
/** Run gcloud, capturing its output. → stdout; throws on failure with gcloud's own message. */
|
|
95
|
+
function gcloud(ctx, args, { json = false, inherit = false } = {}) {
|
|
96
|
+
const argv = [...args, ...(json ? ['--format=json'] : [])];
|
|
97
|
+
if (inherit) {
|
|
98
|
+
const r = ctx.io.exec('gcloud', argv, { inherit: true });
|
|
99
|
+
if (r.status !== 0) throw new ApiError(0, { code: 'gcloud', error: `gcloud ${args.slice(0, 3).join(' ')} failed (exit ${r.status})` });
|
|
100
|
+
return '';
|
|
101
|
+
}
|
|
102
|
+
const r = ctx.io.exec('gcloud', argv);
|
|
103
|
+
if (r.status !== 0) {
|
|
104
|
+
const msg = String(r.stderr || r.stdout || '').trim().split('\n').filter((l) => /ERROR|error/.test(l)).slice(-2).join(' ') || `exit ${r.status}`;
|
|
105
|
+
throw new ApiError(0, { code: 'gcloud', error: `gcloud ${args.slice(0, 3).join(' ')} failed: ${msg}${/auth|login|credential/i.test(msg) ? ' (run `gcloud auth login`)' : ''}` });
|
|
106
|
+
}
|
|
107
|
+
return json ? JSON.parse(r.stdout || 'null') : r.stdout;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function need(s) {
|
|
111
|
+
if (!s.project) throw new UsageError('which Google Cloud project? --project, or "training": { "project": … } in .plaiful.json');
|
|
112
|
+
if (!s.bucket) throw new UsageError('which bucket for job files? --bucket, or "training": { "bucket": … } in .plaiful.json (in the job\'s region)');
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** `plaiful models train check`: is this machine and project ready to run a job? */
|
|
116
|
+
async function check(ctx) {
|
|
117
|
+
const r = repo(ctx, { required: false });
|
|
118
|
+
const s = settings(ctx.flags, r.found && r.found.config);
|
|
119
|
+
const lines = [];
|
|
120
|
+
const ok = (good, text) => { lines.push(`${good ? '✓' : '✗'} ${text}`); return good; };
|
|
121
|
+
let fine = true;
|
|
122
|
+
const ver = ctx.io.exec('gcloud', ['--version']);
|
|
123
|
+
fine = ok(ver.status === 0, ver.status === 0 ? `gcloud: ${String(ver.stdout).split('\n')[0]}` : 'gcloud isn\'t installed (https://cloud.google.com/sdk)') && fine;
|
|
124
|
+
if (ver.status === 0) {
|
|
125
|
+
const acct = ctx.io.exec('gcloud', ['auth', 'print-access-token', '--quiet']);
|
|
126
|
+
fine = ok(acct.status === 0, acct.status === 0 ? 'signed in to Google Cloud' : 'not signed in: run `gcloud auth login`') && fine;
|
|
127
|
+
fine = ok(!!s.project, s.project ? `project: ${s.project}` : 'no project set (--project or .plaiful.json training.project)') && fine;
|
|
128
|
+
fine = ok(!!s.bucket, s.bucket ? `bucket: gs://${s.bucket} (region ${s.region})` : 'no bucket set (--bucket or .plaiful.json training.bucket)') && fine;
|
|
129
|
+
if (acct.status === 0 && s.project) {
|
|
130
|
+
const api = ctx.io.exec('gcloud', ['services', 'list', '--enabled', `--project=${s.project}`, '--filter=config.name=aiplatform.googleapis.com', '--format=value(config.name)']);
|
|
131
|
+
fine = ok(api.status === 0 && /aiplatform/.test(api.stdout), api.status === 0 && /aiplatform/.test(api.stdout) ? 'Vertex AI API enabled' : `Vertex AI API not enabled: gcloud services enable aiplatform.googleapis.com --project=${s.project}`) && fine;
|
|
132
|
+
if (s.bucket) {
|
|
133
|
+
const b = ctx.io.exec('gcloud', ['storage', 'buckets', 'describe', `gs://${s.bucket}`, `--project=${s.project}`, '--format=value(location)']);
|
|
134
|
+
fine = ok(b.status === 0, b.status === 0 ? `bucket found (${String(b.stdout).trim().toLowerCase()})` : `bucket gs://${s.bucket} not found: gcloud storage buckets create gs://${s.bucket} --project=${s.project} --location=${s.region} --uniform-bucket-level-access`) && fine;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
lines.push(` image: ${s.image}`);
|
|
139
|
+
lines.push(' GPU quota (Vertex AI custom training, per region) is in the console: IAM & Admin → Quotas, "custom model training"');
|
|
140
|
+
ctx.out(lines.join('\n'), { success: fine, settings: s });
|
|
141
|
+
return fine ? 0 : 1;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** `plaiful models train submit <task> --code <dir> --entry "<command>" [--data a,b] [--snapshot task/snap] [--gpu l4]` */
|
|
145
|
+
async function submit(ctx, [task]) {
|
|
146
|
+
if (!task) throw new UsageError('plaiful models train submit <task> --code <folder> --entry "python train.py --data {data} --out {out}" [--data <folders>] [--gpu l4|t4|a100] [--hours 12]');
|
|
147
|
+
const r = repo(ctx, { required: false });
|
|
148
|
+
const s = settings(ctx.flags, r.found && r.found.config);
|
|
149
|
+
need(s);
|
|
150
|
+
const gpu = String(ctx.flags.gpu || 'l4').toLowerCase();
|
|
151
|
+
if (!GPUS[gpu]) throw new UsageError(`--gpu must be ${Object.keys(GPUS).join(', ')}`);
|
|
152
|
+
const hours = ctx.flags.hours ? Number(ctx.flags.hours) : DEFAULT_HOURS;
|
|
153
|
+
if (!(hours > 0 && hours <= 72)) throw new UsageError('--hours must be between 0 and 72 (the job is stopped after it)');
|
|
154
|
+
if (!ctx.flags.code || !ctx.flags.entry) throw new UsageError('--code <folder> and --entry "<command>" are required ({data} and {out} are filled in)');
|
|
155
|
+
const code = path.resolve(ctx.io.cwd(), ctx.flags.code);
|
|
156
|
+
if (!fs.existsSync(code) || !fs.statSync(code).isDirectory()) throw new UsageError(`${code} isn't a folder`);
|
|
157
|
+
const id = `${slug(ctx.flags.name || task)}-${stamp()}`;
|
|
158
|
+
|
|
159
|
+
// Data: local folders, and registry snapshots pulled to a local cache first. Each lands in gs://<bucket>/data/<name>/.
|
|
160
|
+
const data = [];
|
|
161
|
+
for (const d of String(ctx.flags.data || '').split(',').map((x) => x.trim()).filter(Boolean)) {
|
|
162
|
+
const abs = path.resolve(ctx.io.cwd(), d);
|
|
163
|
+
if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) throw new UsageError(`--data ${abs} isn't a folder`);
|
|
164
|
+
data.push({ local: abs, name: slug(path.basename(abs)) || 'data' });
|
|
165
|
+
}
|
|
166
|
+
for (const snap of String(ctx.flags.snapshot || '').split(',').map((x) => x.trim()).filter(Boolean)) {
|
|
167
|
+
const [st, sid] = snap.split('/');
|
|
168
|
+
if (!st || !sid) throw new UsageError('--snapshot is <task>/<snapshot>, e.g. voice.radio/voicedata-cast');
|
|
169
|
+
const cache = path.join(process.env.PLAIFUL_HOME || path.join(os.homedir(), '.plaiful'), 'cache', 'datasets', st, sid);
|
|
170
|
+
await models.dataPull({ ...ctx, flags: { ...ctx.flags, out: cache } }, [st, sid]);
|
|
171
|
+
data.push({ local: cache, name: `${slug(st)}-${slug(sid)}` });
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
ctx.io.stderr.write(`Copying the code to gs://${s.bucket}/jobs/${id}/code …\n`);
|
|
175
|
+
gcloud(ctx, ['storage', 'rsync', code, `gs://${s.bucket}/jobs/${id}/code`, '--recursive', `--exclude=${CODE_EXCLUDE}`, `--project=${s.project}`], { inherit: true });
|
|
176
|
+
for (const d of data) {
|
|
177
|
+
ctx.io.stderr.write(`Syncing ${d.local} → gs://${s.bucket}/data/${d.name} (only what changed) …\n`);
|
|
178
|
+
gcloud(ctx, ['storage', 'rsync', d.local, `gs://${s.bucket}/data/${d.name}`, '--recursive', `--project=${s.project}`], { inherit: true });
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// gcloud reads the config as YAML, and JSON is YAML.
|
|
182
|
+
const spec = jobSpec({ bucket: s.bucket, id, entry: ctx.flags.entry, gpu, machine: ctx.flags.machine, image: s.image, hours, task });
|
|
183
|
+
const cfg = path.join(os.tmpdir(), `plaiful-job-${id}.json`);
|
|
184
|
+
fs.writeFileSync(cfg, JSON.stringify(spec, null, 2));
|
|
185
|
+
let job;
|
|
186
|
+
try {
|
|
187
|
+
job = gcloud(ctx, ['ai', 'custom-jobs', 'create', `--project=${s.project}`, `--region=${s.region}`, `--display-name=${id}`, `--config=${cfg}`,
|
|
188
|
+
`--labels=plaiful-job=${id},plaiful-task=${slug(task)},plaiful-gpu=${gpu}`], { json: true });
|
|
189
|
+
} finally { try { fs.unlinkSync(cfg); } catch { /* temp file */ } }
|
|
190
|
+
const record = { id, name: job && job.name, task, gpu, machine: spec.workerPoolSpecs[0].machineSpec.machineType, hours, project: s.project, region: s.region, bucket: s.bucket, data: data.map((d) => d.name), submitted_at: new Date().toISOString() };
|
|
191
|
+
writeJobs([...readJobs(), record]);
|
|
192
|
+
ctx.out([
|
|
193
|
+
`Submitted ${id} on one ${gpu.toUpperCase()} (${record.machine}), stopped after ${hours} h at most.`,
|
|
194
|
+
` output: gs://${s.bucket}/jobs/${id}/out`,
|
|
195
|
+
` plaiful models train status ${id} · plaiful models train logs ${id} · plaiful models train fetch ${id}`,
|
|
196
|
+
].join('\n'), { success: true, job: record });
|
|
197
|
+
return 0;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function findJob(id) {
|
|
201
|
+
const jobs = readJobs();
|
|
202
|
+
const j = id ? jobs.find((x) => x.id === id) : jobs[jobs.length - 1];
|
|
203
|
+
if (!j) throw new UsageError(id ? `no job ${id} submitted from this machine (plaiful models train list)` : 'no jobs submitted from this machine yet');
|
|
204
|
+
return j;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** Vertex AI's view of a job → { state, started, ended, gpu_hours, error } */
|
|
208
|
+
function describe(ctx, j) {
|
|
209
|
+
const d = gcloud(ctx, ['ai', 'custom-jobs', 'describe', j.name, `--region=${j.region}`, `--project=${j.project}`], { json: true });
|
|
210
|
+
const state = String(d.state || '').replace(/^JOB_STATE_/, '').toLowerCase();
|
|
211
|
+
// Vertex AI sets startTime as soon as a job is accepted, so a job still waiting for its machine isn't GPU time yet.
|
|
212
|
+
const waiting = ['queued', 'pending'].includes(state);
|
|
213
|
+
const start = d.startTime && !waiting ? Date.parse(d.startTime) : null;
|
|
214
|
+
const end = d.endTime ? Date.parse(d.endTime) : (start ? Date.now() : null);
|
|
215
|
+
return { state, created: d.createTime, started: d.startTime || null, ended: d.endTime || null, gpu_hours: start ? Math.round(((end - start) / 3.6e6) * 100) / 100 : 0, error: d.error && d.error.message };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async function status(ctx, [id]) {
|
|
219
|
+
const j = findJob(id);
|
|
220
|
+
const d = describe(ctx, j);
|
|
221
|
+
ctx.out([
|
|
222
|
+
`${j.id}: ${d.state} (${j.gpu.toUpperCase()}, ${j.task})`,
|
|
223
|
+
` submitted ${j.submitted_at}${d.started ? `, started ${d.started}` : ', waiting for a GPU'}${d.ended ? `, ended ${d.ended}` : ''}`,
|
|
224
|
+
` GPU time so far: ${d.gpu_hours} h (billed to ${j.project}; Billing → Reports, label plaiful-job=${j.id})`,
|
|
225
|
+
...(d.error ? [` error: ${d.error}`] : []),
|
|
226
|
+
].join('\n'), { success: true, job: j, ...d });
|
|
227
|
+
return 0;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
async function list(ctx) {
|
|
231
|
+
const jobs = readJobs();
|
|
232
|
+
if (!jobs.length) { ctx.out('No training jobs submitted from this machine yet.', { success: true, jobs: [] }); return 0; }
|
|
233
|
+
const rows = jobs.slice(-20).map((j) => { let d = {}; try { d = describe(ctx, j); } catch (e) { d = { state: `unknown (${e.message})` }; } return { ...j, ...d }; });
|
|
234
|
+
ctx.out(rows.map((j) => `${j.id} ${j.state} ${j.gpu} ${j.gpu_hours != null ? `${j.gpu_hours} GPU-h` : ''} ${j.task}`).join('\n'), { success: true, jobs: rows });
|
|
235
|
+
return 0;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
async function logs(ctx, [id]) {
|
|
239
|
+
const j = findJob(id);
|
|
240
|
+
gcloud(ctx, ['ai', 'custom-jobs', 'stream-logs', j.name, `--region=${j.region}`, `--project=${j.project}`], { inherit: true });
|
|
241
|
+
return 0;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
async function fetch(ctx, [id]) {
|
|
245
|
+
const j = findJob(id);
|
|
246
|
+
const out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('model-runs', j.id));
|
|
247
|
+
fs.mkdirSync(out, { recursive: true });
|
|
248
|
+
gcloud(ctx, ['storage', 'rsync', `gs://${j.bucket}/jobs/${j.id}/out`, out, '--recursive', `--project=${j.project}`], { inherit: true });
|
|
249
|
+
ctx.out(`${j.id}'s output is in ${out}. Card it, then plaiful models publish <card.json>.`, { success: true, out });
|
|
250
|
+
return 0;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
async function cancel(ctx, [id]) {
|
|
254
|
+
const j = findJob(id);
|
|
255
|
+
if (!id) throw new UsageError('cancel which job? plaiful models train cancel <job>');
|
|
256
|
+
gcloud(ctx, ['ai', 'custom-jobs', 'cancel', j.name, `--region=${j.region}`, `--project=${j.project}`]);
|
|
257
|
+
ctx.out(`Asked Vertex AI to cancel ${j.id}.`, { success: true });
|
|
258
|
+
return 0;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
return { check, submit, status, list, logs, fetch, cancel };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
module.exports = { makeTrain, jobSpec, jobScript, settings, GPUS, DEFAULT_IMAGE, slug };
|