@plaiful/cli 0.16.1 → 0.16.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/package.json +1 -1
  2. package/src/train.js +264 -260
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@plaiful/cli",
3
- "version": "0.16.1",
3
+ "version": "0.16.2",
4
4
  "description": "The Plaiful CLI: work on your Plaiful product from your game repo, with no AWS access",
5
5
  "bin": {
6
6
  "plaiful": "bin/plaiful.js"
package/src/train.js CHANGED
@@ -1,260 +1,264 @@
1
- 'use strict';
2
-
3
- /**
4
- * `plaiful models train …`: run a training job on a Google Cloud GPU (Vertex AI custom training) instead of the
5
- * developer's own machine, so the GPU time draws on Plaiful's Google Cloud credits (PlaifulCMS
6
- * docs/design/model-registry.md §5). Plaiful staff only for now, and it runs on this machine's `gcloud` sign-in: the
7
- * platform holds no Google credentials.
8
- *
9
- * A job: the code folder and the data folders are copied to the project's bucket (data is cached across jobs), Vertex
10
- * AI runs the entry command on one GPU with the bucket mounted at /gcs/<bucket>, the job writes to {out}, and
11
- * `train fetch` brings the output home, ready to card and `plaiful models publish`.
12
- *
13
- * Settings, from flags, else .plaiful.json "training": { project, region, bucket, image }, else PLAIFUL_GCP_* env.
14
- */
15
-
16
- const fs = require('fs');
17
- const os = require('os');
18
- const path = require('path');
19
-
20
- /** One GPU per job, by name: the machine Vertex AI pairs with it. */
21
- const GPUS = {
22
- l4: { machineType: 'g2-standard-8', acceleratorType: 'NVIDIA_L4' },
23
- t4: { machineType: 'n1-standard-8', acceleratorType: 'NVIDIA_TESLA_T4' },
24
- a100: { machineType: 'a2-highgpu-1g', acceleratorType: 'NVIDIA_TESLA_A100' },
25
- };
26
- // Google's prebuilt PyTorch training image. Check it against Google's current list on the first run (model-registry.md §5).
27
- const DEFAULT_IMAGE = 'us-docker.pkg.dev/vertex-ai/training/pytorch-gpu.2-1.py310:latest';
28
- const DEFAULT_REGION = 'us-central1';
29
- const DEFAULT_HOURS = 12;
30
- const CODE_EXCLUDE = '(^|/)(\\.git|node_modules|__pycache__|\\.venv|venv)(/|$)';
31
-
32
- const jobsFile = () => path.join(process.env.PLAIFUL_HOME || path.join(os.homedir(), '.plaiful'), 'train-jobs.json');
33
- function readJobs() { try { return JSON.parse(fs.readFileSync(jobsFile(), 'utf8')); } catch { return []; } }
34
- function writeJobs(jobs) { fs.mkdirSync(path.dirname(jobsFile()), { recursive: true }); fs.writeFileSync(jobsFile(), `${JSON.stringify(jobs, null, 2)}\n`); }
35
-
36
- /** A name gcloud and the bucket accept: lowercase letters, digits and dashes. */
37
- const slug = (s) => String(s).toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40);
38
- const stamp = (d = new Date()) => d.toISOString().replace(/[-:]/g, '').replace(/\..*/, '').replace('T', '-').toLowerCase();
39
-
40
- /** The job's settings → { project, region, bucket, image } (bucket without gs://). */
41
- function settings(flags, repoConfig) {
42
- const t = (repoConfig && repoConfig.training) || {};
43
- const s = {
44
- project: flags.project || t.project || process.env.PLAIFUL_GCP_PROJECT,
45
- region: flags.region || t.region || process.env.PLAIFUL_GCP_REGION || DEFAULT_REGION,
46
- bucket: String(flags.bucket || t.bucket || process.env.PLAIFUL_GCP_BUCKET || '').replace(/^gs:\/\//, '').replace(/\/+$/, ''),
47
- image: flags.image || t.image || DEFAULT_IMAGE,
48
- };
49
- return s;
50
- }
51
-
52
- /** The shell script the container runs: copy the code out of the bucket, install, run the entry. */
53
- function jobScript({ bucket, id, entry }) {
54
- const root = `/gcs/${bucket}`;
55
- const cmd = entry.replace(/\{data\}/g, `${root}/data`).replace(/\{out\}/g, `${root}/jobs/${id}/out`);
56
- return [
57
- 'set -euo pipefail',
58
- `mkdir -p /tmp/code && cp -r ${root}/jobs/${id}/code/. /tmp/code/`,
59
- 'cd /tmp/code',
60
- 'if [ -f requirements.txt ]; then pip install -q -r requirements.txt; fi',
61
- 'mkdir -p "$PLAIFUL_OUT"',
62
- 'nvidia-smi --query-gpu=name,memory.total --format=csv || true',
63
- cmd,
64
- ].join('\n');
65
- }
66
-
67
- /** The Vertex AI CustomJobSpec (gcloud ai custom-jobs create --config). */
68
- function jobSpec({ bucket, id, entry, gpu, machine, image, hours, task }) {
69
- const g = GPUS[gpu];
70
- const root = `/gcs/${bucket}`;
71
- return {
72
- workerPoolSpecs: [{
73
- machineSpec: { machineType: machine || g.machineType, acceleratorType: g.acceleratorType, acceleratorCount: 1 },
74
- replicaCount: 1,
75
- diskSpec: { bootDiskType: 'pd-ssd', bootDiskSizeGb: 100 },
76
- containerSpec: {
77
- imageUri: image,
78
- command: ['bash', '-c'],
79
- args: [jobScript({ bucket, id, entry })],
80
- env: [
81
- { name: 'PLAIFUL_JOB', value: id },
82
- { name: 'PLAIFUL_TASK', value: task },
83
- { name: 'PLAIFUL_DATA', value: `${root}/data` },
84
- { name: 'PLAIFUL_OUT', value: `${root}/jobs/${id}/out` },
85
- ],
86
- },
87
- }],
88
- scheduling: { timeout: `${Math.round(hours * 3600)}s` },
89
- };
90
- }
91
-
92
- function makeTrain({ UsageError, ApiError, repo, models }) {
93
- /** Run gcloud, capturing its output. → stdout; throws on failure with gcloud's own message. */
94
- function gcloud(ctx, args, { json = false, inherit = false } = {}) {
95
- const argv = [...args, ...(json ? ['--format=json'] : [])];
96
- if (inherit) {
97
- const r = ctx.io.exec('gcloud', argv, { inherit: true });
98
- if (r.status !== 0) throw new ApiError(0, { code: 'gcloud', error: `gcloud ${args.slice(0, 3).join(' ')} failed (exit ${r.status})` });
99
- return '';
100
- }
101
- const r = ctx.io.exec('gcloud', argv);
102
- if (r.status !== 0) {
103
- const msg = String(r.stderr || r.stdout || '').trim().split('\n').filter((l) => /ERROR|error/.test(l)).slice(-2).join(' ') || `exit ${r.status}`;
104
- throw new ApiError(0, { code: 'gcloud', error: `gcloud ${args.slice(0, 3).join(' ')} failed: ${msg}${/auth|login|credential/i.test(msg) ? ' (run `gcloud auth login`)' : ''}` });
105
- }
106
- return json ? JSON.parse(r.stdout || 'null') : r.stdout;
107
- }
108
-
109
- function need(s) {
110
- if (!s.project) throw new UsageError('which Google Cloud project? --project, or "training": { "project": … } in .plaiful.json');
111
- if (!s.bucket) throw new UsageError('which bucket for job files? --bucket, or "training": { "bucket": … } in .plaiful.json (in the job\'s region)');
112
- }
113
-
114
- /** `plaiful models train check`: is this machine and project ready to run a job? */
115
- async function check(ctx) {
116
- const r = repo(ctx, { required: false });
117
- const s = settings(ctx.flags, r.found && r.found.config);
118
- const lines = [];
119
- const ok = (good, text) => { lines.push(`${good ? '✓' : '✗'} ${text}`); return good; };
120
- let fine = true;
121
- const ver = ctx.io.exec('gcloud', ['--version']);
122
- fine = ok(ver.status === 0, ver.status === 0 ? `gcloud: ${String(ver.stdout).split('\n')[0]}` : 'gcloud isn\'t installed (https://cloud.google.com/sdk)') && fine;
123
- if (ver.status === 0) {
124
- const acct = ctx.io.exec('gcloud', ['auth', 'print-access-token', '--quiet']);
125
- fine = ok(acct.status === 0, acct.status === 0 ? 'signed in to Google Cloud' : 'not signed in: run `gcloud auth login`') && fine;
126
- fine = ok(!!s.project, s.project ? `project: ${s.project}` : 'no project set (--project or .plaiful.json training.project)') && fine;
127
- fine = ok(!!s.bucket, s.bucket ? `bucket: gs://${s.bucket} (region ${s.region})` : 'no bucket set (--bucket or .plaiful.json training.bucket)') && fine;
128
- if (acct.status === 0 && s.project) {
129
- const api = ctx.io.exec('gcloud', ['services', 'list', '--enabled', `--project=${s.project}`, '--filter=config.name=aiplatform.googleapis.com', '--format=value(config.name)']);
130
- fine = ok(api.status === 0 && /aiplatform/.test(api.stdout), api.status === 0 && /aiplatform/.test(api.stdout) ? 'Vertex AI API enabled' : `Vertex AI API not enabled: gcloud services enable aiplatform.googleapis.com --project=${s.project}`) && fine;
131
- if (s.bucket) {
132
- const b = ctx.io.exec('gcloud', ['storage', 'buckets', 'describe', `gs://${s.bucket}`, `--project=${s.project}`, '--format=value(location)']);
133
- fine = ok(b.status === 0, b.status === 0 ? `bucket found (${String(b.stdout).trim().toLowerCase()})` : `bucket gs://${s.bucket} not found: gcloud storage buckets create gs://${s.bucket} --project=${s.project} --location=${s.region} --uniform-bucket-level-access`) && fine;
134
- }
135
- }
136
- }
137
- lines.push(` image: ${s.image}`);
138
- lines.push(' GPU quota (Vertex AI custom training, per region) is in the console: IAM & Admin → Quotas, "custom model training"');
139
- ctx.out(lines.join('\n'), { success: fine, settings: s });
140
- return fine ? 0 : 1;
141
- }
142
-
143
- /** `plaiful models train submit <task> --code <dir> --entry "<command>" [--data a,b] [--snapshot task/snap] [--gpu l4]` */
144
- async function submit(ctx, [task]) {
145
- if (!task) throw new UsageError('plaiful models train submit <task> --code <folder> --entry "python train.py --data {data} --out {out}" [--data <folders>] [--gpu l4|t4|a100] [--hours 12]');
146
- const r = repo(ctx, { required: false });
147
- const s = settings(ctx.flags, r.found && r.found.config);
148
- need(s);
149
- const gpu = String(ctx.flags.gpu || 'l4').toLowerCase();
150
- if (!GPUS[gpu]) throw new UsageError(`--gpu must be ${Object.keys(GPUS).join(', ')}`);
151
- const hours = ctx.flags.hours ? Number(ctx.flags.hours) : DEFAULT_HOURS;
152
- if (!(hours > 0 && hours <= 72)) throw new UsageError('--hours must be between 0 and 72 (the job is stopped after it)');
153
- if (!ctx.flags.code || !ctx.flags.entry) throw new UsageError('--code <folder> and --entry "<command>" are required ({data} and {out} are filled in)');
154
- const code = path.resolve(ctx.io.cwd(), ctx.flags.code);
155
- if (!fs.existsSync(code) || !fs.statSync(code).isDirectory()) throw new UsageError(`${code} isn't a folder`);
156
- const id = `${slug(ctx.flags.name || task)}-${stamp()}`;
157
-
158
- // Data: local folders, and registry snapshots pulled to a local cache first. Each lands in gs://<bucket>/data/<name>/.
159
- const data = [];
160
- for (const d of String(ctx.flags.data || '').split(',').map((x) => x.trim()).filter(Boolean)) {
161
- const abs = path.resolve(ctx.io.cwd(), d);
162
- if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) throw new UsageError(`--data ${abs} isn't a folder`);
163
- data.push({ local: abs, name: slug(path.basename(abs)) || 'data' });
164
- }
165
- for (const snap of String(ctx.flags.snapshot || '').split(',').map((x) => x.trim()).filter(Boolean)) {
166
- const [st, sid] = snap.split('/');
167
- if (!st || !sid) throw new UsageError('--snapshot is <task>/<snapshot>, e.g. voice.radio/voicedata-cast');
168
- const cache = path.join(process.env.PLAIFUL_HOME || path.join(os.homedir(), '.plaiful'), 'cache', 'datasets', st, sid);
169
- await models.dataPull({ ...ctx, flags: { ...ctx.flags, out: cache } }, [st, sid]);
170
- data.push({ local: cache, name: `${slug(st)}-${slug(sid)}` });
171
- }
172
-
173
- ctx.io.stderr.write(`Copying the code to gs://${s.bucket}/jobs/${id}/code …\n`);
174
- gcloud(ctx, ['storage', 'rsync', code, `gs://${s.bucket}/jobs/${id}/code`, '--recursive', `--exclude=${CODE_EXCLUDE}`, `--project=${s.project}`], { inherit: true });
175
- for (const d of data) {
176
- ctx.io.stderr.write(`Syncing ${d.local} → gs://${s.bucket}/data/${d.name} (only what changed) …\n`);
177
- gcloud(ctx, ['storage', 'rsync', d.local, `gs://${s.bucket}/data/${d.name}`, '--recursive', `--project=${s.project}`], { inherit: true });
178
- }
179
-
180
- // gcloud reads the config as YAML, and JSON is YAML.
181
- const spec = jobSpec({ bucket: s.bucket, id, entry: ctx.flags.entry, gpu, machine: ctx.flags.machine, image: s.image, hours, task });
182
- const cfg = path.join(os.tmpdir(), `plaiful-job-${id}.json`);
183
- fs.writeFileSync(cfg, JSON.stringify(spec, null, 2));
184
- let job;
185
- try {
186
- job = gcloud(ctx, ['ai', 'custom-jobs', 'create', `--project=${s.project}`, `--region=${s.region}`, `--display-name=${id}`, `--config=${cfg}`,
187
- `--labels=plaiful-job=${id},plaiful-task=${slug(task)},plaiful-gpu=${gpu}`], { json: true });
188
- } finally { try { fs.unlinkSync(cfg); } catch { /* temp file */ } }
189
- const record = { id, name: job && job.name, task, gpu, machine: spec.workerPoolSpecs[0].machineSpec.machineType, hours, project: s.project, region: s.region, bucket: s.bucket, data: data.map((d) => d.name), submitted_at: new Date().toISOString() };
190
- writeJobs([...readJobs(), record]);
191
- ctx.out([
192
- `Submitted ${id} on one ${gpu.toUpperCase()} (${record.machine}), stopped after ${hours} h at most.`,
193
- ` output: gs://${s.bucket}/jobs/${id}/out`,
194
- ` plaiful models train status ${id} · plaiful models train logs ${id} · plaiful models train fetch ${id}`,
195
- ].join('\n'), { success: true, job: record });
196
- return 0;
197
- }
198
-
199
- function findJob(id) {
200
- const jobs = readJobs();
201
- const j = id ? jobs.find((x) => x.id === id) : jobs[jobs.length - 1];
202
- if (!j) throw new UsageError(id ? `no job ${id} submitted from this machine (plaiful models train list)` : 'no jobs submitted from this machine yet');
203
- return j;
204
- }
205
-
206
- /** Vertex AI's view of a job → { state, started, ended, gpu_hours, error } */
207
- function describe(ctx, j) {
208
- const d = gcloud(ctx, ['ai', 'custom-jobs', 'describe', j.name, `--region=${j.region}`, `--project=${j.project}`], { json: true });
209
- const start = d.startTime ? Date.parse(d.startTime) : null;
210
- const end = d.endTime ? Date.parse(d.endTime) : (start ? Date.now() : null);
211
- return { state: String(d.state || '').replace(/^JOB_STATE_/, '').toLowerCase(), created: d.createTime, started: d.startTime || null, ended: d.endTime || null, gpu_hours: start ? Math.round(((end - start) / 3.6e6) * 100) / 100 : 0, error: d.error && d.error.message };
212
- }
213
-
214
- async function status(ctx, [id]) {
215
- const j = findJob(id);
216
- const d = describe(ctx, j);
217
- ctx.out([
218
- `${j.id}: ${d.state} (${j.gpu.toUpperCase()}, ${j.task})`,
219
- ` submitted ${j.submitted_at}${d.started ? `, started ${d.started}` : ', waiting for a GPU'}${d.ended ? `, ended ${d.ended}` : ''}`,
220
- ` GPU time so far: ${d.gpu_hours} h (billed to ${j.project}; Billing → Reports, label plaiful-job=${j.id})`,
221
- ...(d.error ? [` error: ${d.error}`] : []),
222
- ].join('\n'), { success: true, job: j, ...d });
223
- return 0;
224
- }
225
-
226
- async function list(ctx) {
227
- const jobs = readJobs();
228
- if (!jobs.length) { ctx.out('No training jobs submitted from this machine yet.', { success: true, jobs: [] }); return 0; }
229
- const rows = jobs.slice(-20).map((j) => { let d = {}; try { d = describe(ctx, j); } catch (e) { d = { state: `unknown (${e.message})` }; } return { ...j, ...d }; });
230
- ctx.out(rows.map((j) => `${j.id} ${j.state} ${j.gpu} ${j.gpu_hours != null ? `${j.gpu_hours} GPU-h` : ''} ${j.task}`).join('\n'), { success: true, jobs: rows });
231
- return 0;
232
- }
233
-
234
- async function logs(ctx, [id]) {
235
- const j = findJob(id);
236
- gcloud(ctx, ['ai', 'custom-jobs', 'stream-logs', j.name, `--region=${j.region}`, `--project=${j.project}`], { inherit: true });
237
- return 0;
238
- }
239
-
240
- async function fetch(ctx, [id]) {
241
- const j = findJob(id);
242
- const out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('model-runs', j.id));
243
- fs.mkdirSync(out, { recursive: true });
244
- gcloud(ctx, ['storage', 'rsync', `gs://${j.bucket}/jobs/${j.id}/out`, out, '--recursive', `--project=${j.project}`], { inherit: true });
245
- ctx.out(`${j.id}'s output is in ${out}. Card it, then plaiful models publish <card.json>.`, { success: true, out });
246
- return 0;
247
- }
248
-
249
- async function cancel(ctx, [id]) {
250
- const j = findJob(id);
251
- if (!id) throw new UsageError('cancel which job? plaiful models train cancel <job>');
252
- gcloud(ctx, ['ai', 'custom-jobs', 'cancel', j.name, `--region=${j.region}`, `--project=${j.project}`]);
253
- ctx.out(`Asked Vertex AI to cancel ${j.id}.`, { success: true });
254
- return 0;
255
- }
256
-
257
- return { check, submit, status, list, logs, fetch, cancel };
258
- }
259
-
260
- module.exports = { makeTrain, jobSpec, jobScript, settings, GPUS, DEFAULT_IMAGE, slug };
1
+ 'use strict';
2
+
3
+ /**
4
+ * `plaiful models train …`: run a training job on a Google Cloud GPU (Vertex AI custom training) instead of the
5
+ * developer's own machine, so the GPU time draws on Plaiful's Google Cloud credits (PlaifulCMS
6
+ * docs/design/model-registry.md §5). Plaiful staff only for now, and it runs on this machine's `gcloud` sign-in: the
7
+ * platform holds no Google credentials.
8
+ *
9
+ * A job: the code folder and the data folders are copied to the project's bucket (data is cached across jobs), Vertex
10
+ * AI runs the entry command on one GPU with the bucket mounted at /gcs/<bucket>, the job writes to {out}, and
11
+ * `train fetch` brings the output home, ready to card and `plaiful models publish`.
12
+ *
13
+ * Settings, from flags, else .plaiful.json "training": { project, region, bucket, image }, else PLAIFUL_GCP_* env.
14
+ */
15
+
16
+ const fs = require('fs');
17
+ const os = require('os');
18
+ const path = require('path');
19
+
20
+ /** One GPU per job, by name: the machine Vertex AI pairs with it. */
21
+ const GPUS = {
22
+ l4: { machineType: 'g2-standard-8', acceleratorType: 'NVIDIA_L4' },
23
+ t4: { machineType: 'n1-standard-8', acceleratorType: 'NVIDIA_TESLA_T4' },
24
+ a100: { machineType: 'a2-highgpu-1g', acceleratorType: 'NVIDIA_TESLA_A100' },
25
+ };
26
+ // Google's newest prebuilt PyTorch training image. The 2.1 image is deprecated, and Vertex AI failed every job on it
27
+ // with only "Internal error" (2026-11-16).
28
+ const DEFAULT_IMAGE = 'us-docker.pkg.dev/vertex-ai/training/pytorch-gpu.2-4.py310:latest';
29
+ const DEFAULT_REGION = 'us-central1';
30
+ const DEFAULT_HOURS = 12;
31
+ const CODE_EXCLUDE = '(^|/)(\\.git|node_modules|__pycache__|\\.venv|venv)(/|$)';
32
+
33
+ const jobsFile = () => path.join(process.env.PLAIFUL_HOME || path.join(os.homedir(), '.plaiful'), 'train-jobs.json');
34
+ function readJobs() { try { return JSON.parse(fs.readFileSync(jobsFile(), 'utf8')); } catch { return []; } }
35
+ function writeJobs(jobs) { fs.mkdirSync(path.dirname(jobsFile()), { recursive: true }); fs.writeFileSync(jobsFile(), `${JSON.stringify(jobs, null, 2)}\n`); }
36
+
37
+ /** A name gcloud and the bucket accept: lowercase letters, digits and dashes. */
38
+ const slug = (s) => String(s).toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40);
39
+ const stamp = (d = new Date()) => d.toISOString().replace(/[-:]/g, '').replace(/\..*/, '').replace('T', '-').toLowerCase();
40
+
41
+ /** The job's settings → { project, region, bucket, image } (bucket without gs://). */
42
+ function settings(flags, repoConfig) {
43
+ const t = (repoConfig && repoConfig.training) || {};
44
+ const s = {
45
+ project: flags.project || t.project || process.env.PLAIFUL_GCP_PROJECT,
46
+ region: flags.region || t.region || process.env.PLAIFUL_GCP_REGION || DEFAULT_REGION,
47
+ bucket: String(flags.bucket || t.bucket || process.env.PLAIFUL_GCP_BUCKET || '').replace(/^gs:\/\//, '').replace(/\/+$/, ''),
48
+ image: flags.image || t.image || DEFAULT_IMAGE,
49
+ };
50
+ return s;
51
+ }
52
+
53
+ /** The shell script the container runs: copy the code out of the bucket, install, run the entry. */
54
+ function jobScript({ bucket, id, entry }) {
55
+ const root = `/gcs/${bucket}`;
56
+ const cmd = entry.replace(/\{data\}/g, `${root}/data`).replace(/\{out\}/g, `${root}/jobs/${id}/out`);
57
+ return [
58
+ 'set -euo pipefail',
59
+ `mkdir -p /tmp/code && cp -r ${root}/jobs/${id}/code/. /tmp/code/`,
60
+ 'cd /tmp/code',
61
+ 'if [ -f requirements.txt ]; then pip install -q -r requirements.txt; fi',
62
+ 'mkdir -p "$PLAIFUL_OUT"',
63
+ 'nvidia-smi --query-gpu=name,memory.total --format=csv || true',
64
+ cmd,
65
+ ].join('\n');
66
+ }
67
+
68
+ /** The Vertex AI CustomJobSpec (gcloud ai custom-jobs create --config). */
69
+ function jobSpec({ bucket, id, entry, gpu, machine, image, hours, task }) {
70
+ const g = GPUS[gpu];
71
+ const root = `/gcs/${bucket}`;
72
+ return {
73
+ workerPoolSpecs: [{
74
+ machineSpec: { machineType: machine || g.machineType, acceleratorType: g.acceleratorType, acceleratorCount: 1 },
75
+ replicaCount: 1,
76
+ diskSpec: { bootDiskType: 'pd-ssd', bootDiskSizeGb: 100 },
77
+ containerSpec: {
78
+ imageUri: image,
79
+ command: ['bash', '-c'],
80
+ args: [jobScript({ bucket, id, entry })],
81
+ env: [
82
+ { name: 'PLAIFUL_JOB', value: id },
83
+ { name: 'PLAIFUL_TASK', value: task },
84
+ { name: 'PLAIFUL_DATA', value: `${root}/data` },
85
+ { name: 'PLAIFUL_OUT', value: `${root}/jobs/${id}/out` },
86
+ ],
87
+ },
88
+ }],
89
+ scheduling: { timeout: `${Math.round(hours * 3600)}s` },
90
+ };
91
+ }
92
+
93
+ function makeTrain({ UsageError, ApiError, repo, models }) {
94
+ /** Run gcloud, capturing its output. → stdout; throws on failure with gcloud's own message. */
95
+ function gcloud(ctx, args, { json = false, inherit = false } = {}) {
96
+ const argv = [...args, ...(json ? ['--format=json'] : [])];
97
+ if (inherit) {
98
+ const r = ctx.io.exec('gcloud', argv, { inherit: true });
99
+ if (r.status !== 0) throw new ApiError(0, { code: 'gcloud', error: `gcloud ${args.slice(0, 3).join(' ')} failed (exit ${r.status})` });
100
+ return '';
101
+ }
102
+ const r = ctx.io.exec('gcloud', argv);
103
+ if (r.status !== 0) {
104
+ const msg = String(r.stderr || r.stdout || '').trim().split('\n').filter((l) => /ERROR|error/.test(l)).slice(-2).join(' ') || `exit ${r.status}`;
105
+ throw new ApiError(0, { code: 'gcloud', error: `gcloud ${args.slice(0, 3).join(' ')} failed: ${msg}${/auth|login|credential/i.test(msg) ? ' (run `gcloud auth login`)' : ''}` });
106
+ }
107
+ return json ? JSON.parse(r.stdout || 'null') : r.stdout;
108
+ }
109
+
110
+ function need(s) {
111
+ if (!s.project) throw new UsageError('which Google Cloud project? --project, or "training": { "project": … } in .plaiful.json');
112
+ if (!s.bucket) throw new UsageError('which bucket for job files? --bucket, or "training": { "bucket": … } in .plaiful.json (in the job\'s region)');
113
+ }
114
+
115
+ /** `plaiful models train check`: is this machine and project ready to run a job? */
116
+ async function check(ctx) {
117
+ const r = repo(ctx, { required: false });
118
+ const s = settings(ctx.flags, r.found && r.found.config);
119
+ const lines = [];
120
+ const ok = (good, text) => { lines.push(`${good ? '✓' : '✗'} ${text}`); return good; };
121
+ let fine = true;
122
+ const ver = ctx.io.exec('gcloud', ['--version']);
123
+ fine = ok(ver.status === 0, ver.status === 0 ? `gcloud: ${String(ver.stdout).split('\n')[0]}` : 'gcloud isn\'t installed (https://cloud.google.com/sdk)') && fine;
124
+ if (ver.status === 0) {
125
+ const acct = ctx.io.exec('gcloud', ['auth', 'print-access-token', '--quiet']);
126
+ fine = ok(acct.status === 0, acct.status === 0 ? 'signed in to Google Cloud' : 'not signed in: run `gcloud auth login`') && fine;
127
+ fine = ok(!!s.project, s.project ? `project: ${s.project}` : 'no project set (--project or .plaiful.json training.project)') && fine;
128
+ fine = ok(!!s.bucket, s.bucket ? `bucket: gs://${s.bucket} (region ${s.region})` : 'no bucket set (--bucket or .plaiful.json training.bucket)') && fine;
129
+ if (acct.status === 0 && s.project) {
130
+ const api = ctx.io.exec('gcloud', ['services', 'list', '--enabled', `--project=${s.project}`, '--filter=config.name=aiplatform.googleapis.com', '--format=value(config.name)']);
131
+ fine = ok(api.status === 0 && /aiplatform/.test(api.stdout), api.status === 0 && /aiplatform/.test(api.stdout) ? 'Vertex AI API enabled' : `Vertex AI API not enabled: gcloud services enable aiplatform.googleapis.com --project=${s.project}`) && fine;
132
+ if (s.bucket) {
133
+ const b = ctx.io.exec('gcloud', ['storage', 'buckets', 'describe', `gs://${s.bucket}`, `--project=${s.project}`, '--format=value(location)']);
134
+ fine = ok(b.status === 0, b.status === 0 ? `bucket found (${String(b.stdout).trim().toLowerCase()})` : `bucket gs://${s.bucket} not found: gcloud storage buckets create gs://${s.bucket} --project=${s.project} --location=${s.region} --uniform-bucket-level-access`) && fine;
135
+ }
136
+ }
137
+ }
138
+ lines.push(` image: ${s.image}`);
139
+ lines.push(' GPU quota (Vertex AI custom training, per region) is in the console: IAM & Admin → Quotas, "custom model training"');
140
+ ctx.out(lines.join('\n'), { success: fine, settings: s });
141
+ return fine ? 0 : 1;
142
+ }
143
+
144
+ /** `plaiful models train submit <task> --code <dir> --entry "<command>" [--data a,b] [--snapshot task/snap] [--gpu l4]` */
145
+ async function submit(ctx, [task]) {
146
+ if (!task) throw new UsageError('plaiful models train submit <task> --code <folder> --entry "python train.py --data {data} --out {out}" [--data <folders>] [--gpu l4|t4|a100] [--hours 12]');
147
+ const r = repo(ctx, { required: false });
148
+ const s = settings(ctx.flags, r.found && r.found.config);
149
+ need(s);
150
+ const gpu = String(ctx.flags.gpu || 'l4').toLowerCase();
151
+ if (!GPUS[gpu]) throw new UsageError(`--gpu must be ${Object.keys(GPUS).join(', ')}`);
152
+ const hours = ctx.flags.hours ? Number(ctx.flags.hours) : DEFAULT_HOURS;
153
+ if (!(hours > 0 && hours <= 72)) throw new UsageError('--hours must be between 0 and 72 (the job is stopped after it)');
154
+ if (!ctx.flags.code || !ctx.flags.entry) throw new UsageError('--code <folder> and --entry "<command>" are required ({data} and {out} are filled in)');
155
+ const code = path.resolve(ctx.io.cwd(), ctx.flags.code);
156
+ if (!fs.existsSync(code) || !fs.statSync(code).isDirectory()) throw new UsageError(`${code} isn't a folder`);
157
+ const id = `${slug(ctx.flags.name || task)}-${stamp()}`;
158
+
159
+ // Data: local folders, and registry snapshots pulled to a local cache first. Each lands in gs://<bucket>/data/<name>/.
160
+ const data = [];
161
+ for (const d of String(ctx.flags.data || '').split(',').map((x) => x.trim()).filter(Boolean)) {
162
+ const abs = path.resolve(ctx.io.cwd(), d);
163
+ if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) throw new UsageError(`--data ${abs} isn't a folder`);
164
+ data.push({ local: abs, name: slug(path.basename(abs)) || 'data' });
165
+ }
166
+ for (const snap of String(ctx.flags.snapshot || '').split(',').map((x) => x.trim()).filter(Boolean)) {
167
+ const [st, sid] = snap.split('/');
168
+ if (!st || !sid) throw new UsageError('--snapshot is <task>/<snapshot>, e.g. voice.radio/voicedata-cast');
169
+ const cache = path.join(process.env.PLAIFUL_HOME || path.join(os.homedir(), '.plaiful'), 'cache', 'datasets', st, sid);
170
+ await models.dataPull({ ...ctx, flags: { ...ctx.flags, out: cache } }, [st, sid]);
171
+ data.push({ local: cache, name: `${slug(st)}-${slug(sid)}` });
172
+ }
173
+
174
+ ctx.io.stderr.write(`Copying the code to gs://${s.bucket}/jobs/${id}/code …\n`);
175
+ gcloud(ctx, ['storage', 'rsync', code, `gs://${s.bucket}/jobs/${id}/code`, '--recursive', `--exclude=${CODE_EXCLUDE}`, `--project=${s.project}`], { inherit: true });
176
+ for (const d of data) {
177
+ ctx.io.stderr.write(`Syncing ${d.local} → gs://${s.bucket}/data/${d.name} (only what changed) …\n`);
178
+ gcloud(ctx, ['storage', 'rsync', d.local, `gs://${s.bucket}/data/${d.name}`, '--recursive', `--project=${s.project}`], { inherit: true });
179
+ }
180
+
181
+ // gcloud reads the config as YAML, and JSON is YAML.
182
+ const spec = jobSpec({ bucket: s.bucket, id, entry: ctx.flags.entry, gpu, machine: ctx.flags.machine, image: s.image, hours, task });
183
+ const cfg = path.join(os.tmpdir(), `plaiful-job-${id}.json`);
184
+ fs.writeFileSync(cfg, JSON.stringify(spec, null, 2));
185
+ let job;
186
+ try {
187
+ job = gcloud(ctx, ['ai', 'custom-jobs', 'create', `--project=${s.project}`, `--region=${s.region}`, `--display-name=${id}`, `--config=${cfg}`,
188
+ `--labels=plaiful-job=${id},plaiful-task=${slug(task)},plaiful-gpu=${gpu}`], { json: true });
189
+ } finally { try { fs.unlinkSync(cfg); } catch { /* temp file */ } }
190
+ const record = { id, name: job && job.name, task, gpu, machine: spec.workerPoolSpecs[0].machineSpec.machineType, hours, project: s.project, region: s.region, bucket: s.bucket, data: data.map((d) => d.name), submitted_at: new Date().toISOString() };
191
+ writeJobs([...readJobs(), record]);
192
+ ctx.out([
193
+ `Submitted ${id} on one ${gpu.toUpperCase()} (${record.machine}), stopped after ${hours} h at most.`,
194
+ ` output: gs://${s.bucket}/jobs/${id}/out`,
195
+ ` plaiful models train status ${id} · plaiful models train logs ${id} · plaiful models train fetch ${id}`,
196
+ ].join('\n'), { success: true, job: record });
197
+ return 0;
198
+ }
199
+
200
+ function findJob(id) {
201
+ const jobs = readJobs();
202
+ const j = id ? jobs.find((x) => x.id === id) : jobs[jobs.length - 1];
203
+ if (!j) throw new UsageError(id ? `no job ${id} submitted from this machine (plaiful models train list)` : 'no jobs submitted from this machine yet');
204
+ return j;
205
+ }
206
+
207
+ /** Vertex AI's view of a job → { state, started, ended, gpu_hours, error } */
208
+ function describe(ctx, j) {
209
+ const d = gcloud(ctx, ['ai', 'custom-jobs', 'describe', j.name, `--region=${j.region}`, `--project=${j.project}`], { json: true });
210
+ const state = String(d.state || '').replace(/^JOB_STATE_/, '').toLowerCase();
211
+ // Vertex AI sets startTime as soon as a job is accepted, so a job still waiting for its machine isn't GPU time yet.
212
+ const waiting = ['queued', 'pending'].includes(state);
213
+ const start = d.startTime && !waiting ? Date.parse(d.startTime) : null;
214
+ const end = d.endTime ? Date.parse(d.endTime) : (start ? Date.now() : null);
215
+ return { state, created: d.createTime, started: d.startTime || null, ended: d.endTime || null, gpu_hours: start ? Math.round(((end - start) / 3.6e6) * 100) / 100 : 0, error: d.error && d.error.message };
216
+ }
217
+
218
+ async function status(ctx, [id]) {
219
+ const j = findJob(id);
220
+ const d = describe(ctx, j);
221
+ ctx.out([
222
+ `${j.id}: ${d.state} (${j.gpu.toUpperCase()}, ${j.task})`,
223
+ ` submitted ${j.submitted_at}${d.started ? `, started ${d.started}` : ', waiting for a GPU'}${d.ended ? `, ended ${d.ended}` : ''}`,
224
+ ` GPU time so far: ${d.gpu_hours} h (billed to ${j.project}; Billing → Reports, label plaiful-job=${j.id})`,
225
+ ...(d.error ? [` error: ${d.error}`] : []),
226
+ ].join('\n'), { success: true, job: j, ...d });
227
+ return 0;
228
+ }
229
+
230
+ async function list(ctx) {
231
+ const jobs = readJobs();
232
+ if (!jobs.length) { ctx.out('No training jobs submitted from this machine yet.', { success: true, jobs: [] }); return 0; }
233
+ const rows = jobs.slice(-20).map((j) => { let d = {}; try { d = describe(ctx, j); } catch (e) { d = { state: `unknown (${e.message})` }; } return { ...j, ...d }; });
234
+ ctx.out(rows.map((j) => `${j.id} ${j.state} ${j.gpu} ${j.gpu_hours != null ? `${j.gpu_hours} GPU-h` : ''} ${j.task}`).join('\n'), { success: true, jobs: rows });
235
+ return 0;
236
+ }
237
+
238
+ async function logs(ctx, [id]) {
239
+ const j = findJob(id);
240
+ gcloud(ctx, ['ai', 'custom-jobs', 'stream-logs', j.name, `--region=${j.region}`, `--project=${j.project}`], { inherit: true });
241
+ return 0;
242
+ }
243
+
244
+ async function fetch(ctx, [id]) {
245
+ const j = findJob(id);
246
+ const out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('model-runs', j.id));
247
+ fs.mkdirSync(out, { recursive: true });
248
+ gcloud(ctx, ['storage', 'rsync', `gs://${j.bucket}/jobs/${j.id}/out`, out, '--recursive', `--project=${j.project}`], { inherit: true });
249
+ ctx.out(`${j.id}'s output is in ${out}. Card it, then plaiful models publish <card.json>.`, { success: true, out });
250
+ return 0;
251
+ }
252
+
253
+ async function cancel(ctx, [id]) {
254
+ const j = findJob(id);
255
+ if (!id) throw new UsageError('cancel which job? plaiful models train cancel <job>');
256
+ gcloud(ctx, ['ai', 'custom-jobs', 'cancel', j.name, `--region=${j.region}`, `--project=${j.project}`]);
257
+ ctx.out(`Asked Vertex AI to cancel ${j.id}.`, { success: true });
258
+ return 0;
259
+ }
260
+
261
+ return { check, submit, status, list, logs, fetch, cancel };
262
+ }
263
+
264
+ module.exports = { makeTrain, jobSpec, jobScript, settings, GPUS, DEFAULT_IMAGE, slug };