@plaiful/cli 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -75,6 +75,7 @@ plaiful assets # per agent: records, and how many have art made /
75
75
  plaiful assets crew-portrait --state failed # one agent's records: state, why a render failed, where the files are
76
76
  plaiful assets battlefield --json # also: any step that failed ({step}_step_error), and a part's source_crop and depth_min / depth_max
77
77
  plaiful assets regenerate battlefield --state step_failed --steps depth # re-run a step (or, without --steps, re-render: a new picture)
78
+ plaiful assets regenerate building --state no_model --model # a 3D model from each record's picture (a 2d+3d Art Style; Regenerate → 3D model)
78
79
  plaiful quests # the quests OVID wrote: steps, and what each pays (--track, --series); quests show <id>
79
80
  plaiful lore # the lore sets (lore show <setId>: a set's snippets)
80
81
  plaiful world # the world's states and events in story order, and the clock
@@ -151,3 +152,7 @@ same screen shows the product's activity: everything the CLI and its keys did.
151
152
  npm as a member of the `plaiful` org.
152
153
  - **Tests:** the end-to-end tests run this CLI against the partner API handler:
153
154
  `lambda/functions/partner-api/__tests__/partner-api.test.js`.
155
+ - **The model registry (`plaiful models`, staff only):** publish release cards with their files, push training data,
156
+ and run GPU training on Google Cloud (`models train`, with your own `gcloud` sign-in). Guide:
157
+ `docs/design/model-registry.md`; card format: `docs/design/model-card.md`. Tests:
158
+ `lambda/functions/partner-api/__tests__/models.test.js` and `models-cli.test.js`.
package/package.json CHANGED
@@ -1,21 +1,21 @@
1
- {
2
- "name": "@plaiful/cli",
3
- "version": "0.14.0",
4
- "description": "The Plaiful CLI: work on your Plaiful product from your game repo, with no AWS access",
5
- "bin": {
6
- "plaiful": "bin/plaiful.js"
7
- },
8
- "files": [
9
- "bin",
10
- "src",
11
- "README.md"
12
- ],
13
- "engines": {
14
- "node": ">=18"
15
- },
16
- "license": "UNLICENSED",
17
- "publishConfig": {
18
- "registry": "https://registry.npmjs.org/",
19
- "access": "public"
20
- }
21
- }
1
+ {
2
+ "name": "@plaiful/cli",
3
+ "version": "0.15.0",
4
+ "description": "The Plaiful CLI: work on your Plaiful product from your game repo, with no AWS access",
5
+ "bin": {
6
+ "plaiful": "bin/plaiful.js"
7
+ },
8
+ "files": [
9
+ "bin",
10
+ "src",
11
+ "README.md"
12
+ ],
13
+ "engines": {
14
+ "node": ">=18"
15
+ },
16
+ "license": "UNLICENSED",
17
+ "publishConfig": {
18
+ "registry": "https://registry.npmjs.org/",
19
+ "access": "public"
20
+ }
21
+ }
package/src/cli.js CHANGED
@@ -102,6 +102,9 @@ const HELP = `plaiful ${version}: work on your Plaiful product from your game re
102
102
  plaiful assets regenerate <agent> [uuids...] re-render records with the table's Art Style, as the Library's
103
103
  Regenerate does: a new picture (--all, or --state failed|none|made|step_failed)
104
104
  --steps depth[,normal] | --steps all only re-run those steps on the current picture (Regenerate → Steps)
105
+ --model a 3D model from each record's current picture, with the table's 2d+3d Art Style
106
+ (Regenerate → 3D model; --state no_model: every record with a picture and no model;
107
+ at most 50 a run, each a paid build)
105
108
  --state made|pending|failed|none --limit <n> (default 50)
106
109
  plaiful animation sets the animation sets you can use (a set = a list of animation states, one sprite
107
110
  sheet each, run together on a character): the built-ins, Plaiful's and yours
@@ -111,6 +114,22 @@ const HELP = `plaiful ${version}: work on your Plaiful product from your game re
111
114
  plaiful animation sets delete <id> delete one of your sets
112
115
  plaiful animation run <set> <agent> [uuids...] run a set on an agent's records (--all: every record with its art
113
116
  made; --style <sprite style>, else the set's; --force: make them again)
117
+ plaiful models the model registry (Plaiful staff for now): every release, its status and licence
118
+ plaiful models check <card|dir> check release cards and their files on disk (a dry run: nothing uploads)
119
+ plaiful models publish <card|dir> upload each card's files (checked against its sha256s) and publish the card;
120
+ a published release never changes, only its card can be corrected
121
+ (--files <folder> if the card has no archive_path; owner from .plaiful.json)
122
+ plaiful models show <task> <release> one release: card, files, status history
123
+ plaiful models status <task> <release> active|candidate|retired (--note "…"; retired, never deleted)
124
+ plaiful models pull <task> <release> download a release, every sha256 checked (--out <dir>, --role shipped)
125
+ plaiful models data push <task> <snapshot> <folder> store a training-data folder as a sealed snapshot
126
+ (--exclude <regex>, --source "…", --licence "…", --note "…", --releases a,b)
127
+ plaiful models data [list] the training-data snapshots; models data pull <task> <snapshot> [--out <dir>]
128
+ plaiful models train check is this machine ready to train on Google Cloud (gcloud, project, bucket, API)
129
+ plaiful models train submit <task> --code <folder> --entry "python train.py --data {data} --out {out}"
130
+ run it on a Google Cloud GPU (--gpu l4|t4|a100, --hours 12, --data <folders>,
131
+ --snapshot <task>/<snapshot>, --project, --region, --bucket, --image, --name)
132
+ plaiful models train list|status|logs|fetch|cancel [job] follow a job; fetch brings its output home (--out)
114
133
  plaiful upload <files...> upload map tiles into your dev assets bucket (map-data/; --dest map-data)
115
134
  plaiful clone [product] [dir] clone the game repo, write .env.development, docs pull, npm install, open Zed
116
135
  --no-install --no-open
@@ -141,8 +160,8 @@ function parse(argv) {
141
160
  const args = [];
142
161
  for (let i = 0; i < argv.length; i += 1) {
143
162
  const a = argv[i];
144
- if (['--json', '--no-browser', '--help', '-h', '--version', '--apply', '--reset', '--rows', '--no-install', '--no-open', '--copies', '--no-render', '--wait', '--force', '--fountain', '--vo', '--all', '--retire', '--known'].includes(a)) flags[a.replace(/^-+/, '')] = true;
145
- else if (['--product', '--api', '--dir', '--dest', '--count', '--base', '--per-effort', '--track', '--series', '--state', '--limit', '--section', '--since', '--note', '--out', '--style', '--aspect', '--frame', '--title', '--description', '--category', '--order', '--directives', '--narrative', '--hint', '--rarity', '--statement', '--name', '--reason', '--type', '--homeland', '--scope', '--status', '--steps'].includes(a)) {
163
+ if (['--json', '--no-browser', '--help', '-h', '--version', '--apply', '--reset', '--rows', '--no-install', '--no-open', '--copies', '--no-render', '--wait', '--force', '--fountain', '--vo', '--all', '--retire', '--known', '--model'].includes(a)) flags[a.replace(/^-+/, '')] = true;
164
+ else if (['--product', '--api', '--dir', '--dest', '--count', '--base', '--per-effort', '--track', '--series', '--state', '--limit', '--section', '--since', '--note', '--out', '--style', '--aspect', '--frame', '--title', '--description', '--category', '--order', '--directives', '--narrative', '--hint', '--rarity', '--statement', '--name', '--reason', '--type', '--homeland', '--scope', '--status', '--steps', '--files', '--role', '--exclude', '--source', '--licence', '--releases', '--code', '--entry', '--data', '--snapshot', '--gpu', '--machine', '--hours', '--image', '--project', '--region', '--bucket'].includes(a)) {
146
165
  if (argv[i + 1] === undefined) throw new UsageError(`${a} needs a value`);
147
166
  flags[a.slice(2)] = argv[i += 1];
148
167
  } else if (a.startsWith('--')) throw new UsageError(`unknown option ${a}`);
@@ -1266,15 +1285,22 @@ async function assetsRegenerate(ctx, [agent, ...uuids]) {
1266
1285
  if (uuids.length) body.records = uuids;
1267
1286
  if (ctx.flags.all) body.all = true;
1268
1287
  if (ctx.flags.state) body.state = String(ctx.flags.state);
1269
- if (!body.records && !body.all && !body.state) throw new UsageError('which records? list their uuids, or --all, or --state failed|none|made|step_failed');
1288
+ if (!body.records && !body.all && !body.state) throw new UsageError('which records? list their uuids, or --all, or --state failed|none|made|step_failed (with --model, also no_model)');
1289
+ if (ctx.flags.model) {
1290
+ if (ctx.flags.steps !== undefined) throw new UsageError('--model and --steps are separate runs: one or the other');
1291
+ body.model = true;
1292
+ }
1270
1293
  if (ctx.flags.steps !== undefined) {
1271
1294
  if (ctx.flags.steps === true) throw new UsageError('--steps takes step names (--steps depth,normal) or all');
1272
1295
  body.steps = String(ctx.flags.steps) === 'all' ? 'all' : String(ctx.flags.steps).split(',').map((x) => x.trim()).filter(Boolean);
1273
1296
  }
1274
1297
  const { data } = await ctx.call('POST', `/partner/${encodeURIComponent(r.product)}/assets/${encodeURIComponent(agent)}/regenerate`, body);
1275
- const what = data.mode === 'steps' ? `the ${data.steps.join(', ')} step${data.steps.length !== 1 ? 's' : ''} of ${data.style}` : `a new picture in ${data.style}`;
1276
- ctx.out(`Started ${what} for ${data.records} ${agent} record${data.records !== 1 ? 's' : ''}. They render in the background: plaiful assets ${agent} shows each one as it lands.`, data);
1277
- return 0;
1298
+ const what = data.mode === 'steps' ? `the ${data.steps.join(', ')} step${data.steps.length !== 1 ? 's' : ''} of ${data.style}`
1299
+ : data.mode === 'model' ? `a 3D model with ${data.style}` : `a new picture in ${data.style}`;
1300
+ const lines = [`Started ${what} for ${data.records} ${agent} record${data.records !== 1 ? 's' : ''}. They ${data.mode === 'model' ? 'build' : 'render'} in the background: plaiful assets ${agent} shows each one as it lands.`];
1301
+ for (const s of data.skipped || []) lines.push(` skipped ${s.uuid}: ${s.reason}`);
1302
+ ctx.out(lines.join('\n'), data);
1303
+ return data.records ? 0 : 1;
1278
1304
  }
1279
1305
 
1280
1306
  async function assets(ctx, [agent, ...rest]) {
@@ -1464,6 +1490,24 @@ async function upload(ctx, files) {
1464
1490
  return 0;
1465
1491
  }
1466
1492
 
1493
+ // ── The model registry and training (models.js, train.js) ──────────────────────────────────────────────────────
1494
+
1495
+ const modelsCmd = require('./models').makeModels({ UsageError, ApiError, repo });
1496
+ const trainCmd = require('./train').makeTrain({ UsageError, ApiError, repo, models: modelsCmd });
1497
+
1498
+ function modelsData(ctx, [sub, ...rest]) {
1499
+ if (sub === 'push') return modelsCmd.dataPush(ctx, rest);
1500
+ if (sub === 'pull') return modelsCmd.dataPull(ctx, rest);
1501
+ if (!sub || sub === 'list') return modelsCmd.dataList(ctx, rest);
1502
+ throw new UsageError(`plaiful models data list | push | pull (not "${sub}")`);
1503
+ }
1504
+
1505
+ function modelsTrain(ctx, [sub, ...rest]) {
1506
+ const run = { check: trainCmd.check, submit: trainCmd.submit, list: trainCmd.list, status: trainCmd.status, logs: trainCmd.logs, fetch: trainCmd.fetch, cancel: trainCmd.cancel }[sub || 'list'];
1507
+ if (!run) throw new UsageError(`plaiful models train check | submit | list | status | logs | fetch | cancel (not "${sub}")`);
1508
+ return run(ctx, rest);
1509
+ }
1510
+
1467
1511
  // ── Clone and .env (the New-Game Flow) ───────────────────────────────────────────────────────────────────────────
1468
1512
 
1469
1513
  /** The game template's .env variables (plaiful-kit packages/client config.ts ENV_KEYS) ← the client config. */
@@ -1622,6 +1666,15 @@ const COMMANDS = {
1622
1666
  'events storyboard': (ctx, rest) => eventsStoryboard(ctx, rest),
1623
1667
  'animation sets': (ctx, rest) => animationSets(ctx, rest),
1624
1668
  'animation run': (ctx, rest) => animationRun(ctx, rest),
1669
+ 'models check': (ctx, rest) => modelsCmd.publish(ctx, rest, { apply: false }),
1670
+ 'models publish': (ctx, rest) => modelsCmd.publish(ctx, rest, { apply: true }),
1671
+ 'models list': (ctx, rest) => modelsCmd.list(ctx, rest),
1672
+ 'models show': (ctx, rest) => modelsCmd.show(ctx, rest),
1673
+ 'models status': (ctx, rest) => modelsCmd.status(ctx, rest),
1674
+ 'models pull': (ctx, rest) => modelsCmd.pull(ctx, rest),
1675
+ 'models data': (ctx, rest) => modelsData(ctx, rest),
1676
+ 'models train': (ctx, rest) => modelsTrain(ctx, rest),
1677
+ models: (ctx, rest) => modelsCmd.list(ctx, rest),
1625
1678
  upload: (ctx, files) => upload(ctx, files),
1626
1679
  clone: (ctx, rest) => clone(ctx, rest),
1627
1680
  env: (ctx) => envCmd(ctx),
@@ -1648,6 +1701,16 @@ function defaultIo() {
1648
1701
  return { status: r.status === null ? 1 : r.status };
1649
1702
  } catch { return { status: 1 }; }
1650
1703
  },
1704
+ // gcloud for `plaiful models train`: its output captured (or shown, with `inherit`). On Windows gcloud is a .cmd,
1705
+ // which needs a shell, so arguments with spaces are quoted. → { status, stdout, stderr }
1706
+ exec: (cmd, args, { inherit = false } = {}) => {
1707
+ const shell = process.platform === 'win32';
1708
+ const argv = shell ? args.map((a) => (/^[A-Za-z0-9_./:=@,-]+$/.test(a) ? a : `"${String(a).replace(/"/g, '\\"')}"`)) : args;
1709
+ try {
1710
+ const r = spawnSync(cmd, argv, { shell, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, stdio: inherit ? 'inherit' : 'pipe' });
1711
+ return { status: r.status === null ? 1 : r.status, stdout: r.stdout || '', stderr: r.stderr || '' };
1712
+ } catch (e) { return { status: 1, stdout: '', stderr: e.message }; }
1713
+ },
1651
1714
  openUrl: (url) => {
1652
1715
  const [cmd, args] = process.platform === 'win32' ? ['rundll32', ['url.dll,FileProtocolHandler', url]]
1653
1716
  : process.platform === 'darwin' ? ['open', [url]] : ['xdg-open', [url]];
package/src/models.js ADDED
@@ -0,0 +1,284 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * `plaiful models …`: the model registry from a game repo (PlaifulCMS docs/design/model-registry.md). Plaiful staff
5
+ * only for now. A release is a card (`<release>.card.json`, Ironheads' format or version 2) plus the files it lists;
6
+ * a training-data snapshot is a folder, pushed as is with a manifest of every file's size and sha256.
7
+ *
8
+ * Files go straight to storage on short-lived links the platform signs with each file's sha256, so storage refuses
9
+ * bytes that don't match the card. Nothing here needs AWS.
10
+ */
11
+
12
+ const fs = require('fs');
13
+ const path = require('path');
14
+ const crypto = require('crypto');
15
+
16
+ const BATCH = 500;
17
+ const CONCURRENCY = 6;
18
+
19
+ function sha256File(file) {
20
+ return new Promise((resolve, reject) => {
21
+ const h = crypto.createHash('sha256');
22
+ fs.createReadStream(file).on('error', reject).on('data', (d) => h.update(d)).on('end', () => resolve(h.digest('hex')));
23
+ });
24
+ }
25
+
26
+ async function pool(items, n, fn) {
27
+ let i = 0;
28
+ await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
29
+ while (i < items.length) { const k = i; i += 1; await fn(items[k], k); }
30
+ }));
31
+ }
32
+
33
+ const mb = (b) => (b >= 1e9 ? `${(b / 1e9).toFixed(2)} GB` : b >= 1e6 ? `${(b / 1e6).toFixed(1)} MB` : `${Math.max(1, Math.round(b / 1e3))} KB`);
34
+ const enc = (s) => encodeURIComponent(s);
35
+
36
+ /** Every file under `dir` (relative, forward slashes), skipping names that match `exclude`. */
37
+ function walk(dir, exclude) {
38
+ const out = [];
39
+ const visit = (rel) => {
40
+ for (const ent of fs.readdirSync(path.join(dir, rel), { withFileTypes: true })) {
41
+ const r = rel ? `${rel}/${ent.name}` : ent.name;
42
+ if (exclude && exclude.test(r)) continue;
43
+ if (ent.isDirectory()) visit(r);
44
+ else if (ent.isFile()) out.push(r);
45
+ }
46
+ };
47
+ visit('');
48
+ return out.sort();
49
+ }
50
+
51
+ /** The card files a path names: one card, or every *.card.json under a folder. */
52
+ function cardFiles(target) {
53
+ if (fs.statSync(target).isFile()) return [target];
54
+ const found = [];
55
+ const visit = (d) => {
56
+ for (const ent of fs.readdirSync(d, { withFileTypes: true })) {
57
+ const p = path.join(d, ent.name);
58
+ if (ent.isDirectory()) visit(p);
59
+ else if (ent.name.endsWith('.card.json')) found.push(p);
60
+ }
61
+ };
62
+ visit(target);
63
+ return found.sort();
64
+ }
65
+
66
+ /** Where a card's files are on this machine: its archive folder (or, without one, where the game ships it). */
67
+ function filesRoot(root, card, flags) {
68
+ const base = flags.files || (card.files && (card.files.archive_path || card.files.shipped_path));
69
+ if (!base) return null;
70
+ return path.resolve(root, base);
71
+ }
72
+
73
+ /** PUT one file to a signed link; the link's headers carry the sha256 storage checks. */
74
+ async function putFile(io, file, u) {
75
+ const body = fs.readFileSync(file);
76
+ let res;
77
+ try { res = await io.fetch(u.url, { method: u.method || 'PUT', headers: u.headers, body }); } catch (e) {
78
+ throw Object.assign(new Error(`the upload of ${u.path} didn't go through (${e.message}); run it again`), { code: 'network' });
79
+ }
80
+ if (!res.ok) {
81
+ const why = res.status === 400 ? 'its bytes don\'t match the sha256 (did the file change after the card was written?)' : `storage answered ${res.status}`;
82
+ throw Object.assign(new Error(`storage refused ${u.path}: ${why}`), { code: 'upload_failed' });
83
+ }
84
+ }
85
+
86
+ async function getFile(io, url, file, sha) {
87
+ const res = await io.fetch(url);
88
+ if (!res.ok) throw Object.assign(new Error(`couldn't download ${path.basename(file)} (${res.status})`), { code: 'download_failed' });
89
+ const buf = Buffer.from(await res.arrayBuffer());
90
+ const got = crypto.createHash('sha256').update(buf).digest('hex');
91
+ if (sha && got !== sha) throw Object.assign(new Error(`${file} doesn't match its sha256 after download`), { code: 'checksum' });
92
+ fs.mkdirSync(path.dirname(file), { recursive: true });
93
+ fs.writeFileSync(file, buf);
94
+ }
95
+
96
+ function makeModels({ UsageError, ApiError, repo }) {
97
+ const apiFail = (code, error) => new ApiError(409, { code, error });
98
+
99
+ /** Read a card and check its files on disk against it. → { card, root, problems } */
100
+ async function localCheck(ctx, file) {
101
+ let card;
102
+ try { card = JSON.parse(fs.readFileSync(file, 'utf8')); } catch (e) { throw new UsageError(`${file} isn't a JSON card (${e.message})`); }
103
+ const r = repo(ctx, { required: false });
104
+ const root = filesRoot(r.root, card, ctx.flags);
105
+ const problems = [];
106
+ const items = (card.files && card.files.items) || [];
107
+ if (!root) problems.push('the card has no files.archive_path or shipped_path: pass --files <folder>');
108
+ else {
109
+ await pool(items, CONCURRENCY, async (f) => {
110
+ const p = path.join(root, f.path);
111
+ if (!fs.existsSync(p)) { problems.push(`${f.path}: not found under ${root}`); return; }
112
+ const size = fs.statSync(p).size;
113
+ if (size !== f.bytes) { problems.push(`${f.path}: ${size} bytes, the card says ${f.bytes}`); return; }
114
+ if (await sha256File(p) !== f.sha256) problems.push(`${f.path}: sha256 differs from the card`);
115
+ });
116
+ }
117
+ return { card, root, problems, owner: card.owner || r.product || null };
118
+ }
119
+
120
+ /** `plaiful models check|publish <card.json | folder of cards>` */
121
+ async function publish(ctx, [target], { apply }) {
122
+ if (!target) throw new UsageError(`${apply ? 'publish' : 'check'} what? plaiful models ${apply ? 'publish' : 'check'} <card.json | folder of cards>`);
123
+ const abs = path.resolve(ctx.io.cwd(), target);
124
+ if (!fs.existsSync(abs)) throw new UsageError(`${abs} doesn't exist`);
125
+ const files = cardFiles(abs);
126
+ if (!files.length) throw new UsageError(`no *.card.json under ${abs}`);
127
+ const results = [];
128
+ let failed = 0;
129
+ for (const file of files) {
130
+ const { card, root, problems, owner } = await localCheck(ctx, file);
131
+ const id = `${card.task_type}/${card.release_id}`;
132
+ const row = { card: path.relative(ctx.io.cwd(), file), release: id, problems, warnings: [] };
133
+ results.push(row);
134
+ if (problems.length) { failed += 1; ctx.io.stderr.write(`✗ ${id}: ${problems.length} file problem(s)\n${problems.slice(0, 5).map((p) => ` ${p}`).join('\n')}\n`); continue; }
135
+ const route = `/partner/models/releases/${enc(card.task_type)}/${enc(card.release_id)}`;
136
+ let up;
137
+ try {
138
+ up = (await ctx.call('POST', `${route}/upload`, { card, owner })).data;
139
+ } catch (e) {
140
+ if (!(e instanceof ApiError) || e.status >= 500 || e.status === 401 || e.status === 403) throw e;
141
+ failed += 1; row.problems.push(e.message); ctx.io.stderr.write(`✗ ${id}: ${e.message}\n`); continue;
142
+ }
143
+ row.warnings = up.warnings || [];
144
+ if (!apply) {
145
+ row.would_upload = up.uploads.length;
146
+ row.stored = up.stored;
147
+ ctx.io.stderr.write(`✓ ${id}: valid; ${up.uploads.length} file(s) to upload, ${up.stored} stored${up.published ? ' (published: a card correction)' : ''}${row.warnings.length ? `; ${row.warnings.length} warning(s)` : ''}\n`);
148
+ continue;
149
+ }
150
+ let sent = 0;
151
+ await pool(up.uploads, CONCURRENCY, async (u) => { await putFile(ctx.io, path.join(root, u.path), u); sent += u.bytes; });
152
+ const pub = (await ctx.call('POST', `${route}/publish`, { card, owner })).data;
153
+ Object.assign(row, { uploaded: up.uploads.length, uploaded_bytes: sent, card_rev: pub.card_rev, corrected: pub.corrected, status: pub.status });
154
+ ctx.io.stderr.write(`✓ ${id}: published (card ${pub.card_rev}${pub.corrected ? ', corrected' : ''}, ${pub.status}); ${up.uploads.length} file(s), ${mb(sent)} uploaded\n`);
155
+ }
156
+ const ok = results.length - failed;
157
+ ctx.out(`${apply ? 'Published' : 'Checked'} ${ok} of ${results.length} release(s)${failed ? `; ${failed} with problems (above)` : ''}.${apply ? '' : ' Nothing was uploaded: plaiful models publish to upload.'}`, { success: failed === 0, results });
158
+ return failed ? 1 : 0;
159
+ }
160
+
161
+ async function list(ctx, [task]) {
162
+ const { data } = await ctx.call('GET', `/partner/models/releases${task ? `?task=${enc(task)}` : ''}`);
163
+ const rows = data.releases;
164
+ const lines = rows.length ? [] : ['No releases in the registry yet: plaiful models publish <card.json>'];
165
+ let last = null;
166
+ for (const r of rows.sort((a, b) => a.task_type.localeCompare(b.task_type) || String(b.date).localeCompare(String(a.date)))) {
167
+ if (r.task_type !== last) { lines.push(`${r.task_type}`); last = r.task_type; }
168
+ lines.push(` ${r.release_id} ${r.status} ${r.date} ${mb(r.bytes)} cleared: ${r.cleared_for_other_games} (${r.owner})`);
169
+ }
170
+ ctx.out(lines.join('\n'), data);
171
+ return 0;
172
+ }
173
+
174
+ async function show(ctx, [task, release]) {
175
+ if (!task || !release) throw new UsageError('plaiful models show <task> <release>');
176
+ const { data } = await ctx.call('GET', `/partner/models/releases/${enc(task)}/${enc(release)}`);
177
+ const r = data.release;
178
+ const lines = [
179
+ `${r.task_type} / ${r.release_id} (${r.status}, card ${r.card_rev}, owner ${r.owner})`,
180
+ ` ${r.name}: ${r.one_line}`,
181
+ ` cleared for other games: ${r.cleared_for_other_games}${r.attribution.length ? `; credit: ${r.attribution.join(' · ')}` : ''}`,
182
+ ` ${r.files} file(s), ${mb(r.bytes)} (shipped ${mb(r.shipped_bytes)}); ${r.prefix}`,
183
+ ` published ${r.published_at} by ${r.published_by}`,
184
+ ...(r.status_history || []).map((h) => ` ${h.at.slice(0, 10)} ${h.status} ${h.by}${h.note ? `: ${h.note}` : ''}`),
185
+ ];
186
+ ctx.out(lines.join('\n'), data);
187
+ return 0;
188
+ }
189
+
190
+ async function status(ctx, [task, release, value]) {
191
+ if (!task || !release || !value) throw new UsageError('plaiful models status <task> <release> active|candidate|retired [--note "…"]');
192
+ const { data } = await ctx.call('POST', `/partner/models/releases/${enc(task)}/${enc(release)}/status`, { status: value, note: ctx.flags.note });
193
+ ctx.out(data.changed ? `${task}/${release}: ${data.was} → ${data.status}. Update the card's status too, so the two agree.` : `${task}/${release} is already ${data.status}.`, data);
194
+ return 0;
195
+ }
196
+
197
+ async function pull(ctx, [task, release]) {
198
+ if (!task || !release) throw new UsageError('plaiful models pull <task> <release> [--out <dir>] [--role shipped]');
199
+ const roles = ctx.flags.role ? String(ctx.flags.role).split(',') : undefined;
200
+ const { data } = await ctx.call('POST', `/partner/models/releases/${enc(task)}/${enc(release)}/download`, { roles });
201
+ const out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('models', task, release));
202
+ await pool(data.files, CONCURRENCY, (f) => getFile(ctx.io, f.url, path.join(out, f.path), f.sha256));
203
+ fs.writeFileSync(path.join(out, `${release}.card.json`), `${JSON.stringify(data.card, null, 1)}\n`);
204
+ ctx.out(`Downloaded ${data.files.length} file(s) of ${task}/${release} (${data.status}) to ${out}, every sha256 checked.`, { success: true, out, files: data.files.length });
205
+ return 0;
206
+ }
207
+
208
+ // ── Training-data snapshots ──────────────────────────────────────────────────────────────────────────────────
209
+
210
+ async function dataPush(ctx, [task, snapshot, dir]) {
211
+ if (!task || !snapshot || !dir) throw new UsageError('plaiful models data push <task> <snapshot> <folder> [--exclude <regex>] [--source "…"] [--licence "…"] [--note "…"] [--releases a,b]');
212
+ const abs = path.resolve(ctx.io.cwd(), dir);
213
+ if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) throw new UsageError(`${abs} isn't a folder`);
214
+ let exclude = null;
215
+ if (ctx.flags.exclude) { try { exclude = new RegExp(ctx.flags.exclude); } catch (e) { throw new UsageError(`--exclude isn't a regular expression (${e.message})`); } }
216
+ const rels = walk(abs, exclude);
217
+ if (!rels.length) throw new UsageError(`no files under ${abs}`);
218
+ ctx.io.stderr.write(`Hashing ${rels.length} file(s) under ${abs}…\n`);
219
+ const entries = new Array(rels.length);
220
+ await pool(rels, CONCURRENCY, async (rel, i) => {
221
+ const p = path.join(abs, rel);
222
+ entries[i] = { path: rel, bytes: fs.statSync(p).size, sha256: await sha256File(p) };
223
+ });
224
+ const total = entries.reduce((n, e) => n + e.bytes, 0);
225
+ const manifest = `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`;
226
+ const mBytes = Buffer.byteLength(manifest);
227
+ const mSha = crypto.createHash('sha256').update(manifest).digest('hex');
228
+ const route = `/partner/models/datasets/${enc(task)}/${enc(snapshot)}`;
229
+ let uploaded = 0;
230
+ let stored = 0;
231
+ let sent = 0;
232
+ for (let i = 0; i < entries.length; i += BATCH) {
233
+ const batch = entries.slice(i, i + BATCH);
234
+ const last = i + BATCH >= entries.length;
235
+ const { data } = await ctx.call('POST', `${route}/upload`, { files: batch, ...(last ? { manifest: { bytes: mBytes, sha256: mSha } } : {}) });
236
+ stored += data.stored;
237
+ await pool(data.uploads, CONCURRENCY, async (u) => { await putFile(ctx.io, path.join(abs, u.path), u); uploaded += 1; sent += u.bytes; });
238
+ if (last) {
239
+ const res = await ctx.io.fetch(data.manifest.url, { method: 'PUT', headers: data.manifest.headers, body: Buffer.from(manifest) });
240
+ if (!res.ok) throw apiFail('upload_failed', `storage refused the manifest (${res.status}); run the push again`);
241
+ }
242
+ ctx.io.stderr.write(` ${Math.min(i + BATCH, entries.length)} / ${entries.length} (${uploaded} uploaded, ${stored} already stored)\n`);
243
+ }
244
+ const about = { source: ctx.flags.source || abs, licence: ctx.flags.licence || '', note: ctx.flags.note || '', releases: ctx.flags.releases ? String(ctx.flags.releases).split(',') : [] };
245
+ const { data } = await ctx.call('POST', `${route}/finish`, { about });
246
+ ctx.out(`${task}/${snapshot}: sealed, ${data.files} file(s), ${mb(data.bytes)} (${uploaded} uploaded, ${mb(sent)}; ${stored} were already stored). Backed up to the Management account by replication.`, { success: true, ...data, uploaded, total_bytes: total });
247
+ return 0;
248
+ }
249
+
250
+ async function dataList(ctx, [task]) {
251
+ const { data } = await ctx.call('GET', `/partner/models/datasets${task ? `?task=${enc(task)}` : ''}`);
252
+ const lines = data.datasets.length ? data.datasets.map((d) => `${d.task_type}/${d.snapshot} ${d.state} ${d.files != null ? `${d.files} file(s), ${mb(d.bytes)}` : '(push not finished)'}${d.about && d.about.releases && d.about.releases.length ? ` for ${d.about.releases.join(', ')}` : ''}`)
253
+ : ['No training-data snapshots yet: plaiful models data push <task> <snapshot> <folder>'];
254
+ ctx.out(lines.join('\n'), data);
255
+ return 0;
256
+ }
257
+
258
+ async function dataPull(ctx, [task, snapshot]) {
259
+ if (!task || !snapshot) throw new UsageError('plaiful models data pull <task> <snapshot> [--out <dir>]');
260
+ const out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('datasets', task, snapshot));
261
+ const route = `/partner/models/datasets/${enc(task)}/${enc(snapshot)}/download`;
262
+ let after = 0;
263
+ let n = 0;
264
+ let skipped = 0;
265
+ for (;;) {
266
+ const { data } = await ctx.call('POST', route, { after });
267
+ await pool(data.files, CONCURRENCY, async (f) => {
268
+ const p = path.join(out, f.path);
269
+ if (fs.existsSync(p) && fs.statSync(p).size === f.bytes && await sha256File(p) === f.sha256) { skipped += 1; return; }
270
+ await getFile(ctx.io, f.url, p, f.sha256);
271
+ n += 1;
272
+ });
273
+ ctx.io.stderr.write(` ${data.after + data.files.length} / ${data.total}\n`);
274
+ if (data.next == null) break;
275
+ after = data.next;
276
+ }
277
+ ctx.out(`${task}/${snapshot} → ${out}: ${n} downloaded, ${skipped} already here, every sha256 checked.`, { success: true, out, downloaded: n, skipped });
278
+ return 0;
279
+ }
280
+
281
+ return { publish, list, show, status, pull, dataPush, dataList, dataPull };
282
+ }
283
+
284
+ module.exports = { makeModels, walk, sha256File, cardFiles, filesRoot, mb };
package/src/train.js ADDED
@@ -0,0 +1,260 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * `plaiful models train …`: run a training job on a Google Cloud GPU (Vertex AI custom training) instead of the
5
+ * developer's own machine, so the GPU time draws on Plaiful's Google Cloud credits (PlaifulCMS
6
+ * docs/design/model-registry.md §5). Plaiful staff only for now, and it runs on this machine's `gcloud` sign-in: the
7
+ * platform holds no Google credentials.
8
+ *
9
+ * A job: the code folder and the data folders are copied to the project's bucket (data is cached across jobs), Vertex
10
+ * AI runs the entry command on one GPU with the bucket mounted at /gcs/<bucket>, the job writes to {out}, and
11
+ * `train fetch` brings the output home, ready to card and `plaiful models publish`.
12
+ *
13
+ * Settings, from flags, else .plaiful.json "training": { project, region, bucket, image }, else PLAIFUL_GCP_* env.
14
+ */
15
+
16
+ const fs = require('fs');
17
+ const os = require('os');
18
+ const path = require('path');
19
+
20
+ /** One GPU per job, by name: the machine Vertex AI pairs with it. */
21
+ const GPUS = {
22
+ l4: { machineType: 'g2-standard-8', acceleratorType: 'NVIDIA_L4' },
23
+ t4: { machineType: 'n1-standard-8', acceleratorType: 'NVIDIA_TESLA_T4' },
24
+ a100: { machineType: 'a2-highgpu-1g', acceleratorType: 'NVIDIA_TESLA_A100' },
25
+ };
26
+ // Google's prebuilt PyTorch training image. Check it against Google's current list on the first run (model-registry.md §5).
27
+ const DEFAULT_IMAGE = 'us-docker.pkg.dev/vertex-ai/training/pytorch-gpu.2-1.py310:latest';
28
+ const DEFAULT_REGION = 'us-central1';
29
+ const DEFAULT_HOURS = 12;
30
+ const CODE_EXCLUDE = '(^|/)(\\.git|node_modules|__pycache__|\\.venv|venv)(/|$)';
31
+
32
+ const jobsFile = () => path.join(process.env.PLAIFUL_HOME || path.join(os.homedir(), '.plaiful'), 'train-jobs.json');
33
+ function readJobs() { try { return JSON.parse(fs.readFileSync(jobsFile(), 'utf8')); } catch { return []; } }
34
+ function writeJobs(jobs) { fs.mkdirSync(path.dirname(jobsFile()), { recursive: true }); fs.writeFileSync(jobsFile(), `${JSON.stringify(jobs, null, 2)}\n`); }
35
+
36
+ /** A name gcloud and the bucket accept: lowercase letters, digits and dashes. */
37
+ const slug = (s) => String(s).toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40);
38
+ const stamp = (d = new Date()) => d.toISOString().replace(/[-:]/g, '').replace(/\..*/, '').replace('T', '-').toLowerCase();
39
+
40
+ /** The job's settings → { project, region, bucket, image } (bucket without gs://). */
41
+ function settings(flags, repoConfig) {
42
+ const t = (repoConfig && repoConfig.training) || {};
43
+ const s = {
44
+ project: flags.project || t.project || process.env.PLAIFUL_GCP_PROJECT,
45
+ region: flags.region || t.region || process.env.PLAIFUL_GCP_REGION || DEFAULT_REGION,
46
+ bucket: String(flags.bucket || t.bucket || process.env.PLAIFUL_GCP_BUCKET || '').replace(/^gs:\/\//, '').replace(/\/+$/, ''),
47
+ image: flags.image || t.image || DEFAULT_IMAGE,
48
+ };
49
+ return s;
50
+ }
51
+
52
+ /** The shell script the container runs: copy the code out of the bucket, install, run the entry. */
53
+ function jobScript({ bucket, id, entry }) {
54
+ const root = `/gcs/${bucket}`;
55
+ const cmd = entry.replace(/\{data\}/g, `${root}/data`).replace(/\{out\}/g, `${root}/jobs/${id}/out`);
56
+ return [
57
+ 'set -euo pipefail',
58
+ `mkdir -p /tmp/code && cp -r ${root}/jobs/${id}/code/. /tmp/code/`,
59
+ 'cd /tmp/code',
60
+ 'if [ -f requirements.txt ]; then pip install -q -r requirements.txt; fi',
61
+ 'mkdir -p "$PLAIFUL_OUT"',
62
+ 'nvidia-smi --query-gpu=name,memory.total --format=csv || true',
63
+ cmd,
64
+ ].join('\n');
65
+ }
66
+
67
+ /** The Vertex AI CustomJobSpec (gcloud ai custom-jobs create --config). */
68
+ function jobSpec({ bucket, id, entry, gpu, machine, image, hours, task }) {
69
+ const g = GPUS[gpu];
70
+ const root = `/gcs/${bucket}`;
71
+ return {
72
+ workerPoolSpecs: [{
73
+ machineSpec: { machineType: machine || g.machineType, acceleratorType: g.acceleratorType, acceleratorCount: 1 },
74
+ replicaCount: 1,
75
+ diskSpec: { bootDiskType: 'pd-ssd', bootDiskSizeGb: 100 },
76
+ containerSpec: {
77
+ imageUri: image,
78
+ command: ['bash', '-c'],
79
+ args: [jobScript({ bucket, id, entry })],
80
+ env: [
81
+ { name: 'PLAIFUL_JOB', value: id },
82
+ { name: 'PLAIFUL_TASK', value: task },
83
+ { name: 'PLAIFUL_DATA', value: `${root}/data` },
84
+ { name: 'PLAIFUL_OUT', value: `${root}/jobs/${id}/out` },
85
+ ],
86
+ },
87
+ }],
88
+ scheduling: { timeout: `${Math.round(hours * 3600)}s` },
89
+ };
90
+ }
91
+
92
+ function makeTrain({ UsageError, ApiError, repo, models }) {
93
+ /** Run gcloud, capturing its output. → stdout; throws on failure with gcloud's own message. */
94
+ function gcloud(ctx, args, { json = false, inherit = false } = {}) {
95
+ const argv = [...args, ...(json ? ['--format=json'] : [])];
96
+ if (inherit) {
97
+ const r = ctx.io.exec('gcloud', argv, { inherit: true });
98
+ if (r.status !== 0) throw new ApiError(0, { code: 'gcloud', error: `gcloud ${args.slice(0, 3).join(' ')} failed (exit ${r.status})` });
99
+ return '';
100
+ }
101
+ const r = ctx.io.exec('gcloud', argv);
102
+ if (r.status !== 0) {
103
+ const msg = String(r.stderr || r.stdout || '').trim().split('\n').filter((l) => /ERROR|error/.test(l)).slice(-2).join(' ') || `exit ${r.status}`;
104
+ throw new ApiError(0, { code: 'gcloud', error: `gcloud ${args.slice(0, 3).join(' ')} failed: ${msg}${/auth|login|credential/i.test(msg) ? ' (run `gcloud auth login`)' : ''}` });
105
+ }
106
+ return json ? JSON.parse(r.stdout || 'null') : r.stdout;
107
+ }
108
+
109
+ function need(s) {
110
+ if (!s.project) throw new UsageError('which Google Cloud project? --project, or "training": { "project": … } in .plaiful.json');
111
+ if (!s.bucket) throw new UsageError('which bucket for job files? --bucket, or "training": { "bucket": … } in .plaiful.json (in the job\'s region)');
112
+ }
113
+
114
+ /** `plaiful models train check`: is this machine and project ready to run a job? */
115
+ async function check(ctx) {
116
+ const r = repo(ctx, { required: false });
117
+ const s = settings(ctx.flags, r.found && r.found.config);
118
+ const lines = [];
119
+ const ok = (good, text) => { lines.push(`${good ? '✓' : '✗'} ${text}`); return good; };
120
+ let fine = true;
121
+ const ver = ctx.io.exec('gcloud', ['--version']);
122
+ fine = ok(ver.status === 0, ver.status === 0 ? `gcloud: ${String(ver.stdout).split('\n')[0]}` : 'gcloud isn\'t installed (https://cloud.google.com/sdk)') && fine;
123
+ if (ver.status === 0) {
124
+ const acct = ctx.io.exec('gcloud', ['auth', 'print-access-token', '--quiet']);
125
+ fine = ok(acct.status === 0, acct.status === 0 ? 'signed in to Google Cloud' : 'not signed in: run `gcloud auth login`') && fine;
126
+ fine = ok(!!s.project, s.project ? `project: ${s.project}` : 'no project set (--project or .plaiful.json training.project)') && fine;
127
+ fine = ok(!!s.bucket, s.bucket ? `bucket: gs://${s.bucket} (region ${s.region})` : 'no bucket set (--bucket or .plaiful.json training.bucket)') && fine;
128
+ if (acct.status === 0 && s.project) {
129
+ const api = ctx.io.exec('gcloud', ['services', 'list', '--enabled', `--project=${s.project}`, '--filter=config.name=aiplatform.googleapis.com', '--format=value(config.name)']);
130
+ fine = ok(api.status === 0 && /aiplatform/.test(api.stdout), api.status === 0 && /aiplatform/.test(api.stdout) ? 'Vertex AI API enabled' : `Vertex AI API not enabled: gcloud services enable aiplatform.googleapis.com --project=${s.project}`) && fine;
131
+ if (s.bucket) {
132
+ const b = ctx.io.exec('gcloud', ['storage', 'buckets', 'describe', `gs://${s.bucket}`, `--project=${s.project}`, '--format=value(location)']);
133
+ fine = ok(b.status === 0, b.status === 0 ? `bucket found (${String(b.stdout).trim().toLowerCase()})` : `bucket gs://${s.bucket} not found: gcloud storage buckets create gs://${s.bucket} --project=${s.project} --location=${s.region} --uniform-bucket-level-access`) && fine;
134
+ }
135
+ }
136
+ }
137
+ lines.push(` image: ${s.image}`);
138
+ lines.push(' GPU quota (Vertex AI custom training, per region) is in the console: IAM & Admin → Quotas, "custom model training"');
139
+ ctx.out(lines.join('\n'), { success: fine, settings: s });
140
+ return fine ? 0 : 1;
141
+ }
142
+
143
+ /** `plaiful models train submit <task> --code <dir> --entry "<command>" [--data a,b] [--snapshot task/snap] [--gpu l4]` */
144
+ async function submit(ctx, [task]) {
145
+ if (!task) throw new UsageError('plaiful models train submit <task> --code <folder> --entry "python train.py --data {data} --out {out}" [--data <folders>] [--gpu l4|t4|a100] [--hours 12]');
146
+ const r = repo(ctx, { required: false });
147
+ const s = settings(ctx.flags, r.found && r.found.config);
148
+ need(s);
149
+ const gpu = String(ctx.flags.gpu || 'l4').toLowerCase();
150
+ if (!GPUS[gpu]) throw new UsageError(`--gpu must be ${Object.keys(GPUS).join(', ')}`);
151
+ const hours = ctx.flags.hours ? Number(ctx.flags.hours) : DEFAULT_HOURS;
152
+ if (!(hours > 0 && hours <= 72)) throw new UsageError('--hours must be between 0 and 72 (the job is stopped after it)');
153
+ if (!ctx.flags.code || !ctx.flags.entry) throw new UsageError('--code <folder> and --entry "<command>" are required ({data} and {out} are filled in)');
154
+ const code = path.resolve(ctx.io.cwd(), ctx.flags.code);
155
+ if (!fs.existsSync(code) || !fs.statSync(code).isDirectory()) throw new UsageError(`${code} isn't a folder`);
156
+ const id = `${slug(ctx.flags.name || task)}-${stamp()}`;
157
+
158
+ // Data: local folders, and registry snapshots pulled to a local cache first. Each lands in gs://<bucket>/data/<name>/.
159
+ const data = [];
160
+ for (const d of String(ctx.flags.data || '').split(',').map((x) => x.trim()).filter(Boolean)) {
161
+ const abs = path.resolve(ctx.io.cwd(), d);
162
+ if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) throw new UsageError(`--data ${abs} isn't a folder`);
163
+ data.push({ local: abs, name: slug(path.basename(abs)) || 'data' });
164
+ }
165
+ for (const snap of String(ctx.flags.snapshot || '').split(',').map((x) => x.trim()).filter(Boolean)) {
166
+ const [st, sid] = snap.split('/');
167
+ if (!st || !sid) throw new UsageError('--snapshot is <task>/<snapshot>, e.g. voice.radio/voicedata-cast');
168
+ const cache = path.join(process.env.PLAIFUL_HOME || path.join(os.homedir(), '.plaiful'), 'cache', 'datasets', st, sid);
169
+ await models.dataPull({ ...ctx, flags: { ...ctx.flags, out: cache } }, [st, sid]);
170
+ data.push({ local: cache, name: `${slug(st)}-${slug(sid)}` });
171
+ }
172
+
173
+ ctx.io.stderr.write(`Copying the code to gs://${s.bucket}/jobs/${id}/code …\n`);
174
+ gcloud(ctx, ['storage', 'rsync', code, `gs://${s.bucket}/jobs/${id}/code`, '--recursive', `--exclude=${CODE_EXCLUDE}`, `--project=${s.project}`], { inherit: true });
175
+ for (const d of data) {
176
+ ctx.io.stderr.write(`Syncing ${d.local} → gs://${s.bucket}/data/${d.name} (only what changed) …\n`);
177
+ gcloud(ctx, ['storage', 'rsync', d.local, `gs://${s.bucket}/data/${d.name}`, '--recursive', `--project=${s.project}`], { inherit: true });
178
+ }
179
+
180
+ // gcloud reads the config as YAML, and JSON is YAML.
181
+ const spec = jobSpec({ bucket: s.bucket, id, entry: ctx.flags.entry, gpu, machine: ctx.flags.machine, image: s.image, hours, task });
182
+ const cfg = path.join(os.tmpdir(), `plaiful-job-${id}.json`);
183
+ fs.writeFileSync(cfg, JSON.stringify(spec, null, 2));
184
+ let job;
185
+ try {
186
+ job = gcloud(ctx, ['ai', 'custom-jobs', 'create', `--project=${s.project}`, `--region=${s.region}`, `--display-name=${id}`, `--config=${cfg}`,
187
+ `--labels=plaiful-job=${id},plaiful-task=${slug(task)},plaiful-gpu=${gpu}`], { json: true });
188
+ } finally { try { fs.unlinkSync(cfg); } catch { /* temp file */ } }
189
+ const record = { id, name: job && job.name, task, gpu, machine: spec.workerPoolSpecs[0].machineSpec.machineType, hours, project: s.project, region: s.region, bucket: s.bucket, data: data.map((d) => d.name), submitted_at: new Date().toISOString() };
190
+ writeJobs([...readJobs(), record]);
191
+ ctx.out([
192
+ `Submitted ${id} on one ${gpu.toUpperCase()} (${record.machine}), stopped after ${hours} h at most.`,
193
+ ` output: gs://${s.bucket}/jobs/${id}/out`,
194
+ ` plaiful models train status ${id} · plaiful models train logs ${id} · plaiful models train fetch ${id}`,
195
+ ].join('\n'), { success: true, job: record });
196
+ return 0;
197
+ }
198
+
199
+ function findJob(id) {
200
+ const jobs = readJobs();
201
+ const j = id ? jobs.find((x) => x.id === id) : jobs[jobs.length - 1];
202
+ if (!j) throw new UsageError(id ? `no job ${id} submitted from this machine (plaiful models train list)` : 'no jobs submitted from this machine yet');
203
+ return j;
204
+ }
205
+
206
+ /** Vertex AI's view of a job → { state, started, ended, gpu_hours, error } */
207
+ function describe(ctx, j) {
208
+ const d = gcloud(ctx, ['ai', 'custom-jobs', 'describe', j.name, `--region=${j.region}`, `--project=${j.project}`], { json: true });
209
+ const start = d.startTime ? Date.parse(d.startTime) : null;
210
+ const end = d.endTime ? Date.parse(d.endTime) : (start ? Date.now() : null);
211
+ return { state: String(d.state || '').replace(/^JOB_STATE_/, '').toLowerCase(), created: d.createTime, started: d.startTime || null, ended: d.endTime || null, gpu_hours: start ? Math.round(((end - start) / 3.6e6) * 100) / 100 : 0, error: d.error && d.error.message };
212
+ }
213
+
214
+ async function status(ctx, [id]) {
215
+ const j = findJob(id);
216
+ const d = describe(ctx, j);
217
+ ctx.out([
218
+ `${j.id}: ${d.state} (${j.gpu.toUpperCase()}, ${j.task})`,
219
+ ` submitted ${j.submitted_at}${d.started ? `, started ${d.started}` : ', waiting for a GPU'}${d.ended ? `, ended ${d.ended}` : ''}`,
220
+ ` GPU time so far: ${d.gpu_hours} h (billed to ${j.project}; Billing → Reports, label plaiful-job=${j.id})`,
221
+ ...(d.error ? [` error: ${d.error}`] : []),
222
+ ].join('\n'), { success: true, job: j, ...d });
223
+ return 0;
224
+ }
225
+
226
+ async function list(ctx) {
227
+ const jobs = readJobs();
228
+ if (!jobs.length) { ctx.out('No training jobs submitted from this machine yet.', { success: true, jobs: [] }); return 0; }
229
+ const rows = jobs.slice(-20).map((j) => { let d = {}; try { d = describe(ctx, j); } catch (e) { d = { state: `unknown (${e.message})` }; } return { ...j, ...d }; });
230
+ ctx.out(rows.map((j) => `${j.id} ${j.state} ${j.gpu} ${j.gpu_hours != null ? `${j.gpu_hours} GPU-h` : ''} ${j.task}`).join('\n'), { success: true, jobs: rows });
231
+ return 0;
232
+ }
233
+
234
+ async function logs(ctx, [id]) {
235
+ const j = findJob(id);
236
+ gcloud(ctx, ['ai', 'custom-jobs', 'stream-logs', j.name, `--region=${j.region}`, `--project=${j.project}`], { inherit: true });
237
+ return 0;
238
+ }
239
+
240
+ async function fetch(ctx, [id]) {
241
+ const j = findJob(id);
242
+ const out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('model-runs', j.id));
243
+ fs.mkdirSync(out, { recursive: true });
244
+ gcloud(ctx, ['storage', 'rsync', `gs://${j.bucket}/jobs/${j.id}/out`, out, '--recursive', `--project=${j.project}`], { inherit: true });
245
+ ctx.out(`${j.id}'s output is in ${out}. Card it, then plaiful models publish <card.json>.`, { success: true, out });
246
+ return 0;
247
+ }
248
+
249
+ async function cancel(ctx, [id]) {
250
+ const j = findJob(id);
251
+ if (!id) throw new UsageError('cancel which job? plaiful models train cancel <job>');
252
+ gcloud(ctx, ['ai', 'custom-jobs', 'cancel', j.name, `--region=${j.region}`, `--project=${j.project}`]);
253
+ ctx.out(`Asked Vertex AI to cancel ${j.id}.`, { success: true });
254
+ return 0;
255
+ }
256
+
257
+ return { check, submit, status, list, logs, fetch, cancel };
258
+ }
259
+
260
+ module.exports = { makeTrain, jobSpec, jobScript, settings, GPUS, DEFAULT_IMAGE, slug };