@plaiful/cli 0.10.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/package.json +1 -1
- package/src/cli.js +328 -7
- package/src/models.js +284 -0
- package/src/train.js +260 -0
package/README.md
CHANGED
|
@@ -37,6 +37,7 @@ plaiful login
|
|
|
37
37
|
| Get a cinematic script for a world event | `plaiful events script write <eventId> --wait`, then `plaiful events script export <eventId> --fountain` |
|
|
38
38
|
| Animate characters: a set of sprite sheets (idle, attacks, hurt, poses, a walk sheet) per record | `plaiful animation sets`, then `plaiful animation run <set> <agent> --all` |
|
|
39
39
|
| Publish your OVID declaration | `plaiful ovid check`, then `plaiful ovid publish` |
|
|
40
|
+
| Tell HERMES your studio's strategy | `plaiful hermes strategy check`, then `plaiful hermes strategy publish` (your studio's admins) |
|
|
40
41
|
| Deploy and run worker agents | `plaiful agents deploy`, `plaiful agents run <name>` |
|
|
41
42
|
|
|
42
43
|
Every load starts as a dry run: it shows what would change, and changes nothing until you add `--apply`.
|
|
@@ -72,6 +73,9 @@ plaiful seed personas # THESPIS personas and situations from resources/p
|
|
|
72
73
|
plaiful status # what OVID and THESPIS have in dev: counts per store, the clock, the declaration
|
|
73
74
|
plaiful assets # per agent: records, and how many have art made / pending / failed / none yet
|
|
74
75
|
plaiful assets crew-portrait --state failed # one agent's records: state, why a render failed, where the files are
|
|
76
|
+
plaiful assets battlefield --json # also: any step that failed ({step}_step_error), and a part's source_crop and depth_min / depth_max
|
|
77
|
+
plaiful assets regenerate battlefield --state step_failed --steps depth # re-run a step (or, without --steps, re-render: a new picture)
|
|
78
|
+
plaiful assets regenerate building --state no_model --model # a 3D model from each record's picture (a 2d+3d Art Style; Regenerate → 3D model)
|
|
75
79
|
plaiful quests # the quests OVID wrote: steps, and what each pays (--track, --series); quests show <id>
|
|
76
80
|
plaiful lore # the lore sets (lore show <setId>: a set's snippets)
|
|
77
81
|
plaiful world # the world's states and events in story order, and the clock
|
|
@@ -98,8 +102,12 @@ plaiful records import crew-portrait # dry run: records you wrote (reso
|
|
|
98
102
|
# into the agent's table; --apply writes them, --no-render skips the art
|
|
99
103
|
plaiful lists # the product's SSM lists
|
|
100
104
|
plaiful lists set shop_type tavern blacksmith healer # write one (lists get shop_type reads it)
|
|
105
|
+
plaiful midas check # check docs/plaiful/roadforge-midas-declaration.json (publish: plaiful midas publish; show)
|
|
106
|
+
plaiful midas health # MIDAS's analysis, per kind (balance: what's flagged; proposals; assess: run it now)
|
|
107
|
+
plaiful midas wallet <uid> # a player's balances and history (wallet "platform"); midas ledger check <file>: a report's dry run
|
|
101
108
|
plaiful midas pricing # what each quest pays, in your reward kinds (MIDAS); delivered in the quest's rewards
|
|
102
109
|
plaiful midas pricing set scrap --base 20 --per-effort 10 # price a kind (unset <kind>, weight <step> <n>, apply [file])
|
|
110
|
+
plaiful hermes strategy check # check docs/plaiful/hermes-strategy.json (publish: your studio's admins; no word: what HERMES has)
|
|
103
111
|
plaiful upload ovid-map-biomes-tiles.json ovid-map-kingdoms-tiles.json ovid-map-colors.json # map tiles → map-data/
|
|
104
112
|
plaiful whoami # who you are (or which key), and your products
|
|
105
113
|
plaiful logout
|
|
@@ -144,3 +152,7 @@ same screen shows the product's activity: everything the CLI and its keys did.
|
|
|
144
152
|
npm as a member of the `plaiful` org.
|
|
145
153
|
- **Tests:** the end-to-end tests run this CLI against the partner API handler:
|
|
146
154
|
`lambda/functions/partner-api/__tests__/partner-api.test.js`.
|
|
155
|
+
- **The model registry (`plaiful models`, staff only):** publish release cards with their files, push training data,
|
|
156
|
+
and run GPU training on Google Cloud (`models train`, with your own `gcloud` sign-in). Guide:
|
|
157
|
+
`docs/design/model-registry.md`; card format: `docs/design/model-card.md`. Tests:
|
|
158
|
+
`lambda/functions/partner-api/__tests__/models.test.js` and `models-cli.test.js`.
|
package/package.json
CHANGED
package/src/cli.js
CHANGED
|
@@ -77,14 +77,34 @@ const HELP = `plaiful ${version}: work on your Plaiful product from your game re
|
|
|
77
77
|
plaiful records import <agent> [file] dry run: records you wrote, into the agent's table (default file:
|
|
78
78
|
resources/records/<product>/<agent>.json; matched on the agent's duplicate field)
|
|
79
79
|
--apply write them; --no-render don't make new records' art or sound
|
|
80
|
+
plaiful midas check [file] check your MIDAS declaration: your economy's kinds, wallet, items, schedule
|
|
81
|
+
(default: <docsDir>/<product>-midas-declaration.json)
|
|
82
|
+
plaiful midas publish [file] check it and publish it
|
|
83
|
+
plaiful midas show the declaration MIDAS has, with its defaults filled in
|
|
84
|
+
plaiful midas health MIDAS's analysis of your economy: per kind, what quests make available (and with a
|
|
85
|
+
wallet, what players earn and spend)
|
|
86
|
+
plaiful midas balance what MIDAS flagged, and why (--all for everything it checked)
|
|
87
|
+
plaiful midas proposals what MIDAS proposes (--status pending|approved|applied|dismissed)
|
|
88
|
+
plaiful midas assess run MIDAS's analysis now
|
|
89
|
+
plaiful midas wallet <player> one player's balances and recent earning / spending (wallet "platform")
|
|
90
|
+
plaiful midas ledger check <file> would your game's report be accepted (wallet "game"): a dry run
|
|
80
91
|
plaiful midas pricing what MIDAS pays per quest, in your reward kinds (delivered in each quest's rewards)
|
|
81
92
|
plaiful midas pricing set <kind> --base <n> --per-effort <n> price a kind (either option alone keeps the other)
|
|
82
93
|
plaiful midas pricing unset <kind> stop pricing a kind
|
|
83
94
|
plaiful midas pricing weight <step> <n> how much a quest step counts (combat, gather, explore, ...)
|
|
84
95
|
plaiful midas pricing apply [file] replace it all from a file (default:
|
|
85
96
|
resources/midas/<product>/quest-pricing.json; { kinds, objective_weights, ... })
|
|
97
|
+
plaiful hermes strategy your studio's strategy, the one HERMES judges the market against
|
|
98
|
+
plaiful hermes strategy check [file] check it (default: <docsDir>/hermes-strategy.json)
|
|
99
|
+
plaiful hermes strategy publish [file] check it and publish it as a new version (the studio's admins)
|
|
86
100
|
plaiful assets per agent: records, and how many have their art made, pending, failed or none
|
|
87
101
|
plaiful assets <agent> its records: state, why a render failed, where the files are
|
|
102
|
+
plaiful assets regenerate <agent> [uuids...] re-render records with the table's Art Style, as the Library's
|
|
103
|
+
Regenerate does: a new picture (--all, or --state failed|none|made|step_failed)
|
|
104
|
+
--steps depth[,normal] | --steps all only re-run those steps on the current picture (Regenerate → Steps)
|
|
105
|
+
--model a 3D model from each record's current picture, with the table's 2d+3d Art Style
|
|
106
|
+
(Regenerate → 3D model; --state no_model: every record with a picture and no model;
|
|
107
|
+
at most 50 a run, each a paid build)
|
|
88
108
|
--state made|pending|failed|none --limit <n> (default 50)
|
|
89
109
|
plaiful animation sets the animation sets you can use (a set = a list of animation states, one sprite
|
|
90
110
|
sheet each, run together on a character): the built-ins, Plaiful's and yours
|
|
@@ -94,6 +114,22 @@ const HELP = `plaiful ${version}: work on your Plaiful product from your game re
|
|
|
94
114
|
plaiful animation sets delete <id> delete one of your sets
|
|
95
115
|
plaiful animation run <set> <agent> [uuids...] run a set on an agent's records (--all: every record with its art
|
|
96
116
|
made; --style <sprite style>, else the set's; --force: make them again)
|
|
117
|
+
plaiful models the model registry (Plaiful staff for now): every release, its status and licence
|
|
118
|
+
plaiful models check <card|dir> check release cards and their files on disk (a dry run: nothing uploads)
|
|
119
|
+
plaiful models publish <card|dir> upload each card's files (checked against its sha256s) and publish the card;
|
|
120
|
+
a published release never changes, only its card can be corrected
|
|
121
|
+
(--files <folder> if the card has no archive_path; owner from .plaiful.json)
|
|
122
|
+
plaiful models show <task> <release> one release: card, files, status history
|
|
123
|
+
plaiful models status <task> <release> active|candidate|retired (--note "…"; retired, never deleted)
|
|
124
|
+
plaiful models pull <task> <release> download a release, every sha256 checked (--out <dir>, --role shipped)
|
|
125
|
+
plaiful models data push <task> <snapshot> <folder> store a training-data folder as a sealed snapshot
|
|
126
|
+
(--exclude <regex>, --source "…", --licence "…", --note "…", --releases a,b)
|
|
127
|
+
plaiful models data [list] the training-data snapshots; models data pull <task> <snapshot> [--out <dir>]
|
|
128
|
+
plaiful models train check is this machine ready to train on Google Cloud (gcloud, project, bucket, API)
|
|
129
|
+
plaiful models train submit <task> --code <folder> --entry "python train.py --data {data} --out {out}"
|
|
130
|
+
run it on a Google Cloud GPU (--gpu l4|t4|a100, --hours 12, --data <folders>,
|
|
131
|
+
--snapshot <task>/<snapshot>, --project, --region, --bucket, --image, --name)
|
|
132
|
+
plaiful models train list|status|logs|fetch|cancel [job] follow a job; fetch brings its output home (--out)
|
|
97
133
|
plaiful upload <files...> upload map tiles into your dev assets bucket (map-data/; --dest map-data)
|
|
98
134
|
plaiful clone [product] [dir] clone the game repo, write .env.development, docs pull, npm install, open Zed
|
|
99
135
|
--no-install --no-open
|
|
@@ -124,8 +160,8 @@ function parse(argv) {
|
|
|
124
160
|
const args = [];
|
|
125
161
|
for (let i = 0; i < argv.length; i += 1) {
|
|
126
162
|
const a = argv[i];
|
|
127
|
-
if (['--json', '--no-browser', '--help', '-h', '--version', '--apply', '--reset', '--rows', '--no-install', '--no-open', '--copies', '--no-render', '--wait', '--force', '--fountain', '--vo', '--all', '--retire', '--known'].includes(a)) flags[a.replace(/^-+/, '')] = true;
|
|
128
|
-
else if (['--product', '--api', '--dir', '--dest', '--count', '--base', '--per-effort', '--track', '--series', '--state', '--limit', '--section', '--since', '--note', '--out', '--style', '--aspect', '--frame', '--title', '--description', '--category', '--order', '--directives', '--narrative', '--hint', '--rarity', '--statement', '--name', '--reason', '--type', '--homeland', '--scope'].includes(a)) {
|
|
163
|
+
if (['--json', '--no-browser', '--help', '-h', '--version', '--apply', '--reset', '--rows', '--no-install', '--no-open', '--copies', '--no-render', '--wait', '--force', '--fountain', '--vo', '--all', '--retire', '--known', '--model'].includes(a)) flags[a.replace(/^-+/, '')] = true;
|
|
164
|
+
else if (['--product', '--api', '--dir', '--dest', '--count', '--base', '--per-effort', '--track', '--series', '--state', '--limit', '--section', '--since', '--note', '--out', '--style', '--aspect', '--frame', '--title', '--description', '--category', '--order', '--directives', '--narrative', '--hint', '--rarity', '--statement', '--name', '--reason', '--type', '--homeland', '--scope', '--status', '--steps', '--files', '--role', '--exclude', '--source', '--licence', '--releases', '--code', '--entry', '--data', '--snapshot', '--gpu', '--machine', '--hours', '--image', '--project', '--region', '--bucket'].includes(a)) {
|
|
129
165
|
if (argv[i + 1] === undefined) throw new UsageError(`${a} needs a value`);
|
|
130
166
|
flags[a.slice(2)] = argv[i += 1];
|
|
131
167
|
} else if (a.startsWith('--')) throw new UsageError(`unknown option ${a}`);
|
|
@@ -451,9 +487,9 @@ function ignoreClientConfig(docsDir) {
|
|
|
451
487
|
return { added, tracked };
|
|
452
488
|
}
|
|
453
489
|
|
|
454
|
-
function readDeclaration(ctx, r, file) {
|
|
455
|
-
const target = path.resolve(ctx.io.cwd(), file || path.join(r.docsDir, `${r.product}-
|
|
456
|
-
if (!fs.existsSync(target)) throw new UsageError(`no declaration at ${target} (write it first:
|
|
490
|
+
function readDeclaration(ctx, r, file, agent = 'ovid') {
|
|
491
|
+
const target = path.resolve(ctx.io.cwd(), file || path.join(r.docsDir, `${r.product}-${agent}-declaration.json`));
|
|
492
|
+
if (!fs.existsSync(target)) throw new UsageError(`no declaration at ${target} (write it first: ${agent}-declaration.md)`);
|
|
457
493
|
try { return { target, declaration: JSON.parse(fs.readFileSync(target, 'utf8')) }; } catch (e) {
|
|
458
494
|
throw new UsageError(`${target} isn't valid JSON: ${e.message}`);
|
|
459
495
|
}
|
|
@@ -495,6 +531,51 @@ async function ovidPublish(ctx, [file]) {
|
|
|
495
531
|
return 0;
|
|
496
532
|
}
|
|
497
533
|
|
|
534
|
+
// ── The studio's HERMES strategy (hermes-strategy.md) ──
|
|
535
|
+
function readStrategy(ctx, r, file) {
|
|
536
|
+
const target = path.resolve(ctx.io.cwd(), file || path.join(r.docsDir, 'hermes-strategy.json'));
|
|
537
|
+
if (!fs.existsSync(target)) throw new UsageError(`no strategy at ${target} (write it first: hermes-strategy.md)`);
|
|
538
|
+
try { return { target, strategy: JSON.parse(fs.readFileSync(target, 'utf8')) }; } catch (e) {
|
|
539
|
+
throw new UsageError(`${target} isn't valid JSON: ${e.message}`);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
const strategyErrors = (data) => (data.errors || []).map((e) => ` ✗ ${e.path}: ${e.message}`);
|
|
544
|
+
|
|
545
|
+
async function hermesStrategyCheck(ctx, [file]) {
|
|
546
|
+
const r = repo(ctx);
|
|
547
|
+
const { target, strategy } = readStrategy(ctx, r, file);
|
|
548
|
+
const { data } = await ctx.call('POST', `/partner/${encodeURIComponent(r.product)}/hermes/strategy/check`, { strategy });
|
|
549
|
+
const head = data.success
|
|
550
|
+
? `${path.basename(target)} is valid. Publish it with: plaiful hermes strategy publish`
|
|
551
|
+
: `${path.basename(target)} has ${(data.errors || []).length} error(s); fix them and check again.`;
|
|
552
|
+
ctx.out([head, ...strategyErrors(data)].join('\n'), data);
|
|
553
|
+
return data.success ? 0 : 1;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
async function hermesStrategyPublish(ctx, [file]) {
|
|
557
|
+
const r = repo(ctx);
|
|
558
|
+
const { target, strategy } = readStrategy(ctx, r, file);
|
|
559
|
+
const { data } = await ctx.call('PUT', `/partner/${encodeURIComponent(r.product)}/hermes/strategy`, { strategy });
|
|
560
|
+
if (!data.published) {
|
|
561
|
+
ctx.out([`Not published: ${data.error || `${path.basename(target)} has ${(data.errors || []).length} error(s)`}`, ...strategyErrors(data)].join('\n'), data);
|
|
562
|
+
return 1;
|
|
563
|
+
}
|
|
564
|
+
ctx.out(`Published ${path.basename(target)} as version ${data.strategy.version} of studio ${data.studio_id}'s strategy; HERMES uses it within about 5 minutes.`, data);
|
|
565
|
+
return 0;
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
async function hermesStrategyShow(ctx, rest) {
|
|
569
|
+
if (rest.length) throw new UsageError(`plaiful hermes strategy, or strategy check | publish (not "${rest[0]}")`);
|
|
570
|
+
const r = repo(ctx);
|
|
571
|
+
const { data } = await ctx.call('GET', `/partner/${encodeURIComponent(r.product)}/hermes/strategy`);
|
|
572
|
+
if (ctx.flags.json) { ctx.out(data); return 0; }
|
|
573
|
+
const studio = data.studio_name || data.studio_id;
|
|
574
|
+
if (!data.strategy) ctx.out(`${studio} has no strategy yet: HERMES says so, and judges the market without one. Write <docsDir>/hermes-strategy.json, then plaiful hermes strategy publish.`);
|
|
575
|
+
else ctx.out(`${studio}'s strategy, version ${data.strategy.version} (${data.strategy.updated_at} by ${data.strategy.updated_by}):\n${JSON.stringify(data.strategy.strategy, null, 2)}`);
|
|
576
|
+
return 0;
|
|
577
|
+
}
|
|
578
|
+
|
|
498
579
|
async function ovidShow(ctx) {
|
|
499
580
|
const r = repo(ctx);
|
|
500
581
|
const { data } = await ctx.call('GET', `/partner/${encodeURIComponent(r.product)}/ovid/declaration`);
|
|
@@ -504,6 +585,166 @@ async function ovidShow(ctx) {
|
|
|
504
585
|
return 0;
|
|
505
586
|
}
|
|
506
587
|
|
|
588
|
+
// ── The MIDAS declaration (docs/agents/midas/multi-product-economy-plan.md §3) ──────────────────────────────────
|
|
589
|
+
|
|
590
|
+
const midasDeclRoute = (r, rest = '') => `/partner/${encodeURIComponent(r.product)}/midas/declaration${rest}`;
|
|
591
|
+
|
|
592
|
+
/** "scrap, ore, xp (only builds up)" */
|
|
593
|
+
const kindsText = (kinds) => (kinds || []).map((k) => `${k.label && k.label !== k.id ? `${k.label} (${k.id})` : k.id}${k.spendable === false ? ', only builds up' : ''}`).join('; ') || 'none';
|
|
594
|
+
|
|
595
|
+
function midasEffectiveText(e) {
|
|
596
|
+
if (!e) return [];
|
|
597
|
+
const items = e.items && e.items.enabled ? `on: ${e.items.collections.map((c) => c.collection).join(', ')}` : 'off';
|
|
598
|
+
return [
|
|
599
|
+
` kinds ${kindsText(e.kinds)}${e.kinds_from === 'ovid' ? ' (from your OVID reward kinds)' : ''}`,
|
|
600
|
+
` wallet ${e.wallet}`,
|
|
601
|
+
` items ${items}`,
|
|
602
|
+
` schedule ${e.analysis && e.analysis.schedule === 'daily' ? 'daily' : 'off (Assess Now)'}`,
|
|
603
|
+
];
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
async function midasCheck(ctx, [file]) {
|
|
607
|
+
const r = repo(ctx);
|
|
608
|
+
const { target, declaration } = readDeclaration(ctx, r, file, 'midas');
|
|
609
|
+
const { data } = await ctx.call('POST', midasDeclRoute(r, '/check'), { declaration });
|
|
610
|
+
const head = data.ok
|
|
611
|
+
? `${path.basename(target)} is valid${(data.warnings || []).length ? `, with ${data.warnings.length} warning(s)` : ''}. Publish it with: plaiful midas publish`
|
|
612
|
+
: `${path.basename(target)} has ${data.errors.length} error(s); fix them and check again.`;
|
|
613
|
+
ctx.out([head, ...(data.ok ? midasEffectiveText(data.effective) : []), ...findingsText(data)].join('\n'), data);
|
|
614
|
+
return data.ok ? 0 : 1;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
async function midasPublish(ctx, [file]) {
|
|
618
|
+
const r = repo(ctx);
|
|
619
|
+
const { target, declaration } = readDeclaration(ctx, r, file, 'midas');
|
|
620
|
+
const { data } = await ctx.call('PUT', midasDeclRoute(r), { declaration });
|
|
621
|
+
if (!data.published) {
|
|
622
|
+
ctx.out([`Not published: ${data.error || `${path.basename(target)} has ${(data.errors || []).length} error(s)`}`, ...findingsText(data)].join('\n'), data);
|
|
623
|
+
return 1;
|
|
624
|
+
}
|
|
625
|
+
ctx.out([`Published ${path.basename(target)} for ${r.product}.`, ...midasEffectiveText(data.effective), ...findingsText(data),
|
|
626
|
+
'Check it took: plaiful midas show'].join('\n'), data);
|
|
627
|
+
return 0;
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
async function midasShow(ctx) {
|
|
631
|
+
const r = repo(ctx);
|
|
632
|
+
const { data } = await ctx.call('GET', midasDeclRoute(r));
|
|
633
|
+
if (ctx.flags.json) { ctx.out(data); return 0; }
|
|
634
|
+
const head = data.declaration
|
|
635
|
+
? `${r.product}'s MIDAS declaration, published ${data.updated_at || '(date unknown)'}${data.published_by ? ` by ${data.published_by}` : ''}:`
|
|
636
|
+
: `${r.product} has no MIDAS declaration yet; MIDAS uses its defaults:`;
|
|
637
|
+
ctx.out([head, ...midasEffectiveText(data.effective), ...findingsText({ warnings: data.warnings })].join('\n'));
|
|
638
|
+
return 0;
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
// ── MIDAS's analysis of the game (multi-product-economy-plan.md §9.1; the plaiful-economy-review skill) ──────────
|
|
642
|
+
|
|
643
|
+
const num = (v) => (v == null ? '—' : Number(v).toLocaleString('en-US', { maximumFractionDigits: 1 }));
|
|
644
|
+
const HEALTH_WORDS = { green: 'healthy', amber: 'needs a look', red: 'out of balance' };
|
|
645
|
+
|
|
646
|
+
/** What a flag means, in plain words (the CMS's Balance tab says the same). */
|
|
647
|
+
function flagWords(f) {
|
|
648
|
+
const k = f.kind || '';
|
|
649
|
+
switch (f.metric) {
|
|
650
|
+
case 'unpaidKind': return f.priced ? `no quest pays ${k} (it's priced: check OVID's rewards.types)` : `no quest pays ${k}: it has no quest price`;
|
|
651
|
+
case 'neverSpent': return `${num(f.value)} ${k} earned, none spent`;
|
|
652
|
+
case 'earnExceedsSpend': return `players earn ${f.value}× the ${k} they spend`;
|
|
653
|
+
case 'spendExceedsEarn': return `players spend ${f.value}× the ${k} they earn`;
|
|
654
|
+
case 'noReward': return 'pays nothing (no step carries effort)';
|
|
655
|
+
case 'rewardOutlier': return `pays ${num(f.value)} ${k}, over ${Math.round(f.value / f.median)}× the median quest (${num(f.median)})`;
|
|
656
|
+
case 'itemValueOutlier': return `worth ${num(f.value)} ${k}, ${f.direction} its group's median (${num(f.median)})`;
|
|
657
|
+
case 'unaffordable': return `costs ${num(f.value)} typical quests' ${k}`;
|
|
658
|
+
case 'trivialCost': return `costs under a quarter of one typical quest's ${k}`;
|
|
659
|
+
default: return f.metric;
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
async function midasHealth(ctx) {
|
|
664
|
+
const r = repo(ctx);
|
|
665
|
+
const { data } = await ctx.call('GET', `/partner/${encodeURIComponent(r.product)}/midas/health`);
|
|
666
|
+
if (ctx.flags.json) { ctx.out(data); return 0; }
|
|
667
|
+
const s = data.summary;
|
|
668
|
+
if (!s || data.error) { ctx.out(`${r.product} has no economy analysis yet. Run one: plaiful midas assess`); return 0; }
|
|
669
|
+
const lines = [`${r.product}'s economy (dev): ${HEALTH_WORDS[s.overallHealth] || s.overallHealth}. ${s.flaggedItems} of ${s.totalItems} flagged; analyzed ${data.analyzedAt || '?'}`];
|
|
670
|
+
if (!s.priced) lines.push(' ! quests aren\'t priced yet: plaiful midas pricing set <kind> --base <n> --per-effort <n>');
|
|
671
|
+
for (const k of s.kinds || []) {
|
|
672
|
+
lines.push(` ${k.label}${k.label !== k.id ? ` (${k.id})` : ''}${k.spendable ? '' : ', builds up'}: ${num(k.supply)} from quests, median ${num(k.medianReward)} per quest, ${k.questsPaying} quests pay it`
|
|
673
|
+
+ (s.wallet && s.wallet !== 'none' ? `; last 30 days earned ${num(k.earned)}, spent ${num(k.spent)}` : '')
|
|
674
|
+
+ (k.held !== undefined ? `; players hold ${num(k.held)} (${k.holders} of ${k.players})` : ''));
|
|
675
|
+
}
|
|
676
|
+
lines.push('Details: plaiful midas balance; proposals: plaiful midas proposals');
|
|
677
|
+
ctx.out(lines.join('\n'), data);
|
|
678
|
+
return 0;
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
async function midasBalance(ctx) {
|
|
682
|
+
const r = repo(ctx);
|
|
683
|
+
const { data } = await ctx.call('GET', `/partner/${encodeURIComponent(r.product)}/midas/balance`);
|
|
684
|
+
if (ctx.flags.json) { ctx.out(data); return 0; }
|
|
685
|
+
const items = (data.items || []).filter((si) => ctx.flags.all || !si.healthy);
|
|
686
|
+
if (!data.items || data.error) { ctx.out(`${r.product} has no economy analysis yet. Run one: plaiful midas assess`); return 0; }
|
|
687
|
+
const lines = [`${r.product}: ${items.length} ${ctx.flags.all ? 'checked' : 'flagged'} of ${data.items.length}${ctx.flags.all ? '' : ' (--all for everything)'}`];
|
|
688
|
+
for (const si of items) {
|
|
689
|
+
const it = si.item;
|
|
690
|
+
lines.push(` ${si.healthy ? '·' : si.severity === 'high' ? '✗' : '!'} ${it.name} [${it.category}]${si.flags.length ? ` ${si.flags.map(flagWords).join('; ')}` : ''}`);
|
|
691
|
+
}
|
|
692
|
+
ctx.out(lines.join('\n'), data);
|
|
693
|
+
return 0;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
async function midasProposals(ctx) {
|
|
697
|
+
const r = repo(ctx);
|
|
698
|
+
const qs = ctx.flags.status ? `?status=${encodeURIComponent(ctx.flags.status)}` : '';
|
|
699
|
+
const { data } = await ctx.call('GET', `/partner/${encodeURIComponent(r.product)}/midas/proposals${qs}`);
|
|
700
|
+
if (ctx.flags.json) { ctx.out(data); return 0; }
|
|
701
|
+
const list = data.proposals || [];
|
|
702
|
+
const lines = [`${r.product}: ${list.length} ${data.status || 'pending'} proposal(s)${list.length ? '. An editor approves them on the Economy page in the CMS (Proposals).' : ''}`];
|
|
703
|
+
for (const p of list) {
|
|
704
|
+
const change = p.proposedValue == null ? 'for the team to act on'
|
|
705
|
+
: `${p.field || 'value'}: ${p.currentValue == null ? 'none' : p.currentValue} → ${p.proposedValue}`;
|
|
706
|
+
lines.push(` [${p.severity}] ${p.type.replace(/_/g, ' ').toLowerCase()} · ${p.targetName}: ${change}`);
|
|
707
|
+
if (p.rationale) lines.push(` ${p.rationale}`);
|
|
708
|
+
}
|
|
709
|
+
ctx.out(lines.join('\n'), data);
|
|
710
|
+
return 0;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
/** `plaiful midas wallet <player>`: one player's balances and recent earning and spending (wallet "platform"). */
|
|
714
|
+
async function midasWallet(ctx, [player]) {
|
|
715
|
+
const r = repo(ctx);
|
|
716
|
+
if (!player) throw new UsageError('which player? plaiful midas wallet <player id> (their Firebase uid)');
|
|
717
|
+
const { data } = await ctx.call('GET', `/partner/${encodeURIComponent(r.product)}/midas/wallet/${encodeURIComponent(player)}`);
|
|
718
|
+
if (ctx.flags.json) { ctx.out(data); return 0; }
|
|
719
|
+
const label = Object.fromEntries((data.kinds || []).map((k) => [k.id, k.label]));
|
|
720
|
+
const lines = [`${player} in ${r.product} (dev): ${Object.entries(data.balances || {}).map(([k, v]) => `${num(v)} ${label[k] || k}`).join(' · ') || 'nothing yet'}`];
|
|
721
|
+
for (const e of data.history || []) lines.push(` ${e.ts} ${e.delta > 0 ? '+' : ''}${num(e.delta)} ${label[e.kind] || e.kind} ${e.reason}${e.source_id ? ` (${e.source_id})` : ''}`);
|
|
722
|
+
ctx.out(lines.join('\n'), data);
|
|
723
|
+
return 0;
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
/** `plaiful midas ledger check <file>`: would POST /game/wallet/report accept these entries (wallet "game")? */
|
|
727
|
+
async function midasLedger(ctx, [sub, file]) {
|
|
728
|
+
if (sub !== 'check' || !file) throw new UsageError('plaiful midas ledger check <file.json> (a list of { kind, delta, reason, idempotency_key }, or { "entries": [...] })');
|
|
729
|
+
const r = repo(ctx);
|
|
730
|
+
const { json } = readJsonFile(ctx, file);
|
|
731
|
+
const entries = Array.isArray(json) ? json : json && json.entries;
|
|
732
|
+
const { data } = await ctx.call('POST', `/partner/${encodeURIComponent(r.product)}/midas/ledger/check`, { entries });
|
|
733
|
+
if (ctx.flags.json) { ctx.out(data); return data.success ? 0 : 1; }
|
|
734
|
+
const lines = [data.success ? `${data.count} entries: all valid. Your game sends them with POST /game/wallet/report.` : `${(data.problems || []).length} problem(s):`];
|
|
735
|
+
for (const p of data.problems || []) lines.push(` ✗ ${p}`);
|
|
736
|
+
if (data.note) lines.push(` ! ${data.note}`);
|
|
737
|
+
ctx.out(lines.join('\n'), data);
|
|
738
|
+
return data.success ? 0 : 1;
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
async function midasAssess(ctx) {
|
|
742
|
+
const r = repo(ctx);
|
|
743
|
+
const { data } = await ctx.call('POST', `/partner/${encodeURIComponent(r.product)}/midas/assess`, {});
|
|
744
|
+
ctx.out(`MIDAS is analyzing ${r.product}'s economy (about a minute). Then: plaiful midas health`, data);
|
|
745
|
+
return 0;
|
|
746
|
+
}
|
|
747
|
+
|
|
507
748
|
// ── Seeding (phase 2) ──────────────────────────────────────────────────────────────────────────────────────────
|
|
508
749
|
|
|
509
750
|
const STORES = ['world', 'lore', 'map', 'personas'];
|
|
@@ -1035,9 +1276,35 @@ async function eventsStoryboard(ctx, [sub, eventId, version]) {
|
|
|
1035
1276
|
|
|
1036
1277
|
// ── What the agents made ────────────────────────────────────────────────────────────────────────────────────────
|
|
1037
1278
|
|
|
1038
|
-
const countsText = (a) => `${a.records} records: ${a.made} made, ${a.pending} pending, ${a.failed} failed, ${a.none} none yet${a.truncated ? ' (first 2000)' : ''}`;
|
|
1279
|
+
const countsText = (a) => `${a.records} records: ${a.made} made, ${a.pending} pending, ${a.failed} failed, ${a.none} none yet${a.step_failed ? `; ${a.step_failed} with a failed step` : ''}${a.truncated ? ' (first 2000)' : ''}`;
|
|
1039
1280
|
|
|
1040
|
-
async function
|
|
1281
|
+
async function assetsRegenerate(ctx, [agent, ...uuids]) {
|
|
1282
|
+
if (!agent) throw new UsageError('which agent? plaiful assets regenerate <agent> [uuids...] (or --all, --state <state>)');
|
|
1283
|
+
const r = repo(ctx);
|
|
1284
|
+
const body = {};
|
|
1285
|
+
if (uuids.length) body.records = uuids;
|
|
1286
|
+
if (ctx.flags.all) body.all = true;
|
|
1287
|
+
if (ctx.flags.state) body.state = String(ctx.flags.state);
|
|
1288
|
+
if (!body.records && !body.all && !body.state) throw new UsageError('which records? list their uuids, or --all, or --state failed|none|made|step_failed (with --model, also no_model)');
|
|
1289
|
+
if (ctx.flags.model) {
|
|
1290
|
+
if (ctx.flags.steps !== undefined) throw new UsageError('--model and --steps are separate runs: one or the other');
|
|
1291
|
+
body.model = true;
|
|
1292
|
+
}
|
|
1293
|
+
if (ctx.flags.steps !== undefined) {
|
|
1294
|
+
if (ctx.flags.steps === true) throw new UsageError('--steps takes step names (--steps depth,normal) or all');
|
|
1295
|
+
body.steps = String(ctx.flags.steps) === 'all' ? 'all' : String(ctx.flags.steps).split(',').map((x) => x.trim()).filter(Boolean);
|
|
1296
|
+
}
|
|
1297
|
+
const { data } = await ctx.call('POST', `/partner/${encodeURIComponent(r.product)}/assets/${encodeURIComponent(agent)}/regenerate`, body);
|
|
1298
|
+
const what = data.mode === 'steps' ? `the ${data.steps.join(', ')} step${data.steps.length !== 1 ? 's' : ''} of ${data.style}`
|
|
1299
|
+
: data.mode === 'model' ? `a 3D model with ${data.style}` : `a new picture in ${data.style}`;
|
|
1300
|
+
const lines = [`Started ${what} for ${data.records} ${agent} record${data.records !== 1 ? 's' : ''}. They ${data.mode === 'model' ? 'build' : 'render'} in the background: plaiful assets ${agent} shows each one as it lands.`];
|
|
1301
|
+
for (const s of data.skipped || []) lines.push(` skipped ${s.uuid}: ${s.reason}`);
|
|
1302
|
+
ctx.out(lines.join('\n'), data);
|
|
1303
|
+
return data.records ? 0 : 1;
|
|
1304
|
+
}
|
|
1305
|
+
|
|
1306
|
+
async function assets(ctx, [agent, ...rest]) {
|
|
1307
|
+
if (agent === 'regenerate') return assetsRegenerate(ctx, rest);
|
|
1041
1308
|
const r = repo(ctx);
|
|
1042
1309
|
const base = `/partner/${encodeURIComponent(r.product)}/assets`;
|
|
1043
1310
|
if (!agent) {
|
|
@@ -1060,6 +1327,9 @@ async function assets(ctx, [agent]) {
|
|
|
1060
1327
|
const files = [rec.files && `${rec.files.base}.{${rec.files.try.join('|')}}`, ...Object.values(rec.media_files || {})].filter(Boolean);
|
|
1061
1328
|
const where = rec.state === 'made' ? files.join(' ') : '';
|
|
1062
1329
|
lines.push(` ${rec.state.padEnd(8)} ${rec.name || '(unnamed)'} ${rec.uuid}${why ? ` ${why}` : ''}${where ? ` ${where}` : ''}`);
|
|
1330
|
+
// A step that failed (a depth map): the art is made, the step's file isn't. Re-run it in the Library: Regenerate → Steps.
|
|
1331
|
+
for (const [step, err] of Object.entries(rec.step_errors || {})) lines.push(` ! step ${step} failed: ${err}`);
|
|
1332
|
+
if (rec.depth_match_error) lines.push(` ! depth not matched to the overview: ${rec.depth_match_error}`);
|
|
1063
1333
|
}
|
|
1064
1334
|
ctx.out(lines.join('\n'), data);
|
|
1065
1335
|
return 0;
|
|
@@ -1220,6 +1490,24 @@ async function upload(ctx, files) {
|
|
|
1220
1490
|
return 0;
|
|
1221
1491
|
}
|
|
1222
1492
|
|
|
1493
|
+
// ── The model registry and training (models.js, train.js) ──────────────────────────────────────────────────────
|
|
1494
|
+
|
|
1495
|
+
const modelsCmd = require('./models').makeModels({ UsageError, ApiError, repo });
|
|
1496
|
+
const trainCmd = require('./train').makeTrain({ UsageError, ApiError, repo, models: modelsCmd });
|
|
1497
|
+
|
|
1498
|
+
function modelsData(ctx, [sub, ...rest]) {
|
|
1499
|
+
if (sub === 'push') return modelsCmd.dataPush(ctx, rest);
|
|
1500
|
+
if (sub === 'pull') return modelsCmd.dataPull(ctx, rest);
|
|
1501
|
+
if (!sub || sub === 'list') return modelsCmd.dataList(ctx, rest);
|
|
1502
|
+
throw new UsageError(`plaiful models data list | push | pull (not "${sub}")`);
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
function modelsTrain(ctx, [sub, ...rest]) {
|
|
1506
|
+
const run = { check: trainCmd.check, submit: trainCmd.submit, list: trainCmd.list, status: trainCmd.status, logs: trainCmd.logs, fetch: trainCmd.fetch, cancel: trainCmd.cancel }[sub || 'list'];
|
|
1507
|
+
if (!run) throw new UsageError(`plaiful models train check | submit | list | status | logs | fetch | cancel (not "${sub}")`);
|
|
1508
|
+
return run(ctx, rest);
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1223
1511
|
// ── Clone and .env (the New-Game Flow) ───────────────────────────────────────────────────────────────────────────
|
|
1224
1512
|
|
|
1225
1513
|
/** The game template's .env variables (plaiful-kit packages/client config.ts ENV_KEYS) ← the client config. */
|
|
@@ -1347,12 +1635,26 @@ const COMMANDS = {
|
|
|
1347
1635
|
'ovid check': (ctx, rest) => ovidCheck(ctx, rest),
|
|
1348
1636
|
'ovid publish': (ctx, rest) => ovidPublish(ctx, rest),
|
|
1349
1637
|
'ovid show': (ctx) => ovidShow(ctx),
|
|
1638
|
+
'hermes strategy': (ctx, [sub, ...rest]) => {
|
|
1639
|
+
if (sub === 'check') return hermesStrategyCheck(ctx, rest);
|
|
1640
|
+
if (sub === 'publish') return hermesStrategyPublish(ctx, rest);
|
|
1641
|
+
return hermesStrategyShow(ctx, sub ? [sub, ...rest] : []);
|
|
1642
|
+
},
|
|
1350
1643
|
seed: (ctx, [store, ...paths]) => seedStore(ctx, store, paths),
|
|
1351
1644
|
status: (ctx) => status(ctx),
|
|
1352
1645
|
lists: (ctx, rest) => lists(ctx, rest),
|
|
1353
1646
|
agents: (ctx, rest) => agents(ctx, rest),
|
|
1354
1647
|
records: (ctx, rest) => records(ctx, rest),
|
|
1355
1648
|
'midas pricing': (ctx, rest) => midasPricing(ctx, rest),
|
|
1649
|
+
'midas check': (ctx, rest) => midasCheck(ctx, rest),
|
|
1650
|
+
'midas publish': (ctx, rest) => midasPublish(ctx, rest),
|
|
1651
|
+
'midas show': (ctx) => midasShow(ctx),
|
|
1652
|
+
'midas health': (ctx) => midasHealth(ctx),
|
|
1653
|
+
'midas balance': (ctx) => midasBalance(ctx),
|
|
1654
|
+
'midas proposals': (ctx) => midasProposals(ctx),
|
|
1655
|
+
'midas assess': (ctx) => midasAssess(ctx),
|
|
1656
|
+
'midas wallet': (ctx, rest) => midasWallet(ctx, rest),
|
|
1657
|
+
'midas ledger': (ctx, rest) => midasLedger(ctx, rest),
|
|
1356
1658
|
quests: (ctx, rest) => quests(ctx, rest),
|
|
1357
1659
|
assets: (ctx, rest) => assets(ctx, rest),
|
|
1358
1660
|
lore: (ctx, rest) => lore(ctx, rest),
|
|
@@ -1364,6 +1666,15 @@ const COMMANDS = {
|
|
|
1364
1666
|
'events storyboard': (ctx, rest) => eventsStoryboard(ctx, rest),
|
|
1365
1667
|
'animation sets': (ctx, rest) => animationSets(ctx, rest),
|
|
1366
1668
|
'animation run': (ctx, rest) => animationRun(ctx, rest),
|
|
1669
|
+
'models check': (ctx, rest) => modelsCmd.publish(ctx, rest, { apply: false }),
|
|
1670
|
+
'models publish': (ctx, rest) => modelsCmd.publish(ctx, rest, { apply: true }),
|
|
1671
|
+
'models list': (ctx, rest) => modelsCmd.list(ctx, rest),
|
|
1672
|
+
'models show': (ctx, rest) => modelsCmd.show(ctx, rest),
|
|
1673
|
+
'models status': (ctx, rest) => modelsCmd.status(ctx, rest),
|
|
1674
|
+
'models pull': (ctx, rest) => modelsCmd.pull(ctx, rest),
|
|
1675
|
+
'models data': (ctx, rest) => modelsData(ctx, rest),
|
|
1676
|
+
'models train': (ctx, rest) => modelsTrain(ctx, rest),
|
|
1677
|
+
models: (ctx, rest) => modelsCmd.list(ctx, rest),
|
|
1367
1678
|
upload: (ctx, files) => upload(ctx, files),
|
|
1368
1679
|
clone: (ctx, rest) => clone(ctx, rest),
|
|
1369
1680
|
env: (ctx) => envCmd(ctx),
|
|
@@ -1390,6 +1701,16 @@ function defaultIo() {
|
|
|
1390
1701
|
return { status: r.status === null ? 1 : r.status };
|
|
1391
1702
|
} catch { return { status: 1 }; }
|
|
1392
1703
|
},
|
|
1704
|
+
// gcloud for `plaiful models train`: its output captured (or shown, with `inherit`). On Windows gcloud is a .cmd,
|
|
1705
|
+
// which needs a shell, so arguments with spaces are quoted. → { status, stdout, stderr }
|
|
1706
|
+
exec: (cmd, args, { inherit = false } = {}) => {
|
|
1707
|
+
const shell = process.platform === 'win32';
|
|
1708
|
+
const argv = shell ? args.map((a) => (/^[A-Za-z0-9_./:=@,-]+$/.test(a) ? a : `"${String(a).replace(/"/g, '\\"')}"`)) : args;
|
|
1709
|
+
try {
|
|
1710
|
+
const r = spawnSync(cmd, argv, { shell, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, stdio: inherit ? 'inherit' : 'pipe' });
|
|
1711
|
+
return { status: r.status === null ? 1 : r.status, stdout: r.stdout || '', stderr: r.stderr || '' };
|
|
1712
|
+
} catch (e) { return { status: 1, stdout: '', stderr: e.message }; }
|
|
1713
|
+
},
|
|
1393
1714
|
openUrl: (url) => {
|
|
1394
1715
|
const [cmd, args] = process.platform === 'win32' ? ['rundll32', ['url.dll,FileProtocolHandler', url]]
|
|
1395
1716
|
: process.platform === 'darwin' ? ['open', [url]] : ['xdg-open', [url]];
|
package/src/models.js
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* `plaiful models …`: the model registry from a game repo (PlaifulCMS docs/design/model-registry.md). Plaiful staff
|
|
5
|
+
* only for now. A release is a card (`<release>.card.json`, Ironheads' format or version 2) plus the files it lists;
|
|
6
|
+
* a training-data snapshot is a folder, pushed as is with a manifest of every file's size and sha256.
|
|
7
|
+
*
|
|
8
|
+
* Files go straight to storage on short-lived links the platform signs with each file's sha256, so storage refuses
|
|
9
|
+
* bytes that don't match the card. Nothing here needs AWS.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const fs = require('fs');
|
|
13
|
+
const path = require('path');
|
|
14
|
+
const crypto = require('crypto');
|
|
15
|
+
|
|
16
|
+
const BATCH = 500;
|
|
17
|
+
const CONCURRENCY = 6;
|
|
18
|
+
|
|
19
|
+
function sha256File(file) {
|
|
20
|
+
return new Promise((resolve, reject) => {
|
|
21
|
+
const h = crypto.createHash('sha256');
|
|
22
|
+
fs.createReadStream(file).on('error', reject).on('data', (d) => h.update(d)).on('end', () => resolve(h.digest('hex')));
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
async function pool(items, n, fn) {
|
|
27
|
+
let i = 0;
|
|
28
|
+
await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
|
|
29
|
+
while (i < items.length) { const k = i; i += 1; await fn(items[k], k); }
|
|
30
|
+
}));
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const mb = (b) => (b >= 1e9 ? `${(b / 1e9).toFixed(2)} GB` : b >= 1e6 ? `${(b / 1e6).toFixed(1)} MB` : `${Math.max(1, Math.round(b / 1e3))} KB`);
|
|
34
|
+
const enc = (s) => encodeURIComponent(s);
|
|
35
|
+
|
|
36
|
+
/** Every file under `dir` (relative, forward slashes), skipping names that match `exclude`. */
|
|
37
|
+
function walk(dir, exclude) {
|
|
38
|
+
const out = [];
|
|
39
|
+
const visit = (rel) => {
|
|
40
|
+
for (const ent of fs.readdirSync(path.join(dir, rel), { withFileTypes: true })) {
|
|
41
|
+
const r = rel ? `${rel}/${ent.name}` : ent.name;
|
|
42
|
+
if (exclude && exclude.test(r)) continue;
|
|
43
|
+
if (ent.isDirectory()) visit(r);
|
|
44
|
+
else if (ent.isFile()) out.push(r);
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
visit('');
|
|
48
|
+
return out.sort();
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** The card files a path names: one card, or every *.card.json under a folder. */
|
|
52
|
+
function cardFiles(target) {
|
|
53
|
+
if (fs.statSync(target).isFile()) return [target];
|
|
54
|
+
const found = [];
|
|
55
|
+
const visit = (d) => {
|
|
56
|
+
for (const ent of fs.readdirSync(d, { withFileTypes: true })) {
|
|
57
|
+
const p = path.join(d, ent.name);
|
|
58
|
+
if (ent.isDirectory()) visit(p);
|
|
59
|
+
else if (ent.name.endsWith('.card.json')) found.push(p);
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
visit(target);
|
|
63
|
+
return found.sort();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Where a card's files are on this machine: its archive folder (or, without one, where the game ships it). */
|
|
67
|
+
function filesRoot(root, card, flags) {
|
|
68
|
+
const base = flags.files || (card.files && (card.files.archive_path || card.files.shipped_path));
|
|
69
|
+
if (!base) return null;
|
|
70
|
+
return path.resolve(root, base);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** PUT one file to a signed link; the link's headers carry the sha256 storage checks. */
|
|
74
|
+
async function putFile(io, file, u) {
|
|
75
|
+
const body = fs.readFileSync(file);
|
|
76
|
+
let res;
|
|
77
|
+
try { res = await io.fetch(u.url, { method: u.method || 'PUT', headers: u.headers, body }); } catch (e) {
|
|
78
|
+
throw Object.assign(new Error(`the upload of ${u.path} didn't go through (${e.message}); run it again`), { code: 'network' });
|
|
79
|
+
}
|
|
80
|
+
if (!res.ok) {
|
|
81
|
+
const why = res.status === 400 ? 'its bytes don\'t match the sha256 (did the file change after the card was written?)' : `storage answered ${res.status}`;
|
|
82
|
+
throw Object.assign(new Error(`storage refused ${u.path}: ${why}`), { code: 'upload_failed' });
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function getFile(io, url, file, sha) {
|
|
87
|
+
const res = await io.fetch(url);
|
|
88
|
+
if (!res.ok) throw Object.assign(new Error(`couldn't download ${path.basename(file)} (${res.status})`), { code: 'download_failed' });
|
|
89
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
90
|
+
const got = crypto.createHash('sha256').update(buf).digest('hex');
|
|
91
|
+
if (sha && got !== sha) throw Object.assign(new Error(`${file} doesn't match its sha256 after download`), { code: 'checksum' });
|
|
92
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
93
|
+
fs.writeFileSync(file, buf);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function makeModels({ UsageError, ApiError, repo }) {
|
|
97
|
+
const apiFail = (code, error) => new ApiError(409, { code, error });
|
|
98
|
+
|
|
99
|
+
/** Read a card and check its files on disk against it. → { card, root, problems } */
|
|
100
|
+
async function localCheck(ctx, file) {
|
|
101
|
+
let card;
|
|
102
|
+
try { card = JSON.parse(fs.readFileSync(file, 'utf8')); } catch (e) { throw new UsageError(`${file} isn't a JSON card (${e.message})`); }
|
|
103
|
+
const r = repo(ctx, { required: false });
|
|
104
|
+
const root = filesRoot(r.root, card, ctx.flags);
|
|
105
|
+
const problems = [];
|
|
106
|
+
const items = (card.files && card.files.items) || [];
|
|
107
|
+
if (!root) problems.push('the card has no files.archive_path or shipped_path: pass --files <folder>');
|
|
108
|
+
else {
|
|
109
|
+
await pool(items, CONCURRENCY, async (f) => {
|
|
110
|
+
const p = path.join(root, f.path);
|
|
111
|
+
if (!fs.existsSync(p)) { problems.push(`${f.path}: not found under ${root}`); return; }
|
|
112
|
+
const size = fs.statSync(p).size;
|
|
113
|
+
if (size !== f.bytes) { problems.push(`${f.path}: ${size} bytes, the card says ${f.bytes}`); return; }
|
|
114
|
+
if (await sha256File(p) !== f.sha256) problems.push(`${f.path}: sha256 differs from the card`);
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
return { card, root, problems, owner: card.owner || r.product || null };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** `plaiful models check|publish <card.json | folder of cards>` */
|
|
121
|
+
async function publish(ctx, [target], { apply }) {
|
|
122
|
+
if (!target) throw new UsageError(`${apply ? 'publish' : 'check'} what? plaiful models ${apply ? 'publish' : 'check'} <card.json | folder of cards>`);
|
|
123
|
+
const abs = path.resolve(ctx.io.cwd(), target);
|
|
124
|
+
if (!fs.existsSync(abs)) throw new UsageError(`${abs} doesn't exist`);
|
|
125
|
+
const files = cardFiles(abs);
|
|
126
|
+
if (!files.length) throw new UsageError(`no *.card.json under ${abs}`);
|
|
127
|
+
const results = [];
|
|
128
|
+
let failed = 0;
|
|
129
|
+
for (const file of files) {
|
|
130
|
+
const { card, root, problems, owner } = await localCheck(ctx, file);
|
|
131
|
+
const id = `${card.task_type}/${card.release_id}`;
|
|
132
|
+
const row = { card: path.relative(ctx.io.cwd(), file), release: id, problems, warnings: [] };
|
|
133
|
+
results.push(row);
|
|
134
|
+
if (problems.length) { failed += 1; ctx.io.stderr.write(`✗ ${id}: ${problems.length} file problem(s)\n${problems.slice(0, 5).map((p) => ` ${p}`).join('\n')}\n`); continue; }
|
|
135
|
+
const route = `/partner/models/releases/${enc(card.task_type)}/${enc(card.release_id)}`;
|
|
136
|
+
let up;
|
|
137
|
+
try {
|
|
138
|
+
up = (await ctx.call('POST', `${route}/upload`, { card, owner })).data;
|
|
139
|
+
} catch (e) {
|
|
140
|
+
if (!(e instanceof ApiError) || e.status >= 500 || e.status === 401 || e.status === 403) throw e;
|
|
141
|
+
failed += 1; row.problems.push(e.message); ctx.io.stderr.write(`✗ ${id}: ${e.message}\n`); continue;
|
|
142
|
+
}
|
|
143
|
+
row.warnings = up.warnings || [];
|
|
144
|
+
if (!apply) {
|
|
145
|
+
row.would_upload = up.uploads.length;
|
|
146
|
+
row.stored = up.stored;
|
|
147
|
+
ctx.io.stderr.write(`✓ ${id}: valid; ${up.uploads.length} file(s) to upload, ${up.stored} stored${up.published ? ' (published: a card correction)' : ''}${row.warnings.length ? `; ${row.warnings.length} warning(s)` : ''}\n`);
|
|
148
|
+
continue;
|
|
149
|
+
}
|
|
150
|
+
let sent = 0;
|
|
151
|
+
await pool(up.uploads, CONCURRENCY, async (u) => { await putFile(ctx.io, path.join(root, u.path), u); sent += u.bytes; });
|
|
152
|
+
const pub = (await ctx.call('POST', `${route}/publish`, { card, owner })).data;
|
|
153
|
+
Object.assign(row, { uploaded: up.uploads.length, uploaded_bytes: sent, card_rev: pub.card_rev, corrected: pub.corrected, status: pub.status });
|
|
154
|
+
ctx.io.stderr.write(`✓ ${id}: published (card ${pub.card_rev}${pub.corrected ? ', corrected' : ''}, ${pub.status}); ${up.uploads.length} file(s), ${mb(sent)} uploaded\n`);
|
|
155
|
+
}
|
|
156
|
+
const ok = results.length - failed;
|
|
157
|
+
ctx.out(`${apply ? 'Published' : 'Checked'} ${ok} of ${results.length} release(s)${failed ? `; ${failed} with problems (above)` : ''}.${apply ? '' : ' Nothing was uploaded: plaiful models publish to upload.'}`, { success: failed === 0, results });
|
|
158
|
+
return failed ? 1 : 0;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
async function list(ctx, [task]) {
|
|
162
|
+
const { data } = await ctx.call('GET', `/partner/models/releases${task ? `?task=${enc(task)}` : ''}`);
|
|
163
|
+
const rows = data.releases;
|
|
164
|
+
const lines = rows.length ? [] : ['No releases in the registry yet: plaiful models publish <card.json>'];
|
|
165
|
+
let last = null;
|
|
166
|
+
for (const r of rows.sort((a, b) => a.task_type.localeCompare(b.task_type) || String(b.date).localeCompare(String(a.date)))) {
|
|
167
|
+
if (r.task_type !== last) { lines.push(`${r.task_type}`); last = r.task_type; }
|
|
168
|
+
lines.push(` ${r.release_id} ${r.status} ${r.date} ${mb(r.bytes)} cleared: ${r.cleared_for_other_games} (${r.owner})`);
|
|
169
|
+
}
|
|
170
|
+
ctx.out(lines.join('\n'), data);
|
|
171
|
+
return 0;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async function show(ctx, [task, release]) {
|
|
175
|
+
if (!task || !release) throw new UsageError('plaiful models show <task> <release>');
|
|
176
|
+
const { data } = await ctx.call('GET', `/partner/models/releases/${enc(task)}/${enc(release)}`);
|
|
177
|
+
const r = data.release;
|
|
178
|
+
const lines = [
|
|
179
|
+
`${r.task_type} / ${r.release_id} (${r.status}, card ${r.card_rev}, owner ${r.owner})`,
|
|
180
|
+
` ${r.name}: ${r.one_line}`,
|
|
181
|
+
` cleared for other games: ${r.cleared_for_other_games}${r.attribution.length ? `; credit: ${r.attribution.join(' · ')}` : ''}`,
|
|
182
|
+
` ${r.files} file(s), ${mb(r.bytes)} (shipped ${mb(r.shipped_bytes)}); ${r.prefix}`,
|
|
183
|
+
` published ${r.published_at} by ${r.published_by}`,
|
|
184
|
+
...(r.status_history || []).map((h) => ` ${h.at.slice(0, 10)} ${h.status} ${h.by}${h.note ? `: ${h.note}` : ''}`),
|
|
185
|
+
];
|
|
186
|
+
ctx.out(lines.join('\n'), data);
|
|
187
|
+
return 0;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
async function status(ctx, [task, release, value]) {
|
|
191
|
+
if (!task || !release || !value) throw new UsageError('plaiful models status <task> <release> active|candidate|retired [--note "…"]');
|
|
192
|
+
const { data } = await ctx.call('POST', `/partner/models/releases/${enc(task)}/${enc(release)}/status`, { status: value, note: ctx.flags.note });
|
|
193
|
+
ctx.out(data.changed ? `${task}/${release}: ${data.was} → ${data.status}. Update the card's status too, so the two agree.` : `${task}/${release} is already ${data.status}.`, data);
|
|
194
|
+
return 0;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
async function pull(ctx, [task, release]) {
|
|
198
|
+
if (!task || !release) throw new UsageError('plaiful models pull <task> <release> [--out <dir>] [--role shipped]');
|
|
199
|
+
const roles = ctx.flags.role ? String(ctx.flags.role).split(',') : undefined;
|
|
200
|
+
const { data } = await ctx.call('POST', `/partner/models/releases/${enc(task)}/${enc(release)}/download`, { roles });
|
|
201
|
+
const out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('models', task, release));
|
|
202
|
+
await pool(data.files, CONCURRENCY, (f) => getFile(ctx.io, f.url, path.join(out, f.path), f.sha256));
|
|
203
|
+
fs.writeFileSync(path.join(out, `${release}.card.json`), `${JSON.stringify(data.card, null, 1)}\n`);
|
|
204
|
+
ctx.out(`Downloaded ${data.files.length} file(s) of ${task}/${release} (${data.status}) to ${out}, every sha256 checked.`, { success: true, out, files: data.files.length });
|
|
205
|
+
return 0;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// ── Training-data snapshots ──────────────────────────────────────────────────────────────────────────────────
|
|
209
|
+
|
|
210
|
+
async function dataPush(ctx, [task, snapshot, dir]) {
|
|
211
|
+
if (!task || !snapshot || !dir) throw new UsageError('plaiful models data push <task> <snapshot> <folder> [--exclude <regex>] [--source "…"] [--licence "…"] [--note "…"] [--releases a,b]');
|
|
212
|
+
const abs = path.resolve(ctx.io.cwd(), dir);
|
|
213
|
+
if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) throw new UsageError(`${abs} isn't a folder`);
|
|
214
|
+
let exclude = null;
|
|
215
|
+
if (ctx.flags.exclude) { try { exclude = new RegExp(ctx.flags.exclude); } catch (e) { throw new UsageError(`--exclude isn't a regular expression (${e.message})`); } }
|
|
216
|
+
const rels = walk(abs, exclude);
|
|
217
|
+
if (!rels.length) throw new UsageError(`no files under ${abs}`);
|
|
218
|
+
ctx.io.stderr.write(`Hashing ${rels.length} file(s) under ${abs}…\n`);
|
|
219
|
+
const entries = new Array(rels.length);
|
|
220
|
+
await pool(rels, CONCURRENCY, async (rel, i) => {
|
|
221
|
+
const p = path.join(abs, rel);
|
|
222
|
+
entries[i] = { path: rel, bytes: fs.statSync(p).size, sha256: await sha256File(p) };
|
|
223
|
+
});
|
|
224
|
+
const total = entries.reduce((n, e) => n + e.bytes, 0);
|
|
225
|
+
const manifest = `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`;
|
|
226
|
+
const mBytes = Buffer.byteLength(manifest);
|
|
227
|
+
const mSha = crypto.createHash('sha256').update(manifest).digest('hex');
|
|
228
|
+
const route = `/partner/models/datasets/${enc(task)}/${enc(snapshot)}`;
|
|
229
|
+
let uploaded = 0;
|
|
230
|
+
let stored = 0;
|
|
231
|
+
let sent = 0;
|
|
232
|
+
for (let i = 0; i < entries.length; i += BATCH) {
|
|
233
|
+
const batch = entries.slice(i, i + BATCH);
|
|
234
|
+
const last = i + BATCH >= entries.length;
|
|
235
|
+
const { data } = await ctx.call('POST', `${route}/upload`, { files: batch, ...(last ? { manifest: { bytes: mBytes, sha256: mSha } } : {}) });
|
|
236
|
+
stored += data.stored;
|
|
237
|
+
await pool(data.uploads, CONCURRENCY, async (u) => { await putFile(ctx.io, path.join(abs, u.path), u); uploaded += 1; sent += u.bytes; });
|
|
238
|
+
if (last) {
|
|
239
|
+
const res = await ctx.io.fetch(data.manifest.url, { method: 'PUT', headers: data.manifest.headers, body: Buffer.from(manifest) });
|
|
240
|
+
if (!res.ok) throw apiFail('upload_failed', `storage refused the manifest (${res.status}); run the push again`);
|
|
241
|
+
}
|
|
242
|
+
ctx.io.stderr.write(` ${Math.min(i + BATCH, entries.length)} / ${entries.length} (${uploaded} uploaded, ${stored} already stored)\n`);
|
|
243
|
+
}
|
|
244
|
+
const about = { source: ctx.flags.source || abs, licence: ctx.flags.licence || '', note: ctx.flags.note || '', releases: ctx.flags.releases ? String(ctx.flags.releases).split(',') : [] };
|
|
245
|
+
const { data } = await ctx.call('POST', `${route}/finish`, { about });
|
|
246
|
+
ctx.out(`${task}/${snapshot}: sealed, ${data.files} file(s), ${mb(data.bytes)} (${uploaded} uploaded, ${mb(sent)}; ${stored} were already stored). Backed up to the Management account by replication.`, { success: true, ...data, uploaded, total_bytes: total });
|
|
247
|
+
return 0;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
async function dataList(ctx, [task]) {
|
|
251
|
+
const { data } = await ctx.call('GET', `/partner/models/datasets${task ? `?task=${enc(task)}` : ''}`);
|
|
252
|
+
const lines = data.datasets.length ? data.datasets.map((d) => `${d.task_type}/${d.snapshot} ${d.state} ${d.files != null ? `${d.files} file(s), ${mb(d.bytes)}` : '(push not finished)'}${d.about && d.about.releases && d.about.releases.length ? ` for ${d.about.releases.join(', ')}` : ''}`)
|
|
253
|
+
: ['No training-data snapshots yet: plaiful models data push <task> <snapshot> <folder>'];
|
|
254
|
+
ctx.out(lines.join('\n'), data);
|
|
255
|
+
return 0;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
async function dataPull(ctx, [task, snapshot]) {
|
|
259
|
+
if (!task || !snapshot) throw new UsageError('plaiful models data pull <task> <snapshot> [--out <dir>]');
|
|
260
|
+
const out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('datasets', task, snapshot));
|
|
261
|
+
const route = `/partner/models/datasets/${enc(task)}/${enc(snapshot)}/download`;
|
|
262
|
+
let after = 0;
|
|
263
|
+
let n = 0;
|
|
264
|
+
let skipped = 0;
|
|
265
|
+
for (;;) {
|
|
266
|
+
const { data } = await ctx.call('POST', route, { after });
|
|
267
|
+
await pool(data.files, CONCURRENCY, async (f) => {
|
|
268
|
+
const p = path.join(out, f.path);
|
|
269
|
+
if (fs.existsSync(p) && fs.statSync(p).size === f.bytes && await sha256File(p) === f.sha256) { skipped += 1; return; }
|
|
270
|
+
await getFile(ctx.io, f.url, p, f.sha256);
|
|
271
|
+
n += 1;
|
|
272
|
+
});
|
|
273
|
+
ctx.io.stderr.write(` ${data.after + data.files.length} / ${data.total}\n`);
|
|
274
|
+
if (data.next == null) break;
|
|
275
|
+
after = data.next;
|
|
276
|
+
}
|
|
277
|
+
ctx.out(`${task}/${snapshot} → ${out}: ${n} downloaded, ${skipped} already here, every sha256 checked.`, { success: true, out, downloaded: n, skipped });
|
|
278
|
+
return 0;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
return { publish, list, show, status, pull, dataPush, dataList, dataPull };
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
module.exports = { makeModels, walk, sha256File, cardFiles, filesRoot, mb };
|
package/src/train.js
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* `plaiful models train …`: run a training job on a Google Cloud GPU (Vertex AI custom training) instead of the
|
|
5
|
+
* developer's own machine, so the GPU time draws on Plaiful's Google Cloud credits (PlaifulCMS
|
|
6
|
+
* docs/design/model-registry.md §5). Plaiful staff only for now, and it runs on this machine's `gcloud` sign-in: the
|
|
7
|
+
* platform holds no Google credentials.
|
|
8
|
+
*
|
|
9
|
+
* A job: the code folder and the data folders are copied to the project's bucket (data is cached across jobs), Vertex
|
|
10
|
+
* AI runs the entry command on one GPU with the bucket mounted at /gcs/<bucket>, the job writes to {out}, and
|
|
11
|
+
* `train fetch` brings the output home, ready to card and `plaiful models publish`.
|
|
12
|
+
*
|
|
13
|
+
* Settings, from flags, else .plaiful.json "training": { project, region, bucket, image }, else PLAIFUL_GCP_* env.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const fs = require('fs');
|
|
17
|
+
const os = require('os');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
|
|
20
|
+
/** One GPU per job, by name: the machine Vertex AI pairs with it. */
|
|
21
|
+
const GPUS = {
|
|
22
|
+
l4: { machineType: 'g2-standard-8', acceleratorType: 'NVIDIA_L4' },
|
|
23
|
+
t4: { machineType: 'n1-standard-8', acceleratorType: 'NVIDIA_TESLA_T4' },
|
|
24
|
+
a100: { machineType: 'a2-highgpu-1g', acceleratorType: 'NVIDIA_TESLA_A100' },
|
|
25
|
+
};
|
|
26
|
+
// Google's prebuilt PyTorch training image. Check it against Google's current list on the first run (model-registry.md §5).
|
|
27
|
+
const DEFAULT_IMAGE = 'us-docker.pkg.dev/vertex-ai/training/pytorch-gpu.2-1.py310:latest';
|
|
28
|
+
const DEFAULT_REGION = 'us-central1';
|
|
29
|
+
const DEFAULT_HOURS = 12;
|
|
30
|
+
const CODE_EXCLUDE = '(^|/)(\\.git|node_modules|__pycache__|\\.venv|venv)(/|$)';
|
|
31
|
+
|
|
32
|
+
const jobsFile = () => path.join(process.env.PLAIFUL_HOME || path.join(os.homedir(), '.plaiful'), 'train-jobs.json');
|
|
33
|
+
function readJobs() { try { return JSON.parse(fs.readFileSync(jobsFile(), 'utf8')); } catch { return []; } }
|
|
34
|
+
function writeJobs(jobs) { fs.mkdirSync(path.dirname(jobsFile()), { recursive: true }); fs.writeFileSync(jobsFile(), `${JSON.stringify(jobs, null, 2)}\n`); }
|
|
35
|
+
|
|
36
|
+
/** A name gcloud and the bucket accept: lowercase letters, digits and dashes. */
|
|
37
|
+
const slug = (s) => String(s).toLowerCase().replace(/[^a-z0-9-]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 40);
|
|
38
|
+
const stamp = (d = new Date()) => d.toISOString().replace(/[-:]/g, '').replace(/\..*/, '').replace('T', '-').toLowerCase();
|
|
39
|
+
|
|
40
|
+
/** The job's settings → { project, region, bucket, image } (bucket without gs://). */
|
|
41
|
+
function settings(flags, repoConfig) {
|
|
42
|
+
const t = (repoConfig && repoConfig.training) || {};
|
|
43
|
+
const s = {
|
|
44
|
+
project: flags.project || t.project || process.env.PLAIFUL_GCP_PROJECT,
|
|
45
|
+
region: flags.region || t.region || process.env.PLAIFUL_GCP_REGION || DEFAULT_REGION,
|
|
46
|
+
bucket: String(flags.bucket || t.bucket || process.env.PLAIFUL_GCP_BUCKET || '').replace(/^gs:\/\//, '').replace(/\/+$/, ''),
|
|
47
|
+
image: flags.image || t.image || DEFAULT_IMAGE,
|
|
48
|
+
};
|
|
49
|
+
return s;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** The shell script the container runs: copy the code out of the bucket, install, run the entry. */
|
|
53
|
+
function jobScript({ bucket, id, entry }) {
|
|
54
|
+
const root = `/gcs/${bucket}`;
|
|
55
|
+
const cmd = entry.replace(/\{data\}/g, `${root}/data`).replace(/\{out\}/g, `${root}/jobs/${id}/out`);
|
|
56
|
+
return [
|
|
57
|
+
'set -euo pipefail',
|
|
58
|
+
`mkdir -p /tmp/code && cp -r ${root}/jobs/${id}/code/. /tmp/code/`,
|
|
59
|
+
'cd /tmp/code',
|
|
60
|
+
'if [ -f requirements.txt ]; then pip install -q -r requirements.txt; fi',
|
|
61
|
+
'mkdir -p "$PLAIFUL_OUT"',
|
|
62
|
+
'nvidia-smi --query-gpu=name,memory.total --format=csv || true',
|
|
63
|
+
cmd,
|
|
64
|
+
].join('\n');
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** The Vertex AI CustomJobSpec (gcloud ai custom-jobs create --config). */
|
|
68
|
+
function jobSpec({ bucket, id, entry, gpu, machine, image, hours, task }) {
|
|
69
|
+
const g = GPUS[gpu];
|
|
70
|
+
const root = `/gcs/${bucket}`;
|
|
71
|
+
return {
|
|
72
|
+
workerPoolSpecs: [{
|
|
73
|
+
machineSpec: { machineType: machine || g.machineType, acceleratorType: g.acceleratorType, acceleratorCount: 1 },
|
|
74
|
+
replicaCount: 1,
|
|
75
|
+
diskSpec: { bootDiskType: 'pd-ssd', bootDiskSizeGb: 100 },
|
|
76
|
+
containerSpec: {
|
|
77
|
+
imageUri: image,
|
|
78
|
+
command: ['bash', '-c'],
|
|
79
|
+
args: [jobScript({ bucket, id, entry })],
|
|
80
|
+
env: [
|
|
81
|
+
{ name: 'PLAIFUL_JOB', value: id },
|
|
82
|
+
{ name: 'PLAIFUL_TASK', value: task },
|
|
83
|
+
{ name: 'PLAIFUL_DATA', value: `${root}/data` },
|
|
84
|
+
{ name: 'PLAIFUL_OUT', value: `${root}/jobs/${id}/out` },
|
|
85
|
+
],
|
|
86
|
+
},
|
|
87
|
+
}],
|
|
88
|
+
scheduling: { timeout: `${Math.round(hours * 3600)}s` },
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function makeTrain({ UsageError, ApiError, repo, models }) {
|
|
93
|
+
/** Run gcloud, capturing its output. → stdout; throws on failure with gcloud's own message. */
|
|
94
|
+
function gcloud(ctx, args, { json = false, inherit = false } = {}) {
|
|
95
|
+
const argv = [...args, ...(json ? ['--format=json'] : [])];
|
|
96
|
+
if (inherit) {
|
|
97
|
+
const r = ctx.io.exec('gcloud', argv, { inherit: true });
|
|
98
|
+
if (r.status !== 0) throw new ApiError(0, { code: 'gcloud', error: `gcloud ${args.slice(0, 3).join(' ')} failed (exit ${r.status})` });
|
|
99
|
+
return '';
|
|
100
|
+
}
|
|
101
|
+
const r = ctx.io.exec('gcloud', argv);
|
|
102
|
+
if (r.status !== 0) {
|
|
103
|
+
const msg = String(r.stderr || r.stdout || '').trim().split('\n').filter((l) => /ERROR|error/.test(l)).slice(-2).join(' ') || `exit ${r.status}`;
|
|
104
|
+
throw new ApiError(0, { code: 'gcloud', error: `gcloud ${args.slice(0, 3).join(' ')} failed: ${msg}${/auth|login|credential/i.test(msg) ? ' (run `gcloud auth login`)' : ''}` });
|
|
105
|
+
}
|
|
106
|
+
return json ? JSON.parse(r.stdout || 'null') : r.stdout;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function need(s) {
|
|
110
|
+
if (!s.project) throw new UsageError('which Google Cloud project? --project, or "training": { "project": … } in .plaiful.json');
|
|
111
|
+
if (!s.bucket) throw new UsageError('which bucket for job files? --bucket, or "training": { "bucket": … } in .plaiful.json (in the job\'s region)');
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** `plaiful models train check`: is this machine and project ready to run a job? */
|
|
115
|
+
async function check(ctx) {
|
|
116
|
+
const r = repo(ctx, { required: false });
|
|
117
|
+
const s = settings(ctx.flags, r.found && r.found.config);
|
|
118
|
+
const lines = [];
|
|
119
|
+
const ok = (good, text) => { lines.push(`${good ? '✓' : '✗'} ${text}`); return good; };
|
|
120
|
+
let fine = true;
|
|
121
|
+
const ver = ctx.io.exec('gcloud', ['--version']);
|
|
122
|
+
fine = ok(ver.status === 0, ver.status === 0 ? `gcloud: ${String(ver.stdout).split('\n')[0]}` : 'gcloud isn\'t installed (https://cloud.google.com/sdk)') && fine;
|
|
123
|
+
if (ver.status === 0) {
|
|
124
|
+
const acct = ctx.io.exec('gcloud', ['auth', 'print-access-token', '--quiet']);
|
|
125
|
+
fine = ok(acct.status === 0, acct.status === 0 ? 'signed in to Google Cloud' : 'not signed in: run `gcloud auth login`') && fine;
|
|
126
|
+
fine = ok(!!s.project, s.project ? `project: ${s.project}` : 'no project set (--project or .plaiful.json training.project)') && fine;
|
|
127
|
+
fine = ok(!!s.bucket, s.bucket ? `bucket: gs://${s.bucket} (region ${s.region})` : 'no bucket set (--bucket or .plaiful.json training.bucket)') && fine;
|
|
128
|
+
if (acct.status === 0 && s.project) {
|
|
129
|
+
const api = ctx.io.exec('gcloud', ['services', 'list', '--enabled', `--project=${s.project}`, '--filter=config.name=aiplatform.googleapis.com', '--format=value(config.name)']);
|
|
130
|
+
fine = ok(api.status === 0 && /aiplatform/.test(api.stdout), api.status === 0 && /aiplatform/.test(api.stdout) ? 'Vertex AI API enabled' : `Vertex AI API not enabled: gcloud services enable aiplatform.googleapis.com --project=${s.project}`) && fine;
|
|
131
|
+
if (s.bucket) {
|
|
132
|
+
const b = ctx.io.exec('gcloud', ['storage', 'buckets', 'describe', `gs://${s.bucket}`, `--project=${s.project}`, '--format=value(location)']);
|
|
133
|
+
fine = ok(b.status === 0, b.status === 0 ? `bucket found (${String(b.stdout).trim().toLowerCase()})` : `bucket gs://${s.bucket} not found: gcloud storage buckets create gs://${s.bucket} --project=${s.project} --location=${s.region} --uniform-bucket-level-access`) && fine;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
lines.push(` image: ${s.image}`);
|
|
138
|
+
lines.push(' GPU quota (Vertex AI custom training, per region) is in the console: IAM & Admin → Quotas, "custom model training"');
|
|
139
|
+
ctx.out(lines.join('\n'), { success: fine, settings: s });
|
|
140
|
+
return fine ? 0 : 1;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** `plaiful models train submit <task> --code <dir> --entry "<command>" [--data a,b] [--snapshot task/snap] [--gpu l4]` */
|
|
144
|
+
async function submit(ctx, [task]) {
|
|
145
|
+
if (!task) throw new UsageError('plaiful models train submit <task> --code <folder> --entry "python train.py --data {data} --out {out}" [--data <folders>] [--gpu l4|t4|a100] [--hours 12]');
|
|
146
|
+
const r = repo(ctx, { required: false });
|
|
147
|
+
const s = settings(ctx.flags, r.found && r.found.config);
|
|
148
|
+
need(s);
|
|
149
|
+
const gpu = String(ctx.flags.gpu || 'l4').toLowerCase();
|
|
150
|
+
if (!GPUS[gpu]) throw new UsageError(`--gpu must be ${Object.keys(GPUS).join(', ')}`);
|
|
151
|
+
const hours = ctx.flags.hours ? Number(ctx.flags.hours) : DEFAULT_HOURS;
|
|
152
|
+
if (!(hours > 0 && hours <= 72)) throw new UsageError('--hours must be between 0 and 72 (the job is stopped after it)');
|
|
153
|
+
if (!ctx.flags.code || !ctx.flags.entry) throw new UsageError('--code <folder> and --entry "<command>" are required ({data} and {out} are filled in)');
|
|
154
|
+
const code = path.resolve(ctx.io.cwd(), ctx.flags.code);
|
|
155
|
+
if (!fs.existsSync(code) || !fs.statSync(code).isDirectory()) throw new UsageError(`${code} isn't a folder`);
|
|
156
|
+
const id = `${slug(ctx.flags.name || task)}-${stamp()}`;
|
|
157
|
+
|
|
158
|
+
// Data: local folders, and registry snapshots pulled to a local cache first. Each lands in gs://<bucket>/data/<name>/.
|
|
159
|
+
const data = [];
|
|
160
|
+
for (const d of String(ctx.flags.data || '').split(',').map((x) => x.trim()).filter(Boolean)) {
|
|
161
|
+
const abs = path.resolve(ctx.io.cwd(), d);
|
|
162
|
+
if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) throw new UsageError(`--data ${abs} isn't a folder`);
|
|
163
|
+
data.push({ local: abs, name: slug(path.basename(abs)) || 'data' });
|
|
164
|
+
}
|
|
165
|
+
for (const snap of String(ctx.flags.snapshot || '').split(',').map((x) => x.trim()).filter(Boolean)) {
|
|
166
|
+
const [st, sid] = snap.split('/');
|
|
167
|
+
if (!st || !sid) throw new UsageError('--snapshot is <task>/<snapshot>, e.g. voice.radio/voicedata-cast');
|
|
168
|
+
const cache = path.join(process.env.PLAIFUL_HOME || path.join(os.homedir(), '.plaiful'), 'cache', 'datasets', st, sid);
|
|
169
|
+
await models.dataPull({ ...ctx, flags: { ...ctx.flags, out: cache } }, [st, sid]);
|
|
170
|
+
data.push({ local: cache, name: `${slug(st)}-${slug(sid)}` });
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
ctx.io.stderr.write(`Copying the code to gs://${s.bucket}/jobs/${id}/code …\n`);
|
|
174
|
+
gcloud(ctx, ['storage', 'rsync', code, `gs://${s.bucket}/jobs/${id}/code`, '--recursive', `--exclude=${CODE_EXCLUDE}`, `--project=${s.project}`], { inherit: true });
|
|
175
|
+
for (const d of data) {
|
|
176
|
+
ctx.io.stderr.write(`Syncing ${d.local} → gs://${s.bucket}/data/${d.name} (only what changed) …\n`);
|
|
177
|
+
gcloud(ctx, ['storage', 'rsync', d.local, `gs://${s.bucket}/data/${d.name}`, '--recursive', `--project=${s.project}`], { inherit: true });
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// gcloud reads the config as YAML, and JSON is YAML.
|
|
181
|
+
const spec = jobSpec({ bucket: s.bucket, id, entry: ctx.flags.entry, gpu, machine: ctx.flags.machine, image: s.image, hours, task });
|
|
182
|
+
const cfg = path.join(os.tmpdir(), `plaiful-job-${id}.json`);
|
|
183
|
+
fs.writeFileSync(cfg, JSON.stringify(spec, null, 2));
|
|
184
|
+
let job;
|
|
185
|
+
try {
|
|
186
|
+
job = gcloud(ctx, ['ai', 'custom-jobs', 'create', `--project=${s.project}`, `--region=${s.region}`, `--display-name=${id}`, `--config=${cfg}`,
|
|
187
|
+
`--labels=plaiful-job=${id},plaiful-task=${slug(task)},plaiful-gpu=${gpu}`], { json: true });
|
|
188
|
+
} finally { try { fs.unlinkSync(cfg); } catch { /* temp file */ } }
|
|
189
|
+
const record = { id, name: job && job.name, task, gpu, machine: spec.workerPoolSpecs[0].machineSpec.machineType, hours, project: s.project, region: s.region, bucket: s.bucket, data: data.map((d) => d.name), submitted_at: new Date().toISOString() };
|
|
190
|
+
writeJobs([...readJobs(), record]);
|
|
191
|
+
ctx.out([
|
|
192
|
+
`Submitted ${id} on one ${gpu.toUpperCase()} (${record.machine}), stopped after ${hours} h at most.`,
|
|
193
|
+
` output: gs://${s.bucket}/jobs/${id}/out`,
|
|
194
|
+
` plaiful models train status ${id} · plaiful models train logs ${id} · plaiful models train fetch ${id}`,
|
|
195
|
+
].join('\n'), { success: true, job: record });
|
|
196
|
+
return 0;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
function findJob(id) {
|
|
200
|
+
const jobs = readJobs();
|
|
201
|
+
const j = id ? jobs.find((x) => x.id === id) : jobs[jobs.length - 1];
|
|
202
|
+
if (!j) throw new UsageError(id ? `no job ${id} submitted from this machine (plaiful models train list)` : 'no jobs submitted from this machine yet');
|
|
203
|
+
return j;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/** Vertex AI's view of a job → { state, started, ended, gpu_hours, error } */
|
|
207
|
+
function describe(ctx, j) {
|
|
208
|
+
const d = gcloud(ctx, ['ai', 'custom-jobs', 'describe', j.name, `--region=${j.region}`, `--project=${j.project}`], { json: true });
|
|
209
|
+
const start = d.startTime ? Date.parse(d.startTime) : null;
|
|
210
|
+
const end = d.endTime ? Date.parse(d.endTime) : (start ? Date.now() : null);
|
|
211
|
+
return { state: String(d.state || '').replace(/^JOB_STATE_/, '').toLowerCase(), created: d.createTime, started: d.startTime || null, ended: d.endTime || null, gpu_hours: start ? Math.round(((end - start) / 3.6e6) * 100) / 100 : 0, error: d.error && d.error.message };
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
async function status(ctx, [id]) {
|
|
215
|
+
const j = findJob(id);
|
|
216
|
+
const d = describe(ctx, j);
|
|
217
|
+
ctx.out([
|
|
218
|
+
`${j.id}: ${d.state} (${j.gpu.toUpperCase()}, ${j.task})`,
|
|
219
|
+
` submitted ${j.submitted_at}${d.started ? `, started ${d.started}` : ', waiting for a GPU'}${d.ended ? `, ended ${d.ended}` : ''}`,
|
|
220
|
+
` GPU time so far: ${d.gpu_hours} h (billed to ${j.project}; Billing → Reports, label plaiful-job=${j.id})`,
|
|
221
|
+
...(d.error ? [` error: ${d.error}`] : []),
|
|
222
|
+
].join('\n'), { success: true, job: j, ...d });
|
|
223
|
+
return 0;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
async function list(ctx) {
|
|
227
|
+
const jobs = readJobs();
|
|
228
|
+
if (!jobs.length) { ctx.out('No training jobs submitted from this machine yet.', { success: true, jobs: [] }); return 0; }
|
|
229
|
+
const rows = jobs.slice(-20).map((j) => { let d = {}; try { d = describe(ctx, j); } catch (e) { d = { state: `unknown (${e.message})` }; } return { ...j, ...d }; });
|
|
230
|
+
ctx.out(rows.map((j) => `${j.id} ${j.state} ${j.gpu} ${j.gpu_hours != null ? `${j.gpu_hours} GPU-h` : ''} ${j.task}`).join('\n'), { success: true, jobs: rows });
|
|
231
|
+
return 0;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
async function logs(ctx, [id]) {
|
|
235
|
+
const j = findJob(id);
|
|
236
|
+
gcloud(ctx, ['ai', 'custom-jobs', 'stream-logs', j.name, `--region=${j.region}`, `--project=${j.project}`], { inherit: true });
|
|
237
|
+
return 0;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async function fetch(ctx, [id]) {
|
|
241
|
+
const j = findJob(id);
|
|
242
|
+
const out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('model-runs', j.id));
|
|
243
|
+
fs.mkdirSync(out, { recursive: true });
|
|
244
|
+
gcloud(ctx, ['storage', 'rsync', `gs://${j.bucket}/jobs/${j.id}/out`, out, '--recursive', `--project=${j.project}`], { inherit: true });
|
|
245
|
+
ctx.out(`${j.id}'s output is in ${out}. Card it, then plaiful models publish <card.json>.`, { success: true, out });
|
|
246
|
+
return 0;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
async function cancel(ctx, [id]) {
|
|
250
|
+
const j = findJob(id);
|
|
251
|
+
if (!id) throw new UsageError('cancel which job? plaiful models train cancel <job>');
|
|
252
|
+
gcloud(ctx, ['ai', 'custom-jobs', 'cancel', j.name, `--region=${j.region}`, `--project=${j.project}`]);
|
|
253
|
+
ctx.out(`Asked Vertex AI to cancel ${j.id}.`, { success: true });
|
|
254
|
+
return 0;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
return { check, submit, status, list, logs, fetch, cancel };
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
module.exports = { makeTrain, jobSpec, jobScript, settings, GPUS, DEFAULT_IMAGE, slug };
|