@plaiful/cli 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +179 -153
- package/package.json +1 -1
- package/src/cli.js +2159 -1688
- package/src/models.js +531 -0
- package/src/train.js +260 -0
package/src/models.js
ADDED
|
@@ -0,0 +1,531 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* `plaiful models …`: the model registry from a game repo (PlaifulCMS docs/design/model-registry.md). Plaiful staff
|
|
5
|
+
* only for now. A release is a card (`<release>.card.json`, the first card format or version 2) plus the files it lists;
|
|
6
|
+
* a training-data snapshot is a folder, pushed as is with a manifest of every file's size and sha256.
|
|
7
|
+
*
|
|
8
|
+
* Files go straight to storage on short-lived links the platform signs with each file's sha256, so storage refuses
|
|
9
|
+
* bytes that don't match the card. A file bigger than one part (64 MiB) goes up in parts, each signed the same way, and
|
|
10
|
+
* a push that stops carries on from the last stored part. Nothing here needs AWS.
|
|
11
|
+
*
|
|
12
|
+
* `plaiful models archive …` pushes and pulls the plaiful-models archive (`<kind>/<role>/<name>`: sources, teachers,
|
|
13
|
+
* corpora, experiments), one sealed snapshot per entry, each with the licence record its catalog, manifest or README
|
|
14
|
+
* gives (PLAI-185).
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const fs = require('fs');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
const crypto = require('crypto');
|
|
20
|
+
const { Readable } = require('stream');
|
|
21
|
+
const { pipeline } = require('stream/promises');
|
|
22
|
+
|
|
23
|
+
const BATCH = 500;
|
|
24
|
+
const CONCURRENCY = 6;
|
|
25
|
+
|
|
26
|
+
function sha256File(file) {
|
|
27
|
+
return new Promise((resolve, reject) => {
|
|
28
|
+
const h = crypto.createHash('sha256');
|
|
29
|
+
fs.createReadStream(file).on('error', reject).on('data', (d) => h.update(d)).on('end', () => resolve(h.digest('hex')));
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function pool(items, n, fn) {
|
|
34
|
+
let i = 0;
|
|
35
|
+
await Promise.all(Array.from({ length: Math.min(n, items.length) }, async () => {
|
|
36
|
+
while (i < items.length) { const k = i; i += 1; await fn(items[k], k); }
|
|
37
|
+
}));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const mb = (b) => (b >= 1e9 ? `${(b / 1e9).toFixed(2)} GB` : b >= 1e6 ? `${(b / 1e6).toFixed(1)} MB` : `${Math.max(1, Math.round(b / 1e3))} KB`);
|
|
41
|
+
const enc = (s) => encodeURIComponent(s);
|
|
42
|
+
|
|
43
|
+
/** The sha256 of each `partBytes` slice of a file, in order (how a big file is stored in parts). */
|
|
44
|
+
function partHashes(file, partBytes) {
|
|
45
|
+
return new Promise((resolve, reject) => {
|
|
46
|
+
const out = [];
|
|
47
|
+
let h = crypto.createHash('sha256');
|
|
48
|
+
let inPart = 0;
|
|
49
|
+
fs.createReadStream(file, { highWaterMark: 4 * 1024 * 1024 }).on('error', reject).on('data', (d) => {
|
|
50
|
+
let at = 0;
|
|
51
|
+
while (at < d.length) {
|
|
52
|
+
const take = Math.min(partBytes - inPart, d.length - at);
|
|
53
|
+
h.update(d.subarray(at, at + take));
|
|
54
|
+
inPart += take;
|
|
55
|
+
at += take;
|
|
56
|
+
if (inPart === partBytes) { out.push(h.digest('hex')); h = crypto.createHash('sha256'); inPart = 0; }
|
|
57
|
+
}
|
|
58
|
+
}).on('end', () => { if (inPart > 0) out.push(h.digest('hex')); resolve(out); });
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Every file under `dir` (relative, forward slashes), skipping names that match `exclude`. Links and junctions are
|
|
64
|
+
* never followed (a game reaches moved archive folders through junctions at their old paths): they're listed in
|
|
65
|
+
* `skipped` instead.
|
|
66
|
+
*/
|
|
67
|
+
function walk(dir, exclude, skipped = []) {
|
|
68
|
+
const out = [];
|
|
69
|
+
const visit = (rel) => {
|
|
70
|
+
for (const ent of fs.readdirSync(path.join(dir, rel), { withFileTypes: true })) {
|
|
71
|
+
const r = rel ? `${rel}/${ent.name}` : ent.name;
|
|
72
|
+
if (exclude && exclude.test(r)) continue;
|
|
73
|
+
if (ent.isSymbolicLink()) skipped.push(r);
|
|
74
|
+
else if (ent.isDirectory()) visit(r);
|
|
75
|
+
else if (ent.isFile()) out.push(r);
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
visit('');
|
|
79
|
+
return out.sort();
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** The card files a path names: one card, or every *.card.json under a folder. */
|
|
83
|
+
function cardFiles(target) {
|
|
84
|
+
if (fs.statSync(target).isFile()) return [target];
|
|
85
|
+
const found = [];
|
|
86
|
+
const visit = (d) => {
|
|
87
|
+
for (const ent of fs.readdirSync(d, { withFileTypes: true })) {
|
|
88
|
+
const p = path.join(d, ent.name);
|
|
89
|
+
if (ent.isDirectory()) visit(p);
|
|
90
|
+
else if (ent.name.endsWith('.card.json')) found.push(p);
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
visit(target);
|
|
94
|
+
return found.sort();
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Where a card's files are on this machine: its archive folder (or, without one, where the game ships it). */
|
|
98
|
+
function filesRoot(root, card, flags) {
|
|
99
|
+
const base = flags.files || (card.files && (card.files.archive_path || card.files.shipped_path));
|
|
100
|
+
if (!base) return null;
|
|
101
|
+
return path.resolve(root, base);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** PUT one file to a signed link; the link's headers carry the sha256 storage checks. */
|
|
105
|
+
async function putFile(io, file, u) {
|
|
106
|
+
const body = fs.readFileSync(file);
|
|
107
|
+
let res;
|
|
108
|
+
try { res = await io.fetch(u.url, { method: u.method || 'PUT', headers: u.headers, body }); } catch (e) {
|
|
109
|
+
throw Object.assign(new Error(`the upload of ${u.path} didn't go through (${e.message}); run it again`), { code: 'network' });
|
|
110
|
+
}
|
|
111
|
+
if (!res.ok) {
|
|
112
|
+
const why = res.status === 400 ? 'its bytes don\'t match the sha256 (did the file change after the card was written?)' : `storage answered ${res.status}`;
|
|
113
|
+
throw Object.assign(new Error(`storage refused ${u.path}: ${why}`), { code: 'upload_failed' });
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** PUT one part of a big file (its bytes read from the file at the part's offset) to its signed link. */
|
|
118
|
+
async function putPart(io, file, rel, link) {
|
|
119
|
+
const body = Buffer.alloc(link.bytes);
|
|
120
|
+
const fd = fs.openSync(file, 'r');
|
|
121
|
+
try { fs.readSync(fd, body, 0, link.bytes, link.offset); } finally { fs.closeSync(fd); }
|
|
122
|
+
let res;
|
|
123
|
+
try { res = await io.fetch(link.url, { method: link.method || 'PUT', headers: link.headers, body }); } catch (e) {
|
|
124
|
+
throw Object.assign(new Error(`part ${link.part} of ${rel} didn't go through (${e.message}); run it again (stored parts are kept)`), { code: 'network' });
|
|
125
|
+
}
|
|
126
|
+
if (!res.ok) throw Object.assign(new Error(`storage refused part ${link.part} of ${rel} (${res.status}${res.status === 400 ? ': the file changed while it was being pushed?' : ''})`), { code: 'upload_failed' });
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Download to `file`, streamed (a file may be bigger than memory), checked against its sha256 before it's kept. */
|
|
130
|
+
async function getFile(io, url, file, sha) {
|
|
131
|
+
const res = await io.fetch(url);
|
|
132
|
+
if (!res.ok) throw Object.assign(new Error(`couldn't download ${path.basename(file)} (${res.status})`), { code: 'download_failed' });
|
|
133
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
134
|
+
const tmp = `${file}.part`;
|
|
135
|
+
const h = crypto.createHash('sha256');
|
|
136
|
+
if (res.body && typeof res.body.getReader === 'function') {
|
|
137
|
+
const src = Readable.fromWeb(res.body);
|
|
138
|
+
src.on('data', (d) => h.update(d));
|
|
139
|
+
await pipeline(src, fs.createWriteStream(tmp));
|
|
140
|
+
} else {
|
|
141
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
142
|
+
h.update(buf);
|
|
143
|
+
fs.writeFileSync(tmp, buf);
|
|
144
|
+
}
|
|
145
|
+
if (sha && h.digest('hex') !== sha) {
|
|
146
|
+
fs.rmSync(tmp, { force: true });
|
|
147
|
+
throw Object.assign(new Error(`${file} doesn't match its sha256 after download`), { code: 'checksum' });
|
|
148
|
+
}
|
|
149
|
+
fs.renameSync(tmp, file);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** The archive's roles: one sealed snapshot per `<kind>/<role>/<name>` folder. */
|
|
153
|
+
const ROLES = ['sources', 'teachers', 'corpora', 'experiments'];
|
|
154
|
+
const GAME_RE = /^[a-z0-9]+$/;
|
|
155
|
+
|
|
156
|
+
const readJson = (file) => { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return null; } };
|
|
157
|
+
const firstParagraph = (file) => {
|
|
158
|
+
if (!fs.existsSync(file)) return '';
|
|
159
|
+
const body = fs.readFileSync(file, 'utf8').split(/\r?\n\r?\n/).map((x) => x.trim()).filter((x) => x && !x.startsWith('#'));
|
|
160
|
+
return (body[0] || '').replace(/\s+/g, ' ').slice(0, 1500);
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
/** Every entry of the archive at `root` (`<kind>/<role>/<name>` folders, never links), optionally under `filter`. */
|
|
164
|
+
function archiveEntries(root, filter) {
|
|
165
|
+
const out = [];
|
|
166
|
+
const dirs = (d) => fs.readdirSync(d, { withFileTypes: true }).filter((e) => e.isDirectory() && !e.isSymbolicLink() && !/^[._]/.test(e.name)).map((e) => e.name).sort();
|
|
167
|
+
for (const kind of dirs(root)) {
|
|
168
|
+
if (!/^[a-z]+$/.test(kind)) continue;
|
|
169
|
+
for (const role of dirs(path.join(root, kind)).filter((r) => ROLES.includes(r))) {
|
|
170
|
+
for (const name of dirs(path.join(root, kind, role))) {
|
|
171
|
+
const rel = `${kind}/${role}/${name}`;
|
|
172
|
+
if (filter && rel !== filter && !rel.startsWith(`${String(filter).replace(/\/+$/, '')}/`)) continue;
|
|
173
|
+
out.push({ kind, role, name, rel, abs: path.join(root, kind, role, name), task: `${kind}.${role}` });
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return out;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* The licence record for an archive entry, from the records the archive keeps: a source's row in
|
|
182
|
+
* `<kind>/sources/catalog.json`, a teacher's `manifest.json`, a corpus's or experiment's README. → { about } or
|
|
183
|
+
* { problem } (an entry with no record isn't pushed: its licence must travel with it).
|
|
184
|
+
*/
|
|
185
|
+
function archiveRecord(root, e, { games } = {}) {
|
|
186
|
+
const base = { kind: e.kind, role: e.role, path: e.rel, source: e.rel };
|
|
187
|
+
if (e.role === 'sources') {
|
|
188
|
+
const cat = readJson(path.join(root, e.kind, 'sources', 'catalog.json'));
|
|
189
|
+
const row = cat && Array.isArray(cat.sources) ? cat.sources.find((r) => r && r.id === e.name) : null;
|
|
190
|
+
if (!row) return { problem: `no row with id "${e.name}" in ${e.kind}/sources/catalog.json` };
|
|
191
|
+
if (!row.licence) return { problem: `its catalog row has no licence` };
|
|
192
|
+
return {
|
|
193
|
+
about: {
|
|
194
|
+
...base, name: row.name || e.name, url: row.url || '', version: row.version || '', licence: row.licence,
|
|
195
|
+
licence_url: row.licence_url || '', attribution: row.attribution || '', commercial_ok: row.commercial_ok,
|
|
196
|
+
training_ok: row.training_ok, share_alike: row.share_alike, consent: row.consent || '',
|
|
197
|
+
note: [row.use, row.coverage].filter(Boolean).join(' · '),
|
|
198
|
+
},
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
if (e.role === 'teachers') {
|
|
202
|
+
const m = readJson(path.join(e.abs, 'manifest.json'));
|
|
203
|
+
if (!m) return { problem: 'no manifest.json (its exact revision and every file\'s sha256)' };
|
|
204
|
+
if (!m.licence) return { problem: 'its manifest.json has no licence' };
|
|
205
|
+
const version = [m.package && m.package.name && `${m.package.name} ${m.package.version || ''}`.trim(), m.github && m.github.commit && `github ${String(m.github.commit).slice(0, 12)}`, m.version].filter(Boolean).join('; ');
|
|
206
|
+
return {
|
|
207
|
+
about: {
|
|
208
|
+
...base, name: m.name || e.name, url: (m.github && m.github.repo) || m.url || m.repo || '', version, licence: m.licence,
|
|
209
|
+
share_alike: m.share_alike === true, note: `A teacher: an outside model we learn from, never a product. ${m.why || ''}`.trim(),
|
|
210
|
+
},
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
// A corpus or experiment is ours: `<game>-<name>`, described by its README (else the role's README). A snapshot can't
|
|
214
|
+
// be renamed once sealed, so a name that doesn't start with one of the archive's games is refused, not guessed.
|
|
215
|
+
const game = e.name.split('-')[0];
|
|
216
|
+
if (!games || !games.size) return { problem: 'say which games the archive holds (--games my-game,other-game,…) so its game can be checked' };
|
|
217
|
+
if (!e.name.includes('-') || !games.has(game) || !GAME_RE.test(game)) return { problem: `"${e.name}" doesn't start with one of the games (${[...games].join(', ')}): name it <game>-<name> before pushing, since a snapshot can't be renamed` };
|
|
218
|
+
const own = path.join(e.abs, 'README.md');
|
|
219
|
+
const shared = path.join(root, e.kind, e.role, 'README.md');
|
|
220
|
+
if (!fs.existsSync(own) && !fs.existsSync(shared)) return { problem: `no README.md (what it is, which release it trained, how it was made)` };
|
|
221
|
+
return {
|
|
222
|
+
about: {
|
|
223
|
+
...base, name: e.name, game,
|
|
224
|
+
licence: 'Plaiful\'s own training data (see its README for what made it)',
|
|
225
|
+
note: firstParagraph(own) || `See ${e.kind}/${e.role}/README.md`,
|
|
226
|
+
},
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function makeModels({ UsageError, ApiError, repo }) {
|
|
231
|
+
const apiFail = (code, error) => new ApiError(409, { code, error });
|
|
232
|
+
|
|
233
|
+
/** Read a card and check its files on disk against it. → { card, root, problems } */
|
|
234
|
+
async function localCheck(ctx, file) {
|
|
235
|
+
let card;
|
|
236
|
+
try { card = JSON.parse(fs.readFileSync(file, 'utf8')); } catch (e) { throw new UsageError(`${file} isn't a JSON card (${e.message})`); }
|
|
237
|
+
const r = repo(ctx, { required: false });
|
|
238
|
+
const root = filesRoot(r.root, card, ctx.flags);
|
|
239
|
+
const problems = [];
|
|
240
|
+
const items = (card.files && card.files.items) || [];
|
|
241
|
+
if (!root) problems.push('the card has no files.archive_path or shipped_path: pass --files <folder>');
|
|
242
|
+
else {
|
|
243
|
+
await pool(items, CONCURRENCY, async (f) => {
|
|
244
|
+
const p = path.join(root, f.path);
|
|
245
|
+
if (!fs.existsSync(p)) { problems.push(`${f.path}: not found under ${root}`); return; }
|
|
246
|
+
const size = fs.statSync(p).size;
|
|
247
|
+
if (size !== f.bytes) { problems.push(`${f.path}: ${size} bytes, the card says ${f.bytes}`); return; }
|
|
248
|
+
if (await sha256File(p) !== f.sha256) problems.push(`${f.path}: sha256 differs from the card`);
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
return { card, root, problems, owner: card.owner || r.product || null };
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** `plaiful models check|publish <card.json | folder of cards>` */
|
|
255
|
+
async function publish(ctx, [target], { apply }) {
|
|
256
|
+
if (!target) throw new UsageError(`${apply ? 'publish' : 'check'} what? plaiful models ${apply ? 'publish' : 'check'} <card.json | folder of cards>`);
|
|
257
|
+
const abs = path.resolve(ctx.io.cwd(), target);
|
|
258
|
+
if (!fs.existsSync(abs)) throw new UsageError(`${abs} doesn't exist`);
|
|
259
|
+
const files = cardFiles(abs);
|
|
260
|
+
if (!files.length) throw new UsageError(`no *.card.json under ${abs}`);
|
|
261
|
+
const results = [];
|
|
262
|
+
let failed = 0;
|
|
263
|
+
for (const file of files) {
|
|
264
|
+
const { card, root, problems, owner } = await localCheck(ctx, file);
|
|
265
|
+
const id = `${card.task_type}/${card.release_id}`;
|
|
266
|
+
const row = { card: path.relative(ctx.io.cwd(), file), release: id, problems, warnings: [] };
|
|
267
|
+
results.push(row);
|
|
268
|
+
if (problems.length) { failed += 1; ctx.io.stderr.write(`✗ ${id}: ${problems.length} file problem(s)\n${problems.slice(0, 5).map((p) => ` ${p}`).join('\n')}\n`); continue; }
|
|
269
|
+
const route = `/partner/models/releases/${enc(card.task_type)}/${enc(card.release_id)}`;
|
|
270
|
+
let up;
|
|
271
|
+
try {
|
|
272
|
+
const r = await ctx.call('POST', `${route}/upload`, { card, owner });
|
|
273
|
+
// A refused card (invalid, or training data it may not use) comes back as a 422 answer, not a thrown error.
|
|
274
|
+
if (r.status >= 400) throw new ApiError(r.status, r.data);
|
|
275
|
+
up = r.data;
|
|
276
|
+
} catch (e) {
|
|
277
|
+
if (!(e instanceof ApiError) || e.status >= 500 || e.status === 401 || e.status === 403) throw e;
|
|
278
|
+
failed += 1; row.problems.push(e.message); ctx.io.stderr.write(`✗ ${id}: ${e.message}\n`); continue;
|
|
279
|
+
}
|
|
280
|
+
row.warnings = up.warnings || [];
|
|
281
|
+
if (!apply) {
|
|
282
|
+
row.would_upload = up.uploads.length;
|
|
283
|
+
row.stored = up.stored;
|
|
284
|
+
ctx.io.stderr.write(`✓ ${id}: valid; ${up.uploads.length} file(s) to upload, ${up.stored} stored${up.published ? ' (published: a card correction)' : ''}${row.warnings.length ? `; ${row.warnings.length} warning(s)` : ''}\n`);
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
let sent = 0;
|
|
288
|
+
await pool(up.uploads, CONCURRENCY, async (u) => { await putFile(ctx.io, path.join(root, u.path), u); sent += u.bytes; });
|
|
289
|
+
const pub = (await ctx.call('POST', `${route}/publish`, { card, owner })).data;
|
|
290
|
+
Object.assign(row, { uploaded: up.uploads.length, uploaded_bytes: sent, card_rev: pub.card_rev, corrected: pub.corrected, status: pub.status });
|
|
291
|
+
ctx.io.stderr.write(`✓ ${id}: published (card ${pub.card_rev}${pub.corrected ? ', corrected' : ''}, ${pub.status}); ${up.uploads.length} file(s), ${mb(sent)} uploaded\n`);
|
|
292
|
+
}
|
|
293
|
+
const ok = results.length - failed;
|
|
294
|
+
ctx.out(`${apply ? 'Published' : 'Checked'} ${ok} of ${results.length} release(s)${failed ? `; ${failed} with problems (above)` : ''}.${apply ? '' : ' Nothing was uploaded: plaiful models publish to upload.'}`, { success: failed === 0, results });
|
|
295
|
+
return failed ? 1 : 0;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
async function list(ctx, [task]) {
|
|
299
|
+
const { data } = await ctx.call('GET', `/partner/models/releases${task ? `?task=${enc(task)}` : ''}`);
|
|
300
|
+
const rows = data.releases;
|
|
301
|
+
const lines = rows.length ? [] : ['No releases in the registry yet: plaiful models publish <card.json>'];
|
|
302
|
+
let last = null;
|
|
303
|
+
for (const r of rows.sort((a, b) => a.task_type.localeCompare(b.task_type) || String(b.date).localeCompare(String(a.date)))) {
|
|
304
|
+
if (r.task_type !== last) { lines.push(`${r.task_type}`); last = r.task_type; }
|
|
305
|
+
lines.push(` ${r.release_id} ${r.status} ${r.date} ${mb(r.bytes)} cleared: ${r.cleared_for_other_games} (${r.owner})`);
|
|
306
|
+
}
|
|
307
|
+
ctx.out(lines.join('\n'), data);
|
|
308
|
+
return 0;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
async function show(ctx, [task, release]) {
|
|
312
|
+
if (!task || !release) throw new UsageError('plaiful models show <task> <release>');
|
|
313
|
+
const { data } = await ctx.call('GET', `/partner/models/releases/${enc(task)}/${enc(release)}`);
|
|
314
|
+
const r = data.release;
|
|
315
|
+
const lines = [
|
|
316
|
+
`${r.task_type} / ${r.release_id} (${r.status}, card ${r.card_rev}, owner ${r.owner})`,
|
|
317
|
+
` ${r.name}: ${r.one_line}`,
|
|
318
|
+
` cleared for other games: ${r.cleared_for_other_games}${r.attribution.length ? `; credit: ${r.attribution.join(' · ')}` : ''}`,
|
|
319
|
+
` ${r.files} file(s), ${mb(r.bytes)} (shipped ${mb(r.shipped_bytes)}); ${r.prefix}`,
|
|
320
|
+
` published ${r.published_at} by ${r.published_by}`,
|
|
321
|
+
...(r.status_history || []).map((h) => ` ${h.at.slice(0, 10)} ${h.status} ${h.by}${h.note ? `: ${h.note}` : ''}`),
|
|
322
|
+
];
|
|
323
|
+
ctx.out(lines.join('\n'), data);
|
|
324
|
+
return 0;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
async function status(ctx, [task, release, value]) {
|
|
328
|
+
if (!task || !release || !value) throw new UsageError('plaiful models status <task> <release> active|candidate|retired [--note "…"]');
|
|
329
|
+
const { data } = await ctx.call('POST', `/partner/models/releases/${enc(task)}/${enc(release)}/status`, { status: value, note: ctx.flags.note });
|
|
330
|
+
ctx.out(data.changed ? `${task}/${release}: ${data.was} → ${data.status}. Update the card's status too, so the two agree.` : `${task}/${release} is already ${data.status}.`, data);
|
|
331
|
+
return 0;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
async function pull(ctx, [task, release]) {
|
|
335
|
+
if (!task || !release) throw new UsageError('plaiful models pull <task> <release> [--out <dir>] [--role shipped]');
|
|
336
|
+
const roles = ctx.flags.role ? String(ctx.flags.role).split(',') : undefined;
|
|
337
|
+
const { data } = await ctx.call('POST', `/partner/models/releases/${enc(task)}/${enc(release)}/download`, { roles });
|
|
338
|
+
const out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('models', task, release));
|
|
339
|
+
await pool(data.files, CONCURRENCY, (f) => getFile(ctx.io, f.url, path.join(out, f.path), f.sha256));
|
|
340
|
+
fs.writeFileSync(path.join(out, `${release}.card.json`), `${JSON.stringify(data.card, null, 1)}\n`);
|
|
341
|
+
ctx.out(`Downloaded ${data.files.length} file(s) of ${task}/${release} (${data.status}) to ${out}, every sha256 checked.`, { success: true, out, files: data.files.length });
|
|
342
|
+
return 0;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// ── Training-data snapshots ──────────────────────────────────────────────────────────────────────────────────
|
|
346
|
+
|
|
347
|
+
/** One big file: its parts' links, a batch at a time (a link lasts 15 minutes), then the join. */
|
|
348
|
+
async function pushBig(ctx, route, abs, entry) {
|
|
349
|
+
for (;;) {
|
|
350
|
+
const { data } = await ctx.call('POST', `${route}/multipart`, { file: entry, want: 24 });
|
|
351
|
+
if (data.stored || !data.links.length) {
|
|
352
|
+
if (!data.stored) await ctx.call('POST', `${route}/multipart/complete`, { file: entry, upload_id: data.upload_id });
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
await pool(data.links, CONCURRENCY, (l) => putPart(ctx.io, path.join(abs, entry.path), entry.path, l));
|
|
356
|
+
ctx.io.stderr.write(` ${entry.path}: ${Math.min(data.parts, data.done + data.links.length)} / ${data.parts} parts\n`);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/** Push the folder `abs` as the sealed snapshot task/snapshot. Safe to re-run: stored files and parts are skipped. */
|
|
361
|
+
async function pushSnapshot(ctx, { task, snapshot, abs, exclude = null, about }) {
|
|
362
|
+
const skipped = [];
|
|
363
|
+
const rels = walk(abs, exclude, skipped);
|
|
364
|
+
if (skipped.length) ctx.io.stderr.write(` not followed (links or junctions): ${skipped.slice(0, 5).join(', ')}${skipped.length > 5 ? ` and ${skipped.length - 5} more` : ''}\n`);
|
|
365
|
+
if (!rels.length) throw new UsageError(`no files under ${abs}`);
|
|
366
|
+
ctx.io.stderr.write(`Hashing ${rels.length} file(s) under ${abs}…\n`);
|
|
367
|
+
const entries = new Array(rels.length);
|
|
368
|
+
await pool(rels, CONCURRENCY, async (rel, i) => {
|
|
369
|
+
const p = path.join(abs, rel);
|
|
370
|
+
entries[i] = { path: rel, bytes: fs.statSync(p).size, sha256: await sha256File(p) };
|
|
371
|
+
});
|
|
372
|
+
const total = entries.reduce((n, e) => n + e.bytes, 0);
|
|
373
|
+
const route = `/partner/models/datasets/${enc(task)}/${enc(snapshot)}`;
|
|
374
|
+
let uploaded = 0;
|
|
375
|
+
let stored = 0;
|
|
376
|
+
let sent = 0;
|
|
377
|
+
let partBytes = null;
|
|
378
|
+
for (let i = 0; i < entries.length; i += BATCH) {
|
|
379
|
+
const batch = entries.slice(i, i + BATCH);
|
|
380
|
+
const { data } = await ctx.call('POST', `${route}/upload`, { files: batch });
|
|
381
|
+
partBytes = data.part_bytes || partBytes;
|
|
382
|
+
stored += data.stored;
|
|
383
|
+
// A big file's manifest entry lists its parts' sha256s, so the registry can check the stored file whole.
|
|
384
|
+
if (partBytes) await pool(batch.filter((e) => e.bytes > partBytes), 2, async (e) => { e.parts = await partHashes(path.join(abs, e.path), partBytes); });
|
|
385
|
+
const small = data.uploads.filter((u) => !u.multipart);
|
|
386
|
+
await pool(small, CONCURRENCY, async (u) => { await putFile(ctx.io, path.join(abs, u.path), u); uploaded += 1; sent += u.bytes; });
|
|
387
|
+
for (const u of data.uploads.filter((x) => x.multipart)) {
|
|
388
|
+
await pushBig(ctx, route, abs, batch.find((e) => e.path === u.path));
|
|
389
|
+
uploaded += 1; sent += u.bytes;
|
|
390
|
+
}
|
|
391
|
+
ctx.io.stderr.write(` ${Math.min(i + BATCH, entries.length)} / ${entries.length} (${uploaded} uploaded, ${stored} already stored)\n`);
|
|
392
|
+
}
|
|
393
|
+
const manifest = `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`;
|
|
394
|
+
const mSha = crypto.createHash('sha256').update(manifest).digest('hex');
|
|
395
|
+
const { data: m } = await ctx.call('POST', `${route}/upload`, { manifest: { bytes: Buffer.byteLength(manifest), sha256: mSha } });
|
|
396
|
+
const res = await ctx.io.fetch(m.manifest.url, { method: 'PUT', headers: m.manifest.headers, body: Buffer.from(manifest) });
|
|
397
|
+
if (!res.ok) throw apiFail('upload_failed', `storage refused the manifest (${res.status}); run the push again`);
|
|
398
|
+
const { data } = await ctx.call('POST', `${route}/finish`, { about });
|
|
399
|
+
return { ...data, uploaded, sent, stored, total_bytes: total, skipped_links: skipped.length };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
async function dataPush(ctx, [task, snapshot, dir]) {
|
|
403
|
+
if (!task || !snapshot || !dir) throw new UsageError('plaiful models data push <task> <snapshot> <folder> [--exclude <regex>] [--source "…"] [--licence "…"] [--note "…"] [--releases a,b]');
|
|
404
|
+
const abs = path.resolve(ctx.io.cwd(), dir);
|
|
405
|
+
if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) throw new UsageError(`${abs} isn't a folder`);
|
|
406
|
+
let exclude = null;
|
|
407
|
+
if (ctx.flags.exclude) { try { exclude = new RegExp(ctx.flags.exclude); } catch (e) { throw new UsageError(`--exclude isn't a regular expression (${e.message})`); } }
|
|
408
|
+
const about = { source: ctx.flags.source || abs, licence: ctx.flags.licence || '', note: ctx.flags.note || '', releases: ctx.flags.releases ? String(ctx.flags.releases).split(',') : [] };
|
|
409
|
+
const data = await pushSnapshot(ctx, { task, snapshot, abs, exclude, about });
|
|
410
|
+
ctx.out(`${task}/${snapshot}: sealed, ${data.files} file(s), ${mb(data.bytes)} (${data.uploaded} uploaded, ${mb(data.sent)}; ${data.stored} were already stored). Backed up to the Management account by replication.`, { success: true, ...data });
|
|
411
|
+
return 0;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/** `plaiful models archive push <root> [<kind>[/<role>[/<name>]]]`: every entry not sealed yet, one snapshot each. */
|
|
415
|
+
async function archivePush(ctx, [dir, filter]) {
|
|
416
|
+
if (!dir) throw new UsageError('plaiful models archive push <archive folder> [<kind>[/<role>[/<name>]]] --games my-game,other-game,…');
|
|
417
|
+
const games = new Set(String(ctx.flags.games || '').split(',').map((g) => g.trim()).filter(Boolean));
|
|
418
|
+
const root = path.resolve(ctx.io.cwd(), dir);
|
|
419
|
+
if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) throw new UsageError(`${root} isn't a folder`);
|
|
420
|
+
const entries = archiveEntries(root, filter);
|
|
421
|
+
if (!entries.length) throw new UsageError(`no <kind>/<role>/<name> entries under ${root}${filter ? ` matching ${filter}` : ''}`);
|
|
422
|
+
const { data: listed } = await ctx.call('GET', '/partner/models/datasets');
|
|
423
|
+
const sealed = new Set(listed.datasets.filter((d) => d.state === 'sealed').map((d) => `${d.task_type}/${d.snapshot}`));
|
|
424
|
+
const results = [];
|
|
425
|
+
for (const e of entries) {
|
|
426
|
+
const id = `${e.task}/${e.name}`;
|
|
427
|
+
if (sealed.has(id)) { results.push({ entry: e.rel, snapshot: id, state: 'sealed already' }); continue; }
|
|
428
|
+
const rec = archiveRecord(root, e, { games });
|
|
429
|
+
if (rec.problem) {
|
|
430
|
+
results.push({ entry: e.rel, snapshot: id, state: 'not pushed', problem: rec.problem });
|
|
431
|
+
ctx.io.stderr.write(`✗ ${e.rel}: not pushed: ${rec.problem}\n`);
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
ctx.io.stderr.write(`→ ${e.rel} as ${id}${rec.about.share_alike ? ' (share-alike: reference only)' : ''}\n`);
|
|
435
|
+
try {
|
|
436
|
+
const data = await pushSnapshot(ctx, { task: e.task, snapshot: e.name, abs: e.abs, about: rec.about });
|
|
437
|
+
results.push({ entry: e.rel, snapshot: id, state: 'sealed', files: data.files, bytes: data.bytes, uploaded_bytes: data.sent });
|
|
438
|
+
ctx.io.stderr.write(`✓ ${id}: sealed, ${data.files} file(s), ${mb(data.bytes)}\n`);
|
|
439
|
+
} catch (err) {
|
|
440
|
+
if (err instanceof UsageError) { results.push({ entry: e.rel, snapshot: id, state: 'not pushed', problem: err.message }); ctx.io.stderr.write(`✗ ${e.rel}: ${err.message}\n`); continue; }
|
|
441
|
+
throw err;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
const failed = results.filter((r) => r.state === 'not pushed').length;
|
|
445
|
+
const done = results.filter((r) => r.state === 'sealed').length;
|
|
446
|
+
ctx.out(`${done} pushed, ${results.length - done - failed} already sealed, ${failed} not pushed${failed ? ' (above: add the missing record, then run it again)' : ''}. Backed up to the Management account by replication.`, { success: failed === 0, results });
|
|
447
|
+
return failed ? 1 : 0;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/** `plaiful models archive list [<root>]`: every archive snapshot in the registry (and, with a root, what's not pushed). */
|
|
451
|
+
async function archiveList(ctx, [dir, filter]) {
|
|
452
|
+
const { data } = await ctx.call('GET', '/partner/models/datasets');
|
|
453
|
+
const inRegistry = new Map(data.datasets.filter((d) => d.about && d.about.path).map((d) => [d.about.path, d]));
|
|
454
|
+
const rows = [];
|
|
455
|
+
const local = dir ? archiveEntries(path.resolve(ctx.io.cwd(), dir), filter) : [];
|
|
456
|
+
for (const e of local) if (!inRegistry.has(e.rel)) rows.push({ path: e.rel, state: 'not pushed' });
|
|
457
|
+
for (const [p, d] of inRegistry) {
|
|
458
|
+
if (filter && p !== filter && !p.startsWith(`${String(filter).replace(/\/+$/, '')}/`)) continue;
|
|
459
|
+
rows.push({ path: p, state: d.state, snapshot: `${d.task_type}/${d.snapshot}`, files: d.files, bytes: d.bytes, licence: d.about.licence, share_alike: !!d.about.share_alike, role: d.about.role, game: d.about.game });
|
|
460
|
+
}
|
|
461
|
+
rows.sort((a, b) => a.path.localeCompare(b.path));
|
|
462
|
+
const lines = rows.length ? rows.map((r) => `${r.path} ${r.state}${r.files != null ? ` ${r.files} file(s), ${mb(r.bytes)}` : ''}${r.licence ? ` ${r.licence}` : ''}${r.share_alike ? ' SHARE-ALIKE: reference only, not training data for a release' : ''}${r.role === 'teachers' ? ' (a teacher, never a product)' : ''}`)
|
|
463
|
+
: ['No archive snapshots yet: plaiful models archive push <archive folder>'];
|
|
464
|
+
ctx.out(lines.join('\n'), { rows });
|
|
465
|
+
return 0;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/** `plaiful models archive pull <root> [<kind>[/<role>[/<name>]]]`: each snapshot back at its path under root. */
|
|
469
|
+
async function archivePull(ctx, [dir, filter]) {
|
|
470
|
+
if (!dir) throw new UsageError('plaiful models archive pull <folder> [<kind>[/<role>[/<name>]]]');
|
|
471
|
+
const root = path.resolve(ctx.io.cwd(), dir);
|
|
472
|
+
const { data } = await ctx.call('GET', '/partner/models/datasets');
|
|
473
|
+
const picked = data.datasets.filter((d) => d.state === 'sealed' && d.about && d.about.path
|
|
474
|
+
&& (!filter || d.about.path === filter || d.about.path.startsWith(`${String(filter).replace(/\/+$/, '')}/`)));
|
|
475
|
+
if (!picked.length) throw new UsageError(`no sealed archive snapshots${filter ? ` under ${filter}` : ''}`);
|
|
476
|
+
let n = 0;
|
|
477
|
+
for (const d of picked) {
|
|
478
|
+
ctx.io.stderr.write(`→ ${d.about.path}\n`);
|
|
479
|
+
n += (await pullSnapshot(ctx, d.task_type, d.snapshot, path.join(root, ...d.about.path.split('/')))).downloaded;
|
|
480
|
+
}
|
|
481
|
+
ctx.out(`${picked.length} snapshot(s) under ${root}, ${n} file(s) downloaded, every sha256 checked.`, { success: true, root, snapshots: picked.length, downloaded: n });
|
|
482
|
+
return 0;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
async function dataList(ctx, [task]) {
|
|
486
|
+
const { data } = await ctx.call('GET', `/partner/models/datasets${task ? `?task=${enc(task)}` : ''}`);
|
|
487
|
+
const lines = data.datasets.length ? data.datasets.map((d) => `${d.task_type}/${d.snapshot} ${d.state} ${d.files != null ? `${d.files} file(s), ${mb(d.bytes)}` : '(push not finished)'}${d.about && d.about.releases && d.about.releases.length ? ` for ${d.about.releases.join(', ')}` : ''}${d.about && d.about.share_alike ? ' SHARE-ALIKE: reference only' : ''}`)
|
|
488
|
+
: ['No training-data snapshots yet: plaiful models data push <task> <snapshot> <folder>'];
|
|
489
|
+
ctx.out(lines.join('\n'), data);
|
|
490
|
+
return 0;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
async function dataPull(ctx, [task, snapshot]) {
|
|
494
|
+
if (!task || !snapshot) throw new UsageError('plaiful models data pull <task> <snapshot> [--out <dir> | --into <archive folder>]');
|
|
495
|
+
let out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('datasets', task, snapshot));
|
|
496
|
+
if (ctx.flags.into) {
|
|
497
|
+
// Back at its archive path, so the catalogs' relative paths still hold.
|
|
498
|
+
const { data } = await ctx.call('GET', `/partner/models/datasets/${enc(task)}/${enc(snapshot)}`);
|
|
499
|
+
const at = data.dataset.about && data.dataset.about.path;
|
|
500
|
+
if (!at) throw new UsageError(`${task}/${snapshot} has no archive path (it wasn't pushed from the archive): use --out`);
|
|
501
|
+
out = path.join(path.resolve(ctx.io.cwd(), ctx.flags.into), ...at.split('/'));
|
|
502
|
+
}
|
|
503
|
+
const r = await pullSnapshot(ctx, task, snapshot, out);
|
|
504
|
+
ctx.out(`${task}/${snapshot} → ${out}: ${r.downloaded} downloaded, ${r.skipped} already here, every sha256 checked.`, { success: true, out, ...r });
|
|
505
|
+
return 0;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
async function pullSnapshot(ctx, task, snapshot, out) {
|
|
509
|
+
const route = `/partner/models/datasets/${enc(task)}/${enc(snapshot)}/download`;
|
|
510
|
+
let after = 0;
|
|
511
|
+
let n = 0;
|
|
512
|
+
let skipped = 0;
|
|
513
|
+
for (;;) {
|
|
514
|
+
const { data } = await ctx.call('POST', route, { after });
|
|
515
|
+
await pool(data.files, CONCURRENCY, async (f) => {
|
|
516
|
+
const p = path.join(out, f.path);
|
|
517
|
+
if (fs.existsSync(p) && fs.statSync(p).size === f.bytes && await sha256File(p) === f.sha256) { skipped += 1; return; }
|
|
518
|
+
await getFile(ctx.io, f.url, p, f.sha256);
|
|
519
|
+
n += 1;
|
|
520
|
+
});
|
|
521
|
+
ctx.io.stderr.write(` ${data.after + data.files.length} / ${data.total}\n`);
|
|
522
|
+
if (data.next == null) break;
|
|
523
|
+
after = data.next;
|
|
524
|
+
}
|
|
525
|
+
return { downloaded: n, skipped };
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
return { publish, list, show, status, pull, dataPush, dataList, dataPull, archivePush, archiveList, archivePull };
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
module.exports = { makeModels, walk, sha256File, partHashes, archiveEntries, archiveRecord, cardFiles, filesRoot, mb };
|