@plaiful/cli 0.15.0 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +179 -158
- package/package.json +21 -21
- package/src/cli.js +2164 -1751
- package/src/models.js +282 -35
package/src/models.js
CHANGED
|
@@ -2,16 +2,23 @@
|
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* `plaiful models …`: the model registry from a game repo (PlaifulCMS docs/design/model-registry.md). Plaiful staff
|
|
5
|
-
* only for now. A release is a card (`<release>.card.json`,
|
|
5
|
+
* only for now. A release is a card (`<release>.card.json`, the first card format or version 2) plus the files it lists;
|
|
6
6
|
* a training-data snapshot is a folder, pushed as is with a manifest of every file's size and sha256.
|
|
7
7
|
*
|
|
8
8
|
* Files go straight to storage on short-lived links the platform signs with each file's sha256, so storage refuses
|
|
9
|
-
* bytes that don't match the card.
|
|
9
|
+
* bytes that don't match the card. A file bigger than one part (64 MiB) goes up in parts, each signed the same way, and
|
|
10
|
+
* a push that stops carries on from the last stored part. Nothing here needs AWS.
|
|
11
|
+
*
|
|
12
|
+
* `plaiful models archive …` pushes and pulls the plaiful-models archive (`<kind>/<role>/<name>`: sources, teachers,
|
|
13
|
+
* corpora, experiments), one sealed snapshot per entry, each with the licence record its catalog, manifest or README
|
|
14
|
+
* gives (PLAI-185).
|
|
10
15
|
*/
|
|
11
16
|
|
|
12
17
|
const fs = require('fs');
|
|
13
18
|
const path = require('path');
|
|
14
19
|
const crypto = require('crypto');
|
|
20
|
+
const { Readable } = require('stream');
|
|
21
|
+
const { pipeline } = require('stream/promises');
|
|
15
22
|
|
|
16
23
|
const BATCH = 500;
|
|
17
24
|
const CONCURRENCY = 6;
|
|
@@ -33,14 +40,38 @@ async function pool(items, n, fn) {
|
|
|
33
40
|
const mb = (b) => (b >= 1e9 ? `${(b / 1e9).toFixed(2)} GB` : b >= 1e6 ? `${(b / 1e6).toFixed(1)} MB` : `${Math.max(1, Math.round(b / 1e3))} KB`);
|
|
34
41
|
const enc = (s) => encodeURIComponent(s);
|
|
35
42
|
|
|
36
|
-
/**
|
|
37
|
-
function
|
|
43
|
+
/** The sha256 of each `partBytes` slice of a file, in order (how a big file is stored in parts). */
|
|
44
|
+
function partHashes(file, partBytes) {
|
|
45
|
+
return new Promise((resolve, reject) => {
|
|
46
|
+
const out = [];
|
|
47
|
+
let h = crypto.createHash('sha256');
|
|
48
|
+
let inPart = 0;
|
|
49
|
+
fs.createReadStream(file, { highWaterMark: 4 * 1024 * 1024 }).on('error', reject).on('data', (d) => {
|
|
50
|
+
let at = 0;
|
|
51
|
+
while (at < d.length) {
|
|
52
|
+
const take = Math.min(partBytes - inPart, d.length - at);
|
|
53
|
+
h.update(d.subarray(at, at + take));
|
|
54
|
+
inPart += take;
|
|
55
|
+
at += take;
|
|
56
|
+
if (inPart === partBytes) { out.push(h.digest('hex')); h = crypto.createHash('sha256'); inPart = 0; }
|
|
57
|
+
}
|
|
58
|
+
}).on('end', () => { if (inPart > 0) out.push(h.digest('hex')); resolve(out); });
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Every file under `dir` (relative, forward slashes), skipping names that match `exclude`. Links and junctions are
|
|
64
|
+
* never followed (a game reaches moved archive folders through junctions at their old paths): they're listed in
|
|
65
|
+
* `skipped` instead.
|
|
66
|
+
*/
|
|
67
|
+
function walk(dir, exclude, skipped = []) {
|
|
38
68
|
const out = [];
|
|
39
69
|
const visit = (rel) => {
|
|
40
70
|
for (const ent of fs.readdirSync(path.join(dir, rel), { withFileTypes: true })) {
|
|
41
71
|
const r = rel ? `${rel}/${ent.name}` : ent.name;
|
|
42
72
|
if (exclude && exclude.test(r)) continue;
|
|
43
|
-
if (ent.
|
|
73
|
+
if (ent.isSymbolicLink()) skipped.push(r);
|
|
74
|
+
else if (ent.isDirectory()) visit(r);
|
|
44
75
|
else if (ent.isFile()) out.push(r);
|
|
45
76
|
}
|
|
46
77
|
};
|
|
@@ -83,14 +114,117 @@ async function putFile(io, file, u) {
|
|
|
83
114
|
}
|
|
84
115
|
}
|
|
85
116
|
|
|
117
|
+
/** PUT one part of a big file (its bytes read from the file at the part's offset) to its signed link. */
|
|
118
|
+
async function putPart(io, file, rel, link) {
|
|
119
|
+
const body = Buffer.alloc(link.bytes);
|
|
120
|
+
const fd = fs.openSync(file, 'r');
|
|
121
|
+
try { fs.readSync(fd, body, 0, link.bytes, link.offset); } finally { fs.closeSync(fd); }
|
|
122
|
+
let res;
|
|
123
|
+
try { res = await io.fetch(link.url, { method: link.method || 'PUT', headers: link.headers, body }); } catch (e) {
|
|
124
|
+
throw Object.assign(new Error(`part ${link.part} of ${rel} didn't go through (${e.message}); run it again (stored parts are kept)`), { code: 'network' });
|
|
125
|
+
}
|
|
126
|
+
if (!res.ok) throw Object.assign(new Error(`storage refused part ${link.part} of ${rel} (${res.status}${res.status === 400 ? ': the file changed while it was being pushed?' : ''})`), { code: 'upload_failed' });
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Download to `file`, streamed (a file may be bigger than memory), checked against its sha256 before it's kept. */
|
|
86
130
|
async function getFile(io, url, file, sha) {
|
|
87
131
|
const res = await io.fetch(url);
|
|
88
132
|
if (!res.ok) throw Object.assign(new Error(`couldn't download ${path.basename(file)} (${res.status})`), { code: 'download_failed' });
|
|
89
|
-
const buf = Buffer.from(await res.arrayBuffer());
|
|
90
|
-
const got = crypto.createHash('sha256').update(buf).digest('hex');
|
|
91
|
-
if (sha && got !== sha) throw Object.assign(new Error(`${file} doesn't match its sha256 after download`), { code: 'checksum' });
|
|
92
133
|
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
93
|
-
|
|
134
|
+
const tmp = `${file}.part`;
|
|
135
|
+
const h = crypto.createHash('sha256');
|
|
136
|
+
if (res.body && typeof res.body.getReader === 'function') {
|
|
137
|
+
const src = Readable.fromWeb(res.body);
|
|
138
|
+
src.on('data', (d) => h.update(d));
|
|
139
|
+
await pipeline(src, fs.createWriteStream(tmp));
|
|
140
|
+
} else {
|
|
141
|
+
const buf = Buffer.from(await res.arrayBuffer());
|
|
142
|
+
h.update(buf);
|
|
143
|
+
fs.writeFileSync(tmp, buf);
|
|
144
|
+
}
|
|
145
|
+
if (sha && h.digest('hex') !== sha) {
|
|
146
|
+
fs.rmSync(tmp, { force: true });
|
|
147
|
+
throw Object.assign(new Error(`${file} doesn't match its sha256 after download`), { code: 'checksum' });
|
|
148
|
+
}
|
|
149
|
+
fs.renameSync(tmp, file);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** The archive's roles: one sealed snapshot per `<kind>/<role>/<name>` folder. */
|
|
153
|
+
const ROLES = ['sources', 'teachers', 'corpora', 'experiments'];
|
|
154
|
+
const GAME_RE = /^[a-z0-9]+$/;
|
|
155
|
+
|
|
156
|
+
const readJson = (file) => { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return null; } };
|
|
157
|
+
const firstParagraph = (file) => {
|
|
158
|
+
if (!fs.existsSync(file)) return '';
|
|
159
|
+
const body = fs.readFileSync(file, 'utf8').split(/\r?\n\r?\n/).map((x) => x.trim()).filter((x) => x && !x.startsWith('#'));
|
|
160
|
+
return (body[0] || '').replace(/\s+/g, ' ').slice(0, 1500);
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
/** Every entry of the archive at `root` (`<kind>/<role>/<name>` folders, never links), optionally under `filter`. */
|
|
164
|
+
function archiveEntries(root, filter) {
|
|
165
|
+
const out = [];
|
|
166
|
+
const dirs = (d) => fs.readdirSync(d, { withFileTypes: true }).filter((e) => e.isDirectory() && !e.isSymbolicLink() && !/^[._]/.test(e.name)).map((e) => e.name).sort();
|
|
167
|
+
for (const kind of dirs(root)) {
|
|
168
|
+
if (!/^[a-z]+$/.test(kind)) continue;
|
|
169
|
+
for (const role of dirs(path.join(root, kind)).filter((r) => ROLES.includes(r))) {
|
|
170
|
+
for (const name of dirs(path.join(root, kind, role))) {
|
|
171
|
+
const rel = `${kind}/${role}/${name}`;
|
|
172
|
+
if (filter && rel !== filter && !rel.startsWith(`${String(filter).replace(/\/+$/, '')}/`)) continue;
|
|
173
|
+
out.push({ kind, role, name, rel, abs: path.join(root, kind, role, name), task: `${kind}.${role}` });
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return out;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* The licence record for an archive entry, from the records the archive keeps: a source's row in
|
|
182
|
+
* `<kind>/sources/catalog.json`, a teacher's `manifest.json`, a corpus's or experiment's README. → { about } or
|
|
183
|
+
* { problem } (an entry with no record isn't pushed: its licence must travel with it).
|
|
184
|
+
*/
|
|
185
|
+
function archiveRecord(root, e, { games } = {}) {
|
|
186
|
+
const base = { kind: e.kind, role: e.role, path: e.rel, source: e.rel };
|
|
187
|
+
if (e.role === 'sources') {
|
|
188
|
+
const cat = readJson(path.join(root, e.kind, 'sources', 'catalog.json'));
|
|
189
|
+
const row = cat && Array.isArray(cat.sources) ? cat.sources.find((r) => r && r.id === e.name) : null;
|
|
190
|
+
if (!row) return { problem: `no row with id "${e.name}" in ${e.kind}/sources/catalog.json` };
|
|
191
|
+
if (!row.licence) return { problem: `its catalog row has no licence` };
|
|
192
|
+
return {
|
|
193
|
+
about: {
|
|
194
|
+
...base, name: row.name || e.name, url: row.url || '', version: row.version || '', licence: row.licence,
|
|
195
|
+
licence_url: row.licence_url || '', attribution: row.attribution || '', commercial_ok: row.commercial_ok,
|
|
196
|
+
training_ok: row.training_ok, share_alike: row.share_alike, consent: row.consent || '',
|
|
197
|
+
note: [row.use, row.coverage].filter(Boolean).join(' · '),
|
|
198
|
+
},
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
if (e.role === 'teachers') {
|
|
202
|
+
const m = readJson(path.join(e.abs, 'manifest.json'));
|
|
203
|
+
if (!m) return { problem: 'no manifest.json (its exact revision and every file\'s sha256)' };
|
|
204
|
+
if (!m.licence) return { problem: 'its manifest.json has no licence' };
|
|
205
|
+
const version = [m.package && m.package.name && `${m.package.name} ${m.package.version || ''}`.trim(), m.github && m.github.commit && `github ${String(m.github.commit).slice(0, 12)}`, m.version].filter(Boolean).join('; ');
|
|
206
|
+
return {
|
|
207
|
+
about: {
|
|
208
|
+
...base, name: m.name || e.name, url: (m.github && m.github.repo) || m.url || m.repo || '', version, licence: m.licence,
|
|
209
|
+
share_alike: m.share_alike === true, note: `A teacher: an outside model we learn from, never a product. ${m.why || ''}`.trim(),
|
|
210
|
+
},
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
// A corpus or experiment is ours: `<game>-<name>`, described by its README (else the role's README). A snapshot can't
|
|
214
|
+
// be renamed once sealed, so a name that doesn't start with one of the archive's games is refused, not guessed.
|
|
215
|
+
const game = e.name.split('-')[0];
|
|
216
|
+
if (!games || !games.size) return { problem: 'say which games the archive holds (--games my-game,other-game,…) so its game can be checked' };
|
|
217
|
+
if (!e.name.includes('-') || !games.has(game) || !GAME_RE.test(game)) return { problem: `"${e.name}" doesn't start with one of the games (${[...games].join(', ')}): name it <game>-<name> before pushing, since a snapshot can't be renamed` };
|
|
218
|
+
const own = path.join(e.abs, 'README.md');
|
|
219
|
+
const shared = path.join(root, e.kind, e.role, 'README.md');
|
|
220
|
+
if (!fs.existsSync(own) && !fs.existsSync(shared)) return { problem: `no README.md (what it is, which release it trained, how it was made)` };
|
|
221
|
+
return {
|
|
222
|
+
about: {
|
|
223
|
+
...base, name: e.name, game,
|
|
224
|
+
licence: 'Plaiful\'s own training data (see its README for what made it)',
|
|
225
|
+
note: firstParagraph(own) || `See ${e.kind}/${e.role}/README.md`,
|
|
226
|
+
},
|
|
227
|
+
};
|
|
94
228
|
}
|
|
95
229
|
|
|
96
230
|
function makeModels({ UsageError, ApiError, repo }) {
|
|
@@ -135,7 +269,10 @@ function makeModels({ UsageError, ApiError, repo }) {
|
|
|
135
269
|
const route = `/partner/models/releases/${enc(card.task_type)}/${enc(card.release_id)}`;
|
|
136
270
|
let up;
|
|
137
271
|
try {
|
|
138
|
-
|
|
272
|
+
const r = await ctx.call('POST', `${route}/upload`, { card, owner });
|
|
273
|
+
// A refused card (invalid, or training data it may not use) comes back as a 422 answer, not a thrown error.
|
|
274
|
+
if (r.status >= 400) throw new ApiError(r.status, r.data);
|
|
275
|
+
up = r.data;
|
|
139
276
|
} catch (e) {
|
|
140
277
|
if (!(e instanceof ApiError) || e.status >= 500 || e.status === 401 || e.status === 403) throw e;
|
|
141
278
|
failed += 1; row.problems.push(e.message); ctx.io.stderr.write(`✗ ${id}: ${e.message}\n`); continue;
|
|
@@ -207,13 +344,24 @@ function makeModels({ UsageError, ApiError, repo }) {
|
|
|
207
344
|
|
|
208
345
|
// ── Training-data snapshots ──────────────────────────────────────────────────────────────────────────────────
|
|
209
346
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
347
|
+
/** One big file: its parts' links, a batch at a time (a link lasts 15 minutes), then the join. */
|
|
348
|
+
async function pushBig(ctx, route, abs, entry) {
|
|
349
|
+
for (;;) {
|
|
350
|
+
const { data } = await ctx.call('POST', `${route}/multipart`, { file: entry, want: 24 });
|
|
351
|
+
if (data.stored || !data.links.length) {
|
|
352
|
+
if (!data.stored) await ctx.call('POST', `${route}/multipart/complete`, { file: entry, upload_id: data.upload_id });
|
|
353
|
+
return;
|
|
354
|
+
}
|
|
355
|
+
await pool(data.links, CONCURRENCY, (l) => putPart(ctx.io, path.join(abs, entry.path), entry.path, l));
|
|
356
|
+
ctx.io.stderr.write(` ${entry.path}: ${Math.min(data.parts, data.done + data.links.length)} / ${data.parts} parts\n`);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/** Push the folder `abs` as the sealed snapshot task/snapshot. Safe to re-run: stored files and parts are skipped. */
|
|
361
|
+
async function pushSnapshot(ctx, { task, snapshot, abs, exclude = null, about }) {
|
|
362
|
+
const skipped = [];
|
|
363
|
+
const rels = walk(abs, exclude, skipped);
|
|
364
|
+
if (skipped.length) ctx.io.stderr.write(` not followed (links or junctions): ${skipped.slice(0, 5).join(', ')}${skipped.length > 5 ? ` and ${skipped.length - 5} more` : ''}\n`);
|
|
217
365
|
if (!rels.length) throw new UsageError(`no files under ${abs}`);
|
|
218
366
|
ctx.io.stderr.write(`Hashing ${rels.length} file(s) under ${abs}…\n`);
|
|
219
367
|
const entries = new Array(rels.length);
|
|
@@ -222,42 +370,142 @@ function makeModels({ UsageError, ApiError, repo }) {
|
|
|
222
370
|
entries[i] = { path: rel, bytes: fs.statSync(p).size, sha256: await sha256File(p) };
|
|
223
371
|
});
|
|
224
372
|
const total = entries.reduce((n, e) => n + e.bytes, 0);
|
|
225
|
-
const manifest = `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`;
|
|
226
|
-
const mBytes = Buffer.byteLength(manifest);
|
|
227
|
-
const mSha = crypto.createHash('sha256').update(manifest).digest('hex');
|
|
228
373
|
const route = `/partner/models/datasets/${enc(task)}/${enc(snapshot)}`;
|
|
229
374
|
let uploaded = 0;
|
|
230
375
|
let stored = 0;
|
|
231
376
|
let sent = 0;
|
|
377
|
+
let partBytes = null;
|
|
232
378
|
for (let i = 0; i < entries.length; i += BATCH) {
|
|
233
379
|
const batch = entries.slice(i, i + BATCH);
|
|
234
|
-
const
|
|
235
|
-
|
|
380
|
+
const { data } = await ctx.call('POST', `${route}/upload`, { files: batch });
|
|
381
|
+
partBytes = data.part_bytes || partBytes;
|
|
236
382
|
stored += data.stored;
|
|
237
|
-
|
|
238
|
-
if (
|
|
239
|
-
|
|
240
|
-
|
|
383
|
+
// A big file's manifest entry lists its parts' sha256s, so the registry can check the stored file whole.
|
|
384
|
+
if (partBytes) await pool(batch.filter((e) => e.bytes > partBytes), 2, async (e) => { e.parts = await partHashes(path.join(abs, e.path), partBytes); });
|
|
385
|
+
const small = data.uploads.filter((u) => !u.multipart);
|
|
386
|
+
await pool(small, CONCURRENCY, async (u) => { await putFile(ctx.io, path.join(abs, u.path), u); uploaded += 1; sent += u.bytes; });
|
|
387
|
+
for (const u of data.uploads.filter((x) => x.multipart)) {
|
|
388
|
+
await pushBig(ctx, route, abs, batch.find((e) => e.path === u.path));
|
|
389
|
+
uploaded += 1; sent += u.bytes;
|
|
241
390
|
}
|
|
242
391
|
ctx.io.stderr.write(` ${Math.min(i + BATCH, entries.length)} / ${entries.length} (${uploaded} uploaded, ${stored} already stored)\n`);
|
|
243
392
|
}
|
|
244
|
-
const
|
|
393
|
+
const manifest = `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`;
|
|
394
|
+
const mSha = crypto.createHash('sha256').update(manifest).digest('hex');
|
|
395
|
+
const { data: m } = await ctx.call('POST', `${route}/upload`, { manifest: { bytes: Buffer.byteLength(manifest), sha256: mSha } });
|
|
396
|
+
const res = await ctx.io.fetch(m.manifest.url, { method: 'PUT', headers: m.manifest.headers, body: Buffer.from(manifest) });
|
|
397
|
+
if (!res.ok) throw apiFail('upload_failed', `storage refused the manifest (${res.status}); run the push again`);
|
|
245
398
|
const { data } = await ctx.call('POST', `${route}/finish`, { about });
|
|
246
|
-
|
|
399
|
+
return { ...data, uploaded, sent, stored, total_bytes: total, skipped_links: skipped.length };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
async function dataPush(ctx, [task, snapshot, dir]) {
|
|
403
|
+
if (!task || !snapshot || !dir) throw new UsageError('plaiful models data push <task> <snapshot> <folder> [--exclude <regex>] [--source "…"] [--licence "…"] [--note "…"] [--releases a,b]');
|
|
404
|
+
const abs = path.resolve(ctx.io.cwd(), dir);
|
|
405
|
+
if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) throw new UsageError(`${abs} isn't a folder`);
|
|
406
|
+
let exclude = null;
|
|
407
|
+
if (ctx.flags.exclude) { try { exclude = new RegExp(ctx.flags.exclude); } catch (e) { throw new UsageError(`--exclude isn't a regular expression (${e.message})`); } }
|
|
408
|
+
const about = { source: ctx.flags.source || abs, licence: ctx.flags.licence || '', note: ctx.flags.note || '', releases: ctx.flags.releases ? String(ctx.flags.releases).split(',') : [] };
|
|
409
|
+
const data = await pushSnapshot(ctx, { task, snapshot, abs, exclude, about });
|
|
410
|
+
ctx.out(`${task}/${snapshot}: sealed, ${data.files} file(s), ${mb(data.bytes)} (${data.uploaded} uploaded, ${mb(data.sent)}; ${data.stored} were already stored). Backed up to the Management account by replication.`, { success: true, ...data });
|
|
411
|
+
return 0;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/** `plaiful models archive push <root> [<kind>[/<role>[/<name>]]]`: every entry not sealed yet, one snapshot each. */
|
|
415
|
+
async function archivePush(ctx, [dir, filter]) {
|
|
416
|
+
if (!dir) throw new UsageError('plaiful models archive push <archive folder> [<kind>[/<role>[/<name>]]] --games my-game,other-game,…');
|
|
417
|
+
const games = new Set(String(ctx.flags.games || '').split(',').map((g) => g.trim()).filter(Boolean));
|
|
418
|
+
const root = path.resolve(ctx.io.cwd(), dir);
|
|
419
|
+
if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) throw new UsageError(`${root} isn't a folder`);
|
|
420
|
+
const entries = archiveEntries(root, filter);
|
|
421
|
+
if (!entries.length) throw new UsageError(`no <kind>/<role>/<name> entries under ${root}${filter ? ` matching ${filter}` : ''}`);
|
|
422
|
+
const { data: listed } = await ctx.call('GET', '/partner/models/datasets');
|
|
423
|
+
const sealed = new Set(listed.datasets.filter((d) => d.state === 'sealed').map((d) => `${d.task_type}/${d.snapshot}`));
|
|
424
|
+
const results = [];
|
|
425
|
+
for (const e of entries) {
|
|
426
|
+
const id = `${e.task}/${e.name}`;
|
|
427
|
+
if (sealed.has(id)) { results.push({ entry: e.rel, snapshot: id, state: 'sealed already' }); continue; }
|
|
428
|
+
const rec = archiveRecord(root, e, { games });
|
|
429
|
+
if (rec.problem) {
|
|
430
|
+
results.push({ entry: e.rel, snapshot: id, state: 'not pushed', problem: rec.problem });
|
|
431
|
+
ctx.io.stderr.write(`✗ ${e.rel}: not pushed: ${rec.problem}\n`);
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
ctx.io.stderr.write(`→ ${e.rel} as ${id}${rec.about.share_alike ? ' (share-alike: reference only)' : ''}\n`);
|
|
435
|
+
try {
|
|
436
|
+
const data = await pushSnapshot(ctx, { task: e.task, snapshot: e.name, abs: e.abs, about: rec.about });
|
|
437
|
+
results.push({ entry: e.rel, snapshot: id, state: 'sealed', files: data.files, bytes: data.bytes, uploaded_bytes: data.sent });
|
|
438
|
+
ctx.io.stderr.write(`✓ ${id}: sealed, ${data.files} file(s), ${mb(data.bytes)}\n`);
|
|
439
|
+
} catch (err) {
|
|
440
|
+
if (err instanceof UsageError) { results.push({ entry: e.rel, snapshot: id, state: 'not pushed', problem: err.message }); ctx.io.stderr.write(`✗ ${e.rel}: ${err.message}\n`); continue; }
|
|
441
|
+
throw err;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
const failed = results.filter((r) => r.state === 'not pushed').length;
|
|
445
|
+
const done = results.filter((r) => r.state === 'sealed').length;
|
|
446
|
+
ctx.out(`${done} pushed, ${results.length - done - failed} already sealed, ${failed} not pushed${failed ? ' (above: add the missing record, then run it again)' : ''}. Backed up to the Management account by replication.`, { success: failed === 0, results });
|
|
447
|
+
return failed ? 1 : 0;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/** `plaiful models archive list [<root>]`: every archive snapshot in the registry (and, with a root, what's not pushed). */
|
|
451
|
+
async function archiveList(ctx, [dir, filter]) {
|
|
452
|
+
const { data } = await ctx.call('GET', '/partner/models/datasets');
|
|
453
|
+
const inRegistry = new Map(data.datasets.filter((d) => d.about && d.about.path).map((d) => [d.about.path, d]));
|
|
454
|
+
const rows = [];
|
|
455
|
+
const local = dir ? archiveEntries(path.resolve(ctx.io.cwd(), dir), filter) : [];
|
|
456
|
+
for (const e of local) if (!inRegistry.has(e.rel)) rows.push({ path: e.rel, state: 'not pushed' });
|
|
457
|
+
for (const [p, d] of inRegistry) {
|
|
458
|
+
if (filter && p !== filter && !p.startsWith(`${String(filter).replace(/\/+$/, '')}/`)) continue;
|
|
459
|
+
rows.push({ path: p, state: d.state, snapshot: `${d.task_type}/${d.snapshot}`, files: d.files, bytes: d.bytes, licence: d.about.licence, share_alike: !!d.about.share_alike, role: d.about.role, game: d.about.game });
|
|
460
|
+
}
|
|
461
|
+
rows.sort((a, b) => a.path.localeCompare(b.path));
|
|
462
|
+
const lines = rows.length ? rows.map((r) => `${r.path} ${r.state}${r.files != null ? ` ${r.files} file(s), ${mb(r.bytes)}` : ''}${r.licence ? ` ${r.licence}` : ''}${r.share_alike ? ' SHARE-ALIKE: reference only, not training data for a release' : ''}${r.role === 'teachers' ? ' (a teacher, never a product)' : ''}`)
|
|
463
|
+
: ['No archive snapshots yet: plaiful models archive push <archive folder>'];
|
|
464
|
+
ctx.out(lines.join('\n'), { rows });
|
|
465
|
+
return 0;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/** `plaiful models archive pull <root> [<kind>[/<role>[/<name>]]]`: each snapshot back at its path under root. */
|
|
469
|
+
async function archivePull(ctx, [dir, filter]) {
|
|
470
|
+
if (!dir) throw new UsageError('plaiful models archive pull <folder> [<kind>[/<role>[/<name>]]]');
|
|
471
|
+
const root = path.resolve(ctx.io.cwd(), dir);
|
|
472
|
+
const { data } = await ctx.call('GET', '/partner/models/datasets');
|
|
473
|
+
const picked = data.datasets.filter((d) => d.state === 'sealed' && d.about && d.about.path
|
|
474
|
+
&& (!filter || d.about.path === filter || d.about.path.startsWith(`${String(filter).replace(/\/+$/, '')}/`)));
|
|
475
|
+
if (!picked.length) throw new UsageError(`no sealed archive snapshots${filter ? ` under ${filter}` : ''}`);
|
|
476
|
+
let n = 0;
|
|
477
|
+
for (const d of picked) {
|
|
478
|
+
ctx.io.stderr.write(`→ ${d.about.path}\n`);
|
|
479
|
+
n += (await pullSnapshot(ctx, d.task_type, d.snapshot, path.join(root, ...d.about.path.split('/')))).downloaded;
|
|
480
|
+
}
|
|
481
|
+
ctx.out(`${picked.length} snapshot(s) under ${root}, ${n} file(s) downloaded, every sha256 checked.`, { success: true, root, snapshots: picked.length, downloaded: n });
|
|
247
482
|
return 0;
|
|
248
483
|
}
|
|
249
484
|
|
|
250
485
|
async function dataList(ctx, [task]) {
|
|
251
486
|
const { data } = await ctx.call('GET', `/partner/models/datasets${task ? `?task=${enc(task)}` : ''}`);
|
|
252
|
-
const lines = data.datasets.length ? data.datasets.map((d) => `${d.task_type}/${d.snapshot} ${d.state} ${d.files != null ? `${d.files} file(s), ${mb(d.bytes)}` : '(push not finished)'}${d.about && d.about.releases && d.about.releases.length ? ` for ${d.about.releases.join(', ')}` : ''}`)
|
|
487
|
+
const lines = data.datasets.length ? data.datasets.map((d) => `${d.task_type}/${d.snapshot} ${d.state} ${d.files != null ? `${d.files} file(s), ${mb(d.bytes)}` : '(push not finished)'}${d.about && d.about.releases && d.about.releases.length ? ` for ${d.about.releases.join(', ')}` : ''}${d.about && d.about.share_alike ? ' SHARE-ALIKE: reference only' : ''}`)
|
|
253
488
|
: ['No training-data snapshots yet: plaiful models data push <task> <snapshot> <folder>'];
|
|
254
489
|
ctx.out(lines.join('\n'), data);
|
|
255
490
|
return 0;
|
|
256
491
|
}
|
|
257
492
|
|
|
258
493
|
async function dataPull(ctx, [task, snapshot]) {
|
|
259
|
-
if (!task || !snapshot) throw new UsageError('plaiful models data pull <task> <snapshot> [--out <dir>]');
|
|
260
|
-
|
|
494
|
+
if (!task || !snapshot) throw new UsageError('plaiful models data pull <task> <snapshot> [--out <dir> | --into <archive folder>]');
|
|
495
|
+
let out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('datasets', task, snapshot));
|
|
496
|
+
if (ctx.flags.into) {
|
|
497
|
+
// Back at its archive path, so the catalogs' relative paths still hold.
|
|
498
|
+
const { data } = await ctx.call('GET', `/partner/models/datasets/${enc(task)}/${enc(snapshot)}`);
|
|
499
|
+
const at = data.dataset.about && data.dataset.about.path;
|
|
500
|
+
if (!at) throw new UsageError(`${task}/${snapshot} has no archive path (it wasn't pushed from the archive): use --out`);
|
|
501
|
+
out = path.join(path.resolve(ctx.io.cwd(), ctx.flags.into), ...at.split('/'));
|
|
502
|
+
}
|
|
503
|
+
const r = await pullSnapshot(ctx, task, snapshot, out);
|
|
504
|
+
ctx.out(`${task}/${snapshot} → ${out}: ${r.downloaded} downloaded, ${r.skipped} already here, every sha256 checked.`, { success: true, out, ...r });
|
|
505
|
+
return 0;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
async function pullSnapshot(ctx, task, snapshot, out) {
|
|
261
509
|
const route = `/partner/models/datasets/${enc(task)}/${enc(snapshot)}/download`;
|
|
262
510
|
let after = 0;
|
|
263
511
|
let n = 0;
|
|
@@ -274,11 +522,10 @@ function makeModels({ UsageError, ApiError, repo }) {
|
|
|
274
522
|
if (data.next == null) break;
|
|
275
523
|
after = data.next;
|
|
276
524
|
}
|
|
277
|
-
|
|
278
|
-
return 0;
|
|
525
|
+
return { downloaded: n, skipped };
|
|
279
526
|
}
|
|
280
527
|
|
|
281
|
-
return { publish, list, show, status, pull, dataPush, dataList, dataPull };
|
|
528
|
+
return { publish, list, show, status, pull, dataPush, dataList, dataPull, archivePush, archiveList, archivePull };
|
|
282
529
|
}
|
|
283
530
|
|
|
284
|
-
module.exports = { makeModels, walk, sha256File, cardFiles, filesRoot, mb };
|
|
531
|
+
module.exports = { makeModels, walk, sha256File, partHashes, archiveEntries, archiveRecord, cardFiles, filesRoot, mb };
|