@plaiful/cli 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/README.md +179 -158
  2. package/package.json +21 -21
  3. package/src/cli.js +2159 -1751
  4. package/src/models.js +282 -35
package/src/models.js CHANGED
@@ -2,16 +2,23 @@
2
2
 
3
3
  /**
4
4
  * `plaiful models …`: the model registry from a game repo (PlaifulCMS docs/design/model-registry.md). Plaiful staff
5
- * only for now. A release is a card (`<release>.card.json`, Ironheads' format or version 2) plus the files it lists;
5
+ * only for now. A release is a card (`<release>.card.json`, the first card format or version 2) plus the files it lists;
6
6
  * a training-data snapshot is a folder, pushed as is with a manifest of every file's size and sha256.
7
7
  *
8
8
  * Files go straight to storage on short-lived links the platform signs with each file's sha256, so storage refuses
9
- * bytes that don't match the card. Nothing here needs AWS.
9
+ * bytes that don't match the card. A file bigger than one part (64 MiB) goes up in parts, each signed the same way, and
10
+ * a push that stops carries on from the last stored part. Nothing here needs AWS.
11
+ *
12
+ * `plaiful models archive …` pushes and pulls the plaiful-models archive (`<kind>/<role>/<name>`: sources, teachers,
13
+ * corpora, experiments), one sealed snapshot per entry, each with the licence record its catalog, manifest or README
14
+ * gives (PLAI-185).
10
15
  */
11
16
 
12
17
  const fs = require('fs');
13
18
  const path = require('path');
14
19
  const crypto = require('crypto');
20
+ const { Readable } = require('stream');
21
+ const { pipeline } = require('stream/promises');
15
22
 
16
23
  const BATCH = 500;
17
24
  const CONCURRENCY = 6;
@@ -33,14 +40,38 @@ async function pool(items, n, fn) {
33
40
  const mb = (b) => (b >= 1e9 ? `${(b / 1e9).toFixed(2)} GB` : b >= 1e6 ? `${(b / 1e6).toFixed(1)} MB` : `${Math.max(1, Math.round(b / 1e3))} KB`);
34
41
  const enc = (s) => encodeURIComponent(s);
35
42
 
36
- /** Every file under `dir` (relative, forward slashes), skipping names that match `exclude`. */
37
- function walk(dir, exclude) {
43
+ /** The sha256 of each `partBytes` slice of a file, in order (how a big file is stored in parts). */
44
+ function partHashes(file, partBytes) {
45
+ return new Promise((resolve, reject) => {
46
+ const out = [];
47
+ let h = crypto.createHash('sha256');
48
+ let inPart = 0;
49
+ fs.createReadStream(file, { highWaterMark: 4 * 1024 * 1024 }).on('error', reject).on('data', (d) => {
50
+ let at = 0;
51
+ while (at < d.length) {
52
+ const take = Math.min(partBytes - inPart, d.length - at);
53
+ h.update(d.subarray(at, at + take));
54
+ inPart += take;
55
+ at += take;
56
+ if (inPart === partBytes) { out.push(h.digest('hex')); h = crypto.createHash('sha256'); inPart = 0; }
57
+ }
58
+ }).on('end', () => { if (inPart > 0) out.push(h.digest('hex')); resolve(out); });
59
+ });
60
+ }
61
+
62
+ /**
63
+ * Every file under `dir` (relative, forward slashes), skipping names that match `exclude`. Links and junctions are
64
+ * never followed (a game reaches moved archive folders through junctions at their old paths): they're listed in
65
+ * `skipped` instead.
66
+ */
67
+ function walk(dir, exclude, skipped = []) {
38
68
  const out = [];
39
69
  const visit = (rel) => {
40
70
  for (const ent of fs.readdirSync(path.join(dir, rel), { withFileTypes: true })) {
41
71
  const r = rel ? `${rel}/${ent.name}` : ent.name;
42
72
  if (exclude && exclude.test(r)) continue;
43
- if (ent.isDirectory()) visit(r);
73
+ if (ent.isSymbolicLink()) skipped.push(r);
74
+ else if (ent.isDirectory()) visit(r);
44
75
  else if (ent.isFile()) out.push(r);
45
76
  }
46
77
  };
@@ -83,14 +114,117 @@ async function putFile(io, file, u) {
83
114
  }
84
115
  }
85
116
 
117
+ /** PUT one part of a big file (its bytes read from the file at the part's offset) to its signed link. */
118
+ async function putPart(io, file, rel, link) {
119
+ const body = Buffer.alloc(link.bytes);
120
+ const fd = fs.openSync(file, 'r');
121
+ try { fs.readSync(fd, body, 0, link.bytes, link.offset); } finally { fs.closeSync(fd); }
122
+ let res;
123
+ try { res = await io.fetch(link.url, { method: link.method || 'PUT', headers: link.headers, body }); } catch (e) {
124
+ throw Object.assign(new Error(`part ${link.part} of ${rel} didn't go through (${e.message}); run it again (stored parts are kept)`), { code: 'network' });
125
+ }
126
+ if (!res.ok) throw Object.assign(new Error(`storage refused part ${link.part} of ${rel} (${res.status}${res.status === 400 ? ': the file changed while it was being pushed?' : ''})`), { code: 'upload_failed' });
127
+ }
128
+
129
+ /** Download to `file`, streamed (a file may be bigger than memory), checked against its sha256 before it's kept. */
86
130
  async function getFile(io, url, file, sha) {
87
131
  const res = await io.fetch(url);
88
132
  if (!res.ok) throw Object.assign(new Error(`couldn't download ${path.basename(file)} (${res.status})`), { code: 'download_failed' });
89
- const buf = Buffer.from(await res.arrayBuffer());
90
- const got = crypto.createHash('sha256').update(buf).digest('hex');
91
- if (sha && got !== sha) throw Object.assign(new Error(`${file} doesn't match its sha256 after download`), { code: 'checksum' });
92
133
  fs.mkdirSync(path.dirname(file), { recursive: true });
93
- fs.writeFileSync(file, buf);
134
+ const tmp = `${file}.part`;
135
+ const h = crypto.createHash('sha256');
136
+ if (res.body && typeof res.body.getReader === 'function') {
137
+ const src = Readable.fromWeb(res.body);
138
+ src.on('data', (d) => h.update(d));
139
+ await pipeline(src, fs.createWriteStream(tmp));
140
+ } else {
141
+ const buf = Buffer.from(await res.arrayBuffer());
142
+ h.update(buf);
143
+ fs.writeFileSync(tmp, buf);
144
+ }
145
+ if (sha && h.digest('hex') !== sha) {
146
+ fs.rmSync(tmp, { force: true });
147
+ throw Object.assign(new Error(`${file} doesn't match its sha256 after download`), { code: 'checksum' });
148
+ }
149
+ fs.renameSync(tmp, file);
150
+ }
151
+
152
+ /** The archive's roles: one sealed snapshot per `<kind>/<role>/<name>` folder. */
153
+ const ROLES = ['sources', 'teachers', 'corpora', 'experiments'];
154
+ const GAME_RE = /^[a-z0-9]+$/;
155
+
156
+ const readJson = (file) => { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return null; } };
157
+ const firstParagraph = (file) => {
158
+ if (!fs.existsSync(file)) return '';
159
+ const body = fs.readFileSync(file, 'utf8').split(/\r?\n\r?\n/).map((x) => x.trim()).filter((x) => x && !x.startsWith('#'));
160
+ return (body[0] || '').replace(/\s+/g, ' ').slice(0, 1500);
161
+ };
162
+
163
+ /** Every entry of the archive at `root` (`<kind>/<role>/<name>` folders, never links), optionally under `filter`. */
164
+ function archiveEntries(root, filter) {
165
+ const out = [];
166
+ const dirs = (d) => fs.readdirSync(d, { withFileTypes: true }).filter((e) => e.isDirectory() && !e.isSymbolicLink() && !/^[._]/.test(e.name)).map((e) => e.name).sort();
167
+ for (const kind of dirs(root)) {
168
+ if (!/^[a-z]+$/.test(kind)) continue;
169
+ for (const role of dirs(path.join(root, kind)).filter((r) => ROLES.includes(r))) {
170
+ for (const name of dirs(path.join(root, kind, role))) {
171
+ const rel = `${kind}/${role}/${name}`;
172
+ if (filter && rel !== filter && !rel.startsWith(`${String(filter).replace(/\/+$/, '')}/`)) continue;
173
+ out.push({ kind, role, name, rel, abs: path.join(root, kind, role, name), task: `${kind}.${role}` });
174
+ }
175
+ }
176
+ }
177
+ return out;
178
+ }
179
+
180
+ /**
181
+ * The licence record for an archive entry, from the records the archive keeps: a source's row in
182
+ * `<kind>/sources/catalog.json`, a teacher's `manifest.json`, a corpus's or experiment's README. → { about } or
183
+ * { problem } (an entry with no record isn't pushed: its licence must travel with it).
184
+ */
185
+ function archiveRecord(root, e, { games } = {}) {
186
+ const base = { kind: e.kind, role: e.role, path: e.rel, source: e.rel };
187
+ if (e.role === 'sources') {
188
+ const cat = readJson(path.join(root, e.kind, 'sources', 'catalog.json'));
189
+ const row = cat && Array.isArray(cat.sources) ? cat.sources.find((r) => r && r.id === e.name) : null;
190
+ if (!row) return { problem: `no row with id "${e.name}" in ${e.kind}/sources/catalog.json` };
191
+ if (!row.licence) return { problem: `its catalog row has no licence` };
192
+ return {
193
+ about: {
194
+ ...base, name: row.name || e.name, url: row.url || '', version: row.version || '', licence: row.licence,
195
+ licence_url: row.licence_url || '', attribution: row.attribution || '', commercial_ok: row.commercial_ok,
196
+ training_ok: row.training_ok, share_alike: row.share_alike, consent: row.consent || '',
197
+ note: [row.use, row.coverage].filter(Boolean).join(' · '),
198
+ },
199
+ };
200
+ }
201
+ if (e.role === 'teachers') {
202
+ const m = readJson(path.join(e.abs, 'manifest.json'));
203
+ if (!m) return { problem: 'no manifest.json (its exact revision and every file\'s sha256)' };
204
+ if (!m.licence) return { problem: 'its manifest.json has no licence' };
205
+ const version = [m.package && m.package.name && `${m.package.name} ${m.package.version || ''}`.trim(), m.github && m.github.commit && `github ${String(m.github.commit).slice(0, 12)}`, m.version].filter(Boolean).join('; ');
206
+ return {
207
+ about: {
208
+ ...base, name: m.name || e.name, url: (m.github && m.github.repo) || m.url || m.repo || '', version, licence: m.licence,
209
+ share_alike: m.share_alike === true, note: `A teacher: an outside model we learn from, never a product. ${m.why || ''}`.trim(),
210
+ },
211
+ };
212
+ }
213
+ // A corpus or experiment is ours: `<game>-<name>`, described by its README (else the role's README). A snapshot can't
214
+ // be renamed once sealed, so a name that doesn't start with one of the archive's games is refused, not guessed.
215
+ const game = e.name.split('-')[0];
216
+ if (!games || !games.size) return { problem: 'say which games the archive holds (--games my-game,other-game,…) so its game can be checked' };
217
+ if (!e.name.includes('-') || !games.has(game) || !GAME_RE.test(game)) return { problem: `"${e.name}" doesn't start with one of the games (${[...games].join(', ')}): name it <game>-<name> before pushing, since a snapshot can't be renamed` };
218
+ const own = path.join(e.abs, 'README.md');
219
+ const shared = path.join(root, e.kind, e.role, 'README.md');
220
+ if (!fs.existsSync(own) && !fs.existsSync(shared)) return { problem: `no README.md (what it is, which release it trained, how it was made)` };
221
+ return {
222
+ about: {
223
+ ...base, name: e.name, game,
224
+ licence: 'Plaiful\'s own training data (see its README for what made it)',
225
+ note: firstParagraph(own) || `See ${e.kind}/${e.role}/README.md`,
226
+ },
227
+ };
94
228
  }
95
229
 
96
230
  function makeModels({ UsageError, ApiError, repo }) {
@@ -135,7 +269,10 @@ function makeModels({ UsageError, ApiError, repo }) {
135
269
  const route = `/partner/models/releases/${enc(card.task_type)}/${enc(card.release_id)}`;
136
270
  let up;
137
271
  try {
138
- up = (await ctx.call('POST', `${route}/upload`, { card, owner })).data;
272
+ const r = await ctx.call('POST', `${route}/upload`, { card, owner });
273
+ // A refused card (invalid, or training data it may not use) comes back as a 422 answer, not a thrown error.
274
+ if (r.status >= 400) throw new ApiError(r.status, r.data);
275
+ up = r.data;
139
276
  } catch (e) {
140
277
  if (!(e instanceof ApiError) || e.status >= 500 || e.status === 401 || e.status === 403) throw e;
141
278
  failed += 1; row.problems.push(e.message); ctx.io.stderr.write(`✗ ${id}: ${e.message}\n`); continue;
@@ -207,13 +344,24 @@ function makeModels({ UsageError, ApiError, repo }) {
207
344
 
208
345
  // ── Training-data snapshots ──────────────────────────────────────────────────────────────────────────────────
209
346
 
210
- async function dataPush(ctx, [task, snapshot, dir]) {
211
- if (!task || !snapshot || !dir) throw new UsageError('plaiful models data push <task> <snapshot> <folder> [--exclude <regex>] [--source "…"] [--licence "…"] [--note "…"] [--releases a,b]');
212
- const abs = path.resolve(ctx.io.cwd(), dir);
213
- if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) throw new UsageError(`${abs} isn't a folder`);
214
- let exclude = null;
215
- if (ctx.flags.exclude) { try { exclude = new RegExp(ctx.flags.exclude); } catch (e) { throw new UsageError(`--exclude isn't a regular expression (${e.message})`); } }
216
- const rels = walk(abs, exclude);
347
+ /** One big file: its parts' links, a batch at a time (a link lasts 15 minutes), then the join. */
348
+ async function pushBig(ctx, route, abs, entry) {
349
+ for (;;) {
350
+ const { data } = await ctx.call('POST', `${route}/multipart`, { file: entry, want: 24 });
351
+ if (data.stored || !data.links.length) {
352
+ if (!data.stored) await ctx.call('POST', `${route}/multipart/complete`, { file: entry, upload_id: data.upload_id });
353
+ return;
354
+ }
355
+ await pool(data.links, CONCURRENCY, (l) => putPart(ctx.io, path.join(abs, entry.path), entry.path, l));
356
+ ctx.io.stderr.write(` ${entry.path}: ${Math.min(data.parts, data.done + data.links.length)} / ${data.parts} parts\n`);
357
+ }
358
+ }
359
+
360
+ /** Push the folder `abs` as the sealed snapshot task/snapshot. Safe to re-run: stored files and parts are skipped. */
361
+ async function pushSnapshot(ctx, { task, snapshot, abs, exclude = null, about }) {
362
+ const skipped = [];
363
+ const rels = walk(abs, exclude, skipped);
364
+ if (skipped.length) ctx.io.stderr.write(` not followed (links or junctions): ${skipped.slice(0, 5).join(', ')}${skipped.length > 5 ? ` and ${skipped.length - 5} more` : ''}\n`);
217
365
  if (!rels.length) throw new UsageError(`no files under ${abs}`);
218
366
  ctx.io.stderr.write(`Hashing ${rels.length} file(s) under ${abs}…\n`);
219
367
  const entries = new Array(rels.length);
@@ -222,42 +370,142 @@ function makeModels({ UsageError, ApiError, repo }) {
222
370
  entries[i] = { path: rel, bytes: fs.statSync(p).size, sha256: await sha256File(p) };
223
371
  });
224
372
  const total = entries.reduce((n, e) => n + e.bytes, 0);
225
- const manifest = `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`;
226
- const mBytes = Buffer.byteLength(manifest);
227
- const mSha = crypto.createHash('sha256').update(manifest).digest('hex');
228
373
  const route = `/partner/models/datasets/${enc(task)}/${enc(snapshot)}`;
229
374
  let uploaded = 0;
230
375
  let stored = 0;
231
376
  let sent = 0;
377
+ let partBytes = null;
232
378
  for (let i = 0; i < entries.length; i += BATCH) {
233
379
  const batch = entries.slice(i, i + BATCH);
234
- const last = i + BATCH >= entries.length;
235
- const { data } = await ctx.call('POST', `${route}/upload`, { files: batch, ...(last ? { manifest: { bytes: mBytes, sha256: mSha } } : {}) });
380
+ const { data } = await ctx.call('POST', `${route}/upload`, { files: batch });
381
+ partBytes = data.part_bytes || partBytes;
236
382
  stored += data.stored;
237
- await pool(data.uploads, CONCURRENCY, async (u) => { await putFile(ctx.io, path.join(abs, u.path), u); uploaded += 1; sent += u.bytes; });
238
- if (last) {
239
- const res = await ctx.io.fetch(data.manifest.url, { method: 'PUT', headers: data.manifest.headers, body: Buffer.from(manifest) });
240
- if (!res.ok) throw apiFail('upload_failed', `storage refused the manifest (${res.status}); run the push again`);
383
+ // A big file's manifest entry lists its parts' sha256s, so the registry can check the stored file whole.
384
+ if (partBytes) await pool(batch.filter((e) => e.bytes > partBytes), 2, async (e) => { e.parts = await partHashes(path.join(abs, e.path), partBytes); });
385
+ const small = data.uploads.filter((u) => !u.multipart);
386
+ await pool(small, CONCURRENCY, async (u) => { await putFile(ctx.io, path.join(abs, u.path), u); uploaded += 1; sent += u.bytes; });
387
+ for (const u of data.uploads.filter((x) => x.multipart)) {
388
+ await pushBig(ctx, route, abs, batch.find((e) => e.path === u.path));
389
+ uploaded += 1; sent += u.bytes;
241
390
  }
242
391
  ctx.io.stderr.write(` ${Math.min(i + BATCH, entries.length)} / ${entries.length} (${uploaded} uploaded, ${stored} already stored)\n`);
243
392
  }
244
- const about = { source: ctx.flags.source || abs, licence: ctx.flags.licence || '', note: ctx.flags.note || '', releases: ctx.flags.releases ? String(ctx.flags.releases).split(',') : [] };
393
+ const manifest = `${entries.map((e) => JSON.stringify(e)).join('\n')}\n`;
394
+ const mSha = crypto.createHash('sha256').update(manifest).digest('hex');
395
+ const { data: m } = await ctx.call('POST', `${route}/upload`, { manifest: { bytes: Buffer.byteLength(manifest), sha256: mSha } });
396
+ const res = await ctx.io.fetch(m.manifest.url, { method: 'PUT', headers: m.manifest.headers, body: Buffer.from(manifest) });
397
+ if (!res.ok) throw apiFail('upload_failed', `storage refused the manifest (${res.status}); run the push again`);
245
398
  const { data } = await ctx.call('POST', `${route}/finish`, { about });
246
- ctx.out(`${task}/${snapshot}: sealed, ${data.files} file(s), ${mb(data.bytes)} (${uploaded} uploaded, ${mb(sent)}; ${stored} were already stored). Backed up to the Management account by replication.`, { success: true, ...data, uploaded, total_bytes: total });
399
+ return { ...data, uploaded, sent, stored, total_bytes: total, skipped_links: skipped.length };
400
+ }
401
+
402
+ async function dataPush(ctx, [task, snapshot, dir]) {
403
+ if (!task || !snapshot || !dir) throw new UsageError('plaiful models data push <task> <snapshot> <folder> [--exclude <regex>] [--source "…"] [--licence "…"] [--note "…"] [--releases a,b]');
404
+ const abs = path.resolve(ctx.io.cwd(), dir);
405
+ if (!fs.existsSync(abs) || !fs.statSync(abs).isDirectory()) throw new UsageError(`${abs} isn't a folder`);
406
+ let exclude = null;
407
+ if (ctx.flags.exclude) { try { exclude = new RegExp(ctx.flags.exclude); } catch (e) { throw new UsageError(`--exclude isn't a regular expression (${e.message})`); } }
408
+ const about = { source: ctx.flags.source || abs, licence: ctx.flags.licence || '', note: ctx.flags.note || '', releases: ctx.flags.releases ? String(ctx.flags.releases).split(',') : [] };
409
+ const data = await pushSnapshot(ctx, { task, snapshot, abs, exclude, about });
410
+ ctx.out(`${task}/${snapshot}: sealed, ${data.files} file(s), ${mb(data.bytes)} (${data.uploaded} uploaded, ${mb(data.sent)}; ${data.stored} were already stored). Backed up to the Management account by replication.`, { success: true, ...data });
411
+ return 0;
412
+ }
413
+
414
+ /** `plaiful models archive push <root> [<kind>[/<role>[/<name>]]]`: every entry not sealed yet, one snapshot each. */
415
+ async function archivePush(ctx, [dir, filter]) {
416
+ if (!dir) throw new UsageError('plaiful models archive push <archive folder> [<kind>[/<role>[/<name>]]] --games my-game,other-game,…');
417
+ const games = new Set(String(ctx.flags.games || '').split(',').map((g) => g.trim()).filter(Boolean));
418
+ const root = path.resolve(ctx.io.cwd(), dir);
419
+ if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) throw new UsageError(`${root} isn't a folder`);
420
+ const entries = archiveEntries(root, filter);
421
+ if (!entries.length) throw new UsageError(`no <kind>/<role>/<name> entries under ${root}${filter ? ` matching ${filter}` : ''}`);
422
+ const { data: listed } = await ctx.call('GET', '/partner/models/datasets');
423
+ const sealed = new Set(listed.datasets.filter((d) => d.state === 'sealed').map((d) => `${d.task_type}/${d.snapshot}`));
424
+ const results = [];
425
+ for (const e of entries) {
426
+ const id = `${e.task}/${e.name}`;
427
+ if (sealed.has(id)) { results.push({ entry: e.rel, snapshot: id, state: 'sealed already' }); continue; }
428
+ const rec = archiveRecord(root, e, { games });
429
+ if (rec.problem) {
430
+ results.push({ entry: e.rel, snapshot: id, state: 'not pushed', problem: rec.problem });
431
+ ctx.io.stderr.write(`✗ ${e.rel}: not pushed: ${rec.problem}\n`);
432
+ continue;
433
+ }
434
+ ctx.io.stderr.write(`→ ${e.rel} as ${id}${rec.about.share_alike ? ' (share-alike: reference only)' : ''}\n`);
435
+ try {
436
+ const data = await pushSnapshot(ctx, { task: e.task, snapshot: e.name, abs: e.abs, about: rec.about });
437
+ results.push({ entry: e.rel, snapshot: id, state: 'sealed', files: data.files, bytes: data.bytes, uploaded_bytes: data.sent });
438
+ ctx.io.stderr.write(`✓ ${id}: sealed, ${data.files} file(s), ${mb(data.bytes)}\n`);
439
+ } catch (err) {
440
+ if (err instanceof UsageError) { results.push({ entry: e.rel, snapshot: id, state: 'not pushed', problem: err.message }); ctx.io.stderr.write(`✗ ${e.rel}: ${err.message}\n`); continue; }
441
+ throw err;
442
+ }
443
+ }
444
+ const failed = results.filter((r) => r.state === 'not pushed').length;
445
+ const done = results.filter((r) => r.state === 'sealed').length;
446
+ ctx.out(`${done} pushed, ${results.length - done - failed} already sealed, ${failed} not pushed${failed ? ' (above: add the missing record, then run it again)' : ''}. Backed up to the Management account by replication.`, { success: failed === 0, results });
447
+ return failed ? 1 : 0;
448
+ }
449
+
450
+ /** `plaiful models archive list [<root>]`: every archive snapshot in the registry (and, with a root, what's not pushed). */
451
+ async function archiveList(ctx, [dir, filter]) {
452
+ const { data } = await ctx.call('GET', '/partner/models/datasets');
453
+ const inRegistry = new Map(data.datasets.filter((d) => d.about && d.about.path).map((d) => [d.about.path, d]));
454
+ const rows = [];
455
+ const local = dir ? archiveEntries(path.resolve(ctx.io.cwd(), dir), filter) : [];
456
+ for (const e of local) if (!inRegistry.has(e.rel)) rows.push({ path: e.rel, state: 'not pushed' });
457
+ for (const [p, d] of inRegistry) {
458
+ if (filter && p !== filter && !p.startsWith(`${String(filter).replace(/\/+$/, '')}/`)) continue;
459
+ rows.push({ path: p, state: d.state, snapshot: `${d.task_type}/${d.snapshot}`, files: d.files, bytes: d.bytes, licence: d.about.licence, share_alike: !!d.about.share_alike, role: d.about.role, game: d.about.game });
460
+ }
461
+ rows.sort((a, b) => a.path.localeCompare(b.path));
462
+ const lines = rows.length ? rows.map((r) => `${r.path} ${r.state}${r.files != null ? ` ${r.files} file(s), ${mb(r.bytes)}` : ''}${r.licence ? ` ${r.licence}` : ''}${r.share_alike ? ' SHARE-ALIKE: reference only, not training data for a release' : ''}${r.role === 'teachers' ? ' (a teacher, never a product)' : ''}`)
463
+ : ['No archive snapshots yet: plaiful models archive push <archive folder>'];
464
+ ctx.out(lines.join('\n'), { rows });
465
+ return 0;
466
+ }
467
+
468
+ /** `plaiful models archive pull <root> [<kind>[/<role>[/<name>]]]`: each snapshot back at its path under root. */
469
+ async function archivePull(ctx, [dir, filter]) {
470
+ if (!dir) throw new UsageError('plaiful models archive pull <folder> [<kind>[/<role>[/<name>]]]');
471
+ const root = path.resolve(ctx.io.cwd(), dir);
472
+ const { data } = await ctx.call('GET', '/partner/models/datasets');
473
+ const picked = data.datasets.filter((d) => d.state === 'sealed' && d.about && d.about.path
474
+ && (!filter || d.about.path === filter || d.about.path.startsWith(`${String(filter).replace(/\/+$/, '')}/`)));
475
+ if (!picked.length) throw new UsageError(`no sealed archive snapshots${filter ? ` under ${filter}` : ''}`);
476
+ let n = 0;
477
+ for (const d of picked) {
478
+ ctx.io.stderr.write(`→ ${d.about.path}\n`);
479
+ n += (await pullSnapshot(ctx, d.task_type, d.snapshot, path.join(root, ...d.about.path.split('/')))).downloaded;
480
+ }
481
+ ctx.out(`${picked.length} snapshot(s) under ${root}, ${n} file(s) downloaded, every sha256 checked.`, { success: true, root, snapshots: picked.length, downloaded: n });
247
482
  return 0;
248
483
  }
249
484
 
250
485
  async function dataList(ctx, [task]) {
251
486
  const { data } = await ctx.call('GET', `/partner/models/datasets${task ? `?task=${enc(task)}` : ''}`);
252
- const lines = data.datasets.length ? data.datasets.map((d) => `${d.task_type}/${d.snapshot} ${d.state} ${d.files != null ? `${d.files} file(s), ${mb(d.bytes)}` : '(push not finished)'}${d.about && d.about.releases && d.about.releases.length ? ` for ${d.about.releases.join(', ')}` : ''}`)
487
+ const lines = data.datasets.length ? data.datasets.map((d) => `${d.task_type}/${d.snapshot} ${d.state} ${d.files != null ? `${d.files} file(s), ${mb(d.bytes)}` : '(push not finished)'}${d.about && d.about.releases && d.about.releases.length ? ` for ${d.about.releases.join(', ')}` : ''}${d.about && d.about.share_alike ? ' SHARE-ALIKE: reference only' : ''}`)
253
488
  : ['No training-data snapshots yet: plaiful models data push <task> <snapshot> <folder>'];
254
489
  ctx.out(lines.join('\n'), data);
255
490
  return 0;
256
491
  }
257
492
 
258
493
  async function dataPull(ctx, [task, snapshot]) {
259
- if (!task || !snapshot) throw new UsageError('plaiful models data pull <task> <snapshot> [--out <dir>]');
260
- const out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('datasets', task, snapshot));
494
+ if (!task || !snapshot) throw new UsageError('plaiful models data pull <task> <snapshot> [--out <dir> | --into <archive folder>]');
495
+ let out = path.resolve(ctx.io.cwd(), ctx.flags.out || path.join('datasets', task, snapshot));
496
+ if (ctx.flags.into) {
497
+ // Back at its archive path, so the catalogs' relative paths still hold.
498
+ const { data } = await ctx.call('GET', `/partner/models/datasets/${enc(task)}/${enc(snapshot)}`);
499
+ const at = data.dataset.about && data.dataset.about.path;
500
+ if (!at) throw new UsageError(`${task}/${snapshot} has no archive path (it wasn't pushed from the archive): use --out`);
501
+ out = path.join(path.resolve(ctx.io.cwd(), ctx.flags.into), ...at.split('/'));
502
+ }
503
+ const r = await pullSnapshot(ctx, task, snapshot, out);
504
+ ctx.out(`${task}/${snapshot} → ${out}: ${r.downloaded} downloaded, ${r.skipped} already here, every sha256 checked.`, { success: true, out, ...r });
505
+ return 0;
506
+ }
507
+
508
+ async function pullSnapshot(ctx, task, snapshot, out) {
261
509
  const route = `/partner/models/datasets/${enc(task)}/${enc(snapshot)}/download`;
262
510
  let after = 0;
263
511
  let n = 0;
@@ -274,11 +522,10 @@ function makeModels({ UsageError, ApiError, repo }) {
274
522
  if (data.next == null) break;
275
523
  after = data.next;
276
524
  }
277
- ctx.out(`${task}/${snapshot} → ${out}: ${n} downloaded, ${skipped} already here, every sha256 checked.`, { success: true, out, downloaded: n, skipped });
278
- return 0;
525
+ return { downloaded: n, skipped };
279
526
  }
280
527
 
281
- return { publish, list, show, status, pull, dataPush, dataList, dataPull };
528
+ return { publish, list, show, status, pull, dataPush, dataList, dataPull, archivePush, archiveList, archivePull };
282
529
  }
283
530
 
284
- module.exports = { makeModels, walk, sha256File, cardFiles, filesRoot, mb };
531
+ module.exports = { makeModels, walk, sha256File, partHashes, archiveEntries, archiveRecord, cardFiles, filesRoot, mb };