ainize 0.1.0

package/dist/commands/dataset.js

@@ -0,0 +1,157 @@
+/**
+ * `ainize dataset get <patch-id|sha>` — the TRAINING SET behind a published knowledge (lineage design §13, §12.3).
+ *
+ * Different from `ainize teach dataset get <id>`, which is one of MY datasets on this node: this asks a knowledge
+ * for the questions it was taught from, under the access level its creator chose (§6.1):
+ *
+ *   public       anyone may download and build on them
+ *   derivative   only people building on this knowledge — the CLI posts a signed derive intent and fetches with the
+ *                token it gets back, which is also what the creator's "built on N times" counter is made of
+ *   private      nobody: the refusal says so in plain words and exits 3
+ *
+ *   ainize dataset get krx-all-2761                      # what it is: questions, licence, access, where it came from
+ *   ainize dataset get krx-all-2761 -o questions.jsonl   # the exact canonical bytes (re-uploadable as my own dataset)
+ *   ainize dataset get krx-all-2761 --manifest           # row origin, benchmark hash, PII scan, declaration
+ */
+import { createHash } from 'node:crypto';
+import { writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { signMessage } from '@ainize/core';
+import { CliError, PROG } from '../context.js';
+import { c, emit, fmtBytes, kv, shortHash, warn } from '../output.js';
+import { TeachSession } from './teach-dataset.js';
+const sha256 = (b) => createHash('sha256').update(b).digest('hex');
+const SHA_RE = /^[0-9a-f]{64}$/i;
+/** A refusal a person can act on: the node's reason, plus what to do about it. Exit 3, like a missing login. */
+function refusal(e) {
+    const err = e;
+    const msg = String(err?.message ?? e);
+    if (/^dataset_private/.test(msg))
+        throw new CliError(`${msg}\nOnly the verification questions on the record are public. Ask the creator, or build on a knowledge whose training set is shared.`, 3, err.details);
+    if (/^dataset_derivative_only/.test(msg))
+        throw new CliError(`${msg}\nSign the request with a teaching key (this CLI does) — if it still refuses, the creator's node no longer serves it.`, 3, err.details);
+    if (/^dataset_unavailable/.test(msg))
+        throw new CliError(`${msg}\nNo node here holds the bytes. Try again against a node that does: \`${PROG} dataset get <id> --node <url>\`.`, 3, err.details);
+    throw e;
+}
+/**
+ * The training set of a published knowledge. With `-o` the bytes are written: the public path is a plain download,
+ * a `derivative` set goes through a signed derive intent and `/p2p/dataset/<sha>` with the token it returns.
+ */
+export async function datasetGetPublished(ctx, target, opts = {}) {
+    const s = await TeachSession.open(ctx, opts);
+    const { id, name } = await resolveTarget(s, target);
+    const dataset = await s.get(`/api/patches/${encodeURIComponent(id)}/dataset`).catch(refusal);
+    const out = { node: s.client.baseUrl, patch_id: id, ...(name ? { name } : {}), dataset };
+    if (opts.manifest)
+        out.manifest = (await s.get(`/api/patches/${encodeURIComponent(id)}/dataset/manifest`).catch(refusal)).manifest;
+    if (opts.out)
+        out.saved = await download(s, id, dataset, opts, ctx);
+    emit(ctx, out, renderPublishedDataset);
+    return out;
+}
+/** A sha256 names the bytes, not the knowledge: find the knowledge on this node that published them. */
+async function resolveTarget(s, target) {
+    if (!SHA_RE.test(target))
+        return { id: target };
+    const sha = target.toLowerCase();
+    const cat = await s.client.get('/api/catalog?limit=200', { auth: false }).catch(() => ({ items: [] }));
+    const hit = cat.items.map((e) => e.anchor).find((a) => a.dataset?.sha256 === sha);
+    if (!hit)
+        throw new CliError(`no knowledge on ${s.client.baseUrl} published the training set ${shortHash(sha, 12)} — give the knowledge id instead`, 3);
+    return { id: hit.id, name: hit.name };
+}
+async function download(s, id, d, opts, ctx) {
+    let bytes = null;
+    let via = 'node';
+    const direct = await s.raw(`/api/patches/${encodeURIComponent(id)}/dataset/rows`);
+    if (direct.ok)
+        bytes = Buffer.from(await direct.arrayBuffer());
+    else if (direct.status === 403 && d.access === 'derivative') {
+        // the derive intent IS the record that someone is building on this knowledge (§6.1) — it is counted on the parent
+        const intent = await s.post(`/api/patches/${encodeURIComponent(id)}/derive-intent`, {}).catch(refusal);
+        // Item 312: the intent is a commitment, not a formality — say what was just promised, in the node's own words.
+        if (intent.terms)
+            warn(ctx, intent.terms);
+        const ts = Date.now();
+        const auth = `${s.key.address}:${ts}:${signMessage(`dataset:${intent.sha256}:${ts}`, s.key.privateKey)}`;
+        const res = await fetch(`${s.client.baseUrl}/p2p/dataset/${intent.sha256}`, { headers: { 'x-ainize-auth': auth, 'x-ainize-derive': intent.token } });
+        if (!res.ok)
+            throw new CliError(`the training set was not served: HTTP ${res.status} ${(await res.text()).slice(0, 200)}`, 3);
+        bytes = Buffer.from(await res.arrayBuffer());
+        via = 'derive';
+    }
+    else {
+        const body = await direct.text();
+        let msg = `HTTP ${direct.status}`;
+        try {
+            msg = String(JSON.parse(body).error ?? msg);
+        }
+        catch { /* not json */ }
+        refusal(new CliError(msg, 3));
+    }
+    // notes are the publisher's own words about a row; they are in the file only because the publisher opted in, and
+    // they are written to disk only if this caller asks for them too (§6.2)
+    let notesRemoved = 0;
+    let written = bytes;
+    if (!opts.includeNotes) {
+        const lines = written.toString('utf8').split('\n').filter((l) => l.trim());
+        const stripped = lines.map((l) => {
+            try {
+                const o = JSON.parse(l);
+                if (o.note === undefined)
+                    return l;
+                notesRemoved++;
+                const { note: _n, ...rest } = o;
+                return JSON.stringify(rest);
+            }
+            catch {
+                return l;
+            }
+        });
+        if (notesRemoved)
+            written = Buffer.from(stripped.join('\n') + '\n', 'utf8');
+    }
+    const path = resolve(opts.out);
+    writeFileSync(path, written);
+    const verified = sha256(written) === d.sha256;
+    if (!verified && !notesRemoved)
+        warn(ctx, 'the bytes do not hash to the fingerprint on the record — do not build on this file');
+    return { path, bytes: written.length, sha256: sha256(written), verified, notes_removed: notesRemoved, via };
+}
+export function renderPublishedDataset(r) {
+    const d = r.dataset;
+    const access = d.access === 'public' ? 'public — anyone can download and build on them'
+        : d.access === 'derivative' ? 'shared with people building on this knowledge'
+            : 'private — nobody can build on this knowledge';
+    const lines = [
+        c.head(`training set of ${c.id(r.patch_id)}${r.name ? ` · ${r.name}` : ''}`),
+        kv([
+            ['questions', String(d.rows)],
+            ['access', access],
+            ['licence', d.license ?? '—'],
+            ['fingerprint', shortHash(d.sha256, 16)],
+            ['on this node', d.held ? 'yes' : 'no — fetched from a peer when needed'],
+            ...(d.parents.length ? [['built from', d.parents.map((p) => `${p.patch_id} (${p.rows} questions)`).join(', ')]] : []),
+            ...(d.benchmark_samples ? [['checked with', `${d.benchmark_samples} questions (the full list travels with the set)`]] : []),
+        ]),
+    ];
+    if (d.preview?.length) {
+        lines.push('', c.head(`first ${d.preview.length} questions`));
+        for (const row of d.preview)
+            lines.push(`  ${c.dim('Q')} ${row.prompt}\n  ${c.dim('A')} ${row.answer}`);
+    }
+    if (r.manifest)
+        lines.push('', c.head('manifest'), JSON.stringify(r.manifest, null, 1));
+    if (r.saved) {
+        lines.push('', c.ok('✓ ') + `saved ${r.saved.path} (${fmtBytes(r.saved.bytes)})${r.saved.via === 'derive' ? c.dim(' · fetched as a derivative — recorded against your teaching key') : ''}`);
+        if (r.saved.via === 'derive')
+            lines.push(c.dim(`  a lesson trained on these questions must name ${r.patch_id} as its base (\`${PROG} teach train <dataset> --on ${r.patch_id}\`), or the node will refuse to publish it`));
+        if (r.saved.notes_removed)
+            lines.push(c.dim(`  ${r.saved.notes_removed} note(s) left out — pass --include-notes to keep them (and the published fingerprint)`));
+        else if (r.saved.verified)
+            lines.push(c.dim('  fingerprint verified — uploading this file gives you the same questions to build on'));
+    }
+    return lines.join('\n');
+}
+//# sourceMappingURL=dataset.js.map

package/dist/commands/dataset.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/commands/dataset.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAmB,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGlD,MAAM,MAAM,GAAG,CAAC,CAAkB,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACpF,MAAM,MAAM,GAAG,iBAAiB,CAAC;AAyBjC,gHAAgH;AAChH,SAAS,OAAO,CAAC,CAAU;IACzB,MAAM,GAAG,GAAG,CAAa,CAAC;IAC1B,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC;IACtC,IAAI,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,oIAAoI,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IACjN,IAAI,0BAA0B,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,uHAAuH,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5M,IAAI,sBAAsB,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,MAAM,IAAI,QAAQ,CAAC,GAAG,GAAG,yEAAyE,IAAI,mCAAmC,EAAE,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IACjM,MAAM,CAAC,CAAC;AACV,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,GAAe,EAAE,MAAc,EAAE,OAAuB,EAAE;IAClG,MAAM,CAAC,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IAC7C,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,MAAM,aAAa,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,GAAG,CAAmB,gBAAgB,kBAAkB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/G,MAAM,GAAG,GAA2B,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC;IACjH,IAAI,IAAI,CAAC,QAAQ;QAAE,GAAG,CAAC,QAAQ,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAwC,gBAAgB,kBAAkB,CAAC,EAAE,CAAC,mBAAmB,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;IAC1K,IAAI,IAAI,CAAC,GAAG;QAAE,GAAG,CAAC,KAAK,GAAG,MAAM,QAAQ,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC;IACpE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,sBAAsB,CAAC,CAAC;IACvC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,wGAAwG;AACxG,KAAK,UAAU,aAAa,CAAC,CAAe,EAAE,MAAc;IAC1D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC;IAChD,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,CAAuF,wBAAwB,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7L,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,GAAG,CAAC,CAAC;IAClF,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,QAAQ,CAAC,mBAAmB,CAAC,CAAC,MAAM,CAAC,OAAO,+BAA+B,SAAS,CAAC,GAAG,EAAE,EAAE,CAAC,kCAAkC,EAAE,CAAC,CAAC,CAAC;IACxJ,OAAO,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;AACxC,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,CAAe,EAAE,EAAU,EAAE,CAAmB,EAAE,IAAoB,EAAE,GAAe;IAC7G,IAAI,KAAK,GAAkB,IAAI,CAAC;IAChC,IAAI,GAAG,GAAsB,MAAM,CAAC;IACpC,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,gBAAgB,kBAAkB,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC;IAClF,IAAI,MAAM,CAAC,EAAE;QAAE,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;SAC1D,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;QAC5D,kHAAkH;QAClH,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,IAAI,CAAoD,gBAAgB,kBAAkB,CAAC,EAAE,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC1J,+GAA+G;QAC/G,IAAI,MAAM,CAAC,KAAK;YAAE,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,IAAI,WAAW,CAAC,WAAW,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACzG,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,gBAAgB,MAAM,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,eAAe,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QACrJ,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,QAAQ,CAAC,yCAAyC,GAAG,CAAC,MAAM,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC9H,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;QAC7C,GAAG,GAAG,QAAQ,CAAC;IACjB,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,GAAG,GAAG,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,IAAI,CAAC;YAAC,GAAG,GAAG,MAAM,CAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAwB,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,cAAc,CAAC,CAAC;QACrG,OAAO,CAAC,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,iHAAiH;IACjH,wEAAwE;IACxE,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,OAAO,GAAG,KAAM,CAAC;IACrB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC/B,IAAI,CAAC;gBACH,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAA4B,CAAC;gBACnD,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS;oBAAE,OAAO,CAAC,CAAC;gBACnC,YAAY,EAAE,CAAC;gBACf,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;gBAChC,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;YAAC,MAAM,CAAC;gBAAC,OAAO,CAAC,CAAC;YAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QACH,IAAI,YAAY;YAAE,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,GAAI,CAAC,CAAC;IAChC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC7B,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAC9C,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY;QAAE,IAAI,CAAC,GAAG,EAAE,oFAAoF,CAAC,CAAC;IAChI,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,aAAa,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC;AAC9G,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,CAAyB;IAC9D,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;IACpB,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,gDAAgD;QACrF,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,CAAC,+CAA+C;YAC3E,CAAC,CAAC,8CAA8C,CAAC;IACrD,MAAM,KAAK,GAAG;QACZ,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC5E,EAAE,CAAC;YACD,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7B,CAAC,QAAQ,EAAE,MAAM,CAAC;YAClB,CAAC,SAAS,EAAE,CAAC,CAAC,OAAO,IAAI,GAAG,CAAC;YAC7B,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACxC,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,sCAAsC,CAAC;YACzE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAqB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACzI,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC,iBAAiB,iDAAiD,CAAqB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAChJ,CAAC;KACH,CAAC;IACF,IAAI,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,MAAM,YAAY,CAAC,CAAC,CAAC;QAC9D,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1G,CAAC;IACD,IAAI,CAAC,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACxF,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACZ,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,iEAAiE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7L,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC,QAAQ,mBAAmB,IAAI,+BAA+B,CAAC,CAAC,QAAQ,4CAA4C,CAAC,CAAC,CAAC;QAC3N,IAAI,CAAC,CAAC,KAAK,CAAC,aAAa;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,aAAa,uFAAuF,CAAC,CAAC,CAAC;aAC3J,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,uFAAuF,CAAC,CAAC,CAAC;IACxI,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/commands/drive.d.ts

@@ -0,0 +1,32 @@
+import { type CliContext } from '../context.js';
+import { ok } from '../output.js';
+export declare const DEFAULT_AINDRIVE_SERVER: string;
+export declare function aindriveBin(): string;
+export interface DriveStatus {
+    configured: boolean;
+    running: boolean;
+    pid: number | null;
+    folder: string;
+    server: string | null;
+    drive_id: string | null;
+    url: string | null;
+    login_hint: string;
+    files: {
+        path: string;
+        size: number;
+        mtime: number;
+    }[];
+    cli?: string;
+}
+export declare function driveStatus(ctx: CliContext, a?: {
+    files?: boolean;
+}): Promise<DriveStatus>;
+export declare function driveAction(ctx: CliContext, action: 'up' | 'stop' | 'sync' | 'status'): Promise<unknown>;
+/** One-time browser pairing: runs `aindrive login` interactively inside the node's drive folder. */
+export declare function driveLogin(ctx: CliContext, a?: {
+    server?: string;
+    name?: string;
+    noOpen?: boolean;
+}): Promise<number>;
+export { ok };
+//# sourceMappingURL=drive.d.ts.map

package/dist/commands/drive.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"drive.d.ts","sourceRoot":"","sources":["../../src/commands/drive.ts"],"names":[],"mappings":"AAQA,OAAO,EAAkB,KAAK,UAAU,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAkC,EAAE,EAAS,MAAM,cAAc,CAAC;AAKzE,eAAO,MAAM,uBAAuB,QAAiE,CAAC;AAEtG,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED,MAAM,WAAW,WAAW;IAAG,UAAU,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAAC,GAAG,CAAC,EAAE,MAAM,CAAC;CAAE;AAEzQ,wBAAsB,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,CAAC,GAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,OAAO,CAAC,WAAW,CAAC,CAYpG;AAED,wBAAsB,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAO9G;AAED,oGAAoG;AACpG,wBAAsB,UAAU,CAAC,GAAG,EAAE,UAAU,EAAE,CAAC,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAkB/H;AAED,OAAO,EAAE,EAAE,EAAE,CAAC"}

package/dist/commands/drive.js

@@ -0,0 +1,68 @@
+/**
+ * `ainize drive status|up|stop|sync|login` — aindrive integration (files & change history of the node).
+ */
+import { spawn } from 'node:child_process';
+import { existsSync, mkdirSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { join } from 'node:path';
+import { NodeClient } from '../client.js';
+import { CliError, PROG } from '../context.js';
+import { c, emit, fmtBytes, fmtTime, kv, ok, table } from '../output.js';
+import { requireConfig } from './init.js';
+const require = createRequire(import.meta.url);
+export const DEFAULT_AINDRIVE_SERVER = process.env.AINDRIVE_SERVER ?? 'https://aindrive.ainetwork.ai';
+export function aindriveBin() {
+    try {
+        return require.resolve('aindrive/dist/aindrive.mjs');
+    }
+    catch {
+        throw new CliError('aindrive CLI not installed (npm install aindrive)');
+    }
+}
+export async function driveStatus(ctx, a = {}) {
+    const d = await new NodeClient(ctx).get('/api/drive', { auth: false });
+    emit(ctx, d, (x) => [
+        kv([
+            ['folder', x.folder], ['paired', x.configured ? c.ok('yes') : c.warn(`no — run \`${PROG} drive login\``)], ['agent', x.running ? c.ok(`running (pid ${x.pid})`) : c.dim('stopped')],
+            ['server', x.server ?? '-'], ['drive id', x.drive_id ?? '-'], ['url', x.url ? c.id(x.url) : '-'], ['files', x.files.length],
+        ]),
+        a.files ? '\n' + table(x.files.slice(0, 200), [
+            { key: 'p', title: 'PATH', get: (f) => f.path }, { key: 's', title: 'SIZE', get: (f) => fmtBytes(f.size), align: 'right' }, { key: 't', title: 'MODIFIED', get: (f) => fmtTime(f.mtime) },
+        ]) : '',
+    ].filter(Boolean).join('\n'));
+    return d;
+}
+export async function driveAction(ctx, action) {
+    const r = await new NodeClient(ctx).post('/api/drive', { action }, { timeoutMs: 120_000 });
+    emit(ctx, r, (x) => {
+        if (action === 'sync')
+            return c.ok('✓ ') + `drive folder synced (${x.written ?? 0} file(s) written)`;
+        return (x.ok === false ? c.warn('! ') : c.ok('✓ ')) + (x.message ?? JSON.stringify(x));
+    });
+    return r;
+}
+/** One-time browser pairing: runs `aindrive login` interactively inside the node's drive folder. */
+export async function driveLogin(ctx, a = {}) {
+    const cfg = requireConfig(ctx);
+    const folder = join(cfg.dataDir, 'drive');
+    if (!existsSync(folder))
+        mkdirSync(folder, { recursive: true });
+    const server = a.server ?? DEFAULT_AINDRIVE_SERVER;
+    const args = [aindriveBin(), 'login', '--server', server];
+    if (a.name)
+        args.push('--name', a.name);
+    if (a.noOpen || !process.env.DISPLAY)
+        args.push('--no-open');
+    process.stdout.write([
+        c.head('aindrive pairing'),
+        `  folder : ${folder}`, `  server : ${server}`,
+        c.dim('  A browser sign-in link will be printed — open it, click Authorize, and this folder becomes a drive.'),
+        c.dim(`  After pairing, Ctrl+C here and run \`${PROG} drive up\` to serve it in the background.`), '',
+    ].join('\n'));
+    return new Promise((resolve) => {
+        const child = spawn(process.execPath, args, { cwd: folder, stdio: 'inherit', env: { ...process.env, AINDRIVE_SERVER: server } });
+        child.on('exit', (code) => resolve(code ?? 0));
+    });
+}
+export { ok };
+//# sourceMappingURL=drive.js.map

package/dist/commands/drive.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"drive.js","sourceRoot":"","sources":["../../src/commands/drive.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAmB,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE1C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE/C,MAAM,CAAC,MAAM,uBAAuB,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,+BAA+B,CAAC;AAEtG,MAAM,UAAU,WAAW;IACzB,IAAI,CAAC;QAAC,OAAO,OAAO,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,MAAM,IAAI,QAAQ,CAAC,mDAAmD,CAAC,CAAC;IAAC,CAAC;AAClJ,CAAC;AAID,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAe,EAAE,IAAyB,EAAE;IAC5E,MAAM,CAAC,GAAG,MAAM,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,CAAc,YAAY,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACpF,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;QAClB,EAAE,CAAC;YACD,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,IAAI,gBAAgB,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACnL,CAAC,QAAQ,EAAE,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,IAAI,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;SAC5H,CAAC;QACF,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YAC5C,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE;SAC1L,CAAC,CAAC,CAAC,CAAC,EAAE;KACR,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9B,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAe,EAAE,MAAyC;IAC1F,MAAM,CAAC,GAAG,MAAM,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,CAAuD,YAAY,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;IACjJ,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE;QACjB,IAAI,MAAM,KAAK,MAAM;YAAE,OAAO,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,CAAC,OAAO,IAAI,CAAC,mBAAmB,CAAC;QACrG,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,CAAC;AACX,CAAC;AAED,oGAAoG;AACpG,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAe,EAAE,IAA0D,EAAE;IAC5G,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,IAAI,uBAAuB,CAAC;IACnD,MAAM,IAAI,GAAG,CAAC,WAAW,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,CAAC,CAAC,IAAI;QAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;IACxC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO;QAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;QACnB,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC;QAC1B,cAAc,MAAM,EAAE,EAAE,cAAc,MAAM,EAAE;QAC9C,CAAC,CAAC,GAAG,CAAC,uGAAuG,CAAC;QAC9G,CAAC,CAAC,GAAG,CAAC,0CAA0C,IAAI,4CAA4C,CAAC,EAAE,EAAE;KACtG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACd,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,eAAe,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;QACjI,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,OAAO,EAAE,EAAE,EAAE,CAAC"}

package/dist/commands/huggingface-dataset.d.ts

@@ -0,0 +1,63 @@
+import { type CliContext } from '../context.js';
+import { type DatasetOpts, type TrainOpts } from './teach-dataset.js';
+type Fetcher = typeof fetch;
+export interface HuggingFaceOptions extends DatasetOpts, TrainOpts {
+    config?: string;
+    split?: string;
+    revision?: string;
+    file?: string;
+    limit?: number;
+    offset?: number;
+    hfTokenFile?: string;
+    train?: boolean;
+}
+export interface HuggingFaceSource {
+    kind: 'huggingface';
+    url: string;
+    repository: string;
+    revision: string;
+    config: string | null;
+    split: string | null;
+    file: string | null;
+    offset: number | null;
+    importedRows: number | null;
+    totalRows: number | null;
+    sampled: boolean | null;
+    format: string;
+    sha256: string;
+    bytes: number;
+    inputSha256: string;
+    inputBytes: number;
+    license: unknown;
+    columns: unknown;
+}
+export declare function parseHuggingFaceUrl(input: string): {
+    repository: string;
+    file?: string;
+    revision?: string;
+    config?: string;
+    split?: string;
+};
+export declare function mapHuggingFaceColumns(bytes: Buffer, format: string, columns?: string): Buffer;
+export declare function readHuggingFaceDataset(input: string, opts?: HuggingFaceOptions, fetcher?: Fetcher): Promise<{
+    bytes: Buffer;
+    inputBytes: Buffer;
+    filename: string;
+    source: HuggingFaceSource;
+}>;
+export declare function datasetImportHuggingFace(ctx: CliContext, input: string, opts?: HuggingFaceOptions): Promise<{
+    dataset_id: string;
+    source: HuggingFaceSource;
+    provenance: string;
+    node: string;
+    file: string;
+    bytes: number;
+    sha256: string;
+    job?: import("./teach-dataset.js").CreateJobResult;
+    waited?: boolean;
+    dataset: import("@ainize/core").TeachDataset;
+    report: import("./teach-dataset.js").DatasetReport;
+    created: boolean;
+}>;
+export {};
+//# sourceMappingURL=huggingface-dataset.d.ts.map

package/dist/commands/huggingface-dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"huggingface-dataset.d.ts","sourceRoot":"","sources":["../../src/commands/huggingface-dataset.ts"],"names":[],"mappings":"AAGA,OAAO,EAAY,KAAK,UAAU,EAAE,MAAM,eAAe,CAAC;AAE1D,OAAO,EAA+B,KAAK,WAAW,EAAE,KAAK,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAOnG,KAAK,OAAO,GAAG,OAAO,KAAK,CAAC;AAG5B,MAAM,WAAW,kBAAmB,SAAQ,WAAW,EAAE,SAAS;IAChE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAClE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAC;CACxE;AACD,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,aAAa,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IACvE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACjE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACtG,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;CAC5H;AAOD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAe5I;AAgDD,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAoB7F;AAED,wBAAsB,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,kBAAuB,EAAE,OAAO,GAAE,OAAe,GAAG,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,iBAAiB,CAAA;CAAE,CAAC,CAyFhN;AAED,wBAAsB,wBAAwB,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,kBAAuB;;;;;;;;;;;;;GAe3G"}

package/dist/commands/huggingface-dataset.js

@@ -0,0 +1,254 @@
+import { createHash } from 'node:crypto';
+import { mkdirSync, mkdtempSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { extname, join } from 'node:path';
+import { CliError } from '../context.js';
+import { emit } from '../output.js';
+import { datasetUpload, renderUpload } from './teach-dataset.js';
+const HUB = 'https://huggingface.co';
+const VIEWER = 'https://datasets-server.huggingface.co';
+const MAX_BYTES = 32 * 1024 * 1024;
+const MAX_ROWS = 10000;
+const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
+function object(value, label) {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        throw new CliError(`invalid Hugging Face ${label}`);
+    return value;
+}
+export function parseHuggingFaceUrl(input) {
+    let url;
+    try {
+        url = new URL(input);
+    }
+    catch {
+        throw new CliError('expected https://huggingface.co/datasets/<owner>/<name>');
+    }
+    if (url.origin !== HUB || url.username || url.password || url.search || url.hash || /%2f|%5c|\\/i.test(url.pathname)) {
+        throw new CliError('use an HTTPS huggingface.co dataset URL without credentials, query parameters or fragments');
+    }
+    const parts = url.pathname.replace(/\/$/, '').split('/').slice(1).map(part => decodeURIComponent(part));
+    if (parts[0] !== 'datasets' || !parts[1] || !parts[2] || ![parts[1], parts[2]].every(part => /^[A-Za-z0-9_.-]+$/.test(part))) {
+        throw new CliError('expected a dataset URL, not a model or Space URL: https://huggingface.co/datasets/<owner>/<name>');
+    }
+    const repository = `${parts[1]}/${parts[2]}`;
+    if (parts.length === 3)
+        return { repository };
+    if (parts[3] === 'resolve' && parts.length >= 6 && parts[4])
+        return { repository, revision: parts[4], file: parts.slice(5).join('/') };
+    if (parts[3] === 'viewer' && parts.length === 6 && parts[4] && parts[5])
+        return { repository, config: parts[4], split: parts[5] };
+    throw new CliError('use the dataset repository URL, /viewer/<config>/<split>, or /resolve/<revision>/<file>');
+}
+function filePath(value) {
+    if (!value || value.includes('\\') || value.split('/').some(part => !part || part === '.' || part === '..'))
+        throw new CliError('invalid Hugging Face file path');
+    return value.split('/').map(encodeURIComponent).join('/');
+}
+async function download(url, fetcher, token) {
+    for (let redirects = 0; redirects <= 5; redirects++) {
+        const trusted = url.origin === HUB || url.origin === VIEWER;
+        const fileHost = url.hostname.endsWith('.hf.co') || url.hostname.endsWith('.huggingface.co');
+        if (url.protocol !== 'https:' || url.port || url.username || url.password || (!trusted && !fileHost))
+            throw new CliError('Hugging Face redirected to an unsupported host');
+        const headers = { accept: 'application/json, text/plain, */*' };
+        if (trusted && token)
+            headers.authorization = `Bearer ${token}`;
+        const response = await fetcher(url, { headers, redirect: 'manual', signal: AbortSignal.timeout(60000) });
+        if ([301, 302, 303, 307, 308].includes(response.status)) {
+            const location = response.headers.get('location');
+            await response.body?.cancel();
+            if (!location)
+                throw new CliError('Hugging Face redirect has no location');
+            url = new URL(location, url);
+            continue;
+        }
+        if (!response.ok) {
+            await response.body?.cancel();
+            throw new CliError(`Hugging Face HTTP ${response.status}${response.status === 401 || response.status === 403 ? '; use --hf-token-file for an account with access (gated terms must already be accepted)' : '; dataset viewer may be unavailable: use --file with a supported data file'}`);
+        }
+        if (Number(response.headers.get('content-length')) > MAX_BYTES) {
+            await response.body?.cancel();
+            throw new CliError('Hugging Face response exceeds the 32 MiB import limit');
+        }
+        if (!response.body)
+            throw new CliError('empty Hugging Face response');
+        const chunks = [];
+        let length = 0;
+        const reader = response.body.getReader();
+        try {
+            for (;;) {
+                const chunk = await reader.read();
+                if (chunk.done)
+                    break;
+                length += chunk.value.byteLength;
+                if (length > MAX_BYTES)
+                    throw new CliError('Hugging Face response exceeds the 32 MiB import limit');
+                chunks.push(Buffer.from(chunk.value));
+            }
+        }
+        finally {
+            await reader.cancel();
+            reader.releaseLock();
+        }
+        return { bytes: Buffer.concat(chunks), revision: response.headers.get('x-revision') };
+    }
+    throw new CliError('too many Hugging Face redirects');
+}
+export function mapHuggingFaceColumns(bytes, format, columns) {
+    if (!columns || !['jsonl', 'json'].includes(format))
+        return bytes;
+    const mapping = object(JSON.parse(columns), 'column mapping');
+    const fields = ['prompt', 'answer', 'alt_prompt', 'note'];
+    if (!mapping.prompt || !mapping.answer || Object.keys(mapping).some(key => !fields.includes(key)) || Object.values(mapping).some(value => typeof value !== 'string' || !value))
+        throw new CliError('--columns must map prompt/answer and optional alt_prompt/note to existing column names');
+    const parsed = format === 'jsonl' ? bytes.toString('utf8').trim().split('\n').map(line => JSON.parse(line)) : JSON.parse(bytes.toString('utf8'));
+    if (!Array.isArray(parsed))
+        throw new CliError('JSON column mapping requires an array of objects');
+    const rows = parsed.map((value, index) => {
+        const original = object(value, `row ${index}`);
+        const mapped = {};
+        for (const [field, column] of Object.entries(mapping)) {
+            if (!Object.hasOwn(original, column))
+                throw new CliError(`missing column ${column} at row ${index}`);
+            const cell = original[column];
+            if ((cell === null || cell === undefined) && !['prompt', 'answer'].includes(field))
+                continue;
+            if (typeof cell !== 'string' && typeof cell !== 'boolean' && !(typeof cell === 'number' && Number.isFinite(cell)))
+                throw new CliError(`column ${column} at row ${index} must be a scalar, not an object or array`);
+            mapped[field] = String(cell);
+        }
+        return mapped;
+    });
+    return Buffer.from(format === 'jsonl' ? rows.map(row => JSON.stringify(row)).join('\n') + '\n' : JSON.stringify(rows));
+}
+export async function readHuggingFaceDataset(input, opts = {}, fetcher = fetch) {
+    const target = parseHuggingFaceUrl(input);
+    for (const key of ['revision', 'file', 'config', 'split']) {
+        if (target[key] && opts[key] && target[key] !== opts[key])
+            throw new CliError(`URL and --${key} disagree`);
+    }
+    const selected = { ...target, ...Object.fromEntries(Object.entries(opts).filter(([, value]) => value !== undefined)) };
+    let token;
+    if (opts.hfTokenFile) {
+        const stats = statSync(opts.hfTokenFile);
+        if (!stats.isFile() || (stats.mode & 0o077) !== 0)
+            throw new CliError('--hf-token-file must be a private file (chmod 600)');
+        token = readFileSync(opts.hfTokenFile, 'utf8').trim();
+        if (!token || /\s/.test(token))
+            throw new CliError('invalid Hugging Face token file');
+    }
+    const request = (url) => download(url, fetcher, token);
+    const metadataUrl = new URL(`/api/datasets/${target.repository}/revision/${encodeURIComponent(selected.revision || 'main')}`, HUB);
+    const metadata = object(JSON.parse((await request(metadataUrl)).bytes.toString('utf8')), 'repository metadata');
+    if (typeof metadata.sha !== 'string' || !/^[a-f0-9]{40}$/.test(metadata.sha))
+        throw new CliError('Hugging Face did not return an immutable revision');
+    if (metadata.id !== target.repository)
+        throw new CliError('Hugging Face repository identity changed');
+    const source = { kind: 'huggingface', url: input, repository: target.repository, revision: metadata.sha,
+        config: null, split: null, file: null, offset: null, importedRows: null, totalRows: null, sampled: null,
+        format: '', sha256: '', bytes: 0, inputSha256: '', inputBytes: 0, license: metadata.cardData?.license ?? null,
+        columns: opts.columns ? JSON.parse(opts.columns) : null };
+    let bytes;
+    let filename;
+    if (selected.file) {
+        if (opts.offset !== undefined || opts.limit !== undefined || selected.config || selected.split)
+            throw new CliError('--file cannot be combined with config/split/offset/limit; it imports the file as supplied');
+        const encoded = filePath(selected.file);
+        source.format = opts.format || extname(selected.file).slice(1).toLowerCase();
+        if (!['jsonl', 'json', 'csv', 'tsv', 'txt'].includes(source.format))
+            throw new CliError('file imports support jsonl/json/csv/tsv/txt; use a repository URL and --config/--split for viewer-backed Parquet datasets');
+        bytes = (await request(new URL(`/datasets/${target.repository}/resolve/${metadata.sha}/${encoded}`, HUB))).bytes;
+        filename = `data.${source.format}`;
+        source.file = selected.file;
+    }
+    else {
+        const offset = opts.offset ?? 0;
+        if (!Number.isSafeInteger(offset) || offset < 0 || (opts.limit !== undefined && (!Number.isSafeInteger(opts.limit) || opts.limit < 1 || opts.limit > MAX_ROWS)))
+            throw new CliError('offset must be a nonnegative integer; limit must be 1..10000');
+        const splitsUrl = new URL('/splits', VIEWER);
+        splitsUrl.searchParams.set('dataset', target.repository);
+        const splitsResponse = await request(splitsUrl);
+        if (splitsResponse.revision !== metadata.sha)
+            throw new CliError('dataset viewer revision differs from the requested source; use --file for an immutable file import');
+        const splitData = object(JSON.parse(splitsResponse.bytes.toString('utf8')), 'splits');
+        if (!Array.isArray(splitData.splits))
+            throw new CliError('invalid Hugging Face split list');
+        const splits = splitData.splits.map(value => object(value, 'split'));
+        const configs = [...new Set(splits.map(split => split.config))];
+        const config = selected.config ?? (configs.length === 1 ? configs[0] : undefined);
+        if (typeof config !== 'string')
+            throw new CliError(`choose --config; available configurations: ${configs.slice(0, 12).join(', ') || 'not ready in the viewer; use --file'}`);
+        const candidates = splits.filter(split => split.config === config);
+        const splitName = selected.split ?? (candidates.some(split => split.split === 'train') ? 'train' : candidates.length === 1 ? candidates[0].split : undefined);
+        if (typeof splitName !== 'string' || !candidates.some(split => split.split === splitName))
+            throw new CliError('choose an available --split for the selected --config');
+        const rows = [];
+        let serializedBytes = 0;
+        let totalRows = null;
+        let targetRows = opts.limit ?? MAX_ROWS;
+        while (rows.length < targetRows) {
+            const pageOffset = offset + rows.length;
+            const pageLength = Math.min(100, targetRows - rows.length);
+            const rowsUrl = new URL('/rows', VIEWER);
+            for (const [key, value] of Object.entries({ dataset: target.repository, config, split: splitName, offset: pageOffset, length: pageLength }))
+                rowsUrl.searchParams.set(key, String(value));
+            const pageResponse = await request(rowsUrl);
+            if (pageResponse.revision !== metadata.sha)
+                throw new CliError('dataset revision changed during pagination; no dataset uploaded');
+            const page = object(JSON.parse(pageResponse.bytes.toString('utf8')), 'rows');
+            if (page.partial !== false || !Number.isSafeInteger(page.num_rows_total) || page.num_rows_total < 0 || !Array.isArray(page.rows))
+                throw new CliError('incomplete or invalid Hugging Face rows response');
+            if (totalRows !== null && totalRows !== page.num_rows_total)
+                throw new CliError('dataset row count changed during pagination');
+            totalRows = page.num_rows_total;
+            if (offset >= totalRows)
+                throw new CliError('offset selects no rows');
+            if (opts.limit === undefined && totalRows - offset > MAX_ROWS)
+                throw new CliError('dataset exceeds 10000 rows; choose an explicit --limit (and optionally --offset), or import a supported file');
+            targetRows = Math.min(opts.limit ?? totalRows - offset, totalRows - offset);
+            if (page.rows.length !== Math.min(pageLength, targetRows - rows.length))
+                throw new CliError('missing or unexpected Hugging Face rows');
+            for (const [index, value] of page.rows.entries()) {
+                const item = object(value, 'row wrapper');
+                if (item.row_idx !== pageOffset + index || !Array.isArray(item.truncated_cells) || item.truncated_cells.length)
+                    throw new CliError('out-of-order or truncated Hugging Face row');
+                const row = object(item.row, 'row');
+                serializedBytes += Buffer.byteLength(JSON.stringify(row)) + 1;
+                if (serializedBytes > MAX_BYTES)
+                    throw new CliError('import exceeds the 32 MiB limit; select fewer rows with --limit');
+                rows.push(row);
+            }
+        }
+        bytes = Buffer.from(rows.map(row => JSON.stringify(row)).join('\n') + '\n');
+        filename = `huggingface-${target.repository.replace('/', '-')}.jsonl`;
+        Object.assign(source, { config, split: splitName, offset, importedRows: rows.length, totalRows, sampled: offset > 0 || rows.length !== totalRows, format: 'jsonl' });
+    }
+    if (!bytes.length || bytes.length > MAX_BYTES)
+        throw new CliError('import must contain 1 byte..32 MiB');
+    const inputBytes = bytes;
+    source.inputSha256 = sha256(inputBytes);
+    source.inputBytes = inputBytes.length;
+    bytes = mapHuggingFaceColumns(inputBytes, source.format, opts.columns);
+    if (bytes.length > MAX_BYTES)
+        throw new CliError('mapped import exceeds the 32 MiB limit');
+    source.sha256 = sha256(bytes);
+    source.bytes = bytes.length;
+    return { bytes, inputBytes, filename, source };
+}
+export async function datasetImportHuggingFace(ctx, input, opts = {}) {
+    if (opts.wait && !opts.train)
+        throw new CliError('--wait requires --train');
+    const imported = await readHuggingFaceDataset(input, opts);
+    const cache = join(ctx.home, 'hf-imports');
+    mkdirSync(cache, { recursive: true, mode: 0o700 });
+    const folder = mkdtempSync(join(cache, 'import-'));
+    const filename = join(folder, imported.filename);
+    const provenance = join(folder, 'source.json');
+    writeFileSync(filename, imported.bytes, { flag: 'wx', mode: 0o600 });
+    writeFileSync(join(folder, `input.${imported.source.format}`), imported.inputBytes, { flag: 'wx', mode: 0o600 });
+    writeFileSync(provenance, JSON.stringify(imported.source, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
+    const result = await datasetUpload(ctx, filename, { ...opts, columns: ['json', 'jsonl'].includes(imported.source.format) ? undefined : opts.columns, format: imported.source.format, silent: true });
+    const output = { ...result, dataset_id: result.dataset.id, source: imported.source, provenance };
+    emit(ctx, output, value => `${renderUpload(value)}\nHugging Face: ${value.source.repository}@${value.source.revision}\nSource evidence: ${value.provenance}\nImported into this node; not published to Hugging Face or the public catalog.`);
+    return output;
+}
+//# sourceMappingURL=huggingface-dataset.js.map