@voltro/data-transfer 0.86.0 → 0.87.0

package/dist/exporter.js

@@ -1,257 +1,271 @@
 import { snapshotPrelude as e } from "./consistency.js";
 import { classificationFromSnapshot as t, maskRow as n, planMasking as r } from "./masking.js";
-import { BundleError as i, MaskingError as a, ScopeError as o } from "./errors.js";
-import { computeSubsetIds as s, subsetToScope as c } from "./subset.js";
-import { resolveScope as l } from "./scope.js";
-import { captureAssetsToCas as u } from "./casAssetPhase.js";
-import { compressionExt as d, makeCompressor as f, pickCompression as p } from "./compress.js";
-import { ASSETS_DIR as m, ASSET_INDEX_FILE as h, DATA_DIR as g, LEDGER_FILE as _, MANIFEST_FILE as v, README_FILE as y, encodeManifest as b } from "./manifest.js";
-import { buildReadme as x } from "./readme.js";
-import { reportProgress as S, withTableSpan as C } from "./progress.js";
-import { readLedger as w, tableDone as T, writeLedger as E } from "./ledger.js";
-import { rowsToNdjson as D } from "./ndjson.js";
-import { topoSortTables as O } from "./topo.js";
-import { Cause as k, Effect as A, Exit as j, Schedule as M, Stream as N } from "effect";
-import { streamTable as P } from "@voltro/database";
-import { fingerprintSchema as F } from "@voltro/database/sql";
+import { emptyTally as i, mergeTally as a, tallyRow as o } from "./sealedKeys.js";
+import { BundleError as s, MaskingError as c, ScopeError as l } from "./errors.js";
+import { computeSubsetIds as u, subsetToScope as d } from "./subset.js";
+import { resolveScope as f } from "./scope.js";
+import { ASSETS_DIR as p, ASSET_INDEX_FILE as m, DATA_DIR as h, LEDGER_FILE as g, MANIFEST_FILE as _, README_FILE as v, encodeManifest as y } from "./manifest.js";
+import { captureAssetsToCas as b } from "./casAssetPhase.js";
+import { compressionExt as x, makeCompressor as S, pickCompression as C } from "./compress.js";
+import { buildReadme as w } from "./readme.js";
+import { reportProgress as T, withTableSpan as E } from "./progress.js";
+import { readLedger as D, tableDone as O, writeLedger as k } from "./ledger.js";
+import { rowsToNdjson as A } from "./ndjson.js";
+import { topoSortTables as j } from "./topo.js";
+import { Cause as M, Effect as N, Exit as P, Schedule as F, Stream as I } from "effect";
+import { streamTable as L } from "@voltro/database";
+import { fingerprintSchema as R } from "@voltro/database/sql";
 //#region src/exporter.ts
-var I = (e) => {
+var z = (e) => {
 	let t = e, n = [];
 	typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
 	let r = typeof t?.message == "string" ? t.message.trim() : "";
-	if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(I(t.cause)) : n.push(r), n.length === 0) {
+	if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(z(t.cause)) : n.push(r), n.length === 0) {
 		let t = String(e ?? "").trim();
-		return t === "" || t === "[object Object]" ? L(e) : t;
+		return t === "" || t === "[object Object]" ? B(e) : t;
 	}
 	return n.join(": ");
-}, L = (e) => {
+}, B = (e) => {
 	try {
 		let t = JSON.stringify(e);
 		return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
 	} catch {
 		return "unknown error (not serialisable)";
 	}
-}, R = (e, t) => {
+}, V = (e, t) => {
 	let n = e.tables.find((e) => e.name === t), r = {};
 	for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
 	return r;
-}, z = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), B = (e, t) => {
+}, H = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), U = (e, t) => {
 	if (t.length === 0) return e;
 	let n = { ...e };
 	for (let e of t) delete n[e];
 	return n;
-}, V = (e, t) => {
+}, W = (e, t) => {
 	let n = e.tables.find((e) => e.name === t);
 	if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
 	let r = n.columns.find((e) => e.type === "id");
 	if (r !== void 0) return { column: r.name };
 	let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
 	return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
-}, H = (e, t, n) => A.tryPromise({
+}, G = (e, t, n) => N.tryPromise({
 	try: async () => {
-		let { createWriteStream: r } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: l } = await import("node:stream"), u = a("sha256"), d = 0, p = new l({ transform(e, t, n) {
+		let { createWriteStream: r } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: l } = await import("node:stream"), u = a("sha256"), d = 0, f = new l({ transform(e, t, n) {
 			u.update(e);
 			let r = e;
 			for (let e = 0; e < r.length; e++) r[e] === 10 && d++;
 			n(null, e);
-		} }), m = c.fromWeb(N.toReadableStream(e)), h = `${t}.tmp-${o()}`;
+		} }), p = c.fromWeb(I.toReadableStream(e)), m = `${t}.tmp-${o()}`;
 		try {
-			await s(m, p, f(n), r(h)), await i.rename(h, t);
+			await s(p, f, S(n), r(m)), await i.rename(m, t);
 		} catch (e) {
-			throw await i.rm(h, { force: !0 }).catch(() => {}), e;
+			throw await i.rm(m, { force: !0 }).catch(() => {}), e;
 		}
 		return {
 			rowCount: d,
 			checksum: u.digest("hex")
 		};
 	},
-	catch: (e) => new i({
-		reason: `write table failed: ${I(e)}`,
+	catch: (e) => new s({
+		reason: `write table failed: ${z(e)}`,
 		path: t
 	})
-}), U = (e, t) => A.tryPromise({
+}), K = (e, t) => N.tryPromise({
 	try: async () => {
 		await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
 	},
-	catch: (t) => new i({
-		reason: `append failed: ${I(t)}`,
+	catch: (t) => new s({
+		reason: `append failed: ${z(t)}`,
 		path: e
 	})
-}), W = (e, t) => A.gen(function* () {
-	let r = [], i = t.ordered.length;
-	for (let a = 0; a < t.ordered.length; a++) {
-		let s = t.ordered[a], c = R(t.snapshot, s), l = V(t.snapshot, s);
-		if ("refusal" in l) return yield* A.fail(new o({ reason: `export: ${l.refusal}` }));
-		let u = l.column, f = `${g}/${s}.ndjson${d(t.compression)}`;
-		if (T(t.ledger, s)) {
-			let e = t.ledger.tables[s];
-			S(t.onProgress, {
+}), q = (e, t) => N.gen(function* () {
+	let r = [], s = t.ordered.length;
+	for (let c = 0; c < t.ordered.length; c++) {
+		let u = t.ordered[c], d = V(t.snapshot, u), f = W(t.snapshot, u);
+		if ("refusal" in f) return yield* N.fail(new l({ reason: `export: ${f.refusal}` }));
+		let p = f.column, m = `${h}/${u}.ndjson${x(t.compression)}`;
+		if (O(t.ledger, u)) {
+			let e = t.ledger.tables[u];
+			e.sealed !== void 0 && a(t.sealed, e.sealed), T(t.onProgress, {
 				phase: "export",
-				table: s,
-				index: a,
-				tableCount: i,
+				table: u,
+				index: c,
+				tableCount: s,
 				event: "done",
 				rowsDone: e.rowCount,
 				total: e.rowCount
 			}), r.push({
-				name: s,
-				file: f,
+				name: u,
+				file: m,
 				rowCount: e.rowCount,
 				checksum: e.checksum,
-				columnTypes: c,
-				primaryKey: u
+				columnTypes: d,
+				primaryKey: p
 			});
 			continue;
 		}
-		S(t.onProgress, {
+		T(t.onProgress, {
 			phase: "export",
-			table: s,
-			index: a,
-			tableCount: i,
+			table: u,
+			index: c,
+			tableCount: s,
 			event: "start",
 			rowsDone: 0
 		});
-		let p = t.resolved.predicateFor(s), m = P(e, {
-			table: s,
-			pk: u,
+		let g = t.resolved.predicateFor(u), _ = L(e, {
+			table: u,
+			pk: p,
 			chunkSize: t.chunkSize,
 			retry: t.retry,
-			...p === void 0 ? {} : { where: p }
-		}), h = z(t.snapshot, s), _ = h.length === 0 ? m : m.pipe(N.map((e) => B(e, h))), v = t.mask ? _.pipe(N.map((e) => n(e, s, t.mask.actions, t.mask.seed, c))) : _, { rowCount: y, checksum: b } = yield* C("export", s, a, i, (e) => e.rowCount, H(D(v, c), t.bundleFile(f), t.compression));
-		t.ledger.tables[s] = {
-			rowCount: y,
-			checksum: b
-		}, yield* E(t.ledgerPath, t.ledger), S(t.onProgress, {
+			...g === void 0 ? {} : { where: g }
+		}), v = H(t.snapshot, u), y = v.length === 0 ? _ : _.pipe(I.map((e) => U(e, v))), b = t.mask ? y.pipe(I.map((e) => n(e, u, t.mask.actions, t.mask.seed, d))) : y, S = i(), C = b.pipe(I.map((e) => (o(e, S), e))), { rowCount: w, checksum: D } = yield* E("export", u, c, s, (e) => e.rowCount, G(A(C, d), t.bundleFile(m), t.compression));
+		t.ledger.tables[u] = {
+			rowCount: w,
+			checksum: D,
+			sealed: S
+		}, a(t.sealed, S), yield* k(t.ledgerPath, t.ledger), T(t.onProgress, {
 			phase: "export",
-			table: s,
-			index: a,
-			tableCount: i,
+			table: u,
+			index: c,
+			tableCount: s,
 			event: "done",
-			rowsDone: y,
-			total: y
+			rowsDone: w,
+			total: w
 		}), r.push({
-			name: s,
-			file: f,
-			rowCount: y,
-			checksum: b,
-			columnTypes: c,
-			primaryKey: u
+			name: u,
+			file: m,
+			rowCount: w,
+			checksum: D,
+			columnTypes: d,
+			primaryKey: p
 		});
 	}
 	return r;
-}), G = (e) => ({
+}), J = (e) => ({
 	_tag: "RawSqlFragment",
 	strings: [e],
 	values: []
-}), K = (t, n, r) => A.tryPromise({
+}), Y = (t, n, r) => N.tryPromise({
 	try: () => t.transactional(async (t) => {
-		if (t.raw) for (let r of e(n)) await t.raw(G(r));
-		let i = await A.runPromiseExit(r(t));
-		if (j.isFailure(i)) throw Error(k.pretty(i.cause));
+		if (t.raw) for (let r of e(n)) await t.raw(J(r));
+		let i = await N.runPromiseExit(r(t));
+		if (P.isFailure(i)) throw Error(M.pretty(i.cause));
 		return i.value;
 	}),
-	catch: (e) => e instanceof i ? e : new i({ reason: `snapshot export failed: ${I(e)}` })
-}), q = (e) => A.gen(function* () {
-	let n = e.scope ?? { kind: "all" }, o = p(e.compression), d = e.chunkSize ?? 1e3, f = e.retryTimes ?? 5, S = M.max([M.exponential("50 millis").pipe(M.jittered), M.recurs(f)]), C = yield* A.promise(() => import("node:path")), T = (t) => C.join(e.outDir, t);
-	yield* A.tryPromise({
+	catch: (e) => e instanceof s ? e : new s({ reason: `snapshot export failed: ${z(e)}` })
+}), X = (e) => N.gen(function* () {
+	let n = e.scope ?? { kind: "all" }, a = C(e.compression), o = e.chunkSize ?? 1e3, l = e.retryTimes ?? 5, x = F.max([F.exponential("50 millis").pipe(F.jittered), F.recurs(l)]), S = yield* N.promise(() => import("node:path")), T = (t) => S.join(e.outDir, t);
+	yield* N.tryPromise({
 		try: async () => {
 			let t = await import("node:fs/promises");
-			await t.mkdir(T(g), { recursive: !0 }), e.assets && await t.mkdir(T(m), { recursive: !0 });
+			await t.mkdir(T(h), { recursive: !0 }), e.assets && await t.mkdir(T(p), { recursive: !0 });
 		},
-		catch: (t) => new i({
-			reason: `mkdir failed: ${I(t)}`,
+		catch: (t) => new s({
+			reason: `mkdir failed: ${z(t)}`,
 			path: e.outDir
 		})
 	});
-	let D = T(_), k = yield* w(D), j = e.subset ? c(yield* s(e.store, e.snapshot, e.subset), e.snapshot) : yield* l(n, e.snapshot, {
+	let E = T(g), O = yield* D(E), A = e.subset ? d(yield* u(e.store, e.snapshot, e.subset), e.snapshot) : yield* f(n, e.snapshot, {
 		store: e.store,
 		...e.tenantTables ? { tenantTables: e.tenantTables } : {}
-	}), N = O(j.tables, e.snapshot), P = e.consistency ?? "live", L, R;
+	}), M = j(A.tables, e.snapshot), P = e.consistency ?? "live", I, L;
 	if (e.masking) {
-		let n = e.classification ?? t(e.snapshot), i = N.map((t) => ({
+		let n = e.classification ?? t(e.snapshot), i = M.map((t) => ({
 			name: t,
 			columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
 				name: e.name,
 				type: e.type
 			}))
-		})), o = r(i, n, e.masking);
-		if (o.unclassified.length > 0 || o.invalid.length > 0) return yield* A.fail(new a({
-			unclassified: o.unclassified,
-			...o.invalid.length > 0 ? { invalidActions: o.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
+		})), a = r(i, n, e.masking);
+		if (a.unclassified.length > 0 || a.invalid.length > 0) return yield* N.fail(new c({
+			unclassified: a.unclassified,
+			...a.invalid.length > 0 ? { invalidActions: a.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
 		}));
-		R = {
-			actions: o.actions,
+		L = {
+			actions: a.actions,
 			seed: e.masking.seed
 		};
-		let s = Object.entries(o.actions).filter(([, e]) => e !== "keep").map(([e]) => e);
-		L = {
+		let o = Object.entries(a.actions).filter(([, e]) => e !== "keep").map(([e]) => e);
+		I = {
 			...e.masking.id ? { policyId: e.masking.id } : {},
-			transformed: s
+			transformed: o
 		};
 	}
-	let z = {
+	let B = i(), V = {
+		sealed: B,
 		snapshot: e.snapshot,
-		ordered: N,
-		resolved: j,
-		compression: o,
-		chunkSize: d,
-		retry: S,
-		ledger: k,
-		ledgerPath: D,
+		ordered: M,
+		resolved: A,
+		compression: a,
+		chunkSize: o,
+		retry: x,
+		ledger: O,
+		ledgerPath: E,
 		bundleFile: T,
-		...R ? { mask: R } : {},
+		...L ? { mask: L } : {},
 		...e.onProgress ? { onProgress: e.onProgress } : {}
-	}, B = P === "snapshot" ? yield* K(e.store, e.dialect, (e) => W(e, z)) : yield* W(e.store, z), V = 0, H = 0;
+	}, H = P === "snapshot" ? yield* Y(e.store, e.dialect, (e) => q(e, V)) : yield* q(e.store, V), U = 0, W = 0;
 	if (e.assets) {
-		let t = T(h), n = yield* u({
-			source: e.assets,
-			assetsDir: T(m),
+		let t = T(m), n = yield* b({
+			source: {
+				...e.assets,
+				list: () => e.assets.list({
+					dir: e.outDir,
+					tables: H,
+					compression: a
+				})
+			},
+			assetsDir: T(p),
 			progress: {
-				done: (e) => e in k.assets,
-				mark: (e, n) => (k.assets[e] = !0, U(t, JSON.stringify(n)).pipe(A.andThen(E(D, k))))
+				done: (e) => e in O.assets,
+				mark: (e, n) => (O.assets[e] = !0, K(t, JSON.stringify(n)).pipe(N.andThen(k(E, O))))
 			}
 		});
-		V = n.count, H = n.totalBytes, n.missing.length > 0 && (yield* A.logWarning(`${n.missing.length} of ${n.missing.length + n.count} blob reference(s) point at objects the storage provider does not have; they are NOT in this bundle and no import can bring them back. e.g. ${n.missing.slice(0, 3).map((e) => `'${e}'`).join(", ")}` + (n.missing.length > 3 ? `, and ${n.missing.length - 3} more` : "")));
+		U = n.count, W = n.totalBytes, n.missing.length > 0 && (yield* N.logWarning(`${n.missing.length} of ${n.missing.length + n.count} blob reference(s) point at objects the storage provider does not have; they are NOT in this bundle and no import can bring them back. e.g. ${n.missing.slice(0, 3).map((e) => `'${e}'`).join(", ")}` + (n.missing.length > 3 ? `, and ${n.missing.length - 3} more` : "")));
 	}
 	let G = {
-		formatVersion: 1,
+		formatVersion: 2,
 		createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
 		source: {
 			dialect: e.dialect,
-			schemaFingerprint: F(e.snapshot)
+			schemaFingerprint: R(e.snapshot)
 		},
 		scope: n,
 		consistency: P,
-		compression: o,
-		tables: B,
-		assets: V > 0 ? {
-			count: V,
-			totalBytes: H,
-			index: h
+		compression: a,
+		tables: H,
+		assets: U > 0 ? {
+			count: U,
+			totalBytes: W,
+			index: m
 		} : {
 			count: 0,
 			totalBytes: 0
 		},
-		...L ? { masking: L } : {}
+		...I ? { masking: I } : {},
+		fieldKeys: {
+			keyIds: Object.entries(B.byKey).filter(([, e]) => e > 0).map(([e]) => e).sort(),
+			unnamed: B.unnamed
+		}
 	};
-	return yield* A.tryPromise({
+	return yield* N.tryPromise({
 		try: async () => {
-			await (await import("node:fs/promises")).writeFile(T(v), b(G));
+			await (await import("node:fs/promises")).writeFile(T(_), y(G));
 		},
-		catch: (e) => new i({
-			reason: `manifest write failed: ${I(e)}`,
-			path: v
+		catch: (e) => new s({
+			reason: `manifest write failed: ${z(e)}`,
+			path: _
 		})
-	}), yield* A.tryPromise({
+	}), yield* N.tryPromise({
 		try: async () => {
 			let e = await import("node:fs/promises"), t = await import("node:os");
-			await e.writeFile(T(y), x(G, { hostname: t.hostname() }));
+			await e.writeFile(T(v), w(G, { hostname: t.hostname() }));
 		},
-		catch: (e) => new i({
-			reason: `readme write failed: ${I(e)}`,
-			path: y
+		catch: (e) => new s({
+			reason: `readme write failed: ${z(e)}`,
+			path: v
 		})
 	}), G;
 });
 //#endregion
-export { q as runExport };
+export { X as runExport };

package/dist/filesystemAdmission.js

@@ -0,0 +1,33 @@
+import { constants as e } from "node:fs";
+import { copyFile as t, lstat as n, mkdir as r, mkdtemp as i, rename as a, rm as o } from "node:fs/promises";
+import { dirname as s, join as c, parse as l, resolve as u, sep as d } from "node:path";
+//#region src/filesystemAdmission.ts
+var f = async (e, t) => {
+	let r = u(e), i = l(r).root, a = r.slice(i.length).split(d).filter(Boolean), o = i;
+	for (let e = 0; e < a.length; e++) {
+		o = c(o, a[e]);
+		let r;
+		try {
+			r = await n(o);
+		} catch (e) {
+			if (typeof e == "object" && e && "code" in e && e.code === "ENOENT") return;
+			throw e;
+		}
+		if (r.isSymbolicLink()) throw Error("bundle path contains a symbolic link");
+		if (e === a.length - 1 && t === "file" ? !r.isFile() : !r.isDirectory()) throw Error("bundle path has an incompatible filesystem entry");
+	}
+}, p = async (n, l) => {
+	await f(l, "file"), await r(s(l), { recursive: !0 }), await f(l, "file");
+	let u = await i(c(s(l), ".voltro-unpack-"));
+	try {
+		let r = c(u, "file");
+		await t(n, r, e.COPYFILE_EXCL), await f(l, "file"), await a(r, l);
+	} finally {
+		await o(u, {
+			recursive: !0,
+			force: !0
+		});
+	}
+};
+//#endregion
+export { f as admitFilesystemPath, p as publishBundleFile };