@voltro/data-transfer 0.85.0 → 0.87.0

package/dist/exporter.js

@@ -0,0 +1,271 @@
+import { snapshotPrelude as e } from "./consistency.js";
+import { classificationFromSnapshot as t, maskRow as n, planMasking as r } from "./masking.js";
+import { emptyTally as i, mergeTally as a, tallyRow as o } from "./sealedKeys.js";
+import { BundleError as s, MaskingError as c, ScopeError as l } from "./errors.js";
+import { computeSubsetIds as u, subsetToScope as d } from "./subset.js";
+import { resolveScope as f } from "./scope.js";
+import { ASSETS_DIR as p, ASSET_INDEX_FILE as m, DATA_DIR as h, LEDGER_FILE as g, MANIFEST_FILE as _, README_FILE as v, encodeManifest as y } from "./manifest.js";
+import { captureAssetsToCas as b } from "./casAssetPhase.js";
+import { compressionExt as x, makeCompressor as S, pickCompression as C } from "./compress.js";
+import { buildReadme as w } from "./readme.js";
+import { reportProgress as T, withTableSpan as E } from "./progress.js";
+import { readLedger as D, tableDone as O, writeLedger as k } from "./ledger.js";
+import { rowsToNdjson as A } from "./ndjson.js";
+import { topoSortTables as j } from "./topo.js";
+import { Cause as M, Effect as N, Exit as P, Schedule as F, Stream as I } from "effect";
+import { streamTable as L } from "@voltro/database";
+import { fingerprintSchema as R } from "@voltro/database/sql";
+//#region src/exporter.ts
+var z = (e) => {
+	let t = e, n = [];
+	typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
+	let r = typeof t?.message == "string" ? t.message.trim() : "";
+	if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(z(t.cause)) : n.push(r), n.length === 0) {
+		let t = String(e ?? "").trim();
+		return t === "" || t === "[object Object]" ? B(e) : t;
+	}
+	return n.join(": ");
+}, B = (e) => {
+	try {
+		let t = JSON.stringify(e);
+		return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
+	} catch {
+		return "unknown error (not serialisable)";
+	}
+}, V = (e, t) => {
+	let n = e.tables.find((e) => e.name === t), r = {};
+	for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
+	return r;
+}, H = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), U = (e, t) => {
+	if (t.length === 0) return e;
+	let n = { ...e };
+	for (let e of t) delete n[e];
+	return n;
+}, W = (e, t) => {
+	let n = e.tables.find((e) => e.name === t);
+	if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
+	let r = n.columns.find((e) => e.type === "id");
+	if (r !== void 0) return { column: r.name };
+	let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
+	return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
+}, G = (e, t, n) => N.tryPromise({
+	try: async () => {
+		let { createWriteStream: r } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: l } = await import("node:stream"), u = a("sha256"), d = 0, f = new l({ transform(e, t, n) {
+			u.update(e);
+			let r = e;
+			for (let e = 0; e < r.length; e++) r[e] === 10 && d++;
+			n(null, e);
+		} }), p = c.fromWeb(I.toReadableStream(e)), m = `${t}.tmp-${o()}`;
+		try {
+			await s(p, f, S(n), r(m)), await i.rename(m, t);
+		} catch (e) {
+			throw await i.rm(m, { force: !0 }).catch(() => {}), e;
+		}
+		return {
+			rowCount: d,
+			checksum: u.digest("hex")
+		};
+	},
+	catch: (e) => new s({
+		reason: `write table failed: ${z(e)}`,
+		path: t
+	})
+}), K = (e, t) => N.tryPromise({
+	try: async () => {
+		await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
+	},
+	catch: (t) => new s({
+		reason: `append failed: ${z(t)}`,
+		path: e
+	})
+}), q = (e, t) => N.gen(function* () {
+	let r = [], s = t.ordered.length;
+	for (let c = 0; c < t.ordered.length; c++) {
+		let u = t.ordered[c], d = V(t.snapshot, u), f = W(t.snapshot, u);
+		if ("refusal" in f) return yield* N.fail(new l({ reason: `export: ${f.refusal}` }));
+		let p = f.column, m = `${h}/${u}.ndjson${x(t.compression)}`;
+		if (O(t.ledger, u)) {
+			let e = t.ledger.tables[u];
+			e.sealed !== void 0 && a(t.sealed, e.sealed), T(t.onProgress, {
+				phase: "export",
+				table: u,
+				index: c,
+				tableCount: s,
+				event: "done",
+				rowsDone: e.rowCount,
+				total: e.rowCount
+			}), r.push({
+				name: u,
+				file: m,
+				rowCount: e.rowCount,
+				checksum: e.checksum,
+				columnTypes: d,
+				primaryKey: p
+			});
+			continue;
+		}
+		T(t.onProgress, {
+			phase: "export",
+			table: u,
+			index: c,
+			tableCount: s,
+			event: "start",
+			rowsDone: 0
+		});
+		let g = t.resolved.predicateFor(u), _ = L(e, {
+			table: u,
+			pk: p,
+			chunkSize: t.chunkSize,
+			retry: t.retry,
+			...g === void 0 ? {} : { where: g }
+		}), v = H(t.snapshot, u), y = v.length === 0 ? _ : _.pipe(I.map((e) => U(e, v))), b = t.mask ? y.pipe(I.map((e) => n(e, u, t.mask.actions, t.mask.seed, d))) : y, S = i(), C = b.pipe(I.map((e) => (o(e, S), e))), { rowCount: w, checksum: D } = yield* E("export", u, c, s, (e) => e.rowCount, G(A(C, d), t.bundleFile(m), t.compression));
+		t.ledger.tables[u] = {
+			rowCount: w,
+			checksum: D,
+			sealed: S
+		}, a(t.sealed, S), yield* k(t.ledgerPath, t.ledger), T(t.onProgress, {
+			phase: "export",
+			table: u,
+			index: c,
+			tableCount: s,
+			event: "done",
+			rowsDone: w,
+			total: w
+		}), r.push({
+			name: u,
+			file: m,
+			rowCount: w,
+			checksum: D,
+			columnTypes: d,
+			primaryKey: p
+		});
+	}
+	return r;
+}), J = (e) => ({
+	_tag: "RawSqlFragment",
+	strings: [e],
+	values: []
+}), Y = (t, n, r) => N.tryPromise({
+	try: () => t.transactional(async (t) => {
+		if (t.raw) for (let r of e(n)) await t.raw(J(r));
+		let i = await N.runPromiseExit(r(t));
+		if (P.isFailure(i)) throw Error(M.pretty(i.cause));
+		return i.value;
+	}),
+	catch: (e) => e instanceof s ? e : new s({ reason: `snapshot export failed: ${z(e)}` })
+}), X = (e) => N.gen(function* () {
+	let n = e.scope ?? { kind: "all" }, a = C(e.compression), o = e.chunkSize ?? 1e3, l = e.retryTimes ?? 5, x = F.max([F.exponential("50 millis").pipe(F.jittered), F.recurs(l)]), S = yield* N.promise(() => import("node:path")), T = (t) => S.join(e.outDir, t);
+	yield* N.tryPromise({
+		try: async () => {
+			let t = await import("node:fs/promises");
+			await t.mkdir(T(h), { recursive: !0 }), e.assets && await t.mkdir(T(p), { recursive: !0 });
+		},
+		catch: (t) => new s({
+			reason: `mkdir failed: ${z(t)}`,
+			path: e.outDir
+		})
+	});
+	let E = T(g), O = yield* D(E), A = e.subset ? d(yield* u(e.store, e.snapshot, e.subset), e.snapshot) : yield* f(n, e.snapshot, {
+		store: e.store,
+		...e.tenantTables ? { tenantTables: e.tenantTables } : {}
+	}), M = j(A.tables, e.snapshot), P = e.consistency ?? "live", I, L;
+	if (e.masking) {
+		let n = e.classification ?? t(e.snapshot), i = M.map((t) => ({
+			name: t,
+			columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
+				name: e.name,
+				type: e.type
+			}))
+		})), a = r(i, n, e.masking);
+		if (a.unclassified.length > 0 || a.invalid.length > 0) return yield* N.fail(new c({
+			unclassified: a.unclassified,
+			...a.invalid.length > 0 ? { invalidActions: a.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
+		}));
+		L = {
+			actions: a.actions,
+			seed: e.masking.seed
+		};
+		let o = Object.entries(a.actions).filter(([, e]) => e !== "keep").map(([e]) => e);
+		I = {
+			...e.masking.id ? { policyId: e.masking.id } : {},
+			transformed: o
+		};
+	}
+	let B = i(), V = {
+		sealed: B,
+		snapshot: e.snapshot,
+		ordered: M,
+		resolved: A,
+		compression: a,
+		chunkSize: o,
+		retry: x,
+		ledger: O,
+		ledgerPath: E,
+		bundleFile: T,
+		...L ? { mask: L } : {},
+		...e.onProgress ? { onProgress: e.onProgress } : {}
+	}, H = P === "snapshot" ? yield* Y(e.store, e.dialect, (e) => q(e, V)) : yield* q(e.store, V), U = 0, W = 0;
+	if (e.assets) {
+		let t = T(m), n = yield* b({
+			source: {
+				...e.assets,
+				list: () => e.assets.list({
+					dir: e.outDir,
+					tables: H,
+					compression: a
+				})
+			},
+			assetsDir: T(p),
+			progress: {
+				done: (e) => e in O.assets,
+				mark: (e, n) => (O.assets[e] = !0, K(t, JSON.stringify(n)).pipe(N.andThen(k(E, O))))
+			}
+		});
+		U = n.count, W = n.totalBytes, n.missing.length > 0 && (yield* N.logWarning(`${n.missing.length} of ${n.missing.length + n.count} blob reference(s) point at objects the storage provider does not have; they are NOT in this bundle and no import can bring them back. e.g. ${n.missing.slice(0, 3).map((e) => `'${e}'`).join(", ")}` + (n.missing.length > 3 ? `, and ${n.missing.length - 3} more` : "")));
+	}
+	let G = {
+		formatVersion: 2,
+		createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
+		source: {
+			dialect: e.dialect,
+			schemaFingerprint: R(e.snapshot)
+		},
+		scope: n,
+		consistency: P,
+		compression: a,
+		tables: H,
+		assets: U > 0 ? {
+			count: U,
+			totalBytes: W,
+			index: m
+		} : {
+			count: 0,
+			totalBytes: 0
+		},
+		...I ? { masking: I } : {},
+		fieldKeys: {
+			keyIds: Object.entries(B.byKey).filter(([, e]) => e > 0).map(([e]) => e).sort(),
+			unnamed: B.unnamed
+		}
+	};
+	return yield* N.tryPromise({
+		try: async () => {
+			await (await import("node:fs/promises")).writeFile(T(_), y(G));
+		},
+		catch: (e) => new s({
+			reason: `manifest write failed: ${z(e)}`,
+			path: _
+		})
+	}), yield* N.tryPromise({
+		try: async () => {
+			let e = await import("node:fs/promises"), t = await import("node:os");
+			await e.writeFile(T(v), w(G, { hostname: t.hostname() }));
+		},
+		catch: (e) => new s({
+			reason: `readme write failed: ${z(e)}`,
+			path: v
+		})
+	}), G;
+});
+//#endregion
+export { X as runExport };

package/dist/filesystemAdmission.js

@@ -0,0 +1,33 @@
+import { constants as e } from "node:fs";
+import { copyFile as t, lstat as n, mkdir as r, mkdtemp as i, rename as a, rm as o } from "node:fs/promises";
+import { dirname as s, join as c, parse as l, resolve as u, sep as d } from "node:path";
+//#region src/filesystemAdmission.ts
+var f = async (e, t) => {
+	let r = u(e), i = l(r).root, a = r.slice(i.length).split(d).filter(Boolean), o = i;
+	for (let e = 0; e < a.length; e++) {
+		o = c(o, a[e]);
+		let r;
+		try {
+			r = await n(o);
+		} catch (e) {
+			if (typeof e == "object" && e && "code" in e && e.code === "ENOENT") return;
+			throw e;
+		}
+		if (r.isSymbolicLink()) throw Error("bundle path contains a symbolic link");
+		if (e === a.length - 1 && t === "file" ? !r.isFile() : !r.isDirectory()) throw Error("bundle path has an incompatible filesystem entry");
+	}
+}, p = async (n, l) => {
+	await f(l, "file"), await r(s(l), { recursive: !0 }), await f(l, "file");
+	let u = await i(c(s(l), ".voltro-unpack-"));
+	try {
+		let r = c(u, "file");
+		await t(n, r, e.COPYFILE_EXCL), await f(l, "file"), await a(r, l);
+	} finally {
+		await o(u, {
+			recursive: !0,
+			force: !0
+		});
+	}
+};
+//#endregion
+export { f as admitFilesystemPath, p as publishBundleFile };