@voltro/data-transfer 0.84.0 → 0.86.0

package/dist/errors.js

@@ -0,0 +1,82 @@
+import { Data as e, Schema as t } from "effect";
+//#region src/errors.ts
+var n = class extends t.TaggedError()("BundleError", {
+	reason: t.String,
+	path: t.optional(t.String)
+}) {
+	get message() {
+		return this.path === void 0 ? this.reason : `${this.reason} (${this.path})`;
+	}
+}, r = class extends t.TaggedError()("CodecError", {
+	table: t.String,
+	reason: t.String
+}) {}, i = class extends t.TaggedError()("IntegrityError", {
+	what: t.String,
+	expected: t.String,
+	actual: t.String
+}) {
+	get message() {
+		return `${this.what}: expected ${this.expected}, got ${this.actual}`;
+	}
+}, a = class extends e.TaggedError("CompressionError") {}, o = class extends t.TaggedError()("ScopeError", { reason: t.String }) {
+	get message() {
+		return this.reason;
+	}
+}, s = class extends t.TaggedError()("RowsRefusedError", {
+	rows: t.Array(t.Struct({
+		table: t.String,
+		id: t.String,
+		reason: t.String,
+		derived: t.Boolean
+	})),
+	primaryCount: t.Number,
+	totalCount: t.Number,
+	byTable: t.Array(t.Struct({
+		table: t.String,
+		refused: t.Number,
+		primary: t.Number
+	}))
+}) {
+	get message() {
+		return `RowsRefusedError: ${JSON.stringify({
+			rows: this.rows,
+			primaryCount: this.primaryCount,
+			totalCount: this.totalCount,
+			byTable: this.byTable
+		})}`;
+	}
+}, c = class extends t.TaggedError()("SchemaDriftError", {
+	bundleFingerprint: t.String,
+	targetFingerprint: t.String,
+	diff: t.Array(t.String)
+}) {
+	get message() {
+		return `bundle schema ${this.bundleFingerprint} does not match target ${this.targetFingerprint}: ${this.diff.join("; ")}`;
+	}
+}, l = class extends e.TaggedError("NativeToolError") {
+	get message() {
+		let e = this.stderr?.trim(), t = this.hint?.trim();
+		return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}${t ? `\n\nhint: ${t}` : ""}`;
+	}
+}, u = class extends t.TaggedError()("MaskingError", {
+	unclassified: t.Array(t.String),
+	invalidActions: t.optional(t.Array(t.String))
+}) {
+	get message() {
+		let e = [];
+		return this.unclassified.length > 0 && e.push(`unclassified columns: ${this.unclassified.join(", ")}`), this.invalidActions !== void 0 && this.invalidActions.length > 0 && e.push(`invalidActions: ${this.invalidActions.join(", ")}`), `masking refused — ${e.join("; ") || "no column may be exported"}`;
+	}
+}, d = class extends t.TaggedError()("ImportModeError", {
+	mode: t.String,
+	reason: t.String
+}) {
+	get message() {
+		return `import mode '${this.mode}' refused: ${this.reason}`;
+	}
+}, f = class extends t.TaggedError()("CrossDialectError", {
+	sourceDialect: t.String,
+	targetDialect: t.String,
+	blocking: t.Array(t.String)
+}) {};
+//#endregion
+export { n as BundleError, r as CodecError, a as CompressionError, f as CrossDialectError, d as ImportModeError, i as IntegrityError, u as MaskingError, l as NativeToolError, s as RowsRefusedError, c as SchemaDriftError, o as ScopeError };

package/dist/exporter.js

@@ -0,0 +1,257 @@
+import { snapshotPrelude as e } from "./consistency.js";
+import { classificationFromSnapshot as t, maskRow as n, planMasking as r } from "./masking.js";
+import { BundleError as i, MaskingError as a, ScopeError as o } from "./errors.js";
+import { computeSubsetIds as s, subsetToScope as c } from "./subset.js";
+import { resolveScope as l } from "./scope.js";
+import { captureAssetsToCas as u } from "./casAssetPhase.js";
+import { compressionExt as d, makeCompressor as f, pickCompression as p } from "./compress.js";
+import { ASSETS_DIR as m, ASSET_INDEX_FILE as h, DATA_DIR as g, LEDGER_FILE as _, MANIFEST_FILE as v, README_FILE as y, encodeManifest as b } from "./manifest.js";
+import { buildReadme as x } from "./readme.js";
+import { reportProgress as S, withTableSpan as C } from "./progress.js";
+import { readLedger as w, tableDone as T, writeLedger as E } from "./ledger.js";
+import { rowsToNdjson as D } from "./ndjson.js";
+import { topoSortTables as O } from "./topo.js";
+import { Cause as k, Effect as A, Exit as j, Schedule as M, Stream as N } from "effect";
+import { streamTable as P } from "@voltro/database";
+import { fingerprintSchema as F } from "@voltro/database/sql";
+//#region src/exporter.ts
+var I = (e) => {
+	let t = e, n = [];
+	typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
+	let r = typeof t?.message == "string" ? t.message.trim() : "";
+	if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(I(t.cause)) : n.push(r), n.length === 0) {
+		let t = String(e ?? "").trim();
+		return t === "" || t === "[object Object]" ? L(e) : t;
+	}
+	return n.join(": ");
+}, L = (e) => {
+	try {
+		let t = JSON.stringify(e);
+		return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
+	} catch {
+		return "unknown error (not serialisable)";
+	}
+}, R = (e, t) => {
+	let n = e.tables.find((e) => e.name === t), r = {};
+	for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
+	return r;
+}, z = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), B = (e, t) => {
+	if (t.length === 0) return e;
+	let n = { ...e };
+	for (let e of t) delete n[e];
+	return n;
+}, V = (e, t) => {
+	let n = e.tables.find((e) => e.name === t);
+	if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
+	let r = n.columns.find((e) => e.type === "id");
+	if (r !== void 0) return { column: r.name };
+	let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
+	return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
+}, H = (e, t, n) => A.tryPromise({
+	try: async () => {
+		let { createWriteStream: r } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: l } = await import("node:stream"), u = a("sha256"), d = 0, p = new l({ transform(e, t, n) {
+			u.update(e);
+			let r = e;
+			for (let e = 0; e < r.length; e++) r[e] === 10 && d++;
+			n(null, e);
+		} }), m = c.fromWeb(N.toReadableStream(e)), h = `${t}.tmp-${o()}`;
+		try {
+			await s(m, p, f(n), r(h)), await i.rename(h, t);
+		} catch (e) {
+			throw await i.rm(h, { force: !0 }).catch(() => {}), e;
+		}
+		return {
+			rowCount: d,
+			checksum: u.digest("hex")
+		};
+	},
+	catch: (e) => new i({
+		reason: `write table failed: ${I(e)}`,
+		path: t
+	})
+}), U = (e, t) => A.tryPromise({
+	try: async () => {
+		await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
+	},
+	catch: (t) => new i({
+		reason: `append failed: ${I(t)}`,
+		path: e
+	})
+}), W = (e, t) => A.gen(function* () {
+	let r = [], i = t.ordered.length;
+	for (let a = 0; a < t.ordered.length; a++) {
+		let s = t.ordered[a], c = R(t.snapshot, s), l = V(t.snapshot, s);
+		if ("refusal" in l) return yield* A.fail(new o({ reason: `export: ${l.refusal}` }));
+		let u = l.column, f = `${g}/${s}.ndjson${d(t.compression)}`;
+		if (T(t.ledger, s)) {
+			let e = t.ledger.tables[s];
+			S(t.onProgress, {
+				phase: "export",
+				table: s,
+				index: a,
+				tableCount: i,
+				event: "done",
+				rowsDone: e.rowCount,
+				total: e.rowCount
+			}), r.push({
+				name: s,
+				file: f,
+				rowCount: e.rowCount,
+				checksum: e.checksum,
+				columnTypes: c,
+				primaryKey: u
+			});
+			continue;
+		}
+		S(t.onProgress, {
+			phase: "export",
+			table: s,
+			index: a,
+			tableCount: i,
+			event: "start",
+			rowsDone: 0
+		});
+		let p = t.resolved.predicateFor(s), m = P(e, {
+			table: s,
+			pk: u,
+			chunkSize: t.chunkSize,
+			retry: t.retry,
+			...p === void 0 ? {} : { where: p }
+		}), h = z(t.snapshot, s), _ = h.length === 0 ? m : m.pipe(N.map((e) => B(e, h))), v = t.mask ? _.pipe(N.map((e) => n(e, s, t.mask.actions, t.mask.seed, c))) : _, { rowCount: y, checksum: b } = yield* C("export", s, a, i, (e) => e.rowCount, H(D(v, c), t.bundleFile(f), t.compression));
+		t.ledger.tables[s] = {
+			rowCount: y,
+			checksum: b
+		}, yield* E(t.ledgerPath, t.ledger), S(t.onProgress, {
+			phase: "export",
+			table: s,
+			index: a,
+			tableCount: i,
+			event: "done",
+			rowsDone: y,
+			total: y
+		}), r.push({
+			name: s,
+			file: f,
+			rowCount: y,
+			checksum: b,
+			columnTypes: c,
+			primaryKey: u
+		});
+	}
+	return r;
+}), G = (e) => ({
+	_tag: "RawSqlFragment",
+	strings: [e],
+	values: []
+}), K = (t, n, r) => A.tryPromise({
+	try: () => t.transactional(async (t) => {
+		if (t.raw) for (let r of e(n)) await t.raw(G(r));
+		let i = await A.runPromiseExit(r(t));
+		if (j.isFailure(i)) throw Error(k.pretty(i.cause));
+		return i.value;
+	}),
+	catch: (e) => e instanceof i ? e : new i({ reason: `snapshot export failed: ${I(e)}` })
+}), q = (e) => A.gen(function* () {
+	let n = e.scope ?? { kind: "all" }, o = p(e.compression), d = e.chunkSize ?? 1e3, f = e.retryTimes ?? 5, S = M.max([M.exponential("50 millis").pipe(M.jittered), M.recurs(f)]), C = yield* A.promise(() => import("node:path")), T = (t) => C.join(e.outDir, t);
+	yield* A.tryPromise({
+		try: async () => {
+			let t = await import("node:fs/promises");
+			await t.mkdir(T(g), { recursive: !0 }), e.assets && await t.mkdir(T(m), { recursive: !0 });
+		},
+		catch: (t) => new i({
+			reason: `mkdir failed: ${I(t)}`,
+			path: e.outDir
+		})
+	});
+	let D = T(_), k = yield* w(D), j = e.subset ? c(yield* s(e.store, e.snapshot, e.subset), e.snapshot) : yield* l(n, e.snapshot, {
+		store: e.store,
+		...e.tenantTables ? { tenantTables: e.tenantTables } : {}
+	}), N = O(j.tables, e.snapshot), P = e.consistency ?? "live", L, R;
+	if (e.masking) {
+		let n = e.classification ?? t(e.snapshot), i = N.map((t) => ({
+			name: t,
+			columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
+				name: e.name,
+				type: e.type
+			}))
+		})), o = r(i, n, e.masking);
+		if (o.unclassified.length > 0 || o.invalid.length > 0) return yield* A.fail(new a({
+			unclassified: o.unclassified,
+			...o.invalid.length > 0 ? { invalidActions: o.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
+		}));
+		R = {
+			actions: o.actions,
+			seed: e.masking.seed
+		};
+		let s = Object.entries(o.actions).filter(([, e]) => e !== "keep").map(([e]) => e);
+		L = {
+			...e.masking.id ? { policyId: e.masking.id } : {},
+			transformed: s
+		};
+	}
+	let z = {
+		snapshot: e.snapshot,
+		ordered: N,
+		resolved: j,
+		compression: o,
+		chunkSize: d,
+		retry: S,
+		ledger: k,
+		ledgerPath: D,
+		bundleFile: T,
+		...R ? { mask: R } : {},
+		...e.onProgress ? { onProgress: e.onProgress } : {}
+	}, B = P === "snapshot" ? yield* K(e.store, e.dialect, (e) => W(e, z)) : yield* W(e.store, z), V = 0, H = 0;
+	if (e.assets) {
+		let t = T(h), n = yield* u({
+			source: e.assets,
+			assetsDir: T(m),
+			progress: {
+				done: (e) => e in k.assets,
+				mark: (e, n) => (k.assets[e] = !0, U(t, JSON.stringify(n)).pipe(A.andThen(E(D, k))))
+			}
+		});
+		V = n.count, H = n.totalBytes, n.missing.length > 0 && (yield* A.logWarning(`${n.missing.length} of ${n.missing.length + n.count} blob reference(s) point at objects the storage provider does not have; they are NOT in this bundle and no import can bring them back. e.g. ${n.missing.slice(0, 3).map((e) => `'${e}'`).join(", ")}` + (n.missing.length > 3 ? `, and ${n.missing.length - 3} more` : "")));
+	}
+	let G = {
+		formatVersion: 1,
+		createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
+		source: {
+			dialect: e.dialect,
+			schemaFingerprint: F(e.snapshot)
+		},
+		scope: n,
+		consistency: P,
+		compression: o,
+		tables: B,
+		assets: V > 0 ? {
+			count: V,
+			totalBytes: H,
+			index: h
+		} : {
+			count: 0,
+			totalBytes: 0
+		},
+		...L ? { masking: L } : {}
+	};
+	return yield* A.tryPromise({
+		try: async () => {
+			await (await import("node:fs/promises")).writeFile(T(v), b(G));
+		},
+		catch: (e) => new i({
+			reason: `manifest write failed: ${I(e)}`,
+			path: v
+		})
+	}), yield* A.tryPromise({
+		try: async () => {
+			let e = await import("node:fs/promises"), t = await import("node:os");
+			await e.writeFile(T(y), x(G, { hostname: t.hostname() }));
+		},
+		catch: (e) => new i({
+			reason: `readme write failed: ${I(e)}`,
+			path: y
+		})
+	}), G;
+});
+//#endregion
+export { q as runExport };