@voltro/data-transfer 0.85.0 → 0.87.0

package/dist/ledger.js

@@ -0,0 +1,21 @@
+import { BundleError as e } from "./errors.js";
+import { Effect as t } from "effect";
+//#region src/ledger.ts
+var n = () => ({
+	tables: {},
+	assets: {}
+}), r = (e) => t.tryPromise(async () => {
+	let t = await (await import("node:fs/promises")).readFile(e, "utf8");
+	return JSON.parse(t);
+}).pipe(t.orElseSucceed(n)), i = (n, r) => t.tryPromise({
+	try: async () => {
+		let e = await import("node:fs/promises"), { randomUUID: t } = await import("node:crypto"), i = `${n}.tmp-${t()}`;
+		await e.writeFile(i, JSON.stringify(r)), await e.rename(i, n);
+	},
+	catch: (t) => new e({
+		reason: `ledger write failed: ${String(t?.message ?? t)}`,
+		path: n
+	})
+}), a = (e, t) => t in e.tables, o = (e, t) => t in e.assets;
+//#endregion
+export { o as assetDone, n as emptyLedger, r as readLedger, a as tableDone, i as writeLedger };

package/dist/liveGuard.js

@@ -0,0 +1,11 @@
+//#region src/liveGuard.ts
+var e = (e) => {
+	if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
+	let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
+	return {
+		refuse: !0,
+		message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
+	};
+};
+//#endregion
+export { e as assessLiveImport };

package/dist/manifest.js

@@ -0,0 +1,56 @@
+import { isSafeBundlePath as e } from "./bundlePaths.js";
+import { Schema as t } from "effect";
+import { StorageAddress as n } from "@voltro/plugin-storage/errors";
+//#region src/manifest.ts
+var r = 2, i = t.Struct({
+	name: t.String,
+	file: t.String,
+	rowCount: t.Number,
+	checksum: t.String,
+	columnTypes: t.Record(t.String, t.String),
+	primaryKey: t.String
+}), a = t.Struct({
+	kind: t.Literals([
+		"all",
+		"tenant",
+		"tables"
+	]),
+	tenantId: t.optional(t.String),
+	tables: t.optional(t.Array(t.String)),
+	exclude: t.optional(t.Array(t.String))
+}), o = t.Struct({
+	count: t.Number,
+	totalBytes: t.Number,
+	index: t.optional(t.String.pipe(t.check(t.makeFilter(e))))
+}), s = n, c = t.Struct({
+	...s.fields,
+	sha256: t.String.pipe(t.check(t.isPattern(/^[a-f0-9]{64}$/))),
+	size: t.Number.pipe(t.check(t.isInt()), t.check(t.isGreaterThanOrEqualTo(0)), t.check(t.isLessThanOrEqualTo(2 ** 53 - 1))),
+	contentType: t.String.pipe(t.check(t.isMinLength(1)), t.check(t.makeFilter((e) => !/[\uD800-\uDFFF]/u.test(e))))
+}), l = t.Struct({
+	formatVersion: t.Number,
+	createdAt: t.String,
+	source: t.Struct({
+		dialect: t.String,
+		schemaFingerprint: t.String
+	}),
+	scope: a,
+	consistency: t.Literals(["live", "snapshot"]),
+	compression: t.Literals([
+		"zstd",
+		"gzip",
+		"none"
+	]),
+	tables: t.Array(i),
+	assets: o,
+	fieldKeys: t.optional(t.Struct({
+		keyIds: t.Array(t.String),
+		unnamed: t.Number
+	})),
+	masking: t.optional(t.Struct({
+		policyId: t.optional(t.String),
+		transformed: t.Array(t.String)
+	}))
+}), u = t.encodeSync(t.fromJsonString(l)), d = t.decodeUnknownSync(t.fromJsonString(l)), f = "manifest.json", p = "README.md", m = "data", h = "assets", g = "assets/index.ndjson", _ = ".ledger.json";
+//#endregion
+export { h as ASSETS_DIR, g as ASSET_INDEX_FILE, s as AssetAddress, c as AssetEntry, o as AssetSummary, m as DATA_DIR, a as ExportScope, r as FORMAT_VERSION, _ as LEDGER_FILE, f as MANIFEST_FILE, l as Manifest, p as README_FILE, i as TableEntry, d as decodeManifest, u as encodeManifest };

package/dist/masking.js

@@ -0,0 +1,246 @@
+import { createHmac as e } from "node:crypto";
+//#region src/masking.ts
+var t = {
+	email: { fake: "email" },
+	fullName: { fake: "fullName" },
+	firstName: { fake: "firstName" },
+	lastName: { fake: "lastName" },
+	username: { fake: "username" },
+	phone: { fake: "phone" },
+	address: { fake: "address" },
+	company: { fake: "company" },
+	url: { fake: "url" },
+	ip: { fake: "ip" },
+	creditCard: { fake: "creditCard" },
+	date: "dateShift",
+	secret: "null",
+	freeText: "redact"
+}, n = (e) => {
+	let t = {};
+	for (let n of e.tables) {
+		let e = {};
+		for (let t of n.columns) e[t.name] = {
+			...t.sensitive ? { sensitive: t.sensitive } : {},
+			...t.safe ? { safe: !0 } : {},
+			columnType: t.type
+		};
+		t[n.name] = e;
+	}
+	return t;
+}, r = (e, n, r, i) => {
+	let a = i.columns?.[`${e}.${n}`];
+	if (a) return a;
+	if (r?.sensitive) {
+		let e = r.sensitive.class;
+		return i.classes?.[e] ?? t[e] ?? "redact";
+	}
+	return r?.safe ? i.onSafe ?? "keep" : r?.columnType === "id" || r?.columnType === "reference" ? "keep" : i.onUnclassified ?? "error";
+}, i = (t, n) => e("sha256", t).update(n == null ? "∅" : String(n)).digest(), a = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), o = [
+	"Ada",
+	"Alan",
+	"Grace",
+	"Linus",
+	"Ida",
+	"Ken",
+	"Barbara",
+	"Dennis",
+	"Radia",
+	"Guido",
+	"Margaret",
+	"Bjarne"
+], s = [
+	"Lovelace",
+	"Turing",
+	"Hopper",
+	"Torvalds",
+	"Rhodes",
+	"Thompson",
+	"Liskov",
+	"Ritchie",
+	"Perlman",
+	"Rossum",
+	"Hamilton",
+	"Stroustrup"
+], c = [
+	"Maple",
+	"Oak",
+	"Cedar",
+	"Pine",
+	"Elm",
+	"Birch",
+	"Willow",
+	"Ash"
+], l = [
+	"Initech",
+	"Hooli",
+	"Umbrella",
+	"Acme",
+	"Globex",
+	"Soylent",
+	"Stark",
+	"Wayne"
+], u = (e, t, n) => e[a(t, n) % e.length], d = (e, t, n) => {
+	let r = i(n, `${e}:${String(t)}`), d = a(r, 1) % 1e4;
+	switch (e) {
+		case "email": return `${u(o, r, 0).toLowerCase()}.${u(s, r, 2).toLowerCase()}${d}@example.com`;
+		case "fullName": return `${u(o, r, 0)} ${u(s, r, 1)}`;
+		case "firstName": return u(o, r, 0);
+		case "lastName": return u(s, r, 1);
+		case "username": return `${u(o, r, 0).toLowerCase()}${d}`;
+		case "phone": return `+1${String(a(r, 0) % 1e10).padStart(10, "0")}`;
+		case "address": return `${a(r, 0) % 9899 + 100} ${u(c, r, 1)} St`;
+		case "company": return u(l, r, 0);
+		case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
+		case "ip": return `${a(r, 0) % 256}.${a(r, 1) % 256}.${a(r, 2) % 256}.${a(r, 3) % 254 + 1}`;
+		case "creditCard": return f(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
+		default: return `${e}_${r.toString("hex").slice(0, 8)}`;
+	}
+}, f = (e) => {
+	let t = e.slice(0, 15).split("").map(Number), n = 0;
+	t.forEach((e, t) => {
+		let r = e;
+		(15 - t) % 2 == 1 && (r *= 2, r > 9 && (r -= 9)), n += r;
+	});
+	let r = (10 - n % 10) % 10;
+	return e.slice(0, 15) + String(r);
+}, p = 864e5, m = (e, t) => {
+	let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
+	if (Number.isNaN(n)) return e;
+	let r = a(i(t, "date-shift-offset"), 0) % 730 - 365, o = new Date(n + r * p);
+	return e instanceof Date ? o : o.toISOString();
+}, h = (e, t) => {
+	if (t.value === null || t.value === void 0) return null;
+	if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
+	if (e === "keep") return t.value;
+	if (e === "null") return null;
+	if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
+	if (e === "hash") return i(t.seed, t.value).toString("hex").slice(0, 16);
+	if (e === "dateShift") return m(t.value, t.seed);
+	if (typeof e == "object" && "fake" in e) return d(e.fake, t.value, t.seed);
+	if (typeof e == "object" && "custom" in e) return e.custom(t);
+	if (typeof e == "object" && "json" in e) return g(e, t);
+	throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn } | { json: { fields, rest? } }.`);
+}, g = (e, t) => {
+	let { fields: n, rest: r = "redact" } = e.json, i = typeof t.value == "string" ? _(t.value) : {
+		ok: !0,
+		value: t.value
+	};
+	if (!i.ok) return h(r, {
+		...t,
+		columnType: "text"
+	});
+	let a = (e, i) => {
+		if (e == null) return null;
+		if (Array.isArray(e)) return e.map((e, t) => a(e, `${i}[${t}]`));
+		if (typeof e == "object") {
+			let a = {};
+			for (let [o, s] of Object.entries(e)) {
+				let e = n[o] ?? r, c = `${i}.${o}`;
+				a[o] = typeof e == "object" && "json" in e ? g(e, {
+					...t,
+					value: s,
+					column: c,
+					columnType: "json"
+				}) : h(e, {
+					...t,
+					value: s,
+					column: c,
+					columnType: typeof s == "string" ? "text" : "json"
+				});
+			}
+			return a;
+		}
+		return h(r, {
+			...t,
+			value: e,
+			column: i,
+			columnType: typeof e == "string" ? "text" : "json"
+		});
+	}, o = a(i.value, t.column);
+	return typeof t.value == "string" ? JSON.stringify(o) : o;
+}, _ = (e) => {
+	try {
+		return {
+			ok: !0,
+			value: JSON.parse(e)
+		};
+	} catch {
+		return { ok: !1 };
+	}
+}, v = (e) => {
+	if (typeof e == "string") return e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error";
+	if (typeof e != "object" || !e) return !1;
+	if ("fake" in e) return typeof e.fake == "string";
+	if ("custom" in e) return typeof e.custom == "function";
+	if ("json" in e) {
+		let t = e.json;
+		if (typeof t != "object" || !t) return !1;
+		let { fields: n, rest: r } = t;
+		return typeof n != "object" || !n ? !1 : Object.values(n).every(v) && (r === void 0 || v(r));
+	}
+	return !1;
+}, y = (e, t, n) => {
+	let i = {}, a = [], o = [];
+	for (let s of e) for (let e of s.columns) {
+		let c = `${s.name}.${e.name}`, l = r(s.name, e.name, t[s.name]?.[e.name], n);
+		l === "error" ? a.push(c) : v(l) ? i[c] = l : o.push({
+			column: c,
+			action: l
+		});
+	}
+	return {
+		actions: i,
+		unclassified: a,
+		invalid: o
+	};
+}, b = [
+	{
+		name: "email",
+		re: /[\w.+-]+@[\w-]+\.[\w.-]+/
+	},
+	{
+		name: "phone",
+		re: /(?:\+?\d[\d\s().-]{8,}\d)/
+	},
+	{
+		name: "ipv4",
+		re: /\b(?:\d{1,3}\.){3}\d{1,3}\b/
+	},
+	{
+		name: "creditCard",
+		re: /\b(?:\d[ -]?){13,16}\b/
+	}
+], x = (e, t, n) => {
+	let r = [], i = /* @__PURE__ */ new Set();
+	for (let [a, o] of Object.entries(n)) {
+		if (!a.startsWith(`${t}.`) || o !== "keep") continue;
+		let n = a.slice(t.length + 1);
+		for (let a of e) {
+			let e = a[n];
+			if (typeof e != "string" || e.length === 0) continue;
+			let o = b.find((t) => t.re.test(e));
+			if (o && !i.has(n)) {
+				i.add(n), r.push({
+					table: t,
+					column: n,
+					pattern: o.name,
+					sample: e.slice(0, 40)
+				});
+				break;
+			}
+		}
+	}
+	return r;
+}, S = (e, t, n, r, i) => {
+	let a = {};
+	for (let [o, s] of Object.entries(e)) a[o] = h(n[`${t}.${o}`] ?? "keep", {
+		value: s,
+		table: t,
+		column: o,
+		seed: r,
+		...i?.[o] ? { columnType: i[o] } : {}
+	});
+	return a;
+};
+//#endregion
+export { t as DEFAULT_CLASS_ACTIONS, h as applyAction, n as classificationFromSnapshot, v as isMaskAction, S as maskRow, y as planMasking, r as resolveAction, x as scanForLeaks };

package/dist/maskingPreview.js

@@ -0,0 +1,48 @@
+import { classificationFromSnapshot as e, maskRow as t, planMasking as n, scanForLeaks as r } from "./masking.js";
+import { resolveScope as i } from "./scope.js";
+import { topoSortTables as a } from "./topo.js";
+import { Effect as o, Stream as s } from "effect";
+import { streamTable as c } from "@voltro/database";
+//#region src/maskingPreview.ts
+var l = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), u = (u) => o.gen(function* () {
+	let d = u.scope ?? { kind: "all" }, f = yield* i(d, u.snapshot, {
+		store: u.store,
+		...u.tenantTables ? { tenantTables: u.tenantTables } : {}
+	}), p = a(f.tables, u.snapshot), m = u.classification ?? e(u.snapshot), h = u.sampleSize ?? 20, g = p.map((e) => ({
+		name: e,
+		columns: (u.snapshot.tables.find((t) => t.name === e)?.columns ?? []).map((e) => ({
+			name: e.name,
+			type: e.type
+		}))
+	})), _ = n(g, m, u.masking), v = [], y = [];
+	for (let e of p) {
+		let n = f.predicateFor(e), i = yield* c(u.store, {
+			table: e,
+			chunkSize: h,
+			...n === void 0 ? {} : { where: n }
+		}).pipe(s.take(h), s.runCollect, o.map((e) => [...e])), a = i.map((n) => t(n, e, _.actions, u.masking.seed));
+		y.push(...r(a, e, _.actions));
+		let d = i[0], p = a[0], m = [];
+		if (d && p) for (let [t, n] of Object.entries(_.actions)) {
+			if (!t.startsWith(`${e}.`) || n === "keep") continue;
+			let r = t.slice(e.length + 1);
+			m.push({
+				column: r,
+				action: l(n),
+				before: d[r],
+				after: p[r]
+			});
+		}
+		v.push({
+			table: e,
+			samples: m
+		});
+	}
+	return {
+		tables: v,
+		leaks: y,
+		unclassified: _.unclassified
+	};
+});
+//#endregion
+export { u as previewMasking };