@voltro/data-transfer 0.34.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +167 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +294 -255
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -4,14 +4,14 @@ import { fingerprintSchema as m } from "@voltro/database/sql";
|
|
|
4
4
|
import { createCipheriv as h, createDecipheriv as g, createHash as _, createHmac as v, randomBytes as y, scrypt as b } from "node:crypto";
|
|
5
5
|
import { STORAGE_REFS_TABLE as x, fromNodeReadable as S, getObjectStream as C, putObjectStream as ee, toNodeReadable as te } from "@voltro/plugin-storage";
|
|
6
6
|
import * as w from "node:zlib";
|
|
7
|
-
import { PassThrough as
|
|
8
|
-
import { copyFile as
|
|
9
|
-
import { mkdir as
|
|
10
|
-
import { basename as
|
|
11
|
-
import { promisify as
|
|
12
|
-
import { spawn as
|
|
7
|
+
import { PassThrough as T, Readable as ne } from "node:stream";
|
|
8
|
+
import { copyFile as re, createReadStream as E, createWriteStream as D, existsSync as ie, unlinkSync as ae } from "node:fs";
|
|
9
|
+
import { mkdir as oe, readdir as se, stat as ce } from "node:fs/promises";
|
|
10
|
+
import { basename as le, delimiter as ue, dirname as de, join as O, resolve as fe, sep as pe } from "node:path";
|
|
11
|
+
import { promisify as me } from "node:util";
|
|
12
|
+
import { spawn as he } from "node:child_process";
|
|
13
13
|
//#region src/consistency.ts
|
|
14
|
-
var
|
|
14
|
+
var ge = (e) => {
|
|
15
15
|
switch (e) {
|
|
16
16
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
17
17
|
case "mysql":
|
|
@@ -21,7 +21,7 @@ var ve = (e) => {
|
|
|
21
21
|
case "turso": return [];
|
|
22
22
|
default: return [];
|
|
23
23
|
}
|
|
24
|
-
},
|
|
24
|
+
}, _e = {
|
|
25
25
|
email: { fake: "email" },
|
|
26
26
|
fullName: { fake: "fullName" },
|
|
27
27
|
firstName: { fake: "firstName" },
|
|
@@ -36,7 +36,7 @@ var ve = (e) => {
|
|
|
36
36
|
date: "dateShift",
|
|
37
37
|
secret: "null",
|
|
38
38
|
freeText: "redact"
|
|
39
|
-
},
|
|
39
|
+
}, ve = (e) => {
|
|
40
40
|
let t = {};
|
|
41
41
|
for (let n of e.tables) {
|
|
42
42
|
let e = {};
|
|
@@ -48,15 +48,15 @@ var ve = (e) => {
|
|
|
48
48
|
t[n.name] = e;
|
|
49
49
|
}
|
|
50
50
|
return t;
|
|
51
|
-
},
|
|
51
|
+
}, ye = (e, t, n, r) => {
|
|
52
52
|
let i = r.columns?.[`${e}.${t}`];
|
|
53
53
|
if (i) return i;
|
|
54
54
|
if (n?.sensitive) {
|
|
55
55
|
let e = n.sensitive.class;
|
|
56
|
-
return r.classes?.[e] ??
|
|
56
|
+
return r.classes?.[e] ?? _e[e] ?? "redact";
|
|
57
57
|
}
|
|
58
58
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
59
|
-
},
|
|
59
|
+
}, k = (e, t) => v("sha256", e).update(t == null ? "∅" : String(t)).digest(), A = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), j = [
|
|
60
60
|
"Ada",
|
|
61
61
|
"Alan",
|
|
62
62
|
"Grace",
|
|
@@ -69,7 +69,7 @@ var ve = (e) => {
|
|
|
69
69
|
"Guido",
|
|
70
70
|
"Margaret",
|
|
71
71
|
"Bjarne"
|
|
72
|
-
],
|
|
72
|
+
], M = [
|
|
73
73
|
"Lovelace",
|
|
74
74
|
"Turing",
|
|
75
75
|
"Hopper",
|
|
@@ -82,7 +82,7 @@ var ve = (e) => {
|
|
|
82
82
|
"Rossum",
|
|
83
83
|
"Hamilton",
|
|
84
84
|
"Stroustrup"
|
|
85
|
-
],
|
|
85
|
+
], be = [
|
|
86
86
|
"Maple",
|
|
87
87
|
"Oak",
|
|
88
88
|
"Cedar",
|
|
@@ -91,7 +91,7 @@ var ve = (e) => {
|
|
|
91
91
|
"Birch",
|
|
92
92
|
"Willow",
|
|
93
93
|
"Ash"
|
|
94
|
-
],
|
|
94
|
+
], xe = [
|
|
95
95
|
"Initech",
|
|
96
96
|
"Hooli",
|
|
97
97
|
"Umbrella",
|
|
@@ -100,23 +100,23 @@ var ve = (e) => {
|
|
|
100
100
|
"Soylent",
|
|
101
101
|
"Stark",
|
|
102
102
|
"Wayne"
|
|
103
|
-
],
|
|
104
|
-
let r =
|
|
103
|
+
], N = (e, t, n) => e[A(t, n) % e.length], Se = (e, t, n) => {
|
|
104
|
+
let r = k(n, `${e}:${String(t)}`), i = A(r, 1) % 1e4;
|
|
105
105
|
switch (e) {
|
|
106
|
-
case "email": return `${j
|
|
107
|
-
case "fullName": return `${j
|
|
108
|
-
case "firstName": return j
|
|
109
|
-
case "lastName": return
|
|
110
|
-
case "username": return `${j
|
|
111
|
-
case "phone": return `+1${String(
|
|
112
|
-
case "address": return `${
|
|
113
|
-
case "company": return
|
|
106
|
+
case "email": return `${N(j, r, 0).toLowerCase()}.${N(M, r, 2).toLowerCase()}${i}@example.com`;
|
|
107
|
+
case "fullName": return `${N(j, r, 0)} ${N(M, r, 1)}`;
|
|
108
|
+
case "firstName": return N(j, r, 0);
|
|
109
|
+
case "lastName": return N(M, r, 1);
|
|
110
|
+
case "username": return `${N(j, r, 0).toLowerCase()}${i}`;
|
|
111
|
+
case "phone": return `+1${String(A(r, 0) % 1e10).padStart(10, "0")}`;
|
|
112
|
+
case "address": return `${A(r, 0) % 9899 + 100} ${N(be, r, 1)} St`;
|
|
113
|
+
case "company": return N(xe, r, 0);
|
|
114
114
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
115
|
-
case "ip": return `${
|
|
116
|
-
case "creditCard": return
|
|
115
|
+
case "ip": return `${A(r, 0) % 256}.${A(r, 1) % 256}.${A(r, 2) % 256}.${A(r, 3) % 254 + 1}`;
|
|
116
|
+
case "creditCard": return Ce(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
117
117
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
118
118
|
}
|
|
119
|
-
},
|
|
119
|
+
}, Ce = (e) => {
|
|
120
120
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
121
121
|
t.forEach((e, t) => {
|
|
122
122
|
let r = e;
|
|
@@ -124,26 +124,26 @@ var ve = (e) => {
|
|
|
124
124
|
});
|
|
125
125
|
let r = (10 - n % 10) % 10;
|
|
126
126
|
return e.slice(0, 15) + String(r);
|
|
127
|
-
},
|
|
127
|
+
}, we = 864e5, Te = (e, t) => {
|
|
128
128
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
129
129
|
if (Number.isNaN(n)) return e;
|
|
130
|
-
let r =
|
|
130
|
+
let r = A(k(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * we);
|
|
131
131
|
return e instanceof Date ? i : i.toISOString();
|
|
132
|
-
},
|
|
132
|
+
}, Ee = (e, t) => {
|
|
133
133
|
if (t.value === null || t.value === void 0) return null;
|
|
134
134
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
135
|
-
return e === "keep" ? t.value : e === "null" ? null : e === "redact" ? t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null : e === "hash" ?
|
|
136
|
-
},
|
|
135
|
+
return e === "keep" ? t.value : e === "null" ? null : e === "redact" ? t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null : e === "hash" ? k(t.seed, t.value).toString("hex").slice(0, 16) : e === "dateShift" ? Te(t.value, t.seed) : typeof e == "object" && "fake" in e ? Se(e.fake, t.value, t.seed) : typeof e == "object" && "custom" in e ? e.custom(t) : t.value;
|
|
136
|
+
}, De = (e, t, n) => {
|
|
137
137
|
let r = {}, i = [];
|
|
138
138
|
for (let a of e) for (let e of a.columns) {
|
|
139
|
-
let o = `${a.name}.${e.name}`, s =
|
|
139
|
+
let o = `${a.name}.${e.name}`, s = ye(a.name, e.name, t[a.name]?.[e.name], n);
|
|
140
140
|
s === "error" ? i.push(o) : r[o] = s;
|
|
141
141
|
}
|
|
142
142
|
return {
|
|
143
143
|
actions: r,
|
|
144
144
|
unclassified: i
|
|
145
145
|
};
|
|
146
|
-
},
|
|
146
|
+
}, Oe = [
|
|
147
147
|
{
|
|
148
148
|
name: "email",
|
|
149
149
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -160,7 +160,7 @@ var ve = (e) => {
|
|
|
160
160
|
name: "creditCard",
|
|
161
161
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
162
162
|
}
|
|
163
|
-
],
|
|
163
|
+
], ke = (e, t, n) => {
|
|
164
164
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
165
165
|
for (let [a, o] of Object.entries(n)) {
|
|
166
166
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -168,7 +168,7 @@ var ve = (e) => {
|
|
|
168
168
|
for (let a of e) {
|
|
169
169
|
let e = a[n];
|
|
170
170
|
if (typeof e != "string" || e.length === 0) continue;
|
|
171
|
-
let o =
|
|
171
|
+
let o = Oe.find((t) => t.re.test(e));
|
|
172
172
|
if (o && !i.has(n)) {
|
|
173
173
|
i.add(n), r.push({
|
|
174
174
|
table: t,
|
|
@@ -181,9 +181,9 @@ var ve = (e) => {
|
|
|
181
181
|
}
|
|
182
182
|
}
|
|
183
183
|
return r;
|
|
184
|
-
},
|
|
184
|
+
}, P = (e, t, n, r, i) => {
|
|
185
185
|
let a = {};
|
|
186
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
186
|
+
for (let [o, s] of Object.entries(e)) a[o] = Ee(n[`${t}.${o}`] ?? "keep", {
|
|
187
187
|
value: s,
|
|
188
188
|
table: t,
|
|
189
189
|
column: o,
|
|
@@ -191,43 +191,43 @@ var ve = (e) => {
|
|
|
191
191
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
192
192
|
});
|
|
193
193
|
return a;
|
|
194
|
-
},
|
|
194
|
+
}, F = class extends s.TaggedError()("BundleError", {
|
|
195
195
|
reason: s.String,
|
|
196
196
|
path: s.optional(s.String)
|
|
197
|
-
}) {},
|
|
197
|
+
}) {}, Ae = class extends s.TaggedError()("CodecError", {
|
|
198
198
|
table: s.String,
|
|
199
199
|
reason: s.String
|
|
200
|
-
}) {},
|
|
200
|
+
}) {}, I = class extends s.TaggedError()("IntegrityError", {
|
|
201
201
|
what: s.String,
|
|
202
202
|
expected: s.String,
|
|
203
203
|
actual: s.String
|
|
204
|
-
}) {},
|
|
204
|
+
}) {}, je = class extends t.TaggedError("CompressionError") {}, Me = class extends s.TaggedError()("ScopeError", { reason: s.String }) {}, Ne = class extends s.TaggedError()("DanglingReferenceError", {
|
|
205
205
|
rows: s.Array(s.Struct({
|
|
206
206
|
table: s.String,
|
|
207
207
|
id: s.String,
|
|
208
208
|
reason: s.String
|
|
209
209
|
})),
|
|
210
210
|
totalCount: s.Number
|
|
211
|
-
}) {},
|
|
211
|
+
}) {}, Pe = class extends s.TaggedError()("SchemaDriftError", {
|
|
212
212
|
bundleFingerprint: s.String,
|
|
213
213
|
targetFingerprint: s.String,
|
|
214
214
|
diff: s.Array(s.String)
|
|
215
|
-
}) {},
|
|
215
|
+
}) {}, L = class extends t.TaggedError("NativeToolError") {
|
|
216
216
|
get message() {
|
|
217
217
|
let e = this.stderr?.trim();
|
|
218
218
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
219
219
|
}
|
|
220
|
-
},
|
|
220
|
+
}, Fe = class extends s.TaggedError()("MaskingError", { unclassified: s.Array(s.String) }) {}, Ie = class extends s.TaggedError()("ImportModeError", {
|
|
221
221
|
mode: s.String,
|
|
222
222
|
reason: s.String
|
|
223
|
-
}) {},
|
|
223
|
+
}) {}, Le = class extends s.TaggedError()("CrossDialectError", {
|
|
224
224
|
sourceDialect: s.String,
|
|
225
225
|
targetDialect: s.String,
|
|
226
226
|
blocking: s.Array(s.String)
|
|
227
|
-
}) {},
|
|
227
|
+
}) {}, R = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Re = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
228
228
|
col: e.name,
|
|
229
229
|
refTable: e.references.table
|
|
230
|
-
})),
|
|
230
|
+
})), z = (e, t, r) => n.gen(function* () {
|
|
231
231
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map(), d = (e) => {
|
|
232
232
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
233
233
|
return s.set(e, t), t;
|
|
@@ -245,10 +245,10 @@ var ve = (e) => {
|
|
|
245
245
|
for (let t of n) m(t.refTable, e[t.col]);
|
|
246
246
|
};
|
|
247
247
|
}, g = (r, i) => {
|
|
248
|
-
let a = h(r,
|
|
248
|
+
let a = h(r, R(t, r), Re(t, r));
|
|
249
249
|
return c.runForEach(p(e, {
|
|
250
250
|
table: r,
|
|
251
|
-
pk:
|
|
251
|
+
pk: R(t, r),
|
|
252
252
|
...i === void 0 ? {} : { where: i }
|
|
253
253
|
}), (e) => n.sync(() => a(e)));
|
|
254
254
|
}, _ = n.gen(function* () {
|
|
@@ -257,7 +257,7 @@ var ve = (e) => {
|
|
|
257
257
|
l.clear();
|
|
258
258
|
for (let [n, r] of e) {
|
|
259
259
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
260
|
-
e.length !== 0 && (yield* g(n, u(
|
|
260
|
+
e.length !== 0 && (yield* g(n, u(R(t, n), e)));
|
|
261
261
|
}
|
|
262
262
|
}
|
|
263
263
|
});
|
|
@@ -280,13 +280,13 @@ var ve = (e) => {
|
|
|
280
280
|
}
|
|
281
281
|
}
|
|
282
282
|
return o;
|
|
283
|
-
}),
|
|
283
|
+
}), ze = (e, t) => ({
|
|
284
284
|
tables: [...e.keys()],
|
|
285
285
|
predicateFor: (n) => {
|
|
286
286
|
let r = e.get(n);
|
|
287
|
-
return r ? u(
|
|
287
|
+
return r ? u(R(t, n), [...r]) : void 0;
|
|
288
288
|
}
|
|
289
|
-
}),
|
|
289
|
+
}), B = "tenantId", Be = (e, t, r) => n.gen(function* () {
|
|
290
290
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
291
291
|
if (e.kind === "all") return {
|
|
292
292
|
tables: i,
|
|
@@ -300,25 +300,25 @@ var ve = (e) => {
|
|
|
300
300
|
};
|
|
301
301
|
}
|
|
302
302
|
let o = e.tenantId;
|
|
303
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
304
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
305
|
-
if (s.length === 0) return yield* n.fail(new
|
|
306
|
-
let c = yield*
|
|
307
|
-
seeds: Object.fromEntries(s.map((e) => [e, l(
|
|
303
|
+
if (o === void 0 || o === "") return yield* n.fail(new Me({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
304
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === B) ?? !1);
|
|
305
|
+
if (s.length === 0) return yield* n.fail(new Me({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${B}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
306
|
+
let c = yield* z(r.store, t, {
|
|
307
|
+
seeds: Object.fromEntries(s.map((e) => [e, l(B, o)])),
|
|
308
308
|
pinned: s,
|
|
309
309
|
children: { roots: s }
|
|
310
310
|
}), d = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), f = new Set(s);
|
|
311
311
|
return {
|
|
312
312
|
tables: [...s, ...d],
|
|
313
|
-
predicateFor: (e) => f.has(e) ? l(
|
|
313
|
+
predicateFor: (e) => f.has(e) ? l(B, o) : c.has(e) ? u(R(t, e), [...c.get(e)]) : void 0
|
|
314
314
|
};
|
|
315
|
-
}),
|
|
315
|
+
}), Ve = (e, t) => ({
|
|
316
316
|
list: t,
|
|
317
317
|
open: (t) => C(e, t)
|
|
318
|
-
}),
|
|
318
|
+
}), He = (e) => ({
|
|
319
319
|
write: (t, n, r) => ee(e, t, n, r),
|
|
320
320
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
321
|
-
}),
|
|
321
|
+
}), Ue = (e, t) => n.tryPromise({
|
|
322
322
|
try: async () => {
|
|
323
323
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
324
324
|
u.update(e), d += e.length, n(null, e);
|
|
@@ -336,23 +336,23 @@ var ve = (e) => {
|
|
|
336
336
|
contentType: e.contentType
|
|
337
337
|
};
|
|
338
338
|
},
|
|
339
|
-
catch: (e) => new
|
|
339
|
+
catch: (e) => new F({
|
|
340
340
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
341
341
|
path: t
|
|
342
342
|
})
|
|
343
|
-
}),
|
|
343
|
+
}), We = (e, t, r) => n.gen(function* () {
|
|
344
344
|
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), l = o("sha256"), u = S("bundle", () => a(s)).pipe(c.tap((e) => n.sync(() => l.update(e))));
|
|
345
345
|
yield* e.write(r.key, u, {
|
|
346
346
|
contentType: r.contentType,
|
|
347
347
|
totalSize: r.size
|
|
348
348
|
});
|
|
349
349
|
let d = l.digest("hex");
|
|
350
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
350
|
+
if (d !== r.sha256) return yield* n.fail(new I({
|
|
351
351
|
what: `asset ${r.key}`,
|
|
352
352
|
expected: r.sha256,
|
|
353
353
|
actual: d
|
|
354
354
|
}));
|
|
355
|
-
}),
|
|
355
|
+
}), Ge = () => typeof w.createZstdCompress == "function", Ke = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : Ge() ? "zstd" : "gzip", qe = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", Je = (e) => e === "zstd" ? w.createZstdCompress() : e === "gzip" ? w.createGzip() : new T(), Ye = (e) => e === "zstd" ? w.createZstdDecompress() : e === "gzip" ? w.createGunzip() : new T(), Xe = 1, Ze = s.Struct({
|
|
356
356
|
name: s.String,
|
|
357
357
|
file: s.String,
|
|
358
358
|
rowCount: s.Number,
|
|
@@ -362,11 +362,11 @@ var ve = (e) => {
|
|
|
362
362
|
value: s.String
|
|
363
363
|
}),
|
|
364
364
|
primaryKey: s.String
|
|
365
|
-
}),
|
|
365
|
+
}), Qe = s.Struct({
|
|
366
366
|
kind: s.Literal("all", "tenant", "tables"),
|
|
367
367
|
tenantId: s.optional(s.String),
|
|
368
368
|
tables: s.optional(s.Array(s.String))
|
|
369
|
-
}),
|
|
369
|
+
}), $e = s.Struct({
|
|
370
370
|
count: s.Number,
|
|
371
371
|
totalBytes: s.Number,
|
|
372
372
|
index: s.optional(s.String)
|
|
@@ -377,23 +377,23 @@ s.Struct({
|
|
|
377
377
|
size: s.Number,
|
|
378
378
|
contentType: s.String
|
|
379
379
|
});
|
|
380
|
-
var
|
|
380
|
+
var et = s.Struct({
|
|
381
381
|
formatVersion: s.Number,
|
|
382
382
|
createdAt: s.String,
|
|
383
383
|
source: s.Struct({
|
|
384
384
|
dialect: s.String,
|
|
385
385
|
schemaFingerprint: s.String
|
|
386
386
|
}),
|
|
387
|
-
scope:
|
|
387
|
+
scope: Qe,
|
|
388
388
|
consistency: s.Literal("live", "snapshot"),
|
|
389
389
|
compression: s.Literal("zstd", "gzip", "none"),
|
|
390
|
-
tables: s.Array(
|
|
391
|
-
assets:
|
|
390
|
+
tables: s.Array(Ze),
|
|
391
|
+
assets: $e,
|
|
392
392
|
masking: s.optional(s.Struct({
|
|
393
393
|
policyId: s.optional(s.String),
|
|
394
394
|
transformed: s.Array(s.String)
|
|
395
395
|
}))
|
|
396
|
-
}),
|
|
396
|
+
}), tt = s.encodeSync(s.parseJson(et)), nt = s.decodeUnknownSync(s.parseJson(et)), V = "manifest.json", rt = "README.md", it = "data", H = "assets", U = "assets/index.ndjson", at = ".ledger.json", ot = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", st = (e, t) => {
|
|
397
397
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
398
398
|
return `# Voltro data export
|
|
399
399
|
|
|
@@ -407,7 +407,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
407
407
|
| **Format version** | ${e.formatVersion} |
|
|
408
408
|
| **Consistency** | ${e.consistency} |
|
|
409
409
|
| **Compression** | ${e.compression} |
|
|
410
|
-
| **Scope** | ${
|
|
410
|
+
| **Scope** | ${ot(e.scope)} |
|
|
411
411
|
| **Tables** | ${e.tables.length} |
|
|
412
412
|
| **Rows (total)** | ${n} |
|
|
413
413
|
| **Assets** | ${a} |
|
|
@@ -432,39 +432,39 @@ voltro data import <this-directory>
|
|
|
432
432
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
433
433
|
\`\`\`
|
|
434
434
|
`;
|
|
435
|
-
},
|
|
435
|
+
}, ct = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), lt = a.exponential({
|
|
436
436
|
start: .01,
|
|
437
437
|
factor: 2,
|
|
438
438
|
count: 16
|
|
439
|
-
}),
|
|
439
|
+
}), ut = i.histogram("voltro_data_transfer_table_seconds", lt, "Per-table export/import duration in seconds, tagged by phase."), W = (e, t) => {
|
|
440
440
|
if (e) try {
|
|
441
441
|
e(t);
|
|
442
442
|
} catch {}
|
|
443
|
-
},
|
|
443
|
+
}, dt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(ut.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), ct.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
444
444
|
table: t,
|
|
445
445
|
index: r,
|
|
446
446
|
tableCount: a,
|
|
447
447
|
phase: e
|
|
448
|
-
} })),
|
|
448
|
+
} })), ft = () => ({
|
|
449
449
|
tables: {},
|
|
450
450
|
assets: {}
|
|
451
|
-
}),
|
|
451
|
+
}), G = (e) => n.tryPromise(async () => {
|
|
452
452
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
453
453
|
return JSON.parse(t);
|
|
454
|
-
}).pipe(n.orElseSucceed(
|
|
454
|
+
}).pipe(n.orElseSucceed(ft)), K = (e, t) => n.tryPromise({
|
|
455
455
|
try: async () => {
|
|
456
456
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
457
457
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
458
458
|
},
|
|
459
|
-
catch: (t) => new
|
|
459
|
+
catch: (t) => new F({
|
|
460
460
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
461
461
|
path: e
|
|
462
462
|
})
|
|
463
|
-
}),
|
|
463
|
+
}), pt = (e, t) => t in e.tables, mt = (e, t) => t in e.assets, ht = /* @__PURE__ */ new Set([
|
|
464
464
|
"json",
|
|
465
465
|
"array",
|
|
466
466
|
"vector"
|
|
467
|
-
]),
|
|
467
|
+
]), gt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", _t = (e) => Buffer.from(e).toString("base64"), vt = (e) => new Uint8Array(Buffer.from(e, "base64")), yt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), bt = (e, t) => e == null ? null : t !== void 0 && ht.has(t) ? e : t === "bytes" ? _t(e) : t === "timestamp" || t === "date" ? yt(e) : typeof e == "bigint" ? {
|
|
468
468
|
$v: "bigint",
|
|
469
469
|
d: e.toString()
|
|
470
470
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -475,30 +475,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
475
475
|
d: e.toISOString()
|
|
476
476
|
} : e instanceof Uint8Array ? {
|
|
477
477
|
$v: "bytes",
|
|
478
|
-
d:
|
|
479
|
-
} : e,
|
|
478
|
+
d: _t(e)
|
|
479
|
+
} : e, xt = (e) => {
|
|
480
480
|
switch (e.$v) {
|
|
481
481
|
case "bigint": return BigInt(e.d);
|
|
482
482
|
case "number": return Number(e.d);
|
|
483
483
|
case "date": return new Date(e.d);
|
|
484
|
-
case "bytes": return
|
|
484
|
+
case "bytes": return vt(e.d);
|
|
485
485
|
default: return e;
|
|
486
486
|
}
|
|
487
|
-
},
|
|
487
|
+
}, St = (e, t) => e == null ? null : t !== void 0 && ht.has(t) ? e : gt(e) ? xt(e) : t === "bytes" ? vt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Ct = (e, t) => {
|
|
488
488
|
let n = {};
|
|
489
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
489
|
+
for (let [r, i] of Object.entries(e)) n[r] = bt(i, t[r]);
|
|
490
490
|
return n;
|
|
491
|
-
},
|
|
491
|
+
}, wt = (e, t) => {
|
|
492
492
|
let n = {};
|
|
493
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
493
|
+
for (let [r, i] of Object.entries(e)) n[r] = St(i, t[r]);
|
|
494
494
|
return n;
|
|
495
|
-
},
|
|
496
|
-
try: () =>
|
|
497
|
-
catch: (e) => new
|
|
495
|
+
}, Tt = new TextEncoder(), Et = (e, t) => e.pipe(c.map((e) => Tt.encode(`${JSON.stringify(Ct(e, t))}\n`))), Dt = (e, t, r) => e.pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.mapEffect((e) => n.try({
|
|
496
|
+
try: () => wt(JSON.parse(e), r),
|
|
497
|
+
catch: (e) => new Ae({
|
|
498
498
|
table: t,
|
|
499
499
|
reason: String(e?.message ?? e)
|
|
500
500
|
})
|
|
501
|
-
}))),
|
|
501
|
+
}))), q = (e, t) => {
|
|
502
502
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
503
503
|
for (let e of t.tables) {
|
|
504
504
|
if (!n.has(e.name)) continue;
|
|
@@ -518,11 +518,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
518
518
|
};
|
|
519
519
|
for (let t of e) n.has(t) && s(t);
|
|
520
520
|
return i;
|
|
521
|
-
},
|
|
521
|
+
}, Ot = (e, t) => {
|
|
522
522
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
523
523
|
for (let e of n?.columns ?? []) r[e.name] = e.type;
|
|
524
524
|
return r;
|
|
525
|
-
},
|
|
525
|
+
}, kt = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", At = (e, t, r) => n.tryPromise({
|
|
526
526
|
try: async () => {
|
|
527
527
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: l, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
528
528
|
d.update(e);
|
|
@@ -531,7 +531,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
531
531
|
n(null, e);
|
|
532
532
|
} }), m = l.fromWeb(c.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
533
533
|
try {
|
|
534
|
-
await s(m, p,
|
|
534
|
+
await s(m, p, Je(r), n(h)), await i.rename(h, t);
|
|
535
535
|
} catch (e) {
|
|
536
536
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
537
537
|
}
|
|
@@ -540,25 +540,25 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
540
540
|
checksum: d.digest("hex")
|
|
541
541
|
};
|
|
542
542
|
},
|
|
543
|
-
catch: (e) => new
|
|
543
|
+
catch: (e) => new F({
|
|
544
544
|
reason: `write table failed: ${String(e?.message ?? e)}`,
|
|
545
545
|
path: t
|
|
546
546
|
})
|
|
547
|
-
}),
|
|
547
|
+
}), jt = (e, t) => n.tryPromise({
|
|
548
548
|
try: async () => {
|
|
549
549
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
550
550
|
},
|
|
551
|
-
catch: (t) => new
|
|
551
|
+
catch: (t) => new F({
|
|
552
552
|
reason: `append failed: ${String(t?.message ?? t)}`,
|
|
553
553
|
path: e
|
|
554
554
|
})
|
|
555
|
-
}),
|
|
555
|
+
}), Mt = (e, t) => n.gen(function* () {
|
|
556
556
|
let n = [], r = t.ordered.length;
|
|
557
557
|
for (let i = 0; i < t.ordered.length; i++) {
|
|
558
|
-
let a = t.ordered[i], o =
|
|
559
|
-
if (
|
|
558
|
+
let a = t.ordered[i], o = Ot(t.snapshot, a), s = kt(t.snapshot, a), l = `${it}/${a}.ndjson${qe(t.compression)}`;
|
|
559
|
+
if (pt(t.ledger, a)) {
|
|
560
560
|
let e = t.ledger.tables[a];
|
|
561
|
-
|
|
561
|
+
W(t.onProgress, {
|
|
562
562
|
phase: "export",
|
|
563
563
|
table: a,
|
|
564
564
|
index: i,
|
|
@@ -576,7 +576,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
576
576
|
});
|
|
577
577
|
continue;
|
|
578
578
|
}
|
|
579
|
-
|
|
579
|
+
W(t.onProgress, {
|
|
580
580
|
phase: "export",
|
|
581
581
|
table: a,
|
|
582
582
|
index: i,
|
|
@@ -590,11 +590,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
590
590
|
chunkSize: t.chunkSize,
|
|
591
591
|
retry: t.retry,
|
|
592
592
|
...u === void 0 ? {} : { where: u }
|
|
593
|
-
}), f = t.mask ? d.pipe(c.map((e) =>
|
|
593
|
+
}), f = t.mask ? d.pipe(c.map((e) => P(e, a, t.mask.actions, t.mask.seed, o))) : d, { rowCount: m, checksum: h } = yield* dt("export", a, i, r, (e) => e.rowCount, At(Et(f, o), t.bundleFile(l), t.compression));
|
|
594
594
|
t.ledger.tables[a] = {
|
|
595
595
|
rowCount: m,
|
|
596
596
|
checksum: h
|
|
597
|
-
}, yield*
|
|
597
|
+
}, yield* K(t.ledgerPath, t.ledger), W(t.onProgress, {
|
|
598
598
|
phase: "export",
|
|
599
599
|
table: a,
|
|
600
600
|
index: i,
|
|
@@ -612,43 +612,43 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
612
612
|
});
|
|
613
613
|
}
|
|
614
614
|
return n;
|
|
615
|
-
}),
|
|
615
|
+
}), Nt = (e) => ({
|
|
616
616
|
_tag: "RawSqlFragment",
|
|
617
617
|
strings: [e],
|
|
618
618
|
values: []
|
|
619
|
-
}),
|
|
619
|
+
}), Pt = (t, i, a) => n.tryPromise({
|
|
620
620
|
try: () => t.transactional(async (t) => {
|
|
621
|
-
if (t.raw) for (let e of
|
|
621
|
+
if (t.raw) for (let e of ge(i)) await t.raw(Nt(e));
|
|
622
622
|
let o = await n.runPromiseExit(a(t));
|
|
623
623
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
624
624
|
return o.value;
|
|
625
625
|
}),
|
|
626
|
-
catch: (e) => e instanceof
|
|
627
|
-
}),
|
|
628
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
626
|
+
catch: (e) => e instanceof F ? e : new F({ reason: `snapshot export failed: ${String(e?.message ?? e)}` })
|
|
627
|
+
}), Ft = (e) => n.gen(function* () {
|
|
628
|
+
let t = e.scope ?? { kind: "all" }, r = Ke(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, s = o.exponential("50 millis").pipe(o.jittered, o.intersect(o.recurs(a))), l = yield* n.promise(() => import("node:path")), u = (t) => l.join(e.outDir, t);
|
|
629
629
|
yield* n.tryPromise({
|
|
630
630
|
try: async () => {
|
|
631
631
|
let t = await import("node:fs/promises");
|
|
632
|
-
await t.mkdir(u(
|
|
632
|
+
await t.mkdir(u(it), { recursive: !0 }), e.assets && await t.mkdir(u(H), { recursive: !0 });
|
|
633
633
|
},
|
|
634
|
-
catch: (t) => new
|
|
634
|
+
catch: (t) => new F({
|
|
635
635
|
reason: `mkdir failed: ${String(t?.message ?? t)}`,
|
|
636
636
|
path: e.outDir
|
|
637
637
|
})
|
|
638
638
|
});
|
|
639
|
-
let d = u(
|
|
639
|
+
let d = u(at), f = yield* G(d), p = e.subset ? ze(yield* z(e.store, e.snapshot, e.subset), e.snapshot) : yield* Be(t, e.snapshot, {
|
|
640
640
|
store: e.store,
|
|
641
641
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
642
|
-
}), h =
|
|
642
|
+
}), h = q(p.tables, e.snapshot), g = e.consistency ?? "live", _, v;
|
|
643
643
|
if (e.masking) {
|
|
644
|
-
let t = e.classification ??
|
|
644
|
+
let t = e.classification ?? ve(e.snapshot), r = De(h.map((t) => ({
|
|
645
645
|
name: t,
|
|
646
646
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
647
647
|
name: e.name,
|
|
648
648
|
type: e.type
|
|
649
649
|
}))
|
|
650
650
|
})), t, e.masking);
|
|
651
|
-
if (r.unclassified.length > 0) return yield* n.fail(new
|
|
651
|
+
if (r.unclassified.length > 0) return yield* n.fail(new Fe({ unclassified: r.unclassified }));
|
|
652
652
|
v = {
|
|
653
653
|
actions: r.actions,
|
|
654
654
|
seed: e.masking.seed
|
|
@@ -671,18 +671,18 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
671
671
|
bundleFile: u,
|
|
672
672
|
...v ? { mask: v } : {},
|
|
673
673
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
674
|
-
}, b = g === "snapshot" ? yield*
|
|
674
|
+
}, b = g === "snapshot" ? yield* Pt(e.store, e.dialect, (e) => Mt(e, y)) : yield* Mt(e.store, y), x = 0, S = 0;
|
|
675
675
|
if (e.assets) {
|
|
676
|
-
let t = u(
|
|
677
|
-
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
676
|
+
let t = u(U);
|
|
677
|
+
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => Ue(e, u(H))), n.flatMap((e) => {
|
|
678
678
|
let i = {
|
|
679
679
|
key: r.key,
|
|
680
680
|
sha256: e.sha256,
|
|
681
681
|
size: e.size,
|
|
682
682
|
contentType: e.contentType
|
|
683
683
|
};
|
|
684
|
-
return x++, S += e.size, f.assets[r.key] = !0,
|
|
685
|
-
}))).pipe(n.mapError((e) => e instanceof
|
|
684
|
+
return x++, S += e.size, f.assets[r.key] = !0, jt(t, JSON.stringify(i)).pipe(n.andThen(K(d, f)));
|
|
685
|
+
}))).pipe(n.mapError((e) => e instanceof F ? e : new F({ reason: `asset export failed: ${String(e)}` })));
|
|
686
686
|
}
|
|
687
687
|
let C = {
|
|
688
688
|
formatVersion: 1,
|
|
@@ -698,7 +698,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
698
698
|
assets: x > 0 ? {
|
|
699
699
|
count: x,
|
|
700
700
|
totalBytes: S,
|
|
701
|
-
index:
|
|
701
|
+
index: U
|
|
702
702
|
} : {
|
|
703
703
|
count: 0,
|
|
704
704
|
totalBytes: 0
|
|
@@ -707,23 +707,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
707
707
|
};
|
|
708
708
|
return yield* n.tryPromise({
|
|
709
709
|
try: async () => {
|
|
710
|
-
await (await import("node:fs/promises")).writeFile(u(
|
|
710
|
+
await (await import("node:fs/promises")).writeFile(u(V), tt(C));
|
|
711
711
|
},
|
|
712
|
-
catch: (e) => new
|
|
712
|
+
catch: (e) => new F({
|
|
713
713
|
reason: `manifest write failed: ${String(e?.message ?? e)}`,
|
|
714
|
-
path:
|
|
714
|
+
path: V
|
|
715
715
|
})
|
|
716
716
|
}), yield* n.tryPromise({
|
|
717
717
|
try: async () => {
|
|
718
718
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
719
|
-
await e.writeFile(u(
|
|
719
|
+
await e.writeFile(u(rt), st(C, { hostname: t.hostname() }));
|
|
720
720
|
},
|
|
721
|
-
catch: (e) => new
|
|
721
|
+
catch: (e) => new F({
|
|
722
722
|
reason: `readme write failed: ${String(e?.message ?? e)}`,
|
|
723
|
-
path:
|
|
723
|
+
path: rt
|
|
724
724
|
})
|
|
725
725
|
}), C;
|
|
726
|
-
}),
|
|
726
|
+
}), It = (e) => {
|
|
727
727
|
switch (e) {
|
|
728
728
|
case "id":
|
|
729
729
|
case "text":
|
|
@@ -742,12 +742,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
742
742
|
case "raw": return "raw";
|
|
743
743
|
default: return "string";
|
|
744
744
|
}
|
|
745
|
-
},
|
|
745
|
+
}, Lt = {
|
|
746
746
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
747
747
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
748
748
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
749
|
-
},
|
|
750
|
-
let i =
|
|
749
|
+
}, Rt = (e, t, n, r) => {
|
|
750
|
+
let i = It(n);
|
|
751
751
|
if (i === "raw") return {
|
|
752
752
|
table: e,
|
|
753
753
|
column: t,
|
|
@@ -755,7 +755,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
755
755
|
severity: "block",
|
|
756
756
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
757
757
|
};
|
|
758
|
-
let a =
|
|
758
|
+
let a = Lt[i];
|
|
759
759
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
760
760
|
table: e,
|
|
761
761
|
column: t,
|
|
@@ -769,20 +769,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
769
769
|
severity: "block",
|
|
770
770
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
771
771
|
};
|
|
772
|
-
},
|
|
772
|
+
}, zt = (e, t, n) => {
|
|
773
773
|
if (t === n) return [];
|
|
774
774
|
let r = [];
|
|
775
775
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
776
|
-
let a =
|
|
776
|
+
let a = Rt(t.name, e, i, n);
|
|
777
777
|
a && r.push(a);
|
|
778
778
|
}
|
|
779
779
|
return r;
|
|
780
|
-
},
|
|
780
|
+
}, Bt = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, Vt = (e, t) => new Map((e.tables.find((e) => e.name === t)?.columns ?? []).map((e) => [e.name, e.type])), Ht = (e, t) => {
|
|
781
781
|
let n = [], r = new Set(e.tables.map((e) => e.name)), i = new Set(t.tables.map((e) => e.name));
|
|
782
782
|
for (let e of [...r].filter((e) => !i.has(e)).sort()) n.push(`table '${e}' is in the bundle but MISSING from the target`);
|
|
783
783
|
for (let e of [...i].filter((e) => !r.has(e)).sort()) n.push(`table '${e}' is in the target but NOT in the bundle (import leaves it untouched)`);
|
|
784
784
|
for (let a of [...r].filter((e) => i.has(e)).sort()) {
|
|
785
|
-
let r =
|
|
785
|
+
let r = Vt(e, a), i = Vt(t, a);
|
|
786
786
|
for (let [e] of r) i.has(e) || n.push(`${a}.${e}: in the bundle, MISSING from the target`);
|
|
787
787
|
for (let [e] of i) r.has(e) || n.push(`${a}.${e}: in the target, not in the bundle`);
|
|
788
788
|
for (let [e, t] of r) {
|
|
@@ -791,7 +791,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
791
791
|
}
|
|
792
792
|
}
|
|
793
793
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n.length > 0 ? n : ["the schema shape differs (indexes, nullability, or defaults) — the specific change is below the table/column granularity shown here"];
|
|
794
|
-
},
|
|
794
|
+
}, Ut = (e, t, n) => {
|
|
795
795
|
let r = m(t);
|
|
796
796
|
return r === e ? {
|
|
797
797
|
drifted: !1,
|
|
@@ -802,20 +802,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
802
802
|
drifted: !0,
|
|
803
803
|
bundleFingerprint: e,
|
|
804
804
|
targetFingerprint: r,
|
|
805
|
-
diff: n ?
|
|
805
|
+
diff: n ? Ht(n, t) : ["the target schema fingerprint does not match the bundle (no source schema available to itemise the diff — the manifest records only the fingerprint)"]
|
|
806
806
|
};
|
|
807
|
-
},
|
|
807
|
+
}, Wt = (e) => n.tryPromise({
|
|
808
808
|
try: async () => {
|
|
809
809
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
810
|
-
return
|
|
810
|
+
return nt(await t.readFile(n.join(e, V), "utf8"));
|
|
811
811
|
},
|
|
812
|
-
catch: (e) => new
|
|
812
|
+
catch: (e) => new F({
|
|
813
813
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
814
|
-
path:
|
|
814
|
+
path: V
|
|
815
815
|
})
|
|
816
|
-
}),
|
|
817
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
818
|
-
if (a.formatVersion !== 1) return yield* n.fail(new
|
|
816
|
+
}), Gt = (t) => n.gen(function* () {
|
|
817
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* Wt(t.bundleDir);
|
|
818
|
+
if (a.formatVersion !== 1) return yield* n.fail(new F({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
819
819
|
if (t.targetSnapshot) {
|
|
820
820
|
let e = { tables: a.tables.map((e) => ({
|
|
821
821
|
name: e.name,
|
|
@@ -827,9 +827,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
827
827
|
unique: !1,
|
|
828
828
|
hasDefault: !1
|
|
829
829
|
}))
|
|
830
|
-
})) }, r =
|
|
830
|
+
})) }, r = Ut(a.source.schemaFingerprint, t.targetSnapshot, e);
|
|
831
831
|
if (r.drifted) {
|
|
832
|
-
if (!t.force) return yield* n.fail(new
|
|
832
|
+
if (!t.force) return yield* n.fail(new Pe({
|
|
833
833
|
bundleFingerprint: r.bundleFingerprint,
|
|
834
834
|
targetFingerprint: r.targetFingerprint,
|
|
835
835
|
diff: r.diff
|
|
@@ -838,15 +838,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
838
838
|
}
|
|
839
839
|
}
|
|
840
840
|
if (t.targetDialect && t.targetDialect !== a.source.dialect) {
|
|
841
|
-
let e =
|
|
842
|
-
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new
|
|
841
|
+
let e = zt(a.tables, a.source.dialect, t.targetDialect).filter((e) => e.severity === "block");
|
|
842
|
+
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new Le({
|
|
843
843
|
sourceDialect: a.source.dialect,
|
|
844
844
|
targetDialect: t.targetDialect,
|
|
845
|
-
blocking: e.map(
|
|
845
|
+
blocking: e.map(Bt)
|
|
846
846
|
}));
|
|
847
847
|
}
|
|
848
|
-
let o = t.verify ?? !0, s = t.mode ?? "upsert", l = t.onConflict ?? "skip", u = t.ledgerPath ?? i.join(t.bundleDir, ".import..ledger.json"), f = yield*
|
|
849
|
-
if (s === "replace" && a.scope.kind !== "all") return yield* n.fail(new
|
|
848
|
+
let o = t.verify ?? !0, s = t.mode ?? "upsert", l = t.onConflict ?? "skip", u = t.ledgerPath ?? i.join(t.bundleDir, ".import..ledger.json"), f = yield* G(u);
|
|
849
|
+
if (s === "replace" && a.scope.kind !== "all") return yield* n.fail(new Ie({
|
|
850
850
|
mode: s,
|
|
851
851
|
reason: `replace refuses a partial bundle (scope='${a.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
852
852
|
}));
|
|
@@ -875,7 +875,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
875
875
|
r = t;
|
|
876
876
|
}
|
|
877
877
|
for (let t of i) (yield* m(() => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] }), t)) || (r = [...r, t]);
|
|
878
|
-
if (r.length > 0) return yield* n.fail(new
|
|
878
|
+
if (r.length > 0) return yield* n.fail(new Ne({
|
|
879
879
|
rows: r.slice(0, 20).map((e) => ({
|
|
880
880
|
table: e.entry.name,
|
|
881
881
|
id: String(e.row[e.entry.primaryKey]),
|
|
@@ -887,89 +887,128 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
887
887
|
if (s === "replace" && !(r && f.truncated)) {
|
|
888
888
|
for (let t of [...a.tables].reverse()) yield* n.tryPromise({
|
|
889
889
|
try: () => e.deleteMany(t.name, { where: d(t.primaryKey) }),
|
|
890
|
-
catch: (e) => new
|
|
890
|
+
catch: (e) => new F({ reason: `truncate ${t.name} failed: ${String(e?.message ?? e)}` })
|
|
891
891
|
});
|
|
892
|
-
r && (f.truncated = !0, yield*
|
|
892
|
+
r && (f.truncated = !0, yield* K(u, f));
|
|
893
893
|
}
|
|
894
|
-
let l = [],
|
|
895
|
-
for (let
|
|
896
|
-
let
|
|
897
|
-
if (r &&
|
|
898
|
-
|
|
894
|
+
let l = [], m = [], h = a.tables.length;
|
|
895
|
+
for (let d = 0; d < a.tables.length; d++) {
|
|
896
|
+
let g = a.tables[d];
|
|
897
|
+
if (r && g.name in f.tables) {
|
|
898
|
+
W(t.onProgress, {
|
|
899
899
|
phase: "import",
|
|
900
|
-
table:
|
|
901
|
-
index:
|
|
902
|
-
tableCount:
|
|
900
|
+
table: g.name,
|
|
901
|
+
index: d,
|
|
902
|
+
tableCount: h,
|
|
903
903
|
event: "done",
|
|
904
|
-
rowsDone: f.tables[
|
|
905
|
-
total:
|
|
904
|
+
rowsDone: f.tables[g.name].rowCount,
|
|
905
|
+
total: g.rowCount
|
|
906
906
|
});
|
|
907
907
|
continue;
|
|
908
908
|
}
|
|
909
|
-
let
|
|
910
|
-
|
|
909
|
+
let _ = i.join(t.bundleDir, g.file);
|
|
910
|
+
W(t.onProgress, {
|
|
911
911
|
phase: "import",
|
|
912
|
-
table:
|
|
913
|
-
index:
|
|
914
|
-
tableCount:
|
|
912
|
+
table: g.name,
|
|
913
|
+
index: d,
|
|
914
|
+
tableCount: h,
|
|
915
915
|
event: "start",
|
|
916
916
|
rowsDone: 0,
|
|
917
|
-
total:
|
|
917
|
+
total: g.rowCount
|
|
918
918
|
});
|
|
919
919
|
let v = l.length, y = n.gen(function* () {
|
|
920
|
-
let { createHash:
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
let r = {
|
|
924
|
-
entry: d,
|
|
920
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, u = S("bundle", () => E(_).pipe(Ye(a.compression))).pipe(c.tap((e) => n.sync(() => i.update(e)))), d = async (t) => {
|
|
921
|
+
let n = {
|
|
922
|
+
entry: g,
|
|
925
923
|
row: t,
|
|
926
924
|
reason: ""
|
|
927
925
|
};
|
|
928
|
-
|
|
929
|
-
e
|
|
930
|
-
})
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
926
|
+
try {
|
|
927
|
+
await p(e, g, t);
|
|
928
|
+
} catch (e) {
|
|
929
|
+
n.reason = String(e?.message ?? e), l.push(n);
|
|
930
|
+
}
|
|
931
|
+
}, f = yield* n.promise(async () => {
|
|
932
|
+
let n = t.copyLoader;
|
|
933
|
+
if (!n || n.dialect !== "postgres" || t.atomic) return !1;
|
|
934
|
+
if (s === "replace") return !0;
|
|
935
|
+
if (s !== "upsert") return !1;
|
|
936
|
+
try {
|
|
937
|
+
return (await e.query({
|
|
938
|
+
table: g.name,
|
|
939
|
+
predicate: void 0,
|
|
940
|
+
order: [],
|
|
941
|
+
take: 1,
|
|
942
|
+
skip: void 0,
|
|
943
|
+
projection: void 0
|
|
944
|
+
})).length === 0;
|
|
945
|
+
} catch {
|
|
946
|
+
return !1;
|
|
947
|
+
}
|
|
948
|
+
}), m = Object.keys(g.columnTypes), h = Math.max(1, t.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
949
|
+
if (v.length === 0) return;
|
|
950
|
+
let e = v;
|
|
951
|
+
v = [];
|
|
952
|
+
try {
|
|
953
|
+
await t.copyLoader.copyInto({
|
|
954
|
+
table: g.name,
|
|
955
|
+
columns: m,
|
|
956
|
+
columnTypes: g.columnTypes,
|
|
957
|
+
rows: e
|
|
958
|
+
});
|
|
959
|
+
} catch (n) {
|
|
960
|
+
f = !1, t.onWarn?.(`bulk COPY into ${g.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
961
|
+
for (let t of e) await d(t);
|
|
962
|
+
}
|
|
963
|
+
};
|
|
964
|
+
return yield* c.runForEach(Dt(u, g.name, g.columnTypes), (e) => n.promise(async () => {
|
|
965
|
+
if (o++, f) {
|
|
966
|
+
v.push(e), v.length >= h && await y();
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
await d(e);
|
|
970
|
+
})), yield* n.promise(y), {
|
|
971
|
+
rowCount: o,
|
|
972
|
+
checksum: i.digest("hex")
|
|
934
973
|
};
|
|
935
|
-
}), { rowCount: b, checksum: x } = yield*
|
|
974
|
+
}), { rowCount: b, checksum: x } = yield* dt("import", g.name, d, h, (e) => e.rowCount, y);
|
|
936
975
|
if (o) {
|
|
937
|
-
if (x !==
|
|
938
|
-
what: `table ${
|
|
939
|
-
expected:
|
|
976
|
+
if (x !== g.checksum) return yield* n.fail(new I({
|
|
977
|
+
what: `table ${g.name} checksum`,
|
|
978
|
+
expected: g.checksum,
|
|
940
979
|
actual: x
|
|
941
980
|
}));
|
|
942
|
-
if (b !==
|
|
943
|
-
what: `table ${
|
|
944
|
-
expected: String(
|
|
981
|
+
if (b !== g.rowCount) return yield* n.fail(new I({
|
|
982
|
+
what: `table ${g.name} rowCount`,
|
|
983
|
+
expected: String(g.rowCount),
|
|
945
984
|
actual: String(b)
|
|
946
985
|
}));
|
|
947
986
|
}
|
|
948
|
-
|
|
987
|
+
W(t.onProgress, {
|
|
949
988
|
phase: "import",
|
|
950
|
-
table:
|
|
951
|
-
index:
|
|
952
|
-
tableCount:
|
|
989
|
+
table: g.name,
|
|
990
|
+
index: d,
|
|
991
|
+
tableCount: h,
|
|
953
992
|
event: "done",
|
|
954
993
|
rowsDone: b,
|
|
955
|
-
total:
|
|
956
|
-
}), r && (l.length > v ?
|
|
957
|
-
name:
|
|
994
|
+
total: g.rowCount
|
|
995
|
+
}), r && (l.length > v ? m.push({
|
|
996
|
+
name: g.name,
|
|
958
997
|
rowCount: b
|
|
959
|
-
}) : (f.tables[
|
|
998
|
+
}) : (f.tables[g.name] = {
|
|
960
999
|
rowCount: b,
|
|
961
|
-
checksum:
|
|
962
|
-
}, yield*
|
|
1000
|
+
checksum: g.checksum
|
|
1001
|
+
}, yield* K(u, f)));
|
|
963
1002
|
}
|
|
964
|
-
if (l.length > 0 && (yield* g(e, l)), r &&
|
|
965
|
-
for (let e of
|
|
1003
|
+
if (l.length > 0 && (yield* g(e, l)), r && m.length > 0) {
|
|
1004
|
+
for (let e of m) {
|
|
966
1005
|
let t = a.tables.find((t) => t.name === e.name);
|
|
967
1006
|
f.tables[e.name] = {
|
|
968
1007
|
rowCount: e.rowCount,
|
|
969
1008
|
checksum: t.checksum
|
|
970
1009
|
};
|
|
971
1010
|
}
|
|
972
|
-
yield*
|
|
1011
|
+
yield* K(u, f);
|
|
973
1012
|
}
|
|
974
1013
|
});
|
|
975
1014
|
if (t.atomic ? yield* n.tryPromise({
|
|
@@ -977,26 +1016,26 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
977
1016
|
let i = await n.runPromiseExit(_(t, !1));
|
|
978
1017
|
if (r.isFailure(i)) throw Error(e.pretty(i.cause));
|
|
979
1018
|
}),
|
|
980
|
-
catch: (e) => new
|
|
1019
|
+
catch: (e) => new F({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
981
1020
|
}) : yield* _(t.store, !0), t.assets && a.assets.index && !t.assetsAlreadyRestored) {
|
|
982
|
-
let e = t.assets, r = i.join(t.bundleDir,
|
|
983
|
-
yield* c.runForEach(s, (t) => t.key in f.assets ? n.void :
|
|
1021
|
+
let e = t.assets, r = i.join(t.bundleDir, H), o = i.join(t.bundleDir, a.assets.index ?? "assets/index.ndjson"), s = S("bundle", () => E(o)).pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.map((e) => JSON.parse(e)));
|
|
1022
|
+
yield* c.runForEach(s, (t) => t.key in f.assets ? n.void : We(e, r, t).pipe(n.andThen(n.sync(() => {
|
|
984
1023
|
f.assets[t.key] = !0;
|
|
985
|
-
})), n.andThen(
|
|
1024
|
+
})), n.andThen(K(u, f))));
|
|
986
1025
|
}
|
|
987
1026
|
return a;
|
|
988
|
-
}),
|
|
1027
|
+
}), Kt = (e) => e, qt = (e) => {
|
|
989
1028
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
990
1029
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
991
1030
|
return {
|
|
992
1031
|
refuse: !0,
|
|
993
1032
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
994
1033
|
};
|
|
995
|
-
},
|
|
1034
|
+
}, J = Buffer.from("VBUNDLE2"), Y = 0, Jt = 1, Yt = 2, Xt = 32, Zt = async (e) => {
|
|
996
1035
|
let t = [], n = async (r) => {
|
|
997
1036
|
let i;
|
|
998
1037
|
try {
|
|
999
|
-
i = await
|
|
1038
|
+
i = await se(O(e, r), { withFileTypes: !0 });
|
|
1000
1039
|
} catch {
|
|
1001
1040
|
return;
|
|
1002
1041
|
}
|
|
@@ -1008,16 +1047,16 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1008
1047
|
return await n(""), t.sort();
|
|
1009
1048
|
}, Qt = (e, t) => {
|
|
1010
1049
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1011
|
-
return r.writeUInt8(
|
|
1050
|
+
return r.writeUInt8(Jt, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1012
1051
|
}, $t = (e, t, n) => {
|
|
1013
1052
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1014
1053
|
return a.writeUInt8(Yt, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1015
1054
|
};
|
|
1016
1055
|
async function* en(e) {
|
|
1017
|
-
yield new Uint8Array(
|
|
1056
|
+
yield new Uint8Array(J);
|
|
1018
1057
|
for (let t of await Zt(e.dir)) {
|
|
1019
|
-
let n =
|
|
1020
|
-
if (yield new Uint8Array(Qt(t, r.size)), r.size > 0) for await (let e of
|
|
1058
|
+
let n = O(e.dir, t), r = await ce(n);
|
|
1059
|
+
if (yield new Uint8Array(Qt(t, r.size)), r.size > 0) for await (let e of E(n)) yield new Uint8Array(e);
|
|
1021
1060
|
}
|
|
1022
1061
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1023
1062
|
yield new Uint8Array($t(t.key, t.contentType, t.size));
|
|
@@ -1029,11 +1068,11 @@ async function* en(e) {
|
|
|
1029
1068
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1030
1069
|
yield new Uint8Array(n.digest());
|
|
1031
1070
|
}
|
|
1032
|
-
yield new Uint8Array(Buffer.from([
|
|
1071
|
+
yield new Uint8Array(Buffer.from([Y]));
|
|
1033
1072
|
}
|
|
1034
1073
|
var tn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), nn = (e) => new Promise((t) => e.end(t)), rn = async (e, t) => {
|
|
1035
|
-
let n =
|
|
1036
|
-
await
|
|
1074
|
+
let n = fe(t.dir);
|
|
1075
|
+
await oe(n, { recursive: !0 });
|
|
1037
1076
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1038
1077
|
let { value: e, done: t } = await i.next();
|
|
1039
1078
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1044,21 +1083,21 @@ var tn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), nn
|
|
|
1044
1083
|
return r = r.subarray(e), t;
|
|
1045
1084
|
};
|
|
1046
1085
|
try {
|
|
1047
|
-
if (await s(
|
|
1086
|
+
if (await s(J.length), !c(J.length).equals(J)) throw Error("not a bundle archive (bad magic)");
|
|
1048
1087
|
for (;;) {
|
|
1049
1088
|
await s(1);
|
|
1050
1089
|
let e = c(1).readUInt8(0);
|
|
1051
|
-
if (e ===
|
|
1052
|
-
if (e ===
|
|
1090
|
+
if (e === Y) return;
|
|
1091
|
+
if (e === Jt) {
|
|
1053
1092
|
await s(4);
|
|
1054
1093
|
let e = c(4).readUInt32BE(0);
|
|
1055
1094
|
await s(e);
|
|
1056
1095
|
let t = c(e).toString("utf8");
|
|
1057
1096
|
await s(8);
|
|
1058
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1059
|
-
if (a !== n && !a.startsWith(n +
|
|
1060
|
-
await
|
|
1061
|
-
let o =
|
|
1097
|
+
let i = c(8).readBigUInt64BE(0), a = fe(n, t);
|
|
1098
|
+
if (a !== n && !a.startsWith(n + pe)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1099
|
+
await oe(de(a), { recursive: !0 });
|
|
1100
|
+
let o = D(a);
|
|
1062
1101
|
for (; i > 0n;) {
|
|
1063
1102
|
r.length === 0 && await s(1);
|
|
1064
1103
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
@@ -1075,7 +1114,7 @@ var tn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), nn
|
|
|
1075
1114
|
await s(i);
|
|
1076
1115
|
let a = c(i).toString("utf8");
|
|
1077
1116
|
await s(8);
|
|
1078
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = _("sha256"), d = new
|
|
1117
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = _("sha256"), d = new T(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1079
1118
|
contentType: a,
|
|
1080
1119
|
size: Number(o)
|
|
1081
1120
|
}), p = o;
|
|
@@ -1111,11 +1150,11 @@ var tn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), nn
|
|
|
1111
1150
|
}
|
|
1112
1151
|
return Buffer.concat(t);
|
|
1113
1152
|
};
|
|
1114
|
-
if (!await s(
|
|
1153
|
+
if (!await s(J.length) || !c(J.length).equals(J)) throw Error("not a bundle archive (bad magic)");
|
|
1115
1154
|
for (; await s(1);) {
|
|
1116
1155
|
let e = c(1).readUInt8(0);
|
|
1117
|
-
if (e ===
|
|
1118
|
-
if (e !==
|
|
1156
|
+
if (e === Y || e === Yt) break;
|
|
1157
|
+
if (e !== Jt) throw Error(`unknown bundle entry kind ${e}`);
|
|
1119
1158
|
if (!await s(4)) break;
|
|
1120
1159
|
let t = c(4).readUInt32BE(0);
|
|
1121
1160
|
if (!await s(t)) break;
|
|
@@ -1133,14 +1172,14 @@ var tn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), nn
|
|
|
1133
1172
|
return {
|
|
1134
1173
|
has: async (e) => t.has(e),
|
|
1135
1174
|
write: async (n, r, i) => {
|
|
1136
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1137
|
-
await
|
|
1175
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = O(e, "assets");
|
|
1176
|
+
await oe(c, { recursive: !0 });
|
|
1138
1177
|
let l = _("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1139
1178
|
l.update(e), u += e.length, n(null, e);
|
|
1140
|
-
} }), f =
|
|
1141
|
-
await o(r, d,
|
|
1179
|
+
} }), f = O(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1180
|
+
await o(r, d, D(f));
|
|
1142
1181
|
let p = l.digest("hex");
|
|
1143
|
-
await a.rename(f,
|
|
1182
|
+
await a.rename(f, O(c, p)), await a.appendFile(O(c, "index.ndjson"), `${JSON.stringify({
|
|
1144
1183
|
key: n,
|
|
1145
1184
|
sha256: p,
|
|
1146
1185
|
size: u,
|
|
@@ -1148,7 +1187,7 @@ var tn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), nn
|
|
|
1148
1187
|
})}\n`), t.add(n);
|
|
1149
1188
|
}
|
|
1150
1189
|
};
|
|
1151
|
-
}, cn =
|
|
1190
|
+
}, cn = me(b), ln = Buffer.from("VENC1\0\0\0"), un = 32, dn = 16, fn = 7, X = 16, pn = 256 * 1024, Z = {
|
|
1152
1191
|
N: 32768,
|
|
1153
1192
|
r: 8,
|
|
1154
1193
|
p: 1
|
|
@@ -1239,7 +1278,7 @@ var vn = (e, t) => ({
|
|
|
1239
1278
|
})))), bn = (e) => ({
|
|
1240
1279
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1241
1280
|
write: async (t, r, i) => {
|
|
1242
|
-
let a = S("storage", () =>
|
|
1281
|
+
let a = S("storage", () => ne.from(r));
|
|
1243
1282
|
await n.runPromise(ee(e, t, a, {
|
|
1244
1283
|
contentType: i.contentType,
|
|
1245
1284
|
totalSize: i.size
|
|
@@ -1248,7 +1287,7 @@ var vn = (e, t) => ({
|
|
|
1248
1287
|
}), xn = (e, t) => {
|
|
1249
1288
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1250
1289
|
return e === "mariadb" && Sn(n) ? n : r;
|
|
1251
|
-
}, Sn = (e) => (process.env.PATH ?? "").split(
|
|
1290
|
+
}, Sn = (e) => (process.env.PATH ?? "").split(ue).filter((e) => e !== "").some((t) => ie(O(t, e))), Cn = (e) => {
|
|
1252
1291
|
switch (e) {
|
|
1253
1292
|
case "postgres": return "db.dump";
|
|
1254
1293
|
case "mysql":
|
|
@@ -1359,7 +1398,7 @@ var vn = (e, t) => ({
|
|
|
1359
1398
|
],
|
|
1360
1399
|
env: {}
|
|
1361
1400
|
};
|
|
1362
|
-
default: throw new
|
|
1401
|
+
default: throw new L({
|
|
1363
1402
|
tool: "backup",
|
|
1364
1403
|
reason: `no native backup for dialect '${e}'`
|
|
1365
1404
|
});
|
|
@@ -1443,18 +1482,18 @@ var vn = (e, t) => ({
|
|
|
1443
1482
|
],
|
|
1444
1483
|
env: {}
|
|
1445
1484
|
};
|
|
1446
|
-
default: throw new
|
|
1485
|
+
default: throw new L({
|
|
1447
1486
|
tool: "restore",
|
|
1448
1487
|
reason: `no native restore for dialect '${e}'`
|
|
1449
1488
|
});
|
|
1450
1489
|
}
|
|
1451
1490
|
}, On = (e) => e.kind === "copy" ? n.async((t) => {
|
|
1452
|
-
|
|
1491
|
+
re(e.from, e.to, (r) => t(r ? n.fail(new L({
|
|
1453
1492
|
tool: "copy",
|
|
1454
1493
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1455
1494
|
})) : n.void));
|
|
1456
1495
|
}) : n.async((t) => {
|
|
1457
|
-
let r =
|
|
1496
|
+
let r = he(e.tool, [...e.args], {
|
|
1458
1497
|
env: {
|
|
1459
1498
|
...process.env,
|
|
1460
1499
|
...e.env
|
|
@@ -1467,14 +1506,14 @@ var vn = (e, t) => ({
|
|
|
1467
1506
|
}), i = "";
|
|
1468
1507
|
r.stderr?.on("data", (e) => {
|
|
1469
1508
|
i += String(e);
|
|
1470
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(
|
|
1509
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(D(e.stdoutFile)), e.stdinFile && r.stdin && E(e.stdinFile).pipe(r.stdin);
|
|
1471
1510
|
let a = () => {
|
|
1472
1511
|
if (e.stdoutFile !== void 0) try {
|
|
1473
|
-
|
|
1512
|
+
ae(e.stdoutFile);
|
|
1474
1513
|
} catch {}
|
|
1475
1514
|
};
|
|
1476
1515
|
r.on("error", (r) => {
|
|
1477
|
-
a(), t(n.fail(new
|
|
1516
|
+
a(), t(n.fail(new L({
|
|
1478
1517
|
tool: e.tool,
|
|
1479
1518
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`
|
|
1480
1519
|
})));
|
|
@@ -1483,17 +1522,17 @@ var vn = (e, t) => ({
|
|
|
1483
1522
|
t(n.void);
|
|
1484
1523
|
return;
|
|
1485
1524
|
}
|
|
1486
|
-
a(), t(n.fail(new
|
|
1525
|
+
a(), t(n.fail(new L({
|
|
1487
1526
|
tool: e.tool,
|
|
1488
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
1527
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${le(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
1489
1528
|
stderr: i.slice(-2e3)
|
|
1490
1529
|
})));
|
|
1491
1530
|
});
|
|
1492
1531
|
}), kn = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), An = (e) => n.gen(function* () {
|
|
1493
|
-
let t = yield*
|
|
1532
|
+
let t = yield* Be(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
1494
1533
|
store: e.store,
|
|
1495
1534
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
1496
|
-
}), r =
|
|
1535
|
+
}), r = q(t.tables, e.snapshot), i = e.classification ?? ve(e.snapshot), a = e.sampleSize ?? 20, o = De(r.map((t) => ({
|
|
1497
1536
|
name: t,
|
|
1498
1537
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
1499
1538
|
name: e.name,
|
|
@@ -1505,8 +1544,8 @@ var vn = (e, t) => ({
|
|
|
1505
1544
|
table: i,
|
|
1506
1545
|
chunkSize: a,
|
|
1507
1546
|
...r === void 0 ? {} : { where: r }
|
|
1508
|
-
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
1509
|
-
l.push(...
|
|
1547
|
+
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) => P(t, i, o.actions, e.masking.seed));
|
|
1548
|
+
l.push(...ke(d, i, o.actions));
|
|
1510
1549
|
let f = u[0], m = d[0], h = [];
|
|
1511
1550
|
if (f && m) for (let [e, t] of Object.entries(o.actions)) {
|
|
1512
1551
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
@@ -1594,4 +1633,4 @@ var vn = (e, t) => ({
|
|
|
1594
1633
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
1595
1634
|
};
|
|
1596
1635
|
//#endregion
|
|
1597
|
-
export {
|
|
1636
|
+
export { H as ASSETS_DIR, U as ASSET_INDEX_FILE, F as BundleError, Ae as CodecError, je as CompressionError, Le as CrossDialectError, it as DATA_DIR, _e as DEFAULT_CLASS_ACTIONS, Ne as DanglingReferenceError, Xe as FORMAT_VERSION, Ie as ImportModeError, I as IntegrityError, at as LEDGER_FILE, V as MANIFEST_FILE, Fe as MaskingError, L as NativeToolError, Pe as SchemaDriftError, Me as ScopeError, Ee as applyAction, qt as assessLiveImport, mt as assetDone, Cn as backupArtifactName, En as backupCommand, It as canonicalType, zt as checkPortability, Ut as checkSchemaDrift, ve as classificationFromSnapshot, jn as classifyImportDrift, qe as compressionExt, z as computeSubsetIds, nt as decodeManifest, wt as decodeRow, St as decodeValue, _n as decryptStream, Kt as defineDataProfile, Ht as diffSnapshots, sn as dirBlobSink, ft as emptyLedger, tt as encodeManifest, Ct as encodeRow, bt as encodeValue, gn as encryptStream, Mn as formatImportDrift, Bt as formatIssue, Ge as hasZstd, Je as makeCompressor, Ye as makeDecompressor, P as maskRow, Dt as ndjsonToRows, en as packBundle, an as peekBundle, Ke as pickCompression, De as planMasking, An as previewMasking, G as readLedger, ye as resolveAction, Be as resolveScope, We as restoreAssetFromCas, Dn as restoreCommand, Et as rowsToNdjson, Ft as runExport, Gt as runImport, On as runNativeStep, ke as scanForLeaks, He as storageAssetSink, Ve as storageAssetSource, bn as storageBlobSink, vn as storageBlobSource, yn as storageRefsBlobSource, ze as subsetToScope, pt as tableDone, q as topoSortTables, rn as unpackBundle, on as unpackBundleToDir, Ue as writeAssetToCas, K as writeLedger };
|