@voltro/data-transfer 0.44.0 → 0.44.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +364 -336
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -4,14 +4,14 @@ import { fingerprintSchema as g } from "@voltro/database/sql";
|
|
|
4
4
|
import { createCipheriv as _, createDecipheriv as v, createHash as y, createHmac as b, randomBytes as x, scrypt as S } from "node:crypto";
|
|
5
5
|
import { STORAGE_REFS_TABLE as C, fromNodeReadable as w, getObjectStream as T, putObjectStream as E, toNodeReadable as D } from "@voltro/plugin-storage";
|
|
6
6
|
import * as O from "node:zlib";
|
|
7
|
-
import { PassThrough as k, Readable as
|
|
8
|
-
import { copyFile as
|
|
9
|
-
import { mkdir as
|
|
10
|
-
import { basename as
|
|
11
|
-
import { promisify as
|
|
12
|
-
import { spawn as
|
|
7
|
+
import { PassThrough as k, Readable as ee } from "node:stream";
|
|
8
|
+
import { copyFile as A, createReadStream as j, createWriteStream as M, existsSync as te, unlinkSync as ne } from "node:fs";
|
|
9
|
+
import { mkdir as re, readdir as ie, stat as ae } from "node:fs/promises";
|
|
10
|
+
import { basename as oe, delimiter as se, dirname as ce, join as N, resolve as le, sep as ue } from "node:path";
|
|
11
|
+
import { promisify as de } from "node:util";
|
|
12
|
+
import { spawn as fe } from "node:child_process";
|
|
13
13
|
//#region src/consistency.ts
|
|
14
|
-
var
|
|
14
|
+
var pe = (e) => {
|
|
15
15
|
switch (e) {
|
|
16
16
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
17
17
|
case "mysql":
|
|
@@ -21,7 +21,7 @@ var me = (e) => {
|
|
|
21
21
|
case "turso": return [];
|
|
22
22
|
default: return [];
|
|
23
23
|
}
|
|
24
|
-
},
|
|
24
|
+
}, me = {
|
|
25
25
|
email: { fake: "email" },
|
|
26
26
|
fullName: { fake: "fullName" },
|
|
27
27
|
firstName: { fake: "firstName" },
|
|
@@ -36,7 +36,7 @@ var me = (e) => {
|
|
|
36
36
|
date: "dateShift",
|
|
37
37
|
secret: "null",
|
|
38
38
|
freeText: "redact"
|
|
39
|
-
},
|
|
39
|
+
}, he = (e) => {
|
|
40
40
|
let t = {};
|
|
41
41
|
for (let n of e.tables) {
|
|
42
42
|
let e = {};
|
|
@@ -53,10 +53,10 @@ var me = (e) => {
|
|
|
53
53
|
if (i) return i;
|
|
54
54
|
if (n?.sensitive) {
|
|
55
55
|
let e = n.sensitive.class;
|
|
56
|
-
return r.classes?.[e] ??
|
|
56
|
+
return r.classes?.[e] ?? me[e] ?? "redact";
|
|
57
57
|
}
|
|
58
58
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
59
|
-
},
|
|
59
|
+
}, _e = (e, t) => b("sha256", e).update(t == null ? "∅" : String(t)).digest(), P = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), F = [
|
|
60
60
|
"Ada",
|
|
61
61
|
"Alan",
|
|
62
62
|
"Grace",
|
|
@@ -69,7 +69,7 @@ var me = (e) => {
|
|
|
69
69
|
"Guido",
|
|
70
70
|
"Margaret",
|
|
71
71
|
"Bjarne"
|
|
72
|
-
],
|
|
72
|
+
], ve = [
|
|
73
73
|
"Lovelace",
|
|
74
74
|
"Turing",
|
|
75
75
|
"Hopper",
|
|
@@ -82,7 +82,7 @@ var me = (e) => {
|
|
|
82
82
|
"Rossum",
|
|
83
83
|
"Hamilton",
|
|
84
84
|
"Stroustrup"
|
|
85
|
-
],
|
|
85
|
+
], ye = [
|
|
86
86
|
"Maple",
|
|
87
87
|
"Oak",
|
|
88
88
|
"Cedar",
|
|
@@ -91,7 +91,7 @@ var me = (e) => {
|
|
|
91
91
|
"Birch",
|
|
92
92
|
"Willow",
|
|
93
93
|
"Ash"
|
|
94
|
-
],
|
|
94
|
+
], be = [
|
|
95
95
|
"Initech",
|
|
96
96
|
"Hooli",
|
|
97
97
|
"Umbrella",
|
|
@@ -100,23 +100,23 @@ var me = (e) => {
|
|
|
100
100
|
"Soylent",
|
|
101
101
|
"Stark",
|
|
102
102
|
"Wayne"
|
|
103
|
-
],
|
|
104
|
-
let r =
|
|
103
|
+
], I = (e, t, n) => e[P(t, n) % e.length], xe = (e, t, n) => {
|
|
104
|
+
let r = _e(n, `${e}:${String(t)}`), i = P(r, 1) % 1e4;
|
|
105
105
|
switch (e) {
|
|
106
|
-
case "email": return `${
|
|
107
|
-
case "fullName": return `${
|
|
108
|
-
case "firstName": return
|
|
109
|
-
case "lastName": return
|
|
110
|
-
case "username": return `${
|
|
111
|
-
case "phone": return `+1${String(
|
|
112
|
-
case "address": return `${
|
|
113
|
-
case "company": return
|
|
106
|
+
case "email": return `${I(F, r, 0).toLowerCase()}.${I(ve, r, 2).toLowerCase()}${i}@example.com`;
|
|
107
|
+
case "fullName": return `${I(F, r, 0)} ${I(ve, r, 1)}`;
|
|
108
|
+
case "firstName": return I(F, r, 0);
|
|
109
|
+
case "lastName": return I(ve, r, 1);
|
|
110
|
+
case "username": return `${I(F, r, 0).toLowerCase()}${i}`;
|
|
111
|
+
case "phone": return `+1${String(P(r, 0) % 1e10).padStart(10, "0")}`;
|
|
112
|
+
case "address": return `${P(r, 0) % 9899 + 100} ${I(ye, r, 1)} St`;
|
|
113
|
+
case "company": return I(be, r, 0);
|
|
114
114
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
115
|
-
case "ip": return `${
|
|
116
|
-
case "creditCard": return
|
|
115
|
+
case "ip": return `${P(r, 0) % 256}.${P(r, 1) % 256}.${P(r, 2) % 256}.${P(r, 3) % 254 + 1}`;
|
|
116
|
+
case "creditCard": return Se(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
117
117
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
118
118
|
}
|
|
119
|
-
},
|
|
119
|
+
}, Se = (e) => {
|
|
120
120
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
121
121
|
t.forEach((e, t) => {
|
|
122
122
|
let r = e;
|
|
@@ -124,23 +124,23 @@ var me = (e) => {
|
|
|
124
124
|
});
|
|
125
125
|
let r = (10 - n % 10) % 10;
|
|
126
126
|
return e.slice(0, 15) + String(r);
|
|
127
|
-
},
|
|
127
|
+
}, Ce = 864e5, we = (e, t) => {
|
|
128
128
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
129
129
|
if (Number.isNaN(n)) return e;
|
|
130
|
-
let r =
|
|
130
|
+
let r = P(_e(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Ce);
|
|
131
131
|
return e instanceof Date ? i : i.toISOString();
|
|
132
|
-
},
|
|
132
|
+
}, Te = (e, t) => {
|
|
133
133
|
if (t.value === null || t.value === void 0) return null;
|
|
134
134
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
135
135
|
if (e === "keep") return t.value;
|
|
136
136
|
if (e === "null") return null;
|
|
137
137
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
138
|
-
if (e === "hash") return
|
|
139
|
-
if (e === "dateShift") return
|
|
140
|
-
if (typeof e == "object" && "fake" in e) return
|
|
138
|
+
if (e === "hash") return _e(t.seed, t.value).toString("hex").slice(0, 16);
|
|
139
|
+
if (e === "dateShift") return we(t.value, t.seed);
|
|
140
|
+
if (typeof e == "object" && "fake" in e) return xe(e.fake, t.value, t.seed);
|
|
141
141
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
142
142
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
143
|
-
},
|
|
143
|
+
}, Ee = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", De = (e, t, n) => {
|
|
144
144
|
let r = {}, i = [], a = [];
|
|
145
145
|
for (let o of e) for (let e of o.columns) {
|
|
146
146
|
let s = `${o.name}.${e.name}`, c = ge(o.name, e.name, t[o.name]?.[e.name], n);
|
|
@@ -148,7 +148,7 @@ var me = (e) => {
|
|
|
148
148
|
i.push(s);
|
|
149
149
|
continue;
|
|
150
150
|
}
|
|
151
|
-
if (!
|
|
151
|
+
if (!Ee(c)) {
|
|
152
152
|
a.push({
|
|
153
153
|
column: s,
|
|
154
154
|
action: c
|
|
@@ -162,7 +162,7 @@ var me = (e) => {
|
|
|
162
162
|
unclassified: i,
|
|
163
163
|
invalid: a
|
|
164
164
|
};
|
|
165
|
-
},
|
|
165
|
+
}, Oe = [
|
|
166
166
|
{
|
|
167
167
|
name: "email",
|
|
168
168
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -179,7 +179,7 @@ var me = (e) => {
|
|
|
179
179
|
name: "creditCard",
|
|
180
180
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
181
181
|
}
|
|
182
|
-
],
|
|
182
|
+
], ke = (e, t, n) => {
|
|
183
183
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
184
184
|
for (let [a, o] of Object.entries(n)) {
|
|
185
185
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -187,7 +187,7 @@ var me = (e) => {
|
|
|
187
187
|
for (let a of e) {
|
|
188
188
|
let e = a[n];
|
|
189
189
|
if (typeof e != "string" || e.length === 0) continue;
|
|
190
|
-
let o =
|
|
190
|
+
let o = Oe.find((t) => t.re.test(e));
|
|
191
191
|
if (o && !i.has(n)) {
|
|
192
192
|
i.add(n), r.push({
|
|
193
193
|
table: t,
|
|
@@ -200,9 +200,9 @@ var me = (e) => {
|
|
|
200
200
|
}
|
|
201
201
|
}
|
|
202
202
|
return r;
|
|
203
|
-
},
|
|
203
|
+
}, Ae = (e, t, n, r, i) => {
|
|
204
204
|
let a = {};
|
|
205
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
205
|
+
for (let [o, s] of Object.entries(e)) a[o] = Te(n[`${t}.${o}`] ?? "keep", {
|
|
206
206
|
value: s,
|
|
207
207
|
table: t,
|
|
208
208
|
column: o,
|
|
@@ -210,17 +210,17 @@ var me = (e) => {
|
|
|
210
210
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
211
211
|
});
|
|
212
212
|
return a;
|
|
213
|
-
},
|
|
213
|
+
}, L = class extends s.TaggedError()("BundleError", {
|
|
214
214
|
reason: s.String,
|
|
215
215
|
path: s.optional(s.String)
|
|
216
|
-
}) {},
|
|
216
|
+
}) {}, je = class extends s.TaggedError()("CodecError", {
|
|
217
217
|
table: s.String,
|
|
218
218
|
reason: s.String
|
|
219
|
-
}) {},
|
|
219
|
+
}) {}, R = class extends s.TaggedError()("IntegrityError", {
|
|
220
220
|
what: s.String,
|
|
221
221
|
expected: s.String,
|
|
222
222
|
actual: s.String
|
|
223
|
-
}) {},
|
|
223
|
+
}) {}, Me = class extends t.TaggedError("CompressionError") {}, z = class extends s.TaggedError()("ScopeError", { reason: s.String }) {}, Ne = class extends s.TaggedError()("RowsRefusedError", {
|
|
224
224
|
rows: s.Array(s.Struct({
|
|
225
225
|
table: s.String,
|
|
226
226
|
id: s.String,
|
|
@@ -228,30 +228,35 @@ var me = (e) => {
|
|
|
228
228
|
derived: s.Boolean
|
|
229
229
|
})),
|
|
230
230
|
primaryCount: s.Number,
|
|
231
|
-
totalCount: s.Number
|
|
232
|
-
|
|
231
|
+
totalCount: s.Number,
|
|
232
|
+
byTable: s.Array(s.Struct({
|
|
233
|
+
table: s.String,
|
|
234
|
+
refused: s.Number,
|
|
235
|
+
primary: s.Number
|
|
236
|
+
}))
|
|
237
|
+
}) {}, Pe = class extends s.TaggedError()("SchemaDriftError", {
|
|
233
238
|
bundleFingerprint: s.String,
|
|
234
239
|
targetFingerprint: s.String,
|
|
235
240
|
diff: s.Array(s.String)
|
|
236
|
-
}) {},
|
|
241
|
+
}) {}, B = class extends t.TaggedError("NativeToolError") {
|
|
237
242
|
get message() {
|
|
238
243
|
let e = this.stderr?.trim();
|
|
239
244
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
240
245
|
}
|
|
241
|
-
},
|
|
246
|
+
}, Fe = class extends s.TaggedError()("MaskingError", {
|
|
242
247
|
unclassified: s.Array(s.String),
|
|
243
248
|
invalidActions: s.optional(s.Array(s.String))
|
|
244
|
-
}) {},
|
|
249
|
+
}) {}, Ie = class extends s.TaggedError()("ImportModeError", {
|
|
245
250
|
mode: s.String,
|
|
246
251
|
reason: s.String
|
|
247
|
-
}) {},
|
|
252
|
+
}) {}, Le = class extends s.TaggedError()("CrossDialectError", {
|
|
248
253
|
sourceDialect: s.String,
|
|
249
254
|
targetDialect: s.String,
|
|
250
255
|
blocking: s.Array(s.String)
|
|
251
|
-
}) {},
|
|
256
|
+
}) {}, V = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Re = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
252
257
|
col: e.name,
|
|
253
258
|
refTable: e.references.table
|
|
254
|
-
})),
|
|
259
|
+
})), ze = (e, t, r) => n.gen(function* () {
|
|
255
260
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map(), u = (e) => {
|
|
256
261
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
257
262
|
return s.set(e, t), t;
|
|
@@ -269,10 +274,10 @@ var me = (e) => {
|
|
|
269
274
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
270
275
|
};
|
|
271
276
|
}, g = (r, i) => {
|
|
272
|
-
let a = p(r,
|
|
277
|
+
let a = p(r, V(t, r), Re(t, r));
|
|
273
278
|
return c.runForEach(h(e, {
|
|
274
279
|
table: r,
|
|
275
|
-
pk:
|
|
280
|
+
pk: V(t, r),
|
|
276
281
|
...i === void 0 ? {} : { where: i }
|
|
277
282
|
}), (e) => n.sync(() => a(e)));
|
|
278
283
|
}, _ = n.gen(function* () {
|
|
@@ -281,7 +286,7 @@ var me = (e) => {
|
|
|
281
286
|
l.clear();
|
|
282
287
|
for (let [n, r] of e) {
|
|
283
288
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
284
|
-
e.length !== 0 && (yield* g(n, f(
|
|
289
|
+
e.length !== 0 && (yield* g(n, f(V(t, n), e)));
|
|
285
290
|
}
|
|
286
291
|
}
|
|
287
292
|
});
|
|
@@ -304,13 +309,13 @@ var me = (e) => {
|
|
|
304
309
|
}
|
|
305
310
|
}
|
|
306
311
|
return o;
|
|
307
|
-
}),
|
|
312
|
+
}), Be = (e, t) => ({
|
|
308
313
|
tables: [...e.keys()],
|
|
309
314
|
predicateFor: (n) => {
|
|
310
315
|
let r = e.get(n);
|
|
311
|
-
return r ? f(
|
|
316
|
+
return r ? f(V(t, n), [...r]) : void 0;
|
|
312
317
|
}
|
|
313
|
-
}),
|
|
318
|
+
}), H = "tenantId", Ve = (e, t, r) => n.gen(function* () {
|
|
314
319
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
315
320
|
if (e.kind === "all") return {
|
|
316
321
|
tables: i,
|
|
@@ -318,7 +323,7 @@ var me = (e) => {
|
|
|
318
323
|
};
|
|
319
324
|
if (e.kind === "tables") {
|
|
320
325
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
321
|
-
if (a.length > 0) return yield* n.fail(new
|
|
326
|
+
if (a.length > 0) return yield* n.fail(new z({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
322
327
|
let o = new Set(t);
|
|
323
328
|
return {
|
|
324
329
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -326,25 +331,25 @@ var me = (e) => {
|
|
|
326
331
|
};
|
|
327
332
|
}
|
|
328
333
|
let o = e.tenantId;
|
|
329
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
330
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
331
|
-
if (s.length === 0) return yield* n.fail(new
|
|
332
|
-
let c = yield*
|
|
333
|
-
seeds: Object.fromEntries(s.map((e) => [e, u(
|
|
334
|
+
if (o === void 0 || o === "") return yield* n.fail(new z({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
335
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === H) ?? !1);
|
|
336
|
+
if (s.length === 0) return yield* n.fail(new z({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${H}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
337
|
+
let c = yield* ze(r.store, t, {
|
|
338
|
+
seeds: Object.fromEntries(s.map((e) => [e, u(H, o)])),
|
|
334
339
|
pinned: s,
|
|
335
340
|
children: { roots: s }
|
|
336
341
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), d = new Set(s);
|
|
337
342
|
return {
|
|
338
343
|
tables: [...s, ...l],
|
|
339
|
-
predicateFor: (e) => d.has(e) ? u(
|
|
344
|
+
predicateFor: (e) => d.has(e) ? u(H, o) : c.has(e) ? f(V(t, e), [...c.get(e)]) : void 0
|
|
340
345
|
};
|
|
341
|
-
}),
|
|
346
|
+
}), He = (e, t) => ({
|
|
342
347
|
list: t,
|
|
343
348
|
open: (t) => T(e, t)
|
|
344
|
-
}),
|
|
349
|
+
}), Ue = (e) => ({
|
|
345
350
|
write: (t, n, r) => E(e, t, n, r),
|
|
346
351
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
347
|
-
}),
|
|
352
|
+
}), We = (e, t) => n.tryPromise({
|
|
348
353
|
try: async () => {
|
|
349
354
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
350
355
|
u.update(e), d += e.length, n(null, e);
|
|
@@ -362,23 +367,23 @@ var me = (e) => {
|
|
|
362
367
|
contentType: e.contentType
|
|
363
368
|
};
|
|
364
369
|
},
|
|
365
|
-
catch: (e) => new
|
|
370
|
+
catch: (e) => new L({
|
|
366
371
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
367
372
|
path: t
|
|
368
373
|
})
|
|
369
|
-
}),
|
|
374
|
+
}), Ge = (e, t, r) => n.gen(function* () {
|
|
370
375
|
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), l = o("sha256"), u = w("bundle", () => a(s)).pipe(c.tap((e) => n.sync(() => l.update(e))));
|
|
371
376
|
yield* e.write(r.key, u, {
|
|
372
377
|
contentType: r.contentType,
|
|
373
378
|
totalSize: r.size
|
|
374
379
|
});
|
|
375
380
|
let d = l.digest("hex");
|
|
376
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
381
|
+
if (d !== r.sha256) return yield* n.fail(new R({
|
|
377
382
|
what: `asset ${r.key}`,
|
|
378
383
|
expected: r.sha256,
|
|
379
384
|
actual: d
|
|
380
385
|
}));
|
|
381
|
-
}),
|
|
386
|
+
}), Ke = () => typeof O.createZstdCompress == "function", qe = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : Ke() ? "zstd" : "gzip", Je = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", Ye = (e) => e === "zstd" ? O.createZstdCompress() : e === "gzip" ? O.createGzip() : new k(), Xe = (e) => e === "zstd" ? O.createZstdDecompress() : e === "gzip" ? O.createGunzip() : new k(), Ze = 1, Qe = s.Struct({
|
|
382
387
|
name: s.String,
|
|
383
388
|
file: s.String,
|
|
384
389
|
rowCount: s.Number,
|
|
@@ -388,11 +393,11 @@ var me = (e) => {
|
|
|
388
393
|
value: s.String
|
|
389
394
|
}),
|
|
390
395
|
primaryKey: s.String
|
|
391
|
-
}),
|
|
396
|
+
}), $e = s.Struct({
|
|
392
397
|
kind: s.Literal("all", "tenant", "tables"),
|
|
393
398
|
tenantId: s.optional(s.String),
|
|
394
399
|
tables: s.optional(s.Array(s.String))
|
|
395
|
-
}),
|
|
400
|
+
}), et = s.Struct({
|
|
396
401
|
count: s.Number,
|
|
397
402
|
totalBytes: s.Number,
|
|
398
403
|
index: s.optional(s.String)
|
|
@@ -403,23 +408,23 @@ s.Struct({
|
|
|
403
408
|
size: s.Number,
|
|
404
409
|
contentType: s.String
|
|
405
410
|
});
|
|
406
|
-
var
|
|
411
|
+
var tt = s.Struct({
|
|
407
412
|
formatVersion: s.Number,
|
|
408
413
|
createdAt: s.String,
|
|
409
414
|
source: s.Struct({
|
|
410
415
|
dialect: s.String,
|
|
411
416
|
schemaFingerprint: s.String
|
|
412
417
|
}),
|
|
413
|
-
scope:
|
|
418
|
+
scope: $e,
|
|
414
419
|
consistency: s.Literal("live", "snapshot"),
|
|
415
420
|
compression: s.Literal("zstd", "gzip", "none"),
|
|
416
|
-
tables: s.Array(
|
|
417
|
-
assets:
|
|
421
|
+
tables: s.Array(Qe),
|
|
422
|
+
assets: et,
|
|
418
423
|
masking: s.optional(s.Struct({
|
|
419
424
|
policyId: s.optional(s.String),
|
|
420
425
|
transformed: s.Array(s.String)
|
|
421
426
|
}))
|
|
422
|
-
}),
|
|
427
|
+
}), nt = s.encodeSync(s.parseJson(tt)), rt = s.decodeUnknownSync(s.parseJson(tt)), U = "manifest.json", it = "README.md", at = "data", W = "assets", ot = "assets/index.ndjson", st = ".ledger.json", ct = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", lt = (e, t) => {
|
|
423
428
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
424
429
|
return `# Voltro data export
|
|
425
430
|
|
|
@@ -433,7 +438,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
433
438
|
| **Format version** | ${e.formatVersion} |
|
|
434
439
|
| **Consistency** | ${e.consistency} |
|
|
435
440
|
| **Compression** | ${e.compression} |
|
|
436
|
-
| **Scope** | ${
|
|
441
|
+
| **Scope** | ${ct(e.scope)} |
|
|
437
442
|
| **Tables** | ${e.tables.length} |
|
|
438
443
|
| **Rows (total)** | ${n} |
|
|
439
444
|
| **Assets** | ${a} |
|
|
@@ -458,39 +463,39 @@ voltro data import <this-directory>
|
|
|
458
463
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
459
464
|
\`\`\`
|
|
460
465
|
`;
|
|
461
|
-
},
|
|
466
|
+
}, ut = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), dt = a.exponential({
|
|
462
467
|
start: .01,
|
|
463
468
|
factor: 2,
|
|
464
469
|
count: 16
|
|
465
|
-
}),
|
|
470
|
+
}), ft = i.histogram("voltro_data_transfer_table_seconds", dt, "Per-table export/import duration in seconds, tagged by phase."), G = (e, t) => {
|
|
466
471
|
if (e) try {
|
|
467
472
|
e(t);
|
|
468
473
|
} catch {}
|
|
469
|
-
},
|
|
474
|
+
}, pt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(ft.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), ut.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
470
475
|
table: t,
|
|
471
476
|
index: r,
|
|
472
477
|
tableCount: a,
|
|
473
478
|
phase: e
|
|
474
|
-
} })),
|
|
479
|
+
} })), mt = () => ({
|
|
475
480
|
tables: {},
|
|
476
481
|
assets: {}
|
|
477
|
-
}),
|
|
482
|
+
}), ht = (e) => n.tryPromise(async () => {
|
|
478
483
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
479
484
|
return JSON.parse(t);
|
|
480
|
-
}).pipe(n.orElseSucceed(
|
|
485
|
+
}).pipe(n.orElseSucceed(mt)), K = (e, t) => n.tryPromise({
|
|
481
486
|
try: async () => {
|
|
482
487
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
483
488
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
484
489
|
},
|
|
485
|
-
catch: (t) => new
|
|
490
|
+
catch: (t) => new L({
|
|
486
491
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
487
492
|
path: e
|
|
488
493
|
})
|
|
489
|
-
}),
|
|
494
|
+
}), gt = (e, t) => t in e.tables, _t = (e, t) => t in e.assets, vt = /* @__PURE__ */ new Set([
|
|
490
495
|
"json",
|
|
491
496
|
"array",
|
|
492
497
|
"vector"
|
|
493
|
-
]),
|
|
498
|
+
]), yt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", bt = (e) => Buffer.from(e).toString("base64"), xt = (e) => new Uint8Array(Buffer.from(e, "base64")), St = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), Ct = (e, t) => e == null ? null : t !== void 0 && vt.has(t) ? e : t === "bytes" ? bt(e) : t === "timestamp" || t === "date" ? St(e) : typeof e == "bigint" ? {
|
|
494
499
|
$v: "bigint",
|
|
495
500
|
d: e.toString()
|
|
496
501
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -501,30 +506,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
501
506
|
d: e.toISOString()
|
|
502
507
|
} : e instanceof Uint8Array ? {
|
|
503
508
|
$v: "bytes",
|
|
504
|
-
d:
|
|
505
|
-
} : e,
|
|
509
|
+
d: bt(e)
|
|
510
|
+
} : e, wt = (e) => {
|
|
506
511
|
switch (e.$v) {
|
|
507
512
|
case "bigint": return BigInt(e.d);
|
|
508
513
|
case "number": return Number(e.d);
|
|
509
514
|
case "date": return new Date(e.d);
|
|
510
|
-
case "bytes": return
|
|
515
|
+
case "bytes": return xt(e.d);
|
|
511
516
|
default: return e;
|
|
512
517
|
}
|
|
513
|
-
},
|
|
518
|
+
}, Tt = (e, t) => e == null ? null : t !== void 0 && vt.has(t) ? e : yt(e) ? wt(e) : t === "bytes" ? xt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Et = (e, t) => {
|
|
514
519
|
let n = {};
|
|
515
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
520
|
+
for (let [r, i] of Object.entries(e)) n[r] = Ct(i, t[r]);
|
|
516
521
|
return n;
|
|
517
|
-
},
|
|
522
|
+
}, Dt = (e, t) => {
|
|
518
523
|
let n = {};
|
|
519
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
524
|
+
for (let [r, i] of Object.entries(e)) n[r] = Tt(i, t[r]);
|
|
520
525
|
return n;
|
|
521
|
-
},
|
|
522
|
-
try: () =>
|
|
523
|
-
catch: (e) => new
|
|
526
|
+
}, Ot = new TextEncoder(), kt = (e, t) => e.pipe(c.map((e) => Ot.encode(`${JSON.stringify(Et(e, t))}\n`))), At = (e, t, r) => e.pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.mapEffect((e) => n.try({
|
|
527
|
+
try: () => Dt(JSON.parse(e), r),
|
|
528
|
+
catch: (e) => new je({
|
|
524
529
|
table: t,
|
|
525
530
|
reason: String(e?.message ?? e)
|
|
526
531
|
})
|
|
527
|
-
}))),
|
|
532
|
+
}))), jt = (e, t) => {
|
|
528
533
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
529
534
|
for (let e of t.tables) {
|
|
530
535
|
if (!n.has(e.name)) continue;
|
|
@@ -544,39 +549,39 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
544
549
|
};
|
|
545
550
|
for (let t of e) n.has(t) && s(t);
|
|
546
551
|
return i;
|
|
547
|
-
},
|
|
552
|
+
}, q = (e) => {
|
|
548
553
|
let t = e, n = [];
|
|
549
554
|
typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
|
|
550
555
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
551
|
-
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(
|
|
556
|
+
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(q(t.cause)) : n.push(r), n.length === 0) {
|
|
552
557
|
let t = String(e ?? "").trim();
|
|
553
|
-
return t === "" || t === "[object Object]" ?
|
|
558
|
+
return t === "" || t === "[object Object]" ? Mt(e) : t;
|
|
554
559
|
}
|
|
555
560
|
return n.join(": ");
|
|
556
|
-
},
|
|
561
|
+
}, Mt = (e) => {
|
|
557
562
|
try {
|
|
558
563
|
let t = JSON.stringify(e);
|
|
559
564
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
560
565
|
} catch {
|
|
561
566
|
return "unknown error (not serialisable)";
|
|
562
567
|
}
|
|
563
|
-
},
|
|
568
|
+
}, Nt = (e, t) => {
|
|
564
569
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
565
570
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
566
571
|
return r;
|
|
567
|
-
},
|
|
572
|
+
}, Pt = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), Ft = (e, t) => {
|
|
568
573
|
if (t.length === 0) return e;
|
|
569
574
|
let n = { ...e };
|
|
570
575
|
for (let e of t) delete n[e];
|
|
571
576
|
return n;
|
|
572
|
-
},
|
|
577
|
+
}, It = (e, t) => {
|
|
573
578
|
let n = e.tables.find((e) => e.name === t);
|
|
574
579
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
575
580
|
let r = n.columns.find((e) => e.type === "id");
|
|
576
581
|
if (r !== void 0) return { column: r.name };
|
|
577
582
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
578
583
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
579
|
-
},
|
|
584
|
+
}, Lt = (e, t, r) => n.tryPromise({
|
|
580
585
|
try: async () => {
|
|
581
586
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: l, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
582
587
|
d.update(e);
|
|
@@ -585,7 +590,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
585
590
|
n(null, e);
|
|
586
591
|
} }), m = l.fromWeb(c.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
587
592
|
try {
|
|
588
|
-
await s(m, p,
|
|
593
|
+
await s(m, p, Ye(r), n(h)), await i.rename(h, t);
|
|
589
594
|
} catch (e) {
|
|
590
595
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
591
596
|
}
|
|
@@ -594,27 +599,27 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
594
599
|
checksum: d.digest("hex")
|
|
595
600
|
};
|
|
596
601
|
},
|
|
597
|
-
catch: (e) => new
|
|
598
|
-
reason: `write table failed: ${
|
|
602
|
+
catch: (e) => new L({
|
|
603
|
+
reason: `write table failed: ${q(e)}`,
|
|
599
604
|
path: t
|
|
600
605
|
})
|
|
601
|
-
}),
|
|
606
|
+
}), Rt = (e, t) => n.tryPromise({
|
|
602
607
|
try: async () => {
|
|
603
608
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
604
609
|
},
|
|
605
|
-
catch: (t) => new
|
|
606
|
-
reason: `append failed: ${
|
|
610
|
+
catch: (t) => new L({
|
|
611
|
+
reason: `append failed: ${q(t)}`,
|
|
607
612
|
path: e
|
|
608
613
|
})
|
|
609
|
-
}),
|
|
614
|
+
}), zt = (e, t) => n.gen(function* () {
|
|
610
615
|
let r = [], i = t.ordered.length;
|
|
611
616
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
612
|
-
let o = t.ordered[a], s =
|
|
613
|
-
if ("refusal" in l) return yield* n.fail(new
|
|
614
|
-
let u = l.column, d = `${
|
|
615
|
-
if (
|
|
617
|
+
let o = t.ordered[a], s = Nt(t.snapshot, o), l = It(t.snapshot, o);
|
|
618
|
+
if ("refusal" in l) return yield* n.fail(new z({ reason: `export: ${l.refusal}` }));
|
|
619
|
+
let u = l.column, d = `${at}/${o}.ndjson${Je(t.compression)}`;
|
|
620
|
+
if (gt(t.ledger, o)) {
|
|
616
621
|
let e = t.ledger.tables[o];
|
|
617
|
-
|
|
622
|
+
G(t.onProgress, {
|
|
618
623
|
phase: "export",
|
|
619
624
|
table: o,
|
|
620
625
|
index: a,
|
|
@@ -632,7 +637,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
632
637
|
});
|
|
633
638
|
continue;
|
|
634
639
|
}
|
|
635
|
-
|
|
640
|
+
G(t.onProgress, {
|
|
636
641
|
phase: "export",
|
|
637
642
|
table: o,
|
|
638
643
|
index: a,
|
|
@@ -646,11 +651,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
646
651
|
chunkSize: t.chunkSize,
|
|
647
652
|
retry: t.retry,
|
|
648
653
|
...f === void 0 ? {} : { where: f }
|
|
649
|
-
}), m =
|
|
654
|
+
}), m = Pt(t.snapshot, o), g = m.length === 0 ? p : p.pipe(c.map((e) => Ft(e, m))), _ = t.mask ? g.pipe(c.map((e) => Ae(e, o, t.mask.actions, t.mask.seed, s))) : g, { rowCount: v, checksum: y } = yield* pt("export", o, a, i, (e) => e.rowCount, Lt(kt(_, s), t.bundleFile(d), t.compression));
|
|
650
655
|
t.ledger.tables[o] = {
|
|
651
656
|
rowCount: v,
|
|
652
657
|
checksum: y
|
|
653
|
-
}, yield*
|
|
658
|
+
}, yield* K(t.ledgerPath, t.ledger), G(t.onProgress, {
|
|
654
659
|
phase: "export",
|
|
655
660
|
table: o,
|
|
656
661
|
index: a,
|
|
@@ -668,43 +673,43 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
668
673
|
});
|
|
669
674
|
}
|
|
670
675
|
return r;
|
|
671
|
-
}),
|
|
676
|
+
}), Bt = (e) => ({
|
|
672
677
|
_tag: "RawSqlFragment",
|
|
673
678
|
strings: [e],
|
|
674
679
|
values: []
|
|
675
|
-
}),
|
|
680
|
+
}), Vt = (t, i, a) => n.tryPromise({
|
|
676
681
|
try: () => t.transactional(async (t) => {
|
|
677
|
-
if (t.raw) for (let e of
|
|
682
|
+
if (t.raw) for (let e of pe(i)) await t.raw(Bt(e));
|
|
678
683
|
let o = await n.runPromiseExit(a(t));
|
|
679
684
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
680
685
|
return o.value;
|
|
681
686
|
}),
|
|
682
|
-
catch: (e) => e instanceof
|
|
683
|
-
}),
|
|
684
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
687
|
+
catch: (e) => e instanceof L ? e : new L({ reason: `snapshot export failed: ${q(e)}` })
|
|
688
|
+
}), Ht = (e) => n.gen(function* () {
|
|
689
|
+
let t = e.scope ?? { kind: "all" }, r = qe(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, s = o.exponential("50 millis").pipe(o.jittered, o.intersect(o.recurs(a))), l = yield* n.promise(() => import("node:path")), u = (t) => l.join(e.outDir, t);
|
|
685
690
|
yield* n.tryPromise({
|
|
686
691
|
try: async () => {
|
|
687
692
|
let t = await import("node:fs/promises");
|
|
688
|
-
await t.mkdir(u(
|
|
693
|
+
await t.mkdir(u(at), { recursive: !0 }), e.assets && await t.mkdir(u(W), { recursive: !0 });
|
|
689
694
|
},
|
|
690
|
-
catch: (t) => new
|
|
691
|
-
reason: `mkdir failed: ${
|
|
695
|
+
catch: (t) => new L({
|
|
696
|
+
reason: `mkdir failed: ${q(t)}`,
|
|
692
697
|
path: e.outDir
|
|
693
698
|
})
|
|
694
699
|
});
|
|
695
|
-
let d = u(
|
|
700
|
+
let d = u(st), f = yield* ht(d), p = e.subset ? Be(yield* ze(e.store, e.snapshot, e.subset), e.snapshot) : yield* Ve(t, e.snapshot, {
|
|
696
701
|
store: e.store,
|
|
697
702
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
698
|
-
}), m =
|
|
703
|
+
}), m = jt(p.tables, e.snapshot), h = e.consistency ?? "live", _, v;
|
|
699
704
|
if (e.masking) {
|
|
700
|
-
let t = e.classification ??
|
|
705
|
+
let t = e.classification ?? he(e.snapshot), r = De(m.map((t) => ({
|
|
701
706
|
name: t,
|
|
702
707
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
703
708
|
name: e.name,
|
|
704
709
|
type: e.type
|
|
705
710
|
}))
|
|
706
711
|
})), t, e.masking);
|
|
707
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
712
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new Fe({
|
|
708
713
|
unclassified: r.unclassified,
|
|
709
714
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
710
715
|
}));
|
|
@@ -730,18 +735,18 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
730
735
|
bundleFile: u,
|
|
731
736
|
...v ? { mask: v } : {},
|
|
732
737
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
733
|
-
}, b = h === "snapshot" ? yield*
|
|
738
|
+
}, b = h === "snapshot" ? yield* Vt(e.store, e.dialect, (e) => zt(e, y)) : yield* zt(e.store, y), x = 0, S = 0;
|
|
734
739
|
if (e.assets) {
|
|
735
|
-
let t = u(
|
|
736
|
-
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
740
|
+
let t = u(ot);
|
|
741
|
+
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => We(e, u(W))), n.flatMap((e) => {
|
|
737
742
|
let i = {
|
|
738
743
|
key: r.key,
|
|
739
744
|
sha256: e.sha256,
|
|
740
745
|
size: e.size,
|
|
741
746
|
contentType: e.contentType
|
|
742
747
|
};
|
|
743
|
-
return x++, S += e.size, f.assets[r.key] = !0,
|
|
744
|
-
}))).pipe(n.mapError((e) => e instanceof
|
|
748
|
+
return x++, S += e.size, f.assets[r.key] = !0, Rt(t, JSON.stringify(i)).pipe(n.andThen(K(d, f)));
|
|
749
|
+
}))).pipe(n.mapError((e) => e instanceof L ? e : new L({ reason: `asset export failed: ${String(e)}` })));
|
|
745
750
|
}
|
|
746
751
|
let C = {
|
|
747
752
|
formatVersion: 1,
|
|
@@ -757,7 +762,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
757
762
|
assets: x > 0 ? {
|
|
758
763
|
count: x,
|
|
759
764
|
totalBytes: S,
|
|
760
|
-
index:
|
|
765
|
+
index: ot
|
|
761
766
|
} : {
|
|
762
767
|
count: 0,
|
|
763
768
|
totalBytes: 0
|
|
@@ -766,23 +771,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
766
771
|
};
|
|
767
772
|
return yield* n.tryPromise({
|
|
768
773
|
try: async () => {
|
|
769
|
-
await (await import("node:fs/promises")).writeFile(u(
|
|
774
|
+
await (await import("node:fs/promises")).writeFile(u(U), nt(C));
|
|
770
775
|
},
|
|
771
|
-
catch: (e) => new
|
|
772
|
-
reason: `manifest write failed: ${
|
|
773
|
-
path:
|
|
776
|
+
catch: (e) => new L({
|
|
777
|
+
reason: `manifest write failed: ${q(e)}`,
|
|
778
|
+
path: U
|
|
774
779
|
})
|
|
775
780
|
}), yield* n.tryPromise({
|
|
776
781
|
try: async () => {
|
|
777
782
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
778
|
-
await e.writeFile(u(
|
|
783
|
+
await e.writeFile(u(it), lt(C, { hostname: t.hostname() }));
|
|
779
784
|
},
|
|
780
|
-
catch: (e) => new
|
|
781
|
-
reason: `readme write failed: ${
|
|
782
|
-
path:
|
|
785
|
+
catch: (e) => new L({
|
|
786
|
+
reason: `readme write failed: ${q(e)}`,
|
|
787
|
+
path: it
|
|
783
788
|
})
|
|
784
789
|
}), C;
|
|
785
|
-
}),
|
|
790
|
+
}), Ut = (e, t) => {
|
|
786
791
|
let n = new Set(t), r = [];
|
|
787
792
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
788
793
|
let i = e.references?.table;
|
|
@@ -793,7 +798,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
793
798
|
});
|
|
794
799
|
}
|
|
795
800
|
return r;
|
|
796
|
-
},
|
|
801
|
+
}, Wt = (e) => {
|
|
797
802
|
let t = /* @__PURE__ */ new Map();
|
|
798
803
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
799
804
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -801,7 +806,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
801
806
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
802
807
|
});
|
|
803
808
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
804
|
-
},
|
|
809
|
+
}, Gt = (e) => {
|
|
805
810
|
switch (e) {
|
|
806
811
|
case "id":
|
|
807
812
|
case "text":
|
|
@@ -820,12 +825,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
820
825
|
case "raw": return "raw";
|
|
821
826
|
default: return "string";
|
|
822
827
|
}
|
|
823
|
-
},
|
|
828
|
+
}, Kt = {
|
|
824
829
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
825
830
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
826
831
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
827
|
-
},
|
|
828
|
-
let i =
|
|
832
|
+
}, qt = (e, t, n, r) => {
|
|
833
|
+
let i = Gt(n);
|
|
829
834
|
if (i === "raw") return {
|
|
830
835
|
table: e,
|
|
831
836
|
column: t,
|
|
@@ -833,7 +838,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
833
838
|
severity: "block",
|
|
834
839
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
835
840
|
};
|
|
836
|
-
let a =
|
|
841
|
+
let a = Kt[i];
|
|
837
842
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
838
843
|
table: e,
|
|
839
844
|
column: t,
|
|
@@ -847,15 +852,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
847
852
|
severity: "block",
|
|
848
853
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
849
854
|
};
|
|
850
|
-
},
|
|
855
|
+
}, Jt = (e, t, n) => {
|
|
851
856
|
if (t === n) return [];
|
|
852
857
|
let r = [];
|
|
853
858
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
854
|
-
let a =
|
|
859
|
+
let a = qt(t.name, e, i, n);
|
|
855
860
|
a && r.push(a);
|
|
856
861
|
}
|
|
857
862
|
return r;
|
|
858
|
-
},
|
|
863
|
+
}, Yt = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, Xt = (e) => {
|
|
859
864
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
860
865
|
switch (e.kind) {
|
|
861
866
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -864,29 +869,37 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
864
869
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
865
870
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
866
871
|
}
|
|
867
|
-
},
|
|
872
|
+
}, Zt = (e) => {
|
|
868
873
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
869
874
|
return t.length > 0 ? t.join("/") : void 0;
|
|
870
|
-
},
|
|
871
|
-
let t =
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
875
|
+
}, J = "Failed to execute statement", Qt = (e) => {
|
|
876
|
+
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
877
|
+
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
878
|
+
return t === `SqlError: ${J}` ? J : t;
|
|
879
|
+
}, $t = (e) => {
|
|
880
|
+
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
881
|
+
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
882
|
+
unclassified: t,
|
|
883
|
+
raw: n
|
|
884
|
+
}), i = d(e), a = l(i), o = Zt(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
885
|
+
if (a !== void 0) return r(`${Xt(a)}${s}`, !1);
|
|
886
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, u = e?.message, f = Qt(typeof u == "string" && u.length > 0 ? u : String(e)), p = c !== void 0 && c !== f && !c.startsWith("SqlError:") ? c : f, m = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, h = m !== void 0 && /[←‖]/.test(m) ? ` (via ${m})` : "";
|
|
887
|
+
return r(p === J || p.length === 0 ? `the database refused the write and the driver gave no detail${s}${h}` : `${p}${s}`, p === J || p.length === 0);
|
|
888
|
+
}, en = (e) => $t(e).reason, tn = (e) => {
|
|
876
889
|
let t = d(e), n = l(t);
|
|
877
890
|
if (n?.kind === "foreignKeyInUse") {
|
|
878
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
891
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = Zt(t);
|
|
879
892
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
880
893
|
}
|
|
881
|
-
return
|
|
882
|
-
},
|
|
894
|
+
return en(e);
|
|
895
|
+
}, nn = (e, t) => e.tables.find((e) => e.name === t), rn = (e, t) => new Map((nn(e, t)?.columns ?? []).map((e) => [e.name, e])), an = (e, t) => {
|
|
883
896
|
let n = [];
|
|
884
897
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
885
|
-
if (
|
|
898
|
+
if (nn(t, r.name) === void 0) {
|
|
886
899
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
887
900
|
continue;
|
|
888
901
|
}
|
|
889
|
-
let e =
|
|
902
|
+
let e = rn(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
890
903
|
for (let [t, a] of i) {
|
|
891
904
|
let i = e.get(t);
|
|
892
905
|
if (i === void 0) {
|
|
@@ -898,7 +911,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
898
911
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
899
912
|
}
|
|
900
913
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
901
|
-
},
|
|
914
|
+
}, on = (e, t, n) => {
|
|
902
915
|
let r = g(t);
|
|
903
916
|
if (r === e || n === void 0) return {
|
|
904
917
|
drifted: !1,
|
|
@@ -906,28 +919,28 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
906
919
|
targetFingerprint: r,
|
|
907
920
|
diff: []
|
|
908
921
|
};
|
|
909
|
-
let i =
|
|
922
|
+
let i = an(n, t);
|
|
910
923
|
return {
|
|
911
924
|
drifted: i.length > 0,
|
|
912
925
|
bundleFingerprint: e,
|
|
913
926
|
targetFingerprint: r,
|
|
914
927
|
diff: i
|
|
915
928
|
};
|
|
916
|
-
},
|
|
929
|
+
}, sn = 200, cn = (e) => n.tryPromise({
|
|
917
930
|
try: async () => {
|
|
918
931
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
919
|
-
return
|
|
932
|
+
return rt(await t.readFile(n.join(e, U), "utf8"));
|
|
920
933
|
},
|
|
921
|
-
catch: (e) => new
|
|
934
|
+
catch: (e) => new L({
|
|
922
935
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
923
|
-
path:
|
|
936
|
+
path: U
|
|
924
937
|
})
|
|
925
|
-
}),
|
|
926
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
927
|
-
if (a.formatVersion !== 1) return yield* n.fail(new
|
|
938
|
+
}), ln = (t) => n.gen(function* () {
|
|
939
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* cn(t.bundleDir);
|
|
940
|
+
if (a.formatVersion !== 1) return yield* n.fail(new L({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
928
941
|
if (t.tables !== void 0) {
|
|
929
942
|
let e = new Set(a.tables.map((e) => e.name)), r = t.tables.filter((t) => !e.has(t));
|
|
930
|
-
if (r.length > 0) return yield* n.fail(new
|
|
943
|
+
if (r.length > 0) return yield* n.fail(new L({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...e].sort().join(", ")}.` }));
|
|
931
944
|
}
|
|
932
945
|
let o = t.tables === void 0 ? void 0 : new Set(t.tables), s = o === void 0 ? a : {
|
|
933
946
|
...a,
|
|
@@ -944,9 +957,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
944
957
|
unique: !1,
|
|
945
958
|
hasDefault: !1
|
|
946
959
|
}))
|
|
947
|
-
})) }, r =
|
|
960
|
+
})) }, r = on(s.source.schemaFingerprint, t.targetSnapshot, e);
|
|
948
961
|
if (r.drifted) {
|
|
949
|
-
if (!t.force) return yield* n.fail(new
|
|
962
|
+
if (!t.force) return yield* n.fail(new Pe({
|
|
950
963
|
bundleFingerprint: r.bundleFingerprint,
|
|
951
964
|
targetFingerprint: r.targetFingerprint,
|
|
952
965
|
diff: r.diff
|
|
@@ -955,23 +968,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
955
968
|
}
|
|
956
969
|
}
|
|
957
970
|
if (t.targetDialect && t.targetDialect !== s.source.dialect) {
|
|
958
|
-
let e =
|
|
959
|
-
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new
|
|
971
|
+
let e = Jt(s.tables, s.source.dialect, t.targetDialect).filter((e) => e.severity === "block");
|
|
972
|
+
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new Le({
|
|
960
973
|
sourceDialect: s.source.dialect,
|
|
961
974
|
targetDialect: t.targetDialect,
|
|
962
|
-
blocking: e.map(
|
|
975
|
+
blocking: e.map(Yt)
|
|
963
976
|
}));
|
|
964
977
|
}
|
|
965
|
-
let l = t.verify ?? !0, u = t.mode ?? "upsert", d = t.atomic ?? u === "replace", f = t.onConflict ?? "skip", m = t.ledgerPath ?? i.join(t.bundleDir, ".import..ledger.json"), h = yield*
|
|
966
|
-
if (u === "replace" && s.scope.kind !== "all") return yield* n.fail(new
|
|
978
|
+
let l = t.verify ?? !0, u = t.mode ?? "upsert", d = t.atomic ?? u === "replace", f = t.onConflict ?? "skip", m = t.ledgerPath ?? i.join(t.bundleDir, ".import..ledger.json"), h = yield* ht(m);
|
|
979
|
+
if (u === "replace" && s.scope.kind !== "all") return yield* n.fail(new Ie({
|
|
967
980
|
mode: u,
|
|
968
981
|
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
969
982
|
}));
|
|
970
983
|
if (u === "replace") {
|
|
971
984
|
let e = s.tables.map((e) => e.name), r = t.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
972
985
|
try: () => t.store.incomingForeignKeys(e),
|
|
973
|
-
catch: (e) => new
|
|
974
|
-
}) : t.targetSnapshot === void 0 ? [] :
|
|
986
|
+
catch: (e) => new L({ reason: `foreign-key pre-flight failed: ${tn(e)}` })
|
|
987
|
+
}) : t.targetSnapshot === void 0 ? [] : Ut(t.targetSnapshot, e), i = [], a = /* @__PURE__ */ new Map();
|
|
975
988
|
for (let e of r) {
|
|
976
989
|
if (!a.has(e.from)) {
|
|
977
990
|
let r = yield* n.tryPromise({
|
|
@@ -983,7 +996,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
983
996
|
skip: void 0,
|
|
984
997
|
projection: []
|
|
985
998
|
}),
|
|
986
|
-
catch: () => new
|
|
999
|
+
catch: () => new L({ reason: `unreadable: ${e.from}` })
|
|
987
1000
|
}).pipe(n.map((e) => e.length), n.orElseSucceed(() => 0));
|
|
988
1001
|
a.set(e.from, r);
|
|
989
1002
|
}
|
|
@@ -993,9 +1006,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
993
1006
|
rowCount: r
|
|
994
1007
|
});
|
|
995
1008
|
}
|
|
996
|
-
if (i.length > 0) return yield* n.fail(new
|
|
1009
|
+
if (i.length > 0) return yield* n.fail(new Ie({
|
|
997
1010
|
mode: u,
|
|
998
|
-
reason:
|
|
1011
|
+
reason: Wt(i)
|
|
999
1012
|
}));
|
|
1000
1013
|
}
|
|
1001
1014
|
if (t.dryRun === !0) return {
|
|
@@ -1020,16 +1033,18 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1020
1033
|
}, v = (e, t, n, r = !1) => {
|
|
1021
1034
|
let i = t.name, a = t.primaryKey, o = _(i, n);
|
|
1022
1035
|
return u === "append" ? f === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : u === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1023
|
-
}, y = (e, t) => e.savepoint ? e.savepoint(t) : t(), b = (e,
|
|
1036
|
+
}, y = (e, t) => e.savepoint ? e.savepoint(t) : t(), b = 0, x = (e, n, r) => {
|
|
1037
|
+
r.unclassified && (b++, !(b > 3) && t.onWarn?.(`row refused with no driver detail — table '${e.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (b === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1038
|
+
}, S = (e, t, r = !0) => n.tryPromise({
|
|
1024
1039
|
try: e,
|
|
1025
|
-
catch: (e) =>
|
|
1026
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e), !1)))),
|
|
1040
|
+
catch: (e) => $t(e)
|
|
1041
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, x(t.entry, t.row, e)), !1)))), C = (e) => {
|
|
1027
1042
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1028
1043
|
if (t.length !== 0) return {
|
|
1029
1044
|
...e.row,
|
|
1030
1045
|
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1031
1046
|
};
|
|
1032
|
-
},
|
|
1047
|
+
}, T = (e) => {
|
|
1033
1048
|
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
1034
1049
|
if (r !== "reference") return !1;
|
|
1035
1050
|
let i = e.row[n];
|
|
@@ -1039,27 +1054,39 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1039
1054
|
id: String(e.row[e.entry.primaryKey]),
|
|
1040
1055
|
reason: e.reason,
|
|
1041
1056
|
derived: n(e)
|
|
1042
|
-
}));
|
|
1043
|
-
|
|
1044
|
-
|
|
1057
|
+
})), i = [...r.filter((e) => !e.derived), ...r.filter((e) => e.derived)], a = /* @__PURE__ */ new Map();
|
|
1058
|
+
for (let e of r) {
|
|
1059
|
+
let t = a.get(e.table) ?? {
|
|
1060
|
+
refused: 0,
|
|
1061
|
+
primary: 0
|
|
1062
|
+
};
|
|
1063
|
+
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1064
|
+
}
|
|
1065
|
+
return new Ne({
|
|
1066
|
+
rows: i.slice(0, 20),
|
|
1045
1067
|
primaryCount: r.filter((e) => !e.derived).length,
|
|
1046
|
-
totalCount: e.length
|
|
1068
|
+
totalCount: e.length,
|
|
1069
|
+
byTable: [...a.entries()].map(([e, t]) => ({
|
|
1070
|
+
table: e,
|
|
1071
|
+
refused: t.refused,
|
|
1072
|
+
primary: t.primary
|
|
1073
|
+
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1047
1074
|
});
|
|
1048
|
-
},
|
|
1075
|
+
}, E = (e, t) => n.gen(function* () {
|
|
1049
1076
|
let r = t, i = [], a = !0;
|
|
1050
1077
|
for (; r.length > 0 && a;) {
|
|
1051
1078
|
a = !1;
|
|
1052
1079
|
let t = [];
|
|
1053
|
-
for (let n of r) (yield*
|
|
1080
|
+
for (let n of r) (yield* S(() => y(e, () => v(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1054
1081
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1055
|
-
let r = t[n], o =
|
|
1056
|
-
o !== void 0 && (yield*
|
|
1082
|
+
let r = t[n], o = C(r);
|
|
1083
|
+
o !== void 0 && (yield* S(() => y(e, () => v(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1057
1084
|
}
|
|
1058
1085
|
r = t;
|
|
1059
1086
|
}
|
|
1060
|
-
for (let t of i) (yield*
|
|
1061
|
-
if (r.length > 0) return yield* n.fail(
|
|
1062
|
-
}),
|
|
1087
|
+
for (let t of i) (yield* S(() => y(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1088
|
+
if (r.length > 0) return yield* n.fail(T(r));
|
|
1089
|
+
}), D = !1, O = 0, k = 0, ee = 0, A = 0, M = (e, r) => n.gen(function* () {
|
|
1063
1090
|
if (u === "replace" && !(r && h.truncated)) {
|
|
1064
1091
|
let t = [...s.tables].reverse().map((e) => e.name), i = !r, a = e.emptyTables ? () => e.emptyTables(t) : i ? async () => {
|
|
1065
1092
|
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: p(t.primaryKey) });
|
|
@@ -1068,14 +1095,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1068
1095
|
});
|
|
1069
1096
|
yield* n.tryPromise({
|
|
1070
1097
|
try: a,
|
|
1071
|
-
catch: (e) => new
|
|
1072
|
-
}), r && (h.truncated = !0, yield*
|
|
1098
|
+
catch: (e) => new L({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${tn(e)}` })
|
|
1099
|
+
}), r && (h.truncated = !0, yield* K(m, h));
|
|
1073
1100
|
}
|
|
1074
1101
|
let a = [], o = [], f = [], g = s.tables.length;
|
|
1075
1102
|
for (let p = 0; p < s.tables.length; p++) {
|
|
1076
1103
|
let _ = s.tables[p];
|
|
1077
1104
|
if (r && _.name in h.tables) {
|
|
1078
|
-
f.push(_.name),
|
|
1105
|
+
f.push(_.name), A++, ee += h.tables[_.name].rowCount, G(t.onProgress, {
|
|
1079
1106
|
phase: "import",
|
|
1080
1107
|
table: _.name,
|
|
1081
1108
|
index: p,
|
|
@@ -1087,7 +1114,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1087
1114
|
continue;
|
|
1088
1115
|
}
|
|
1089
1116
|
let b = i.join(t.bundleDir, _.file);
|
|
1090
|
-
|
|
1117
|
+
G(t.onProgress, {
|
|
1091
1118
|
phase: "import",
|
|
1092
1119
|
table: _.name,
|
|
1093
1120
|
index: p,
|
|
@@ -1096,8 +1123,8 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1096
1123
|
rowsDone: 0,
|
|
1097
1124
|
total: _.rowCount
|
|
1098
1125
|
});
|
|
1099
|
-
let
|
|
1100
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, l = w("bundle", () => j(b).pipe(
|
|
1126
|
+
let S = a.length, C = n.gen(function* () {
|
|
1127
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, l = w("bundle", () => j(b).pipe(Xe(s.compression))).pipe(c.tap((e) => n.sync(() => i.update(e)))), f = async (t) => {
|
|
1101
1128
|
let n = {
|
|
1102
1129
|
entry: _,
|
|
1103
1130
|
row: t,
|
|
@@ -1106,12 +1133,13 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1106
1133
|
try {
|
|
1107
1134
|
await y(e, () => v(e, _, t));
|
|
1108
1135
|
} catch (e) {
|
|
1109
|
-
|
|
1136
|
+
let r = $t(e);
|
|
1137
|
+
n.reason = r.reason, x(_, t, r), a.push(n);
|
|
1110
1138
|
}
|
|
1111
1139
|
}, p = yield* n.promise(async () => {
|
|
1112
1140
|
let n = t.copyLoader;
|
|
1113
1141
|
if (!n || n.dialect !== "postgres") return !1;
|
|
1114
|
-
if (d) return
|
|
1142
|
+
if (d) return D || (D = !0, t.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1115
1143
|
if (u === "replace") return !0;
|
|
1116
1144
|
if (u !== "upsert") return !1;
|
|
1117
1145
|
try {
|
|
@@ -1126,7 +1154,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1126
1154
|
} catch {
|
|
1127
1155
|
return !1;
|
|
1128
1156
|
}
|
|
1129
|
-
}), m = Object.keys(_.columnTypes), h = Math.max(1, t.copyBatchSize ?? 5e3), g = [],
|
|
1157
|
+
}), m = Object.keys(_.columnTypes), h = Math.max(1, t.copyBatchSize ?? 5e3), g = [], S = async () => {
|
|
1130
1158
|
if (g.length === 0) return;
|
|
1131
1159
|
let e = g;
|
|
1132
1160
|
g = [];
|
|
@@ -1141,7 +1169,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1141
1169
|
p = !1, t.onWarn?.(`bulk COPY into ${_.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1142
1170
|
for (let t of e) await f(t);
|
|
1143
1171
|
}
|
|
1144
|
-
},
|
|
1172
|
+
}, C = Math.max(1, t.savepointBatchSize ?? 200), T = typeof e.savepoint == "function", E = [], O = async () => {
|
|
1145
1173
|
if (E.length === 0) return;
|
|
1146
1174
|
let t = E;
|
|
1147
1175
|
E = [];
|
|
@@ -1153,50 +1181,50 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1153
1181
|
for (let e of t) await f(e);
|
|
1154
1182
|
}
|
|
1155
1183
|
};
|
|
1156
|
-
return yield* c.runForEach(
|
|
1184
|
+
return yield* c.runForEach(At(l, _.name, _.columnTypes), (e) => n.promise(async () => {
|
|
1157
1185
|
if (o++, p) {
|
|
1158
|
-
g.push(e), g.length >= h && await
|
|
1186
|
+
g.push(e), g.length >= h && await S();
|
|
1159
1187
|
return;
|
|
1160
1188
|
}
|
|
1161
|
-
if (
|
|
1162
|
-
E.push(e), E.length >=
|
|
1189
|
+
if (T) {
|
|
1190
|
+
E.push(e), E.length >= C && await O();
|
|
1163
1191
|
return;
|
|
1164
1192
|
}
|
|
1165
1193
|
await f(e);
|
|
1166
|
-
})), yield* n.promise(
|
|
1194
|
+
})), yield* n.promise(O), yield* n.promise(S), {
|
|
1167
1195
|
rowCount: o,
|
|
1168
1196
|
checksum: i.digest("hex")
|
|
1169
1197
|
};
|
|
1170
|
-
}), { rowCount:
|
|
1198
|
+
}), { rowCount: T, checksum: E } = yield* pt("import", _.name, p, g, (e) => e.rowCount, C);
|
|
1171
1199
|
if (l) {
|
|
1172
|
-
if (
|
|
1200
|
+
if (E !== _.checksum) return yield* n.fail(new R({
|
|
1173
1201
|
what: `table ${_.name} checksum`,
|
|
1174
1202
|
expected: _.checksum,
|
|
1175
|
-
actual:
|
|
1203
|
+
actual: E
|
|
1176
1204
|
}));
|
|
1177
|
-
if (
|
|
1205
|
+
if (T !== _.rowCount) return yield* n.fail(new R({
|
|
1178
1206
|
what: `table ${_.name} rowCount`,
|
|
1179
1207
|
expected: String(_.rowCount),
|
|
1180
|
-
actual: String(
|
|
1208
|
+
actual: String(T)
|
|
1181
1209
|
}));
|
|
1182
1210
|
}
|
|
1183
|
-
|
|
1211
|
+
G(t.onProgress, {
|
|
1184
1212
|
phase: "import",
|
|
1185
1213
|
table: _.name,
|
|
1186
1214
|
index: p,
|
|
1187
1215
|
tableCount: g,
|
|
1188
1216
|
event: "done",
|
|
1189
|
-
rowsDone:
|
|
1217
|
+
rowsDone: T,
|
|
1190
1218
|
total: _.rowCount
|
|
1191
|
-
}),
|
|
1219
|
+
}), k++, O += T, r && (a.length > S ? o.push({
|
|
1192
1220
|
name: _.name,
|
|
1193
|
-
rowCount:
|
|
1221
|
+
rowCount: T
|
|
1194
1222
|
}) : (h.tables[_.name] = {
|
|
1195
|
-
rowCount:
|
|
1223
|
+
rowCount: T,
|
|
1196
1224
|
checksum: _.checksum
|
|
1197
|
-
}, yield*
|
|
1225
|
+
}, yield* K(m, h)));
|
|
1198
1226
|
}
|
|
1199
|
-
if (a.length > 0 && (yield*
|
|
1227
|
+
if (a.length > 0 && (yield* E(e, a)), r && o.length > 0) {
|
|
1200
1228
|
for (let e of o) {
|
|
1201
1229
|
let t = s.tables.find((t) => t.name === e.name);
|
|
1202
1230
|
h.tables[e.name] = {
|
|
@@ -1204,7 +1232,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1204
1232
|
checksum: t.checksum
|
|
1205
1233
|
};
|
|
1206
1234
|
}
|
|
1207
|
-
yield*
|
|
1235
|
+
yield* K(m, h);
|
|
1208
1236
|
}
|
|
1209
1237
|
if (f.length > 0) {
|
|
1210
1238
|
let e = f.length === s.tables.length;
|
|
@@ -1213,36 +1241,36 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1213
1241
|
});
|
|
1214
1242
|
if (d ? yield* n.tryPromise({
|
|
1215
1243
|
try: () => t.store.transactional(async (t) => {
|
|
1216
|
-
let i = await n.runPromiseExit(
|
|
1244
|
+
let i = await n.runPromiseExit(M(t, !1));
|
|
1217
1245
|
if (r.isFailure(i)) throw Error(e.pretty(i.cause));
|
|
1218
1246
|
}),
|
|
1219
|
-
catch: (e) => new
|
|
1220
|
-
}) : yield*
|
|
1221
|
-
let e = t.assets, r = i.join(t.bundleDir,
|
|
1222
|
-
yield* c.runForEach(o, (t) => t.key in h.assets ? n.void :
|
|
1247
|
+
catch: (e) => new L({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1248
|
+
}) : yield* M(t.store, !0), t.assets && s.assets.index && !t.assetsAlreadyRestored) {
|
|
1249
|
+
let e = t.assets, r = i.join(t.bundleDir, W), a = i.join(t.bundleDir, s.assets.index ?? "assets/index.ndjson"), o = w("bundle", () => j(a)).pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.map((e) => JSON.parse(e)));
|
|
1250
|
+
yield* c.runForEach(o, (t) => t.key in h.assets ? n.void : Ge(e, r, t).pipe(n.andThen(n.sync(() => {
|
|
1223
1251
|
h.assets[t.key] = !0;
|
|
1224
|
-
})), n.andThen(
|
|
1252
|
+
})), n.andThen(K(m, h))));
|
|
1225
1253
|
}
|
|
1226
1254
|
return {
|
|
1227
1255
|
manifest: s,
|
|
1228
|
-
tablesWritten:
|
|
1229
|
-
rowsWritten:
|
|
1230
|
-
tablesSkipped:
|
|
1231
|
-
rowsSkipped:
|
|
1232
|
-
fullyResumed:
|
|
1256
|
+
tablesWritten: k,
|
|
1257
|
+
rowsWritten: O,
|
|
1258
|
+
tablesSkipped: A,
|
|
1259
|
+
rowsSkipped: ee,
|
|
1260
|
+
fullyResumed: k === 0 && A > 0
|
|
1233
1261
|
};
|
|
1234
|
-
}),
|
|
1262
|
+
}), un = (e) => e, dn = (e) => {
|
|
1235
1263
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1236
1264
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1237
1265
|
return {
|
|
1238
1266
|
refuse: !0,
|
|
1239
1267
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1240
1268
|
};
|
|
1241
|
-
}, Y = Buffer.from("VBUNDLE2"),
|
|
1269
|
+
}, Y = Buffer.from("VBUNDLE2"), fn = 0, pn = 1, mn = 2, hn = 32, gn = async (e) => {
|
|
1242
1270
|
let t = [], n = async (r) => {
|
|
1243
1271
|
let i;
|
|
1244
1272
|
try {
|
|
1245
|
-
i = await
|
|
1273
|
+
i = await ie(N(e, r), { withFileTypes: !0 });
|
|
1246
1274
|
} catch {
|
|
1247
1275
|
return;
|
|
1248
1276
|
}
|
|
@@ -1252,21 +1280,21 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1252
1280
|
}
|
|
1253
1281
|
};
|
|
1254
1282
|
return await n(""), t.sort();
|
|
1255
|
-
},
|
|
1283
|
+
}, _n = (e, t) => {
|
|
1256
1284
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1257
|
-
return r.writeUInt8(
|
|
1258
|
-
},
|
|
1285
|
+
return r.writeUInt8(pn, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1286
|
+
}, vn = (e, t, n) => {
|
|
1259
1287
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1260
|
-
return a.writeUInt8(
|
|
1288
|
+
return a.writeUInt8(mn, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1261
1289
|
};
|
|
1262
|
-
async function*
|
|
1290
|
+
async function* yn(e) {
|
|
1263
1291
|
yield new Uint8Array(Y);
|
|
1264
|
-
for (let t of await
|
|
1265
|
-
let n =
|
|
1266
|
-
if (yield new Uint8Array(
|
|
1292
|
+
for (let t of await gn(e.dir)) {
|
|
1293
|
+
let n = N(e.dir, t), r = await ae(n);
|
|
1294
|
+
if (yield new Uint8Array(_n(t, r.size)), r.size > 0) for await (let e of j(n)) yield new Uint8Array(e);
|
|
1267
1295
|
}
|
|
1268
1296
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1269
|
-
yield new Uint8Array(
|
|
1297
|
+
yield new Uint8Array(vn(t.key, t.contentType, t.size));
|
|
1270
1298
|
let n = y("sha256"), r = 0;
|
|
1271
1299
|
for await (let i of e.blobs.open(t.key)) {
|
|
1272
1300
|
let e = Buffer.from(i);
|
|
@@ -1275,11 +1303,11 @@ async function* gn(e) {
|
|
|
1275
1303
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1276
1304
|
yield new Uint8Array(n.digest());
|
|
1277
1305
|
}
|
|
1278
|
-
yield new Uint8Array(Buffer.from([
|
|
1306
|
+
yield new Uint8Array(Buffer.from([fn]));
|
|
1279
1307
|
}
|
|
1280
|
-
var
|
|
1281
|
-
let n =
|
|
1282
|
-
await
|
|
1308
|
+
var bn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), xn = (e) => new Promise((t) => e.end(t)), Sn = async (e, t) => {
|
|
1309
|
+
let n = le(t.dir);
|
|
1310
|
+
await re(n, { recursive: !0 });
|
|
1283
1311
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1284
1312
|
let { value: e, done: t } = await i.next();
|
|
1285
1313
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1294,24 +1322,24 @@ var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn
|
|
|
1294
1322
|
for (;;) {
|
|
1295
1323
|
await s(1);
|
|
1296
1324
|
let e = c(1).readUInt8(0);
|
|
1297
|
-
if (e ===
|
|
1298
|
-
if (e ===
|
|
1325
|
+
if (e === fn) return;
|
|
1326
|
+
if (e === pn) {
|
|
1299
1327
|
await s(4);
|
|
1300
1328
|
let e = c(4).readUInt32BE(0);
|
|
1301
1329
|
await s(e);
|
|
1302
1330
|
let t = c(e).toString("utf8");
|
|
1303
1331
|
await s(8);
|
|
1304
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1305
|
-
if (a !== n && !a.startsWith(n +
|
|
1306
|
-
await
|
|
1307
|
-
let o =
|
|
1332
|
+
let i = c(8).readBigUInt64BE(0), a = le(n, t);
|
|
1333
|
+
if (a !== n && !a.startsWith(n + ue)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1334
|
+
await re(ce(a), { recursive: !0 });
|
|
1335
|
+
let o = M(a);
|
|
1308
1336
|
for (; i > 0n;) {
|
|
1309
1337
|
r.length === 0 && await s(1);
|
|
1310
1338
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1311
|
-
await
|
|
1339
|
+
await bn(o, c(e)), i -= BigInt(e);
|
|
1312
1340
|
}
|
|
1313
|
-
await
|
|
1314
|
-
} else if (e ===
|
|
1341
|
+
await xn(o);
|
|
1342
|
+
} else if (e === mn) {
|
|
1315
1343
|
await s(4);
|
|
1316
1344
|
let e = c(4).readUInt32BE(0);
|
|
1317
1345
|
await s(e);
|
|
@@ -1328,17 +1356,17 @@ var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn
|
|
|
1328
1356
|
for (; p > 0n;) {
|
|
1329
1357
|
r.length === 0 && await s(1);
|
|
1330
1358
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1331
|
-
u.update(t), l || await
|
|
1359
|
+
u.update(t), l || await bn(d, t), p -= BigInt(e);
|
|
1332
1360
|
}
|
|
1333
|
-
l ? d.destroy() : await
|
|
1334
|
-
let m = c(
|
|
1361
|
+
l ? d.destroy() : await xn(d), await f, await s(hn);
|
|
1362
|
+
let m = c(hn);
|
|
1335
1363
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1336
1364
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1337
1365
|
}
|
|
1338
1366
|
} finally {
|
|
1339
1367
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1340
1368
|
}
|
|
1341
|
-
},
|
|
1369
|
+
}, Cn = async (e, t) => {
|
|
1342
1370
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1343
1371
|
let { value: e, done: t } = await a.next();
|
|
1344
1372
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1360,8 +1388,8 @@ var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn
|
|
|
1360
1388
|
if (!await s(Y.length) || !c(Y.length).equals(Y)) throw Error("not a bundle archive (bad magic)");
|
|
1361
1389
|
for (; await s(1);) {
|
|
1362
1390
|
let e = c(1).readUInt8(0);
|
|
1363
|
-
if (e ===
|
|
1364
|
-
if (e !==
|
|
1391
|
+
if (e === fn || e === mn) break;
|
|
1392
|
+
if (e !== pn) throw Error(`unknown bundle entry kind ${e}`);
|
|
1365
1393
|
if (!await s(4)) break;
|
|
1366
1394
|
let t = c(4).readUInt32BE(0);
|
|
1367
1395
|
if (!await s(t)) break;
|
|
@@ -1371,22 +1399,22 @@ var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn
|
|
|
1371
1399
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1372
1400
|
}
|
|
1373
1401
|
return r;
|
|
1374
|
-
},
|
|
1402
|
+
}, wn = (e, t) => Sn(e, {
|
|
1375
1403
|
dir: t,
|
|
1376
|
-
blobs:
|
|
1377
|
-
}),
|
|
1404
|
+
blobs: Tn(t)
|
|
1405
|
+
}), Tn = (e) => {
|
|
1378
1406
|
let t = /* @__PURE__ */ new Set();
|
|
1379
1407
|
return {
|
|
1380
1408
|
has: async (e) => t.has(e),
|
|
1381
1409
|
write: async (n, r, i) => {
|
|
1382
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1383
|
-
await
|
|
1410
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = N(e, "assets");
|
|
1411
|
+
await re(c, { recursive: !0 });
|
|
1384
1412
|
let l = y("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1385
1413
|
l.update(e), u += e.length, n(null, e);
|
|
1386
|
-
} }), f =
|
|
1387
|
-
await o(r, d,
|
|
1414
|
+
} }), f = N(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1415
|
+
await o(r, d, M(f));
|
|
1388
1416
|
let p = l.digest("hex");
|
|
1389
|
-
await a.rename(f,
|
|
1417
|
+
await a.rename(f, N(c, p)), await a.appendFile(N(c, "index.ndjson"), `${JSON.stringify({
|
|
1390
1418
|
key: n,
|
|
1391
1419
|
sha256: p,
|
|
1392
1420
|
size: u,
|
|
@@ -1394,33 +1422,33 @@ var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn
|
|
|
1394
1422
|
})}\n`), t.add(n);
|
|
1395
1423
|
}
|
|
1396
1424
|
};
|
|
1397
|
-
},
|
|
1425
|
+
}, En = de(S), Dn = Buffer.from("VENC1\0\0\0"), On = 32, kn = 16, An = 7, X = 16, jn = 262144, Z = {
|
|
1398
1426
|
N: 32768,
|
|
1399
1427
|
r: 8,
|
|
1400
1428
|
p: 1
|
|
1401
|
-
},
|
|
1429
|
+
}, Mn = 134217728, Q = 43, Nn = (e, t, n) => {
|
|
1402
1430
|
let r = Buffer.alloc(12);
|
|
1403
|
-
return e.copy(r, 0, 0,
|
|
1431
|
+
return e.copy(r, 0, 0, An), r.writeUInt32BE(t >>> 0, An), r.writeUInt8(+!!n, 11), r;
|
|
1404
1432
|
};
|
|
1405
|
-
async function*
|
|
1433
|
+
async function* Pn(e, t) {
|
|
1406
1434
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1407
|
-
let n = x(
|
|
1435
|
+
let n = x(kn), r = x(An), i = await En(t, n, On, {
|
|
1408
1436
|
...Z,
|
|
1409
|
-
maxmem:
|
|
1437
|
+
maxmem: Mn
|
|
1410
1438
|
}), a = Buffer.alloc(Q);
|
|
1411
|
-
|
|
1439
|
+
Dn.copy(a, 0), a.writeUInt32BE(Z.N, 8), a.writeUInt32BE(Z.r, 12), a.writeUInt32BE(Z.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1412
1440
|
let o = 0, s = (e, t) => {
|
|
1413
|
-
let n = _("aes-256-gcm", i,
|
|
1441
|
+
let n = _("aes-256-gcm", i, Nn(r, o, t));
|
|
1414
1442
|
o === 0 && n.setAAD(a);
|
|
1415
1443
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1416
1444
|
o++;
|
|
1417
1445
|
let l = Buffer.alloc(5 + s.length + X);
|
|
1418
1446
|
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + X, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1419
1447
|
}, c = Buffer.alloc(0);
|
|
1420
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1448
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > jn;) yield s(c.subarray(0, jn), !1), c = c.subarray(jn);
|
|
1421
1449
|
yield s(c, !0);
|
|
1422
1450
|
}
|
|
1423
|
-
async function*
|
|
1451
|
+
async function* Fn(e, t) {
|
|
1424
1452
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1425
1453
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1426
1454
|
let { value: e, done: t } = await n.next();
|
|
@@ -1431,12 +1459,12 @@ async function* Mn(e, t) {
|
|
|
1431
1459
|
};
|
|
1432
1460
|
if (!await o(Q)) throw Error("encrypted bundle truncated (no header)");
|
|
1433
1461
|
let s = r.subarray(0, Q);
|
|
1434
|
-
if (!s.subarray(0, 8).equals(
|
|
1435
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await
|
|
1462
|
+
if (!s.subarray(0, 8).equals(Dn)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1463
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await En(t, d, On, {
|
|
1436
1464
|
N: c,
|
|
1437
1465
|
r: l,
|
|
1438
1466
|
p: u,
|
|
1439
|
-
maxmem:
|
|
1467
|
+
maxmem: Mn
|
|
1440
1468
|
});
|
|
1441
1469
|
r = r.subarray(Q);
|
|
1442
1470
|
let m = 0, h = !1;
|
|
@@ -1451,7 +1479,7 @@ async function* Mn(e, t) {
|
|
|
1451
1479
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1452
1480
|
let n = r.subarray(5, 5 + t - X), i = r.subarray(5 + t - X, 5 + t);
|
|
1453
1481
|
r = r.subarray(5 + t);
|
|
1454
|
-
let a = v("aes-256-gcm", p,
|
|
1482
|
+
let a = v("aes-256-gcm", p, Nn(f, m, e));
|
|
1455
1483
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1456
1484
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1457
1485
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1459,7 +1487,7 @@ async function* Mn(e, t) {
|
|
|
1459
1487
|
}
|
|
1460
1488
|
//#endregion
|
|
1461
1489
|
//#region src/blobStorage.ts
|
|
1462
|
-
var
|
|
1490
|
+
var In = (e, t) => ({
|
|
1463
1491
|
list: () => {
|
|
1464
1492
|
let e = t();
|
|
1465
1493
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1475,26 +1503,26 @@ var Nn = (e, t) => ({
|
|
|
1475
1503
|
let r = await n.runPromise(T(e, t)), i = await D(r.stream);
|
|
1476
1504
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1477
1505
|
} })
|
|
1478
|
-
}),
|
|
1506
|
+
}), Ln = (e, t) => In(e, () => h(t, {
|
|
1479
1507
|
table: C,
|
|
1480
1508
|
chunkSize: 500
|
|
1481
1509
|
}).pipe(c.map((e) => ({
|
|
1482
1510
|
key: String(e.key),
|
|
1483
1511
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1484
1512
|
size: Number(e.size ?? 0)
|
|
1485
|
-
})))),
|
|
1513
|
+
})))), Rn = (e) => ({
|
|
1486
1514
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1487
1515
|
write: async (t, r, i) => {
|
|
1488
|
-
let a = w("storage", () =>
|
|
1516
|
+
let a = w("storage", () => ee.from(r));
|
|
1489
1517
|
await n.runPromise(E(e, t, a, {
|
|
1490
1518
|
contentType: i.contentType,
|
|
1491
1519
|
totalSize: i.size
|
|
1492
1520
|
}));
|
|
1493
1521
|
}
|
|
1494
|
-
}),
|
|
1522
|
+
}), zn = (e, t) => {
|
|
1495
1523
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1496
|
-
return e === "mariadb" &&
|
|
1497
|
-
},
|
|
1524
|
+
return e === "mariadb" && Bn(n) ? n : r;
|
|
1525
|
+
}, Bn = (e) => (process.env.PATH ?? "").split(se).filter((e) => e !== "").some((t) => te(N(t, e))), Vn = (e) => {
|
|
1498
1526
|
switch (e) {
|
|
1499
1527
|
case "postgres": return "db.dump";
|
|
1500
1528
|
case "mysql":
|
|
@@ -1504,7 +1532,7 @@ var Nn = (e, t) => ({
|
|
|
1504
1532
|
case "mssql": return "db.bacpac";
|
|
1505
1533
|
default: return "db.dump";
|
|
1506
1534
|
}
|
|
1507
|
-
},
|
|
1535
|
+
}, Hn = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1508
1536
|
if (e.host || e.database) return {
|
|
1509
1537
|
host: e.host ?? "localhost",
|
|
1510
1538
|
port: e.port ?? 3306,
|
|
@@ -1520,7 +1548,7 @@ var Nn = (e, t) => ({
|
|
|
1520
1548
|
password: decodeURIComponent(t.password) || "",
|
|
1521
1549
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1522
1550
|
};
|
|
1523
|
-
},
|
|
1551
|
+
}, Un = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Wn = (e, t, n) => {
|
|
1524
1552
|
switch (e) {
|
|
1525
1553
|
case "postgres": {
|
|
1526
1554
|
if (t.url) return {
|
|
@@ -1567,7 +1595,7 @@ var Nn = (e, t) => ({
|
|
|
1567
1595
|
let r = $(t);
|
|
1568
1596
|
return {
|
|
1569
1597
|
kind: "spawn",
|
|
1570
|
-
tool:
|
|
1598
|
+
tool: zn(e, "dump"),
|
|
1571
1599
|
args: [
|
|
1572
1600
|
"--single-transaction",
|
|
1573
1601
|
"--routines",
|
|
@@ -1589,7 +1617,7 @@ var Nn = (e, t) => ({
|
|
|
1589
1617
|
case "sqlite":
|
|
1590
1618
|
case "turso": return {
|
|
1591
1619
|
kind: "copy",
|
|
1592
|
-
from:
|
|
1620
|
+
from: Hn(t),
|
|
1593
1621
|
to: n
|
|
1594
1622
|
};
|
|
1595
1623
|
case "mssql": return {
|
|
@@ -1597,7 +1625,7 @@ var Nn = (e, t) => ({
|
|
|
1597
1625
|
tool: "sqlpackage",
|
|
1598
1626
|
args: [
|
|
1599
1627
|
"/Action:Export",
|
|
1600
|
-
`/SourceConnectionString:${
|
|
1628
|
+
`/SourceConnectionString:${Un($({
|
|
1601
1629
|
...t,
|
|
1602
1630
|
port: t.port ?? 1433
|
|
1603
1631
|
}))}`,
|
|
@@ -1605,12 +1633,12 @@ var Nn = (e, t) => ({
|
|
|
1605
1633
|
],
|
|
1606
1634
|
env: {}
|
|
1607
1635
|
};
|
|
1608
|
-
default: throw new
|
|
1636
|
+
default: throw new B({
|
|
1609
1637
|
tool: "backup",
|
|
1610
1638
|
reason: `no native backup for dialect '${e}'`
|
|
1611
1639
|
});
|
|
1612
1640
|
}
|
|
1613
|
-
},
|
|
1641
|
+
}, Gn = (e, t, n) => {
|
|
1614
1642
|
switch (e) {
|
|
1615
1643
|
case "postgres": {
|
|
1616
1644
|
if (t.url) return {
|
|
@@ -1655,7 +1683,7 @@ var Nn = (e, t) => ({
|
|
|
1655
1683
|
let r = $(t);
|
|
1656
1684
|
return {
|
|
1657
1685
|
kind: "spawn",
|
|
1658
|
-
tool:
|
|
1686
|
+
tool: zn(e, "client"),
|
|
1659
1687
|
args: [
|
|
1660
1688
|
"--protocol=TCP",
|
|
1661
1689
|
"-h",
|
|
@@ -1674,14 +1702,14 @@ var Nn = (e, t) => ({
|
|
|
1674
1702
|
case "turso": return {
|
|
1675
1703
|
kind: "copy",
|
|
1676
1704
|
from: n,
|
|
1677
|
-
to:
|
|
1705
|
+
to: Hn(t)
|
|
1678
1706
|
};
|
|
1679
1707
|
case "mssql": return {
|
|
1680
1708
|
kind: "spawn",
|
|
1681
1709
|
tool: "sqlpackage",
|
|
1682
1710
|
args: [
|
|
1683
1711
|
"/Action:Import",
|
|
1684
|
-
`/TargetConnectionString:${
|
|
1712
|
+
`/TargetConnectionString:${Un($({
|
|
1685
1713
|
...t,
|
|
1686
1714
|
port: t.port ?? 1433
|
|
1687
1715
|
}))}`,
|
|
@@ -1689,18 +1717,18 @@ var Nn = (e, t) => ({
|
|
|
1689
1717
|
],
|
|
1690
1718
|
env: {}
|
|
1691
1719
|
};
|
|
1692
|
-
default: throw new
|
|
1720
|
+
default: throw new B({
|
|
1693
1721
|
tool: "restore",
|
|
1694
1722
|
reason: `no native restore for dialect '${e}'`
|
|
1695
1723
|
});
|
|
1696
1724
|
}
|
|
1697
|
-
},
|
|
1698
|
-
|
|
1725
|
+
}, Kn = (e) => e.kind === "copy" ? n.async((t) => {
|
|
1726
|
+
A(e.from, e.to, (r) => t(r ? n.fail(new B({
|
|
1699
1727
|
tool: "copy",
|
|
1700
1728
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1701
1729
|
})) : n.void));
|
|
1702
1730
|
}) : n.async((t) => {
|
|
1703
|
-
let r =
|
|
1731
|
+
let r = fe(e.tool, [...e.args], {
|
|
1704
1732
|
env: {
|
|
1705
1733
|
...process.env,
|
|
1706
1734
|
...e.env
|
|
@@ -1713,14 +1741,14 @@ var Nn = (e, t) => ({
|
|
|
1713
1741
|
}), i = "";
|
|
1714
1742
|
r.stderr?.on("data", (e) => {
|
|
1715
1743
|
i += String(e);
|
|
1716
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(
|
|
1744
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin && j(e.stdinFile).pipe(r.stdin);
|
|
1717
1745
|
let a = () => {
|
|
1718
1746
|
if (e.stdoutFile !== void 0) try {
|
|
1719
|
-
|
|
1747
|
+
ne(e.stdoutFile);
|
|
1720
1748
|
} catch {}
|
|
1721
1749
|
};
|
|
1722
1750
|
r.on("error", (r) => {
|
|
1723
|
-
a(), t(n.fail(new
|
|
1751
|
+
a(), t(n.fail(new B({
|
|
1724
1752
|
tool: e.tool,
|
|
1725
1753
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`
|
|
1726
1754
|
})));
|
|
@@ -1729,17 +1757,17 @@ var Nn = (e, t) => ({
|
|
|
1729
1757
|
t(n.void);
|
|
1730
1758
|
return;
|
|
1731
1759
|
}
|
|
1732
|
-
a(), t(n.fail(new
|
|
1760
|
+
a(), t(n.fail(new B({
|
|
1733
1761
|
tool: e.tool,
|
|
1734
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
1762
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${oe(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
1735
1763
|
stderr: i.slice(-2e3)
|
|
1736
1764
|
})));
|
|
1737
1765
|
});
|
|
1738
|
-
}),
|
|
1739
|
-
let t = yield*
|
|
1766
|
+
}), qn = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Jn = (e) => n.gen(function* () {
|
|
1767
|
+
let t = yield* Ve(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
1740
1768
|
store: e.store,
|
|
1741
1769
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
1742
|
-
}), r =
|
|
1770
|
+
}), r = jt(t.tables, e.snapshot), i = e.classification ?? he(e.snapshot), a = e.sampleSize ?? 20, o = De(r.map((t) => ({
|
|
1743
1771
|
name: t,
|
|
1744
1772
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
1745
1773
|
name: e.name,
|
|
@@ -1751,15 +1779,15 @@ var Nn = (e, t) => ({
|
|
|
1751
1779
|
table: i,
|
|
1752
1780
|
chunkSize: a,
|
|
1753
1781
|
...r === void 0 ? {} : { where: r }
|
|
1754
|
-
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
1755
|
-
l.push(...
|
|
1782
|
+
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) => Ae(t, i, o.actions, e.masking.seed));
|
|
1783
|
+
l.push(...ke(d, i, o.actions));
|
|
1756
1784
|
let f = u[0], p = d[0], m = [];
|
|
1757
1785
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
1758
1786
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
1759
1787
|
let n = e.slice(i.length + 1);
|
|
1760
1788
|
m.push({
|
|
1761
1789
|
column: n,
|
|
1762
|
-
action:
|
|
1790
|
+
action: qn(t),
|
|
1763
1791
|
before: f[n],
|
|
1764
1792
|
after: p[n]
|
|
1765
1793
|
});
|
|
@@ -1774,7 +1802,7 @@ var Nn = (e, t) => ({
|
|
|
1774
1802
|
leaks: l,
|
|
1775
1803
|
unclassified: o.unclassified
|
|
1776
1804
|
};
|
|
1777
|
-
}),
|
|
1805
|
+
}), Yn = (e, t) => {
|
|
1778
1806
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
1779
1807
|
for (let t of e) {
|
|
1780
1808
|
let e = n.get(t.name);
|
|
@@ -1829,7 +1857,7 @@ var Nn = (e, t) => ({
|
|
|
1829
1857
|
skipColumns: i,
|
|
1830
1858
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
1831
1859
|
};
|
|
1832
|
-
},
|
|
1860
|
+
}, Xn = (e) => {
|
|
1833
1861
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
1834
1862
|
let t = {
|
|
1835
1863
|
safe: "✓",
|
|
@@ -1840,4 +1868,4 @@ var Nn = (e, t) => ({
|
|
|
1840
1868
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
1841
1869
|
};
|
|
1842
1870
|
//#endregion
|
|
1843
|
-
export {
|
|
1871
|
+
export { W as ASSETS_DIR, ot as ASSET_INDEX_FILE, L as BundleError, je as CodecError, Me as CompressionError, Le as CrossDialectError, at as DATA_DIR, me as DEFAULT_CLASS_ACTIONS, Ze as FORMAT_VERSION, Ie as ImportModeError, R as IntegrityError, st as LEDGER_FILE, U as MANIFEST_FILE, Fe as MaskingError, B as NativeToolError, Ne as RowsRefusedError, sn as SAVEPOINT_BATCH_SIZE, Pe as SchemaDriftError, z as ScopeError, Te as applyAction, dn as assessLiveImport, _t as assetDone, Vn as backupArtifactName, Wn as backupCommand, Gt as canonicalType, Jt as checkPortability, on as checkSchemaDrift, he as classificationFromSnapshot, Yn as classifyImportDrift, Je as compressionExt, ze as computeSubsetIds, rt as decodeManifest, Dt as decodeRow, Tt as decodeValue, Fn as decryptStream, un as defineDataProfile, an as diffSnapshots, Tn as dirBlobSink, mt as emptyLedger, nt as encodeManifest, Et as encodeRow, Ct as encodeValue, Pn as encryptStream, Xn as formatImportDrift, Yt as formatIssue, Ke as hasZstd, Ye as makeCompressor, Xe as makeDecompressor, Ae as maskRow, At as ndjsonToRows, Wt as outsideReferenceRefusal, Ut as outsideReferencesInto, yn as packBundle, Cn as peekBundle, qe as pickCompression, De as planMasking, Jn as previewMasking, ht as readLedger, ge as resolveAction, Ve as resolveScope, Ge as restoreAssetFromCas, Gn as restoreCommand, kt as rowsToNdjson, Ht as runExport, ln as runImport, Kn as runNativeStep, ke as scanForLeaks, Ue as storageAssetSink, He as storageAssetSource, Rn as storageBlobSink, In as storageBlobSource, Ln as storageRefsBlobSource, Be as subsetToScope, gt as tableDone, jt as topoSortTables, Sn as unpackBundle, wn as unpackBundleToDir, We as writeAssetToCas, K as writeLedger };
|