@voltro/data-transfer 0.43.1 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +65 -0
- package/dist/index.d.ts +79 -1
- package/dist/index.js +419 -328
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -2,16 +2,16 @@ import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBound
|
|
|
2
2
|
import { classifyConstraintViolation as l, eq as u, extractDbCause as d, inSet as f, isNotNull as p, or as m, streamTable as h } from "@voltro/database";
|
|
3
3
|
import { fingerprintSchema as g } from "@voltro/database/sql";
|
|
4
4
|
import { createCipheriv as _, createDecipheriv as v, createHash as y, createHmac as b, randomBytes as x, scrypt as S } from "node:crypto";
|
|
5
|
-
import { STORAGE_REFS_TABLE as C, fromNodeReadable as w, getObjectStream as
|
|
6
|
-
import * as
|
|
7
|
-
import { PassThrough as
|
|
8
|
-
import { copyFile as
|
|
9
|
-
import { mkdir as
|
|
10
|
-
import { basename as
|
|
11
|
-
import { promisify as
|
|
12
|
-
import { spawn as
|
|
5
|
+
import { STORAGE_REFS_TABLE as C, fromNodeReadable as w, getObjectStream as T, putObjectStream as E, toNodeReadable as D } from "@voltro/plugin-storage";
|
|
6
|
+
import * as O from "node:zlib";
|
|
7
|
+
import { PassThrough as k, Readable as A } from "node:stream";
|
|
8
|
+
import { copyFile as ee, createReadStream as j, createWriteStream as te, existsSync as ne, unlinkSync as re } from "node:fs";
|
|
9
|
+
import { mkdir as ie, readdir as ae, stat as oe } from "node:fs/promises";
|
|
10
|
+
import { basename as se, delimiter as ce, dirname as le, join as M, resolve as ue, sep as de } from "node:path";
|
|
11
|
+
import { promisify as fe } from "node:util";
|
|
12
|
+
import { spawn as pe } from "node:child_process";
|
|
13
13
|
//#region src/consistency.ts
|
|
14
|
-
var
|
|
14
|
+
var me = (e) => {
|
|
15
15
|
switch (e) {
|
|
16
16
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
17
17
|
case "mysql":
|
|
@@ -21,7 +21,7 @@ var _e = (e) => {
|
|
|
21
21
|
case "turso": return [];
|
|
22
22
|
default: return [];
|
|
23
23
|
}
|
|
24
|
-
},
|
|
24
|
+
}, he = {
|
|
25
25
|
email: { fake: "email" },
|
|
26
26
|
fullName: { fake: "fullName" },
|
|
27
27
|
firstName: { fake: "firstName" },
|
|
@@ -36,7 +36,7 @@ var _e = (e) => {
|
|
|
36
36
|
date: "dateShift",
|
|
37
37
|
secret: "null",
|
|
38
38
|
freeText: "redact"
|
|
39
|
-
},
|
|
39
|
+
}, N = (e) => {
|
|
40
40
|
let t = {};
|
|
41
41
|
for (let n of e.tables) {
|
|
42
42
|
let e = {};
|
|
@@ -48,15 +48,15 @@ var _e = (e) => {
|
|
|
48
48
|
t[n.name] = e;
|
|
49
49
|
}
|
|
50
50
|
return t;
|
|
51
|
-
},
|
|
51
|
+
}, ge = (e, t, n, r) => {
|
|
52
52
|
let i = r.columns?.[`${e}.${t}`];
|
|
53
53
|
if (i) return i;
|
|
54
54
|
if (n?.sensitive) {
|
|
55
55
|
let e = n.sensitive.class;
|
|
56
|
-
return r.classes?.[e] ??
|
|
56
|
+
return r.classes?.[e] ?? he[e] ?? "redact";
|
|
57
57
|
}
|
|
58
58
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
59
|
-
},
|
|
59
|
+
}, P = (e, t) => b("sha256", e).update(t == null ? "∅" : String(t)).digest(), F = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), I = [
|
|
60
60
|
"Ada",
|
|
61
61
|
"Alan",
|
|
62
62
|
"Grace",
|
|
@@ -69,7 +69,7 @@ var _e = (e) => {
|
|
|
69
69
|
"Guido",
|
|
70
70
|
"Margaret",
|
|
71
71
|
"Bjarne"
|
|
72
|
-
],
|
|
72
|
+
], _e = [
|
|
73
73
|
"Lovelace",
|
|
74
74
|
"Turing",
|
|
75
75
|
"Hopper",
|
|
@@ -82,7 +82,7 @@ var _e = (e) => {
|
|
|
82
82
|
"Rossum",
|
|
83
83
|
"Hamilton",
|
|
84
84
|
"Stroustrup"
|
|
85
|
-
],
|
|
85
|
+
], ve = [
|
|
86
86
|
"Maple",
|
|
87
87
|
"Oak",
|
|
88
88
|
"Cedar",
|
|
@@ -91,7 +91,7 @@ var _e = (e) => {
|
|
|
91
91
|
"Birch",
|
|
92
92
|
"Willow",
|
|
93
93
|
"Ash"
|
|
94
|
-
],
|
|
94
|
+
], ye = [
|
|
95
95
|
"Initech",
|
|
96
96
|
"Hooli",
|
|
97
97
|
"Umbrella",
|
|
@@ -100,23 +100,23 @@ var _e = (e) => {
|
|
|
100
100
|
"Soylent",
|
|
101
101
|
"Stark",
|
|
102
102
|
"Wayne"
|
|
103
|
-
],
|
|
104
|
-
let r =
|
|
103
|
+
], L = (e, t, n) => e[F(t, n) % e.length], be = (e, t, n) => {
|
|
104
|
+
let r = P(n, `${e}:${String(t)}`), i = F(r, 1) % 1e4;
|
|
105
105
|
switch (e) {
|
|
106
|
-
case "email": return `${
|
|
107
|
-
case "fullName": return `${
|
|
108
|
-
case "firstName": return
|
|
109
|
-
case "lastName": return
|
|
110
|
-
case "username": return `${
|
|
111
|
-
case "phone": return `+1${String(
|
|
112
|
-
case "address": return `${
|
|
113
|
-
case "company": return
|
|
106
|
+
case "email": return `${L(I, r, 0).toLowerCase()}.${L(_e, r, 2).toLowerCase()}${i}@example.com`;
|
|
107
|
+
case "fullName": return `${L(I, r, 0)} ${L(_e, r, 1)}`;
|
|
108
|
+
case "firstName": return L(I, r, 0);
|
|
109
|
+
case "lastName": return L(_e, r, 1);
|
|
110
|
+
case "username": return `${L(I, r, 0).toLowerCase()}${i}`;
|
|
111
|
+
case "phone": return `+1${String(F(r, 0) % 1e10).padStart(10, "0")}`;
|
|
112
|
+
case "address": return `${F(r, 0) % 9899 + 100} ${L(ve, r, 1)} St`;
|
|
113
|
+
case "company": return L(ye, r, 0);
|
|
114
114
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
115
|
-
case "ip": return `${
|
|
116
|
-
case "creditCard": return
|
|
115
|
+
case "ip": return `${F(r, 0) % 256}.${F(r, 1) % 256}.${F(r, 2) % 256}.${F(r, 3) % 254 + 1}`;
|
|
116
|
+
case "creditCard": return xe(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
117
117
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
118
118
|
}
|
|
119
|
-
},
|
|
119
|
+
}, xe = (e) => {
|
|
120
120
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
121
121
|
t.forEach((e, t) => {
|
|
122
122
|
let r = e;
|
|
@@ -124,31 +124,31 @@ var _e = (e) => {
|
|
|
124
124
|
});
|
|
125
125
|
let r = (10 - n % 10) % 10;
|
|
126
126
|
return e.slice(0, 15) + String(r);
|
|
127
|
-
},
|
|
127
|
+
}, Se = 864e5, Ce = (e, t) => {
|
|
128
128
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
129
129
|
if (Number.isNaN(n)) return e;
|
|
130
|
-
let r =
|
|
130
|
+
let r = F(P(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Se);
|
|
131
131
|
return e instanceof Date ? i : i.toISOString();
|
|
132
|
-
},
|
|
132
|
+
}, we = (e, t) => {
|
|
133
133
|
if (t.value === null || t.value === void 0) return null;
|
|
134
134
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
135
135
|
if (e === "keep") return t.value;
|
|
136
136
|
if (e === "null") return null;
|
|
137
137
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
138
|
-
if (e === "hash") return
|
|
139
|
-
if (e === "dateShift") return
|
|
140
|
-
if (typeof e == "object" && "fake" in e) return
|
|
138
|
+
if (e === "hash") return P(t.seed, t.value).toString("hex").slice(0, 16);
|
|
139
|
+
if (e === "dateShift") return Ce(t.value, t.seed);
|
|
140
|
+
if (typeof e == "object" && "fake" in e) return be(e.fake, t.value, t.seed);
|
|
141
141
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
142
142
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
143
|
-
},
|
|
143
|
+
}, Te = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Ee = (e, t, n) => {
|
|
144
144
|
let r = {}, i = [], a = [];
|
|
145
145
|
for (let o of e) for (let e of o.columns) {
|
|
146
|
-
let s = `${o.name}.${e.name}`, c =
|
|
146
|
+
let s = `${o.name}.${e.name}`, c = ge(o.name, e.name, t[o.name]?.[e.name], n);
|
|
147
147
|
if (c === "error") {
|
|
148
148
|
i.push(s);
|
|
149
149
|
continue;
|
|
150
150
|
}
|
|
151
|
-
if (!
|
|
151
|
+
if (!Te(c)) {
|
|
152
152
|
a.push({
|
|
153
153
|
column: s,
|
|
154
154
|
action: c
|
|
@@ -162,7 +162,7 @@ var _e = (e) => {
|
|
|
162
162
|
unclassified: i,
|
|
163
163
|
invalid: a
|
|
164
164
|
};
|
|
165
|
-
},
|
|
165
|
+
}, De = [
|
|
166
166
|
{
|
|
167
167
|
name: "email",
|
|
168
168
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -179,7 +179,7 @@ var _e = (e) => {
|
|
|
179
179
|
name: "creditCard",
|
|
180
180
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
181
181
|
}
|
|
182
|
-
],
|
|
182
|
+
], Oe = (e, t, n) => {
|
|
183
183
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
184
184
|
for (let [a, o] of Object.entries(n)) {
|
|
185
185
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -187,7 +187,7 @@ var _e = (e) => {
|
|
|
187
187
|
for (let a of e) {
|
|
188
188
|
let e = a[n];
|
|
189
189
|
if (typeof e != "string" || e.length === 0) continue;
|
|
190
|
-
let o =
|
|
190
|
+
let o = De.find((t) => t.re.test(e));
|
|
191
191
|
if (o && !i.has(n)) {
|
|
192
192
|
i.add(n), r.push({
|
|
193
193
|
table: t,
|
|
@@ -200,9 +200,9 @@ var _e = (e) => {
|
|
|
200
200
|
}
|
|
201
201
|
}
|
|
202
202
|
return r;
|
|
203
|
-
},
|
|
203
|
+
}, ke = (e, t, n, r, i) => {
|
|
204
204
|
let a = {};
|
|
205
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
205
|
+
for (let [o, s] of Object.entries(e)) a[o] = we(n[`${t}.${o}`] ?? "keep", {
|
|
206
206
|
value: s,
|
|
207
207
|
table: t,
|
|
208
208
|
column: o,
|
|
@@ -210,17 +210,17 @@ var _e = (e) => {
|
|
|
210
210
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
211
211
|
});
|
|
212
212
|
return a;
|
|
213
|
-
},
|
|
213
|
+
}, R = class extends s.TaggedError()("BundleError", {
|
|
214
214
|
reason: s.String,
|
|
215
215
|
path: s.optional(s.String)
|
|
216
|
-
}) {},
|
|
216
|
+
}) {}, Ae = class extends s.TaggedError()("CodecError", {
|
|
217
217
|
table: s.String,
|
|
218
218
|
reason: s.String
|
|
219
|
-
}) {},
|
|
219
|
+
}) {}, z = class extends s.TaggedError()("IntegrityError", {
|
|
220
220
|
what: s.String,
|
|
221
221
|
expected: s.String,
|
|
222
222
|
actual: s.String
|
|
223
|
-
}) {},
|
|
223
|
+
}) {}, je = class extends t.TaggedError("CompressionError") {}, B = class extends s.TaggedError()("ScopeError", { reason: s.String }) {}, Me = class extends s.TaggedError()("RowsRefusedError", {
|
|
224
224
|
rows: s.Array(s.Struct({
|
|
225
225
|
table: s.String,
|
|
226
226
|
id: s.String,
|
|
@@ -229,29 +229,29 @@ var _e = (e) => {
|
|
|
229
229
|
})),
|
|
230
230
|
primaryCount: s.Number,
|
|
231
231
|
totalCount: s.Number
|
|
232
|
-
}) {},
|
|
232
|
+
}) {}, Ne = class extends s.TaggedError()("SchemaDriftError", {
|
|
233
233
|
bundleFingerprint: s.String,
|
|
234
234
|
targetFingerprint: s.String,
|
|
235
235
|
diff: s.Array(s.String)
|
|
236
|
-
}) {},
|
|
236
|
+
}) {}, V = class extends t.TaggedError("NativeToolError") {
|
|
237
237
|
get message() {
|
|
238
238
|
let e = this.stderr?.trim();
|
|
239
239
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
240
240
|
}
|
|
241
|
-
},
|
|
241
|
+
}, Pe = class extends s.TaggedError()("MaskingError", {
|
|
242
242
|
unclassified: s.Array(s.String),
|
|
243
243
|
invalidActions: s.optional(s.Array(s.String))
|
|
244
|
-
}) {},
|
|
244
|
+
}) {}, Fe = class extends s.TaggedError()("ImportModeError", {
|
|
245
245
|
mode: s.String,
|
|
246
246
|
reason: s.String
|
|
247
|
-
}) {},
|
|
247
|
+
}) {}, Ie = class extends s.TaggedError()("CrossDialectError", {
|
|
248
248
|
sourceDialect: s.String,
|
|
249
249
|
targetDialect: s.String,
|
|
250
250
|
blocking: s.Array(s.String)
|
|
251
|
-
}) {},
|
|
251
|
+
}) {}, H = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Le = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
252
252
|
col: e.name,
|
|
253
253
|
refTable: e.references.table
|
|
254
|
-
})),
|
|
254
|
+
})), Re = (e, t, r) => n.gen(function* () {
|
|
255
255
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map(), u = (e) => {
|
|
256
256
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
257
257
|
return s.set(e, t), t;
|
|
@@ -269,10 +269,10 @@ var _e = (e) => {
|
|
|
269
269
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
270
270
|
};
|
|
271
271
|
}, g = (r, i) => {
|
|
272
|
-
let a = p(r,
|
|
272
|
+
let a = p(r, H(t, r), Le(t, r));
|
|
273
273
|
return c.runForEach(h(e, {
|
|
274
274
|
table: r,
|
|
275
|
-
pk:
|
|
275
|
+
pk: H(t, r),
|
|
276
276
|
...i === void 0 ? {} : { where: i }
|
|
277
277
|
}), (e) => n.sync(() => a(e)));
|
|
278
278
|
}, _ = n.gen(function* () {
|
|
@@ -281,7 +281,7 @@ var _e = (e) => {
|
|
|
281
281
|
l.clear();
|
|
282
282
|
for (let [n, r] of e) {
|
|
283
283
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
284
|
-
e.length !== 0 && (yield* g(n, f(
|
|
284
|
+
e.length !== 0 && (yield* g(n, f(H(t, n), e)));
|
|
285
285
|
}
|
|
286
286
|
}
|
|
287
287
|
});
|
|
@@ -304,13 +304,13 @@ var _e = (e) => {
|
|
|
304
304
|
}
|
|
305
305
|
}
|
|
306
306
|
return o;
|
|
307
|
-
}),
|
|
307
|
+
}), ze = (e, t) => ({
|
|
308
308
|
tables: [...e.keys()],
|
|
309
309
|
predicateFor: (n) => {
|
|
310
310
|
let r = e.get(n);
|
|
311
|
-
return r ? f(
|
|
311
|
+
return r ? f(H(t, n), [...r]) : void 0;
|
|
312
312
|
}
|
|
313
|
-
}),
|
|
313
|
+
}), U = "tenantId", Be = (e, t, r) => n.gen(function* () {
|
|
314
314
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
315
315
|
if (e.kind === "all") return {
|
|
316
316
|
tables: i,
|
|
@@ -318,7 +318,7 @@ var _e = (e) => {
|
|
|
318
318
|
};
|
|
319
319
|
if (e.kind === "tables") {
|
|
320
320
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
321
|
-
if (a.length > 0) return yield* n.fail(new
|
|
321
|
+
if (a.length > 0) return yield* n.fail(new B({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
322
322
|
let o = new Set(t);
|
|
323
323
|
return {
|
|
324
324
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -326,30 +326,30 @@ var _e = (e) => {
|
|
|
326
326
|
};
|
|
327
327
|
}
|
|
328
328
|
let o = e.tenantId;
|
|
329
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
330
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
331
|
-
if (s.length === 0) return yield* n.fail(new
|
|
332
|
-
let c = yield*
|
|
333
|
-
seeds: Object.fromEntries(s.map((e) => [e, u(
|
|
329
|
+
if (o === void 0 || o === "") return yield* n.fail(new B({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
330
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === U) ?? !1);
|
|
331
|
+
if (s.length === 0) return yield* n.fail(new B({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${U}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
332
|
+
let c = yield* Re(r.store, t, {
|
|
333
|
+
seeds: Object.fromEntries(s.map((e) => [e, u(U, o)])),
|
|
334
334
|
pinned: s,
|
|
335
335
|
children: { roots: s }
|
|
336
336
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), d = new Set(s);
|
|
337
337
|
return {
|
|
338
338
|
tables: [...s, ...l],
|
|
339
|
-
predicateFor: (e) => d.has(e) ? u(
|
|
339
|
+
predicateFor: (e) => d.has(e) ? u(U, o) : c.has(e) ? f(H(t, e), [...c.get(e)]) : void 0
|
|
340
340
|
};
|
|
341
|
-
}),
|
|
341
|
+
}), Ve = (e, t) => ({
|
|
342
342
|
list: t,
|
|
343
|
-
open: (t) =>
|
|
344
|
-
}),
|
|
345
|
-
write: (t, n, r) =>
|
|
343
|
+
open: (t) => T(e, t)
|
|
344
|
+
}), He = (e) => ({
|
|
345
|
+
write: (t, n, r) => E(e, t, n, r),
|
|
346
346
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
347
|
-
}),
|
|
347
|
+
}), Ue = (e, t) => n.tryPromise({
|
|
348
348
|
try: async () => {
|
|
349
349
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
350
350
|
u.update(e), d += e.length, n(null, e);
|
|
351
351
|
} });
|
|
352
|
-
await o(await
|
|
352
|
+
await o(await D(e.stream), f, n(l));
|
|
353
353
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
354
354
|
try {
|
|
355
355
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -362,23 +362,23 @@ var _e = (e) => {
|
|
|
362
362
|
contentType: e.contentType
|
|
363
363
|
};
|
|
364
364
|
},
|
|
365
|
-
catch: (e) => new
|
|
365
|
+
catch: (e) => new R({
|
|
366
366
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
367
367
|
path: t
|
|
368
368
|
})
|
|
369
|
-
}),
|
|
369
|
+
}), We = (e, t, r) => n.gen(function* () {
|
|
370
370
|
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), l = o("sha256"), u = w("bundle", () => a(s)).pipe(c.tap((e) => n.sync(() => l.update(e))));
|
|
371
371
|
yield* e.write(r.key, u, {
|
|
372
372
|
contentType: r.contentType,
|
|
373
373
|
totalSize: r.size
|
|
374
374
|
});
|
|
375
375
|
let d = l.digest("hex");
|
|
376
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
376
|
+
if (d !== r.sha256) return yield* n.fail(new z({
|
|
377
377
|
what: `asset ${r.key}`,
|
|
378
378
|
expected: r.sha256,
|
|
379
379
|
actual: d
|
|
380
380
|
}));
|
|
381
|
-
}),
|
|
381
|
+
}), Ge = () => typeof O.createZstdCompress == "function", Ke = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : Ge() ? "zstd" : "gzip", qe = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", Je = (e) => e === "zstd" ? O.createZstdCompress() : e === "gzip" ? O.createGzip() : new k(), Ye = (e) => e === "zstd" ? O.createZstdDecompress() : e === "gzip" ? O.createGunzip() : new k(), Xe = 1, Ze = s.Struct({
|
|
382
382
|
name: s.String,
|
|
383
383
|
file: s.String,
|
|
384
384
|
rowCount: s.Number,
|
|
@@ -388,11 +388,11 @@ var _e = (e) => {
|
|
|
388
388
|
value: s.String
|
|
389
389
|
}),
|
|
390
390
|
primaryKey: s.String
|
|
391
|
-
}),
|
|
391
|
+
}), Qe = s.Struct({
|
|
392
392
|
kind: s.Literal("all", "tenant", "tables"),
|
|
393
393
|
tenantId: s.optional(s.String),
|
|
394
394
|
tables: s.optional(s.Array(s.String))
|
|
395
|
-
}),
|
|
395
|
+
}), $e = s.Struct({
|
|
396
396
|
count: s.Number,
|
|
397
397
|
totalBytes: s.Number,
|
|
398
398
|
index: s.optional(s.String)
|
|
@@ -403,23 +403,23 @@ s.Struct({
|
|
|
403
403
|
size: s.Number,
|
|
404
404
|
contentType: s.String
|
|
405
405
|
});
|
|
406
|
-
var
|
|
406
|
+
var et = s.Struct({
|
|
407
407
|
formatVersion: s.Number,
|
|
408
408
|
createdAt: s.String,
|
|
409
409
|
source: s.Struct({
|
|
410
410
|
dialect: s.String,
|
|
411
411
|
schemaFingerprint: s.String
|
|
412
412
|
}),
|
|
413
|
-
scope:
|
|
413
|
+
scope: Qe,
|
|
414
414
|
consistency: s.Literal("live", "snapshot"),
|
|
415
415
|
compression: s.Literal("zstd", "gzip", "none"),
|
|
416
|
-
tables: s.Array(
|
|
417
|
-
assets:
|
|
416
|
+
tables: s.Array(Ze),
|
|
417
|
+
assets: $e,
|
|
418
418
|
masking: s.optional(s.Struct({
|
|
419
419
|
policyId: s.optional(s.String),
|
|
420
420
|
transformed: s.Array(s.String)
|
|
421
421
|
}))
|
|
422
|
-
}),
|
|
422
|
+
}), tt = s.encodeSync(s.parseJson(et)), nt = s.decodeUnknownSync(s.parseJson(et)), W = "manifest.json", rt = "README.md", it = "data", G = "assets", at = "assets/index.ndjson", ot = ".ledger.json", st = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", ct = (e, t) => {
|
|
423
423
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
424
424
|
return `# Voltro data export
|
|
425
425
|
|
|
@@ -482,7 +482,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
482
482
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
483
483
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
484
484
|
},
|
|
485
|
-
catch: (t) => new
|
|
485
|
+
catch: (t) => new R({
|
|
486
486
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
487
487
|
path: e
|
|
488
488
|
})
|
|
@@ -520,7 +520,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
520
520
|
return n;
|
|
521
521
|
}, Dt = new TextEncoder(), Ot = (e, t) => e.pipe(c.map((e) => Dt.encode(`${JSON.stringify(Tt(e, t))}\n`))), kt = (e, t, r) => e.pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.mapEffect((e) => n.try({
|
|
522
522
|
try: () => Et(JSON.parse(e), r),
|
|
523
|
-
catch: (e) => new
|
|
523
|
+
catch: (e) => new Ae({
|
|
524
524
|
table: t,
|
|
525
525
|
reason: String(e?.message ?? e)
|
|
526
526
|
})
|
|
@@ -585,7 +585,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
585
585
|
n(null, e);
|
|
586
586
|
} }), m = l.fromWeb(c.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
587
587
|
try {
|
|
588
|
-
await s(m, p,
|
|
588
|
+
await s(m, p, Je(r), n(h)), await i.rename(h, t);
|
|
589
589
|
} catch (e) {
|
|
590
590
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
591
591
|
}
|
|
@@ -594,7 +594,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
594
594
|
checksum: d.digest("hex")
|
|
595
595
|
};
|
|
596
596
|
},
|
|
597
|
-
catch: (e) => new
|
|
597
|
+
catch: (e) => new R({
|
|
598
598
|
reason: `write table failed: ${J(e)}`,
|
|
599
599
|
path: t
|
|
600
600
|
})
|
|
@@ -602,7 +602,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
602
602
|
try: async () => {
|
|
603
603
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
604
604
|
},
|
|
605
|
-
catch: (t) => new
|
|
605
|
+
catch: (t) => new R({
|
|
606
606
|
reason: `append failed: ${J(t)}`,
|
|
607
607
|
path: e
|
|
608
608
|
})
|
|
@@ -610,8 +610,8 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
610
610
|
let r = [], i = t.ordered.length;
|
|
611
611
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
612
612
|
let o = t.ordered[a], s = Mt(t.snapshot, o), l = Ft(t.snapshot, o);
|
|
613
|
-
if ("refusal" in l) return yield* n.fail(new
|
|
614
|
-
let u = l.column, d = `${
|
|
613
|
+
if ("refusal" in l) return yield* n.fail(new B({ reason: `export: ${l.refusal}` }));
|
|
614
|
+
let u = l.column, d = `${it}/${o}.ndjson${qe(t.compression)}`;
|
|
615
615
|
if (ht(t.ledger, o)) {
|
|
616
616
|
let e = t.ledger.tables[o];
|
|
617
617
|
K(t.onProgress, {
|
|
@@ -646,7 +646,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
646
646
|
chunkSize: t.chunkSize,
|
|
647
647
|
retry: t.retry,
|
|
648
648
|
...f === void 0 ? {} : { where: f }
|
|
649
|
-
}), m = Nt(t.snapshot, o), g = m.length === 0 ? p : p.pipe(c.map((e) => Pt(e, m))), _ = t.mask ? g.pipe(c.map((e) =>
|
|
649
|
+
}), m = Nt(t.snapshot, o), g = m.length === 0 ? p : p.pipe(c.map((e) => Pt(e, m))), _ = t.mask ? g.pipe(c.map((e) => ke(e, o, t.mask.actions, t.mask.seed, s))) : g, { rowCount: v, checksum: y } = yield* ft("export", o, a, i, (e) => e.rowCount, It(Ot(_, s), t.bundleFile(d), t.compression));
|
|
650
650
|
t.ledger.tables[o] = {
|
|
651
651
|
rowCount: v,
|
|
652
652
|
checksum: y
|
|
@@ -674,37 +674,37 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
674
674
|
values: []
|
|
675
675
|
}), Bt = (t, i, a) => n.tryPromise({
|
|
676
676
|
try: () => t.transactional(async (t) => {
|
|
677
|
-
if (t.raw) for (let e of
|
|
677
|
+
if (t.raw) for (let e of me(i)) await t.raw(zt(e));
|
|
678
678
|
let o = await n.runPromiseExit(a(t));
|
|
679
679
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
680
680
|
return o.value;
|
|
681
681
|
}),
|
|
682
|
-
catch: (e) => e instanceof
|
|
682
|
+
catch: (e) => e instanceof R ? e : new R({ reason: `snapshot export failed: ${J(e)}` })
|
|
683
683
|
}), Vt = (e) => n.gen(function* () {
|
|
684
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
684
|
+
let t = e.scope ?? { kind: "all" }, r = Ke(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, s = o.exponential("50 millis").pipe(o.jittered, o.intersect(o.recurs(a))), l = yield* n.promise(() => import("node:path")), u = (t) => l.join(e.outDir, t);
|
|
685
685
|
yield* n.tryPromise({
|
|
686
686
|
try: async () => {
|
|
687
687
|
let t = await import("node:fs/promises");
|
|
688
|
-
await t.mkdir(u(
|
|
688
|
+
await t.mkdir(u(it), { recursive: !0 }), e.assets && await t.mkdir(u(G), { recursive: !0 });
|
|
689
689
|
},
|
|
690
|
-
catch: (t) => new
|
|
690
|
+
catch: (t) => new R({
|
|
691
691
|
reason: `mkdir failed: ${J(t)}`,
|
|
692
692
|
path: e.outDir
|
|
693
693
|
})
|
|
694
694
|
});
|
|
695
|
-
let d = u(ot), f = yield* mt(d), p = e.subset ?
|
|
695
|
+
let d = u(ot), f = yield* mt(d), p = e.subset ? ze(yield* Re(e.store, e.snapshot, e.subset), e.snapshot) : yield* Be(t, e.snapshot, {
|
|
696
696
|
store: e.store,
|
|
697
697
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
698
698
|
}), m = At(p.tables, e.snapshot), h = e.consistency ?? "live", _, v;
|
|
699
699
|
if (e.masking) {
|
|
700
|
-
let t = e.classification ??
|
|
700
|
+
let t = e.classification ?? N(e.snapshot), r = Ee(m.map((t) => ({
|
|
701
701
|
name: t,
|
|
702
702
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
703
703
|
name: e.name,
|
|
704
704
|
type: e.type
|
|
705
705
|
}))
|
|
706
706
|
})), t, e.masking);
|
|
707
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
707
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new Pe({
|
|
708
708
|
unclassified: r.unclassified,
|
|
709
709
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
710
710
|
}));
|
|
@@ -732,8 +732,8 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
732
732
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
733
733
|
}, b = h === "snapshot" ? yield* Bt(e.store, e.dialect, (e) => Rt(e, y)) : yield* Rt(e.store, y), x = 0, S = 0;
|
|
734
734
|
if (e.assets) {
|
|
735
|
-
let t = u(
|
|
736
|
-
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
735
|
+
let t = u(at);
|
|
736
|
+
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => Ue(e, u(G))), n.flatMap((e) => {
|
|
737
737
|
let i = {
|
|
738
738
|
key: r.key,
|
|
739
739
|
sha256: e.sha256,
|
|
@@ -741,7 +741,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
741
741
|
contentType: e.contentType
|
|
742
742
|
};
|
|
743
743
|
return x++, S += e.size, f.assets[r.key] = !0, Lt(t, JSON.stringify(i)).pipe(n.andThen(q(d, f)));
|
|
744
|
-
}))).pipe(n.mapError((e) => e instanceof
|
|
744
|
+
}))).pipe(n.mapError((e) => e instanceof R ? e : new R({ reason: `asset export failed: ${String(e)}` })));
|
|
745
745
|
}
|
|
746
746
|
let C = {
|
|
747
747
|
formatVersion: 1,
|
|
@@ -757,7 +757,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
757
757
|
assets: x > 0 ? {
|
|
758
758
|
count: x,
|
|
759
759
|
totalBytes: S,
|
|
760
|
-
index:
|
|
760
|
+
index: at
|
|
761
761
|
} : {
|
|
762
762
|
count: 0,
|
|
763
763
|
totalBytes: 0
|
|
@@ -766,23 +766,42 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
766
766
|
};
|
|
767
767
|
return yield* n.tryPromise({
|
|
768
768
|
try: async () => {
|
|
769
|
-
await (await import("node:fs/promises")).writeFile(u(
|
|
769
|
+
await (await import("node:fs/promises")).writeFile(u(W), tt(C));
|
|
770
770
|
},
|
|
771
|
-
catch: (e) => new
|
|
771
|
+
catch: (e) => new R({
|
|
772
772
|
reason: `manifest write failed: ${J(e)}`,
|
|
773
|
-
path:
|
|
773
|
+
path: W
|
|
774
774
|
})
|
|
775
775
|
}), yield* n.tryPromise({
|
|
776
776
|
try: async () => {
|
|
777
777
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
778
|
-
await e.writeFile(u(
|
|
778
|
+
await e.writeFile(u(rt), ct(C, { hostname: t.hostname() }));
|
|
779
779
|
},
|
|
780
|
-
catch: (e) => new
|
|
780
|
+
catch: (e) => new R({
|
|
781
781
|
reason: `readme write failed: ${J(e)}`,
|
|
782
|
-
path:
|
|
782
|
+
path: rt
|
|
783
783
|
})
|
|
784
784
|
}), C;
|
|
785
|
-
}), Ht = (e) => {
|
|
785
|
+
}), Ht = (e, t) => {
|
|
786
|
+
let n = new Set(t), r = [];
|
|
787
|
+
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
788
|
+
let i = e.references?.table;
|
|
789
|
+
i !== void 0 && n.has(i) && r.push({
|
|
790
|
+
from: t.name,
|
|
791
|
+
column: e.name,
|
|
792
|
+
to: i
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
return r;
|
|
796
|
+
}, Ut = (e) => {
|
|
797
|
+
let t = /* @__PURE__ */ new Map();
|
|
798
|
+
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
799
|
+
let n = [...t.entries()].map(([e, t]) => {
|
|
800
|
+
let n = t[0].rowCount, r = t.map((e) => `${e.column} → ${e.to}`).join(", ");
|
|
801
|
+
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
802
|
+
});
|
|
803
|
+
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
804
|
+
}, Wt = (e) => {
|
|
786
805
|
switch (e) {
|
|
787
806
|
case "id":
|
|
788
807
|
case "text":
|
|
@@ -801,12 +820,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
801
820
|
case "raw": return "raw";
|
|
802
821
|
default: return "string";
|
|
803
822
|
}
|
|
804
|
-
},
|
|
823
|
+
}, Gt = {
|
|
805
824
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
806
825
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
807
826
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
808
|
-
},
|
|
809
|
-
let i =
|
|
827
|
+
}, Kt = (e, t, n, r) => {
|
|
828
|
+
let i = Wt(n);
|
|
810
829
|
if (i === "raw") return {
|
|
811
830
|
table: e,
|
|
812
831
|
column: t,
|
|
@@ -814,7 +833,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
814
833
|
severity: "block",
|
|
815
834
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
816
835
|
};
|
|
817
|
-
let a =
|
|
836
|
+
let a = Gt[i];
|
|
818
837
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
819
838
|
table: e,
|
|
820
839
|
column: t,
|
|
@@ -828,15 +847,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
828
847
|
severity: "block",
|
|
829
848
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
830
849
|
};
|
|
831
|
-
},
|
|
850
|
+
}, qt = (e, t, n) => {
|
|
832
851
|
if (t === n) return [];
|
|
833
852
|
let r = [];
|
|
834
853
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
835
|
-
let a =
|
|
854
|
+
let a = Kt(t.name, e, i, n);
|
|
836
855
|
a && r.push(a);
|
|
837
856
|
}
|
|
838
857
|
return r;
|
|
839
|
-
},
|
|
858
|
+
}, Jt = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, Yt = (e) => {
|
|
840
859
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
841
860
|
switch (e.kind) {
|
|
842
861
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -845,22 +864,29 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
845
864
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
846
865
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
847
866
|
}
|
|
848
|
-
},
|
|
867
|
+
}, Xt = (e) => {
|
|
849
868
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
850
869
|
return t.length > 0 ? t.join("/") : void 0;
|
|
851
|
-
},
|
|
852
|
-
let t = d(e), n = l(t), r =
|
|
853
|
-
if (n !== void 0) return `${
|
|
870
|
+
}, Zt = (e) => {
|
|
871
|
+
let t = d(e), n = l(t), r = Xt(t), i = r === void 0 ? "" : ` [${r}]`;
|
|
872
|
+
if (n !== void 0) return `${Yt(n)}${i}`;
|
|
854
873
|
let a = typeof t.dbMessage == "string" ? t.dbMessage : void 0, o = e?.message, s = typeof o == "string" && o.length > 0 ? o : String(e), c = a !== void 0 && a !== s && !a.startsWith("SqlError:") ? a : s, u = c === "Failed to execute statement" ? `the database refused the write and the driver gave no detail${i}` : `${c}${i}`;
|
|
855
874
|
return u.length <= 300 ? u : `${u.slice(0, 300)}…`;
|
|
856
|
-
},
|
|
875
|
+
}, Qt = (e) => {
|
|
876
|
+
let t = d(e), n = l(t);
|
|
877
|
+
if (n?.kind === "foreignKeyInUse") {
|
|
878
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = Xt(t);
|
|
879
|
+
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
880
|
+
}
|
|
881
|
+
return Zt(e);
|
|
882
|
+
}, $t = (e, t) => e.tables.find((e) => e.name === t), en = (e, t) => new Map(($t(e, t)?.columns ?? []).map((e) => [e.name, e])), tn = (e, t) => {
|
|
857
883
|
let n = [];
|
|
858
884
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
859
|
-
if (
|
|
885
|
+
if ($t(t, r.name) === void 0) {
|
|
860
886
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
861
887
|
continue;
|
|
862
888
|
}
|
|
863
|
-
let e =
|
|
889
|
+
let e = en(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
864
890
|
for (let [t, a] of i) {
|
|
865
891
|
let i = e.get(t);
|
|
866
892
|
if (i === void 0) {
|
|
@@ -872,7 +898,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
872
898
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
873
899
|
}
|
|
874
900
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
875
|
-
},
|
|
901
|
+
}, nn = (e, t, n) => {
|
|
876
902
|
let r = g(t);
|
|
877
903
|
if (r === e || n === void 0) return {
|
|
878
904
|
drifted: !1,
|
|
@@ -880,28 +906,28 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
880
906
|
targetFingerprint: r,
|
|
881
907
|
diff: []
|
|
882
908
|
};
|
|
883
|
-
let i =
|
|
909
|
+
let i = tn(n, t);
|
|
884
910
|
return {
|
|
885
911
|
drifted: i.length > 0,
|
|
886
912
|
bundleFingerprint: e,
|
|
887
913
|
targetFingerprint: r,
|
|
888
914
|
diff: i
|
|
889
915
|
};
|
|
890
|
-
},
|
|
916
|
+
}, rn = 200, an = (e) => n.tryPromise({
|
|
891
917
|
try: async () => {
|
|
892
918
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
893
|
-
return
|
|
919
|
+
return nt(await t.readFile(n.join(e, W), "utf8"));
|
|
894
920
|
},
|
|
895
|
-
catch: (e) => new
|
|
921
|
+
catch: (e) => new R({
|
|
896
922
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
897
|
-
path:
|
|
923
|
+
path: W
|
|
898
924
|
})
|
|
899
|
-
}),
|
|
900
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
901
|
-
if (a.formatVersion !== 1) return yield* n.fail(new
|
|
925
|
+
}), on = (t) => n.gen(function* () {
|
|
926
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* an(t.bundleDir);
|
|
927
|
+
if (a.formatVersion !== 1) return yield* n.fail(new R({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
902
928
|
if (t.tables !== void 0) {
|
|
903
929
|
let e = new Set(a.tables.map((e) => e.name)), r = t.tables.filter((t) => !e.has(t));
|
|
904
|
-
if (r.length > 0) return yield* n.fail(new
|
|
930
|
+
if (r.length > 0) return yield* n.fail(new R({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...e].sort().join(", ")}.` }));
|
|
905
931
|
}
|
|
906
932
|
let o = t.tables === void 0 ? void 0 : new Set(t.tables), s = o === void 0 ? a : {
|
|
907
933
|
...a,
|
|
@@ -918,9 +944,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
918
944
|
unique: !1,
|
|
919
945
|
hasDefault: !1
|
|
920
946
|
}))
|
|
921
|
-
})) }, r =
|
|
947
|
+
})) }, r = nn(s.source.schemaFingerprint, t.targetSnapshot, e);
|
|
922
948
|
if (r.drifted) {
|
|
923
|
-
if (!t.force) return yield* n.fail(new
|
|
949
|
+
if (!t.force) return yield* n.fail(new Ne({
|
|
924
950
|
bundleFingerprint: r.bundleFingerprint,
|
|
925
951
|
targetFingerprint: r.targetFingerprint,
|
|
926
952
|
diff: r.diff
|
|
@@ -929,43 +955,81 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
929
955
|
}
|
|
930
956
|
}
|
|
931
957
|
if (t.targetDialect && t.targetDialect !== s.source.dialect) {
|
|
932
|
-
let e =
|
|
933
|
-
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new
|
|
958
|
+
let e = qt(s.tables, s.source.dialect, t.targetDialect).filter((e) => e.severity === "block");
|
|
959
|
+
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new Ie({
|
|
934
960
|
sourceDialect: s.source.dialect,
|
|
935
961
|
targetDialect: t.targetDialect,
|
|
936
|
-
blocking: e.map(
|
|
962
|
+
blocking: e.map(Jt)
|
|
937
963
|
}));
|
|
938
964
|
}
|
|
939
|
-
let l = t.verify ?? !0, u = t.mode ?? "upsert", d = t.onConflict ?? "skip",
|
|
940
|
-
if (u === "replace" && s.scope.kind !== "all") return yield* n.fail(new
|
|
965
|
+
let l = t.verify ?? !0, u = t.mode ?? "upsert", d = t.atomic ?? u === "replace", f = t.onConflict ?? "skip", m = t.ledgerPath ?? i.join(t.bundleDir, ".import..ledger.json"), h = yield* mt(m);
|
|
966
|
+
if (u === "replace" && s.scope.kind !== "all") return yield* n.fail(new Fe({
|
|
941
967
|
mode: u,
|
|
942
968
|
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
943
969
|
}));
|
|
944
|
-
if (
|
|
945
|
-
|
|
970
|
+
if (u === "replace") {
|
|
971
|
+
let e = s.tables.map((e) => e.name), r = t.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
972
|
+
try: () => t.store.incomingForeignKeys(e),
|
|
973
|
+
catch: (e) => new R({ reason: `foreign-key pre-flight failed: ${Qt(e)}` })
|
|
974
|
+
}) : t.targetSnapshot === void 0 ? [] : Ht(t.targetSnapshot, e), i = [], a = /* @__PURE__ */ new Map();
|
|
975
|
+
for (let e of r) {
|
|
976
|
+
if (!a.has(e.from)) {
|
|
977
|
+
let r = yield* n.tryPromise({
|
|
978
|
+
try: () => t.store.query({
|
|
979
|
+
table: e.from,
|
|
980
|
+
predicate: void 0,
|
|
981
|
+
order: [],
|
|
982
|
+
take: void 0,
|
|
983
|
+
skip: void 0,
|
|
984
|
+
projection: []
|
|
985
|
+
}),
|
|
986
|
+
catch: () => new R({ reason: `unreadable: ${e.from}` })
|
|
987
|
+
}).pipe(n.map((e) => e.length), n.orElseSucceed(() => 0));
|
|
988
|
+
a.set(e.from, r);
|
|
989
|
+
}
|
|
990
|
+
let r = a.get(e.from) ?? 0;
|
|
991
|
+
r > 0 && i.push({
|
|
992
|
+
...e,
|
|
993
|
+
rowCount: r
|
|
994
|
+
});
|
|
995
|
+
}
|
|
996
|
+
if (i.length > 0) return yield* n.fail(new Fe({
|
|
997
|
+
mode: u,
|
|
998
|
+
reason: Ut(i)
|
|
999
|
+
}));
|
|
1000
|
+
}
|
|
1001
|
+
if (t.dryRun === !0) return {
|
|
1002
|
+
manifest: s,
|
|
1003
|
+
tablesWritten: 0,
|
|
1004
|
+
rowsWritten: 0,
|
|
1005
|
+
tablesSkipped: 0,
|
|
1006
|
+
rowsSkipped: 0,
|
|
1007
|
+
fullyResumed: !1
|
|
1008
|
+
};
|
|
1009
|
+
let g = /* @__PURE__ */ new Map();
|
|
946
1010
|
for (let e of t.targetSnapshot?.tables ?? []) {
|
|
947
1011
|
let t = e.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
948
|
-
t.length > 0 &&
|
|
1012
|
+
t.length > 0 && g.set(e.name, t);
|
|
949
1013
|
}
|
|
950
|
-
let
|
|
951
|
-
let n =
|
|
1014
|
+
let _ = (e, t) => {
|
|
1015
|
+
let n = g.get(e);
|
|
952
1016
|
if (n === void 0) return t;
|
|
953
1017
|
let r;
|
|
954
1018
|
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
955
1019
|
return r ?? t;
|
|
956
|
-
},
|
|
957
|
-
let
|
|
958
|
-
return u === "append" ?
|
|
959
|
-
},
|
|
1020
|
+
}, v = (e, t, n, r = !1) => {
|
|
1021
|
+
let i = t.name, a = t.primaryKey, o = _(i, n);
|
|
1022
|
+
return u === "append" ? f === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : u === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1023
|
+
}, y = (e, t) => e.savepoint ? e.savepoint(t) : t(), b = (e, t, r = !0) => n.tryPromise({
|
|
960
1024
|
try: e,
|
|
961
|
-
catch: (e) =>
|
|
962
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e), !1)))),
|
|
1025
|
+
catch: (e) => Zt(e)
|
|
1026
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e), !1)))), x = (e) => {
|
|
963
1027
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
964
1028
|
if (t.length !== 0) return {
|
|
965
1029
|
...e.row,
|
|
966
1030
|
...Object.fromEntries(t.map((e) => [e, null]))
|
|
967
1031
|
};
|
|
968
|
-
},
|
|
1032
|
+
}, S = (e) => {
|
|
969
1033
|
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
970
1034
|
if (r !== "reference") return !1;
|
|
971
1035
|
let i = e.row[n];
|
|
@@ -976,78 +1040,83 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
976
1040
|
reason: e.reason,
|
|
977
1041
|
derived: n(e)
|
|
978
1042
|
}));
|
|
979
|
-
return new
|
|
1043
|
+
return new Me({
|
|
980
1044
|
rows: [...r.filter((e) => !e.derived), ...r.filter((e) => e.derived)].slice(0, 20),
|
|
981
1045
|
primaryCount: r.filter((e) => !e.derived).length,
|
|
982
1046
|
totalCount: e.length
|
|
983
1047
|
});
|
|
984
|
-
},
|
|
1048
|
+
}, C = (e, t) => n.gen(function* () {
|
|
985
1049
|
let r = t, i = [], a = !0;
|
|
986
1050
|
for (; r.length > 0 && a;) {
|
|
987
1051
|
a = !1;
|
|
988
1052
|
let t = [];
|
|
989
|
-
for (let n of r) (yield*
|
|
1053
|
+
for (let n of r) (yield* b(() => y(e, () => v(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
990
1054
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
991
|
-
let r = t[n], o =
|
|
992
|
-
o !== void 0 && (yield*
|
|
1055
|
+
let r = t[n], o = x(r);
|
|
1056
|
+
o !== void 0 && (yield* b(() => y(e, () => v(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
993
1057
|
}
|
|
994
1058
|
r = t;
|
|
995
1059
|
}
|
|
996
|
-
for (let t of i) (yield*
|
|
997
|
-
if (r.length > 0) return yield* n.fail(
|
|
998
|
-
}),
|
|
999
|
-
if (u === "replace" && !(r &&
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1060
|
+
for (let t of i) (yield* b(() => y(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1061
|
+
if (r.length > 0) return yield* n.fail(S(r));
|
|
1062
|
+
}), T = !1, E = 0, D = 0, O = 0, k = 0, A = (e, r) => n.gen(function* () {
|
|
1063
|
+
if (u === "replace" && !(r && h.truncated)) {
|
|
1064
|
+
let t = [...s.tables].reverse().map((e) => e.name), i = !r, a = e.emptyTables ? () => e.emptyTables(t) : i ? async () => {
|
|
1065
|
+
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: p(t.primaryKey) });
|
|
1066
|
+
} : () => e.transactional(async (e) => {
|
|
1067
|
+
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: p(t.primaryKey) });
|
|
1003
1068
|
});
|
|
1004
|
-
|
|
1069
|
+
yield* n.tryPromise({
|
|
1070
|
+
try: a,
|
|
1071
|
+
catch: (e) => new R({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${Qt(e)}` })
|
|
1072
|
+
}), r && (h.truncated = !0, yield* q(m, h));
|
|
1005
1073
|
}
|
|
1006
|
-
let a = [], o = [],
|
|
1074
|
+
let a = [], o = [], f = [], g = s.tables.length;
|
|
1007
1075
|
for (let p = 0; p < s.tables.length; p++) {
|
|
1008
|
-
let
|
|
1009
|
-
if (r &&
|
|
1010
|
-
|
|
1076
|
+
let _ = s.tables[p];
|
|
1077
|
+
if (r && _.name in h.tables) {
|
|
1078
|
+
f.push(_.name), k++, O += h.tables[_.name].rowCount, K(t.onProgress, {
|
|
1011
1079
|
phase: "import",
|
|
1012
|
-
table:
|
|
1080
|
+
table: _.name,
|
|
1013
1081
|
index: p,
|
|
1014
|
-
tableCount:
|
|
1082
|
+
tableCount: g,
|
|
1015
1083
|
event: "done",
|
|
1016
|
-
rowsDone:
|
|
1017
|
-
total:
|
|
1084
|
+
rowsDone: h.tables[_.name].rowCount,
|
|
1085
|
+
total: _.rowCount
|
|
1018
1086
|
});
|
|
1019
1087
|
continue;
|
|
1020
1088
|
}
|
|
1021
|
-
let
|
|
1089
|
+
let b = i.join(t.bundleDir, _.file);
|
|
1022
1090
|
K(t.onProgress, {
|
|
1023
1091
|
phase: "import",
|
|
1024
|
-
table:
|
|
1092
|
+
table: _.name,
|
|
1025
1093
|
index: p,
|
|
1026
|
-
tableCount:
|
|
1094
|
+
tableCount: g,
|
|
1027
1095
|
event: "start",
|
|
1028
1096
|
rowsDone: 0,
|
|
1029
|
-
total:
|
|
1097
|
+
total: _.rowCount
|
|
1030
1098
|
});
|
|
1031
|
-
let
|
|
1032
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, l = w("bundle", () =>
|
|
1099
|
+
let x = a.length, S = n.gen(function* () {
|
|
1100
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, l = w("bundle", () => j(b).pipe(Ye(s.compression))).pipe(c.tap((e) => n.sync(() => i.update(e)))), f = async (t) => {
|
|
1033
1101
|
let n = {
|
|
1034
|
-
entry:
|
|
1102
|
+
entry: _,
|
|
1035
1103
|
row: t,
|
|
1036
1104
|
reason: ""
|
|
1037
1105
|
};
|
|
1038
1106
|
try {
|
|
1039
|
-
await
|
|
1107
|
+
await y(e, () => v(e, _, t));
|
|
1040
1108
|
} catch (e) {
|
|
1041
|
-
n.reason =
|
|
1109
|
+
n.reason = Zt(e), a.push(n);
|
|
1042
1110
|
}
|
|
1043
|
-
},
|
|
1111
|
+
}, p = yield* n.promise(async () => {
|
|
1044
1112
|
let n = t.copyLoader;
|
|
1045
|
-
if (!n || n.dialect !== "postgres"
|
|
1113
|
+
if (!n || n.dialect !== "postgres") return !1;
|
|
1114
|
+
if (d) return T || (T = !0, t.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1046
1115
|
if (u === "replace") return !0;
|
|
1047
1116
|
if (u !== "upsert") return !1;
|
|
1048
1117
|
try {
|
|
1049
1118
|
return (await e.query({
|
|
1050
|
-
table:
|
|
1119
|
+
table: _.name,
|
|
1051
1120
|
predicate: void 0,
|
|
1052
1121
|
order: [],
|
|
1053
1122
|
take: 1,
|
|
@@ -1057,101 +1126,123 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1057
1126
|
} catch {
|
|
1058
1127
|
return !1;
|
|
1059
1128
|
}
|
|
1060
|
-
}),
|
|
1061
|
-
if (
|
|
1062
|
-
let e =
|
|
1063
|
-
|
|
1129
|
+
}), m = Object.keys(_.columnTypes), h = Math.max(1, t.copyBatchSize ?? 5e3), g = [], x = async () => {
|
|
1130
|
+
if (g.length === 0) return;
|
|
1131
|
+
let e = g;
|
|
1132
|
+
g = [];
|
|
1064
1133
|
try {
|
|
1065
1134
|
await t.copyLoader.copyInto({
|
|
1066
|
-
table:
|
|
1067
|
-
columns:
|
|
1068
|
-
columnTypes:
|
|
1135
|
+
table: _.name,
|
|
1136
|
+
columns: m,
|
|
1137
|
+
columnTypes: _.columnTypes,
|
|
1069
1138
|
rows: e
|
|
1070
1139
|
});
|
|
1071
1140
|
} catch (n) {
|
|
1072
|
-
|
|
1073
|
-
for (let t of e) await
|
|
1141
|
+
p = !1, t.onWarn?.(`bulk COPY into ${_.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1142
|
+
for (let t of e) await f(t);
|
|
1143
|
+
}
|
|
1144
|
+
}, S = Math.max(1, t.savepointBatchSize ?? 200), C = typeof e.savepoint == "function", E = [], D = async () => {
|
|
1145
|
+
if (E.length === 0) return;
|
|
1146
|
+
let t = E;
|
|
1147
|
+
E = [];
|
|
1148
|
+
try {
|
|
1149
|
+
await e.savepoint(async () => {
|
|
1150
|
+
for (let n of t) await v(e, _, n);
|
|
1151
|
+
});
|
|
1152
|
+
} catch {
|
|
1153
|
+
for (let e of t) await f(e);
|
|
1074
1154
|
}
|
|
1075
1155
|
};
|
|
1076
|
-
return yield* c.runForEach(kt(l,
|
|
1077
|
-
if (o++,
|
|
1078
|
-
|
|
1156
|
+
return yield* c.runForEach(kt(l, _.name, _.columnTypes), (e) => n.promise(async () => {
|
|
1157
|
+
if (o++, p) {
|
|
1158
|
+
g.push(e), g.length >= h && await x();
|
|
1079
1159
|
return;
|
|
1080
1160
|
}
|
|
1081
|
-
|
|
1082
|
-
|
|
1161
|
+
if (C) {
|
|
1162
|
+
E.push(e), E.length >= S && await D();
|
|
1163
|
+
return;
|
|
1164
|
+
}
|
|
1165
|
+
await f(e);
|
|
1166
|
+
})), yield* n.promise(D), yield* n.promise(x), {
|
|
1083
1167
|
rowCount: o,
|
|
1084
1168
|
checksum: i.digest("hex")
|
|
1085
1169
|
};
|
|
1086
|
-
}), { rowCount:
|
|
1170
|
+
}), { rowCount: C, checksum: A } = yield* ft("import", _.name, p, g, (e) => e.rowCount, S);
|
|
1087
1171
|
if (l) {
|
|
1088
|
-
if (
|
|
1089
|
-
what: `table ${
|
|
1090
|
-
expected:
|
|
1091
|
-
actual:
|
|
1172
|
+
if (A !== _.checksum) return yield* n.fail(new z({
|
|
1173
|
+
what: `table ${_.name} checksum`,
|
|
1174
|
+
expected: _.checksum,
|
|
1175
|
+
actual: A
|
|
1092
1176
|
}));
|
|
1093
|
-
if (
|
|
1094
|
-
what: `table ${
|
|
1095
|
-
expected: String(
|
|
1096
|
-
actual: String(
|
|
1177
|
+
if (C !== _.rowCount) return yield* n.fail(new z({
|
|
1178
|
+
what: `table ${_.name} rowCount`,
|
|
1179
|
+
expected: String(_.rowCount),
|
|
1180
|
+
actual: String(C)
|
|
1097
1181
|
}));
|
|
1098
1182
|
}
|
|
1099
1183
|
K(t.onProgress, {
|
|
1100
1184
|
phase: "import",
|
|
1101
|
-
table:
|
|
1185
|
+
table: _.name,
|
|
1102
1186
|
index: p,
|
|
1103
|
-
tableCount:
|
|
1187
|
+
tableCount: g,
|
|
1104
1188
|
event: "done",
|
|
1105
|
-
rowsDone:
|
|
1106
|
-
total:
|
|
1107
|
-
}), r && (a.length >
|
|
1108
|
-
name:
|
|
1109
|
-
rowCount:
|
|
1110
|
-
}) : (
|
|
1111
|
-
rowCount:
|
|
1112
|
-
checksum:
|
|
1113
|
-
}, yield* q(
|
|
1189
|
+
rowsDone: C,
|
|
1190
|
+
total: _.rowCount
|
|
1191
|
+
}), D++, E += C, r && (a.length > x ? o.push({
|
|
1192
|
+
name: _.name,
|
|
1193
|
+
rowCount: C
|
|
1194
|
+
}) : (h.tables[_.name] = {
|
|
1195
|
+
rowCount: C,
|
|
1196
|
+
checksum: _.checksum
|
|
1197
|
+
}, yield* q(m, h)));
|
|
1114
1198
|
}
|
|
1115
|
-
if (a.length > 0 && (yield*
|
|
1199
|
+
if (a.length > 0 && (yield* C(e, a)), r && o.length > 0) {
|
|
1116
1200
|
for (let e of o) {
|
|
1117
1201
|
let t = s.tables.find((t) => t.name === e.name);
|
|
1118
|
-
|
|
1202
|
+
h.tables[e.name] = {
|
|
1119
1203
|
rowCount: e.rowCount,
|
|
1120
1204
|
checksum: t.checksum
|
|
1121
1205
|
};
|
|
1122
1206
|
}
|
|
1123
|
-
yield* q(
|
|
1207
|
+
yield* q(m, h);
|
|
1124
1208
|
}
|
|
1125
|
-
if (
|
|
1126
|
-
let e =
|
|
1127
|
-
t.onWarn?.(`resume: ${
|
|
1209
|
+
if (f.length > 0) {
|
|
1210
|
+
let e = f.length === s.tables.length;
|
|
1211
|
+
t.onWarn?.(`resume: ${f.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${e ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${f.join(", ")}. Delete ${m} to force a full re-import.`);
|
|
1128
1212
|
}
|
|
1129
1213
|
});
|
|
1130
|
-
if (
|
|
1214
|
+
if (d ? yield* n.tryPromise({
|
|
1131
1215
|
try: () => t.store.transactional(async (t) => {
|
|
1132
|
-
let i = await n.runPromiseExit(
|
|
1216
|
+
let i = await n.runPromiseExit(A(t, !1));
|
|
1133
1217
|
if (r.isFailure(i)) throw Error(e.pretty(i.cause));
|
|
1134
1218
|
}),
|
|
1135
|
-
catch: (e) => new
|
|
1136
|
-
}) : yield*
|
|
1137
|
-
let e = t.assets, r = i.join(t.bundleDir,
|
|
1138
|
-
yield* c.runForEach(o, (t) => t.key in
|
|
1139
|
-
|
|
1140
|
-
})), n.andThen(q(
|
|
1219
|
+
catch: (e) => new R({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1220
|
+
}) : yield* A(t.store, !0), t.assets && s.assets.index && !t.assetsAlreadyRestored) {
|
|
1221
|
+
let e = t.assets, r = i.join(t.bundleDir, G), a = i.join(t.bundleDir, s.assets.index ?? "assets/index.ndjson"), o = w("bundle", () => j(a)).pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.map((e) => JSON.parse(e)));
|
|
1222
|
+
yield* c.runForEach(o, (t) => t.key in h.assets ? n.void : We(e, r, t).pipe(n.andThen(n.sync(() => {
|
|
1223
|
+
h.assets[t.key] = !0;
|
|
1224
|
+
})), n.andThen(q(m, h))));
|
|
1141
1225
|
}
|
|
1142
|
-
return
|
|
1143
|
-
|
|
1226
|
+
return {
|
|
1227
|
+
manifest: s,
|
|
1228
|
+
tablesWritten: D,
|
|
1229
|
+
rowsWritten: E,
|
|
1230
|
+
tablesSkipped: k,
|
|
1231
|
+
rowsSkipped: O,
|
|
1232
|
+
fullyResumed: D === 0 && k > 0
|
|
1233
|
+
};
|
|
1234
|
+
}), sn = (e) => e, cn = (e) => {
|
|
1144
1235
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1145
1236
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1146
1237
|
return {
|
|
1147
1238
|
refuse: !0,
|
|
1148
1239
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1149
1240
|
};
|
|
1150
|
-
}, Y = Buffer.from("VBUNDLE2"),
|
|
1241
|
+
}, Y = Buffer.from("VBUNDLE2"), ln = 0, un = 1, dn = 2, fn = 32, pn = async (e) => {
|
|
1151
1242
|
let t = [], n = async (r) => {
|
|
1152
1243
|
let i;
|
|
1153
1244
|
try {
|
|
1154
|
-
i = await
|
|
1245
|
+
i = await ae(M(e, r), { withFileTypes: !0 });
|
|
1155
1246
|
} catch {
|
|
1156
1247
|
return;
|
|
1157
1248
|
}
|
|
@@ -1161,21 +1252,21 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1161
1252
|
}
|
|
1162
1253
|
};
|
|
1163
1254
|
return await n(""), t.sort();
|
|
1164
|
-
},
|
|
1255
|
+
}, mn = (e, t) => {
|
|
1165
1256
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1166
|
-
return r.writeUInt8(
|
|
1167
|
-
},
|
|
1257
|
+
return r.writeUInt8(un, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1258
|
+
}, hn = (e, t, n) => {
|
|
1168
1259
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1169
|
-
return a.writeUInt8(
|
|
1260
|
+
return a.writeUInt8(dn, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1170
1261
|
};
|
|
1171
|
-
async function*
|
|
1262
|
+
async function* gn(e) {
|
|
1172
1263
|
yield new Uint8Array(Y);
|
|
1173
|
-
for (let t of await
|
|
1174
|
-
let n =
|
|
1175
|
-
if (yield new Uint8Array(
|
|
1264
|
+
for (let t of await pn(e.dir)) {
|
|
1265
|
+
let n = M(e.dir, t), r = await oe(n);
|
|
1266
|
+
if (yield new Uint8Array(mn(t, r.size)), r.size > 0) for await (let e of j(n)) yield new Uint8Array(e);
|
|
1176
1267
|
}
|
|
1177
1268
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1178
|
-
yield new Uint8Array(
|
|
1269
|
+
yield new Uint8Array(hn(t.key, t.contentType, t.size));
|
|
1179
1270
|
let n = y("sha256"), r = 0;
|
|
1180
1271
|
for await (let i of e.blobs.open(t.key)) {
|
|
1181
1272
|
let e = Buffer.from(i);
|
|
@@ -1184,11 +1275,11 @@ async function* fn(e) {
|
|
|
1184
1275
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1185
1276
|
yield new Uint8Array(n.digest());
|
|
1186
1277
|
}
|
|
1187
|
-
yield new Uint8Array(Buffer.from([
|
|
1278
|
+
yield new Uint8Array(Buffer.from([ln]));
|
|
1188
1279
|
}
|
|
1189
|
-
var
|
|
1190
|
-
let n =
|
|
1191
|
-
await
|
|
1280
|
+
var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn = (e) => new Promise((t) => e.end(t)), yn = async (e, t) => {
|
|
1281
|
+
let n = ue(t.dir);
|
|
1282
|
+
await ie(n, { recursive: !0 });
|
|
1192
1283
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1193
1284
|
let { value: e, done: t } = await i.next();
|
|
1194
1285
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1203,24 +1294,24 @@ var pn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), mn
|
|
|
1203
1294
|
for (;;) {
|
|
1204
1295
|
await s(1);
|
|
1205
1296
|
let e = c(1).readUInt8(0);
|
|
1206
|
-
if (e ===
|
|
1207
|
-
if (e ===
|
|
1297
|
+
if (e === ln) return;
|
|
1298
|
+
if (e === un) {
|
|
1208
1299
|
await s(4);
|
|
1209
1300
|
let e = c(4).readUInt32BE(0);
|
|
1210
1301
|
await s(e);
|
|
1211
1302
|
let t = c(e).toString("utf8");
|
|
1212
1303
|
await s(8);
|
|
1213
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1214
|
-
if (a !== n && !a.startsWith(n +
|
|
1215
|
-
await
|
|
1216
|
-
let o =
|
|
1304
|
+
let i = c(8).readBigUInt64BE(0), a = ue(n, t);
|
|
1305
|
+
if (a !== n && !a.startsWith(n + de)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1306
|
+
await ie(le(a), { recursive: !0 });
|
|
1307
|
+
let o = te(a);
|
|
1217
1308
|
for (; i > 0n;) {
|
|
1218
1309
|
r.length === 0 && await s(1);
|
|
1219
1310
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1220
|
-
await
|
|
1311
|
+
await _n(o, c(e)), i -= BigInt(e);
|
|
1221
1312
|
}
|
|
1222
|
-
await
|
|
1223
|
-
} else if (e ===
|
|
1313
|
+
await vn(o);
|
|
1314
|
+
} else if (e === dn) {
|
|
1224
1315
|
await s(4);
|
|
1225
1316
|
let e = c(4).readUInt32BE(0);
|
|
1226
1317
|
await s(e);
|
|
@@ -1230,24 +1321,24 @@ var pn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), mn
|
|
|
1230
1321
|
await s(i);
|
|
1231
1322
|
let a = c(i).toString("utf8");
|
|
1232
1323
|
await s(8);
|
|
1233
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = y("sha256"), d = new
|
|
1324
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = y("sha256"), d = new k(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1234
1325
|
contentType: a,
|
|
1235
1326
|
size: Number(o)
|
|
1236
1327
|
}), p = o;
|
|
1237
1328
|
for (; p > 0n;) {
|
|
1238
1329
|
r.length === 0 && await s(1);
|
|
1239
1330
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1240
|
-
u.update(t), l || await
|
|
1331
|
+
u.update(t), l || await _n(d, t), p -= BigInt(e);
|
|
1241
1332
|
}
|
|
1242
|
-
l ? d.destroy() : await
|
|
1243
|
-
let m = c(
|
|
1333
|
+
l ? d.destroy() : await vn(d), await f, await s(fn);
|
|
1334
|
+
let m = c(fn);
|
|
1244
1335
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1245
1336
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1246
1337
|
}
|
|
1247
1338
|
} finally {
|
|
1248
1339
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1249
1340
|
}
|
|
1250
|
-
},
|
|
1341
|
+
}, bn = async (e, t) => {
|
|
1251
1342
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1252
1343
|
let { value: e, done: t } = await a.next();
|
|
1253
1344
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1269,8 +1360,8 @@ var pn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), mn
|
|
|
1269
1360
|
if (!await s(Y.length) || !c(Y.length).equals(Y)) throw Error("not a bundle archive (bad magic)");
|
|
1270
1361
|
for (; await s(1);) {
|
|
1271
1362
|
let e = c(1).readUInt8(0);
|
|
1272
|
-
if (e ===
|
|
1273
|
-
if (e !==
|
|
1363
|
+
if (e === ln || e === dn) break;
|
|
1364
|
+
if (e !== un) throw Error(`unknown bundle entry kind ${e}`);
|
|
1274
1365
|
if (!await s(4)) break;
|
|
1275
1366
|
let t = c(4).readUInt32BE(0);
|
|
1276
1367
|
if (!await s(t)) break;
|
|
@@ -1280,22 +1371,22 @@ var pn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), mn
|
|
|
1280
1371
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1281
1372
|
}
|
|
1282
1373
|
return r;
|
|
1283
|
-
},
|
|
1374
|
+
}, xn = (e, t) => yn(e, {
|
|
1284
1375
|
dir: t,
|
|
1285
|
-
blobs:
|
|
1286
|
-
}),
|
|
1376
|
+
blobs: Sn(t)
|
|
1377
|
+
}), Sn = (e) => {
|
|
1287
1378
|
let t = /* @__PURE__ */ new Set();
|
|
1288
1379
|
return {
|
|
1289
1380
|
has: async (e) => t.has(e),
|
|
1290
1381
|
write: async (n, r, i) => {
|
|
1291
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1292
|
-
await
|
|
1382
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = M(e, "assets");
|
|
1383
|
+
await ie(c, { recursive: !0 });
|
|
1293
1384
|
let l = y("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1294
1385
|
l.update(e), u += e.length, n(null, e);
|
|
1295
|
-
} }), f =
|
|
1296
|
-
await o(r, d,
|
|
1386
|
+
} }), f = M(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1387
|
+
await o(r, d, te(f));
|
|
1297
1388
|
let p = l.digest("hex");
|
|
1298
|
-
await a.rename(f,
|
|
1389
|
+
await a.rename(f, M(c, p)), await a.appendFile(M(c, "index.ndjson"), `${JSON.stringify({
|
|
1299
1390
|
key: n,
|
|
1300
1391
|
sha256: p,
|
|
1301
1392
|
size: u,
|
|
@@ -1303,33 +1394,33 @@ var pn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), mn
|
|
|
1303
1394
|
})}\n`), t.add(n);
|
|
1304
1395
|
}
|
|
1305
1396
|
};
|
|
1306
|
-
},
|
|
1397
|
+
}, Cn = fe(S), wn = Buffer.from("VENC1\0\0\0"), Tn = 32, En = 16, Dn = 7, X = 16, On = 262144, Z = {
|
|
1307
1398
|
N: 32768,
|
|
1308
1399
|
r: 8,
|
|
1309
1400
|
p: 1
|
|
1310
|
-
},
|
|
1401
|
+
}, kn = 134217728, Q = 43, An = (e, t, n) => {
|
|
1311
1402
|
let r = Buffer.alloc(12);
|
|
1312
|
-
return e.copy(r, 0, 0,
|
|
1403
|
+
return e.copy(r, 0, 0, Dn), r.writeUInt32BE(t >>> 0, Dn), r.writeUInt8(+!!n, 11), r;
|
|
1313
1404
|
};
|
|
1314
|
-
async function*
|
|
1405
|
+
async function* jn(e, t) {
|
|
1315
1406
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1316
|
-
let n = x(
|
|
1407
|
+
let n = x(En), r = x(Dn), i = await Cn(t, n, Tn, {
|
|
1317
1408
|
...Z,
|
|
1318
|
-
maxmem:
|
|
1409
|
+
maxmem: kn
|
|
1319
1410
|
}), a = Buffer.alloc(Q);
|
|
1320
|
-
|
|
1411
|
+
wn.copy(a, 0), a.writeUInt32BE(Z.N, 8), a.writeUInt32BE(Z.r, 12), a.writeUInt32BE(Z.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1321
1412
|
let o = 0, s = (e, t) => {
|
|
1322
|
-
let n = _("aes-256-gcm", i,
|
|
1413
|
+
let n = _("aes-256-gcm", i, An(r, o, t));
|
|
1323
1414
|
o === 0 && n.setAAD(a);
|
|
1324
1415
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1325
1416
|
o++;
|
|
1326
1417
|
let l = Buffer.alloc(5 + s.length + X);
|
|
1327
1418
|
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + X, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1328
1419
|
}, c = Buffer.alloc(0);
|
|
1329
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1420
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > On;) yield s(c.subarray(0, On), !1), c = c.subarray(On);
|
|
1330
1421
|
yield s(c, !0);
|
|
1331
1422
|
}
|
|
1332
|
-
async function*
|
|
1423
|
+
async function* Mn(e, t) {
|
|
1333
1424
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1334
1425
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1335
1426
|
let { value: e, done: t } = await n.next();
|
|
@@ -1340,12 +1431,12 @@ async function* On(e, t) {
|
|
|
1340
1431
|
};
|
|
1341
1432
|
if (!await o(Q)) throw Error("encrypted bundle truncated (no header)");
|
|
1342
1433
|
let s = r.subarray(0, Q);
|
|
1343
|
-
if (!s.subarray(0, 8).equals(
|
|
1344
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await
|
|
1434
|
+
if (!s.subarray(0, 8).equals(wn)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1435
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await Cn(t, d, Tn, {
|
|
1345
1436
|
N: c,
|
|
1346
1437
|
r: l,
|
|
1347
1438
|
p: u,
|
|
1348
|
-
maxmem:
|
|
1439
|
+
maxmem: kn
|
|
1349
1440
|
});
|
|
1350
1441
|
r = r.subarray(Q);
|
|
1351
1442
|
let m = 0, h = !1;
|
|
@@ -1360,7 +1451,7 @@ async function* On(e, t) {
|
|
|
1360
1451
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1361
1452
|
let n = r.subarray(5, 5 + t - X), i = r.subarray(5 + t - X, 5 + t);
|
|
1362
1453
|
r = r.subarray(5 + t);
|
|
1363
|
-
let a = v("aes-256-gcm", p,
|
|
1454
|
+
let a = v("aes-256-gcm", p, An(f, m, e));
|
|
1364
1455
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1365
1456
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1366
1457
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1368,7 +1459,7 @@ async function* On(e, t) {
|
|
|
1368
1459
|
}
|
|
1369
1460
|
//#endregion
|
|
1370
1461
|
//#region src/blobStorage.ts
|
|
1371
|
-
var
|
|
1462
|
+
var Nn = (e, t) => ({
|
|
1372
1463
|
list: () => {
|
|
1373
1464
|
let e = t();
|
|
1374
1465
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1381,29 +1472,29 @@ var kn = (e, t) => ({
|
|
|
1381
1472
|
} };
|
|
1382
1473
|
},
|
|
1383
1474
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1384
|
-
let r = await n.runPromise(
|
|
1475
|
+
let r = await n.runPromise(T(e, t)), i = await D(r.stream);
|
|
1385
1476
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1386
1477
|
} })
|
|
1387
|
-
}),
|
|
1478
|
+
}), Pn = (e, t) => Nn(e, () => h(t, {
|
|
1388
1479
|
table: C,
|
|
1389
1480
|
chunkSize: 500
|
|
1390
1481
|
}).pipe(c.map((e) => ({
|
|
1391
1482
|
key: String(e.key),
|
|
1392
1483
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1393
1484
|
size: Number(e.size ?? 0)
|
|
1394
|
-
})))),
|
|
1485
|
+
})))), Fn = (e) => ({
|
|
1395
1486
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1396
1487
|
write: async (t, r, i) => {
|
|
1397
|
-
let a = w("storage", () =>
|
|
1398
|
-
await n.runPromise(
|
|
1488
|
+
let a = w("storage", () => A.from(r));
|
|
1489
|
+
await n.runPromise(E(e, t, a, {
|
|
1399
1490
|
contentType: i.contentType,
|
|
1400
1491
|
totalSize: i.size
|
|
1401
1492
|
}));
|
|
1402
1493
|
}
|
|
1403
|
-
}),
|
|
1494
|
+
}), In = (e, t) => {
|
|
1404
1495
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1405
|
-
return e === "mariadb" &&
|
|
1406
|
-
},
|
|
1496
|
+
return e === "mariadb" && Ln(n) ? n : r;
|
|
1497
|
+
}, Ln = (e) => (process.env.PATH ?? "").split(ce).filter((e) => e !== "").some((t) => ne(M(t, e))), Rn = (e) => {
|
|
1407
1498
|
switch (e) {
|
|
1408
1499
|
case "postgres": return "db.dump";
|
|
1409
1500
|
case "mysql":
|
|
@@ -1413,7 +1504,7 @@ var kn = (e, t) => ({
|
|
|
1413
1504
|
case "mssql": return "db.bacpac";
|
|
1414
1505
|
default: return "db.dump";
|
|
1415
1506
|
}
|
|
1416
|
-
},
|
|
1507
|
+
}, zn = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1417
1508
|
if (e.host || e.database) return {
|
|
1418
1509
|
host: e.host ?? "localhost",
|
|
1419
1510
|
port: e.port ?? 3306,
|
|
@@ -1429,7 +1520,7 @@ var kn = (e, t) => ({
|
|
|
1429
1520
|
password: decodeURIComponent(t.password) || "",
|
|
1430
1521
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1431
1522
|
};
|
|
1432
|
-
},
|
|
1523
|
+
}, Bn = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Vn = (e, t, n) => {
|
|
1433
1524
|
switch (e) {
|
|
1434
1525
|
case "postgres": {
|
|
1435
1526
|
if (t.url) return {
|
|
@@ -1476,7 +1567,7 @@ var kn = (e, t) => ({
|
|
|
1476
1567
|
let r = $(t);
|
|
1477
1568
|
return {
|
|
1478
1569
|
kind: "spawn",
|
|
1479
|
-
tool:
|
|
1570
|
+
tool: In(e, "dump"),
|
|
1480
1571
|
args: [
|
|
1481
1572
|
"--single-transaction",
|
|
1482
1573
|
"--routines",
|
|
@@ -1498,7 +1589,7 @@ var kn = (e, t) => ({
|
|
|
1498
1589
|
case "sqlite":
|
|
1499
1590
|
case "turso": return {
|
|
1500
1591
|
kind: "copy",
|
|
1501
|
-
from:
|
|
1592
|
+
from: zn(t),
|
|
1502
1593
|
to: n
|
|
1503
1594
|
};
|
|
1504
1595
|
case "mssql": return {
|
|
@@ -1506,7 +1597,7 @@ var kn = (e, t) => ({
|
|
|
1506
1597
|
tool: "sqlpackage",
|
|
1507
1598
|
args: [
|
|
1508
1599
|
"/Action:Export",
|
|
1509
|
-
`/SourceConnectionString:${
|
|
1600
|
+
`/SourceConnectionString:${Bn($({
|
|
1510
1601
|
...t,
|
|
1511
1602
|
port: t.port ?? 1433
|
|
1512
1603
|
}))}`,
|
|
@@ -1514,12 +1605,12 @@ var kn = (e, t) => ({
|
|
|
1514
1605
|
],
|
|
1515
1606
|
env: {}
|
|
1516
1607
|
};
|
|
1517
|
-
default: throw new
|
|
1608
|
+
default: throw new V({
|
|
1518
1609
|
tool: "backup",
|
|
1519
1610
|
reason: `no native backup for dialect '${e}'`
|
|
1520
1611
|
});
|
|
1521
1612
|
}
|
|
1522
|
-
},
|
|
1613
|
+
}, Hn = (e, t, n) => {
|
|
1523
1614
|
switch (e) {
|
|
1524
1615
|
case "postgres": {
|
|
1525
1616
|
if (t.url) return {
|
|
@@ -1564,7 +1655,7 @@ var kn = (e, t) => ({
|
|
|
1564
1655
|
let r = $(t);
|
|
1565
1656
|
return {
|
|
1566
1657
|
kind: "spawn",
|
|
1567
|
-
tool:
|
|
1658
|
+
tool: In(e, "client"),
|
|
1568
1659
|
args: [
|
|
1569
1660
|
"--protocol=TCP",
|
|
1570
1661
|
"-h",
|
|
@@ -1583,14 +1674,14 @@ var kn = (e, t) => ({
|
|
|
1583
1674
|
case "turso": return {
|
|
1584
1675
|
kind: "copy",
|
|
1585
1676
|
from: n,
|
|
1586
|
-
to:
|
|
1677
|
+
to: zn(t)
|
|
1587
1678
|
};
|
|
1588
1679
|
case "mssql": return {
|
|
1589
1680
|
kind: "spawn",
|
|
1590
1681
|
tool: "sqlpackage",
|
|
1591
1682
|
args: [
|
|
1592
1683
|
"/Action:Import",
|
|
1593
|
-
`/TargetConnectionString:${
|
|
1684
|
+
`/TargetConnectionString:${Bn($({
|
|
1594
1685
|
...t,
|
|
1595
1686
|
port: t.port ?? 1433
|
|
1596
1687
|
}))}`,
|
|
@@ -1598,18 +1689,18 @@ var kn = (e, t) => ({
|
|
|
1598
1689
|
],
|
|
1599
1690
|
env: {}
|
|
1600
1691
|
};
|
|
1601
|
-
default: throw new
|
|
1692
|
+
default: throw new V({
|
|
1602
1693
|
tool: "restore",
|
|
1603
1694
|
reason: `no native restore for dialect '${e}'`
|
|
1604
1695
|
});
|
|
1605
1696
|
}
|
|
1606
|
-
},
|
|
1607
|
-
|
|
1697
|
+
}, Un = (e) => e.kind === "copy" ? n.async((t) => {
|
|
1698
|
+
ee(e.from, e.to, (r) => t(r ? n.fail(new V({
|
|
1608
1699
|
tool: "copy",
|
|
1609
1700
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1610
1701
|
})) : n.void));
|
|
1611
1702
|
}) : n.async((t) => {
|
|
1612
|
-
let r =
|
|
1703
|
+
let r = pe(e.tool, [...e.args], {
|
|
1613
1704
|
env: {
|
|
1614
1705
|
...process.env,
|
|
1615
1706
|
...e.env
|
|
@@ -1622,14 +1713,14 @@ var kn = (e, t) => ({
|
|
|
1622
1713
|
}), i = "";
|
|
1623
1714
|
r.stderr?.on("data", (e) => {
|
|
1624
1715
|
i += String(e);
|
|
1625
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(
|
|
1716
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(te(e.stdoutFile)), e.stdinFile && r.stdin && j(e.stdinFile).pipe(r.stdin);
|
|
1626
1717
|
let a = () => {
|
|
1627
1718
|
if (e.stdoutFile !== void 0) try {
|
|
1628
|
-
|
|
1719
|
+
re(e.stdoutFile);
|
|
1629
1720
|
} catch {}
|
|
1630
1721
|
};
|
|
1631
1722
|
r.on("error", (r) => {
|
|
1632
|
-
a(), t(n.fail(new
|
|
1723
|
+
a(), t(n.fail(new V({
|
|
1633
1724
|
tool: e.tool,
|
|
1634
1725
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`
|
|
1635
1726
|
})));
|
|
@@ -1638,17 +1729,17 @@ var kn = (e, t) => ({
|
|
|
1638
1729
|
t(n.void);
|
|
1639
1730
|
return;
|
|
1640
1731
|
}
|
|
1641
|
-
a(), t(n.fail(new
|
|
1732
|
+
a(), t(n.fail(new V({
|
|
1642
1733
|
tool: e.tool,
|
|
1643
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
1734
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${se(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
1644
1735
|
stderr: i.slice(-2e3)
|
|
1645
1736
|
})));
|
|
1646
1737
|
});
|
|
1647
|
-
}),
|
|
1648
|
-
let t = yield*
|
|
1738
|
+
}), Wn = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Gn = (e) => n.gen(function* () {
|
|
1739
|
+
let t = yield* Be(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
1649
1740
|
store: e.store,
|
|
1650
1741
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
1651
|
-
}), r = At(t.tables, e.snapshot), i = e.classification ??
|
|
1742
|
+
}), r = At(t.tables, e.snapshot), i = e.classification ?? N(e.snapshot), a = e.sampleSize ?? 20, o = Ee(r.map((t) => ({
|
|
1652
1743
|
name: t,
|
|
1653
1744
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
1654
1745
|
name: e.name,
|
|
@@ -1660,15 +1751,15 @@ var kn = (e, t) => ({
|
|
|
1660
1751
|
table: i,
|
|
1661
1752
|
chunkSize: a,
|
|
1662
1753
|
...r === void 0 ? {} : { where: r }
|
|
1663
|
-
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
1664
|
-
l.push(...
|
|
1754
|
+
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) => ke(t, i, o.actions, e.masking.seed));
|
|
1755
|
+
l.push(...Oe(d, i, o.actions));
|
|
1665
1756
|
let f = u[0], p = d[0], m = [];
|
|
1666
1757
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
1667
1758
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
1668
1759
|
let n = e.slice(i.length + 1);
|
|
1669
1760
|
m.push({
|
|
1670
1761
|
column: n,
|
|
1671
|
-
action:
|
|
1762
|
+
action: Wn(t),
|
|
1672
1763
|
before: f[n],
|
|
1673
1764
|
after: p[n]
|
|
1674
1765
|
});
|
|
@@ -1683,7 +1774,7 @@ var kn = (e, t) => ({
|
|
|
1683
1774
|
leaks: l,
|
|
1684
1775
|
unclassified: o.unclassified
|
|
1685
1776
|
};
|
|
1686
|
-
}),
|
|
1777
|
+
}), Kn = (e, t) => {
|
|
1687
1778
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
1688
1779
|
for (let t of e) {
|
|
1689
1780
|
let e = n.get(t.name);
|
|
@@ -1738,7 +1829,7 @@ var kn = (e, t) => ({
|
|
|
1738
1829
|
skipColumns: i,
|
|
1739
1830
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
1740
1831
|
};
|
|
1741
|
-
},
|
|
1832
|
+
}, qn = (e) => {
|
|
1742
1833
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
1743
1834
|
let t = {
|
|
1744
1835
|
safe: "✓",
|
|
@@ -1749,4 +1840,4 @@ var kn = (e, t) => ({
|
|
|
1749
1840
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
1750
1841
|
};
|
|
1751
1842
|
//#endregion
|
|
1752
|
-
export {
|
|
1843
|
+
export { G as ASSETS_DIR, at as ASSET_INDEX_FILE, R as BundleError, Ae as CodecError, je as CompressionError, Ie as CrossDialectError, it as DATA_DIR, he as DEFAULT_CLASS_ACTIONS, Xe as FORMAT_VERSION, Fe as ImportModeError, z as IntegrityError, ot as LEDGER_FILE, W as MANIFEST_FILE, Pe as MaskingError, V as NativeToolError, Me as RowsRefusedError, rn as SAVEPOINT_BATCH_SIZE, Ne as SchemaDriftError, B as ScopeError, we as applyAction, cn as assessLiveImport, gt as assetDone, Rn as backupArtifactName, Vn as backupCommand, Wt as canonicalType, qt as checkPortability, nn as checkSchemaDrift, N as classificationFromSnapshot, Kn as classifyImportDrift, qe as compressionExt, Re as computeSubsetIds, nt as decodeManifest, Et as decodeRow, wt as decodeValue, Mn as decryptStream, sn as defineDataProfile, tn as diffSnapshots, Sn as dirBlobSink, pt as emptyLedger, tt as encodeManifest, Tt as encodeRow, St as encodeValue, jn as encryptStream, qn as formatImportDrift, Jt as formatIssue, Ge as hasZstd, Je as makeCompressor, Ye as makeDecompressor, ke as maskRow, kt as ndjsonToRows, Ut as outsideReferenceRefusal, Ht as outsideReferencesInto, gn as packBundle, bn as peekBundle, Ke as pickCompression, Ee as planMasking, Gn as previewMasking, mt as readLedger, ge as resolveAction, Be as resolveScope, We as restoreAssetFromCas, Hn as restoreCommand, Ot as rowsToNdjson, Vt as runExport, on as runImport, Un as runNativeStep, Oe as scanForLeaks, He as storageAssetSink, Ve as storageAssetSource, Fn as storageBlobSink, Nn as storageBlobSource, Pn as storageRefsBlobSource, ze as subsetToScope, ht as tableDone, At as topoSortTables, yn as unpackBundle, xn as unpackBundleToDir, Ue as writeAssetToCas, q as writeLedger };
|