@voltro/data-transfer 0.41.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +97 -0
- package/dist/index.d.ts +65 -7
- package/dist/index.js +407 -345
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Schedule as o, Schema as s, Stream as c } from "effect";
|
|
2
|
-
import {
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { STORAGE_REFS_TABLE as
|
|
2
|
+
import { classifyConstraintViolation as l, eq as u, extractDbCause as d, inSet as f, isNotNull as p, or as m, streamTable as h } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as g } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as _, createDecipheriv as v, createHash as y, createHmac as ee, randomBytes as b, scrypt as x } from "node:crypto";
|
|
5
|
+
import { STORAGE_REFS_TABLE as S, fromNodeReadable as C, getObjectStream as te, putObjectStream as ne, toNodeReadable as re } from "@voltro/plugin-storage";
|
|
6
6
|
import * as w from "node:zlib";
|
|
7
|
-
import { PassThrough as
|
|
8
|
-
import { copyFile as
|
|
9
|
-
import { mkdir as
|
|
10
|
-
import { basename as
|
|
11
|
-
import { promisify as
|
|
12
|
-
import { spawn as
|
|
7
|
+
import { PassThrough as ie, Readable as ae } from "node:stream";
|
|
8
|
+
import { copyFile as oe, createReadStream as T, createWriteStream as E, existsSync as se, unlinkSync as ce } from "node:fs";
|
|
9
|
+
import { mkdir as D, readdir as le, stat as ue } from "node:fs/promises";
|
|
10
|
+
import { basename as de, delimiter as fe, dirname as pe, join as O, resolve as me, sep as he } from "node:path";
|
|
11
|
+
import { promisify as ge } from "node:util";
|
|
12
|
+
import { spawn as _e } from "node:child_process";
|
|
13
13
|
//#region src/consistency.ts
|
|
14
|
-
var
|
|
14
|
+
var ve = (e) => {
|
|
15
15
|
switch (e) {
|
|
16
16
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
17
17
|
case "mysql":
|
|
@@ -21,7 +21,7 @@ var ge = (e) => {
|
|
|
21
21
|
case "turso": return [];
|
|
22
22
|
default: return [];
|
|
23
23
|
}
|
|
24
|
-
},
|
|
24
|
+
}, ye = {
|
|
25
25
|
email: { fake: "email" },
|
|
26
26
|
fullName: { fake: "fullName" },
|
|
27
27
|
firstName: { fake: "firstName" },
|
|
@@ -36,7 +36,7 @@ var ge = (e) => {
|
|
|
36
36
|
date: "dateShift",
|
|
37
37
|
secret: "null",
|
|
38
38
|
freeText: "redact"
|
|
39
|
-
},
|
|
39
|
+
}, k = (e) => {
|
|
40
40
|
let t = {};
|
|
41
41
|
for (let n of e.tables) {
|
|
42
42
|
let e = {};
|
|
@@ -48,15 +48,15 @@ var ge = (e) => {
|
|
|
48
48
|
t[n.name] = e;
|
|
49
49
|
}
|
|
50
50
|
return t;
|
|
51
|
-
},
|
|
51
|
+
}, be = (e, t, n, r) => {
|
|
52
52
|
let i = r.columns?.[`${e}.${t}`];
|
|
53
53
|
if (i) return i;
|
|
54
54
|
if (n?.sensitive) {
|
|
55
55
|
let e = n.sensitive.class;
|
|
56
|
-
return r.classes?.[e] ??
|
|
56
|
+
return r.classes?.[e] ?? ye[e] ?? "redact";
|
|
57
57
|
}
|
|
58
58
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
59
|
-
},
|
|
59
|
+
}, A = (e, t) => ee("sha256", e).update(t == null ? "∅" : String(t)).digest(), j = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), M = [
|
|
60
60
|
"Ada",
|
|
61
61
|
"Alan",
|
|
62
62
|
"Grace",
|
|
@@ -100,19 +100,19 @@ var ge = (e) => {
|
|
|
100
100
|
"Soylent",
|
|
101
101
|
"Stark",
|
|
102
102
|
"Wayne"
|
|
103
|
-
],
|
|
104
|
-
let r =
|
|
103
|
+
], N = (e, t, n) => e[j(t, n) % e.length], we = (e, t, n) => {
|
|
104
|
+
let r = A(n, `${e}:${String(t)}`), i = j(r, 1) % 1e4;
|
|
105
105
|
switch (e) {
|
|
106
|
-
case "email": return `${
|
|
107
|
-
case "fullName": return `${
|
|
108
|
-
case "firstName": return
|
|
109
|
-
case "lastName": return
|
|
110
|
-
case "username": return `${
|
|
111
|
-
case "phone": return `+1${String(
|
|
112
|
-
case "address": return `${
|
|
113
|
-
case "company": return
|
|
106
|
+
case "email": return `${N(M, r, 0).toLowerCase()}.${N(xe, r, 2).toLowerCase()}${i}@example.com`;
|
|
107
|
+
case "fullName": return `${N(M, r, 0)} ${N(xe, r, 1)}`;
|
|
108
|
+
case "firstName": return N(M, r, 0);
|
|
109
|
+
case "lastName": return N(xe, r, 1);
|
|
110
|
+
case "username": return `${N(M, r, 0).toLowerCase()}${i}`;
|
|
111
|
+
case "phone": return `+1${String(j(r, 0) % 1e10).padStart(10, "0")}`;
|
|
112
|
+
case "address": return `${j(r, 0) % 9899 + 100} ${N(Se, r, 1)} St`;
|
|
113
|
+
case "company": return N(Ce, r, 0);
|
|
114
114
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
115
|
-
case "ip": return `${
|
|
115
|
+
case "ip": return `${j(r, 0) % 256}.${j(r, 1) % 256}.${j(r, 2) % 256}.${j(r, 3) % 254 + 1}`;
|
|
116
116
|
case "creditCard": return Te(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
117
117
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
118
118
|
}
|
|
@@ -127,23 +127,42 @@ var ge = (e) => {
|
|
|
127
127
|
}, Ee = 864e5, De = (e, t) => {
|
|
128
128
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
129
129
|
if (Number.isNaN(n)) return e;
|
|
130
|
-
let r =
|
|
130
|
+
let r = j(A(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Ee);
|
|
131
131
|
return e instanceof Date ? i : i.toISOString();
|
|
132
132
|
}, Oe = (e, t) => {
|
|
133
133
|
if (t.value === null || t.value === void 0) return null;
|
|
134
134
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
135
|
+
if (e === "keep") return t.value;
|
|
136
|
+
if (e === "null") return null;
|
|
137
|
+
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
138
|
+
if (e === "hash") return A(t.seed, t.value).toString("hex").slice(0, 16);
|
|
139
|
+
if (e === "dateShift") return De(t.value, t.seed);
|
|
140
|
+
if (typeof e == "object" && "fake" in e) return we(e.fake, t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
142
|
+
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
143
|
+
}, ke = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Ae = (e, t, n) => {
|
|
144
|
+
let r = {}, i = [], a = [];
|
|
145
|
+
for (let o of e) for (let e of o.columns) {
|
|
146
|
+
let s = `${o.name}.${e.name}`, c = be(o.name, e.name, t[o.name]?.[e.name], n);
|
|
147
|
+
if (c === "error") {
|
|
148
|
+
i.push(s);
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
if (!ke(c)) {
|
|
152
|
+
a.push({
|
|
153
|
+
column: s,
|
|
154
|
+
action: c
|
|
155
|
+
});
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
r[s] = c;
|
|
141
159
|
}
|
|
142
160
|
return {
|
|
143
161
|
actions: r,
|
|
144
|
-
unclassified: i
|
|
162
|
+
unclassified: i,
|
|
163
|
+
invalid: a
|
|
145
164
|
};
|
|
146
|
-
},
|
|
165
|
+
}, je = [
|
|
147
166
|
{
|
|
148
167
|
name: "email",
|
|
149
168
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -160,7 +179,7 @@ var ge = (e) => {
|
|
|
160
179
|
name: "creditCard",
|
|
161
180
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
162
181
|
}
|
|
163
|
-
],
|
|
182
|
+
], Me = (e, t, n) => {
|
|
164
183
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
165
184
|
for (let [a, o] of Object.entries(n)) {
|
|
166
185
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -168,7 +187,7 @@ var ge = (e) => {
|
|
|
168
187
|
for (let a of e) {
|
|
169
188
|
let e = a[n];
|
|
170
189
|
if (typeof e != "string" || e.length === 0) continue;
|
|
171
|
-
let o =
|
|
190
|
+
let o = je.find((t) => t.re.test(e));
|
|
172
191
|
if (o && !i.has(n)) {
|
|
173
192
|
i.add(n), r.push({
|
|
174
193
|
table: t,
|
|
@@ -181,7 +200,7 @@ var ge = (e) => {
|
|
|
181
200
|
}
|
|
182
201
|
}
|
|
183
202
|
return r;
|
|
184
|
-
},
|
|
203
|
+
}, Ne = (e, t, n, r, i) => {
|
|
185
204
|
let a = {};
|
|
186
205
|
for (let [o, s] of Object.entries(e)) a[o] = Oe(n[`${t}.${o}`] ?? "keep", {
|
|
187
206
|
value: s,
|
|
@@ -194,21 +213,21 @@ var ge = (e) => {
|
|
|
194
213
|
}, P = class extends s.TaggedError()("BundleError", {
|
|
195
214
|
reason: s.String,
|
|
196
215
|
path: s.optional(s.String)
|
|
197
|
-
}) {},
|
|
216
|
+
}) {}, Pe = class extends s.TaggedError()("CodecError", {
|
|
198
217
|
table: s.String,
|
|
199
218
|
reason: s.String
|
|
200
219
|
}) {}, F = class extends s.TaggedError()("IntegrityError", {
|
|
201
220
|
what: s.String,
|
|
202
221
|
expected: s.String,
|
|
203
222
|
actual: s.String
|
|
204
|
-
}) {},
|
|
223
|
+
}) {}, Fe = class extends t.TaggedError("CompressionError") {}, I = class extends s.TaggedError()("ScopeError", { reason: s.String }) {}, Ie = class extends s.TaggedError()("DanglingReferenceError", {
|
|
205
224
|
rows: s.Array(s.Struct({
|
|
206
225
|
table: s.String,
|
|
207
226
|
id: s.String,
|
|
208
227
|
reason: s.String
|
|
209
228
|
})),
|
|
210
229
|
totalCount: s.Number
|
|
211
|
-
}) {},
|
|
230
|
+
}) {}, Le = class extends s.TaggedError()("SchemaDriftError", {
|
|
212
231
|
bundleFingerprint: s.String,
|
|
213
232
|
targetFingerprint: s.String,
|
|
214
233
|
diff: s.Array(s.String)
|
|
@@ -217,36 +236,39 @@ var ge = (e) => {
|
|
|
217
236
|
let e = this.stderr?.trim();
|
|
218
237
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
219
238
|
}
|
|
220
|
-
},
|
|
239
|
+
}, Re = class extends s.TaggedError()("MaskingError", {
|
|
240
|
+
unclassified: s.Array(s.String),
|
|
241
|
+
invalidActions: s.optional(s.Array(s.String))
|
|
242
|
+
}) {}, ze = class extends s.TaggedError()("ImportModeError", {
|
|
221
243
|
mode: s.String,
|
|
222
244
|
reason: s.String
|
|
223
|
-
}) {},
|
|
245
|
+
}) {}, Be = class extends s.TaggedError()("CrossDialectError", {
|
|
224
246
|
sourceDialect: s.String,
|
|
225
247
|
targetDialect: s.String,
|
|
226
248
|
blocking: s.Array(s.String)
|
|
227
|
-
}) {}, R = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id",
|
|
249
|
+
}) {}, R = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Ve = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
228
250
|
col: e.name,
|
|
229
251
|
refTable: e.references.table
|
|
230
252
|
})), z = (e, t, r) => n.gen(function* () {
|
|
231
|
-
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map(),
|
|
253
|
+
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map(), u = (e) => {
|
|
232
254
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
233
255
|
return s.set(e, t), t;
|
|
234
|
-
},
|
|
256
|
+
}, d = (e, t) => {
|
|
235
257
|
if (t == null || a.has(e) || s.get(e)?.has(t)) return;
|
|
236
258
|
let n = l.get(e) ?? /* @__PURE__ */ new Set();
|
|
237
259
|
n.add(t), l.set(e, n);
|
|
238
|
-
},
|
|
239
|
-
let r =
|
|
260
|
+
}, p = (e, t, n) => {
|
|
261
|
+
let r = u(e), i = a.has(e) ? void 0 : o.get(e) ?? (() => {
|
|
240
262
|
let t = /* @__PURE__ */ new Set();
|
|
241
263
|
return o.set(e, t), t;
|
|
242
264
|
})();
|
|
243
265
|
return (e) => {
|
|
244
266
|
r.add(e[t]), i?.add(e[t]);
|
|
245
|
-
for (let t of n)
|
|
267
|
+
for (let t of n) d(t.refTable, e[t.col]);
|
|
246
268
|
};
|
|
247
269
|
}, g = (r, i) => {
|
|
248
|
-
let a =
|
|
249
|
-
return c.runForEach(
|
|
270
|
+
let a = p(r, R(t, r), Ve(t, r));
|
|
271
|
+
return c.runForEach(h(e, {
|
|
250
272
|
table: r,
|
|
251
273
|
pk: R(t, r),
|
|
252
274
|
...i === void 0 ? {} : { where: i }
|
|
@@ -257,7 +279,7 @@ var ge = (e) => {
|
|
|
257
279
|
l.clear();
|
|
258
280
|
for (let [n, r] of e) {
|
|
259
281
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
260
|
-
e.length !== 0 && (yield* g(n,
|
|
282
|
+
e.length !== 0 && (yield* g(n, f(R(t, n), e)));
|
|
261
283
|
}
|
|
262
284
|
}
|
|
263
285
|
});
|
|
@@ -272,19 +294,19 @@ var ge = (e) => {
|
|
|
272
294
|
refTable: e.references?.table
|
|
273
295
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
274
296
|
if (a.length === 0) continue;
|
|
275
|
-
let o = s.get(t)?.size ?? 0, c = a.map((e) =>
|
|
276
|
-
yield* g(t, c.length === 1 ? c[0] :
|
|
297
|
+
let o = s.get(t)?.size ?? 0, c = a.map((e) => f(e.col, [...s.get(e.refTable)]));
|
|
298
|
+
yield* g(t, c.length === 1 ? c[0] : m(...c)), (s.get(t)?.size ?? 0) > o && !e.has(t) && r.push(t), (s.get(t)?.size ?? 0) > o && (n = !0);
|
|
277
299
|
}
|
|
278
300
|
for (let t of r) e.add(t);
|
|
279
301
|
if (yield* _, !n) break;
|
|
280
302
|
}
|
|
281
303
|
}
|
|
282
304
|
return o;
|
|
283
|
-
}),
|
|
305
|
+
}), He = (e, t) => ({
|
|
284
306
|
tables: [...e.keys()],
|
|
285
307
|
predicateFor: (n) => {
|
|
286
308
|
let r = e.get(n);
|
|
287
|
-
return r ?
|
|
309
|
+
return r ? f(R(t, n), [...r]) : void 0;
|
|
288
310
|
}
|
|
289
311
|
}), B = "tenantId", V = (e, t, r) => n.gen(function* () {
|
|
290
312
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
@@ -306,26 +328,26 @@ var ge = (e) => {
|
|
|
306
328
|
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === B) ?? !1);
|
|
307
329
|
if (s.length === 0) return yield* n.fail(new I({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${B}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
308
330
|
let c = yield* z(r.store, t, {
|
|
309
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
331
|
+
seeds: Object.fromEntries(s.map((e) => [e, u(B, o)])),
|
|
310
332
|
pinned: s,
|
|
311
333
|
children: { roots: s }
|
|
312
|
-
}),
|
|
334
|
+
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), d = new Set(s);
|
|
313
335
|
return {
|
|
314
|
-
tables: [...s, ...
|
|
315
|
-
predicateFor: (e) =>
|
|
336
|
+
tables: [...s, ...l],
|
|
337
|
+
predicateFor: (e) => d.has(e) ? u(B, o) : c.has(e) ? f(R(t, e), [...c.get(e)]) : void 0
|
|
316
338
|
};
|
|
317
|
-
}),
|
|
339
|
+
}), Ue = (e, t) => ({
|
|
318
340
|
list: t,
|
|
319
|
-
open: (t) =>
|
|
320
|
-
}),
|
|
321
|
-
write: (t, n, r) =>
|
|
341
|
+
open: (t) => te(e, t)
|
|
342
|
+
}), We = (e) => ({
|
|
343
|
+
write: (t, n, r) => ne(e, t, n, r),
|
|
322
344
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
323
|
-
}),
|
|
345
|
+
}), Ge = (e, t) => n.tryPromise({
|
|
324
346
|
try: async () => {
|
|
325
347
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
326
348
|
u.update(e), d += e.length, n(null, e);
|
|
327
349
|
} });
|
|
328
|
-
await o(await
|
|
350
|
+
await o(await re(e.stream), f, n(l));
|
|
329
351
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
330
352
|
try {
|
|
331
353
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -342,8 +364,8 @@ var ge = (e) => {
|
|
|
342
364
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
343
365
|
path: t
|
|
344
366
|
})
|
|
345
|
-
}),
|
|
346
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), l = o("sha256"), u =
|
|
367
|
+
}), Ke = (e, t, r) => n.gen(function* () {
|
|
368
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), l = o("sha256"), u = C("bundle", () => a(s)).pipe(c.tap((e) => n.sync(() => l.update(e))));
|
|
347
369
|
yield* e.write(r.key, u, {
|
|
348
370
|
contentType: r.contentType,
|
|
349
371
|
totalSize: r.size
|
|
@@ -354,7 +376,7 @@ var ge = (e) => {
|
|
|
354
376
|
expected: r.sha256,
|
|
355
377
|
actual: d
|
|
356
378
|
}));
|
|
357
|
-
}),
|
|
379
|
+
}), qe = () => typeof w.createZstdCompress == "function", Je = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : qe() ? "zstd" : "gzip", Ye = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", Xe = (e) => e === "zstd" ? w.createZstdCompress() : e === "gzip" ? w.createGzip() : new ie(), Ze = (e) => e === "zstd" ? w.createZstdDecompress() : e === "gzip" ? w.createGunzip() : new ie(), Qe = 1, $e = s.Struct({
|
|
358
380
|
name: s.String,
|
|
359
381
|
file: s.String,
|
|
360
382
|
rowCount: s.Number,
|
|
@@ -364,11 +386,11 @@ var ge = (e) => {
|
|
|
364
386
|
value: s.String
|
|
365
387
|
}),
|
|
366
388
|
primaryKey: s.String
|
|
367
|
-
}),
|
|
389
|
+
}), et = s.Struct({
|
|
368
390
|
kind: s.Literal("all", "tenant", "tables"),
|
|
369
391
|
tenantId: s.optional(s.String),
|
|
370
392
|
tables: s.optional(s.Array(s.String))
|
|
371
|
-
}),
|
|
393
|
+
}), tt = s.Struct({
|
|
372
394
|
count: s.Number,
|
|
373
395
|
totalBytes: s.Number,
|
|
374
396
|
index: s.optional(s.String)
|
|
@@ -379,23 +401,23 @@ s.Struct({
|
|
|
379
401
|
size: s.Number,
|
|
380
402
|
contentType: s.String
|
|
381
403
|
});
|
|
382
|
-
var
|
|
404
|
+
var nt = s.Struct({
|
|
383
405
|
formatVersion: s.Number,
|
|
384
406
|
createdAt: s.String,
|
|
385
407
|
source: s.Struct({
|
|
386
408
|
dialect: s.String,
|
|
387
409
|
schemaFingerprint: s.String
|
|
388
410
|
}),
|
|
389
|
-
scope:
|
|
411
|
+
scope: et,
|
|
390
412
|
consistency: s.Literal("live", "snapshot"),
|
|
391
413
|
compression: s.Literal("zstd", "gzip", "none"),
|
|
392
|
-
tables: s.Array(
|
|
393
|
-
assets:
|
|
414
|
+
tables: s.Array($e),
|
|
415
|
+
assets: tt,
|
|
394
416
|
masking: s.optional(s.Struct({
|
|
395
417
|
policyId: s.optional(s.String),
|
|
396
418
|
transformed: s.Array(s.String)
|
|
397
419
|
}))
|
|
398
|
-
}),
|
|
420
|
+
}), rt = s.encodeSync(s.parseJson(nt)), it = s.decodeUnknownSync(s.parseJson(nt)), H = "manifest.json", at = "README.md", U = "data", W = "assets", G = "assets/index.ndjson", ot = ".ledger.json", st = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", ct = (e, t) => {
|
|
399
421
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
400
422
|
return `# Voltro data export
|
|
401
423
|
|
|
@@ -409,7 +431,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
409
431
|
| **Format version** | ${e.formatVersion} |
|
|
410
432
|
| **Consistency** | ${e.consistency} |
|
|
411
433
|
| **Compression** | ${e.compression} |
|
|
412
|
-
| **Scope** | ${
|
|
434
|
+
| **Scope** | ${st(e.scope)} |
|
|
413
435
|
| **Tables** | ${e.tables.length} |
|
|
414
436
|
| **Rows (total)** | ${n} |
|
|
415
437
|
| **Assets** | ${a} |
|
|
@@ -434,26 +456,26 @@ voltro data import <this-directory>
|
|
|
434
456
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
435
457
|
\`\`\`
|
|
436
458
|
`;
|
|
437
|
-
},
|
|
459
|
+
}, lt = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), ut = a.exponential({
|
|
438
460
|
start: .01,
|
|
439
461
|
factor: 2,
|
|
440
462
|
count: 16
|
|
441
|
-
}),
|
|
463
|
+
}), dt = i.histogram("voltro_data_transfer_table_seconds", ut, "Per-table export/import duration in seconds, tagged by phase."), K = (e, t) => {
|
|
442
464
|
if (e) try {
|
|
443
465
|
e(t);
|
|
444
466
|
} catch {}
|
|
445
|
-
},
|
|
467
|
+
}, ft = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(dt.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), lt.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
446
468
|
table: t,
|
|
447
469
|
index: r,
|
|
448
470
|
tableCount: a,
|
|
449
471
|
phase: e
|
|
450
|
-
} })),
|
|
472
|
+
} })), pt = () => ({
|
|
451
473
|
tables: {},
|
|
452
474
|
assets: {}
|
|
453
|
-
}),
|
|
475
|
+
}), mt = (e) => n.tryPromise(async () => {
|
|
454
476
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
455
477
|
return JSON.parse(t);
|
|
456
|
-
}).pipe(n.orElseSucceed(
|
|
478
|
+
}).pipe(n.orElseSucceed(pt)), q = (e, t) => n.tryPromise({
|
|
457
479
|
try: async () => {
|
|
458
480
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
459
481
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
@@ -462,11 +484,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
462
484
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
463
485
|
path: e
|
|
464
486
|
})
|
|
465
|
-
}),
|
|
487
|
+
}), ht = (e, t) => t in e.tables, gt = (e, t) => t in e.assets, _t = /* @__PURE__ */ new Set([
|
|
466
488
|
"json",
|
|
467
489
|
"array",
|
|
468
490
|
"vector"
|
|
469
|
-
]),
|
|
491
|
+
]), vt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", yt = (e) => Buffer.from(e).toString("base64"), bt = (e) => new Uint8Array(Buffer.from(e, "base64")), xt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), St = (e, t) => e == null ? null : t !== void 0 && _t.has(t) ? e : t === "bytes" ? yt(e) : t === "timestamp" || t === "date" ? xt(e) : typeof e == "bigint" ? {
|
|
470
492
|
$v: "bigint",
|
|
471
493
|
d: e.toString()
|
|
472
494
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -477,30 +499,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
477
499
|
d: e.toISOString()
|
|
478
500
|
} : e instanceof Uint8Array ? {
|
|
479
501
|
$v: "bytes",
|
|
480
|
-
d:
|
|
481
|
-
} : e,
|
|
502
|
+
d: yt(e)
|
|
503
|
+
} : e, Ct = (e) => {
|
|
482
504
|
switch (e.$v) {
|
|
483
505
|
case "bigint": return BigInt(e.d);
|
|
484
506
|
case "number": return Number(e.d);
|
|
485
507
|
case "date": return new Date(e.d);
|
|
486
|
-
case "bytes": return
|
|
508
|
+
case "bytes": return bt(e.d);
|
|
487
509
|
default: return e;
|
|
488
510
|
}
|
|
489
|
-
},
|
|
511
|
+
}, wt = (e, t) => e == null ? null : t !== void 0 && _t.has(t) ? e : vt(e) ? Ct(e) : t === "bytes" ? bt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Tt = (e, t) => {
|
|
490
512
|
let n = {};
|
|
491
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
513
|
+
for (let [r, i] of Object.entries(e)) n[r] = St(i, t[r]);
|
|
492
514
|
return n;
|
|
493
|
-
},
|
|
515
|
+
}, Et = (e, t) => {
|
|
494
516
|
let n = {};
|
|
495
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
517
|
+
for (let [r, i] of Object.entries(e)) n[r] = wt(i, t[r]);
|
|
496
518
|
return n;
|
|
497
|
-
},
|
|
498
|
-
try: () =>
|
|
499
|
-
catch: (e) => new
|
|
519
|
+
}, Dt = new TextEncoder(), Ot = (e, t) => e.pipe(c.map((e) => Dt.encode(`${JSON.stringify(Tt(e, t))}\n`))), kt = (e, t, r) => e.pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.mapEffect((e) => n.try({
|
|
520
|
+
try: () => Et(JSON.parse(e), r),
|
|
521
|
+
catch: (e) => new Pe({
|
|
500
522
|
table: t,
|
|
501
523
|
reason: String(e?.message ?? e)
|
|
502
524
|
})
|
|
503
|
-
}))),
|
|
525
|
+
}))), At = (e, t) => {
|
|
504
526
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
505
527
|
for (let e of t.tables) {
|
|
506
528
|
if (!n.has(e.name)) continue;
|
|
@@ -526,28 +548,28 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
526
548
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
527
549
|
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(J(t.cause)) : n.push(r), n.length === 0) {
|
|
528
550
|
let t = String(e ?? "").trim();
|
|
529
|
-
return t === "" || t === "[object Object]" ?
|
|
551
|
+
return t === "" || t === "[object Object]" ? jt(e) : t;
|
|
530
552
|
}
|
|
531
553
|
return n.join(": ");
|
|
532
|
-
},
|
|
554
|
+
}, jt = (e) => {
|
|
533
555
|
try {
|
|
534
556
|
let t = JSON.stringify(e);
|
|
535
557
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
536
558
|
} catch {
|
|
537
559
|
return "unknown error (not serialisable)";
|
|
538
560
|
}
|
|
539
|
-
},
|
|
561
|
+
}, Mt = (e, t) => {
|
|
540
562
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
541
563
|
for (let e of n?.columns ?? []) r[e.name] = e.type;
|
|
542
564
|
return r;
|
|
543
|
-
},
|
|
565
|
+
}, Nt = (e, t) => {
|
|
544
566
|
let n = e.tables.find((e) => e.name === t);
|
|
545
567
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
546
568
|
let r = n.columns.find((e) => e.type === "id");
|
|
547
569
|
if (r !== void 0) return { column: r.name };
|
|
548
570
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
549
571
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
550
|
-
},
|
|
572
|
+
}, Pt = (e, t, r) => n.tryPromise({
|
|
551
573
|
try: async () => {
|
|
552
574
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: l, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
553
575
|
d.update(e);
|
|
@@ -556,7 +578,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
556
578
|
n(null, e);
|
|
557
579
|
} }), m = l.fromWeb(c.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
558
580
|
try {
|
|
559
|
-
await s(m, p,
|
|
581
|
+
await s(m, p, Xe(r), n(h)), await i.rename(h, t);
|
|
560
582
|
} catch (e) {
|
|
561
583
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
562
584
|
}
|
|
@@ -569,7 +591,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
569
591
|
reason: `write table failed: ${J(e)}`,
|
|
570
592
|
path: t
|
|
571
593
|
})
|
|
572
|
-
}),
|
|
594
|
+
}), Ft = (e, t) => n.tryPromise({
|
|
573
595
|
try: async () => {
|
|
574
596
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
575
597
|
},
|
|
@@ -577,13 +599,13 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
577
599
|
reason: `append failed: ${J(t)}`,
|
|
578
600
|
path: e
|
|
579
601
|
})
|
|
580
|
-
}),
|
|
602
|
+
}), It = (e, t) => n.gen(function* () {
|
|
581
603
|
let r = [], i = t.ordered.length;
|
|
582
604
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
583
|
-
let o = t.ordered[a], s =
|
|
605
|
+
let o = t.ordered[a], s = Mt(t.snapshot, o), l = Nt(t.snapshot, o);
|
|
584
606
|
if ("refusal" in l) return yield* n.fail(new I({ reason: `export: ${l.refusal}` }));
|
|
585
|
-
let u = l.column, d = `${U}/${o}.ndjson${
|
|
586
|
-
if (
|
|
607
|
+
let u = l.column, d = `${U}/${o}.ndjson${Ye(t.compression)}`;
|
|
608
|
+
if (ht(t.ledger, o)) {
|
|
587
609
|
let e = t.ledger.tables[o];
|
|
588
610
|
K(t.onProgress, {
|
|
589
611
|
phase: "export",
|
|
@@ -611,13 +633,13 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
611
633
|
event: "start",
|
|
612
634
|
rowsDone: 0
|
|
613
635
|
});
|
|
614
|
-
let f = t.resolved.predicateFor(o),
|
|
636
|
+
let f = t.resolved.predicateFor(o), p = h(e, {
|
|
615
637
|
table: o,
|
|
616
638
|
pk: u,
|
|
617
639
|
chunkSize: t.chunkSize,
|
|
618
640
|
retry: t.retry,
|
|
619
641
|
...f === void 0 ? {} : { where: f }
|
|
620
|
-
}),
|
|
642
|
+
}), m = t.mask ? p.pipe(c.map((e) => Ne(e, o, t.mask.actions, t.mask.seed, s))) : p, { rowCount: g, checksum: _ } = yield* ft("export", o, a, i, (e) => e.rowCount, Pt(Ot(m, s), t.bundleFile(d), t.compression));
|
|
621
643
|
t.ledger.tables[o] = {
|
|
622
644
|
rowCount: g,
|
|
623
645
|
checksum: _
|
|
@@ -639,20 +661,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
639
661
|
});
|
|
640
662
|
}
|
|
641
663
|
return r;
|
|
642
|
-
}),
|
|
664
|
+
}), Lt = (e) => ({
|
|
643
665
|
_tag: "RawSqlFragment",
|
|
644
666
|
strings: [e],
|
|
645
667
|
values: []
|
|
646
|
-
}),
|
|
668
|
+
}), Rt = (t, i, a) => n.tryPromise({
|
|
647
669
|
try: () => t.transactional(async (t) => {
|
|
648
|
-
if (t.raw) for (let e of
|
|
670
|
+
if (t.raw) for (let e of ve(i)) await t.raw(Lt(e));
|
|
649
671
|
let o = await n.runPromiseExit(a(t));
|
|
650
672
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
651
673
|
return o.value;
|
|
652
674
|
}),
|
|
653
675
|
catch: (e) => e instanceof P ? e : new P({ reason: `snapshot export failed: ${J(e)}` })
|
|
654
|
-
}),
|
|
655
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
676
|
+
}), zt = (e) => n.gen(function* () {
|
|
677
|
+
let t = e.scope ?? { kind: "all" }, r = Je(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, s = o.exponential("50 millis").pipe(o.jittered, o.intersect(o.recurs(a))), l = yield* n.promise(() => import("node:path")), u = (t) => l.join(e.outDir, t);
|
|
656
678
|
yield* n.tryPromise({
|
|
657
679
|
try: async () => {
|
|
658
680
|
let t = await import("node:fs/promises");
|
|
@@ -663,19 +685,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
663
685
|
path: e.outDir
|
|
664
686
|
})
|
|
665
687
|
});
|
|
666
|
-
let d = u(
|
|
688
|
+
let d = u(ot), f = yield* mt(d), p = e.subset ? He(yield* z(e.store, e.snapshot, e.subset), e.snapshot) : yield* V(t, e.snapshot, {
|
|
667
689
|
store: e.store,
|
|
668
690
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
669
|
-
}),
|
|
691
|
+
}), m = At(p.tables, e.snapshot), h = e.consistency ?? "live", _, v;
|
|
670
692
|
if (e.masking) {
|
|
671
|
-
let t = e.classification ??
|
|
693
|
+
let t = e.classification ?? k(e.snapshot), r = Ae(m.map((t) => ({
|
|
672
694
|
name: t,
|
|
673
695
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
674
696
|
name: e.name,
|
|
675
697
|
type: e.type
|
|
676
698
|
}))
|
|
677
699
|
})), t, e.masking);
|
|
678
|
-
if (r.unclassified.length > 0) return yield* n.fail(new
|
|
700
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new Re({
|
|
701
|
+
unclassified: r.unclassified,
|
|
702
|
+
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
703
|
+
}));
|
|
679
704
|
v = {
|
|
680
705
|
actions: r.actions,
|
|
681
706
|
seed: e.masking.seed
|
|
@@ -688,7 +713,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
688
713
|
}
|
|
689
714
|
let y = {
|
|
690
715
|
snapshot: e.snapshot,
|
|
691
|
-
ordered:
|
|
716
|
+
ordered: m,
|
|
692
717
|
resolved: p,
|
|
693
718
|
compression: r,
|
|
694
719
|
chunkSize: i,
|
|
@@ -698,33 +723,33 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
698
723
|
bundleFile: u,
|
|
699
724
|
...v ? { mask: v } : {},
|
|
700
725
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
701
|
-
},
|
|
726
|
+
}, ee = h === "snapshot" ? yield* Rt(e.store, e.dialect, (e) => It(e, y)) : yield* It(e.store, y), b = 0, x = 0;
|
|
702
727
|
if (e.assets) {
|
|
703
728
|
let t = u(G);
|
|
704
|
-
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
729
|
+
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => Ge(e, u(W))), n.flatMap((e) => {
|
|
705
730
|
let i = {
|
|
706
731
|
key: r.key,
|
|
707
732
|
sha256: e.sha256,
|
|
708
733
|
size: e.size,
|
|
709
734
|
contentType: e.contentType
|
|
710
735
|
};
|
|
711
|
-
return
|
|
736
|
+
return b++, x += e.size, f.assets[r.key] = !0, Ft(t, JSON.stringify(i)).pipe(n.andThen(q(d, f)));
|
|
712
737
|
}))).pipe(n.mapError((e) => e instanceof P ? e : new P({ reason: `asset export failed: ${String(e)}` })));
|
|
713
738
|
}
|
|
714
|
-
let
|
|
739
|
+
let S = {
|
|
715
740
|
formatVersion: 1,
|
|
716
741
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
717
742
|
source: {
|
|
718
743
|
dialect: e.dialect,
|
|
719
|
-
schemaFingerprint:
|
|
744
|
+
schemaFingerprint: g(e.snapshot)
|
|
720
745
|
},
|
|
721
746
|
scope: t,
|
|
722
|
-
consistency:
|
|
747
|
+
consistency: h,
|
|
723
748
|
compression: r,
|
|
724
|
-
tables:
|
|
725
|
-
assets:
|
|
726
|
-
count:
|
|
727
|
-
totalBytes:
|
|
749
|
+
tables: ee,
|
|
750
|
+
assets: b > 0 ? {
|
|
751
|
+
count: b,
|
|
752
|
+
totalBytes: x,
|
|
728
753
|
index: G
|
|
729
754
|
} : {
|
|
730
755
|
count: 0,
|
|
@@ -734,7 +759,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
734
759
|
};
|
|
735
760
|
return yield* n.tryPromise({
|
|
736
761
|
try: async () => {
|
|
737
|
-
await (await import("node:fs/promises")).writeFile(u(H),
|
|
762
|
+
await (await import("node:fs/promises")).writeFile(u(H), rt(S));
|
|
738
763
|
},
|
|
739
764
|
catch: (e) => new P({
|
|
740
765
|
reason: `manifest write failed: ${J(e)}`,
|
|
@@ -743,14 +768,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
743
768
|
}), yield* n.tryPromise({
|
|
744
769
|
try: async () => {
|
|
745
770
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
746
|
-
await e.writeFile(u(
|
|
771
|
+
await e.writeFile(u(at), ct(S, { hostname: t.hostname() }));
|
|
747
772
|
},
|
|
748
773
|
catch: (e) => new P({
|
|
749
774
|
reason: `readme write failed: ${J(e)}`,
|
|
750
|
-
path:
|
|
775
|
+
path: at
|
|
751
776
|
})
|
|
752
|
-
}),
|
|
753
|
-
}),
|
|
777
|
+
}), S;
|
|
778
|
+
}), Bt = (e) => {
|
|
754
779
|
switch (e) {
|
|
755
780
|
case "id":
|
|
756
781
|
case "text":
|
|
@@ -769,12 +794,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
769
794
|
case "raw": return "raw";
|
|
770
795
|
default: return "string";
|
|
771
796
|
}
|
|
772
|
-
},
|
|
797
|
+
}, Vt = {
|
|
773
798
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
774
799
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
775
800
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
776
|
-
},
|
|
777
|
-
let i =
|
|
801
|
+
}, Ht = (e, t, n, r) => {
|
|
802
|
+
let i = Bt(n);
|
|
778
803
|
if (i === "raw") return {
|
|
779
804
|
table: e,
|
|
780
805
|
column: t,
|
|
@@ -782,7 +807,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
782
807
|
severity: "block",
|
|
783
808
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
784
809
|
};
|
|
785
|
-
let a =
|
|
810
|
+
let a = Vt[i];
|
|
786
811
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
787
812
|
table: e,
|
|
788
813
|
column: t,
|
|
@@ -796,55 +821,87 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
796
821
|
severity: "block",
|
|
797
822
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
798
823
|
};
|
|
799
|
-
},
|
|
824
|
+
}, Ut = (e, t, n) => {
|
|
800
825
|
if (t === n) return [];
|
|
801
826
|
let r = [];
|
|
802
827
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
803
|
-
let a =
|
|
828
|
+
let a = Ht(t.name, e, i, n);
|
|
804
829
|
a && r.push(a);
|
|
805
830
|
}
|
|
806
831
|
return r;
|
|
807
|
-
},
|
|
808
|
-
let
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
832
|
+
}, Wt = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, Gt = (e) => {
|
|
833
|
+
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
834
|
+
switch (e.kind) {
|
|
835
|
+
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
836
|
+
case "foreignKeyInUse": return `foreign key${n}: the row is still referenced by other rows`;
|
|
837
|
+
case "unique": return `unique constraint${n}: a row with this value already exists`;
|
|
838
|
+
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
839
|
+
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
840
|
+
}
|
|
841
|
+
}, Kt = (e) => {
|
|
842
|
+
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
843
|
+
return t.length > 0 ? t.join("/") : void 0;
|
|
844
|
+
}, qt = (e) => {
|
|
845
|
+
let t = d(e), n = l(t), r = Kt(t), i = r === void 0 ? "" : ` [${r}]`;
|
|
846
|
+
if (n !== void 0) return `${Gt(n)}${i}`;
|
|
847
|
+
let a = typeof t.dbMessage == "string" ? t.dbMessage : void 0, o = e?.message, s = typeof o == "string" && o.length > 0 ? o : String(e), c = a !== void 0 && a !== s && !a.startsWith("SqlError:") ? a : s, u = c === "Failed to execute statement" ? `the database refused the write and the driver gave no detail${i}` : `${c}${i}`;
|
|
848
|
+
return u.length <= 300 ? u : `${u.slice(0, 300)}…`;
|
|
849
|
+
}, Jt = (e, t) => e.tables.find((e) => e.name === t), Yt = (e, t) => new Map((Jt(e, t)?.columns ?? []).map((e) => [e.name, e])), Xt = (e, t) => {
|
|
850
|
+
let n = [];
|
|
851
|
+
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
852
|
+
if (Jt(t, r.name) === void 0) {
|
|
853
|
+
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
854
|
+
continue;
|
|
855
|
+
}
|
|
856
|
+
let e = Yt(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
857
|
+
for (let [t, a] of i) {
|
|
858
|
+
let i = e.get(t);
|
|
859
|
+
if (i === void 0) {
|
|
860
|
+
n.push(`${r.name}.${t}: in the bundle, MISSING from the target — the value has nowhere to go`);
|
|
861
|
+
continue;
|
|
862
|
+
}
|
|
863
|
+
i.type !== a.type && n.push(`${r.name}.${t}: type '${a.type}' (bundle) vs '${i.type}' (target)`);
|
|
818
864
|
}
|
|
865
|
+
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
819
866
|
}
|
|
820
|
-
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n
|
|
821
|
-
},
|
|
822
|
-
let r =
|
|
823
|
-
|
|
867
|
+
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
868
|
+
}, Zt = (e, t, n) => {
|
|
869
|
+
let r = g(t);
|
|
870
|
+
if (r === e || n === void 0) return {
|
|
824
871
|
drifted: !1,
|
|
825
872
|
bundleFingerprint: e,
|
|
826
873
|
targetFingerprint: r,
|
|
827
874
|
diff: []
|
|
828
|
-
}
|
|
829
|
-
|
|
875
|
+
};
|
|
876
|
+
let i = Xt(n, t);
|
|
877
|
+
return {
|
|
878
|
+
drifted: i.length > 0,
|
|
830
879
|
bundleFingerprint: e,
|
|
831
880
|
targetFingerprint: r,
|
|
832
|
-
diff:
|
|
881
|
+
diff: i
|
|
833
882
|
};
|
|
834
|
-
},
|
|
883
|
+
}, Qt = (e) => n.tryPromise({
|
|
835
884
|
try: async () => {
|
|
836
885
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
837
|
-
return
|
|
886
|
+
return it(await t.readFile(n.join(e, H), "utf8"));
|
|
838
887
|
},
|
|
839
888
|
catch: (e) => new P({
|
|
840
889
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
841
890
|
path: H
|
|
842
891
|
})
|
|
843
|
-
}),
|
|
844
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
892
|
+
}), $t = (t) => n.gen(function* () {
|
|
893
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* Qt(t.bundleDir);
|
|
845
894
|
if (a.formatVersion !== 1) return yield* n.fail(new P({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
895
|
+
if (t.tables !== void 0) {
|
|
896
|
+
let e = new Set(a.tables.map((e) => e.name)), r = t.tables.filter((t) => !e.has(t));
|
|
897
|
+
if (r.length > 0) return yield* n.fail(new P({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...e].sort().join(", ")}.` }));
|
|
898
|
+
}
|
|
899
|
+
let o = t.tables === void 0 ? void 0 : new Set(t.tables), s = o === void 0 ? a : {
|
|
900
|
+
...a,
|
|
901
|
+
tables: a.tables.filter((e) => o.has(e.name))
|
|
902
|
+
};
|
|
846
903
|
if (t.targetSnapshot) {
|
|
847
|
-
let e = { tables:
|
|
904
|
+
let e = { tables: s.tables.map((e) => ({
|
|
848
905
|
name: e.name,
|
|
849
906
|
indexes: [],
|
|
850
907
|
columns: Object.entries(e.columnTypes).map(([e, t]) => ({
|
|
@@ -854,9 +911,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
854
911
|
unique: !1,
|
|
855
912
|
hasDefault: !1
|
|
856
913
|
}))
|
|
857
|
-
})) }, r =
|
|
914
|
+
})) }, r = Zt(s.source.schemaFingerprint, t.targetSnapshot, e);
|
|
858
915
|
if (r.drifted) {
|
|
859
|
-
if (!t.force) return yield* n.fail(new
|
|
916
|
+
if (!t.force) return yield* n.fail(new Le({
|
|
860
917
|
bundleFingerprint: r.bundleFingerprint,
|
|
861
918
|
targetFingerprint: r.targetFingerprint,
|
|
862
919
|
diff: r.diff
|
|
@@ -864,45 +921,46 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
864
921
|
t.onWarn?.(`schema drift: the target schema (fingerprint ${r.targetFingerprint.slice(0, 16)}) differs from the bundle's source (${r.bundleFingerprint.slice(0, 16)}); importing anyway (--force). ${r.diff.join("; ")}`);
|
|
865
922
|
}
|
|
866
923
|
}
|
|
867
|
-
if (t.targetDialect && t.targetDialect !==
|
|
868
|
-
let e =
|
|
869
|
-
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new
|
|
870
|
-
sourceDialect:
|
|
924
|
+
if (t.targetDialect && t.targetDialect !== s.source.dialect) {
|
|
925
|
+
let e = Ut(s.tables, s.source.dialect, t.targetDialect).filter((e) => e.severity === "block");
|
|
926
|
+
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new Be({
|
|
927
|
+
sourceDialect: s.source.dialect,
|
|
871
928
|
targetDialect: t.targetDialect,
|
|
872
|
-
blocking: e.map(
|
|
929
|
+
blocking: e.map(Wt)
|
|
873
930
|
}));
|
|
874
931
|
}
|
|
875
|
-
let
|
|
876
|
-
if (
|
|
877
|
-
mode:
|
|
878
|
-
reason: `replace refuses a partial bundle (scope='${
|
|
932
|
+
let l = t.verify ?? !0, u = t.mode ?? "upsert", d = t.onConflict ?? "skip", f = t.ledgerPath ?? i.join(t.bundleDir, ".import..ledger.json"), m = yield* mt(f);
|
|
933
|
+
if (u === "replace" && s.scope.kind !== "all") return yield* n.fail(new ze({
|
|
934
|
+
mode: u,
|
|
935
|
+
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
879
936
|
}));
|
|
880
|
-
|
|
937
|
+
if (t.dryRun === !0) return s;
|
|
938
|
+
let h = (e, t, n) => {
|
|
881
939
|
let r = t.name, i = t.primaryKey;
|
|
882
|
-
return
|
|
883
|
-
},
|
|
940
|
+
return u === "append" ? d === "fail" ? e.insert(r, n) : e.insertIgnore(r, n, { conflictColumns: [i] }) : u === "replace" ? e.insert(r, n) : e.upsert(r, n, { conflictColumns: [i] });
|
|
941
|
+
}, g = (e, t, r = !0) => n.tryPromise({
|
|
884
942
|
try: e,
|
|
885
|
-
catch: (e) =>
|
|
886
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (t.reason = e, !1)))),
|
|
943
|
+
catch: (e) => qt(e)
|
|
944
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e), !1)))), _ = (e) => {
|
|
887
945
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
888
946
|
if (t.length !== 0) return {
|
|
889
947
|
...e.row,
|
|
890
948
|
...Object.fromEntries(t.map((e) => [e, null]))
|
|
891
949
|
};
|
|
892
|
-
},
|
|
950
|
+
}, v = (e, t) => n.gen(function* () {
|
|
893
951
|
let r = t, i = [], a = !0;
|
|
894
952
|
for (; r.length > 0 && a;) {
|
|
895
953
|
a = !1;
|
|
896
954
|
let t = [];
|
|
897
|
-
for (let n of r) (yield*
|
|
955
|
+
for (let n of r) (yield* g(() => h(e, n.entry, n.row), n)) ? a = !0 : t.push(n);
|
|
898
956
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
899
|
-
let r = t[n], o =
|
|
900
|
-
o !== void 0 && (yield*
|
|
957
|
+
let r = t[n], o = _(r);
|
|
958
|
+
o !== void 0 && (yield* g(() => h(e, r.entry, o), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
901
959
|
}
|
|
902
960
|
r = t;
|
|
903
961
|
}
|
|
904
|
-
for (let t of i) (yield*
|
|
905
|
-
if (r.length > 0) return yield* n.fail(new
|
|
962
|
+
for (let t of i) (yield* g(() => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] }), t)) || (r = [...r, t]);
|
|
963
|
+
if (r.length > 0) return yield* n.fail(new Ie({
|
|
906
964
|
rows: r.slice(0, 20).map((e) => ({
|
|
907
965
|
table: e.entry.name,
|
|
908
966
|
id: String(e.row[e.entry.primaryKey]),
|
|
@@ -910,59 +968,59 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
910
968
|
})),
|
|
911
969
|
totalCount: r.length
|
|
912
970
|
}));
|
|
913
|
-
}),
|
|
914
|
-
if (
|
|
915
|
-
for (let t of [...
|
|
916
|
-
try: () => e.deleteMany(t.name, { where:
|
|
971
|
+
}), y = (e, r) => n.gen(function* () {
|
|
972
|
+
if (u === "replace" && !(r && m.truncated)) {
|
|
973
|
+
for (let t of [...s.tables].reverse()) yield* n.tryPromise({
|
|
974
|
+
try: () => e.deleteMany(t.name, { where: p(t.primaryKey) }),
|
|
917
975
|
catch: (e) => new P({ reason: `truncate ${t.name} failed: ${String(e?.message ?? e)}` })
|
|
918
976
|
});
|
|
919
|
-
r && (
|
|
977
|
+
r && (m.truncated = !0, yield* q(f, m));
|
|
920
978
|
}
|
|
921
|
-
let
|
|
922
|
-
for (let
|
|
923
|
-
let
|
|
924
|
-
if (r &&
|
|
925
|
-
K(t.onProgress, {
|
|
979
|
+
let a = [], o = [], d = [], g = s.tables.length;
|
|
980
|
+
for (let p = 0; p < s.tables.length; p++) {
|
|
981
|
+
let _ = s.tables[p];
|
|
982
|
+
if (r && _.name in m.tables) {
|
|
983
|
+
d.push(_.name), K(t.onProgress, {
|
|
926
984
|
phase: "import",
|
|
927
|
-
table:
|
|
928
|
-
index:
|
|
929
|
-
tableCount:
|
|
985
|
+
table: _.name,
|
|
986
|
+
index: p,
|
|
987
|
+
tableCount: g,
|
|
930
988
|
event: "done",
|
|
931
|
-
rowsDone:
|
|
932
|
-
total:
|
|
989
|
+
rowsDone: m.tables[_.name].rowCount,
|
|
990
|
+
total: _.rowCount
|
|
933
991
|
});
|
|
934
992
|
continue;
|
|
935
993
|
}
|
|
936
|
-
let
|
|
994
|
+
let v = i.join(t.bundleDir, _.file);
|
|
937
995
|
K(t.onProgress, {
|
|
938
996
|
phase: "import",
|
|
939
|
-
table:
|
|
940
|
-
index:
|
|
941
|
-
tableCount:
|
|
997
|
+
table: _.name,
|
|
998
|
+
index: p,
|
|
999
|
+
tableCount: g,
|
|
942
1000
|
event: "start",
|
|
943
1001
|
rowsDone: 0,
|
|
944
|
-
total:
|
|
1002
|
+
total: _.rowCount
|
|
945
1003
|
});
|
|
946
|
-
let
|
|
947
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0,
|
|
1004
|
+
let y = a.length, ee = n.gen(function* () {
|
|
1005
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, l = C("bundle", () => T(v).pipe(Ze(s.compression))).pipe(c.tap((e) => n.sync(() => i.update(e)))), d = async (t) => {
|
|
948
1006
|
let n = {
|
|
949
|
-
entry:
|
|
1007
|
+
entry: _,
|
|
950
1008
|
row: t,
|
|
951
1009
|
reason: ""
|
|
952
1010
|
};
|
|
953
1011
|
try {
|
|
954
|
-
await
|
|
1012
|
+
await h(e, _, t);
|
|
955
1013
|
} catch (e) {
|
|
956
|
-
n.reason =
|
|
1014
|
+
n.reason = qt(e), a.push(n);
|
|
957
1015
|
}
|
|
958
1016
|
}, f = yield* n.promise(async () => {
|
|
959
1017
|
let n = t.copyLoader;
|
|
960
1018
|
if (!n || n.dialect !== "postgres" || t.atomic) return !1;
|
|
961
|
-
if (
|
|
962
|
-
if (
|
|
1019
|
+
if (u === "replace") return !0;
|
|
1020
|
+
if (u !== "upsert") return !1;
|
|
963
1021
|
try {
|
|
964
1022
|
return (await e.query({
|
|
965
|
-
table:
|
|
1023
|
+
table: _.name,
|
|
966
1024
|
predicate: void 0,
|
|
967
1025
|
order: [],
|
|
968
1026
|
take: 1,
|
|
@@ -972,25 +1030,25 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
972
1030
|
} catch {
|
|
973
1031
|
return !1;
|
|
974
1032
|
}
|
|
975
|
-
}),
|
|
976
|
-
if (
|
|
977
|
-
let e =
|
|
978
|
-
|
|
1033
|
+
}), p = Object.keys(_.columnTypes), m = Math.max(1, t.copyBatchSize ?? 5e3), g = [], y = async () => {
|
|
1034
|
+
if (g.length === 0) return;
|
|
1035
|
+
let e = g;
|
|
1036
|
+
g = [];
|
|
979
1037
|
try {
|
|
980
1038
|
await t.copyLoader.copyInto({
|
|
981
|
-
table:
|
|
982
|
-
columns:
|
|
983
|
-
columnTypes:
|
|
1039
|
+
table: _.name,
|
|
1040
|
+
columns: p,
|
|
1041
|
+
columnTypes: _.columnTypes,
|
|
984
1042
|
rows: e
|
|
985
1043
|
});
|
|
986
1044
|
} catch (n) {
|
|
987
|
-
f = !1, t.onWarn?.(`bulk COPY into ${
|
|
1045
|
+
f = !1, t.onWarn?.(`bulk COPY into ${_.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
988
1046
|
for (let t of e) await d(t);
|
|
989
1047
|
}
|
|
990
1048
|
};
|
|
991
|
-
return yield* c.runForEach(
|
|
1049
|
+
return yield* c.runForEach(kt(l, _.name, _.columnTypes), (e) => n.promise(async () => {
|
|
992
1050
|
if (o++, f) {
|
|
993
|
-
|
|
1051
|
+
g.push(e), g.length >= m && await y();
|
|
994
1052
|
return;
|
|
995
1053
|
}
|
|
996
1054
|
await d(e);
|
|
@@ -998,71 +1056,75 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
998
1056
|
rowCount: o,
|
|
999
1057
|
checksum: i.digest("hex")
|
|
1000
1058
|
};
|
|
1001
|
-
}), { rowCount: b, checksum: x } = yield*
|
|
1002
|
-
if (
|
|
1003
|
-
if (x !==
|
|
1004
|
-
what: `table ${
|
|
1005
|
-
expected:
|
|
1059
|
+
}), { rowCount: b, checksum: x } = yield* ft("import", _.name, p, g, (e) => e.rowCount, ee);
|
|
1060
|
+
if (l) {
|
|
1061
|
+
if (x !== _.checksum) return yield* n.fail(new F({
|
|
1062
|
+
what: `table ${_.name} checksum`,
|
|
1063
|
+
expected: _.checksum,
|
|
1006
1064
|
actual: x
|
|
1007
1065
|
}));
|
|
1008
|
-
if (b !==
|
|
1009
|
-
what: `table ${
|
|
1010
|
-
expected: String(
|
|
1066
|
+
if (b !== _.rowCount) return yield* n.fail(new F({
|
|
1067
|
+
what: `table ${_.name} rowCount`,
|
|
1068
|
+
expected: String(_.rowCount),
|
|
1011
1069
|
actual: String(b)
|
|
1012
1070
|
}));
|
|
1013
1071
|
}
|
|
1014
1072
|
K(t.onProgress, {
|
|
1015
1073
|
phase: "import",
|
|
1016
|
-
table:
|
|
1017
|
-
index:
|
|
1018
|
-
tableCount:
|
|
1074
|
+
table: _.name,
|
|
1075
|
+
index: p,
|
|
1076
|
+
tableCount: g,
|
|
1019
1077
|
event: "done",
|
|
1020
1078
|
rowsDone: b,
|
|
1021
|
-
total:
|
|
1022
|
-
}), r && (
|
|
1023
|
-
name:
|
|
1079
|
+
total: _.rowCount
|
|
1080
|
+
}), r && (a.length > y ? o.push({
|
|
1081
|
+
name: _.name,
|
|
1024
1082
|
rowCount: b
|
|
1025
|
-
}) : (
|
|
1083
|
+
}) : (m.tables[_.name] = {
|
|
1026
1084
|
rowCount: b,
|
|
1027
|
-
checksum:
|
|
1028
|
-
}, yield* q(
|
|
1085
|
+
checksum: _.checksum
|
|
1086
|
+
}, yield* q(f, m)));
|
|
1029
1087
|
}
|
|
1030
|
-
if (
|
|
1031
|
-
for (let e of
|
|
1032
|
-
let t =
|
|
1033
|
-
|
|
1088
|
+
if (a.length > 0 && (yield* v(e, a)), r && o.length > 0) {
|
|
1089
|
+
for (let e of o) {
|
|
1090
|
+
let t = s.tables.find((t) => t.name === e.name);
|
|
1091
|
+
m.tables[e.name] = {
|
|
1034
1092
|
rowCount: e.rowCount,
|
|
1035
1093
|
checksum: t.checksum
|
|
1036
1094
|
};
|
|
1037
1095
|
}
|
|
1038
|
-
yield* q(
|
|
1096
|
+
yield* q(f, m);
|
|
1097
|
+
}
|
|
1098
|
+
if (d.length > 0) {
|
|
1099
|
+
let e = d.length === s.tables.length;
|
|
1100
|
+
t.onWarn?.(`resume: ${d.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${e ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${d.join(", ")}. Delete ${f} to force a full re-import.`);
|
|
1039
1101
|
}
|
|
1040
1102
|
});
|
|
1041
1103
|
if (t.atomic ? yield* n.tryPromise({
|
|
1042
1104
|
try: () => t.store.transactional(async (t) => {
|
|
1043
|
-
let i = await n.runPromiseExit(
|
|
1105
|
+
let i = await n.runPromiseExit(y(t, !1));
|
|
1044
1106
|
if (r.isFailure(i)) throw Error(e.pretty(i.cause));
|
|
1045
1107
|
}),
|
|
1046
1108
|
catch: (e) => new P({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1047
|
-
}) : yield*
|
|
1048
|
-
let e = t.assets, r = i.join(t.bundleDir, W),
|
|
1049
|
-
yield* c.runForEach(
|
|
1050
|
-
|
|
1051
|
-
})), n.andThen(q(
|
|
1109
|
+
}) : yield* y(t.store, !0), t.assets && s.assets.index && !t.assetsAlreadyRestored) {
|
|
1110
|
+
let e = t.assets, r = i.join(t.bundleDir, W), a = i.join(t.bundleDir, s.assets.index ?? "assets/index.ndjson"), o = C("bundle", () => T(a)).pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.map((e) => JSON.parse(e)));
|
|
1111
|
+
yield* c.runForEach(o, (t) => t.key in m.assets ? n.void : Ke(e, r, t).pipe(n.andThen(n.sync(() => {
|
|
1112
|
+
m.assets[t.key] = !0;
|
|
1113
|
+
})), n.andThen(q(f, m))));
|
|
1052
1114
|
}
|
|
1053
|
-
return
|
|
1054
|
-
}),
|
|
1115
|
+
return s;
|
|
1116
|
+
}), en = (e) => e, tn = (e) => {
|
|
1055
1117
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1056
1118
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1057
1119
|
return {
|
|
1058
1120
|
refuse: !0,
|
|
1059
1121
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1060
1122
|
};
|
|
1061
|
-
}, Y = Buffer.from("VBUNDLE2"),
|
|
1123
|
+
}, Y = Buffer.from("VBUNDLE2"), nn = 0, rn = 1, an = 2, on = 32, sn = async (e) => {
|
|
1062
1124
|
let t = [], n = async (r) => {
|
|
1063
1125
|
let i;
|
|
1064
1126
|
try {
|
|
1065
|
-
i = await
|
|
1127
|
+
i = await le(O(e, r), { withFileTypes: !0 });
|
|
1066
1128
|
} catch {
|
|
1067
1129
|
return;
|
|
1068
1130
|
}
|
|
@@ -1072,22 +1134,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1072
1134
|
}
|
|
1073
1135
|
};
|
|
1074
1136
|
return await n(""), t.sort();
|
|
1075
|
-
},
|
|
1137
|
+
}, cn = (e, t) => {
|
|
1076
1138
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1077
|
-
return r.writeUInt8(
|
|
1078
|
-
},
|
|
1139
|
+
return r.writeUInt8(rn, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1140
|
+
}, ln = (e, t, n) => {
|
|
1079
1141
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1080
|
-
return a.writeUInt8(
|
|
1142
|
+
return a.writeUInt8(an, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1081
1143
|
};
|
|
1082
|
-
async function*
|
|
1144
|
+
async function* un(e) {
|
|
1083
1145
|
yield new Uint8Array(Y);
|
|
1084
|
-
for (let t of await
|
|
1085
|
-
let n = O(e.dir, t), r = await
|
|
1086
|
-
if (yield new Uint8Array(
|
|
1146
|
+
for (let t of await sn(e.dir)) {
|
|
1147
|
+
let n = O(e.dir, t), r = await ue(n);
|
|
1148
|
+
if (yield new Uint8Array(cn(t, r.size)), r.size > 0) for await (let e of T(n)) yield new Uint8Array(e);
|
|
1087
1149
|
}
|
|
1088
1150
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1089
|
-
yield new Uint8Array(
|
|
1090
|
-
let n =
|
|
1151
|
+
yield new Uint8Array(ln(t.key, t.contentType, t.size));
|
|
1152
|
+
let n = y("sha256"), r = 0;
|
|
1091
1153
|
for await (let i of e.blobs.open(t.key)) {
|
|
1092
1154
|
let e = Buffer.from(i);
|
|
1093
1155
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1095,11 +1157,11 @@ async function* nn(e) {
|
|
|
1095
1157
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1096
1158
|
yield new Uint8Array(n.digest());
|
|
1097
1159
|
}
|
|
1098
|
-
yield new Uint8Array(Buffer.from([
|
|
1160
|
+
yield new Uint8Array(Buffer.from([nn]));
|
|
1099
1161
|
}
|
|
1100
|
-
var
|
|
1101
|
-
let n =
|
|
1102
|
-
await
|
|
1162
|
+
var dn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), fn = (e) => new Promise((t) => e.end(t)), pn = async (e, t) => {
|
|
1163
|
+
let n = me(t.dir);
|
|
1164
|
+
await D(n, { recursive: !0 });
|
|
1103
1165
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1104
1166
|
let { value: e, done: t } = await i.next();
|
|
1105
1167
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1114,24 +1176,24 @@ var rn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), an
|
|
|
1114
1176
|
for (;;) {
|
|
1115
1177
|
await s(1);
|
|
1116
1178
|
let e = c(1).readUInt8(0);
|
|
1117
|
-
if (e ===
|
|
1118
|
-
if (e ===
|
|
1179
|
+
if (e === nn) return;
|
|
1180
|
+
if (e === rn) {
|
|
1119
1181
|
await s(4);
|
|
1120
1182
|
let e = c(4).readUInt32BE(0);
|
|
1121
1183
|
await s(e);
|
|
1122
1184
|
let t = c(e).toString("utf8");
|
|
1123
1185
|
await s(8);
|
|
1124
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1125
|
-
if (a !== n && !a.startsWith(n +
|
|
1126
|
-
await
|
|
1127
|
-
let o =
|
|
1186
|
+
let i = c(8).readBigUInt64BE(0), a = me(n, t);
|
|
1187
|
+
if (a !== n && !a.startsWith(n + he)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1188
|
+
await D(pe(a), { recursive: !0 });
|
|
1189
|
+
let o = E(a);
|
|
1128
1190
|
for (; i > 0n;) {
|
|
1129
1191
|
r.length === 0 && await s(1);
|
|
1130
1192
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1131
|
-
await
|
|
1193
|
+
await dn(o, c(e)), i -= BigInt(e);
|
|
1132
1194
|
}
|
|
1133
|
-
await
|
|
1134
|
-
} else if (e ===
|
|
1195
|
+
await fn(o);
|
|
1196
|
+
} else if (e === an) {
|
|
1135
1197
|
await s(4);
|
|
1136
1198
|
let e = c(4).readUInt32BE(0);
|
|
1137
1199
|
await s(e);
|
|
@@ -1141,24 +1203,24 @@ var rn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), an
|
|
|
1141
1203
|
await s(i);
|
|
1142
1204
|
let a = c(i).toString("utf8");
|
|
1143
1205
|
await s(8);
|
|
1144
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1206
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = y("sha256"), d = new ie(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1145
1207
|
contentType: a,
|
|
1146
1208
|
size: Number(o)
|
|
1147
1209
|
}), p = o;
|
|
1148
1210
|
for (; p > 0n;) {
|
|
1149
1211
|
r.length === 0 && await s(1);
|
|
1150
1212
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1151
|
-
u.update(t), l || await
|
|
1213
|
+
u.update(t), l || await dn(d, t), p -= BigInt(e);
|
|
1152
1214
|
}
|
|
1153
|
-
l ? d.destroy() : await
|
|
1154
|
-
let m = c(
|
|
1215
|
+
l ? d.destroy() : await fn(d), await f, await s(on);
|
|
1216
|
+
let m = c(on);
|
|
1155
1217
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1156
1218
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1157
1219
|
}
|
|
1158
1220
|
} finally {
|
|
1159
1221
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1160
1222
|
}
|
|
1161
|
-
},
|
|
1223
|
+
}, mn = async (e, t) => {
|
|
1162
1224
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1163
1225
|
let { value: e, done: t } = await a.next();
|
|
1164
1226
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1180,8 +1242,8 @@ var rn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), an
|
|
|
1180
1242
|
if (!await s(Y.length) || !c(Y.length).equals(Y)) throw Error("not a bundle archive (bad magic)");
|
|
1181
1243
|
for (; await s(1);) {
|
|
1182
1244
|
let e = c(1).readUInt8(0);
|
|
1183
|
-
if (e ===
|
|
1184
|
-
if (e !==
|
|
1245
|
+
if (e === nn || e === an) break;
|
|
1246
|
+
if (e !== rn) throw Error(`unknown bundle entry kind ${e}`);
|
|
1185
1247
|
if (!await s(4)) break;
|
|
1186
1248
|
let t = c(4).readUInt32BE(0);
|
|
1187
1249
|
if (!await s(t)) break;
|
|
@@ -1191,20 +1253,20 @@ var rn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), an
|
|
|
1191
1253
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1192
1254
|
}
|
|
1193
1255
|
return r;
|
|
1194
|
-
},
|
|
1256
|
+
}, hn = (e, t) => pn(e, {
|
|
1195
1257
|
dir: t,
|
|
1196
|
-
blobs:
|
|
1197
|
-
}),
|
|
1258
|
+
blobs: gn(t)
|
|
1259
|
+
}), gn = (e) => {
|
|
1198
1260
|
let t = /* @__PURE__ */ new Set();
|
|
1199
1261
|
return {
|
|
1200
1262
|
has: async (e) => t.has(e),
|
|
1201
1263
|
write: async (n, r, i) => {
|
|
1202
1264
|
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = O(e, "assets");
|
|
1203
|
-
await
|
|
1204
|
-
let l =
|
|
1265
|
+
await D(c, { recursive: !0 });
|
|
1266
|
+
let l = y("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1205
1267
|
l.update(e), u += e.length, n(null, e);
|
|
1206
1268
|
} }), f = O(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1207
|
-
await o(r, d,
|
|
1269
|
+
await o(r, d, E(f));
|
|
1208
1270
|
let p = l.digest("hex");
|
|
1209
1271
|
await a.rename(f, O(c, p)), await a.appendFile(O(c, "index.ndjson"), `${JSON.stringify({
|
|
1210
1272
|
key: n,
|
|
@@ -1214,33 +1276,33 @@ var rn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), an
|
|
|
1214
1276
|
})}\n`), t.add(n);
|
|
1215
1277
|
}
|
|
1216
1278
|
};
|
|
1217
|
-
},
|
|
1279
|
+
}, _n = ge(x), vn = Buffer.from("VENC1\0\0\0"), yn = 32, bn = 16, xn = 7, X = 16, Sn = 262144, Z = {
|
|
1218
1280
|
N: 32768,
|
|
1219
1281
|
r: 8,
|
|
1220
1282
|
p: 1
|
|
1221
|
-
},
|
|
1283
|
+
}, Cn = 134217728, Q = 43, wn = (e, t, n) => {
|
|
1222
1284
|
let r = Buffer.alloc(12);
|
|
1223
|
-
return e.copy(r, 0, 0,
|
|
1285
|
+
return e.copy(r, 0, 0, xn), r.writeUInt32BE(t >>> 0, xn), r.writeUInt8(+!!n, 11), r;
|
|
1224
1286
|
};
|
|
1225
|
-
async function*
|
|
1287
|
+
async function* Tn(e, t) {
|
|
1226
1288
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1227
|
-
let n =
|
|
1289
|
+
let n = b(bn), r = b(xn), i = await _n(t, n, yn, {
|
|
1228
1290
|
...Z,
|
|
1229
|
-
maxmem:
|
|
1291
|
+
maxmem: Cn
|
|
1230
1292
|
}), a = Buffer.alloc(Q);
|
|
1231
|
-
|
|
1293
|
+
vn.copy(a, 0), a.writeUInt32BE(Z.N, 8), a.writeUInt32BE(Z.r, 12), a.writeUInt32BE(Z.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1232
1294
|
let o = 0, s = (e, t) => {
|
|
1233
|
-
let n =
|
|
1295
|
+
let n = _("aes-256-gcm", i, wn(r, o, t));
|
|
1234
1296
|
o === 0 && n.setAAD(a);
|
|
1235
1297
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1236
1298
|
o++;
|
|
1237
1299
|
let l = Buffer.alloc(5 + s.length + X);
|
|
1238
1300
|
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + X, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1239
1301
|
}, c = Buffer.alloc(0);
|
|
1240
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1302
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > Sn;) yield s(c.subarray(0, Sn), !1), c = c.subarray(Sn);
|
|
1241
1303
|
yield s(c, !0);
|
|
1242
1304
|
}
|
|
1243
|
-
async function*
|
|
1305
|
+
async function* En(e, t) {
|
|
1244
1306
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1245
1307
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1246
1308
|
let { value: e, done: t } = await n.next();
|
|
@@ -1251,12 +1313,12 @@ async function* yn(e, t) {
|
|
|
1251
1313
|
};
|
|
1252
1314
|
if (!await o(Q)) throw Error("encrypted bundle truncated (no header)");
|
|
1253
1315
|
let s = r.subarray(0, Q);
|
|
1254
|
-
if (!s.subarray(0, 8).equals(
|
|
1255
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await
|
|
1316
|
+
if (!s.subarray(0, 8).equals(vn)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1317
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await _n(t, d, yn, {
|
|
1256
1318
|
N: c,
|
|
1257
1319
|
r: l,
|
|
1258
1320
|
p: u,
|
|
1259
|
-
maxmem:
|
|
1321
|
+
maxmem: Cn
|
|
1260
1322
|
});
|
|
1261
1323
|
r = r.subarray(Q);
|
|
1262
1324
|
let m = 0, h = !1;
|
|
@@ -1271,7 +1333,7 @@ async function* yn(e, t) {
|
|
|
1271
1333
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1272
1334
|
let n = r.subarray(5, 5 + t - X), i = r.subarray(5 + t - X, 5 + t);
|
|
1273
1335
|
r = r.subarray(5 + t);
|
|
1274
|
-
let a =
|
|
1336
|
+
let a = v("aes-256-gcm", p, wn(f, m, e));
|
|
1275
1337
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1276
1338
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1277
1339
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1279,7 +1341,7 @@ async function* yn(e, t) {
|
|
|
1279
1341
|
}
|
|
1280
1342
|
//#endregion
|
|
1281
1343
|
//#region src/blobStorage.ts
|
|
1282
|
-
var
|
|
1344
|
+
var Dn = (e, t) => ({
|
|
1283
1345
|
list: () => {
|
|
1284
1346
|
let e = t();
|
|
1285
1347
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1292,29 +1354,29 @@ var bn = (e, t) => ({
|
|
|
1292
1354
|
} };
|
|
1293
1355
|
},
|
|
1294
1356
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1295
|
-
let r = await n.runPromise(
|
|
1357
|
+
let r = await n.runPromise(te(e, t)), i = await re(r.stream);
|
|
1296
1358
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1297
1359
|
} })
|
|
1298
|
-
}),
|
|
1299
|
-
table:
|
|
1360
|
+
}), On = (e, t) => Dn(e, () => h(t, {
|
|
1361
|
+
table: S,
|
|
1300
1362
|
chunkSize: 500
|
|
1301
1363
|
}).pipe(c.map((e) => ({
|
|
1302
1364
|
key: String(e.key),
|
|
1303
1365
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1304
1366
|
size: Number(e.size ?? 0)
|
|
1305
|
-
})))),
|
|
1367
|
+
})))), kn = (e) => ({
|
|
1306
1368
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1307
1369
|
write: async (t, r, i) => {
|
|
1308
|
-
let a =
|
|
1309
|
-
await n.runPromise(
|
|
1370
|
+
let a = C("storage", () => ae.from(r));
|
|
1371
|
+
await n.runPromise(ne(e, t, a, {
|
|
1310
1372
|
contentType: i.contentType,
|
|
1311
1373
|
totalSize: i.size
|
|
1312
1374
|
}));
|
|
1313
1375
|
}
|
|
1314
|
-
}),
|
|
1376
|
+
}), An = (e, t) => {
|
|
1315
1377
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1316
|
-
return e === "mariadb" &&
|
|
1317
|
-
},
|
|
1378
|
+
return e === "mariadb" && jn(n) ? n : r;
|
|
1379
|
+
}, jn = (e) => (process.env.PATH ?? "").split(fe).filter((e) => e !== "").some((t) => se(O(t, e))), Mn = (e) => {
|
|
1318
1380
|
switch (e) {
|
|
1319
1381
|
case "postgres": return "db.dump";
|
|
1320
1382
|
case "mysql":
|
|
@@ -1324,7 +1386,7 @@ var bn = (e, t) => ({
|
|
|
1324
1386
|
case "mssql": return "db.bacpac";
|
|
1325
1387
|
default: return "db.dump";
|
|
1326
1388
|
}
|
|
1327
|
-
},
|
|
1389
|
+
}, Nn = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1328
1390
|
if (e.host || e.database) return {
|
|
1329
1391
|
host: e.host ?? "localhost",
|
|
1330
1392
|
port: e.port ?? 3306,
|
|
@@ -1340,7 +1402,7 @@ var bn = (e, t) => ({
|
|
|
1340
1402
|
password: decodeURIComponent(t.password) || "",
|
|
1341
1403
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1342
1404
|
};
|
|
1343
|
-
},
|
|
1405
|
+
}, Pn = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Fn = (e, t, n) => {
|
|
1344
1406
|
switch (e) {
|
|
1345
1407
|
case "postgres": {
|
|
1346
1408
|
if (t.url) return {
|
|
@@ -1387,7 +1449,7 @@ var bn = (e, t) => ({
|
|
|
1387
1449
|
let r = $(t);
|
|
1388
1450
|
return {
|
|
1389
1451
|
kind: "spawn",
|
|
1390
|
-
tool:
|
|
1452
|
+
tool: An(e, "dump"),
|
|
1391
1453
|
args: [
|
|
1392
1454
|
"--single-transaction",
|
|
1393
1455
|
"--routines",
|
|
@@ -1409,7 +1471,7 @@ var bn = (e, t) => ({
|
|
|
1409
1471
|
case "sqlite":
|
|
1410
1472
|
case "turso": return {
|
|
1411
1473
|
kind: "copy",
|
|
1412
|
-
from:
|
|
1474
|
+
from: Nn(t),
|
|
1413
1475
|
to: n
|
|
1414
1476
|
};
|
|
1415
1477
|
case "mssql": return {
|
|
@@ -1417,7 +1479,7 @@ var bn = (e, t) => ({
|
|
|
1417
1479
|
tool: "sqlpackage",
|
|
1418
1480
|
args: [
|
|
1419
1481
|
"/Action:Export",
|
|
1420
|
-
`/SourceConnectionString:${
|
|
1482
|
+
`/SourceConnectionString:${Pn($({
|
|
1421
1483
|
...t,
|
|
1422
1484
|
port: t.port ?? 1433
|
|
1423
1485
|
}))}`,
|
|
@@ -1430,7 +1492,7 @@ var bn = (e, t) => ({
|
|
|
1430
1492
|
reason: `no native backup for dialect '${e}'`
|
|
1431
1493
|
});
|
|
1432
1494
|
}
|
|
1433
|
-
},
|
|
1495
|
+
}, In = (e, t, n) => {
|
|
1434
1496
|
switch (e) {
|
|
1435
1497
|
case "postgres": {
|
|
1436
1498
|
if (t.url) return {
|
|
@@ -1475,7 +1537,7 @@ var bn = (e, t) => ({
|
|
|
1475
1537
|
let r = $(t);
|
|
1476
1538
|
return {
|
|
1477
1539
|
kind: "spawn",
|
|
1478
|
-
tool:
|
|
1540
|
+
tool: An(e, "client"),
|
|
1479
1541
|
args: [
|
|
1480
1542
|
"--protocol=TCP",
|
|
1481
1543
|
"-h",
|
|
@@ -1494,14 +1556,14 @@ var bn = (e, t) => ({
|
|
|
1494
1556
|
case "turso": return {
|
|
1495
1557
|
kind: "copy",
|
|
1496
1558
|
from: n,
|
|
1497
|
-
to:
|
|
1559
|
+
to: Nn(t)
|
|
1498
1560
|
};
|
|
1499
1561
|
case "mssql": return {
|
|
1500
1562
|
kind: "spawn",
|
|
1501
1563
|
tool: "sqlpackage",
|
|
1502
1564
|
args: [
|
|
1503
1565
|
"/Action:Import",
|
|
1504
|
-
`/TargetConnectionString:${
|
|
1566
|
+
`/TargetConnectionString:${Pn($({
|
|
1505
1567
|
...t,
|
|
1506
1568
|
port: t.port ?? 1433
|
|
1507
1569
|
}))}`,
|
|
@@ -1514,13 +1576,13 @@ var bn = (e, t) => ({
|
|
|
1514
1576
|
reason: `no native restore for dialect '${e}'`
|
|
1515
1577
|
});
|
|
1516
1578
|
}
|
|
1517
|
-
},
|
|
1518
|
-
|
|
1579
|
+
}, Ln = (e) => e.kind === "copy" ? n.async((t) => {
|
|
1580
|
+
oe(e.from, e.to, (r) => t(r ? n.fail(new L({
|
|
1519
1581
|
tool: "copy",
|
|
1520
1582
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1521
1583
|
})) : n.void));
|
|
1522
1584
|
}) : n.async((t) => {
|
|
1523
|
-
let r =
|
|
1585
|
+
let r = _e(e.tool, [...e.args], {
|
|
1524
1586
|
env: {
|
|
1525
1587
|
...process.env,
|
|
1526
1588
|
...e.env
|
|
@@ -1533,10 +1595,10 @@ var bn = (e, t) => ({
|
|
|
1533
1595
|
}), i = "";
|
|
1534
1596
|
r.stderr?.on("data", (e) => {
|
|
1535
1597
|
i += String(e);
|
|
1536
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(
|
|
1598
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(E(e.stdoutFile)), e.stdinFile && r.stdin && T(e.stdinFile).pipe(r.stdin);
|
|
1537
1599
|
let a = () => {
|
|
1538
1600
|
if (e.stdoutFile !== void 0) try {
|
|
1539
|
-
|
|
1601
|
+
ce(e.stdoutFile);
|
|
1540
1602
|
} catch {}
|
|
1541
1603
|
};
|
|
1542
1604
|
r.on("error", (r) => {
|
|
@@ -1551,15 +1613,15 @@ var bn = (e, t) => ({
|
|
|
1551
1613
|
}
|
|
1552
1614
|
a(), t(n.fail(new L({
|
|
1553
1615
|
tool: e.tool,
|
|
1554
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
1616
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${de(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
1555
1617
|
stderr: i.slice(-2e3)
|
|
1556
1618
|
})));
|
|
1557
1619
|
});
|
|
1558
|
-
}),
|
|
1620
|
+
}), Rn = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), zn = (e) => n.gen(function* () {
|
|
1559
1621
|
let t = yield* V(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
1560
1622
|
store: e.store,
|
|
1561
1623
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
1562
|
-
}), r =
|
|
1624
|
+
}), r = At(t.tables, e.snapshot), i = e.classification ?? k(e.snapshot), a = e.sampleSize ?? 20, o = Ae(r.map((t) => ({
|
|
1563
1625
|
name: t,
|
|
1564
1626
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
1565
1627
|
name: e.name,
|
|
@@ -1567,26 +1629,26 @@ var bn = (e, t) => ({
|
|
|
1567
1629
|
}))
|
|
1568
1630
|
})), i, e.masking), s = [], l = [];
|
|
1569
1631
|
for (let i of r) {
|
|
1570
|
-
let r = t.predicateFor(i), u = yield*
|
|
1632
|
+
let r = t.predicateFor(i), u = yield* h(e.store, {
|
|
1571
1633
|
table: i,
|
|
1572
1634
|
chunkSize: a,
|
|
1573
1635
|
...r === void 0 ? {} : { where: r }
|
|
1574
|
-
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
1575
|
-
l.push(...
|
|
1576
|
-
let f = u[0],
|
|
1577
|
-
if (f &&
|
|
1636
|
+
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) => Ne(t, i, o.actions, e.masking.seed));
|
|
1637
|
+
l.push(...Me(d, i, o.actions));
|
|
1638
|
+
let f = u[0], p = d[0], m = [];
|
|
1639
|
+
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
1578
1640
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
1579
1641
|
let n = e.slice(i.length + 1);
|
|
1580
|
-
|
|
1642
|
+
m.push({
|
|
1581
1643
|
column: n,
|
|
1582
|
-
action:
|
|
1644
|
+
action: Rn(t),
|
|
1583
1645
|
before: f[n],
|
|
1584
|
-
after:
|
|
1646
|
+
after: p[n]
|
|
1585
1647
|
});
|
|
1586
1648
|
}
|
|
1587
1649
|
s.push({
|
|
1588
1650
|
table: i,
|
|
1589
|
-
samples:
|
|
1651
|
+
samples: m
|
|
1590
1652
|
});
|
|
1591
1653
|
}
|
|
1592
1654
|
return {
|
|
@@ -1594,7 +1656,7 @@ var bn = (e, t) => ({
|
|
|
1594
1656
|
leaks: l,
|
|
1595
1657
|
unclassified: o.unclassified
|
|
1596
1658
|
};
|
|
1597
|
-
}),
|
|
1659
|
+
}), Bn = (e, t) => {
|
|
1598
1660
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
1599
1661
|
for (let t of e) {
|
|
1600
1662
|
let e = n.get(t.name);
|
|
@@ -1649,7 +1711,7 @@ var bn = (e, t) => ({
|
|
|
1649
1711
|
skipColumns: i,
|
|
1650
1712
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
1651
1713
|
};
|
|
1652
|
-
},
|
|
1714
|
+
}, Vn = (e) => {
|
|
1653
1715
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
1654
1716
|
let t = {
|
|
1655
1717
|
safe: "✓",
|
|
@@ -1660,4 +1722,4 @@ var bn = (e, t) => ({
|
|
|
1660
1722
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
1661
1723
|
};
|
|
1662
1724
|
//#endregion
|
|
1663
|
-
export { W as ASSETS_DIR, G as ASSET_INDEX_FILE, P as BundleError,
|
|
1725
|
+
export { W as ASSETS_DIR, G as ASSET_INDEX_FILE, P as BundleError, Pe as CodecError, Fe as CompressionError, Be as CrossDialectError, U as DATA_DIR, ye as DEFAULT_CLASS_ACTIONS, Ie as DanglingReferenceError, Qe as FORMAT_VERSION, ze as ImportModeError, F as IntegrityError, ot as LEDGER_FILE, H as MANIFEST_FILE, Re as MaskingError, L as NativeToolError, Le as SchemaDriftError, I as ScopeError, Oe as applyAction, tn as assessLiveImport, gt as assetDone, Mn as backupArtifactName, Fn as backupCommand, Bt as canonicalType, Ut as checkPortability, Zt as checkSchemaDrift, k as classificationFromSnapshot, Bn as classifyImportDrift, Ye as compressionExt, z as computeSubsetIds, it as decodeManifest, Et as decodeRow, wt as decodeValue, En as decryptStream, en as defineDataProfile, Xt as diffSnapshots, gn as dirBlobSink, pt as emptyLedger, rt as encodeManifest, Tt as encodeRow, St as encodeValue, Tn as encryptStream, Vn as formatImportDrift, Wt as formatIssue, qe as hasZstd, Xe as makeCompressor, Ze as makeDecompressor, Ne as maskRow, kt as ndjsonToRows, un as packBundle, mn as peekBundle, Je as pickCompression, Ae as planMasking, zn as previewMasking, mt as readLedger, be as resolveAction, V as resolveScope, Ke as restoreAssetFromCas, In as restoreCommand, Ot as rowsToNdjson, zt as runExport, $t as runImport, Ln as runNativeStep, Me as scanForLeaks, We as storageAssetSink, Ue as storageAssetSource, kn as storageBlobSink, Dn as storageBlobSource, On as storageRefsBlobSource, He as subsetToScope, ht as tableDone, At as topoSortTables, pn as unpackBundle, hn as unpackBundleToDir, Ge as writeAssetToCas, q as writeLedger };
|