@voltro/data-transfer 0.22.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +316 -0
- package/dist/index.d.ts +59 -0
- package/dist/index.js +271 -199
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -5,13 +5,13 @@ import { createCipheriv as h, createDecipheriv as g, createHash as _, createHmac
|
|
|
5
5
|
import { STORAGE_REFS_TABLE as x, fromNodeReadable as S, getObjectStream as C, putObjectStream as ee, toNodeReadable as te } from "@voltro/plugin-storage";
|
|
6
6
|
import * as w from "node:zlib";
|
|
7
7
|
import { PassThrough as T, Readable as ne } from "node:stream";
|
|
8
|
-
import { copyFile as re, createReadStream as E, createWriteStream as ie } from "node:fs";
|
|
9
|
-
import { mkdir as
|
|
10
|
-
import { dirname as
|
|
11
|
-
import { promisify as
|
|
12
|
-
import { spawn as
|
|
8
|
+
import { copyFile as re, createReadStream as E, createWriteStream as ie, existsSync as ae } from "node:fs";
|
|
9
|
+
import { mkdir as oe, readdir as se, stat as ce } from "node:fs/promises";
|
|
10
|
+
import { delimiter as le, dirname as ue, join as D, resolve as de, sep as fe } from "node:path";
|
|
11
|
+
import { promisify as pe } from "node:util";
|
|
12
|
+
import { spawn as me } from "node:child_process";
|
|
13
13
|
//#region src/consistency.ts
|
|
14
|
-
var
|
|
14
|
+
var he = (e) => {
|
|
15
15
|
switch (e) {
|
|
16
16
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
17
17
|
case "mysql":
|
|
@@ -21,7 +21,7 @@ var pe = (e) => {
|
|
|
21
21
|
case "turso": return [];
|
|
22
22
|
default: return [];
|
|
23
23
|
}
|
|
24
|
-
},
|
|
24
|
+
}, ge = {
|
|
25
25
|
email: { fake: "email" },
|
|
26
26
|
fullName: { fake: "fullName" },
|
|
27
27
|
firstName: { fake: "firstName" },
|
|
@@ -48,12 +48,12 @@ var pe = (e) => {
|
|
|
48
48
|
t[n.name] = e;
|
|
49
49
|
}
|
|
50
50
|
return t;
|
|
51
|
-
},
|
|
51
|
+
}, _e = (e, t, n, r) => {
|
|
52
52
|
let i = r.columns?.[`${e}.${t}`];
|
|
53
53
|
if (i) return i;
|
|
54
54
|
if (n?.sensitive) {
|
|
55
55
|
let e = n.sensitive.class;
|
|
56
|
-
return r.classes?.[e] ??
|
|
56
|
+
return r.classes?.[e] ?? ge[e] ?? "redact";
|
|
57
57
|
}
|
|
58
58
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
59
59
|
}, k = (e, t) => v("sha256", e).update(t == null ? "∅" : String(t)).digest(), A = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), j = [
|
|
@@ -69,7 +69,7 @@ var pe = (e) => {
|
|
|
69
69
|
"Guido",
|
|
70
70
|
"Margaret",
|
|
71
71
|
"Bjarne"
|
|
72
|
-
],
|
|
72
|
+
], ve = [
|
|
73
73
|
"Lovelace",
|
|
74
74
|
"Turing",
|
|
75
75
|
"Hopper",
|
|
@@ -82,7 +82,7 @@ var pe = (e) => {
|
|
|
82
82
|
"Rossum",
|
|
83
83
|
"Hamilton",
|
|
84
84
|
"Stroustrup"
|
|
85
|
-
],
|
|
85
|
+
], ye = [
|
|
86
86
|
"Maple",
|
|
87
87
|
"Oak",
|
|
88
88
|
"Cedar",
|
|
@@ -91,7 +91,7 @@ var pe = (e) => {
|
|
|
91
91
|
"Birch",
|
|
92
92
|
"Willow",
|
|
93
93
|
"Ash"
|
|
94
|
-
],
|
|
94
|
+
], be = [
|
|
95
95
|
"Initech",
|
|
96
96
|
"Hooli",
|
|
97
97
|
"Umbrella",
|
|
@@ -100,23 +100,23 @@ var pe = (e) => {
|
|
|
100
100
|
"Soylent",
|
|
101
101
|
"Stark",
|
|
102
102
|
"Wayne"
|
|
103
|
-
], M = (e, t, n) => e[A(t, n) % e.length],
|
|
103
|
+
], M = (e, t, n) => e[A(t, n) % e.length], xe = (e, t, n) => {
|
|
104
104
|
let r = k(n, `${e}:${String(t)}`), i = A(r, 1) % 1e4;
|
|
105
105
|
switch (e) {
|
|
106
|
-
case "email": return `${M(j, r, 0).toLowerCase()}.${M(
|
|
107
|
-
case "fullName": return `${M(j, r, 0)} ${M(
|
|
106
|
+
case "email": return `${M(j, r, 0).toLowerCase()}.${M(ve, r, 2).toLowerCase()}${i}@example.com`;
|
|
107
|
+
case "fullName": return `${M(j, r, 0)} ${M(ve, r, 1)}`;
|
|
108
108
|
case "firstName": return M(j, r, 0);
|
|
109
|
-
case "lastName": return M(
|
|
109
|
+
case "lastName": return M(ve, r, 1);
|
|
110
110
|
case "username": return `${M(j, r, 0).toLowerCase()}${i}`;
|
|
111
111
|
case "phone": return `+1${String(A(r, 0) % 1e10).padStart(10, "0")}`;
|
|
112
|
-
case "address": return `${A(r, 0) % 9899 + 100} ${M(
|
|
113
|
-
case "company": return M(
|
|
112
|
+
case "address": return `${A(r, 0) % 9899 + 100} ${M(ye, r, 1)} St`;
|
|
113
|
+
case "company": return M(be, r, 0);
|
|
114
114
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
115
115
|
case "ip": return `${A(r, 0) % 256}.${A(r, 1) % 256}.${A(r, 2) % 256}.${A(r, 3) % 254 + 1}`;
|
|
116
|
-
case "creditCard": return
|
|
116
|
+
case "creditCard": return Se(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
117
117
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
118
118
|
}
|
|
119
|
-
},
|
|
119
|
+
}, Se = (e) => {
|
|
120
120
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
121
121
|
t.forEach((e, t) => {
|
|
122
122
|
let r = e;
|
|
@@ -124,26 +124,26 @@ var pe = (e) => {
|
|
|
124
124
|
});
|
|
125
125
|
let r = (10 - n % 10) % 10;
|
|
126
126
|
return e.slice(0, 15) + String(r);
|
|
127
|
-
},
|
|
127
|
+
}, Ce = 864e5, we = (e, t) => {
|
|
128
128
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
129
129
|
if (Number.isNaN(n)) return e;
|
|
130
|
-
let r = A(k(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r *
|
|
130
|
+
let r = A(k(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Ce);
|
|
131
131
|
return e instanceof Date ? i : i.toISOString();
|
|
132
|
-
},
|
|
132
|
+
}, Te = (e, t) => {
|
|
133
133
|
if (t.value === null || t.value === void 0) return null;
|
|
134
134
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
135
|
-
return e === "keep" ? t.value : e === "null" ? null : e === "redact" ? t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null : e === "hash" ? k(t.seed, t.value).toString("hex").slice(0, 16) : e === "dateShift" ?
|
|
135
|
+
return e === "keep" ? t.value : e === "null" ? null : e === "redact" ? t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null : e === "hash" ? k(t.seed, t.value).toString("hex").slice(0, 16) : e === "dateShift" ? we(t.value, t.seed) : typeof e == "object" && "fake" in e ? xe(e.fake, t.value, t.seed) : typeof e == "object" && "custom" in e ? e.custom(t) : t.value;
|
|
136
136
|
}, N = (e, t, n) => {
|
|
137
137
|
let r = {}, i = [];
|
|
138
138
|
for (let a of e) for (let e of a.columns) {
|
|
139
|
-
let o = `${a.name}.${e.name}`, s =
|
|
139
|
+
let o = `${a.name}.${e.name}`, s = _e(a.name, e.name, t[a.name]?.[e.name], n);
|
|
140
140
|
s === "error" ? i.push(o) : r[o] = s;
|
|
141
141
|
}
|
|
142
142
|
return {
|
|
143
143
|
actions: r,
|
|
144
144
|
unclassified: i
|
|
145
145
|
};
|
|
146
|
-
},
|
|
146
|
+
}, Ee = [
|
|
147
147
|
{
|
|
148
148
|
name: "email",
|
|
149
149
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -160,7 +160,7 @@ var pe = (e) => {
|
|
|
160
160
|
name: "creditCard",
|
|
161
161
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
162
162
|
}
|
|
163
|
-
],
|
|
163
|
+
], De = (e, t, n) => {
|
|
164
164
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
165
165
|
for (let [a, o] of Object.entries(n)) {
|
|
166
166
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -168,7 +168,7 @@ var pe = (e) => {
|
|
|
168
168
|
for (let a of e) {
|
|
169
169
|
let e = a[n];
|
|
170
170
|
if (typeof e != "string" || e.length === 0) continue;
|
|
171
|
-
let o =
|
|
171
|
+
let o = Ee.find((t) => t.re.test(e));
|
|
172
172
|
if (o && !i.has(n)) {
|
|
173
173
|
i.add(n), r.push({
|
|
174
174
|
table: t,
|
|
@@ -183,7 +183,7 @@ var pe = (e) => {
|
|
|
183
183
|
return r;
|
|
184
184
|
}, P = (e, t, n, r, i) => {
|
|
185
185
|
let a = {};
|
|
186
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
186
|
+
for (let [o, s] of Object.entries(e)) a[o] = Te(n[`${t}.${o}`] ?? "keep", {
|
|
187
187
|
value: s,
|
|
188
188
|
table: t,
|
|
189
189
|
column: o,
|
|
@@ -194,35 +194,40 @@ var pe = (e) => {
|
|
|
194
194
|
}, F = class extends s.TaggedError()("BundleError", {
|
|
195
195
|
reason: s.String,
|
|
196
196
|
path: s.optional(s.String)
|
|
197
|
-
}) {},
|
|
197
|
+
}) {}, Oe = class extends s.TaggedError()("CodecError", {
|
|
198
198
|
table: s.String,
|
|
199
199
|
reason: s.String
|
|
200
200
|
}) {}, I = class extends s.TaggedError()("IntegrityError", {
|
|
201
201
|
what: s.String,
|
|
202
202
|
expected: s.String,
|
|
203
203
|
actual: s.String
|
|
204
|
-
}) {},
|
|
204
|
+
}) {}, ke = class extends t.TaggedError("CompressionError") {}, Ae = class extends s.TaggedError()("ScopeError", { reason: s.String }) {}, je = class extends s.TaggedError()("DanglingReferenceError", {
|
|
205
205
|
rows: s.Array(s.Struct({
|
|
206
206
|
table: s.String,
|
|
207
207
|
id: s.String,
|
|
208
208
|
reason: s.String
|
|
209
209
|
})),
|
|
210
210
|
totalCount: s.Number
|
|
211
|
-
}) {},
|
|
211
|
+
}) {}, Me = class extends s.TaggedError()("SchemaDriftError", {
|
|
212
212
|
bundleFingerprint: s.String,
|
|
213
213
|
targetFingerprint: s.String,
|
|
214
214
|
diff: s.Array(s.String)
|
|
215
|
-
}) {}, L = class extends t.TaggedError("NativeToolError") {
|
|
215
|
+
}) {}, L = class extends t.TaggedError("NativeToolError") {
|
|
216
|
+
get message() {
|
|
217
|
+
let e = this.stderr?.trim();
|
|
218
|
+
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
219
|
+
}
|
|
220
|
+
}, Ne = class extends s.TaggedError()("MaskingError", { unclassified: s.Array(s.String) }) {}, Pe = class extends s.TaggedError()("ImportModeError", {
|
|
216
221
|
mode: s.String,
|
|
217
222
|
reason: s.String
|
|
218
|
-
}) {},
|
|
223
|
+
}) {}, Fe = class extends s.TaggedError()("CrossDialectError", {
|
|
219
224
|
sourceDialect: s.String,
|
|
220
225
|
targetDialect: s.String,
|
|
221
226
|
blocking: s.Array(s.String)
|
|
222
|
-
}) {}, R = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id",
|
|
227
|
+
}) {}, R = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Ie = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
223
228
|
col: e.name,
|
|
224
229
|
refTable: e.references.table
|
|
225
|
-
})),
|
|
230
|
+
})), Le = (e, t, r) => n.gen(function* () {
|
|
226
231
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map(), d = (e) => {
|
|
227
232
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
228
233
|
return s.set(e, t), t;
|
|
@@ -240,7 +245,7 @@ var pe = (e) => {
|
|
|
240
245
|
for (let t of n) m(t.refTable, e[t.col]);
|
|
241
246
|
};
|
|
242
247
|
}, g = (r, i) => {
|
|
243
|
-
let a = h(r, R(t, r),
|
|
248
|
+
let a = h(r, R(t, r), Ie(t, r));
|
|
244
249
|
return c.runForEach(p(e, {
|
|
245
250
|
table: r,
|
|
246
251
|
pk: R(t, r),
|
|
@@ -275,13 +280,13 @@ var pe = (e) => {
|
|
|
275
280
|
}
|
|
276
281
|
}
|
|
277
282
|
return o;
|
|
278
|
-
}),
|
|
283
|
+
}), Re = (e, t) => ({
|
|
279
284
|
tables: [...e.keys()],
|
|
280
285
|
predicateFor: (n) => {
|
|
281
286
|
let r = e.get(n);
|
|
282
287
|
return r ? u(R(t, n), [...r]) : void 0;
|
|
283
288
|
}
|
|
284
|
-
}), z = "tenantId",
|
|
289
|
+
}), z = "tenantId", ze = (e, t, r) => n.gen(function* () {
|
|
285
290
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
286
291
|
if (e.kind === "all") return {
|
|
287
292
|
tables: i,
|
|
@@ -295,10 +300,10 @@ var pe = (e) => {
|
|
|
295
300
|
};
|
|
296
301
|
}
|
|
297
302
|
let o = e.tenantId;
|
|
298
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
303
|
+
if (o === void 0 || o === "") return yield* n.fail(new Ae({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
299
304
|
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === z) ?? !1);
|
|
300
|
-
if (s.length === 0) return yield* n.fail(new
|
|
301
|
-
let c = yield*
|
|
305
|
+
if (s.length === 0) return yield* n.fail(new Ae({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${z}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
306
|
+
let c = yield* Le(r.store, t, {
|
|
302
307
|
seeds: Object.fromEntries(s.map((e) => [e, l(z, o)])),
|
|
303
308
|
pinned: s,
|
|
304
309
|
children: { roots: s }
|
|
@@ -307,13 +312,13 @@ var pe = (e) => {
|
|
|
307
312
|
tables: [...s, ...d],
|
|
308
313
|
predicateFor: (e) => f.has(e) ? l(z, o) : c.has(e) ? u(R(t, e), [...c.get(e)]) : void 0
|
|
309
314
|
};
|
|
310
|
-
}),
|
|
315
|
+
}), Be = (e, t) => ({
|
|
311
316
|
list: t,
|
|
312
317
|
open: (t) => C(e, t)
|
|
313
|
-
}),
|
|
318
|
+
}), Ve = (e) => ({
|
|
314
319
|
write: (t, n, r) => ee(e, t, n, r),
|
|
315
320
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
316
|
-
}),
|
|
321
|
+
}), He = (e, t) => n.tryPromise({
|
|
317
322
|
try: async () => {
|
|
318
323
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
319
324
|
u.update(e), d += e.length, n(null, e);
|
|
@@ -335,7 +340,7 @@ var pe = (e) => {
|
|
|
335
340
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
336
341
|
path: t
|
|
337
342
|
})
|
|
338
|
-
}),
|
|
343
|
+
}), Ue = (e, t, r) => n.gen(function* () {
|
|
339
344
|
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), l = o("sha256"), u = S("bundle", () => a(s)).pipe(c.tap((e) => n.sync(() => l.update(e))));
|
|
340
345
|
yield* e.write(r.key, u, {
|
|
341
346
|
contentType: r.contentType,
|
|
@@ -347,7 +352,7 @@ var pe = (e) => {
|
|
|
347
352
|
expected: r.sha256,
|
|
348
353
|
actual: d
|
|
349
354
|
}));
|
|
350
|
-
}),
|
|
355
|
+
}), We = () => typeof w.createZstdCompress == "function", Ge = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : We() ? "zstd" : "gzip", Ke = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", qe = (e) => e === "zstd" ? w.createZstdCompress() : e === "gzip" ? w.createGzip() : new T(), Je = (e) => e === "zstd" ? w.createZstdDecompress() : e === "gzip" ? w.createGunzip() : new T(), Ye = 1, Xe = s.Struct({
|
|
351
356
|
name: s.String,
|
|
352
357
|
file: s.String,
|
|
353
358
|
rowCount: s.Number,
|
|
@@ -357,11 +362,11 @@ var pe = (e) => {
|
|
|
357
362
|
value: s.String
|
|
358
363
|
}),
|
|
359
364
|
primaryKey: s.String
|
|
360
|
-
}),
|
|
365
|
+
}), Ze = s.Struct({
|
|
361
366
|
kind: s.Literal("all", "tenant", "tables"),
|
|
362
367
|
tenantId: s.optional(s.String),
|
|
363
368
|
tables: s.optional(s.Array(s.String))
|
|
364
|
-
}),
|
|
369
|
+
}), Qe = s.Struct({
|
|
365
370
|
count: s.Number,
|
|
366
371
|
totalBytes: s.Number,
|
|
367
372
|
index: s.optional(s.String)
|
|
@@ -372,23 +377,23 @@ s.Struct({
|
|
|
372
377
|
size: s.Number,
|
|
373
378
|
contentType: s.String
|
|
374
379
|
});
|
|
375
|
-
var
|
|
380
|
+
var $e = s.Struct({
|
|
376
381
|
formatVersion: s.Number,
|
|
377
382
|
createdAt: s.String,
|
|
378
383
|
source: s.Struct({
|
|
379
384
|
dialect: s.String,
|
|
380
385
|
schemaFingerprint: s.String
|
|
381
386
|
}),
|
|
382
|
-
scope:
|
|
387
|
+
scope: Ze,
|
|
383
388
|
consistency: s.Literal("live", "snapshot"),
|
|
384
389
|
compression: s.Literal("zstd", "gzip", "none"),
|
|
385
|
-
tables: s.Array(
|
|
386
|
-
assets:
|
|
390
|
+
tables: s.Array(Xe),
|
|
391
|
+
assets: Qe,
|
|
387
392
|
masking: s.optional(s.Struct({
|
|
388
393
|
policyId: s.optional(s.String),
|
|
389
394
|
transformed: s.Array(s.String)
|
|
390
395
|
}))
|
|
391
|
-
}),
|
|
396
|
+
}), et = s.encodeSync(s.parseJson($e)), tt = s.decodeUnknownSync(s.parseJson($e)), B = "manifest.json", nt = "README.md", V = "data", H = "assets", U = "assets/index.ndjson", rt = ".ledger.json", it = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", at = (e, t) => {
|
|
392
397
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
393
398
|
return `# Voltro data export
|
|
394
399
|
|
|
@@ -402,7 +407,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
402
407
|
| **Format version** | ${e.formatVersion} |
|
|
403
408
|
| **Consistency** | ${e.consistency} |
|
|
404
409
|
| **Compression** | ${e.compression} |
|
|
405
|
-
| **Scope** | ${
|
|
410
|
+
| **Scope** | ${it(e.scope)} |
|
|
406
411
|
| **Tables** | ${e.tables.length} |
|
|
407
412
|
| **Rows (total)** | ${n} |
|
|
408
413
|
| **Assets** | ${a} |
|
|
@@ -427,26 +432,26 @@ voltro data import <this-directory>
|
|
|
427
432
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
428
433
|
\`\`\`
|
|
429
434
|
`;
|
|
430
|
-
},
|
|
435
|
+
}, ot = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), st = a.exponential({
|
|
431
436
|
start: .01,
|
|
432
437
|
factor: 2,
|
|
433
438
|
count: 16
|
|
434
|
-
}),
|
|
439
|
+
}), ct = i.histogram("voltro_data_transfer_table_seconds", st, "Per-table export/import duration in seconds, tagged by phase."), W = (e, t) => {
|
|
435
440
|
if (e) try {
|
|
436
441
|
e(t);
|
|
437
442
|
} catch {}
|
|
438
|
-
},
|
|
443
|
+
}, lt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(ct.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), ot.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
439
444
|
table: t,
|
|
440
445
|
index: r,
|
|
441
446
|
tableCount: a,
|
|
442
447
|
phase: e
|
|
443
|
-
} })),
|
|
448
|
+
} })), ut = () => ({
|
|
444
449
|
tables: {},
|
|
445
450
|
assets: {}
|
|
446
451
|
}), G = (e) => n.tryPromise(async () => {
|
|
447
452
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
448
453
|
return JSON.parse(t);
|
|
449
|
-
}).pipe(n.orElseSucceed(
|
|
454
|
+
}).pipe(n.orElseSucceed(ut)), K = (e, t) => n.tryPromise({
|
|
450
455
|
try: async () => {
|
|
451
456
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
452
457
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
@@ -455,11 +460,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
455
460
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
456
461
|
path: e
|
|
457
462
|
})
|
|
458
|
-
}),
|
|
463
|
+
}), dt = (e, t) => t in e.tables, ft = (e, t) => t in e.assets, pt = /* @__PURE__ */ new Set([
|
|
459
464
|
"json",
|
|
460
465
|
"array",
|
|
461
466
|
"vector"
|
|
462
|
-
]),
|
|
467
|
+
]), mt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", ht = (e) => Buffer.from(e).toString("base64"), gt = (e) => new Uint8Array(Buffer.from(e, "base64")), _t = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), vt = (e, t) => e == null ? null : t !== void 0 && pt.has(t) ? e : t === "bytes" ? ht(e) : t === "timestamp" || t === "date" ? _t(e) : typeof e == "bigint" ? {
|
|
463
468
|
$v: "bigint",
|
|
464
469
|
d: e.toString()
|
|
465
470
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -470,26 +475,26 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
470
475
|
d: e.toISOString()
|
|
471
476
|
} : e instanceof Uint8Array ? {
|
|
472
477
|
$v: "bytes",
|
|
473
|
-
d:
|
|
474
|
-
} : e,
|
|
478
|
+
d: ht(e)
|
|
479
|
+
} : e, yt = (e) => {
|
|
475
480
|
switch (e.$v) {
|
|
476
481
|
case "bigint": return BigInt(e.d);
|
|
477
482
|
case "number": return Number(e.d);
|
|
478
483
|
case "date": return new Date(e.d);
|
|
479
|
-
case "bytes": return
|
|
484
|
+
case "bytes": return gt(e.d);
|
|
480
485
|
default: return e;
|
|
481
486
|
}
|
|
482
|
-
},
|
|
487
|
+
}, bt = (e, t) => e == null ? null : t !== void 0 && pt.has(t) ? e : mt(e) ? yt(e) : t === "bytes" ? gt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, xt = (e, t) => {
|
|
483
488
|
let n = {};
|
|
484
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
489
|
+
for (let [r, i] of Object.entries(e)) n[r] = vt(i, t[r]);
|
|
485
490
|
return n;
|
|
486
|
-
},
|
|
491
|
+
}, St = (e, t) => {
|
|
487
492
|
let n = {};
|
|
488
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
493
|
+
for (let [r, i] of Object.entries(e)) n[r] = bt(i, t[r]);
|
|
489
494
|
return n;
|
|
490
|
-
},
|
|
491
|
-
try: () =>
|
|
492
|
-
catch: (e) => new
|
|
495
|
+
}, Ct = new TextEncoder(), wt = (e, t) => e.pipe(c.map((e) => Ct.encode(`${JSON.stringify(xt(e, t))}\n`))), Tt = (e, t, r) => e.pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.mapEffect((e) => n.try({
|
|
496
|
+
try: () => St(JSON.parse(e), r),
|
|
497
|
+
catch: (e) => new Oe({
|
|
493
498
|
table: t,
|
|
494
499
|
reason: String(e?.message ?? e)
|
|
495
500
|
})
|
|
@@ -513,11 +518,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
513
518
|
};
|
|
514
519
|
for (let t of e) n.has(t) && s(t);
|
|
515
520
|
return i;
|
|
516
|
-
},
|
|
521
|
+
}, Et = (e, t) => {
|
|
517
522
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
518
523
|
for (let e of n?.columns ?? []) r[e.name] = e.type;
|
|
519
524
|
return r;
|
|
520
|
-
},
|
|
525
|
+
}, Dt = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Ot = (e, t, r) => n.tryPromise({
|
|
521
526
|
try: async () => {
|
|
522
527
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: l, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
523
528
|
d.update(e);
|
|
@@ -526,7 +531,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
526
531
|
n(null, e);
|
|
527
532
|
} }), m = l.fromWeb(c.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
528
533
|
try {
|
|
529
|
-
await s(m, p,
|
|
534
|
+
await s(m, p, qe(r), n(h)), await i.rename(h, t);
|
|
530
535
|
} catch (e) {
|
|
531
536
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
532
537
|
}
|
|
@@ -539,7 +544,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
539
544
|
reason: `write table failed: ${String(e?.message ?? e)}`,
|
|
540
545
|
path: t
|
|
541
546
|
})
|
|
542
|
-
}),
|
|
547
|
+
}), kt = (e, t) => n.tryPromise({
|
|
543
548
|
try: async () => {
|
|
544
549
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
545
550
|
},
|
|
@@ -547,11 +552,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
547
552
|
reason: `append failed: ${String(t?.message ?? t)}`,
|
|
548
553
|
path: e
|
|
549
554
|
})
|
|
550
|
-
}),
|
|
555
|
+
}), At = (e, t) => n.gen(function* () {
|
|
551
556
|
let n = [], r = t.ordered.length;
|
|
552
557
|
for (let i = 0; i < t.ordered.length; i++) {
|
|
553
|
-
let a = t.ordered[i], o =
|
|
554
|
-
if (
|
|
558
|
+
let a = t.ordered[i], o = Et(t.snapshot, a), s = Dt(t.snapshot, a), l = `${V}/${a}.ndjson${Ke(t.compression)}`;
|
|
559
|
+
if (dt(t.ledger, a)) {
|
|
555
560
|
let e = t.ledger.tables[a];
|
|
556
561
|
W(t.onProgress, {
|
|
557
562
|
phase: "export",
|
|
@@ -585,7 +590,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
585
590
|
chunkSize: t.chunkSize,
|
|
586
591
|
retry: t.retry,
|
|
587
592
|
...u === void 0 ? {} : { where: u }
|
|
588
|
-
}), f = t.mask ? d.pipe(c.map((e) => P(e, a, t.mask.actions, t.mask.seed, o))) : d, { rowCount: m, checksum: h } = yield*
|
|
593
|
+
}), f = t.mask ? d.pipe(c.map((e) => P(e, a, t.mask.actions, t.mask.seed, o))) : d, { rowCount: m, checksum: h } = yield* lt("export", a, i, r, (e) => e.rowCount, Ot(wt(f, o), t.bundleFile(l), t.compression));
|
|
589
594
|
t.ledger.tables[a] = {
|
|
590
595
|
rowCount: m,
|
|
591
596
|
checksum: h
|
|
@@ -607,20 +612,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
607
612
|
});
|
|
608
613
|
}
|
|
609
614
|
return n;
|
|
610
|
-
}),
|
|
615
|
+
}), jt = (e) => ({
|
|
611
616
|
_tag: "RawSqlFragment",
|
|
612
617
|
strings: [e],
|
|
613
618
|
values: []
|
|
614
|
-
}),
|
|
619
|
+
}), Mt = (t, i, a) => n.tryPromise({
|
|
615
620
|
try: () => t.transactional(async (t) => {
|
|
616
|
-
if (t.raw) for (let e of
|
|
621
|
+
if (t.raw) for (let e of he(i)) await t.raw(jt(e));
|
|
617
622
|
let o = await n.runPromiseExit(a(t));
|
|
618
623
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
619
624
|
return o.value;
|
|
620
625
|
}),
|
|
621
626
|
catch: (e) => e instanceof F ? e : new F({ reason: `snapshot export failed: ${String(e?.message ?? e)}` })
|
|
622
|
-
}),
|
|
623
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
627
|
+
}), Nt = (e) => n.gen(function* () {
|
|
628
|
+
let t = e.scope ?? { kind: "all" }, r = Ge(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, s = o.exponential("50 millis").pipe(o.jittered, o.intersect(o.recurs(a))), l = yield* n.promise(() => import("node:path")), u = (t) => l.join(e.outDir, t);
|
|
624
629
|
yield* n.tryPromise({
|
|
625
630
|
try: async () => {
|
|
626
631
|
let t = await import("node:fs/promises");
|
|
@@ -631,7 +636,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
631
636
|
path: e.outDir
|
|
632
637
|
})
|
|
633
638
|
});
|
|
634
|
-
let d = u(
|
|
639
|
+
let d = u(rt), f = yield* G(d), p = e.subset ? Re(yield* Le(e.store, e.snapshot, e.subset), e.snapshot) : yield* ze(t, e.snapshot, {
|
|
635
640
|
store: e.store,
|
|
636
641
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
637
642
|
}), h = q(p.tables, e.snapshot), g = e.consistency ?? "live", _, v;
|
|
@@ -643,7 +648,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
643
648
|
type: e.type
|
|
644
649
|
}))
|
|
645
650
|
})), t, e.masking);
|
|
646
|
-
if (r.unclassified.length > 0) return yield* n.fail(new
|
|
651
|
+
if (r.unclassified.length > 0) return yield* n.fail(new Ne({ unclassified: r.unclassified }));
|
|
647
652
|
v = {
|
|
648
653
|
actions: r.actions,
|
|
649
654
|
seed: e.masking.seed
|
|
@@ -666,17 +671,17 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
666
671
|
bundleFile: u,
|
|
667
672
|
...v ? { mask: v } : {},
|
|
668
673
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
669
|
-
}, b = g === "snapshot" ? yield*
|
|
674
|
+
}, b = g === "snapshot" ? yield* Mt(e.store, e.dialect, (e) => At(e, y)) : yield* At(e.store, y), x = 0, S = 0;
|
|
670
675
|
if (e.assets) {
|
|
671
676
|
let t = u(U);
|
|
672
|
-
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
677
|
+
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => He(e, u(H))), n.flatMap((e) => {
|
|
673
678
|
let i = {
|
|
674
679
|
key: r.key,
|
|
675
680
|
sha256: e.sha256,
|
|
676
681
|
size: e.size,
|
|
677
682
|
contentType: e.contentType
|
|
678
683
|
};
|
|
679
|
-
return x++, S += e.size, f.assets[r.key] = !0,
|
|
684
|
+
return x++, S += e.size, f.assets[r.key] = !0, kt(t, JSON.stringify(i)).pipe(n.andThen(K(d, f)));
|
|
680
685
|
}))).pipe(n.mapError((e) => e instanceof F ? e : new F({ reason: `asset export failed: ${String(e)}` })));
|
|
681
686
|
}
|
|
682
687
|
let C = {
|
|
@@ -702,7 +707,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
702
707
|
};
|
|
703
708
|
return yield* n.tryPromise({
|
|
704
709
|
try: async () => {
|
|
705
|
-
await (await import("node:fs/promises")).writeFile(u(B),
|
|
710
|
+
await (await import("node:fs/promises")).writeFile(u(B), et(C));
|
|
706
711
|
},
|
|
707
712
|
catch: (e) => new F({
|
|
708
713
|
reason: `manifest write failed: ${String(e?.message ?? e)}`,
|
|
@@ -711,14 +716,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
711
716
|
}), yield* n.tryPromise({
|
|
712
717
|
try: async () => {
|
|
713
718
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
714
|
-
await e.writeFile(u(
|
|
719
|
+
await e.writeFile(u(nt), at(C, { hostname: t.hostname() }));
|
|
715
720
|
},
|
|
716
721
|
catch: (e) => new F({
|
|
717
722
|
reason: `readme write failed: ${String(e?.message ?? e)}`,
|
|
718
|
-
path:
|
|
723
|
+
path: nt
|
|
719
724
|
})
|
|
720
725
|
}), C;
|
|
721
|
-
}),
|
|
726
|
+
}), Pt = (e) => {
|
|
722
727
|
switch (e) {
|
|
723
728
|
case "id":
|
|
724
729
|
case "text":
|
|
@@ -737,12 +742,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
737
742
|
case "raw": return "raw";
|
|
738
743
|
default: return "string";
|
|
739
744
|
}
|
|
740
|
-
},
|
|
745
|
+
}, Ft = {
|
|
741
746
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
742
747
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
743
748
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
744
|
-
},
|
|
745
|
-
let i =
|
|
749
|
+
}, It = (e, t, n, r) => {
|
|
750
|
+
let i = Pt(n);
|
|
746
751
|
if (i === "raw") return {
|
|
747
752
|
table: e,
|
|
748
753
|
column: t,
|
|
@@ -750,7 +755,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
750
755
|
severity: "block",
|
|
751
756
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
752
757
|
};
|
|
753
|
-
let a =
|
|
758
|
+
let a = Ft[i];
|
|
754
759
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
755
760
|
table: e,
|
|
756
761
|
column: t,
|
|
@@ -764,20 +769,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
764
769
|
severity: "block",
|
|
765
770
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
766
771
|
};
|
|
767
|
-
},
|
|
772
|
+
}, Lt = (e, t, n) => {
|
|
768
773
|
if (t === n) return [];
|
|
769
774
|
let r = [];
|
|
770
775
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
771
|
-
let a =
|
|
776
|
+
let a = It(t.name, e, i, n);
|
|
772
777
|
a && r.push(a);
|
|
773
778
|
}
|
|
774
779
|
return r;
|
|
775
|
-
},
|
|
780
|
+
}, Rt = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, zt = (e, t) => new Map((e.tables.find((e) => e.name === t)?.columns ?? []).map((e) => [e.name, e.type])), Bt = (e, t) => {
|
|
776
781
|
let n = [], r = new Set(e.tables.map((e) => e.name)), i = new Set(t.tables.map((e) => e.name));
|
|
777
782
|
for (let e of [...r].filter((e) => !i.has(e)).sort()) n.push(`table '${e}' is in the bundle but MISSING from the target`);
|
|
778
783
|
for (let e of [...i].filter((e) => !r.has(e)).sort()) n.push(`table '${e}' is in the target but NOT in the bundle (import leaves it untouched)`);
|
|
779
784
|
for (let a of [...r].filter((e) => i.has(e)).sort()) {
|
|
780
|
-
let r =
|
|
785
|
+
let r = zt(e, a), i = zt(t, a);
|
|
781
786
|
for (let [e] of r) i.has(e) || n.push(`${a}.${e}: in the bundle, MISSING from the target`);
|
|
782
787
|
for (let [e] of i) r.has(e) || n.push(`${a}.${e}: in the target, not in the bundle`);
|
|
783
788
|
for (let [e, t] of r) {
|
|
@@ -786,7 +791,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
786
791
|
}
|
|
787
792
|
}
|
|
788
793
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n.length > 0 ? n : ["the schema shape differs (indexes, nullability, or defaults) — the specific change is below the table/column granularity shown here"];
|
|
789
|
-
},
|
|
794
|
+
}, Vt = (e, t, n) => {
|
|
790
795
|
let r = m(t);
|
|
791
796
|
return r === e ? {
|
|
792
797
|
drifted: !1,
|
|
@@ -797,19 +802,19 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
797
802
|
drifted: !0,
|
|
798
803
|
bundleFingerprint: e,
|
|
799
804
|
targetFingerprint: r,
|
|
800
|
-
diff: n ?
|
|
805
|
+
diff: n ? Bt(n, t) : ["the target schema fingerprint does not match the bundle (no source schema available to itemise the diff — the manifest records only the fingerprint)"]
|
|
801
806
|
};
|
|
802
|
-
},
|
|
807
|
+
}, Ht = (e) => n.tryPromise({
|
|
803
808
|
try: async () => {
|
|
804
809
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
805
|
-
return
|
|
810
|
+
return tt(await t.readFile(n.join(e, B), "utf8"));
|
|
806
811
|
},
|
|
807
812
|
catch: (e) => new F({
|
|
808
813
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
809
814
|
path: B
|
|
810
815
|
})
|
|
811
|
-
}),
|
|
812
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
816
|
+
}), Ut = (t) => n.gen(function* () {
|
|
817
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* Ht(t.bundleDir);
|
|
813
818
|
if (a.formatVersion !== 1) return yield* n.fail(new F({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
814
819
|
if (t.targetSnapshot) {
|
|
815
820
|
let e = { tables: a.tables.map((e) => ({
|
|
@@ -822,9 +827,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
822
827
|
unique: !1,
|
|
823
828
|
hasDefault: !1
|
|
824
829
|
}))
|
|
825
|
-
})) }, r =
|
|
830
|
+
})) }, r = Vt(a.source.schemaFingerprint, t.targetSnapshot, e);
|
|
826
831
|
if (r.drifted) {
|
|
827
|
-
if (!t.force) return yield* n.fail(new
|
|
832
|
+
if (!t.force) return yield* n.fail(new Me({
|
|
828
833
|
bundleFingerprint: r.bundleFingerprint,
|
|
829
834
|
targetFingerprint: r.targetFingerprint,
|
|
830
835
|
diff: r.diff
|
|
@@ -833,15 +838,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
833
838
|
}
|
|
834
839
|
}
|
|
835
840
|
if (t.targetDialect && t.targetDialect !== a.source.dialect) {
|
|
836
|
-
let e =
|
|
837
|
-
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new
|
|
841
|
+
let e = Lt(a.tables, a.source.dialect, t.targetDialect).filter((e) => e.severity === "block");
|
|
842
|
+
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new Fe({
|
|
838
843
|
sourceDialect: a.source.dialect,
|
|
839
844
|
targetDialect: t.targetDialect,
|
|
840
|
-
blocking: e.map(
|
|
845
|
+
blocking: e.map(Rt)
|
|
841
846
|
}));
|
|
842
847
|
}
|
|
843
848
|
let o = t.verify ?? !0, s = t.mode ?? "upsert", l = t.onConflict ?? "skip", u = t.ledgerPath ?? i.join(t.bundleDir, ".import..ledger.json"), f = yield* G(u);
|
|
844
|
-
if (s === "replace" && a.scope.kind !== "all") return yield* n.fail(new
|
|
849
|
+
if (s === "replace" && a.scope.kind !== "all") return yield* n.fail(new Pe({
|
|
845
850
|
mode: s,
|
|
846
851
|
reason: `replace refuses a partial bundle (scope='${a.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
847
852
|
}));
|
|
@@ -870,7 +875,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
870
875
|
r = t;
|
|
871
876
|
}
|
|
872
877
|
for (let t of i) (yield* m(() => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] }), t)) || (r = [...r, t]);
|
|
873
|
-
if (r.length > 0) return yield* n.fail(new
|
|
878
|
+
if (r.length > 0) return yield* n.fail(new je({
|
|
874
879
|
rows: r.slice(0, 20).map((e) => ({
|
|
875
880
|
table: e.entry.name,
|
|
876
881
|
id: String(e.row[e.entry.primaryKey]),
|
|
@@ -912,8 +917,8 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
912
917
|
total: d.rowCount
|
|
913
918
|
});
|
|
914
919
|
let v = l.length, y = n.gen(function* () {
|
|
915
|
-
let { createHash: t } = yield* n.promise(() => import("node:crypto")), r = t("sha256"), i = 0, o = S("bundle", () => E(g).pipe(
|
|
916
|
-
return yield* c.runForEach(
|
|
920
|
+
let { createHash: t } = yield* n.promise(() => import("node:crypto")), r = t("sha256"), i = 0, o = S("bundle", () => E(g).pipe(Je(a.compression))).pipe(c.tap((e) => n.sync(() => r.update(e))));
|
|
921
|
+
return yield* c.runForEach(Tt(o, d.name, d.columnTypes), (t) => {
|
|
917
922
|
i++;
|
|
918
923
|
let r = {
|
|
919
924
|
entry: d,
|
|
@@ -927,7 +932,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
927
932
|
rowCount: i,
|
|
928
933
|
checksum: r.digest("hex")
|
|
929
934
|
};
|
|
930
|
-
}), { rowCount: b, checksum: x } = yield*
|
|
935
|
+
}), { rowCount: b, checksum: x } = yield* lt("import", d.name, s, _, (e) => e.rowCount, y);
|
|
931
936
|
if (o) {
|
|
932
937
|
if (x !== d.checksum) return yield* n.fail(new I({
|
|
933
938
|
what: `table ${d.name} checksum`,
|
|
@@ -975,19 +980,19 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
975
980
|
catch: (e) => new F({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
976
981
|
}) : yield* _(t.store, !0), t.assets && a.assets.index && !t.assetsAlreadyRestored) {
|
|
977
982
|
let e = t.assets, r = i.join(t.bundleDir, H), o = i.join(t.bundleDir, a.assets.index ?? "assets/index.ndjson"), s = S("bundle", () => E(o)).pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.map((e) => JSON.parse(e)));
|
|
978
|
-
yield* c.runForEach(s, (t) => t.key in f.assets ? n.void :
|
|
983
|
+
yield* c.runForEach(s, (t) => t.key in f.assets ? n.void : Ue(e, r, t).pipe(n.andThen(n.sync(() => {
|
|
979
984
|
f.assets[t.key] = !0;
|
|
980
985
|
})), n.andThen(K(u, f))));
|
|
981
986
|
}
|
|
982
987
|
return a;
|
|
983
|
-
}),
|
|
988
|
+
}), Wt = (e) => e, Gt = (e) => e.allowLive || e.liveInstance === null ? { refuse: !1 } : {
|
|
984
989
|
refuse: !0,
|
|
985
990
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}). A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
986
|
-
}, J = Buffer.from("VBUNDLE2"),
|
|
991
|
+
}, J = Buffer.from("VBUNDLE2"), Kt = 0, qt = 1, Jt = 2, Yt = 32, Xt = async (e) => {
|
|
987
992
|
let t = [], n = async (r) => {
|
|
988
993
|
let i;
|
|
989
994
|
try {
|
|
990
|
-
i = await
|
|
995
|
+
i = await se(D(e, r), { withFileTypes: !0 });
|
|
991
996
|
} catch {
|
|
992
997
|
return;
|
|
993
998
|
}
|
|
@@ -997,21 +1002,21 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
997
1002
|
}
|
|
998
1003
|
};
|
|
999
1004
|
return await n(""), t.sort();
|
|
1000
|
-
},
|
|
1005
|
+
}, Zt = (e, t) => {
|
|
1001
1006
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1002
|
-
return r.writeUInt8(
|
|
1003
|
-
},
|
|
1007
|
+
return r.writeUInt8(qt, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1008
|
+
}, Qt = (e, t, n) => {
|
|
1004
1009
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1005
|
-
return a.writeUInt8(
|
|
1010
|
+
return a.writeUInt8(Jt, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1006
1011
|
};
|
|
1007
|
-
async function*
|
|
1012
|
+
async function* $t(e) {
|
|
1008
1013
|
yield new Uint8Array(J);
|
|
1009
|
-
for (let t of await
|
|
1010
|
-
let n = D(e.dir, t), r = await
|
|
1011
|
-
if (yield new Uint8Array(
|
|
1014
|
+
for (let t of await Xt(e.dir)) {
|
|
1015
|
+
let n = D(e.dir, t), r = await ce(n);
|
|
1016
|
+
if (yield new Uint8Array(Zt(t, r.size)), r.size > 0) for await (let e of E(n)) yield new Uint8Array(e);
|
|
1012
1017
|
}
|
|
1013
1018
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1014
|
-
yield new Uint8Array(
|
|
1019
|
+
yield new Uint8Array(Qt(t.key, t.contentType, t.size));
|
|
1015
1020
|
let n = _("sha256"), r = 0;
|
|
1016
1021
|
for await (let i of e.blobs.open(t.key)) {
|
|
1017
1022
|
let e = Buffer.from(i);
|
|
@@ -1020,11 +1025,11 @@ async function* Zt(e) {
|
|
|
1020
1025
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1021
1026
|
yield new Uint8Array(n.digest());
|
|
1022
1027
|
}
|
|
1023
|
-
yield new Uint8Array(Buffer.from([
|
|
1028
|
+
yield new Uint8Array(Buffer.from([Kt]));
|
|
1024
1029
|
}
|
|
1025
|
-
var
|
|
1026
|
-
let n =
|
|
1027
|
-
await
|
|
1030
|
+
var en = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tn = (e) => new Promise((t) => e.end(t)), nn = async (e, t) => {
|
|
1031
|
+
let n = de(t.dir);
|
|
1032
|
+
await oe(n, { recursive: !0 });
|
|
1028
1033
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1029
1034
|
let { value: e, done: t } = await i.next();
|
|
1030
1035
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1039,24 +1044,24 @@ var Qt = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), $t
|
|
|
1039
1044
|
for (;;) {
|
|
1040
1045
|
await s(1);
|
|
1041
1046
|
let e = c(1).readUInt8(0);
|
|
1042
|
-
if (e ===
|
|
1043
|
-
if (e ===
|
|
1047
|
+
if (e === Kt) return;
|
|
1048
|
+
if (e === qt) {
|
|
1044
1049
|
await s(4);
|
|
1045
1050
|
let e = c(4).readUInt32BE(0);
|
|
1046
1051
|
await s(e);
|
|
1047
1052
|
let t = c(e).toString("utf8");
|
|
1048
1053
|
await s(8);
|
|
1049
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1050
|
-
if (a !== n && !a.startsWith(n +
|
|
1051
|
-
await
|
|
1054
|
+
let i = c(8).readBigUInt64BE(0), a = de(n, t);
|
|
1055
|
+
if (a !== n && !a.startsWith(n + fe)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1056
|
+
await oe(ue(a), { recursive: !0 });
|
|
1052
1057
|
let o = ie(a);
|
|
1053
1058
|
for (; i > 0n;) {
|
|
1054
1059
|
r.length === 0 && await s(1);
|
|
1055
1060
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1056
|
-
await
|
|
1061
|
+
await en(o, c(e)), i -= BigInt(e);
|
|
1057
1062
|
}
|
|
1058
|
-
await
|
|
1059
|
-
} else if (e ===
|
|
1063
|
+
await tn(o);
|
|
1064
|
+
} else if (e === Jt) {
|
|
1060
1065
|
await s(4);
|
|
1061
1066
|
let e = c(4).readUInt32BE(0);
|
|
1062
1067
|
await s(e);
|
|
@@ -1073,17 +1078,17 @@ var Qt = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), $t
|
|
|
1073
1078
|
for (; p > 0n;) {
|
|
1074
1079
|
r.length === 0 && await s(1);
|
|
1075
1080
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1076
|
-
u.update(t), l || await
|
|
1081
|
+
u.update(t), l || await en(d, t), p -= BigInt(e);
|
|
1077
1082
|
}
|
|
1078
|
-
l ? d.destroy() : await
|
|
1079
|
-
let m = c(
|
|
1083
|
+
l ? d.destroy() : await tn(d), await f, await s(Yt);
|
|
1084
|
+
let m = c(Yt);
|
|
1080
1085
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1081
1086
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1082
1087
|
}
|
|
1083
1088
|
} finally {
|
|
1084
1089
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1085
1090
|
}
|
|
1086
|
-
},
|
|
1091
|
+
}, rn = async (e, t) => {
|
|
1087
1092
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1088
1093
|
let { value: e, done: t } = await a.next();
|
|
1089
1094
|
return t ? !1 : (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1105,8 +1110,8 @@ var Qt = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), $t
|
|
|
1105
1110
|
if (!await s(J.length) || !c(J.length).equals(J)) throw Error("not a bundle archive (bad magic)");
|
|
1106
1111
|
for (; await s(1);) {
|
|
1107
1112
|
let e = c(1).readUInt8(0);
|
|
1108
|
-
if (e ===
|
|
1109
|
-
if (e !==
|
|
1113
|
+
if (e === Kt || e === Jt) break;
|
|
1114
|
+
if (e !== qt) throw Error(`unknown bundle entry kind ${e}`);
|
|
1110
1115
|
if (!await s(4)) break;
|
|
1111
1116
|
let t = c(4).readUInt32BE(0);
|
|
1112
1117
|
if (!await s(t)) break;
|
|
@@ -1116,16 +1121,16 @@ var Qt = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), $t
|
|
|
1116
1121
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1117
1122
|
}
|
|
1118
1123
|
return r;
|
|
1119
|
-
},
|
|
1124
|
+
}, an = (e, t) => nn(e, {
|
|
1120
1125
|
dir: t,
|
|
1121
|
-
blobs:
|
|
1122
|
-
}),
|
|
1126
|
+
blobs: on(t)
|
|
1127
|
+
}), on = (e) => {
|
|
1123
1128
|
let t = /* @__PURE__ */ new Set();
|
|
1124
1129
|
return {
|
|
1125
1130
|
has: async (e) => t.has(e),
|
|
1126
1131
|
write: async (n, r, i) => {
|
|
1127
1132
|
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = D(e, "assets");
|
|
1128
|
-
await
|
|
1133
|
+
await oe(c, { recursive: !0 });
|
|
1129
1134
|
let l = _("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1130
1135
|
l.update(e), u += e.length, n(null, e);
|
|
1131
1136
|
} }), f = D(c, `.tmp-${Date.now()}-${t.size}`);
|
|
@@ -1139,23 +1144,23 @@ var Qt = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), $t
|
|
|
1139
1144
|
})}\n`), t.add(n);
|
|
1140
1145
|
}
|
|
1141
1146
|
};
|
|
1142
|
-
},
|
|
1147
|
+
}, sn = pe(b), cn = Buffer.from("VENC1\0\0\0"), ln = 32, un = 16, dn = 7, Y = 16, X = 256 * 1024, Z = {
|
|
1143
1148
|
N: 32768,
|
|
1144
1149
|
r: 8,
|
|
1145
1150
|
p: 1
|
|
1146
|
-
},
|
|
1151
|
+
}, fn = 128 * 1024 * 1024, Q = 43, pn = (e, t, n) => {
|
|
1147
1152
|
let r = Buffer.alloc(12);
|
|
1148
|
-
return e.copy(r, 0, 0,
|
|
1153
|
+
return e.copy(r, 0, 0, dn), r.writeUInt32BE(t >>> 0, dn), r.writeUInt8(+!!n, 11), r;
|
|
1149
1154
|
};
|
|
1150
|
-
async function*
|
|
1155
|
+
async function* mn(e, t) {
|
|
1151
1156
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1152
|
-
let n = y(
|
|
1157
|
+
let n = y(un), r = y(dn), i = await sn(t, n, ln, {
|
|
1153
1158
|
...Z,
|
|
1154
|
-
maxmem:
|
|
1159
|
+
maxmem: fn
|
|
1155
1160
|
}), a = Buffer.alloc(Q);
|
|
1156
|
-
|
|
1161
|
+
cn.copy(a, 0), a.writeUInt32BE(Z.N, 8), a.writeUInt32BE(Z.r, 12), a.writeUInt32BE(Z.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1157
1162
|
let o = 0, s = (e, t) => {
|
|
1158
|
-
let n = h("aes-256-gcm", i,
|
|
1163
|
+
let n = h("aes-256-gcm", i, pn(r, o, t));
|
|
1159
1164
|
o === 0 && n.setAAD(a);
|
|
1160
1165
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1161
1166
|
o++;
|
|
@@ -1165,7 +1170,7 @@ async function* fn(e, t) {
|
|
|
1165
1170
|
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > X;) yield s(c.subarray(0, X), !1), c = c.subarray(X);
|
|
1166
1171
|
yield s(c, !0);
|
|
1167
1172
|
}
|
|
1168
|
-
async function*
|
|
1173
|
+
async function* hn(e, t) {
|
|
1169
1174
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1170
1175
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1171
1176
|
let { value: e, done: t } = await n.next();
|
|
@@ -1176,12 +1181,12 @@ async function* pn(e, t) {
|
|
|
1176
1181
|
};
|
|
1177
1182
|
if (!await o(Q)) throw Error("encrypted bundle truncated (no header)");
|
|
1178
1183
|
let s = r.subarray(0, Q);
|
|
1179
|
-
if (!s.subarray(0, 8).equals(
|
|
1180
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await
|
|
1184
|
+
if (!s.subarray(0, 8).equals(cn)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1185
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await sn(t, d, ln, {
|
|
1181
1186
|
N: c,
|
|
1182
1187
|
r: l,
|
|
1183
1188
|
p: u,
|
|
1184
|
-
maxmem:
|
|
1189
|
+
maxmem: fn
|
|
1185
1190
|
});
|
|
1186
1191
|
r = r.subarray(Q);
|
|
1187
1192
|
let m = 0, h = !1;
|
|
@@ -1196,7 +1201,7 @@ async function* pn(e, t) {
|
|
|
1196
1201
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1197
1202
|
let n = r.subarray(5, 5 + t - Y), i = r.subarray(5 + t - Y, 5 + t);
|
|
1198
1203
|
r = r.subarray(5 + t);
|
|
1199
|
-
let a = g("aes-256-gcm", p,
|
|
1204
|
+
let a = g("aes-256-gcm", p, pn(f, m, e));
|
|
1200
1205
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1201
1206
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1202
1207
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1204,7 +1209,7 @@ async function* pn(e, t) {
|
|
|
1204
1209
|
}
|
|
1205
1210
|
//#endregion
|
|
1206
1211
|
//#region src/blobStorage.ts
|
|
1207
|
-
var
|
|
1212
|
+
var gn = (e, t) => ({
|
|
1208
1213
|
list: () => {
|
|
1209
1214
|
let e = t();
|
|
1210
1215
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1220,14 +1225,14 @@ var mn = (e, t) => ({
|
|
|
1220
1225
|
let r = await te((await n.runPromise(C(e, t))).stream);
|
|
1221
1226
|
for await (let e of r) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1222
1227
|
} })
|
|
1223
|
-
}),
|
|
1228
|
+
}), _n = (e, t) => gn(e, () => p(t, {
|
|
1224
1229
|
table: x,
|
|
1225
1230
|
chunkSize: 500
|
|
1226
1231
|
}).pipe(c.map((e) => ({
|
|
1227
1232
|
key: String(e.key),
|
|
1228
1233
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1229
1234
|
size: Number(e.size ?? 0)
|
|
1230
|
-
})))),
|
|
1235
|
+
})))), vn = (e) => ({
|
|
1231
1236
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1232
1237
|
write: async (t, r, i) => {
|
|
1233
1238
|
let a = S("storage", () => ne.from(r));
|
|
@@ -1236,7 +1241,10 @@ var mn = (e, t) => ({
|
|
|
1236
1241
|
totalSize: i.size
|
|
1237
1242
|
}));
|
|
1238
1243
|
}
|
|
1239
|
-
}),
|
|
1244
|
+
}), yn = (e, t) => {
|
|
1245
|
+
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1246
|
+
return e === "mariadb" && bn(n) ? n : r;
|
|
1247
|
+
}, bn = (e) => (process.env.PATH ?? "").split(le).filter((e) => e !== "").some((t) => ae(D(t, e))), xn = (e) => {
|
|
1240
1248
|
switch (e) {
|
|
1241
1249
|
case "postgres": return "db.dump";
|
|
1242
1250
|
case "mysql":
|
|
@@ -1246,7 +1254,7 @@ var mn = (e, t) => ({
|
|
|
1246
1254
|
case "mssql": return "db.bacpac";
|
|
1247
1255
|
default: return "db.dump";
|
|
1248
1256
|
}
|
|
1249
|
-
},
|
|
1257
|
+
}, Sn = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1250
1258
|
if (e.host || e.database) return {
|
|
1251
1259
|
host: e.host ?? "localhost",
|
|
1252
1260
|
port: e.port ?? 3306,
|
|
@@ -1262,7 +1270,7 @@ var mn = (e, t) => ({
|
|
|
1262
1270
|
password: decodeURIComponent(t.password) || "",
|
|
1263
1271
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1264
1272
|
};
|
|
1265
|
-
},
|
|
1273
|
+
}, Cn = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, wn = (e, t, n) => {
|
|
1266
1274
|
switch (e) {
|
|
1267
1275
|
case "postgres": {
|
|
1268
1276
|
if (t.url) return {
|
|
@@ -1306,31 +1314,31 @@ var mn = (e, t) => ({
|
|
|
1306
1314
|
}
|
|
1307
1315
|
case "mysql":
|
|
1308
1316
|
case "mariadb": {
|
|
1309
|
-
let
|
|
1317
|
+
let r = $(t);
|
|
1310
1318
|
return {
|
|
1311
1319
|
kind: "spawn",
|
|
1312
|
-
tool: "
|
|
1320
|
+
tool: yn(e, "dump"),
|
|
1313
1321
|
args: [
|
|
1314
1322
|
"--single-transaction",
|
|
1315
1323
|
"--routines",
|
|
1316
1324
|
"--triggers",
|
|
1317
1325
|
"--no-tablespaces",
|
|
1318
1326
|
"-h",
|
|
1319
|
-
|
|
1327
|
+
r.host,
|
|
1320
1328
|
"-P",
|
|
1321
|
-
String(
|
|
1329
|
+
String(r.port),
|
|
1322
1330
|
"-u",
|
|
1323
|
-
|
|
1324
|
-
|
|
1331
|
+
r.username,
|
|
1332
|
+
r.database
|
|
1325
1333
|
],
|
|
1326
|
-
env: { MYSQL_PWD:
|
|
1334
|
+
env: { MYSQL_PWD: r.password },
|
|
1327
1335
|
stdoutFile: n
|
|
1328
1336
|
};
|
|
1329
1337
|
}
|
|
1330
1338
|
case "sqlite":
|
|
1331
1339
|
case "turso": return {
|
|
1332
1340
|
kind: "copy",
|
|
1333
|
-
from:
|
|
1341
|
+
from: Sn(t),
|
|
1334
1342
|
to: n
|
|
1335
1343
|
};
|
|
1336
1344
|
case "mssql": return {
|
|
@@ -1338,7 +1346,7 @@ var mn = (e, t) => ({
|
|
|
1338
1346
|
tool: "sqlpackage",
|
|
1339
1347
|
args: [
|
|
1340
1348
|
"/Action:Export",
|
|
1341
|
-
`/SourceConnectionString:${
|
|
1349
|
+
`/SourceConnectionString:${Cn($({
|
|
1342
1350
|
...t,
|
|
1343
1351
|
port: t.port ?? 1433
|
|
1344
1352
|
}))}`,
|
|
@@ -1351,7 +1359,7 @@ var mn = (e, t) => ({
|
|
|
1351
1359
|
reason: `no native backup for dialect '${e}'`
|
|
1352
1360
|
});
|
|
1353
1361
|
}
|
|
1354
|
-
},
|
|
1362
|
+
}, Tn = (e, t, n) => {
|
|
1355
1363
|
switch (e) {
|
|
1356
1364
|
case "postgres": {
|
|
1357
1365
|
if (t.url) return {
|
|
@@ -1393,20 +1401,20 @@ var mn = (e, t) => ({
|
|
|
1393
1401
|
}
|
|
1394
1402
|
case "mysql":
|
|
1395
1403
|
case "mariadb": {
|
|
1396
|
-
let
|
|
1404
|
+
let r = $(t);
|
|
1397
1405
|
return {
|
|
1398
1406
|
kind: "spawn",
|
|
1399
|
-
tool: "
|
|
1407
|
+
tool: yn(e, "client"),
|
|
1400
1408
|
args: [
|
|
1401
1409
|
"-h",
|
|
1402
|
-
|
|
1410
|
+
r.host,
|
|
1403
1411
|
"-P",
|
|
1404
|
-
String(
|
|
1412
|
+
String(r.port),
|
|
1405
1413
|
"-u",
|
|
1406
|
-
|
|
1407
|
-
|
|
1414
|
+
r.username,
|
|
1415
|
+
r.database
|
|
1408
1416
|
],
|
|
1409
|
-
env: { MYSQL_PWD:
|
|
1417
|
+
env: { MYSQL_PWD: r.password },
|
|
1410
1418
|
stdinFile: n
|
|
1411
1419
|
};
|
|
1412
1420
|
}
|
|
@@ -1414,14 +1422,14 @@ var mn = (e, t) => ({
|
|
|
1414
1422
|
case "turso": return {
|
|
1415
1423
|
kind: "copy",
|
|
1416
1424
|
from: n,
|
|
1417
|
-
to:
|
|
1425
|
+
to: Sn(t)
|
|
1418
1426
|
};
|
|
1419
1427
|
case "mssql": return {
|
|
1420
1428
|
kind: "spawn",
|
|
1421
1429
|
tool: "sqlpackage",
|
|
1422
1430
|
args: [
|
|
1423
1431
|
"/Action:Import",
|
|
1424
|
-
`/TargetConnectionString:${
|
|
1432
|
+
`/TargetConnectionString:${Cn($({
|
|
1425
1433
|
...t,
|
|
1426
1434
|
port: t.port ?? 1433
|
|
1427
1435
|
}))}`,
|
|
@@ -1434,13 +1442,13 @@ var mn = (e, t) => ({
|
|
|
1434
1442
|
reason: `no native restore for dialect '${e}'`
|
|
1435
1443
|
});
|
|
1436
1444
|
}
|
|
1437
|
-
},
|
|
1445
|
+
}, En = (e) => e.kind === "copy" ? n.async((t) => {
|
|
1438
1446
|
re(e.from, e.to, (r) => t(r ? n.fail(new L({
|
|
1439
1447
|
tool: "copy",
|
|
1440
1448
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1441
1449
|
})) : n.void));
|
|
1442
1450
|
}) : n.async((t) => {
|
|
1443
|
-
let r =
|
|
1451
|
+
let r = me(e.tool, [...e.args], {
|
|
1444
1452
|
env: {
|
|
1445
1453
|
...process.env,
|
|
1446
1454
|
...e.env
|
|
@@ -1461,8 +1469,8 @@ var mn = (e, t) => ({
|
|
|
1461
1469
|
reason: `exited with code ${r}`,
|
|
1462
1470
|
stderr: i.slice(-2e3)
|
|
1463
1471
|
}))));
|
|
1464
|
-
}),
|
|
1465
|
-
let t = yield*
|
|
1472
|
+
}), Dn = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), On = (e) => n.gen(function* () {
|
|
1473
|
+
let t = yield* ze(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
1466
1474
|
store: e.store,
|
|
1467
1475
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
1468
1476
|
}), r = q(t.tables, e.snapshot), i = e.classification ?? O(e.snapshot), a = e.sampleSize ?? 20, o = N(r.map((t) => ({
|
|
@@ -1478,14 +1486,14 @@ var mn = (e, t) => ({
|
|
|
1478
1486
|
chunkSize: a,
|
|
1479
1487
|
...r === void 0 ? {} : { where: r }
|
|
1480
1488
|
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) => P(t, i, o.actions, e.masking.seed));
|
|
1481
|
-
l.push(...
|
|
1489
|
+
l.push(...De(d, i, o.actions));
|
|
1482
1490
|
let f = u[0], m = d[0], h = [];
|
|
1483
1491
|
if (f && m) for (let [e, t] of Object.entries(o.actions)) {
|
|
1484
1492
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
1485
1493
|
let n = e.slice(i.length + 1);
|
|
1486
1494
|
h.push({
|
|
1487
1495
|
column: n,
|
|
1488
|
-
action:
|
|
1496
|
+
action: Dn(t),
|
|
1489
1497
|
before: f[n],
|
|
1490
1498
|
after: m[n]
|
|
1491
1499
|
});
|
|
@@ -1500,6 +1508,70 @@ var mn = (e, t) => ({
|
|
|
1500
1508
|
leaks: l,
|
|
1501
1509
|
unclassified: o.unclassified
|
|
1502
1510
|
};
|
|
1503
|
-
})
|
|
1511
|
+
}), kn = (e, t) => {
|
|
1512
|
+
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
1513
|
+
for (let t of e) {
|
|
1514
|
+
let e = n.get(t.name);
|
|
1515
|
+
if (e === void 0) {
|
|
1516
|
+
r.push({
|
|
1517
|
+
verdict: "refuse",
|
|
1518
|
+
table: t.name,
|
|
1519
|
+
detail: `the bundle carries '${t.name}' and the target schema has no such table — nowhere to put ${t.name}'s rows`
|
|
1520
|
+
});
|
|
1521
|
+
continue;
|
|
1522
|
+
}
|
|
1523
|
+
let a = new Map(e.columns.map((e) => [e.name, e])), o = new Map(t.columns.map((e) => [e.name, e])), s = [];
|
|
1524
|
+
for (let e of t.columns) {
|
|
1525
|
+
let n = a.get(e.name);
|
|
1526
|
+
if (n === void 0) {
|
|
1527
|
+
s.push(e.name), r.push({
|
|
1528
|
+
verdict: "value-dropped",
|
|
1529
|
+
table: t.name,
|
|
1530
|
+
column: e.name,
|
|
1531
|
+
detail: `'${t.name}.${e.name}' is in the bundle and not in the schema — its values are DISCARDED`
|
|
1532
|
+
});
|
|
1533
|
+
continue;
|
|
1534
|
+
}
|
|
1535
|
+
n.type !== e.type && r.push({
|
|
1536
|
+
verdict: "refuse",
|
|
1537
|
+
table: t.name,
|
|
1538
|
+
column: e.name,
|
|
1539
|
+
detail: `'${t.name}.${e.name}' changed type (${e.type} → ${n.type}) — the bundle's values would be written through a codec that no longer matches them`
|
|
1540
|
+
});
|
|
1541
|
+
}
|
|
1542
|
+
for (let n of e.columns) if (!o.has(n.name)) {
|
|
1543
|
+
if (n.nullable || n.hasDefault) {
|
|
1544
|
+
r.push({
|
|
1545
|
+
verdict: "filled-by-default",
|
|
1546
|
+
table: t.name,
|
|
1547
|
+
column: n.name,
|
|
1548
|
+
detail: `'${t.name}.${n.name}' is in the schema and not in the bundle — every imported row gets ${n.hasDefault ? "its default" : "NULL"}`
|
|
1549
|
+
});
|
|
1550
|
+
continue;
|
|
1551
|
+
}
|
|
1552
|
+
r.push({
|
|
1553
|
+
verdict: "refuse",
|
|
1554
|
+
table: t.name,
|
|
1555
|
+
column: n.name,
|
|
1556
|
+
detail: `'${t.name}.${n.name}' is NOT NULL with no default and the bundle has no value for it`
|
|
1557
|
+
});
|
|
1558
|
+
}
|
|
1559
|
+
s.length > 0 && (i[t.name] = s);
|
|
1560
|
+
}
|
|
1561
|
+
return {
|
|
1562
|
+
findings: r,
|
|
1563
|
+
skipColumns: i,
|
|
1564
|
+
refuse: r.some((e) => e.verdict === "refuse")
|
|
1565
|
+
};
|
|
1566
|
+
}, An = (e) => {
|
|
1567
|
+
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
1568
|
+
let t = {
|
|
1569
|
+
safe: "✓",
|
|
1570
|
+
"value-dropped": "⊖",
|
|
1571
|
+
"filled-by-default": "⊕",
|
|
1572
|
+
refuse: "✗"
|
|
1573
|
+
}, n = e.findings.map((e) => ` ${t[e.verdict]} ${e.detail}`), r = (t) => e.findings.filter((e) => e.verdict === t).length;
|
|
1574
|
+
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
1575
|
+
};
|
|
1504
1576
|
//#endregion
|
|
1505
|
-
export { H as ASSETS_DIR, U as ASSET_INDEX_FILE, F as BundleError,
|
|
1577
|
+
export { H as ASSETS_DIR, U as ASSET_INDEX_FILE, F as BundleError, Oe as CodecError, ke as CompressionError, Fe as CrossDialectError, V as DATA_DIR, ge as DEFAULT_CLASS_ACTIONS, je as DanglingReferenceError, Ye as FORMAT_VERSION, Pe as ImportModeError, I as IntegrityError, rt as LEDGER_FILE, B as MANIFEST_FILE, Ne as MaskingError, L as NativeToolError, Me as SchemaDriftError, Ae as ScopeError, Te as applyAction, Gt as assessLiveImport, ft as assetDone, xn as backupArtifactName, wn as backupCommand, Pt as canonicalType, Lt as checkPortability, Vt as checkSchemaDrift, O as classificationFromSnapshot, kn as classifyImportDrift, Ke as compressionExt, Le as computeSubsetIds, tt as decodeManifest, St as decodeRow, bt as decodeValue, hn as decryptStream, Wt as defineDataProfile, Bt as diffSnapshots, on as dirBlobSink, ut as emptyLedger, et as encodeManifest, xt as encodeRow, vt as encodeValue, mn as encryptStream, An as formatImportDrift, Rt as formatIssue, We as hasZstd, qe as makeCompressor, Je as makeDecompressor, P as maskRow, Tt as ndjsonToRows, $t as packBundle, rn as peekBundle, Ge as pickCompression, N as planMasking, On as previewMasking, G as readLedger, _e as resolveAction, ze as resolveScope, Ue as restoreAssetFromCas, Tn as restoreCommand, wt as rowsToNdjson, Nt as runExport, Ut as runImport, En as runNativeStep, De as scanForLeaks, Ve as storageAssetSink, Be as storageAssetSource, vn as storageBlobSink, gn as storageBlobSource, _n as storageRefsBlobSource, Re as subsetToScope, dt as tableDone, q as topoSortTables, nn as unpackBundle, an as unpackBundleToDir, He as writeAssetToCas, K as writeLedger };
|