@voltro/data-transfer 0.33.0 → 0.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1968 -0
- package/dist/index.d.ts +50 -5
- package/dist/index.js +317 -258
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -5,13 +5,13 @@ import { createCipheriv as h, createDecipheriv as g, createHash as _, createHmac
|
|
|
5
5
|
import { STORAGE_REFS_TABLE as x, fromNodeReadable as S, getObjectStream as C, putObjectStream as ee, toNodeReadable as te } from "@voltro/plugin-storage";
|
|
6
6
|
import * as w from "node:zlib";
|
|
7
7
|
import { PassThrough as T, Readable as ne } from "node:stream";
|
|
8
|
-
import { copyFile as re, createReadStream as E, createWriteStream as
|
|
8
|
+
import { copyFile as re, createReadStream as E, createWriteStream as D, existsSync as ie, unlinkSync as ae } from "node:fs";
|
|
9
9
|
import { mkdir as oe, readdir as se, stat as ce } from "node:fs/promises";
|
|
10
|
-
import {
|
|
11
|
-
import { promisify as
|
|
12
|
-
import { spawn as
|
|
10
|
+
import { basename as le, delimiter as ue, dirname as de, join as O, resolve as fe, sep as pe } from "node:path";
|
|
11
|
+
import { promisify as me } from "node:util";
|
|
12
|
+
import { spawn as he } from "node:child_process";
|
|
13
13
|
//#region src/consistency.ts
|
|
14
|
-
var
|
|
14
|
+
var ge = (e) => {
|
|
15
15
|
switch (e) {
|
|
16
16
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
17
17
|
case "mysql":
|
|
@@ -21,7 +21,7 @@ var he = (e) => {
|
|
|
21
21
|
case "turso": return [];
|
|
22
22
|
default: return [];
|
|
23
23
|
}
|
|
24
|
-
},
|
|
24
|
+
}, _e = {
|
|
25
25
|
email: { fake: "email" },
|
|
26
26
|
fullName: { fake: "fullName" },
|
|
27
27
|
firstName: { fake: "firstName" },
|
|
@@ -36,7 +36,7 @@ var he = (e) => {
|
|
|
36
36
|
date: "dateShift",
|
|
37
37
|
secret: "null",
|
|
38
38
|
freeText: "redact"
|
|
39
|
-
},
|
|
39
|
+
}, ve = (e) => {
|
|
40
40
|
let t = {};
|
|
41
41
|
for (let n of e.tables) {
|
|
42
42
|
let e = {};
|
|
@@ -48,12 +48,12 @@ var he = (e) => {
|
|
|
48
48
|
t[n.name] = e;
|
|
49
49
|
}
|
|
50
50
|
return t;
|
|
51
|
-
},
|
|
51
|
+
}, ye = (e, t, n, r) => {
|
|
52
52
|
let i = r.columns?.[`${e}.${t}`];
|
|
53
53
|
if (i) return i;
|
|
54
54
|
if (n?.sensitive) {
|
|
55
55
|
let e = n.sensitive.class;
|
|
56
|
-
return r.classes?.[e] ??
|
|
56
|
+
return r.classes?.[e] ?? _e[e] ?? "redact";
|
|
57
57
|
}
|
|
58
58
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
59
59
|
}, k = (e, t) => v("sha256", e).update(t == null ? "∅" : String(t)).digest(), A = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), j = [
|
|
@@ -69,7 +69,7 @@ var he = (e) => {
|
|
|
69
69
|
"Guido",
|
|
70
70
|
"Margaret",
|
|
71
71
|
"Bjarne"
|
|
72
|
-
],
|
|
72
|
+
], M = [
|
|
73
73
|
"Lovelace",
|
|
74
74
|
"Turing",
|
|
75
75
|
"Hopper",
|
|
@@ -82,7 +82,7 @@ var he = (e) => {
|
|
|
82
82
|
"Rossum",
|
|
83
83
|
"Hamilton",
|
|
84
84
|
"Stroustrup"
|
|
85
|
-
],
|
|
85
|
+
], be = [
|
|
86
86
|
"Maple",
|
|
87
87
|
"Oak",
|
|
88
88
|
"Cedar",
|
|
@@ -91,7 +91,7 @@ var he = (e) => {
|
|
|
91
91
|
"Birch",
|
|
92
92
|
"Willow",
|
|
93
93
|
"Ash"
|
|
94
|
-
],
|
|
94
|
+
], xe = [
|
|
95
95
|
"Initech",
|
|
96
96
|
"Hooli",
|
|
97
97
|
"Umbrella",
|
|
@@ -100,23 +100,23 @@ var he = (e) => {
|
|
|
100
100
|
"Soylent",
|
|
101
101
|
"Stark",
|
|
102
102
|
"Wayne"
|
|
103
|
-
],
|
|
103
|
+
], N = (e, t, n) => e[A(t, n) % e.length], Se = (e, t, n) => {
|
|
104
104
|
let r = k(n, `${e}:${String(t)}`), i = A(r, 1) % 1e4;
|
|
105
105
|
switch (e) {
|
|
106
|
-
case "email": return `${
|
|
107
|
-
case "fullName": return `${
|
|
108
|
-
case "firstName": return
|
|
109
|
-
case "lastName": return M
|
|
110
|
-
case "username": return `${
|
|
106
|
+
case "email": return `${N(j, r, 0).toLowerCase()}.${N(M, r, 2).toLowerCase()}${i}@example.com`;
|
|
107
|
+
case "fullName": return `${N(j, r, 0)} ${N(M, r, 1)}`;
|
|
108
|
+
case "firstName": return N(j, r, 0);
|
|
109
|
+
case "lastName": return N(M, r, 1);
|
|
110
|
+
case "username": return `${N(j, r, 0).toLowerCase()}${i}`;
|
|
111
111
|
case "phone": return `+1${String(A(r, 0) % 1e10).padStart(10, "0")}`;
|
|
112
|
-
case "address": return `${A(r, 0) % 9899 + 100} ${
|
|
113
|
-
case "company": return
|
|
112
|
+
case "address": return `${A(r, 0) % 9899 + 100} ${N(be, r, 1)} St`;
|
|
113
|
+
case "company": return N(xe, r, 0);
|
|
114
114
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
115
115
|
case "ip": return `${A(r, 0) % 256}.${A(r, 1) % 256}.${A(r, 2) % 256}.${A(r, 3) % 254 + 1}`;
|
|
116
|
-
case "creditCard": return
|
|
116
|
+
case "creditCard": return Ce(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
117
117
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
118
118
|
}
|
|
119
|
-
},
|
|
119
|
+
}, Ce = (e) => {
|
|
120
120
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
121
121
|
t.forEach((e, t) => {
|
|
122
122
|
let r = e;
|
|
@@ -124,26 +124,26 @@ var he = (e) => {
|
|
|
124
124
|
});
|
|
125
125
|
let r = (10 - n % 10) % 10;
|
|
126
126
|
return e.slice(0, 15) + String(r);
|
|
127
|
-
},
|
|
127
|
+
}, we = 864e5, Te = (e, t) => {
|
|
128
128
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
129
129
|
if (Number.isNaN(n)) return e;
|
|
130
|
-
let r = A(k(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r *
|
|
130
|
+
let r = A(k(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * we);
|
|
131
131
|
return e instanceof Date ? i : i.toISOString();
|
|
132
|
-
},
|
|
132
|
+
}, Ee = (e, t) => {
|
|
133
133
|
if (t.value === null || t.value === void 0) return null;
|
|
134
134
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
135
|
-
return e === "keep" ? t.value : e === "null" ? null : e === "redact" ? t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null : e === "hash" ? k(t.seed, t.value).toString("hex").slice(0, 16) : e === "dateShift" ?
|
|
136
|
-
},
|
|
135
|
+
return e === "keep" ? t.value : e === "null" ? null : e === "redact" ? t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null : e === "hash" ? k(t.seed, t.value).toString("hex").slice(0, 16) : e === "dateShift" ? Te(t.value, t.seed) : typeof e == "object" && "fake" in e ? Se(e.fake, t.value, t.seed) : typeof e == "object" && "custom" in e ? e.custom(t) : t.value;
|
|
136
|
+
}, De = (e, t, n) => {
|
|
137
137
|
let r = {}, i = [];
|
|
138
138
|
for (let a of e) for (let e of a.columns) {
|
|
139
|
-
let o = `${a.name}.${e.name}`, s =
|
|
139
|
+
let o = `${a.name}.${e.name}`, s = ye(a.name, e.name, t[a.name]?.[e.name], n);
|
|
140
140
|
s === "error" ? i.push(o) : r[o] = s;
|
|
141
141
|
}
|
|
142
142
|
return {
|
|
143
143
|
actions: r,
|
|
144
144
|
unclassified: i
|
|
145
145
|
};
|
|
146
|
-
},
|
|
146
|
+
}, Oe = [
|
|
147
147
|
{
|
|
148
148
|
name: "email",
|
|
149
149
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -160,7 +160,7 @@ var he = (e) => {
|
|
|
160
160
|
name: "creditCard",
|
|
161
161
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
162
162
|
}
|
|
163
|
-
],
|
|
163
|
+
], ke = (e, t, n) => {
|
|
164
164
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
165
165
|
for (let [a, o] of Object.entries(n)) {
|
|
166
166
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -168,7 +168,7 @@ var he = (e) => {
|
|
|
168
168
|
for (let a of e) {
|
|
169
169
|
let e = a[n];
|
|
170
170
|
if (typeof e != "string" || e.length === 0) continue;
|
|
171
|
-
let o =
|
|
171
|
+
let o = Oe.find((t) => t.re.test(e));
|
|
172
172
|
if (o && !i.has(n)) {
|
|
173
173
|
i.add(n), r.push({
|
|
174
174
|
table: t,
|
|
@@ -183,7 +183,7 @@ var he = (e) => {
|
|
|
183
183
|
return r;
|
|
184
184
|
}, P = (e, t, n, r, i) => {
|
|
185
185
|
let a = {};
|
|
186
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
186
|
+
for (let [o, s] of Object.entries(e)) a[o] = Ee(n[`${t}.${o}`] ?? "keep", {
|
|
187
187
|
value: s,
|
|
188
188
|
table: t,
|
|
189
189
|
column: o,
|
|
@@ -194,21 +194,21 @@ var he = (e) => {
|
|
|
194
194
|
}, F = class extends s.TaggedError()("BundleError", {
|
|
195
195
|
reason: s.String,
|
|
196
196
|
path: s.optional(s.String)
|
|
197
|
-
}) {},
|
|
197
|
+
}) {}, Ae = class extends s.TaggedError()("CodecError", {
|
|
198
198
|
table: s.String,
|
|
199
199
|
reason: s.String
|
|
200
200
|
}) {}, I = class extends s.TaggedError()("IntegrityError", {
|
|
201
201
|
what: s.String,
|
|
202
202
|
expected: s.String,
|
|
203
203
|
actual: s.String
|
|
204
|
-
}) {},
|
|
204
|
+
}) {}, je = class extends t.TaggedError("CompressionError") {}, Me = class extends s.TaggedError()("ScopeError", { reason: s.String }) {}, Ne = class extends s.TaggedError()("DanglingReferenceError", {
|
|
205
205
|
rows: s.Array(s.Struct({
|
|
206
206
|
table: s.String,
|
|
207
207
|
id: s.String,
|
|
208
208
|
reason: s.String
|
|
209
209
|
})),
|
|
210
210
|
totalCount: s.Number
|
|
211
|
-
}) {},
|
|
211
|
+
}) {}, Pe = class extends s.TaggedError()("SchemaDriftError", {
|
|
212
212
|
bundleFingerprint: s.String,
|
|
213
213
|
targetFingerprint: s.String,
|
|
214
214
|
diff: s.Array(s.String)
|
|
@@ -217,17 +217,17 @@ var he = (e) => {
|
|
|
217
217
|
let e = this.stderr?.trim();
|
|
218
218
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
219
219
|
}
|
|
220
|
-
},
|
|
220
|
+
}, Fe = class extends s.TaggedError()("MaskingError", { unclassified: s.Array(s.String) }) {}, Ie = class extends s.TaggedError()("ImportModeError", {
|
|
221
221
|
mode: s.String,
|
|
222
222
|
reason: s.String
|
|
223
|
-
}) {},
|
|
223
|
+
}) {}, Le = class extends s.TaggedError()("CrossDialectError", {
|
|
224
224
|
sourceDialect: s.String,
|
|
225
225
|
targetDialect: s.String,
|
|
226
226
|
blocking: s.Array(s.String)
|
|
227
|
-
}) {}, R = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id",
|
|
227
|
+
}) {}, R = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Re = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
228
228
|
col: e.name,
|
|
229
229
|
refTable: e.references.table
|
|
230
|
-
})),
|
|
230
|
+
})), z = (e, t, r) => n.gen(function* () {
|
|
231
231
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map(), d = (e) => {
|
|
232
232
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
233
233
|
return s.set(e, t), t;
|
|
@@ -245,7 +245,7 @@ var he = (e) => {
|
|
|
245
245
|
for (let t of n) m(t.refTable, e[t.col]);
|
|
246
246
|
};
|
|
247
247
|
}, g = (r, i) => {
|
|
248
|
-
let a = h(r, R(t, r),
|
|
248
|
+
let a = h(r, R(t, r), Re(t, r));
|
|
249
249
|
return c.runForEach(p(e, {
|
|
250
250
|
table: r,
|
|
251
251
|
pk: R(t, r),
|
|
@@ -280,13 +280,13 @@ var he = (e) => {
|
|
|
280
280
|
}
|
|
281
281
|
}
|
|
282
282
|
return o;
|
|
283
|
-
}),
|
|
283
|
+
}), ze = (e, t) => ({
|
|
284
284
|
tables: [...e.keys()],
|
|
285
285
|
predicateFor: (n) => {
|
|
286
286
|
let r = e.get(n);
|
|
287
287
|
return r ? u(R(t, n), [...r]) : void 0;
|
|
288
288
|
}
|
|
289
|
-
}),
|
|
289
|
+
}), B = "tenantId", Be = (e, t, r) => n.gen(function* () {
|
|
290
290
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
291
291
|
if (e.kind === "all") return {
|
|
292
292
|
tables: i,
|
|
@@ -300,25 +300,25 @@ var he = (e) => {
|
|
|
300
300
|
};
|
|
301
301
|
}
|
|
302
302
|
let o = e.tenantId;
|
|
303
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
304
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
305
|
-
if (s.length === 0) return yield* n.fail(new
|
|
306
|
-
let c = yield*
|
|
307
|
-
seeds: Object.fromEntries(s.map((e) => [e, l(
|
|
303
|
+
if (o === void 0 || o === "") return yield* n.fail(new Me({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
304
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === B) ?? !1);
|
|
305
|
+
if (s.length === 0) return yield* n.fail(new Me({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${B}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
306
|
+
let c = yield* z(r.store, t, {
|
|
307
|
+
seeds: Object.fromEntries(s.map((e) => [e, l(B, o)])),
|
|
308
308
|
pinned: s,
|
|
309
309
|
children: { roots: s }
|
|
310
310
|
}), d = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), f = new Set(s);
|
|
311
311
|
return {
|
|
312
312
|
tables: [...s, ...d],
|
|
313
|
-
predicateFor: (e) => f.has(e) ? l(
|
|
313
|
+
predicateFor: (e) => f.has(e) ? l(B, o) : c.has(e) ? u(R(t, e), [...c.get(e)]) : void 0
|
|
314
314
|
};
|
|
315
|
-
}),
|
|
315
|
+
}), Ve = (e, t) => ({
|
|
316
316
|
list: t,
|
|
317
317
|
open: (t) => C(e, t)
|
|
318
|
-
}),
|
|
318
|
+
}), He = (e) => ({
|
|
319
319
|
write: (t, n, r) => ee(e, t, n, r),
|
|
320
320
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
321
|
-
}),
|
|
321
|
+
}), Ue = (e, t) => n.tryPromise({
|
|
322
322
|
try: async () => {
|
|
323
323
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
324
324
|
u.update(e), d += e.length, n(null, e);
|
|
@@ -340,7 +340,7 @@ var he = (e) => {
|
|
|
340
340
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
341
341
|
path: t
|
|
342
342
|
})
|
|
343
|
-
}),
|
|
343
|
+
}), We = (e, t, r) => n.gen(function* () {
|
|
344
344
|
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), l = o("sha256"), u = S("bundle", () => a(s)).pipe(c.tap((e) => n.sync(() => l.update(e))));
|
|
345
345
|
yield* e.write(r.key, u, {
|
|
346
346
|
contentType: r.contentType,
|
|
@@ -352,7 +352,7 @@ var he = (e) => {
|
|
|
352
352
|
expected: r.sha256,
|
|
353
353
|
actual: d
|
|
354
354
|
}));
|
|
355
|
-
}),
|
|
355
|
+
}), Ge = () => typeof w.createZstdCompress == "function", Ke = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : Ge() ? "zstd" : "gzip", qe = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", Je = (e) => e === "zstd" ? w.createZstdCompress() : e === "gzip" ? w.createGzip() : new T(), Ye = (e) => e === "zstd" ? w.createZstdDecompress() : e === "gzip" ? w.createGunzip() : new T(), Xe = 1, Ze = s.Struct({
|
|
356
356
|
name: s.String,
|
|
357
357
|
file: s.String,
|
|
358
358
|
rowCount: s.Number,
|
|
@@ -362,11 +362,11 @@ var he = (e) => {
|
|
|
362
362
|
value: s.String
|
|
363
363
|
}),
|
|
364
364
|
primaryKey: s.String
|
|
365
|
-
}),
|
|
365
|
+
}), Qe = s.Struct({
|
|
366
366
|
kind: s.Literal("all", "tenant", "tables"),
|
|
367
367
|
tenantId: s.optional(s.String),
|
|
368
368
|
tables: s.optional(s.Array(s.String))
|
|
369
|
-
}),
|
|
369
|
+
}), $e = s.Struct({
|
|
370
370
|
count: s.Number,
|
|
371
371
|
totalBytes: s.Number,
|
|
372
372
|
index: s.optional(s.String)
|
|
@@ -377,23 +377,23 @@ s.Struct({
|
|
|
377
377
|
size: s.Number,
|
|
378
378
|
contentType: s.String
|
|
379
379
|
});
|
|
380
|
-
var
|
|
380
|
+
var et = s.Struct({
|
|
381
381
|
formatVersion: s.Number,
|
|
382
382
|
createdAt: s.String,
|
|
383
383
|
source: s.Struct({
|
|
384
384
|
dialect: s.String,
|
|
385
385
|
schemaFingerprint: s.String
|
|
386
386
|
}),
|
|
387
|
-
scope:
|
|
387
|
+
scope: Qe,
|
|
388
388
|
consistency: s.Literal("live", "snapshot"),
|
|
389
389
|
compression: s.Literal("zstd", "gzip", "none"),
|
|
390
|
-
tables: s.Array(
|
|
391
|
-
assets:
|
|
390
|
+
tables: s.Array(Ze),
|
|
391
|
+
assets: $e,
|
|
392
392
|
masking: s.optional(s.Struct({
|
|
393
393
|
policyId: s.optional(s.String),
|
|
394
394
|
transformed: s.Array(s.String)
|
|
395
395
|
}))
|
|
396
|
-
}),
|
|
396
|
+
}), tt = s.encodeSync(s.parseJson(et)), nt = s.decodeUnknownSync(s.parseJson(et)), V = "manifest.json", rt = "README.md", it = "data", H = "assets", U = "assets/index.ndjson", at = ".ledger.json", ot = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", st = (e, t) => {
|
|
397
397
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
398
398
|
return `# Voltro data export
|
|
399
399
|
|
|
@@ -407,7 +407,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
407
407
|
| **Format version** | ${e.formatVersion} |
|
|
408
408
|
| **Consistency** | ${e.consistency} |
|
|
409
409
|
| **Compression** | ${e.compression} |
|
|
410
|
-
| **Scope** | ${
|
|
410
|
+
| **Scope** | ${ot(e.scope)} |
|
|
411
411
|
| **Tables** | ${e.tables.length} |
|
|
412
412
|
| **Rows (total)** | ${n} |
|
|
413
413
|
| **Assets** | ${a} |
|
|
@@ -432,26 +432,26 @@ voltro data import <this-directory>
|
|
|
432
432
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
433
433
|
\`\`\`
|
|
434
434
|
`;
|
|
435
|
-
},
|
|
435
|
+
}, ct = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), lt = a.exponential({
|
|
436
436
|
start: .01,
|
|
437
437
|
factor: 2,
|
|
438
438
|
count: 16
|
|
439
|
-
}),
|
|
439
|
+
}), ut = i.histogram("voltro_data_transfer_table_seconds", lt, "Per-table export/import duration in seconds, tagged by phase."), W = (e, t) => {
|
|
440
440
|
if (e) try {
|
|
441
441
|
e(t);
|
|
442
442
|
} catch {}
|
|
443
|
-
},
|
|
443
|
+
}, dt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(ut.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), ct.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
444
444
|
table: t,
|
|
445
445
|
index: r,
|
|
446
446
|
tableCount: a,
|
|
447
447
|
phase: e
|
|
448
|
-
} })),
|
|
448
|
+
} })), ft = () => ({
|
|
449
449
|
tables: {},
|
|
450
450
|
assets: {}
|
|
451
451
|
}), G = (e) => n.tryPromise(async () => {
|
|
452
452
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
453
453
|
return JSON.parse(t);
|
|
454
|
-
}).pipe(n.orElseSucceed(
|
|
454
|
+
}).pipe(n.orElseSucceed(ft)), K = (e, t) => n.tryPromise({
|
|
455
455
|
try: async () => {
|
|
456
456
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
457
457
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
@@ -460,11 +460,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
460
460
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
461
461
|
path: e
|
|
462
462
|
})
|
|
463
|
-
}),
|
|
463
|
+
}), pt = (e, t) => t in e.tables, mt = (e, t) => t in e.assets, ht = /* @__PURE__ */ new Set([
|
|
464
464
|
"json",
|
|
465
465
|
"array",
|
|
466
466
|
"vector"
|
|
467
|
-
]),
|
|
467
|
+
]), gt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", _t = (e) => Buffer.from(e).toString("base64"), vt = (e) => new Uint8Array(Buffer.from(e, "base64")), yt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), bt = (e, t) => e == null ? null : t !== void 0 && ht.has(t) ? e : t === "bytes" ? _t(e) : t === "timestamp" || t === "date" ? yt(e) : typeof e == "bigint" ? {
|
|
468
468
|
$v: "bigint",
|
|
469
469
|
d: e.toString()
|
|
470
470
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -475,26 +475,26 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
475
475
|
d: e.toISOString()
|
|
476
476
|
} : e instanceof Uint8Array ? {
|
|
477
477
|
$v: "bytes",
|
|
478
|
-
d:
|
|
479
|
-
} : e,
|
|
478
|
+
d: _t(e)
|
|
479
|
+
} : e, xt = (e) => {
|
|
480
480
|
switch (e.$v) {
|
|
481
481
|
case "bigint": return BigInt(e.d);
|
|
482
482
|
case "number": return Number(e.d);
|
|
483
483
|
case "date": return new Date(e.d);
|
|
484
|
-
case "bytes": return
|
|
484
|
+
case "bytes": return vt(e.d);
|
|
485
485
|
default: return e;
|
|
486
486
|
}
|
|
487
|
-
},
|
|
487
|
+
}, St = (e, t) => e == null ? null : t !== void 0 && ht.has(t) ? e : gt(e) ? xt(e) : t === "bytes" ? vt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Ct = (e, t) => {
|
|
488
488
|
let n = {};
|
|
489
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
489
|
+
for (let [r, i] of Object.entries(e)) n[r] = bt(i, t[r]);
|
|
490
490
|
return n;
|
|
491
|
-
},
|
|
491
|
+
}, wt = (e, t) => {
|
|
492
492
|
let n = {};
|
|
493
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
493
|
+
for (let [r, i] of Object.entries(e)) n[r] = St(i, t[r]);
|
|
494
494
|
return n;
|
|
495
|
-
},
|
|
496
|
-
try: () =>
|
|
497
|
-
catch: (e) => new
|
|
495
|
+
}, Tt = new TextEncoder(), Et = (e, t) => e.pipe(c.map((e) => Tt.encode(`${JSON.stringify(Ct(e, t))}\n`))), Dt = (e, t, r) => e.pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.mapEffect((e) => n.try({
|
|
496
|
+
try: () => wt(JSON.parse(e), r),
|
|
497
|
+
catch: (e) => new Ae({
|
|
498
498
|
table: t,
|
|
499
499
|
reason: String(e?.message ?? e)
|
|
500
500
|
})
|
|
@@ -518,11 +518,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
518
518
|
};
|
|
519
519
|
for (let t of e) n.has(t) && s(t);
|
|
520
520
|
return i;
|
|
521
|
-
},
|
|
521
|
+
}, Ot = (e, t) => {
|
|
522
522
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
523
523
|
for (let e of n?.columns ?? []) r[e.name] = e.type;
|
|
524
524
|
return r;
|
|
525
|
-
},
|
|
525
|
+
}, kt = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", At = (e, t, r) => n.tryPromise({
|
|
526
526
|
try: async () => {
|
|
527
527
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: l, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
528
528
|
d.update(e);
|
|
@@ -531,7 +531,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
531
531
|
n(null, e);
|
|
532
532
|
} }), m = l.fromWeb(c.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
533
533
|
try {
|
|
534
|
-
await s(m, p,
|
|
534
|
+
await s(m, p, Je(r), n(h)), await i.rename(h, t);
|
|
535
535
|
} catch (e) {
|
|
536
536
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
537
537
|
}
|
|
@@ -544,7 +544,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
544
544
|
reason: `write table failed: ${String(e?.message ?? e)}`,
|
|
545
545
|
path: t
|
|
546
546
|
})
|
|
547
|
-
}),
|
|
547
|
+
}), jt = (e, t) => n.tryPromise({
|
|
548
548
|
try: async () => {
|
|
549
549
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
550
550
|
},
|
|
@@ -552,11 +552,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
552
552
|
reason: `append failed: ${String(t?.message ?? t)}`,
|
|
553
553
|
path: e
|
|
554
554
|
})
|
|
555
|
-
}),
|
|
555
|
+
}), Mt = (e, t) => n.gen(function* () {
|
|
556
556
|
let n = [], r = t.ordered.length;
|
|
557
557
|
for (let i = 0; i < t.ordered.length; i++) {
|
|
558
|
-
let a = t.ordered[i], o =
|
|
559
|
-
if (
|
|
558
|
+
let a = t.ordered[i], o = Ot(t.snapshot, a), s = kt(t.snapshot, a), l = `${it}/${a}.ndjson${qe(t.compression)}`;
|
|
559
|
+
if (pt(t.ledger, a)) {
|
|
560
560
|
let e = t.ledger.tables[a];
|
|
561
561
|
W(t.onProgress, {
|
|
562
562
|
phase: "export",
|
|
@@ -590,7 +590,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
590
590
|
chunkSize: t.chunkSize,
|
|
591
591
|
retry: t.retry,
|
|
592
592
|
...u === void 0 ? {} : { where: u }
|
|
593
|
-
}), f = t.mask ? d.pipe(c.map((e) => P(e, a, t.mask.actions, t.mask.seed, o))) : d, { rowCount: m, checksum: h } = yield*
|
|
593
|
+
}), f = t.mask ? d.pipe(c.map((e) => P(e, a, t.mask.actions, t.mask.seed, o))) : d, { rowCount: m, checksum: h } = yield* dt("export", a, i, r, (e) => e.rowCount, At(Et(f, o), t.bundleFile(l), t.compression));
|
|
594
594
|
t.ledger.tables[a] = {
|
|
595
595
|
rowCount: m,
|
|
596
596
|
checksum: h
|
|
@@ -612,43 +612,43 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
612
612
|
});
|
|
613
613
|
}
|
|
614
614
|
return n;
|
|
615
|
-
}),
|
|
615
|
+
}), Nt = (e) => ({
|
|
616
616
|
_tag: "RawSqlFragment",
|
|
617
617
|
strings: [e],
|
|
618
618
|
values: []
|
|
619
|
-
}),
|
|
619
|
+
}), Pt = (t, i, a) => n.tryPromise({
|
|
620
620
|
try: () => t.transactional(async (t) => {
|
|
621
|
-
if (t.raw) for (let e of
|
|
621
|
+
if (t.raw) for (let e of ge(i)) await t.raw(Nt(e));
|
|
622
622
|
let o = await n.runPromiseExit(a(t));
|
|
623
623
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
624
624
|
return o.value;
|
|
625
625
|
}),
|
|
626
626
|
catch: (e) => e instanceof F ? e : new F({ reason: `snapshot export failed: ${String(e?.message ?? e)}` })
|
|
627
|
-
}),
|
|
628
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
627
|
+
}), Ft = (e) => n.gen(function* () {
|
|
628
|
+
let t = e.scope ?? { kind: "all" }, r = Ke(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, s = o.exponential("50 millis").pipe(o.jittered, o.intersect(o.recurs(a))), l = yield* n.promise(() => import("node:path")), u = (t) => l.join(e.outDir, t);
|
|
629
629
|
yield* n.tryPromise({
|
|
630
630
|
try: async () => {
|
|
631
631
|
let t = await import("node:fs/promises");
|
|
632
|
-
await t.mkdir(u(
|
|
632
|
+
await t.mkdir(u(it), { recursive: !0 }), e.assets && await t.mkdir(u(H), { recursive: !0 });
|
|
633
633
|
},
|
|
634
634
|
catch: (t) => new F({
|
|
635
635
|
reason: `mkdir failed: ${String(t?.message ?? t)}`,
|
|
636
636
|
path: e.outDir
|
|
637
637
|
})
|
|
638
638
|
});
|
|
639
|
-
let d = u(
|
|
639
|
+
let d = u(at), f = yield* G(d), p = e.subset ? ze(yield* z(e.store, e.snapshot, e.subset), e.snapshot) : yield* Be(t, e.snapshot, {
|
|
640
640
|
store: e.store,
|
|
641
641
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
642
642
|
}), h = q(p.tables, e.snapshot), g = e.consistency ?? "live", _, v;
|
|
643
643
|
if (e.masking) {
|
|
644
|
-
let t = e.classification ??
|
|
644
|
+
let t = e.classification ?? ve(e.snapshot), r = De(h.map((t) => ({
|
|
645
645
|
name: t,
|
|
646
646
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
647
647
|
name: e.name,
|
|
648
648
|
type: e.type
|
|
649
649
|
}))
|
|
650
650
|
})), t, e.masking);
|
|
651
|
-
if (r.unclassified.length > 0) return yield* n.fail(new
|
|
651
|
+
if (r.unclassified.length > 0) return yield* n.fail(new Fe({ unclassified: r.unclassified }));
|
|
652
652
|
v = {
|
|
653
653
|
actions: r.actions,
|
|
654
654
|
seed: e.masking.seed
|
|
@@ -671,17 +671,17 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
671
671
|
bundleFile: u,
|
|
672
672
|
...v ? { mask: v } : {},
|
|
673
673
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
674
|
-
}, b = g === "snapshot" ? yield*
|
|
674
|
+
}, b = g === "snapshot" ? yield* Pt(e.store, e.dialect, (e) => Mt(e, y)) : yield* Mt(e.store, y), x = 0, S = 0;
|
|
675
675
|
if (e.assets) {
|
|
676
676
|
let t = u(U);
|
|
677
|
-
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
677
|
+
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => Ue(e, u(H))), n.flatMap((e) => {
|
|
678
678
|
let i = {
|
|
679
679
|
key: r.key,
|
|
680
680
|
sha256: e.sha256,
|
|
681
681
|
size: e.size,
|
|
682
682
|
contentType: e.contentType
|
|
683
683
|
};
|
|
684
|
-
return x++, S += e.size, f.assets[r.key] = !0,
|
|
684
|
+
return x++, S += e.size, f.assets[r.key] = !0, jt(t, JSON.stringify(i)).pipe(n.andThen(K(d, f)));
|
|
685
685
|
}))).pipe(n.mapError((e) => e instanceof F ? e : new F({ reason: `asset export failed: ${String(e)}` })));
|
|
686
686
|
}
|
|
687
687
|
let C = {
|
|
@@ -707,23 +707,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
707
707
|
};
|
|
708
708
|
return yield* n.tryPromise({
|
|
709
709
|
try: async () => {
|
|
710
|
-
await (await import("node:fs/promises")).writeFile(u(
|
|
710
|
+
await (await import("node:fs/promises")).writeFile(u(V), tt(C));
|
|
711
711
|
},
|
|
712
712
|
catch: (e) => new F({
|
|
713
713
|
reason: `manifest write failed: ${String(e?.message ?? e)}`,
|
|
714
|
-
path:
|
|
714
|
+
path: V
|
|
715
715
|
})
|
|
716
716
|
}), yield* n.tryPromise({
|
|
717
717
|
try: async () => {
|
|
718
718
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
719
|
-
await e.writeFile(u(
|
|
719
|
+
await e.writeFile(u(rt), st(C, { hostname: t.hostname() }));
|
|
720
720
|
},
|
|
721
721
|
catch: (e) => new F({
|
|
722
722
|
reason: `readme write failed: ${String(e?.message ?? e)}`,
|
|
723
|
-
path:
|
|
723
|
+
path: rt
|
|
724
724
|
})
|
|
725
725
|
}), C;
|
|
726
|
-
}),
|
|
726
|
+
}), It = (e) => {
|
|
727
727
|
switch (e) {
|
|
728
728
|
case "id":
|
|
729
729
|
case "text":
|
|
@@ -742,12 +742,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
742
742
|
case "raw": return "raw";
|
|
743
743
|
default: return "string";
|
|
744
744
|
}
|
|
745
|
-
},
|
|
745
|
+
}, Lt = {
|
|
746
746
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
747
747
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
748
748
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
749
|
-
},
|
|
750
|
-
let i =
|
|
749
|
+
}, Rt = (e, t, n, r) => {
|
|
750
|
+
let i = It(n);
|
|
751
751
|
if (i === "raw") return {
|
|
752
752
|
table: e,
|
|
753
753
|
column: t,
|
|
@@ -755,7 +755,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
755
755
|
severity: "block",
|
|
756
756
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
757
757
|
};
|
|
758
|
-
let a =
|
|
758
|
+
let a = Lt[i];
|
|
759
759
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
760
760
|
table: e,
|
|
761
761
|
column: t,
|
|
@@ -769,20 +769,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
769
769
|
severity: "block",
|
|
770
770
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
771
771
|
};
|
|
772
|
-
},
|
|
772
|
+
}, zt = (e, t, n) => {
|
|
773
773
|
if (t === n) return [];
|
|
774
774
|
let r = [];
|
|
775
775
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
776
|
-
let a =
|
|
776
|
+
let a = Rt(t.name, e, i, n);
|
|
777
777
|
a && r.push(a);
|
|
778
778
|
}
|
|
779
779
|
return r;
|
|
780
|
-
},
|
|
780
|
+
}, Bt = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, Vt = (e, t) => new Map((e.tables.find((e) => e.name === t)?.columns ?? []).map((e) => [e.name, e.type])), Ht = (e, t) => {
|
|
781
781
|
let n = [], r = new Set(e.tables.map((e) => e.name)), i = new Set(t.tables.map((e) => e.name));
|
|
782
782
|
for (let e of [...r].filter((e) => !i.has(e)).sort()) n.push(`table '${e}' is in the bundle but MISSING from the target`);
|
|
783
783
|
for (let e of [...i].filter((e) => !r.has(e)).sort()) n.push(`table '${e}' is in the target but NOT in the bundle (import leaves it untouched)`);
|
|
784
784
|
for (let a of [...r].filter((e) => i.has(e)).sort()) {
|
|
785
|
-
let r =
|
|
785
|
+
let r = Vt(e, a), i = Vt(t, a);
|
|
786
786
|
for (let [e] of r) i.has(e) || n.push(`${a}.${e}: in the bundle, MISSING from the target`);
|
|
787
787
|
for (let [e] of i) r.has(e) || n.push(`${a}.${e}: in the target, not in the bundle`);
|
|
788
788
|
for (let [e, t] of r) {
|
|
@@ -791,7 +791,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
791
791
|
}
|
|
792
792
|
}
|
|
793
793
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n.length > 0 ? n : ["the schema shape differs (indexes, nullability, or defaults) — the specific change is below the table/column granularity shown here"];
|
|
794
|
-
},
|
|
794
|
+
}, Ut = (e, t, n) => {
|
|
795
795
|
let r = m(t);
|
|
796
796
|
return r === e ? {
|
|
797
797
|
drifted: !1,
|
|
@@ -802,19 +802,19 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
802
802
|
drifted: !0,
|
|
803
803
|
bundleFingerprint: e,
|
|
804
804
|
targetFingerprint: r,
|
|
805
|
-
diff: n ?
|
|
805
|
+
diff: n ? Ht(n, t) : ["the target schema fingerprint does not match the bundle (no source schema available to itemise the diff — the manifest records only the fingerprint)"]
|
|
806
806
|
};
|
|
807
|
-
},
|
|
807
|
+
}, Wt = (e) => n.tryPromise({
|
|
808
808
|
try: async () => {
|
|
809
809
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
810
|
-
return
|
|
810
|
+
return nt(await t.readFile(n.join(e, V), "utf8"));
|
|
811
811
|
},
|
|
812
812
|
catch: (e) => new F({
|
|
813
813
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
814
|
-
path:
|
|
814
|
+
path: V
|
|
815
815
|
})
|
|
816
|
-
}),
|
|
817
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
816
|
+
}), Gt = (t) => n.gen(function* () {
|
|
817
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* Wt(t.bundleDir);
|
|
818
818
|
if (a.formatVersion !== 1) return yield* n.fail(new F({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
819
819
|
if (t.targetSnapshot) {
|
|
820
820
|
let e = { tables: a.tables.map((e) => ({
|
|
@@ -827,9 +827,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
827
827
|
unique: !1,
|
|
828
828
|
hasDefault: !1
|
|
829
829
|
}))
|
|
830
|
-
})) }, r =
|
|
830
|
+
})) }, r = Ut(a.source.schemaFingerprint, t.targetSnapshot, e);
|
|
831
831
|
if (r.drifted) {
|
|
832
|
-
if (!t.force) return yield* n.fail(new
|
|
832
|
+
if (!t.force) return yield* n.fail(new Pe({
|
|
833
833
|
bundleFingerprint: r.bundleFingerprint,
|
|
834
834
|
targetFingerprint: r.targetFingerprint,
|
|
835
835
|
diff: r.diff
|
|
@@ -838,15 +838,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
838
838
|
}
|
|
839
839
|
}
|
|
840
840
|
if (t.targetDialect && t.targetDialect !== a.source.dialect) {
|
|
841
|
-
let e =
|
|
842
|
-
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new
|
|
841
|
+
let e = zt(a.tables, a.source.dialect, t.targetDialect).filter((e) => e.severity === "block");
|
|
842
|
+
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new Le({
|
|
843
843
|
sourceDialect: a.source.dialect,
|
|
844
844
|
targetDialect: t.targetDialect,
|
|
845
|
-
blocking: e.map(
|
|
845
|
+
blocking: e.map(Bt)
|
|
846
846
|
}));
|
|
847
847
|
}
|
|
848
848
|
let o = t.verify ?? !0, s = t.mode ?? "upsert", l = t.onConflict ?? "skip", u = t.ledgerPath ?? i.join(t.bundleDir, ".import..ledger.json"), f = yield* G(u);
|
|
849
|
-
if (s === "replace" && a.scope.kind !== "all") return yield* n.fail(new
|
|
849
|
+
if (s === "replace" && a.scope.kind !== "all") return yield* n.fail(new Ie({
|
|
850
850
|
mode: s,
|
|
851
851
|
reason: `replace refuses a partial bundle (scope='${a.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
852
852
|
}));
|
|
@@ -875,7 +875,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
875
875
|
r = t;
|
|
876
876
|
}
|
|
877
877
|
for (let t of i) (yield* m(() => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] }), t)) || (r = [...r, t]);
|
|
878
|
-
if (r.length > 0) return yield* n.fail(new
|
|
878
|
+
if (r.length > 0) return yield* n.fail(new Ne({
|
|
879
879
|
rows: r.slice(0, 20).map((e) => ({
|
|
880
880
|
table: e.entry.name,
|
|
881
881
|
id: String(e.row[e.entry.primaryKey]),
|
|
@@ -891,78 +891,117 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
891
891
|
});
|
|
892
892
|
r && (f.truncated = !0, yield* K(u, f));
|
|
893
893
|
}
|
|
894
|
-
let l = [],
|
|
895
|
-
for (let
|
|
896
|
-
let
|
|
897
|
-
if (r &&
|
|
894
|
+
let l = [], m = [], h = a.tables.length;
|
|
895
|
+
for (let d = 0; d < a.tables.length; d++) {
|
|
896
|
+
let g = a.tables[d];
|
|
897
|
+
if (r && g.name in f.tables) {
|
|
898
898
|
W(t.onProgress, {
|
|
899
899
|
phase: "import",
|
|
900
|
-
table:
|
|
901
|
-
index:
|
|
902
|
-
tableCount:
|
|
900
|
+
table: g.name,
|
|
901
|
+
index: d,
|
|
902
|
+
tableCount: h,
|
|
903
903
|
event: "done",
|
|
904
|
-
rowsDone: f.tables[
|
|
905
|
-
total:
|
|
904
|
+
rowsDone: f.tables[g.name].rowCount,
|
|
905
|
+
total: g.rowCount
|
|
906
906
|
});
|
|
907
907
|
continue;
|
|
908
908
|
}
|
|
909
|
-
let
|
|
909
|
+
let _ = i.join(t.bundleDir, g.file);
|
|
910
910
|
W(t.onProgress, {
|
|
911
911
|
phase: "import",
|
|
912
|
-
table:
|
|
913
|
-
index:
|
|
914
|
-
tableCount:
|
|
912
|
+
table: g.name,
|
|
913
|
+
index: d,
|
|
914
|
+
tableCount: h,
|
|
915
915
|
event: "start",
|
|
916
916
|
rowsDone: 0,
|
|
917
|
-
total:
|
|
917
|
+
total: g.rowCount
|
|
918
918
|
});
|
|
919
919
|
let v = l.length, y = n.gen(function* () {
|
|
920
|
-
let { createHash:
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
let r = {
|
|
924
|
-
entry: d,
|
|
920
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, u = S("bundle", () => E(_).pipe(Ye(a.compression))).pipe(c.tap((e) => n.sync(() => i.update(e)))), d = async (t) => {
|
|
921
|
+
let n = {
|
|
922
|
+
entry: g,
|
|
925
923
|
row: t,
|
|
926
924
|
reason: ""
|
|
927
925
|
};
|
|
928
|
-
|
|
929
|
-
e
|
|
930
|
-
})
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
926
|
+
try {
|
|
927
|
+
await p(e, g, t);
|
|
928
|
+
} catch (e) {
|
|
929
|
+
n.reason = String(e?.message ?? e), l.push(n);
|
|
930
|
+
}
|
|
931
|
+
}, f = yield* n.promise(async () => {
|
|
932
|
+
let n = t.copyLoader;
|
|
933
|
+
if (!n || n.dialect !== "postgres" || t.atomic) return !1;
|
|
934
|
+
if (s === "replace") return !0;
|
|
935
|
+
if (s !== "upsert") return !1;
|
|
936
|
+
try {
|
|
937
|
+
return (await e.query({
|
|
938
|
+
table: g.name,
|
|
939
|
+
predicate: void 0,
|
|
940
|
+
order: [],
|
|
941
|
+
take: 1,
|
|
942
|
+
skip: void 0,
|
|
943
|
+
projection: void 0
|
|
944
|
+
})).length === 0;
|
|
945
|
+
} catch {
|
|
946
|
+
return !1;
|
|
947
|
+
}
|
|
948
|
+
}), m = Object.keys(g.columnTypes), h = Math.max(1, t.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
949
|
+
if (v.length === 0) return;
|
|
950
|
+
let e = v;
|
|
951
|
+
v = [];
|
|
952
|
+
try {
|
|
953
|
+
await t.copyLoader.copyInto({
|
|
954
|
+
table: g.name,
|
|
955
|
+
columns: m,
|
|
956
|
+
columnTypes: g.columnTypes,
|
|
957
|
+
rows: e
|
|
958
|
+
});
|
|
959
|
+
} catch (n) {
|
|
960
|
+
f = !1, t.onWarn?.(`bulk COPY into ${g.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
961
|
+
for (let t of e) await d(t);
|
|
962
|
+
}
|
|
963
|
+
};
|
|
964
|
+
return yield* c.runForEach(Dt(u, g.name, g.columnTypes), (e) => n.promise(async () => {
|
|
965
|
+
if (o++, f) {
|
|
966
|
+
v.push(e), v.length >= h && await y();
|
|
967
|
+
return;
|
|
968
|
+
}
|
|
969
|
+
await d(e);
|
|
970
|
+
})), yield* n.promise(y), {
|
|
971
|
+
rowCount: o,
|
|
972
|
+
checksum: i.digest("hex")
|
|
934
973
|
};
|
|
935
|
-
}), { rowCount: b, checksum: x } = yield*
|
|
974
|
+
}), { rowCount: b, checksum: x } = yield* dt("import", g.name, d, h, (e) => e.rowCount, y);
|
|
936
975
|
if (o) {
|
|
937
|
-
if (x !==
|
|
938
|
-
what: `table ${
|
|
939
|
-
expected:
|
|
976
|
+
if (x !== g.checksum) return yield* n.fail(new I({
|
|
977
|
+
what: `table ${g.name} checksum`,
|
|
978
|
+
expected: g.checksum,
|
|
940
979
|
actual: x
|
|
941
980
|
}));
|
|
942
|
-
if (b !==
|
|
943
|
-
what: `table ${
|
|
944
|
-
expected: String(
|
|
981
|
+
if (b !== g.rowCount) return yield* n.fail(new I({
|
|
982
|
+
what: `table ${g.name} rowCount`,
|
|
983
|
+
expected: String(g.rowCount),
|
|
945
984
|
actual: String(b)
|
|
946
985
|
}));
|
|
947
986
|
}
|
|
948
987
|
W(t.onProgress, {
|
|
949
988
|
phase: "import",
|
|
950
|
-
table:
|
|
951
|
-
index:
|
|
952
|
-
tableCount:
|
|
989
|
+
table: g.name,
|
|
990
|
+
index: d,
|
|
991
|
+
tableCount: h,
|
|
953
992
|
event: "done",
|
|
954
993
|
rowsDone: b,
|
|
955
|
-
total:
|
|
956
|
-
}), r && (l.length > v ?
|
|
957
|
-
name:
|
|
994
|
+
total: g.rowCount
|
|
995
|
+
}), r && (l.length > v ? m.push({
|
|
996
|
+
name: g.name,
|
|
958
997
|
rowCount: b
|
|
959
|
-
}) : (f.tables[
|
|
998
|
+
}) : (f.tables[g.name] = {
|
|
960
999
|
rowCount: b,
|
|
961
|
-
checksum:
|
|
1000
|
+
checksum: g.checksum
|
|
962
1001
|
}, yield* K(u, f)));
|
|
963
1002
|
}
|
|
964
|
-
if (l.length > 0 && (yield* g(e, l)), r &&
|
|
965
|
-
for (let e of
|
|
1003
|
+
if (l.length > 0 && (yield* g(e, l)), r && m.length > 0) {
|
|
1004
|
+
for (let e of m) {
|
|
966
1005
|
let t = a.tables.find((t) => t.name === e.name);
|
|
967
1006
|
f.tables[e.name] = {
|
|
968
1007
|
rowCount: e.rowCount,
|
|
@@ -980,19 +1019,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
980
1019
|
catch: (e) => new F({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
981
1020
|
}) : yield* _(t.store, !0), t.assets && a.assets.index && !t.assetsAlreadyRestored) {
|
|
982
1021
|
let e = t.assets, r = i.join(t.bundleDir, H), o = i.join(t.bundleDir, a.assets.index ?? "assets/index.ndjson"), s = S("bundle", () => E(o)).pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.map((e) => JSON.parse(e)));
|
|
983
|
-
yield* c.runForEach(s, (t) => t.key in f.assets ? n.void :
|
|
1022
|
+
yield* c.runForEach(s, (t) => t.key in f.assets ? n.void : We(e, r, t).pipe(n.andThen(n.sync(() => {
|
|
984
1023
|
f.assets[t.key] = !0;
|
|
985
1024
|
})), n.andThen(K(u, f))));
|
|
986
1025
|
}
|
|
987
1026
|
return a;
|
|
988
|
-
}),
|
|
989
|
-
refuse: !
|
|
990
|
-
|
|
991
|
-
|
|
1027
|
+
}), Kt = (e) => e, qt = (e) => {
|
|
1028
|
+
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1029
|
+
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1030
|
+
return {
|
|
1031
|
+
refuse: !0,
|
|
1032
|
+
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1033
|
+
};
|
|
1034
|
+
}, J = Buffer.from("VBUNDLE2"), Y = 0, Jt = 1, Yt = 2, Xt = 32, Zt = async (e) => {
|
|
992
1035
|
let t = [], n = async (r) => {
|
|
993
1036
|
let i;
|
|
994
1037
|
try {
|
|
995
|
-
i = await se(
|
|
1038
|
+
i = await se(O(e, r), { withFileTypes: !0 });
|
|
996
1039
|
} catch {
|
|
997
1040
|
return;
|
|
998
1041
|
}
|
|
@@ -1002,21 +1045,21 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1002
1045
|
}
|
|
1003
1046
|
};
|
|
1004
1047
|
return await n(""), t.sort();
|
|
1005
|
-
},
|
|
1048
|
+
}, Qt = (e, t) => {
|
|
1006
1049
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1007
|
-
return r.writeUInt8(
|
|
1008
|
-
},
|
|
1050
|
+
return r.writeUInt8(Jt, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1051
|
+
}, $t = (e, t, n) => {
|
|
1009
1052
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1010
|
-
return a.writeUInt8(
|
|
1053
|
+
return a.writeUInt8(Yt, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1011
1054
|
};
|
|
1012
|
-
async function*
|
|
1055
|
+
async function* en(e) {
|
|
1013
1056
|
yield new Uint8Array(J);
|
|
1014
|
-
for (let t of await
|
|
1015
|
-
let n =
|
|
1016
|
-
if (yield new Uint8Array(
|
|
1057
|
+
for (let t of await Zt(e.dir)) {
|
|
1058
|
+
let n = O(e.dir, t), r = await ce(n);
|
|
1059
|
+
if (yield new Uint8Array(Qt(t, r.size)), r.size > 0) for await (let e of E(n)) yield new Uint8Array(e);
|
|
1017
1060
|
}
|
|
1018
1061
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1019
|
-
yield new Uint8Array(
|
|
1062
|
+
yield new Uint8Array($t(t.key, t.contentType, t.size));
|
|
1020
1063
|
let n = _("sha256"), r = 0;
|
|
1021
1064
|
for await (let i of e.blobs.open(t.key)) {
|
|
1022
1065
|
let e = Buffer.from(i);
|
|
@@ -1025,10 +1068,10 @@ async function* $t(e) {
|
|
|
1025
1068
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1026
1069
|
yield new Uint8Array(n.digest());
|
|
1027
1070
|
}
|
|
1028
|
-
yield new Uint8Array(Buffer.from([
|
|
1071
|
+
yield new Uint8Array(Buffer.from([Y]));
|
|
1029
1072
|
}
|
|
1030
|
-
var
|
|
1031
|
-
let n =
|
|
1073
|
+
var tn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), nn = (e) => new Promise((t) => e.end(t)), rn = async (e, t) => {
|
|
1074
|
+
let n = fe(t.dir);
|
|
1032
1075
|
await oe(n, { recursive: !0 });
|
|
1033
1076
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1034
1077
|
let { value: e, done: t } = await i.next();
|
|
@@ -1044,24 +1087,24 @@ var en = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tn
|
|
|
1044
1087
|
for (;;) {
|
|
1045
1088
|
await s(1);
|
|
1046
1089
|
let e = c(1).readUInt8(0);
|
|
1047
|
-
if (e ===
|
|
1048
|
-
if (e ===
|
|
1090
|
+
if (e === Y) return;
|
|
1091
|
+
if (e === Jt) {
|
|
1049
1092
|
await s(4);
|
|
1050
1093
|
let e = c(4).readUInt32BE(0);
|
|
1051
1094
|
await s(e);
|
|
1052
1095
|
let t = c(e).toString("utf8");
|
|
1053
1096
|
await s(8);
|
|
1054
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1055
|
-
if (a !== n && !a.startsWith(n +
|
|
1056
|
-
await oe(
|
|
1057
|
-
let o =
|
|
1097
|
+
let i = c(8).readBigUInt64BE(0), a = fe(n, t);
|
|
1098
|
+
if (a !== n && !a.startsWith(n + pe)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1099
|
+
await oe(de(a), { recursive: !0 });
|
|
1100
|
+
let o = D(a);
|
|
1058
1101
|
for (; i > 0n;) {
|
|
1059
1102
|
r.length === 0 && await s(1);
|
|
1060
1103
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1061
|
-
await
|
|
1104
|
+
await tn(o, c(e)), i -= BigInt(e);
|
|
1062
1105
|
}
|
|
1063
|
-
await
|
|
1064
|
-
} else if (e ===
|
|
1106
|
+
await nn(o);
|
|
1107
|
+
} else if (e === Yt) {
|
|
1065
1108
|
await s(4);
|
|
1066
1109
|
let e = c(4).readUInt32BE(0);
|
|
1067
1110
|
await s(e);
|
|
@@ -1078,17 +1121,17 @@ var en = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tn
|
|
|
1078
1121
|
for (; p > 0n;) {
|
|
1079
1122
|
r.length === 0 && await s(1);
|
|
1080
1123
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1081
|
-
u.update(t), l || await
|
|
1124
|
+
u.update(t), l || await tn(d, t), p -= BigInt(e);
|
|
1082
1125
|
}
|
|
1083
|
-
l ? d.destroy() : await
|
|
1084
|
-
let m = c(
|
|
1126
|
+
l ? d.destroy() : await nn(d), await f, await s(Xt);
|
|
1127
|
+
let m = c(Xt);
|
|
1085
1128
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1086
1129
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1087
1130
|
}
|
|
1088
1131
|
} finally {
|
|
1089
1132
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1090
1133
|
}
|
|
1091
|
-
},
|
|
1134
|
+
}, an = async (e, t) => {
|
|
1092
1135
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1093
1136
|
let { value: e, done: t } = await a.next();
|
|
1094
1137
|
return t ? !1 : (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1110,8 +1153,8 @@ var en = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tn
|
|
|
1110
1153
|
if (!await s(J.length) || !c(J.length).equals(J)) throw Error("not a bundle archive (bad magic)");
|
|
1111
1154
|
for (; await s(1);) {
|
|
1112
1155
|
let e = c(1).readUInt8(0);
|
|
1113
|
-
if (e ===
|
|
1114
|
-
if (e !==
|
|
1156
|
+
if (e === Y || e === Yt) break;
|
|
1157
|
+
if (e !== Jt) throw Error(`unknown bundle entry kind ${e}`);
|
|
1115
1158
|
if (!await s(4)) break;
|
|
1116
1159
|
let t = c(4).readUInt32BE(0);
|
|
1117
1160
|
if (!await s(t)) break;
|
|
@@ -1121,22 +1164,22 @@ var en = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tn
|
|
|
1121
1164
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1122
1165
|
}
|
|
1123
1166
|
return r;
|
|
1124
|
-
},
|
|
1167
|
+
}, on = (e, t) => rn(e, {
|
|
1125
1168
|
dir: t,
|
|
1126
|
-
blobs:
|
|
1127
|
-
}),
|
|
1169
|
+
blobs: sn(t)
|
|
1170
|
+
}), sn = (e) => {
|
|
1128
1171
|
let t = /* @__PURE__ */ new Set();
|
|
1129
1172
|
return {
|
|
1130
1173
|
has: async (e) => t.has(e),
|
|
1131
1174
|
write: async (n, r, i) => {
|
|
1132
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1175
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = O(e, "assets");
|
|
1133
1176
|
await oe(c, { recursive: !0 });
|
|
1134
1177
|
let l = _("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1135
1178
|
l.update(e), u += e.length, n(null, e);
|
|
1136
|
-
} }), f =
|
|
1137
|
-
await o(r, d,
|
|
1179
|
+
} }), f = O(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1180
|
+
await o(r, d, D(f));
|
|
1138
1181
|
let p = l.digest("hex");
|
|
1139
|
-
await a.rename(f,
|
|
1182
|
+
await a.rename(f, O(c, p)), await a.appendFile(O(c, "index.ndjson"), `${JSON.stringify({
|
|
1140
1183
|
key: n,
|
|
1141
1184
|
sha256: p,
|
|
1142
1185
|
size: u,
|
|
@@ -1144,33 +1187,33 @@ var en = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tn
|
|
|
1144
1187
|
})}\n`), t.add(n);
|
|
1145
1188
|
}
|
|
1146
1189
|
};
|
|
1147
|
-
},
|
|
1190
|
+
}, cn = me(b), ln = Buffer.from("VENC1\0\0\0"), un = 32, dn = 16, fn = 7, X = 16, pn = 256 * 1024, Z = {
|
|
1148
1191
|
N: 32768,
|
|
1149
1192
|
r: 8,
|
|
1150
1193
|
p: 1
|
|
1151
|
-
},
|
|
1194
|
+
}, mn = 128 * 1024 * 1024, Q = 43, hn = (e, t, n) => {
|
|
1152
1195
|
let r = Buffer.alloc(12);
|
|
1153
|
-
return e.copy(r, 0, 0,
|
|
1196
|
+
return e.copy(r, 0, 0, fn), r.writeUInt32BE(t >>> 0, fn), r.writeUInt8(+!!n, 11), r;
|
|
1154
1197
|
};
|
|
1155
|
-
async function*
|
|
1198
|
+
async function* gn(e, t) {
|
|
1156
1199
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1157
|
-
let n = y(
|
|
1200
|
+
let n = y(dn), r = y(fn), i = await cn(t, n, un, {
|
|
1158
1201
|
...Z,
|
|
1159
|
-
maxmem:
|
|
1202
|
+
maxmem: mn
|
|
1160
1203
|
}), a = Buffer.alloc(Q);
|
|
1161
|
-
|
|
1204
|
+
ln.copy(a, 0), a.writeUInt32BE(Z.N, 8), a.writeUInt32BE(Z.r, 12), a.writeUInt32BE(Z.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1162
1205
|
let o = 0, s = (e, t) => {
|
|
1163
|
-
let n = h("aes-256-gcm", i,
|
|
1206
|
+
let n = h("aes-256-gcm", i, hn(r, o, t));
|
|
1164
1207
|
o === 0 && n.setAAD(a);
|
|
1165
1208
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1166
1209
|
o++;
|
|
1167
|
-
let l = Buffer.alloc(5 + s.length +
|
|
1168
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
1210
|
+
let l = Buffer.alloc(5 + s.length + X);
|
|
1211
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + X, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1169
1212
|
}, c = Buffer.alloc(0);
|
|
1170
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1213
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > pn;) yield s(c.subarray(0, pn), !1), c = c.subarray(pn);
|
|
1171
1214
|
yield s(c, !0);
|
|
1172
1215
|
}
|
|
1173
|
-
async function*
|
|
1216
|
+
async function* _n(e, t) {
|
|
1174
1217
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1175
1218
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1176
1219
|
let { value: e, done: t } = await n.next();
|
|
@@ -1181,12 +1224,12 @@ async function* hn(e, t) {
|
|
|
1181
1224
|
};
|
|
1182
1225
|
if (!await o(Q)) throw Error("encrypted bundle truncated (no header)");
|
|
1183
1226
|
let s = r.subarray(0, Q);
|
|
1184
|
-
if (!s.subarray(0, 8).equals(
|
|
1185
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await
|
|
1227
|
+
if (!s.subarray(0, 8).equals(ln)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1228
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await cn(t, d, un, {
|
|
1186
1229
|
N: c,
|
|
1187
1230
|
r: l,
|
|
1188
1231
|
p: u,
|
|
1189
|
-
maxmem:
|
|
1232
|
+
maxmem: mn
|
|
1190
1233
|
});
|
|
1191
1234
|
r = r.subarray(Q);
|
|
1192
1235
|
let m = 0, h = !1;
|
|
@@ -1197,11 +1240,11 @@ async function* hn(e, t) {
|
|
|
1197
1240
|
}
|
|
1198
1241
|
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
1199
1242
|
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
1200
|
-
if (t <
|
|
1243
|
+
if (t < X) throw Error("encrypted bundle corrupt (short frame)");
|
|
1201
1244
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1202
|
-
let n = r.subarray(5, 5 + t -
|
|
1245
|
+
let n = r.subarray(5, 5 + t - X), i = r.subarray(5 + t - X, 5 + t);
|
|
1203
1246
|
r = r.subarray(5 + t);
|
|
1204
|
-
let a = g("aes-256-gcm", p,
|
|
1247
|
+
let a = g("aes-256-gcm", p, hn(f, m, e));
|
|
1205
1248
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1206
1249
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1207
1250
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1209,7 +1252,7 @@ async function* hn(e, t) {
|
|
|
1209
1252
|
}
|
|
1210
1253
|
//#endregion
|
|
1211
1254
|
//#region src/blobStorage.ts
|
|
1212
|
-
var
|
|
1255
|
+
var vn = (e, t) => ({
|
|
1213
1256
|
list: () => {
|
|
1214
1257
|
let e = t();
|
|
1215
1258
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1225,14 +1268,14 @@ var gn = (e, t) => ({
|
|
|
1225
1268
|
let r = await te((await n.runPromise(C(e, t))).stream);
|
|
1226
1269
|
for await (let e of r) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1227
1270
|
} })
|
|
1228
|
-
}),
|
|
1271
|
+
}), yn = (e, t) => vn(e, () => p(t, {
|
|
1229
1272
|
table: x,
|
|
1230
1273
|
chunkSize: 500
|
|
1231
1274
|
}).pipe(c.map((e) => ({
|
|
1232
1275
|
key: String(e.key),
|
|
1233
1276
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1234
1277
|
size: Number(e.size ?? 0)
|
|
1235
|
-
})))),
|
|
1278
|
+
})))), bn = (e) => ({
|
|
1236
1279
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1237
1280
|
write: async (t, r, i) => {
|
|
1238
1281
|
let a = S("storage", () => ne.from(r));
|
|
@@ -1241,10 +1284,10 @@ var gn = (e, t) => ({
|
|
|
1241
1284
|
totalSize: i.size
|
|
1242
1285
|
}));
|
|
1243
1286
|
}
|
|
1244
|
-
}),
|
|
1287
|
+
}), xn = (e, t) => {
|
|
1245
1288
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1246
|
-
return e === "mariadb" &&
|
|
1247
|
-
},
|
|
1289
|
+
return e === "mariadb" && Sn(n) ? n : r;
|
|
1290
|
+
}, Sn = (e) => (process.env.PATH ?? "").split(ue).filter((e) => e !== "").some((t) => ie(O(t, e))), Cn = (e) => {
|
|
1248
1291
|
switch (e) {
|
|
1249
1292
|
case "postgres": return "db.dump";
|
|
1250
1293
|
case "mysql":
|
|
@@ -1254,7 +1297,7 @@ var gn = (e, t) => ({
|
|
|
1254
1297
|
case "mssql": return "db.bacpac";
|
|
1255
1298
|
default: return "db.dump";
|
|
1256
1299
|
}
|
|
1257
|
-
},
|
|
1300
|
+
}, wn = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1258
1301
|
if (e.host || e.database) return {
|
|
1259
1302
|
host: e.host ?? "localhost",
|
|
1260
1303
|
port: e.port ?? 3306,
|
|
@@ -1270,7 +1313,7 @@ var gn = (e, t) => ({
|
|
|
1270
1313
|
password: decodeURIComponent(t.password) || "",
|
|
1271
1314
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1272
1315
|
};
|
|
1273
|
-
},
|
|
1316
|
+
}, Tn = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, En = (e, t, n) => {
|
|
1274
1317
|
switch (e) {
|
|
1275
1318
|
case "postgres": {
|
|
1276
1319
|
if (t.url) return {
|
|
@@ -1317,12 +1360,13 @@ var gn = (e, t) => ({
|
|
|
1317
1360
|
let r = $(t);
|
|
1318
1361
|
return {
|
|
1319
1362
|
kind: "spawn",
|
|
1320
|
-
tool:
|
|
1363
|
+
tool: xn(e, "dump"),
|
|
1321
1364
|
args: [
|
|
1322
1365
|
"--single-transaction",
|
|
1323
1366
|
"--routines",
|
|
1324
1367
|
"--triggers",
|
|
1325
1368
|
"--no-tablespaces",
|
|
1369
|
+
"--protocol=TCP",
|
|
1326
1370
|
"-h",
|
|
1327
1371
|
r.host,
|
|
1328
1372
|
"-P",
|
|
@@ -1338,7 +1382,7 @@ var gn = (e, t) => ({
|
|
|
1338
1382
|
case "sqlite":
|
|
1339
1383
|
case "turso": return {
|
|
1340
1384
|
kind: "copy",
|
|
1341
|
-
from:
|
|
1385
|
+
from: wn(t),
|
|
1342
1386
|
to: n
|
|
1343
1387
|
};
|
|
1344
1388
|
case "mssql": return {
|
|
@@ -1346,7 +1390,7 @@ var gn = (e, t) => ({
|
|
|
1346
1390
|
tool: "sqlpackage",
|
|
1347
1391
|
args: [
|
|
1348
1392
|
"/Action:Export",
|
|
1349
|
-
`/SourceConnectionString:${
|
|
1393
|
+
`/SourceConnectionString:${Tn($({
|
|
1350
1394
|
...t,
|
|
1351
1395
|
port: t.port ?? 1433
|
|
1352
1396
|
}))}`,
|
|
@@ -1359,7 +1403,7 @@ var gn = (e, t) => ({
|
|
|
1359
1403
|
reason: `no native backup for dialect '${e}'`
|
|
1360
1404
|
});
|
|
1361
1405
|
}
|
|
1362
|
-
},
|
|
1406
|
+
}, Dn = (e, t, n) => {
|
|
1363
1407
|
switch (e) {
|
|
1364
1408
|
case "postgres": {
|
|
1365
1409
|
if (t.url) return {
|
|
@@ -1404,8 +1448,9 @@ var gn = (e, t) => ({
|
|
|
1404
1448
|
let r = $(t);
|
|
1405
1449
|
return {
|
|
1406
1450
|
kind: "spawn",
|
|
1407
|
-
tool:
|
|
1451
|
+
tool: xn(e, "client"),
|
|
1408
1452
|
args: [
|
|
1453
|
+
"--protocol=TCP",
|
|
1409
1454
|
"-h",
|
|
1410
1455
|
r.host,
|
|
1411
1456
|
"-P",
|
|
@@ -1422,14 +1467,14 @@ var gn = (e, t) => ({
|
|
|
1422
1467
|
case "turso": return {
|
|
1423
1468
|
kind: "copy",
|
|
1424
1469
|
from: n,
|
|
1425
|
-
to:
|
|
1470
|
+
to: wn(t)
|
|
1426
1471
|
};
|
|
1427
1472
|
case "mssql": return {
|
|
1428
1473
|
kind: "spawn",
|
|
1429
1474
|
tool: "sqlpackage",
|
|
1430
1475
|
args: [
|
|
1431
1476
|
"/Action:Import",
|
|
1432
|
-
`/TargetConnectionString:${
|
|
1477
|
+
`/TargetConnectionString:${Tn($({
|
|
1433
1478
|
...t,
|
|
1434
1479
|
port: t.port ?? 1433
|
|
1435
1480
|
}))}`,
|
|
@@ -1442,13 +1487,13 @@ var gn = (e, t) => ({
|
|
|
1442
1487
|
reason: `no native restore for dialect '${e}'`
|
|
1443
1488
|
});
|
|
1444
1489
|
}
|
|
1445
|
-
},
|
|
1490
|
+
}, On = (e) => e.kind === "copy" ? n.async((t) => {
|
|
1446
1491
|
re(e.from, e.to, (r) => t(r ? n.fail(new L({
|
|
1447
1492
|
tool: "copy",
|
|
1448
1493
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1449
1494
|
})) : n.void));
|
|
1450
1495
|
}) : n.async((t) => {
|
|
1451
|
-
let r =
|
|
1496
|
+
let r = he(e.tool, [...e.args], {
|
|
1452
1497
|
env: {
|
|
1453
1498
|
...process.env,
|
|
1454
1499
|
...e.env
|
|
@@ -1461,19 +1506,33 @@ var gn = (e, t) => ({
|
|
|
1461
1506
|
}), i = "";
|
|
1462
1507
|
r.stderr?.on("data", (e) => {
|
|
1463
1508
|
i += String(e);
|
|
1464
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1509
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(D(e.stdoutFile)), e.stdinFile && r.stdin && E(e.stdinFile).pipe(r.stdin);
|
|
1510
|
+
let a = () => {
|
|
1511
|
+
if (e.stdoutFile !== void 0) try {
|
|
1512
|
+
ae(e.stdoutFile);
|
|
1513
|
+
} catch {}
|
|
1514
|
+
};
|
|
1515
|
+
r.on("error", (r) => {
|
|
1516
|
+
a(), t(n.fail(new L({
|
|
1517
|
+
tool: e.tool,
|
|
1518
|
+
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`
|
|
1519
|
+
})));
|
|
1520
|
+
}), r.on("close", (r) => {
|
|
1521
|
+
if (r === 0) {
|
|
1522
|
+
t(n.void);
|
|
1523
|
+
return;
|
|
1524
|
+
}
|
|
1525
|
+
a(), t(n.fail(new L({
|
|
1526
|
+
tool: e.tool,
|
|
1527
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${le(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
1528
|
+
stderr: i.slice(-2e3)
|
|
1529
|
+
})));
|
|
1530
|
+
});
|
|
1531
|
+
}), kn = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), An = (e) => n.gen(function* () {
|
|
1532
|
+
let t = yield* Be(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
1474
1533
|
store: e.store,
|
|
1475
1534
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
1476
|
-
}), r = q(t.tables, e.snapshot), i = e.classification ??
|
|
1535
|
+
}), r = q(t.tables, e.snapshot), i = e.classification ?? ve(e.snapshot), a = e.sampleSize ?? 20, o = De(r.map((t) => ({
|
|
1477
1536
|
name: t,
|
|
1478
1537
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
1479
1538
|
name: e.name,
|
|
@@ -1486,14 +1545,14 @@ var gn = (e, t) => ({
|
|
|
1486
1545
|
chunkSize: a,
|
|
1487
1546
|
...r === void 0 ? {} : { where: r }
|
|
1488
1547
|
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) => P(t, i, o.actions, e.masking.seed));
|
|
1489
|
-
l.push(...
|
|
1548
|
+
l.push(...ke(d, i, o.actions));
|
|
1490
1549
|
let f = u[0], m = d[0], h = [];
|
|
1491
1550
|
if (f && m) for (let [e, t] of Object.entries(o.actions)) {
|
|
1492
1551
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
1493
1552
|
let n = e.slice(i.length + 1);
|
|
1494
1553
|
h.push({
|
|
1495
1554
|
column: n,
|
|
1496
|
-
action:
|
|
1555
|
+
action: kn(t),
|
|
1497
1556
|
before: f[n],
|
|
1498
1557
|
after: m[n]
|
|
1499
1558
|
});
|
|
@@ -1508,7 +1567,7 @@ var gn = (e, t) => ({
|
|
|
1508
1567
|
leaks: l,
|
|
1509
1568
|
unclassified: o.unclassified
|
|
1510
1569
|
};
|
|
1511
|
-
}),
|
|
1570
|
+
}), jn = (e, t) => {
|
|
1512
1571
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
1513
1572
|
for (let t of e) {
|
|
1514
1573
|
let e = n.get(t.name);
|
|
@@ -1563,7 +1622,7 @@ var gn = (e, t) => ({
|
|
|
1563
1622
|
skipColumns: i,
|
|
1564
1623
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
1565
1624
|
};
|
|
1566
|
-
},
|
|
1625
|
+
}, Mn = (e) => {
|
|
1567
1626
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
1568
1627
|
let t = {
|
|
1569
1628
|
safe: "✓",
|
|
@@ -1574,4 +1633,4 @@ var gn = (e, t) => ({
|
|
|
1574
1633
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
1575
1634
|
};
|
|
1576
1635
|
//#endregion
|
|
1577
|
-
export { H as ASSETS_DIR, U as ASSET_INDEX_FILE, F as BundleError,
|
|
1636
|
+
export { H as ASSETS_DIR, U as ASSET_INDEX_FILE, F as BundleError, Ae as CodecError, je as CompressionError, Le as CrossDialectError, it as DATA_DIR, _e as DEFAULT_CLASS_ACTIONS, Ne as DanglingReferenceError, Xe as FORMAT_VERSION, Ie as ImportModeError, I as IntegrityError, at as LEDGER_FILE, V as MANIFEST_FILE, Fe as MaskingError, L as NativeToolError, Pe as SchemaDriftError, Me as ScopeError, Ee as applyAction, qt as assessLiveImport, mt as assetDone, Cn as backupArtifactName, En as backupCommand, It as canonicalType, zt as checkPortability, Ut as checkSchemaDrift, ve as classificationFromSnapshot, jn as classifyImportDrift, qe as compressionExt, z as computeSubsetIds, nt as decodeManifest, wt as decodeRow, St as decodeValue, _n as decryptStream, Kt as defineDataProfile, Ht as diffSnapshots, sn as dirBlobSink, ft as emptyLedger, tt as encodeManifest, Ct as encodeRow, bt as encodeValue, gn as encryptStream, Mn as formatImportDrift, Bt as formatIssue, Ge as hasZstd, Je as makeCompressor, Ye as makeDecompressor, P as maskRow, Dt as ndjsonToRows, en as packBundle, an as peekBundle, Ke as pickCompression, De as planMasking, An as previewMasking, G as readLedger, ye as resolveAction, Be as resolveScope, We as restoreAssetFromCas, Dn as restoreCommand, Et as rowsToNdjson, Ft as runExport, Gt as runImport, On as runNativeStep, ke as scanForLeaks, He as storageAssetSink, Ve as storageAssetSource, bn as storageBlobSink, vn as storageBlobSource, yn as storageRefsBlobSource, ze as subsetToScope, pt as tableDone, q as topoSortTables, rn as unpackBundle, on as unpackBundleToDir, Ue as writeAssetToCas, K as writeLedger };
|