@voltro/data-transfer 0.41.0 → 0.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +188 -0
- package/dist/index.d.ts +106 -30
- package/dist/index.js +429 -340
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Schedule as o, Schema as s, Stream as c } from "effect";
|
|
2
|
-
import {
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { STORAGE_REFS_TABLE as
|
|
6
|
-
import * as
|
|
7
|
-
import { PassThrough as
|
|
8
|
-
import { copyFile as
|
|
9
|
-
import { mkdir as
|
|
10
|
-
import { basename as
|
|
11
|
-
import { promisify as
|
|
12
|
-
import { spawn as
|
|
2
|
+
import { classifyConstraintViolation as l, eq as u, extractDbCause as d, inSet as f, isNotNull as p, or as m, streamTable as h } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as g } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as _, createDecipheriv as v, createHash as y, createHmac as b, randomBytes as x, scrypt as S } from "node:crypto";
|
|
5
|
+
import { STORAGE_REFS_TABLE as C, fromNodeReadable as w, getObjectStream as ee, putObjectStream as te, toNodeReadable as ne } from "@voltro/plugin-storage";
|
|
6
|
+
import * as T from "node:zlib";
|
|
7
|
+
import { PassThrough as re, Readable as ie } from "node:stream";
|
|
8
|
+
import { copyFile as ae, createReadStream as E, createWriteStream as D, existsSync as oe, unlinkSync as se } from "node:fs";
|
|
9
|
+
import { mkdir as O, readdir as ce, stat as le } from "node:fs/promises";
|
|
10
|
+
import { basename as ue, delimiter as de, dirname as fe, join as k, resolve as pe, sep as me } from "node:path";
|
|
11
|
+
import { promisify as he } from "node:util";
|
|
12
|
+
import { spawn as ge } from "node:child_process";
|
|
13
13
|
//#region src/consistency.ts
|
|
14
|
-
var
|
|
14
|
+
var _e = (e) => {
|
|
15
15
|
switch (e) {
|
|
16
16
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
17
17
|
case "mysql":
|
|
@@ -21,7 +21,7 @@ var ge = (e) => {
|
|
|
21
21
|
case "turso": return [];
|
|
22
22
|
default: return [];
|
|
23
23
|
}
|
|
24
|
-
},
|
|
24
|
+
}, ve = {
|
|
25
25
|
email: { fake: "email" },
|
|
26
26
|
fullName: { fake: "fullName" },
|
|
27
27
|
firstName: { fake: "firstName" },
|
|
@@ -36,7 +36,7 @@ var ge = (e) => {
|
|
|
36
36
|
date: "dateShift",
|
|
37
37
|
secret: "null",
|
|
38
38
|
freeText: "redact"
|
|
39
|
-
},
|
|
39
|
+
}, A = (e) => {
|
|
40
40
|
let t = {};
|
|
41
41
|
for (let n of e.tables) {
|
|
42
42
|
let e = {};
|
|
@@ -53,10 +53,10 @@ var ge = (e) => {
|
|
|
53
53
|
if (i) return i;
|
|
54
54
|
if (n?.sensitive) {
|
|
55
55
|
let e = n.sensitive.class;
|
|
56
|
-
return r.classes?.[e] ??
|
|
56
|
+
return r.classes?.[e] ?? ve[e] ?? "redact";
|
|
57
57
|
}
|
|
58
58
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
59
|
-
}, be = (e, t) =>
|
|
59
|
+
}, be = (e, t) => b("sha256", e).update(t == null ? "∅" : String(t)).digest(), j = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), M = [
|
|
60
60
|
"Ada",
|
|
61
61
|
"Alan",
|
|
62
62
|
"Grace",
|
|
@@ -100,19 +100,19 @@ var ge = (e) => {
|
|
|
100
100
|
"Soylent",
|
|
101
101
|
"Stark",
|
|
102
102
|
"Wayne"
|
|
103
|
-
],
|
|
104
|
-
let r = be(n, `${e}:${String(t)}`), i =
|
|
103
|
+
], N = (e, t, n) => e[j(t, n) % e.length], we = (e, t, n) => {
|
|
104
|
+
let r = be(n, `${e}:${String(t)}`), i = j(r, 1) % 1e4;
|
|
105
105
|
switch (e) {
|
|
106
|
-
case "email": return `${
|
|
107
|
-
case "fullName": return `${
|
|
108
|
-
case "firstName": return
|
|
109
|
-
case "lastName": return
|
|
110
|
-
case "username": return `${
|
|
111
|
-
case "phone": return `+1${String(
|
|
112
|
-
case "address": return `${
|
|
113
|
-
case "company": return
|
|
106
|
+
case "email": return `${N(M, r, 0).toLowerCase()}.${N(xe, r, 2).toLowerCase()}${i}@example.com`;
|
|
107
|
+
case "fullName": return `${N(M, r, 0)} ${N(xe, r, 1)}`;
|
|
108
|
+
case "firstName": return N(M, r, 0);
|
|
109
|
+
case "lastName": return N(xe, r, 1);
|
|
110
|
+
case "username": return `${N(M, r, 0).toLowerCase()}${i}`;
|
|
111
|
+
case "phone": return `+1${String(j(r, 0) % 1e10).padStart(10, "0")}`;
|
|
112
|
+
case "address": return `${j(r, 0) % 9899 + 100} ${N(Se, r, 1)} St`;
|
|
113
|
+
case "company": return N(Ce, r, 0);
|
|
114
114
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
115
|
-
case "ip": return `${
|
|
115
|
+
case "ip": return `${j(r, 0) % 256}.${j(r, 1) % 256}.${j(r, 2) % 256}.${j(r, 3) % 254 + 1}`;
|
|
116
116
|
case "creditCard": return Te(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
117
117
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
118
118
|
}
|
|
@@ -127,23 +127,42 @@ var ge = (e) => {
|
|
|
127
127
|
}, Ee = 864e5, De = (e, t) => {
|
|
128
128
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
129
129
|
if (Number.isNaN(n)) return e;
|
|
130
|
-
let r =
|
|
130
|
+
let r = j(be(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Ee);
|
|
131
131
|
return e instanceof Date ? i : i.toISOString();
|
|
132
132
|
}, Oe = (e, t) => {
|
|
133
133
|
if (t.value === null || t.value === void 0) return null;
|
|
134
134
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
135
|
+
if (e === "keep") return t.value;
|
|
136
|
+
if (e === "null") return null;
|
|
137
|
+
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
138
|
+
if (e === "hash") return be(t.seed, t.value).toString("hex").slice(0, 16);
|
|
139
|
+
if (e === "dateShift") return De(t.value, t.seed);
|
|
140
|
+
if (typeof e == "object" && "fake" in e) return we(e.fake, t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
142
|
+
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
143
|
+
}, ke = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Ae = (e, t, n) => {
|
|
144
|
+
let r = {}, i = [], a = [];
|
|
145
|
+
for (let o of e) for (let e of o.columns) {
|
|
146
|
+
let s = `${o.name}.${e.name}`, c = ye(o.name, e.name, t[o.name]?.[e.name], n);
|
|
147
|
+
if (c === "error") {
|
|
148
|
+
i.push(s);
|
|
149
|
+
continue;
|
|
150
|
+
}
|
|
151
|
+
if (!ke(c)) {
|
|
152
|
+
a.push({
|
|
153
|
+
column: s,
|
|
154
|
+
action: c
|
|
155
|
+
});
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
r[s] = c;
|
|
141
159
|
}
|
|
142
160
|
return {
|
|
143
161
|
actions: r,
|
|
144
|
-
unclassified: i
|
|
162
|
+
unclassified: i,
|
|
163
|
+
invalid: a
|
|
145
164
|
};
|
|
146
|
-
},
|
|
165
|
+
}, je = [
|
|
147
166
|
{
|
|
148
167
|
name: "email",
|
|
149
168
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -160,7 +179,7 @@ var ge = (e) => {
|
|
|
160
179
|
name: "creditCard",
|
|
161
180
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
162
181
|
}
|
|
163
|
-
],
|
|
182
|
+
], Me = (e, t, n) => {
|
|
164
183
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
165
184
|
for (let [a, o] of Object.entries(n)) {
|
|
166
185
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -168,7 +187,7 @@ var ge = (e) => {
|
|
|
168
187
|
for (let a of e) {
|
|
169
188
|
let e = a[n];
|
|
170
189
|
if (typeof e != "string" || e.length === 0) continue;
|
|
171
|
-
let o =
|
|
190
|
+
let o = je.find((t) => t.re.test(e));
|
|
172
191
|
if (o && !i.has(n)) {
|
|
173
192
|
i.add(n), r.push({
|
|
174
193
|
table: t,
|
|
@@ -181,7 +200,7 @@ var ge = (e) => {
|
|
|
181
200
|
}
|
|
182
201
|
}
|
|
183
202
|
return r;
|
|
184
|
-
},
|
|
203
|
+
}, Ne = (e, t, n, r, i) => {
|
|
185
204
|
let a = {};
|
|
186
205
|
for (let [o, s] of Object.entries(e)) a[o] = Oe(n[`${t}.${o}`] ?? "keep", {
|
|
187
206
|
value: s,
|
|
@@ -194,21 +213,23 @@ var ge = (e) => {
|
|
|
194
213
|
}, P = class extends s.TaggedError()("BundleError", {
|
|
195
214
|
reason: s.String,
|
|
196
215
|
path: s.optional(s.String)
|
|
197
|
-
}) {},
|
|
216
|
+
}) {}, Pe = class extends s.TaggedError()("CodecError", {
|
|
198
217
|
table: s.String,
|
|
199
218
|
reason: s.String
|
|
200
219
|
}) {}, F = class extends s.TaggedError()("IntegrityError", {
|
|
201
220
|
what: s.String,
|
|
202
221
|
expected: s.String,
|
|
203
222
|
actual: s.String
|
|
204
|
-
}) {},
|
|
223
|
+
}) {}, Fe = class extends t.TaggedError("CompressionError") {}, I = class extends s.TaggedError()("ScopeError", { reason: s.String }) {}, Ie = class extends s.TaggedError()("RowsRefusedError", {
|
|
205
224
|
rows: s.Array(s.Struct({
|
|
206
225
|
table: s.String,
|
|
207
226
|
id: s.String,
|
|
208
|
-
reason: s.String
|
|
227
|
+
reason: s.String,
|
|
228
|
+
derived: s.Boolean
|
|
209
229
|
})),
|
|
230
|
+
primaryCount: s.Number,
|
|
210
231
|
totalCount: s.Number
|
|
211
|
-
}) {},
|
|
232
|
+
}) {}, Le = class extends s.TaggedError()("SchemaDriftError", {
|
|
212
233
|
bundleFingerprint: s.String,
|
|
213
234
|
targetFingerprint: s.String,
|
|
214
235
|
diff: s.Array(s.String)
|
|
@@ -217,36 +238,39 @@ var ge = (e) => {
|
|
|
217
238
|
let e = this.stderr?.trim();
|
|
218
239
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
219
240
|
}
|
|
220
|
-
},
|
|
241
|
+
}, Re = class extends s.TaggedError()("MaskingError", {
|
|
242
|
+
unclassified: s.Array(s.String),
|
|
243
|
+
invalidActions: s.optional(s.Array(s.String))
|
|
244
|
+
}) {}, ze = class extends s.TaggedError()("ImportModeError", {
|
|
221
245
|
mode: s.String,
|
|
222
246
|
reason: s.String
|
|
223
|
-
}) {},
|
|
247
|
+
}) {}, Be = class extends s.TaggedError()("CrossDialectError", {
|
|
224
248
|
sourceDialect: s.String,
|
|
225
249
|
targetDialect: s.String,
|
|
226
250
|
blocking: s.Array(s.String)
|
|
227
|
-
}) {}, R = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id",
|
|
251
|
+
}) {}, R = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Ve = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
228
252
|
col: e.name,
|
|
229
253
|
refTable: e.references.table
|
|
230
254
|
})), z = (e, t, r) => n.gen(function* () {
|
|
231
|
-
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map(),
|
|
255
|
+
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map(), u = (e) => {
|
|
232
256
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
233
257
|
return s.set(e, t), t;
|
|
234
|
-
},
|
|
258
|
+
}, d = (e, t) => {
|
|
235
259
|
if (t == null || a.has(e) || s.get(e)?.has(t)) return;
|
|
236
260
|
let n = l.get(e) ?? /* @__PURE__ */ new Set();
|
|
237
261
|
n.add(t), l.set(e, n);
|
|
238
|
-
},
|
|
239
|
-
let r =
|
|
262
|
+
}, p = (e, t, n) => {
|
|
263
|
+
let r = u(e), i = a.has(e) ? void 0 : o.get(e) ?? (() => {
|
|
240
264
|
let t = /* @__PURE__ */ new Set();
|
|
241
265
|
return o.set(e, t), t;
|
|
242
266
|
})();
|
|
243
267
|
return (e) => {
|
|
244
268
|
r.add(e[t]), i?.add(e[t]);
|
|
245
|
-
for (let t of n)
|
|
269
|
+
for (let t of n) d(t.refTable, e[t.col]);
|
|
246
270
|
};
|
|
247
271
|
}, g = (r, i) => {
|
|
248
|
-
let a =
|
|
249
|
-
return c.runForEach(
|
|
272
|
+
let a = p(r, R(t, r), Ve(t, r));
|
|
273
|
+
return c.runForEach(h(e, {
|
|
250
274
|
table: r,
|
|
251
275
|
pk: R(t, r),
|
|
252
276
|
...i === void 0 ? {} : { where: i }
|
|
@@ -257,7 +281,7 @@ var ge = (e) => {
|
|
|
257
281
|
l.clear();
|
|
258
282
|
for (let [n, r] of e) {
|
|
259
283
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
260
|
-
e.length !== 0 && (yield* g(n,
|
|
284
|
+
e.length !== 0 && (yield* g(n, f(R(t, n), e)));
|
|
261
285
|
}
|
|
262
286
|
}
|
|
263
287
|
});
|
|
@@ -272,19 +296,19 @@ var ge = (e) => {
|
|
|
272
296
|
refTable: e.references?.table
|
|
273
297
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
274
298
|
if (a.length === 0) continue;
|
|
275
|
-
let o = s.get(t)?.size ?? 0, c = a.map((e) =>
|
|
276
|
-
yield* g(t, c.length === 1 ? c[0] :
|
|
299
|
+
let o = s.get(t)?.size ?? 0, c = a.map((e) => f(e.col, [...s.get(e.refTable)]));
|
|
300
|
+
yield* g(t, c.length === 1 ? c[0] : m(...c)), (s.get(t)?.size ?? 0) > o && !e.has(t) && r.push(t), (s.get(t)?.size ?? 0) > o && (n = !0);
|
|
277
301
|
}
|
|
278
302
|
for (let t of r) e.add(t);
|
|
279
303
|
if (yield* _, !n) break;
|
|
280
304
|
}
|
|
281
305
|
}
|
|
282
306
|
return o;
|
|
283
|
-
}),
|
|
307
|
+
}), He = (e, t) => ({
|
|
284
308
|
tables: [...e.keys()],
|
|
285
309
|
predicateFor: (n) => {
|
|
286
310
|
let r = e.get(n);
|
|
287
|
-
return r ?
|
|
311
|
+
return r ? f(R(t, n), [...r]) : void 0;
|
|
288
312
|
}
|
|
289
313
|
}), B = "tenantId", V = (e, t, r) => n.gen(function* () {
|
|
290
314
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
@@ -306,26 +330,26 @@ var ge = (e) => {
|
|
|
306
330
|
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === B) ?? !1);
|
|
307
331
|
if (s.length === 0) return yield* n.fail(new I({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${B}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
308
332
|
let c = yield* z(r.store, t, {
|
|
309
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
333
|
+
seeds: Object.fromEntries(s.map((e) => [e, u(B, o)])),
|
|
310
334
|
pinned: s,
|
|
311
335
|
children: { roots: s }
|
|
312
|
-
}),
|
|
336
|
+
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), d = new Set(s);
|
|
313
337
|
return {
|
|
314
|
-
tables: [...s, ...
|
|
315
|
-
predicateFor: (e) =>
|
|
338
|
+
tables: [...s, ...l],
|
|
339
|
+
predicateFor: (e) => d.has(e) ? u(B, o) : c.has(e) ? f(R(t, e), [...c.get(e)]) : void 0
|
|
316
340
|
};
|
|
317
|
-
}),
|
|
341
|
+
}), Ue = (e, t) => ({
|
|
318
342
|
list: t,
|
|
319
|
-
open: (t) =>
|
|
320
|
-
}),
|
|
321
|
-
write: (t, n, r) =>
|
|
343
|
+
open: (t) => ee(e, t)
|
|
344
|
+
}), We = (e) => ({
|
|
345
|
+
write: (t, n, r) => te(e, t, n, r),
|
|
322
346
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
323
|
-
}),
|
|
347
|
+
}), Ge = (e, t) => n.tryPromise({
|
|
324
348
|
try: async () => {
|
|
325
349
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
326
350
|
u.update(e), d += e.length, n(null, e);
|
|
327
351
|
} });
|
|
328
|
-
await o(await
|
|
352
|
+
await o(await ne(e.stream), f, n(l));
|
|
329
353
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
330
354
|
try {
|
|
331
355
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -342,8 +366,8 @@ var ge = (e) => {
|
|
|
342
366
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
343
367
|
path: t
|
|
344
368
|
})
|
|
345
|
-
}),
|
|
346
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), l = o("sha256"), u =
|
|
369
|
+
}), Ke = (e, t, r) => n.gen(function* () {
|
|
370
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), l = o("sha256"), u = w("bundle", () => a(s)).pipe(c.tap((e) => n.sync(() => l.update(e))));
|
|
347
371
|
yield* e.write(r.key, u, {
|
|
348
372
|
contentType: r.contentType,
|
|
349
373
|
totalSize: r.size
|
|
@@ -354,7 +378,7 @@ var ge = (e) => {
|
|
|
354
378
|
expected: r.sha256,
|
|
355
379
|
actual: d
|
|
356
380
|
}));
|
|
357
|
-
}),
|
|
381
|
+
}), qe = () => typeof T.createZstdCompress == "function", Je = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : qe() ? "zstd" : "gzip", Ye = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", Xe = (e) => e === "zstd" ? T.createZstdCompress() : e === "gzip" ? T.createGzip() : new re(), Ze = (e) => e === "zstd" ? T.createZstdDecompress() : e === "gzip" ? T.createGunzip() : new re(), Qe = 1, $e = s.Struct({
|
|
358
382
|
name: s.String,
|
|
359
383
|
file: s.String,
|
|
360
384
|
rowCount: s.Number,
|
|
@@ -364,11 +388,11 @@ var ge = (e) => {
|
|
|
364
388
|
value: s.String
|
|
365
389
|
}),
|
|
366
390
|
primaryKey: s.String
|
|
367
|
-
}),
|
|
391
|
+
}), et = s.Struct({
|
|
368
392
|
kind: s.Literal("all", "tenant", "tables"),
|
|
369
393
|
tenantId: s.optional(s.String),
|
|
370
394
|
tables: s.optional(s.Array(s.String))
|
|
371
|
-
}),
|
|
395
|
+
}), tt = s.Struct({
|
|
372
396
|
count: s.Number,
|
|
373
397
|
totalBytes: s.Number,
|
|
374
398
|
index: s.optional(s.String)
|
|
@@ -379,23 +403,23 @@ s.Struct({
|
|
|
379
403
|
size: s.Number,
|
|
380
404
|
contentType: s.String
|
|
381
405
|
});
|
|
382
|
-
var
|
|
406
|
+
var nt = s.Struct({
|
|
383
407
|
formatVersion: s.Number,
|
|
384
408
|
createdAt: s.String,
|
|
385
409
|
source: s.Struct({
|
|
386
410
|
dialect: s.String,
|
|
387
411
|
schemaFingerprint: s.String
|
|
388
412
|
}),
|
|
389
|
-
scope:
|
|
413
|
+
scope: et,
|
|
390
414
|
consistency: s.Literal("live", "snapshot"),
|
|
391
415
|
compression: s.Literal("zstd", "gzip", "none"),
|
|
392
|
-
tables: s.Array(
|
|
393
|
-
assets:
|
|
416
|
+
tables: s.Array($e),
|
|
417
|
+
assets: tt,
|
|
394
418
|
masking: s.optional(s.Struct({
|
|
395
419
|
policyId: s.optional(s.String),
|
|
396
420
|
transformed: s.Array(s.String)
|
|
397
421
|
}))
|
|
398
|
-
}),
|
|
422
|
+
}), rt = s.encodeSync(s.parseJson(nt)), it = s.decodeUnknownSync(s.parseJson(nt)), H = "manifest.json", at = "README.md", U = "data", W = "assets", G = "assets/index.ndjson", ot = ".ledger.json", st = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", ct = (e, t) => {
|
|
399
423
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
400
424
|
return `# Voltro data export
|
|
401
425
|
|
|
@@ -409,7 +433,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
409
433
|
| **Format version** | ${e.formatVersion} |
|
|
410
434
|
| **Consistency** | ${e.consistency} |
|
|
411
435
|
| **Compression** | ${e.compression} |
|
|
412
|
-
| **Scope** | ${
|
|
436
|
+
| **Scope** | ${st(e.scope)} |
|
|
413
437
|
| **Tables** | ${e.tables.length} |
|
|
414
438
|
| **Rows (total)** | ${n} |
|
|
415
439
|
| **Assets** | ${a} |
|
|
@@ -434,26 +458,26 @@ voltro data import <this-directory>
|
|
|
434
458
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
435
459
|
\`\`\`
|
|
436
460
|
`;
|
|
437
|
-
},
|
|
461
|
+
}, lt = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), ut = a.exponential({
|
|
438
462
|
start: .01,
|
|
439
463
|
factor: 2,
|
|
440
464
|
count: 16
|
|
441
|
-
}),
|
|
465
|
+
}), dt = i.histogram("voltro_data_transfer_table_seconds", ut, "Per-table export/import duration in seconds, tagged by phase."), K = (e, t) => {
|
|
442
466
|
if (e) try {
|
|
443
467
|
e(t);
|
|
444
468
|
} catch {}
|
|
445
|
-
},
|
|
469
|
+
}, ft = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(dt.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), lt.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
446
470
|
table: t,
|
|
447
471
|
index: r,
|
|
448
472
|
tableCount: a,
|
|
449
473
|
phase: e
|
|
450
|
-
} })),
|
|
474
|
+
} })), pt = () => ({
|
|
451
475
|
tables: {},
|
|
452
476
|
assets: {}
|
|
453
|
-
}),
|
|
477
|
+
}), mt = (e) => n.tryPromise(async () => {
|
|
454
478
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
455
479
|
return JSON.parse(t);
|
|
456
|
-
}).pipe(n.orElseSucceed(
|
|
480
|
+
}).pipe(n.orElseSucceed(pt)), q = (e, t) => n.tryPromise({
|
|
457
481
|
try: async () => {
|
|
458
482
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
459
483
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
@@ -462,11 +486,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
462
486
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
463
487
|
path: e
|
|
464
488
|
})
|
|
465
|
-
}),
|
|
489
|
+
}), ht = (e, t) => t in e.tables, gt = (e, t) => t in e.assets, _t = /* @__PURE__ */ new Set([
|
|
466
490
|
"json",
|
|
467
491
|
"array",
|
|
468
492
|
"vector"
|
|
469
|
-
]),
|
|
493
|
+
]), vt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", yt = (e) => Buffer.from(e).toString("base64"), bt = (e) => new Uint8Array(Buffer.from(e, "base64")), xt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), St = (e, t) => e == null ? null : t !== void 0 && _t.has(t) ? e : t === "bytes" ? yt(e) : t === "timestamp" || t === "date" ? xt(e) : typeof e == "bigint" ? {
|
|
470
494
|
$v: "bigint",
|
|
471
495
|
d: e.toString()
|
|
472
496
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -477,30 +501,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
477
501
|
d: e.toISOString()
|
|
478
502
|
} : e instanceof Uint8Array ? {
|
|
479
503
|
$v: "bytes",
|
|
480
|
-
d:
|
|
481
|
-
} : e,
|
|
504
|
+
d: yt(e)
|
|
505
|
+
} : e, Ct = (e) => {
|
|
482
506
|
switch (e.$v) {
|
|
483
507
|
case "bigint": return BigInt(e.d);
|
|
484
508
|
case "number": return Number(e.d);
|
|
485
509
|
case "date": return new Date(e.d);
|
|
486
|
-
case "bytes": return
|
|
510
|
+
case "bytes": return bt(e.d);
|
|
487
511
|
default: return e;
|
|
488
512
|
}
|
|
489
|
-
},
|
|
513
|
+
}, wt = (e, t) => e == null ? null : t !== void 0 && _t.has(t) ? e : vt(e) ? Ct(e) : t === "bytes" ? bt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Tt = (e, t) => {
|
|
490
514
|
let n = {};
|
|
491
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
515
|
+
for (let [r, i] of Object.entries(e)) n[r] = St(i, t[r]);
|
|
492
516
|
return n;
|
|
493
|
-
},
|
|
517
|
+
}, Et = (e, t) => {
|
|
494
518
|
let n = {};
|
|
495
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
519
|
+
for (let [r, i] of Object.entries(e)) n[r] = wt(i, t[r]);
|
|
496
520
|
return n;
|
|
497
|
-
},
|
|
498
|
-
try: () =>
|
|
499
|
-
catch: (e) => new
|
|
521
|
+
}, Dt = new TextEncoder(), Ot = (e, t) => e.pipe(c.map((e) => Dt.encode(`${JSON.stringify(Tt(e, t))}\n`))), kt = (e, t, r) => e.pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.mapEffect((e) => n.try({
|
|
522
|
+
try: () => Et(JSON.parse(e), r),
|
|
523
|
+
catch: (e) => new Pe({
|
|
500
524
|
table: t,
|
|
501
525
|
reason: String(e?.message ?? e)
|
|
502
526
|
})
|
|
503
|
-
}))),
|
|
527
|
+
}))), At = (e, t) => {
|
|
504
528
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
505
529
|
for (let e of t.tables) {
|
|
506
530
|
if (!n.has(e.name)) continue;
|
|
@@ -526,28 +550,33 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
526
550
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
527
551
|
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(J(t.cause)) : n.push(r), n.length === 0) {
|
|
528
552
|
let t = String(e ?? "").trim();
|
|
529
|
-
return t === "" || t === "[object Object]" ?
|
|
553
|
+
return t === "" || t === "[object Object]" ? jt(e) : t;
|
|
530
554
|
}
|
|
531
555
|
return n.join(": ");
|
|
532
|
-
},
|
|
556
|
+
}, jt = (e) => {
|
|
533
557
|
try {
|
|
534
558
|
let t = JSON.stringify(e);
|
|
535
559
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
536
560
|
} catch {
|
|
537
561
|
return "unknown error (not serialisable)";
|
|
538
562
|
}
|
|
539
|
-
},
|
|
563
|
+
}, Mt = (e, t) => {
|
|
540
564
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
541
|
-
for (let e of n?.columns ?? []) r[e.name] = e.type;
|
|
565
|
+
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
542
566
|
return r;
|
|
543
|
-
},
|
|
567
|
+
}, Nt = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), Pt = (e, t) => {
|
|
568
|
+
if (t.length === 0) return e;
|
|
569
|
+
let n = { ...e };
|
|
570
|
+
for (let e of t) delete n[e];
|
|
571
|
+
return n;
|
|
572
|
+
}, Ft = (e, t) => {
|
|
544
573
|
let n = e.tables.find((e) => e.name === t);
|
|
545
574
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
546
575
|
let r = n.columns.find((e) => e.type === "id");
|
|
547
576
|
if (r !== void 0) return { column: r.name };
|
|
548
577
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
549
578
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
550
|
-
},
|
|
579
|
+
}, It = (e, t, r) => n.tryPromise({
|
|
551
580
|
try: async () => {
|
|
552
581
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: l, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
553
582
|
d.update(e);
|
|
@@ -556,7 +585,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
556
585
|
n(null, e);
|
|
557
586
|
} }), m = l.fromWeb(c.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
558
587
|
try {
|
|
559
|
-
await s(m, p,
|
|
588
|
+
await s(m, p, Xe(r), n(h)), await i.rename(h, t);
|
|
560
589
|
} catch (e) {
|
|
561
590
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
562
591
|
}
|
|
@@ -569,7 +598,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
569
598
|
reason: `write table failed: ${J(e)}`,
|
|
570
599
|
path: t
|
|
571
600
|
})
|
|
572
|
-
}),
|
|
601
|
+
}), Lt = (e, t) => n.tryPromise({
|
|
573
602
|
try: async () => {
|
|
574
603
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
575
604
|
},
|
|
@@ -577,13 +606,13 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
577
606
|
reason: `append failed: ${J(t)}`,
|
|
578
607
|
path: e
|
|
579
608
|
})
|
|
580
|
-
}),
|
|
609
|
+
}), Rt = (e, t) => n.gen(function* () {
|
|
581
610
|
let r = [], i = t.ordered.length;
|
|
582
611
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
583
|
-
let o = t.ordered[a], s =
|
|
612
|
+
let o = t.ordered[a], s = Mt(t.snapshot, o), l = Ft(t.snapshot, o);
|
|
584
613
|
if ("refusal" in l) return yield* n.fail(new I({ reason: `export: ${l.refusal}` }));
|
|
585
|
-
let u = l.column, d = `${U}/${o}.ndjson${
|
|
586
|
-
if (
|
|
614
|
+
let u = l.column, d = `${U}/${o}.ndjson${Ye(t.compression)}`;
|
|
615
|
+
if (ht(t.ledger, o)) {
|
|
587
616
|
let e = t.ledger.tables[o];
|
|
588
617
|
K(t.onProgress, {
|
|
589
618
|
phase: "export",
|
|
@@ -611,48 +640,48 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
611
640
|
event: "start",
|
|
612
641
|
rowsDone: 0
|
|
613
642
|
});
|
|
614
|
-
let f = t.resolved.predicateFor(o),
|
|
643
|
+
let f = t.resolved.predicateFor(o), p = h(e, {
|
|
615
644
|
table: o,
|
|
616
645
|
pk: u,
|
|
617
646
|
chunkSize: t.chunkSize,
|
|
618
647
|
retry: t.retry,
|
|
619
648
|
...f === void 0 ? {} : { where: f }
|
|
620
|
-
}),
|
|
649
|
+
}), m = Nt(t.snapshot, o), g = m.length === 0 ? p : p.pipe(c.map((e) => Pt(e, m))), _ = t.mask ? g.pipe(c.map((e) => Ne(e, o, t.mask.actions, t.mask.seed, s))) : g, { rowCount: v, checksum: y } = yield* ft("export", o, a, i, (e) => e.rowCount, It(Ot(_, s), t.bundleFile(d), t.compression));
|
|
621
650
|
t.ledger.tables[o] = {
|
|
622
|
-
rowCount:
|
|
623
|
-
checksum:
|
|
651
|
+
rowCount: v,
|
|
652
|
+
checksum: y
|
|
624
653
|
}, yield* q(t.ledgerPath, t.ledger), K(t.onProgress, {
|
|
625
654
|
phase: "export",
|
|
626
655
|
table: o,
|
|
627
656
|
index: a,
|
|
628
657
|
tableCount: i,
|
|
629
658
|
event: "done",
|
|
630
|
-
rowsDone:
|
|
631
|
-
total:
|
|
659
|
+
rowsDone: v,
|
|
660
|
+
total: v
|
|
632
661
|
}), r.push({
|
|
633
662
|
name: o,
|
|
634
663
|
file: d,
|
|
635
|
-
rowCount:
|
|
636
|
-
checksum:
|
|
664
|
+
rowCount: v,
|
|
665
|
+
checksum: y,
|
|
637
666
|
columnTypes: s,
|
|
638
667
|
primaryKey: u
|
|
639
668
|
});
|
|
640
669
|
}
|
|
641
670
|
return r;
|
|
642
|
-
}),
|
|
671
|
+
}), zt = (e) => ({
|
|
643
672
|
_tag: "RawSqlFragment",
|
|
644
673
|
strings: [e],
|
|
645
674
|
values: []
|
|
646
|
-
}),
|
|
675
|
+
}), Bt = (t, i, a) => n.tryPromise({
|
|
647
676
|
try: () => t.transactional(async (t) => {
|
|
648
|
-
if (t.raw) for (let e of
|
|
677
|
+
if (t.raw) for (let e of _e(i)) await t.raw(zt(e));
|
|
649
678
|
let o = await n.runPromiseExit(a(t));
|
|
650
679
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
651
680
|
return o.value;
|
|
652
681
|
}),
|
|
653
682
|
catch: (e) => e instanceof P ? e : new P({ reason: `snapshot export failed: ${J(e)}` })
|
|
654
|
-
}),
|
|
655
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
683
|
+
}), Vt = (e) => n.gen(function* () {
|
|
684
|
+
let t = e.scope ?? { kind: "all" }, r = Je(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, s = o.exponential("50 millis").pipe(o.jittered, o.intersect(o.recurs(a))), l = yield* n.promise(() => import("node:path")), u = (t) => l.join(e.outDir, t);
|
|
656
685
|
yield* n.tryPromise({
|
|
657
686
|
try: async () => {
|
|
658
687
|
let t = await import("node:fs/promises");
|
|
@@ -663,19 +692,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
663
692
|
path: e.outDir
|
|
664
693
|
})
|
|
665
694
|
});
|
|
666
|
-
let d = u(
|
|
695
|
+
let d = u(ot), f = yield* mt(d), p = e.subset ? He(yield* z(e.store, e.snapshot, e.subset), e.snapshot) : yield* V(t, e.snapshot, {
|
|
667
696
|
store: e.store,
|
|
668
697
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
669
|
-
}),
|
|
698
|
+
}), m = At(p.tables, e.snapshot), h = e.consistency ?? "live", _, v;
|
|
670
699
|
if (e.masking) {
|
|
671
|
-
let t = e.classification ??
|
|
700
|
+
let t = e.classification ?? A(e.snapshot), r = Ae(m.map((t) => ({
|
|
672
701
|
name: t,
|
|
673
702
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
674
703
|
name: e.name,
|
|
675
704
|
type: e.type
|
|
676
705
|
}))
|
|
677
706
|
})), t, e.masking);
|
|
678
|
-
if (r.unclassified.length > 0) return yield* n.fail(new
|
|
707
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new Re({
|
|
708
|
+
unclassified: r.unclassified,
|
|
709
|
+
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
710
|
+
}));
|
|
679
711
|
v = {
|
|
680
712
|
actions: r.actions,
|
|
681
713
|
seed: e.masking.seed
|
|
@@ -688,7 +720,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
688
720
|
}
|
|
689
721
|
let y = {
|
|
690
722
|
snapshot: e.snapshot,
|
|
691
|
-
ordered:
|
|
723
|
+
ordered: m,
|
|
692
724
|
resolved: p,
|
|
693
725
|
compression: r,
|
|
694
726
|
chunkSize: i,
|
|
@@ -698,17 +730,17 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
698
730
|
bundleFile: u,
|
|
699
731
|
...v ? { mask: v } : {},
|
|
700
732
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
701
|
-
}, b =
|
|
733
|
+
}, b = h === "snapshot" ? yield* Bt(e.store, e.dialect, (e) => Rt(e, y)) : yield* Rt(e.store, y), x = 0, S = 0;
|
|
702
734
|
if (e.assets) {
|
|
703
735
|
let t = u(G);
|
|
704
|
-
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
736
|
+
yield* c.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => Ge(e, u(W))), n.flatMap((e) => {
|
|
705
737
|
let i = {
|
|
706
738
|
key: r.key,
|
|
707
739
|
sha256: e.sha256,
|
|
708
740
|
size: e.size,
|
|
709
741
|
contentType: e.contentType
|
|
710
742
|
};
|
|
711
|
-
return x++, S += e.size, f.assets[r.key] = !0,
|
|
743
|
+
return x++, S += e.size, f.assets[r.key] = !0, Lt(t, JSON.stringify(i)).pipe(n.andThen(q(d, f)));
|
|
712
744
|
}))).pipe(n.mapError((e) => e instanceof P ? e : new P({ reason: `asset export failed: ${String(e)}` })));
|
|
713
745
|
}
|
|
714
746
|
let C = {
|
|
@@ -716,10 +748,10 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
716
748
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
717
749
|
source: {
|
|
718
750
|
dialect: e.dialect,
|
|
719
|
-
schemaFingerprint:
|
|
751
|
+
schemaFingerprint: g(e.snapshot)
|
|
720
752
|
},
|
|
721
753
|
scope: t,
|
|
722
|
-
consistency:
|
|
754
|
+
consistency: h,
|
|
723
755
|
compression: r,
|
|
724
756
|
tables: b,
|
|
725
757
|
assets: x > 0 ? {
|
|
@@ -734,7 +766,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
734
766
|
};
|
|
735
767
|
return yield* n.tryPromise({
|
|
736
768
|
try: async () => {
|
|
737
|
-
await (await import("node:fs/promises")).writeFile(u(H),
|
|
769
|
+
await (await import("node:fs/promises")).writeFile(u(H), rt(C));
|
|
738
770
|
},
|
|
739
771
|
catch: (e) => new P({
|
|
740
772
|
reason: `manifest write failed: ${J(e)}`,
|
|
@@ -743,14 +775,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
743
775
|
}), yield* n.tryPromise({
|
|
744
776
|
try: async () => {
|
|
745
777
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
746
|
-
await e.writeFile(u(
|
|
778
|
+
await e.writeFile(u(at), ct(C, { hostname: t.hostname() }));
|
|
747
779
|
},
|
|
748
780
|
catch: (e) => new P({
|
|
749
781
|
reason: `readme write failed: ${J(e)}`,
|
|
750
|
-
path:
|
|
782
|
+
path: at
|
|
751
783
|
})
|
|
752
784
|
}), C;
|
|
753
|
-
}),
|
|
785
|
+
}), Ht = (e) => {
|
|
754
786
|
switch (e) {
|
|
755
787
|
case "id":
|
|
756
788
|
case "text":
|
|
@@ -769,12 +801,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
769
801
|
case "raw": return "raw";
|
|
770
802
|
default: return "string";
|
|
771
803
|
}
|
|
772
|
-
},
|
|
804
|
+
}, Ut = {
|
|
773
805
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
774
806
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
775
807
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
776
|
-
},
|
|
777
|
-
let i =
|
|
808
|
+
}, Wt = (e, t, n, r) => {
|
|
809
|
+
let i = Ht(n);
|
|
778
810
|
if (i === "raw") return {
|
|
779
811
|
table: e,
|
|
780
812
|
column: t,
|
|
@@ -782,7 +814,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
782
814
|
severity: "block",
|
|
783
815
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
784
816
|
};
|
|
785
|
-
let a =
|
|
817
|
+
let a = Ut[i];
|
|
786
818
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
787
819
|
table: e,
|
|
788
820
|
column: t,
|
|
@@ -796,55 +828,87 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
796
828
|
severity: "block",
|
|
797
829
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
798
830
|
};
|
|
799
|
-
},
|
|
831
|
+
}, Gt = (e, t, n) => {
|
|
800
832
|
if (t === n) return [];
|
|
801
833
|
let r = [];
|
|
802
834
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
803
|
-
let a =
|
|
835
|
+
let a = Wt(t.name, e, i, n);
|
|
804
836
|
a && r.push(a);
|
|
805
837
|
}
|
|
806
838
|
return r;
|
|
807
|
-
},
|
|
808
|
-
let
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
839
|
+
}, Kt = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, qt = (e) => {
|
|
840
|
+
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
841
|
+
switch (e.kind) {
|
|
842
|
+
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
843
|
+
case "foreignKeyInUse": return `foreign key${n}: the row is still referenced by other rows`;
|
|
844
|
+
case "unique": return `unique constraint${n}: a row with this value already exists`;
|
|
845
|
+
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
846
|
+
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
847
|
+
}
|
|
848
|
+
}, Jt = (e) => {
|
|
849
|
+
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
850
|
+
return t.length > 0 ? t.join("/") : void 0;
|
|
851
|
+
}, Yt = (e) => {
|
|
852
|
+
let t = d(e), n = l(t), r = Jt(t), i = r === void 0 ? "" : ` [${r}]`;
|
|
853
|
+
if (n !== void 0) return `${qt(n)}${i}`;
|
|
854
|
+
let a = typeof t.dbMessage == "string" ? t.dbMessage : void 0, o = e?.message, s = typeof o == "string" && o.length > 0 ? o : String(e), c = a !== void 0 && a !== s && !a.startsWith("SqlError:") ? a : s, u = c === "Failed to execute statement" ? `the database refused the write and the driver gave no detail${i}` : `${c}${i}`;
|
|
855
|
+
return u.length <= 300 ? u : `${u.slice(0, 300)}…`;
|
|
856
|
+
}, Xt = (e, t) => e.tables.find((e) => e.name === t), Zt = (e, t) => new Map((Xt(e, t)?.columns ?? []).map((e) => [e.name, e])), Qt = (e, t) => {
|
|
857
|
+
let n = [];
|
|
858
|
+
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
859
|
+
if (Xt(t, r.name) === void 0) {
|
|
860
|
+
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
861
|
+
continue;
|
|
862
|
+
}
|
|
863
|
+
let e = Zt(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
864
|
+
for (let [t, a] of i) {
|
|
865
|
+
let i = e.get(t);
|
|
866
|
+
if (i === void 0) {
|
|
867
|
+
n.push(`${r.name}.${t}: in the bundle, MISSING from the target — the value has nowhere to go`);
|
|
868
|
+
continue;
|
|
869
|
+
}
|
|
870
|
+
i.type !== a.type && n.push(`${r.name}.${t}: type '${a.type}' (bundle) vs '${i.type}' (target)`);
|
|
818
871
|
}
|
|
872
|
+
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
819
873
|
}
|
|
820
|
-
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n
|
|
821
|
-
},
|
|
822
|
-
let r =
|
|
823
|
-
|
|
874
|
+
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
875
|
+
}, $t = (e, t, n) => {
|
|
876
|
+
let r = g(t);
|
|
877
|
+
if (r === e || n === void 0) return {
|
|
824
878
|
drifted: !1,
|
|
825
879
|
bundleFingerprint: e,
|
|
826
880
|
targetFingerprint: r,
|
|
827
881
|
diff: []
|
|
828
|
-
}
|
|
829
|
-
|
|
882
|
+
};
|
|
883
|
+
let i = Qt(n, t);
|
|
884
|
+
return {
|
|
885
|
+
drifted: i.length > 0,
|
|
830
886
|
bundleFingerprint: e,
|
|
831
887
|
targetFingerprint: r,
|
|
832
|
-
diff:
|
|
888
|
+
diff: i
|
|
833
889
|
};
|
|
834
|
-
},
|
|
890
|
+
}, en = (e) => n.tryPromise({
|
|
835
891
|
try: async () => {
|
|
836
892
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
837
|
-
return
|
|
893
|
+
return it(await t.readFile(n.join(e, H), "utf8"));
|
|
838
894
|
},
|
|
839
895
|
catch: (e) => new P({
|
|
840
896
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
841
897
|
path: H
|
|
842
898
|
})
|
|
843
|
-
}),
|
|
844
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
899
|
+
}), tn = (t) => n.gen(function* () {
|
|
900
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* en(t.bundleDir);
|
|
845
901
|
if (a.formatVersion !== 1) return yield* n.fail(new P({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
902
|
+
if (t.tables !== void 0) {
|
|
903
|
+
let e = new Set(a.tables.map((e) => e.name)), r = t.tables.filter((t) => !e.has(t));
|
|
904
|
+
if (r.length > 0) return yield* n.fail(new P({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...e].sort().join(", ")}.` }));
|
|
905
|
+
}
|
|
906
|
+
let o = t.tables === void 0 ? void 0 : new Set(t.tables), s = o === void 0 ? a : {
|
|
907
|
+
...a,
|
|
908
|
+
tables: a.tables.filter((e) => o.has(e.name))
|
|
909
|
+
};
|
|
846
910
|
if (t.targetSnapshot) {
|
|
847
|
-
let e = { tables:
|
|
911
|
+
let e = { tables: s.tables.map((e) => ({
|
|
848
912
|
name: e.name,
|
|
849
913
|
indexes: [],
|
|
850
914
|
columns: Object.entries(e.columnTypes).map(([e, t]) => ({
|
|
@@ -854,9 +918,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
854
918
|
unique: !1,
|
|
855
919
|
hasDefault: !1
|
|
856
920
|
}))
|
|
857
|
-
})) }, r =
|
|
921
|
+
})) }, r = $t(s.source.schemaFingerprint, t.targetSnapshot, e);
|
|
858
922
|
if (r.drifted) {
|
|
859
|
-
if (!t.force) return yield* n.fail(new
|
|
923
|
+
if (!t.force) return yield* n.fail(new Le({
|
|
860
924
|
bundleFingerprint: r.bundleFingerprint,
|
|
861
925
|
targetFingerprint: r.targetFingerprint,
|
|
862
926
|
diff: r.diff
|
|
@@ -864,102 +928,123 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
864
928
|
t.onWarn?.(`schema drift: the target schema (fingerprint ${r.targetFingerprint.slice(0, 16)}) differs from the bundle's source (${r.bundleFingerprint.slice(0, 16)}); importing anyway (--force). ${r.diff.join("; ")}`);
|
|
865
929
|
}
|
|
866
930
|
}
|
|
867
|
-
if (t.targetDialect && t.targetDialect !==
|
|
868
|
-
let e =
|
|
869
|
-
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new
|
|
870
|
-
sourceDialect:
|
|
931
|
+
if (t.targetDialect && t.targetDialect !== s.source.dialect) {
|
|
932
|
+
let e = Gt(s.tables, s.source.dialect, t.targetDialect).filter((e) => e.severity === "block");
|
|
933
|
+
if (e.length > 0 && !t.allowIncompatible) return yield* n.fail(new Be({
|
|
934
|
+
sourceDialect: s.source.dialect,
|
|
871
935
|
targetDialect: t.targetDialect,
|
|
872
|
-
blocking: e.map(
|
|
936
|
+
blocking: e.map(Kt)
|
|
873
937
|
}));
|
|
874
938
|
}
|
|
875
|
-
let
|
|
876
|
-
if (
|
|
877
|
-
mode:
|
|
878
|
-
reason: `replace refuses a partial bundle (scope='${
|
|
939
|
+
let l = t.verify ?? !0, u = t.mode ?? "upsert", d = t.onConflict ?? "skip", f = t.ledgerPath ?? i.join(t.bundleDir, ".import..ledger.json"), m = yield* mt(f);
|
|
940
|
+
if (u === "replace" && s.scope.kind !== "all") return yield* n.fail(new ze({
|
|
941
|
+
mode: u,
|
|
942
|
+
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
879
943
|
}));
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
944
|
+
if (t.dryRun === !0) return s;
|
|
945
|
+
let h = /* @__PURE__ */ new Map();
|
|
946
|
+
for (let e of t.targetSnapshot?.tables ?? []) {
|
|
947
|
+
let t = e.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
948
|
+
t.length > 0 && h.set(e.name, t);
|
|
949
|
+
}
|
|
950
|
+
let g = (e, t) => {
|
|
951
|
+
let n = h.get(e);
|
|
952
|
+
if (n === void 0) return t;
|
|
953
|
+
let r;
|
|
954
|
+
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
955
|
+
return r ?? t;
|
|
956
|
+
}, _ = (e, t, n) => {
|
|
957
|
+
let r = t.name, i = t.primaryKey, a = g(r, n);
|
|
958
|
+
return u === "append" ? d === "fail" ? e.insert(r, a) : e.insertIgnore(r, a, { conflictColumns: [i] }) : u === "replace" ? e.insert(r, a) : e.upsert(r, a, { conflictColumns: [i] });
|
|
959
|
+
}, v = (e, t, r = !0) => n.tryPromise({
|
|
884
960
|
try: e,
|
|
885
|
-
catch: (e) =>
|
|
886
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (t.reason = e, !1)))),
|
|
961
|
+
catch: (e) => Yt(e)
|
|
962
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e), !1)))), y = (e) => {
|
|
887
963
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
888
964
|
if (t.length !== 0) return {
|
|
889
965
|
...e.row,
|
|
890
966
|
...Object.fromEntries(t.map((e) => [e, null]))
|
|
891
967
|
};
|
|
892
|
-
},
|
|
968
|
+
}, b = (e) => {
|
|
969
|
+
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
970
|
+
if (r !== "reference") return !1;
|
|
971
|
+
let i = e.row[n];
|
|
972
|
+
return i != null && t.has(String(i));
|
|
973
|
+
}), r = e.map((e) => ({
|
|
974
|
+
table: e.entry.name,
|
|
975
|
+
id: String(e.row[e.entry.primaryKey]),
|
|
976
|
+
reason: e.reason,
|
|
977
|
+
derived: n(e)
|
|
978
|
+
}));
|
|
979
|
+
return new Ie({
|
|
980
|
+
rows: [...r.filter((e) => !e.derived), ...r.filter((e) => e.derived)].slice(0, 20),
|
|
981
|
+
primaryCount: r.filter((e) => !e.derived).length,
|
|
982
|
+
totalCount: e.length
|
|
983
|
+
});
|
|
984
|
+
}, x = (e, t) => n.gen(function* () {
|
|
893
985
|
let r = t, i = [], a = !0;
|
|
894
986
|
for (; r.length > 0 && a;) {
|
|
895
987
|
a = !1;
|
|
896
988
|
let t = [];
|
|
897
|
-
for (let n of r) (yield*
|
|
989
|
+
for (let n of r) (yield* v(() => _(e, n.entry, n.row), n)) ? a = !0 : t.push(n);
|
|
898
990
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
899
|
-
let r = t[n], o =
|
|
900
|
-
o !== void 0 && (yield*
|
|
991
|
+
let r = t[n], o = y(r);
|
|
992
|
+
o !== void 0 && (yield* v(() => _(e, r.entry, o), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
901
993
|
}
|
|
902
994
|
r = t;
|
|
903
995
|
}
|
|
904
|
-
for (let t of i) (yield*
|
|
905
|
-
if (r.length > 0) return yield* n.fail(
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
})),
|
|
911
|
-
totalCount: r.length
|
|
912
|
-
}));
|
|
913
|
-
}), _ = (e, r) => n.gen(function* () {
|
|
914
|
-
if (s === "replace" && !(r && f.truncated)) {
|
|
915
|
-
for (let t of [...a.tables].reverse()) yield* n.tryPromise({
|
|
916
|
-
try: () => e.deleteMany(t.name, { where: d(t.primaryKey) }),
|
|
996
|
+
for (let t of i) (yield* v(() => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] }), t)) || (r = [...r, t]);
|
|
997
|
+
if (r.length > 0) return yield* n.fail(b(r));
|
|
998
|
+
}), S = (e, r) => n.gen(function* () {
|
|
999
|
+
if (u === "replace" && !(r && m.truncated)) {
|
|
1000
|
+
for (let t of [...s.tables].reverse()) yield* n.tryPromise({
|
|
1001
|
+
try: () => e.deleteMany(t.name, { where: p(t.primaryKey) }),
|
|
917
1002
|
catch: (e) => new P({ reason: `truncate ${t.name} failed: ${String(e?.message ?? e)}` })
|
|
918
1003
|
});
|
|
919
|
-
r && (
|
|
1004
|
+
r && (m.truncated = !0, yield* q(f, m));
|
|
920
1005
|
}
|
|
921
|
-
let
|
|
922
|
-
for (let
|
|
923
|
-
let g =
|
|
924
|
-
if (r && g.name in
|
|
925
|
-
K(t.onProgress, {
|
|
1006
|
+
let a = [], o = [], d = [], h = s.tables.length;
|
|
1007
|
+
for (let p = 0; p < s.tables.length; p++) {
|
|
1008
|
+
let g = s.tables[p];
|
|
1009
|
+
if (r && g.name in m.tables) {
|
|
1010
|
+
d.push(g.name), K(t.onProgress, {
|
|
926
1011
|
phase: "import",
|
|
927
1012
|
table: g.name,
|
|
928
|
-
index:
|
|
1013
|
+
index: p,
|
|
929
1014
|
tableCount: h,
|
|
930
1015
|
event: "done",
|
|
931
|
-
rowsDone:
|
|
1016
|
+
rowsDone: m.tables[g.name].rowCount,
|
|
932
1017
|
total: g.rowCount
|
|
933
1018
|
});
|
|
934
1019
|
continue;
|
|
935
1020
|
}
|
|
936
|
-
let
|
|
1021
|
+
let v = i.join(t.bundleDir, g.file);
|
|
937
1022
|
K(t.onProgress, {
|
|
938
1023
|
phase: "import",
|
|
939
1024
|
table: g.name,
|
|
940
|
-
index:
|
|
1025
|
+
index: p,
|
|
941
1026
|
tableCount: h,
|
|
942
1027
|
event: "start",
|
|
943
1028
|
rowsDone: 0,
|
|
944
1029
|
total: g.rowCount
|
|
945
1030
|
});
|
|
946
|
-
let
|
|
947
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0,
|
|
1031
|
+
let y = a.length, b = n.gen(function* () {
|
|
1032
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, l = w("bundle", () => E(v).pipe(Ze(s.compression))).pipe(c.tap((e) => n.sync(() => i.update(e)))), d = async (t) => {
|
|
948
1033
|
let n = {
|
|
949
1034
|
entry: g,
|
|
950
1035
|
row: t,
|
|
951
1036
|
reason: ""
|
|
952
1037
|
};
|
|
953
1038
|
try {
|
|
954
|
-
await
|
|
1039
|
+
await _(e, g, t);
|
|
955
1040
|
} catch (e) {
|
|
956
|
-
n.reason =
|
|
1041
|
+
n.reason = Yt(e), a.push(n);
|
|
957
1042
|
}
|
|
958
1043
|
}, f = yield* n.promise(async () => {
|
|
959
1044
|
let n = t.copyLoader;
|
|
960
1045
|
if (!n || n.dialect !== "postgres" || t.atomic) return !1;
|
|
961
|
-
if (
|
|
962
|
-
if (
|
|
1046
|
+
if (u === "replace") return !0;
|
|
1047
|
+
if (u !== "upsert") return !1;
|
|
963
1048
|
try {
|
|
964
1049
|
return (await e.query({
|
|
965
1050
|
table: g.name,
|
|
@@ -972,14 +1057,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
972
1057
|
} catch {
|
|
973
1058
|
return !1;
|
|
974
1059
|
}
|
|
975
|
-
}),
|
|
976
|
-
if (
|
|
977
|
-
let e =
|
|
978
|
-
|
|
1060
|
+
}), p = Object.keys(g.columnTypes), m = Math.max(1, t.copyBatchSize ?? 5e3), h = [], y = async () => {
|
|
1061
|
+
if (h.length === 0) return;
|
|
1062
|
+
let e = h;
|
|
1063
|
+
h = [];
|
|
979
1064
|
try {
|
|
980
1065
|
await t.copyLoader.copyInto({
|
|
981
1066
|
table: g.name,
|
|
982
|
-
columns:
|
|
1067
|
+
columns: p,
|
|
983
1068
|
columnTypes: g.columnTypes,
|
|
984
1069
|
rows: e
|
|
985
1070
|
});
|
|
@@ -988,9 +1073,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
988
1073
|
for (let t of e) await d(t);
|
|
989
1074
|
}
|
|
990
1075
|
};
|
|
991
|
-
return yield* c.runForEach(
|
|
1076
|
+
return yield* c.runForEach(kt(l, g.name, g.columnTypes), (e) => n.promise(async () => {
|
|
992
1077
|
if (o++, f) {
|
|
993
|
-
|
|
1078
|
+
h.push(e), h.length >= m && await y();
|
|
994
1079
|
return;
|
|
995
1080
|
}
|
|
996
1081
|
await d(e);
|
|
@@ -998,71 +1083,75 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
998
1083
|
rowCount: o,
|
|
999
1084
|
checksum: i.digest("hex")
|
|
1000
1085
|
};
|
|
1001
|
-
}), { rowCount:
|
|
1002
|
-
if (
|
|
1003
|
-
if (
|
|
1086
|
+
}), { rowCount: x, checksum: S } = yield* ft("import", g.name, p, h, (e) => e.rowCount, b);
|
|
1087
|
+
if (l) {
|
|
1088
|
+
if (S !== g.checksum) return yield* n.fail(new F({
|
|
1004
1089
|
what: `table ${g.name} checksum`,
|
|
1005
1090
|
expected: g.checksum,
|
|
1006
|
-
actual:
|
|
1091
|
+
actual: S
|
|
1007
1092
|
}));
|
|
1008
|
-
if (
|
|
1093
|
+
if (x !== g.rowCount) return yield* n.fail(new F({
|
|
1009
1094
|
what: `table ${g.name} rowCount`,
|
|
1010
1095
|
expected: String(g.rowCount),
|
|
1011
|
-
actual: String(
|
|
1096
|
+
actual: String(x)
|
|
1012
1097
|
}));
|
|
1013
1098
|
}
|
|
1014
1099
|
K(t.onProgress, {
|
|
1015
1100
|
phase: "import",
|
|
1016
1101
|
table: g.name,
|
|
1017
|
-
index:
|
|
1102
|
+
index: p,
|
|
1018
1103
|
tableCount: h,
|
|
1019
1104
|
event: "done",
|
|
1020
|
-
rowsDone:
|
|
1105
|
+
rowsDone: x,
|
|
1021
1106
|
total: g.rowCount
|
|
1022
|
-
}), r && (
|
|
1107
|
+
}), r && (a.length > y ? o.push({
|
|
1023
1108
|
name: g.name,
|
|
1024
|
-
rowCount:
|
|
1025
|
-
}) : (
|
|
1026
|
-
rowCount:
|
|
1109
|
+
rowCount: x
|
|
1110
|
+
}) : (m.tables[g.name] = {
|
|
1111
|
+
rowCount: x,
|
|
1027
1112
|
checksum: g.checksum
|
|
1028
|
-
}, yield* q(
|
|
1113
|
+
}, yield* q(f, m)));
|
|
1029
1114
|
}
|
|
1030
|
-
if (
|
|
1031
|
-
for (let e of
|
|
1032
|
-
let t =
|
|
1033
|
-
|
|
1115
|
+
if (a.length > 0 && (yield* x(e, a)), r && o.length > 0) {
|
|
1116
|
+
for (let e of o) {
|
|
1117
|
+
let t = s.tables.find((t) => t.name === e.name);
|
|
1118
|
+
m.tables[e.name] = {
|
|
1034
1119
|
rowCount: e.rowCount,
|
|
1035
1120
|
checksum: t.checksum
|
|
1036
1121
|
};
|
|
1037
1122
|
}
|
|
1038
|
-
yield* q(
|
|
1123
|
+
yield* q(f, m);
|
|
1124
|
+
}
|
|
1125
|
+
if (d.length > 0) {
|
|
1126
|
+
let e = d.length === s.tables.length;
|
|
1127
|
+
t.onWarn?.(`resume: ${d.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${e ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${d.join(", ")}. Delete ${f} to force a full re-import.`);
|
|
1039
1128
|
}
|
|
1040
1129
|
});
|
|
1041
1130
|
if (t.atomic ? yield* n.tryPromise({
|
|
1042
1131
|
try: () => t.store.transactional(async (t) => {
|
|
1043
|
-
let i = await n.runPromiseExit(
|
|
1132
|
+
let i = await n.runPromiseExit(S(t, !1));
|
|
1044
1133
|
if (r.isFailure(i)) throw Error(e.pretty(i.cause));
|
|
1045
1134
|
}),
|
|
1046
1135
|
catch: (e) => new P({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1047
|
-
}) : yield*
|
|
1048
|
-
let e = t.assets, r = i.join(t.bundleDir, W),
|
|
1049
|
-
yield* c.runForEach(
|
|
1050
|
-
|
|
1051
|
-
})), n.andThen(q(
|
|
1136
|
+
}) : yield* S(t.store, !0), t.assets && s.assets.index && !t.assetsAlreadyRestored) {
|
|
1137
|
+
let e = t.assets, r = i.join(t.bundleDir, W), a = i.join(t.bundleDir, s.assets.index ?? "assets/index.ndjson"), o = w("bundle", () => E(a)).pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.map((e) => JSON.parse(e)));
|
|
1138
|
+
yield* c.runForEach(o, (t) => t.key in m.assets ? n.void : Ke(e, r, t).pipe(n.andThen(n.sync(() => {
|
|
1139
|
+
m.assets[t.key] = !0;
|
|
1140
|
+
})), n.andThen(q(f, m))));
|
|
1052
1141
|
}
|
|
1053
|
-
return
|
|
1054
|
-
}),
|
|
1142
|
+
return s;
|
|
1143
|
+
}), nn = (e) => e, rn = (e) => {
|
|
1055
1144
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1056
1145
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1057
1146
|
return {
|
|
1058
1147
|
refuse: !0,
|
|
1059
1148
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1060
1149
|
};
|
|
1061
|
-
}, Y = Buffer.from("VBUNDLE2"),
|
|
1150
|
+
}, Y = Buffer.from("VBUNDLE2"), an = 0, on = 1, sn = 2, cn = 32, ln = async (e) => {
|
|
1062
1151
|
let t = [], n = async (r) => {
|
|
1063
1152
|
let i;
|
|
1064
1153
|
try {
|
|
1065
|
-
i = await
|
|
1154
|
+
i = await ce(k(e, r), { withFileTypes: !0 });
|
|
1066
1155
|
} catch {
|
|
1067
1156
|
return;
|
|
1068
1157
|
}
|
|
@@ -1072,22 +1161,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1072
1161
|
}
|
|
1073
1162
|
};
|
|
1074
1163
|
return await n(""), t.sort();
|
|
1075
|
-
},
|
|
1164
|
+
}, un = (e, t) => {
|
|
1076
1165
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1077
|
-
return r.writeUInt8(
|
|
1078
|
-
},
|
|
1166
|
+
return r.writeUInt8(on, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1167
|
+
}, dn = (e, t, n) => {
|
|
1079
1168
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1080
|
-
return a.writeUInt8(
|
|
1169
|
+
return a.writeUInt8(sn, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1081
1170
|
};
|
|
1082
|
-
async function*
|
|
1171
|
+
async function* fn(e) {
|
|
1083
1172
|
yield new Uint8Array(Y);
|
|
1084
|
-
for (let t of await
|
|
1085
|
-
let n =
|
|
1086
|
-
if (yield new Uint8Array(
|
|
1173
|
+
for (let t of await ln(e.dir)) {
|
|
1174
|
+
let n = k(e.dir, t), r = await le(n);
|
|
1175
|
+
if (yield new Uint8Array(un(t, r.size)), r.size > 0) for await (let e of E(n)) yield new Uint8Array(e);
|
|
1087
1176
|
}
|
|
1088
1177
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1089
|
-
yield new Uint8Array(
|
|
1090
|
-
let n =
|
|
1178
|
+
yield new Uint8Array(dn(t.key, t.contentType, t.size));
|
|
1179
|
+
let n = y("sha256"), r = 0;
|
|
1091
1180
|
for await (let i of e.blobs.open(t.key)) {
|
|
1092
1181
|
let e = Buffer.from(i);
|
|
1093
1182
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1095,11 +1184,11 @@ async function* nn(e) {
|
|
|
1095
1184
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1096
1185
|
yield new Uint8Array(n.digest());
|
|
1097
1186
|
}
|
|
1098
|
-
yield new Uint8Array(Buffer.from([
|
|
1187
|
+
yield new Uint8Array(Buffer.from([an]));
|
|
1099
1188
|
}
|
|
1100
|
-
var
|
|
1101
|
-
let n =
|
|
1102
|
-
await
|
|
1189
|
+
var pn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), mn = (e) => new Promise((t) => e.end(t)), hn = async (e, t) => {
|
|
1190
|
+
let n = pe(t.dir);
|
|
1191
|
+
await O(n, { recursive: !0 });
|
|
1103
1192
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1104
1193
|
let { value: e, done: t } = await i.next();
|
|
1105
1194
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1114,24 +1203,24 @@ var rn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), an
|
|
|
1114
1203
|
for (;;) {
|
|
1115
1204
|
await s(1);
|
|
1116
1205
|
let e = c(1).readUInt8(0);
|
|
1117
|
-
if (e ===
|
|
1118
|
-
if (e ===
|
|
1206
|
+
if (e === an) return;
|
|
1207
|
+
if (e === on) {
|
|
1119
1208
|
await s(4);
|
|
1120
1209
|
let e = c(4).readUInt32BE(0);
|
|
1121
1210
|
await s(e);
|
|
1122
1211
|
let t = c(e).toString("utf8");
|
|
1123
1212
|
await s(8);
|
|
1124
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1125
|
-
if (a !== n && !a.startsWith(n +
|
|
1126
|
-
await
|
|
1213
|
+
let i = c(8).readBigUInt64BE(0), a = pe(n, t);
|
|
1214
|
+
if (a !== n && !a.startsWith(n + me)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1215
|
+
await O(fe(a), { recursive: !0 });
|
|
1127
1216
|
let o = D(a);
|
|
1128
1217
|
for (; i > 0n;) {
|
|
1129
1218
|
r.length === 0 && await s(1);
|
|
1130
1219
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1131
|
-
await
|
|
1220
|
+
await pn(o, c(e)), i -= BigInt(e);
|
|
1132
1221
|
}
|
|
1133
|
-
await
|
|
1134
|
-
} else if (e ===
|
|
1222
|
+
await mn(o);
|
|
1223
|
+
} else if (e === sn) {
|
|
1135
1224
|
await s(4);
|
|
1136
1225
|
let e = c(4).readUInt32BE(0);
|
|
1137
1226
|
await s(e);
|
|
@@ -1141,24 +1230,24 @@ var rn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), an
|
|
|
1141
1230
|
await s(i);
|
|
1142
1231
|
let a = c(i).toString("utf8");
|
|
1143
1232
|
await s(8);
|
|
1144
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1233
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = y("sha256"), d = new re(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1145
1234
|
contentType: a,
|
|
1146
1235
|
size: Number(o)
|
|
1147
1236
|
}), p = o;
|
|
1148
1237
|
for (; p > 0n;) {
|
|
1149
1238
|
r.length === 0 && await s(1);
|
|
1150
1239
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1151
|
-
u.update(t), l || await
|
|
1240
|
+
u.update(t), l || await pn(d, t), p -= BigInt(e);
|
|
1152
1241
|
}
|
|
1153
|
-
l ? d.destroy() : await
|
|
1154
|
-
let m = c(
|
|
1242
|
+
l ? d.destroy() : await mn(d), await f, await s(cn);
|
|
1243
|
+
let m = c(cn);
|
|
1155
1244
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1156
1245
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1157
1246
|
}
|
|
1158
1247
|
} finally {
|
|
1159
1248
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1160
1249
|
}
|
|
1161
|
-
},
|
|
1250
|
+
}, gn = async (e, t) => {
|
|
1162
1251
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1163
1252
|
let { value: e, done: t } = await a.next();
|
|
1164
1253
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1180,8 +1269,8 @@ var rn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), an
|
|
|
1180
1269
|
if (!await s(Y.length) || !c(Y.length).equals(Y)) throw Error("not a bundle archive (bad magic)");
|
|
1181
1270
|
for (; await s(1);) {
|
|
1182
1271
|
let e = c(1).readUInt8(0);
|
|
1183
|
-
if (e ===
|
|
1184
|
-
if (e !==
|
|
1272
|
+
if (e === an || e === sn) break;
|
|
1273
|
+
if (e !== on) throw Error(`unknown bundle entry kind ${e}`);
|
|
1185
1274
|
if (!await s(4)) break;
|
|
1186
1275
|
let t = c(4).readUInt32BE(0);
|
|
1187
1276
|
if (!await s(t)) break;
|
|
@@ -1191,22 +1280,22 @@ var rn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), an
|
|
|
1191
1280
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1192
1281
|
}
|
|
1193
1282
|
return r;
|
|
1194
|
-
},
|
|
1283
|
+
}, _n = (e, t) => hn(e, {
|
|
1195
1284
|
dir: t,
|
|
1196
|
-
blobs:
|
|
1197
|
-
}),
|
|
1285
|
+
blobs: vn(t)
|
|
1286
|
+
}), vn = (e) => {
|
|
1198
1287
|
let t = /* @__PURE__ */ new Set();
|
|
1199
1288
|
return {
|
|
1200
1289
|
has: async (e) => t.has(e),
|
|
1201
1290
|
write: async (n, r, i) => {
|
|
1202
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1203
|
-
await
|
|
1204
|
-
let l =
|
|
1291
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = k(e, "assets");
|
|
1292
|
+
await O(c, { recursive: !0 });
|
|
1293
|
+
let l = y("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1205
1294
|
l.update(e), u += e.length, n(null, e);
|
|
1206
|
-
} }), f =
|
|
1295
|
+
} }), f = k(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1207
1296
|
await o(r, d, D(f));
|
|
1208
1297
|
let p = l.digest("hex");
|
|
1209
|
-
await a.rename(f,
|
|
1298
|
+
await a.rename(f, k(c, p)), await a.appendFile(k(c, "index.ndjson"), `${JSON.stringify({
|
|
1210
1299
|
key: n,
|
|
1211
1300
|
sha256: p,
|
|
1212
1301
|
size: u,
|
|
@@ -1214,33 +1303,33 @@ var rn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), an
|
|
|
1214
1303
|
})}\n`), t.add(n);
|
|
1215
1304
|
}
|
|
1216
1305
|
};
|
|
1217
|
-
},
|
|
1306
|
+
}, yn = he(S), bn = Buffer.from("VENC1\0\0\0"), xn = 32, Sn = 16, Cn = 7, X = 16, wn = 262144, Z = {
|
|
1218
1307
|
N: 32768,
|
|
1219
1308
|
r: 8,
|
|
1220
1309
|
p: 1
|
|
1221
|
-
},
|
|
1310
|
+
}, Tn = 134217728, Q = 43, En = (e, t, n) => {
|
|
1222
1311
|
let r = Buffer.alloc(12);
|
|
1223
|
-
return e.copy(r, 0, 0,
|
|
1312
|
+
return e.copy(r, 0, 0, Cn), r.writeUInt32BE(t >>> 0, Cn), r.writeUInt8(+!!n, 11), r;
|
|
1224
1313
|
};
|
|
1225
|
-
async function*
|
|
1314
|
+
async function* Dn(e, t) {
|
|
1226
1315
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1227
|
-
let n =
|
|
1316
|
+
let n = x(Sn), r = x(Cn), i = await yn(t, n, xn, {
|
|
1228
1317
|
...Z,
|
|
1229
|
-
maxmem:
|
|
1318
|
+
maxmem: Tn
|
|
1230
1319
|
}), a = Buffer.alloc(Q);
|
|
1231
|
-
|
|
1320
|
+
bn.copy(a, 0), a.writeUInt32BE(Z.N, 8), a.writeUInt32BE(Z.r, 12), a.writeUInt32BE(Z.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1232
1321
|
let o = 0, s = (e, t) => {
|
|
1233
|
-
let n =
|
|
1322
|
+
let n = _("aes-256-gcm", i, En(r, o, t));
|
|
1234
1323
|
o === 0 && n.setAAD(a);
|
|
1235
1324
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1236
1325
|
o++;
|
|
1237
1326
|
let l = Buffer.alloc(5 + s.length + X);
|
|
1238
1327
|
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + X, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1239
1328
|
}, c = Buffer.alloc(0);
|
|
1240
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1329
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > wn;) yield s(c.subarray(0, wn), !1), c = c.subarray(wn);
|
|
1241
1330
|
yield s(c, !0);
|
|
1242
1331
|
}
|
|
1243
|
-
async function*
|
|
1332
|
+
async function* On(e, t) {
|
|
1244
1333
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1245
1334
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1246
1335
|
let { value: e, done: t } = await n.next();
|
|
@@ -1251,12 +1340,12 @@ async function* yn(e, t) {
|
|
|
1251
1340
|
};
|
|
1252
1341
|
if (!await o(Q)) throw Error("encrypted bundle truncated (no header)");
|
|
1253
1342
|
let s = r.subarray(0, Q);
|
|
1254
|
-
if (!s.subarray(0, 8).equals(
|
|
1255
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await
|
|
1343
|
+
if (!s.subarray(0, 8).equals(bn)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1344
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await yn(t, d, xn, {
|
|
1256
1345
|
N: c,
|
|
1257
1346
|
r: l,
|
|
1258
1347
|
p: u,
|
|
1259
|
-
maxmem:
|
|
1348
|
+
maxmem: Tn
|
|
1260
1349
|
});
|
|
1261
1350
|
r = r.subarray(Q);
|
|
1262
1351
|
let m = 0, h = !1;
|
|
@@ -1271,7 +1360,7 @@ async function* yn(e, t) {
|
|
|
1271
1360
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1272
1361
|
let n = r.subarray(5, 5 + t - X), i = r.subarray(5 + t - X, 5 + t);
|
|
1273
1362
|
r = r.subarray(5 + t);
|
|
1274
|
-
let a =
|
|
1363
|
+
let a = v("aes-256-gcm", p, En(f, m, e));
|
|
1275
1364
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1276
1365
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1277
1366
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1279,7 +1368,7 @@ async function* yn(e, t) {
|
|
|
1279
1368
|
}
|
|
1280
1369
|
//#endregion
|
|
1281
1370
|
//#region src/blobStorage.ts
|
|
1282
|
-
var
|
|
1371
|
+
var kn = (e, t) => ({
|
|
1283
1372
|
list: () => {
|
|
1284
1373
|
let e = t();
|
|
1285
1374
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1292,29 +1381,29 @@ var bn = (e, t) => ({
|
|
|
1292
1381
|
} };
|
|
1293
1382
|
},
|
|
1294
1383
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1295
|
-
let r = await n.runPromise(
|
|
1384
|
+
let r = await n.runPromise(ee(e, t)), i = await ne(r.stream);
|
|
1296
1385
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1297
1386
|
} })
|
|
1298
|
-
}),
|
|
1299
|
-
table:
|
|
1387
|
+
}), An = (e, t) => kn(e, () => h(t, {
|
|
1388
|
+
table: C,
|
|
1300
1389
|
chunkSize: 500
|
|
1301
1390
|
}).pipe(c.map((e) => ({
|
|
1302
1391
|
key: String(e.key),
|
|
1303
1392
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1304
1393
|
size: Number(e.size ?? 0)
|
|
1305
|
-
})))),
|
|
1394
|
+
})))), jn = (e) => ({
|
|
1306
1395
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1307
1396
|
write: async (t, r, i) => {
|
|
1308
|
-
let a =
|
|
1309
|
-
await n.runPromise(
|
|
1397
|
+
let a = w("storage", () => ie.from(r));
|
|
1398
|
+
await n.runPromise(te(e, t, a, {
|
|
1310
1399
|
contentType: i.contentType,
|
|
1311
1400
|
totalSize: i.size
|
|
1312
1401
|
}));
|
|
1313
1402
|
}
|
|
1314
|
-
}),
|
|
1403
|
+
}), Mn = (e, t) => {
|
|
1315
1404
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1316
|
-
return e === "mariadb" &&
|
|
1317
|
-
},
|
|
1405
|
+
return e === "mariadb" && Nn(n) ? n : r;
|
|
1406
|
+
}, Nn = (e) => (process.env.PATH ?? "").split(de).filter((e) => e !== "").some((t) => oe(k(t, e))), Pn = (e) => {
|
|
1318
1407
|
switch (e) {
|
|
1319
1408
|
case "postgres": return "db.dump";
|
|
1320
1409
|
case "mysql":
|
|
@@ -1324,7 +1413,7 @@ var bn = (e, t) => ({
|
|
|
1324
1413
|
case "mssql": return "db.bacpac";
|
|
1325
1414
|
default: return "db.dump";
|
|
1326
1415
|
}
|
|
1327
|
-
},
|
|
1416
|
+
}, Fn = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1328
1417
|
if (e.host || e.database) return {
|
|
1329
1418
|
host: e.host ?? "localhost",
|
|
1330
1419
|
port: e.port ?? 3306,
|
|
@@ -1340,7 +1429,7 @@ var bn = (e, t) => ({
|
|
|
1340
1429
|
password: decodeURIComponent(t.password) || "",
|
|
1341
1430
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1342
1431
|
};
|
|
1343
|
-
},
|
|
1432
|
+
}, In = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Ln = (e, t, n) => {
|
|
1344
1433
|
switch (e) {
|
|
1345
1434
|
case "postgres": {
|
|
1346
1435
|
if (t.url) return {
|
|
@@ -1387,7 +1476,7 @@ var bn = (e, t) => ({
|
|
|
1387
1476
|
let r = $(t);
|
|
1388
1477
|
return {
|
|
1389
1478
|
kind: "spawn",
|
|
1390
|
-
tool:
|
|
1479
|
+
tool: Mn(e, "dump"),
|
|
1391
1480
|
args: [
|
|
1392
1481
|
"--single-transaction",
|
|
1393
1482
|
"--routines",
|
|
@@ -1409,7 +1498,7 @@ var bn = (e, t) => ({
|
|
|
1409
1498
|
case "sqlite":
|
|
1410
1499
|
case "turso": return {
|
|
1411
1500
|
kind: "copy",
|
|
1412
|
-
from:
|
|
1501
|
+
from: Fn(t),
|
|
1413
1502
|
to: n
|
|
1414
1503
|
};
|
|
1415
1504
|
case "mssql": return {
|
|
@@ -1417,7 +1506,7 @@ var bn = (e, t) => ({
|
|
|
1417
1506
|
tool: "sqlpackage",
|
|
1418
1507
|
args: [
|
|
1419
1508
|
"/Action:Export",
|
|
1420
|
-
`/SourceConnectionString:${
|
|
1509
|
+
`/SourceConnectionString:${In($({
|
|
1421
1510
|
...t,
|
|
1422
1511
|
port: t.port ?? 1433
|
|
1423
1512
|
}))}`,
|
|
@@ -1430,7 +1519,7 @@ var bn = (e, t) => ({
|
|
|
1430
1519
|
reason: `no native backup for dialect '${e}'`
|
|
1431
1520
|
});
|
|
1432
1521
|
}
|
|
1433
|
-
},
|
|
1522
|
+
}, Rn = (e, t, n) => {
|
|
1434
1523
|
switch (e) {
|
|
1435
1524
|
case "postgres": {
|
|
1436
1525
|
if (t.url) return {
|
|
@@ -1475,7 +1564,7 @@ var bn = (e, t) => ({
|
|
|
1475
1564
|
let r = $(t);
|
|
1476
1565
|
return {
|
|
1477
1566
|
kind: "spawn",
|
|
1478
|
-
tool:
|
|
1567
|
+
tool: Mn(e, "client"),
|
|
1479
1568
|
args: [
|
|
1480
1569
|
"--protocol=TCP",
|
|
1481
1570
|
"-h",
|
|
@@ -1494,14 +1583,14 @@ var bn = (e, t) => ({
|
|
|
1494
1583
|
case "turso": return {
|
|
1495
1584
|
kind: "copy",
|
|
1496
1585
|
from: n,
|
|
1497
|
-
to:
|
|
1586
|
+
to: Fn(t)
|
|
1498
1587
|
};
|
|
1499
1588
|
case "mssql": return {
|
|
1500
1589
|
kind: "spawn",
|
|
1501
1590
|
tool: "sqlpackage",
|
|
1502
1591
|
args: [
|
|
1503
1592
|
"/Action:Import",
|
|
1504
|
-
`/TargetConnectionString:${
|
|
1593
|
+
`/TargetConnectionString:${In($({
|
|
1505
1594
|
...t,
|
|
1506
1595
|
port: t.port ?? 1433
|
|
1507
1596
|
}))}`,
|
|
@@ -1514,13 +1603,13 @@ var bn = (e, t) => ({
|
|
|
1514
1603
|
reason: `no native restore for dialect '${e}'`
|
|
1515
1604
|
});
|
|
1516
1605
|
}
|
|
1517
|
-
},
|
|
1518
|
-
|
|
1606
|
+
}, zn = (e) => e.kind === "copy" ? n.async((t) => {
|
|
1607
|
+
ae(e.from, e.to, (r) => t(r ? n.fail(new L({
|
|
1519
1608
|
tool: "copy",
|
|
1520
1609
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1521
1610
|
})) : n.void));
|
|
1522
1611
|
}) : n.async((t) => {
|
|
1523
|
-
let r =
|
|
1612
|
+
let r = ge(e.tool, [...e.args], {
|
|
1524
1613
|
env: {
|
|
1525
1614
|
...process.env,
|
|
1526
1615
|
...e.env
|
|
@@ -1536,7 +1625,7 @@ var bn = (e, t) => ({
|
|
|
1536
1625
|
}), e.stdoutFile && r.stdout && r.stdout.pipe(D(e.stdoutFile)), e.stdinFile && r.stdin && E(e.stdinFile).pipe(r.stdin);
|
|
1537
1626
|
let a = () => {
|
|
1538
1627
|
if (e.stdoutFile !== void 0) try {
|
|
1539
|
-
|
|
1628
|
+
se(e.stdoutFile);
|
|
1540
1629
|
} catch {}
|
|
1541
1630
|
};
|
|
1542
1631
|
r.on("error", (r) => {
|
|
@@ -1551,15 +1640,15 @@ var bn = (e, t) => ({
|
|
|
1551
1640
|
}
|
|
1552
1641
|
a(), t(n.fail(new L({
|
|
1553
1642
|
tool: e.tool,
|
|
1554
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
1643
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${ue(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
1555
1644
|
stderr: i.slice(-2e3)
|
|
1556
1645
|
})));
|
|
1557
1646
|
});
|
|
1558
|
-
}),
|
|
1647
|
+
}), Bn = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Vn = (e) => n.gen(function* () {
|
|
1559
1648
|
let t = yield* V(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
1560
1649
|
store: e.store,
|
|
1561
1650
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
1562
|
-
}), r =
|
|
1651
|
+
}), r = At(t.tables, e.snapshot), i = e.classification ?? A(e.snapshot), a = e.sampleSize ?? 20, o = Ae(r.map((t) => ({
|
|
1563
1652
|
name: t,
|
|
1564
1653
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
1565
1654
|
name: e.name,
|
|
@@ -1567,26 +1656,26 @@ var bn = (e, t) => ({
|
|
|
1567
1656
|
}))
|
|
1568
1657
|
})), i, e.masking), s = [], l = [];
|
|
1569
1658
|
for (let i of r) {
|
|
1570
|
-
let r = t.predicateFor(i), u = yield*
|
|
1659
|
+
let r = t.predicateFor(i), u = yield* h(e.store, {
|
|
1571
1660
|
table: i,
|
|
1572
1661
|
chunkSize: a,
|
|
1573
1662
|
...r === void 0 ? {} : { where: r }
|
|
1574
|
-
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
1575
|
-
l.push(...
|
|
1576
|
-
let f = u[0],
|
|
1577
|
-
if (f &&
|
|
1663
|
+
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) => Ne(t, i, o.actions, e.masking.seed));
|
|
1664
|
+
l.push(...Me(d, i, o.actions));
|
|
1665
|
+
let f = u[0], p = d[0], m = [];
|
|
1666
|
+
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
1578
1667
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
1579
1668
|
let n = e.slice(i.length + 1);
|
|
1580
|
-
|
|
1669
|
+
m.push({
|
|
1581
1670
|
column: n,
|
|
1582
|
-
action:
|
|
1671
|
+
action: Bn(t),
|
|
1583
1672
|
before: f[n],
|
|
1584
|
-
after:
|
|
1673
|
+
after: p[n]
|
|
1585
1674
|
});
|
|
1586
1675
|
}
|
|
1587
1676
|
s.push({
|
|
1588
1677
|
table: i,
|
|
1589
|
-
samples:
|
|
1678
|
+
samples: m
|
|
1590
1679
|
});
|
|
1591
1680
|
}
|
|
1592
1681
|
return {
|
|
@@ -1594,7 +1683,7 @@ var bn = (e, t) => ({
|
|
|
1594
1683
|
leaks: l,
|
|
1595
1684
|
unclassified: o.unclassified
|
|
1596
1685
|
};
|
|
1597
|
-
}),
|
|
1686
|
+
}), Hn = (e, t) => {
|
|
1598
1687
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
1599
1688
|
for (let t of e) {
|
|
1600
1689
|
let e = n.get(t.name);
|
|
@@ -1649,7 +1738,7 @@ var bn = (e, t) => ({
|
|
|
1649
1738
|
skipColumns: i,
|
|
1650
1739
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
1651
1740
|
};
|
|
1652
|
-
},
|
|
1741
|
+
}, Un = (e) => {
|
|
1653
1742
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
1654
1743
|
let t = {
|
|
1655
1744
|
safe: "✓",
|
|
@@ -1660,4 +1749,4 @@ var bn = (e, t) => ({
|
|
|
1660
1749
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
1661
1750
|
};
|
|
1662
1751
|
//#endregion
|
|
1663
|
-
export { W as ASSETS_DIR, G as ASSET_INDEX_FILE, P as BundleError,
|
|
1752
|
+
export { W as ASSETS_DIR, G as ASSET_INDEX_FILE, P as BundleError, Pe as CodecError, Fe as CompressionError, Be as CrossDialectError, U as DATA_DIR, ve as DEFAULT_CLASS_ACTIONS, Qe as FORMAT_VERSION, ze as ImportModeError, F as IntegrityError, ot as LEDGER_FILE, H as MANIFEST_FILE, Re as MaskingError, L as NativeToolError, Ie as RowsRefusedError, Le as SchemaDriftError, I as ScopeError, Oe as applyAction, rn as assessLiveImport, gt as assetDone, Pn as backupArtifactName, Ln as backupCommand, Ht as canonicalType, Gt as checkPortability, $t as checkSchemaDrift, A as classificationFromSnapshot, Hn as classifyImportDrift, Ye as compressionExt, z as computeSubsetIds, it as decodeManifest, Et as decodeRow, wt as decodeValue, On as decryptStream, nn as defineDataProfile, Qt as diffSnapshots, vn as dirBlobSink, pt as emptyLedger, rt as encodeManifest, Tt as encodeRow, St as encodeValue, Dn as encryptStream, Un as formatImportDrift, Kt as formatIssue, qe as hasZstd, Xe as makeCompressor, Ze as makeDecompressor, Ne as maskRow, kt as ndjsonToRows, fn as packBundle, gn as peekBundle, Je as pickCompression, Ae as planMasking, Vn as previewMasking, mt as readLedger, ye as resolveAction, V as resolveScope, Ke as restoreAssetFromCas, Rn as restoreCommand, Ot as rowsToNdjson, Vt as runExport, tn as runImport, zn as runNativeStep, Me as scanForLeaks, We as storageAssetSink, Ue as storageAssetSource, jn as storageBlobSink, kn as storageBlobSource, An as storageRefsBlobSource, He as subsetToScope, ht as tableDone, At as topoSortTables, hn as unpackBundle, _n as unpackBundleToDir, Ge as writeAssetToCas, q as writeLedger };
|