@voltro/data-transfer 0.44.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +120 -1
- package/dist/index.d.ts +31 -0
- package/dist/index.js +564 -517
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a,
|
|
2
|
-
import { classifyConstraintViolation as
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { STORAGE_REFS_TABLE as
|
|
6
|
-
import * as
|
|
7
|
-
import { PassThrough as
|
|
8
|
-
import { copyFile as ee, createReadStream as
|
|
1
|
+
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Runtime as o, Schedule as s, Schema as c, Stream as l } from "effect";
|
|
2
|
+
import { classifyConstraintViolation as u, eq as d, extractDbCause as f, inSet as p, isNotNull as m, or as h, streamTable as g } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as _ } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as v, createDecipheriv as y, createHash as b, createHmac as x, randomBytes as S, scrypt as C } from "node:crypto";
|
|
5
|
+
import { STORAGE_REFS_TABLE as w, fromNodeReadable as T, getObjectStream as E, putObjectStream as D, toNodeReadable as O } from "@voltro/plugin-storage";
|
|
6
|
+
import * as k from "node:zlib";
|
|
7
|
+
import { PassThrough as A, Readable as j } from "node:stream";
|
|
8
|
+
import { copyFile as ee, createReadStream as M, createWriteStream as te, existsSync as ne, unlinkSync as re } from "node:fs";
|
|
9
9
|
import { mkdir as ie, readdir as ae, stat as oe } from "node:fs/promises";
|
|
10
|
-
import { basename as se, delimiter as ce, dirname as le, join as
|
|
10
|
+
import { basename as se, delimiter as ce, dirname as le, join as N, resolve as ue, sep as de } from "node:path";
|
|
11
11
|
import { promisify as fe } from "node:util";
|
|
12
12
|
import { spawn as pe } from "node:child_process";
|
|
13
13
|
//#region src/consistency.ts
|
|
@@ -36,7 +36,7 @@ var me = (e) => {
|
|
|
36
36
|
date: "dateShift",
|
|
37
37
|
secret: "null",
|
|
38
38
|
freeText: "redact"
|
|
39
|
-
},
|
|
39
|
+
}, ge = (e) => {
|
|
40
40
|
let t = {};
|
|
41
41
|
for (let n of e.tables) {
|
|
42
42
|
let e = {};
|
|
@@ -48,7 +48,7 @@ var me = (e) => {
|
|
|
48
48
|
t[n.name] = e;
|
|
49
49
|
}
|
|
50
50
|
return t;
|
|
51
|
-
},
|
|
51
|
+
}, _e = (e, t, n, r) => {
|
|
52
52
|
let i = r.columns?.[`${e}.${t}`];
|
|
53
53
|
if (i) return i;
|
|
54
54
|
if (n?.sensitive) {
|
|
@@ -56,7 +56,7 @@ var me = (e) => {
|
|
|
56
56
|
return r.classes?.[e] ?? he[e] ?? "redact";
|
|
57
57
|
}
|
|
58
58
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
59
|
-
},
|
|
59
|
+
}, ve = (e, t) => x("sha256", e).update(t == null ? "∅" : String(t)).digest(), P = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), F = [
|
|
60
60
|
"Ada",
|
|
61
61
|
"Alan",
|
|
62
62
|
"Grace",
|
|
@@ -69,7 +69,7 @@ var me = (e) => {
|
|
|
69
69
|
"Guido",
|
|
70
70
|
"Margaret",
|
|
71
71
|
"Bjarne"
|
|
72
|
-
],
|
|
72
|
+
], ye = [
|
|
73
73
|
"Lovelace",
|
|
74
74
|
"Turing",
|
|
75
75
|
"Hopper",
|
|
@@ -82,7 +82,7 @@ var me = (e) => {
|
|
|
82
82
|
"Rossum",
|
|
83
83
|
"Hamilton",
|
|
84
84
|
"Stroustrup"
|
|
85
|
-
],
|
|
85
|
+
], be = [
|
|
86
86
|
"Maple",
|
|
87
87
|
"Oak",
|
|
88
88
|
"Cedar",
|
|
@@ -91,7 +91,7 @@ var me = (e) => {
|
|
|
91
91
|
"Birch",
|
|
92
92
|
"Willow",
|
|
93
93
|
"Ash"
|
|
94
|
-
],
|
|
94
|
+
], xe = [
|
|
95
95
|
"Initech",
|
|
96
96
|
"Hooli",
|
|
97
97
|
"Umbrella",
|
|
@@ -100,23 +100,23 @@ var me = (e) => {
|
|
|
100
100
|
"Soylent",
|
|
101
101
|
"Stark",
|
|
102
102
|
"Wayne"
|
|
103
|
-
],
|
|
104
|
-
let r =
|
|
103
|
+
], I = (e, t, n) => e[P(t, n) % e.length], Se = (e, t, n) => {
|
|
104
|
+
let r = ve(n, `${e}:${String(t)}`), i = P(r, 1) % 1e4;
|
|
105
105
|
switch (e) {
|
|
106
|
-
case "email": return `${
|
|
107
|
-
case "fullName": return `${
|
|
108
|
-
case "firstName": return
|
|
109
|
-
case "lastName": return
|
|
110
|
-
case "username": return `${
|
|
111
|
-
case "phone": return `+1${String(
|
|
112
|
-
case "address": return `${
|
|
113
|
-
case "company": return
|
|
106
|
+
case "email": return `${I(F, r, 0).toLowerCase()}.${I(ye, r, 2).toLowerCase()}${i}@example.com`;
|
|
107
|
+
case "fullName": return `${I(F, r, 0)} ${I(ye, r, 1)}`;
|
|
108
|
+
case "firstName": return I(F, r, 0);
|
|
109
|
+
case "lastName": return I(ye, r, 1);
|
|
110
|
+
case "username": return `${I(F, r, 0).toLowerCase()}${i}`;
|
|
111
|
+
case "phone": return `+1${String(P(r, 0) % 1e10).padStart(10, "0")}`;
|
|
112
|
+
case "address": return `${P(r, 0) % 9899 + 100} ${I(be, r, 1)} St`;
|
|
113
|
+
case "company": return I(xe, r, 0);
|
|
114
114
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
115
|
-
case "ip": return `${
|
|
116
|
-
case "creditCard": return
|
|
115
|
+
case "ip": return `${P(r, 0) % 256}.${P(r, 1) % 256}.${P(r, 2) % 256}.${P(r, 3) % 254 + 1}`;
|
|
116
|
+
case "creditCard": return Ce(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
117
117
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
118
118
|
}
|
|
119
|
-
},
|
|
119
|
+
}, Ce = (e) => {
|
|
120
120
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
121
121
|
t.forEach((e, t) => {
|
|
122
122
|
let r = e;
|
|
@@ -124,31 +124,31 @@ var me = (e) => {
|
|
|
124
124
|
});
|
|
125
125
|
let r = (10 - n % 10) % 10;
|
|
126
126
|
return e.slice(0, 15) + String(r);
|
|
127
|
-
},
|
|
127
|
+
}, we = 864e5, Te = (e, t) => {
|
|
128
128
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
129
129
|
if (Number.isNaN(n)) return e;
|
|
130
|
-
let r =
|
|
130
|
+
let r = P(ve(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * we);
|
|
131
131
|
return e instanceof Date ? i : i.toISOString();
|
|
132
|
-
},
|
|
132
|
+
}, Ee = (e, t) => {
|
|
133
133
|
if (t.value === null || t.value === void 0) return null;
|
|
134
134
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
135
135
|
if (e === "keep") return t.value;
|
|
136
136
|
if (e === "null") return null;
|
|
137
137
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
138
|
-
if (e === "hash") return
|
|
139
|
-
if (e === "dateShift") return
|
|
140
|
-
if (typeof e == "object" && "fake" in e) return
|
|
138
|
+
if (e === "hash") return ve(t.seed, t.value).toString("hex").slice(0, 16);
|
|
139
|
+
if (e === "dateShift") return Te(t.value, t.seed);
|
|
140
|
+
if (typeof e == "object" && "fake" in e) return Se(e.fake, t.value, t.seed);
|
|
141
141
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
142
142
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
143
|
-
},
|
|
143
|
+
}, De = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Oe = (e, t, n) => {
|
|
144
144
|
let r = {}, i = [], a = [];
|
|
145
145
|
for (let o of e) for (let e of o.columns) {
|
|
146
|
-
let s = `${o.name}.${e.name}`, c =
|
|
146
|
+
let s = `${o.name}.${e.name}`, c = _e(o.name, e.name, t[o.name]?.[e.name], n);
|
|
147
147
|
if (c === "error") {
|
|
148
148
|
i.push(s);
|
|
149
149
|
continue;
|
|
150
150
|
}
|
|
151
|
-
if (!
|
|
151
|
+
if (!De(c)) {
|
|
152
152
|
a.push({
|
|
153
153
|
column: s,
|
|
154
154
|
action: c
|
|
@@ -162,7 +162,7 @@ var me = (e) => {
|
|
|
162
162
|
unclassified: i,
|
|
163
163
|
invalid: a
|
|
164
164
|
};
|
|
165
|
-
},
|
|
165
|
+
}, ke = [
|
|
166
166
|
{
|
|
167
167
|
name: "email",
|
|
168
168
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -179,7 +179,7 @@ var me = (e) => {
|
|
|
179
179
|
name: "creditCard",
|
|
180
180
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
181
181
|
}
|
|
182
|
-
],
|
|
182
|
+
], Ae = (e, t, n) => {
|
|
183
183
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
184
184
|
for (let [a, o] of Object.entries(n)) {
|
|
185
185
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -187,7 +187,7 @@ var me = (e) => {
|
|
|
187
187
|
for (let a of e) {
|
|
188
188
|
let e = a[n];
|
|
189
189
|
if (typeof e != "string" || e.length === 0) continue;
|
|
190
|
-
let o =
|
|
190
|
+
let o = ke.find((t) => t.re.test(e));
|
|
191
191
|
if (o && !i.has(n)) {
|
|
192
192
|
i.add(n), r.push({
|
|
193
193
|
table: t,
|
|
@@ -200,9 +200,9 @@ var me = (e) => {
|
|
|
200
200
|
}
|
|
201
201
|
}
|
|
202
202
|
return r;
|
|
203
|
-
},
|
|
203
|
+
}, je = (e, t, n, r, i) => {
|
|
204
204
|
let a = {};
|
|
205
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
205
|
+
for (let [o, s] of Object.entries(e)) a[o] = Ee(n[`${t}.${o}`] ?? "keep", {
|
|
206
206
|
value: s,
|
|
207
207
|
table: t,
|
|
208
208
|
column: o,
|
|
@@ -210,56 +210,61 @@ var me = (e) => {
|
|
|
210
210
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
211
211
|
});
|
|
212
212
|
return a;
|
|
213
|
-
},
|
|
214
|
-
reason:
|
|
215
|
-
path:
|
|
216
|
-
}) {},
|
|
217
|
-
table:
|
|
218
|
-
reason:
|
|
219
|
-
}) {},
|
|
220
|
-
what:
|
|
221
|
-
expected:
|
|
222
|
-
actual:
|
|
223
|
-
}) {},
|
|
224
|
-
rows:
|
|
225
|
-
table:
|
|
226
|
-
id:
|
|
227
|
-
reason:
|
|
228
|
-
derived:
|
|
213
|
+
}, L = class extends c.TaggedError()("BundleError", {
|
|
214
|
+
reason: c.String,
|
|
215
|
+
path: c.optional(c.String)
|
|
216
|
+
}) {}, Me = class extends c.TaggedError()("CodecError", {
|
|
217
|
+
table: c.String,
|
|
218
|
+
reason: c.String
|
|
219
|
+
}) {}, R = class extends c.TaggedError()("IntegrityError", {
|
|
220
|
+
what: c.String,
|
|
221
|
+
expected: c.String,
|
|
222
|
+
actual: c.String
|
|
223
|
+
}) {}, Ne = class extends t.TaggedError("CompressionError") {}, z = class extends c.TaggedError()("ScopeError", { reason: c.String }) {}, Pe = class extends c.TaggedError()("RowsRefusedError", {
|
|
224
|
+
rows: c.Array(c.Struct({
|
|
225
|
+
table: c.String,
|
|
226
|
+
id: c.String,
|
|
227
|
+
reason: c.String,
|
|
228
|
+
derived: c.Boolean
|
|
229
229
|
})),
|
|
230
|
-
primaryCount:
|
|
231
|
-
totalCount:
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
})
|
|
230
|
+
primaryCount: c.Number,
|
|
231
|
+
totalCount: c.Number,
|
|
232
|
+
byTable: c.Array(c.Struct({
|
|
233
|
+
table: c.String,
|
|
234
|
+
refused: c.Number,
|
|
235
|
+
primary: c.Number
|
|
236
|
+
}))
|
|
237
|
+
}) {}, Fe = class extends c.TaggedError()("SchemaDriftError", {
|
|
238
|
+
bundleFingerprint: c.String,
|
|
239
|
+
targetFingerprint: c.String,
|
|
240
|
+
diff: c.Array(c.String)
|
|
241
|
+
}) {}, B = class extends t.TaggedError("NativeToolError") {
|
|
237
242
|
get message() {
|
|
238
243
|
let e = this.stderr?.trim();
|
|
239
244
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
240
245
|
}
|
|
241
|
-
},
|
|
242
|
-
unclassified:
|
|
243
|
-
invalidActions:
|
|
244
|
-
}) {},
|
|
245
|
-
mode:
|
|
246
|
-
reason:
|
|
247
|
-
}) {},
|
|
248
|
-
sourceDialect:
|
|
249
|
-
targetDialect:
|
|
250
|
-
blocking:
|
|
251
|
-
}) {},
|
|
246
|
+
}, Ie = class extends c.TaggedError()("MaskingError", {
|
|
247
|
+
unclassified: c.Array(c.String),
|
|
248
|
+
invalidActions: c.optional(c.Array(c.String))
|
|
249
|
+
}) {}, Le = class extends c.TaggedError()("ImportModeError", {
|
|
250
|
+
mode: c.String,
|
|
251
|
+
reason: c.String
|
|
252
|
+
}) {}, Re = class extends c.TaggedError()("CrossDialectError", {
|
|
253
|
+
sourceDialect: c.String,
|
|
254
|
+
targetDialect: c.String,
|
|
255
|
+
blocking: c.Array(c.String)
|
|
256
|
+
}) {}, V = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", ze = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
252
257
|
col: e.name,
|
|
253
258
|
refTable: e.references.table
|
|
254
|
-
})),
|
|
255
|
-
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(),
|
|
259
|
+
})), Be = (e, t, r) => n.gen(function* () {
|
|
260
|
+
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), u = (e) => {
|
|
256
261
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
257
262
|
return s.set(e, t), t;
|
|
258
263
|
}, d = (e, t) => {
|
|
259
264
|
if (t == null || a.has(e) || s.get(e)?.has(t)) return;
|
|
260
|
-
let n =
|
|
261
|
-
n.add(t),
|
|
262
|
-
},
|
|
265
|
+
let n = c.get(e) ?? /* @__PURE__ */ new Set();
|
|
266
|
+
n.add(t), c.set(e, n);
|
|
267
|
+
}, f = (e, t, n) => {
|
|
263
268
|
let r = u(e), i = a.has(e) ? void 0 : o.get(e) ?? (() => {
|
|
264
269
|
let t = /* @__PURE__ */ new Set();
|
|
265
270
|
return o.set(e, t), t;
|
|
@@ -268,24 +273,24 @@ var me = (e) => {
|
|
|
268
273
|
r.add(e[t]), i?.add(e[t]);
|
|
269
274
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
270
275
|
};
|
|
271
|
-
},
|
|
272
|
-
let a =
|
|
273
|
-
return
|
|
276
|
+
}, m = (r, i) => {
|
|
277
|
+
let a = f(r, V(t, r), ze(t, r));
|
|
278
|
+
return l.runForEach(g(e, {
|
|
274
279
|
table: r,
|
|
275
|
-
pk:
|
|
280
|
+
pk: V(t, r),
|
|
276
281
|
...i === void 0 ? {} : { where: i }
|
|
277
282
|
}), (e) => n.sync(() => a(e)));
|
|
278
283
|
}, _ = n.gen(function* () {
|
|
279
|
-
for (let e = 0; e < i &&
|
|
280
|
-
let e = [...
|
|
281
|
-
|
|
284
|
+
for (let e = 0; e < i && c.size > 0; e++) {
|
|
285
|
+
let e = [...c.entries()];
|
|
286
|
+
c.clear();
|
|
282
287
|
for (let [n, r] of e) {
|
|
283
288
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
284
|
-
e.length !== 0 && (yield*
|
|
289
|
+
e.length !== 0 && (yield* m(n, p(V(t, n), e)));
|
|
285
290
|
}
|
|
286
291
|
}
|
|
287
292
|
});
|
|
288
|
-
for (let [e, t] of Object.entries(r.seeds)) yield*
|
|
293
|
+
for (let [e, t] of Object.entries(r.seeds)) yield* m(e, t);
|
|
289
294
|
if (yield* _, r.children) {
|
|
290
295
|
let e = new Set(r.children === !0 ? [...s.keys()] : r.children.roots.filter((e) => s.has(e)));
|
|
291
296
|
for (let n = 0; n < i; n++) {
|
|
@@ -296,21 +301,21 @@ var me = (e) => {
|
|
|
296
301
|
refTable: e.references?.table
|
|
297
302
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
298
303
|
if (a.length === 0) continue;
|
|
299
|
-
let o = s.get(t)?.size ?? 0, c = a.map((e) =>
|
|
300
|
-
yield*
|
|
304
|
+
let o = s.get(t)?.size ?? 0, c = a.map((e) => p(e.col, [...s.get(e.refTable)]));
|
|
305
|
+
yield* m(t, c.length === 1 ? c[0] : h(...c)), (s.get(t)?.size ?? 0) > o && !e.has(t) && r.push(t), (s.get(t)?.size ?? 0) > o && (n = !0);
|
|
301
306
|
}
|
|
302
307
|
for (let t of r) e.add(t);
|
|
303
308
|
if (yield* _, !n) break;
|
|
304
309
|
}
|
|
305
310
|
}
|
|
306
311
|
return o;
|
|
307
|
-
}),
|
|
312
|
+
}), Ve = (e, t) => ({
|
|
308
313
|
tables: [...e.keys()],
|
|
309
314
|
predicateFor: (n) => {
|
|
310
315
|
let r = e.get(n);
|
|
311
|
-
return r ?
|
|
316
|
+
return r ? p(V(t, n), [...r]) : void 0;
|
|
312
317
|
}
|
|
313
|
-
}),
|
|
318
|
+
}), H = "tenantId", He = (e, t, r) => n.gen(function* () {
|
|
314
319
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
315
320
|
if (e.kind === "all") return {
|
|
316
321
|
tables: i,
|
|
@@ -318,7 +323,7 @@ var me = (e) => {
|
|
|
318
323
|
};
|
|
319
324
|
if (e.kind === "tables") {
|
|
320
325
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
321
|
-
if (a.length > 0) return yield* n.fail(new
|
|
326
|
+
if (a.length > 0) return yield* n.fail(new z({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
322
327
|
let o = new Set(t);
|
|
323
328
|
return {
|
|
324
329
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -326,30 +331,30 @@ var me = (e) => {
|
|
|
326
331
|
};
|
|
327
332
|
}
|
|
328
333
|
let o = e.tenantId;
|
|
329
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
330
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
331
|
-
if (s.length === 0) return yield* n.fail(new
|
|
332
|
-
let c = yield*
|
|
333
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
334
|
+
if (o === void 0 || o === "") return yield* n.fail(new z({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
335
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === H) ?? !1);
|
|
336
|
+
if (s.length === 0) return yield* n.fail(new z({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${H}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
337
|
+
let c = yield* Be(r.store, t, {
|
|
338
|
+
seeds: Object.fromEntries(s.map((e) => [e, d(H, o)])),
|
|
334
339
|
pinned: s,
|
|
335
340
|
children: { roots: s }
|
|
336
|
-
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e),
|
|
341
|
+
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
337
342
|
return {
|
|
338
343
|
tables: [...s, ...l],
|
|
339
|
-
predicateFor: (e) =>
|
|
344
|
+
predicateFor: (e) => u.has(e) ? d(H, o) : c.has(e) ? p(V(t, e), [...c.get(e)]) : void 0
|
|
340
345
|
};
|
|
341
|
-
}),
|
|
346
|
+
}), Ue = (e, t) => ({
|
|
342
347
|
list: t,
|
|
343
|
-
open: (t) =>
|
|
344
|
-
}),
|
|
345
|
-
write: (t, n, r) =>
|
|
348
|
+
open: (t) => E(e, t)
|
|
349
|
+
}), We = (e) => ({
|
|
350
|
+
write: (t, n, r) => D(e, t, n, r),
|
|
346
351
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
347
|
-
}),
|
|
352
|
+
}), Ge = (e, t) => n.tryPromise({
|
|
348
353
|
try: async () => {
|
|
349
354
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
350
355
|
u.update(e), d += e.length, n(null, e);
|
|
351
356
|
} });
|
|
352
|
-
await o(await
|
|
357
|
+
await o(await O(e.stream), f, n(l));
|
|
353
358
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
354
359
|
try {
|
|
355
360
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -362,64 +367,64 @@ var me = (e) => {
|
|
|
362
367
|
contentType: e.contentType
|
|
363
368
|
};
|
|
364
369
|
},
|
|
365
|
-
catch: (e) => new
|
|
370
|
+
catch: (e) => new L({
|
|
366
371
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
367
372
|
path: t
|
|
368
373
|
})
|
|
369
|
-
}),
|
|
370
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256),
|
|
374
|
+
}), Ke = (e, t, r) => n.gen(function* () {
|
|
375
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u = T("bundle", () => a(s)).pipe(l.tap((e) => n.sync(() => c.update(e))));
|
|
371
376
|
yield* e.write(r.key, u, {
|
|
372
377
|
contentType: r.contentType,
|
|
373
378
|
totalSize: r.size
|
|
374
379
|
});
|
|
375
|
-
let d =
|
|
376
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
380
|
+
let d = c.digest("hex");
|
|
381
|
+
if (d !== r.sha256) return yield* n.fail(new R({
|
|
377
382
|
what: `asset ${r.key}`,
|
|
378
383
|
expected: r.sha256,
|
|
379
384
|
actual: d
|
|
380
385
|
}));
|
|
381
|
-
}),
|
|
382
|
-
name:
|
|
383
|
-
file:
|
|
384
|
-
rowCount:
|
|
385
|
-
checksum:
|
|
386
|
-
columnTypes:
|
|
387
|
-
key:
|
|
388
|
-
value:
|
|
386
|
+
}), qe = () => typeof k.createZstdCompress == "function", Je = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : qe() ? "zstd" : "gzip", Ye = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", Xe = (e) => e === "zstd" ? k.createZstdCompress() : e === "gzip" ? k.createGzip() : new A(), Ze = (e) => e === "zstd" ? k.createZstdDecompress() : e === "gzip" ? k.createGunzip() : new A(), Qe = 1, $e = c.Struct({
|
|
387
|
+
name: c.String,
|
|
388
|
+
file: c.String,
|
|
389
|
+
rowCount: c.Number,
|
|
390
|
+
checksum: c.String,
|
|
391
|
+
columnTypes: c.Record({
|
|
392
|
+
key: c.String,
|
|
393
|
+
value: c.String
|
|
389
394
|
}),
|
|
390
|
-
primaryKey:
|
|
391
|
-
}),
|
|
392
|
-
kind:
|
|
393
|
-
tenantId:
|
|
394
|
-
tables:
|
|
395
|
-
}),
|
|
396
|
-
count:
|
|
397
|
-
totalBytes:
|
|
398
|
-
index:
|
|
395
|
+
primaryKey: c.String
|
|
396
|
+
}), et = c.Struct({
|
|
397
|
+
kind: c.Literal("all", "tenant", "tables"),
|
|
398
|
+
tenantId: c.optional(c.String),
|
|
399
|
+
tables: c.optional(c.Array(c.String))
|
|
400
|
+
}), tt = c.Struct({
|
|
401
|
+
count: c.Number,
|
|
402
|
+
totalBytes: c.Number,
|
|
403
|
+
index: c.optional(c.String)
|
|
399
404
|
});
|
|
400
|
-
|
|
401
|
-
key:
|
|
402
|
-
sha256:
|
|
403
|
-
size:
|
|
404
|
-
contentType:
|
|
405
|
+
c.Struct({
|
|
406
|
+
key: c.String,
|
|
407
|
+
sha256: c.String,
|
|
408
|
+
size: c.Number,
|
|
409
|
+
contentType: c.String
|
|
405
410
|
});
|
|
406
|
-
var
|
|
407
|
-
formatVersion:
|
|
408
|
-
createdAt:
|
|
409
|
-
source:
|
|
410
|
-
dialect:
|
|
411
|
-
schemaFingerprint:
|
|
411
|
+
var nt = c.Struct({
|
|
412
|
+
formatVersion: c.Number,
|
|
413
|
+
createdAt: c.String,
|
|
414
|
+
source: c.Struct({
|
|
415
|
+
dialect: c.String,
|
|
416
|
+
schemaFingerprint: c.String
|
|
412
417
|
}),
|
|
413
|
-
scope:
|
|
414
|
-
consistency:
|
|
415
|
-
compression:
|
|
416
|
-
tables:
|
|
417
|
-
assets:
|
|
418
|
-
masking:
|
|
419
|
-
policyId:
|
|
420
|
-
transformed:
|
|
418
|
+
scope: et,
|
|
419
|
+
consistency: c.Literal("live", "snapshot"),
|
|
420
|
+
compression: c.Literal("zstd", "gzip", "none"),
|
|
421
|
+
tables: c.Array($e),
|
|
422
|
+
assets: tt,
|
|
423
|
+
masking: c.optional(c.Struct({
|
|
424
|
+
policyId: c.optional(c.String),
|
|
425
|
+
transformed: c.Array(c.String)
|
|
421
426
|
}))
|
|
422
|
-
}),
|
|
427
|
+
}), rt = c.encodeSync(c.parseJson(nt)), it = c.decodeUnknownSync(c.parseJson(nt)), U = "manifest.json", at = "README.md", ot = "data", W = "assets", st = "assets/index.ndjson", ct = ".ledger.json", lt = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", ut = (e, t) => {
|
|
423
428
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
424
429
|
return `# Voltro data export
|
|
425
430
|
|
|
@@ -433,7 +438,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
433
438
|
| **Format version** | ${e.formatVersion} |
|
|
434
439
|
| **Consistency** | ${e.consistency} |
|
|
435
440
|
| **Compression** | ${e.compression} |
|
|
436
|
-
| **Scope** | ${
|
|
441
|
+
| **Scope** | ${lt(e.scope)} |
|
|
437
442
|
| **Tables** | ${e.tables.length} |
|
|
438
443
|
| **Rows (total)** | ${n} |
|
|
439
444
|
| **Assets** | ${a} |
|
|
@@ -458,39 +463,39 @@ voltro data import <this-directory>
|
|
|
458
463
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
459
464
|
\`\`\`
|
|
460
465
|
`;
|
|
461
|
-
},
|
|
466
|
+
}, dt = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), ft = a.exponential({
|
|
462
467
|
start: .01,
|
|
463
468
|
factor: 2,
|
|
464
469
|
count: 16
|
|
465
|
-
}),
|
|
470
|
+
}), pt = i.histogram("voltro_data_transfer_table_seconds", ft, "Per-table export/import duration in seconds, tagged by phase."), G = (e, t) => {
|
|
466
471
|
if (e) try {
|
|
467
472
|
e(t);
|
|
468
473
|
} catch {}
|
|
469
|
-
},
|
|
474
|
+
}, mt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(pt.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), dt.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
470
475
|
table: t,
|
|
471
476
|
index: r,
|
|
472
477
|
tableCount: a,
|
|
473
478
|
phase: e
|
|
474
|
-
} })),
|
|
479
|
+
} })), ht = () => ({
|
|
475
480
|
tables: {},
|
|
476
481
|
assets: {}
|
|
477
|
-
}),
|
|
482
|
+
}), gt = (e) => n.tryPromise(async () => {
|
|
478
483
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
479
484
|
return JSON.parse(t);
|
|
480
|
-
}).pipe(n.orElseSucceed(
|
|
485
|
+
}).pipe(n.orElseSucceed(ht)), K = (e, t) => n.tryPromise({
|
|
481
486
|
try: async () => {
|
|
482
487
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
483
488
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
484
489
|
},
|
|
485
|
-
catch: (t) => new
|
|
490
|
+
catch: (t) => new L({
|
|
486
491
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
487
492
|
path: e
|
|
488
493
|
})
|
|
489
|
-
}),
|
|
494
|
+
}), _t = (e, t) => t in e.tables, vt = (e, t) => t in e.assets, yt = /* @__PURE__ */ new Set([
|
|
490
495
|
"json",
|
|
491
496
|
"array",
|
|
492
497
|
"vector"
|
|
493
|
-
]),
|
|
498
|
+
]), bt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", xt = (e) => Buffer.from(e).toString("base64"), St = (e) => new Uint8Array(Buffer.from(e, "base64")), Ct = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), wt = (e, t) => e == null ? null : t !== void 0 && yt.has(t) ? e : t === "bytes" ? xt(e) : t === "timestamp" || t === "date" ? Ct(e) : typeof e == "bigint" ? {
|
|
494
499
|
$v: "bigint",
|
|
495
500
|
d: e.toString()
|
|
496
501
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -501,30 +506,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
501
506
|
d: e.toISOString()
|
|
502
507
|
} : e instanceof Uint8Array ? {
|
|
503
508
|
$v: "bytes",
|
|
504
|
-
d:
|
|
505
|
-
} : e,
|
|
509
|
+
d: xt(e)
|
|
510
|
+
} : e, Tt = (e) => {
|
|
506
511
|
switch (e.$v) {
|
|
507
512
|
case "bigint": return BigInt(e.d);
|
|
508
513
|
case "number": return Number(e.d);
|
|
509
514
|
case "date": return new Date(e.d);
|
|
510
|
-
case "bytes": return
|
|
515
|
+
case "bytes": return St(e.d);
|
|
511
516
|
default: return e;
|
|
512
517
|
}
|
|
513
|
-
},
|
|
518
|
+
}, Et = (e, t) => e == null ? null : t !== void 0 && yt.has(t) ? e : bt(e) ? Tt(e) : t === "bytes" ? St(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Dt = (e, t) => {
|
|
514
519
|
let n = {};
|
|
515
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
520
|
+
for (let [r, i] of Object.entries(e)) n[r] = wt(i, t[r]);
|
|
516
521
|
return n;
|
|
517
|
-
},
|
|
522
|
+
}, Ot = (e, t) => {
|
|
518
523
|
let n = {};
|
|
519
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
524
|
+
for (let [r, i] of Object.entries(e)) n[r] = Et(i, t[r]);
|
|
520
525
|
return n;
|
|
521
|
-
},
|
|
522
|
-
try: () =>
|
|
523
|
-
catch: (e) => new
|
|
526
|
+
}, kt = new TextEncoder(), At = (e, t) => e.pipe(l.map((e) => kt.encode(`${JSON.stringify(Dt(e, t))}\n`))), jt = (e, t, r) => e.pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.mapEffect((e) => n.try({
|
|
527
|
+
try: () => Ot(JSON.parse(e), r),
|
|
528
|
+
catch: (e) => new Me({
|
|
524
529
|
table: t,
|
|
525
530
|
reason: String(e?.message ?? e)
|
|
526
531
|
})
|
|
527
|
-
}))),
|
|
532
|
+
}))), Mt = (e, t) => {
|
|
528
533
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
529
534
|
for (let e of t.tables) {
|
|
530
535
|
if (!n.has(e.name)) continue;
|
|
@@ -544,48 +549,48 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
544
549
|
};
|
|
545
550
|
for (let t of e) n.has(t) && s(t);
|
|
546
551
|
return i;
|
|
547
|
-
},
|
|
552
|
+
}, q = (e) => {
|
|
548
553
|
let t = e, n = [];
|
|
549
554
|
typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
|
|
550
555
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
551
|
-
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(
|
|
556
|
+
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(q(t.cause)) : n.push(r), n.length === 0) {
|
|
552
557
|
let t = String(e ?? "").trim();
|
|
553
|
-
return t === "" || t === "[object Object]" ?
|
|
558
|
+
return t === "" || t === "[object Object]" ? Nt(e) : t;
|
|
554
559
|
}
|
|
555
560
|
return n.join(": ");
|
|
556
|
-
},
|
|
561
|
+
}, Nt = (e) => {
|
|
557
562
|
try {
|
|
558
563
|
let t = JSON.stringify(e);
|
|
559
564
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
560
565
|
} catch {
|
|
561
566
|
return "unknown error (not serialisable)";
|
|
562
567
|
}
|
|
563
|
-
},
|
|
568
|
+
}, Pt = (e, t) => {
|
|
564
569
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
565
570
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
566
571
|
return r;
|
|
567
|
-
},
|
|
572
|
+
}, Ft = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), It = (e, t) => {
|
|
568
573
|
if (t.length === 0) return e;
|
|
569
574
|
let n = { ...e };
|
|
570
575
|
for (let e of t) delete n[e];
|
|
571
576
|
return n;
|
|
572
|
-
},
|
|
577
|
+
}, Lt = (e, t) => {
|
|
573
578
|
let n = e.tables.find((e) => e.name === t);
|
|
574
579
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
575
580
|
let r = n.columns.find((e) => e.type === "id");
|
|
576
581
|
if (r !== void 0) return { column: r.name };
|
|
577
582
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
578
583
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
579
|
-
},
|
|
584
|
+
}, Rt = (e, t, r) => n.tryPromise({
|
|
580
585
|
try: async () => {
|
|
581
|
-
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable:
|
|
586
|
+
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
582
587
|
d.update(e);
|
|
583
588
|
let r = e;
|
|
584
589
|
for (let e = 0; e < r.length; e++) r[e] === 10 && f++;
|
|
585
590
|
n(null, e);
|
|
586
|
-
} }), m =
|
|
591
|
+
} }), m = c.fromWeb(l.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
587
592
|
try {
|
|
588
|
-
await s(m, p,
|
|
593
|
+
await s(m, p, Xe(r), n(h)), await i.rename(h, t);
|
|
589
594
|
} catch (e) {
|
|
590
595
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
591
596
|
}
|
|
@@ -594,27 +599,27 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
594
599
|
checksum: d.digest("hex")
|
|
595
600
|
};
|
|
596
601
|
},
|
|
597
|
-
catch: (e) => new
|
|
598
|
-
reason: `write table failed: ${
|
|
602
|
+
catch: (e) => new L({
|
|
603
|
+
reason: `write table failed: ${q(e)}`,
|
|
599
604
|
path: t
|
|
600
605
|
})
|
|
601
|
-
}),
|
|
606
|
+
}), zt = (e, t) => n.tryPromise({
|
|
602
607
|
try: async () => {
|
|
603
608
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
604
609
|
},
|
|
605
|
-
catch: (t) => new
|
|
606
|
-
reason: `append failed: ${
|
|
610
|
+
catch: (t) => new L({
|
|
611
|
+
reason: `append failed: ${q(t)}`,
|
|
607
612
|
path: e
|
|
608
613
|
})
|
|
609
|
-
}),
|
|
614
|
+
}), Bt = (e, t) => n.gen(function* () {
|
|
610
615
|
let r = [], i = t.ordered.length;
|
|
611
616
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
612
|
-
let o = t.ordered[a], s =
|
|
613
|
-
if ("refusal" in
|
|
614
|
-
let u =
|
|
615
|
-
if (
|
|
617
|
+
let o = t.ordered[a], s = Pt(t.snapshot, o), c = Lt(t.snapshot, o);
|
|
618
|
+
if ("refusal" in c) return yield* n.fail(new z({ reason: `export: ${c.refusal}` }));
|
|
619
|
+
let u = c.column, d = `${ot}/${o}.ndjson${Ye(t.compression)}`;
|
|
620
|
+
if (_t(t.ledger, o)) {
|
|
616
621
|
let e = t.ledger.tables[o];
|
|
617
|
-
|
|
622
|
+
G(t.onProgress, {
|
|
618
623
|
phase: "export",
|
|
619
624
|
table: o,
|
|
620
625
|
index: a,
|
|
@@ -632,7 +637,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
632
637
|
});
|
|
633
638
|
continue;
|
|
634
639
|
}
|
|
635
|
-
|
|
640
|
+
G(t.onProgress, {
|
|
636
641
|
phase: "export",
|
|
637
642
|
table: o,
|
|
638
643
|
index: a,
|
|
@@ -640,17 +645,17 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
640
645
|
event: "start",
|
|
641
646
|
rowsDone: 0
|
|
642
647
|
});
|
|
643
|
-
let f = t.resolved.predicateFor(o), p =
|
|
648
|
+
let f = t.resolved.predicateFor(o), p = g(e, {
|
|
644
649
|
table: o,
|
|
645
650
|
pk: u,
|
|
646
651
|
chunkSize: t.chunkSize,
|
|
647
652
|
retry: t.retry,
|
|
648
653
|
...f === void 0 ? {} : { where: f }
|
|
649
|
-
}), m =
|
|
654
|
+
}), m = Ft(t.snapshot, o), h = m.length === 0 ? p : p.pipe(l.map((e) => It(e, m))), _ = t.mask ? h.pipe(l.map((e) => je(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: v, checksum: y } = yield* mt("export", o, a, i, (e) => e.rowCount, Rt(At(_, s), t.bundleFile(d), t.compression));
|
|
650
655
|
t.ledger.tables[o] = {
|
|
651
656
|
rowCount: v,
|
|
652
657
|
checksum: y
|
|
653
|
-
}, yield*
|
|
658
|
+
}, yield* K(t.ledgerPath, t.ledger), G(t.onProgress, {
|
|
654
659
|
phase: "export",
|
|
655
660
|
table: o,
|
|
656
661
|
index: a,
|
|
@@ -668,43 +673,43 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
668
673
|
});
|
|
669
674
|
}
|
|
670
675
|
return r;
|
|
671
|
-
}),
|
|
676
|
+
}), Vt = (e) => ({
|
|
672
677
|
_tag: "RawSqlFragment",
|
|
673
678
|
strings: [e],
|
|
674
679
|
values: []
|
|
675
|
-
}),
|
|
680
|
+
}), Ht = (t, i, a) => n.tryPromise({
|
|
676
681
|
try: () => t.transactional(async (t) => {
|
|
677
|
-
if (t.raw) for (let e of me(i)) await t.raw(
|
|
682
|
+
if (t.raw) for (let e of me(i)) await t.raw(Vt(e));
|
|
678
683
|
let o = await n.runPromiseExit(a(t));
|
|
679
684
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
680
685
|
return o.value;
|
|
681
686
|
}),
|
|
682
|
-
catch: (e) => e instanceof
|
|
683
|
-
}),
|
|
684
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
687
|
+
catch: (e) => e instanceof L ? e : new L({ reason: `snapshot export failed: ${q(e)}` })
|
|
688
|
+
}), Ut = (e) => n.gen(function* () {
|
|
689
|
+
let t = e.scope ?? { kind: "all" }, r = Je(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, o = s.exponential("50 millis").pipe(s.jittered, s.intersect(s.recurs(a))), c = yield* n.promise(() => import("node:path")), u = (t) => c.join(e.outDir, t);
|
|
685
690
|
yield* n.tryPromise({
|
|
686
691
|
try: async () => {
|
|
687
692
|
let t = await import("node:fs/promises");
|
|
688
|
-
await t.mkdir(u(
|
|
693
|
+
await t.mkdir(u(ot), { recursive: !0 }), e.assets && await t.mkdir(u(W), { recursive: !0 });
|
|
689
694
|
},
|
|
690
|
-
catch: (t) => new
|
|
691
|
-
reason: `mkdir failed: ${
|
|
695
|
+
catch: (t) => new L({
|
|
696
|
+
reason: `mkdir failed: ${q(t)}`,
|
|
692
697
|
path: e.outDir
|
|
693
698
|
})
|
|
694
699
|
});
|
|
695
|
-
let d = u(
|
|
700
|
+
let d = u(ct), f = yield* gt(d), p = e.subset ? Ve(yield* Be(e.store, e.snapshot, e.subset), e.snapshot) : yield* He(t, e.snapshot, {
|
|
696
701
|
store: e.store,
|
|
697
702
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
698
|
-
}), m =
|
|
703
|
+
}), m = Mt(p.tables, e.snapshot), h = e.consistency ?? "live", g, v;
|
|
699
704
|
if (e.masking) {
|
|
700
|
-
let t = e.classification ??
|
|
705
|
+
let t = e.classification ?? ge(e.snapshot), r = Oe(m.map((t) => ({
|
|
701
706
|
name: t,
|
|
702
707
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
703
708
|
name: e.name,
|
|
704
709
|
type: e.type
|
|
705
710
|
}))
|
|
706
711
|
})), t, e.masking);
|
|
707
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
712
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new Ie({
|
|
708
713
|
unclassified: r.unclassified,
|
|
709
714
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
710
715
|
}));
|
|
@@ -713,7 +718,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
713
718
|
seed: e.masking.seed
|
|
714
719
|
};
|
|
715
720
|
let i = Object.entries(r.actions).filter(([, e]) => e !== "keep").map(([e]) => e);
|
|
716
|
-
|
|
721
|
+
g = {
|
|
717
722
|
...e.masking.id ? { policyId: e.masking.id } : {},
|
|
718
723
|
transformed: i
|
|
719
724
|
};
|
|
@@ -724,31 +729,31 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
724
729
|
resolved: p,
|
|
725
730
|
compression: r,
|
|
726
731
|
chunkSize: i,
|
|
727
|
-
retry:
|
|
732
|
+
retry: o,
|
|
728
733
|
ledger: f,
|
|
729
734
|
ledgerPath: d,
|
|
730
735
|
bundleFile: u,
|
|
731
736
|
...v ? { mask: v } : {},
|
|
732
737
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
733
|
-
}, b = h === "snapshot" ? yield*
|
|
738
|
+
}, b = h === "snapshot" ? yield* Ht(e.store, e.dialect, (e) => Bt(e, y)) : yield* Bt(e.store, y), x = 0, S = 0;
|
|
734
739
|
if (e.assets) {
|
|
735
|
-
let t = u(
|
|
736
|
-
yield*
|
|
740
|
+
let t = u(st);
|
|
741
|
+
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => Ge(e, u(W))), n.flatMap((e) => {
|
|
737
742
|
let i = {
|
|
738
743
|
key: r.key,
|
|
739
744
|
sha256: e.sha256,
|
|
740
745
|
size: e.size,
|
|
741
746
|
contentType: e.contentType
|
|
742
747
|
};
|
|
743
|
-
return x++, S += e.size, f.assets[r.key] = !0,
|
|
744
|
-
}))).pipe(n.mapError((e) => e instanceof
|
|
748
|
+
return x++, S += e.size, f.assets[r.key] = !0, zt(t, JSON.stringify(i)).pipe(n.andThen(K(d, f)));
|
|
749
|
+
}))).pipe(n.mapError((e) => e instanceof L ? e : new L({ reason: `asset export failed: ${String(e)}` })));
|
|
745
750
|
}
|
|
746
751
|
let C = {
|
|
747
752
|
formatVersion: 1,
|
|
748
753
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
749
754
|
source: {
|
|
750
755
|
dialect: e.dialect,
|
|
751
|
-
schemaFingerprint:
|
|
756
|
+
schemaFingerprint: _(e.snapshot)
|
|
752
757
|
},
|
|
753
758
|
scope: t,
|
|
754
759
|
consistency: h,
|
|
@@ -757,32 +762,32 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
757
762
|
assets: x > 0 ? {
|
|
758
763
|
count: x,
|
|
759
764
|
totalBytes: S,
|
|
760
|
-
index:
|
|
765
|
+
index: st
|
|
761
766
|
} : {
|
|
762
767
|
count: 0,
|
|
763
768
|
totalBytes: 0
|
|
764
769
|
},
|
|
765
|
-
...
|
|
770
|
+
...g ? { masking: g } : {}
|
|
766
771
|
};
|
|
767
772
|
return yield* n.tryPromise({
|
|
768
773
|
try: async () => {
|
|
769
|
-
await (await import("node:fs/promises")).writeFile(u(
|
|
774
|
+
await (await import("node:fs/promises")).writeFile(u(U), rt(C));
|
|
770
775
|
},
|
|
771
|
-
catch: (e) => new
|
|
772
|
-
reason: `manifest write failed: ${
|
|
773
|
-
path:
|
|
776
|
+
catch: (e) => new L({
|
|
777
|
+
reason: `manifest write failed: ${q(e)}`,
|
|
778
|
+
path: U
|
|
774
779
|
})
|
|
775
780
|
}), yield* n.tryPromise({
|
|
776
781
|
try: async () => {
|
|
777
782
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
778
|
-
await e.writeFile(u(
|
|
783
|
+
await e.writeFile(u(at), ut(C, { hostname: t.hostname() }));
|
|
779
784
|
},
|
|
780
|
-
catch: (e) => new
|
|
781
|
-
reason: `readme write failed: ${
|
|
782
|
-
path:
|
|
785
|
+
catch: (e) => new L({
|
|
786
|
+
reason: `readme write failed: ${q(e)}`,
|
|
787
|
+
path: at
|
|
783
788
|
})
|
|
784
789
|
}), C;
|
|
785
|
-
}),
|
|
790
|
+
}), Wt = (e, t) => {
|
|
786
791
|
let n = new Set(t), r = [];
|
|
787
792
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
788
793
|
let i = e.references?.table;
|
|
@@ -793,7 +798,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
793
798
|
});
|
|
794
799
|
}
|
|
795
800
|
return r;
|
|
796
|
-
},
|
|
801
|
+
}, Gt = (e) => {
|
|
797
802
|
let t = /* @__PURE__ */ new Map();
|
|
798
803
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
799
804
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -801,7 +806,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
801
806
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
802
807
|
});
|
|
803
808
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
804
|
-
},
|
|
809
|
+
}, Kt = (e) => {
|
|
805
810
|
switch (e) {
|
|
806
811
|
case "id":
|
|
807
812
|
case "text":
|
|
@@ -820,12 +825,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
820
825
|
case "raw": return "raw";
|
|
821
826
|
default: return "string";
|
|
822
827
|
}
|
|
823
|
-
},
|
|
828
|
+
}, qt = {
|
|
824
829
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
825
830
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
826
831
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
827
|
-
},
|
|
828
|
-
let i =
|
|
832
|
+
}, Jt = (e, t, n, r) => {
|
|
833
|
+
let i = Kt(n);
|
|
829
834
|
if (i === "raw") return {
|
|
830
835
|
table: e,
|
|
831
836
|
column: t,
|
|
@@ -833,7 +838,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
833
838
|
severity: "block",
|
|
834
839
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
835
840
|
};
|
|
836
|
-
let a =
|
|
841
|
+
let a = qt[i];
|
|
837
842
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
838
843
|
table: e,
|
|
839
844
|
column: t,
|
|
@@ -847,15 +852,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
847
852
|
severity: "block",
|
|
848
853
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
849
854
|
};
|
|
850
|
-
},
|
|
855
|
+
}, Yt = (e, t, n) => {
|
|
851
856
|
if (t === n) return [];
|
|
852
857
|
let r = [];
|
|
853
858
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
854
|
-
let a =
|
|
859
|
+
let a = Jt(t.name, e, i, n);
|
|
855
860
|
a && r.push(a);
|
|
856
861
|
}
|
|
857
862
|
return r;
|
|
858
|
-
},
|
|
863
|
+
}, Xt = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, Zt = (e) => {
|
|
859
864
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
860
865
|
switch (e.kind) {
|
|
861
866
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -864,29 +869,37 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
864
869
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
865
870
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
866
871
|
}
|
|
867
|
-
},
|
|
872
|
+
}, Qt = (e) => {
|
|
868
873
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
869
874
|
return t.length > 0 ? t.join("/") : void 0;
|
|
870
|
-
},
|
|
871
|
-
let t =
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
875
|
+
}, J = "Failed to execute statement", $t = (e) => {
|
|
876
|
+
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
877
|
+
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
878
|
+
return t === `SqlError: ${J}` ? J : t;
|
|
879
|
+
}, en = (e) => {
|
|
880
|
+
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
881
|
+
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
882
|
+
unclassified: t,
|
|
883
|
+
raw: n
|
|
884
|
+
}), i = f(e), a = u(i), o = Qt(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
885
|
+
if (a !== void 0) return r(`${Zt(a)}${s}`, !1);
|
|
886
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, d = $t(typeof l == "string" && l.length > 0 ? l : String(e)), p = c !== void 0 && c !== d && !c.startsWith("SqlError:") ? c : d, m = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, h = m !== void 0 && /[←‖]/.test(m) ? ` (via ${m})` : "";
|
|
887
|
+
return r(p === J || p.length === 0 ? `the database refused the write and the driver gave no detail${s}${h}` : `${p}${s}`, p === J || p.length === 0);
|
|
888
|
+
}, tn = (e) => en(e).reason, nn = (e) => {
|
|
889
|
+
let t = f(e), n = u(t);
|
|
877
890
|
if (n?.kind === "foreignKeyInUse") {
|
|
878
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
891
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = Qt(t);
|
|
879
892
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
880
893
|
}
|
|
881
|
-
return
|
|
882
|
-
},
|
|
894
|
+
return tn(e);
|
|
895
|
+
}, rn = (e, t) => e.tables.find((e) => e.name === t), an = (e, t) => new Map((rn(e, t)?.columns ?? []).map((e) => [e.name, e])), on = (e, t) => {
|
|
883
896
|
let n = [];
|
|
884
897
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
885
|
-
if (
|
|
898
|
+
if (rn(t, r.name) === void 0) {
|
|
886
899
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
887
900
|
continue;
|
|
888
901
|
}
|
|
889
|
-
let e =
|
|
902
|
+
let e = an(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
890
903
|
for (let [t, a] of i) {
|
|
891
904
|
let i = e.get(t);
|
|
892
905
|
if (i === void 0) {
|
|
@@ -898,43 +911,62 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
898
911
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
899
912
|
}
|
|
900
913
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
901
|
-
},
|
|
902
|
-
let r =
|
|
914
|
+
}, sn = (e, t, n) => {
|
|
915
|
+
let r = _(t);
|
|
903
916
|
if (r === e || n === void 0) return {
|
|
904
917
|
drifted: !1,
|
|
905
918
|
bundleFingerprint: e,
|
|
906
919
|
targetFingerprint: r,
|
|
907
920
|
diff: []
|
|
908
921
|
};
|
|
909
|
-
let i =
|
|
922
|
+
let i = on(n, t);
|
|
910
923
|
return {
|
|
911
924
|
drifted: i.length > 0,
|
|
912
925
|
bundleFingerprint: e,
|
|
913
926
|
targetFingerprint: r,
|
|
914
927
|
diff: i
|
|
915
928
|
};
|
|
916
|
-
},
|
|
929
|
+
}, cn = 200, ln = /* @__PURE__ */ new Set([
|
|
930
|
+
"BundleError",
|
|
931
|
+
"CodecError",
|
|
932
|
+
"IntegrityError",
|
|
933
|
+
"CrossDialectError",
|
|
934
|
+
"ImportModeError",
|
|
935
|
+
"RowsRefusedError",
|
|
936
|
+
"SchemaDriftError",
|
|
937
|
+
"StorageError"
|
|
938
|
+
]), un = (t) => {
|
|
939
|
+
let n = o.isFiberFailure(t) ? [...e.failures(t[o.FiberFailureCauseId]), ...e.defects(t[o.FiberFailureCauseId])] : [t];
|
|
940
|
+
for (let e of n) {
|
|
941
|
+
let t = e?._tag;
|
|
942
|
+
if (typeof t == "string" && ln.has(t)) return e;
|
|
943
|
+
if (o.isFiberFailure(e)) {
|
|
944
|
+
let t = un(e);
|
|
945
|
+
if (t) return t;
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
}, dn = (t) => [...e.failures(t)][0] ?? Error(e.pretty(t)), fn = (e) => n.tryPromise({
|
|
917
949
|
try: async () => {
|
|
918
950
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
919
|
-
return
|
|
951
|
+
return it(await t.readFile(n.join(e, U), "utf8"));
|
|
920
952
|
},
|
|
921
|
-
catch: (e) => new
|
|
953
|
+
catch: (e) => new L({
|
|
922
954
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
923
|
-
path:
|
|
955
|
+
path: U
|
|
924
956
|
})
|
|
925
|
-
}),
|
|
926
|
-
let
|
|
927
|
-
if (
|
|
928
|
-
if (
|
|
929
|
-
let
|
|
930
|
-
if (r.length > 0) return yield* n.fail(new
|
|
957
|
+
}), pn = (e) => n.gen(function* () {
|
|
958
|
+
let t = yield* n.promise(() => import("node:path")), i = yield* fn(e.bundleDir);
|
|
959
|
+
if (i.formatVersion !== 1) return yield* n.fail(new L({ reason: `unsupported bundle formatVersion ${i.formatVersion} (expected 1)` }));
|
|
960
|
+
if (e.tables !== void 0) {
|
|
961
|
+
let t = new Set(i.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
962
|
+
if (r.length > 0) return yield* n.fail(new L({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...t].sort().join(", ")}.` }));
|
|
931
963
|
}
|
|
932
|
-
let
|
|
933
|
-
...
|
|
934
|
-
tables:
|
|
964
|
+
let a = e.tables === void 0 ? void 0 : new Set(e.tables), o = a === void 0 ? i : {
|
|
965
|
+
...i,
|
|
966
|
+
tables: i.tables.filter((e) => a.has(e.name))
|
|
935
967
|
};
|
|
936
|
-
if (
|
|
937
|
-
let
|
|
968
|
+
if (e.targetSnapshot) {
|
|
969
|
+
let t = { tables: o.tables.map((e) => ({
|
|
938
970
|
name: e.name,
|
|
939
971
|
indexes: [],
|
|
940
972
|
columns: Object.entries(e.columnTypes).map(([e, t]) => ({
|
|
@@ -944,92 +976,94 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
944
976
|
unique: !1,
|
|
945
977
|
hasDefault: !1
|
|
946
978
|
}))
|
|
947
|
-
})) }, r =
|
|
979
|
+
})) }, r = sn(o.source.schemaFingerprint, e.targetSnapshot, t);
|
|
948
980
|
if (r.drifted) {
|
|
949
|
-
if (!
|
|
981
|
+
if (!e.force) return yield* n.fail(new Fe({
|
|
950
982
|
bundleFingerprint: r.bundleFingerprint,
|
|
951
983
|
targetFingerprint: r.targetFingerprint,
|
|
952
984
|
diff: r.diff
|
|
953
985
|
}));
|
|
954
|
-
|
|
986
|
+
e.onWarn?.(`schema drift: the target schema (fingerprint ${r.targetFingerprint.slice(0, 16)}) differs from the bundle's source (${r.bundleFingerprint.slice(0, 16)}); importing anyway (--force). ${r.diff.join("; ")}`);
|
|
955
987
|
}
|
|
956
988
|
}
|
|
957
|
-
if (
|
|
958
|
-
let
|
|
959
|
-
if (
|
|
960
|
-
sourceDialect:
|
|
961
|
-
targetDialect:
|
|
962
|
-
blocking:
|
|
989
|
+
if (e.targetDialect && e.targetDialect !== o.source.dialect) {
|
|
990
|
+
let t = Yt(o.tables, o.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
991
|
+
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new Re({
|
|
992
|
+
sourceDialect: o.source.dialect,
|
|
993
|
+
targetDialect: e.targetDialect,
|
|
994
|
+
blocking: t.map(Xt)
|
|
963
995
|
}));
|
|
964
996
|
}
|
|
965
|
-
let
|
|
966
|
-
if (
|
|
967
|
-
mode:
|
|
968
|
-
reason: `replace refuses a partial bundle (scope='${
|
|
997
|
+
let s = e.verify ?? !0, c = e.mode ?? "upsert", u = e.atomic ?? c === "replace", d = e.onConflict ?? "skip", f = e.ledgerPath ?? t.join(e.bundleDir, ".import..ledger.json"), p = yield* gt(f);
|
|
998
|
+
if (c === "replace" && o.scope.kind !== "all") return yield* n.fail(new Le({
|
|
999
|
+
mode: c,
|
|
1000
|
+
reason: `replace refuses a partial bundle (scope='${o.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
969
1001
|
}));
|
|
970
|
-
if (
|
|
971
|
-
let
|
|
972
|
-
try: () =>
|
|
973
|
-
catch: (e) => new
|
|
974
|
-
}) :
|
|
975
|
-
for (let
|
|
976
|
-
if (!a.has(
|
|
1002
|
+
if (c === "replace") {
|
|
1003
|
+
let t = o.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1004
|
+
try: () => e.store.incomingForeignKeys(t),
|
|
1005
|
+
catch: (e) => new L({ reason: `foreign-key pre-flight failed: ${nn(e)}` })
|
|
1006
|
+
}) : e.targetSnapshot === void 0 ? [] : Wt(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1007
|
+
for (let t of r) {
|
|
1008
|
+
if (!a.has(t.from)) {
|
|
977
1009
|
let r = yield* n.tryPromise({
|
|
978
|
-
try: () =>
|
|
979
|
-
table:
|
|
1010
|
+
try: () => e.store.query({
|
|
1011
|
+
table: t.from,
|
|
980
1012
|
predicate: void 0,
|
|
981
1013
|
order: [],
|
|
982
1014
|
take: void 0,
|
|
983
1015
|
skip: void 0,
|
|
984
1016
|
projection: []
|
|
985
1017
|
}),
|
|
986
|
-
catch: () => new
|
|
1018
|
+
catch: () => new L({ reason: `unreadable: ${t.from}` })
|
|
987
1019
|
}).pipe(n.map((e) => e.length), n.orElseSucceed(() => 0));
|
|
988
|
-
a.set(
|
|
1020
|
+
a.set(t.from, r);
|
|
989
1021
|
}
|
|
990
|
-
let r = a.get(
|
|
1022
|
+
let r = a.get(t.from) ?? 0;
|
|
991
1023
|
r > 0 && i.push({
|
|
992
|
-
...
|
|
1024
|
+
...t,
|
|
993
1025
|
rowCount: r
|
|
994
1026
|
});
|
|
995
1027
|
}
|
|
996
|
-
if (i.length > 0) return yield* n.fail(new
|
|
997
|
-
mode:
|
|
998
|
-
reason:
|
|
1028
|
+
if (i.length > 0) return yield* n.fail(new Le({
|
|
1029
|
+
mode: c,
|
|
1030
|
+
reason: Gt(i)
|
|
999
1031
|
}));
|
|
1000
1032
|
}
|
|
1001
|
-
if (
|
|
1002
|
-
manifest:
|
|
1033
|
+
if (e.dryRun === !0) return {
|
|
1034
|
+
manifest: o,
|
|
1003
1035
|
tablesWritten: 0,
|
|
1004
1036
|
rowsWritten: 0,
|
|
1005
1037
|
tablesSkipped: 0,
|
|
1006
1038
|
rowsSkipped: 0,
|
|
1007
1039
|
fullyResumed: !1
|
|
1008
1040
|
};
|
|
1009
|
-
let
|
|
1010
|
-
for (let
|
|
1011
|
-
let
|
|
1012
|
-
|
|
1041
|
+
let h = /* @__PURE__ */ new Map();
|
|
1042
|
+
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1043
|
+
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1044
|
+
e.length > 0 && h.set(t.name, e);
|
|
1013
1045
|
}
|
|
1014
|
-
let
|
|
1015
|
-
let n =
|
|
1046
|
+
let g = (e, t) => {
|
|
1047
|
+
let n = h.get(e);
|
|
1016
1048
|
if (n === void 0) return t;
|
|
1017
1049
|
let r;
|
|
1018
1050
|
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1019
1051
|
return r ?? t;
|
|
1020
|
-
},
|
|
1021
|
-
let i = t.name, a = t.primaryKey, o =
|
|
1022
|
-
return
|
|
1023
|
-
},
|
|
1052
|
+
}, _ = (e, t, n, r = !1) => {
|
|
1053
|
+
let i = t.name, a = t.primaryKey, o = g(i, n);
|
|
1054
|
+
return c === "append" ? d === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : c === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1055
|
+
}, v = (e, t) => e.savepoint ? e.savepoint(t) : t(), y = 0, b = (t, n, r) => {
|
|
1056
|
+
r.unclassified && (y++, !(y > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (y === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1057
|
+
}, x = (e, t, r = !0) => n.tryPromise({
|
|
1024
1058
|
try: e,
|
|
1025
|
-
catch: (e) =>
|
|
1026
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e), !1)))),
|
|
1059
|
+
catch: (e) => en(e)
|
|
1060
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, b(t.entry, t.row, e)), !1)))), S = (e) => {
|
|
1027
1061
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1028
1062
|
if (t.length !== 0) return {
|
|
1029
1063
|
...e.row,
|
|
1030
1064
|
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1031
1065
|
};
|
|
1032
|
-
},
|
|
1066
|
+
}, C = (e) => {
|
|
1033
1067
|
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
1034
1068
|
if (r !== "reference") return !1;
|
|
1035
1069
|
let i = e.row[n];
|
|
@@ -1039,84 +1073,97 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1039
1073
|
id: String(e.row[e.entry.primaryKey]),
|
|
1040
1074
|
reason: e.reason,
|
|
1041
1075
|
derived: n(e)
|
|
1042
|
-
}));
|
|
1043
|
-
|
|
1044
|
-
|
|
1076
|
+
})), i = [...r.filter((e) => !e.derived), ...r.filter((e) => e.derived)], a = /* @__PURE__ */ new Map();
|
|
1077
|
+
for (let e of r) {
|
|
1078
|
+
let t = a.get(e.table) ?? {
|
|
1079
|
+
refused: 0,
|
|
1080
|
+
primary: 0
|
|
1081
|
+
};
|
|
1082
|
+
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1083
|
+
}
|
|
1084
|
+
return new Pe({
|
|
1085
|
+
rows: i.slice(0, 20),
|
|
1045
1086
|
primaryCount: r.filter((e) => !e.derived).length,
|
|
1046
|
-
totalCount: e.length
|
|
1087
|
+
totalCount: e.length,
|
|
1088
|
+
byTable: [...a.entries()].map(([e, t]) => ({
|
|
1089
|
+
table: e,
|
|
1090
|
+
refused: t.refused,
|
|
1091
|
+
primary: t.primary
|
|
1092
|
+
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1047
1093
|
});
|
|
1048
|
-
},
|
|
1094
|
+
}, w = (e, t) => n.gen(function* () {
|
|
1049
1095
|
let r = t, i = [], a = !0;
|
|
1050
1096
|
for (; r.length > 0 && a;) {
|
|
1051
1097
|
a = !1;
|
|
1052
1098
|
let t = [];
|
|
1053
|
-
for (let n of r) (yield*
|
|
1099
|
+
for (let n of r) (yield* x(() => v(e, () => _(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1054
1100
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1055
|
-
let r = t[n], o =
|
|
1056
|
-
o !== void 0 && (yield*
|
|
1101
|
+
let r = t[n], o = S(r);
|
|
1102
|
+
o !== void 0 && (yield* x(() => v(e, () => _(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1057
1103
|
}
|
|
1058
1104
|
r = t;
|
|
1059
1105
|
}
|
|
1060
|
-
for (let t of i) (yield*
|
|
1061
|
-
if (r.length > 0) return yield* n.fail(
|
|
1062
|
-
}),
|
|
1063
|
-
if (
|
|
1064
|
-
let
|
|
1065
|
-
for (let
|
|
1066
|
-
} : () =>
|
|
1067
|
-
for (let t of [...
|
|
1106
|
+
for (let t of i) (yield* x(() => v(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1107
|
+
if (r.length > 0) return yield* n.fail(C(r));
|
|
1108
|
+
}), E = !1, D = 0, O = 0, k = 0, A = 0, j = (r, i) => n.gen(function* () {
|
|
1109
|
+
if (c === "replace" && !(i && p.truncated)) {
|
|
1110
|
+
let e = [...o.tables].reverse().map((e) => e.name), t = !i, a = r.emptyTables ? () => r.emptyTables(e) : t ? async () => {
|
|
1111
|
+
for (let e of [...o.tables].reverse()) await r.deleteMany(e.name, { where: m(e.primaryKey) });
|
|
1112
|
+
} : () => r.transactional(async (e) => {
|
|
1113
|
+
for (let t of [...o.tables].reverse()) await e.deleteMany(t.name, { where: m(t.primaryKey) });
|
|
1068
1114
|
});
|
|
1069
1115
|
yield* n.tryPromise({
|
|
1070
1116
|
try: a,
|
|
1071
|
-
catch: (e) => new
|
|
1072
|
-
}),
|
|
1117
|
+
catch: (e) => new L({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${nn(e)}` })
|
|
1118
|
+
}), i && (p.truncated = !0, yield* K(f, p));
|
|
1073
1119
|
}
|
|
1074
|
-
let a = [],
|
|
1075
|
-
for (let
|
|
1076
|
-
let
|
|
1077
|
-
if (
|
|
1078
|
-
|
|
1120
|
+
let a = [], d = [], h = [], g = o.tables.length;
|
|
1121
|
+
for (let m = 0; m < o.tables.length; m++) {
|
|
1122
|
+
let y = o.tables[m];
|
|
1123
|
+
if (i && y.name in p.tables) {
|
|
1124
|
+
h.push(y.name), A++, k += p.tables[y.name].rowCount, G(e.onProgress, {
|
|
1079
1125
|
phase: "import",
|
|
1080
|
-
table:
|
|
1081
|
-
index:
|
|
1126
|
+
table: y.name,
|
|
1127
|
+
index: m,
|
|
1082
1128
|
tableCount: g,
|
|
1083
1129
|
event: "done",
|
|
1084
|
-
rowsDone:
|
|
1085
|
-
total:
|
|
1130
|
+
rowsDone: p.tables[y.name].rowCount,
|
|
1131
|
+
total: y.rowCount
|
|
1086
1132
|
});
|
|
1087
1133
|
continue;
|
|
1088
1134
|
}
|
|
1089
|
-
let
|
|
1090
|
-
|
|
1135
|
+
let x = t.join(e.bundleDir, y.file);
|
|
1136
|
+
G(e.onProgress, {
|
|
1091
1137
|
phase: "import",
|
|
1092
|
-
table:
|
|
1093
|
-
index:
|
|
1138
|
+
table: y.name,
|
|
1139
|
+
index: m,
|
|
1094
1140
|
tableCount: g,
|
|
1095
1141
|
event: "start",
|
|
1096
1142
|
rowsDone: 0,
|
|
1097
|
-
total:
|
|
1143
|
+
total: y.rowCount
|
|
1098
1144
|
});
|
|
1099
|
-
let
|
|
1100
|
-
let { createHash:
|
|
1101
|
-
let
|
|
1102
|
-
entry:
|
|
1103
|
-
row:
|
|
1145
|
+
let S = a.length, C = n.gen(function* () {
|
|
1146
|
+
let { createHash: t } = yield* n.promise(() => import("node:crypto")), i = t("sha256"), s = 0, d = T("bundle", () => M(x).pipe(Ze(o.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), f = async (e) => {
|
|
1147
|
+
let t = {
|
|
1148
|
+
entry: y,
|
|
1149
|
+
row: e,
|
|
1104
1150
|
reason: ""
|
|
1105
1151
|
};
|
|
1106
1152
|
try {
|
|
1107
|
-
await
|
|
1108
|
-
} catch (
|
|
1109
|
-
|
|
1153
|
+
await v(r, () => _(r, y, e));
|
|
1154
|
+
} catch (n) {
|
|
1155
|
+
let r = en(n);
|
|
1156
|
+
t.reason = r.reason, b(y, e, r), a.push(t);
|
|
1110
1157
|
}
|
|
1111
1158
|
}, p = yield* n.promise(async () => {
|
|
1112
|
-
let
|
|
1113
|
-
if (!
|
|
1114
|
-
if (
|
|
1115
|
-
if (
|
|
1116
|
-
if (
|
|
1159
|
+
let t = e.copyLoader;
|
|
1160
|
+
if (!t || t.dialect !== "postgres") return !1;
|
|
1161
|
+
if (u) return E || (E = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1162
|
+
if (c === "replace") return !0;
|
|
1163
|
+
if (c !== "upsert") return !1;
|
|
1117
1164
|
try {
|
|
1118
|
-
return (await
|
|
1119
|
-
table:
|
|
1165
|
+
return (await r.query({
|
|
1166
|
+
table: y.name,
|
|
1120
1167
|
predicate: void 0,
|
|
1121
1168
|
order: [],
|
|
1122
1169
|
take: 1,
|
|
@@ -1126,123 +1173,123 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1126
1173
|
} catch {
|
|
1127
1174
|
return !1;
|
|
1128
1175
|
}
|
|
1129
|
-
}), m = Object.keys(
|
|
1176
|
+
}), m = Object.keys(y.columnTypes), h = Math.max(1, e.copyBatchSize ?? 5e3), g = [], S = async () => {
|
|
1130
1177
|
if (g.length === 0) return;
|
|
1131
|
-
let
|
|
1178
|
+
let t = g;
|
|
1132
1179
|
g = [];
|
|
1133
1180
|
try {
|
|
1134
|
-
await
|
|
1135
|
-
table:
|
|
1181
|
+
await e.copyLoader.copyInto({
|
|
1182
|
+
table: y.name,
|
|
1136
1183
|
columns: m,
|
|
1137
|
-
columnTypes:
|
|
1138
|
-
rows:
|
|
1184
|
+
columnTypes: y.columnTypes,
|
|
1185
|
+
rows: t
|
|
1139
1186
|
});
|
|
1140
1187
|
} catch (n) {
|
|
1141
|
-
p = !1,
|
|
1142
|
-
for (let
|
|
1188
|
+
p = !1, e.onWarn?.(`bulk COPY into ${y.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1189
|
+
for (let e of t) await f(e);
|
|
1143
1190
|
}
|
|
1144
|
-
},
|
|
1145
|
-
if (
|
|
1146
|
-
let
|
|
1147
|
-
|
|
1191
|
+
}, C = Math.max(1, e.savepointBatchSize ?? 200), w = typeof r.savepoint == "function", D = [], O = async () => {
|
|
1192
|
+
if (D.length === 0) return;
|
|
1193
|
+
let e = D;
|
|
1194
|
+
D = [];
|
|
1148
1195
|
try {
|
|
1149
|
-
await
|
|
1150
|
-
for (let
|
|
1196
|
+
await r.savepoint(async () => {
|
|
1197
|
+
for (let t of e) await _(r, y, t);
|
|
1151
1198
|
});
|
|
1152
1199
|
} catch {
|
|
1153
|
-
for (let
|
|
1200
|
+
for (let t of e) await f(t);
|
|
1154
1201
|
}
|
|
1155
1202
|
};
|
|
1156
|
-
return yield*
|
|
1157
|
-
if (
|
|
1158
|
-
g.push(e), g.length >= h && await
|
|
1203
|
+
return yield* l.runForEach(jt(d, y.name, y.columnTypes), (e) => n.promise(async () => {
|
|
1204
|
+
if (s++, p) {
|
|
1205
|
+
g.push(e), g.length >= h && await S();
|
|
1159
1206
|
return;
|
|
1160
1207
|
}
|
|
1161
|
-
if (
|
|
1162
|
-
|
|
1208
|
+
if (w) {
|
|
1209
|
+
D.push(e), D.length >= C && await O();
|
|
1163
1210
|
return;
|
|
1164
1211
|
}
|
|
1165
1212
|
await f(e);
|
|
1166
|
-
})), yield* n.promise(
|
|
1167
|
-
rowCount:
|
|
1213
|
+
})), yield* n.promise(O), yield* n.promise(S), {
|
|
1214
|
+
rowCount: s,
|
|
1168
1215
|
checksum: i.digest("hex")
|
|
1169
1216
|
};
|
|
1170
|
-
}), { rowCount:
|
|
1171
|
-
if (
|
|
1172
|
-
if (
|
|
1173
|
-
what: `table ${
|
|
1174
|
-
expected:
|
|
1175
|
-
actual:
|
|
1217
|
+
}), { rowCount: w, checksum: j } = yield* mt("import", y.name, m, g, (e) => e.rowCount, C);
|
|
1218
|
+
if (s) {
|
|
1219
|
+
if (j !== y.checksum) return yield* n.fail(new R({
|
|
1220
|
+
what: `table ${y.name} checksum`,
|
|
1221
|
+
expected: y.checksum,
|
|
1222
|
+
actual: j
|
|
1176
1223
|
}));
|
|
1177
|
-
if (
|
|
1178
|
-
what: `table ${
|
|
1179
|
-
expected: String(
|
|
1180
|
-
actual: String(
|
|
1224
|
+
if (w !== y.rowCount) return yield* n.fail(new R({
|
|
1225
|
+
what: `table ${y.name} rowCount`,
|
|
1226
|
+
expected: String(y.rowCount),
|
|
1227
|
+
actual: String(w)
|
|
1181
1228
|
}));
|
|
1182
1229
|
}
|
|
1183
|
-
|
|
1230
|
+
G(e.onProgress, {
|
|
1184
1231
|
phase: "import",
|
|
1185
|
-
table:
|
|
1186
|
-
index:
|
|
1232
|
+
table: y.name,
|
|
1233
|
+
index: m,
|
|
1187
1234
|
tableCount: g,
|
|
1188
1235
|
event: "done",
|
|
1189
|
-
rowsDone:
|
|
1190
|
-
total:
|
|
1191
|
-
}),
|
|
1192
|
-
name:
|
|
1193
|
-
rowCount:
|
|
1194
|
-
}) : (
|
|
1195
|
-
rowCount:
|
|
1196
|
-
checksum:
|
|
1197
|
-
}, yield*
|
|
1236
|
+
rowsDone: w,
|
|
1237
|
+
total: y.rowCount
|
|
1238
|
+
}), O++, D += w, i && (a.length > S ? d.push({
|
|
1239
|
+
name: y.name,
|
|
1240
|
+
rowCount: w
|
|
1241
|
+
}) : (p.tables[y.name] = {
|
|
1242
|
+
rowCount: w,
|
|
1243
|
+
checksum: y.checksum
|
|
1244
|
+
}, yield* K(f, p)));
|
|
1198
1245
|
}
|
|
1199
|
-
if (a.length > 0 && (yield*
|
|
1200
|
-
for (let e of
|
|
1201
|
-
let t =
|
|
1202
|
-
|
|
1246
|
+
if (a.length > 0 && (yield* w(r, a)), i && d.length > 0) {
|
|
1247
|
+
for (let e of d) {
|
|
1248
|
+
let t = o.tables.find((t) => t.name === e.name);
|
|
1249
|
+
p.tables[e.name] = {
|
|
1203
1250
|
rowCount: e.rowCount,
|
|
1204
1251
|
checksum: t.checksum
|
|
1205
1252
|
};
|
|
1206
1253
|
}
|
|
1207
|
-
yield*
|
|
1254
|
+
yield* K(f, p);
|
|
1208
1255
|
}
|
|
1209
|
-
if (
|
|
1210
|
-
let
|
|
1211
|
-
|
|
1256
|
+
if (h.length > 0) {
|
|
1257
|
+
let t = h.length === o.tables.length;
|
|
1258
|
+
e.onWarn?.(`resume: ${h.length} of ${o.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${h.join(", ")}. Delete ${f} to force a full re-import.`);
|
|
1212
1259
|
}
|
|
1213
1260
|
});
|
|
1214
|
-
if (
|
|
1215
|
-
try: () =>
|
|
1216
|
-
let
|
|
1217
|
-
if (r.isFailure(
|
|
1261
|
+
if (u ? yield* n.tryPromise({
|
|
1262
|
+
try: () => e.store.transactional(async (e) => {
|
|
1263
|
+
let t = await n.runPromiseExit(j(e, !1));
|
|
1264
|
+
if (r.isFailure(t)) throw dn(t.cause);
|
|
1218
1265
|
}),
|
|
1219
|
-
catch: (e) => new
|
|
1220
|
-
}) : yield*
|
|
1221
|
-
let
|
|
1222
|
-
yield*
|
|
1223
|
-
|
|
1224
|
-
})), n.andThen(
|
|
1266
|
+
catch: (e) => un(e) ?? new L({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1267
|
+
}) : yield* j(e.store, !0), e.assets && o.assets.index && !e.assetsAlreadyRestored) {
|
|
1268
|
+
let r = e.assets, i = t.join(e.bundleDir, W), a = t.join(e.bundleDir, o.assets.index ?? "assets/index.ndjson"), s = T("bundle", () => M(a)).pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.map((e) => JSON.parse(e)));
|
|
1269
|
+
yield* l.runForEach(s, (e) => e.key in p.assets ? n.void : Ke(r, i, e).pipe(n.andThen(n.sync(() => {
|
|
1270
|
+
p.assets[e.key] = !0;
|
|
1271
|
+
})), n.andThen(K(f, p))));
|
|
1225
1272
|
}
|
|
1226
1273
|
return {
|
|
1227
|
-
manifest:
|
|
1228
|
-
tablesWritten:
|
|
1229
|
-
rowsWritten:
|
|
1230
|
-
tablesSkipped:
|
|
1231
|
-
rowsSkipped:
|
|
1232
|
-
fullyResumed:
|
|
1274
|
+
manifest: o,
|
|
1275
|
+
tablesWritten: O,
|
|
1276
|
+
rowsWritten: D,
|
|
1277
|
+
tablesSkipped: A,
|
|
1278
|
+
rowsSkipped: k,
|
|
1279
|
+
fullyResumed: O === 0 && A > 0
|
|
1233
1280
|
};
|
|
1234
|
-
}),
|
|
1281
|
+
}), mn = (e) => e, hn = (e) => {
|
|
1235
1282
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1236
1283
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1237
1284
|
return {
|
|
1238
1285
|
refuse: !0,
|
|
1239
1286
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1240
1287
|
};
|
|
1241
|
-
}, Y = Buffer.from("VBUNDLE2"),
|
|
1288
|
+
}, Y = Buffer.from("VBUNDLE2"), gn = 0, _n = 1, vn = 2, yn = 32, bn = async (e) => {
|
|
1242
1289
|
let t = [], n = async (r) => {
|
|
1243
1290
|
let i;
|
|
1244
1291
|
try {
|
|
1245
|
-
i = await ae(
|
|
1292
|
+
i = await ae(N(e, r), { withFileTypes: !0 });
|
|
1246
1293
|
} catch {
|
|
1247
1294
|
return;
|
|
1248
1295
|
}
|
|
@@ -1252,22 +1299,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1252
1299
|
}
|
|
1253
1300
|
};
|
|
1254
1301
|
return await n(""), t.sort();
|
|
1255
|
-
},
|
|
1302
|
+
}, xn = (e, t) => {
|
|
1256
1303
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1257
|
-
return r.writeUInt8(
|
|
1258
|
-
},
|
|
1304
|
+
return r.writeUInt8(_n, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1305
|
+
}, Sn = (e, t, n) => {
|
|
1259
1306
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1260
|
-
return a.writeUInt8(
|
|
1307
|
+
return a.writeUInt8(vn, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1261
1308
|
};
|
|
1262
|
-
async function*
|
|
1309
|
+
async function* Cn(e) {
|
|
1263
1310
|
yield new Uint8Array(Y);
|
|
1264
|
-
for (let t of await
|
|
1265
|
-
let n =
|
|
1266
|
-
if (yield new Uint8Array(
|
|
1311
|
+
for (let t of await bn(e.dir)) {
|
|
1312
|
+
let n = N(e.dir, t), r = await oe(n);
|
|
1313
|
+
if (yield new Uint8Array(xn(t, r.size)), r.size > 0) for await (let e of M(n)) yield new Uint8Array(e);
|
|
1267
1314
|
}
|
|
1268
1315
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1269
|
-
yield new Uint8Array(
|
|
1270
|
-
let n =
|
|
1316
|
+
yield new Uint8Array(Sn(t.key, t.contentType, t.size));
|
|
1317
|
+
let n = b("sha256"), r = 0;
|
|
1271
1318
|
for await (let i of e.blobs.open(t.key)) {
|
|
1272
1319
|
let e = Buffer.from(i);
|
|
1273
1320
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1275,9 +1322,9 @@ async function* gn(e) {
|
|
|
1275
1322
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1276
1323
|
yield new Uint8Array(n.digest());
|
|
1277
1324
|
}
|
|
1278
|
-
yield new Uint8Array(Buffer.from([
|
|
1325
|
+
yield new Uint8Array(Buffer.from([gn]));
|
|
1279
1326
|
}
|
|
1280
|
-
var
|
|
1327
|
+
var wn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Tn = (e) => new Promise((t) => e.end(t)), En = async (e, t) => {
|
|
1281
1328
|
let n = ue(t.dir);
|
|
1282
1329
|
await ie(n, { recursive: !0 });
|
|
1283
1330
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
@@ -1294,8 +1341,8 @@ var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn
|
|
|
1294
1341
|
for (;;) {
|
|
1295
1342
|
await s(1);
|
|
1296
1343
|
let e = c(1).readUInt8(0);
|
|
1297
|
-
if (e ===
|
|
1298
|
-
if (e ===
|
|
1344
|
+
if (e === gn) return;
|
|
1345
|
+
if (e === _n) {
|
|
1299
1346
|
await s(4);
|
|
1300
1347
|
let e = c(4).readUInt32BE(0);
|
|
1301
1348
|
await s(e);
|
|
@@ -1308,10 +1355,10 @@ var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn
|
|
|
1308
1355
|
for (; i > 0n;) {
|
|
1309
1356
|
r.length === 0 && await s(1);
|
|
1310
1357
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1311
|
-
await
|
|
1358
|
+
await wn(o, c(e)), i -= BigInt(e);
|
|
1312
1359
|
}
|
|
1313
|
-
await
|
|
1314
|
-
} else if (e ===
|
|
1360
|
+
await Tn(o);
|
|
1361
|
+
} else if (e === vn) {
|
|
1315
1362
|
await s(4);
|
|
1316
1363
|
let e = c(4).readUInt32BE(0);
|
|
1317
1364
|
await s(e);
|
|
@@ -1321,24 +1368,24 @@ var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn
|
|
|
1321
1368
|
await s(i);
|
|
1322
1369
|
let a = c(i).toString("utf8");
|
|
1323
1370
|
await s(8);
|
|
1324
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1371
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = b("sha256"), d = new A(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1325
1372
|
contentType: a,
|
|
1326
1373
|
size: Number(o)
|
|
1327
1374
|
}), p = o;
|
|
1328
1375
|
for (; p > 0n;) {
|
|
1329
1376
|
r.length === 0 && await s(1);
|
|
1330
1377
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1331
|
-
u.update(t), l || await
|
|
1378
|
+
u.update(t), l || await wn(d, t), p -= BigInt(e);
|
|
1332
1379
|
}
|
|
1333
|
-
l ? d.destroy() : await
|
|
1334
|
-
let m = c(
|
|
1380
|
+
l ? d.destroy() : await Tn(d), await f, await s(yn);
|
|
1381
|
+
let m = c(yn);
|
|
1335
1382
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1336
1383
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1337
1384
|
}
|
|
1338
1385
|
} finally {
|
|
1339
1386
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1340
1387
|
}
|
|
1341
|
-
},
|
|
1388
|
+
}, Dn = async (e, t) => {
|
|
1342
1389
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1343
1390
|
let { value: e, done: t } = await a.next();
|
|
1344
1391
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1360,8 +1407,8 @@ var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn
|
|
|
1360
1407
|
if (!await s(Y.length) || !c(Y.length).equals(Y)) throw Error("not a bundle archive (bad magic)");
|
|
1361
1408
|
for (; await s(1);) {
|
|
1362
1409
|
let e = c(1).readUInt8(0);
|
|
1363
|
-
if (e ===
|
|
1364
|
-
if (e !==
|
|
1410
|
+
if (e === gn || e === vn) break;
|
|
1411
|
+
if (e !== _n) throw Error(`unknown bundle entry kind ${e}`);
|
|
1365
1412
|
if (!await s(4)) break;
|
|
1366
1413
|
let t = c(4).readUInt32BE(0);
|
|
1367
1414
|
if (!await s(t)) break;
|
|
@@ -1371,22 +1418,22 @@ var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn
|
|
|
1371
1418
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1372
1419
|
}
|
|
1373
1420
|
return r;
|
|
1374
|
-
},
|
|
1421
|
+
}, On = (e, t) => En(e, {
|
|
1375
1422
|
dir: t,
|
|
1376
|
-
blobs:
|
|
1377
|
-
}),
|
|
1423
|
+
blobs: kn(t)
|
|
1424
|
+
}), kn = (e) => {
|
|
1378
1425
|
let t = /* @__PURE__ */ new Set();
|
|
1379
1426
|
return {
|
|
1380
1427
|
has: async (e) => t.has(e),
|
|
1381
1428
|
write: async (n, r, i) => {
|
|
1382
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1429
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = N(e, "assets");
|
|
1383
1430
|
await ie(c, { recursive: !0 });
|
|
1384
|
-
let l =
|
|
1431
|
+
let l = b("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1385
1432
|
l.update(e), u += e.length, n(null, e);
|
|
1386
|
-
} }), f =
|
|
1433
|
+
} }), f = N(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1387
1434
|
await o(r, d, te(f));
|
|
1388
1435
|
let p = l.digest("hex");
|
|
1389
|
-
await a.rename(f,
|
|
1436
|
+
await a.rename(f, N(c, p)), await a.appendFile(N(c, "index.ndjson"), `${JSON.stringify({
|
|
1390
1437
|
key: n,
|
|
1391
1438
|
sha256: p,
|
|
1392
1439
|
size: u,
|
|
@@ -1394,33 +1441,33 @@ var _n = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vn
|
|
|
1394
1441
|
})}\n`), t.add(n);
|
|
1395
1442
|
}
|
|
1396
1443
|
};
|
|
1397
|
-
},
|
|
1444
|
+
}, An = fe(C), jn = Buffer.from("VENC1\0\0\0"), Mn = 32, Nn = 16, Pn = 7, X = 16, Fn = 262144, Z = {
|
|
1398
1445
|
N: 32768,
|
|
1399
1446
|
r: 8,
|
|
1400
1447
|
p: 1
|
|
1401
|
-
},
|
|
1448
|
+
}, In = 134217728, Q = 43, Ln = (e, t, n) => {
|
|
1402
1449
|
let r = Buffer.alloc(12);
|
|
1403
|
-
return e.copy(r, 0, 0,
|
|
1450
|
+
return e.copy(r, 0, 0, Pn), r.writeUInt32BE(t >>> 0, Pn), r.writeUInt8(+!!n, 11), r;
|
|
1404
1451
|
};
|
|
1405
|
-
async function*
|
|
1452
|
+
async function* Rn(e, t) {
|
|
1406
1453
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1407
|
-
let n =
|
|
1454
|
+
let n = S(Nn), r = S(Pn), i = await An(t, n, Mn, {
|
|
1408
1455
|
...Z,
|
|
1409
|
-
maxmem:
|
|
1456
|
+
maxmem: In
|
|
1410
1457
|
}), a = Buffer.alloc(Q);
|
|
1411
|
-
|
|
1458
|
+
jn.copy(a, 0), a.writeUInt32BE(Z.N, 8), a.writeUInt32BE(Z.r, 12), a.writeUInt32BE(Z.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1412
1459
|
let o = 0, s = (e, t) => {
|
|
1413
|
-
let n =
|
|
1460
|
+
let n = v("aes-256-gcm", i, Ln(r, o, t));
|
|
1414
1461
|
o === 0 && n.setAAD(a);
|
|
1415
1462
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1416
1463
|
o++;
|
|
1417
1464
|
let l = Buffer.alloc(5 + s.length + X);
|
|
1418
1465
|
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + X, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1419
1466
|
}, c = Buffer.alloc(0);
|
|
1420
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1467
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > Fn;) yield s(c.subarray(0, Fn), !1), c = c.subarray(Fn);
|
|
1421
1468
|
yield s(c, !0);
|
|
1422
1469
|
}
|
|
1423
|
-
async function*
|
|
1470
|
+
async function* zn(e, t) {
|
|
1424
1471
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1425
1472
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1426
1473
|
let { value: e, done: t } = await n.next();
|
|
@@ -1431,12 +1478,12 @@ async function* Mn(e, t) {
|
|
|
1431
1478
|
};
|
|
1432
1479
|
if (!await o(Q)) throw Error("encrypted bundle truncated (no header)");
|
|
1433
1480
|
let s = r.subarray(0, Q);
|
|
1434
|
-
if (!s.subarray(0, 8).equals(
|
|
1435
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await
|
|
1481
|
+
if (!s.subarray(0, 8).equals(jn)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1482
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await An(t, d, Mn, {
|
|
1436
1483
|
N: c,
|
|
1437
1484
|
r: l,
|
|
1438
1485
|
p: u,
|
|
1439
|
-
maxmem:
|
|
1486
|
+
maxmem: In
|
|
1440
1487
|
});
|
|
1441
1488
|
r = r.subarray(Q);
|
|
1442
1489
|
let m = 0, h = !1;
|
|
@@ -1451,7 +1498,7 @@ async function* Mn(e, t) {
|
|
|
1451
1498
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1452
1499
|
let n = r.subarray(5, 5 + t - X), i = r.subarray(5 + t - X, 5 + t);
|
|
1453
1500
|
r = r.subarray(5 + t);
|
|
1454
|
-
let a =
|
|
1501
|
+
let a = y("aes-256-gcm", p, Ln(f, m, e));
|
|
1455
1502
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1456
1503
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1457
1504
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1459,11 +1506,11 @@ async function* Mn(e, t) {
|
|
|
1459
1506
|
}
|
|
1460
1507
|
//#endregion
|
|
1461
1508
|
//#region src/blobStorage.ts
|
|
1462
|
-
var
|
|
1509
|
+
var Bn = (e, t) => ({
|
|
1463
1510
|
list: () => {
|
|
1464
1511
|
let e = t();
|
|
1465
1512
|
return { async *[Symbol.asyncIterator]() {
|
|
1466
|
-
let t = await n.runPromise(
|
|
1513
|
+
let t = await n.runPromise(l.runCollect(e));
|
|
1467
1514
|
for (let e of t) yield {
|
|
1468
1515
|
key: e.key,
|
|
1469
1516
|
contentType: e.contentType,
|
|
@@ -1472,29 +1519,29 @@ var Nn = (e, t) => ({
|
|
|
1472
1519
|
} };
|
|
1473
1520
|
},
|
|
1474
1521
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1475
|
-
let r = await n.runPromise(
|
|
1522
|
+
let r = await n.runPromise(E(e, t)), i = await O(r.stream);
|
|
1476
1523
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1477
1524
|
} })
|
|
1478
|
-
}),
|
|
1479
|
-
table:
|
|
1525
|
+
}), Vn = (e, t) => Bn(e, () => g(t, {
|
|
1526
|
+
table: w,
|
|
1480
1527
|
chunkSize: 500
|
|
1481
|
-
}).pipe(
|
|
1528
|
+
}).pipe(l.map((e) => ({
|
|
1482
1529
|
key: String(e.key),
|
|
1483
1530
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1484
1531
|
size: Number(e.size ?? 0)
|
|
1485
|
-
})))),
|
|
1532
|
+
})))), Hn = (e) => ({
|
|
1486
1533
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1487
1534
|
write: async (t, r, i) => {
|
|
1488
|
-
let a =
|
|
1489
|
-
await n.runPromise(
|
|
1535
|
+
let a = T("storage", () => j.from(r));
|
|
1536
|
+
await n.runPromise(D(e, t, a, {
|
|
1490
1537
|
contentType: i.contentType,
|
|
1491
1538
|
totalSize: i.size
|
|
1492
1539
|
}));
|
|
1493
1540
|
}
|
|
1494
|
-
}),
|
|
1541
|
+
}), Un = (e, t) => {
|
|
1495
1542
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1496
|
-
return e === "mariadb" &&
|
|
1497
|
-
},
|
|
1543
|
+
return e === "mariadb" && Wn(n) ? n : r;
|
|
1544
|
+
}, Wn = (e) => (process.env.PATH ?? "").split(ce).filter((e) => e !== "").some((t) => ne(N(t, e))), Gn = (e) => {
|
|
1498
1545
|
switch (e) {
|
|
1499
1546
|
case "postgres": return "db.dump";
|
|
1500
1547
|
case "mysql":
|
|
@@ -1504,7 +1551,7 @@ var Nn = (e, t) => ({
|
|
|
1504
1551
|
case "mssql": return "db.bacpac";
|
|
1505
1552
|
default: return "db.dump";
|
|
1506
1553
|
}
|
|
1507
|
-
},
|
|
1554
|
+
}, Kn = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1508
1555
|
if (e.host || e.database) return {
|
|
1509
1556
|
host: e.host ?? "localhost",
|
|
1510
1557
|
port: e.port ?? 3306,
|
|
@@ -1520,7 +1567,7 @@ var Nn = (e, t) => ({
|
|
|
1520
1567
|
password: decodeURIComponent(t.password) || "",
|
|
1521
1568
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1522
1569
|
};
|
|
1523
|
-
},
|
|
1570
|
+
}, qn = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Jn = (e, t, n) => {
|
|
1524
1571
|
switch (e) {
|
|
1525
1572
|
case "postgres": {
|
|
1526
1573
|
if (t.url) return {
|
|
@@ -1567,7 +1614,7 @@ var Nn = (e, t) => ({
|
|
|
1567
1614
|
let r = $(t);
|
|
1568
1615
|
return {
|
|
1569
1616
|
kind: "spawn",
|
|
1570
|
-
tool:
|
|
1617
|
+
tool: Un(e, "dump"),
|
|
1571
1618
|
args: [
|
|
1572
1619
|
"--single-transaction",
|
|
1573
1620
|
"--routines",
|
|
@@ -1589,7 +1636,7 @@ var Nn = (e, t) => ({
|
|
|
1589
1636
|
case "sqlite":
|
|
1590
1637
|
case "turso": return {
|
|
1591
1638
|
kind: "copy",
|
|
1592
|
-
from:
|
|
1639
|
+
from: Kn(t),
|
|
1593
1640
|
to: n
|
|
1594
1641
|
};
|
|
1595
1642
|
case "mssql": return {
|
|
@@ -1597,7 +1644,7 @@ var Nn = (e, t) => ({
|
|
|
1597
1644
|
tool: "sqlpackage",
|
|
1598
1645
|
args: [
|
|
1599
1646
|
"/Action:Export",
|
|
1600
|
-
`/SourceConnectionString:${
|
|
1647
|
+
`/SourceConnectionString:${qn($({
|
|
1601
1648
|
...t,
|
|
1602
1649
|
port: t.port ?? 1433
|
|
1603
1650
|
}))}`,
|
|
@@ -1605,12 +1652,12 @@ var Nn = (e, t) => ({
|
|
|
1605
1652
|
],
|
|
1606
1653
|
env: {}
|
|
1607
1654
|
};
|
|
1608
|
-
default: throw new
|
|
1655
|
+
default: throw new B({
|
|
1609
1656
|
tool: "backup",
|
|
1610
1657
|
reason: `no native backup for dialect '${e}'`
|
|
1611
1658
|
});
|
|
1612
1659
|
}
|
|
1613
|
-
},
|
|
1660
|
+
}, Yn = (e, t, n) => {
|
|
1614
1661
|
switch (e) {
|
|
1615
1662
|
case "postgres": {
|
|
1616
1663
|
if (t.url) return {
|
|
@@ -1655,7 +1702,7 @@ var Nn = (e, t) => ({
|
|
|
1655
1702
|
let r = $(t);
|
|
1656
1703
|
return {
|
|
1657
1704
|
kind: "spawn",
|
|
1658
|
-
tool:
|
|
1705
|
+
tool: Un(e, "client"),
|
|
1659
1706
|
args: [
|
|
1660
1707
|
"--protocol=TCP",
|
|
1661
1708
|
"-h",
|
|
@@ -1674,14 +1721,14 @@ var Nn = (e, t) => ({
|
|
|
1674
1721
|
case "turso": return {
|
|
1675
1722
|
kind: "copy",
|
|
1676
1723
|
from: n,
|
|
1677
|
-
to:
|
|
1724
|
+
to: Kn(t)
|
|
1678
1725
|
};
|
|
1679
1726
|
case "mssql": return {
|
|
1680
1727
|
kind: "spawn",
|
|
1681
1728
|
tool: "sqlpackage",
|
|
1682
1729
|
args: [
|
|
1683
1730
|
"/Action:Import",
|
|
1684
|
-
`/TargetConnectionString:${
|
|
1731
|
+
`/TargetConnectionString:${qn($({
|
|
1685
1732
|
...t,
|
|
1686
1733
|
port: t.port ?? 1433
|
|
1687
1734
|
}))}`,
|
|
@@ -1689,13 +1736,13 @@ var Nn = (e, t) => ({
|
|
|
1689
1736
|
],
|
|
1690
1737
|
env: {}
|
|
1691
1738
|
};
|
|
1692
|
-
default: throw new
|
|
1739
|
+
default: throw new B({
|
|
1693
1740
|
tool: "restore",
|
|
1694
1741
|
reason: `no native restore for dialect '${e}'`
|
|
1695
1742
|
});
|
|
1696
1743
|
}
|
|
1697
|
-
},
|
|
1698
|
-
ee(e.from, e.to, (r) => t(r ? n.fail(new
|
|
1744
|
+
}, Xn = (e) => e.kind === "copy" ? n.async((t) => {
|
|
1745
|
+
ee(e.from, e.to, (r) => t(r ? n.fail(new B({
|
|
1699
1746
|
tool: "copy",
|
|
1700
1747
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1701
1748
|
})) : n.void));
|
|
@@ -1713,14 +1760,14 @@ var Nn = (e, t) => ({
|
|
|
1713
1760
|
}), i = "";
|
|
1714
1761
|
r.stderr?.on("data", (e) => {
|
|
1715
1762
|
i += String(e);
|
|
1716
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(te(e.stdoutFile)), e.stdinFile && r.stdin &&
|
|
1763
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(te(e.stdoutFile)), e.stdinFile && r.stdin && M(e.stdinFile).pipe(r.stdin);
|
|
1717
1764
|
let a = () => {
|
|
1718
1765
|
if (e.stdoutFile !== void 0) try {
|
|
1719
1766
|
re(e.stdoutFile);
|
|
1720
1767
|
} catch {}
|
|
1721
1768
|
};
|
|
1722
1769
|
r.on("error", (r) => {
|
|
1723
|
-
a(), t(n.fail(new
|
|
1770
|
+
a(), t(n.fail(new B({
|
|
1724
1771
|
tool: e.tool,
|
|
1725
1772
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`
|
|
1726
1773
|
})));
|
|
@@ -1729,37 +1776,37 @@ var Nn = (e, t) => ({
|
|
|
1729
1776
|
t(n.void);
|
|
1730
1777
|
return;
|
|
1731
1778
|
}
|
|
1732
|
-
a(), t(n.fail(new
|
|
1779
|
+
a(), t(n.fail(new B({
|
|
1733
1780
|
tool: e.tool,
|
|
1734
1781
|
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${se(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
1735
1782
|
stderr: i.slice(-2e3)
|
|
1736
1783
|
})));
|
|
1737
1784
|
});
|
|
1738
|
-
}),
|
|
1739
|
-
let t = yield*
|
|
1785
|
+
}), Zn = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Qn = (e) => n.gen(function* () {
|
|
1786
|
+
let t = yield* He(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
1740
1787
|
store: e.store,
|
|
1741
1788
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
1742
|
-
}), r =
|
|
1789
|
+
}), r = Mt(t.tables, e.snapshot), i = e.classification ?? ge(e.snapshot), a = e.sampleSize ?? 20, o = Oe(r.map((t) => ({
|
|
1743
1790
|
name: t,
|
|
1744
1791
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
1745
1792
|
name: e.name,
|
|
1746
1793
|
type: e.type
|
|
1747
1794
|
}))
|
|
1748
|
-
})), i, e.masking), s = [],
|
|
1795
|
+
})), i, e.masking), s = [], c = [];
|
|
1749
1796
|
for (let i of r) {
|
|
1750
|
-
let r = t.predicateFor(i), u = yield*
|
|
1797
|
+
let r = t.predicateFor(i), u = yield* g(e.store, {
|
|
1751
1798
|
table: i,
|
|
1752
1799
|
chunkSize: a,
|
|
1753
1800
|
...r === void 0 ? {} : { where: r }
|
|
1754
|
-
}).pipe(
|
|
1755
|
-
|
|
1801
|
+
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) => je(t, i, o.actions, e.masking.seed));
|
|
1802
|
+
c.push(...Ae(d, i, o.actions));
|
|
1756
1803
|
let f = u[0], p = d[0], m = [];
|
|
1757
1804
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
1758
1805
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
1759
1806
|
let n = e.slice(i.length + 1);
|
|
1760
1807
|
m.push({
|
|
1761
1808
|
column: n,
|
|
1762
|
-
action:
|
|
1809
|
+
action: Zn(t),
|
|
1763
1810
|
before: f[n],
|
|
1764
1811
|
after: p[n]
|
|
1765
1812
|
});
|
|
@@ -1771,10 +1818,10 @@ var Nn = (e, t) => ({
|
|
|
1771
1818
|
}
|
|
1772
1819
|
return {
|
|
1773
1820
|
tables: s,
|
|
1774
|
-
leaks:
|
|
1821
|
+
leaks: c,
|
|
1775
1822
|
unclassified: o.unclassified
|
|
1776
1823
|
};
|
|
1777
|
-
}),
|
|
1824
|
+
}), $n = (e, t) => {
|
|
1778
1825
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
1779
1826
|
for (let t of e) {
|
|
1780
1827
|
let e = n.get(t.name);
|
|
@@ -1829,7 +1876,7 @@ var Nn = (e, t) => ({
|
|
|
1829
1876
|
skipColumns: i,
|
|
1830
1877
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
1831
1878
|
};
|
|
1832
|
-
},
|
|
1879
|
+
}, er = (e) => {
|
|
1833
1880
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
1834
1881
|
let t = {
|
|
1835
1882
|
safe: "✓",
|
|
@@ -1840,4 +1887,4 @@ var Nn = (e, t) => ({
|
|
|
1840
1887
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
1841
1888
|
};
|
|
1842
1889
|
//#endregion
|
|
1843
|
-
export {
|
|
1890
|
+
export { W as ASSETS_DIR, st as ASSET_INDEX_FILE, L as BundleError, Me as CodecError, Ne as CompressionError, Re as CrossDialectError, ot as DATA_DIR, he as DEFAULT_CLASS_ACTIONS, Qe as FORMAT_VERSION, Le as ImportModeError, R as IntegrityError, ct as LEDGER_FILE, U as MANIFEST_FILE, Ie as MaskingError, B as NativeToolError, Pe as RowsRefusedError, cn as SAVEPOINT_BATCH_SIZE, Fe as SchemaDriftError, z as ScopeError, Ee as applyAction, un as asImportError, hn as assessLiveImport, vt as assetDone, Gn as backupArtifactName, Jn as backupCommand, Kt as canonicalType, Yt as checkPortability, sn as checkSchemaDrift, ge as classificationFromSnapshot, $n as classifyImportDrift, Ye as compressionExt, Be as computeSubsetIds, it as decodeManifest, Ot as decodeRow, Et as decodeValue, zn as decryptStream, mn as defineDataProfile, on as diffSnapshots, kn as dirBlobSink, ht as emptyLedger, rt as encodeManifest, Dt as encodeRow, wt as encodeValue, Rn as encryptStream, er as formatImportDrift, Xt as formatIssue, qe as hasZstd, Xe as makeCompressor, Ze as makeDecompressor, je as maskRow, jt as ndjsonToRows, Gt as outsideReferenceRefusal, Wt as outsideReferencesInto, Cn as packBundle, Dn as peekBundle, Je as pickCompression, Oe as planMasking, Qn as previewMasking, gt as readLedger, _e as resolveAction, He as resolveScope, Ke as restoreAssetFromCas, Yn as restoreCommand, At as rowsToNdjson, Ut as runExport, pn as runImport, Xn as runNativeStep, Ae as scanForLeaks, We as storageAssetSink, Ue as storageAssetSource, Hn as storageBlobSink, Bn as storageBlobSource, Vn as storageRefsBlobSource, Ve as subsetToScope, _t as tableDone, Mt as topoSortTables, En as unpackBundle, On as unpackBundleToDir, Ge as writeAssetToCas, K as writeLedger };
|