@voltro/data-transfer 0.45.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +206 -0
- package/dist/index.d.ts +83 -0
- package/dist/index.js +885 -627
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Runtime as o, Schedule as s, Schema as c, Stream as l } from "effect";
|
|
2
|
-
import {
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { STORAGE_REFS_TABLE as
|
|
2
|
+
import { IMPORT_RUNS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d, classifyConstraintViolation as f, derivedRowId as p, eq as m, extractDbCause as h, getTable as g, inSet as _, isNotNull as v, or as y, recordsTable as b, registerTable as ee, streamTable as x, suspendWriteRecorders as S, suspendWriteRecordersEffect as te, unregisterTable as ne } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as re } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as C, createDecipheriv as w, createHash as T, createHmac as ie, randomBytes as E, scrypt as ae } from "node:crypto";
|
|
5
|
+
import { STORAGE_REFS_TABLE as oe, fromNodeReadable as se, getObjectStream as ce, putObjectStream as D, toNodeReadable as O } from "@voltro/plugin-storage";
|
|
6
6
|
import * as k from "node:zlib";
|
|
7
7
|
import { PassThrough as A, Readable as j } from "node:stream";
|
|
8
|
-
import { copyFile as
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
8
|
+
import { copyFile as le, createReadStream as ue, createWriteStream as M, existsSync as N, unlinkSync as de } from "node:fs";
|
|
9
|
+
import { SqlClient as P } from "@effect/sql";
|
|
10
|
+
import { mkdir as F, readdir as fe, stat as pe } from "node:fs/promises";
|
|
11
|
+
import { basename as I, delimiter as me, dirname as he, join as L, resolve as ge, sep as _e } from "node:path";
|
|
12
|
+
import { promisify as ve } from "node:util";
|
|
13
|
+
import { spawn as ye } from "node:child_process";
|
|
13
14
|
//#region src/consistency.ts
|
|
14
|
-
var
|
|
15
|
+
var be = (e) => {
|
|
15
16
|
switch (e) {
|
|
16
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
17
18
|
case "mysql":
|
|
@@ -21,7 +22,7 @@ var me = (e) => {
|
|
|
21
22
|
case "turso": return [];
|
|
22
23
|
default: return [];
|
|
23
24
|
}
|
|
24
|
-
},
|
|
25
|
+
}, xe = {
|
|
25
26
|
email: { fake: "email" },
|
|
26
27
|
fullName: { fake: "fullName" },
|
|
27
28
|
firstName: { fake: "firstName" },
|
|
@@ -36,7 +37,7 @@ var me = (e) => {
|
|
|
36
37
|
date: "dateShift",
|
|
37
38
|
secret: "null",
|
|
38
39
|
freeText: "redact"
|
|
39
|
-
},
|
|
40
|
+
}, Se = (e) => {
|
|
40
41
|
let t = {};
|
|
41
42
|
for (let n of e.tables) {
|
|
42
43
|
let e = {};
|
|
@@ -48,15 +49,15 @@ var me = (e) => {
|
|
|
48
49
|
t[n.name] = e;
|
|
49
50
|
}
|
|
50
51
|
return t;
|
|
51
|
-
},
|
|
52
|
+
}, Ce = (e, t, n, r) => {
|
|
52
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
53
54
|
if (i) return i;
|
|
54
55
|
if (n?.sensitive) {
|
|
55
56
|
let e = n.sensitive.class;
|
|
56
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? xe[e] ?? "redact";
|
|
57
58
|
}
|
|
58
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
59
|
-
},
|
|
60
|
+
}, we = (e, t) => ie("sha256", e).update(t == null ? "∅" : String(t)).digest(), R = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), Te = [
|
|
60
61
|
"Ada",
|
|
61
62
|
"Alan",
|
|
62
63
|
"Grace",
|
|
@@ -69,7 +70,7 @@ var me = (e) => {
|
|
|
69
70
|
"Guido",
|
|
70
71
|
"Margaret",
|
|
71
72
|
"Bjarne"
|
|
72
|
-
],
|
|
73
|
+
], Ee = [
|
|
73
74
|
"Lovelace",
|
|
74
75
|
"Turing",
|
|
75
76
|
"Hopper",
|
|
@@ -82,7 +83,7 @@ var me = (e) => {
|
|
|
82
83
|
"Rossum",
|
|
83
84
|
"Hamilton",
|
|
84
85
|
"Stroustrup"
|
|
85
|
-
],
|
|
86
|
+
], De = [
|
|
86
87
|
"Maple",
|
|
87
88
|
"Oak",
|
|
88
89
|
"Cedar",
|
|
@@ -91,7 +92,7 @@ var me = (e) => {
|
|
|
91
92
|
"Birch",
|
|
92
93
|
"Willow",
|
|
93
94
|
"Ash"
|
|
94
|
-
],
|
|
95
|
+
], Oe = [
|
|
95
96
|
"Initech",
|
|
96
97
|
"Hooli",
|
|
97
98
|
"Umbrella",
|
|
@@ -100,23 +101,23 @@ var me = (e) => {
|
|
|
100
101
|
"Soylent",
|
|
101
102
|
"Stark",
|
|
102
103
|
"Wayne"
|
|
103
|
-
],
|
|
104
|
-
let r =
|
|
104
|
+
], z = (e, t, n) => e[R(t, n) % e.length], ke = (e, t, n) => {
|
|
105
|
+
let r = we(n, `${e}:${String(t)}`), i = R(r, 1) % 1e4;
|
|
105
106
|
switch (e) {
|
|
106
|
-
case "email": return `${
|
|
107
|
-
case "fullName": return `${
|
|
108
|
-
case "firstName": return
|
|
109
|
-
case "lastName": return
|
|
110
|
-
case "username": return `${
|
|
111
|
-
case "phone": return `+1${String(
|
|
112
|
-
case "address": return `${
|
|
113
|
-
case "company": return
|
|
107
|
+
case "email": return `${z(Te, r, 0).toLowerCase()}.${z(Ee, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${z(Te, r, 0)} ${z(Ee, r, 1)}`;
|
|
109
|
+
case "firstName": return z(Te, r, 0);
|
|
110
|
+
case "lastName": return z(Ee, r, 1);
|
|
111
|
+
case "username": return `${z(Te, r, 0).toLowerCase()}${i}`;
|
|
112
|
+
case "phone": return `+1${String(R(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
+
case "address": return `${R(r, 0) % 9899 + 100} ${z(De, r, 1)} St`;
|
|
114
|
+
case "company": return z(Oe, r, 0);
|
|
114
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
115
|
-
case "ip": return `${
|
|
116
|
-
case "creditCard": return
|
|
116
|
+
case "ip": return `${R(r, 0) % 256}.${R(r, 1) % 256}.${R(r, 2) % 256}.${R(r, 3) % 254 + 1}`;
|
|
117
|
+
case "creditCard": return Ae(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
117
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
118
119
|
}
|
|
119
|
-
},
|
|
120
|
+
}, Ae = (e) => {
|
|
120
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
121
122
|
t.forEach((e, t) => {
|
|
122
123
|
let r = e;
|
|
@@ -124,31 +125,31 @@ var me = (e) => {
|
|
|
124
125
|
});
|
|
125
126
|
let r = (10 - n % 10) % 10;
|
|
126
127
|
return e.slice(0, 15) + String(r);
|
|
127
|
-
},
|
|
128
|
+
}, je = 864e5, Me = (e, t) => {
|
|
128
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
129
130
|
if (Number.isNaN(n)) return e;
|
|
130
|
-
let r =
|
|
131
|
+
let r = R(we(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * je);
|
|
131
132
|
return e instanceof Date ? i : i.toISOString();
|
|
132
|
-
},
|
|
133
|
+
}, Ne = (e, t) => {
|
|
133
134
|
if (t.value === null || t.value === void 0) return null;
|
|
134
135
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
135
136
|
if (e === "keep") return t.value;
|
|
136
137
|
if (e === "null") return null;
|
|
137
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
138
|
-
if (e === "hash") return
|
|
139
|
-
if (e === "dateShift") return
|
|
140
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return we(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Me(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return ke(e.fake, t.value, t.seed);
|
|
141
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
142
143
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
143
|
-
},
|
|
144
|
+
}, Pe = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Fe = (e, t, n) => {
|
|
144
145
|
let r = {}, i = [], a = [];
|
|
145
146
|
for (let o of e) for (let e of o.columns) {
|
|
146
|
-
let s = `${o.name}.${e.name}`, c =
|
|
147
|
+
let s = `${o.name}.${e.name}`, c = Ce(o.name, e.name, t[o.name]?.[e.name], n);
|
|
147
148
|
if (c === "error") {
|
|
148
149
|
i.push(s);
|
|
149
150
|
continue;
|
|
150
151
|
}
|
|
151
|
-
if (!
|
|
152
|
+
if (!Pe(c)) {
|
|
152
153
|
a.push({
|
|
153
154
|
column: s,
|
|
154
155
|
action: c
|
|
@@ -162,7 +163,7 @@ var me = (e) => {
|
|
|
162
163
|
unclassified: i,
|
|
163
164
|
invalid: a
|
|
164
165
|
};
|
|
165
|
-
},
|
|
166
|
+
}, Ie = [
|
|
166
167
|
{
|
|
167
168
|
name: "email",
|
|
168
169
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -179,7 +180,7 @@ var me = (e) => {
|
|
|
179
180
|
name: "creditCard",
|
|
180
181
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
181
182
|
}
|
|
182
|
-
],
|
|
183
|
+
], Le = (e, t, n) => {
|
|
183
184
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
184
185
|
for (let [a, o] of Object.entries(n)) {
|
|
185
186
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -187,7 +188,7 @@ var me = (e) => {
|
|
|
187
188
|
for (let a of e) {
|
|
188
189
|
let e = a[n];
|
|
189
190
|
if (typeof e != "string" || e.length === 0) continue;
|
|
190
|
-
let o =
|
|
191
|
+
let o = Ie.find((t) => t.re.test(e));
|
|
191
192
|
if (o && !i.has(n)) {
|
|
192
193
|
i.add(n), r.push({
|
|
193
194
|
table: t,
|
|
@@ -200,9 +201,9 @@ var me = (e) => {
|
|
|
200
201
|
}
|
|
201
202
|
}
|
|
202
203
|
return r;
|
|
203
|
-
},
|
|
204
|
+
}, Re = (e, t, n, r, i) => {
|
|
204
205
|
let a = {};
|
|
205
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
206
|
+
for (let [o, s] of Object.entries(e)) a[o] = Ne(n[`${t}.${o}`] ?? "keep", {
|
|
206
207
|
value: s,
|
|
207
208
|
table: t,
|
|
208
209
|
column: o,
|
|
@@ -210,17 +211,34 @@ var me = (e) => {
|
|
|
210
211
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
211
212
|
});
|
|
212
213
|
return a;
|
|
213
|
-
},
|
|
214
|
+
}, ze = /* @__PURE__ */ new Map([
|
|
215
|
+
["_voltro_migration_plans", "the schema fingerprint THIS deployment applied — a foreign row makes the next boot refuse"],
|
|
216
|
+
["_voltro_migrations", "which file-based migrations ran HERE — a foreign row marks unrun migrations as done"],
|
|
217
|
+
["_voltro_seeds", "which seeds ran HERE — a foreign row suppresses seeding the target still needs"],
|
|
218
|
+
["_voltro_cdc_offsets", "binlog/WAL positions of the SOURCE server — meaningless and unsafe against another one"],
|
|
219
|
+
["_voltro_schedule_claims", "live leases held by THIS deployment's replicas — a foreign row hands out a lease nobody holds"],
|
|
220
|
+
["_voltro_data_imports", "which bundles were imported HERE — a foreign row claims an import this deployment never ran"],
|
|
221
|
+
["_voltro_replace_in_progress", "the marker a destructive import HERE left behind — a foreign row refuses a boot over a healthy database"],
|
|
222
|
+
["_voltro_wakeups", "pending wakeups addressed to THIS deployment's runners"],
|
|
223
|
+
["_voltro_workflow_watermarks", "how far a consumer has read THIS deployment's event journal — a foreign position skips events"],
|
|
224
|
+
["_voltro_workflow_pending", "starts waiting to be drained HERE — a foreign row makes the target RUN a workflow somebody queued elsewhere"],
|
|
225
|
+
["_voltro_workflow_admissions", "the in-flight accounting a concurrency gate decides on — a foreign row makes the target refuse or admit wrongly"],
|
|
226
|
+
["_voltro_workflow_pauses", "which workflows an operator paused HERE — a foreign row silently pauses them in the target"]
|
|
227
|
+
]), B = (e) => ze.has(e), Be = (e) => ze.get(e), Ve = (e, t) => {
|
|
228
|
+
if (e.length === 0) return;
|
|
229
|
+
let n = e.map((e) => ` ${e} — ${Be(e) ?? "describes this deployment"}`);
|
|
230
|
+
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
231
|
+
}, V = class extends c.TaggedError()("BundleError", {
|
|
214
232
|
reason: c.String,
|
|
215
233
|
path: c.optional(c.String)
|
|
216
|
-
}) {},
|
|
234
|
+
}) {}, He = class extends c.TaggedError()("CodecError", {
|
|
217
235
|
table: c.String,
|
|
218
236
|
reason: c.String
|
|
219
|
-
}) {},
|
|
237
|
+
}) {}, Ue = class extends c.TaggedError()("IntegrityError", {
|
|
220
238
|
what: c.String,
|
|
221
239
|
expected: c.String,
|
|
222
240
|
actual: c.String
|
|
223
|
-
}) {},
|
|
241
|
+
}) {}, We = class extends t.TaggedError("CompressionError") {}, H = class extends c.TaggedError()("ScopeError", { reason: c.String }) {}, Ge = class extends c.TaggedError()("RowsRefusedError", {
|
|
224
242
|
rows: c.Array(c.Struct({
|
|
225
243
|
table: c.String,
|
|
226
244
|
id: c.String,
|
|
@@ -234,29 +252,29 @@ var me = (e) => {
|
|
|
234
252
|
refused: c.Number,
|
|
235
253
|
primary: c.Number
|
|
236
254
|
}))
|
|
237
|
-
}) {},
|
|
255
|
+
}) {}, Ke = class extends c.TaggedError()("SchemaDriftError", {
|
|
238
256
|
bundleFingerprint: c.String,
|
|
239
257
|
targetFingerprint: c.String,
|
|
240
258
|
diff: c.Array(c.String)
|
|
241
|
-
}) {},
|
|
259
|
+
}) {}, U = class extends t.TaggedError("NativeToolError") {
|
|
242
260
|
get message() {
|
|
243
261
|
let e = this.stderr?.trim();
|
|
244
262
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
245
263
|
}
|
|
246
|
-
},
|
|
264
|
+
}, qe = class extends c.TaggedError()("MaskingError", {
|
|
247
265
|
unclassified: c.Array(c.String),
|
|
248
266
|
invalidActions: c.optional(c.Array(c.String))
|
|
249
|
-
}) {},
|
|
267
|
+
}) {}, Je = class extends c.TaggedError()("ImportModeError", {
|
|
250
268
|
mode: c.String,
|
|
251
269
|
reason: c.String
|
|
252
|
-
}) {},
|
|
270
|
+
}) {}, Ye = class extends c.TaggedError()("CrossDialectError", {
|
|
253
271
|
sourceDialect: c.String,
|
|
254
272
|
targetDialect: c.String,
|
|
255
273
|
blocking: c.Array(c.String)
|
|
256
|
-
}) {},
|
|
274
|
+
}) {}, W = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Xe = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
257
275
|
col: e.name,
|
|
258
276
|
refTable: e.references.table
|
|
259
|
-
})),
|
|
277
|
+
})), Ze = (e, t, r) => n.gen(function* () {
|
|
260
278
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), u = (e) => {
|
|
261
279
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
262
280
|
return s.set(e, t), t;
|
|
@@ -273,25 +291,25 @@ var me = (e) => {
|
|
|
273
291
|
r.add(e[t]), i?.add(e[t]);
|
|
274
292
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
275
293
|
};
|
|
276
|
-
},
|
|
277
|
-
let a = f(r,
|
|
278
|
-
return l.runForEach(
|
|
294
|
+
}, p = (r, i) => {
|
|
295
|
+
let a = f(r, W(t, r), Xe(t, r));
|
|
296
|
+
return l.runForEach(x(e, {
|
|
279
297
|
table: r,
|
|
280
|
-
pk:
|
|
298
|
+
pk: W(t, r),
|
|
281
299
|
...i === void 0 ? {} : { where: i }
|
|
282
300
|
}), (e) => n.sync(() => a(e)));
|
|
283
|
-
},
|
|
301
|
+
}, m = n.gen(function* () {
|
|
284
302
|
for (let e = 0; e < i && c.size > 0; e++) {
|
|
285
303
|
let e = [...c.entries()];
|
|
286
304
|
c.clear();
|
|
287
305
|
for (let [n, r] of e) {
|
|
288
306
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
289
|
-
e.length !== 0 && (yield*
|
|
307
|
+
e.length !== 0 && (yield* p(n, _(W(t, n), e)));
|
|
290
308
|
}
|
|
291
309
|
}
|
|
292
310
|
});
|
|
293
|
-
for (let [e, t] of Object.entries(r.seeds)) yield*
|
|
294
|
-
if (yield*
|
|
311
|
+
for (let [e, t] of Object.entries(r.seeds)) yield* p(e, t);
|
|
312
|
+
if (yield* m, r.children) {
|
|
295
313
|
let e = new Set(r.children === !0 ? [...s.keys()] : r.children.roots.filter((e) => s.has(e)));
|
|
296
314
|
for (let n = 0; n < i; n++) {
|
|
297
315
|
let n = !1, r = [];
|
|
@@ -301,29 +319,29 @@ var me = (e) => {
|
|
|
301
319
|
refTable: e.references?.table
|
|
302
320
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
303
321
|
if (a.length === 0) continue;
|
|
304
|
-
let o = s.get(t)?.size ?? 0, c = a.map((e) =>
|
|
305
|
-
yield*
|
|
322
|
+
let o = s.get(t)?.size ?? 0, c = a.map((e) => _(e.col, [...s.get(e.refTable)]));
|
|
323
|
+
yield* p(t, c.length === 1 ? c[0] : y(...c)), (s.get(t)?.size ?? 0) > o && !e.has(t) && r.push(t), (s.get(t)?.size ?? 0) > o && (n = !0);
|
|
306
324
|
}
|
|
307
325
|
for (let t of r) e.add(t);
|
|
308
|
-
if (yield*
|
|
326
|
+
if (yield* m, !n) break;
|
|
309
327
|
}
|
|
310
328
|
}
|
|
311
329
|
return o;
|
|
312
|
-
}),
|
|
330
|
+
}), Qe = (e, t) => ({
|
|
313
331
|
tables: [...e.keys()],
|
|
314
332
|
predicateFor: (n) => {
|
|
315
333
|
let r = e.get(n);
|
|
316
|
-
return r ?
|
|
334
|
+
return r ? _(W(t, n), [...r]) : void 0;
|
|
317
335
|
}
|
|
318
|
-
}),
|
|
336
|
+
}), $e = "tenantId", et = (e, t, r) => n.gen(function* () {
|
|
319
337
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
320
338
|
if (e.kind === "all") return {
|
|
321
|
-
tables: i,
|
|
339
|
+
tables: i.filter((e) => !B(e)),
|
|
322
340
|
predicateFor: () => void 0
|
|
323
341
|
};
|
|
324
342
|
if (e.kind === "tables") {
|
|
325
343
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
326
|
-
if (a.length > 0) return yield* n.fail(new
|
|
344
|
+
if (a.length > 0) return yield* n.fail(new H({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
327
345
|
let o = new Set(t);
|
|
328
346
|
return {
|
|
329
347
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -331,25 +349,25 @@ var me = (e) => {
|
|
|
331
349
|
};
|
|
332
350
|
}
|
|
333
351
|
let o = e.tenantId;
|
|
334
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
335
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
336
|
-
if (s.length === 0) return yield* n.fail(new
|
|
337
|
-
let c = yield*
|
|
338
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
352
|
+
if (o === void 0 || o === "") return yield* n.fail(new H({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
353
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === $e) ?? !1);
|
|
354
|
+
if (s.length === 0) return yield* n.fail(new H({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${$e}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
355
|
+
let c = yield* Ze(r.store, t, {
|
|
356
|
+
seeds: Object.fromEntries(s.map((e) => [e, m($e, o)])),
|
|
339
357
|
pinned: s,
|
|
340
358
|
children: { roots: s }
|
|
341
359
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
342
360
|
return {
|
|
343
361
|
tables: [...s, ...l],
|
|
344
|
-
predicateFor: (e) => u.has(e) ?
|
|
362
|
+
predicateFor: (e) => u.has(e) ? m($e, o) : c.has(e) ? _(W(t, e), [...c.get(e)]) : void 0
|
|
345
363
|
};
|
|
346
|
-
}),
|
|
364
|
+
}), tt = (e, t) => ({
|
|
347
365
|
list: t,
|
|
348
|
-
open: (t) =>
|
|
349
|
-
}),
|
|
366
|
+
open: (t) => ce(e, t)
|
|
367
|
+
}), nt = (e) => ({
|
|
350
368
|
write: (t, n, r) => D(e, t, n, r),
|
|
351
369
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
352
|
-
}),
|
|
370
|
+
}), rt = (e, t) => n.tryPromise({
|
|
353
371
|
try: async () => {
|
|
354
372
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
355
373
|
u.update(e), d += e.length, n(null, e);
|
|
@@ -367,23 +385,23 @@ var me = (e) => {
|
|
|
367
385
|
contentType: e.contentType
|
|
368
386
|
};
|
|
369
387
|
},
|
|
370
|
-
catch: (e) => new
|
|
388
|
+
catch: (e) => new V({
|
|
371
389
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
372
390
|
path: t
|
|
373
391
|
})
|
|
374
|
-
}),
|
|
375
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u =
|
|
392
|
+
}), it = (e, t, r) => n.gen(function* () {
|
|
393
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u = se("bundle", () => a(s)).pipe(l.tap((e) => n.sync(() => c.update(e))));
|
|
376
394
|
yield* e.write(r.key, u, {
|
|
377
395
|
contentType: r.contentType,
|
|
378
396
|
totalSize: r.size
|
|
379
397
|
});
|
|
380
398
|
let d = c.digest("hex");
|
|
381
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
399
|
+
if (d !== r.sha256) return yield* n.fail(new Ue({
|
|
382
400
|
what: `asset ${r.key}`,
|
|
383
401
|
expected: r.sha256,
|
|
384
402
|
actual: d
|
|
385
403
|
}));
|
|
386
|
-
}),
|
|
404
|
+
}), at = () => typeof k.createZstdCompress == "function", ot = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : at() ? "zstd" : "gzip", st = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", ct = (e) => e === "zstd" ? k.createZstdCompress() : e === "gzip" ? k.createGzip() : new A(), lt = (e) => e === "zstd" ? k.createZstdDecompress() : e === "gzip" ? k.createGunzip() : new A(), ut = 1, dt = c.Struct({
|
|
387
405
|
name: c.String,
|
|
388
406
|
file: c.String,
|
|
389
407
|
rowCount: c.Number,
|
|
@@ -393,11 +411,11 @@ var me = (e) => {
|
|
|
393
411
|
value: c.String
|
|
394
412
|
}),
|
|
395
413
|
primaryKey: c.String
|
|
396
|
-
}),
|
|
414
|
+
}), ft = c.Struct({
|
|
397
415
|
kind: c.Literal("all", "tenant", "tables"),
|
|
398
416
|
tenantId: c.optional(c.String),
|
|
399
417
|
tables: c.optional(c.Array(c.String))
|
|
400
|
-
}),
|
|
418
|
+
}), pt = c.Struct({
|
|
401
419
|
count: c.Number,
|
|
402
420
|
totalBytes: c.Number,
|
|
403
421
|
index: c.optional(c.String)
|
|
@@ -408,23 +426,23 @@ c.Struct({
|
|
|
408
426
|
size: c.Number,
|
|
409
427
|
contentType: c.String
|
|
410
428
|
});
|
|
411
|
-
var
|
|
429
|
+
var mt = c.Struct({
|
|
412
430
|
formatVersion: c.Number,
|
|
413
431
|
createdAt: c.String,
|
|
414
432
|
source: c.Struct({
|
|
415
433
|
dialect: c.String,
|
|
416
434
|
schemaFingerprint: c.String
|
|
417
435
|
}),
|
|
418
|
-
scope:
|
|
436
|
+
scope: ft,
|
|
419
437
|
consistency: c.Literal("live", "snapshot"),
|
|
420
438
|
compression: c.Literal("zstd", "gzip", "none"),
|
|
421
|
-
tables: c.Array(
|
|
422
|
-
assets:
|
|
439
|
+
tables: c.Array(dt),
|
|
440
|
+
assets: pt,
|
|
423
441
|
masking: c.optional(c.Struct({
|
|
424
442
|
policyId: c.optional(c.String),
|
|
425
443
|
transformed: c.Array(c.String)
|
|
426
444
|
}))
|
|
427
|
-
}),
|
|
445
|
+
}), ht = c.encodeSync(c.parseJson(mt)), gt = c.decodeUnknownSync(c.parseJson(mt)), G = "manifest.json", _t = "README.md", vt = "data", yt = "assets", bt = "assets/index.ndjson", xt = ".ledger.json", St = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", Ct = (e, t) => {
|
|
428
446
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
429
447
|
return `# Voltro data export
|
|
430
448
|
|
|
@@ -438,7 +456,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
438
456
|
| **Format version** | ${e.formatVersion} |
|
|
439
457
|
| **Consistency** | ${e.consistency} |
|
|
440
458
|
| **Compression** | ${e.compression} |
|
|
441
|
-
| **Scope** | ${
|
|
459
|
+
| **Scope** | ${St(e.scope)} |
|
|
442
460
|
| **Tables** | ${e.tables.length} |
|
|
443
461
|
| **Rows (total)** | ${n} |
|
|
444
462
|
| **Assets** | ${a} |
|
|
@@ -463,39 +481,39 @@ voltro data import <this-directory>
|
|
|
463
481
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
464
482
|
\`\`\`
|
|
465
483
|
`;
|
|
466
|
-
},
|
|
484
|
+
}, wt = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), Tt = a.exponential({
|
|
467
485
|
start: .01,
|
|
468
486
|
factor: 2,
|
|
469
487
|
count: 16
|
|
470
|
-
}),
|
|
488
|
+
}), Et = i.histogram("voltro_data_transfer_table_seconds", Tt, "Per-table export/import duration in seconds, tagged by phase."), K = (e, t) => {
|
|
471
489
|
if (e) try {
|
|
472
490
|
e(t);
|
|
473
491
|
} catch {}
|
|
474
|
-
},
|
|
492
|
+
}, Dt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(Et.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), wt.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
475
493
|
table: t,
|
|
476
494
|
index: r,
|
|
477
495
|
tableCount: a,
|
|
478
496
|
phase: e
|
|
479
|
-
} })),
|
|
497
|
+
} })), Ot = () => ({
|
|
480
498
|
tables: {},
|
|
481
499
|
assets: {}
|
|
482
|
-
}),
|
|
500
|
+
}), kt = (e) => n.tryPromise(async () => {
|
|
483
501
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
484
502
|
return JSON.parse(t);
|
|
485
|
-
}).pipe(n.orElseSucceed(
|
|
503
|
+
}).pipe(n.orElseSucceed(Ot)), q = (e, t) => n.tryPromise({
|
|
486
504
|
try: async () => {
|
|
487
505
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
488
506
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
489
507
|
},
|
|
490
|
-
catch: (t) => new
|
|
508
|
+
catch: (t) => new V({
|
|
491
509
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
492
510
|
path: e
|
|
493
511
|
})
|
|
494
|
-
}),
|
|
512
|
+
}), At = (e, t) => t in e.tables, jt = (e, t) => t in e.assets, Mt = /* @__PURE__ */ new Set([
|
|
495
513
|
"json",
|
|
496
514
|
"array",
|
|
497
515
|
"vector"
|
|
498
|
-
]),
|
|
516
|
+
]), Nt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", Pt = (e) => Buffer.from(e).toString("base64"), Ft = (e) => new Uint8Array(Buffer.from(e, "base64")), It = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), Lt = (e, t) => e == null ? null : t !== void 0 && Mt.has(t) ? e : t === "bytes" ? Pt(e) : t === "timestamp" || t === "date" ? It(e) : typeof e == "bigint" ? {
|
|
499
517
|
$v: "bigint",
|
|
500
518
|
d: e.toString()
|
|
501
519
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -506,30 +524,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
506
524
|
d: e.toISOString()
|
|
507
525
|
} : e instanceof Uint8Array ? {
|
|
508
526
|
$v: "bytes",
|
|
509
|
-
d:
|
|
510
|
-
} : e,
|
|
527
|
+
d: Pt(e)
|
|
528
|
+
} : e, Rt = (e) => {
|
|
511
529
|
switch (e.$v) {
|
|
512
530
|
case "bigint": return BigInt(e.d);
|
|
513
531
|
case "number": return Number(e.d);
|
|
514
532
|
case "date": return new Date(e.d);
|
|
515
|
-
case "bytes": return
|
|
533
|
+
case "bytes": return Ft(e.d);
|
|
516
534
|
default: return e;
|
|
517
535
|
}
|
|
518
|
-
},
|
|
536
|
+
}, zt = (e, t) => e == null ? null : t !== void 0 && Mt.has(t) ? e : Nt(e) ? Rt(e) : t === "bytes" ? Ft(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Bt = (e, t) => {
|
|
519
537
|
let n = {};
|
|
520
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
538
|
+
for (let [r, i] of Object.entries(e)) n[r] = Lt(i, t[r]);
|
|
521
539
|
return n;
|
|
522
|
-
},
|
|
540
|
+
}, Vt = (e, t) => {
|
|
523
541
|
let n = {};
|
|
524
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
542
|
+
for (let [r, i] of Object.entries(e)) n[r] = zt(i, t[r]);
|
|
525
543
|
return n;
|
|
526
|
-
},
|
|
527
|
-
try: () =>
|
|
528
|
-
catch: (e) => new
|
|
544
|
+
}, Ht = new TextEncoder(), Ut = (e, t) => e.pipe(l.map((e) => Ht.encode(`${JSON.stringify(Bt(e, t))}\n`))), Wt = (e, t, r) => e.pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.mapEffect((e) => n.try({
|
|
545
|
+
try: () => Vt(JSON.parse(e), r),
|
|
546
|
+
catch: (e) => new He({
|
|
529
547
|
table: t,
|
|
530
548
|
reason: String(e?.message ?? e)
|
|
531
549
|
})
|
|
532
|
-
}))),
|
|
550
|
+
}))), Gt = (e, t) => {
|
|
533
551
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
534
552
|
for (let e of t.tables) {
|
|
535
553
|
if (!n.has(e.name)) continue;
|
|
@@ -549,39 +567,39 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
549
567
|
};
|
|
550
568
|
for (let t of e) n.has(t) && s(t);
|
|
551
569
|
return i;
|
|
552
|
-
},
|
|
570
|
+
}, J = (e) => {
|
|
553
571
|
let t = e, n = [];
|
|
554
572
|
typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
|
|
555
573
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
556
|
-
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(
|
|
574
|
+
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(J(t.cause)) : n.push(r), n.length === 0) {
|
|
557
575
|
let t = String(e ?? "").trim();
|
|
558
|
-
return t === "" || t === "[object Object]" ?
|
|
576
|
+
return t === "" || t === "[object Object]" ? Kt(e) : t;
|
|
559
577
|
}
|
|
560
578
|
return n.join(": ");
|
|
561
|
-
},
|
|
579
|
+
}, Kt = (e) => {
|
|
562
580
|
try {
|
|
563
581
|
let t = JSON.stringify(e);
|
|
564
582
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
565
583
|
} catch {
|
|
566
584
|
return "unknown error (not serialisable)";
|
|
567
585
|
}
|
|
568
|
-
},
|
|
586
|
+
}, qt = (e, t) => {
|
|
569
587
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
570
588
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
571
589
|
return r;
|
|
572
|
-
},
|
|
590
|
+
}, Jt = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), Yt = (e, t) => {
|
|
573
591
|
if (t.length === 0) return e;
|
|
574
592
|
let n = { ...e };
|
|
575
593
|
for (let e of t) delete n[e];
|
|
576
594
|
return n;
|
|
577
|
-
},
|
|
595
|
+
}, Xt = (e, t) => {
|
|
578
596
|
let n = e.tables.find((e) => e.name === t);
|
|
579
597
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
580
598
|
let r = n.columns.find((e) => e.type === "id");
|
|
581
599
|
if (r !== void 0) return { column: r.name };
|
|
582
600
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
583
601
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
584
|
-
},
|
|
602
|
+
}, Zt = (e, t, r) => n.tryPromise({
|
|
585
603
|
try: async () => {
|
|
586
604
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
587
605
|
d.update(e);
|
|
@@ -590,7 +608,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
590
608
|
n(null, e);
|
|
591
609
|
} }), m = c.fromWeb(l.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
592
610
|
try {
|
|
593
|
-
await s(m, p,
|
|
611
|
+
await s(m, p, ct(r), n(h)), await i.rename(h, t);
|
|
594
612
|
} catch (e) {
|
|
595
613
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
596
614
|
}
|
|
@@ -599,27 +617,27 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
599
617
|
checksum: d.digest("hex")
|
|
600
618
|
};
|
|
601
619
|
},
|
|
602
|
-
catch: (e) => new
|
|
603
|
-
reason: `write table failed: ${
|
|
620
|
+
catch: (e) => new V({
|
|
621
|
+
reason: `write table failed: ${J(e)}`,
|
|
604
622
|
path: t
|
|
605
623
|
})
|
|
606
|
-
}),
|
|
624
|
+
}), Qt = (e, t) => n.tryPromise({
|
|
607
625
|
try: async () => {
|
|
608
626
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
609
627
|
},
|
|
610
|
-
catch: (t) => new
|
|
611
|
-
reason: `append failed: ${
|
|
628
|
+
catch: (t) => new V({
|
|
629
|
+
reason: `append failed: ${J(t)}`,
|
|
612
630
|
path: e
|
|
613
631
|
})
|
|
614
|
-
}),
|
|
632
|
+
}), $t = (e, t) => n.gen(function* () {
|
|
615
633
|
let r = [], i = t.ordered.length;
|
|
616
634
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
617
|
-
let o = t.ordered[a], s =
|
|
618
|
-
if ("refusal" in c) return yield* n.fail(new
|
|
619
|
-
let u = c.column, d = `${
|
|
620
|
-
if (
|
|
635
|
+
let o = t.ordered[a], s = qt(t.snapshot, o), c = Xt(t.snapshot, o);
|
|
636
|
+
if ("refusal" in c) return yield* n.fail(new H({ reason: `export: ${c.refusal}` }));
|
|
637
|
+
let u = c.column, d = `${vt}/${o}.ndjson${st(t.compression)}`;
|
|
638
|
+
if (At(t.ledger, o)) {
|
|
621
639
|
let e = t.ledger.tables[o];
|
|
622
|
-
|
|
640
|
+
K(t.onProgress, {
|
|
623
641
|
phase: "export",
|
|
624
642
|
table: o,
|
|
625
643
|
index: a,
|
|
@@ -637,7 +655,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
637
655
|
});
|
|
638
656
|
continue;
|
|
639
657
|
}
|
|
640
|
-
|
|
658
|
+
K(t.onProgress, {
|
|
641
659
|
phase: "export",
|
|
642
660
|
table: o,
|
|
643
661
|
index: a,
|
|
@@ -645,75 +663,75 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
645
663
|
event: "start",
|
|
646
664
|
rowsDone: 0
|
|
647
665
|
});
|
|
648
|
-
let f = t.resolved.predicateFor(o), p =
|
|
666
|
+
let f = t.resolved.predicateFor(o), p = x(e, {
|
|
649
667
|
table: o,
|
|
650
668
|
pk: u,
|
|
651
669
|
chunkSize: t.chunkSize,
|
|
652
670
|
retry: t.retry,
|
|
653
671
|
...f === void 0 ? {} : { where: f }
|
|
654
|
-
}), m =
|
|
672
|
+
}), m = Jt(t.snapshot, o), h = m.length === 0 ? p : p.pipe(l.map((e) => Yt(e, m))), g = t.mask ? h.pipe(l.map((e) => Re(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* Dt("export", o, a, i, (e) => e.rowCount, Zt(Ut(g, s), t.bundleFile(d), t.compression));
|
|
655
673
|
t.ledger.tables[o] = {
|
|
656
|
-
rowCount:
|
|
657
|
-
checksum:
|
|
658
|
-
}, yield*
|
|
674
|
+
rowCount: _,
|
|
675
|
+
checksum: v
|
|
676
|
+
}, yield* q(t.ledgerPath, t.ledger), K(t.onProgress, {
|
|
659
677
|
phase: "export",
|
|
660
678
|
table: o,
|
|
661
679
|
index: a,
|
|
662
680
|
tableCount: i,
|
|
663
681
|
event: "done",
|
|
664
|
-
rowsDone:
|
|
665
|
-
total:
|
|
682
|
+
rowsDone: _,
|
|
683
|
+
total: _
|
|
666
684
|
}), r.push({
|
|
667
685
|
name: o,
|
|
668
686
|
file: d,
|
|
669
|
-
rowCount:
|
|
670
|
-
checksum:
|
|
687
|
+
rowCount: _,
|
|
688
|
+
checksum: v,
|
|
671
689
|
columnTypes: s,
|
|
672
690
|
primaryKey: u
|
|
673
691
|
});
|
|
674
692
|
}
|
|
675
693
|
return r;
|
|
676
|
-
}),
|
|
694
|
+
}), en = (e) => ({
|
|
677
695
|
_tag: "RawSqlFragment",
|
|
678
696
|
strings: [e],
|
|
679
697
|
values: []
|
|
680
|
-
}),
|
|
698
|
+
}), tn = (t, i, a) => n.tryPromise({
|
|
681
699
|
try: () => t.transactional(async (t) => {
|
|
682
|
-
if (t.raw) for (let e of
|
|
700
|
+
if (t.raw) for (let e of be(i)) await t.raw(en(e));
|
|
683
701
|
let o = await n.runPromiseExit(a(t));
|
|
684
702
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
685
703
|
return o.value;
|
|
686
704
|
}),
|
|
687
|
-
catch: (e) => e instanceof
|
|
688
|
-
}),
|
|
689
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
705
|
+
catch: (e) => e instanceof V ? e : new V({ reason: `snapshot export failed: ${J(e)}` })
|
|
706
|
+
}), nn = (e) => n.gen(function* () {
|
|
707
|
+
let t = e.scope ?? { kind: "all" }, r = ot(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, o = s.exponential("50 millis").pipe(s.jittered, s.intersect(s.recurs(a))), c = yield* n.promise(() => import("node:path")), u = (t) => c.join(e.outDir, t);
|
|
690
708
|
yield* n.tryPromise({
|
|
691
709
|
try: async () => {
|
|
692
710
|
let t = await import("node:fs/promises");
|
|
693
|
-
await t.mkdir(u(
|
|
711
|
+
await t.mkdir(u(vt), { recursive: !0 }), e.assets && await t.mkdir(u(yt), { recursive: !0 });
|
|
694
712
|
},
|
|
695
|
-
catch: (t) => new
|
|
696
|
-
reason: `mkdir failed: ${
|
|
713
|
+
catch: (t) => new V({
|
|
714
|
+
reason: `mkdir failed: ${J(t)}`,
|
|
697
715
|
path: e.outDir
|
|
698
716
|
})
|
|
699
717
|
});
|
|
700
|
-
let d = u(
|
|
718
|
+
let d = u(xt), f = yield* kt(d), p = e.subset ? Qe(yield* Ze(e.store, e.snapshot, e.subset), e.snapshot) : yield* et(t, e.snapshot, {
|
|
701
719
|
store: e.store,
|
|
702
720
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
703
|
-
}), m =
|
|
721
|
+
}), m = Gt(p.tables, e.snapshot), h = e.consistency ?? "live", g, _;
|
|
704
722
|
if (e.masking) {
|
|
705
|
-
let t = e.classification ??
|
|
723
|
+
let t = e.classification ?? Se(e.snapshot), r = Fe(m.map((t) => ({
|
|
706
724
|
name: t,
|
|
707
725
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
708
726
|
name: e.name,
|
|
709
727
|
type: e.type
|
|
710
728
|
}))
|
|
711
729
|
})), t, e.masking);
|
|
712
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
730
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new qe({
|
|
713
731
|
unclassified: r.unclassified,
|
|
714
732
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
715
733
|
}));
|
|
716
|
-
|
|
734
|
+
_ = {
|
|
717
735
|
actions: r.actions,
|
|
718
736
|
seed: e.masking.seed
|
|
719
737
|
};
|
|
@@ -723,7 +741,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
723
741
|
transformed: i
|
|
724
742
|
};
|
|
725
743
|
}
|
|
726
|
-
let
|
|
744
|
+
let v = {
|
|
727
745
|
snapshot: e.snapshot,
|
|
728
746
|
ordered: m,
|
|
729
747
|
resolved: p,
|
|
@@ -733,36 +751,36 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
733
751
|
ledger: f,
|
|
734
752
|
ledgerPath: d,
|
|
735
753
|
bundleFile: u,
|
|
736
|
-
...
|
|
754
|
+
..._ ? { mask: _ } : {},
|
|
737
755
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
738
|
-
},
|
|
756
|
+
}, y = h === "snapshot" ? yield* tn(e.store, e.dialect, (e) => $t(e, v)) : yield* $t(e.store, v), b = 0, ee = 0;
|
|
739
757
|
if (e.assets) {
|
|
740
|
-
let t = u(
|
|
741
|
-
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
758
|
+
let t = u(bt);
|
|
759
|
+
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => rt(e, u(yt))), n.flatMap((e) => {
|
|
742
760
|
let i = {
|
|
743
761
|
key: r.key,
|
|
744
762
|
sha256: e.sha256,
|
|
745
763
|
size: e.size,
|
|
746
764
|
contentType: e.contentType
|
|
747
765
|
};
|
|
748
|
-
return
|
|
749
|
-
}))).pipe(n.mapError((e) => e instanceof
|
|
766
|
+
return b++, ee += e.size, f.assets[r.key] = !0, Qt(t, JSON.stringify(i)).pipe(n.andThen(q(d, f)));
|
|
767
|
+
}))).pipe(n.mapError((e) => e instanceof V ? e : new V({ reason: `asset export failed: ${String(e)}` })));
|
|
750
768
|
}
|
|
751
|
-
let
|
|
769
|
+
let x = {
|
|
752
770
|
formatVersion: 1,
|
|
753
771
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
754
772
|
source: {
|
|
755
773
|
dialect: e.dialect,
|
|
756
|
-
schemaFingerprint:
|
|
774
|
+
schemaFingerprint: re(e.snapshot)
|
|
757
775
|
},
|
|
758
776
|
scope: t,
|
|
759
777
|
consistency: h,
|
|
760
778
|
compression: r,
|
|
761
|
-
tables:
|
|
762
|
-
assets:
|
|
763
|
-
count:
|
|
764
|
-
totalBytes:
|
|
765
|
-
index:
|
|
779
|
+
tables: y,
|
|
780
|
+
assets: b > 0 ? {
|
|
781
|
+
count: b,
|
|
782
|
+
totalBytes: ee,
|
|
783
|
+
index: bt
|
|
766
784
|
} : {
|
|
767
785
|
count: 0,
|
|
768
786
|
totalBytes: 0
|
|
@@ -771,23 +789,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
771
789
|
};
|
|
772
790
|
return yield* n.tryPromise({
|
|
773
791
|
try: async () => {
|
|
774
|
-
await (await import("node:fs/promises")).writeFile(u(
|
|
792
|
+
await (await import("node:fs/promises")).writeFile(u(G), ht(x));
|
|
775
793
|
},
|
|
776
|
-
catch: (e) => new
|
|
777
|
-
reason: `manifest write failed: ${
|
|
778
|
-
path:
|
|
794
|
+
catch: (e) => new V({
|
|
795
|
+
reason: `manifest write failed: ${J(e)}`,
|
|
796
|
+
path: G
|
|
779
797
|
})
|
|
780
798
|
}), yield* n.tryPromise({
|
|
781
799
|
try: async () => {
|
|
782
800
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
783
|
-
await e.writeFile(u(
|
|
801
|
+
await e.writeFile(u(_t), Ct(x, { hostname: t.hostname() }));
|
|
784
802
|
},
|
|
785
|
-
catch: (e) => new
|
|
786
|
-
reason: `readme write failed: ${
|
|
787
|
-
path:
|
|
803
|
+
catch: (e) => new V({
|
|
804
|
+
reason: `readme write failed: ${J(e)}`,
|
|
805
|
+
path: _t
|
|
788
806
|
})
|
|
789
|
-
}),
|
|
790
|
-
}),
|
|
807
|
+
}), x;
|
|
808
|
+
}), rn = (e, t) => {
|
|
791
809
|
let n = new Set(t), r = [];
|
|
792
810
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
793
811
|
let i = e.references?.table;
|
|
@@ -798,7 +816,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
798
816
|
});
|
|
799
817
|
}
|
|
800
818
|
return r;
|
|
801
|
-
},
|
|
819
|
+
}, an = (e) => {
|
|
802
820
|
let t = /* @__PURE__ */ new Map();
|
|
803
821
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
804
822
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -806,7 +824,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
806
824
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
807
825
|
});
|
|
808
826
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
809
|
-
},
|
|
827
|
+
}, on = (e) => {
|
|
810
828
|
switch (e) {
|
|
811
829
|
case "id":
|
|
812
830
|
case "text":
|
|
@@ -825,12 +843,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
825
843
|
case "raw": return "raw";
|
|
826
844
|
default: return "string";
|
|
827
845
|
}
|
|
828
|
-
},
|
|
846
|
+
}, sn = {
|
|
829
847
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
830
848
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
831
849
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
832
|
-
},
|
|
833
|
-
let i =
|
|
850
|
+
}, cn = (e, t, n, r) => {
|
|
851
|
+
let i = on(n);
|
|
834
852
|
if (i === "raw") return {
|
|
835
853
|
table: e,
|
|
836
854
|
column: t,
|
|
@@ -838,7 +856,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
838
856
|
severity: "block",
|
|
839
857
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
840
858
|
};
|
|
841
|
-
let a =
|
|
859
|
+
let a = sn[i];
|
|
842
860
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
843
861
|
table: e,
|
|
844
862
|
column: t,
|
|
@@ -852,15 +870,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
852
870
|
severity: "block",
|
|
853
871
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
854
872
|
};
|
|
855
|
-
},
|
|
873
|
+
}, ln = (e, t, n) => {
|
|
856
874
|
if (t === n) return [];
|
|
857
875
|
let r = [];
|
|
858
876
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
859
|
-
let a =
|
|
877
|
+
let a = cn(t.name, e, i, n);
|
|
860
878
|
a && r.push(a);
|
|
861
879
|
}
|
|
862
880
|
return r;
|
|
863
|
-
},
|
|
881
|
+
}, un = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, dn = (e) => {
|
|
864
882
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
865
883
|
switch (e.kind) {
|
|
866
884
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -869,37 +887,164 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
869
887
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
870
888
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
871
889
|
}
|
|
872
|
-
},
|
|
890
|
+
}, fn = (e) => {
|
|
873
891
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
874
892
|
return t.length > 0 ? t.join("/") : void 0;
|
|
875
|
-
},
|
|
893
|
+
}, pn = "Failed to execute statement", mn = (e) => {
|
|
876
894
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
877
895
|
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
878
|
-
return t === `SqlError: ${
|
|
879
|
-
},
|
|
896
|
+
return t === `SqlError: ${pn}` ? pn : t;
|
|
897
|
+
}, hn = (e) => {
|
|
880
898
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
881
899
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
882
900
|
unclassified: t,
|
|
883
901
|
raw: n
|
|
884
|
-
}), i =
|
|
885
|
-
if (a !== void 0) return r(`${
|
|
886
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message,
|
|
887
|
-
return r(
|
|
888
|
-
},
|
|
889
|
-
let t =
|
|
902
|
+
}), i = h(e), a = f(i), o = fn(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
903
|
+
if (a !== void 0) return r(`${dn(a)}${s}`, !1);
|
|
904
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = mn(typeof l == "string" && l.length > 0 ? l : String(e)), d = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, p = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, m = p !== void 0 && /[←‖]/.test(p) ? ` (via ${p})` : "";
|
|
905
|
+
return r(d === pn || d.length === 0 ? `the database refused the write and the driver gave no detail${s}${m}` : `${d}${s}`, d === pn || d.length === 0);
|
|
906
|
+
}, gn = (e) => hn(e).reason, _n = (e) => {
|
|
907
|
+
let t = h(e), n = f(t);
|
|
890
908
|
if (n?.kind === "foreignKeyInUse") {
|
|
891
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
909
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = fn(t);
|
|
892
910
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
893
911
|
}
|
|
894
|
-
return
|
|
895
|
-
},
|
|
912
|
+
return gn(e);
|
|
913
|
+
}, vn = "_voltro_staging_", Y = (e) => `${vn}${e}`, yn = (e) => typeof e.run == "function", bn = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
914
|
+
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
915
|
+
written before the first delete is what covers that case.`, xn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, X = (e, t) => {
|
|
916
|
+
if (!xn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
917
|
+
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
918
|
+
}, Sn = (e, t) => {
|
|
919
|
+
let n = X(e, t), r = X(Y(e), t);
|
|
920
|
+
switch (t) {
|
|
921
|
+
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
922
|
+
case "mysql":
|
|
923
|
+
case "mariadb": return `CREATE TABLE ${r} LIKE ${n}`;
|
|
924
|
+
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
925
|
+
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
926
|
+
}
|
|
927
|
+
}, Cn = (e, t) => {
|
|
928
|
+
let n = X(Y(e), t);
|
|
929
|
+
return t === "mssql" ? `IF OBJECT_ID('${Y(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
930
|
+
}, wn = (e, t) => {
|
|
931
|
+
let n = t === "mysql" || t === "mariadb", r = [];
|
|
932
|
+
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
933
|
+
for (let n of [...e].reverse()) r.push(`DELETE FROM ${X(n.name, t)}`);
|
|
934
|
+
for (let n of e) {
|
|
935
|
+
if (n.columns.length === 0) throw Error(`refusing to build a swap for '${n.name}' with no columns: an empty list would have to mean \`SELECT *\`, which is the shape that fails on a generated column.`);
|
|
936
|
+
let e = n.columns.map((e) => X(e, t)).join(", ");
|
|
937
|
+
r.push(`INSERT INTO ${X(n.name, t)} (${e}) SELECT ${e} FROM ${X(Y(n.name), t)}`);
|
|
938
|
+
}
|
|
939
|
+
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
940
|
+
}, Tn = (e, t) => {
|
|
941
|
+
let n = [], r = () => {
|
|
942
|
+
for (let e of n) t.unregisterTable(e);
|
|
943
|
+
};
|
|
944
|
+
try {
|
|
945
|
+
for (let r of e) {
|
|
946
|
+
let e = t.getTable(r);
|
|
947
|
+
if (e === void 0) continue;
|
|
948
|
+
let i = Y(r);
|
|
949
|
+
t.registerTable({
|
|
950
|
+
...e,
|
|
951
|
+
tableName: i,
|
|
952
|
+
isReactive: !1
|
|
953
|
+
}), n.push(i);
|
|
954
|
+
}
|
|
955
|
+
} catch (e) {
|
|
956
|
+
throw r(), e;
|
|
957
|
+
}
|
|
958
|
+
return {
|
|
959
|
+
staged: n,
|
|
960
|
+
undo: r
|
|
961
|
+
};
|
|
962
|
+
}, En = (e, t) => {
|
|
963
|
+
let n = new Set(t), r = [];
|
|
964
|
+
for (let t of e.tables) {
|
|
965
|
+
if (!n.has(t.name)) continue;
|
|
966
|
+
let e = t.primaryKey?.[0] ?? "id";
|
|
967
|
+
for (let i of t.columns) {
|
|
968
|
+
let a = i.references;
|
|
969
|
+
a === void 0 || !n.has(a.table) || r.push({
|
|
970
|
+
from: t.name,
|
|
971
|
+
column: i.name,
|
|
972
|
+
to: a.table,
|
|
973
|
+
toColumn: a.column,
|
|
974
|
+
primaryKey: e
|
|
975
|
+
});
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
return r;
|
|
979
|
+
}, Dn = (e, t) => {
|
|
980
|
+
let n = X(Y(e.from), t), r = X(Y(e.to), t), i = X(e.column, t), a = X(e.toColumn, t);
|
|
981
|
+
return `SELECT ${X(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
982
|
+
}, On = (e) => {
|
|
983
|
+
if (e.length === 0) return;
|
|
984
|
+
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
985
|
+
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
986
|
+
}, kn = async (e, t) => {
|
|
987
|
+
t.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
988
|
+
}, An = async (e, t, r) => {
|
|
989
|
+
let i = wn(t, r);
|
|
990
|
+
i.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
991
|
+
}, jn = async (e, t, r) => {
|
|
992
|
+
let i = [];
|
|
993
|
+
for (let a of t) {
|
|
994
|
+
let t = await e.run(P.SqlClient.pipe(n.flatMap((e) => e.unsafe(Dn(a, r)))));
|
|
995
|
+
for (let e of t) i.push({
|
|
996
|
+
table: a.from,
|
|
997
|
+
primaryKey: String(e.pk ?? ""),
|
|
998
|
+
column: a.column,
|
|
999
|
+
value: e.value,
|
|
1000
|
+
references: a.to
|
|
1001
|
+
});
|
|
1002
|
+
}
|
|
1003
|
+
return i;
|
|
1004
|
+
}, Mn = (e, t) => {
|
|
1005
|
+
let n = new Map(t.map((e, t) => [e, t]));
|
|
1006
|
+
return e.filter((e) => {
|
|
1007
|
+
if (e.from === e.to) return !1;
|
|
1008
|
+
let t = n.get(e.from), r = n.get(e.to);
|
|
1009
|
+
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1010
|
+
});
|
|
1011
|
+
}, Nn = (e) => e === "mysql" || e === "mariadb", Pn = (e) => n.gen(function* () {
|
|
1012
|
+
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1013
|
+
if (e.snapshot === void 0) return yield* n.fail(new V({
|
|
1014
|
+
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
1015
|
+
path: e.dir
|
|
1016
|
+
}));
|
|
1017
|
+
let t = {
|
|
1018
|
+
...e.snapshot,
|
|
1019
|
+
tables: e.snapshot.tables.filter((t) => e.tables.includes(t.name))
|
|
1020
|
+
};
|
|
1021
|
+
if (t.tables.length === 0) return;
|
|
1022
|
+
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1023
|
+
so it survives a run whose process does not.`);
|
|
1024
|
+
let r = yield* nn({
|
|
1025
|
+
store: e.store,
|
|
1026
|
+
snapshot: t,
|
|
1027
|
+
dialect: e.dialect ?? "unknown",
|
|
1028
|
+
outDir: e.dir,
|
|
1029
|
+
scope: { kind: "all" },
|
|
1030
|
+
compression: "none"
|
|
1031
|
+
}).pipe(n.mapError((t) => new V({
|
|
1032
|
+
reason: `rollback capture failed, so the import did not start and the target is untouched: ${t.reason ?? String(t)}\n Fix the capture, or re-run with \`--no-rollback\` to proceed without one.`,
|
|
1033
|
+
path: e.dir
|
|
1034
|
+
})));
|
|
1035
|
+
return {
|
|
1036
|
+
dir: e.dir,
|
|
1037
|
+
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1038
|
+
tables: r.tables.length
|
|
1039
|
+
};
|
|
1040
|
+
}), Fn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, In = (e, t) => e.tables.find((e) => e.name === t), Ln = (e, t) => new Map((In(e, t)?.columns ?? []).map((e) => [e.name, e])), Rn = (e, t) => {
|
|
896
1041
|
let n = [];
|
|
897
1042
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
898
|
-
if (
|
|
1043
|
+
if (In(t, r.name) === void 0) {
|
|
899
1044
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
900
1045
|
continue;
|
|
901
1046
|
}
|
|
902
|
-
let e =
|
|
1047
|
+
let e = Ln(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
903
1048
|
for (let [t, a] of i) {
|
|
904
1049
|
let i = e.get(t);
|
|
905
1050
|
if (i === void 0) {
|
|
@@ -911,22 +1056,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
911
1056
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
912
1057
|
}
|
|
913
1058
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
914
|
-
},
|
|
915
|
-
let r =
|
|
1059
|
+
}, zn = (e, t, n) => {
|
|
1060
|
+
let r = re(t);
|
|
916
1061
|
if (r === e || n === void 0) return {
|
|
917
1062
|
drifted: !1,
|
|
918
1063
|
bundleFingerprint: e,
|
|
919
1064
|
targetFingerprint: r,
|
|
920
1065
|
diff: []
|
|
921
1066
|
};
|
|
922
|
-
let i =
|
|
1067
|
+
let i = Rn(n, t);
|
|
923
1068
|
return {
|
|
924
1069
|
drifted: i.length > 0,
|
|
925
1070
|
bundleFingerprint: e,
|
|
926
1071
|
targetFingerprint: r,
|
|
927
1072
|
diff: i
|
|
928
1073
|
};
|
|
929
|
-
},
|
|
1074
|
+
}, Bn = 200, Vn = /* @__PURE__ */ new Set([
|
|
930
1075
|
"BundleError",
|
|
931
1076
|
"CodecError",
|
|
932
1077
|
"IntegrityError",
|
|
@@ -935,361 +1080,474 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
935
1080
|
"RowsRefusedError",
|
|
936
1081
|
"SchemaDriftError",
|
|
937
1082
|
"StorageError"
|
|
938
|
-
]),
|
|
1083
|
+
]), Hn = (t) => {
|
|
939
1084
|
let n = o.isFiberFailure(t) ? [...e.failures(t[o.FiberFailureCauseId]), ...e.defects(t[o.FiberFailureCauseId])] : [t];
|
|
940
1085
|
for (let e of n) {
|
|
941
1086
|
let t = e?._tag;
|
|
942
|
-
if (typeof t == "string" &&
|
|
1087
|
+
if (typeof t == "string" && Vn.has(t)) return e;
|
|
943
1088
|
if (o.isFiberFailure(e)) {
|
|
944
|
-
let t =
|
|
1089
|
+
let t = Hn(e);
|
|
945
1090
|
if (t) return t;
|
|
946
1091
|
}
|
|
947
1092
|
}
|
|
948
|
-
},
|
|
1093
|
+
}, Un = (t) => [...e.failures(t)][0] ?? Error(e.pretty(t)), Wn = (e) => n.tryPromise({
|
|
949
1094
|
try: async () => {
|
|
950
1095
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
951
|
-
return
|
|
1096
|
+
return gt(await t.readFile(n.join(e, G), "utf8"));
|
|
952
1097
|
},
|
|
953
|
-
catch: (e) => new
|
|
1098
|
+
catch: (e) => new V({
|
|
954
1099
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
955
|
-
path:
|
|
1100
|
+
path: G
|
|
956
1101
|
})
|
|
957
|
-
}),
|
|
958
|
-
let t
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
if (
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
tables:
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
1102
|
+
}), Gn = (e) => {
|
|
1103
|
+
let t;
|
|
1104
|
+
return n.gen(function* () {
|
|
1105
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* Wn(e.bundleDir);
|
|
1106
|
+
if (a.formatVersion !== 1) return yield* n.fail(new V({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
1107
|
+
if (e.tables !== void 0) {
|
|
1108
|
+
let t = new Set(a.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
1109
|
+
if (r.length > 0) return yield* n.fail(new V({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...t].sort().join(", ")}.` }));
|
|
1110
|
+
}
|
|
1111
|
+
let o = e.tables === void 0 ? void 0 : new Set(e.tables), s = o === void 0 ? a : {
|
|
1112
|
+
...a,
|
|
1113
|
+
tables: a.tables.filter((e) => o.has(e.name))
|
|
1114
|
+
};
|
|
1115
|
+
if (e.targetSnapshot) {
|
|
1116
|
+
let t = { tables: s.tables.map((e) => ({
|
|
1117
|
+
name: e.name,
|
|
1118
|
+
indexes: [],
|
|
1119
|
+
columns: Object.entries(e.columnTypes).map(([e, t]) => ({
|
|
1120
|
+
name: e,
|
|
1121
|
+
type: t,
|
|
1122
|
+
nullable: !0,
|
|
1123
|
+
unique: !1,
|
|
1124
|
+
hasDefault: !1
|
|
1125
|
+
}))
|
|
1126
|
+
})) }, r = zn(s.source.schemaFingerprint, e.targetSnapshot, t);
|
|
1127
|
+
if (r.drifted) {
|
|
1128
|
+
if (!e.force) return yield* n.fail(new Ke({
|
|
1129
|
+
bundleFingerprint: r.bundleFingerprint,
|
|
1130
|
+
targetFingerprint: r.targetFingerprint,
|
|
1131
|
+
diff: r.diff
|
|
1132
|
+
}));
|
|
1133
|
+
e.onWarn?.(`schema drift: the target schema (fingerprint ${r.targetFingerprint.slice(0, 16)}) differs from the bundle's source (${r.bundleFingerprint.slice(0, 16)}); importing anyway (--force). ${r.diff.join("; ")}`);
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
if (e.targetDialect && e.targetDialect !== s.source.dialect) {
|
|
1137
|
+
let t = ln(s.tables, s.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1138
|
+
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new Ye({
|
|
1139
|
+
sourceDialect: s.source.dialect,
|
|
1140
|
+
targetDialect: e.targetDialect,
|
|
1141
|
+
blocking: t.map(un)
|
|
985
1142
|
}));
|
|
986
|
-
e.onWarn?.(`schema drift: the target schema (fingerprint ${r.targetFingerprint.slice(0, 16)}) differs from the bundle's source (${r.bundleFingerprint.slice(0, 16)}); importing anyway (--force). ${r.diff.join("; ")}`);
|
|
987
1143
|
}
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
sourceDialect: o.source.dialect,
|
|
993
|
-
targetDialect: e.targetDialect,
|
|
994
|
-
blocking: t.map(Xt)
|
|
1144
|
+
let c = e.verify ?? !0, f = e.mode ?? "upsert", m = e.atomic ?? f === "replace", h = e.onConflict ?? "skip", _ = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), y = yield* kt(_);
|
|
1145
|
+
if (f === "replace" && s.scope.kind !== "all") return yield* n.fail(new Je({
|
|
1146
|
+
mode: f,
|
|
1147
|
+
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
995
1148
|
}));
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1149
|
+
if (f === "replace") {
|
|
1150
|
+
let t = s.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1151
|
+
try: () => e.store.incomingForeignKeys(t),
|
|
1152
|
+
catch: (e) => new V({ reason: `foreign-key pre-flight failed: ${_n(e)}` })
|
|
1153
|
+
}) : e.targetSnapshot === void 0 ? [] : rn(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1154
|
+
for (let t of r) {
|
|
1155
|
+
if (!a.has(t.from)) {
|
|
1156
|
+
let r = yield* n.tryPromise({
|
|
1157
|
+
try: () => e.store.query({
|
|
1158
|
+
table: t.from,
|
|
1159
|
+
predicate: void 0,
|
|
1160
|
+
order: [],
|
|
1161
|
+
take: void 0,
|
|
1162
|
+
skip: void 0,
|
|
1163
|
+
projection: []
|
|
1164
|
+
}),
|
|
1165
|
+
catch: () => new V({ reason: `unreadable: ${t.from}` })
|
|
1166
|
+
}).pipe(n.map((e) => e.length), n.orElseSucceed(() => 0));
|
|
1167
|
+
a.set(t.from, r);
|
|
1168
|
+
}
|
|
1169
|
+
let r = a.get(t.from) ?? 0;
|
|
1170
|
+
r > 0 && i.push({
|
|
1171
|
+
...t,
|
|
1172
|
+
rowCount: r
|
|
1173
|
+
});
|
|
1021
1174
|
}
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
});
|
|
1175
|
+
if (i.length > 0) return yield* n.fail(new Je({
|
|
1176
|
+
mode: f,
|
|
1177
|
+
reason: an(i)
|
|
1178
|
+
}));
|
|
1027
1179
|
}
|
|
1028
|
-
if (
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
tablesWritten: 0,
|
|
1036
|
-
rowsWritten: 0,
|
|
1037
|
-
tablesSkipped: 0,
|
|
1038
|
-
rowsSkipped: 0,
|
|
1039
|
-
fullyResumed: !1
|
|
1040
|
-
};
|
|
1041
|
-
let h = /* @__PURE__ */ new Map();
|
|
1042
|
-
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1043
|
-
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1044
|
-
e.length > 0 && h.set(t.name, e);
|
|
1045
|
-
}
|
|
1046
|
-
let g = (e, t) => {
|
|
1047
|
-
let n = h.get(e);
|
|
1048
|
-
if (n === void 0) return t;
|
|
1049
|
-
let r;
|
|
1050
|
-
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1051
|
-
return r ?? t;
|
|
1052
|
-
}, _ = (e, t, n, r = !1) => {
|
|
1053
|
-
let i = t.name, a = t.primaryKey, o = g(i, n);
|
|
1054
|
-
return c === "append" ? d === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : c === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1055
|
-
}, v = (e, t) => e.savepoint ? e.savepoint(t) : t(), y = 0, b = (t, n, r) => {
|
|
1056
|
-
r.unclassified && (y++, !(y > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (y === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1057
|
-
}, x = (e, t, r = !0) => n.tryPromise({
|
|
1058
|
-
try: e,
|
|
1059
|
-
catch: (e) => en(e)
|
|
1060
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, b(t.entry, t.row, e)), !1)))), S = (e) => {
|
|
1061
|
-
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1062
|
-
if (t.length !== 0) return {
|
|
1063
|
-
...e.row,
|
|
1064
|
-
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1180
|
+
if (e.dryRun === !0) return {
|
|
1181
|
+
manifest: s,
|
|
1182
|
+
tablesWritten: 0,
|
|
1183
|
+
rowsWritten: 0,
|
|
1184
|
+
tablesSkipped: 0,
|
|
1185
|
+
rowsSkipped: 0,
|
|
1186
|
+
fullyResumed: !1
|
|
1065
1187
|
};
|
|
1066
|
-
|
|
1067
|
-
let t
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
return i != null && t.has(String(i));
|
|
1071
|
-
}), r = e.map((e) => ({
|
|
1072
|
-
table: e.entry.name,
|
|
1073
|
-
id: String(e.row[e.entry.primaryKey]),
|
|
1074
|
-
reason: e.reason,
|
|
1075
|
-
derived: n(e)
|
|
1076
|
-
})), i = [...r.filter((e) => !e.derived), ...r.filter((e) => e.derived)], a = /* @__PURE__ */ new Map();
|
|
1077
|
-
for (let e of r) {
|
|
1078
|
-
let t = a.get(e.table) ?? {
|
|
1079
|
-
refused: 0,
|
|
1080
|
-
primary: 0
|
|
1081
|
-
};
|
|
1082
|
-
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1188
|
+
let x = /* @__PURE__ */ new Map();
|
|
1189
|
+
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1190
|
+
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1191
|
+
e.length > 0 && x.set(t.name, e);
|
|
1083
1192
|
}
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
})
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
if (
|
|
1101
|
-
|
|
1102
|
-
|
|
1193
|
+
let re = (e, t) => {
|
|
1194
|
+
let n = x.get(e);
|
|
1195
|
+
if (n === void 0) return t;
|
|
1196
|
+
let r;
|
|
1197
|
+
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1198
|
+
return r ?? t;
|
|
1199
|
+
}, C = (e, t, n, r = !1) => {
|
|
1200
|
+
let i = j ? Y(t.name) : t.name, a = t.primaryKey, o = re(t.name, n);
|
|
1201
|
+
return f === "append" ? h === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : f === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1202
|
+
}, w = (e, t) => e.savepoint ? e.savepoint(t) : t(), T = 0, ie = (t, n, r) => {
|
|
1203
|
+
r.unclassified && (T++, !(T > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (T === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1204
|
+
}, E = (e, t, r = !0) => n.tryPromise({
|
|
1205
|
+
try: e,
|
|
1206
|
+
catch: (e) => hn(e)
|
|
1207
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, ie(t.entry, t.row, e)), !1)))), ae = (e) => {
|
|
1208
|
+
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1209
|
+
if (t.length !== 0) return {
|
|
1210
|
+
...e.row,
|
|
1211
|
+
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1212
|
+
};
|
|
1213
|
+
}, oe = (e) => {
|
|
1214
|
+
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
1215
|
+
if (r !== "reference") return !1;
|
|
1216
|
+
let i = e.row[n];
|
|
1217
|
+
return i != null && t.has(String(i));
|
|
1218
|
+
}), r = e.map((e) => ({
|
|
1219
|
+
table: e.entry.name,
|
|
1220
|
+
id: String(e.row[e.entry.primaryKey]),
|
|
1221
|
+
reason: e.reason,
|
|
1222
|
+
derived: n(e)
|
|
1223
|
+
})), i = [...r.filter((e) => !e.derived), ...r.filter((e) => e.derived)], a = /* @__PURE__ */ new Map();
|
|
1224
|
+
for (let e of r) {
|
|
1225
|
+
let t = a.get(e.table) ?? {
|
|
1226
|
+
refused: 0,
|
|
1227
|
+
primary: 0
|
|
1228
|
+
};
|
|
1229
|
+
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1103
1230
|
}
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
for (let t of [...o.tables].reverse()) await e.deleteMany(t.name, { where: m(t.primaryKey) });
|
|
1231
|
+
return new Ge({
|
|
1232
|
+
rows: i.slice(0, 20),
|
|
1233
|
+
primaryCount: r.filter((e) => !e.derived).length,
|
|
1234
|
+
totalCount: e.length,
|
|
1235
|
+
byTable: [...a.entries()].map(([e, t]) => ({
|
|
1236
|
+
table: e,
|
|
1237
|
+
refused: t.refused,
|
|
1238
|
+
primary: t.primary
|
|
1239
|
+
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1114
1240
|
});
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1241
|
+
}, ce = (e, t) => n.gen(function* () {
|
|
1242
|
+
let r = t, i = [], a = !0;
|
|
1243
|
+
for (; r.length > 0 && a;) {
|
|
1244
|
+
a = !1;
|
|
1245
|
+
let t = [];
|
|
1246
|
+
for (let n of r) (yield* E(() => w(e, () => C(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1247
|
+
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1248
|
+
let r = t[n], o = ae(r);
|
|
1249
|
+
o !== void 0 && (yield* E(() => w(e, () => C(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1250
|
+
}
|
|
1251
|
+
r = t;
|
|
1252
|
+
}
|
|
1253
|
+
for (let t of i) (yield* E(() => w(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1254
|
+
if (r.length > 0) return yield* n.fail(oe(r));
|
|
1255
|
+
}), D = (() => {
|
|
1256
|
+
let t = e.targetDialect;
|
|
1257
|
+
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1258
|
+
})(), O = (() => {
|
|
1259
|
+
let t = e.targetSnapshot;
|
|
1260
|
+
if (t === void 0) return [];
|
|
1261
|
+
let n = new Map(t.tables.map((e) => [e.name, e])), r = [];
|
|
1262
|
+
for (let e of s.tables) {
|
|
1263
|
+
if (B(e.name)) continue;
|
|
1264
|
+
let t = n.get(e.name);
|
|
1265
|
+
if (t === void 0) return [];
|
|
1266
|
+
let i = t.columns.filter((e) => e.generatedAs === void 0).map((e) => e.name);
|
|
1267
|
+
if (i.length === 0) return [];
|
|
1268
|
+
r.push({
|
|
1269
|
+
name: e.name,
|
|
1270
|
+
columns: i
|
|
1271
|
+
});
|
|
1272
|
+
}
|
|
1273
|
+
return r;
|
|
1274
|
+
})(), k = s.tables.filter((e) => !B(e.name) && b(e.name)).map((e) => e.name), A = D === void 0 || Nn(D) ? [] : Mn(e.targetSnapshot === void 0 ? [] : En(e.targetSnapshot, O.map((e) => e.name)), O.map((e) => e.name)), j = f === "replace" && D !== void 0 && O.length > 0 && A.length === 0 && yn(e.store), le = !1, M = 0, N = 0, de = 0, P = 0, F = (t, r) => n.gen(function* () {
|
|
1275
|
+
if (f === "replace" && !j && !(r && y.truncated)) {
|
|
1276
|
+
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1277
|
+
try: () => t.insert(d, {
|
|
1278
|
+
id: p("repl", fe),
|
|
1279
|
+
startedAt: pe,
|
|
1280
|
+
tables: String(s.tables.length),
|
|
1281
|
+
capture: I?.dir ?? null,
|
|
1282
|
+
via: e.via ?? "direct"
|
|
1283
|
+
}),
|
|
1284
|
+
catch: (e) => new V({ reason: `could not record that this replace started: ${String(e?.message ?? e)}\n The marker is what makes an interrupted run visible instead of silent, so the run does
|
|
1285
|
+
not proceed without it.` })
|
|
1286
|
+
}));
|
|
1287
|
+
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !B(e)), a = !r, o = t.emptyTables ? () => t.emptyTables(i) : a ? async () => {
|
|
1288
|
+
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where: v(e.primaryKey) });
|
|
1289
|
+
} : () => t.transactional(async (e) => {
|
|
1290
|
+
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: v(t.primaryKey) });
|
|
1291
|
+
});
|
|
1292
|
+
yield* n.tryPromise({
|
|
1293
|
+
try: o,
|
|
1294
|
+
catch: (e) => new V({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${_n(e)}` })
|
|
1295
|
+
}), r && (y.truncated = !0, yield* q(_, y));
|
|
1296
|
+
}
|
|
1297
|
+
let a = [], o = [], u = [], h = [], g = s.tables.length;
|
|
1298
|
+
for (let d = 0; d < s.tables.length; d++) {
|
|
1299
|
+
let p = s.tables[d];
|
|
1300
|
+
if (B(p.name)) {
|
|
1301
|
+
h.push(p.name);
|
|
1302
|
+
continue;
|
|
1303
|
+
}
|
|
1304
|
+
if (r && p.name in y.tables) {
|
|
1305
|
+
u.push(p.name), P++, de += y.tables[p.name].rowCount, K(e.onProgress, {
|
|
1306
|
+
phase: "import",
|
|
1307
|
+
table: p.name,
|
|
1308
|
+
index: d,
|
|
1309
|
+
tableCount: g,
|
|
1310
|
+
event: "done",
|
|
1311
|
+
rowsDone: y.tables[p.name].rowCount,
|
|
1312
|
+
total: p.rowCount
|
|
1313
|
+
});
|
|
1314
|
+
continue;
|
|
1315
|
+
}
|
|
1316
|
+
let v = i.join(e.bundleDir, p.file);
|
|
1317
|
+
K(e.onProgress, {
|
|
1125
1318
|
phase: "import",
|
|
1126
|
-
table:
|
|
1127
|
-
index:
|
|
1319
|
+
table: p.name,
|
|
1320
|
+
index: d,
|
|
1128
1321
|
tableCount: g,
|
|
1129
|
-
event: "
|
|
1130
|
-
rowsDone:
|
|
1131
|
-
total:
|
|
1322
|
+
event: "start",
|
|
1323
|
+
rowsDone: 0,
|
|
1324
|
+
total: p.rowCount
|
|
1132
1325
|
});
|
|
1133
|
-
|
|
1326
|
+
let b = a.length, ee = n.gen(function* () {
|
|
1327
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c = se("bundle", () => ue(v).pipe(lt(s.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), u = async (e) => {
|
|
1328
|
+
let n = {
|
|
1329
|
+
entry: p,
|
|
1330
|
+
row: e,
|
|
1331
|
+
reason: ""
|
|
1332
|
+
};
|
|
1333
|
+
try {
|
|
1334
|
+
await w(t, () => C(t, p, e));
|
|
1335
|
+
} catch (t) {
|
|
1336
|
+
let r = hn(t);
|
|
1337
|
+
n.reason = r.reason, ie(p, e, r), a.push(n);
|
|
1338
|
+
}
|
|
1339
|
+
}, d = yield* n.promise(async () => {
|
|
1340
|
+
let n = e.copyLoader;
|
|
1341
|
+
if (!n || n.dialect !== "postgres") return !1;
|
|
1342
|
+
if (m) return le || (le = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1343
|
+
if (f === "replace") return !0;
|
|
1344
|
+
if (f !== "upsert") return !1;
|
|
1345
|
+
try {
|
|
1346
|
+
return (await t.query({
|
|
1347
|
+
table: p.name,
|
|
1348
|
+
predicate: void 0,
|
|
1349
|
+
order: [],
|
|
1350
|
+
take: 1,
|
|
1351
|
+
skip: void 0,
|
|
1352
|
+
projection: void 0
|
|
1353
|
+
})).length === 0;
|
|
1354
|
+
} catch {
|
|
1355
|
+
return !1;
|
|
1356
|
+
}
|
|
1357
|
+
}), h = Object.keys(p.columnTypes), g = Math.max(1, e.copyBatchSize ?? 5e3), _ = [], y = async () => {
|
|
1358
|
+
if (_.length === 0) return;
|
|
1359
|
+
let t = _;
|
|
1360
|
+
_ = [];
|
|
1361
|
+
try {
|
|
1362
|
+
await e.copyLoader.copyInto({
|
|
1363
|
+
table: p.name,
|
|
1364
|
+
columns: h,
|
|
1365
|
+
columnTypes: p.columnTypes,
|
|
1366
|
+
rows: t
|
|
1367
|
+
});
|
|
1368
|
+
} catch (n) {
|
|
1369
|
+
d = !1, e.onWarn?.(`bulk COPY into ${p.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1370
|
+
for (let e of t) await u(e);
|
|
1371
|
+
}
|
|
1372
|
+
}, b = Math.max(1, e.savepointBatchSize ?? 200), ee = typeof t.savepoint == "function", x = [], S = async () => {
|
|
1373
|
+
if (x.length === 0) return;
|
|
1374
|
+
let e = x;
|
|
1375
|
+
x = [];
|
|
1376
|
+
try {
|
|
1377
|
+
await t.savepoint(async () => {
|
|
1378
|
+
for (let n of e) await C(t, p, n);
|
|
1379
|
+
});
|
|
1380
|
+
} catch {
|
|
1381
|
+
for (let t of e) await u(t);
|
|
1382
|
+
}
|
|
1383
|
+
};
|
|
1384
|
+
return yield* l.runForEach(Wt(c, p.name, p.columnTypes), (e) => n.promise(async () => {
|
|
1385
|
+
if (o++, d) {
|
|
1386
|
+
_.push(e), _.length >= g && await y();
|
|
1387
|
+
return;
|
|
1388
|
+
}
|
|
1389
|
+
if (ee) {
|
|
1390
|
+
x.push(e), x.length >= b && await S();
|
|
1391
|
+
return;
|
|
1392
|
+
}
|
|
1393
|
+
await u(e);
|
|
1394
|
+
})), yield* n.promise(S), yield* n.promise(y), {
|
|
1395
|
+
rowCount: o,
|
|
1396
|
+
checksum: i.digest("hex")
|
|
1397
|
+
};
|
|
1398
|
+
}), { rowCount: x, checksum: S } = yield* Dt("import", p.name, d, g, (e) => e.rowCount, ee);
|
|
1399
|
+
if (c) {
|
|
1400
|
+
if (S !== p.checksum) return yield* n.fail(new Ue({
|
|
1401
|
+
what: `table ${p.name} checksum`,
|
|
1402
|
+
expected: p.checksum,
|
|
1403
|
+
actual: S
|
|
1404
|
+
}));
|
|
1405
|
+
if (x !== p.rowCount) return yield* n.fail(new Ue({
|
|
1406
|
+
what: `table ${p.name} rowCount`,
|
|
1407
|
+
expected: String(p.rowCount),
|
|
1408
|
+
actual: String(x)
|
|
1409
|
+
}));
|
|
1410
|
+
}
|
|
1411
|
+
K(e.onProgress, {
|
|
1412
|
+
phase: "import",
|
|
1413
|
+
table: p.name,
|
|
1414
|
+
index: d,
|
|
1415
|
+
tableCount: g,
|
|
1416
|
+
event: "done",
|
|
1417
|
+
rowsDone: x,
|
|
1418
|
+
total: p.rowCount
|
|
1419
|
+
}), N++, M += x, r && (a.length > b ? o.push({
|
|
1420
|
+
name: p.name,
|
|
1421
|
+
rowCount: x
|
|
1422
|
+
}) : (y.tables[p.name] = {
|
|
1423
|
+
rowCount: x,
|
|
1424
|
+
checksum: p.checksum
|
|
1425
|
+
}, yield* q(_, y)));
|
|
1134
1426
|
}
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
let S = a.length, C = n.gen(function* () {
|
|
1146
|
-
let { createHash: t } = yield* n.promise(() => import("node:crypto")), i = t("sha256"), s = 0, d = T("bundle", () => M(x).pipe(Ze(o.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), f = async (e) => {
|
|
1147
|
-
let t = {
|
|
1148
|
-
entry: y,
|
|
1149
|
-
row: e,
|
|
1150
|
-
reason: ""
|
|
1427
|
+
if (a.length > 0 && (yield* ce(t, a)), f === "replace" && !(r && y.truncated) && (yield* n.promise(() => t.deleteMany(d, { where: {
|
|
1428
|
+
column: "startedAt",
|
|
1429
|
+
op: "lte",
|
|
1430
|
+
value: pe
|
|
1431
|
+
} }).then(() => void 0, () => void 0))), r && o.length > 0) {
|
|
1432
|
+
for (let e of o) {
|
|
1433
|
+
let t = s.tables.find((t) => t.name === e.name);
|
|
1434
|
+
y.tables[e.name] = {
|
|
1435
|
+
rowCount: e.rowCount,
|
|
1436
|
+
checksum: t.checksum
|
|
1151
1437
|
};
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
} catch (n) {
|
|
1155
|
-
let r = en(n);
|
|
1156
|
-
t.reason = r.reason, b(y, e, r), a.push(t);
|
|
1157
|
-
}
|
|
1158
|
-
}, p = yield* n.promise(async () => {
|
|
1159
|
-
let t = e.copyLoader;
|
|
1160
|
-
if (!t || t.dialect !== "postgres") return !1;
|
|
1161
|
-
if (u) return E || (E = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1162
|
-
if (c === "replace") return !0;
|
|
1163
|
-
if (c !== "upsert") return !1;
|
|
1164
|
-
try {
|
|
1165
|
-
return (await r.query({
|
|
1166
|
-
table: y.name,
|
|
1167
|
-
predicate: void 0,
|
|
1168
|
-
order: [],
|
|
1169
|
-
take: 1,
|
|
1170
|
-
skip: void 0,
|
|
1171
|
-
projection: void 0
|
|
1172
|
-
})).length === 0;
|
|
1173
|
-
} catch {
|
|
1174
|
-
return !1;
|
|
1175
|
-
}
|
|
1176
|
-
}), m = Object.keys(y.columnTypes), h = Math.max(1, e.copyBatchSize ?? 5e3), g = [], S = async () => {
|
|
1177
|
-
if (g.length === 0) return;
|
|
1178
|
-
let t = g;
|
|
1179
|
-
g = [];
|
|
1180
|
-
try {
|
|
1181
|
-
await e.copyLoader.copyInto({
|
|
1182
|
-
table: y.name,
|
|
1183
|
-
columns: m,
|
|
1184
|
-
columnTypes: y.columnTypes,
|
|
1185
|
-
rows: t
|
|
1186
|
-
});
|
|
1187
|
-
} catch (n) {
|
|
1188
|
-
p = !1, e.onWarn?.(`bulk COPY into ${y.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1189
|
-
for (let e of t) await f(e);
|
|
1190
|
-
}
|
|
1191
|
-
}, C = Math.max(1, e.savepointBatchSize ?? 200), w = typeof r.savepoint == "function", D = [], O = async () => {
|
|
1192
|
-
if (D.length === 0) return;
|
|
1193
|
-
let e = D;
|
|
1194
|
-
D = [];
|
|
1195
|
-
try {
|
|
1196
|
-
await r.savepoint(async () => {
|
|
1197
|
-
for (let t of e) await _(r, y, t);
|
|
1198
|
-
});
|
|
1199
|
-
} catch {
|
|
1200
|
-
for (let t of e) await f(t);
|
|
1201
|
-
}
|
|
1202
|
-
};
|
|
1203
|
-
return yield* l.runForEach(jt(d, y.name, y.columnTypes), (e) => n.promise(async () => {
|
|
1204
|
-
if (s++, p) {
|
|
1205
|
-
g.push(e), g.length >= h && await S();
|
|
1206
|
-
return;
|
|
1207
|
-
}
|
|
1208
|
-
if (w) {
|
|
1209
|
-
D.push(e), D.length >= C && await O();
|
|
1210
|
-
return;
|
|
1211
|
-
}
|
|
1212
|
-
await f(e);
|
|
1213
|
-
})), yield* n.promise(O), yield* n.promise(S), {
|
|
1214
|
-
rowCount: s,
|
|
1215
|
-
checksum: i.digest("hex")
|
|
1216
|
-
};
|
|
1217
|
-
}), { rowCount: w, checksum: j } = yield* mt("import", y.name, m, g, (e) => e.rowCount, C);
|
|
1218
|
-
if (s) {
|
|
1219
|
-
if (j !== y.checksum) return yield* n.fail(new R({
|
|
1220
|
-
what: `table ${y.name} checksum`,
|
|
1221
|
-
expected: y.checksum,
|
|
1222
|
-
actual: j
|
|
1223
|
-
}));
|
|
1224
|
-
if (w !== y.rowCount) return yield* n.fail(new R({
|
|
1225
|
-
what: `table ${y.name} rowCount`,
|
|
1226
|
-
expected: String(y.rowCount),
|
|
1227
|
-
actual: String(w)
|
|
1228
|
-
}));
|
|
1438
|
+
}
|
|
1439
|
+
yield* q(_, y);
|
|
1229
1440
|
}
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
index: m,
|
|
1234
|
-
tableCount: g,
|
|
1235
|
-
event: "done",
|
|
1236
|
-
rowsDone: w,
|
|
1237
|
-
total: y.rowCount
|
|
1238
|
-
}), O++, D += w, i && (a.length > S ? d.push({
|
|
1239
|
-
name: y.name,
|
|
1240
|
-
rowCount: w
|
|
1241
|
-
}) : (p.tables[y.name] = {
|
|
1242
|
-
rowCount: w,
|
|
1243
|
-
checksum: y.checksum
|
|
1244
|
-
}, yield* K(f, p)));
|
|
1245
|
-
}
|
|
1246
|
-
if (a.length > 0 && (yield* w(r, a)), i && d.length > 0) {
|
|
1247
|
-
for (let e of d) {
|
|
1248
|
-
let t = o.tables.find((t) => t.name === e.name);
|
|
1249
|
-
p.tables[e.name] = {
|
|
1250
|
-
rowCount: e.rowCount,
|
|
1251
|
-
checksum: t.checksum
|
|
1252
|
-
};
|
|
1441
|
+
{
|
|
1442
|
+
let t = Ve(h, "skipped");
|
|
1443
|
+
t !== void 0 && e.onWarn?.(t);
|
|
1253
1444
|
}
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1445
|
+
if (u.length > 0) {
|
|
1446
|
+
let t = u.length === s.tables.length;
|
|
1447
|
+
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${_} to force a full re-import.`);
|
|
1448
|
+
}
|
|
1449
|
+
}), fe = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, pe = /* @__PURE__ */ new Date(), I = yield* Pn({
|
|
1450
|
+
store: e.store,
|
|
1451
|
+
mode: f,
|
|
1452
|
+
dir: e.rollbackDir,
|
|
1453
|
+
snapshot: e.targetSnapshot,
|
|
1454
|
+
dialect: e.targetDialect,
|
|
1455
|
+
tables: s.tables.map((e) => e.name).filter((e) => !B(e)),
|
|
1456
|
+
onWarn: e.onWarn
|
|
1457
|
+
});
|
|
1458
|
+
I !== void 0 && (e.onRollbackCaptured !== void 0 && (yield* n.tryPromise({
|
|
1459
|
+
try: () => e.onRollbackCaptured(I),
|
|
1460
|
+
catch: (e) => new V({
|
|
1461
|
+
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1462
|
+
path: I.dir
|
|
1463
|
+
})
|
|
1464
|
+
})), e.onWarn?.(Fn(I)));
|
|
1465
|
+
let me = (e) => f === "replace" ? S(e) : e(), he = (e) => f === "replace" ? te(e) : e, L = p("dimp", `${Date.now()}-${s.source.schemaFingerprint ?? ""}`), ge = !1;
|
|
1466
|
+
g(u) !== void 0 && (yield* n.promise(() => e.store.insert(u, {
|
|
1467
|
+
id: L,
|
|
1468
|
+
mode: f,
|
|
1469
|
+
via: e.via ?? "direct",
|
|
1470
|
+
bundle: e.bundleDir,
|
|
1471
|
+
sourceFingerprint: s.source.schemaFingerprint ?? null,
|
|
1472
|
+
tablesWritten: 0,
|
|
1473
|
+
rowsWritten: 0
|
|
1474
|
+
}).then(() => {
|
|
1475
|
+
ge = !0;
|
|
1476
|
+
}, (t) => {
|
|
1477
|
+
e.onWarn?.(`the import trace could not be written (${String(t?.message ?? t)}).\n The import continues — this is a record of what happened, not a safety check. Run
|
|
1478
|
+
\`voltro db apply\` so the next one is recorded.`);
|
|
1479
|
+
})));
|
|
1480
|
+
let _e = (t) => ge ? e.store.update(u, L, {
|
|
1481
|
+
finishedAt: t === null ? /* @__PURE__ */ new Date() : null,
|
|
1482
|
+
tablesWritten: N,
|
|
1483
|
+
rowsWritten: M,
|
|
1484
|
+
failure: t
|
|
1485
|
+
}).then(() => void 0, () => void 0) : Promise.resolve();
|
|
1486
|
+
if (t = (e) => _e(e), !j && f === "replace" && m && A.length > 0 && e.onWarn?.(bn(`the bundle's tables reference each other in a cycle (${A.map((e) => `${e.from}.${e.column} → ${e.to}`).join(", ")}), and a bulk copy cannot satisfy both directions on this dialect.`)), f === "replace" && k.length > 0 && e.onWarn?.(`write recorders are SUSPENDED for this replace (${k.length} table(s) have one).\n A replace sets a state rather than changing rows, so a per-row history entry would describe
|
|
1487
|
+
something that did not happen — and this path already writes through the raw store, without
|
|
1488
|
+
tenant scoping or \`audit()\` stamping, for the same reason.`), j) {
|
|
1489
|
+
e.onWarn?.(`staging ${O.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1490
|
+
let t = e.store, r = O.map((e) => e.name), i = Tn(r, {
|
|
1491
|
+
getTable: (e) => g(e),
|
|
1492
|
+
registerTable: (e) => {
|
|
1493
|
+
ee(e);
|
|
1494
|
+
},
|
|
1495
|
+
unregisterTable: ne
|
|
1496
|
+
}), a = !1;
|
|
1497
|
+
try {
|
|
1498
|
+
let i = m ? r : r.filter((e) => !At(y, e));
|
|
1499
|
+
yield* n.tryPromise({
|
|
1500
|
+
try: async () => {
|
|
1501
|
+
await kn(t, i.map((e) => Cn(e, D))), await kn(t, i.map((e) => Sn(e, D)));
|
|
1502
|
+
},
|
|
1503
|
+
catch: (e) => new V({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1504
|
+
}), yield* he(F(e.store, !m)), !m && y.truncated === !0 ? a = !0 : yield* n.tryPromise({
|
|
1505
|
+
try: () => An(t, O, D).then(() => {
|
|
1506
|
+
a = !0;
|
|
1507
|
+
}),
|
|
1508
|
+
catch: (e) => new V({ reason: String(e?.message ?? e) })
|
|
1509
|
+
}).pipe(n.catchAll((i) => n.gen(function* () {
|
|
1510
|
+
let a = On(yield* n.promise(() => jn(t, En(e.targetSnapshot, r), D).then((e) => e, () => [])));
|
|
1511
|
+
return yield* n.fail(new V({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1512
|
+
}))), !m && a && y.truncated !== !0 && (y.truncated = !0, yield* q(_, y));
|
|
1513
|
+
} finally {
|
|
1514
|
+
i.undo(), a || m ? yield* n.promise(() => kn(t, r.map((e) => Cn(e, D))).then(() => void 0, () => void 0)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1515
|
+
}
|
|
1516
|
+
} else m ? yield* n.tryPromise({
|
|
1517
|
+
try: () => e.store.transactional(async (e) => {
|
|
1518
|
+
let t = await me(() => n.runPromiseExit(F(e, !1)));
|
|
1519
|
+
if (r.isFailure(t)) throw Un(t.cause);
|
|
1520
|
+
}),
|
|
1521
|
+
catch: (e) => Hn(e) ?? new V({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1522
|
+
}) : yield* he(F(e.store, !0));
|
|
1523
|
+
if (e.assets && s.assets.index && !e.assetsAlreadyRestored) {
|
|
1524
|
+
let t = e.assets, r = i.join(e.bundleDir, yt), a = i.join(e.bundleDir, s.assets.index ?? "assets/index.ndjson"), o = se("bundle", () => ue(a)).pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.map((e) => JSON.parse(e)));
|
|
1525
|
+
yield* l.runForEach(o, (e) => e.key in y.assets ? n.void : it(t, r, e).pipe(n.andThen(n.sync(() => {
|
|
1526
|
+
y.assets[e.key] = !0;
|
|
1527
|
+
})), n.andThen(q(_, y))));
|
|
1259
1528
|
}
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
})), n.andThen(K(f, p))));
|
|
1272
|
-
}
|
|
1273
|
-
return {
|
|
1274
|
-
manifest: o,
|
|
1275
|
-
tablesWritten: O,
|
|
1276
|
-
rowsWritten: D,
|
|
1277
|
-
tablesSkipped: A,
|
|
1278
|
-
rowsSkipped: k,
|
|
1279
|
-
fullyResumed: O === 0 && A > 0
|
|
1280
|
-
};
|
|
1281
|
-
}), mn = (e) => e, hn = (e) => {
|
|
1529
|
+
return yield* n.promise(() => _e(null)), {
|
|
1530
|
+
manifest: s,
|
|
1531
|
+
tablesWritten: N,
|
|
1532
|
+
rowsWritten: M,
|
|
1533
|
+
tablesSkipped: P,
|
|
1534
|
+
rowsSkipped: de,
|
|
1535
|
+
fullyResumed: N === 0 && P > 0,
|
|
1536
|
+
...I === void 0 ? {} : { rollback: I }
|
|
1537
|
+
};
|
|
1538
|
+
}).pipe(n.tapError((e) => n.promise(() => t?.(e?.reason ?? e?._tag ?? "the import did not finish") ?? Promise.resolve())));
|
|
1539
|
+
}, Kn = (e) => e, qn = (e) => {
|
|
1282
1540
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1283
1541
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1284
1542
|
return {
|
|
1285
1543
|
refuse: !0,
|
|
1286
1544
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1287
1545
|
};
|
|
1288
|
-
},
|
|
1546
|
+
}, Z = Buffer.from("VBUNDLE2"), Jn = 0, Yn = 1, Xn = 2, Zn = 32, Qn = async (e) => {
|
|
1289
1547
|
let t = [], n = async (r) => {
|
|
1290
1548
|
let i;
|
|
1291
1549
|
try {
|
|
1292
|
-
i = await
|
|
1550
|
+
i = await fe(L(e, r), { withFileTypes: !0 });
|
|
1293
1551
|
} catch {
|
|
1294
1552
|
return;
|
|
1295
1553
|
}
|
|
@@ -1299,22 +1557,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1299
1557
|
}
|
|
1300
1558
|
};
|
|
1301
1559
|
return await n(""), t.sort();
|
|
1302
|
-
},
|
|
1560
|
+
}, $n = (e, t) => {
|
|
1303
1561
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1304
|
-
return r.writeUInt8(
|
|
1305
|
-
},
|
|
1562
|
+
return r.writeUInt8(Yn, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1563
|
+
}, er = (e, t, n) => {
|
|
1306
1564
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1307
|
-
return a.writeUInt8(
|
|
1565
|
+
return a.writeUInt8(Xn, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1308
1566
|
};
|
|
1309
|
-
async function*
|
|
1310
|
-
yield new Uint8Array(
|
|
1311
|
-
for (let t of await
|
|
1312
|
-
let n =
|
|
1313
|
-
if (yield new Uint8Array(
|
|
1567
|
+
async function* tr(e) {
|
|
1568
|
+
yield new Uint8Array(Z);
|
|
1569
|
+
for (let t of await Qn(e.dir)) {
|
|
1570
|
+
let n = L(e.dir, t), r = await pe(n);
|
|
1571
|
+
if (yield new Uint8Array($n(t, r.size)), r.size > 0) for await (let e of ue(n)) yield new Uint8Array(e);
|
|
1314
1572
|
}
|
|
1315
1573
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1316
|
-
yield new Uint8Array(
|
|
1317
|
-
let n =
|
|
1574
|
+
yield new Uint8Array(er(t.key, t.contentType, t.size));
|
|
1575
|
+
let n = T("sha256"), r = 0;
|
|
1318
1576
|
for await (let i of e.blobs.open(t.key)) {
|
|
1319
1577
|
let e = Buffer.from(i);
|
|
1320
1578
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1322,11 +1580,11 @@ async function* Cn(e) {
|
|
|
1322
1580
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1323
1581
|
yield new Uint8Array(n.digest());
|
|
1324
1582
|
}
|
|
1325
|
-
yield new Uint8Array(Buffer.from([
|
|
1583
|
+
yield new Uint8Array(Buffer.from([Jn]));
|
|
1326
1584
|
}
|
|
1327
|
-
var
|
|
1328
|
-
let n =
|
|
1329
|
-
await
|
|
1585
|
+
var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr = (e) => new Promise((t) => e.end(t)), ir = async (e, t) => {
|
|
1586
|
+
let n = ge(t.dir);
|
|
1587
|
+
await F(n, { recursive: !0 });
|
|
1330
1588
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1331
1589
|
let { value: e, done: t } = await i.next();
|
|
1332
1590
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1337,28 +1595,28 @@ var wn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Tn
|
|
|
1337
1595
|
return r = r.subarray(e), t;
|
|
1338
1596
|
};
|
|
1339
1597
|
try {
|
|
1340
|
-
if (await s(
|
|
1598
|
+
if (await s(Z.length), !c(Z.length).equals(Z)) throw Error("not a bundle archive (bad magic)");
|
|
1341
1599
|
for (;;) {
|
|
1342
1600
|
await s(1);
|
|
1343
1601
|
let e = c(1).readUInt8(0);
|
|
1344
|
-
if (e ===
|
|
1345
|
-
if (e ===
|
|
1602
|
+
if (e === Jn) return;
|
|
1603
|
+
if (e === Yn) {
|
|
1346
1604
|
await s(4);
|
|
1347
1605
|
let e = c(4).readUInt32BE(0);
|
|
1348
1606
|
await s(e);
|
|
1349
1607
|
let t = c(e).toString("utf8");
|
|
1350
1608
|
await s(8);
|
|
1351
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1352
|
-
if (a !== n && !a.startsWith(n +
|
|
1353
|
-
await
|
|
1354
|
-
let o =
|
|
1609
|
+
let i = c(8).readBigUInt64BE(0), a = ge(n, t);
|
|
1610
|
+
if (a !== n && !a.startsWith(n + _e)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1611
|
+
await F(he(a), { recursive: !0 });
|
|
1612
|
+
let o = M(a);
|
|
1355
1613
|
for (; i > 0n;) {
|
|
1356
1614
|
r.length === 0 && await s(1);
|
|
1357
1615
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1358
|
-
await
|
|
1616
|
+
await nr(o, c(e)), i -= BigInt(e);
|
|
1359
1617
|
}
|
|
1360
|
-
await
|
|
1361
|
-
} else if (e ===
|
|
1618
|
+
await rr(o);
|
|
1619
|
+
} else if (e === Xn) {
|
|
1362
1620
|
await s(4);
|
|
1363
1621
|
let e = c(4).readUInt32BE(0);
|
|
1364
1622
|
await s(e);
|
|
@@ -1368,24 +1626,24 @@ var wn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Tn
|
|
|
1368
1626
|
await s(i);
|
|
1369
1627
|
let a = c(i).toString("utf8");
|
|
1370
1628
|
await s(8);
|
|
1371
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1629
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = T("sha256"), d = new A(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1372
1630
|
contentType: a,
|
|
1373
1631
|
size: Number(o)
|
|
1374
1632
|
}), p = o;
|
|
1375
1633
|
for (; p > 0n;) {
|
|
1376
1634
|
r.length === 0 && await s(1);
|
|
1377
1635
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1378
|
-
u.update(t), l || await
|
|
1636
|
+
u.update(t), l || await nr(d, t), p -= BigInt(e);
|
|
1379
1637
|
}
|
|
1380
|
-
l ? d.destroy() : await
|
|
1381
|
-
let m = c(
|
|
1638
|
+
l ? d.destroy() : await rr(d), await f, await s(Zn);
|
|
1639
|
+
let m = c(Zn);
|
|
1382
1640
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1383
1641
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1384
1642
|
}
|
|
1385
1643
|
} finally {
|
|
1386
1644
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1387
1645
|
}
|
|
1388
|
-
},
|
|
1646
|
+
}, ar = async (e, t) => {
|
|
1389
1647
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1390
1648
|
let { value: e, done: t } = await a.next();
|
|
1391
1649
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1404,11 +1662,11 @@ var wn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Tn
|
|
|
1404
1662
|
}
|
|
1405
1663
|
return Buffer.concat(t);
|
|
1406
1664
|
};
|
|
1407
|
-
if (!await s(
|
|
1665
|
+
if (!await s(Z.length) || !c(Z.length).equals(Z)) throw Error("not a bundle archive (bad magic)");
|
|
1408
1666
|
for (; await s(1);) {
|
|
1409
1667
|
let e = c(1).readUInt8(0);
|
|
1410
|
-
if (e ===
|
|
1411
|
-
if (e !==
|
|
1668
|
+
if (e === Jn || e === Xn) break;
|
|
1669
|
+
if (e !== Yn) throw Error(`unknown bundle entry kind ${e}`);
|
|
1412
1670
|
if (!await s(4)) break;
|
|
1413
1671
|
let t = c(4).readUInt32BE(0);
|
|
1414
1672
|
if (!await s(t)) break;
|
|
@@ -1418,22 +1676,22 @@ var wn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Tn
|
|
|
1418
1676
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1419
1677
|
}
|
|
1420
1678
|
return r;
|
|
1421
|
-
},
|
|
1679
|
+
}, or = (e, t) => ir(e, {
|
|
1422
1680
|
dir: t,
|
|
1423
|
-
blobs:
|
|
1424
|
-
}),
|
|
1681
|
+
blobs: sr(t)
|
|
1682
|
+
}), sr = (e) => {
|
|
1425
1683
|
let t = /* @__PURE__ */ new Set();
|
|
1426
1684
|
return {
|
|
1427
1685
|
has: async (e) => t.has(e),
|
|
1428
1686
|
write: async (n, r, i) => {
|
|
1429
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1430
|
-
await
|
|
1431
|
-
let l =
|
|
1687
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = L(e, "assets");
|
|
1688
|
+
await F(c, { recursive: !0 });
|
|
1689
|
+
let l = T("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1432
1690
|
l.update(e), u += e.length, n(null, e);
|
|
1433
|
-
} }), f =
|
|
1434
|
-
await o(r, d,
|
|
1691
|
+
} }), f = L(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1692
|
+
await o(r, d, M(f));
|
|
1435
1693
|
let p = l.digest("hex");
|
|
1436
|
-
await a.rename(f,
|
|
1694
|
+
await a.rename(f, L(c, p)), await a.appendFile(L(c, "index.ndjson"), `${JSON.stringify({
|
|
1437
1695
|
key: n,
|
|
1438
1696
|
sha256: p,
|
|
1439
1697
|
size: u,
|
|
@@ -1441,33 +1699,33 @@ var wn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Tn
|
|
|
1441
1699
|
})}\n`), t.add(n);
|
|
1442
1700
|
}
|
|
1443
1701
|
};
|
|
1444
|
-
},
|
|
1702
|
+
}, cr = ve(ae), lr = Buffer.from("VENC1\0\0\0"), ur = 32, dr = 16, fr = 7, pr = 16, mr = 262144, hr = {
|
|
1445
1703
|
N: 32768,
|
|
1446
1704
|
r: 8,
|
|
1447
1705
|
p: 1
|
|
1448
|
-
},
|
|
1706
|
+
}, gr = 134217728, Q = 43, _r = (e, t, n) => {
|
|
1449
1707
|
let r = Buffer.alloc(12);
|
|
1450
|
-
return e.copy(r, 0, 0,
|
|
1708
|
+
return e.copy(r, 0, 0, fr), r.writeUInt32BE(t >>> 0, fr), r.writeUInt8(+!!n, 11), r;
|
|
1451
1709
|
};
|
|
1452
|
-
async function*
|
|
1710
|
+
async function* vr(e, t) {
|
|
1453
1711
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1454
|
-
let n =
|
|
1455
|
-
...
|
|
1456
|
-
maxmem:
|
|
1712
|
+
let n = E(dr), r = E(fr), i = await cr(t, n, ur, {
|
|
1713
|
+
...hr,
|
|
1714
|
+
maxmem: gr
|
|
1457
1715
|
}), a = Buffer.alloc(Q);
|
|
1458
|
-
|
|
1716
|
+
lr.copy(a, 0), a.writeUInt32BE(hr.N, 8), a.writeUInt32BE(hr.r, 12), a.writeUInt32BE(hr.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1459
1717
|
let o = 0, s = (e, t) => {
|
|
1460
|
-
let n =
|
|
1718
|
+
let n = C("aes-256-gcm", i, _r(r, o, t));
|
|
1461
1719
|
o === 0 && n.setAAD(a);
|
|
1462
1720
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1463
1721
|
o++;
|
|
1464
|
-
let l = Buffer.alloc(5 + s.length +
|
|
1465
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
1722
|
+
let l = Buffer.alloc(5 + s.length + pr);
|
|
1723
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + pr, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1466
1724
|
}, c = Buffer.alloc(0);
|
|
1467
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1725
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > mr;) yield s(c.subarray(0, mr), !1), c = c.subarray(mr);
|
|
1468
1726
|
yield s(c, !0);
|
|
1469
1727
|
}
|
|
1470
|
-
async function*
|
|
1728
|
+
async function* yr(e, t) {
|
|
1471
1729
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1472
1730
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1473
1731
|
let { value: e, done: t } = await n.next();
|
|
@@ -1478,12 +1736,12 @@ async function* zn(e, t) {
|
|
|
1478
1736
|
};
|
|
1479
1737
|
if (!await o(Q)) throw Error("encrypted bundle truncated (no header)");
|
|
1480
1738
|
let s = r.subarray(0, Q);
|
|
1481
|
-
if (!s.subarray(0, 8).equals(
|
|
1482
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await
|
|
1739
|
+
if (!s.subarray(0, 8).equals(lr)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1740
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await cr(t, d, ur, {
|
|
1483
1741
|
N: c,
|
|
1484
1742
|
r: l,
|
|
1485
1743
|
p: u,
|
|
1486
|
-
maxmem:
|
|
1744
|
+
maxmem: gr
|
|
1487
1745
|
});
|
|
1488
1746
|
r = r.subarray(Q);
|
|
1489
1747
|
let m = 0, h = !1;
|
|
@@ -1494,11 +1752,11 @@ async function* zn(e, t) {
|
|
|
1494
1752
|
}
|
|
1495
1753
|
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
1496
1754
|
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
1497
|
-
if (t <
|
|
1755
|
+
if (t < pr) throw Error("encrypted bundle corrupt (short frame)");
|
|
1498
1756
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1499
|
-
let n = r.subarray(5, 5 + t -
|
|
1757
|
+
let n = r.subarray(5, 5 + t - pr), i = r.subarray(5 + t - pr, 5 + t);
|
|
1500
1758
|
r = r.subarray(5 + t);
|
|
1501
|
-
let a =
|
|
1759
|
+
let a = w("aes-256-gcm", p, _r(f, m, e));
|
|
1502
1760
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1503
1761
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1504
1762
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1506,7 +1764,7 @@ async function* zn(e, t) {
|
|
|
1506
1764
|
}
|
|
1507
1765
|
//#endregion
|
|
1508
1766
|
//#region src/blobStorage.ts
|
|
1509
|
-
var
|
|
1767
|
+
var br = (e, t) => ({
|
|
1510
1768
|
list: () => {
|
|
1511
1769
|
let e = t();
|
|
1512
1770
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1519,29 +1777,29 @@ var Bn = (e, t) => ({
|
|
|
1519
1777
|
} };
|
|
1520
1778
|
},
|
|
1521
1779
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1522
|
-
let r = await n.runPromise(
|
|
1780
|
+
let r = await n.runPromise(ce(e, t)), i = await O(r.stream);
|
|
1523
1781
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1524
1782
|
} })
|
|
1525
|
-
}),
|
|
1526
|
-
table:
|
|
1783
|
+
}), xr = (e, t) => br(e, () => x(t, {
|
|
1784
|
+
table: oe,
|
|
1527
1785
|
chunkSize: 500
|
|
1528
1786
|
}).pipe(l.map((e) => ({
|
|
1529
1787
|
key: String(e.key),
|
|
1530
1788
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1531
1789
|
size: Number(e.size ?? 0)
|
|
1532
|
-
})))),
|
|
1790
|
+
})))), Sr = (e) => ({
|
|
1533
1791
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1534
1792
|
write: async (t, r, i) => {
|
|
1535
|
-
let a =
|
|
1793
|
+
let a = se("storage", () => j.from(r));
|
|
1536
1794
|
await n.runPromise(D(e, t, a, {
|
|
1537
1795
|
contentType: i.contentType,
|
|
1538
1796
|
totalSize: i.size
|
|
1539
1797
|
}));
|
|
1540
1798
|
}
|
|
1541
|
-
}),
|
|
1799
|
+
}), Cr = (e, t) => {
|
|
1542
1800
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1543
|
-
return e === "mariadb" &&
|
|
1544
|
-
},
|
|
1801
|
+
return e === "mariadb" && wr(n) ? n : r;
|
|
1802
|
+
}, wr = (e) => (process.env.PATH ?? "").split(me).filter((e) => e !== "").some((t) => N(L(t, e))), Tr = (e) => {
|
|
1545
1803
|
switch (e) {
|
|
1546
1804
|
case "postgres": return "db.dump";
|
|
1547
1805
|
case "mysql":
|
|
@@ -1551,7 +1809,7 @@ var Bn = (e, t) => ({
|
|
|
1551
1809
|
case "mssql": return "db.bacpac";
|
|
1552
1810
|
default: return "db.dump";
|
|
1553
1811
|
}
|
|
1554
|
-
},
|
|
1812
|
+
}, Er = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1555
1813
|
if (e.host || e.database) return {
|
|
1556
1814
|
host: e.host ?? "localhost",
|
|
1557
1815
|
port: e.port ?? 3306,
|
|
@@ -1567,7 +1825,7 @@ var Bn = (e, t) => ({
|
|
|
1567
1825
|
password: decodeURIComponent(t.password) || "",
|
|
1568
1826
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1569
1827
|
};
|
|
1570
|
-
},
|
|
1828
|
+
}, Dr = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Or = (e, t, n) => {
|
|
1571
1829
|
switch (e) {
|
|
1572
1830
|
case "postgres": {
|
|
1573
1831
|
if (t.url) return {
|
|
@@ -1614,7 +1872,7 @@ var Bn = (e, t) => ({
|
|
|
1614
1872
|
let r = $(t);
|
|
1615
1873
|
return {
|
|
1616
1874
|
kind: "spawn",
|
|
1617
|
-
tool:
|
|
1875
|
+
tool: Cr(e, "dump"),
|
|
1618
1876
|
args: [
|
|
1619
1877
|
"--single-transaction",
|
|
1620
1878
|
"--routines",
|
|
@@ -1636,7 +1894,7 @@ var Bn = (e, t) => ({
|
|
|
1636
1894
|
case "sqlite":
|
|
1637
1895
|
case "turso": return {
|
|
1638
1896
|
kind: "copy",
|
|
1639
|
-
from:
|
|
1897
|
+
from: Er(t),
|
|
1640
1898
|
to: n
|
|
1641
1899
|
};
|
|
1642
1900
|
case "mssql": return {
|
|
@@ -1644,7 +1902,7 @@ var Bn = (e, t) => ({
|
|
|
1644
1902
|
tool: "sqlpackage",
|
|
1645
1903
|
args: [
|
|
1646
1904
|
"/Action:Export",
|
|
1647
|
-
`/SourceConnectionString:${
|
|
1905
|
+
`/SourceConnectionString:${Dr($({
|
|
1648
1906
|
...t,
|
|
1649
1907
|
port: t.port ?? 1433
|
|
1650
1908
|
}))}`,
|
|
@@ -1652,12 +1910,12 @@ var Bn = (e, t) => ({
|
|
|
1652
1910
|
],
|
|
1653
1911
|
env: {}
|
|
1654
1912
|
};
|
|
1655
|
-
default: throw new
|
|
1913
|
+
default: throw new U({
|
|
1656
1914
|
tool: "backup",
|
|
1657
1915
|
reason: `no native backup for dialect '${e}'`
|
|
1658
1916
|
});
|
|
1659
1917
|
}
|
|
1660
|
-
},
|
|
1918
|
+
}, kr = (e, t, n) => {
|
|
1661
1919
|
switch (e) {
|
|
1662
1920
|
case "postgres": {
|
|
1663
1921
|
if (t.url) return {
|
|
@@ -1702,7 +1960,7 @@ var Bn = (e, t) => ({
|
|
|
1702
1960
|
let r = $(t);
|
|
1703
1961
|
return {
|
|
1704
1962
|
kind: "spawn",
|
|
1705
|
-
tool:
|
|
1963
|
+
tool: Cr(e, "client"),
|
|
1706
1964
|
args: [
|
|
1707
1965
|
"--protocol=TCP",
|
|
1708
1966
|
"-h",
|
|
@@ -1721,14 +1979,14 @@ var Bn = (e, t) => ({
|
|
|
1721
1979
|
case "turso": return {
|
|
1722
1980
|
kind: "copy",
|
|
1723
1981
|
from: n,
|
|
1724
|
-
to:
|
|
1982
|
+
to: Er(t)
|
|
1725
1983
|
};
|
|
1726
1984
|
case "mssql": return {
|
|
1727
1985
|
kind: "spawn",
|
|
1728
1986
|
tool: "sqlpackage",
|
|
1729
1987
|
args: [
|
|
1730
1988
|
"/Action:Import",
|
|
1731
|
-
`/TargetConnectionString:${
|
|
1989
|
+
`/TargetConnectionString:${Dr($({
|
|
1732
1990
|
...t,
|
|
1733
1991
|
port: t.port ?? 1433
|
|
1734
1992
|
}))}`,
|
|
@@ -1736,18 +1994,18 @@ var Bn = (e, t) => ({
|
|
|
1736
1994
|
],
|
|
1737
1995
|
env: {}
|
|
1738
1996
|
};
|
|
1739
|
-
default: throw new
|
|
1997
|
+
default: throw new U({
|
|
1740
1998
|
tool: "restore",
|
|
1741
1999
|
reason: `no native restore for dialect '${e}'`
|
|
1742
2000
|
});
|
|
1743
2001
|
}
|
|
1744
|
-
},
|
|
1745
|
-
|
|
2002
|
+
}, Ar = (e) => e.kind === "copy" ? n.async((t) => {
|
|
2003
|
+
le(e.from, e.to, (r) => t(r ? n.fail(new U({
|
|
1746
2004
|
tool: "copy",
|
|
1747
2005
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1748
2006
|
})) : n.void));
|
|
1749
2007
|
}) : n.async((t) => {
|
|
1750
|
-
let r =
|
|
2008
|
+
let r = ye(e.tool, [...e.args], {
|
|
1751
2009
|
env: {
|
|
1752
2010
|
...process.env,
|
|
1753
2011
|
...e.env
|
|
@@ -1760,14 +2018,14 @@ var Bn = (e, t) => ({
|
|
|
1760
2018
|
}), i = "";
|
|
1761
2019
|
r.stderr?.on("data", (e) => {
|
|
1762
2020
|
i += String(e);
|
|
1763
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(
|
|
2021
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin && ue(e.stdinFile).pipe(r.stdin);
|
|
1764
2022
|
let a = () => {
|
|
1765
2023
|
if (e.stdoutFile !== void 0) try {
|
|
1766
|
-
|
|
2024
|
+
de(e.stdoutFile);
|
|
1767
2025
|
} catch {}
|
|
1768
2026
|
};
|
|
1769
2027
|
r.on("error", (r) => {
|
|
1770
|
-
a(), t(n.fail(new
|
|
2028
|
+
a(), t(n.fail(new U({
|
|
1771
2029
|
tool: e.tool,
|
|
1772
2030
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`
|
|
1773
2031
|
})));
|
|
@@ -1776,17 +2034,17 @@ var Bn = (e, t) => ({
|
|
|
1776
2034
|
t(n.void);
|
|
1777
2035
|
return;
|
|
1778
2036
|
}
|
|
1779
|
-
a(), t(n.fail(new
|
|
2037
|
+
a(), t(n.fail(new U({
|
|
1780
2038
|
tool: e.tool,
|
|
1781
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2039
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${I(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
1782
2040
|
stderr: i.slice(-2e3)
|
|
1783
2041
|
})));
|
|
1784
2042
|
});
|
|
1785
|
-
}),
|
|
1786
|
-
let t = yield*
|
|
2043
|
+
}), jr = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Mr = (e) => n.gen(function* () {
|
|
2044
|
+
let t = yield* et(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
1787
2045
|
store: e.store,
|
|
1788
2046
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
1789
|
-
}), r =
|
|
2047
|
+
}), r = Gt(t.tables, e.snapshot), i = e.classification ?? Se(e.snapshot), a = e.sampleSize ?? 20, o = Fe(r.map((t) => ({
|
|
1790
2048
|
name: t,
|
|
1791
2049
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
1792
2050
|
name: e.name,
|
|
@@ -1794,19 +2052,19 @@ var Bn = (e, t) => ({
|
|
|
1794
2052
|
}))
|
|
1795
2053
|
})), i, e.masking), s = [], c = [];
|
|
1796
2054
|
for (let i of r) {
|
|
1797
|
-
let r = t.predicateFor(i), u = yield*
|
|
2055
|
+
let r = t.predicateFor(i), u = yield* x(e.store, {
|
|
1798
2056
|
table: i,
|
|
1799
2057
|
chunkSize: a,
|
|
1800
2058
|
...r === void 0 ? {} : { where: r }
|
|
1801
|
-
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
1802
|
-
c.push(...
|
|
2059
|
+
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) => Re(t, i, o.actions, e.masking.seed));
|
|
2060
|
+
c.push(...Le(d, i, o.actions));
|
|
1803
2061
|
let f = u[0], p = d[0], m = [];
|
|
1804
2062
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
1805
2063
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
1806
2064
|
let n = e.slice(i.length + 1);
|
|
1807
2065
|
m.push({
|
|
1808
2066
|
column: n,
|
|
1809
|
-
action:
|
|
2067
|
+
action: jr(t),
|
|
1810
2068
|
before: f[n],
|
|
1811
2069
|
after: p[n]
|
|
1812
2070
|
});
|
|
@@ -1821,7 +2079,7 @@ var Bn = (e, t) => ({
|
|
|
1821
2079
|
leaks: c,
|
|
1822
2080
|
unclassified: o.unclassified
|
|
1823
2081
|
};
|
|
1824
|
-
}),
|
|
2082
|
+
}), Nr = (e, t) => {
|
|
1825
2083
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
1826
2084
|
for (let t of e) {
|
|
1827
2085
|
let e = n.get(t.name);
|
|
@@ -1876,7 +2134,7 @@ var Bn = (e, t) => ({
|
|
|
1876
2134
|
skipColumns: i,
|
|
1877
2135
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
1878
2136
|
};
|
|
1879
|
-
},
|
|
2137
|
+
}, Pr = (e) => {
|
|
1880
2138
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
1881
2139
|
let t = {
|
|
1882
2140
|
safe: "✓",
|
|
@@ -1887,4 +2145,4 @@ var Bn = (e, t) => ({
|
|
|
1887
2145
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
1888
2146
|
};
|
|
1889
2147
|
//#endregion
|
|
1890
|
-
export {
|
|
2148
|
+
export { yt as ASSETS_DIR, bt as ASSET_INDEX_FILE, V as BundleError, He as CodecError, We as CompressionError, Ye as CrossDialectError, vt as DATA_DIR, xe as DEFAULT_CLASS_ACTIONS, ut as FORMAT_VERSION, Je as ImportModeError, Ue as IntegrityError, xt as LEDGER_FILE, G as MANIFEST_FILE, qe as MaskingError, U as NativeToolError, Ge as RowsRefusedError, Bn as SAVEPOINT_BATCH_SIZE, vn as STAGING_PREFIX, Ke as SchemaDriftError, H as ScopeError, Ne as applyAction, Hn as asImportError, qn as assessLiveImport, jt as assetDone, Tr as backupArtifactName, Or as backupCommand, on as canonicalType, ln as checkPortability, zn as checkSchemaDrift, Se as classificationFromSnapshot, Nr as classifyImportDrift, st as compressionExt, Ze as computeSubsetIds, gt as decodeManifest, Vt as decodeRow, zt as decodeValue, yr as decryptStream, Kn as defineDataProfile, Rn as diffSnapshots, sr as dirBlobSink, Cn as dropStagingSql, Ot as emptyLedger, ht as encodeManifest, Bt as encodeRow, Lt as encodeValue, vr as encryptStream, kn as execStatements, Pr as formatImportDrift, un as formatIssue, at as hasZstd, ct as makeCompressor, lt as makeDecompressor, Re as maskRow, Wt as ndjsonToRows, an as outsideReferenceRefusal, rn as outsideReferencesInto, tr as packBundle, ar as peekBundle, ot as pickCompression, Fe as planMasking, Mr as previewMasking, kt as readLedger, Ce as resolveAction, et as resolveScope, it as restoreAssetFromCas, kr as restoreCommand, Ut as rowsToNdjson, nn as runExport, Gn as runImport, Ar as runNativeStep, Le as scanForLeaks, Y as stagingNameFor, nt as storageAssetSink, tt as storageAssetSource, Sr as storageBlobSink, br as storageBlobSource, xr as storageRefsBlobSource, Qe as subsetToScope, At as tableDone, Gt as topoSortTables, ir as unpackBundle, or as unpackBundleToDir, rt as writeAssetToCas, q as writeLedger };
|