@voltro/data-transfer 0.84.0 → 0.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +549 -0
- package/THIRD-PARTY-NOTICES.md +87 -333
- package/dist/index.d.ts +116 -121
- package/dist/index.js +768 -673
- package/package.json +4 -5
package/dist/index.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i,
|
|
2
|
-
import { DATA_TRANSFERS_TABLE as
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as ae, createDecipheriv as oe, createHash as
|
|
5
|
-
import { copyFile as
|
|
6
|
-
import { STORAGE_REFS_TABLE as
|
|
7
|
-
import * as
|
|
8
|
-
import { PassThrough as
|
|
9
|
-
import { SqlClient as
|
|
10
|
-
import { mkdir as
|
|
11
|
-
import { basename as
|
|
12
|
-
import { promisify as
|
|
13
|
-
import { spawn as
|
|
1
|
+
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, Option as a, Schedule as o, Schema as s, Stream as c } from "effect";
|
|
2
|
+
import { DATA_TRANSFERS_TABLE as l, REPLACE_IN_PROGRESS_TABLE as u, STAGING_TABLE_PREFIX as d, _voltroDataTransfersTable as f, classifyConstraintViolation as p, derivedRowId as m, ensureTableRegistered as h, eq as g, extractDbCause as _, getTable as v, inSet as y, isNotNull as b, or as x, recordsTable as S, registerTable as ee, streamTable as C, suspendWriteRecorders as te, suspendWriteRecordersEffect as ne, unregisterTable as re } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as ie } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as ae, createDecipheriv as oe, createHash as w, createHmac as T, randomBytes as E, randomUUID as se, scrypt as ce } from "node:crypto";
|
|
5
|
+
import { copyFile as le, createReadStream as ue, createWriteStream as de, existsSync as fe, rename as D, statSync as O, unlinkSync as k } from "node:fs";
|
|
6
|
+
import { STORAGE_REFS_TABLE as pe, fromNodeReadable as me, getObjectStream as A, putObjectStream as he, toNodeReadable as j } from "@voltro/plugin-storage";
|
|
7
|
+
import * as M from "node:zlib";
|
|
8
|
+
import { PassThrough as ge, Readable as _e } from "node:stream";
|
|
9
|
+
import { SqlClient as ve } from "effect/sql";
|
|
10
|
+
import { mkdir as N, readdir as ye, stat as be } from "node:fs/promises";
|
|
11
|
+
import { basename as P, delimiter as F, dirname as xe, join as I, resolve as L, sep as Se } from "node:path";
|
|
12
|
+
import { promisify as Ce } from "node:util";
|
|
13
|
+
import { spawn as we } from "node:child_process";
|
|
14
14
|
//#region src/consistency.ts
|
|
15
|
-
var
|
|
15
|
+
var Te = (e) => {
|
|
16
16
|
switch (e) {
|
|
17
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
18
18
|
case "mysql":
|
|
@@ -22,7 +22,7 @@ var Ee = (e) => {
|
|
|
22
22
|
case "turso": return [];
|
|
23
23
|
default: return [];
|
|
24
24
|
}
|
|
25
|
-
},
|
|
25
|
+
}, Ee = {
|
|
26
26
|
email: { fake: "email" },
|
|
27
27
|
fullName: { fake: "fullName" },
|
|
28
28
|
firstName: { fake: "firstName" },
|
|
@@ -37,7 +37,7 @@ var Ee = (e) => {
|
|
|
37
37
|
date: "dateShift",
|
|
38
38
|
secret: "null",
|
|
39
39
|
freeText: "redact"
|
|
40
|
-
},
|
|
40
|
+
}, De = (e) => {
|
|
41
41
|
let t = {};
|
|
42
42
|
for (let n of e.tables) {
|
|
43
43
|
let e = {};
|
|
@@ -49,15 +49,15 @@ var Ee = (e) => {
|
|
|
49
49
|
t[n.name] = e;
|
|
50
50
|
}
|
|
51
51
|
return t;
|
|
52
|
-
},
|
|
52
|
+
}, Oe = (e, t, n, r) => {
|
|
53
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
54
54
|
if (i) return i;
|
|
55
55
|
if (n?.sensitive) {
|
|
56
56
|
let e = n.sensitive.class;
|
|
57
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? Ee[e] ?? "redact";
|
|
58
58
|
}
|
|
59
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
60
|
-
},
|
|
60
|
+
}, ke = (e, t) => T("sha256", e).update(t == null ? "∅" : String(t)).digest(), R = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), Ae = [
|
|
61
61
|
"Ada",
|
|
62
62
|
"Alan",
|
|
63
63
|
"Grace",
|
|
@@ -70,7 +70,7 @@ var Ee = (e) => {
|
|
|
70
70
|
"Guido",
|
|
71
71
|
"Margaret",
|
|
72
72
|
"Bjarne"
|
|
73
|
-
],
|
|
73
|
+
], je = [
|
|
74
74
|
"Lovelace",
|
|
75
75
|
"Turing",
|
|
76
76
|
"Hopper",
|
|
@@ -83,7 +83,7 @@ var Ee = (e) => {
|
|
|
83
83
|
"Rossum",
|
|
84
84
|
"Hamilton",
|
|
85
85
|
"Stroustrup"
|
|
86
|
-
],
|
|
86
|
+
], Me = [
|
|
87
87
|
"Maple",
|
|
88
88
|
"Oak",
|
|
89
89
|
"Cedar",
|
|
@@ -92,7 +92,7 @@ var Ee = (e) => {
|
|
|
92
92
|
"Birch",
|
|
93
93
|
"Willow",
|
|
94
94
|
"Ash"
|
|
95
|
-
],
|
|
95
|
+
], Ne = [
|
|
96
96
|
"Initech",
|
|
97
97
|
"Hooli",
|
|
98
98
|
"Umbrella",
|
|
@@ -101,23 +101,23 @@ var Ee = (e) => {
|
|
|
101
101
|
"Soylent",
|
|
102
102
|
"Stark",
|
|
103
103
|
"Wayne"
|
|
104
|
-
], z = (e, t, n) => e[R(t, n) % e.length],
|
|
105
|
-
let r =
|
|
104
|
+
], z = (e, t, n) => e[R(t, n) % e.length], Pe = (e, t, n) => {
|
|
105
|
+
let r = ke(n, `${e}:${String(t)}`), i = R(r, 1) % 1e4;
|
|
106
106
|
switch (e) {
|
|
107
|
-
case "email": return `${z(
|
|
108
|
-
case "fullName": return `${z(
|
|
109
|
-
case "firstName": return z(
|
|
110
|
-
case "lastName": return z(
|
|
111
|
-
case "username": return `${z(
|
|
107
|
+
case "email": return `${z(Ae, r, 0).toLowerCase()}.${z(je, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${z(Ae, r, 0)} ${z(je, r, 1)}`;
|
|
109
|
+
case "firstName": return z(Ae, r, 0);
|
|
110
|
+
case "lastName": return z(je, r, 1);
|
|
111
|
+
case "username": return `${z(Ae, r, 0).toLowerCase()}${i}`;
|
|
112
112
|
case "phone": return `+1${String(R(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
-
case "address": return `${R(r, 0) % 9899 + 100} ${z(
|
|
114
|
-
case "company": return z(
|
|
113
|
+
case "address": return `${R(r, 0) % 9899 + 100} ${z(Me, r, 1)} St`;
|
|
114
|
+
case "company": return z(Ne, r, 0);
|
|
115
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
116
116
|
case "ip": return `${R(r, 0) % 256}.${R(r, 1) % 256}.${R(r, 2) % 256}.${R(r, 3) % 254 + 1}`;
|
|
117
|
-
case "creditCard": return
|
|
117
|
+
case "creditCard": return Fe(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
118
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
119
119
|
}
|
|
120
|
-
},
|
|
120
|
+
}, Fe = (e) => {
|
|
121
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
122
122
|
t.forEach((e, t) => {
|
|
123
123
|
let r = e;
|
|
@@ -125,10 +125,10 @@ var Ee = (e) => {
|
|
|
125
125
|
});
|
|
126
126
|
let r = (10 - n % 10) % 10;
|
|
127
127
|
return e.slice(0, 15) + String(r);
|
|
128
|
-
},
|
|
128
|
+
}, Ie = 864e5, Le = (e, t) => {
|
|
129
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
130
130
|
if (Number.isNaN(n)) return e;
|
|
131
|
-
let r = R(
|
|
131
|
+
let r = R(ke(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Ie);
|
|
132
132
|
return e instanceof Date ? i : i.toISOString();
|
|
133
133
|
}, B = (e, t) => {
|
|
134
134
|
if (t.value === null || t.value === void 0) return null;
|
|
@@ -136,14 +136,14 @@ var Ee = (e) => {
|
|
|
136
136
|
if (e === "keep") return t.value;
|
|
137
137
|
if (e === "null") return null;
|
|
138
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
139
|
-
if (e === "hash") return
|
|
140
|
-
if (e === "dateShift") return
|
|
141
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return ke(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Le(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return Pe(e.fake, t.value, t.seed);
|
|
142
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
143
|
-
if (typeof e == "object" && "json" in e) return
|
|
143
|
+
if (typeof e == "object" && "json" in e) return Re(e, t);
|
|
144
144
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn } | { json: { fields, rest? } }.`);
|
|
145
|
-
},
|
|
146
|
-
let { fields: n, rest: r = "redact" } = e.json, i = typeof t.value == "string" ?
|
|
145
|
+
}, Re = (e, t) => {
|
|
146
|
+
let { fields: n, rest: r = "redact" } = e.json, i = typeof t.value == "string" ? ze(t.value) : {
|
|
147
147
|
ok: !0,
|
|
148
148
|
value: t.value
|
|
149
149
|
};
|
|
@@ -158,7 +158,7 @@ var Ee = (e) => {
|
|
|
158
158
|
let a = {};
|
|
159
159
|
for (let [o, s] of Object.entries(e)) {
|
|
160
160
|
let e = n[o] ?? r, c = `${i}.${o}`;
|
|
161
|
-
a[o] = typeof e == "object" && "json" in e ?
|
|
161
|
+
a[o] = typeof e == "object" && "json" in e ? Re(e, {
|
|
162
162
|
...t,
|
|
163
163
|
value: s,
|
|
164
164
|
column: c,
|
|
@@ -180,7 +180,7 @@ var Ee = (e) => {
|
|
|
180
180
|
});
|
|
181
181
|
}, o = a(i.value, t.column);
|
|
182
182
|
return typeof t.value == "string" ? JSON.stringify(o) : o;
|
|
183
|
-
},
|
|
183
|
+
}, ze = (e) => {
|
|
184
184
|
try {
|
|
185
185
|
return {
|
|
186
186
|
ok: !0,
|
|
@@ -189,7 +189,7 @@ var Ee = (e) => {
|
|
|
189
189
|
} catch {
|
|
190
190
|
return { ok: !1 };
|
|
191
191
|
}
|
|
192
|
-
},
|
|
192
|
+
}, Be = (e) => {
|
|
193
193
|
if (typeof e == "string") return e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error";
|
|
194
194
|
if (typeof e != "object" || !e) return !1;
|
|
195
195
|
if ("fake" in e) return typeof e.fake == "string";
|
|
@@ -198,18 +198,18 @@ var Ee = (e) => {
|
|
|
198
198
|
let t = e.json;
|
|
199
199
|
if (typeof t != "object" || !t) return !1;
|
|
200
200
|
let { fields: n, rest: r } = t;
|
|
201
|
-
return typeof n != "object" || !n ? !1 : Object.values(n).every(
|
|
201
|
+
return typeof n != "object" || !n ? !1 : Object.values(n).every(Be) && (r === void 0 || Be(r));
|
|
202
202
|
}
|
|
203
203
|
return !1;
|
|
204
|
-
},
|
|
204
|
+
}, Ve = (e, t, n) => {
|
|
205
205
|
let r = {}, i = [], a = [];
|
|
206
206
|
for (let o of e) for (let e of o.columns) {
|
|
207
|
-
let s = `${o.name}.${e.name}`, c =
|
|
207
|
+
let s = `${o.name}.${e.name}`, c = Oe(o.name, e.name, t[o.name]?.[e.name], n);
|
|
208
208
|
if (c === "error") {
|
|
209
209
|
i.push(s);
|
|
210
210
|
continue;
|
|
211
211
|
}
|
|
212
|
-
if (!
|
|
212
|
+
if (!Be(c)) {
|
|
213
213
|
a.push({
|
|
214
214
|
column: s,
|
|
215
215
|
action: c
|
|
@@ -223,7 +223,7 @@ var Ee = (e) => {
|
|
|
223
223
|
unclassified: i,
|
|
224
224
|
invalid: a
|
|
225
225
|
};
|
|
226
|
-
},
|
|
226
|
+
}, He = [
|
|
227
227
|
{
|
|
228
228
|
name: "email",
|
|
229
229
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -240,7 +240,7 @@ var Ee = (e) => {
|
|
|
240
240
|
name: "creditCard",
|
|
241
241
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
242
242
|
}
|
|
243
|
-
],
|
|
243
|
+
], Ue = (e, t, n) => {
|
|
244
244
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
245
245
|
for (let [a, o] of Object.entries(n)) {
|
|
246
246
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -248,7 +248,7 @@ var Ee = (e) => {
|
|
|
248
248
|
for (let a of e) {
|
|
249
249
|
let e = a[n];
|
|
250
250
|
if (typeof e != "string" || e.length === 0) continue;
|
|
251
|
-
let o =
|
|
251
|
+
let o = He.find((t) => t.re.test(e));
|
|
252
252
|
if (o && !i.has(n)) {
|
|
253
253
|
i.add(n), r.push({
|
|
254
254
|
table: t,
|
|
@@ -261,7 +261,7 @@ var Ee = (e) => {
|
|
|
261
261
|
}
|
|
262
262
|
}
|
|
263
263
|
return r;
|
|
264
|
-
},
|
|
264
|
+
}, We = (e, t, n, r, i) => {
|
|
265
265
|
let a = {};
|
|
266
266
|
for (let [o, s] of Object.entries(e)) a[o] = B(n[`${t}.${o}`] ?? "keep", {
|
|
267
267
|
value: s,
|
|
@@ -271,7 +271,7 @@ var Ee = (e) => {
|
|
|
271
271
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
272
272
|
});
|
|
273
273
|
return a;
|
|
274
|
-
},
|
|
274
|
+
}, Ge = /* @__PURE__ */ new Map([
|
|
275
275
|
["_voltro_inspect_create_receipts", "inspect creation commands accepted by THIS deployment — a foreign receipt would replay or suppress a creation the target never accepted"],
|
|
276
276
|
["_voltro_isr_cache", "pages rendered by THIS deployment’s web processes — a foreign row would serve another environment’s HTML here"],
|
|
277
277
|
["_voltro_web_replicas", "heartbeats of the web processes running HERE — a foreign row advertises replicas that do not exist in the target"],
|
|
@@ -345,7 +345,8 @@ var Ee = (e) => {
|
|
|
345
345
|
["cluster_messages", "workflow messages in flight HERE — a foreign row makes the target run work that was started, and may be running, elsewhere"],
|
|
346
346
|
["cluster_replies", "replies to those messages — a foreign row answers a request the target never sent"],
|
|
347
347
|
["cluster_migrations", "which engine migrations ran HERE — a foreign row marks the target's engine tables migrated when they are not"]
|
|
348
|
-
]),
|
|
348
|
+
]), Ke = /* @__PURE__ */ new Map([
|
|
349
|
+
["_voltro_comment_thread_events", "the status history of the app's comment threads — belongs with those threads"],
|
|
349
350
|
["_voltro_agent_threads", "conversations the app's users had — app data"],
|
|
350
351
|
["_voltro_agent_messages", "the messages in those conversations — app data"],
|
|
351
352
|
["_voltro_prompts", "prompt definitions the app authored — app data"],
|
|
@@ -381,117 +382,165 @@ var Ee = (e) => {
|
|
|
381
382
|
["_voltro_comments", "the comments the app's users wrote — app data"],
|
|
382
383
|
["_voltro_comment_threads", "the threads those comments hang on — app data"],
|
|
383
384
|
["_voltro_comment_reactions", "the reactions the app's users left — app data"],
|
|
384
|
-
["_voltro_comment_reads", "which comments each user has read — app data, and losing it re-marks everything unread"]
|
|
385
|
-
|
|
385
|
+
["_voltro_comment_reads", "which comments each user has read — app data, and losing it re-marks everything unread"],
|
|
386
|
+
["_voltro_comment_thread_events", "the status changes the app's users made on a thread — app data"]
|
|
387
|
+
]), V = (e) => Ge.has(e), qe = (e) => Ge.get(e), Je = (e, t) => {
|
|
386
388
|
if (e.length === 0) return;
|
|
387
|
-
let n = e.map((e) => ` ${e} — ${
|
|
389
|
+
let n = e.map((e) => ` ${e} — ${qe(e) ?? "describes this deployment"}`);
|
|
388
390
|
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
389
|
-
}, H = class extends
|
|
390
|
-
reason:
|
|
391
|
-
path:
|
|
392
|
-
}) {
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
}) {},
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
391
|
+
}, H = class extends s.TaggedError()("BundleError", {
|
|
392
|
+
reason: s.String,
|
|
393
|
+
path: s.optional(s.String)
|
|
394
|
+
}) {
|
|
395
|
+
get message() {
|
|
396
|
+
return this.path === void 0 ? this.reason : `${this.reason} (${this.path})`;
|
|
397
|
+
}
|
|
398
|
+
}, Ye = class extends s.TaggedError()("CodecError", {
|
|
399
|
+
table: s.String,
|
|
400
|
+
reason: s.String
|
|
401
|
+
}) {}, Xe = class extends s.TaggedError()("IntegrityError", {
|
|
402
|
+
what: s.String,
|
|
403
|
+
expected: s.String,
|
|
404
|
+
actual: s.String
|
|
405
|
+
}) {
|
|
406
|
+
get message() {
|
|
407
|
+
return `${this.what}: expected ${this.expected}, got ${this.actual}`;
|
|
408
|
+
}
|
|
409
|
+
}, Ze = class extends t.TaggedError("CompressionError") {}, Qe = class extends s.TaggedError()("ScopeError", { reason: s.String }) {
|
|
410
|
+
get message() {
|
|
411
|
+
return this.reason;
|
|
412
|
+
}
|
|
413
|
+
}, $e = class extends s.TaggedError()("RowsRefusedError", {
|
|
414
|
+
rows: s.Array(s.Struct({
|
|
415
|
+
table: s.String,
|
|
416
|
+
id: s.String,
|
|
417
|
+
reason: s.String,
|
|
418
|
+
derived: s.Boolean
|
|
405
419
|
})),
|
|
406
|
-
primaryCount:
|
|
407
|
-
totalCount:
|
|
408
|
-
byTable:
|
|
409
|
-
table:
|
|
410
|
-
refused:
|
|
411
|
-
primary:
|
|
420
|
+
primaryCount: s.Number,
|
|
421
|
+
totalCount: s.Number,
|
|
422
|
+
byTable: s.Array(s.Struct({
|
|
423
|
+
table: s.String,
|
|
424
|
+
refused: s.Number,
|
|
425
|
+
primary: s.Number
|
|
412
426
|
}))
|
|
413
|
-
}) {}, et = class extends
|
|
414
|
-
bundleFingerprint:
|
|
415
|
-
targetFingerprint:
|
|
416
|
-
diff:
|
|
417
|
-
}) {
|
|
427
|
+
}) {}, et = class extends s.TaggedError()("SchemaDriftError", {
|
|
428
|
+
bundleFingerprint: s.String,
|
|
429
|
+
targetFingerprint: s.String,
|
|
430
|
+
diff: s.Array(s.String)
|
|
431
|
+
}) {
|
|
432
|
+
get message() {
|
|
433
|
+
return `bundle schema ${this.bundleFingerprint} does not match target ${this.targetFingerprint}: ${this.diff.join("; ")}`;
|
|
434
|
+
}
|
|
435
|
+
}, U = class extends t.TaggedError("NativeToolError") {
|
|
418
436
|
get message() {
|
|
419
437
|
let e = this.stderr?.trim(), t = this.hint?.trim();
|
|
420
438
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}${t ? `\n\nhint: ${t}` : ""}`;
|
|
421
439
|
}
|
|
422
|
-
}, tt = class extends
|
|
423
|
-
unclassified:
|
|
424
|
-
invalidActions:
|
|
425
|
-
}) {
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
440
|
+
}, tt = class extends s.TaggedError()("MaskingError", {
|
|
441
|
+
unclassified: s.Array(s.String),
|
|
442
|
+
invalidActions: s.optional(s.Array(s.String))
|
|
443
|
+
}) {
|
|
444
|
+
get message() {
|
|
445
|
+
let e = [];
|
|
446
|
+
return this.unclassified.length > 0 && e.push(`unclassified columns: ${this.unclassified.join(", ")}`), this.invalidActions !== void 0 && this.invalidActions.length > 0 && e.push(`invalidActions: ${this.invalidActions.join(", ")}`), `masking refused — ${e.join("; ") || "no column may be exported"}`;
|
|
447
|
+
}
|
|
448
|
+
}, nt = class extends s.TaggedError()("ImportModeError", {
|
|
449
|
+
mode: s.String,
|
|
450
|
+
reason: s.String
|
|
451
|
+
}) {
|
|
452
|
+
get message() {
|
|
453
|
+
return `import mode '${this.mode}' refused: ${this.reason}`;
|
|
454
|
+
}
|
|
455
|
+
}, rt = class extends s.TaggedError()("CrossDialectError", {
|
|
456
|
+
sourceDialect: s.String,
|
|
457
|
+
targetDialect: s.String,
|
|
458
|
+
blocking: s.Array(s.String)
|
|
459
|
+
}) {}, it = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", at = (e, t, r) => n.gen(function* () {
|
|
460
|
+
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map(), u = /* @__PURE__ */ new Map();
|
|
461
|
+
for (let e of t.tables) u.has(e.name) || u.set(e.name, e);
|
|
462
|
+
let d = /* @__PURE__ */ new Map();
|
|
463
|
+
for (let [e, t] of u) d.set(e, {
|
|
464
|
+
pk: t.columns.find((e) => e.type === "id")?.name ?? "id",
|
|
465
|
+
fks: t.columns.filter((e) => e.references !== void 0).map((e) => ({
|
|
466
|
+
col: e.name,
|
|
467
|
+
refTable: e.references.table
|
|
468
|
+
}))
|
|
469
|
+
});
|
|
470
|
+
let f = (e) => d.get(e)?.pk ?? "id", p = (e) => {
|
|
437
471
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
438
472
|
return s.set(e, t), t;
|
|
439
|
-
},
|
|
473
|
+
}, m = (e, t) => {
|
|
440
474
|
if (t == null || a.has(e) || s.get(e)?.has(t)) return;
|
|
441
|
-
let n =
|
|
442
|
-
n.add(t),
|
|
443
|
-
},
|
|
444
|
-
let r =
|
|
475
|
+
let n = l.get(e) ?? /* @__PURE__ */ new Set();
|
|
476
|
+
n.add(t), l.set(e, n);
|
|
477
|
+
}, h = (e, t, n) => {
|
|
478
|
+
let r = p(e), i = a.has(e) ? void 0 : o.get(e) ?? (() => {
|
|
445
479
|
let t = /* @__PURE__ */ new Set();
|
|
446
480
|
return o.set(e, t), t;
|
|
447
481
|
})();
|
|
448
482
|
return (e) => {
|
|
449
483
|
r.add(e[t]), i?.add(e[t]);
|
|
450
|
-
for (let t of n)
|
|
484
|
+
for (let t of n) m(t.refTable, e[t.col]);
|
|
451
485
|
};
|
|
452
|
-
},
|
|
453
|
-
let
|
|
454
|
-
return
|
|
455
|
-
table:
|
|
456
|
-
pk:
|
|
457
|
-
...
|
|
458
|
-
}), (e) => n.sync(() =>
|
|
459
|
-
},
|
|
460
|
-
for (let e = 0; e < i &&
|
|
461
|
-
let e = [...
|
|
462
|
-
|
|
463
|
-
for (let [
|
|
464
|
-
let e = [...
|
|
465
|
-
e.length !== 0
|
|
486
|
+
}, g = (t, r) => {
|
|
487
|
+
let i = h(t, f(t), d.get(t)?.fks ?? []);
|
|
488
|
+
return c.runForEach(C(e, {
|
|
489
|
+
table: t,
|
|
490
|
+
pk: f(t),
|
|
491
|
+
...r === void 0 ? {} : { where: r }
|
|
492
|
+
}), (e) => n.sync(() => i(e)));
|
|
493
|
+
}, _ = n.gen(function* () {
|
|
494
|
+
for (let e = 0; e < i && l.size > 0; e++) {
|
|
495
|
+
let e = [...l.entries()];
|
|
496
|
+
l.clear();
|
|
497
|
+
for (let [t, n] of e) {
|
|
498
|
+
let e = [...n].filter((e) => !s.get(t)?.has(e));
|
|
499
|
+
if (e.length !== 0) for (let n = 0; n < e.length; n += 500) yield* g(t, y(f(t), e.slice(n, n + 500)));
|
|
466
500
|
}
|
|
467
501
|
}
|
|
468
502
|
});
|
|
469
|
-
for (let [e, t] of Object.entries(r.seeds)) yield*
|
|
470
|
-
if (yield*
|
|
471
|
-
let e = new Set(r.children === !0 ? [...s.keys()] : r.children.roots.filter((e) => s.has(e)));
|
|
472
|
-
for (let
|
|
473
|
-
let
|
|
474
|
-
for (let
|
|
475
|
-
let t =
|
|
503
|
+
for (let [e, t] of Object.entries(r.seeds)) yield* g(e, t);
|
|
504
|
+
if (yield* _, r.children) {
|
|
505
|
+
let e = new Set(r.children === !0 ? [...s.keys()] : r.children.roots.filter((e) => s.has(e))), a = /* @__PURE__ */ new Map();
|
|
506
|
+
for (let r = 0; r < i; r++) {
|
|
507
|
+
let r = !1, i = [];
|
|
508
|
+
for (let o of t.tables) {
|
|
509
|
+
let t = o.name, c = o.columns.map((e) => ({
|
|
476
510
|
col: e.name,
|
|
477
511
|
refTable: e.references?.table
|
|
478
512
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
479
|
-
if (
|
|
480
|
-
let
|
|
481
|
-
|
|
513
|
+
if (c.length === 0) continue;
|
|
514
|
+
let l = s.get(t)?.size ?? 0, u = a.get(t);
|
|
515
|
+
u === void 0 && (u = /* @__PURE__ */ new Map(), a.set(t, u));
|
|
516
|
+
let d = [], f = 0, p = () => {
|
|
517
|
+
let e = d;
|
|
518
|
+
return d = [], f = 0, e.length === 0 ? n.void : g(t, e.length === 1 ? e[0] : x(...e));
|
|
519
|
+
};
|
|
520
|
+
for (let e of c) {
|
|
521
|
+
let t = u.get(e.col);
|
|
522
|
+
t === void 0 && (t = /* @__PURE__ */ new Set(), u.set(e.col, t));
|
|
523
|
+
let n = [...s.get(e.refTable)].filter((e) => !t.has(e));
|
|
524
|
+
for (let t = 0; t < n.length; t += 500) {
|
|
525
|
+
let r = n.slice(t, t + 500);
|
|
526
|
+
f + r.length > 500 && (yield* p()), d.push(y(e.col, r)), f += r.length;
|
|
527
|
+
}
|
|
528
|
+
for (let e of n) t.add(e);
|
|
529
|
+
}
|
|
530
|
+
yield* p(), (s.get(t)?.size ?? 0) > l && !e.has(t) && i.push(t), (s.get(t)?.size ?? 0) > l && (r = !0);
|
|
482
531
|
}
|
|
483
|
-
for (let t of
|
|
484
|
-
if (yield*
|
|
532
|
+
for (let t of i) e.add(t);
|
|
533
|
+
if (yield* _, !r) break;
|
|
485
534
|
}
|
|
486
535
|
}
|
|
487
536
|
return o;
|
|
488
|
-
}),
|
|
537
|
+
}), ot = (e, t) => ({
|
|
489
538
|
tables: [...e.keys()],
|
|
490
539
|
predicateFor: (n) => {
|
|
491
540
|
let r = e.get(n);
|
|
492
|
-
return r ?
|
|
541
|
+
return r ? y(it(t, n), [...r]) : void 0;
|
|
493
542
|
}
|
|
494
|
-
}),
|
|
543
|
+
}), st = "tenantId", ct = (e, t, r) => n.gen(function* () {
|
|
495
544
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
496
545
|
if (e.kind === "all") {
|
|
497
546
|
let t = new Set(e.exclude ?? []);
|
|
@@ -502,7 +551,7 @@ var Ee = (e) => {
|
|
|
502
551
|
}
|
|
503
552
|
if (e.kind === "tables") {
|
|
504
553
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
505
|
-
if (a.length > 0) return yield* n.fail(new
|
|
554
|
+
if (a.length > 0) return yield* n.fail(new Qe({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
506
555
|
let o = new Set(t);
|
|
507
556
|
return {
|
|
508
557
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -510,30 +559,30 @@ var Ee = (e) => {
|
|
|
510
559
|
};
|
|
511
560
|
}
|
|
512
561
|
let o = e.tenantId;
|
|
513
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
514
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
515
|
-
if (s.length === 0) return yield* n.fail(new
|
|
516
|
-
let c = yield*
|
|
517
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
562
|
+
if (o === void 0 || o === "") return yield* n.fail(new Qe({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
563
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === st) ?? !1);
|
|
564
|
+
if (s.length === 0) return yield* n.fail(new Qe({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${st}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
565
|
+
let c = yield* at(r.store, t, {
|
|
566
|
+
seeds: Object.fromEntries(s.map((e) => [e, g(st, o)])),
|
|
518
567
|
pinned: s,
|
|
519
568
|
children: { roots: s }
|
|
520
569
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
521
570
|
return {
|
|
522
571
|
tables: [...s, ...l],
|
|
523
|
-
predicateFor: (e) => u.has(e) ?
|
|
572
|
+
predicateFor: (e) => u.has(e) ? g(st, o) : c.has(e) ? y(it(t, e), [...c.get(e)]) : void 0
|
|
524
573
|
};
|
|
525
|
-
}),
|
|
574
|
+
}), lt = (e, t) => ({
|
|
526
575
|
list: t,
|
|
527
|
-
open: (t) =>
|
|
528
|
-
}),
|
|
529
|
-
write: (t, n, r) =>
|
|
576
|
+
open: (t) => A(e, t)
|
|
577
|
+
}), ut = (e) => ({
|
|
578
|
+
write: (t, n, r) => he(e, t, n, r),
|
|
530
579
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
531
|
-
}),
|
|
580
|
+
}), dt = (e, t) => n.tryPromise({
|
|
532
581
|
try: async () => {
|
|
533
582
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
534
583
|
u.update(e), d += e.length, n(null, e);
|
|
535
584
|
} });
|
|
536
|
-
await o(await
|
|
585
|
+
await o(await j(e.stream), f, n(l));
|
|
537
586
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
538
587
|
try {
|
|
539
588
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -550,33 +599,33 @@ var Ee = (e) => {
|
|
|
550
599
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
551
600
|
path: t
|
|
552
601
|
})
|
|
553
|
-
}),
|
|
554
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256),
|
|
602
|
+
}), ft = (e, t, r) => n.gen(function* () {
|
|
603
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), l = o("sha256"), u = me("bundle", () => a(s)).pipe(c.tap((e) => n.sync(() => l.update(e))));
|
|
555
604
|
yield* e.write(r.key, u, {
|
|
556
605
|
contentType: r.contentType,
|
|
557
606
|
totalSize: r.size
|
|
558
607
|
});
|
|
559
|
-
let d =
|
|
560
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
608
|
+
let d = l.digest("hex");
|
|
609
|
+
if (d !== r.sha256) return yield* n.fail(new Xe({
|
|
561
610
|
what: `asset ${r.key}`,
|
|
562
611
|
expected: r.sha256,
|
|
563
612
|
actual: d
|
|
564
613
|
}));
|
|
565
|
-
}),
|
|
614
|
+
}), pt = {
|
|
566
615
|
done: () => !1,
|
|
567
616
|
mark: () => n.void
|
|
568
|
-
},
|
|
569
|
-
let t = e.progress ??
|
|
570
|
-
return yield*
|
|
571
|
-
u.push(
|
|
572
|
-
})), n.flatMap((a) => a === void 0 ? n.void :
|
|
617
|
+
}, mt = (e) => typeof e == "object" && !!e && e.status === 404, ht = (e) => n.gen(function* () {
|
|
618
|
+
let t = e.progress ?? pt, r = 0, i = 0, a = 0, o = 0, s = /* @__PURE__ */ new Set(), l = /* @__PURE__ */ new Set(), u = [];
|
|
619
|
+
return yield* c.runForEach(e.source.list(), (c) => (a++, t.done(c.key) || l.has(c.key) ? n.void : (l.add(c.key), e.source.open(c.key).pipe(n.map((e) => e), n.catchIf(mt, () => n.sync(() => {
|
|
620
|
+
u.push(c.key);
|
|
621
|
+
})), n.flatMap((a) => a === void 0 ? n.void : dt(a, e.assetsDir).pipe(n.flatMap((e) => {
|
|
573
622
|
let n = {
|
|
574
|
-
key:
|
|
623
|
+
key: c.key,
|
|
575
624
|
sha256: e.sha256,
|
|
576
625
|
size: e.size,
|
|
577
626
|
contentType: e.contentType
|
|
578
627
|
};
|
|
579
|
-
return r++, i += e.size, s.has(e.sha256) || (s.add(e.sha256), o += e.size), t.mark(
|
|
628
|
+
return r++, i += e.size, s.has(e.sha256) || (s.add(e.sha256), o += e.size), t.mark(c.key, n);
|
|
580
629
|
}))))))).pipe(n.mapError((e) => e instanceof H ? e : new H({ reason: `asset export failed: ${String(e)}` }))), {
|
|
581
630
|
count: r,
|
|
582
631
|
totalBytes: i,
|
|
@@ -585,60 +634,65 @@ var Ee = (e) => {
|
|
|
585
634
|
objectBytes: o,
|
|
586
635
|
missing: u
|
|
587
636
|
};
|
|
588
|
-
}),
|
|
637
|
+
}), gt = (e) => me("bundle", () => ue(e)).pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.map((e) => JSON.parse(e)), c.mapError((t) => t instanceof H ? t : new H({
|
|
589
638
|
reason: `asset index unreadable: ${String(t)}`,
|
|
590
639
|
path: e
|
|
591
|
-
}))),
|
|
592
|
-
let t = e.progress ??
|
|
593
|
-
return yield*
|
|
640
|
+
}))), _t = (e) => n.gen(function* () {
|
|
641
|
+
let t = e.progress ?? pt, r = 0, i = /* @__PURE__ */ new Set(), a = /* @__PURE__ */ new Set();
|
|
642
|
+
return yield* c.runForEach(gt(e.indexPath), (o) => t.done(o.key) || a.has(o.key) ? n.void : (a.add(o.key), ft(e.sink, e.assetsDir, o)).pipe(n.andThen(n.sync(() => {
|
|
594
643
|
r++, i.add(o.sha256);
|
|
595
644
|
})), n.andThen(t.mark(o.key, o)))), {
|
|
596
645
|
count: r,
|
|
597
646
|
objects: i.size
|
|
598
647
|
};
|
|
599
|
-
}),
|
|
600
|
-
name:
|
|
601
|
-
file:
|
|
602
|
-
rowCount:
|
|
603
|
-
checksum:
|
|
604
|
-
columnTypes:
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
648
|
+
}), vt = () => typeof M.createZstdCompress == "function", yt = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : vt() ? "zstd" : "gzip", bt = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", xt = (e) => e === "zstd" ? M.createZstdCompress() : e === "gzip" ? M.createGzip() : new ge(), St = (e) => e === "zstd" ? M.createZstdDecompress() : e === "gzip" ? M.createGunzip() : new ge(), Ct = 1, wt = s.Struct({
|
|
649
|
+
name: s.String,
|
|
650
|
+
file: s.String,
|
|
651
|
+
rowCount: s.Number,
|
|
652
|
+
checksum: s.String,
|
|
653
|
+
columnTypes: s.Record(s.String, s.String),
|
|
654
|
+
primaryKey: s.String
|
|
655
|
+
}), Tt = s.Struct({
|
|
656
|
+
kind: s.Literals([
|
|
657
|
+
"all",
|
|
658
|
+
"tenant",
|
|
659
|
+
"tables"
|
|
660
|
+
]),
|
|
661
|
+
tenantId: s.optional(s.String),
|
|
662
|
+
tables: s.optional(s.Array(s.String)),
|
|
663
|
+
exclude: s.optional(s.Array(s.String))
|
|
664
|
+
}), Et = s.Struct({
|
|
665
|
+
count: s.Number,
|
|
666
|
+
totalBytes: s.Number,
|
|
667
|
+
index: s.optional(s.String)
|
|
618
668
|
});
|
|
619
|
-
|
|
620
|
-
key:
|
|
621
|
-
sha256:
|
|
622
|
-
size:
|
|
623
|
-
contentType:
|
|
669
|
+
s.Struct({
|
|
670
|
+
key: s.String,
|
|
671
|
+
sha256: s.String,
|
|
672
|
+
size: s.Number,
|
|
673
|
+
contentType: s.String
|
|
624
674
|
});
|
|
625
|
-
var
|
|
626
|
-
formatVersion:
|
|
627
|
-
createdAt:
|
|
628
|
-
source:
|
|
629
|
-
dialect:
|
|
630
|
-
schemaFingerprint:
|
|
675
|
+
var Dt = s.Struct({
|
|
676
|
+
formatVersion: s.Number,
|
|
677
|
+
createdAt: s.String,
|
|
678
|
+
source: s.Struct({
|
|
679
|
+
dialect: s.String,
|
|
680
|
+
schemaFingerprint: s.String
|
|
631
681
|
}),
|
|
632
|
-
scope:
|
|
633
|
-
consistency:
|
|
634
|
-
compression:
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
682
|
+
scope: Tt,
|
|
683
|
+
consistency: s.Literals(["live", "snapshot"]),
|
|
684
|
+
compression: s.Literals([
|
|
685
|
+
"zstd",
|
|
686
|
+
"gzip",
|
|
687
|
+
"none"
|
|
688
|
+
]),
|
|
689
|
+
tables: s.Array(wt),
|
|
690
|
+
assets: Et,
|
|
691
|
+
masking: s.optional(s.Struct({
|
|
692
|
+
policyId: s.optional(s.String),
|
|
693
|
+
transformed: s.Array(s.String)
|
|
640
694
|
}))
|
|
641
|
-
}),
|
|
695
|
+
}), Ot = s.encodeSync(s.fromJsonString(Dt)), kt = s.decodeUnknownSync(s.fromJsonString(Dt)), At = "manifest.json", jt = "README.md", Mt = "data", Nt = "assets", Pt = "assets/index.ndjson", Ft = ".ledger.json", It = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", Lt = (e, t) => {
|
|
642
696
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
643
697
|
return `# Voltro data export
|
|
644
698
|
|
|
@@ -652,7 +706,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
652
706
|
| **Format version** | ${e.formatVersion} |
|
|
653
707
|
| **Consistency** | ${e.consistency} |
|
|
654
708
|
| **Compression** | ${e.compression} |
|
|
655
|
-
| **Scope** | ${
|
|
709
|
+
| **Scope** | ${It(e.scope)} |
|
|
656
710
|
| **Tables** | ${e.tables.length} |
|
|
657
711
|
| **Rows (total)** | ${n} |
|
|
658
712
|
| **Assets** | ${a} |
|
|
@@ -677,26 +731,25 @@ voltro data import <this-directory>
|
|
|
677
731
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
678
732
|
\`\`\`
|
|
679
733
|
`;
|
|
680
|
-
},
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
}), Vt = i.histogram("voltro_data_transfer_table_seconds", Bt, "Per-table export/import duration in seconds, tagged by phase."), G = (e, t) => {
|
|
734
|
+
}, Rt = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), zt = Array.from({ length: 16 }, (e, t) => .01 * 2 ** t), Bt = i.histogram("voltro_data_transfer_table_seconds", {
|
|
735
|
+
boundaries: zt,
|
|
736
|
+
description: "Per-table export/import duration in seconds, tagged by phase."
|
|
737
|
+
}), W = (e, t) => {
|
|
685
738
|
if (e) try {
|
|
686
739
|
e(t);
|
|
687
740
|
} catch {}
|
|
688
|
-
},
|
|
741
|
+
}, Vt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.andThen(Bt.pipe(i.withAttributes({ phase: e }), i.update((s - r) / 1e3)), Rt.pipe(i.withAttributes({ phase: e }), i.withAttributes({ table: t }), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
689
742
|
table: t,
|
|
690
743
|
index: r,
|
|
691
744
|
tableCount: a,
|
|
692
745
|
phase: e
|
|
693
|
-
} })),
|
|
746
|
+
} })), Ht = () => ({
|
|
694
747
|
tables: {},
|
|
695
748
|
assets: {}
|
|
696
|
-
}),
|
|
749
|
+
}), Ut = (e) => n.tryPromise(async () => {
|
|
697
750
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
698
751
|
return JSON.parse(t);
|
|
699
|
-
}).pipe(n.orElseSucceed(
|
|
752
|
+
}).pipe(n.orElseSucceed(Ht)), G = (e, t) => n.tryPromise({
|
|
700
753
|
try: async () => {
|
|
701
754
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
702
755
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
@@ -705,11 +758,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
705
758
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
706
759
|
path: e
|
|
707
760
|
})
|
|
708
|
-
}),
|
|
761
|
+
}), Wt = (e, t) => t in e.tables, Gt = (e, t) => t in e.assets, Kt = /* @__PURE__ */ new Set([
|
|
709
762
|
"json",
|
|
710
763
|
"array",
|
|
711
764
|
"vector"
|
|
712
|
-
]),
|
|
765
|
+
]), qt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", Jt = (e) => Buffer.from(e).toString("base64"), Yt = (e) => new Uint8Array(Buffer.from(e, "base64")), Xt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), Zt = (e, t) => e == null ? null : t !== void 0 && Kt.has(t) ? e : t === "bytes" ? Jt(e) : t === "timestamp" || t === "date" ? Xt(e) : typeof e == "bigint" ? {
|
|
713
766
|
$v: "bigint",
|
|
714
767
|
d: e.toString()
|
|
715
768
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -720,30 +773,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
720
773
|
d: e.toISOString()
|
|
721
774
|
} : e instanceof Uint8Array ? {
|
|
722
775
|
$v: "bytes",
|
|
723
|
-
d:
|
|
724
|
-
} : e,
|
|
776
|
+
d: Jt(e)
|
|
777
|
+
} : e, Qt = (e) => {
|
|
725
778
|
switch (e.$v) {
|
|
726
779
|
case "bigint": return BigInt(e.d);
|
|
727
780
|
case "number": return Number(e.d);
|
|
728
781
|
case "date": return new Date(e.d);
|
|
729
|
-
case "bytes": return
|
|
782
|
+
case "bytes": return Yt(e.d);
|
|
730
783
|
default: return e;
|
|
731
784
|
}
|
|
732
|
-
},
|
|
785
|
+
}, $t = (e, t) => e == null ? null : t !== void 0 && Kt.has(t) ? e : qt(e) ? Qt(e) : t === "bytes" ? Yt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, en = (e, t) => {
|
|
733
786
|
let n = {};
|
|
734
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
787
|
+
for (let [r, i] of Object.entries(e)) n[r] = Zt(i, t[r]);
|
|
735
788
|
return n;
|
|
736
|
-
},
|
|
789
|
+
}, tn = (e, t) => {
|
|
737
790
|
let n = {};
|
|
738
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
791
|
+
for (let [r, i] of Object.entries(e)) n[r] = $t(i, t[r]);
|
|
739
792
|
return n;
|
|
740
|
-
},
|
|
741
|
-
try: () =>
|
|
742
|
-
catch: (e) => new
|
|
793
|
+
}, nn = new TextEncoder(), rn = (e, t) => e.pipe(c.map((e) => nn.encode(`${JSON.stringify(en(e, t))}\n`))), an = (e, t, r) => e.pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.mapEffect((e) => n.try({
|
|
794
|
+
try: () => tn(JSON.parse(e), r),
|
|
795
|
+
catch: (e) => new Ye({
|
|
743
796
|
table: t,
|
|
744
797
|
reason: String(e?.message ?? e)
|
|
745
798
|
})
|
|
746
|
-
}))),
|
|
799
|
+
}))), on = (e, t) => {
|
|
747
800
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
748
801
|
for (let e of t.tables) {
|
|
749
802
|
if (!n.has(e.name)) continue;
|
|
@@ -763,48 +816,48 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
763
816
|
};
|
|
764
817
|
for (let t of e) n.has(t) && s(t);
|
|
765
818
|
return i;
|
|
766
|
-
},
|
|
819
|
+
}, K = (e) => {
|
|
767
820
|
let t = e, n = [];
|
|
768
821
|
typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
|
|
769
822
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
770
|
-
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(
|
|
823
|
+
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(K(t.cause)) : n.push(r), n.length === 0) {
|
|
771
824
|
let t = String(e ?? "").trim();
|
|
772
|
-
return t === "" || t === "[object Object]" ?
|
|
825
|
+
return t === "" || t === "[object Object]" ? sn(e) : t;
|
|
773
826
|
}
|
|
774
827
|
return n.join(": ");
|
|
775
|
-
},
|
|
828
|
+
}, sn = (e) => {
|
|
776
829
|
try {
|
|
777
830
|
let t = JSON.stringify(e);
|
|
778
831
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
779
832
|
} catch {
|
|
780
833
|
return "unknown error (not serialisable)";
|
|
781
834
|
}
|
|
782
|
-
},
|
|
835
|
+
}, cn = (e, t) => {
|
|
783
836
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
784
837
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
785
838
|
return r;
|
|
786
|
-
},
|
|
839
|
+
}, ln = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), un = (e, t) => {
|
|
787
840
|
if (t.length === 0) return e;
|
|
788
841
|
let n = { ...e };
|
|
789
842
|
for (let e of t) delete n[e];
|
|
790
843
|
return n;
|
|
791
|
-
},
|
|
844
|
+
}, dn = (e, t) => {
|
|
792
845
|
let n = e.tables.find((e) => e.name === t);
|
|
793
846
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
794
847
|
let r = n.columns.find((e) => e.type === "id");
|
|
795
848
|
if (r !== void 0) return { column: r.name };
|
|
796
849
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
797
850
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
798
|
-
},
|
|
851
|
+
}, fn = (e, t, r) => n.tryPromise({
|
|
799
852
|
try: async () => {
|
|
800
|
-
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable:
|
|
853
|
+
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: l, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
801
854
|
d.update(e);
|
|
802
855
|
let r = e;
|
|
803
856
|
for (let e = 0; e < r.length; e++) r[e] === 10 && f++;
|
|
804
857
|
n(null, e);
|
|
805
|
-
} }), m =
|
|
858
|
+
} }), m = l.fromWeb(c.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
806
859
|
try {
|
|
807
|
-
await s(m, p,
|
|
860
|
+
await s(m, p, xt(r), n(h)), await i.rename(h, t);
|
|
808
861
|
} catch (e) {
|
|
809
862
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
810
863
|
}
|
|
@@ -814,26 +867,26 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
814
867
|
};
|
|
815
868
|
},
|
|
816
869
|
catch: (e) => new H({
|
|
817
|
-
reason: `write table failed: ${
|
|
870
|
+
reason: `write table failed: ${K(e)}`,
|
|
818
871
|
path: t
|
|
819
872
|
})
|
|
820
|
-
}),
|
|
873
|
+
}), pn = (e, t) => n.tryPromise({
|
|
821
874
|
try: async () => {
|
|
822
875
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
823
876
|
},
|
|
824
877
|
catch: (t) => new H({
|
|
825
|
-
reason: `append failed: ${
|
|
878
|
+
reason: `append failed: ${K(t)}`,
|
|
826
879
|
path: e
|
|
827
880
|
})
|
|
828
|
-
}),
|
|
881
|
+
}), mn = (e, t) => n.gen(function* () {
|
|
829
882
|
let r = [], i = t.ordered.length;
|
|
830
883
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
831
|
-
let o = t.ordered[a], s =
|
|
832
|
-
if ("refusal" in
|
|
833
|
-
let u =
|
|
834
|
-
if (
|
|
884
|
+
let o = t.ordered[a], s = cn(t.snapshot, o), l = dn(t.snapshot, o);
|
|
885
|
+
if ("refusal" in l) return yield* n.fail(new Qe({ reason: `export: ${l.refusal}` }));
|
|
886
|
+
let u = l.column, d = `${Mt}/${o}.ndjson${bt(t.compression)}`;
|
|
887
|
+
if (Wt(t.ledger, o)) {
|
|
835
888
|
let e = t.ledger.tables[o];
|
|
836
|
-
|
|
889
|
+
W(t.onProgress, {
|
|
837
890
|
phase: "export",
|
|
838
891
|
table: o,
|
|
839
892
|
index: a,
|
|
@@ -851,7 +904,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
851
904
|
});
|
|
852
905
|
continue;
|
|
853
906
|
}
|
|
854
|
-
|
|
907
|
+
W(t.onProgress, {
|
|
855
908
|
phase: "export",
|
|
856
909
|
table: o,
|
|
857
910
|
index: a,
|
|
@@ -865,11 +918,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
865
918
|
chunkSize: t.chunkSize,
|
|
866
919
|
retry: t.retry,
|
|
867
920
|
...f === void 0 ? {} : { where: f }
|
|
868
|
-
}), m =
|
|
921
|
+
}), m = ln(t.snapshot, o), h = m.length === 0 ? p : p.pipe(c.map((e) => un(e, m))), g = t.mask ? h.pipe(c.map((e) => We(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* Vt("export", o, a, i, (e) => e.rowCount, fn(rn(g, s), t.bundleFile(d), t.compression));
|
|
869
922
|
t.ledger.tables[o] = {
|
|
870
923
|
rowCount: _,
|
|
871
924
|
checksum: v
|
|
872
|
-
}, yield*
|
|
925
|
+
}, yield* G(t.ledgerPath, t.ledger), W(t.onProgress, {
|
|
873
926
|
phase: "export",
|
|
874
927
|
table: o,
|
|
875
928
|
index: a,
|
|
@@ -887,36 +940,36 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
887
940
|
});
|
|
888
941
|
}
|
|
889
942
|
return r;
|
|
890
|
-
}),
|
|
943
|
+
}), hn = (e) => ({
|
|
891
944
|
_tag: "RawSqlFragment",
|
|
892
945
|
strings: [e],
|
|
893
946
|
values: []
|
|
894
|
-
}),
|
|
947
|
+
}), gn = (t, i, a) => n.tryPromise({
|
|
895
948
|
try: () => t.transactional(async (t) => {
|
|
896
|
-
if (t.raw) for (let e of
|
|
949
|
+
if (t.raw) for (let e of Te(i)) await t.raw(hn(e));
|
|
897
950
|
let o = await n.runPromiseExit(a(t));
|
|
898
951
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
899
952
|
return o.value;
|
|
900
953
|
}),
|
|
901
|
-
catch: (e) => e instanceof H ? e : new H({ reason: `snapshot export failed: ${
|
|
902
|
-
}),
|
|
903
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
954
|
+
catch: (e) => e instanceof H ? e : new H({ reason: `snapshot export failed: ${K(e)}` })
|
|
955
|
+
}), _n = (e) => n.gen(function* () {
|
|
956
|
+
let t = e.scope ?? { kind: "all" }, r = yt(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, s = o.max([o.exponential("50 millis").pipe(o.jittered), o.recurs(a)]), c = yield* n.promise(() => import("node:path")), l = (t) => c.join(e.outDir, t);
|
|
904
957
|
yield* n.tryPromise({
|
|
905
958
|
try: async () => {
|
|
906
959
|
let t = await import("node:fs/promises");
|
|
907
|
-
await t.mkdir(l(
|
|
960
|
+
await t.mkdir(l(Mt), { recursive: !0 }), e.assets && await t.mkdir(l(Nt), { recursive: !0 });
|
|
908
961
|
},
|
|
909
962
|
catch: (t) => new H({
|
|
910
|
-
reason: `mkdir failed: ${
|
|
963
|
+
reason: `mkdir failed: ${K(t)}`,
|
|
911
964
|
path: e.outDir
|
|
912
965
|
})
|
|
913
966
|
});
|
|
914
|
-
let u = l(
|
|
967
|
+
let u = l(Ft), d = yield* Ut(u), f = e.subset ? ot(yield* at(e.store, e.snapshot, e.subset), e.snapshot) : yield* ct(t, e.snapshot, {
|
|
915
968
|
store: e.store,
|
|
916
969
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
917
|
-
}), p =
|
|
970
|
+
}), p = on(f.tables, e.snapshot), m = e.consistency ?? "live", h, g;
|
|
918
971
|
if (e.masking) {
|
|
919
|
-
let t = e.classification ??
|
|
972
|
+
let t = e.classification ?? De(e.snapshot), r = Ve(p.map((t) => ({
|
|
920
973
|
name: t,
|
|
921
974
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
922
975
|
name: e.name,
|
|
@@ -943,20 +996,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
943
996
|
resolved: f,
|
|
944
997
|
compression: r,
|
|
945
998
|
chunkSize: i,
|
|
946
|
-
retry:
|
|
999
|
+
retry: s,
|
|
947
1000
|
ledger: d,
|
|
948
1001
|
ledgerPath: u,
|
|
949
1002
|
bundleFile: l,
|
|
950
1003
|
...g ? { mask: g } : {},
|
|
951
1004
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
952
|
-
}, v = m === "snapshot" ? yield*
|
|
1005
|
+
}, v = m === "snapshot" ? yield* gn(e.store, e.dialect, (e) => mn(e, _)) : yield* mn(e.store, _), y = 0, b = 0;
|
|
953
1006
|
if (e.assets) {
|
|
954
|
-
let t = l(
|
|
1007
|
+
let t = l(Pt), r = yield* ht({
|
|
955
1008
|
source: e.assets,
|
|
956
|
-
assetsDir: l(
|
|
1009
|
+
assetsDir: l(Nt),
|
|
957
1010
|
progress: {
|
|
958
1011
|
done: (e) => e in d.assets,
|
|
959
|
-
mark: (e, r) => (d.assets[e] = !0,
|
|
1012
|
+
mark: (e, r) => (d.assets[e] = !0, pn(t, JSON.stringify(r)).pipe(n.andThen(G(u, d))))
|
|
960
1013
|
}
|
|
961
1014
|
});
|
|
962
1015
|
y = r.count, b = r.totalBytes, r.missing.length > 0 && (yield* n.logWarning(`${r.missing.length} of ${r.missing.length + r.count} blob reference(s) point at objects the storage provider does not have; they are NOT in this bundle and no import can bring them back. e.g. ${r.missing.slice(0, 3).map((e) => `'${e}'`).join(", ")}` + (r.missing.length > 3 ? `, and ${r.missing.length - 3} more` : "")));
|
|
@@ -966,7 +1019,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
966
1019
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
967
1020
|
source: {
|
|
968
1021
|
dialect: e.dialect,
|
|
969
|
-
schemaFingerprint:
|
|
1022
|
+
schemaFingerprint: ie(e.snapshot)
|
|
970
1023
|
},
|
|
971
1024
|
scope: t,
|
|
972
1025
|
consistency: m,
|
|
@@ -975,7 +1028,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
975
1028
|
assets: y > 0 ? {
|
|
976
1029
|
count: y,
|
|
977
1030
|
totalBytes: b,
|
|
978
|
-
index:
|
|
1031
|
+
index: Pt
|
|
979
1032
|
} : {
|
|
980
1033
|
count: 0,
|
|
981
1034
|
totalBytes: 0
|
|
@@ -984,23 +1037,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
984
1037
|
};
|
|
985
1038
|
return yield* n.tryPromise({
|
|
986
1039
|
try: async () => {
|
|
987
|
-
await (await import("node:fs/promises")).writeFile(l(
|
|
1040
|
+
await (await import("node:fs/promises")).writeFile(l(At), Ot(x));
|
|
988
1041
|
},
|
|
989
1042
|
catch: (e) => new H({
|
|
990
|
-
reason: `manifest write failed: ${
|
|
991
|
-
path:
|
|
1043
|
+
reason: `manifest write failed: ${K(e)}`,
|
|
1044
|
+
path: At
|
|
992
1045
|
})
|
|
993
1046
|
}), yield* n.tryPromise({
|
|
994
1047
|
try: async () => {
|
|
995
1048
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
996
|
-
await e.writeFile(l(
|
|
1049
|
+
await e.writeFile(l(jt), Lt(x, { hostname: t.hostname() }));
|
|
997
1050
|
},
|
|
998
1051
|
catch: (e) => new H({
|
|
999
|
-
reason: `readme write failed: ${
|
|
1000
|
-
path:
|
|
1052
|
+
reason: `readme write failed: ${K(e)}`,
|
|
1053
|
+
path: jt
|
|
1001
1054
|
})
|
|
1002
1055
|
}), x;
|
|
1003
|
-
}),
|
|
1056
|
+
}), vn = (e, t) => {
|
|
1004
1057
|
let n = new Set(t), r = [];
|
|
1005
1058
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
1006
1059
|
let i = e.references?.table;
|
|
@@ -1011,7 +1064,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1011
1064
|
});
|
|
1012
1065
|
}
|
|
1013
1066
|
return r;
|
|
1014
|
-
},
|
|
1067
|
+
}, yn = (e) => {
|
|
1015
1068
|
let t = /* @__PURE__ */ new Map();
|
|
1016
1069
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
1017
1070
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -1019,7 +1072,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1019
1072
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
1020
1073
|
});
|
|
1021
1074
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
1022
|
-
},
|
|
1075
|
+
}, bn = (e) => {
|
|
1023
1076
|
switch (e) {
|
|
1024
1077
|
case "id":
|
|
1025
1078
|
case "text":
|
|
@@ -1038,12 +1091,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1038
1091
|
case "raw": return "raw";
|
|
1039
1092
|
default: return "string";
|
|
1040
1093
|
}
|
|
1041
|
-
},
|
|
1094
|
+
}, xn = {
|
|
1042
1095
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
1043
1096
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
1044
1097
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
1045
|
-
},
|
|
1046
|
-
let i =
|
|
1098
|
+
}, Sn = (e, t, n, r) => {
|
|
1099
|
+
let i = bn(n);
|
|
1047
1100
|
if (i === "raw") return {
|
|
1048
1101
|
table: e,
|
|
1049
1102
|
column: t,
|
|
@@ -1051,7 +1104,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1051
1104
|
severity: "block",
|
|
1052
1105
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
1053
1106
|
};
|
|
1054
|
-
let a =
|
|
1107
|
+
let a = xn[i];
|
|
1055
1108
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
1056
1109
|
table: e,
|
|
1057
1110
|
column: t,
|
|
@@ -1065,15 +1118,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1065
1118
|
severity: "block",
|
|
1066
1119
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
1067
1120
|
};
|
|
1068
|
-
},
|
|
1121
|
+
}, Cn = (e, t, n) => {
|
|
1069
1122
|
if (t === n) return [];
|
|
1070
1123
|
let r = [];
|
|
1071
1124
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
1072
|
-
let a =
|
|
1125
|
+
let a = Sn(t.name, e, i, n);
|
|
1073
1126
|
a && r.push(a);
|
|
1074
1127
|
}
|
|
1075
1128
|
return r;
|
|
1076
|
-
},
|
|
1129
|
+
}, wn = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, Tn = (e) => {
|
|
1077
1130
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
1078
1131
|
switch (e.kind) {
|
|
1079
1132
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -1082,36 +1135,35 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1082
1135
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
1083
1136
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
1084
1137
|
}
|
|
1085
|
-
},
|
|
1138
|
+
}, En = (e) => {
|
|
1086
1139
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
1087
1140
|
return t.length > 0 ? t.join("/") : void 0;
|
|
1088
|
-
},
|
|
1141
|
+
}, Dn = "Failed to execute statement", On = (e) => {
|
|
1089
1142
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
}, An = (e) => {
|
|
1143
|
+
return t === `SqlError: ${Dn}` ? Dn : t;
|
|
1144
|
+
}, kn = (e) => {
|
|
1093
1145
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
1094
1146
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
1095
1147
|
unclassified: t,
|
|
1096
1148
|
raw: n
|
|
1097
|
-
}), i =
|
|
1098
|
-
if (a !== void 0) return r(`${
|
|
1099
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u =
|
|
1100
|
-
return r(d ===
|
|
1101
|
-
},
|
|
1102
|
-
let t =
|
|
1149
|
+
}), i = _(e), a = p(i), o = En(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
1150
|
+
if (a !== void 0) return r(`${Tn(a)}${s}`, !1);
|
|
1151
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = On(typeof l == "string" && l.length > 0 ? l : String(e)), d = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, f = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, m = f !== void 0 && /[←‖]/.test(f) ? ` (via ${f})` : "";
|
|
1152
|
+
return r(d === Dn || d.length === 0 ? `the database refused the write and the driver gave no detail${s}${m}` : `${d}${s}`, d === Dn || d.length === 0);
|
|
1153
|
+
}, An = (e) => kn(e).reason, jn = (e) => {
|
|
1154
|
+
let t = _(e), n = p(t);
|
|
1103
1155
|
if (n?.kind === "foreignKeyInUse") {
|
|
1104
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
1156
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = En(t);
|
|
1105
1157
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
1106
1158
|
}
|
|
1107
|
-
return
|
|
1108
|
-
},
|
|
1159
|
+
return An(e);
|
|
1160
|
+
}, Mn = d, q = (e) => `${Mn}${e}`, Nn = (e) => typeof e.run == "function", Pn = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
1109
1161
|
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
1110
|
-
written before the first delete is what covers that case.`,
|
|
1111
|
-
if (!
|
|
1162
|
+
written before the first delete is what covers that case.`, Fn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, J = (e, t) => {
|
|
1163
|
+
if (!Fn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
1112
1164
|
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
1113
|
-
},
|
|
1114
|
-
let n =
|
|
1165
|
+
}, In = (e, t) => {
|
|
1166
|
+
let n = J(e, t), r = J(q(e), t);
|
|
1115
1167
|
switch (t) {
|
|
1116
1168
|
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
1117
1169
|
case "mysql":
|
|
@@ -1119,20 +1171,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1119
1171
|
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
1120
1172
|
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
1121
1173
|
}
|
|
1174
|
+
}, Ln = (e, t) => {
|
|
1175
|
+
let n = J(q(e), t);
|
|
1176
|
+
return t === "mssql" ? `IF OBJECT_ID('${q(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
1122
1177
|
}, Rn = (e, t) => {
|
|
1123
|
-
let n = Y(J(e), t);
|
|
1124
|
-
return t === "mssql" ? `IF OBJECT_ID('${J(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
1125
|
-
}, zn = (e, t) => {
|
|
1126
1178
|
let n = t === "mysql" || t === "mariadb", r = [];
|
|
1127
1179
|
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
1128
|
-
for (let n of [...e].reverse()) r.push(`DELETE FROM ${
|
|
1180
|
+
for (let n of [...e].reverse()) r.push(`DELETE FROM ${J(n.name, t)}`);
|
|
1129
1181
|
for (let n of e) {
|
|
1130
1182
|
if (n.columns.length === 0) throw Error(`refusing to build a swap for '${n.name}' with no columns: an empty list would have to mean \`SELECT *\`, which is the shape that fails on a generated column.`);
|
|
1131
|
-
let e = n.columns.map((e) =>
|
|
1132
|
-
r.push(`INSERT INTO ${
|
|
1183
|
+
let e = n.columns.map((e) => J(e, t)).join(", ");
|
|
1184
|
+
r.push(`INSERT INTO ${J(n.name, t)} (${e}) SELECT ${e} FROM ${J(q(n.name), t)}`);
|
|
1133
1185
|
}
|
|
1134
1186
|
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
1135
|
-
},
|
|
1187
|
+
}, zn = (e, t) => {
|
|
1136
1188
|
let n = [], r = () => {
|
|
1137
1189
|
for (let e of n) t.unregisterTable(e);
|
|
1138
1190
|
};
|
|
@@ -1140,7 +1192,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1140
1192
|
for (let r of e) {
|
|
1141
1193
|
let e = t.getTable(r);
|
|
1142
1194
|
if (e === void 0) continue;
|
|
1143
|
-
let i =
|
|
1195
|
+
let i = q(r);
|
|
1144
1196
|
t.registerTable({
|
|
1145
1197
|
...e,
|
|
1146
1198
|
tableName: i,
|
|
@@ -1154,7 +1206,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1154
1206
|
staged: n,
|
|
1155
1207
|
undo: r
|
|
1156
1208
|
};
|
|
1157
|
-
},
|
|
1209
|
+
}, Bn = (e, t) => {
|
|
1158
1210
|
let n = new Set(t), r = [];
|
|
1159
1211
|
for (let t of e.tables) {
|
|
1160
1212
|
if (!n.has(t.name)) continue;
|
|
@@ -1171,22 +1223,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1171
1223
|
}
|
|
1172
1224
|
}
|
|
1173
1225
|
return r;
|
|
1174
|
-
},
|
|
1175
|
-
let n =
|
|
1176
|
-
return `SELECT ${
|
|
1177
|
-
},
|
|
1226
|
+
}, Vn = (e, t) => {
|
|
1227
|
+
let n = J(q(e.from), t), r = J(q(e.to), t), i = J(e.column, t), a = J(e.toColumn, t);
|
|
1228
|
+
return `SELECT ${J(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
1229
|
+
}, Hn = (e) => {
|
|
1178
1230
|
if (e.length === 0) return;
|
|
1179
1231
|
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
1180
1232
|
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
1181
|
-
},
|
|
1182
|
-
t.length !== 0 && await e.run(
|
|
1233
|
+
}, Un = async (e, t) => {
|
|
1234
|
+
t.length !== 0 && await e.run(ve.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
1235
|
+
}, Wn = async (e, t, r) => {
|
|
1236
|
+
let i = Rn(t, r);
|
|
1237
|
+
i.length !== 0 && await e.run(ve.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
1183
1238
|
}, Gn = async (e, t, r) => {
|
|
1184
|
-
let i = zn(t, r);
|
|
1185
|
-
i.length !== 0 && await e.run(N.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
1186
|
-
}, Kn = async (e, t, r) => {
|
|
1187
1239
|
let i = [];
|
|
1188
1240
|
for (let a of t) {
|
|
1189
|
-
let t = await e.run(
|
|
1241
|
+
let t = await e.run(ve.SqlClient.pipe(n.flatMap((e) => e.unsafe(Vn(a, r)))));
|
|
1190
1242
|
for (let e of t) i.push({
|
|
1191
1243
|
table: a.from,
|
|
1192
1244
|
primaryKey: String(e.pk ?? ""),
|
|
@@ -1196,14 +1248,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1196
1248
|
});
|
|
1197
1249
|
}
|
|
1198
1250
|
return i;
|
|
1199
|
-
},
|
|
1251
|
+
}, Kn = (e, t) => {
|
|
1200
1252
|
let n = new Map(t.map((e, t) => [e, t]));
|
|
1201
1253
|
return e.filter((e) => {
|
|
1202
1254
|
if (e.from === e.to) return !1;
|
|
1203
1255
|
let t = n.get(e.from), r = n.get(e.to);
|
|
1204
1256
|
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1205
1257
|
});
|
|
1206
|
-
},
|
|
1258
|
+
}, qn = (e) => e === "mysql" || e === "mariadb", Jn = (e) => {
|
|
1207
1259
|
if (e.mode !== "replace") return {
|
|
1208
1260
|
active: !1,
|
|
1209
1261
|
tables: [],
|
|
@@ -1253,7 +1305,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1253
1305
|
tables: [],
|
|
1254
1306
|
blocker: "every table in the bundle is one that describes a deployment, so there is nothing for a staged swap to move."
|
|
1255
1307
|
};
|
|
1256
|
-
let s = o.map((e) => e.name), c =
|
|
1308
|
+
let s = o.map((e) => e.name), c = qn(t) ? [] : Kn(Bn(n, s), s);
|
|
1257
1309
|
return c.length > 0 ? {
|
|
1258
1310
|
active: !1,
|
|
1259
1311
|
tables: [],
|
|
@@ -1262,7 +1314,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1262
1314
|
active: !0,
|
|
1263
1315
|
tables: o
|
|
1264
1316
|
};
|
|
1265
|
-
},
|
|
1317
|
+
}, Yn = (e) => n.gen(function* () {
|
|
1266
1318
|
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1267
1319
|
if (e.snapshot === void 0) return yield* n.fail(new H({
|
|
1268
1320
|
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
@@ -1275,7 +1327,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1275
1327
|
if (t.tables.length === 0) return;
|
|
1276
1328
|
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1277
1329
|
so it survives a run whose process does not.`);
|
|
1278
|
-
let r = yield*
|
|
1330
|
+
let r = yield* _n({
|
|
1279
1331
|
store: e.store,
|
|
1280
1332
|
snapshot: t,
|
|
1281
1333
|
dialect: e.dialect ?? "unknown",
|
|
@@ -1291,41 +1343,46 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1291
1343
|
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1292
1344
|
tables: r.tables.length
|
|
1293
1345
|
};
|
|
1294
|
-
}),
|
|
1295
|
-
let n = [];
|
|
1296
|
-
for (let
|
|
1297
|
-
|
|
1298
|
-
|
|
1346
|
+
}), Xn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, Zn = /__dropped_\d+$/, Qn = (e, t) => {
|
|
1347
|
+
let n = [], r = /* @__PURE__ */ new Map();
|
|
1348
|
+
for (let e of t.tables) {
|
|
1349
|
+
let t = e.name;
|
|
1350
|
+
r.has(t) || r.set(t, e);
|
|
1351
|
+
}
|
|
1352
|
+
for (let t of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
1353
|
+
let e = r.get(t.name);
|
|
1354
|
+
if (e === void 0) {
|
|
1355
|
+
n.push(`table '${t.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
1299
1356
|
continue;
|
|
1300
1357
|
}
|
|
1301
|
-
let
|
|
1302
|
-
for (let [
|
|
1303
|
-
let
|
|
1304
|
-
if (
|
|
1305
|
-
n.push(
|
|
1358
|
+
let i = new Map(e.columns.map((e) => [e.name, e])), a = new Map(t.columns.map((e) => [e.name, e]));
|
|
1359
|
+
for (let [e, r] of a) {
|
|
1360
|
+
let a = i.get(e);
|
|
1361
|
+
if (a === void 0) {
|
|
1362
|
+
n.push(Zn.test(e) ? `${t.name}.${e}: a soft-dropped column, present only in the SOURCE — reclaim it there (voltro db gc-snapshots --before <date>) or re-export without it; the target is not missing anything` : `${t.name}.${e}: in the bundle, MISSING from the target — the value has nowhere to go`);
|
|
1306
1363
|
continue;
|
|
1307
1364
|
}
|
|
1308
|
-
|
|
1365
|
+
a.type !== r.type && n.push(`${t.name}.${e}: type '${r.type}' (bundle) vs '${a.type}' (target)`);
|
|
1309
1366
|
}
|
|
1310
|
-
for (let [
|
|
1367
|
+
for (let [e, r] of i) a.has(e) || !r.nullable && !r.hasDefault && n.push(`${t.name}.${e}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
1311
1368
|
}
|
|
1312
1369
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
1313
|
-
},
|
|
1314
|
-
let r =
|
|
1370
|
+
}, $n = (e, t, n) => {
|
|
1371
|
+
let r = ie(t);
|
|
1315
1372
|
if (r === e || n === void 0) return {
|
|
1316
1373
|
drifted: !1,
|
|
1317
1374
|
bundleFingerprint: e,
|
|
1318
1375
|
targetFingerprint: r,
|
|
1319
1376
|
diff: []
|
|
1320
1377
|
};
|
|
1321
|
-
let i =
|
|
1378
|
+
let i = Qn(n, t);
|
|
1322
1379
|
return {
|
|
1323
1380
|
drifted: i.length > 0,
|
|
1324
1381
|
bundleFingerprint: e,
|
|
1325
1382
|
targetFingerprint: r,
|
|
1326
1383
|
diff: i
|
|
1327
1384
|
};
|
|
1328
|
-
},
|
|
1385
|
+
}, er = 2e3, tr = 2e3, nr = 200, rr = /* @__PURE__ */ new Set([
|
|
1329
1386
|
"BundleError",
|
|
1330
1387
|
"CodecError",
|
|
1331
1388
|
"IntegrityError",
|
|
@@ -1334,29 +1391,29 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1334
1391
|
"RowsRefusedError",
|
|
1335
1392
|
"SchemaDriftError",
|
|
1336
1393
|
"StorageError"
|
|
1337
|
-
]),
|
|
1338
|
-
let n =
|
|
1339
|
-
for (let
|
|
1340
|
-
let
|
|
1341
|
-
if (typeof
|
|
1342
|
-
if (
|
|
1343
|
-
let
|
|
1344
|
-
if (
|
|
1394
|
+
]), ir = (t) => {
|
|
1395
|
+
let n = e.isCause(t) ? t : r.isExit(t) && r.isFailure(t) ? t.cause : void 0, i = n === void 0 ? [t] : n.reasons.flatMap((t) => e.isFailReason(t) ? [t.error] : e.isDieReason(t) ? [t.defect] : []);
|
|
1396
|
+
for (let n of i) {
|
|
1397
|
+
let i = n?._tag;
|
|
1398
|
+
if (typeof i == "string" && rr.has(i)) return n;
|
|
1399
|
+
if (n !== t && (e.isCause(n) || r.isExit(n))) {
|
|
1400
|
+
let e = ir(n);
|
|
1401
|
+
if (e) return e;
|
|
1345
1402
|
}
|
|
1346
1403
|
}
|
|
1347
|
-
},
|
|
1404
|
+
}, ar = (t) => a.getOrUndefined(e.findErrorOption(t)) ?? Error(e.pretty(t)), or = (e) => n.tryPromise({
|
|
1348
1405
|
try: async () => {
|
|
1349
1406
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
1350
|
-
return
|
|
1407
|
+
return kt(await t.readFile(n.join(e, At), "utf8"));
|
|
1351
1408
|
},
|
|
1352
1409
|
catch: (e) => new H({
|
|
1353
1410
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
1354
|
-
path:
|
|
1411
|
+
path: At
|
|
1355
1412
|
})
|
|
1356
|
-
}),
|
|
1413
|
+
}), sr = (e) => {
|
|
1357
1414
|
let t;
|
|
1358
1415
|
return n.gen(function* () {
|
|
1359
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
1416
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* or(e.bundleDir);
|
|
1360
1417
|
if (a.formatVersion !== 1) return yield* n.fail(new H({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
1361
1418
|
if (e.tables !== void 0) {
|
|
1362
1419
|
let t = new Set(a.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
@@ -1377,7 +1434,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1377
1434
|
unique: !1,
|
|
1378
1435
|
hasDefault: !1
|
|
1379
1436
|
}))
|
|
1380
|
-
})) }, r =
|
|
1437
|
+
})) }, r = $n(s.source.schemaFingerprint, e.targetSnapshot, t);
|
|
1381
1438
|
if (r.drifted) {
|
|
1382
1439
|
if (!e.force) return yield* n.fail(new et({
|
|
1383
1440
|
bundleFingerprint: r.bundleFingerprint,
|
|
@@ -1388,28 +1445,28 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1388
1445
|
}
|
|
1389
1446
|
}
|
|
1390
1447
|
if (e.targetDialect && e.targetDialect !== s.source.dialect) {
|
|
1391
|
-
let t =
|
|
1448
|
+
let t = Cn(s.tables, s.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1392
1449
|
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new rt({
|
|
1393
1450
|
sourceDialect: s.source.dialect,
|
|
1394
1451
|
targetDialect: e.targetDialect,
|
|
1395
|
-
blocking: t.map(
|
|
1452
|
+
blocking: t.map(wn)
|
|
1396
1453
|
}));
|
|
1397
1454
|
}
|
|
1398
|
-
let
|
|
1399
|
-
if (
|
|
1400
|
-
mode:
|
|
1455
|
+
let d = e.verify ?? !0, p = e.mode ?? "upsert", g = e.atomic ?? p === "replace", _ = e.onConflict ?? "skip", y = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), x = yield* Ut(y);
|
|
1456
|
+
if (p === "replace" && s.scope.kind !== "all") return yield* n.fail(new nt({
|
|
1457
|
+
mode: p,
|
|
1401
1458
|
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope. To leave specific tables out of a FULL export, use \`--exclude a,b\`: it keeps the scope \`all\` and records what it left out.`
|
|
1402
1459
|
}));
|
|
1403
|
-
if (
|
|
1460
|
+
if (p === "replace" && (s.scope.exclude?.length ?? 0) > 0) {
|
|
1404
1461
|
let t = s.scope.exclude;
|
|
1405
1462
|
e.onWarn?.(`this bundle was exported with ${t.length} table(s) excluded: ${t.join(", ")}.\n A replace empties and refills the tables the bundle CARRIES, so those keep whatever rows the
|
|
1406
1463
|
target already holds. That is not "everything the bundle says" — check it is what you meant.`);
|
|
1407
1464
|
}
|
|
1408
|
-
if (
|
|
1465
|
+
if (p === "replace") {
|
|
1409
1466
|
let t = s.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1410
1467
|
try: () => e.store.incomingForeignKeys(t),
|
|
1411
|
-
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${
|
|
1412
|
-
}) : e.targetSnapshot === void 0 ? [] :
|
|
1468
|
+
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${jn(e)}` })
|
|
1469
|
+
}) : e.targetSnapshot === void 0 ? [] : vn(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1413
1470
|
for (let t of r) {
|
|
1414
1471
|
if (!a.has(t.from)) {
|
|
1415
1472
|
let r = yield* n.tryPromise({
|
|
@@ -1432,11 +1489,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1432
1489
|
});
|
|
1433
1490
|
}
|
|
1434
1491
|
if (i.length > 0) return yield* n.fail(new nt({
|
|
1435
|
-
mode:
|
|
1436
|
-
reason:
|
|
1492
|
+
mode: p,
|
|
1493
|
+
reason: yn(i)
|
|
1437
1494
|
}));
|
|
1438
1495
|
}
|
|
1439
|
-
if (e.dryRun === !0) return e.traceId !== void 0 && (yield* n.promise(() => e.store.update(
|
|
1496
|
+
if (e.dryRun === !0) return e.traceId !== void 0 && (yield* n.promise(() => e.store.update(l, e.traceId, {
|
|
1440
1497
|
finishedAt: /* @__PURE__ */ new Date(),
|
|
1441
1498
|
tablesWritten: 0,
|
|
1442
1499
|
rowsWritten: 0,
|
|
@@ -1449,34 +1506,34 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1449
1506
|
rowsSkipped: 0,
|
|
1450
1507
|
fullyResumed: !1
|
|
1451
1508
|
};
|
|
1452
|
-
let
|
|
1509
|
+
let C = /* @__PURE__ */ new Map();
|
|
1453
1510
|
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1454
1511
|
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1455
|
-
e.length > 0 &&
|
|
1512
|
+
e.length > 0 && C.set(t.name, e);
|
|
1456
1513
|
}
|
|
1457
|
-
let
|
|
1458
|
-
let n =
|
|
1514
|
+
let ie = (e, t) => {
|
|
1515
|
+
let n = C.get(e);
|
|
1459
1516
|
if (n === void 0) return t;
|
|
1460
1517
|
let r;
|
|
1461
1518
|
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1462
1519
|
return r ?? t;
|
|
1463
|
-
},
|
|
1464
|
-
let i =
|
|
1465
|
-
return
|
|
1466
|
-
},
|
|
1467
|
-
if (!
|
|
1520
|
+
}, ae = (e, t, n, r = !1) => {
|
|
1521
|
+
let i = A ? q(t.name) : t.name, a = t.primaryKey, o = w(t.name, ie(t.name, n), t.columnTypes);
|
|
1522
|
+
return p === "append" ? _ === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : p === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1523
|
+
}, oe = e.targetDialect !== void 0 && e.targetDialect !== "memory", w = (e, t, n) => {
|
|
1524
|
+
if (!oe || n === void 0 || v(e) !== void 0) return t;
|
|
1468
1525
|
let r = null;
|
|
1469
1526
|
for (let [e, i] of Object.entries(n)) {
|
|
1470
1527
|
let n = t[e];
|
|
1471
1528
|
i === "json" && n != null && (r === null && (r = { ...t }), r[e] = JSON.stringify(n));
|
|
1472
1529
|
}
|
|
1473
1530
|
return r ?? t;
|
|
1474
|
-
}, T = (e, t) => e.savepoint ? e.savepoint(t) : t(),
|
|
1475
|
-
r.unclassified && (
|
|
1476
|
-
},
|
|
1531
|
+
}, T = (e, t) => e.savepoint ? e.savepoint(t) : t(), E = 0, se = (t, n, r) => {
|
|
1532
|
+
r.unclassified && (E++, !(E > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (E === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1533
|
+
}, ce = (e, t, r = !0) => n.tryPromise({
|
|
1477
1534
|
try: e,
|
|
1478
|
-
catch: (e) =>
|
|
1479
|
-
}).pipe(n.map(() => !0), n.
|
|
1535
|
+
catch: (e) => kn(e)
|
|
1536
|
+
}).pipe(n.map(() => !0), n.catch((e) => n.sync(() => (r && (t.reason = e.reason, se(t.entry, t.row, e)), !1)))), le = (e) => {
|
|
1480
1537
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1481
1538
|
if (t.length !== 0) return {
|
|
1482
1539
|
...e.row,
|
|
@@ -1510,36 +1567,36 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1510
1567
|
primary: t.primary
|
|
1511
1568
|
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1512
1569
|
});
|
|
1513
|
-
},
|
|
1570
|
+
}, fe = (e, t) => n.gen(function* () {
|
|
1514
1571
|
let r = t, i = [], a = !0;
|
|
1515
1572
|
for (; r.length > 0 && a;) {
|
|
1516
1573
|
a = !1;
|
|
1517
1574
|
let t = [];
|
|
1518
|
-
for (let n of r) (yield*
|
|
1575
|
+
for (let n of r) (yield* ce(() => T(e, () => ae(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1519
1576
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1520
|
-
let r = t[n], o =
|
|
1521
|
-
o !== void 0 && (yield*
|
|
1577
|
+
let r = t[n], o = le(r);
|
|
1578
|
+
o !== void 0 && (yield* ce(() => T(e, () => ae(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1522
1579
|
}
|
|
1523
1580
|
r = t;
|
|
1524
1581
|
}
|
|
1525
|
-
for (let t of i) (yield*
|
|
1582
|
+
for (let t of i) (yield* ce(() => T(e, () => e.upsert(t.entry.name, w(t.entry.name, t.row, t.entry.columnTypes), { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1526
1583
|
if (r.length > 0) return yield* n.fail(de(r));
|
|
1527
|
-
}),
|
|
1584
|
+
}), D = (() => {
|
|
1528
1585
|
let t = e.targetDialect;
|
|
1529
1586
|
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1530
|
-
})(),
|
|
1531
|
-
mode:
|
|
1587
|
+
})(), O = Jn({
|
|
1588
|
+
mode: p,
|
|
1532
1589
|
...e.targetDialect === void 0 ? {} : { targetDialect: e.targetDialect },
|
|
1533
1590
|
...e.targetSnapshot === void 0 ? {} : { targetSnapshot: e.targetSnapshot },
|
|
1534
1591
|
bundleTables: s.tables.map((e) => e.name),
|
|
1535
1592
|
isSkipped: V,
|
|
1536
|
-
canRunSql:
|
|
1537
|
-
}),
|
|
1538
|
-
if (
|
|
1593
|
+
canRunSql: Nn(e.store)
|
|
1594
|
+
}), k = O.tables, pe = s.tables.filter((e) => !V(e.name) && S(e.name)).map((e) => e.name), A = O.active, he = !1, j = 0, M = 0, ge, _e, ve = 0, N = 0, ye = (t, r) => n.gen(function* () {
|
|
1595
|
+
if (p === "replace" && !A && !(r && x.truncated)) {
|
|
1539
1596
|
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1540
|
-
try: () => t.insert(
|
|
1541
|
-
id:
|
|
1542
|
-
startedAt:
|
|
1597
|
+
try: () => t.insert(u, {
|
|
1598
|
+
id: m("repl", be),
|
|
1599
|
+
startedAt: P,
|
|
1543
1600
|
tables: String(s.tables.length),
|
|
1544
1601
|
capture: F?.dir ?? null,
|
|
1545
1602
|
via: e.via ?? "direct"
|
|
@@ -1548,66 +1605,66 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1548
1605
|
not proceed without it.` })
|
|
1549
1606
|
}));
|
|
1550
1607
|
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !V(e)), a = !r, o = t.emptyTables ? () => t.emptyTables(i) : a ? async () => {
|
|
1551
|
-
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where:
|
|
1608
|
+
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where: b(e.primaryKey) });
|
|
1552
1609
|
} : () => t.transactional(async (e) => {
|
|
1553
|
-
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where:
|
|
1610
|
+
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: b(t.primaryKey) });
|
|
1554
1611
|
});
|
|
1555
1612
|
yield* n.tryPromise({
|
|
1556
1613
|
try: o,
|
|
1557
|
-
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${
|
|
1558
|
-
}), r && (
|
|
1614
|
+
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${jn(e)}` })
|
|
1615
|
+
}), r && (x.truncated = !0, yield* G(y, x));
|
|
1559
1616
|
}
|
|
1560
|
-
let a = [], o = [],
|
|
1561
|
-
for (let
|
|
1562
|
-
let
|
|
1563
|
-
if (V(
|
|
1564
|
-
|
|
1617
|
+
let a = [], o = [], l = [], f = [], h = s.tables.length;
|
|
1618
|
+
for (let u = 0; u < s.tables.length; u++) {
|
|
1619
|
+
let m = s.tables[u];
|
|
1620
|
+
if (V(m.name)) {
|
|
1621
|
+
f.push(m.name);
|
|
1565
1622
|
continue;
|
|
1566
1623
|
}
|
|
1567
|
-
if (r &&
|
|
1568
|
-
|
|
1624
|
+
if (r && m.name in x.tables) {
|
|
1625
|
+
l.push(m.name), N++, ve += x.tables[m.name].rowCount, W(e.onProgress, {
|
|
1569
1626
|
phase: "import",
|
|
1570
|
-
table:
|
|
1571
|
-
index:
|
|
1572
|
-
tableCount:
|
|
1627
|
+
table: m.name,
|
|
1628
|
+
index: u,
|
|
1629
|
+
tableCount: h,
|
|
1573
1630
|
event: "done",
|
|
1574
|
-
rowsDone:
|
|
1575
|
-
total:
|
|
1631
|
+
rowsDone: x.tables[m.name].rowCount,
|
|
1632
|
+
total: m.rowCount
|
|
1576
1633
|
});
|
|
1577
1634
|
continue;
|
|
1578
1635
|
}
|
|
1579
|
-
let _ = i.join(e.bundleDir,
|
|
1580
|
-
|
|
1636
|
+
let _ = i.join(e.bundleDir, m.file);
|
|
1637
|
+
W(e.onProgress, {
|
|
1581
1638
|
phase: "import",
|
|
1582
|
-
table:
|
|
1583
|
-
index:
|
|
1584
|
-
tableCount:
|
|
1639
|
+
table: m.name,
|
|
1640
|
+
index: u,
|
|
1641
|
+
tableCount: h,
|
|
1585
1642
|
event: "start",
|
|
1586
1643
|
rowsDone: 0,
|
|
1587
|
-
total:
|
|
1644
|
+
total: m.rowCount
|
|
1588
1645
|
});
|
|
1589
|
-
let
|
|
1590
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0,
|
|
1646
|
+
let v = a.length, b = n.gen(function* () {
|
|
1647
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, l = me("bundle", () => ue(_).pipe(St(s.compression))).pipe(c.tap((e) => n.sync(() => i.update(e)))), u = async (e) => {
|
|
1591
1648
|
let n = {
|
|
1592
|
-
entry:
|
|
1649
|
+
entry: m,
|
|
1593
1650
|
row: e,
|
|
1594
1651
|
reason: ""
|
|
1595
1652
|
};
|
|
1596
1653
|
try {
|
|
1597
|
-
await T(t, () =>
|
|
1654
|
+
await T(t, () => ae(t, m, e));
|
|
1598
1655
|
} catch (t) {
|
|
1599
|
-
let r =
|
|
1600
|
-
n.reason = r.reason,
|
|
1656
|
+
let r = kn(t);
|
|
1657
|
+
n.reason = r.reason, se(m, e, r), a.push(n);
|
|
1601
1658
|
}
|
|
1602
1659
|
}, d = yield* n.promise(async () => {
|
|
1603
1660
|
let n = e.copyLoader;
|
|
1604
1661
|
if (!n || n.dialect !== "postgres") return !1;
|
|
1605
|
-
if (
|
|
1606
|
-
if (
|
|
1607
|
-
if (
|
|
1662
|
+
if (g) return he || (he = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1663
|
+
if (p === "replace") return !0;
|
|
1664
|
+
if (p !== "upsert") return !1;
|
|
1608
1665
|
try {
|
|
1609
1666
|
return (await t.query({
|
|
1610
|
-
table:
|
|
1667
|
+
table: m.name,
|
|
1611
1668
|
predicate: void 0,
|
|
1612
1669
|
order: [],
|
|
1613
1670
|
take: 1,
|
|
@@ -1617,19 +1674,19 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1617
1674
|
} catch {
|
|
1618
1675
|
return !1;
|
|
1619
1676
|
}
|
|
1620
|
-
}),
|
|
1677
|
+
}), f = Object.keys(m.columnTypes), h = Math.max(1, e.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
1621
1678
|
if (v.length === 0) return;
|
|
1622
1679
|
let t = v;
|
|
1623
1680
|
v = [];
|
|
1624
1681
|
try {
|
|
1625
1682
|
await e.copyLoader.copyInto({
|
|
1626
|
-
table:
|
|
1627
|
-
columns:
|
|
1628
|
-
columnTypes:
|
|
1683
|
+
table: m.name,
|
|
1684
|
+
columns: f,
|
|
1685
|
+
columnTypes: m.columnTypes,
|
|
1629
1686
|
rows: t
|
|
1630
1687
|
});
|
|
1631
1688
|
} catch (n) {
|
|
1632
|
-
d = !1, e.onWarn?.(`bulk COPY into ${
|
|
1689
|
+
d = !1, e.onWarn?.(`bulk COPY into ${m.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1633
1690
|
for (let e of t) await u(e);
|
|
1634
1691
|
}
|
|
1635
1692
|
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof t.savepoint == "function", S = [], ee = async () => {
|
|
@@ -1638,15 +1695,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1638
1695
|
S = [];
|
|
1639
1696
|
try {
|
|
1640
1697
|
await t.savepoint(async () => {
|
|
1641
|
-
for (let n of e) await
|
|
1698
|
+
for (let n of e) await ae(t, m, n);
|
|
1642
1699
|
});
|
|
1643
1700
|
} catch {
|
|
1644
1701
|
for (let t of e) await u(t);
|
|
1645
1702
|
}
|
|
1646
1703
|
};
|
|
1647
|
-
return yield*
|
|
1648
|
-
if (o++,
|
|
1649
|
-
v.push(e), v.length >=
|
|
1704
|
+
return yield* c.runForEach(an(l, m.name, m.columnTypes), (e) => n.promise(async () => {
|
|
1705
|
+
if (o++, _e?.(), d) {
|
|
1706
|
+
v.push(e), v.length >= h && await y();
|
|
1650
1707
|
return;
|
|
1651
1708
|
}
|
|
1652
1709
|
if (x) {
|
|
@@ -1658,60 +1715,60 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1658
1715
|
rowCount: o,
|
|
1659
1716
|
checksum: i.digest("hex")
|
|
1660
1717
|
};
|
|
1661
|
-
}), { rowCount: S, checksum: ee } = yield*
|
|
1662
|
-
if (
|
|
1663
|
-
if (ee !==
|
|
1664
|
-
what: `table ${
|
|
1665
|
-
expected:
|
|
1718
|
+
}), { rowCount: S, checksum: ee } = yield* Vt("import", m.name, u, h, (e) => e.rowCount, b);
|
|
1719
|
+
if (d) {
|
|
1720
|
+
if (ee !== m.checksum) return yield* n.fail(new Xe({
|
|
1721
|
+
what: `table ${m.name} checksum`,
|
|
1722
|
+
expected: m.checksum,
|
|
1666
1723
|
actual: ee
|
|
1667
1724
|
}));
|
|
1668
|
-
if (S !==
|
|
1669
|
-
what: `table ${
|
|
1670
|
-
expected: String(
|
|
1725
|
+
if (S !== m.rowCount) return yield* n.fail(new Xe({
|
|
1726
|
+
what: `table ${m.name} rowCount`,
|
|
1727
|
+
expected: String(m.rowCount),
|
|
1671
1728
|
actual: String(S)
|
|
1672
1729
|
}));
|
|
1673
1730
|
}
|
|
1674
|
-
|
|
1731
|
+
W(e.onProgress, {
|
|
1675
1732
|
phase: "import",
|
|
1676
|
-
table:
|
|
1677
|
-
index:
|
|
1678
|
-
tableCount:
|
|
1733
|
+
table: m.name,
|
|
1734
|
+
index: u,
|
|
1735
|
+
tableCount: h,
|
|
1679
1736
|
event: "done",
|
|
1680
1737
|
rowsDone: S,
|
|
1681
|
-
total:
|
|
1682
|
-
}),
|
|
1683
|
-
name:
|
|
1738
|
+
total: m.rowCount
|
|
1739
|
+
}), M++, j += S, ge?.(), _e?.(), r && (a.length > v ? o.push({
|
|
1740
|
+
name: m.name,
|
|
1684
1741
|
rowCount: S
|
|
1685
|
-
}) : (
|
|
1742
|
+
}) : (x.tables[m.name] = {
|
|
1686
1743
|
rowCount: S,
|
|
1687
|
-
checksum:
|
|
1688
|
-
}, yield*
|
|
1744
|
+
checksum: m.checksum
|
|
1745
|
+
}, yield* G(y, x)));
|
|
1689
1746
|
}
|
|
1690
|
-
if (a.length > 0 && (yield*
|
|
1747
|
+
if (a.length > 0 && (yield* fe(t, a)), p === "replace" && (yield* n.promise(() => t.deleteMany(u, { where: {
|
|
1691
1748
|
column: "startedAt",
|
|
1692
1749
|
op: "lte",
|
|
1693
|
-
value:
|
|
1750
|
+
value: P
|
|
1694
1751
|
} }).then(() => void 0, () => void 0))), r && o.length > 0) {
|
|
1695
1752
|
for (let e of o) {
|
|
1696
1753
|
let t = s.tables.find((t) => t.name === e.name);
|
|
1697
|
-
|
|
1754
|
+
x.tables[e.name] = {
|
|
1698
1755
|
rowCount: e.rowCount,
|
|
1699
1756
|
checksum: t.checksum
|
|
1700
1757
|
};
|
|
1701
1758
|
}
|
|
1702
|
-
yield*
|
|
1759
|
+
yield* G(y, x);
|
|
1703
1760
|
}
|
|
1704
1761
|
{
|
|
1705
|
-
let t =
|
|
1762
|
+
let t = Je(f, "skipped");
|
|
1706
1763
|
t !== void 0 && e.onWarn?.(t);
|
|
1707
1764
|
}
|
|
1708
|
-
if (
|
|
1709
|
-
let t =
|
|
1710
|
-
e.onWarn?.(`resume: ${
|
|
1765
|
+
if (l.length > 0) {
|
|
1766
|
+
let t = l.length === s.tables.length;
|
|
1767
|
+
e.onWarn?.(`resume: ${l.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${l.join(", ")}. Delete ${y} to force a full re-import.`);
|
|
1711
1768
|
}
|
|
1712
|
-
}),
|
|
1769
|
+
}), be = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, P = /* @__PURE__ */ new Date(), F = yield* Yn({
|
|
1713
1770
|
store: e.store,
|
|
1714
|
-
mode:
|
|
1771
|
+
mode: p,
|
|
1715
1772
|
dir: e.rollbackDir,
|
|
1716
1773
|
snapshot: e.targetSnapshot,
|
|
1717
1774
|
dialect: e.targetDialect,
|
|
@@ -1724,60 +1781,60 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1724
1781
|
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1725
1782
|
path: F.dir
|
|
1726
1783
|
})
|
|
1727
|
-
})), e.onWarn?.(
|
|
1728
|
-
let xe = (e) =>
|
|
1729
|
-
Ce &&
|
|
1730
|
-
id:
|
|
1731
|
-
mode:
|
|
1784
|
+
})), e.onWarn?.(Xn(F)));
|
|
1785
|
+
let xe = (e) => p === "replace" ? te(e) : e(), I = (e) => p === "replace" ? ne(e) : e, L = e.traceId ?? m("dimp", `${Date.now()}-${s.source.schemaFingerprint ?? ""}`), Se = e.traceId !== void 0, Ce = (e.targetSnapshot?.tables.some((e) => e.name === l) ?? !1) || v(l) !== void 0;
|
|
1786
|
+
Ce && h(f), Ce && e.traceId === void 0 && (yield* n.promise(() => e.store.insert(l, {
|
|
1787
|
+
id: L,
|
|
1788
|
+
mode: p,
|
|
1732
1789
|
via: e.via ?? "direct",
|
|
1733
1790
|
direction: "import",
|
|
1734
1791
|
bundle: e.bundleDir,
|
|
1735
1792
|
sourceFingerprint: s.source.schemaFingerprint ?? null,
|
|
1736
1793
|
tablesWritten: 0,
|
|
1737
1794
|
rowsWritten: 0,
|
|
1738
|
-
staged:
|
|
1795
|
+
staged: p === "replace" ? A : null
|
|
1739
1796
|
}).then(() => {
|
|
1740
|
-
|
|
1797
|
+
Se = !0;
|
|
1741
1798
|
}, (t) => {
|
|
1742
1799
|
e.onWarn?.(`the import trace could not be written (${String(t?.message ?? t)}).\n The import continues — this is a record of what happened, not a safety check. Run
|
|
1743
1800
|
\`voltro db apply\` so the next one is recorded.`);
|
|
1744
|
-
}))), e.traceId !== void 0 && Ce && (yield* n.promise(() => e.store.update(
|
|
1745
|
-
staged:
|
|
1801
|
+
}))), e.traceId !== void 0 && Ce && (yield* n.promise(() => e.store.update(l, L, {
|
|
1802
|
+
staged: p === "replace" ? A : null,
|
|
1746
1803
|
sourceFingerprint: s.source.schemaFingerprint ?? null
|
|
1747
1804
|
}).then(() => void 0, () => void 0)));
|
|
1748
1805
|
let we = 0, Te = !1;
|
|
1749
|
-
|
|
1750
|
-
if (!
|
|
1806
|
+
ge = () => {
|
|
1807
|
+
if (!Se || Te) return;
|
|
1751
1808
|
let t = Date.now();
|
|
1752
|
-
t - we <
|
|
1753
|
-
tablesWritten:
|
|
1754
|
-
rowsWritten:
|
|
1809
|
+
t - we < er || (we = t, e.store.update(l, L, {
|
|
1810
|
+
tablesWritten: M,
|
|
1811
|
+
rowsWritten: j
|
|
1755
1812
|
}).then(() => void 0, () => {
|
|
1756
1813
|
Te = !0;
|
|
1757
1814
|
}));
|
|
1758
1815
|
};
|
|
1759
|
-
let Ee = (t) =>
|
|
1816
|
+
let Ee = (t) => Se ? e.store.update(l, L, {
|
|
1760
1817
|
finishedAt: t === null ? /* @__PURE__ */ new Date() : null,
|
|
1761
|
-
tablesWritten:
|
|
1762
|
-
rowsWritten:
|
|
1818
|
+
tablesWritten: M,
|
|
1819
|
+
rowsWritten: j,
|
|
1763
1820
|
failure: t
|
|
1764
1821
|
}).then(() => void 0, () => void 0) : Promise.resolve();
|
|
1765
|
-
if (t = (e) => Ee(e),
|
|
1822
|
+
if (t = (e) => Ee(e), p === "replace" && !A && O.blocker !== void 0 && e.onWarn?.(Pn(O.blocker)), p === "replace" && pe.length > 0 && e.onWarn?.(`write recorders are SUSPENDED for this replace (${pe.length} table(s) have one).\n A replace sets a state rather than changing rows, so a per-row history entry would describe
|
|
1766
1823
|
something that did not happen — and this path already writes through the raw store, without
|
|
1767
|
-
tenant scoping or \`audit()\` stamping, for the same reason.`),
|
|
1768
|
-
e.onWarn?.(`staging ${
|
|
1769
|
-
let t = e.store, r =
|
|
1824
|
+
tenant scoping or \`audit()\` stamping, for the same reason.`), A) {
|
|
1825
|
+
e.onWarn?.(`staging ${k.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1826
|
+
let t = e.store, r = k.map((e) => e.name), i = m("repl", `staging-${be}`), a = !1;
|
|
1770
1827
|
if (e.recordInterruption === !0) {
|
|
1771
|
-
yield* n.promise(() => e.store.insert(
|
|
1828
|
+
yield* n.promise(() => e.store.insert(u, {
|
|
1772
1829
|
id: i,
|
|
1773
|
-
startedAt:
|
|
1830
|
+
startedAt: P,
|
|
1774
1831
|
kind: "staging",
|
|
1775
|
-
tables: String(
|
|
1832
|
+
tables: String(k.length),
|
|
1776
1833
|
capture: F?.dir ?? null,
|
|
1777
1834
|
via: e.via ?? "direct",
|
|
1778
|
-
staging: r.map(
|
|
1835
|
+
staging: r.map(q).join(","),
|
|
1779
1836
|
heartbeatAt: /* @__PURE__ */ new Date(),
|
|
1780
|
-
resumable: !
|
|
1837
|
+
resumable: !g
|
|
1781
1838
|
}).then(() => {
|
|
1782
1839
|
a = !0;
|
|
1783
1840
|
}, (t) => {
|
|
@@ -1786,75 +1843,75 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1786
1843
|
them by introspection.`);
|
|
1787
1844
|
}));
|
|
1788
1845
|
let t = Date.now();
|
|
1789
|
-
|
|
1846
|
+
_e = () => {
|
|
1790
1847
|
if (!a) return;
|
|
1791
1848
|
let n = Date.now();
|
|
1792
|
-
n - t <
|
|
1849
|
+
n - t < tr || (t = n, e.store.update(u, i, { heartbeatAt: /* @__PURE__ */ new Date() }).then(() => void 0, () => void 0));
|
|
1793
1850
|
};
|
|
1794
1851
|
}
|
|
1795
|
-
let o = () => a ? e.store.delete(
|
|
1796
|
-
getTable: (e) =>
|
|
1852
|
+
let o = () => a ? e.store.delete(u, i).then(() => void 0, () => void 0) : Promise.resolve(), s = zn(r, {
|
|
1853
|
+
getTable: (e) => v(e),
|
|
1797
1854
|
registerTable: (e) => {
|
|
1798
|
-
|
|
1855
|
+
ee(e);
|
|
1799
1856
|
},
|
|
1800
|
-
unregisterTable:
|
|
1857
|
+
unregisterTable: re
|
|
1801
1858
|
}), c = !1;
|
|
1802
1859
|
try {
|
|
1803
|
-
let i =
|
|
1860
|
+
let i = g ? r : r.filter((e) => !Wt(x, e));
|
|
1804
1861
|
yield* n.tryPromise({
|
|
1805
1862
|
try: async () => {
|
|
1806
|
-
await
|
|
1863
|
+
await Un(t, i.map((e) => Ln(e, D))), await Un(t, i.map((e) => In(e, D)));
|
|
1807
1864
|
},
|
|
1808
1865
|
catch: (e) => new H({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1809
|
-
}), yield*
|
|
1810
|
-
try: () =>
|
|
1866
|
+
}), yield* I(ye(e.store, !g)), !g && x.truncated === !0 ? c = !0 : yield* n.tryPromise({
|
|
1867
|
+
try: () => Wn(t, k, D).then(() => {
|
|
1811
1868
|
c = !0;
|
|
1812
1869
|
}),
|
|
1813
1870
|
catch: (e) => new H({ reason: String(e?.message ?? e) })
|
|
1814
|
-
}).pipe(n.
|
|
1815
|
-
let a =
|
|
1871
|
+
}).pipe(n.catch((i) => n.gen(function* () {
|
|
1872
|
+
let a = Hn(yield* n.promise(() => Gn(t, Bn(e.targetSnapshot, r), D).then((e) => e, () => [])));
|
|
1816
1873
|
return yield* n.fail(new H({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1817
|
-
}))), !
|
|
1874
|
+
}))), !g && c && x.truncated !== !0 && (x.truncated = !0, yield* G(y, x));
|
|
1818
1875
|
} finally {
|
|
1819
|
-
s.undo(), c ||
|
|
1876
|
+
s.undo(), c || g ? (_e = void 0, yield* n.promise(() => Un(t, r.map((e) => Ln(e, D))).then(() => void 0, () => void 0)), yield* n.promise(o)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1820
1877
|
}
|
|
1821
|
-
} else
|
|
1878
|
+
} else g ? yield* n.tryPromise({
|
|
1822
1879
|
try: () => e.store.transactional(async (e) => {
|
|
1823
|
-
let t = await xe(() => n.runPromiseExit(
|
|
1824
|
-
if (r.isFailure(t)) throw
|
|
1880
|
+
let t = await xe(() => n.runPromiseExit(ye(e, !1)));
|
|
1881
|
+
if (r.isFailure(t)) throw ar(t.cause);
|
|
1825
1882
|
}),
|
|
1826
|
-
catch: (e) =>
|
|
1827
|
-
}) : yield*
|
|
1883
|
+
catch: (e) => ir(e) ?? new H({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1884
|
+
}) : yield* I(ye(e.store, !0));
|
|
1828
1885
|
if (e.assets && s.assets.index && !e.assetsAlreadyRestored) {
|
|
1829
1886
|
let t = e.assets;
|
|
1830
|
-
yield*
|
|
1887
|
+
yield* _t({
|
|
1831
1888
|
sink: t,
|
|
1832
|
-
assetsDir: i.join(e.bundleDir,
|
|
1889
|
+
assetsDir: i.join(e.bundleDir, Nt),
|
|
1833
1890
|
indexPath: i.join(e.bundleDir, s.assets.index ?? "assets/index.ndjson"),
|
|
1834
1891
|
progress: {
|
|
1835
|
-
done: (e) => e in
|
|
1836
|
-
mark: (e) => (
|
|
1892
|
+
done: (e) => e in x.assets,
|
|
1893
|
+
mark: (e) => (x.assets[e] = !0, G(y, x))
|
|
1837
1894
|
}
|
|
1838
1895
|
});
|
|
1839
1896
|
}
|
|
1840
1897
|
return yield* n.promise(() => Ee(null)), {
|
|
1841
1898
|
manifest: s,
|
|
1842
|
-
tablesWritten:
|
|
1843
|
-
rowsWritten:
|
|
1899
|
+
tablesWritten: M,
|
|
1900
|
+
rowsWritten: j,
|
|
1844
1901
|
tablesSkipped: N,
|
|
1845
1902
|
rowsSkipped: ve,
|
|
1846
|
-
fullyResumed:
|
|
1903
|
+
fullyResumed: M === 0 && N > 0,
|
|
1847
1904
|
...F === void 0 ? {} : { rollback: F }
|
|
1848
1905
|
};
|
|
1849
1906
|
}).pipe(n.tapError((e) => n.promise(() => t?.(e?.reason ?? e?._tag ?? "the import did not finish") ?? Promise.resolve())));
|
|
1850
|
-
},
|
|
1907
|
+
}, cr = (e) => e, lr = (e) => {
|
|
1851
1908
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1852
1909
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1853
1910
|
return {
|
|
1854
1911
|
refuse: !0,
|
|
1855
1912
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1856
1913
|
};
|
|
1857
|
-
},
|
|
1914
|
+
}, Y = Buffer.from("VBUNDLE2"), ur = 0, dr = 1, fr = 2, pr = 32, mr = async (e) => {
|
|
1858
1915
|
let t = [], n = async (r) => {
|
|
1859
1916
|
let i;
|
|
1860
1917
|
try {
|
|
@@ -1868,22 +1925,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1868
1925
|
}
|
|
1869
1926
|
};
|
|
1870
1927
|
return await n(""), t.sort();
|
|
1871
|
-
},
|
|
1928
|
+
}, hr = (e, t) => {
|
|
1872
1929
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1873
|
-
return r.writeUInt8(
|
|
1874
|
-
},
|
|
1930
|
+
return r.writeUInt8(dr, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1931
|
+
}, gr = (e, t, n) => {
|
|
1875
1932
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1876
|
-
return a.writeUInt8(
|
|
1933
|
+
return a.writeUInt8(fr, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1877
1934
|
};
|
|
1878
|
-
async function*
|
|
1879
|
-
yield new Uint8Array(
|
|
1880
|
-
for (let t of await
|
|
1935
|
+
async function* _r(e) {
|
|
1936
|
+
yield new Uint8Array(Y);
|
|
1937
|
+
for (let t of await mr(e.dir)) {
|
|
1881
1938
|
let n = I(e.dir, t), r = await be(n);
|
|
1882
|
-
if (yield new Uint8Array(
|
|
1939
|
+
if (yield new Uint8Array(hr(t, r.size)), r.size > 0) for await (let e of ue(n)) yield new Uint8Array(e);
|
|
1883
1940
|
}
|
|
1884
1941
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1885
|
-
yield new Uint8Array(
|
|
1886
|
-
let n =
|
|
1942
|
+
yield new Uint8Array(gr(t.key, t.contentType, t.size));
|
|
1943
|
+
let n = w("sha256"), r = 0;
|
|
1887
1944
|
for await (let i of e.blobs.open(t.key)) {
|
|
1888
1945
|
let e = Buffer.from(i);
|
|
1889
1946
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1891,11 +1948,11 @@ async function* br(e) {
|
|
|
1891
1948
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1892
1949
|
yield new Uint8Array(n.digest());
|
|
1893
1950
|
}
|
|
1894
|
-
yield new Uint8Array(Buffer.from([
|
|
1951
|
+
yield new Uint8Array(Buffer.from([ur]));
|
|
1895
1952
|
}
|
|
1896
|
-
var
|
|
1953
|
+
var vr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), yr = (e) => new Promise((t) => e.end(t)), br = async (e, t) => {
|
|
1897
1954
|
let n = L(t.dir);
|
|
1898
|
-
await
|
|
1955
|
+
await N(n, { recursive: !0 });
|
|
1899
1956
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1900
1957
|
let { value: e, done: t } = await i.next();
|
|
1901
1958
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1906,28 +1963,28 @@ var xr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Sr
|
|
|
1906
1963
|
return r = r.subarray(e), t;
|
|
1907
1964
|
};
|
|
1908
1965
|
try {
|
|
1909
|
-
if (await s(
|
|
1966
|
+
if (await s(Y.length), !c(Y.length).equals(Y)) throw Error("not a bundle archive (bad magic)");
|
|
1910
1967
|
for (;;) {
|
|
1911
1968
|
await s(1);
|
|
1912
1969
|
let e = c(1).readUInt8(0);
|
|
1913
|
-
if (e ===
|
|
1914
|
-
if (e ===
|
|
1970
|
+
if (e === ur) return;
|
|
1971
|
+
if (e === dr) {
|
|
1915
1972
|
await s(4);
|
|
1916
1973
|
let e = c(4).readUInt32BE(0);
|
|
1917
1974
|
await s(e);
|
|
1918
1975
|
let t = c(e).toString("utf8");
|
|
1919
1976
|
await s(8);
|
|
1920
1977
|
let i = c(8).readBigUInt64BE(0), a = L(n, t);
|
|
1921
|
-
if (a !== n && !a.startsWith(n +
|
|
1922
|
-
await
|
|
1923
|
-
let o =
|
|
1978
|
+
if (a !== n && !a.startsWith(n + Se)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1979
|
+
await N(xe(a), { recursive: !0 });
|
|
1980
|
+
let o = de(a);
|
|
1924
1981
|
for (; i > 0n;) {
|
|
1925
1982
|
r.length === 0 && await s(1);
|
|
1926
1983
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1927
|
-
await
|
|
1984
|
+
await vr(o, c(e)), i -= BigInt(e);
|
|
1928
1985
|
}
|
|
1929
|
-
await
|
|
1930
|
-
} else if (e ===
|
|
1986
|
+
await yr(o);
|
|
1987
|
+
} else if (e === fr) {
|
|
1931
1988
|
await s(4);
|
|
1932
1989
|
let e = c(4).readUInt32BE(0);
|
|
1933
1990
|
await s(e);
|
|
@@ -1937,24 +1994,24 @@ var xr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Sr
|
|
|
1937
1994
|
await s(i);
|
|
1938
1995
|
let a = c(i).toString("utf8");
|
|
1939
1996
|
await s(8);
|
|
1940
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1997
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = w("sha256"), d = new ge(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1941
1998
|
contentType: a,
|
|
1942
1999
|
size: Number(o)
|
|
1943
2000
|
}), p = o;
|
|
1944
2001
|
for (; p > 0n;) {
|
|
1945
2002
|
r.length === 0 && await s(1);
|
|
1946
2003
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1947
|
-
u.update(t), l || await
|
|
2004
|
+
u.update(t), l || await vr(d, t), p -= BigInt(e);
|
|
1948
2005
|
}
|
|
1949
|
-
l ? d.destroy() : await
|
|
1950
|
-
let m = c(
|
|
2006
|
+
l ? d.destroy() : await yr(d), await f, await s(pr);
|
|
2007
|
+
let m = c(pr);
|
|
1951
2008
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1952
2009
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1953
2010
|
}
|
|
1954
2011
|
} finally {
|
|
1955
2012
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1956
2013
|
}
|
|
1957
|
-
},
|
|
2014
|
+
}, xr = async (e, t) => {
|
|
1958
2015
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1959
2016
|
let { value: e, done: t } = await a.next();
|
|
1960
2017
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1973,11 +2030,11 @@ var xr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Sr
|
|
|
1973
2030
|
}
|
|
1974
2031
|
return Buffer.concat(t);
|
|
1975
2032
|
};
|
|
1976
|
-
if (!await s(
|
|
2033
|
+
if (!await s(Y.length) || !c(Y.length).equals(Y)) throw Error("not a bundle archive (bad magic)");
|
|
1977
2034
|
for (; await s(1);) {
|
|
1978
2035
|
let e = c(1).readUInt8(0);
|
|
1979
|
-
if (e ===
|
|
1980
|
-
if (e !==
|
|
2036
|
+
if (e === ur || e === fr) break;
|
|
2037
|
+
if (e !== dr) throw Error(`unknown bundle entry kind ${e}`);
|
|
1981
2038
|
if (!await s(4)) break;
|
|
1982
2039
|
let t = c(4).readUInt32BE(0);
|
|
1983
2040
|
if (!await s(t)) break;
|
|
@@ -1987,20 +2044,20 @@ var xr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Sr
|
|
|
1987
2044
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1988
2045
|
}
|
|
1989
2046
|
return r;
|
|
1990
|
-
},
|
|
2047
|
+
}, Sr = (e, t) => br(e, {
|
|
1991
2048
|
dir: t,
|
|
1992
|
-
blobs:
|
|
1993
|
-
}),
|
|
2049
|
+
blobs: Cr(t)
|
|
2050
|
+
}), Cr = (e) => {
|
|
1994
2051
|
let t = /* @__PURE__ */ new Set();
|
|
1995
2052
|
return {
|
|
1996
2053
|
has: async (e) => t.has(e),
|
|
1997
2054
|
write: async (n, r, i) => {
|
|
1998
2055
|
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = I(e, "assets");
|
|
1999
|
-
await
|
|
2000
|
-
let l =
|
|
2056
|
+
await N(c, { recursive: !0 });
|
|
2057
|
+
let l = w("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
2001
2058
|
l.update(e), u += e.length, n(null, e);
|
|
2002
2059
|
} }), f = I(c, `.tmp-${Date.now()}-${t.size}`);
|
|
2003
|
-
await o(r, d,
|
|
2060
|
+
await o(r, d, de(f));
|
|
2004
2061
|
let p = l.digest("hex");
|
|
2005
2062
|
await a.rename(f, I(c, p)), await a.appendFile(I(c, "index.ndjson"), `${JSON.stringify({
|
|
2006
2063
|
key: n,
|
|
@@ -2010,76 +2067,114 @@ var xr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Sr
|
|
|
2010
2067
|
})}\n`), t.add(n);
|
|
2011
2068
|
}
|
|
2012
2069
|
};
|
|
2013
|
-
},
|
|
2070
|
+
}, wr = Ce(ce), Tr = Buffer.from("VENC1\0\0\0"), Er = 32, Dr = 16, Or = 7, kr = 16, X = 262144, Ar = {
|
|
2014
2071
|
N: 32768,
|
|
2015
2072
|
r: 8,
|
|
2016
2073
|
p: 1
|
|
2017
|
-
},
|
|
2074
|
+
}, jr = 134217728, Mr = 43, Nr = (e, t, n) => {
|
|
2018
2075
|
let r = Buffer.alloc(12);
|
|
2019
|
-
return e.copy(r, 0, 0,
|
|
2076
|
+
return e.copy(r, 0, 0, Or), r.writeUInt32BE(t >>> 0, Or), r.writeUInt8(+!!n, 11), r;
|
|
2020
2077
|
};
|
|
2021
|
-
async function*
|
|
2078
|
+
async function* Pr(e, t) {
|
|
2022
2079
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
2023
|
-
let n =
|
|
2024
|
-
...
|
|
2025
|
-
maxmem:
|
|
2026
|
-
}), a = Buffer.alloc(
|
|
2027
|
-
|
|
2080
|
+
let n = E(Dr), r = E(Or), i = await wr(t, n, Er, {
|
|
2081
|
+
...Ar,
|
|
2082
|
+
maxmem: jr
|
|
2083
|
+
}), a = Buffer.alloc(Mr);
|
|
2084
|
+
Tr.copy(a, 0), a.writeUInt32BE(Ar.N, 8), a.writeUInt32BE(Ar.r, 12), a.writeUInt32BE(Ar.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
2028
2085
|
let o = 0, s = (e, t) => {
|
|
2029
|
-
let n = ae("aes-256-gcm", i,
|
|
2086
|
+
let n = ae("aes-256-gcm", i, Nr(r, o, t));
|
|
2030
2087
|
o === 0 && n.setAAD(a);
|
|
2031
2088
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
2032
2089
|
o++;
|
|
2033
|
-
let l = Buffer.alloc(5 + s.length +
|
|
2034
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
2035
|
-
}, c = Buffer.alloc(0);
|
|
2036
|
-
for await (let t of e)
|
|
2037
|
-
|
|
2090
|
+
let l = Buffer.alloc(5 + s.length + kr);
|
|
2091
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + kr, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
2092
|
+
}, c = Buffer.alloc(0), l = 0;
|
|
2093
|
+
for await (let t of e) {
|
|
2094
|
+
let e = Buffer.from(t), n = 0;
|
|
2095
|
+
for (; n < e.length;) {
|
|
2096
|
+
if (l === X && (yield s(c, !1), l = 0), l === 0 && e.length - n >= X) {
|
|
2097
|
+
let t = e.subarray(n, n + X);
|
|
2098
|
+
n += X, n < e.length ? yield s(t, !1) : (c = t, l = X);
|
|
2099
|
+
continue;
|
|
2100
|
+
}
|
|
2101
|
+
c.length !== X && (c = Buffer.alloc(X));
|
|
2102
|
+
let t = Math.min(X - l, e.length - n);
|
|
2103
|
+
e.copy(c, l, n, n + t), l += t, n += t;
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
yield s(c.subarray(0, l), !0);
|
|
2038
2107
|
}
|
|
2039
|
-
async function*
|
|
2108
|
+
async function* Fr(e, t) {
|
|
2040
2109
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
2041
|
-
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1,
|
|
2042
|
-
let
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2110
|
+
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = 0, a = !1, o = async (e) => {
|
|
2111
|
+
let t, o = 0;
|
|
2112
|
+
for (; o < e;) {
|
|
2113
|
+
if (i === r.length) {
|
|
2114
|
+
if (a) break;
|
|
2115
|
+
let e = await n.next();
|
|
2116
|
+
if (e.done) {
|
|
2117
|
+
a = !0;
|
|
2118
|
+
break;
|
|
2119
|
+
}
|
|
2120
|
+
if (r = Buffer.from(e.value), i = 0, r.length === 0) continue;
|
|
2121
|
+
}
|
|
2122
|
+
if (o === 0 && r.length - i >= e) {
|
|
2123
|
+
let t = r.subarray(i, i + e);
|
|
2124
|
+
return i += e, t;
|
|
2125
|
+
}
|
|
2126
|
+
t ??= Buffer.alloc(e);
|
|
2127
|
+
let s = Math.min(e - o, r.length - i);
|
|
2128
|
+
r.copy(t, o, i, i + s), i += s, o += s;
|
|
2129
|
+
}
|
|
2130
|
+
return t?.subarray(0, o) ?? Buffer.alloc(0);
|
|
2131
|
+
}, s = !1;
|
|
2132
|
+
try {
|
|
2133
|
+
let e = await o(Mr);
|
|
2134
|
+
if (e.length !== Mr) throw Error("encrypted bundle truncated (no header)");
|
|
2135
|
+
if (!e.subarray(0, 8).equals(Tr)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
2136
|
+
let n = e.readUInt32BE(8), r = e.readUInt32BE(12), i = e.readUInt32BE(16), a = Buffer.from(e.subarray(20, 36)), s = Buffer.from(e.subarray(36, Mr)), c = await wr(t, a, Er, {
|
|
2137
|
+
N: n,
|
|
2138
|
+
r,
|
|
2139
|
+
p: i,
|
|
2140
|
+
maxmem: jr
|
|
2141
|
+
}), l = 0, u = !1;
|
|
2142
|
+
for (;;) {
|
|
2143
|
+
let t = await o(5);
|
|
2144
|
+
if (t.length !== 5) {
|
|
2145
|
+
if (t.length === 0 && u) return;
|
|
2146
|
+
throw Error("encrypted bundle truncated (no authenticated final frame)");
|
|
2147
|
+
}
|
|
2148
|
+
if (u) throw Error("encrypted bundle has trailing data after the final frame");
|
|
2149
|
+
let n = t.readUInt8(0);
|
|
2150
|
+
if (n !== 0 && n !== 1) throw Error("encrypted bundle corrupt (invalid final flag)");
|
|
2151
|
+
let r = n === 1, i = t.readUInt32BE(1);
|
|
2152
|
+
if (i < kr) throw Error("encrypted bundle corrupt (short frame)");
|
|
2153
|
+
if (i > 262160) throw Error("encrypted bundle corrupt (oversized frame)");
|
|
2154
|
+
let a = await o(i);
|
|
2155
|
+
if (a.length !== i) throw Error("encrypted bundle truncated (frame body)");
|
|
2156
|
+
let d = a.subarray(0, i - kr), f = a.subarray(i - kr), p = oe("aes-256-gcm", c, Nr(s, l, r));
|
|
2157
|
+
l === 0 && p.setAAD(e), p.setAuthTag(f);
|
|
2158
|
+
let m = Buffer.concat([p.update(d), p.final()]);
|
|
2159
|
+
l++, r && (u = !0), m.length > 0 && (yield new Uint8Array(m));
|
|
2160
|
+
}
|
|
2161
|
+
} catch (e) {
|
|
2162
|
+
throw s = !0, e;
|
|
2163
|
+
} finally {
|
|
2164
|
+
if (!a) try {
|
|
2165
|
+
await n.return?.();
|
|
2166
|
+
} catch (e) {
|
|
2167
|
+
if (!s) throw e;
|
|
2063
2168
|
}
|
|
2064
|
-
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
2065
|
-
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
2066
|
-
if (t < Mr) throw Error("encrypted bundle corrupt (short frame)");
|
|
2067
|
-
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
2068
|
-
let n = r.subarray(5, 5 + t - Mr), i = r.subarray(5 + t - Mr, 5 + t);
|
|
2069
|
-
r = r.subarray(5 + t);
|
|
2070
|
-
let a = oe("aes-256-gcm", p, Lr(f, m, e));
|
|
2071
|
-
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
2072
|
-
let c = Buffer.concat([a.update(n), a.final()]);
|
|
2073
|
-
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
2074
2169
|
}
|
|
2075
2170
|
}
|
|
2076
2171
|
//#endregion
|
|
2077
2172
|
//#region src/blobStorage.ts
|
|
2078
|
-
var
|
|
2173
|
+
var Ir = (e, t) => ({
|
|
2079
2174
|
list: () => {
|
|
2080
2175
|
let e = t();
|
|
2081
2176
|
return { async *[Symbol.asyncIterator]() {
|
|
2082
|
-
let t = await n.runPromise(
|
|
2177
|
+
let t = await n.runPromise(c.runCollect(e));
|
|
2083
2178
|
for (let e of t) yield {
|
|
2084
2179
|
key: e.key,
|
|
2085
2180
|
contentType: e.contentType,
|
|
@@ -2088,32 +2183,32 @@ var Br = (e, t) => ({
|
|
|
2088
2183
|
} };
|
|
2089
2184
|
},
|
|
2090
2185
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
2091
|
-
let r = await n.runPromise(
|
|
2186
|
+
let r = await n.runPromise(A(e, t)), i = await j(r.stream);
|
|
2092
2187
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
2093
2188
|
} })
|
|
2094
|
-
}),
|
|
2095
|
-
table:
|
|
2189
|
+
}), Lr = (e, t) => Ir(e, () => C(t, {
|
|
2190
|
+
table: pe,
|
|
2096
2191
|
chunkSize: 500
|
|
2097
|
-
}).pipe(
|
|
2192
|
+
}).pipe(c.map((e) => ({
|
|
2098
2193
|
key: String(e.key),
|
|
2099
2194
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
2100
2195
|
size: Number(e.size ?? 0)
|
|
2101
|
-
})))),
|
|
2196
|
+
})))), Rr = (e) => ({
|
|
2102
2197
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
2103
2198
|
write: async (t, r, i) => {
|
|
2104
|
-
let a =
|
|
2105
|
-
await n.runPromise(
|
|
2199
|
+
let a = me("storage", () => _e.from(r));
|
|
2200
|
+
await n.runPromise(he(e, t, a, {
|
|
2106
2201
|
contentType: i.contentType,
|
|
2107
2202
|
totalSize: i.size
|
|
2108
2203
|
}));
|
|
2109
2204
|
}
|
|
2110
|
-
}),
|
|
2205
|
+
}), zr = (e, t) => {
|
|
2111
2206
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
2112
|
-
return e === "mariadb" ?
|
|
2207
|
+
return e === "mariadb" ? Br(n) ? { tool: n } : {
|
|
2113
2208
|
tool: r,
|
|
2114
2209
|
hint: `'${n}' is not on PATH, so this ran '${r}' instead. If that binary came from Oracle's MySQL client it queries information_schema.COLUMN_STATISTICS, which MariaDB does not have, and the run dies with error 1109 after the first table. Install the MariaDB client package (which ships '${n}') and re-run. Do NOT reach for --column-statistics=0: that flag does not exist on '${n}', so it fixes the wrong client and breaks the right one. Note: a MariaDB 12.x client requires TLS by default, so running '${n}' BY HAND against a server without it fails with "TLS/SSL error: SSL is required" (2026) and needs --skip-ssl. This command is not affected.`
|
|
2115
2210
|
} : { tool: r };
|
|
2116
|
-
},
|
|
2211
|
+
}, Br = (e) => (process.env.PATH ?? "").split(F).filter((e) => e !== "").some((t) => fe(I(t, e))), Vr = (e) => {
|
|
2117
2212
|
switch (e) {
|
|
2118
2213
|
case "postgres": return "db.dump";
|
|
2119
2214
|
case "mysql":
|
|
@@ -2123,7 +2218,7 @@ var Br = (e, t) => ({
|
|
|
2123
2218
|
case "mssql": return "db.bacpac";
|
|
2124
2219
|
default: return "db.dump";
|
|
2125
2220
|
}
|
|
2126
|
-
},
|
|
2221
|
+
}, Hr = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), Z = (e, t = 3306) => {
|
|
2127
2222
|
if (e.host || e.database) return {
|
|
2128
2223
|
host: e.host ?? "localhost",
|
|
2129
2224
|
port: e.port ?? t,
|
|
@@ -2139,7 +2234,7 @@ var Br = (e, t) => ({
|
|
|
2139
2234
|
password: decodeURIComponent(n.password) || "",
|
|
2140
2235
|
database: n.pathname.replace(/^\//, "") || ""
|
|
2141
2236
|
};
|
|
2142
|
-
},
|
|
2237
|
+
}, Ur = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Wr = (e, t, n) => {
|
|
2143
2238
|
switch (e) {
|
|
2144
2239
|
case "postgres": {
|
|
2145
2240
|
if (t.url) return {
|
|
@@ -2149,7 +2244,7 @@ var Br = (e, t) => ({
|
|
|
2149
2244
|
"--format=custom",
|
|
2150
2245
|
"--no-owner",
|
|
2151
2246
|
"--no-privileges",
|
|
2152
|
-
...
|
|
2247
|
+
...Kr,
|
|
2153
2248
|
"--dbname",
|
|
2154
2249
|
t.url,
|
|
2155
2250
|
"--file",
|
|
@@ -2165,7 +2260,7 @@ var Br = (e, t) => ({
|
|
|
2165
2260
|
"--format=custom",
|
|
2166
2261
|
"--no-owner",
|
|
2167
2262
|
"--no-privileges",
|
|
2168
|
-
...
|
|
2263
|
+
...Kr,
|
|
2169
2264
|
"-h",
|
|
2170
2265
|
e.host,
|
|
2171
2266
|
"-p",
|
|
@@ -2185,14 +2280,14 @@ var Br = (e, t) => ({
|
|
|
2185
2280
|
let r = Z(t);
|
|
2186
2281
|
return {
|
|
2187
2282
|
kind: "spawn",
|
|
2188
|
-
...
|
|
2283
|
+
...zr(e, "dump"),
|
|
2189
2284
|
args: [
|
|
2190
2285
|
"--single-transaction",
|
|
2191
2286
|
"--routines",
|
|
2192
2287
|
"--triggers",
|
|
2193
2288
|
"--no-tablespaces",
|
|
2194
2289
|
"--protocol=TCP",
|
|
2195
|
-
...
|
|
2290
|
+
...Gr.map((e) => `--ignore-table=${r.database}.${e}`),
|
|
2196
2291
|
"-h",
|
|
2197
2292
|
r.host,
|
|
2198
2293
|
"-P",
|
|
@@ -2208,7 +2303,7 @@ var Br = (e, t) => ({
|
|
|
2208
2303
|
case "sqlite":
|
|
2209
2304
|
case "turso": return {
|
|
2210
2305
|
kind: "copy",
|
|
2211
|
-
from:
|
|
2306
|
+
from: Hr(t),
|
|
2212
2307
|
to: n
|
|
2213
2308
|
};
|
|
2214
2309
|
case "mssql": return {
|
|
@@ -2216,17 +2311,17 @@ var Br = (e, t) => ({
|
|
|
2216
2311
|
tool: "sqlpackage",
|
|
2217
2312
|
args: [
|
|
2218
2313
|
"/Action:Export",
|
|
2219
|
-
`/SourceConnectionString:${
|
|
2314
|
+
`/SourceConnectionString:${Ur(Z(t, 1433))}`,
|
|
2220
2315
|
`/TargetFile:${n}`
|
|
2221
2316
|
],
|
|
2222
2317
|
env: {}
|
|
2223
2318
|
};
|
|
2224
|
-
default: throw new
|
|
2319
|
+
default: throw new U({
|
|
2225
2320
|
tool: "backup",
|
|
2226
2321
|
reason: `no native backup for dialect '${e}'`
|
|
2227
2322
|
});
|
|
2228
2323
|
}
|
|
2229
|
-
},
|
|
2324
|
+
}, Gr = [u, l], Kr = Gr.flatMap((e) => ["--exclude-table", e]), qr = (e) => e === "postgres" || e === "mysql" || e === "mariadb" ? Gr : [], Jr = (e, t, n) => {
|
|
2230
2325
|
switch (e) {
|
|
2231
2326
|
case "postgres": {
|
|
2232
2327
|
if (t.url) return {
|
|
@@ -2268,7 +2363,7 @@ var Br = (e, t) => ({
|
|
|
2268
2363
|
let r = Z(t);
|
|
2269
2364
|
return {
|
|
2270
2365
|
kind: "spawn",
|
|
2271
|
-
...
|
|
2366
|
+
...zr(e, "client"),
|
|
2272
2367
|
args: [
|
|
2273
2368
|
"--protocol=TCP",
|
|
2274
2369
|
"-h",
|
|
@@ -2287,10 +2382,10 @@ var Br = (e, t) => ({
|
|
|
2287
2382
|
case "turso": return {
|
|
2288
2383
|
kind: "copy",
|
|
2289
2384
|
from: n,
|
|
2290
|
-
to:
|
|
2385
|
+
to: Hr(t)
|
|
2291
2386
|
};
|
|
2292
2387
|
case "mssql": {
|
|
2293
|
-
let e = Z(t, 1433), r =
|
|
2388
|
+
let e = Z(t, 1433), r = Xr(e.database);
|
|
2294
2389
|
return {
|
|
2295
2390
|
kind: "mssql-replace",
|
|
2296
2391
|
load: {
|
|
@@ -2298,7 +2393,7 @@ var Br = (e, t) => ({
|
|
|
2298
2393
|
tool: "sqlpackage",
|
|
2299
2394
|
args: [
|
|
2300
2395
|
"/Action:Import",
|
|
2301
|
-
`/TargetConnectionString:${
|
|
2396
|
+
`/TargetConnectionString:${Ur({
|
|
2302
2397
|
...e,
|
|
2303
2398
|
database: r
|
|
2304
2399
|
})}`,
|
|
@@ -2315,25 +2410,25 @@ var Br = (e, t) => ({
|
|
|
2315
2410
|
},
|
|
2316
2411
|
live: e.database,
|
|
2317
2412
|
scratch: r,
|
|
2318
|
-
carry:
|
|
2413
|
+
carry: Gr
|
|
2319
2414
|
};
|
|
2320
2415
|
}
|
|
2321
|
-
default: throw new
|
|
2416
|
+
default: throw new U({
|
|
2322
2417
|
tool: "restore",
|
|
2323
2418
|
reason: `no native restore for dialect '${e}'`
|
|
2324
2419
|
});
|
|
2325
2420
|
}
|
|
2326
|
-
}, Q = (e) => `[${e.replaceAll("]", "]]")}]`, $ = (e) => `N'${e.replaceAll("'", "''")}'`,
|
|
2327
|
-
DECLARE leftovers CURSOR LOCAL FAST_FORWARD FOR SELECT name FROM sys.databases WHERE name LIKE ${$(`${`${e}${
|
|
2421
|
+
}, Q = (e) => `[${e.replaceAll("]", "]]")}]`, $ = (e) => `N'${e.replaceAll("'", "''")}'`, Yr = "__voltro_restore_", Xr = (e, t = se().slice(0, 8)) => `${e}${Yr}${t}`, Zr = (e) => e.replace(Yr, "__voltro_replaced_"), Qr = (e) => `IF DB_ID(${$(e)}) IS NOT NULL BEGIN ALTER DATABASE ${Q(e)} SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE ${Q(e)} END`, $r = (e) => `DECLARE @leftover sysname, @drop nvarchar(max);
|
|
2422
|
+
DECLARE leftovers CURSOR LOCAL FAST_FORWARD FOR SELECT name FROM sys.databases WHERE name LIKE ${$(`${`${e}${Yr}`.replace(/[\\%_[]/g, (e) => `\\${e}`)}%`)} ESCAPE N'\\';
|
|
2328
2423
|
OPEN leftovers; FETCH NEXT FROM leftovers INTO @leftover;
|
|
2329
2424
|
WHILE @@FETCH_STATUS = 0 BEGIN
|
|
2330
2425
|
SET @drop = N'ALTER DATABASE ' + QUOTENAME(@leftover) + N' SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE ' + QUOTENAME(@leftover);
|
|
2331
2426
|
EXEC(@drop);
|
|
2332
2427
|
FETCH NEXT FROM leftovers INTO @leftover;
|
|
2333
|
-
END`,
|
|
2334
|
-
let t = Q(e.live), n = Q(e.scratch), r =
|
|
2428
|
+
END`, ei = (e) => {
|
|
2429
|
+
let t = Q(e.live), n = Q(e.scratch), r = Zr(e.scratch);
|
|
2335
2430
|
return {
|
|
2336
|
-
prepare: [
|
|
2431
|
+
prepare: [$r(e.live)],
|
|
2337
2432
|
carry: e.carry.map((e) => {
|
|
2338
2433
|
let r = $(`${n}.dbo.${Q(e)}`), i = $(`${t}.dbo.${Q(e)}`);
|
|
2339
2434
|
return `IF OBJECT_ID(${r}) IS NOT NULL BEGIN
|
|
@@ -2350,41 +2445,41 @@ END`;
|
|
|
2350
2445
|
`ALTER DATABASE ${t} SET SINGLE_USER WITH ROLLBACK IMMEDIATE`,
|
|
2351
2446
|
`ALTER DATABASE ${t} MODIFY NAME = ${Q(r)}`,
|
|
2352
2447
|
`ALTER DATABASE ${n} MODIFY NAME = ${t}`,
|
|
2353
|
-
|
|
2448
|
+
Qr(r)
|
|
2354
2449
|
],
|
|
2355
2450
|
undoSwap: [`IF DB_ID(${$(e.live)}) IS NULL AND DB_ID(${$(r)}) IS NOT NULL ALTER DATABASE ${Q(r)} MODIFY NAME = ${t}`, `IF DB_ID(${$(e.live)}) IS NOT NULL ALTER DATABASE ${t} SET MULTI_USER`],
|
|
2356
|
-
cleanup: [
|
|
2451
|
+
cleanup: [Qr(e.scratch)]
|
|
2357
2452
|
};
|
|
2358
|
-
},
|
|
2453
|
+
}, ti = (e) => {
|
|
2359
2454
|
let t = [];
|
|
2360
2455
|
for (let n = e, r = 0; n != null && r < 4; r++) {
|
|
2361
2456
|
let e = n instanceof Error ? n.message : typeof n == "object" && "message" in n ? String(n.message) : String(n);
|
|
2362
2457
|
e && !t.includes(e) && t.push(e), n = typeof n == "object" && "cause" in n ? n.cause : void 0;
|
|
2363
2458
|
}
|
|
2364
2459
|
return t.join(": ");
|
|
2365
|
-
},
|
|
2366
|
-
if (t === void 0) return n.fail(new
|
|
2460
|
+
}, ni = (e, t) => {
|
|
2461
|
+
if (t === void 0) return n.fail(new U({
|
|
2367
2462
|
tool: "sqlpackage",
|
|
2368
2463
|
reason: "an mssql restore swaps databases and needs a SQL connection to the server; none was provided"
|
|
2369
2464
|
}));
|
|
2370
|
-
let r =
|
|
2465
|
+
let r = ei(e), i = (r, i) => t(e.server, i).pipe(n.mapError((e) => new U({
|
|
2371
2466
|
tool: "sqlpackage",
|
|
2372
|
-
reason: `${r}: ${
|
|
2467
|
+
reason: `${r}: ${ti(e)}`
|
|
2373
2468
|
}))), a = i("dropping the scratch database", r.cleanup).pipe(n.ignore);
|
|
2374
2469
|
return n.gen(function* () {
|
|
2375
|
-
yield* i(`preparing ${e.scratch}`, r.prepare), yield*
|
|
2470
|
+
yield* i(`preparing ${e.scratch}`, r.prepare), yield* ri(e.load).pipe(n.tapError(() => a)), yield* i(`carrying ${e.carry.join(", ")} into the imported copy`, r.carry).pipe(n.tapError(() => a)), yield* i(`swapping ${e.scratch} in as ${e.live}`, r.swap).pipe(n.tapError(() => i("putting the original database back", r.undoSwap).pipe(n.ignore)));
|
|
2376
2471
|
});
|
|
2377
|
-
},
|
|
2378
|
-
let r = `${e.to}.restoring-${
|
|
2472
|
+
}, ri = (e, t) => e.kind === "mssql-replace" ? ni(e, t) : e.kind === "copy" ? n.callback((t) => {
|
|
2473
|
+
let r = `${e.to}.restoring-${se()}`, i = (e) => {
|
|
2379
2474
|
try {
|
|
2380
|
-
|
|
2475
|
+
k(r);
|
|
2381
2476
|
} catch {}
|
|
2382
|
-
t(n.fail(new
|
|
2477
|
+
t(n.fail(new U({
|
|
2383
2478
|
tool: "copy",
|
|
2384
2479
|
reason: e
|
|
2385
2480
|
})));
|
|
2386
2481
|
};
|
|
2387
|
-
|
|
2482
|
+
le(e.from, r, (a) => {
|
|
2388
2483
|
if (a) {
|
|
2389
2484
|
i(`copy ${e.from} → ${e.to}: ${a.message}`);
|
|
2390
2485
|
return;
|
|
@@ -2394,7 +2489,7 @@ END`;
|
|
|
2394
2489
|
`${e.to}-shm`,
|
|
2395
2490
|
`${e.to}-journal`
|
|
2396
2491
|
]) try {
|
|
2397
|
-
|
|
2492
|
+
k(t);
|
|
2398
2493
|
} catch {}
|
|
2399
2494
|
D(r, e.to, (r) => {
|
|
2400
2495
|
if (r) {
|
|
@@ -2404,8 +2499,8 @@ END`;
|
|
|
2404
2499
|
t(n.void);
|
|
2405
2500
|
});
|
|
2406
2501
|
});
|
|
2407
|
-
}) : n.
|
|
2408
|
-
let r =
|
|
2502
|
+
}) : n.callback((t) => {
|
|
2503
|
+
let r = we(e.tool, [...e.args], {
|
|
2409
2504
|
env: {
|
|
2410
2505
|
...process.env,
|
|
2411
2506
|
...e.env
|
|
@@ -2425,27 +2520,27 @@ END`;
|
|
|
2425
2520
|
s ??= e;
|
|
2426
2521
|
};
|
|
2427
2522
|
if (e.stdoutFile && r.stdout) {
|
|
2428
|
-
let t =
|
|
2429
|
-
t.on("error", (t) => c(`writing ${
|
|
2523
|
+
let t = de(e.stdoutFile);
|
|
2524
|
+
t.on("error", (t) => c(`writing ${P(e.stdoutFile)}: ${t.message}`)), r.stdout.on("error", (e) => c(`reading the tool's output: ${e.message}`)), r.stdout.pipe(t);
|
|
2430
2525
|
}
|
|
2431
2526
|
let l = 0;
|
|
2432
2527
|
if (e.stdinFile && r.stdin) {
|
|
2433
|
-
let t =
|
|
2528
|
+
let t = ue(e.stdinFile);
|
|
2434
2529
|
t.on("data", (e) => {
|
|
2435
2530
|
l += e.length;
|
|
2436
2531
|
}), t.on("error", (t) => {
|
|
2437
|
-
c(`reading ${
|
|
2532
|
+
c(`reading ${P(e.stdinFile)}: ${t.message}`), r.stdin?.end();
|
|
2438
2533
|
}), r.stdin.on("error", (t) => {
|
|
2439
|
-
t.code !== "EPIPE" && c(`feeding ${
|
|
2534
|
+
t.code !== "EPIPE" && c(`feeding ${P(e.stdinFile)} to ${e.tool}: ${t.message}`);
|
|
2440
2535
|
}), t.pipe(r.stdin);
|
|
2441
2536
|
}
|
|
2442
2537
|
let u = () => {
|
|
2443
2538
|
if (e.stdoutFile !== void 0) try {
|
|
2444
|
-
|
|
2539
|
+
k(e.stdoutFile);
|
|
2445
2540
|
} catch {}
|
|
2446
2541
|
};
|
|
2447
2542
|
r.on("error", (t) => {
|
|
2448
|
-
u(), o(n.fail(new
|
|
2543
|
+
u(), o(n.fail(new U({
|
|
2449
2544
|
tool: e.tool,
|
|
2450
2545
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${t.message}`,
|
|
2451
2546
|
...e.hint === void 0 ? {} : { hint: e.hint }
|
|
@@ -2456,45 +2551,45 @@ END`;
|
|
|
2456
2551
|
return;
|
|
2457
2552
|
}
|
|
2458
2553
|
u();
|
|
2459
|
-
let r = e.stdinFile === void 0 ? "" : ` — ${l} of ${
|
|
2460
|
-
o(n.fail(new
|
|
2554
|
+
let r = e.stdinFile === void 0 ? "" : ` — ${l} of ${ii(e.stdinFile)} byte(s) of ${P(e.stdinFile)} had been fed to it`;
|
|
2555
|
+
o(n.fail(new U({
|
|
2461
2556
|
tool: e.tool,
|
|
2462
|
-
reason: (t === 0 ? `exited cleanly, but ${s}` : `exited with code ${t}${s === void 0 ? "" : ` (${s})`}`) + r + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2557
|
+
reason: (t === 0 ? `exited cleanly, but ${s}` : `exited with code ${t}${s === void 0 ? "" : ` (${s})`}`) + r + (e.stdoutFile === void 0 ? "" : ` — the partial ${P(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
2463
2558
|
stderr: i.trim() === "" ? `(${e.tool} printed nothing to stderr)` : i.slice(-2e3),
|
|
2464
2559
|
...e.hint === void 0 ? {} : { hint: e.hint }
|
|
2465
2560
|
})));
|
|
2466
2561
|
});
|
|
2467
|
-
}),
|
|
2562
|
+
}), ii = (e) => {
|
|
2468
2563
|
try {
|
|
2469
|
-
return String(
|
|
2564
|
+
return String(O(e).size);
|
|
2470
2565
|
} catch {
|
|
2471
2566
|
return "?";
|
|
2472
2567
|
}
|
|
2473
|
-
},
|
|
2474
|
-
let t = yield*
|
|
2568
|
+
}, ai = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), oi = (e) => n.gen(function* () {
|
|
2569
|
+
let t = yield* ct(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
2475
2570
|
store: e.store,
|
|
2476
2571
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
2477
|
-
}), r =
|
|
2572
|
+
}), r = on(t.tables, e.snapshot), i = e.classification ?? De(e.snapshot), a = e.sampleSize ?? 20, o = Ve(r.map((t) => ({
|
|
2478
2573
|
name: t,
|
|
2479
2574
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
2480
2575
|
name: e.name,
|
|
2481
2576
|
type: e.type
|
|
2482
2577
|
}))
|
|
2483
|
-
})), i, e.masking), s = [],
|
|
2578
|
+
})), i, e.masking), s = [], l = [];
|
|
2484
2579
|
for (let i of r) {
|
|
2485
2580
|
let r = t.predicateFor(i), u = yield* C(e.store, {
|
|
2486
2581
|
table: i,
|
|
2487
2582
|
chunkSize: a,
|
|
2488
2583
|
...r === void 0 ? {} : { where: r }
|
|
2489
|
-
}).pipe(
|
|
2490
|
-
|
|
2584
|
+
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) => We(t, i, o.actions, e.masking.seed));
|
|
2585
|
+
l.push(...Ue(d, i, o.actions));
|
|
2491
2586
|
let f = u[0], p = d[0], m = [];
|
|
2492
2587
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
2493
2588
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
2494
2589
|
let n = e.slice(i.length + 1);
|
|
2495
2590
|
m.push({
|
|
2496
2591
|
column: n,
|
|
2497
|
-
action:
|
|
2592
|
+
action: ai(t),
|
|
2498
2593
|
before: f[n],
|
|
2499
2594
|
after: p[n]
|
|
2500
2595
|
});
|
|
@@ -2506,10 +2601,10 @@ END`;
|
|
|
2506
2601
|
}
|
|
2507
2602
|
return {
|
|
2508
2603
|
tables: s,
|
|
2509
|
-
leaks:
|
|
2604
|
+
leaks: l,
|
|
2510
2605
|
unclassified: o.unclassified
|
|
2511
2606
|
};
|
|
2512
|
-
}),
|
|
2607
|
+
}), si = (e, t) => {
|
|
2513
2608
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
2514
2609
|
for (let t of e) {
|
|
2515
2610
|
let e = n.get(t.name);
|
|
@@ -2564,7 +2659,7 @@ END`;
|
|
|
2564
2659
|
skipColumns: i,
|
|
2565
2660
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
2566
2661
|
};
|
|
2567
|
-
},
|
|
2662
|
+
}, ci = (e) => {
|
|
2568
2663
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
2569
2664
|
let t = {
|
|
2570
2665
|
safe: "✓",
|
|
@@ -2575,4 +2670,4 @@ END`;
|
|
|
2575
2670
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
2576
2671
|
};
|
|
2577
2672
|
//#endregion
|
|
2578
|
-
export {
|
|
2673
|
+
export { Nt as ASSETS_DIR, Pt as ASSET_INDEX_FILE, H as BundleError, Ye as CodecError, Ze as CompressionError, rt as CrossDialectError, Mt as DATA_DIR, Ee as DEFAULT_CLASS_ACTIONS, Ge as ENVIRONMENT_LOCAL_TABLES, Ke as ENVIRONMENT_PORTABLE_TABLES, Gr as EXCLUDED_FROM_DUMP, Ct as FORMAT_VERSION, nt as ImportModeError, Xe as IntegrityError, Ft as LEDGER_FILE, At as MANIFEST_FILE, tt as MaskingError, U as NativeToolError, $e as RowsRefusedError, nr as SAVEPOINT_BATCH_SIZE, Mn as STAGING_PREFIX, et as SchemaDriftError, Qe as ScopeError, B as applyAction, ir as asImportError, lr as assessLiveImport, Gt as assetDone, Vr as backupArtifactName, Wr as backupCommand, Nn as canStage, bn as canonicalType, ht as captureAssetsToCas, Cn as checkPortability, $n as checkSchemaDrift, De as classificationFromSnapshot, si as classifyImportDrift, bt as compressionExt, at as computeSubsetIds, Jn as decideStaging, kt as decodeManifest, tn as decodeRow, $t as decodeValue, Fr as decryptStream, cr as defineDataProfile, Qn as diffSnapshots, Cr as dirBlobSink, Ln as dropStagingSql, qr as dumpExcludedTables, Ht as emptyLedger, Ot as encodeManifest, en as encodeRow, Zt as encodeValue, Pr as encryptStream, Je as environmentLocalNotice, qe as environmentLocalReason, Un as execStatements, ci as formatImportDrift, wn as formatIssue, vt as hasZstd, V as isEnvironmentLocal, xt as makeCompressor, St as makeDecompressor, We as maskRow, ei as mssqlReplaceStatements, Xr as mssqlScratchName, an as ndjsonToRows, pt as noAssetProgress, yn as outsideReferenceRefusal, vn as outsideReferencesInto, _r as packBundle, xr as peekBundle, yt as pickCompression, Ve as planMasking, oi as previewMasking, gt as readAssetIndex, Ut as readLedger, Oe as resolveAction, ct as resolveScope, ft as restoreAssetFromCas, _t as restoreAssetsFromCas, Jr as restoreCommand, rn as rowsToNdjson, _n as runExport, sr as runImport, ri as runNativeStep, Ue as scanForLeaks, q as stagingNameFor, ut as storageAssetSink, lt as storageAssetSource, Rr as storageBlobSink, Ir as storageBlobSource, Lr as storageRefsBlobSource, ot as subsetToScope, Wt as tableDone, on as topoSortTables, br as unpackBundle, Sr as unpackBundleToDir, dt as writeAssetToCas, G as writeLedger };
|