@voltro/data-transfer 0.83.1 → 0.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1191 -0
- package/THIRD-PARTY-NOTICES.md +90 -336
- package/dist/index.d.ts +116 -121
- package/dist/index.js +770 -673
- package/package.json +4 -5
package/dist/index.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
|
-
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i,
|
|
2
|
-
import { DATA_TRANSFERS_TABLE as
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as ae, createDecipheriv as oe, createHash as
|
|
5
|
-
import { copyFile as
|
|
6
|
-
import { STORAGE_REFS_TABLE as
|
|
7
|
-
import * as
|
|
8
|
-
import { PassThrough as
|
|
9
|
-
import { SqlClient as
|
|
10
|
-
import { mkdir as
|
|
11
|
-
import { basename as
|
|
12
|
-
import { promisify as
|
|
13
|
-
import { spawn as
|
|
1
|
+
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, Option as a, Schedule as o, Schema as s, Stream as c } from "effect";
|
|
2
|
+
import { DATA_TRANSFERS_TABLE as l, REPLACE_IN_PROGRESS_TABLE as u, STAGING_TABLE_PREFIX as d, _voltroDataTransfersTable as f, classifyConstraintViolation as p, derivedRowId as m, ensureTableRegistered as h, eq as g, extractDbCause as _, getTable as v, inSet as y, isNotNull as b, or as x, recordsTable as S, registerTable as ee, streamTable as C, suspendWriteRecorders as te, suspendWriteRecordersEffect as ne, unregisterTable as re } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as ie } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as ae, createDecipheriv as oe, createHash as w, createHmac as T, randomBytes as E, randomUUID as se, scrypt as ce } from "node:crypto";
|
|
5
|
+
import { copyFile as le, createReadStream as ue, createWriteStream as de, existsSync as fe, rename as D, statSync as O, unlinkSync as k } from "node:fs";
|
|
6
|
+
import { STORAGE_REFS_TABLE as pe, fromNodeReadable as me, getObjectStream as A, putObjectStream as he, toNodeReadable as j } from "@voltro/plugin-storage";
|
|
7
|
+
import * as M from "node:zlib";
|
|
8
|
+
import { PassThrough as ge, Readable as _e } from "node:stream";
|
|
9
|
+
import { SqlClient as ve } from "effect/sql";
|
|
10
|
+
import { mkdir as N, readdir as ye, stat as be } from "node:fs/promises";
|
|
11
|
+
import { basename as P, delimiter as F, dirname as xe, join as I, resolve as L, sep as Se } from "node:path";
|
|
12
|
+
import { promisify as Ce } from "node:util";
|
|
13
|
+
import { spawn as we } from "node:child_process";
|
|
14
14
|
//#region src/consistency.ts
|
|
15
|
-
var
|
|
15
|
+
var Te = (e) => {
|
|
16
16
|
switch (e) {
|
|
17
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
18
18
|
case "mysql":
|
|
@@ -22,7 +22,7 @@ var Ee = (e) => {
|
|
|
22
22
|
case "turso": return [];
|
|
23
23
|
default: return [];
|
|
24
24
|
}
|
|
25
|
-
},
|
|
25
|
+
}, Ee = {
|
|
26
26
|
email: { fake: "email" },
|
|
27
27
|
fullName: { fake: "fullName" },
|
|
28
28
|
firstName: { fake: "firstName" },
|
|
@@ -37,7 +37,7 @@ var Ee = (e) => {
|
|
|
37
37
|
date: "dateShift",
|
|
38
38
|
secret: "null",
|
|
39
39
|
freeText: "redact"
|
|
40
|
-
},
|
|
40
|
+
}, De = (e) => {
|
|
41
41
|
let t = {};
|
|
42
42
|
for (let n of e.tables) {
|
|
43
43
|
let e = {};
|
|
@@ -49,15 +49,15 @@ var Ee = (e) => {
|
|
|
49
49
|
t[n.name] = e;
|
|
50
50
|
}
|
|
51
51
|
return t;
|
|
52
|
-
},
|
|
52
|
+
}, Oe = (e, t, n, r) => {
|
|
53
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
54
54
|
if (i) return i;
|
|
55
55
|
if (n?.sensitive) {
|
|
56
56
|
let e = n.sensitive.class;
|
|
57
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? Ee[e] ?? "redact";
|
|
58
58
|
}
|
|
59
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
60
|
-
},
|
|
60
|
+
}, ke = (e, t) => T("sha256", e).update(t == null ? "∅" : String(t)).digest(), R = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), Ae = [
|
|
61
61
|
"Ada",
|
|
62
62
|
"Alan",
|
|
63
63
|
"Grace",
|
|
@@ -70,7 +70,7 @@ var Ee = (e) => {
|
|
|
70
70
|
"Guido",
|
|
71
71
|
"Margaret",
|
|
72
72
|
"Bjarne"
|
|
73
|
-
],
|
|
73
|
+
], je = [
|
|
74
74
|
"Lovelace",
|
|
75
75
|
"Turing",
|
|
76
76
|
"Hopper",
|
|
@@ -83,7 +83,7 @@ var Ee = (e) => {
|
|
|
83
83
|
"Rossum",
|
|
84
84
|
"Hamilton",
|
|
85
85
|
"Stroustrup"
|
|
86
|
-
],
|
|
86
|
+
], Me = [
|
|
87
87
|
"Maple",
|
|
88
88
|
"Oak",
|
|
89
89
|
"Cedar",
|
|
@@ -92,7 +92,7 @@ var Ee = (e) => {
|
|
|
92
92
|
"Birch",
|
|
93
93
|
"Willow",
|
|
94
94
|
"Ash"
|
|
95
|
-
],
|
|
95
|
+
], Ne = [
|
|
96
96
|
"Initech",
|
|
97
97
|
"Hooli",
|
|
98
98
|
"Umbrella",
|
|
@@ -101,23 +101,23 @@ var Ee = (e) => {
|
|
|
101
101
|
"Soylent",
|
|
102
102
|
"Stark",
|
|
103
103
|
"Wayne"
|
|
104
|
-
], z = (e, t, n) => e[R(t, n) % e.length],
|
|
105
|
-
let r =
|
|
104
|
+
], z = (e, t, n) => e[R(t, n) % e.length], Pe = (e, t, n) => {
|
|
105
|
+
let r = ke(n, `${e}:${String(t)}`), i = R(r, 1) % 1e4;
|
|
106
106
|
switch (e) {
|
|
107
|
-
case "email": return `${z(
|
|
108
|
-
case "fullName": return `${z(
|
|
109
|
-
case "firstName": return z(
|
|
110
|
-
case "lastName": return z(
|
|
111
|
-
case "username": return `${z(
|
|
107
|
+
case "email": return `${z(Ae, r, 0).toLowerCase()}.${z(je, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${z(Ae, r, 0)} ${z(je, r, 1)}`;
|
|
109
|
+
case "firstName": return z(Ae, r, 0);
|
|
110
|
+
case "lastName": return z(je, r, 1);
|
|
111
|
+
case "username": return `${z(Ae, r, 0).toLowerCase()}${i}`;
|
|
112
112
|
case "phone": return `+1${String(R(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
-
case "address": return `${R(r, 0) % 9899 + 100} ${z(
|
|
114
|
-
case "company": return z(
|
|
113
|
+
case "address": return `${R(r, 0) % 9899 + 100} ${z(Me, r, 1)} St`;
|
|
114
|
+
case "company": return z(Ne, r, 0);
|
|
115
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
116
116
|
case "ip": return `${R(r, 0) % 256}.${R(r, 1) % 256}.${R(r, 2) % 256}.${R(r, 3) % 254 + 1}`;
|
|
117
|
-
case "creditCard": return
|
|
117
|
+
case "creditCard": return Fe(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
118
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
119
119
|
}
|
|
120
|
-
},
|
|
120
|
+
}, Fe = (e) => {
|
|
121
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
122
122
|
t.forEach((e, t) => {
|
|
123
123
|
let r = e;
|
|
@@ -125,10 +125,10 @@ var Ee = (e) => {
|
|
|
125
125
|
});
|
|
126
126
|
let r = (10 - n % 10) % 10;
|
|
127
127
|
return e.slice(0, 15) + String(r);
|
|
128
|
-
},
|
|
128
|
+
}, Ie = 864e5, Le = (e, t) => {
|
|
129
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
130
130
|
if (Number.isNaN(n)) return e;
|
|
131
|
-
let r = R(
|
|
131
|
+
let r = R(ke(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Ie);
|
|
132
132
|
return e instanceof Date ? i : i.toISOString();
|
|
133
133
|
}, B = (e, t) => {
|
|
134
134
|
if (t.value === null || t.value === void 0) return null;
|
|
@@ -136,14 +136,14 @@ var Ee = (e) => {
|
|
|
136
136
|
if (e === "keep") return t.value;
|
|
137
137
|
if (e === "null") return null;
|
|
138
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
139
|
-
if (e === "hash") return
|
|
140
|
-
if (e === "dateShift") return
|
|
141
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return ke(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Le(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return Pe(e.fake, t.value, t.seed);
|
|
142
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
143
|
-
if (typeof e == "object" && "json" in e) return
|
|
143
|
+
if (typeof e == "object" && "json" in e) return Re(e, t);
|
|
144
144
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn } | { json: { fields, rest? } }.`);
|
|
145
|
-
},
|
|
146
|
-
let { fields: n, rest: r = "redact" } = e.json, i = typeof t.value == "string" ?
|
|
145
|
+
}, Re = (e, t) => {
|
|
146
|
+
let { fields: n, rest: r = "redact" } = e.json, i = typeof t.value == "string" ? ze(t.value) : {
|
|
147
147
|
ok: !0,
|
|
148
148
|
value: t.value
|
|
149
149
|
};
|
|
@@ -158,7 +158,7 @@ var Ee = (e) => {
|
|
|
158
158
|
let a = {};
|
|
159
159
|
for (let [o, s] of Object.entries(e)) {
|
|
160
160
|
let e = n[o] ?? r, c = `${i}.${o}`;
|
|
161
|
-
a[o] = typeof e == "object" && "json" in e ?
|
|
161
|
+
a[o] = typeof e == "object" && "json" in e ? Re(e, {
|
|
162
162
|
...t,
|
|
163
163
|
value: s,
|
|
164
164
|
column: c,
|
|
@@ -180,7 +180,7 @@ var Ee = (e) => {
|
|
|
180
180
|
});
|
|
181
181
|
}, o = a(i.value, t.column);
|
|
182
182
|
return typeof t.value == "string" ? JSON.stringify(o) : o;
|
|
183
|
-
},
|
|
183
|
+
}, ze = (e) => {
|
|
184
184
|
try {
|
|
185
185
|
return {
|
|
186
186
|
ok: !0,
|
|
@@ -189,7 +189,7 @@ var Ee = (e) => {
|
|
|
189
189
|
} catch {
|
|
190
190
|
return { ok: !1 };
|
|
191
191
|
}
|
|
192
|
-
},
|
|
192
|
+
}, Be = (e) => {
|
|
193
193
|
if (typeof e == "string") return e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error";
|
|
194
194
|
if (typeof e != "object" || !e) return !1;
|
|
195
195
|
if ("fake" in e) return typeof e.fake == "string";
|
|
@@ -198,18 +198,18 @@ var Ee = (e) => {
|
|
|
198
198
|
let t = e.json;
|
|
199
199
|
if (typeof t != "object" || !t) return !1;
|
|
200
200
|
let { fields: n, rest: r } = t;
|
|
201
|
-
return typeof n != "object" || !n ? !1 : Object.values(n).every(
|
|
201
|
+
return typeof n != "object" || !n ? !1 : Object.values(n).every(Be) && (r === void 0 || Be(r));
|
|
202
202
|
}
|
|
203
203
|
return !1;
|
|
204
|
-
},
|
|
204
|
+
}, Ve = (e, t, n) => {
|
|
205
205
|
let r = {}, i = [], a = [];
|
|
206
206
|
for (let o of e) for (let e of o.columns) {
|
|
207
|
-
let s = `${o.name}.${e.name}`, c =
|
|
207
|
+
let s = `${o.name}.${e.name}`, c = Oe(o.name, e.name, t[o.name]?.[e.name], n);
|
|
208
208
|
if (c === "error") {
|
|
209
209
|
i.push(s);
|
|
210
210
|
continue;
|
|
211
211
|
}
|
|
212
|
-
if (!
|
|
212
|
+
if (!Be(c)) {
|
|
213
213
|
a.push({
|
|
214
214
|
column: s,
|
|
215
215
|
action: c
|
|
@@ -223,7 +223,7 @@ var Ee = (e) => {
|
|
|
223
223
|
unclassified: i,
|
|
224
224
|
invalid: a
|
|
225
225
|
};
|
|
226
|
-
},
|
|
226
|
+
}, He = [
|
|
227
227
|
{
|
|
228
228
|
name: "email",
|
|
229
229
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -240,7 +240,7 @@ var Ee = (e) => {
|
|
|
240
240
|
name: "creditCard",
|
|
241
241
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
242
242
|
}
|
|
243
|
-
],
|
|
243
|
+
], Ue = (e, t, n) => {
|
|
244
244
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
245
245
|
for (let [a, o] of Object.entries(n)) {
|
|
246
246
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -248,7 +248,7 @@ var Ee = (e) => {
|
|
|
248
248
|
for (let a of e) {
|
|
249
249
|
let e = a[n];
|
|
250
250
|
if (typeof e != "string" || e.length === 0) continue;
|
|
251
|
-
let o =
|
|
251
|
+
let o = He.find((t) => t.re.test(e));
|
|
252
252
|
if (o && !i.has(n)) {
|
|
253
253
|
i.add(n), r.push({
|
|
254
254
|
table: t,
|
|
@@ -261,7 +261,7 @@ var Ee = (e) => {
|
|
|
261
261
|
}
|
|
262
262
|
}
|
|
263
263
|
return r;
|
|
264
|
-
},
|
|
264
|
+
}, We = (e, t, n, r, i) => {
|
|
265
265
|
let a = {};
|
|
266
266
|
for (let [o, s] of Object.entries(e)) a[o] = B(n[`${t}.${o}`] ?? "keep", {
|
|
267
267
|
value: s,
|
|
@@ -271,7 +271,8 @@ var Ee = (e) => {
|
|
|
271
271
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
272
272
|
});
|
|
273
273
|
return a;
|
|
274
|
-
},
|
|
274
|
+
}, Ge = /* @__PURE__ */ new Map([
|
|
275
|
+
["_voltro_inspect_create_receipts", "inspect creation commands accepted by THIS deployment — a foreign receipt would replay or suppress a creation the target never accepted"],
|
|
275
276
|
["_voltro_isr_cache", "pages rendered by THIS deployment’s web processes — a foreign row would serve another environment’s HTML here"],
|
|
276
277
|
["_voltro_web_replicas", "heartbeats of the web processes running HERE — a foreign row advertises replicas that do not exist in the target"],
|
|
277
278
|
["_voltro_aggregate_meta", "materialized refresh generations computed HERE — a foreign row would claim that the target refreshed its own source data"],
|
|
@@ -344,7 +345,8 @@ var Ee = (e) => {
|
|
|
344
345
|
["cluster_messages", "workflow messages in flight HERE — a foreign row makes the target run work that was started, and may be running, elsewhere"],
|
|
345
346
|
["cluster_replies", "replies to those messages — a foreign row answers a request the target never sent"],
|
|
346
347
|
["cluster_migrations", "which engine migrations ran HERE — a foreign row marks the target's engine tables migrated when they are not"]
|
|
347
|
-
]),
|
|
348
|
+
]), Ke = /* @__PURE__ */ new Map([
|
|
349
|
+
["_voltro_comment_thread_events", "the status history of the app's comment threads — belongs with those threads"],
|
|
348
350
|
["_voltro_agent_threads", "conversations the app's users had — app data"],
|
|
349
351
|
["_voltro_agent_messages", "the messages in those conversations — app data"],
|
|
350
352
|
["_voltro_prompts", "prompt definitions the app authored — app data"],
|
|
@@ -373,123 +375,172 @@ var Ee = (e) => {
|
|
|
373
375
|
["_voltro_scim_users", "identities an external directory provisioned into the app — app data"],
|
|
374
376
|
["_voltro_scim_groups", "groups an external directory provisioned into the app — app data"],
|
|
375
377
|
["_voltro_row_history", "the app's own row-version history — the point of keeping versions is that they survive a move"],
|
|
378
|
+
["_voltro_crdt_versions", "the version history of the app's collaborative documents (named snapshots, archived lineages) — it restores the document rows that travel with it"],
|
|
376
379
|
["_voltro_audit_log", "the app's audit trail — compliance data the app owns, not deployment bookkeeping"],
|
|
377
380
|
["_voltro_feature_flags", "the flags the app defined — app configuration"],
|
|
378
381
|
["_voltro_feature_flag_audit", "who changed which flag — belongs with the flags"],
|
|
379
382
|
["_voltro_comments", "the comments the app's users wrote — app data"],
|
|
380
383
|
["_voltro_comment_threads", "the threads those comments hang on — app data"],
|
|
381
384
|
["_voltro_comment_reactions", "the reactions the app's users left — app data"],
|
|
382
|
-
["_voltro_comment_reads", "which comments each user has read — app data, and losing it re-marks everything unread"]
|
|
383
|
-
|
|
385
|
+
["_voltro_comment_reads", "which comments each user has read — app data, and losing it re-marks everything unread"],
|
|
386
|
+
["_voltro_comment_thread_events", "the status changes the app's users made on a thread — app data"]
|
|
387
|
+
]), V = (e) => Ge.has(e), qe = (e) => Ge.get(e), Je = (e, t) => {
|
|
384
388
|
if (e.length === 0) return;
|
|
385
|
-
let n = e.map((e) => ` ${e} — ${
|
|
389
|
+
let n = e.map((e) => ` ${e} — ${qe(e) ?? "describes this deployment"}`);
|
|
386
390
|
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
387
|
-
}, H = class extends
|
|
388
|
-
reason:
|
|
389
|
-
path:
|
|
390
|
-
}) {
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
}
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
}) {},
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
391
|
+
}, H = class extends s.TaggedError()("BundleError", {
|
|
392
|
+
reason: s.String,
|
|
393
|
+
path: s.optional(s.String)
|
|
394
|
+
}) {
|
|
395
|
+
get message() {
|
|
396
|
+
return this.path === void 0 ? this.reason : `${this.reason} (${this.path})`;
|
|
397
|
+
}
|
|
398
|
+
}, Ye = class extends s.TaggedError()("CodecError", {
|
|
399
|
+
table: s.String,
|
|
400
|
+
reason: s.String
|
|
401
|
+
}) {}, Xe = class extends s.TaggedError()("IntegrityError", {
|
|
402
|
+
what: s.String,
|
|
403
|
+
expected: s.String,
|
|
404
|
+
actual: s.String
|
|
405
|
+
}) {
|
|
406
|
+
get message() {
|
|
407
|
+
return `${this.what}: expected ${this.expected}, got ${this.actual}`;
|
|
408
|
+
}
|
|
409
|
+
}, Ze = class extends t.TaggedError("CompressionError") {}, Qe = class extends s.TaggedError()("ScopeError", { reason: s.String }) {
|
|
410
|
+
get message() {
|
|
411
|
+
return this.reason;
|
|
412
|
+
}
|
|
413
|
+
}, $e = class extends s.TaggedError()("RowsRefusedError", {
|
|
414
|
+
rows: s.Array(s.Struct({
|
|
415
|
+
table: s.String,
|
|
416
|
+
id: s.String,
|
|
417
|
+
reason: s.String,
|
|
418
|
+
derived: s.Boolean
|
|
403
419
|
})),
|
|
404
|
-
primaryCount:
|
|
405
|
-
totalCount:
|
|
406
|
-
byTable:
|
|
407
|
-
table:
|
|
408
|
-
refused:
|
|
409
|
-
primary:
|
|
420
|
+
primaryCount: s.Number,
|
|
421
|
+
totalCount: s.Number,
|
|
422
|
+
byTable: s.Array(s.Struct({
|
|
423
|
+
table: s.String,
|
|
424
|
+
refused: s.Number,
|
|
425
|
+
primary: s.Number
|
|
410
426
|
}))
|
|
411
|
-
}) {}, et = class extends
|
|
412
|
-
bundleFingerprint:
|
|
413
|
-
targetFingerprint:
|
|
414
|
-
diff:
|
|
415
|
-
}) {
|
|
427
|
+
}) {}, et = class extends s.TaggedError()("SchemaDriftError", {
|
|
428
|
+
bundleFingerprint: s.String,
|
|
429
|
+
targetFingerprint: s.String,
|
|
430
|
+
diff: s.Array(s.String)
|
|
431
|
+
}) {
|
|
432
|
+
get message() {
|
|
433
|
+
return `bundle schema ${this.bundleFingerprint} does not match target ${this.targetFingerprint}: ${this.diff.join("; ")}`;
|
|
434
|
+
}
|
|
435
|
+
}, U = class extends t.TaggedError("NativeToolError") {
|
|
416
436
|
get message() {
|
|
417
437
|
let e = this.stderr?.trim(), t = this.hint?.trim();
|
|
418
438
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}${t ? `\n\nhint: ${t}` : ""}`;
|
|
419
439
|
}
|
|
420
|
-
}, tt = class extends
|
|
421
|
-
unclassified:
|
|
422
|
-
invalidActions:
|
|
423
|
-
}) {
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
440
|
+
}, tt = class extends s.TaggedError()("MaskingError", {
|
|
441
|
+
unclassified: s.Array(s.String),
|
|
442
|
+
invalidActions: s.optional(s.Array(s.String))
|
|
443
|
+
}) {
|
|
444
|
+
get message() {
|
|
445
|
+
let e = [];
|
|
446
|
+
return this.unclassified.length > 0 && e.push(`unclassified columns: ${this.unclassified.join(", ")}`), this.invalidActions !== void 0 && this.invalidActions.length > 0 && e.push(`invalidActions: ${this.invalidActions.join(", ")}`), `masking refused — ${e.join("; ") || "no column may be exported"}`;
|
|
447
|
+
}
|
|
448
|
+
}, nt = class extends s.TaggedError()("ImportModeError", {
|
|
449
|
+
mode: s.String,
|
|
450
|
+
reason: s.String
|
|
451
|
+
}) {
|
|
452
|
+
get message() {
|
|
453
|
+
return `import mode '${this.mode}' refused: ${this.reason}`;
|
|
454
|
+
}
|
|
455
|
+
}, rt = class extends s.TaggedError()("CrossDialectError", {
|
|
456
|
+
sourceDialect: s.String,
|
|
457
|
+
targetDialect: s.String,
|
|
458
|
+
blocking: s.Array(s.String)
|
|
459
|
+
}) {}, it = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", at = (e, t, r) => n.gen(function* () {
|
|
460
|
+
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), l = /* @__PURE__ */ new Map(), u = /* @__PURE__ */ new Map();
|
|
461
|
+
for (let e of t.tables) u.has(e.name) || u.set(e.name, e);
|
|
462
|
+
let d = /* @__PURE__ */ new Map();
|
|
463
|
+
for (let [e, t] of u) d.set(e, {
|
|
464
|
+
pk: t.columns.find((e) => e.type === "id")?.name ?? "id",
|
|
465
|
+
fks: t.columns.filter((e) => e.references !== void 0).map((e) => ({
|
|
466
|
+
col: e.name,
|
|
467
|
+
refTable: e.references.table
|
|
468
|
+
}))
|
|
469
|
+
});
|
|
470
|
+
let f = (e) => d.get(e)?.pk ?? "id", p = (e) => {
|
|
435
471
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
436
472
|
return s.set(e, t), t;
|
|
437
|
-
},
|
|
473
|
+
}, m = (e, t) => {
|
|
438
474
|
if (t == null || a.has(e) || s.get(e)?.has(t)) return;
|
|
439
|
-
let n =
|
|
440
|
-
n.add(t),
|
|
441
|
-
},
|
|
442
|
-
let r =
|
|
475
|
+
let n = l.get(e) ?? /* @__PURE__ */ new Set();
|
|
476
|
+
n.add(t), l.set(e, n);
|
|
477
|
+
}, h = (e, t, n) => {
|
|
478
|
+
let r = p(e), i = a.has(e) ? void 0 : o.get(e) ?? (() => {
|
|
443
479
|
let t = /* @__PURE__ */ new Set();
|
|
444
480
|
return o.set(e, t), t;
|
|
445
481
|
})();
|
|
446
482
|
return (e) => {
|
|
447
483
|
r.add(e[t]), i?.add(e[t]);
|
|
448
|
-
for (let t of n)
|
|
484
|
+
for (let t of n) m(t.refTable, e[t.col]);
|
|
449
485
|
};
|
|
450
|
-
},
|
|
451
|
-
let
|
|
452
|
-
return
|
|
453
|
-
table:
|
|
454
|
-
pk:
|
|
455
|
-
...
|
|
456
|
-
}), (e) => n.sync(() =>
|
|
457
|
-
},
|
|
458
|
-
for (let e = 0; e < i &&
|
|
459
|
-
let e = [...
|
|
460
|
-
|
|
461
|
-
for (let [
|
|
462
|
-
let e = [...
|
|
463
|
-
e.length !== 0
|
|
486
|
+
}, g = (t, r) => {
|
|
487
|
+
let i = h(t, f(t), d.get(t)?.fks ?? []);
|
|
488
|
+
return c.runForEach(C(e, {
|
|
489
|
+
table: t,
|
|
490
|
+
pk: f(t),
|
|
491
|
+
...r === void 0 ? {} : { where: r }
|
|
492
|
+
}), (e) => n.sync(() => i(e)));
|
|
493
|
+
}, _ = n.gen(function* () {
|
|
494
|
+
for (let e = 0; e < i && l.size > 0; e++) {
|
|
495
|
+
let e = [...l.entries()];
|
|
496
|
+
l.clear();
|
|
497
|
+
for (let [t, n] of e) {
|
|
498
|
+
let e = [...n].filter((e) => !s.get(t)?.has(e));
|
|
499
|
+
if (e.length !== 0) for (let n = 0; n < e.length; n += 500) yield* g(t, y(f(t), e.slice(n, n + 500)));
|
|
464
500
|
}
|
|
465
501
|
}
|
|
466
502
|
});
|
|
467
|
-
for (let [e, t] of Object.entries(r.seeds)) yield*
|
|
468
|
-
if (yield*
|
|
469
|
-
let e = new Set(r.children === !0 ? [...s.keys()] : r.children.roots.filter((e) => s.has(e)));
|
|
470
|
-
for (let
|
|
471
|
-
let
|
|
472
|
-
for (let
|
|
473
|
-
let t =
|
|
503
|
+
for (let [e, t] of Object.entries(r.seeds)) yield* g(e, t);
|
|
504
|
+
if (yield* _, r.children) {
|
|
505
|
+
let e = new Set(r.children === !0 ? [...s.keys()] : r.children.roots.filter((e) => s.has(e))), a = /* @__PURE__ */ new Map();
|
|
506
|
+
for (let r = 0; r < i; r++) {
|
|
507
|
+
let r = !1, i = [];
|
|
508
|
+
for (let o of t.tables) {
|
|
509
|
+
let t = o.name, c = o.columns.map((e) => ({
|
|
474
510
|
col: e.name,
|
|
475
511
|
refTable: e.references?.table
|
|
476
512
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
477
|
-
if (
|
|
478
|
-
let
|
|
479
|
-
|
|
513
|
+
if (c.length === 0) continue;
|
|
514
|
+
let l = s.get(t)?.size ?? 0, u = a.get(t);
|
|
515
|
+
u === void 0 && (u = /* @__PURE__ */ new Map(), a.set(t, u));
|
|
516
|
+
let d = [], f = 0, p = () => {
|
|
517
|
+
let e = d;
|
|
518
|
+
return d = [], f = 0, e.length === 0 ? n.void : g(t, e.length === 1 ? e[0] : x(...e));
|
|
519
|
+
};
|
|
520
|
+
for (let e of c) {
|
|
521
|
+
let t = u.get(e.col);
|
|
522
|
+
t === void 0 && (t = /* @__PURE__ */ new Set(), u.set(e.col, t));
|
|
523
|
+
let n = [...s.get(e.refTable)].filter((e) => !t.has(e));
|
|
524
|
+
for (let t = 0; t < n.length; t += 500) {
|
|
525
|
+
let r = n.slice(t, t + 500);
|
|
526
|
+
f + r.length > 500 && (yield* p()), d.push(y(e.col, r)), f += r.length;
|
|
527
|
+
}
|
|
528
|
+
for (let e of n) t.add(e);
|
|
529
|
+
}
|
|
530
|
+
yield* p(), (s.get(t)?.size ?? 0) > l && !e.has(t) && i.push(t), (s.get(t)?.size ?? 0) > l && (r = !0);
|
|
480
531
|
}
|
|
481
|
-
for (let t of
|
|
482
|
-
if (yield*
|
|
532
|
+
for (let t of i) e.add(t);
|
|
533
|
+
if (yield* _, !r) break;
|
|
483
534
|
}
|
|
484
535
|
}
|
|
485
536
|
return o;
|
|
486
|
-
}),
|
|
537
|
+
}), ot = (e, t) => ({
|
|
487
538
|
tables: [...e.keys()],
|
|
488
539
|
predicateFor: (n) => {
|
|
489
540
|
let r = e.get(n);
|
|
490
|
-
return r ?
|
|
541
|
+
return r ? y(it(t, n), [...r]) : void 0;
|
|
491
542
|
}
|
|
492
|
-
}),
|
|
543
|
+
}), st = "tenantId", ct = (e, t, r) => n.gen(function* () {
|
|
493
544
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
494
545
|
if (e.kind === "all") {
|
|
495
546
|
let t = new Set(e.exclude ?? []);
|
|
@@ -500,7 +551,7 @@ var Ee = (e) => {
|
|
|
500
551
|
}
|
|
501
552
|
if (e.kind === "tables") {
|
|
502
553
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
503
|
-
if (a.length > 0) return yield* n.fail(new
|
|
554
|
+
if (a.length > 0) return yield* n.fail(new Qe({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
504
555
|
let o = new Set(t);
|
|
505
556
|
return {
|
|
506
557
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -508,30 +559,30 @@ var Ee = (e) => {
|
|
|
508
559
|
};
|
|
509
560
|
}
|
|
510
561
|
let o = e.tenantId;
|
|
511
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
512
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
513
|
-
if (s.length === 0) return yield* n.fail(new
|
|
514
|
-
let c = yield*
|
|
515
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
562
|
+
if (o === void 0 || o === "") return yield* n.fail(new Qe({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
563
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === st) ?? !1);
|
|
564
|
+
if (s.length === 0) return yield* n.fail(new Qe({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${st}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
565
|
+
let c = yield* at(r.store, t, {
|
|
566
|
+
seeds: Object.fromEntries(s.map((e) => [e, g(st, o)])),
|
|
516
567
|
pinned: s,
|
|
517
568
|
children: { roots: s }
|
|
518
569
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
519
570
|
return {
|
|
520
571
|
tables: [...s, ...l],
|
|
521
|
-
predicateFor: (e) => u.has(e) ?
|
|
572
|
+
predicateFor: (e) => u.has(e) ? g(st, o) : c.has(e) ? y(it(t, e), [...c.get(e)]) : void 0
|
|
522
573
|
};
|
|
523
|
-
}),
|
|
574
|
+
}), lt = (e, t) => ({
|
|
524
575
|
list: t,
|
|
525
|
-
open: (t) =>
|
|
526
|
-
}),
|
|
527
|
-
write: (t, n, r) =>
|
|
576
|
+
open: (t) => A(e, t)
|
|
577
|
+
}), ut = (e) => ({
|
|
578
|
+
write: (t, n, r) => he(e, t, n, r),
|
|
528
579
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
529
|
-
}),
|
|
580
|
+
}), dt = (e, t) => n.tryPromise({
|
|
530
581
|
try: async () => {
|
|
531
582
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
532
583
|
u.update(e), d += e.length, n(null, e);
|
|
533
584
|
} });
|
|
534
|
-
await o(await
|
|
585
|
+
await o(await j(e.stream), f, n(l));
|
|
535
586
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
536
587
|
try {
|
|
537
588
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -548,33 +599,33 @@ var Ee = (e) => {
|
|
|
548
599
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
549
600
|
path: t
|
|
550
601
|
})
|
|
551
|
-
}),
|
|
552
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256),
|
|
602
|
+
}), ft = (e, t, r) => n.gen(function* () {
|
|
603
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), l = o("sha256"), u = me("bundle", () => a(s)).pipe(c.tap((e) => n.sync(() => l.update(e))));
|
|
553
604
|
yield* e.write(r.key, u, {
|
|
554
605
|
contentType: r.contentType,
|
|
555
606
|
totalSize: r.size
|
|
556
607
|
});
|
|
557
|
-
let d =
|
|
558
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
608
|
+
let d = l.digest("hex");
|
|
609
|
+
if (d !== r.sha256) return yield* n.fail(new Xe({
|
|
559
610
|
what: `asset ${r.key}`,
|
|
560
611
|
expected: r.sha256,
|
|
561
612
|
actual: d
|
|
562
613
|
}));
|
|
563
|
-
}),
|
|
614
|
+
}), pt = {
|
|
564
615
|
done: () => !1,
|
|
565
616
|
mark: () => n.void
|
|
566
|
-
},
|
|
567
|
-
let t = e.progress ??
|
|
568
|
-
return yield*
|
|
569
|
-
u.push(
|
|
570
|
-
})), n.flatMap((a) => a === void 0 ? n.void :
|
|
617
|
+
}, mt = (e) => typeof e == "object" && !!e && e.status === 404, ht = (e) => n.gen(function* () {
|
|
618
|
+
let t = e.progress ?? pt, r = 0, i = 0, a = 0, o = 0, s = /* @__PURE__ */ new Set(), l = /* @__PURE__ */ new Set(), u = [];
|
|
619
|
+
return yield* c.runForEach(e.source.list(), (c) => (a++, t.done(c.key) || l.has(c.key) ? n.void : (l.add(c.key), e.source.open(c.key).pipe(n.map((e) => e), n.catchIf(mt, () => n.sync(() => {
|
|
620
|
+
u.push(c.key);
|
|
621
|
+
})), n.flatMap((a) => a === void 0 ? n.void : dt(a, e.assetsDir).pipe(n.flatMap((e) => {
|
|
571
622
|
let n = {
|
|
572
|
-
key:
|
|
623
|
+
key: c.key,
|
|
573
624
|
sha256: e.sha256,
|
|
574
625
|
size: e.size,
|
|
575
626
|
contentType: e.contentType
|
|
576
627
|
};
|
|
577
|
-
return r++, i += e.size, s.has(e.sha256) || (s.add(e.sha256), o += e.size), t.mark(
|
|
628
|
+
return r++, i += e.size, s.has(e.sha256) || (s.add(e.sha256), o += e.size), t.mark(c.key, n);
|
|
578
629
|
}))))))).pipe(n.mapError((e) => e instanceof H ? e : new H({ reason: `asset export failed: ${String(e)}` }))), {
|
|
579
630
|
count: r,
|
|
580
631
|
totalBytes: i,
|
|
@@ -583,60 +634,65 @@ var Ee = (e) => {
|
|
|
583
634
|
objectBytes: o,
|
|
584
635
|
missing: u
|
|
585
636
|
};
|
|
586
|
-
}),
|
|
637
|
+
}), gt = (e) => me("bundle", () => ue(e)).pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.map((e) => JSON.parse(e)), c.mapError((t) => t instanceof H ? t : new H({
|
|
587
638
|
reason: `asset index unreadable: ${String(t)}`,
|
|
588
639
|
path: e
|
|
589
|
-
}))),
|
|
590
|
-
let t = e.progress ??
|
|
591
|
-
return yield*
|
|
640
|
+
}))), _t = (e) => n.gen(function* () {
|
|
641
|
+
let t = e.progress ?? pt, r = 0, i = /* @__PURE__ */ new Set(), a = /* @__PURE__ */ new Set();
|
|
642
|
+
return yield* c.runForEach(gt(e.indexPath), (o) => t.done(o.key) || a.has(o.key) ? n.void : (a.add(o.key), ft(e.sink, e.assetsDir, o)).pipe(n.andThen(n.sync(() => {
|
|
592
643
|
r++, i.add(o.sha256);
|
|
593
644
|
})), n.andThen(t.mark(o.key, o)))), {
|
|
594
645
|
count: r,
|
|
595
646
|
objects: i.size
|
|
596
647
|
};
|
|
597
|
-
}),
|
|
598
|
-
name:
|
|
599
|
-
file:
|
|
600
|
-
rowCount:
|
|
601
|
-
checksum:
|
|
602
|
-
columnTypes:
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
648
|
+
}), vt = () => typeof M.createZstdCompress == "function", yt = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : vt() ? "zstd" : "gzip", bt = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", xt = (e) => e === "zstd" ? M.createZstdCompress() : e === "gzip" ? M.createGzip() : new ge(), St = (e) => e === "zstd" ? M.createZstdDecompress() : e === "gzip" ? M.createGunzip() : new ge(), Ct = 1, wt = s.Struct({
|
|
649
|
+
name: s.String,
|
|
650
|
+
file: s.String,
|
|
651
|
+
rowCount: s.Number,
|
|
652
|
+
checksum: s.String,
|
|
653
|
+
columnTypes: s.Record(s.String, s.String),
|
|
654
|
+
primaryKey: s.String
|
|
655
|
+
}), Tt = s.Struct({
|
|
656
|
+
kind: s.Literals([
|
|
657
|
+
"all",
|
|
658
|
+
"tenant",
|
|
659
|
+
"tables"
|
|
660
|
+
]),
|
|
661
|
+
tenantId: s.optional(s.String),
|
|
662
|
+
tables: s.optional(s.Array(s.String)),
|
|
663
|
+
exclude: s.optional(s.Array(s.String))
|
|
664
|
+
}), Et = s.Struct({
|
|
665
|
+
count: s.Number,
|
|
666
|
+
totalBytes: s.Number,
|
|
667
|
+
index: s.optional(s.String)
|
|
616
668
|
});
|
|
617
|
-
|
|
618
|
-
key:
|
|
619
|
-
sha256:
|
|
620
|
-
size:
|
|
621
|
-
contentType:
|
|
669
|
+
s.Struct({
|
|
670
|
+
key: s.String,
|
|
671
|
+
sha256: s.String,
|
|
672
|
+
size: s.Number,
|
|
673
|
+
contentType: s.String
|
|
622
674
|
});
|
|
623
|
-
var
|
|
624
|
-
formatVersion:
|
|
625
|
-
createdAt:
|
|
626
|
-
source:
|
|
627
|
-
dialect:
|
|
628
|
-
schemaFingerprint:
|
|
675
|
+
var Dt = s.Struct({
|
|
676
|
+
formatVersion: s.Number,
|
|
677
|
+
createdAt: s.String,
|
|
678
|
+
source: s.Struct({
|
|
679
|
+
dialect: s.String,
|
|
680
|
+
schemaFingerprint: s.String
|
|
629
681
|
}),
|
|
630
|
-
scope:
|
|
631
|
-
consistency:
|
|
632
|
-
compression:
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
682
|
+
scope: Tt,
|
|
683
|
+
consistency: s.Literals(["live", "snapshot"]),
|
|
684
|
+
compression: s.Literals([
|
|
685
|
+
"zstd",
|
|
686
|
+
"gzip",
|
|
687
|
+
"none"
|
|
688
|
+
]),
|
|
689
|
+
tables: s.Array(wt),
|
|
690
|
+
assets: Et,
|
|
691
|
+
masking: s.optional(s.Struct({
|
|
692
|
+
policyId: s.optional(s.String),
|
|
693
|
+
transformed: s.Array(s.String)
|
|
638
694
|
}))
|
|
639
|
-
}),
|
|
695
|
+
}), Ot = s.encodeSync(s.fromJsonString(Dt)), kt = s.decodeUnknownSync(s.fromJsonString(Dt)), At = "manifest.json", jt = "README.md", Mt = "data", Nt = "assets", Pt = "assets/index.ndjson", Ft = ".ledger.json", It = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", Lt = (e, t) => {
|
|
640
696
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
641
697
|
return `# Voltro data export
|
|
642
698
|
|
|
@@ -650,7 +706,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
650
706
|
| **Format version** | ${e.formatVersion} |
|
|
651
707
|
| **Consistency** | ${e.consistency} |
|
|
652
708
|
| **Compression** | ${e.compression} |
|
|
653
|
-
| **Scope** | ${
|
|
709
|
+
| **Scope** | ${It(e.scope)} |
|
|
654
710
|
| **Tables** | ${e.tables.length} |
|
|
655
711
|
| **Rows (total)** | ${n} |
|
|
656
712
|
| **Assets** | ${a} |
|
|
@@ -675,26 +731,25 @@ voltro data import <this-directory>
|
|
|
675
731
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
676
732
|
\`\`\`
|
|
677
733
|
`;
|
|
678
|
-
},
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
}), Vt = i.histogram("voltro_data_transfer_table_seconds", Bt, "Per-table export/import duration in seconds, tagged by phase."), G = (e, t) => {
|
|
734
|
+
}, Rt = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), zt = Array.from({ length: 16 }, (e, t) => .01 * 2 ** t), Bt = i.histogram("voltro_data_transfer_table_seconds", {
|
|
735
|
+
boundaries: zt,
|
|
736
|
+
description: "Per-table export/import duration in seconds, tagged by phase."
|
|
737
|
+
}), W = (e, t) => {
|
|
683
738
|
if (e) try {
|
|
684
739
|
e(t);
|
|
685
740
|
} catch {}
|
|
686
|
-
},
|
|
741
|
+
}, Vt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.andThen(Bt.pipe(i.withAttributes({ phase: e }), i.update((s - r) / 1e3)), Rt.pipe(i.withAttributes({ phase: e }), i.withAttributes({ table: t }), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
687
742
|
table: t,
|
|
688
743
|
index: r,
|
|
689
744
|
tableCount: a,
|
|
690
745
|
phase: e
|
|
691
|
-
} })),
|
|
746
|
+
} })), Ht = () => ({
|
|
692
747
|
tables: {},
|
|
693
748
|
assets: {}
|
|
694
|
-
}),
|
|
749
|
+
}), Ut = (e) => n.tryPromise(async () => {
|
|
695
750
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
696
751
|
return JSON.parse(t);
|
|
697
|
-
}).pipe(n.orElseSucceed(
|
|
752
|
+
}).pipe(n.orElseSucceed(Ht)), G = (e, t) => n.tryPromise({
|
|
698
753
|
try: async () => {
|
|
699
754
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
700
755
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
@@ -703,11 +758,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
703
758
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
704
759
|
path: e
|
|
705
760
|
})
|
|
706
|
-
}),
|
|
761
|
+
}), Wt = (e, t) => t in e.tables, Gt = (e, t) => t in e.assets, Kt = /* @__PURE__ */ new Set([
|
|
707
762
|
"json",
|
|
708
763
|
"array",
|
|
709
764
|
"vector"
|
|
710
|
-
]),
|
|
765
|
+
]), qt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", Jt = (e) => Buffer.from(e).toString("base64"), Yt = (e) => new Uint8Array(Buffer.from(e, "base64")), Xt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), Zt = (e, t) => e == null ? null : t !== void 0 && Kt.has(t) ? e : t === "bytes" ? Jt(e) : t === "timestamp" || t === "date" ? Xt(e) : typeof e == "bigint" ? {
|
|
711
766
|
$v: "bigint",
|
|
712
767
|
d: e.toString()
|
|
713
768
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -718,30 +773,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
718
773
|
d: e.toISOString()
|
|
719
774
|
} : e instanceof Uint8Array ? {
|
|
720
775
|
$v: "bytes",
|
|
721
|
-
d:
|
|
722
|
-
} : e,
|
|
776
|
+
d: Jt(e)
|
|
777
|
+
} : e, Qt = (e) => {
|
|
723
778
|
switch (e.$v) {
|
|
724
779
|
case "bigint": return BigInt(e.d);
|
|
725
780
|
case "number": return Number(e.d);
|
|
726
781
|
case "date": return new Date(e.d);
|
|
727
|
-
case "bytes": return
|
|
782
|
+
case "bytes": return Yt(e.d);
|
|
728
783
|
default: return e;
|
|
729
784
|
}
|
|
730
|
-
},
|
|
785
|
+
}, $t = (e, t) => e == null ? null : t !== void 0 && Kt.has(t) ? e : qt(e) ? Qt(e) : t === "bytes" ? Yt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, en = (e, t) => {
|
|
731
786
|
let n = {};
|
|
732
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
787
|
+
for (let [r, i] of Object.entries(e)) n[r] = Zt(i, t[r]);
|
|
733
788
|
return n;
|
|
734
|
-
},
|
|
789
|
+
}, tn = (e, t) => {
|
|
735
790
|
let n = {};
|
|
736
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
791
|
+
for (let [r, i] of Object.entries(e)) n[r] = $t(i, t[r]);
|
|
737
792
|
return n;
|
|
738
|
-
},
|
|
739
|
-
try: () =>
|
|
740
|
-
catch: (e) => new
|
|
793
|
+
}, nn = new TextEncoder(), rn = (e, t) => e.pipe(c.map((e) => nn.encode(`${JSON.stringify(en(e, t))}\n`))), an = (e, t, r) => e.pipe(c.decodeText(), c.splitLines, c.filter((e) => e.trim().length > 0), c.mapEffect((e) => n.try({
|
|
794
|
+
try: () => tn(JSON.parse(e), r),
|
|
795
|
+
catch: (e) => new Ye({
|
|
741
796
|
table: t,
|
|
742
797
|
reason: String(e?.message ?? e)
|
|
743
798
|
})
|
|
744
|
-
}))),
|
|
799
|
+
}))), on = (e, t) => {
|
|
745
800
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
746
801
|
for (let e of t.tables) {
|
|
747
802
|
if (!n.has(e.name)) continue;
|
|
@@ -761,48 +816,48 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
761
816
|
};
|
|
762
817
|
for (let t of e) n.has(t) && s(t);
|
|
763
818
|
return i;
|
|
764
|
-
},
|
|
819
|
+
}, K = (e) => {
|
|
765
820
|
let t = e, n = [];
|
|
766
821
|
typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
|
|
767
822
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
768
|
-
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(
|
|
823
|
+
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(K(t.cause)) : n.push(r), n.length === 0) {
|
|
769
824
|
let t = String(e ?? "").trim();
|
|
770
|
-
return t === "" || t === "[object Object]" ?
|
|
825
|
+
return t === "" || t === "[object Object]" ? sn(e) : t;
|
|
771
826
|
}
|
|
772
827
|
return n.join(": ");
|
|
773
|
-
},
|
|
828
|
+
}, sn = (e) => {
|
|
774
829
|
try {
|
|
775
830
|
let t = JSON.stringify(e);
|
|
776
831
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
777
832
|
} catch {
|
|
778
833
|
return "unknown error (not serialisable)";
|
|
779
834
|
}
|
|
780
|
-
},
|
|
835
|
+
}, cn = (e, t) => {
|
|
781
836
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
782
837
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
783
838
|
return r;
|
|
784
|
-
},
|
|
839
|
+
}, ln = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), un = (e, t) => {
|
|
785
840
|
if (t.length === 0) return e;
|
|
786
841
|
let n = { ...e };
|
|
787
842
|
for (let e of t) delete n[e];
|
|
788
843
|
return n;
|
|
789
|
-
},
|
|
844
|
+
}, dn = (e, t) => {
|
|
790
845
|
let n = e.tables.find((e) => e.name === t);
|
|
791
846
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
792
847
|
let r = n.columns.find((e) => e.type === "id");
|
|
793
848
|
if (r !== void 0) return { column: r.name };
|
|
794
849
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
795
850
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
796
|
-
},
|
|
851
|
+
}, fn = (e, t, r) => n.tryPromise({
|
|
797
852
|
try: async () => {
|
|
798
|
-
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable:
|
|
853
|
+
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: l, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
799
854
|
d.update(e);
|
|
800
855
|
let r = e;
|
|
801
856
|
for (let e = 0; e < r.length; e++) r[e] === 10 && f++;
|
|
802
857
|
n(null, e);
|
|
803
|
-
} }), m =
|
|
858
|
+
} }), m = l.fromWeb(c.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
804
859
|
try {
|
|
805
|
-
await s(m, p,
|
|
860
|
+
await s(m, p, xt(r), n(h)), await i.rename(h, t);
|
|
806
861
|
} catch (e) {
|
|
807
862
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
808
863
|
}
|
|
@@ -812,26 +867,26 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
812
867
|
};
|
|
813
868
|
},
|
|
814
869
|
catch: (e) => new H({
|
|
815
|
-
reason: `write table failed: ${
|
|
870
|
+
reason: `write table failed: ${K(e)}`,
|
|
816
871
|
path: t
|
|
817
872
|
})
|
|
818
|
-
}),
|
|
873
|
+
}), pn = (e, t) => n.tryPromise({
|
|
819
874
|
try: async () => {
|
|
820
875
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
821
876
|
},
|
|
822
877
|
catch: (t) => new H({
|
|
823
|
-
reason: `append failed: ${
|
|
878
|
+
reason: `append failed: ${K(t)}`,
|
|
824
879
|
path: e
|
|
825
880
|
})
|
|
826
|
-
}),
|
|
881
|
+
}), mn = (e, t) => n.gen(function* () {
|
|
827
882
|
let r = [], i = t.ordered.length;
|
|
828
883
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
829
|
-
let o = t.ordered[a], s =
|
|
830
|
-
if ("refusal" in
|
|
831
|
-
let u =
|
|
832
|
-
if (
|
|
884
|
+
let o = t.ordered[a], s = cn(t.snapshot, o), l = dn(t.snapshot, o);
|
|
885
|
+
if ("refusal" in l) return yield* n.fail(new Qe({ reason: `export: ${l.refusal}` }));
|
|
886
|
+
let u = l.column, d = `${Mt}/${o}.ndjson${bt(t.compression)}`;
|
|
887
|
+
if (Wt(t.ledger, o)) {
|
|
833
888
|
let e = t.ledger.tables[o];
|
|
834
|
-
|
|
889
|
+
W(t.onProgress, {
|
|
835
890
|
phase: "export",
|
|
836
891
|
table: o,
|
|
837
892
|
index: a,
|
|
@@ -849,7 +904,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
849
904
|
});
|
|
850
905
|
continue;
|
|
851
906
|
}
|
|
852
|
-
|
|
907
|
+
W(t.onProgress, {
|
|
853
908
|
phase: "export",
|
|
854
909
|
table: o,
|
|
855
910
|
index: a,
|
|
@@ -863,11 +918,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
863
918
|
chunkSize: t.chunkSize,
|
|
864
919
|
retry: t.retry,
|
|
865
920
|
...f === void 0 ? {} : { where: f }
|
|
866
|
-
}), m =
|
|
921
|
+
}), m = ln(t.snapshot, o), h = m.length === 0 ? p : p.pipe(c.map((e) => un(e, m))), g = t.mask ? h.pipe(c.map((e) => We(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* Vt("export", o, a, i, (e) => e.rowCount, fn(rn(g, s), t.bundleFile(d), t.compression));
|
|
867
922
|
t.ledger.tables[o] = {
|
|
868
923
|
rowCount: _,
|
|
869
924
|
checksum: v
|
|
870
|
-
}, yield*
|
|
925
|
+
}, yield* G(t.ledgerPath, t.ledger), W(t.onProgress, {
|
|
871
926
|
phase: "export",
|
|
872
927
|
table: o,
|
|
873
928
|
index: a,
|
|
@@ -885,36 +940,36 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
885
940
|
});
|
|
886
941
|
}
|
|
887
942
|
return r;
|
|
888
|
-
}),
|
|
943
|
+
}), hn = (e) => ({
|
|
889
944
|
_tag: "RawSqlFragment",
|
|
890
945
|
strings: [e],
|
|
891
946
|
values: []
|
|
892
|
-
}),
|
|
947
|
+
}), gn = (t, i, a) => n.tryPromise({
|
|
893
948
|
try: () => t.transactional(async (t) => {
|
|
894
|
-
if (t.raw) for (let e of
|
|
949
|
+
if (t.raw) for (let e of Te(i)) await t.raw(hn(e));
|
|
895
950
|
let o = await n.runPromiseExit(a(t));
|
|
896
951
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
897
952
|
return o.value;
|
|
898
953
|
}),
|
|
899
|
-
catch: (e) => e instanceof H ? e : new H({ reason: `snapshot export failed: ${
|
|
900
|
-
}),
|
|
901
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
954
|
+
catch: (e) => e instanceof H ? e : new H({ reason: `snapshot export failed: ${K(e)}` })
|
|
955
|
+
}), _n = (e) => n.gen(function* () {
|
|
956
|
+
let t = e.scope ?? { kind: "all" }, r = yt(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, s = o.max([o.exponential("50 millis").pipe(o.jittered), o.recurs(a)]), c = yield* n.promise(() => import("node:path")), l = (t) => c.join(e.outDir, t);
|
|
902
957
|
yield* n.tryPromise({
|
|
903
958
|
try: async () => {
|
|
904
959
|
let t = await import("node:fs/promises");
|
|
905
|
-
await t.mkdir(l(
|
|
960
|
+
await t.mkdir(l(Mt), { recursive: !0 }), e.assets && await t.mkdir(l(Nt), { recursive: !0 });
|
|
906
961
|
},
|
|
907
962
|
catch: (t) => new H({
|
|
908
|
-
reason: `mkdir failed: ${
|
|
963
|
+
reason: `mkdir failed: ${K(t)}`,
|
|
909
964
|
path: e.outDir
|
|
910
965
|
})
|
|
911
966
|
});
|
|
912
|
-
let u = l(
|
|
967
|
+
let u = l(Ft), d = yield* Ut(u), f = e.subset ? ot(yield* at(e.store, e.snapshot, e.subset), e.snapshot) : yield* ct(t, e.snapshot, {
|
|
913
968
|
store: e.store,
|
|
914
969
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
915
|
-
}), p =
|
|
970
|
+
}), p = on(f.tables, e.snapshot), m = e.consistency ?? "live", h, g;
|
|
916
971
|
if (e.masking) {
|
|
917
|
-
let t = e.classification ??
|
|
972
|
+
let t = e.classification ?? De(e.snapshot), r = Ve(p.map((t) => ({
|
|
918
973
|
name: t,
|
|
919
974
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
920
975
|
name: e.name,
|
|
@@ -941,20 +996,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
941
996
|
resolved: f,
|
|
942
997
|
compression: r,
|
|
943
998
|
chunkSize: i,
|
|
944
|
-
retry:
|
|
999
|
+
retry: s,
|
|
945
1000
|
ledger: d,
|
|
946
1001
|
ledgerPath: u,
|
|
947
1002
|
bundleFile: l,
|
|
948
1003
|
...g ? { mask: g } : {},
|
|
949
1004
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
950
|
-
}, v = m === "snapshot" ? yield*
|
|
1005
|
+
}, v = m === "snapshot" ? yield* gn(e.store, e.dialect, (e) => mn(e, _)) : yield* mn(e.store, _), y = 0, b = 0;
|
|
951
1006
|
if (e.assets) {
|
|
952
|
-
let t = l(
|
|
1007
|
+
let t = l(Pt), r = yield* ht({
|
|
953
1008
|
source: e.assets,
|
|
954
|
-
assetsDir: l(
|
|
1009
|
+
assetsDir: l(Nt),
|
|
955
1010
|
progress: {
|
|
956
1011
|
done: (e) => e in d.assets,
|
|
957
|
-
mark: (e, r) => (d.assets[e] = !0,
|
|
1012
|
+
mark: (e, r) => (d.assets[e] = !0, pn(t, JSON.stringify(r)).pipe(n.andThen(G(u, d))))
|
|
958
1013
|
}
|
|
959
1014
|
});
|
|
960
1015
|
y = r.count, b = r.totalBytes, r.missing.length > 0 && (yield* n.logWarning(`${r.missing.length} of ${r.missing.length + r.count} blob reference(s) point at objects the storage provider does not have; they are NOT in this bundle and no import can bring them back. e.g. ${r.missing.slice(0, 3).map((e) => `'${e}'`).join(", ")}` + (r.missing.length > 3 ? `, and ${r.missing.length - 3} more` : "")));
|
|
@@ -964,7 +1019,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
964
1019
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
965
1020
|
source: {
|
|
966
1021
|
dialect: e.dialect,
|
|
967
|
-
schemaFingerprint:
|
|
1022
|
+
schemaFingerprint: ie(e.snapshot)
|
|
968
1023
|
},
|
|
969
1024
|
scope: t,
|
|
970
1025
|
consistency: m,
|
|
@@ -973,7 +1028,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
973
1028
|
assets: y > 0 ? {
|
|
974
1029
|
count: y,
|
|
975
1030
|
totalBytes: b,
|
|
976
|
-
index:
|
|
1031
|
+
index: Pt
|
|
977
1032
|
} : {
|
|
978
1033
|
count: 0,
|
|
979
1034
|
totalBytes: 0
|
|
@@ -982,23 +1037,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
982
1037
|
};
|
|
983
1038
|
return yield* n.tryPromise({
|
|
984
1039
|
try: async () => {
|
|
985
|
-
await (await import("node:fs/promises")).writeFile(l(
|
|
1040
|
+
await (await import("node:fs/promises")).writeFile(l(At), Ot(x));
|
|
986
1041
|
},
|
|
987
1042
|
catch: (e) => new H({
|
|
988
|
-
reason: `manifest write failed: ${
|
|
989
|
-
path:
|
|
1043
|
+
reason: `manifest write failed: ${K(e)}`,
|
|
1044
|
+
path: At
|
|
990
1045
|
})
|
|
991
1046
|
}), yield* n.tryPromise({
|
|
992
1047
|
try: async () => {
|
|
993
1048
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
994
|
-
await e.writeFile(l(
|
|
1049
|
+
await e.writeFile(l(jt), Lt(x, { hostname: t.hostname() }));
|
|
995
1050
|
},
|
|
996
1051
|
catch: (e) => new H({
|
|
997
|
-
reason: `readme write failed: ${
|
|
998
|
-
path:
|
|
1052
|
+
reason: `readme write failed: ${K(e)}`,
|
|
1053
|
+
path: jt
|
|
999
1054
|
})
|
|
1000
1055
|
}), x;
|
|
1001
|
-
}),
|
|
1056
|
+
}), vn = (e, t) => {
|
|
1002
1057
|
let n = new Set(t), r = [];
|
|
1003
1058
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
1004
1059
|
let i = e.references?.table;
|
|
@@ -1009,7 +1064,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1009
1064
|
});
|
|
1010
1065
|
}
|
|
1011
1066
|
return r;
|
|
1012
|
-
},
|
|
1067
|
+
}, yn = (e) => {
|
|
1013
1068
|
let t = /* @__PURE__ */ new Map();
|
|
1014
1069
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
1015
1070
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -1017,7 +1072,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1017
1072
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
1018
1073
|
});
|
|
1019
1074
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
1020
|
-
},
|
|
1075
|
+
}, bn = (e) => {
|
|
1021
1076
|
switch (e) {
|
|
1022
1077
|
case "id":
|
|
1023
1078
|
case "text":
|
|
@@ -1036,12 +1091,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1036
1091
|
case "raw": return "raw";
|
|
1037
1092
|
default: return "string";
|
|
1038
1093
|
}
|
|
1039
|
-
},
|
|
1094
|
+
}, xn = {
|
|
1040
1095
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
1041
1096
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
1042
1097
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
1043
|
-
},
|
|
1044
|
-
let i =
|
|
1098
|
+
}, Sn = (e, t, n, r) => {
|
|
1099
|
+
let i = bn(n);
|
|
1045
1100
|
if (i === "raw") return {
|
|
1046
1101
|
table: e,
|
|
1047
1102
|
column: t,
|
|
@@ -1049,7 +1104,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1049
1104
|
severity: "block",
|
|
1050
1105
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
1051
1106
|
};
|
|
1052
|
-
let a =
|
|
1107
|
+
let a = xn[i];
|
|
1053
1108
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
1054
1109
|
table: e,
|
|
1055
1110
|
column: t,
|
|
@@ -1063,15 +1118,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1063
1118
|
severity: "block",
|
|
1064
1119
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
1065
1120
|
};
|
|
1066
|
-
},
|
|
1121
|
+
}, Cn = (e, t, n) => {
|
|
1067
1122
|
if (t === n) return [];
|
|
1068
1123
|
let r = [];
|
|
1069
1124
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
1070
|
-
let a =
|
|
1125
|
+
let a = Sn(t.name, e, i, n);
|
|
1071
1126
|
a && r.push(a);
|
|
1072
1127
|
}
|
|
1073
1128
|
return r;
|
|
1074
|
-
},
|
|
1129
|
+
}, wn = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, Tn = (e) => {
|
|
1075
1130
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
1076
1131
|
switch (e.kind) {
|
|
1077
1132
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -1080,36 +1135,35 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1080
1135
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
1081
1136
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
1082
1137
|
}
|
|
1083
|
-
},
|
|
1138
|
+
}, En = (e) => {
|
|
1084
1139
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
1085
1140
|
return t.length > 0 ? t.join("/") : void 0;
|
|
1086
|
-
},
|
|
1141
|
+
}, Dn = "Failed to execute statement", On = (e) => {
|
|
1087
1142
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
}, An = (e) => {
|
|
1143
|
+
return t === `SqlError: ${Dn}` ? Dn : t;
|
|
1144
|
+
}, kn = (e) => {
|
|
1091
1145
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
1092
1146
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
1093
1147
|
unclassified: t,
|
|
1094
1148
|
raw: n
|
|
1095
|
-
}), i =
|
|
1096
|
-
if (a !== void 0) return r(`${
|
|
1097
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u =
|
|
1098
|
-
return r(d ===
|
|
1099
|
-
},
|
|
1100
|
-
let t =
|
|
1149
|
+
}), i = _(e), a = p(i), o = En(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
1150
|
+
if (a !== void 0) return r(`${Tn(a)}${s}`, !1);
|
|
1151
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = On(typeof l == "string" && l.length > 0 ? l : String(e)), d = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, f = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, m = f !== void 0 && /[←‖]/.test(f) ? ` (via ${f})` : "";
|
|
1152
|
+
return r(d === Dn || d.length === 0 ? `the database refused the write and the driver gave no detail${s}${m}` : `${d}${s}`, d === Dn || d.length === 0);
|
|
1153
|
+
}, An = (e) => kn(e).reason, jn = (e) => {
|
|
1154
|
+
let t = _(e), n = p(t);
|
|
1101
1155
|
if (n?.kind === "foreignKeyInUse") {
|
|
1102
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
1156
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = En(t);
|
|
1103
1157
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
1104
1158
|
}
|
|
1105
|
-
return
|
|
1106
|
-
},
|
|
1159
|
+
return An(e);
|
|
1160
|
+
}, Mn = d, q = (e) => `${Mn}${e}`, Nn = (e) => typeof e.run == "function", Pn = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
1107
1161
|
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
1108
|
-
written before the first delete is what covers that case.`,
|
|
1109
|
-
if (!
|
|
1162
|
+
written before the first delete is what covers that case.`, Fn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, J = (e, t) => {
|
|
1163
|
+
if (!Fn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
1110
1164
|
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
1111
|
-
},
|
|
1112
|
-
let n =
|
|
1165
|
+
}, In = (e, t) => {
|
|
1166
|
+
let n = J(e, t), r = J(q(e), t);
|
|
1113
1167
|
switch (t) {
|
|
1114
1168
|
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
1115
1169
|
case "mysql":
|
|
@@ -1117,20 +1171,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1117
1171
|
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
1118
1172
|
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
1119
1173
|
}
|
|
1174
|
+
}, Ln = (e, t) => {
|
|
1175
|
+
let n = J(q(e), t);
|
|
1176
|
+
return t === "mssql" ? `IF OBJECT_ID('${q(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
1120
1177
|
}, Rn = (e, t) => {
|
|
1121
|
-
let n = Y(J(e), t);
|
|
1122
|
-
return t === "mssql" ? `IF OBJECT_ID('${J(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
1123
|
-
}, zn = (e, t) => {
|
|
1124
1178
|
let n = t === "mysql" || t === "mariadb", r = [];
|
|
1125
1179
|
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
1126
|
-
for (let n of [...e].reverse()) r.push(`DELETE FROM ${
|
|
1180
|
+
for (let n of [...e].reverse()) r.push(`DELETE FROM ${J(n.name, t)}`);
|
|
1127
1181
|
for (let n of e) {
|
|
1128
1182
|
if (n.columns.length === 0) throw Error(`refusing to build a swap for '${n.name}' with no columns: an empty list would have to mean \`SELECT *\`, which is the shape that fails on a generated column.`);
|
|
1129
|
-
let e = n.columns.map((e) =>
|
|
1130
|
-
r.push(`INSERT INTO ${
|
|
1183
|
+
let e = n.columns.map((e) => J(e, t)).join(", ");
|
|
1184
|
+
r.push(`INSERT INTO ${J(n.name, t)} (${e}) SELECT ${e} FROM ${J(q(n.name), t)}`);
|
|
1131
1185
|
}
|
|
1132
1186
|
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
1133
|
-
},
|
|
1187
|
+
}, zn = (e, t) => {
|
|
1134
1188
|
let n = [], r = () => {
|
|
1135
1189
|
for (let e of n) t.unregisterTable(e);
|
|
1136
1190
|
};
|
|
@@ -1138,7 +1192,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1138
1192
|
for (let r of e) {
|
|
1139
1193
|
let e = t.getTable(r);
|
|
1140
1194
|
if (e === void 0) continue;
|
|
1141
|
-
let i =
|
|
1195
|
+
let i = q(r);
|
|
1142
1196
|
t.registerTable({
|
|
1143
1197
|
...e,
|
|
1144
1198
|
tableName: i,
|
|
@@ -1152,7 +1206,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1152
1206
|
staged: n,
|
|
1153
1207
|
undo: r
|
|
1154
1208
|
};
|
|
1155
|
-
},
|
|
1209
|
+
}, Bn = (e, t) => {
|
|
1156
1210
|
let n = new Set(t), r = [];
|
|
1157
1211
|
for (let t of e.tables) {
|
|
1158
1212
|
if (!n.has(t.name)) continue;
|
|
@@ -1169,22 +1223,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1169
1223
|
}
|
|
1170
1224
|
}
|
|
1171
1225
|
return r;
|
|
1172
|
-
},
|
|
1173
|
-
let n =
|
|
1174
|
-
return `SELECT ${
|
|
1175
|
-
},
|
|
1226
|
+
}, Vn = (e, t) => {
|
|
1227
|
+
let n = J(q(e.from), t), r = J(q(e.to), t), i = J(e.column, t), a = J(e.toColumn, t);
|
|
1228
|
+
return `SELECT ${J(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
1229
|
+
}, Hn = (e) => {
|
|
1176
1230
|
if (e.length === 0) return;
|
|
1177
1231
|
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
1178
1232
|
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
1179
|
-
},
|
|
1180
|
-
t.length !== 0 && await e.run(
|
|
1233
|
+
}, Un = async (e, t) => {
|
|
1234
|
+
t.length !== 0 && await e.run(ve.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
1235
|
+
}, Wn = async (e, t, r) => {
|
|
1236
|
+
let i = Rn(t, r);
|
|
1237
|
+
i.length !== 0 && await e.run(ve.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
1181
1238
|
}, Gn = async (e, t, r) => {
|
|
1182
|
-
let i = zn(t, r);
|
|
1183
|
-
i.length !== 0 && await e.run(N.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
1184
|
-
}, Kn = async (e, t, r) => {
|
|
1185
1239
|
let i = [];
|
|
1186
1240
|
for (let a of t) {
|
|
1187
|
-
let t = await e.run(
|
|
1241
|
+
let t = await e.run(ve.SqlClient.pipe(n.flatMap((e) => e.unsafe(Vn(a, r)))));
|
|
1188
1242
|
for (let e of t) i.push({
|
|
1189
1243
|
table: a.from,
|
|
1190
1244
|
primaryKey: String(e.pk ?? ""),
|
|
@@ -1194,14 +1248,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1194
1248
|
});
|
|
1195
1249
|
}
|
|
1196
1250
|
return i;
|
|
1197
|
-
},
|
|
1251
|
+
}, Kn = (e, t) => {
|
|
1198
1252
|
let n = new Map(t.map((e, t) => [e, t]));
|
|
1199
1253
|
return e.filter((e) => {
|
|
1200
1254
|
if (e.from === e.to) return !1;
|
|
1201
1255
|
let t = n.get(e.from), r = n.get(e.to);
|
|
1202
1256
|
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1203
1257
|
});
|
|
1204
|
-
},
|
|
1258
|
+
}, qn = (e) => e === "mysql" || e === "mariadb", Jn = (e) => {
|
|
1205
1259
|
if (e.mode !== "replace") return {
|
|
1206
1260
|
active: !1,
|
|
1207
1261
|
tables: [],
|
|
@@ -1251,7 +1305,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1251
1305
|
tables: [],
|
|
1252
1306
|
blocker: "every table in the bundle is one that describes a deployment, so there is nothing for a staged swap to move."
|
|
1253
1307
|
};
|
|
1254
|
-
let s = o.map((e) => e.name), c =
|
|
1308
|
+
let s = o.map((e) => e.name), c = qn(t) ? [] : Kn(Bn(n, s), s);
|
|
1255
1309
|
return c.length > 0 ? {
|
|
1256
1310
|
active: !1,
|
|
1257
1311
|
tables: [],
|
|
@@ -1260,7 +1314,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1260
1314
|
active: !0,
|
|
1261
1315
|
tables: o
|
|
1262
1316
|
};
|
|
1263
|
-
},
|
|
1317
|
+
}, Yn = (e) => n.gen(function* () {
|
|
1264
1318
|
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1265
1319
|
if (e.snapshot === void 0) return yield* n.fail(new H({
|
|
1266
1320
|
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
@@ -1273,7 +1327,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1273
1327
|
if (t.tables.length === 0) return;
|
|
1274
1328
|
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1275
1329
|
so it survives a run whose process does not.`);
|
|
1276
|
-
let r = yield*
|
|
1330
|
+
let r = yield* _n({
|
|
1277
1331
|
store: e.store,
|
|
1278
1332
|
snapshot: t,
|
|
1279
1333
|
dialect: e.dialect ?? "unknown",
|
|
@@ -1289,41 +1343,46 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1289
1343
|
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1290
1344
|
tables: r.tables.length
|
|
1291
1345
|
};
|
|
1292
|
-
}),
|
|
1293
|
-
let n = [];
|
|
1294
|
-
for (let
|
|
1295
|
-
|
|
1296
|
-
|
|
1346
|
+
}), Xn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, Zn = /__dropped_\d+$/, Qn = (e, t) => {
|
|
1347
|
+
let n = [], r = /* @__PURE__ */ new Map();
|
|
1348
|
+
for (let e of t.tables) {
|
|
1349
|
+
let t = e.name;
|
|
1350
|
+
r.has(t) || r.set(t, e);
|
|
1351
|
+
}
|
|
1352
|
+
for (let t of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
1353
|
+
let e = r.get(t.name);
|
|
1354
|
+
if (e === void 0) {
|
|
1355
|
+
n.push(`table '${t.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
1297
1356
|
continue;
|
|
1298
1357
|
}
|
|
1299
|
-
let
|
|
1300
|
-
for (let [
|
|
1301
|
-
let
|
|
1302
|
-
if (
|
|
1303
|
-
n.push(
|
|
1358
|
+
let i = new Map(e.columns.map((e) => [e.name, e])), a = new Map(t.columns.map((e) => [e.name, e]));
|
|
1359
|
+
for (let [e, r] of a) {
|
|
1360
|
+
let a = i.get(e);
|
|
1361
|
+
if (a === void 0) {
|
|
1362
|
+
n.push(Zn.test(e) ? `${t.name}.${e}: a soft-dropped column, present only in the SOURCE — reclaim it there (voltro db gc-snapshots --before <date>) or re-export without it; the target is not missing anything` : `${t.name}.${e}: in the bundle, MISSING from the target — the value has nowhere to go`);
|
|
1304
1363
|
continue;
|
|
1305
1364
|
}
|
|
1306
|
-
|
|
1365
|
+
a.type !== r.type && n.push(`${t.name}.${e}: type '${r.type}' (bundle) vs '${a.type}' (target)`);
|
|
1307
1366
|
}
|
|
1308
|
-
for (let [
|
|
1367
|
+
for (let [e, r] of i) a.has(e) || !r.nullable && !r.hasDefault && n.push(`${t.name}.${e}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
1309
1368
|
}
|
|
1310
1369
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
1311
|
-
},
|
|
1312
|
-
let r =
|
|
1370
|
+
}, $n = (e, t, n) => {
|
|
1371
|
+
let r = ie(t);
|
|
1313
1372
|
if (r === e || n === void 0) return {
|
|
1314
1373
|
drifted: !1,
|
|
1315
1374
|
bundleFingerprint: e,
|
|
1316
1375
|
targetFingerprint: r,
|
|
1317
1376
|
diff: []
|
|
1318
1377
|
};
|
|
1319
|
-
let i =
|
|
1378
|
+
let i = Qn(n, t);
|
|
1320
1379
|
return {
|
|
1321
1380
|
drifted: i.length > 0,
|
|
1322
1381
|
bundleFingerprint: e,
|
|
1323
1382
|
targetFingerprint: r,
|
|
1324
1383
|
diff: i
|
|
1325
1384
|
};
|
|
1326
|
-
},
|
|
1385
|
+
}, er = 2e3, tr = 2e3, nr = 200, rr = /* @__PURE__ */ new Set([
|
|
1327
1386
|
"BundleError",
|
|
1328
1387
|
"CodecError",
|
|
1329
1388
|
"IntegrityError",
|
|
@@ -1332,29 +1391,29 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1332
1391
|
"RowsRefusedError",
|
|
1333
1392
|
"SchemaDriftError",
|
|
1334
1393
|
"StorageError"
|
|
1335
|
-
]),
|
|
1336
|
-
let n =
|
|
1337
|
-
for (let
|
|
1338
|
-
let
|
|
1339
|
-
if (typeof
|
|
1340
|
-
if (
|
|
1341
|
-
let
|
|
1342
|
-
if (
|
|
1394
|
+
]), ir = (t) => {
|
|
1395
|
+
let n = e.isCause(t) ? t : r.isExit(t) && r.isFailure(t) ? t.cause : void 0, i = n === void 0 ? [t] : n.reasons.flatMap((t) => e.isFailReason(t) ? [t.error] : e.isDieReason(t) ? [t.defect] : []);
|
|
1396
|
+
for (let n of i) {
|
|
1397
|
+
let i = n?._tag;
|
|
1398
|
+
if (typeof i == "string" && rr.has(i)) return n;
|
|
1399
|
+
if (n !== t && (e.isCause(n) || r.isExit(n))) {
|
|
1400
|
+
let e = ir(n);
|
|
1401
|
+
if (e) return e;
|
|
1343
1402
|
}
|
|
1344
1403
|
}
|
|
1345
|
-
},
|
|
1404
|
+
}, ar = (t) => a.getOrUndefined(e.findErrorOption(t)) ?? Error(e.pretty(t)), or = (e) => n.tryPromise({
|
|
1346
1405
|
try: async () => {
|
|
1347
1406
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
1348
|
-
return
|
|
1407
|
+
return kt(await t.readFile(n.join(e, At), "utf8"));
|
|
1349
1408
|
},
|
|
1350
1409
|
catch: (e) => new H({
|
|
1351
1410
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
1352
|
-
path:
|
|
1411
|
+
path: At
|
|
1353
1412
|
})
|
|
1354
|
-
}),
|
|
1413
|
+
}), sr = (e) => {
|
|
1355
1414
|
let t;
|
|
1356
1415
|
return n.gen(function* () {
|
|
1357
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
1416
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* or(e.bundleDir);
|
|
1358
1417
|
if (a.formatVersion !== 1) return yield* n.fail(new H({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
1359
1418
|
if (e.tables !== void 0) {
|
|
1360
1419
|
let t = new Set(a.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
@@ -1375,7 +1434,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1375
1434
|
unique: !1,
|
|
1376
1435
|
hasDefault: !1
|
|
1377
1436
|
}))
|
|
1378
|
-
})) }, r =
|
|
1437
|
+
})) }, r = $n(s.source.schemaFingerprint, e.targetSnapshot, t);
|
|
1379
1438
|
if (r.drifted) {
|
|
1380
1439
|
if (!e.force) return yield* n.fail(new et({
|
|
1381
1440
|
bundleFingerprint: r.bundleFingerprint,
|
|
@@ -1386,28 +1445,28 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1386
1445
|
}
|
|
1387
1446
|
}
|
|
1388
1447
|
if (e.targetDialect && e.targetDialect !== s.source.dialect) {
|
|
1389
|
-
let t =
|
|
1448
|
+
let t = Cn(s.tables, s.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1390
1449
|
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new rt({
|
|
1391
1450
|
sourceDialect: s.source.dialect,
|
|
1392
1451
|
targetDialect: e.targetDialect,
|
|
1393
|
-
blocking: t.map(
|
|
1452
|
+
blocking: t.map(wn)
|
|
1394
1453
|
}));
|
|
1395
1454
|
}
|
|
1396
|
-
let
|
|
1397
|
-
if (
|
|
1398
|
-
mode:
|
|
1455
|
+
let d = e.verify ?? !0, p = e.mode ?? "upsert", g = e.atomic ?? p === "replace", _ = e.onConflict ?? "skip", y = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), x = yield* Ut(y);
|
|
1456
|
+
if (p === "replace" && s.scope.kind !== "all") return yield* n.fail(new nt({
|
|
1457
|
+
mode: p,
|
|
1399
1458
|
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope. To leave specific tables out of a FULL export, use \`--exclude a,b\`: it keeps the scope \`all\` and records what it left out.`
|
|
1400
1459
|
}));
|
|
1401
|
-
if (
|
|
1460
|
+
if (p === "replace" && (s.scope.exclude?.length ?? 0) > 0) {
|
|
1402
1461
|
let t = s.scope.exclude;
|
|
1403
1462
|
e.onWarn?.(`this bundle was exported with ${t.length} table(s) excluded: ${t.join(", ")}.\n A replace empties and refills the tables the bundle CARRIES, so those keep whatever rows the
|
|
1404
1463
|
target already holds. That is not "everything the bundle says" — check it is what you meant.`);
|
|
1405
1464
|
}
|
|
1406
|
-
if (
|
|
1465
|
+
if (p === "replace") {
|
|
1407
1466
|
let t = s.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1408
1467
|
try: () => e.store.incomingForeignKeys(t),
|
|
1409
|
-
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${
|
|
1410
|
-
}) : e.targetSnapshot === void 0 ? [] :
|
|
1468
|
+
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${jn(e)}` })
|
|
1469
|
+
}) : e.targetSnapshot === void 0 ? [] : vn(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1411
1470
|
for (let t of r) {
|
|
1412
1471
|
if (!a.has(t.from)) {
|
|
1413
1472
|
let r = yield* n.tryPromise({
|
|
@@ -1430,11 +1489,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1430
1489
|
});
|
|
1431
1490
|
}
|
|
1432
1491
|
if (i.length > 0) return yield* n.fail(new nt({
|
|
1433
|
-
mode:
|
|
1434
|
-
reason:
|
|
1492
|
+
mode: p,
|
|
1493
|
+
reason: yn(i)
|
|
1435
1494
|
}));
|
|
1436
1495
|
}
|
|
1437
|
-
if (e.dryRun === !0) return e.traceId !== void 0 && (yield* n.promise(() => e.store.update(
|
|
1496
|
+
if (e.dryRun === !0) return e.traceId !== void 0 && (yield* n.promise(() => e.store.update(l, e.traceId, {
|
|
1438
1497
|
finishedAt: /* @__PURE__ */ new Date(),
|
|
1439
1498
|
tablesWritten: 0,
|
|
1440
1499
|
rowsWritten: 0,
|
|
@@ -1447,34 +1506,34 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1447
1506
|
rowsSkipped: 0,
|
|
1448
1507
|
fullyResumed: !1
|
|
1449
1508
|
};
|
|
1450
|
-
let
|
|
1509
|
+
let C = /* @__PURE__ */ new Map();
|
|
1451
1510
|
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1452
1511
|
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1453
|
-
e.length > 0 &&
|
|
1512
|
+
e.length > 0 && C.set(t.name, e);
|
|
1454
1513
|
}
|
|
1455
|
-
let
|
|
1456
|
-
let n =
|
|
1514
|
+
let ie = (e, t) => {
|
|
1515
|
+
let n = C.get(e);
|
|
1457
1516
|
if (n === void 0) return t;
|
|
1458
1517
|
let r;
|
|
1459
1518
|
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1460
1519
|
return r ?? t;
|
|
1461
|
-
},
|
|
1462
|
-
let i =
|
|
1463
|
-
return
|
|
1464
|
-
},
|
|
1465
|
-
if (!
|
|
1520
|
+
}, ae = (e, t, n, r = !1) => {
|
|
1521
|
+
let i = A ? q(t.name) : t.name, a = t.primaryKey, o = w(t.name, ie(t.name, n), t.columnTypes);
|
|
1522
|
+
return p === "append" ? _ === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : p === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1523
|
+
}, oe = e.targetDialect !== void 0 && e.targetDialect !== "memory", w = (e, t, n) => {
|
|
1524
|
+
if (!oe || n === void 0 || v(e) !== void 0) return t;
|
|
1466
1525
|
let r = null;
|
|
1467
1526
|
for (let [e, i] of Object.entries(n)) {
|
|
1468
1527
|
let n = t[e];
|
|
1469
1528
|
i === "json" && n != null && (r === null && (r = { ...t }), r[e] = JSON.stringify(n));
|
|
1470
1529
|
}
|
|
1471
1530
|
return r ?? t;
|
|
1472
|
-
}, T = (e, t) => e.savepoint ? e.savepoint(t) : t(),
|
|
1473
|
-
r.unclassified && (
|
|
1474
|
-
},
|
|
1531
|
+
}, T = (e, t) => e.savepoint ? e.savepoint(t) : t(), E = 0, se = (t, n, r) => {
|
|
1532
|
+
r.unclassified && (E++, !(E > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (E === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1533
|
+
}, ce = (e, t, r = !0) => n.tryPromise({
|
|
1475
1534
|
try: e,
|
|
1476
|
-
catch: (e) =>
|
|
1477
|
-
}).pipe(n.map(() => !0), n.
|
|
1535
|
+
catch: (e) => kn(e)
|
|
1536
|
+
}).pipe(n.map(() => !0), n.catch((e) => n.sync(() => (r && (t.reason = e.reason, se(t.entry, t.row, e)), !1)))), le = (e) => {
|
|
1478
1537
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1479
1538
|
if (t.length !== 0) return {
|
|
1480
1539
|
...e.row,
|
|
@@ -1508,36 +1567,36 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1508
1567
|
primary: t.primary
|
|
1509
1568
|
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1510
1569
|
});
|
|
1511
|
-
},
|
|
1570
|
+
}, fe = (e, t) => n.gen(function* () {
|
|
1512
1571
|
let r = t, i = [], a = !0;
|
|
1513
1572
|
for (; r.length > 0 && a;) {
|
|
1514
1573
|
a = !1;
|
|
1515
1574
|
let t = [];
|
|
1516
|
-
for (let n of r) (yield*
|
|
1575
|
+
for (let n of r) (yield* ce(() => T(e, () => ae(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1517
1576
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1518
|
-
let r = t[n], o =
|
|
1519
|
-
o !== void 0 && (yield*
|
|
1577
|
+
let r = t[n], o = le(r);
|
|
1578
|
+
o !== void 0 && (yield* ce(() => T(e, () => ae(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1520
1579
|
}
|
|
1521
1580
|
r = t;
|
|
1522
1581
|
}
|
|
1523
|
-
for (let t of i) (yield*
|
|
1582
|
+
for (let t of i) (yield* ce(() => T(e, () => e.upsert(t.entry.name, w(t.entry.name, t.row, t.entry.columnTypes), { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1524
1583
|
if (r.length > 0) return yield* n.fail(de(r));
|
|
1525
|
-
}),
|
|
1584
|
+
}), D = (() => {
|
|
1526
1585
|
let t = e.targetDialect;
|
|
1527
1586
|
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1528
|
-
})(),
|
|
1529
|
-
mode:
|
|
1587
|
+
})(), O = Jn({
|
|
1588
|
+
mode: p,
|
|
1530
1589
|
...e.targetDialect === void 0 ? {} : { targetDialect: e.targetDialect },
|
|
1531
1590
|
...e.targetSnapshot === void 0 ? {} : { targetSnapshot: e.targetSnapshot },
|
|
1532
1591
|
bundleTables: s.tables.map((e) => e.name),
|
|
1533
1592
|
isSkipped: V,
|
|
1534
|
-
canRunSql:
|
|
1535
|
-
}),
|
|
1536
|
-
if (
|
|
1593
|
+
canRunSql: Nn(e.store)
|
|
1594
|
+
}), k = O.tables, pe = s.tables.filter((e) => !V(e.name) && S(e.name)).map((e) => e.name), A = O.active, he = !1, j = 0, M = 0, ge, _e, ve = 0, N = 0, ye = (t, r) => n.gen(function* () {
|
|
1595
|
+
if (p === "replace" && !A && !(r && x.truncated)) {
|
|
1537
1596
|
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1538
|
-
try: () => t.insert(
|
|
1539
|
-
id:
|
|
1540
|
-
startedAt:
|
|
1597
|
+
try: () => t.insert(u, {
|
|
1598
|
+
id: m("repl", be),
|
|
1599
|
+
startedAt: P,
|
|
1541
1600
|
tables: String(s.tables.length),
|
|
1542
1601
|
capture: F?.dir ?? null,
|
|
1543
1602
|
via: e.via ?? "direct"
|
|
@@ -1546,66 +1605,66 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1546
1605
|
not proceed without it.` })
|
|
1547
1606
|
}));
|
|
1548
1607
|
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !V(e)), a = !r, o = t.emptyTables ? () => t.emptyTables(i) : a ? async () => {
|
|
1549
|
-
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where:
|
|
1608
|
+
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where: b(e.primaryKey) });
|
|
1550
1609
|
} : () => t.transactional(async (e) => {
|
|
1551
|
-
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where:
|
|
1610
|
+
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: b(t.primaryKey) });
|
|
1552
1611
|
});
|
|
1553
1612
|
yield* n.tryPromise({
|
|
1554
1613
|
try: o,
|
|
1555
|
-
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${
|
|
1556
|
-
}), r && (
|
|
1614
|
+
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${jn(e)}` })
|
|
1615
|
+
}), r && (x.truncated = !0, yield* G(y, x));
|
|
1557
1616
|
}
|
|
1558
|
-
let a = [], o = [],
|
|
1559
|
-
for (let
|
|
1560
|
-
let
|
|
1561
|
-
if (V(
|
|
1562
|
-
|
|
1617
|
+
let a = [], o = [], l = [], f = [], h = s.tables.length;
|
|
1618
|
+
for (let u = 0; u < s.tables.length; u++) {
|
|
1619
|
+
let m = s.tables[u];
|
|
1620
|
+
if (V(m.name)) {
|
|
1621
|
+
f.push(m.name);
|
|
1563
1622
|
continue;
|
|
1564
1623
|
}
|
|
1565
|
-
if (r &&
|
|
1566
|
-
|
|
1624
|
+
if (r && m.name in x.tables) {
|
|
1625
|
+
l.push(m.name), N++, ve += x.tables[m.name].rowCount, W(e.onProgress, {
|
|
1567
1626
|
phase: "import",
|
|
1568
|
-
table:
|
|
1569
|
-
index:
|
|
1570
|
-
tableCount:
|
|
1627
|
+
table: m.name,
|
|
1628
|
+
index: u,
|
|
1629
|
+
tableCount: h,
|
|
1571
1630
|
event: "done",
|
|
1572
|
-
rowsDone:
|
|
1573
|
-
total:
|
|
1631
|
+
rowsDone: x.tables[m.name].rowCount,
|
|
1632
|
+
total: m.rowCount
|
|
1574
1633
|
});
|
|
1575
1634
|
continue;
|
|
1576
1635
|
}
|
|
1577
|
-
let _ = i.join(e.bundleDir,
|
|
1578
|
-
|
|
1636
|
+
let _ = i.join(e.bundleDir, m.file);
|
|
1637
|
+
W(e.onProgress, {
|
|
1579
1638
|
phase: "import",
|
|
1580
|
-
table:
|
|
1581
|
-
index:
|
|
1582
|
-
tableCount:
|
|
1639
|
+
table: m.name,
|
|
1640
|
+
index: u,
|
|
1641
|
+
tableCount: h,
|
|
1583
1642
|
event: "start",
|
|
1584
1643
|
rowsDone: 0,
|
|
1585
|
-
total:
|
|
1644
|
+
total: m.rowCount
|
|
1586
1645
|
});
|
|
1587
|
-
let
|
|
1588
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0,
|
|
1646
|
+
let v = a.length, b = n.gen(function* () {
|
|
1647
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, l = me("bundle", () => ue(_).pipe(St(s.compression))).pipe(c.tap((e) => n.sync(() => i.update(e)))), u = async (e) => {
|
|
1589
1648
|
let n = {
|
|
1590
|
-
entry:
|
|
1649
|
+
entry: m,
|
|
1591
1650
|
row: e,
|
|
1592
1651
|
reason: ""
|
|
1593
1652
|
};
|
|
1594
1653
|
try {
|
|
1595
|
-
await T(t, () =>
|
|
1654
|
+
await T(t, () => ae(t, m, e));
|
|
1596
1655
|
} catch (t) {
|
|
1597
|
-
let r =
|
|
1598
|
-
n.reason = r.reason,
|
|
1656
|
+
let r = kn(t);
|
|
1657
|
+
n.reason = r.reason, se(m, e, r), a.push(n);
|
|
1599
1658
|
}
|
|
1600
1659
|
}, d = yield* n.promise(async () => {
|
|
1601
1660
|
let n = e.copyLoader;
|
|
1602
1661
|
if (!n || n.dialect !== "postgres") return !1;
|
|
1603
|
-
if (
|
|
1604
|
-
if (
|
|
1605
|
-
if (
|
|
1662
|
+
if (g) return he || (he = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1663
|
+
if (p === "replace") return !0;
|
|
1664
|
+
if (p !== "upsert") return !1;
|
|
1606
1665
|
try {
|
|
1607
1666
|
return (await t.query({
|
|
1608
|
-
table:
|
|
1667
|
+
table: m.name,
|
|
1609
1668
|
predicate: void 0,
|
|
1610
1669
|
order: [],
|
|
1611
1670
|
take: 1,
|
|
@@ -1615,19 +1674,19 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1615
1674
|
} catch {
|
|
1616
1675
|
return !1;
|
|
1617
1676
|
}
|
|
1618
|
-
}),
|
|
1677
|
+
}), f = Object.keys(m.columnTypes), h = Math.max(1, e.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
1619
1678
|
if (v.length === 0) return;
|
|
1620
1679
|
let t = v;
|
|
1621
1680
|
v = [];
|
|
1622
1681
|
try {
|
|
1623
1682
|
await e.copyLoader.copyInto({
|
|
1624
|
-
table:
|
|
1625
|
-
columns:
|
|
1626
|
-
columnTypes:
|
|
1683
|
+
table: m.name,
|
|
1684
|
+
columns: f,
|
|
1685
|
+
columnTypes: m.columnTypes,
|
|
1627
1686
|
rows: t
|
|
1628
1687
|
});
|
|
1629
1688
|
} catch (n) {
|
|
1630
|
-
d = !1, e.onWarn?.(`bulk COPY into ${
|
|
1689
|
+
d = !1, e.onWarn?.(`bulk COPY into ${m.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1631
1690
|
for (let e of t) await u(e);
|
|
1632
1691
|
}
|
|
1633
1692
|
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof t.savepoint == "function", S = [], ee = async () => {
|
|
@@ -1636,15 +1695,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1636
1695
|
S = [];
|
|
1637
1696
|
try {
|
|
1638
1697
|
await t.savepoint(async () => {
|
|
1639
|
-
for (let n of e) await
|
|
1698
|
+
for (let n of e) await ae(t, m, n);
|
|
1640
1699
|
});
|
|
1641
1700
|
} catch {
|
|
1642
1701
|
for (let t of e) await u(t);
|
|
1643
1702
|
}
|
|
1644
1703
|
};
|
|
1645
|
-
return yield*
|
|
1646
|
-
if (o++,
|
|
1647
|
-
v.push(e), v.length >=
|
|
1704
|
+
return yield* c.runForEach(an(l, m.name, m.columnTypes), (e) => n.promise(async () => {
|
|
1705
|
+
if (o++, _e?.(), d) {
|
|
1706
|
+
v.push(e), v.length >= h && await y();
|
|
1648
1707
|
return;
|
|
1649
1708
|
}
|
|
1650
1709
|
if (x) {
|
|
@@ -1656,60 +1715,60 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1656
1715
|
rowCount: o,
|
|
1657
1716
|
checksum: i.digest("hex")
|
|
1658
1717
|
};
|
|
1659
|
-
}), { rowCount: S, checksum: ee } = yield*
|
|
1660
|
-
if (
|
|
1661
|
-
if (ee !==
|
|
1662
|
-
what: `table ${
|
|
1663
|
-
expected:
|
|
1718
|
+
}), { rowCount: S, checksum: ee } = yield* Vt("import", m.name, u, h, (e) => e.rowCount, b);
|
|
1719
|
+
if (d) {
|
|
1720
|
+
if (ee !== m.checksum) return yield* n.fail(new Xe({
|
|
1721
|
+
what: `table ${m.name} checksum`,
|
|
1722
|
+
expected: m.checksum,
|
|
1664
1723
|
actual: ee
|
|
1665
1724
|
}));
|
|
1666
|
-
if (S !==
|
|
1667
|
-
what: `table ${
|
|
1668
|
-
expected: String(
|
|
1725
|
+
if (S !== m.rowCount) return yield* n.fail(new Xe({
|
|
1726
|
+
what: `table ${m.name} rowCount`,
|
|
1727
|
+
expected: String(m.rowCount),
|
|
1669
1728
|
actual: String(S)
|
|
1670
1729
|
}));
|
|
1671
1730
|
}
|
|
1672
|
-
|
|
1731
|
+
W(e.onProgress, {
|
|
1673
1732
|
phase: "import",
|
|
1674
|
-
table:
|
|
1675
|
-
index:
|
|
1676
|
-
tableCount:
|
|
1733
|
+
table: m.name,
|
|
1734
|
+
index: u,
|
|
1735
|
+
tableCount: h,
|
|
1677
1736
|
event: "done",
|
|
1678
1737
|
rowsDone: S,
|
|
1679
|
-
total:
|
|
1680
|
-
}),
|
|
1681
|
-
name:
|
|
1738
|
+
total: m.rowCount
|
|
1739
|
+
}), M++, j += S, ge?.(), _e?.(), r && (a.length > v ? o.push({
|
|
1740
|
+
name: m.name,
|
|
1682
1741
|
rowCount: S
|
|
1683
|
-
}) : (
|
|
1742
|
+
}) : (x.tables[m.name] = {
|
|
1684
1743
|
rowCount: S,
|
|
1685
|
-
checksum:
|
|
1686
|
-
}, yield*
|
|
1744
|
+
checksum: m.checksum
|
|
1745
|
+
}, yield* G(y, x)));
|
|
1687
1746
|
}
|
|
1688
|
-
if (a.length > 0 && (yield*
|
|
1747
|
+
if (a.length > 0 && (yield* fe(t, a)), p === "replace" && (yield* n.promise(() => t.deleteMany(u, { where: {
|
|
1689
1748
|
column: "startedAt",
|
|
1690
1749
|
op: "lte",
|
|
1691
|
-
value:
|
|
1750
|
+
value: P
|
|
1692
1751
|
} }).then(() => void 0, () => void 0))), r && o.length > 0) {
|
|
1693
1752
|
for (let e of o) {
|
|
1694
1753
|
let t = s.tables.find((t) => t.name === e.name);
|
|
1695
|
-
|
|
1754
|
+
x.tables[e.name] = {
|
|
1696
1755
|
rowCount: e.rowCount,
|
|
1697
1756
|
checksum: t.checksum
|
|
1698
1757
|
};
|
|
1699
1758
|
}
|
|
1700
|
-
yield*
|
|
1759
|
+
yield* G(y, x);
|
|
1701
1760
|
}
|
|
1702
1761
|
{
|
|
1703
|
-
let t =
|
|
1762
|
+
let t = Je(f, "skipped");
|
|
1704
1763
|
t !== void 0 && e.onWarn?.(t);
|
|
1705
1764
|
}
|
|
1706
|
-
if (
|
|
1707
|
-
let t =
|
|
1708
|
-
e.onWarn?.(`resume: ${
|
|
1765
|
+
if (l.length > 0) {
|
|
1766
|
+
let t = l.length === s.tables.length;
|
|
1767
|
+
e.onWarn?.(`resume: ${l.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${l.join(", ")}. Delete ${y} to force a full re-import.`);
|
|
1709
1768
|
}
|
|
1710
|
-
}),
|
|
1769
|
+
}), be = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, P = /* @__PURE__ */ new Date(), F = yield* Yn({
|
|
1711
1770
|
store: e.store,
|
|
1712
|
-
mode:
|
|
1771
|
+
mode: p,
|
|
1713
1772
|
dir: e.rollbackDir,
|
|
1714
1773
|
snapshot: e.targetSnapshot,
|
|
1715
1774
|
dialect: e.targetDialect,
|
|
@@ -1722,60 +1781,60 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1722
1781
|
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1723
1782
|
path: F.dir
|
|
1724
1783
|
})
|
|
1725
|
-
})), e.onWarn?.(
|
|
1726
|
-
let xe = (e) =>
|
|
1727
|
-
Ce &&
|
|
1728
|
-
id:
|
|
1729
|
-
mode:
|
|
1784
|
+
})), e.onWarn?.(Xn(F)));
|
|
1785
|
+
let xe = (e) => p === "replace" ? te(e) : e(), I = (e) => p === "replace" ? ne(e) : e, L = e.traceId ?? m("dimp", `${Date.now()}-${s.source.schemaFingerprint ?? ""}`), Se = e.traceId !== void 0, Ce = (e.targetSnapshot?.tables.some((e) => e.name === l) ?? !1) || v(l) !== void 0;
|
|
1786
|
+
Ce && h(f), Ce && e.traceId === void 0 && (yield* n.promise(() => e.store.insert(l, {
|
|
1787
|
+
id: L,
|
|
1788
|
+
mode: p,
|
|
1730
1789
|
via: e.via ?? "direct",
|
|
1731
1790
|
direction: "import",
|
|
1732
1791
|
bundle: e.bundleDir,
|
|
1733
1792
|
sourceFingerprint: s.source.schemaFingerprint ?? null,
|
|
1734
1793
|
tablesWritten: 0,
|
|
1735
1794
|
rowsWritten: 0,
|
|
1736
|
-
staged:
|
|
1795
|
+
staged: p === "replace" ? A : null
|
|
1737
1796
|
}).then(() => {
|
|
1738
|
-
|
|
1797
|
+
Se = !0;
|
|
1739
1798
|
}, (t) => {
|
|
1740
1799
|
e.onWarn?.(`the import trace could not be written (${String(t?.message ?? t)}).\n The import continues — this is a record of what happened, not a safety check. Run
|
|
1741
1800
|
\`voltro db apply\` so the next one is recorded.`);
|
|
1742
|
-
}))), e.traceId !== void 0 && Ce && (yield* n.promise(() => e.store.update(
|
|
1743
|
-
staged:
|
|
1801
|
+
}))), e.traceId !== void 0 && Ce && (yield* n.promise(() => e.store.update(l, L, {
|
|
1802
|
+
staged: p === "replace" ? A : null,
|
|
1744
1803
|
sourceFingerprint: s.source.schemaFingerprint ?? null
|
|
1745
1804
|
}).then(() => void 0, () => void 0)));
|
|
1746
1805
|
let we = 0, Te = !1;
|
|
1747
|
-
|
|
1748
|
-
if (!
|
|
1806
|
+
ge = () => {
|
|
1807
|
+
if (!Se || Te) return;
|
|
1749
1808
|
let t = Date.now();
|
|
1750
|
-
t - we <
|
|
1751
|
-
tablesWritten:
|
|
1752
|
-
rowsWritten:
|
|
1809
|
+
t - we < er || (we = t, e.store.update(l, L, {
|
|
1810
|
+
tablesWritten: M,
|
|
1811
|
+
rowsWritten: j
|
|
1753
1812
|
}).then(() => void 0, () => {
|
|
1754
1813
|
Te = !0;
|
|
1755
1814
|
}));
|
|
1756
1815
|
};
|
|
1757
|
-
let Ee = (t) =>
|
|
1816
|
+
let Ee = (t) => Se ? e.store.update(l, L, {
|
|
1758
1817
|
finishedAt: t === null ? /* @__PURE__ */ new Date() : null,
|
|
1759
|
-
tablesWritten:
|
|
1760
|
-
rowsWritten:
|
|
1818
|
+
tablesWritten: M,
|
|
1819
|
+
rowsWritten: j,
|
|
1761
1820
|
failure: t
|
|
1762
1821
|
}).then(() => void 0, () => void 0) : Promise.resolve();
|
|
1763
|
-
if (t = (e) => Ee(e),
|
|
1822
|
+
if (t = (e) => Ee(e), p === "replace" && !A && O.blocker !== void 0 && e.onWarn?.(Pn(O.blocker)), p === "replace" && pe.length > 0 && e.onWarn?.(`write recorders are SUSPENDED for this replace (${pe.length} table(s) have one).\n A replace sets a state rather than changing rows, so a per-row history entry would describe
|
|
1764
1823
|
something that did not happen — and this path already writes through the raw store, without
|
|
1765
|
-
tenant scoping or \`audit()\` stamping, for the same reason.`),
|
|
1766
|
-
e.onWarn?.(`staging ${
|
|
1767
|
-
let t = e.store, r =
|
|
1824
|
+
tenant scoping or \`audit()\` stamping, for the same reason.`), A) {
|
|
1825
|
+
e.onWarn?.(`staging ${k.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1826
|
+
let t = e.store, r = k.map((e) => e.name), i = m("repl", `staging-${be}`), a = !1;
|
|
1768
1827
|
if (e.recordInterruption === !0) {
|
|
1769
|
-
yield* n.promise(() => e.store.insert(
|
|
1828
|
+
yield* n.promise(() => e.store.insert(u, {
|
|
1770
1829
|
id: i,
|
|
1771
|
-
startedAt:
|
|
1830
|
+
startedAt: P,
|
|
1772
1831
|
kind: "staging",
|
|
1773
|
-
tables: String(
|
|
1832
|
+
tables: String(k.length),
|
|
1774
1833
|
capture: F?.dir ?? null,
|
|
1775
1834
|
via: e.via ?? "direct",
|
|
1776
|
-
staging: r.map(
|
|
1835
|
+
staging: r.map(q).join(","),
|
|
1777
1836
|
heartbeatAt: /* @__PURE__ */ new Date(),
|
|
1778
|
-
resumable: !
|
|
1837
|
+
resumable: !g
|
|
1779
1838
|
}).then(() => {
|
|
1780
1839
|
a = !0;
|
|
1781
1840
|
}, (t) => {
|
|
@@ -1784,75 +1843,75 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1784
1843
|
them by introspection.`);
|
|
1785
1844
|
}));
|
|
1786
1845
|
let t = Date.now();
|
|
1787
|
-
|
|
1846
|
+
_e = () => {
|
|
1788
1847
|
if (!a) return;
|
|
1789
1848
|
let n = Date.now();
|
|
1790
|
-
n - t <
|
|
1849
|
+
n - t < tr || (t = n, e.store.update(u, i, { heartbeatAt: /* @__PURE__ */ new Date() }).then(() => void 0, () => void 0));
|
|
1791
1850
|
};
|
|
1792
1851
|
}
|
|
1793
|
-
let o = () => a ? e.store.delete(
|
|
1794
|
-
getTable: (e) =>
|
|
1852
|
+
let o = () => a ? e.store.delete(u, i).then(() => void 0, () => void 0) : Promise.resolve(), s = zn(r, {
|
|
1853
|
+
getTable: (e) => v(e),
|
|
1795
1854
|
registerTable: (e) => {
|
|
1796
|
-
|
|
1855
|
+
ee(e);
|
|
1797
1856
|
},
|
|
1798
|
-
unregisterTable:
|
|
1857
|
+
unregisterTable: re
|
|
1799
1858
|
}), c = !1;
|
|
1800
1859
|
try {
|
|
1801
|
-
let i =
|
|
1860
|
+
let i = g ? r : r.filter((e) => !Wt(x, e));
|
|
1802
1861
|
yield* n.tryPromise({
|
|
1803
1862
|
try: async () => {
|
|
1804
|
-
await
|
|
1863
|
+
await Un(t, i.map((e) => Ln(e, D))), await Un(t, i.map((e) => In(e, D)));
|
|
1805
1864
|
},
|
|
1806
1865
|
catch: (e) => new H({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1807
|
-
}), yield*
|
|
1808
|
-
try: () =>
|
|
1866
|
+
}), yield* I(ye(e.store, !g)), !g && x.truncated === !0 ? c = !0 : yield* n.tryPromise({
|
|
1867
|
+
try: () => Wn(t, k, D).then(() => {
|
|
1809
1868
|
c = !0;
|
|
1810
1869
|
}),
|
|
1811
1870
|
catch: (e) => new H({ reason: String(e?.message ?? e) })
|
|
1812
|
-
}).pipe(n.
|
|
1813
|
-
let a =
|
|
1871
|
+
}).pipe(n.catch((i) => n.gen(function* () {
|
|
1872
|
+
let a = Hn(yield* n.promise(() => Gn(t, Bn(e.targetSnapshot, r), D).then((e) => e, () => [])));
|
|
1814
1873
|
return yield* n.fail(new H({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1815
|
-
}))), !
|
|
1874
|
+
}))), !g && c && x.truncated !== !0 && (x.truncated = !0, yield* G(y, x));
|
|
1816
1875
|
} finally {
|
|
1817
|
-
s.undo(), c ||
|
|
1876
|
+
s.undo(), c || g ? (_e = void 0, yield* n.promise(() => Un(t, r.map((e) => Ln(e, D))).then(() => void 0, () => void 0)), yield* n.promise(o)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1818
1877
|
}
|
|
1819
|
-
} else
|
|
1878
|
+
} else g ? yield* n.tryPromise({
|
|
1820
1879
|
try: () => e.store.transactional(async (e) => {
|
|
1821
|
-
let t = await xe(() => n.runPromiseExit(
|
|
1822
|
-
if (r.isFailure(t)) throw
|
|
1880
|
+
let t = await xe(() => n.runPromiseExit(ye(e, !1)));
|
|
1881
|
+
if (r.isFailure(t)) throw ar(t.cause);
|
|
1823
1882
|
}),
|
|
1824
|
-
catch: (e) =>
|
|
1825
|
-
}) : yield*
|
|
1883
|
+
catch: (e) => ir(e) ?? new H({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1884
|
+
}) : yield* I(ye(e.store, !0));
|
|
1826
1885
|
if (e.assets && s.assets.index && !e.assetsAlreadyRestored) {
|
|
1827
1886
|
let t = e.assets;
|
|
1828
|
-
yield*
|
|
1887
|
+
yield* _t({
|
|
1829
1888
|
sink: t,
|
|
1830
|
-
assetsDir: i.join(e.bundleDir,
|
|
1889
|
+
assetsDir: i.join(e.bundleDir, Nt),
|
|
1831
1890
|
indexPath: i.join(e.bundleDir, s.assets.index ?? "assets/index.ndjson"),
|
|
1832
1891
|
progress: {
|
|
1833
|
-
done: (e) => e in
|
|
1834
|
-
mark: (e) => (
|
|
1892
|
+
done: (e) => e in x.assets,
|
|
1893
|
+
mark: (e) => (x.assets[e] = !0, G(y, x))
|
|
1835
1894
|
}
|
|
1836
1895
|
});
|
|
1837
1896
|
}
|
|
1838
1897
|
return yield* n.promise(() => Ee(null)), {
|
|
1839
1898
|
manifest: s,
|
|
1840
|
-
tablesWritten:
|
|
1841
|
-
rowsWritten:
|
|
1899
|
+
tablesWritten: M,
|
|
1900
|
+
rowsWritten: j,
|
|
1842
1901
|
tablesSkipped: N,
|
|
1843
1902
|
rowsSkipped: ve,
|
|
1844
|
-
fullyResumed:
|
|
1903
|
+
fullyResumed: M === 0 && N > 0,
|
|
1845
1904
|
...F === void 0 ? {} : { rollback: F }
|
|
1846
1905
|
};
|
|
1847
1906
|
}).pipe(n.tapError((e) => n.promise(() => t?.(e?.reason ?? e?._tag ?? "the import did not finish") ?? Promise.resolve())));
|
|
1848
|
-
},
|
|
1907
|
+
}, cr = (e) => e, lr = (e) => {
|
|
1849
1908
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1850
1909
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1851
1910
|
return {
|
|
1852
1911
|
refuse: !0,
|
|
1853
1912
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1854
1913
|
};
|
|
1855
|
-
},
|
|
1914
|
+
}, Y = Buffer.from("VBUNDLE2"), ur = 0, dr = 1, fr = 2, pr = 32, mr = async (e) => {
|
|
1856
1915
|
let t = [], n = async (r) => {
|
|
1857
1916
|
let i;
|
|
1858
1917
|
try {
|
|
@@ -1866,22 +1925,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1866
1925
|
}
|
|
1867
1926
|
};
|
|
1868
1927
|
return await n(""), t.sort();
|
|
1869
|
-
},
|
|
1928
|
+
}, hr = (e, t) => {
|
|
1870
1929
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1871
|
-
return r.writeUInt8(
|
|
1872
|
-
},
|
|
1930
|
+
return r.writeUInt8(dr, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1931
|
+
}, gr = (e, t, n) => {
|
|
1873
1932
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1874
|
-
return a.writeUInt8(
|
|
1933
|
+
return a.writeUInt8(fr, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1875
1934
|
};
|
|
1876
|
-
async function*
|
|
1877
|
-
yield new Uint8Array(
|
|
1878
|
-
for (let t of await
|
|
1935
|
+
async function* _r(e) {
|
|
1936
|
+
yield new Uint8Array(Y);
|
|
1937
|
+
for (let t of await mr(e.dir)) {
|
|
1879
1938
|
let n = I(e.dir, t), r = await be(n);
|
|
1880
|
-
if (yield new Uint8Array(
|
|
1939
|
+
if (yield new Uint8Array(hr(t, r.size)), r.size > 0) for await (let e of ue(n)) yield new Uint8Array(e);
|
|
1881
1940
|
}
|
|
1882
1941
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1883
|
-
yield new Uint8Array(
|
|
1884
|
-
let n =
|
|
1942
|
+
yield new Uint8Array(gr(t.key, t.contentType, t.size));
|
|
1943
|
+
let n = w("sha256"), r = 0;
|
|
1885
1944
|
for await (let i of e.blobs.open(t.key)) {
|
|
1886
1945
|
let e = Buffer.from(i);
|
|
1887
1946
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1889,11 +1948,11 @@ async function* br(e) {
|
|
|
1889
1948
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1890
1949
|
yield new Uint8Array(n.digest());
|
|
1891
1950
|
}
|
|
1892
|
-
yield new Uint8Array(Buffer.from([
|
|
1951
|
+
yield new Uint8Array(Buffer.from([ur]));
|
|
1893
1952
|
}
|
|
1894
|
-
var
|
|
1953
|
+
var vr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), yr = (e) => new Promise((t) => e.end(t)), br = async (e, t) => {
|
|
1895
1954
|
let n = L(t.dir);
|
|
1896
|
-
await
|
|
1955
|
+
await N(n, { recursive: !0 });
|
|
1897
1956
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1898
1957
|
let { value: e, done: t } = await i.next();
|
|
1899
1958
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1904,28 +1963,28 @@ var xr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Sr
|
|
|
1904
1963
|
return r = r.subarray(e), t;
|
|
1905
1964
|
};
|
|
1906
1965
|
try {
|
|
1907
|
-
if (await s(
|
|
1966
|
+
if (await s(Y.length), !c(Y.length).equals(Y)) throw Error("not a bundle archive (bad magic)");
|
|
1908
1967
|
for (;;) {
|
|
1909
1968
|
await s(1);
|
|
1910
1969
|
let e = c(1).readUInt8(0);
|
|
1911
|
-
if (e ===
|
|
1912
|
-
if (e ===
|
|
1970
|
+
if (e === ur) return;
|
|
1971
|
+
if (e === dr) {
|
|
1913
1972
|
await s(4);
|
|
1914
1973
|
let e = c(4).readUInt32BE(0);
|
|
1915
1974
|
await s(e);
|
|
1916
1975
|
let t = c(e).toString("utf8");
|
|
1917
1976
|
await s(8);
|
|
1918
1977
|
let i = c(8).readBigUInt64BE(0), a = L(n, t);
|
|
1919
|
-
if (a !== n && !a.startsWith(n +
|
|
1920
|
-
await
|
|
1921
|
-
let o =
|
|
1978
|
+
if (a !== n && !a.startsWith(n + Se)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1979
|
+
await N(xe(a), { recursive: !0 });
|
|
1980
|
+
let o = de(a);
|
|
1922
1981
|
for (; i > 0n;) {
|
|
1923
1982
|
r.length === 0 && await s(1);
|
|
1924
1983
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1925
|
-
await
|
|
1984
|
+
await vr(o, c(e)), i -= BigInt(e);
|
|
1926
1985
|
}
|
|
1927
|
-
await
|
|
1928
|
-
} else if (e ===
|
|
1986
|
+
await yr(o);
|
|
1987
|
+
} else if (e === fr) {
|
|
1929
1988
|
await s(4);
|
|
1930
1989
|
let e = c(4).readUInt32BE(0);
|
|
1931
1990
|
await s(e);
|
|
@@ -1935,24 +1994,24 @@ var xr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Sr
|
|
|
1935
1994
|
await s(i);
|
|
1936
1995
|
let a = c(i).toString("utf8");
|
|
1937
1996
|
await s(8);
|
|
1938
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1997
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = w("sha256"), d = new ge(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1939
1998
|
contentType: a,
|
|
1940
1999
|
size: Number(o)
|
|
1941
2000
|
}), p = o;
|
|
1942
2001
|
for (; p > 0n;) {
|
|
1943
2002
|
r.length === 0 && await s(1);
|
|
1944
2003
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1945
|
-
u.update(t), l || await
|
|
2004
|
+
u.update(t), l || await vr(d, t), p -= BigInt(e);
|
|
1946
2005
|
}
|
|
1947
|
-
l ? d.destroy() : await
|
|
1948
|
-
let m = c(
|
|
2006
|
+
l ? d.destroy() : await yr(d), await f, await s(pr);
|
|
2007
|
+
let m = c(pr);
|
|
1949
2008
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1950
2009
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1951
2010
|
}
|
|
1952
2011
|
} finally {
|
|
1953
2012
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1954
2013
|
}
|
|
1955
|
-
},
|
|
2014
|
+
}, xr = async (e, t) => {
|
|
1956
2015
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1957
2016
|
let { value: e, done: t } = await a.next();
|
|
1958
2017
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1971,11 +2030,11 @@ var xr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Sr
|
|
|
1971
2030
|
}
|
|
1972
2031
|
return Buffer.concat(t);
|
|
1973
2032
|
};
|
|
1974
|
-
if (!await s(
|
|
2033
|
+
if (!await s(Y.length) || !c(Y.length).equals(Y)) throw Error("not a bundle archive (bad magic)");
|
|
1975
2034
|
for (; await s(1);) {
|
|
1976
2035
|
let e = c(1).readUInt8(0);
|
|
1977
|
-
if (e ===
|
|
1978
|
-
if (e !==
|
|
2036
|
+
if (e === ur || e === fr) break;
|
|
2037
|
+
if (e !== dr) throw Error(`unknown bundle entry kind ${e}`);
|
|
1979
2038
|
if (!await s(4)) break;
|
|
1980
2039
|
let t = c(4).readUInt32BE(0);
|
|
1981
2040
|
if (!await s(t)) break;
|
|
@@ -1985,20 +2044,20 @@ var xr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Sr
|
|
|
1985
2044
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1986
2045
|
}
|
|
1987
2046
|
return r;
|
|
1988
|
-
},
|
|
2047
|
+
}, Sr = (e, t) => br(e, {
|
|
1989
2048
|
dir: t,
|
|
1990
|
-
blobs:
|
|
1991
|
-
}),
|
|
2049
|
+
blobs: Cr(t)
|
|
2050
|
+
}), Cr = (e) => {
|
|
1992
2051
|
let t = /* @__PURE__ */ new Set();
|
|
1993
2052
|
return {
|
|
1994
2053
|
has: async (e) => t.has(e),
|
|
1995
2054
|
write: async (n, r, i) => {
|
|
1996
2055
|
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = I(e, "assets");
|
|
1997
|
-
await
|
|
1998
|
-
let l =
|
|
2056
|
+
await N(c, { recursive: !0 });
|
|
2057
|
+
let l = w("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1999
2058
|
l.update(e), u += e.length, n(null, e);
|
|
2000
2059
|
} }), f = I(c, `.tmp-${Date.now()}-${t.size}`);
|
|
2001
|
-
await o(r, d,
|
|
2060
|
+
await o(r, d, de(f));
|
|
2002
2061
|
let p = l.digest("hex");
|
|
2003
2062
|
await a.rename(f, I(c, p)), await a.appendFile(I(c, "index.ndjson"), `${JSON.stringify({
|
|
2004
2063
|
key: n,
|
|
@@ -2008,76 +2067,114 @@ var xr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), Sr
|
|
|
2008
2067
|
})}\n`), t.add(n);
|
|
2009
2068
|
}
|
|
2010
2069
|
};
|
|
2011
|
-
},
|
|
2070
|
+
}, wr = Ce(ce), Tr = Buffer.from("VENC1\0\0\0"), Er = 32, Dr = 16, Or = 7, kr = 16, X = 262144, Ar = {
|
|
2012
2071
|
N: 32768,
|
|
2013
2072
|
r: 8,
|
|
2014
2073
|
p: 1
|
|
2015
|
-
},
|
|
2074
|
+
}, jr = 134217728, Mr = 43, Nr = (e, t, n) => {
|
|
2016
2075
|
let r = Buffer.alloc(12);
|
|
2017
|
-
return e.copy(r, 0, 0,
|
|
2076
|
+
return e.copy(r, 0, 0, Or), r.writeUInt32BE(t >>> 0, Or), r.writeUInt8(+!!n, 11), r;
|
|
2018
2077
|
};
|
|
2019
|
-
async function*
|
|
2078
|
+
async function* Pr(e, t) {
|
|
2020
2079
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
2021
|
-
let n =
|
|
2022
|
-
...
|
|
2023
|
-
maxmem:
|
|
2024
|
-
}), a = Buffer.alloc(
|
|
2025
|
-
|
|
2080
|
+
let n = E(Dr), r = E(Or), i = await wr(t, n, Er, {
|
|
2081
|
+
...Ar,
|
|
2082
|
+
maxmem: jr
|
|
2083
|
+
}), a = Buffer.alloc(Mr);
|
|
2084
|
+
Tr.copy(a, 0), a.writeUInt32BE(Ar.N, 8), a.writeUInt32BE(Ar.r, 12), a.writeUInt32BE(Ar.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
2026
2085
|
let o = 0, s = (e, t) => {
|
|
2027
|
-
let n = ae("aes-256-gcm", i,
|
|
2086
|
+
let n = ae("aes-256-gcm", i, Nr(r, o, t));
|
|
2028
2087
|
o === 0 && n.setAAD(a);
|
|
2029
2088
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
2030
2089
|
o++;
|
|
2031
|
-
let l = Buffer.alloc(5 + s.length +
|
|
2032
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
2033
|
-
}, c = Buffer.alloc(0);
|
|
2034
|
-
for await (let t of e)
|
|
2035
|
-
|
|
2090
|
+
let l = Buffer.alloc(5 + s.length + kr);
|
|
2091
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + kr, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
2092
|
+
}, c = Buffer.alloc(0), l = 0;
|
|
2093
|
+
for await (let t of e) {
|
|
2094
|
+
let e = Buffer.from(t), n = 0;
|
|
2095
|
+
for (; n < e.length;) {
|
|
2096
|
+
if (l === X && (yield s(c, !1), l = 0), l === 0 && e.length - n >= X) {
|
|
2097
|
+
let t = e.subarray(n, n + X);
|
|
2098
|
+
n += X, n < e.length ? yield s(t, !1) : (c = t, l = X);
|
|
2099
|
+
continue;
|
|
2100
|
+
}
|
|
2101
|
+
c.length !== X && (c = Buffer.alloc(X));
|
|
2102
|
+
let t = Math.min(X - l, e.length - n);
|
|
2103
|
+
e.copy(c, l, n, n + t), l += t, n += t;
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
yield s(c.subarray(0, l), !0);
|
|
2036
2107
|
}
|
|
2037
|
-
async function*
|
|
2108
|
+
async function* Fr(e, t) {
|
|
2038
2109
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
2039
|
-
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1,
|
|
2040
|
-
let
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2110
|
+
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = 0, a = !1, o = async (e) => {
|
|
2111
|
+
let t, o = 0;
|
|
2112
|
+
for (; o < e;) {
|
|
2113
|
+
if (i === r.length) {
|
|
2114
|
+
if (a) break;
|
|
2115
|
+
let e = await n.next();
|
|
2116
|
+
if (e.done) {
|
|
2117
|
+
a = !0;
|
|
2118
|
+
break;
|
|
2119
|
+
}
|
|
2120
|
+
if (r = Buffer.from(e.value), i = 0, r.length === 0) continue;
|
|
2121
|
+
}
|
|
2122
|
+
if (o === 0 && r.length - i >= e) {
|
|
2123
|
+
let t = r.subarray(i, i + e);
|
|
2124
|
+
return i += e, t;
|
|
2125
|
+
}
|
|
2126
|
+
t ??= Buffer.alloc(e);
|
|
2127
|
+
let s = Math.min(e - o, r.length - i);
|
|
2128
|
+
r.copy(t, o, i, i + s), i += s, o += s;
|
|
2129
|
+
}
|
|
2130
|
+
return t?.subarray(0, o) ?? Buffer.alloc(0);
|
|
2131
|
+
}, s = !1;
|
|
2132
|
+
try {
|
|
2133
|
+
let e = await o(Mr);
|
|
2134
|
+
if (e.length !== Mr) throw Error("encrypted bundle truncated (no header)");
|
|
2135
|
+
if (!e.subarray(0, 8).equals(Tr)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
2136
|
+
let n = e.readUInt32BE(8), r = e.readUInt32BE(12), i = e.readUInt32BE(16), a = Buffer.from(e.subarray(20, 36)), s = Buffer.from(e.subarray(36, Mr)), c = await wr(t, a, Er, {
|
|
2137
|
+
N: n,
|
|
2138
|
+
r,
|
|
2139
|
+
p: i,
|
|
2140
|
+
maxmem: jr
|
|
2141
|
+
}), l = 0, u = !1;
|
|
2142
|
+
for (;;) {
|
|
2143
|
+
let t = await o(5);
|
|
2144
|
+
if (t.length !== 5) {
|
|
2145
|
+
if (t.length === 0 && u) return;
|
|
2146
|
+
throw Error("encrypted bundle truncated (no authenticated final frame)");
|
|
2147
|
+
}
|
|
2148
|
+
if (u) throw Error("encrypted bundle has trailing data after the final frame");
|
|
2149
|
+
let n = t.readUInt8(0);
|
|
2150
|
+
if (n !== 0 && n !== 1) throw Error("encrypted bundle corrupt (invalid final flag)");
|
|
2151
|
+
let r = n === 1, i = t.readUInt32BE(1);
|
|
2152
|
+
if (i < kr) throw Error("encrypted bundle corrupt (short frame)");
|
|
2153
|
+
if (i > 262160) throw Error("encrypted bundle corrupt (oversized frame)");
|
|
2154
|
+
let a = await o(i);
|
|
2155
|
+
if (a.length !== i) throw Error("encrypted bundle truncated (frame body)");
|
|
2156
|
+
let d = a.subarray(0, i - kr), f = a.subarray(i - kr), p = oe("aes-256-gcm", c, Nr(s, l, r));
|
|
2157
|
+
l === 0 && p.setAAD(e), p.setAuthTag(f);
|
|
2158
|
+
let m = Buffer.concat([p.update(d), p.final()]);
|
|
2159
|
+
l++, r && (u = !0), m.length > 0 && (yield new Uint8Array(m));
|
|
2160
|
+
}
|
|
2161
|
+
} catch (e) {
|
|
2162
|
+
throw s = !0, e;
|
|
2163
|
+
} finally {
|
|
2164
|
+
if (!a) try {
|
|
2165
|
+
await n.return?.();
|
|
2166
|
+
} catch (e) {
|
|
2167
|
+
if (!s) throw e;
|
|
2061
2168
|
}
|
|
2062
|
-
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
2063
|
-
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
2064
|
-
if (t < Mr) throw Error("encrypted bundle corrupt (short frame)");
|
|
2065
|
-
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
2066
|
-
let n = r.subarray(5, 5 + t - Mr), i = r.subarray(5 + t - Mr, 5 + t);
|
|
2067
|
-
r = r.subarray(5 + t);
|
|
2068
|
-
let a = oe("aes-256-gcm", p, Lr(f, m, e));
|
|
2069
|
-
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
2070
|
-
let c = Buffer.concat([a.update(n), a.final()]);
|
|
2071
|
-
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
2072
2169
|
}
|
|
2073
2170
|
}
|
|
2074
2171
|
//#endregion
|
|
2075
2172
|
//#region src/blobStorage.ts
|
|
2076
|
-
var
|
|
2173
|
+
var Ir = (e, t) => ({
|
|
2077
2174
|
list: () => {
|
|
2078
2175
|
let e = t();
|
|
2079
2176
|
return { async *[Symbol.asyncIterator]() {
|
|
2080
|
-
let t = await n.runPromise(
|
|
2177
|
+
let t = await n.runPromise(c.runCollect(e));
|
|
2081
2178
|
for (let e of t) yield {
|
|
2082
2179
|
key: e.key,
|
|
2083
2180
|
contentType: e.contentType,
|
|
@@ -2086,32 +2183,32 @@ var Br = (e, t) => ({
|
|
|
2086
2183
|
} };
|
|
2087
2184
|
},
|
|
2088
2185
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
2089
|
-
let r = await n.runPromise(
|
|
2186
|
+
let r = await n.runPromise(A(e, t)), i = await j(r.stream);
|
|
2090
2187
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
2091
2188
|
} })
|
|
2092
|
-
}),
|
|
2093
|
-
table:
|
|
2189
|
+
}), Lr = (e, t) => Ir(e, () => C(t, {
|
|
2190
|
+
table: pe,
|
|
2094
2191
|
chunkSize: 500
|
|
2095
|
-
}).pipe(
|
|
2192
|
+
}).pipe(c.map((e) => ({
|
|
2096
2193
|
key: String(e.key),
|
|
2097
2194
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
2098
2195
|
size: Number(e.size ?? 0)
|
|
2099
|
-
})))),
|
|
2196
|
+
})))), Rr = (e) => ({
|
|
2100
2197
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
2101
2198
|
write: async (t, r, i) => {
|
|
2102
|
-
let a =
|
|
2103
|
-
await n.runPromise(
|
|
2199
|
+
let a = me("storage", () => _e.from(r));
|
|
2200
|
+
await n.runPromise(he(e, t, a, {
|
|
2104
2201
|
contentType: i.contentType,
|
|
2105
2202
|
totalSize: i.size
|
|
2106
2203
|
}));
|
|
2107
2204
|
}
|
|
2108
|
-
}),
|
|
2205
|
+
}), zr = (e, t) => {
|
|
2109
2206
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
2110
|
-
return e === "mariadb" ?
|
|
2207
|
+
return e === "mariadb" ? Br(n) ? { tool: n } : {
|
|
2111
2208
|
tool: r,
|
|
2112
2209
|
hint: `'${n}' is not on PATH, so this ran '${r}' instead. If that binary came from Oracle's MySQL client it queries information_schema.COLUMN_STATISTICS, which MariaDB does not have, and the run dies with error 1109 after the first table. Install the MariaDB client package (which ships '${n}') and re-run. Do NOT reach for --column-statistics=0: that flag does not exist on '${n}', so it fixes the wrong client and breaks the right one. Note: a MariaDB 12.x client requires TLS by default, so running '${n}' BY HAND against a server without it fails with "TLS/SSL error: SSL is required" (2026) and needs --skip-ssl. This command is not affected.`
|
|
2113
2210
|
} : { tool: r };
|
|
2114
|
-
},
|
|
2211
|
+
}, Br = (e) => (process.env.PATH ?? "").split(F).filter((e) => e !== "").some((t) => fe(I(t, e))), Vr = (e) => {
|
|
2115
2212
|
switch (e) {
|
|
2116
2213
|
case "postgres": return "db.dump";
|
|
2117
2214
|
case "mysql":
|
|
@@ -2121,7 +2218,7 @@ var Br = (e, t) => ({
|
|
|
2121
2218
|
case "mssql": return "db.bacpac";
|
|
2122
2219
|
default: return "db.dump";
|
|
2123
2220
|
}
|
|
2124
|
-
},
|
|
2221
|
+
}, Hr = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), Z = (e, t = 3306) => {
|
|
2125
2222
|
if (e.host || e.database) return {
|
|
2126
2223
|
host: e.host ?? "localhost",
|
|
2127
2224
|
port: e.port ?? t,
|
|
@@ -2137,7 +2234,7 @@ var Br = (e, t) => ({
|
|
|
2137
2234
|
password: decodeURIComponent(n.password) || "",
|
|
2138
2235
|
database: n.pathname.replace(/^\//, "") || ""
|
|
2139
2236
|
};
|
|
2140
|
-
},
|
|
2237
|
+
}, Ur = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Wr = (e, t, n) => {
|
|
2141
2238
|
switch (e) {
|
|
2142
2239
|
case "postgres": {
|
|
2143
2240
|
if (t.url) return {
|
|
@@ -2147,7 +2244,7 @@ var Br = (e, t) => ({
|
|
|
2147
2244
|
"--format=custom",
|
|
2148
2245
|
"--no-owner",
|
|
2149
2246
|
"--no-privileges",
|
|
2150
|
-
...
|
|
2247
|
+
...Kr,
|
|
2151
2248
|
"--dbname",
|
|
2152
2249
|
t.url,
|
|
2153
2250
|
"--file",
|
|
@@ -2163,7 +2260,7 @@ var Br = (e, t) => ({
|
|
|
2163
2260
|
"--format=custom",
|
|
2164
2261
|
"--no-owner",
|
|
2165
2262
|
"--no-privileges",
|
|
2166
|
-
...
|
|
2263
|
+
...Kr,
|
|
2167
2264
|
"-h",
|
|
2168
2265
|
e.host,
|
|
2169
2266
|
"-p",
|
|
@@ -2183,14 +2280,14 @@ var Br = (e, t) => ({
|
|
|
2183
2280
|
let r = Z(t);
|
|
2184
2281
|
return {
|
|
2185
2282
|
kind: "spawn",
|
|
2186
|
-
...
|
|
2283
|
+
...zr(e, "dump"),
|
|
2187
2284
|
args: [
|
|
2188
2285
|
"--single-transaction",
|
|
2189
2286
|
"--routines",
|
|
2190
2287
|
"--triggers",
|
|
2191
2288
|
"--no-tablespaces",
|
|
2192
2289
|
"--protocol=TCP",
|
|
2193
|
-
...
|
|
2290
|
+
...Gr.map((e) => `--ignore-table=${r.database}.${e}`),
|
|
2194
2291
|
"-h",
|
|
2195
2292
|
r.host,
|
|
2196
2293
|
"-P",
|
|
@@ -2206,7 +2303,7 @@ var Br = (e, t) => ({
|
|
|
2206
2303
|
case "sqlite":
|
|
2207
2304
|
case "turso": return {
|
|
2208
2305
|
kind: "copy",
|
|
2209
|
-
from:
|
|
2306
|
+
from: Hr(t),
|
|
2210
2307
|
to: n
|
|
2211
2308
|
};
|
|
2212
2309
|
case "mssql": return {
|
|
@@ -2214,17 +2311,17 @@ var Br = (e, t) => ({
|
|
|
2214
2311
|
tool: "sqlpackage",
|
|
2215
2312
|
args: [
|
|
2216
2313
|
"/Action:Export",
|
|
2217
|
-
`/SourceConnectionString:${
|
|
2314
|
+
`/SourceConnectionString:${Ur(Z(t, 1433))}`,
|
|
2218
2315
|
`/TargetFile:${n}`
|
|
2219
2316
|
],
|
|
2220
2317
|
env: {}
|
|
2221
2318
|
};
|
|
2222
|
-
default: throw new
|
|
2319
|
+
default: throw new U({
|
|
2223
2320
|
tool: "backup",
|
|
2224
2321
|
reason: `no native backup for dialect '${e}'`
|
|
2225
2322
|
});
|
|
2226
2323
|
}
|
|
2227
|
-
},
|
|
2324
|
+
}, Gr = [u, l], Kr = Gr.flatMap((e) => ["--exclude-table", e]), qr = (e) => e === "postgres" || e === "mysql" || e === "mariadb" ? Gr : [], Jr = (e, t, n) => {
|
|
2228
2325
|
switch (e) {
|
|
2229
2326
|
case "postgres": {
|
|
2230
2327
|
if (t.url) return {
|
|
@@ -2266,7 +2363,7 @@ var Br = (e, t) => ({
|
|
|
2266
2363
|
let r = Z(t);
|
|
2267
2364
|
return {
|
|
2268
2365
|
kind: "spawn",
|
|
2269
|
-
...
|
|
2366
|
+
...zr(e, "client"),
|
|
2270
2367
|
args: [
|
|
2271
2368
|
"--protocol=TCP",
|
|
2272
2369
|
"-h",
|
|
@@ -2285,10 +2382,10 @@ var Br = (e, t) => ({
|
|
|
2285
2382
|
case "turso": return {
|
|
2286
2383
|
kind: "copy",
|
|
2287
2384
|
from: n,
|
|
2288
|
-
to:
|
|
2385
|
+
to: Hr(t)
|
|
2289
2386
|
};
|
|
2290
2387
|
case "mssql": {
|
|
2291
|
-
let e = Z(t, 1433), r =
|
|
2388
|
+
let e = Z(t, 1433), r = Xr(e.database);
|
|
2292
2389
|
return {
|
|
2293
2390
|
kind: "mssql-replace",
|
|
2294
2391
|
load: {
|
|
@@ -2296,7 +2393,7 @@ var Br = (e, t) => ({
|
|
|
2296
2393
|
tool: "sqlpackage",
|
|
2297
2394
|
args: [
|
|
2298
2395
|
"/Action:Import",
|
|
2299
|
-
`/TargetConnectionString:${
|
|
2396
|
+
`/TargetConnectionString:${Ur({
|
|
2300
2397
|
...e,
|
|
2301
2398
|
database: r
|
|
2302
2399
|
})}`,
|
|
@@ -2313,25 +2410,25 @@ var Br = (e, t) => ({
|
|
|
2313
2410
|
},
|
|
2314
2411
|
live: e.database,
|
|
2315
2412
|
scratch: r,
|
|
2316
|
-
carry:
|
|
2413
|
+
carry: Gr
|
|
2317
2414
|
};
|
|
2318
2415
|
}
|
|
2319
|
-
default: throw new
|
|
2416
|
+
default: throw new U({
|
|
2320
2417
|
tool: "restore",
|
|
2321
2418
|
reason: `no native restore for dialect '${e}'`
|
|
2322
2419
|
});
|
|
2323
2420
|
}
|
|
2324
|
-
}, Q = (e) => `[${e.replaceAll("]", "]]")}]`, $ = (e) => `N'${e.replaceAll("'", "''")}'`,
|
|
2325
|
-
DECLARE leftovers CURSOR LOCAL FAST_FORWARD FOR SELECT name FROM sys.databases WHERE name LIKE ${$(`${`${e}${
|
|
2421
|
+
}, Q = (e) => `[${e.replaceAll("]", "]]")}]`, $ = (e) => `N'${e.replaceAll("'", "''")}'`, Yr = "__voltro_restore_", Xr = (e, t = se().slice(0, 8)) => `${e}${Yr}${t}`, Zr = (e) => e.replace(Yr, "__voltro_replaced_"), Qr = (e) => `IF DB_ID(${$(e)}) IS NOT NULL BEGIN ALTER DATABASE ${Q(e)} SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE ${Q(e)} END`, $r = (e) => `DECLARE @leftover sysname, @drop nvarchar(max);
|
|
2422
|
+
DECLARE leftovers CURSOR LOCAL FAST_FORWARD FOR SELECT name FROM sys.databases WHERE name LIKE ${$(`${`${e}${Yr}`.replace(/[\\%_[]/g, (e) => `\\${e}`)}%`)} ESCAPE N'\\';
|
|
2326
2423
|
OPEN leftovers; FETCH NEXT FROM leftovers INTO @leftover;
|
|
2327
2424
|
WHILE @@FETCH_STATUS = 0 BEGIN
|
|
2328
2425
|
SET @drop = N'ALTER DATABASE ' + QUOTENAME(@leftover) + N' SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE ' + QUOTENAME(@leftover);
|
|
2329
2426
|
EXEC(@drop);
|
|
2330
2427
|
FETCH NEXT FROM leftovers INTO @leftover;
|
|
2331
|
-
END`,
|
|
2332
|
-
let t = Q(e.live), n = Q(e.scratch), r =
|
|
2428
|
+
END`, ei = (e) => {
|
|
2429
|
+
let t = Q(e.live), n = Q(e.scratch), r = Zr(e.scratch);
|
|
2333
2430
|
return {
|
|
2334
|
-
prepare: [
|
|
2431
|
+
prepare: [$r(e.live)],
|
|
2335
2432
|
carry: e.carry.map((e) => {
|
|
2336
2433
|
let r = $(`${n}.dbo.${Q(e)}`), i = $(`${t}.dbo.${Q(e)}`);
|
|
2337
2434
|
return `IF OBJECT_ID(${r}) IS NOT NULL BEGIN
|
|
@@ -2348,41 +2445,41 @@ END`;
|
|
|
2348
2445
|
`ALTER DATABASE ${t} SET SINGLE_USER WITH ROLLBACK IMMEDIATE`,
|
|
2349
2446
|
`ALTER DATABASE ${t} MODIFY NAME = ${Q(r)}`,
|
|
2350
2447
|
`ALTER DATABASE ${n} MODIFY NAME = ${t}`,
|
|
2351
|
-
|
|
2448
|
+
Qr(r)
|
|
2352
2449
|
],
|
|
2353
2450
|
undoSwap: [`IF DB_ID(${$(e.live)}) IS NULL AND DB_ID(${$(r)}) IS NOT NULL ALTER DATABASE ${Q(r)} MODIFY NAME = ${t}`, `IF DB_ID(${$(e.live)}) IS NOT NULL ALTER DATABASE ${t} SET MULTI_USER`],
|
|
2354
|
-
cleanup: [
|
|
2451
|
+
cleanup: [Qr(e.scratch)]
|
|
2355
2452
|
};
|
|
2356
|
-
},
|
|
2453
|
+
}, ti = (e) => {
|
|
2357
2454
|
let t = [];
|
|
2358
2455
|
for (let n = e, r = 0; n != null && r < 4; r++) {
|
|
2359
2456
|
let e = n instanceof Error ? n.message : typeof n == "object" && "message" in n ? String(n.message) : String(n);
|
|
2360
2457
|
e && !t.includes(e) && t.push(e), n = typeof n == "object" && "cause" in n ? n.cause : void 0;
|
|
2361
2458
|
}
|
|
2362
2459
|
return t.join(": ");
|
|
2363
|
-
},
|
|
2364
|
-
if (t === void 0) return n.fail(new
|
|
2460
|
+
}, ni = (e, t) => {
|
|
2461
|
+
if (t === void 0) return n.fail(new U({
|
|
2365
2462
|
tool: "sqlpackage",
|
|
2366
2463
|
reason: "an mssql restore swaps databases and needs a SQL connection to the server; none was provided"
|
|
2367
2464
|
}));
|
|
2368
|
-
let r =
|
|
2465
|
+
let r = ei(e), i = (r, i) => t(e.server, i).pipe(n.mapError((e) => new U({
|
|
2369
2466
|
tool: "sqlpackage",
|
|
2370
|
-
reason: `${r}: ${
|
|
2467
|
+
reason: `${r}: ${ti(e)}`
|
|
2371
2468
|
}))), a = i("dropping the scratch database", r.cleanup).pipe(n.ignore);
|
|
2372
2469
|
return n.gen(function* () {
|
|
2373
|
-
yield* i(`preparing ${e.scratch}`, r.prepare), yield*
|
|
2470
|
+
yield* i(`preparing ${e.scratch}`, r.prepare), yield* ri(e.load).pipe(n.tapError(() => a)), yield* i(`carrying ${e.carry.join(", ")} into the imported copy`, r.carry).pipe(n.tapError(() => a)), yield* i(`swapping ${e.scratch} in as ${e.live}`, r.swap).pipe(n.tapError(() => i("putting the original database back", r.undoSwap).pipe(n.ignore)));
|
|
2374
2471
|
});
|
|
2375
|
-
},
|
|
2376
|
-
let r = `${e.to}.restoring-${
|
|
2472
|
+
}, ri = (e, t) => e.kind === "mssql-replace" ? ni(e, t) : e.kind === "copy" ? n.callback((t) => {
|
|
2473
|
+
let r = `${e.to}.restoring-${se()}`, i = (e) => {
|
|
2377
2474
|
try {
|
|
2378
|
-
|
|
2475
|
+
k(r);
|
|
2379
2476
|
} catch {}
|
|
2380
|
-
t(n.fail(new
|
|
2477
|
+
t(n.fail(new U({
|
|
2381
2478
|
tool: "copy",
|
|
2382
2479
|
reason: e
|
|
2383
2480
|
})));
|
|
2384
2481
|
};
|
|
2385
|
-
|
|
2482
|
+
le(e.from, r, (a) => {
|
|
2386
2483
|
if (a) {
|
|
2387
2484
|
i(`copy ${e.from} → ${e.to}: ${a.message}`);
|
|
2388
2485
|
return;
|
|
@@ -2392,7 +2489,7 @@ END`;
|
|
|
2392
2489
|
`${e.to}-shm`,
|
|
2393
2490
|
`${e.to}-journal`
|
|
2394
2491
|
]) try {
|
|
2395
|
-
|
|
2492
|
+
k(t);
|
|
2396
2493
|
} catch {}
|
|
2397
2494
|
D(r, e.to, (r) => {
|
|
2398
2495
|
if (r) {
|
|
@@ -2402,8 +2499,8 @@ END`;
|
|
|
2402
2499
|
t(n.void);
|
|
2403
2500
|
});
|
|
2404
2501
|
});
|
|
2405
|
-
}) : n.
|
|
2406
|
-
let r =
|
|
2502
|
+
}) : n.callback((t) => {
|
|
2503
|
+
let r = we(e.tool, [...e.args], {
|
|
2407
2504
|
env: {
|
|
2408
2505
|
...process.env,
|
|
2409
2506
|
...e.env
|
|
@@ -2423,27 +2520,27 @@ END`;
|
|
|
2423
2520
|
s ??= e;
|
|
2424
2521
|
};
|
|
2425
2522
|
if (e.stdoutFile && r.stdout) {
|
|
2426
|
-
let t =
|
|
2427
|
-
t.on("error", (t) => c(`writing ${
|
|
2523
|
+
let t = de(e.stdoutFile);
|
|
2524
|
+
t.on("error", (t) => c(`writing ${P(e.stdoutFile)}: ${t.message}`)), r.stdout.on("error", (e) => c(`reading the tool's output: ${e.message}`)), r.stdout.pipe(t);
|
|
2428
2525
|
}
|
|
2429
2526
|
let l = 0;
|
|
2430
2527
|
if (e.stdinFile && r.stdin) {
|
|
2431
|
-
let t =
|
|
2528
|
+
let t = ue(e.stdinFile);
|
|
2432
2529
|
t.on("data", (e) => {
|
|
2433
2530
|
l += e.length;
|
|
2434
2531
|
}), t.on("error", (t) => {
|
|
2435
|
-
c(`reading ${
|
|
2532
|
+
c(`reading ${P(e.stdinFile)}: ${t.message}`), r.stdin?.end();
|
|
2436
2533
|
}), r.stdin.on("error", (t) => {
|
|
2437
|
-
t.code !== "EPIPE" && c(`feeding ${
|
|
2534
|
+
t.code !== "EPIPE" && c(`feeding ${P(e.stdinFile)} to ${e.tool}: ${t.message}`);
|
|
2438
2535
|
}), t.pipe(r.stdin);
|
|
2439
2536
|
}
|
|
2440
2537
|
let u = () => {
|
|
2441
2538
|
if (e.stdoutFile !== void 0) try {
|
|
2442
|
-
|
|
2539
|
+
k(e.stdoutFile);
|
|
2443
2540
|
} catch {}
|
|
2444
2541
|
};
|
|
2445
2542
|
r.on("error", (t) => {
|
|
2446
|
-
u(), o(n.fail(new
|
|
2543
|
+
u(), o(n.fail(new U({
|
|
2447
2544
|
tool: e.tool,
|
|
2448
2545
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${t.message}`,
|
|
2449
2546
|
...e.hint === void 0 ? {} : { hint: e.hint }
|
|
@@ -2454,45 +2551,45 @@ END`;
|
|
|
2454
2551
|
return;
|
|
2455
2552
|
}
|
|
2456
2553
|
u();
|
|
2457
|
-
let r = e.stdinFile === void 0 ? "" : ` — ${l} of ${
|
|
2458
|
-
o(n.fail(new
|
|
2554
|
+
let r = e.stdinFile === void 0 ? "" : ` — ${l} of ${ii(e.stdinFile)} byte(s) of ${P(e.stdinFile)} had been fed to it`;
|
|
2555
|
+
o(n.fail(new U({
|
|
2459
2556
|
tool: e.tool,
|
|
2460
|
-
reason: (t === 0 ? `exited cleanly, but ${s}` : `exited with code ${t}${s === void 0 ? "" : ` (${s})`}`) + r + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2557
|
+
reason: (t === 0 ? `exited cleanly, but ${s}` : `exited with code ${t}${s === void 0 ? "" : ` (${s})`}`) + r + (e.stdoutFile === void 0 ? "" : ` — the partial ${P(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
2461
2558
|
stderr: i.trim() === "" ? `(${e.tool} printed nothing to stderr)` : i.slice(-2e3),
|
|
2462
2559
|
...e.hint === void 0 ? {} : { hint: e.hint }
|
|
2463
2560
|
})));
|
|
2464
2561
|
});
|
|
2465
|
-
}),
|
|
2562
|
+
}), ii = (e) => {
|
|
2466
2563
|
try {
|
|
2467
|
-
return String(
|
|
2564
|
+
return String(O(e).size);
|
|
2468
2565
|
} catch {
|
|
2469
2566
|
return "?";
|
|
2470
2567
|
}
|
|
2471
|
-
},
|
|
2472
|
-
let t = yield*
|
|
2568
|
+
}, ai = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), oi = (e) => n.gen(function* () {
|
|
2569
|
+
let t = yield* ct(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
2473
2570
|
store: e.store,
|
|
2474
2571
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
2475
|
-
}), r =
|
|
2572
|
+
}), r = on(t.tables, e.snapshot), i = e.classification ?? De(e.snapshot), a = e.sampleSize ?? 20, o = Ve(r.map((t) => ({
|
|
2476
2573
|
name: t,
|
|
2477
2574
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
2478
2575
|
name: e.name,
|
|
2479
2576
|
type: e.type
|
|
2480
2577
|
}))
|
|
2481
|
-
})), i, e.masking), s = [],
|
|
2578
|
+
})), i, e.masking), s = [], l = [];
|
|
2482
2579
|
for (let i of r) {
|
|
2483
2580
|
let r = t.predicateFor(i), u = yield* C(e.store, {
|
|
2484
2581
|
table: i,
|
|
2485
2582
|
chunkSize: a,
|
|
2486
2583
|
...r === void 0 ? {} : { where: r }
|
|
2487
|
-
}).pipe(
|
|
2488
|
-
|
|
2584
|
+
}).pipe(c.take(a), c.runCollect, n.map((e) => [...e])), d = u.map((t) => We(t, i, o.actions, e.masking.seed));
|
|
2585
|
+
l.push(...Ue(d, i, o.actions));
|
|
2489
2586
|
let f = u[0], p = d[0], m = [];
|
|
2490
2587
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
2491
2588
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
2492
2589
|
let n = e.slice(i.length + 1);
|
|
2493
2590
|
m.push({
|
|
2494
2591
|
column: n,
|
|
2495
|
-
action:
|
|
2592
|
+
action: ai(t),
|
|
2496
2593
|
before: f[n],
|
|
2497
2594
|
after: p[n]
|
|
2498
2595
|
});
|
|
@@ -2504,10 +2601,10 @@ END`;
|
|
|
2504
2601
|
}
|
|
2505
2602
|
return {
|
|
2506
2603
|
tables: s,
|
|
2507
|
-
leaks:
|
|
2604
|
+
leaks: l,
|
|
2508
2605
|
unclassified: o.unclassified
|
|
2509
2606
|
};
|
|
2510
|
-
}),
|
|
2607
|
+
}), si = (e, t) => {
|
|
2511
2608
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
2512
2609
|
for (let t of e) {
|
|
2513
2610
|
let e = n.get(t.name);
|
|
@@ -2562,7 +2659,7 @@ END`;
|
|
|
2562
2659
|
skipColumns: i,
|
|
2563
2660
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
2564
2661
|
};
|
|
2565
|
-
},
|
|
2662
|
+
}, ci = (e) => {
|
|
2566
2663
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
2567
2664
|
let t = {
|
|
2568
2665
|
safe: "✓",
|
|
@@ -2573,4 +2670,4 @@ END`;
|
|
|
2573
2670
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
2574
2671
|
};
|
|
2575
2672
|
//#endregion
|
|
2576
|
-
export {
|
|
2673
|
+
export { Nt as ASSETS_DIR, Pt as ASSET_INDEX_FILE, H as BundleError, Ye as CodecError, Ze as CompressionError, rt as CrossDialectError, Mt as DATA_DIR, Ee as DEFAULT_CLASS_ACTIONS, Ge as ENVIRONMENT_LOCAL_TABLES, Ke as ENVIRONMENT_PORTABLE_TABLES, Gr as EXCLUDED_FROM_DUMP, Ct as FORMAT_VERSION, nt as ImportModeError, Xe as IntegrityError, Ft as LEDGER_FILE, At as MANIFEST_FILE, tt as MaskingError, U as NativeToolError, $e as RowsRefusedError, nr as SAVEPOINT_BATCH_SIZE, Mn as STAGING_PREFIX, et as SchemaDriftError, Qe as ScopeError, B as applyAction, ir as asImportError, lr as assessLiveImport, Gt as assetDone, Vr as backupArtifactName, Wr as backupCommand, Nn as canStage, bn as canonicalType, ht as captureAssetsToCas, Cn as checkPortability, $n as checkSchemaDrift, De as classificationFromSnapshot, si as classifyImportDrift, bt as compressionExt, at as computeSubsetIds, Jn as decideStaging, kt as decodeManifest, tn as decodeRow, $t as decodeValue, Fr as decryptStream, cr as defineDataProfile, Qn as diffSnapshots, Cr as dirBlobSink, Ln as dropStagingSql, qr as dumpExcludedTables, Ht as emptyLedger, Ot as encodeManifest, en as encodeRow, Zt as encodeValue, Pr as encryptStream, Je as environmentLocalNotice, qe as environmentLocalReason, Un as execStatements, ci as formatImportDrift, wn as formatIssue, vt as hasZstd, V as isEnvironmentLocal, xt as makeCompressor, St as makeDecompressor, We as maskRow, ei as mssqlReplaceStatements, Xr as mssqlScratchName, an as ndjsonToRows, pt as noAssetProgress, yn as outsideReferenceRefusal, vn as outsideReferencesInto, _r as packBundle, xr as peekBundle, yt as pickCompression, Ve as planMasking, oi as previewMasking, gt as readAssetIndex, Ut as readLedger, Oe as resolveAction, ct as resolveScope, ft as restoreAssetFromCas, _t as restoreAssetsFromCas, Jr as restoreCommand, rn as rowsToNdjson, _n as runExport, sr as runImport, ri as runNativeStep, Ue as scanForLeaks, q as stagingNameFor, ut as storageAssetSink, lt as storageAssetSource, Rr as storageBlobSink, Ir as storageBlobSource, Lr as storageRefsBlobSource, ot as subsetToScope, Wt as tableDone, on as topoSortTables, br as unpackBundle, Sr as unpackBundleToDir, dt as writeAssetToCas, G as writeLedger };
|