@voltro/data-transfer 0.63.0 → 0.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +90 -0
- package/dist/index.d.ts +21 -1
- package/dist/index.js +520 -460
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Runtime as o, Schedule as s, Schema as c, Stream as l } from "effect";
|
|
2
|
-
import { DATA_TRANSFERS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d, STAGING_TABLE_PREFIX as f, _voltroDataTransfersTable as p, classifyConstraintViolation as m, derivedRowId as h, ensureTableRegistered as g, eq as _, extractDbCause as v, getTable as y, inSet as b, isNotNull as x, or as S, recordsTable as
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { copyFile as
|
|
6
|
-
import { STORAGE_REFS_TABLE as
|
|
7
|
-
import * as
|
|
8
|
-
import { PassThrough as
|
|
9
|
-
import { SqlClient as
|
|
10
|
-
import { mkdir as
|
|
11
|
-
import { basename as
|
|
12
|
-
import { promisify as
|
|
13
|
-
import { spawn as
|
|
2
|
+
import { DATA_TRANSFERS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d, STAGING_TABLE_PREFIX as f, _voltroDataTransfersTable as p, classifyConstraintViolation as m, derivedRowId as h, ensureTableRegistered as g, eq as _, extractDbCause as v, getTable as y, inSet as b, isNotNull as x, or as S, recordsTable as C, registerTable as ee, streamTable as w, suspendWriteRecorders as te, suspendWriteRecordersEffect as ne, unregisterTable as re } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as T } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as E, createDecipheriv as ie, createHash as ae, createHmac as oe, randomBytes as se, randomUUID as ce, scrypt as le } from "node:crypto";
|
|
5
|
+
import { copyFile as D, createReadStream as ue, createWriteStream as O, existsSync as de, rename as fe, statSync as k, unlinkSync as pe } from "node:fs";
|
|
6
|
+
import { STORAGE_REFS_TABLE as A, fromNodeReadable as me, getObjectStream as j, putObjectStream as he, toNodeReadable as M } from "@voltro/plugin-storage";
|
|
7
|
+
import * as N from "node:zlib";
|
|
8
|
+
import { PassThrough as P, Readable as ge } from "node:stream";
|
|
9
|
+
import { SqlClient as F } from "@effect/sql";
|
|
10
|
+
import { mkdir as I, readdir as L, stat as _e } from "node:fs/promises";
|
|
11
|
+
import { basename as R, delimiter as z, dirname as ve, join as B, resolve as ye, sep as be } from "node:path";
|
|
12
|
+
import { promisify as xe } from "node:util";
|
|
13
|
+
import { spawn as Se } from "node:child_process";
|
|
14
14
|
//#region src/consistency.ts
|
|
15
|
-
var
|
|
15
|
+
var Ce = (e) => {
|
|
16
16
|
switch (e) {
|
|
17
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
18
18
|
case "mysql":
|
|
@@ -22,7 +22,7 @@ var Te = (e) => {
|
|
|
22
22
|
case "turso": return [];
|
|
23
23
|
default: return [];
|
|
24
24
|
}
|
|
25
|
-
},
|
|
25
|
+
}, we = {
|
|
26
26
|
email: { fake: "email" },
|
|
27
27
|
fullName: { fake: "fullName" },
|
|
28
28
|
firstName: { fake: "firstName" },
|
|
@@ -37,7 +37,7 @@ var Te = (e) => {
|
|
|
37
37
|
date: "dateShift",
|
|
38
38
|
secret: "null",
|
|
39
39
|
freeText: "redact"
|
|
40
|
-
},
|
|
40
|
+
}, Te = (e) => {
|
|
41
41
|
let t = {};
|
|
42
42
|
for (let n of e.tables) {
|
|
43
43
|
let e = {};
|
|
@@ -49,15 +49,15 @@ var Te = (e) => {
|
|
|
49
49
|
t[n.name] = e;
|
|
50
50
|
}
|
|
51
51
|
return t;
|
|
52
|
-
},
|
|
52
|
+
}, Ee = (e, t, n, r) => {
|
|
53
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
54
54
|
if (i) return i;
|
|
55
55
|
if (n?.sensitive) {
|
|
56
56
|
let e = n.sensitive.class;
|
|
57
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? we[e] ?? "redact";
|
|
58
58
|
}
|
|
59
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
60
|
-
},
|
|
60
|
+
}, De = (e, t) => oe("sha256", e).update(t == null ? "∅" : String(t)).digest(), V = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), Oe = [
|
|
61
61
|
"Ada",
|
|
62
62
|
"Alan",
|
|
63
63
|
"Grace",
|
|
@@ -70,7 +70,7 @@ var Te = (e) => {
|
|
|
70
70
|
"Guido",
|
|
71
71
|
"Margaret",
|
|
72
72
|
"Bjarne"
|
|
73
|
-
],
|
|
73
|
+
], ke = [
|
|
74
74
|
"Lovelace",
|
|
75
75
|
"Turing",
|
|
76
76
|
"Hopper",
|
|
@@ -83,7 +83,7 @@ var Te = (e) => {
|
|
|
83
83
|
"Rossum",
|
|
84
84
|
"Hamilton",
|
|
85
85
|
"Stroustrup"
|
|
86
|
-
],
|
|
86
|
+
], Ae = [
|
|
87
87
|
"Maple",
|
|
88
88
|
"Oak",
|
|
89
89
|
"Cedar",
|
|
@@ -92,7 +92,7 @@ var Te = (e) => {
|
|
|
92
92
|
"Birch",
|
|
93
93
|
"Willow",
|
|
94
94
|
"Ash"
|
|
95
|
-
],
|
|
95
|
+
], je = [
|
|
96
96
|
"Initech",
|
|
97
97
|
"Hooli",
|
|
98
98
|
"Umbrella",
|
|
@@ -101,23 +101,23 @@ var Te = (e) => {
|
|
|
101
101
|
"Soylent",
|
|
102
102
|
"Stark",
|
|
103
103
|
"Wayne"
|
|
104
|
-
],
|
|
105
|
-
let r =
|
|
104
|
+
], H = (e, t, n) => e[V(t, n) % e.length], Me = (e, t, n) => {
|
|
105
|
+
let r = De(n, `${e}:${String(t)}`), i = V(r, 1) % 1e4;
|
|
106
106
|
switch (e) {
|
|
107
|
-
case "email": return `${
|
|
108
|
-
case "fullName": return `${
|
|
109
|
-
case "firstName": return
|
|
110
|
-
case "lastName": return
|
|
111
|
-
case "username": return `${
|
|
112
|
-
case "phone": return `+1${String(
|
|
113
|
-
case "address": return `${
|
|
114
|
-
case "company": return
|
|
107
|
+
case "email": return `${H(Oe, r, 0).toLowerCase()}.${H(ke, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${H(Oe, r, 0)} ${H(ke, r, 1)}`;
|
|
109
|
+
case "firstName": return H(Oe, r, 0);
|
|
110
|
+
case "lastName": return H(ke, r, 1);
|
|
111
|
+
case "username": return `${H(Oe, r, 0).toLowerCase()}${i}`;
|
|
112
|
+
case "phone": return `+1${String(V(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
+
case "address": return `${V(r, 0) % 9899 + 100} ${H(Ae, r, 1)} St`;
|
|
114
|
+
case "company": return H(je, r, 0);
|
|
115
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
116
|
-
case "ip": return `${
|
|
117
|
-
case "creditCard": return
|
|
116
|
+
case "ip": return `${V(r, 0) % 256}.${V(r, 1) % 256}.${V(r, 2) % 256}.${V(r, 3) % 254 + 1}`;
|
|
117
|
+
case "creditCard": return Ne(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
118
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
119
119
|
}
|
|
120
|
-
},
|
|
120
|
+
}, Ne = (e) => {
|
|
121
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
122
122
|
t.forEach((e, t) => {
|
|
123
123
|
let r = e;
|
|
@@ -125,31 +125,91 @@ var Te = (e) => {
|
|
|
125
125
|
});
|
|
126
126
|
let r = (10 - n % 10) % 10;
|
|
127
127
|
return e.slice(0, 15) + String(r);
|
|
128
|
-
},
|
|
128
|
+
}, Pe = 864e5, Fe = (e, t) => {
|
|
129
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
130
130
|
if (Number.isNaN(n)) return e;
|
|
131
|
-
let r =
|
|
131
|
+
let r = V(De(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Pe);
|
|
132
132
|
return e instanceof Date ? i : i.toISOString();
|
|
133
|
-
},
|
|
133
|
+
}, U = (e, t) => {
|
|
134
134
|
if (t.value === null || t.value === void 0) return null;
|
|
135
135
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
136
136
|
if (e === "keep") return t.value;
|
|
137
137
|
if (e === "null") return null;
|
|
138
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
139
|
-
if (e === "hash") return
|
|
140
|
-
if (e === "dateShift") return
|
|
141
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return De(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Fe(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return Me(e.fake, t.value, t.seed);
|
|
142
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
143
|
-
|
|
144
|
-
}
|
|
143
|
+
if (typeof e == "object" && "json" in e) return Ie(e, t);
|
|
144
|
+
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn } | { json: { fields, rest? } }.`);
|
|
145
|
+
}, Ie = (e, t) => {
|
|
146
|
+
let { fields: n, rest: r = "redact" } = e.json, i = typeof t.value == "string" ? Le(t.value) : {
|
|
147
|
+
ok: !0,
|
|
148
|
+
value: t.value
|
|
149
|
+
};
|
|
150
|
+
if (!i.ok) return U(r, {
|
|
151
|
+
...t,
|
|
152
|
+
columnType: "text"
|
|
153
|
+
});
|
|
154
|
+
let a = (e, i) => {
|
|
155
|
+
if (e == null) return null;
|
|
156
|
+
if (Array.isArray(e)) return e.map((e, t) => a(e, `${i}[${t}]`));
|
|
157
|
+
if (typeof e == "object") {
|
|
158
|
+
let a = {};
|
|
159
|
+
for (let [o, s] of Object.entries(e)) {
|
|
160
|
+
let e = n[o] ?? r, c = `${i}.${o}`;
|
|
161
|
+
a[o] = typeof e == "object" && "json" in e ? Ie(e, {
|
|
162
|
+
...t,
|
|
163
|
+
value: s,
|
|
164
|
+
column: c,
|
|
165
|
+
columnType: "json"
|
|
166
|
+
}) : U(e, {
|
|
167
|
+
...t,
|
|
168
|
+
value: s,
|
|
169
|
+
column: c,
|
|
170
|
+
columnType: typeof s == "string" ? "text" : "json"
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
return a;
|
|
174
|
+
}
|
|
175
|
+
return U(r, {
|
|
176
|
+
...t,
|
|
177
|
+
value: e,
|
|
178
|
+
column: i,
|
|
179
|
+
columnType: typeof e == "string" ? "text" : "json"
|
|
180
|
+
});
|
|
181
|
+
}, o = a(i.value, t.column);
|
|
182
|
+
return typeof t.value == "string" ? JSON.stringify(o) : o;
|
|
183
|
+
}, Le = (e) => {
|
|
184
|
+
try {
|
|
185
|
+
return {
|
|
186
|
+
ok: !0,
|
|
187
|
+
value: JSON.parse(e)
|
|
188
|
+
};
|
|
189
|
+
} catch {
|
|
190
|
+
return { ok: !1 };
|
|
191
|
+
}
|
|
192
|
+
}, Re = (e) => {
|
|
193
|
+
if (typeof e == "string") return e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error";
|
|
194
|
+
if (typeof e != "object" || !e) return !1;
|
|
195
|
+
if ("fake" in e) return typeof e.fake == "string";
|
|
196
|
+
if ("custom" in e) return typeof e.custom == "function";
|
|
197
|
+
if ("json" in e) {
|
|
198
|
+
let t = e.json;
|
|
199
|
+
if (typeof t != "object" || !t) return !1;
|
|
200
|
+
let { fields: n, rest: r } = t;
|
|
201
|
+
return typeof n != "object" || !n ? !1 : Object.values(n).every(Re) && (r === void 0 || Re(r));
|
|
202
|
+
}
|
|
203
|
+
return !1;
|
|
204
|
+
}, ze = (e, t, n) => {
|
|
145
205
|
let r = {}, i = [], a = [];
|
|
146
206
|
for (let o of e) for (let e of o.columns) {
|
|
147
|
-
let s = `${o.name}.${e.name}`, c =
|
|
207
|
+
let s = `${o.name}.${e.name}`, c = Ee(o.name, e.name, t[o.name]?.[e.name], n);
|
|
148
208
|
if (c === "error") {
|
|
149
209
|
i.push(s);
|
|
150
210
|
continue;
|
|
151
211
|
}
|
|
152
|
-
if (!
|
|
212
|
+
if (!Re(c)) {
|
|
153
213
|
a.push({
|
|
154
214
|
column: s,
|
|
155
215
|
action: c
|
|
@@ -163,7 +223,7 @@ var Te = (e) => {
|
|
|
163
223
|
unclassified: i,
|
|
164
224
|
invalid: a
|
|
165
225
|
};
|
|
166
|
-
},
|
|
226
|
+
}, Be = [
|
|
167
227
|
{
|
|
168
228
|
name: "email",
|
|
169
229
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -180,7 +240,7 @@ var Te = (e) => {
|
|
|
180
240
|
name: "creditCard",
|
|
181
241
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
182
242
|
}
|
|
183
|
-
],
|
|
243
|
+
], Ve = (e, t, n) => {
|
|
184
244
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
185
245
|
for (let [a, o] of Object.entries(n)) {
|
|
186
246
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -188,7 +248,7 @@ var Te = (e) => {
|
|
|
188
248
|
for (let a of e) {
|
|
189
249
|
let e = a[n];
|
|
190
250
|
if (typeof e != "string" || e.length === 0) continue;
|
|
191
|
-
let o =
|
|
251
|
+
let o = Be.find((t) => t.re.test(e));
|
|
192
252
|
if (o && !i.has(n)) {
|
|
193
253
|
i.add(n), r.push({
|
|
194
254
|
table: t,
|
|
@@ -201,9 +261,9 @@ var Te = (e) => {
|
|
|
201
261
|
}
|
|
202
262
|
}
|
|
203
263
|
return r;
|
|
204
|
-
},
|
|
264
|
+
}, He = (e, t, n, r, i) => {
|
|
205
265
|
let a = {};
|
|
206
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
266
|
+
for (let [o, s] of Object.entries(e)) a[o] = U(n[`${t}.${o}`] ?? "keep", {
|
|
207
267
|
value: s,
|
|
208
268
|
table: t,
|
|
209
269
|
column: o,
|
|
@@ -211,7 +271,7 @@ var Te = (e) => {
|
|
|
211
271
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
212
272
|
});
|
|
213
273
|
return a;
|
|
214
|
-
},
|
|
274
|
+
}, Ue = /* @__PURE__ */ new Map([
|
|
215
275
|
["_voltro_migration_plans", "the schema fingerprint THIS deployment applied — a foreign row makes the next boot refuse"],
|
|
216
276
|
["_voltro_migrations", "which file-based migrations ran HERE — a foreign row marks unrun migrations as done"],
|
|
217
277
|
["_voltro_seeds", "which seeds ran HERE — a foreign row suppresses seeding the target still needs"],
|
|
@@ -246,7 +306,7 @@ var Te = (e) => {
|
|
|
246
306
|
["_voltro_notification_held", "notifications waiting to be SENT from here — a foreign row makes the target deliver something queued elsewhere"],
|
|
247
307
|
["_voltro_webhook_event_stats", "per-deployment delivery counters a backoff decision reads — foreign counts steer the target's retries"],
|
|
248
308
|
["_voltro_webhook_rate_windows", "the live rate-limit windows this deployment is inside — a foreign window throttles or releases wrongly"]
|
|
249
|
-
]),
|
|
309
|
+
]), We = /* @__PURE__ */ new Map([
|
|
250
310
|
["_voltro_agent_threads", "conversations the app's users had — app data"],
|
|
251
311
|
["_voltro_agent_messages", "the messages in those conversations — app data"],
|
|
252
312
|
["_voltro_prompts", "prompt definitions the app authored — app data"],
|
|
@@ -281,21 +341,21 @@ var Te = (e) => {
|
|
|
281
341
|
["_voltro_comment_threads", "the threads those comments hang on — app data"],
|
|
282
342
|
["_voltro_comment_reactions", "the reactions the app's users left — app data"],
|
|
283
343
|
["_voltro_comment_reads", "which comments each user has read — app data, and losing it re-marks everything unread"]
|
|
284
|
-
]),
|
|
344
|
+
]), W = (e) => Ue.has(e), Ge = (e) => Ue.get(e), Ke = (e, t) => {
|
|
285
345
|
if (e.length === 0) return;
|
|
286
|
-
let n = e.map((e) => ` ${e} — ${
|
|
346
|
+
let n = e.map((e) => ` ${e} — ${Ge(e) ?? "describes this deployment"}`);
|
|
287
347
|
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
288
|
-
},
|
|
348
|
+
}, G = class extends c.TaggedError()("BundleError", {
|
|
289
349
|
reason: c.String,
|
|
290
350
|
path: c.optional(c.String)
|
|
291
|
-
}) {},
|
|
351
|
+
}) {}, qe = class extends c.TaggedError()("CodecError", {
|
|
292
352
|
table: c.String,
|
|
293
353
|
reason: c.String
|
|
294
|
-
}) {},
|
|
354
|
+
}) {}, Je = class extends c.TaggedError()("IntegrityError", {
|
|
295
355
|
what: c.String,
|
|
296
356
|
expected: c.String,
|
|
297
357
|
actual: c.String
|
|
298
|
-
}) {},
|
|
358
|
+
}) {}, Ye = class extends t.TaggedError("CompressionError") {}, Xe = class extends c.TaggedError()("ScopeError", { reason: c.String }) {}, Ze = class extends c.TaggedError()("RowsRefusedError", {
|
|
299
359
|
rows: c.Array(c.Struct({
|
|
300
360
|
table: c.String,
|
|
301
361
|
id: c.String,
|
|
@@ -313,7 +373,7 @@ var Te = (e) => {
|
|
|
313
373
|
bundleFingerprint: c.String,
|
|
314
374
|
targetFingerprint: c.String,
|
|
315
375
|
diff: c.Array(c.String)
|
|
316
|
-
}) {},
|
|
376
|
+
}) {}, K = class extends t.TaggedError("NativeToolError") {
|
|
317
377
|
get message() {
|
|
318
378
|
let e = this.stderr?.trim(), t = this.hint?.trim();
|
|
319
379
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}${t ? `\n\nhint: ${t}` : ""}`;
|
|
@@ -328,10 +388,10 @@ var Te = (e) => {
|
|
|
328
388
|
sourceDialect: c.String,
|
|
329
389
|
targetDialect: c.String,
|
|
330
390
|
blocking: c.Array(c.String)
|
|
331
|
-
}) {},
|
|
391
|
+
}) {}, nt = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", rt = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
332
392
|
col: e.name,
|
|
333
393
|
refTable: e.references.table
|
|
334
|
-
})),
|
|
394
|
+
})), it = (e, t, r) => n.gen(function* () {
|
|
335
395
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), u = (e) => {
|
|
336
396
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
337
397
|
return s.set(e, t), t;
|
|
@@ -349,10 +409,10 @@ var Te = (e) => {
|
|
|
349
409
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
350
410
|
};
|
|
351
411
|
}, p = (r, i) => {
|
|
352
|
-
let a = f(r,
|
|
353
|
-
return l.runForEach(
|
|
412
|
+
let a = f(r, nt(t, r), rt(t, r));
|
|
413
|
+
return l.runForEach(w(e, {
|
|
354
414
|
table: r,
|
|
355
|
-
pk:
|
|
415
|
+
pk: nt(t, r),
|
|
356
416
|
...i === void 0 ? {} : { where: i }
|
|
357
417
|
}), (e) => n.sync(() => a(e)));
|
|
358
418
|
}, m = n.gen(function* () {
|
|
@@ -361,7 +421,7 @@ var Te = (e) => {
|
|
|
361
421
|
c.clear();
|
|
362
422
|
for (let [n, r] of e) {
|
|
363
423
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
364
|
-
e.length !== 0 && (yield* p(n, b(
|
|
424
|
+
e.length !== 0 && (yield* p(n, b(nt(t, n), e)));
|
|
365
425
|
}
|
|
366
426
|
}
|
|
367
427
|
});
|
|
@@ -384,24 +444,24 @@ var Te = (e) => {
|
|
|
384
444
|
}
|
|
385
445
|
}
|
|
386
446
|
return o;
|
|
387
|
-
}),
|
|
447
|
+
}), at = (e, t) => ({
|
|
388
448
|
tables: [...e.keys()],
|
|
389
449
|
predicateFor: (n) => {
|
|
390
450
|
let r = e.get(n);
|
|
391
|
-
return r ? b(
|
|
451
|
+
return r ? b(nt(t, n), [...r]) : void 0;
|
|
392
452
|
}
|
|
393
|
-
}),
|
|
453
|
+
}), ot = "tenantId", st = (e, t, r) => n.gen(function* () {
|
|
394
454
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
395
455
|
if (e.kind === "all") {
|
|
396
456
|
let t = new Set(e.exclude ?? []);
|
|
397
457
|
return {
|
|
398
|
-
tables: i.filter((e) => !
|
|
458
|
+
tables: i.filter((e) => !W(e) && !t.has(e)),
|
|
399
459
|
predicateFor: () => void 0
|
|
400
460
|
};
|
|
401
461
|
}
|
|
402
462
|
if (e.kind === "tables") {
|
|
403
463
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
404
|
-
if (a.length > 0) return yield* n.fail(new
|
|
464
|
+
if (a.length > 0) return yield* n.fail(new Xe({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
405
465
|
let o = new Set(t);
|
|
406
466
|
return {
|
|
407
467
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -409,30 +469,30 @@ var Te = (e) => {
|
|
|
409
469
|
};
|
|
410
470
|
}
|
|
411
471
|
let o = e.tenantId;
|
|
412
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
413
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
414
|
-
if (s.length === 0) return yield* n.fail(new
|
|
415
|
-
let c = yield*
|
|
416
|
-
seeds: Object.fromEntries(s.map((e) => [e, _(
|
|
472
|
+
if (o === void 0 || o === "") return yield* n.fail(new Xe({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
473
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === ot) ?? !1);
|
|
474
|
+
if (s.length === 0) return yield* n.fail(new Xe({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${ot}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
475
|
+
let c = yield* it(r.store, t, {
|
|
476
|
+
seeds: Object.fromEntries(s.map((e) => [e, _(ot, o)])),
|
|
417
477
|
pinned: s,
|
|
418
478
|
children: { roots: s }
|
|
419
479
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
420
480
|
return {
|
|
421
481
|
tables: [...s, ...l],
|
|
422
|
-
predicateFor: (e) => u.has(e) ? _(
|
|
482
|
+
predicateFor: (e) => u.has(e) ? _(ot, o) : c.has(e) ? b(nt(t, e), [...c.get(e)]) : void 0
|
|
423
483
|
};
|
|
424
|
-
}),
|
|
484
|
+
}), ct = (e, t) => ({
|
|
425
485
|
list: t,
|
|
426
|
-
open: (t) =>
|
|
427
|
-
}),
|
|
428
|
-
write: (t, n, r) =>
|
|
486
|
+
open: (t) => j(e, t)
|
|
487
|
+
}), lt = (e) => ({
|
|
488
|
+
write: (t, n, r) => he(e, t, n, r),
|
|
429
489
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
430
|
-
}),
|
|
490
|
+
}), ut = (e, t) => n.tryPromise({
|
|
431
491
|
try: async () => {
|
|
432
492
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
433
493
|
u.update(e), d += e.length, n(null, e);
|
|
434
494
|
} });
|
|
435
|
-
await o(await
|
|
495
|
+
await o(await M(e.stream), f, n(l));
|
|
436
496
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
437
497
|
try {
|
|
438
498
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -445,30 +505,30 @@ var Te = (e) => {
|
|
|
445
505
|
contentType: e.contentType
|
|
446
506
|
};
|
|
447
507
|
},
|
|
448
|
-
catch: (e) => new
|
|
508
|
+
catch: (e) => new G({
|
|
449
509
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
450
510
|
path: t
|
|
451
511
|
})
|
|
452
|
-
}),
|
|
453
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u =
|
|
512
|
+
}), dt = (e, t, r) => n.gen(function* () {
|
|
513
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u = me("bundle", () => a(s)).pipe(l.tap((e) => n.sync(() => c.update(e))));
|
|
454
514
|
yield* e.write(r.key, u, {
|
|
455
515
|
contentType: r.contentType,
|
|
456
516
|
totalSize: r.size
|
|
457
517
|
});
|
|
458
518
|
let d = c.digest("hex");
|
|
459
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
519
|
+
if (d !== r.sha256) return yield* n.fail(new Je({
|
|
460
520
|
what: `asset ${r.key}`,
|
|
461
521
|
expected: r.sha256,
|
|
462
522
|
actual: d
|
|
463
523
|
}));
|
|
464
|
-
}),
|
|
524
|
+
}), ft = {
|
|
465
525
|
done: () => !1,
|
|
466
526
|
mark: () => n.void
|
|
467
|
-
},
|
|
468
|
-
let t = e.progress ??
|
|
469
|
-
return yield* l.runForEach(e.source.list(), (l) => (a++, t.done(l.key) || c.has(l.key) ? n.void : (c.add(l.key), e.source.open(l.key).pipe(n.map((e) => e), n.catchIf(
|
|
527
|
+
}, pt = (e) => typeof e == "object" && !!e && e.status === 404, mt = (e) => n.gen(function* () {
|
|
528
|
+
let t = e.progress ?? ft, r = 0, i = 0, a = 0, o = 0, s = /* @__PURE__ */ new Set(), c = /* @__PURE__ */ new Set(), u = [];
|
|
529
|
+
return yield* l.runForEach(e.source.list(), (l) => (a++, t.done(l.key) || c.has(l.key) ? n.void : (c.add(l.key), e.source.open(l.key).pipe(n.map((e) => e), n.catchIf(pt, () => n.sync(() => {
|
|
470
530
|
u.push(l.key);
|
|
471
|
-
})), n.flatMap((a) => a === void 0 ? n.void :
|
|
531
|
+
})), n.flatMap((a) => a === void 0 ? n.void : ut(a, e.assetsDir).pipe(n.flatMap((e) => {
|
|
472
532
|
let n = {
|
|
473
533
|
key: l.key,
|
|
474
534
|
sha256: e.sha256,
|
|
@@ -476,7 +536,7 @@ var Te = (e) => {
|
|
|
476
536
|
contentType: e.contentType
|
|
477
537
|
};
|
|
478
538
|
return r++, i += e.size, s.has(e.sha256) || (s.add(e.sha256), o += e.size), t.mark(l.key, n);
|
|
479
|
-
}))))))).pipe(n.mapError((e) => e instanceof
|
|
539
|
+
}))))))).pipe(n.mapError((e) => e instanceof G ? e : new G({ reason: `asset export failed: ${String(e)}` }))), {
|
|
480
540
|
count: r,
|
|
481
541
|
totalBytes: i,
|
|
482
542
|
references: a,
|
|
@@ -484,18 +544,18 @@ var Te = (e) => {
|
|
|
484
544
|
objectBytes: o,
|
|
485
545
|
missing: u
|
|
486
546
|
};
|
|
487
|
-
}),
|
|
547
|
+
}), ht = (e) => me("bundle", () => ue(e)).pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.map((e) => JSON.parse(e)), l.mapError((t) => t instanceof G ? t : new G({
|
|
488
548
|
reason: `asset index unreadable: ${String(t)}`,
|
|
489
549
|
path: e
|
|
490
|
-
}))),
|
|
491
|
-
let t = e.progress ??
|
|
492
|
-
return yield* l.runForEach(
|
|
550
|
+
}))), gt = (e) => n.gen(function* () {
|
|
551
|
+
let t = e.progress ?? ft, r = 0, i = /* @__PURE__ */ new Set(), a = /* @__PURE__ */ new Set();
|
|
552
|
+
return yield* l.runForEach(ht(e.indexPath), (o) => t.done(o.key) || a.has(o.key) ? n.void : (a.add(o.key), dt(e.sink, e.assetsDir, o)).pipe(n.andThen(n.sync(() => {
|
|
493
553
|
r++, i.add(o.sha256);
|
|
494
554
|
})), n.andThen(t.mark(o.key, o)))), {
|
|
495
555
|
count: r,
|
|
496
556
|
objects: i.size
|
|
497
557
|
};
|
|
498
|
-
}),
|
|
558
|
+
}), _t = () => typeof N.createZstdCompress == "function", vt = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : _t() ? "zstd" : "gzip", yt = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", bt = (e) => e === "zstd" ? N.createZstdCompress() : e === "gzip" ? N.createGzip() : new P(), xt = (e) => e === "zstd" ? N.createZstdDecompress() : e === "gzip" ? N.createGunzip() : new P(), St = 1, Ct = c.Struct({
|
|
499
559
|
name: c.String,
|
|
500
560
|
file: c.String,
|
|
501
561
|
rowCount: c.Number,
|
|
@@ -505,12 +565,12 @@ var Te = (e) => {
|
|
|
505
565
|
value: c.String
|
|
506
566
|
}),
|
|
507
567
|
primaryKey: c.String
|
|
508
|
-
}),
|
|
568
|
+
}), wt = c.Struct({
|
|
509
569
|
kind: c.Literal("all", "tenant", "tables"),
|
|
510
570
|
tenantId: c.optional(c.String),
|
|
511
571
|
tables: c.optional(c.Array(c.String)),
|
|
512
572
|
exclude: c.optional(c.Array(c.String))
|
|
513
|
-
}),
|
|
573
|
+
}), Tt = c.Struct({
|
|
514
574
|
count: c.Number,
|
|
515
575
|
totalBytes: c.Number,
|
|
516
576
|
index: c.optional(c.String)
|
|
@@ -521,23 +581,23 @@ c.Struct({
|
|
|
521
581
|
size: c.Number,
|
|
522
582
|
contentType: c.String
|
|
523
583
|
});
|
|
524
|
-
var
|
|
584
|
+
var Et = c.Struct({
|
|
525
585
|
formatVersion: c.Number,
|
|
526
586
|
createdAt: c.String,
|
|
527
587
|
source: c.Struct({
|
|
528
588
|
dialect: c.String,
|
|
529
589
|
schemaFingerprint: c.String
|
|
530
590
|
}),
|
|
531
|
-
scope:
|
|
591
|
+
scope: wt,
|
|
532
592
|
consistency: c.Literal("live", "snapshot"),
|
|
533
593
|
compression: c.Literal("zstd", "gzip", "none"),
|
|
534
|
-
tables: c.Array(
|
|
535
|
-
assets:
|
|
594
|
+
tables: c.Array(Ct),
|
|
595
|
+
assets: Tt,
|
|
536
596
|
masking: c.optional(c.Struct({
|
|
537
597
|
policyId: c.optional(c.String),
|
|
538
598
|
transformed: c.Array(c.String)
|
|
539
599
|
}))
|
|
540
|
-
}),
|
|
600
|
+
}), Dt = c.encodeSync(c.parseJson(Et)), Ot = c.decodeUnknownSync(c.parseJson(Et)), kt = "manifest.json", At = "README.md", jt = "data", Mt = "assets", Nt = "assets/index.ndjson", Pt = ".ledger.json", Ft = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", It = (e, t) => {
|
|
541
601
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
542
602
|
return `# Voltro data export
|
|
543
603
|
|
|
@@ -551,7 +611,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
551
611
|
| **Format version** | ${e.formatVersion} |
|
|
552
612
|
| **Consistency** | ${e.consistency} |
|
|
553
613
|
| **Compression** | ${e.compression} |
|
|
554
|
-
| **Scope** | ${
|
|
614
|
+
| **Scope** | ${Ft(e.scope)} |
|
|
555
615
|
| **Tables** | ${e.tables.length} |
|
|
556
616
|
| **Rows (total)** | ${n} |
|
|
557
617
|
| **Assets** | ${a} |
|
|
@@ -576,39 +636,39 @@ voltro data import <this-directory>
|
|
|
576
636
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
577
637
|
\`\`\`
|
|
578
638
|
`;
|
|
579
|
-
},
|
|
639
|
+
}, Lt = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), Rt = a.exponential({
|
|
580
640
|
start: .01,
|
|
581
641
|
factor: 2,
|
|
582
642
|
count: 16
|
|
583
|
-
}),
|
|
643
|
+
}), zt = i.histogram("voltro_data_transfer_table_seconds", Rt, "Per-table export/import duration in seconds, tagged by phase."), q = (e, t) => {
|
|
584
644
|
if (e) try {
|
|
585
645
|
e(t);
|
|
586
646
|
} catch {}
|
|
587
|
-
},
|
|
647
|
+
}, Bt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(zt.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), Lt.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
588
648
|
table: t,
|
|
589
649
|
index: r,
|
|
590
650
|
tableCount: a,
|
|
591
651
|
phase: e
|
|
592
|
-
} })),
|
|
652
|
+
} })), Vt = () => ({
|
|
593
653
|
tables: {},
|
|
594
654
|
assets: {}
|
|
595
|
-
}),
|
|
655
|
+
}), Ht = (e) => n.tryPromise(async () => {
|
|
596
656
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
597
657
|
return JSON.parse(t);
|
|
598
|
-
}).pipe(n.orElseSucceed(
|
|
658
|
+
}).pipe(n.orElseSucceed(Vt)), J = (e, t) => n.tryPromise({
|
|
599
659
|
try: async () => {
|
|
600
660
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
601
661
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
602
662
|
},
|
|
603
|
-
catch: (t) => new
|
|
663
|
+
catch: (t) => new G({
|
|
604
664
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
605
665
|
path: e
|
|
606
666
|
})
|
|
607
|
-
}),
|
|
667
|
+
}), Ut = (e, t) => t in e.tables, Wt = (e, t) => t in e.assets, Gt = /* @__PURE__ */ new Set([
|
|
608
668
|
"json",
|
|
609
669
|
"array",
|
|
610
670
|
"vector"
|
|
611
|
-
]),
|
|
671
|
+
]), Kt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", qt = (e) => Buffer.from(e).toString("base64"), Jt = (e) => new Uint8Array(Buffer.from(e, "base64")), Yt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), Xt = (e, t) => e == null ? null : t !== void 0 && Gt.has(t) ? e : t === "bytes" ? qt(e) : t === "timestamp" || t === "date" ? Yt(e) : typeof e == "bigint" ? {
|
|
612
672
|
$v: "bigint",
|
|
613
673
|
d: e.toString()
|
|
614
674
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -619,30 +679,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
619
679
|
d: e.toISOString()
|
|
620
680
|
} : e instanceof Uint8Array ? {
|
|
621
681
|
$v: "bytes",
|
|
622
|
-
d:
|
|
623
|
-
} : e,
|
|
682
|
+
d: qt(e)
|
|
683
|
+
} : e, Zt = (e) => {
|
|
624
684
|
switch (e.$v) {
|
|
625
685
|
case "bigint": return BigInt(e.d);
|
|
626
686
|
case "number": return Number(e.d);
|
|
627
687
|
case "date": return new Date(e.d);
|
|
628
|
-
case "bytes": return
|
|
688
|
+
case "bytes": return Jt(e.d);
|
|
629
689
|
default: return e;
|
|
630
690
|
}
|
|
631
|
-
},
|
|
691
|
+
}, Qt = (e, t) => e == null ? null : t !== void 0 && Gt.has(t) ? e : Kt(e) ? Zt(e) : t === "bytes" ? Jt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, $t = (e, t) => {
|
|
632
692
|
let n = {};
|
|
633
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
693
|
+
for (let [r, i] of Object.entries(e)) n[r] = Xt(i, t[r]);
|
|
634
694
|
return n;
|
|
635
|
-
},
|
|
695
|
+
}, en = (e, t) => {
|
|
636
696
|
let n = {};
|
|
637
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
697
|
+
for (let [r, i] of Object.entries(e)) n[r] = Qt(i, t[r]);
|
|
638
698
|
return n;
|
|
639
|
-
},
|
|
640
|
-
try: () =>
|
|
641
|
-
catch: (e) => new
|
|
699
|
+
}, tn = new TextEncoder(), nn = (e, t) => e.pipe(l.map((e) => tn.encode(`${JSON.stringify($t(e, t))}\n`))), rn = (e, t, r) => e.pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.mapEffect((e) => n.try({
|
|
700
|
+
try: () => en(JSON.parse(e), r),
|
|
701
|
+
catch: (e) => new qe({
|
|
642
702
|
table: t,
|
|
643
703
|
reason: String(e?.message ?? e)
|
|
644
704
|
})
|
|
645
|
-
}))),
|
|
705
|
+
}))), an = (e, t) => {
|
|
646
706
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
647
707
|
for (let e of t.tables) {
|
|
648
708
|
if (!n.has(e.name)) continue;
|
|
@@ -668,33 +728,33 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
668
728
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
669
729
|
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(Y(t.cause)) : n.push(r), n.length === 0) {
|
|
670
730
|
let t = String(e ?? "").trim();
|
|
671
|
-
return t === "" || t === "[object Object]" ?
|
|
731
|
+
return t === "" || t === "[object Object]" ? on(e) : t;
|
|
672
732
|
}
|
|
673
733
|
return n.join(": ");
|
|
674
|
-
},
|
|
734
|
+
}, on = (e) => {
|
|
675
735
|
try {
|
|
676
736
|
let t = JSON.stringify(e);
|
|
677
737
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
678
738
|
} catch {
|
|
679
739
|
return "unknown error (not serialisable)";
|
|
680
740
|
}
|
|
681
|
-
},
|
|
741
|
+
}, sn = (e, t) => {
|
|
682
742
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
683
743
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
684
744
|
return r;
|
|
685
|
-
},
|
|
745
|
+
}, cn = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), ln = (e, t) => {
|
|
686
746
|
if (t.length === 0) return e;
|
|
687
747
|
let n = { ...e };
|
|
688
748
|
for (let e of t) delete n[e];
|
|
689
749
|
return n;
|
|
690
|
-
},
|
|
750
|
+
}, un = (e, t) => {
|
|
691
751
|
let n = e.tables.find((e) => e.name === t);
|
|
692
752
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
693
753
|
let r = n.columns.find((e) => e.type === "id");
|
|
694
754
|
if (r !== void 0) return { column: r.name };
|
|
695
755
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
696
756
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
697
|
-
},
|
|
757
|
+
}, dn = (e, t, r) => n.tryPromise({
|
|
698
758
|
try: async () => {
|
|
699
759
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
700
760
|
d.update(e);
|
|
@@ -703,7 +763,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
703
763
|
n(null, e);
|
|
704
764
|
} }), m = c.fromWeb(l.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
705
765
|
try {
|
|
706
|
-
await s(m, p,
|
|
766
|
+
await s(m, p, bt(r), n(h)), await i.rename(h, t);
|
|
707
767
|
} catch (e) {
|
|
708
768
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
709
769
|
}
|
|
@@ -712,25 +772,25 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
712
772
|
checksum: d.digest("hex")
|
|
713
773
|
};
|
|
714
774
|
},
|
|
715
|
-
catch: (e) => new
|
|
775
|
+
catch: (e) => new G({
|
|
716
776
|
reason: `write table failed: ${Y(e)}`,
|
|
717
777
|
path: t
|
|
718
778
|
})
|
|
719
|
-
}),
|
|
779
|
+
}), fn = (e, t) => n.tryPromise({
|
|
720
780
|
try: async () => {
|
|
721
781
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
722
782
|
},
|
|
723
|
-
catch: (t) => new
|
|
783
|
+
catch: (t) => new G({
|
|
724
784
|
reason: `append failed: ${Y(t)}`,
|
|
725
785
|
path: e
|
|
726
786
|
})
|
|
727
|
-
}),
|
|
787
|
+
}), pn = (e, t) => n.gen(function* () {
|
|
728
788
|
let r = [], i = t.ordered.length;
|
|
729
789
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
730
|
-
let o = t.ordered[a], s =
|
|
731
|
-
if ("refusal" in c) return yield* n.fail(new
|
|
732
|
-
let u = c.column, d = `${
|
|
733
|
-
if (
|
|
790
|
+
let o = t.ordered[a], s = sn(t.snapshot, o), c = un(t.snapshot, o);
|
|
791
|
+
if ("refusal" in c) return yield* n.fail(new Xe({ reason: `export: ${c.refusal}` }));
|
|
792
|
+
let u = c.column, d = `${jt}/${o}.ndjson${yt(t.compression)}`;
|
|
793
|
+
if (Ut(t.ledger, o)) {
|
|
734
794
|
let e = t.ledger.tables[o];
|
|
735
795
|
q(t.onProgress, {
|
|
736
796
|
phase: "export",
|
|
@@ -758,13 +818,13 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
758
818
|
event: "start",
|
|
759
819
|
rowsDone: 0
|
|
760
820
|
});
|
|
761
|
-
let f = t.resolved.predicateFor(o), p =
|
|
821
|
+
let f = t.resolved.predicateFor(o), p = w(e, {
|
|
762
822
|
table: o,
|
|
763
823
|
pk: u,
|
|
764
824
|
chunkSize: t.chunkSize,
|
|
765
825
|
retry: t.retry,
|
|
766
826
|
...f === void 0 ? {} : { where: f }
|
|
767
|
-
}), m =
|
|
827
|
+
}), m = cn(t.snapshot, o), h = m.length === 0 ? p : p.pipe(l.map((e) => ln(e, m))), g = t.mask ? h.pipe(l.map((e) => He(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* Bt("export", o, a, i, (e) => e.rowCount, dn(nn(g, s), t.bundleFile(d), t.compression));
|
|
768
828
|
t.ledger.tables[o] = {
|
|
769
829
|
rowCount: _,
|
|
770
830
|
checksum: v
|
|
@@ -786,36 +846,36 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
786
846
|
});
|
|
787
847
|
}
|
|
788
848
|
return r;
|
|
789
|
-
}),
|
|
849
|
+
}), mn = (e) => ({
|
|
790
850
|
_tag: "RawSqlFragment",
|
|
791
851
|
strings: [e],
|
|
792
852
|
values: []
|
|
793
|
-
}),
|
|
853
|
+
}), hn = (t, i, a) => n.tryPromise({
|
|
794
854
|
try: () => t.transactional(async (t) => {
|
|
795
|
-
if (t.raw) for (let e of
|
|
855
|
+
if (t.raw) for (let e of Ce(i)) await t.raw(mn(e));
|
|
796
856
|
let o = await n.runPromiseExit(a(t));
|
|
797
857
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
798
858
|
return o.value;
|
|
799
859
|
}),
|
|
800
|
-
catch: (e) => e instanceof
|
|
801
|
-
}),
|
|
802
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
860
|
+
catch: (e) => e instanceof G ? e : new G({ reason: `snapshot export failed: ${Y(e)}` })
|
|
861
|
+
}), gn = (e) => n.gen(function* () {
|
|
862
|
+
let t = e.scope ?? { kind: "all" }, r = vt(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, o = s.exponential("50 millis").pipe(s.jittered, s.intersect(s.recurs(a))), c = yield* n.promise(() => import("node:path")), l = (t) => c.join(e.outDir, t);
|
|
803
863
|
yield* n.tryPromise({
|
|
804
864
|
try: async () => {
|
|
805
865
|
let t = await import("node:fs/promises");
|
|
806
|
-
await t.mkdir(l(
|
|
866
|
+
await t.mkdir(l(jt), { recursive: !0 }), e.assets && await t.mkdir(l(Mt), { recursive: !0 });
|
|
807
867
|
},
|
|
808
|
-
catch: (t) => new
|
|
868
|
+
catch: (t) => new G({
|
|
809
869
|
reason: `mkdir failed: ${Y(t)}`,
|
|
810
870
|
path: e.outDir
|
|
811
871
|
})
|
|
812
872
|
});
|
|
813
|
-
let u = l(
|
|
873
|
+
let u = l(Pt), d = yield* Ht(u), f = e.subset ? at(yield* it(e.store, e.snapshot, e.subset), e.snapshot) : yield* st(t, e.snapshot, {
|
|
814
874
|
store: e.store,
|
|
815
875
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
816
|
-
}), p =
|
|
876
|
+
}), p = an(f.tables, e.snapshot), m = e.consistency ?? "live", h, g;
|
|
817
877
|
if (e.masking) {
|
|
818
|
-
let t = e.classification ??
|
|
878
|
+
let t = e.classification ?? Te(e.snapshot), r = ze(p.map((t) => ({
|
|
819
879
|
name: t,
|
|
820
880
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
821
881
|
name: e.name,
|
|
@@ -848,14 +908,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
848
908
|
bundleFile: l,
|
|
849
909
|
...g ? { mask: g } : {},
|
|
850
910
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
851
|
-
}, v = m === "snapshot" ? yield*
|
|
911
|
+
}, v = m === "snapshot" ? yield* hn(e.store, e.dialect, (e) => pn(e, _)) : yield* pn(e.store, _), y = 0, b = 0;
|
|
852
912
|
if (e.assets) {
|
|
853
|
-
let t = l(
|
|
913
|
+
let t = l(Nt), r = yield* mt({
|
|
854
914
|
source: e.assets,
|
|
855
|
-
assetsDir: l(
|
|
915
|
+
assetsDir: l(Mt),
|
|
856
916
|
progress: {
|
|
857
917
|
done: (e) => e in d.assets,
|
|
858
|
-
mark: (e, r) => (d.assets[e] = !0,
|
|
918
|
+
mark: (e, r) => (d.assets[e] = !0, fn(t, JSON.stringify(r)).pipe(n.andThen(J(u, d))))
|
|
859
919
|
}
|
|
860
920
|
});
|
|
861
921
|
y = r.count, b = r.totalBytes, r.missing.length > 0 && (yield* n.logWarning(`${r.missing.length} of ${r.missing.length + r.count} blob reference(s) point at objects the storage provider does not have; they are NOT in this bundle and no import can bring them back. e.g. ${r.missing.slice(0, 3).map((e) => `'${e}'`).join(", ")}` + (r.missing.length > 3 ? `, and ${r.missing.length - 3} more` : "")));
|
|
@@ -865,7 +925,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
865
925
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
866
926
|
source: {
|
|
867
927
|
dialect: e.dialect,
|
|
868
|
-
schemaFingerprint:
|
|
928
|
+
schemaFingerprint: T(e.snapshot)
|
|
869
929
|
},
|
|
870
930
|
scope: t,
|
|
871
931
|
consistency: m,
|
|
@@ -874,7 +934,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
874
934
|
assets: y > 0 ? {
|
|
875
935
|
count: y,
|
|
876
936
|
totalBytes: b,
|
|
877
|
-
index:
|
|
937
|
+
index: Nt
|
|
878
938
|
} : {
|
|
879
939
|
count: 0,
|
|
880
940
|
totalBytes: 0
|
|
@@ -883,23 +943,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
883
943
|
};
|
|
884
944
|
return yield* n.tryPromise({
|
|
885
945
|
try: async () => {
|
|
886
|
-
await (await import("node:fs/promises")).writeFile(l(
|
|
946
|
+
await (await import("node:fs/promises")).writeFile(l(kt), Dt(x));
|
|
887
947
|
},
|
|
888
|
-
catch: (e) => new
|
|
948
|
+
catch: (e) => new G({
|
|
889
949
|
reason: `manifest write failed: ${Y(e)}`,
|
|
890
|
-
path:
|
|
950
|
+
path: kt
|
|
891
951
|
})
|
|
892
952
|
}), yield* n.tryPromise({
|
|
893
953
|
try: async () => {
|
|
894
954
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
895
|
-
await e.writeFile(l(
|
|
955
|
+
await e.writeFile(l(At), It(x, { hostname: t.hostname() }));
|
|
896
956
|
},
|
|
897
|
-
catch: (e) => new
|
|
957
|
+
catch: (e) => new G({
|
|
898
958
|
reason: `readme write failed: ${Y(e)}`,
|
|
899
|
-
path:
|
|
959
|
+
path: At
|
|
900
960
|
})
|
|
901
961
|
}), x;
|
|
902
|
-
}),
|
|
962
|
+
}), _n = (e, t) => {
|
|
903
963
|
let n = new Set(t), r = [];
|
|
904
964
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
905
965
|
let i = e.references?.table;
|
|
@@ -910,7 +970,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
910
970
|
});
|
|
911
971
|
}
|
|
912
972
|
return r;
|
|
913
|
-
},
|
|
973
|
+
}, vn = (e) => {
|
|
914
974
|
let t = /* @__PURE__ */ new Map();
|
|
915
975
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
916
976
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -918,7 +978,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
918
978
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
919
979
|
});
|
|
920
980
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
921
|
-
},
|
|
981
|
+
}, yn = (e) => {
|
|
922
982
|
switch (e) {
|
|
923
983
|
case "id":
|
|
924
984
|
case "text":
|
|
@@ -937,12 +997,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
937
997
|
case "raw": return "raw";
|
|
938
998
|
default: return "string";
|
|
939
999
|
}
|
|
940
|
-
},
|
|
1000
|
+
}, bn = {
|
|
941
1001
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
942
1002
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
943
1003
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
944
|
-
},
|
|
945
|
-
let i =
|
|
1004
|
+
}, xn = (e, t, n, r) => {
|
|
1005
|
+
let i = yn(n);
|
|
946
1006
|
if (i === "raw") return {
|
|
947
1007
|
table: e,
|
|
948
1008
|
column: t,
|
|
@@ -950,7 +1010,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
950
1010
|
severity: "block",
|
|
951
1011
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
952
1012
|
};
|
|
953
|
-
let a =
|
|
1013
|
+
let a = bn[i];
|
|
954
1014
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
955
1015
|
table: e,
|
|
956
1016
|
column: t,
|
|
@@ -964,15 +1024,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
964
1024
|
severity: "block",
|
|
965
1025
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
966
1026
|
};
|
|
967
|
-
},
|
|
1027
|
+
}, Sn = (e, t, n) => {
|
|
968
1028
|
if (t === n) return [];
|
|
969
1029
|
let r = [];
|
|
970
1030
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
971
|
-
let a =
|
|
1031
|
+
let a = xn(t.name, e, i, n);
|
|
972
1032
|
a && r.push(a);
|
|
973
1033
|
}
|
|
974
1034
|
return r;
|
|
975
|
-
},
|
|
1035
|
+
}, Cn = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, wn = (e) => {
|
|
976
1036
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
977
1037
|
switch (e.kind) {
|
|
978
1038
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -981,35 +1041,35 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
981
1041
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
982
1042
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
983
1043
|
}
|
|
984
|
-
},
|
|
1044
|
+
}, Tn = (e) => {
|
|
985
1045
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
986
1046
|
return t.length > 0 ? t.join("/") : void 0;
|
|
987
|
-
},
|
|
1047
|
+
}, En = "Failed to execute statement", Dn = (e) => {
|
|
988
1048
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
989
1049
|
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
990
|
-
return t === `SqlError: ${
|
|
991
|
-
},
|
|
1050
|
+
return t === `SqlError: ${En}` ? En : t;
|
|
1051
|
+
}, On = (e) => {
|
|
992
1052
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
993
1053
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
994
1054
|
unclassified: t,
|
|
995
1055
|
raw: n
|
|
996
|
-
}), i = v(e), a = m(i), o =
|
|
997
|
-
if (a !== void 0) return r(`${
|
|
998
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u =
|
|
999
|
-
return r(d ===
|
|
1000
|
-
},
|
|
1056
|
+
}), i = v(e), a = m(i), o = Tn(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
1057
|
+
if (a !== void 0) return r(`${wn(a)}${s}`, !1);
|
|
1058
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = Dn(typeof l == "string" && l.length > 0 ? l : String(e)), d = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, f = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, p = f !== void 0 && /[←‖]/.test(f) ? ` (via ${f})` : "";
|
|
1059
|
+
return r(d === En || d.length === 0 ? `the database refused the write and the driver gave no detail${s}${p}` : `${d}${s}`, d === En || d.length === 0);
|
|
1060
|
+
}, kn = (e) => On(e).reason, An = (e) => {
|
|
1001
1061
|
let t = v(e), n = m(t);
|
|
1002
1062
|
if (n?.kind === "foreignKeyInUse") {
|
|
1003
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
1063
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = Tn(t);
|
|
1004
1064
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
1005
1065
|
}
|
|
1006
|
-
return
|
|
1007
|
-
},
|
|
1066
|
+
return kn(e);
|
|
1067
|
+
}, jn = f, X = (e) => `${jn}${e}`, Mn = (e) => typeof e.run == "function", Nn = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
1008
1068
|
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
1009
|
-
written before the first delete is what covers that case.`,
|
|
1010
|
-
if (!
|
|
1069
|
+
written before the first delete is what covers that case.`, Pn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, Z = (e, t) => {
|
|
1070
|
+
if (!Pn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
1011
1071
|
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
1012
|
-
},
|
|
1072
|
+
}, Fn = (e, t) => {
|
|
1013
1073
|
let n = Z(e, t), r = Z(X(e), t);
|
|
1014
1074
|
switch (t) {
|
|
1015
1075
|
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
@@ -1018,10 +1078,10 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1018
1078
|
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
1019
1079
|
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
1020
1080
|
}
|
|
1021
|
-
},
|
|
1081
|
+
}, In = (e, t) => {
|
|
1022
1082
|
let n = Z(X(e), t);
|
|
1023
1083
|
return t === "mssql" ? `IF OBJECT_ID('${X(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
1024
|
-
},
|
|
1084
|
+
}, Ln = (e, t) => {
|
|
1025
1085
|
let n = t === "mysql" || t === "mariadb", r = [];
|
|
1026
1086
|
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
1027
1087
|
for (let n of [...e].reverse()) r.push(`DELETE FROM ${Z(n.name, t)}`);
|
|
@@ -1031,7 +1091,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1031
1091
|
r.push(`INSERT INTO ${Z(n.name, t)} (${e}) SELECT ${e} FROM ${Z(X(n.name), t)}`);
|
|
1032
1092
|
}
|
|
1033
1093
|
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
1034
|
-
},
|
|
1094
|
+
}, Rn = (e, t) => {
|
|
1035
1095
|
let n = [], r = () => {
|
|
1036
1096
|
for (let e of n) t.unregisterTable(e);
|
|
1037
1097
|
};
|
|
@@ -1053,7 +1113,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1053
1113
|
staged: n,
|
|
1054
1114
|
undo: r
|
|
1055
1115
|
};
|
|
1056
|
-
},
|
|
1116
|
+
}, zn = (e, t) => {
|
|
1057
1117
|
let n = new Set(t), r = [];
|
|
1058
1118
|
for (let t of e.tables) {
|
|
1059
1119
|
if (!n.has(t.name)) continue;
|
|
@@ -1070,22 +1130,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1070
1130
|
}
|
|
1071
1131
|
}
|
|
1072
1132
|
return r;
|
|
1073
|
-
},
|
|
1133
|
+
}, Bn = (e, t) => {
|
|
1074
1134
|
let n = Z(X(e.from), t), r = Z(X(e.to), t), i = Z(e.column, t), a = Z(e.toColumn, t);
|
|
1075
1135
|
return `SELECT ${Z(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
1076
|
-
},
|
|
1136
|
+
}, Vn = (e) => {
|
|
1077
1137
|
if (e.length === 0) return;
|
|
1078
1138
|
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
1079
1139
|
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
1080
|
-
},
|
|
1081
|
-
t.length !== 0 && await e.run(
|
|
1082
|
-
},
|
|
1083
|
-
let i =
|
|
1084
|
-
i.length !== 0 && await e.run(
|
|
1085
|
-
},
|
|
1140
|
+
}, Hn = async (e, t) => {
|
|
1141
|
+
t.length !== 0 && await e.run(F.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
1142
|
+
}, Un = async (e, t, r) => {
|
|
1143
|
+
let i = Ln(t, r);
|
|
1144
|
+
i.length !== 0 && await e.run(F.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
1145
|
+
}, Wn = async (e, t, r) => {
|
|
1086
1146
|
let i = [];
|
|
1087
1147
|
for (let a of t) {
|
|
1088
|
-
let t = await e.run(
|
|
1148
|
+
let t = await e.run(F.SqlClient.pipe(n.flatMap((e) => e.unsafe(Bn(a, r)))));
|
|
1089
1149
|
for (let e of t) i.push({
|
|
1090
1150
|
table: a.from,
|
|
1091
1151
|
primaryKey: String(e.pk ?? ""),
|
|
@@ -1095,14 +1155,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1095
1155
|
});
|
|
1096
1156
|
}
|
|
1097
1157
|
return i;
|
|
1098
|
-
},
|
|
1158
|
+
}, Gn = (e, t) => {
|
|
1099
1159
|
let n = new Map(t.map((e, t) => [e, t]));
|
|
1100
1160
|
return e.filter((e) => {
|
|
1101
1161
|
if (e.from === e.to) return !1;
|
|
1102
1162
|
let t = n.get(e.from), r = n.get(e.to);
|
|
1103
1163
|
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1104
1164
|
});
|
|
1105
|
-
},
|
|
1165
|
+
}, Kn = (e) => e === "mysql" || e === "mariadb", qn = (e) => {
|
|
1106
1166
|
if (e.mode !== "replace") return {
|
|
1107
1167
|
active: !1,
|
|
1108
1168
|
tables: [],
|
|
@@ -1152,7 +1212,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1152
1212
|
tables: [],
|
|
1153
1213
|
blocker: "every table in the bundle is one that describes a deployment, so there is nothing for a staged swap to move."
|
|
1154
1214
|
};
|
|
1155
|
-
let s = o.map((e) => e.name), c =
|
|
1215
|
+
let s = o.map((e) => e.name), c = Kn(t) ? [] : Gn(zn(n, s), s);
|
|
1156
1216
|
return c.length > 0 ? {
|
|
1157
1217
|
active: !1,
|
|
1158
1218
|
tables: [],
|
|
@@ -1161,9 +1221,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1161
1221
|
active: !0,
|
|
1162
1222
|
tables: o
|
|
1163
1223
|
};
|
|
1164
|
-
},
|
|
1224
|
+
}, Jn = (e) => n.gen(function* () {
|
|
1165
1225
|
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1166
|
-
if (e.snapshot === void 0) return yield* n.fail(new
|
|
1226
|
+
if (e.snapshot === void 0) return yield* n.fail(new G({
|
|
1167
1227
|
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
1168
1228
|
path: e.dir
|
|
1169
1229
|
}));
|
|
@@ -1174,14 +1234,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1174
1234
|
if (t.tables.length === 0) return;
|
|
1175
1235
|
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1176
1236
|
so it survives a run whose process does not.`);
|
|
1177
|
-
let r = yield*
|
|
1237
|
+
let r = yield* gn({
|
|
1178
1238
|
store: e.store,
|
|
1179
1239
|
snapshot: t,
|
|
1180
1240
|
dialect: e.dialect ?? "unknown",
|
|
1181
1241
|
outDir: e.dir,
|
|
1182
1242
|
scope: { kind: "all" },
|
|
1183
1243
|
compression: "none"
|
|
1184
|
-
}).pipe(n.mapError((t) => new
|
|
1244
|
+
}).pipe(n.mapError((t) => new G({
|
|
1185
1245
|
reason: `rollback capture failed, so the import did not start and the target is untouched: ${t.reason ?? String(t)}\n Fix the capture, or re-run with \`--no-rollback\` to proceed without one.`,
|
|
1186
1246
|
path: e.dir
|
|
1187
1247
|
})));
|
|
@@ -1190,18 +1250,18 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1190
1250
|
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1191
1251
|
tables: r.tables.length
|
|
1192
1252
|
};
|
|
1193
|
-
}),
|
|
1253
|
+
}), Yn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, Xn = (e, t) => e.tables.find((e) => e.name === t), Zn = (e, t) => new Map((Xn(e, t)?.columns ?? []).map((e) => [e.name, e])), Qn = /__dropped_\d+$/, $n = (e, t) => {
|
|
1194
1254
|
let n = [];
|
|
1195
1255
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
1196
|
-
if (
|
|
1256
|
+
if (Xn(t, r.name) === void 0) {
|
|
1197
1257
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
1198
1258
|
continue;
|
|
1199
1259
|
}
|
|
1200
|
-
let e =
|
|
1260
|
+
let e = Zn(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
1201
1261
|
for (let [t, a] of i) {
|
|
1202
1262
|
let i = e.get(t);
|
|
1203
1263
|
if (i === void 0) {
|
|
1204
|
-
n.push(
|
|
1264
|
+
n.push(Qn.test(t) ? `${r.name}.${t}: a soft-dropped column, present only in the SOURCE — reclaim it there (voltro db gc-snapshots --before <date>) or re-export without it; the target is not missing anything` : `${r.name}.${t}: in the bundle, MISSING from the target — the value has nowhere to go`);
|
|
1205
1265
|
continue;
|
|
1206
1266
|
}
|
|
1207
1267
|
i.type !== a.type && n.push(`${r.name}.${t}: type '${a.type}' (bundle) vs '${i.type}' (target)`);
|
|
@@ -1209,22 +1269,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1209
1269
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
1210
1270
|
}
|
|
1211
1271
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
1212
|
-
},
|
|
1213
|
-
let r =
|
|
1272
|
+
}, er = (e, t, n) => {
|
|
1273
|
+
let r = T(t);
|
|
1214
1274
|
if (r === e || n === void 0) return {
|
|
1215
1275
|
drifted: !1,
|
|
1216
1276
|
bundleFingerprint: e,
|
|
1217
1277
|
targetFingerprint: r,
|
|
1218
1278
|
diff: []
|
|
1219
1279
|
};
|
|
1220
|
-
let i =
|
|
1280
|
+
let i = $n(n, t);
|
|
1221
1281
|
return {
|
|
1222
1282
|
drifted: i.length > 0,
|
|
1223
1283
|
bundleFingerprint: e,
|
|
1224
1284
|
targetFingerprint: r,
|
|
1225
1285
|
diff: i
|
|
1226
1286
|
};
|
|
1227
|
-
},
|
|
1287
|
+
}, tr = 2e3, nr = 2e3, rr = 200, ir = /* @__PURE__ */ new Set([
|
|
1228
1288
|
"BundleError",
|
|
1229
1289
|
"CodecError",
|
|
1230
1290
|
"IntegrityError",
|
|
@@ -1233,33 +1293,33 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1233
1293
|
"RowsRefusedError",
|
|
1234
1294
|
"SchemaDriftError",
|
|
1235
1295
|
"StorageError"
|
|
1236
|
-
]),
|
|
1296
|
+
]), ar = (t) => {
|
|
1237
1297
|
let n = o.isFiberFailure(t) ? [...e.failures(t[o.FiberFailureCauseId]), ...e.defects(t[o.FiberFailureCauseId])] : [t];
|
|
1238
1298
|
for (let e of n) {
|
|
1239
1299
|
let t = e?._tag;
|
|
1240
|
-
if (typeof t == "string" &&
|
|
1300
|
+
if (typeof t == "string" && ir.has(t)) return e;
|
|
1241
1301
|
if (o.isFiberFailure(e)) {
|
|
1242
|
-
let t =
|
|
1302
|
+
let t = ar(e);
|
|
1243
1303
|
if (t) return t;
|
|
1244
1304
|
}
|
|
1245
1305
|
}
|
|
1246
|
-
},
|
|
1306
|
+
}, or = (t) => [...e.failures(t)][0] ?? Error(e.pretty(t)), sr = (e) => n.tryPromise({
|
|
1247
1307
|
try: async () => {
|
|
1248
1308
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
1249
|
-
return
|
|
1309
|
+
return Ot(await t.readFile(n.join(e, kt), "utf8"));
|
|
1250
1310
|
},
|
|
1251
|
-
catch: (e) => new
|
|
1311
|
+
catch: (e) => new G({
|
|
1252
1312
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
1253
|
-
path:
|
|
1313
|
+
path: kt
|
|
1254
1314
|
})
|
|
1255
|
-
}),
|
|
1315
|
+
}), cr = (e) => {
|
|
1256
1316
|
let t;
|
|
1257
1317
|
return n.gen(function* () {
|
|
1258
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
1259
|
-
if (a.formatVersion !== 1) return yield* n.fail(new
|
|
1318
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* sr(e.bundleDir);
|
|
1319
|
+
if (a.formatVersion !== 1) return yield* n.fail(new G({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
1260
1320
|
if (e.tables !== void 0) {
|
|
1261
1321
|
let t = new Set(a.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
1262
|
-
if (r.length > 0) return yield* n.fail(new
|
|
1322
|
+
if (r.length > 0) return yield* n.fail(new G({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...t].sort().join(", ")}.` }));
|
|
1263
1323
|
}
|
|
1264
1324
|
let o = e.tables === void 0 ? void 0 : new Set(e.tables), s = o === void 0 ? a : {
|
|
1265
1325
|
...a,
|
|
@@ -1276,7 +1336,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1276
1336
|
unique: !1,
|
|
1277
1337
|
hasDefault: !1
|
|
1278
1338
|
}))
|
|
1279
|
-
})) }, r =
|
|
1339
|
+
})) }, r = er(s.source.schemaFingerprint, e.targetSnapshot, t);
|
|
1280
1340
|
if (r.drifted) {
|
|
1281
1341
|
if (!e.force) return yield* n.fail(new Qe({
|
|
1282
1342
|
bundleFingerprint: r.bundleFingerprint,
|
|
@@ -1287,14 +1347,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1287
1347
|
}
|
|
1288
1348
|
}
|
|
1289
1349
|
if (e.targetDialect && e.targetDialect !== s.source.dialect) {
|
|
1290
|
-
let t =
|
|
1350
|
+
let t = Sn(s.tables, s.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1291
1351
|
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new tt({
|
|
1292
1352
|
sourceDialect: s.source.dialect,
|
|
1293
1353
|
targetDialect: e.targetDialect,
|
|
1294
|
-
blocking: t.map(
|
|
1354
|
+
blocking: t.map(Cn)
|
|
1295
1355
|
}));
|
|
1296
1356
|
}
|
|
1297
|
-
let c = e.verify ?? !0, f = e.mode ?? "upsert", m = e.atomic ?? f === "replace", _ = e.onConflict ?? "skip", v = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), b = yield*
|
|
1357
|
+
let c = e.verify ?? !0, f = e.mode ?? "upsert", m = e.atomic ?? f === "replace", _ = e.onConflict ?? "skip", v = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), b = yield* Ht(v);
|
|
1298
1358
|
if (f === "replace" && s.scope.kind !== "all") return yield* n.fail(new et({
|
|
1299
1359
|
mode: f,
|
|
1300
1360
|
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope. To leave specific tables out of a FULL export, use \`--exclude a,b\`: it keeps the scope \`all\` and records what it left out.`
|
|
@@ -1307,8 +1367,8 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1307
1367
|
if (f === "replace") {
|
|
1308
1368
|
let t = s.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1309
1369
|
try: () => e.store.incomingForeignKeys(t),
|
|
1310
|
-
catch: (e) => new
|
|
1311
|
-
}) : e.targetSnapshot === void 0 ? [] :
|
|
1370
|
+
catch: (e) => new G({ reason: `foreign-key pre-flight failed: ${An(e)}` })
|
|
1371
|
+
}) : e.targetSnapshot === void 0 ? [] : _n(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1312
1372
|
for (let t of r) {
|
|
1313
1373
|
if (!a.has(t.from)) {
|
|
1314
1374
|
let r = yield* n.tryPromise({
|
|
@@ -1320,7 +1380,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1320
1380
|
skip: void 0,
|
|
1321
1381
|
projection: []
|
|
1322
1382
|
}),
|
|
1323
|
-
catch: () => new
|
|
1383
|
+
catch: () => new G({ reason: `unreadable: ${t.from}` })
|
|
1324
1384
|
}).pipe(n.map((e) => e.length), n.orElseSucceed(() => 0));
|
|
1325
1385
|
a.set(t.from, r);
|
|
1326
1386
|
}
|
|
@@ -1332,7 +1392,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1332
1392
|
}
|
|
1333
1393
|
if (i.length > 0) return yield* n.fail(new et({
|
|
1334
1394
|
mode: f,
|
|
1335
|
-
reason:
|
|
1395
|
+
reason: vn(i)
|
|
1336
1396
|
}));
|
|
1337
1397
|
}
|
|
1338
1398
|
if (e.dryRun === !0) return e.traceId !== void 0 && (yield* n.promise(() => e.store.update(u, e.traceId, {
|
|
@@ -1353,27 +1413,27 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1353
1413
|
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1354
1414
|
e.length > 0 && S.set(t.name, e);
|
|
1355
1415
|
}
|
|
1356
|
-
let
|
|
1416
|
+
let w = (e, t) => {
|
|
1357
1417
|
let n = S.get(e);
|
|
1358
1418
|
if (n === void 0) return t;
|
|
1359
1419
|
let r;
|
|
1360
1420
|
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1361
1421
|
return r ?? t;
|
|
1362
|
-
},
|
|
1363
|
-
let i =
|
|
1422
|
+
}, T = (e, t, n, r = !1) => {
|
|
1423
|
+
let i = k ? X(t.name) : t.name, a = t.primaryKey, o = w(t.name, n);
|
|
1364
1424
|
return f === "append" ? _ === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : f === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1365
|
-
},
|
|
1366
|
-
r.unclassified && (
|
|
1367
|
-
},
|
|
1425
|
+
}, E = (e, t) => e.savepoint ? e.savepoint(t) : t(), ie = 0, ae = (t, n, r) => {
|
|
1426
|
+
r.unclassified && (ie++, !(ie > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (ie === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1427
|
+
}, oe = (e, t, r = !0) => n.tryPromise({
|
|
1368
1428
|
try: e,
|
|
1369
|
-
catch: (e) =>
|
|
1370
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason,
|
|
1429
|
+
catch: (e) => On(e)
|
|
1430
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, ae(t.entry, t.row, e)), !1)))), se = (e) => {
|
|
1371
1431
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1372
1432
|
if (t.length !== 0) return {
|
|
1373
1433
|
...e.row,
|
|
1374
1434
|
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1375
1435
|
};
|
|
1376
|
-
},
|
|
1436
|
+
}, ce = (e) => {
|
|
1377
1437
|
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
1378
1438
|
if (r !== "reference") return !1;
|
|
1379
1439
|
let i = e.row[n];
|
|
@@ -1401,62 +1461,62 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1401
1461
|
primary: t.primary
|
|
1402
1462
|
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1403
1463
|
});
|
|
1404
|
-
},
|
|
1464
|
+
}, le = (e, t) => n.gen(function* () {
|
|
1405
1465
|
let r = t, i = [], a = !0;
|
|
1406
1466
|
for (; r.length > 0 && a;) {
|
|
1407
1467
|
a = !1;
|
|
1408
1468
|
let t = [];
|
|
1409
|
-
for (let n of r) (yield*
|
|
1469
|
+
for (let n of r) (yield* oe(() => E(e, () => T(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1410
1470
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1411
|
-
let r = t[n], o =
|
|
1412
|
-
o !== void 0 && (yield*
|
|
1471
|
+
let r = t[n], o = se(r);
|
|
1472
|
+
o !== void 0 && (yield* oe(() => E(e, () => T(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1413
1473
|
}
|
|
1414
1474
|
r = t;
|
|
1415
1475
|
}
|
|
1416
|
-
for (let t of i) (yield*
|
|
1417
|
-
if (r.length > 0) return yield* n.fail(
|
|
1418
|
-
}),
|
|
1476
|
+
for (let t of i) (yield* oe(() => E(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1477
|
+
if (r.length > 0) return yield* n.fail(ce(r));
|
|
1478
|
+
}), D = (() => {
|
|
1419
1479
|
let t = e.targetDialect;
|
|
1420
1480
|
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1421
|
-
})(),
|
|
1481
|
+
})(), O = qn({
|
|
1422
1482
|
mode: f,
|
|
1423
1483
|
...e.targetDialect === void 0 ? {} : { targetDialect: e.targetDialect },
|
|
1424
1484
|
...e.targetSnapshot === void 0 ? {} : { targetSnapshot: e.targetSnapshot },
|
|
1425
1485
|
bundleTables: s.tables.map((e) => e.name),
|
|
1426
|
-
isSkipped:
|
|
1427
|
-
canRunSql:
|
|
1428
|
-
}),
|
|
1429
|
-
if (f === "replace" && !
|
|
1486
|
+
isSkipped: W,
|
|
1487
|
+
canRunSql: Mn(e.store)
|
|
1488
|
+
}), de = O.tables, fe = s.tables.filter((e) => !W(e.name) && C(e.name)).map((e) => e.name), k = O.active, pe = !1, A = 0, j = 0, he, M, N = 0, P = 0, ge = (t, r) => n.gen(function* () {
|
|
1489
|
+
if (f === "replace" && !k && !(r && b.truncated)) {
|
|
1430
1490
|
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1431
1491
|
try: () => t.insert(d, {
|
|
1432
|
-
id: h("repl",
|
|
1433
|
-
startedAt:
|
|
1492
|
+
id: h("repl", F),
|
|
1493
|
+
startedAt: I,
|
|
1434
1494
|
tables: String(s.tables.length),
|
|
1435
|
-
capture:
|
|
1495
|
+
capture: L?.dir ?? null,
|
|
1436
1496
|
via: e.via ?? "direct"
|
|
1437
1497
|
}),
|
|
1438
|
-
catch: (e) => new
|
|
1498
|
+
catch: (e) => new G({ reason: `could not record that this replace started: ${String(e?.message ?? e)}\n The marker is what makes an interrupted run visible instead of silent, so the run does
|
|
1439
1499
|
not proceed without it.` })
|
|
1440
1500
|
}));
|
|
1441
|
-
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !
|
|
1501
|
+
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !W(e)), a = !r, o = t.emptyTables ? () => t.emptyTables(i) : a ? async () => {
|
|
1442
1502
|
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where: x(e.primaryKey) });
|
|
1443
1503
|
} : () => t.transactional(async (e) => {
|
|
1444
1504
|
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: x(t.primaryKey) });
|
|
1445
1505
|
});
|
|
1446
1506
|
yield* n.tryPromise({
|
|
1447
1507
|
try: o,
|
|
1448
|
-
catch: (e) => new
|
|
1508
|
+
catch: (e) => new G({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${An(e)}` })
|
|
1449
1509
|
}), r && (b.truncated = !0, yield* J(v, b));
|
|
1450
1510
|
}
|
|
1451
1511
|
let a = [], o = [], u = [], p = [], g = s.tables.length;
|
|
1452
1512
|
for (let d = 0; d < s.tables.length; d++) {
|
|
1453
1513
|
let h = s.tables[d];
|
|
1454
|
-
if (
|
|
1514
|
+
if (W(h.name)) {
|
|
1455
1515
|
p.push(h.name);
|
|
1456
1516
|
continue;
|
|
1457
1517
|
}
|
|
1458
1518
|
if (r && h.name in b.tables) {
|
|
1459
|
-
u.push(h.name),
|
|
1519
|
+
u.push(h.name), P++, N += b.tables[h.name].rowCount, q(e.onProgress, {
|
|
1460
1520
|
phase: "import",
|
|
1461
1521
|
table: h.name,
|
|
1462
1522
|
index: d,
|
|
@@ -1478,22 +1538,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1478
1538
|
total: h.rowCount
|
|
1479
1539
|
});
|
|
1480
1540
|
let y = a.length, x = n.gen(function* () {
|
|
1481
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c =
|
|
1541
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c = me("bundle", () => ue(_).pipe(xt(s.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), u = async (e) => {
|
|
1482
1542
|
let n = {
|
|
1483
1543
|
entry: h,
|
|
1484
1544
|
row: e,
|
|
1485
1545
|
reason: ""
|
|
1486
1546
|
};
|
|
1487
1547
|
try {
|
|
1488
|
-
await
|
|
1548
|
+
await E(t, () => T(t, h, e));
|
|
1489
1549
|
} catch (t) {
|
|
1490
|
-
let r =
|
|
1491
|
-
n.reason = r.reason,
|
|
1550
|
+
let r = On(t);
|
|
1551
|
+
n.reason = r.reason, ae(h, e, r), a.push(n);
|
|
1492
1552
|
}
|
|
1493
1553
|
}, d = yield* n.promise(async () => {
|
|
1494
1554
|
let n = e.copyLoader;
|
|
1495
1555
|
if (!n || n.dialect !== "postgres") return !1;
|
|
1496
|
-
if (m) return
|
|
1556
|
+
if (m) return pe || (pe = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1497
1557
|
if (f === "replace") return !0;
|
|
1498
1558
|
if (f !== "upsert") return !1;
|
|
1499
1559
|
try {
|
|
@@ -1523,40 +1583,40 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1523
1583
|
d = !1, e.onWarn?.(`bulk COPY into ${h.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1524
1584
|
for (let e of t) await u(e);
|
|
1525
1585
|
}
|
|
1526
|
-
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof t.savepoint == "function", S = [],
|
|
1586
|
+
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof t.savepoint == "function", S = [], C = async () => {
|
|
1527
1587
|
if (S.length === 0) return;
|
|
1528
1588
|
let e = S;
|
|
1529
1589
|
S = [];
|
|
1530
1590
|
try {
|
|
1531
1591
|
await t.savepoint(async () => {
|
|
1532
|
-
for (let n of e) await
|
|
1592
|
+
for (let n of e) await T(t, h, n);
|
|
1533
1593
|
});
|
|
1534
1594
|
} catch {
|
|
1535
1595
|
for (let t of e) await u(t);
|
|
1536
1596
|
}
|
|
1537
1597
|
};
|
|
1538
|
-
return yield* l.runForEach(
|
|
1539
|
-
if (o++,
|
|
1598
|
+
return yield* l.runForEach(rn(c, h.name, h.columnTypes), (e) => n.promise(async () => {
|
|
1599
|
+
if (o++, M?.(), d) {
|
|
1540
1600
|
v.push(e), v.length >= g && await y();
|
|
1541
1601
|
return;
|
|
1542
1602
|
}
|
|
1543
1603
|
if (x) {
|
|
1544
|
-
S.push(e), S.length >= b && await
|
|
1604
|
+
S.push(e), S.length >= b && await C();
|
|
1545
1605
|
return;
|
|
1546
1606
|
}
|
|
1547
1607
|
await u(e);
|
|
1548
|
-
})), yield* n.promise(
|
|
1608
|
+
})), yield* n.promise(C), yield* n.promise(y), {
|
|
1549
1609
|
rowCount: o,
|
|
1550
1610
|
checksum: i.digest("hex")
|
|
1551
1611
|
};
|
|
1552
|
-
}), { rowCount: S, checksum:
|
|
1612
|
+
}), { rowCount: S, checksum: C } = yield* Bt("import", h.name, d, g, (e) => e.rowCount, x);
|
|
1553
1613
|
if (c) {
|
|
1554
|
-
if (
|
|
1614
|
+
if (C !== h.checksum) return yield* n.fail(new Je({
|
|
1555
1615
|
what: `table ${h.name} checksum`,
|
|
1556
1616
|
expected: h.checksum,
|
|
1557
|
-
actual:
|
|
1617
|
+
actual: C
|
|
1558
1618
|
}));
|
|
1559
|
-
if (S !== h.rowCount) return yield* n.fail(new
|
|
1619
|
+
if (S !== h.rowCount) return yield* n.fail(new Je({
|
|
1560
1620
|
what: `table ${h.name} rowCount`,
|
|
1561
1621
|
expected: String(h.rowCount),
|
|
1562
1622
|
actual: String(S)
|
|
@@ -1570,7 +1630,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1570
1630
|
event: "done",
|
|
1571
1631
|
rowsDone: S,
|
|
1572
1632
|
total: h.rowCount
|
|
1573
|
-
}),
|
|
1633
|
+
}), j++, A += S, he?.(), M?.(), r && (a.length > y ? o.push({
|
|
1574
1634
|
name: h.name,
|
|
1575
1635
|
rowCount: S
|
|
1576
1636
|
}) : (b.tables[h.name] = {
|
|
@@ -1578,10 +1638,10 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1578
1638
|
checksum: h.checksum
|
|
1579
1639
|
}, yield* J(v, b)));
|
|
1580
1640
|
}
|
|
1581
|
-
if (a.length > 0 && (yield*
|
|
1641
|
+
if (a.length > 0 && (yield* le(t, a)), f === "replace" && (yield* n.promise(() => t.deleteMany(d, { where: {
|
|
1582
1642
|
column: "startedAt",
|
|
1583
1643
|
op: "lte",
|
|
1584
|
-
value:
|
|
1644
|
+
value: I
|
|
1585
1645
|
} }).then(() => void 0, () => void 0))), r && o.length > 0) {
|
|
1586
1646
|
for (let e of o) {
|
|
1587
1647
|
let t = s.tables.find((t) => t.name === e.name);
|
|
@@ -1593,32 +1653,32 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1593
1653
|
yield* J(v, b);
|
|
1594
1654
|
}
|
|
1595
1655
|
{
|
|
1596
|
-
let t =
|
|
1656
|
+
let t = Ke(p, "skipped");
|
|
1597
1657
|
t !== void 0 && e.onWarn?.(t);
|
|
1598
1658
|
}
|
|
1599
1659
|
if (u.length > 0) {
|
|
1600
1660
|
let t = u.length === s.tables.length;
|
|
1601
1661
|
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${v} to force a full re-import.`);
|
|
1602
1662
|
}
|
|
1603
|
-
}),
|
|
1663
|
+
}), F = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, I = /* @__PURE__ */ new Date(), L = yield* Jn({
|
|
1604
1664
|
store: e.store,
|
|
1605
1665
|
mode: f,
|
|
1606
1666
|
dir: e.rollbackDir,
|
|
1607
1667
|
snapshot: e.targetSnapshot,
|
|
1608
1668
|
dialect: e.targetDialect,
|
|
1609
|
-
tables: s.tables.map((e) => e.name).filter((e) => !
|
|
1669
|
+
tables: s.tables.map((e) => e.name).filter((e) => !W(e)),
|
|
1610
1670
|
onWarn: e.onWarn
|
|
1611
1671
|
});
|
|
1612
|
-
|
|
1613
|
-
try: () => e.onRollbackCaptured(
|
|
1614
|
-
catch: (e) => new
|
|
1672
|
+
L !== void 0 && (e.onRollbackCaptured !== void 0 && (yield* n.tryPromise({
|
|
1673
|
+
try: () => e.onRollbackCaptured(L),
|
|
1674
|
+
catch: (e) => new G({
|
|
1615
1675
|
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1616
|
-
path:
|
|
1676
|
+
path: L.dir
|
|
1617
1677
|
})
|
|
1618
|
-
})), e.onWarn?.(
|
|
1619
|
-
let
|
|
1620
|
-
|
|
1621
|
-
id:
|
|
1678
|
+
})), e.onWarn?.(Yn(L)));
|
|
1679
|
+
let _e = (e) => f === "replace" ? te(e) : e(), R = (e) => f === "replace" ? ne(e) : e, z = e.traceId ?? h("dimp", `${Date.now()}-${s.source.schemaFingerprint ?? ""}`), ve = e.traceId !== void 0, B = (e.targetSnapshot?.tables.some((e) => e.name === u) ?? !1) || y(u) !== void 0;
|
|
1680
|
+
B && g(p), B && e.traceId === void 0 && (yield* n.promise(() => e.store.insert(u, {
|
|
1681
|
+
id: z,
|
|
1622
1682
|
mode: f,
|
|
1623
1683
|
via: e.via ?? "direct",
|
|
1624
1684
|
direction: "import",
|
|
@@ -1626,45 +1686,45 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1626
1686
|
sourceFingerprint: s.source.schemaFingerprint ?? null,
|
|
1627
1687
|
tablesWritten: 0,
|
|
1628
1688
|
rowsWritten: 0,
|
|
1629
|
-
staged: f === "replace" ?
|
|
1689
|
+
staged: f === "replace" ? k : null
|
|
1630
1690
|
}).then(() => {
|
|
1631
|
-
|
|
1691
|
+
ve = !0;
|
|
1632
1692
|
}, (t) => {
|
|
1633
1693
|
e.onWarn?.(`the import trace could not be written (${String(t?.message ?? t)}).\n The import continues — this is a record of what happened, not a safety check. Run
|
|
1634
1694
|
\`voltro db apply\` so the next one is recorded.`);
|
|
1635
|
-
}))), e.traceId !== void 0 &&
|
|
1636
|
-
staged: f === "replace" ?
|
|
1695
|
+
}))), e.traceId !== void 0 && B && (yield* n.promise(() => e.store.update(u, z, {
|
|
1696
|
+
staged: f === "replace" ? k : null,
|
|
1637
1697
|
sourceFingerprint: s.source.schemaFingerprint ?? null
|
|
1638
1698
|
}).then(() => void 0, () => void 0)));
|
|
1639
|
-
let
|
|
1640
|
-
|
|
1641
|
-
if (!
|
|
1699
|
+
let ye = 0, be = !1;
|
|
1700
|
+
he = () => {
|
|
1701
|
+
if (!ve || be) return;
|
|
1642
1702
|
let t = Date.now();
|
|
1643
|
-
t -
|
|
1644
|
-
tablesWritten:
|
|
1645
|
-
rowsWritten:
|
|
1703
|
+
t - ye < tr || (ye = t, e.store.update(u, z, {
|
|
1704
|
+
tablesWritten: j,
|
|
1705
|
+
rowsWritten: A
|
|
1646
1706
|
}).then(() => void 0, () => {
|
|
1647
|
-
|
|
1707
|
+
be = !0;
|
|
1648
1708
|
}));
|
|
1649
1709
|
};
|
|
1650
|
-
let
|
|
1710
|
+
let xe = (t) => ve ? e.store.update(u, z, {
|
|
1651
1711
|
finishedAt: t === null ? /* @__PURE__ */ new Date() : null,
|
|
1652
|
-
tablesWritten:
|
|
1653
|
-
rowsWritten:
|
|
1712
|
+
tablesWritten: j,
|
|
1713
|
+
rowsWritten: A,
|
|
1654
1714
|
failure: t
|
|
1655
1715
|
}).then(() => void 0, () => void 0) : Promise.resolve();
|
|
1656
|
-
if (t = (e) =>
|
|
1716
|
+
if (t = (e) => xe(e), f === "replace" && !k && O.blocker !== void 0 && e.onWarn?.(Nn(O.blocker)), f === "replace" && fe.length > 0 && e.onWarn?.(`write recorders are SUSPENDED for this replace (${fe.length} table(s) have one).\n A replace sets a state rather than changing rows, so a per-row history entry would describe
|
|
1657
1717
|
something that did not happen — and this path already writes through the raw store, without
|
|
1658
|
-
tenant scoping or \`audit()\` stamping, for the same reason.`),
|
|
1659
|
-
e.onWarn?.(`staging ${
|
|
1660
|
-
let t = e.store, r =
|
|
1718
|
+
tenant scoping or \`audit()\` stamping, for the same reason.`), k) {
|
|
1719
|
+
e.onWarn?.(`staging ${de.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1720
|
+
let t = e.store, r = de.map((e) => e.name), i = h("repl", `staging-${F}`), a = !1;
|
|
1661
1721
|
if (e.recordInterruption === !0) {
|
|
1662
1722
|
yield* n.promise(() => e.store.insert(d, {
|
|
1663
1723
|
id: i,
|
|
1664
|
-
startedAt:
|
|
1724
|
+
startedAt: I,
|
|
1665
1725
|
kind: "staging",
|
|
1666
|
-
tables: String(
|
|
1667
|
-
capture:
|
|
1726
|
+
tables: String(de.length),
|
|
1727
|
+
capture: L?.dir ?? null,
|
|
1668
1728
|
via: e.via ?? "direct",
|
|
1669
1729
|
staging: r.map(X).join(","),
|
|
1670
1730
|
heartbeatAt: /* @__PURE__ */ new Date(),
|
|
@@ -1677,50 +1737,50 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1677
1737
|
them by introspection.`);
|
|
1678
1738
|
}));
|
|
1679
1739
|
let t = Date.now();
|
|
1680
|
-
|
|
1740
|
+
M = () => {
|
|
1681
1741
|
if (!a) return;
|
|
1682
1742
|
let n = Date.now();
|
|
1683
|
-
n - t <
|
|
1743
|
+
n - t < nr || (t = n, e.store.update(d, i, { heartbeatAt: /* @__PURE__ */ new Date() }).then(() => void 0, () => void 0));
|
|
1684
1744
|
};
|
|
1685
1745
|
}
|
|
1686
|
-
let o = () => a ? e.store.delete(d, i).then(() => void 0, () => void 0) : Promise.resolve(), s =
|
|
1746
|
+
let o = () => a ? e.store.delete(d, i).then(() => void 0, () => void 0) : Promise.resolve(), s = Rn(r, {
|
|
1687
1747
|
getTable: (e) => y(e),
|
|
1688
1748
|
registerTable: (e) => {
|
|
1689
|
-
|
|
1749
|
+
ee(e);
|
|
1690
1750
|
},
|
|
1691
|
-
unregisterTable:
|
|
1751
|
+
unregisterTable: re
|
|
1692
1752
|
}), c = !1;
|
|
1693
1753
|
try {
|
|
1694
|
-
let i = m ? r : r.filter((e) => !
|
|
1754
|
+
let i = m ? r : r.filter((e) => !Ut(b, e));
|
|
1695
1755
|
yield* n.tryPromise({
|
|
1696
1756
|
try: async () => {
|
|
1697
|
-
await
|
|
1757
|
+
await Hn(t, i.map((e) => In(e, D))), await Hn(t, i.map((e) => Fn(e, D)));
|
|
1698
1758
|
},
|
|
1699
|
-
catch: (e) => new
|
|
1700
|
-
}), yield*
|
|
1701
|
-
try: () =>
|
|
1759
|
+
catch: (e) => new G({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1760
|
+
}), yield* R(ge(e.store, !m)), !m && b.truncated === !0 ? c = !0 : yield* n.tryPromise({
|
|
1761
|
+
try: () => Un(t, de, D).then(() => {
|
|
1702
1762
|
c = !0;
|
|
1703
1763
|
}),
|
|
1704
|
-
catch: (e) => new
|
|
1764
|
+
catch: (e) => new G({ reason: String(e?.message ?? e) })
|
|
1705
1765
|
}).pipe(n.catchAll((i) => n.gen(function* () {
|
|
1706
|
-
let a =
|
|
1707
|
-
return yield* n.fail(new
|
|
1766
|
+
let a = Vn(yield* n.promise(() => Wn(t, zn(e.targetSnapshot, r), D).then((e) => e, () => [])));
|
|
1767
|
+
return yield* n.fail(new G({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1708
1768
|
}))), !m && c && b.truncated !== !0 && (b.truncated = !0, yield* J(v, b));
|
|
1709
1769
|
} finally {
|
|
1710
|
-
s.undo(), c || m ? (
|
|
1770
|
+
s.undo(), c || m ? (M = void 0, yield* n.promise(() => Hn(t, r.map((e) => In(e, D))).then(() => void 0, () => void 0)), yield* n.promise(o)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1711
1771
|
}
|
|
1712
1772
|
} else m ? yield* n.tryPromise({
|
|
1713
1773
|
try: () => e.store.transactional(async (e) => {
|
|
1714
|
-
let t = await
|
|
1715
|
-
if (r.isFailure(t)) throw
|
|
1774
|
+
let t = await _e(() => n.runPromiseExit(ge(e, !1)));
|
|
1775
|
+
if (r.isFailure(t)) throw or(t.cause);
|
|
1716
1776
|
}),
|
|
1717
|
-
catch: (e) =>
|
|
1718
|
-
}) : yield*
|
|
1777
|
+
catch: (e) => ar(e) ?? new G({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1778
|
+
}) : yield* R(ge(e.store, !0));
|
|
1719
1779
|
if (e.assets && s.assets.index && !e.assetsAlreadyRestored) {
|
|
1720
1780
|
let t = e.assets;
|
|
1721
|
-
yield*
|
|
1781
|
+
yield* gt({
|
|
1722
1782
|
sink: t,
|
|
1723
|
-
assetsDir: i.join(e.bundleDir,
|
|
1783
|
+
assetsDir: i.join(e.bundleDir, Mt),
|
|
1724
1784
|
indexPath: i.join(e.bundleDir, s.assets.index ?? "assets/index.ndjson"),
|
|
1725
1785
|
progress: {
|
|
1726
1786
|
done: (e) => e in b.assets,
|
|
@@ -1728,28 +1788,28 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1728
1788
|
}
|
|
1729
1789
|
});
|
|
1730
1790
|
}
|
|
1731
|
-
return yield* n.promise(() =>
|
|
1791
|
+
return yield* n.promise(() => xe(null)), {
|
|
1732
1792
|
manifest: s,
|
|
1733
|
-
tablesWritten:
|
|
1734
|
-
rowsWritten:
|
|
1735
|
-
tablesSkipped:
|
|
1736
|
-
rowsSkipped:
|
|
1737
|
-
fullyResumed:
|
|
1738
|
-
...
|
|
1793
|
+
tablesWritten: j,
|
|
1794
|
+
rowsWritten: A,
|
|
1795
|
+
tablesSkipped: P,
|
|
1796
|
+
rowsSkipped: N,
|
|
1797
|
+
fullyResumed: j === 0 && P > 0,
|
|
1798
|
+
...L === void 0 ? {} : { rollback: L }
|
|
1739
1799
|
};
|
|
1740
1800
|
}).pipe(n.tapError((e) => n.promise(() => t?.(e?.reason ?? e?._tag ?? "the import did not finish") ?? Promise.resolve())));
|
|
1741
|
-
},
|
|
1801
|
+
}, lr = (e) => e, ur = (e) => {
|
|
1742
1802
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1743
1803
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1744
1804
|
return {
|
|
1745
1805
|
refuse: !0,
|
|
1746
1806
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1747
1807
|
};
|
|
1748
|
-
}, Q = Buffer.from("VBUNDLE2"),
|
|
1808
|
+
}, Q = Buffer.from("VBUNDLE2"), dr = 0, fr = 1, pr = 2, mr = 32, hr = async (e) => {
|
|
1749
1809
|
let t = [], n = async (r) => {
|
|
1750
1810
|
let i;
|
|
1751
1811
|
try {
|
|
1752
|
-
i = await
|
|
1812
|
+
i = await L(B(e, r), { withFileTypes: !0 });
|
|
1753
1813
|
} catch {
|
|
1754
1814
|
return;
|
|
1755
1815
|
}
|
|
@@ -1759,22 +1819,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1759
1819
|
}
|
|
1760
1820
|
};
|
|
1761
1821
|
return await n(""), t.sort();
|
|
1762
|
-
},
|
|
1822
|
+
}, gr = (e, t) => {
|
|
1763
1823
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1764
|
-
return r.writeUInt8(
|
|
1765
|
-
},
|
|
1824
|
+
return r.writeUInt8(fr, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1825
|
+
}, _r = (e, t, n) => {
|
|
1766
1826
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1767
|
-
return a.writeUInt8(
|
|
1827
|
+
return a.writeUInt8(pr, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1768
1828
|
};
|
|
1769
|
-
async function*
|
|
1829
|
+
async function* vr(e) {
|
|
1770
1830
|
yield new Uint8Array(Q);
|
|
1771
|
-
for (let t of await
|
|
1772
|
-
let n =
|
|
1773
|
-
if (yield new Uint8Array(
|
|
1831
|
+
for (let t of await hr(e.dir)) {
|
|
1832
|
+
let n = B(e.dir, t), r = await _e(n);
|
|
1833
|
+
if (yield new Uint8Array(gr(t, r.size)), r.size > 0) for await (let e of ue(n)) yield new Uint8Array(e);
|
|
1774
1834
|
}
|
|
1775
1835
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1776
|
-
yield new Uint8Array(
|
|
1777
|
-
let n =
|
|
1836
|
+
yield new Uint8Array(_r(t.key, t.contentType, t.size));
|
|
1837
|
+
let n = ae("sha256"), r = 0;
|
|
1778
1838
|
for await (let i of e.blobs.open(t.key)) {
|
|
1779
1839
|
let e = Buffer.from(i);
|
|
1780
1840
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1782,11 +1842,11 @@ async function* gr(e) {
|
|
|
1782
1842
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1783
1843
|
yield new Uint8Array(n.digest());
|
|
1784
1844
|
}
|
|
1785
|
-
yield new Uint8Array(Buffer.from([
|
|
1845
|
+
yield new Uint8Array(Buffer.from([dr]));
|
|
1786
1846
|
}
|
|
1787
|
-
var
|
|
1788
|
-
let n =
|
|
1789
|
-
await
|
|
1847
|
+
var yr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), br = (e) => new Promise((t) => e.end(t)), xr = async (e, t) => {
|
|
1848
|
+
let n = ye(t.dir);
|
|
1849
|
+
await I(n, { recursive: !0 });
|
|
1790
1850
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1791
1851
|
let { value: e, done: t } = await i.next();
|
|
1792
1852
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1801,24 +1861,24 @@ var _r = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vr
|
|
|
1801
1861
|
for (;;) {
|
|
1802
1862
|
await s(1);
|
|
1803
1863
|
let e = c(1).readUInt8(0);
|
|
1804
|
-
if (e ===
|
|
1805
|
-
if (e ===
|
|
1864
|
+
if (e === dr) return;
|
|
1865
|
+
if (e === fr) {
|
|
1806
1866
|
await s(4);
|
|
1807
1867
|
let e = c(4).readUInt32BE(0);
|
|
1808
1868
|
await s(e);
|
|
1809
1869
|
let t = c(e).toString("utf8");
|
|
1810
1870
|
await s(8);
|
|
1811
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1812
|
-
if (a !== n && !a.startsWith(n +
|
|
1813
|
-
await
|
|
1814
|
-
let o =
|
|
1871
|
+
let i = c(8).readBigUInt64BE(0), a = ye(n, t);
|
|
1872
|
+
if (a !== n && !a.startsWith(n + be)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1873
|
+
await I(ve(a), { recursive: !0 });
|
|
1874
|
+
let o = O(a);
|
|
1815
1875
|
for (; i > 0n;) {
|
|
1816
1876
|
r.length === 0 && await s(1);
|
|
1817
1877
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1818
|
-
await
|
|
1878
|
+
await yr(o, c(e)), i -= BigInt(e);
|
|
1819
1879
|
}
|
|
1820
|
-
await
|
|
1821
|
-
} else if (e ===
|
|
1880
|
+
await br(o);
|
|
1881
|
+
} else if (e === pr) {
|
|
1822
1882
|
await s(4);
|
|
1823
1883
|
let e = c(4).readUInt32BE(0);
|
|
1824
1884
|
await s(e);
|
|
@@ -1828,24 +1888,24 @@ var _r = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vr
|
|
|
1828
1888
|
await s(i);
|
|
1829
1889
|
let a = c(i).toString("utf8");
|
|
1830
1890
|
await s(8);
|
|
1831
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1891
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = ae("sha256"), d = new P(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1832
1892
|
contentType: a,
|
|
1833
1893
|
size: Number(o)
|
|
1834
1894
|
}), p = o;
|
|
1835
1895
|
for (; p > 0n;) {
|
|
1836
1896
|
r.length === 0 && await s(1);
|
|
1837
1897
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1838
|
-
u.update(t), l || await
|
|
1898
|
+
u.update(t), l || await yr(d, t), p -= BigInt(e);
|
|
1839
1899
|
}
|
|
1840
|
-
l ? d.destroy() : await
|
|
1841
|
-
let m = c(
|
|
1900
|
+
l ? d.destroy() : await br(d), await f, await s(mr);
|
|
1901
|
+
let m = c(mr);
|
|
1842
1902
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1843
1903
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1844
1904
|
}
|
|
1845
1905
|
} finally {
|
|
1846
1906
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1847
1907
|
}
|
|
1848
|
-
},
|
|
1908
|
+
}, Sr = async (e, t) => {
|
|
1849
1909
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1850
1910
|
let { value: e, done: t } = await a.next();
|
|
1851
1911
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1867,8 +1927,8 @@ var _r = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vr
|
|
|
1867
1927
|
if (!await s(Q.length) || !c(Q.length).equals(Q)) throw Error("not a bundle archive (bad magic)");
|
|
1868
1928
|
for (; await s(1);) {
|
|
1869
1929
|
let e = c(1).readUInt8(0);
|
|
1870
|
-
if (e ===
|
|
1871
|
-
if (e !==
|
|
1930
|
+
if (e === dr || e === pr) break;
|
|
1931
|
+
if (e !== fr) throw Error(`unknown bundle entry kind ${e}`);
|
|
1872
1932
|
if (!await s(4)) break;
|
|
1873
1933
|
let t = c(4).readUInt32BE(0);
|
|
1874
1934
|
if (!await s(t)) break;
|
|
@@ -1878,22 +1938,22 @@ var _r = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vr
|
|
|
1878
1938
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1879
1939
|
}
|
|
1880
1940
|
return r;
|
|
1881
|
-
},
|
|
1941
|
+
}, Cr = (e, t) => xr(e, {
|
|
1882
1942
|
dir: t,
|
|
1883
|
-
blobs:
|
|
1884
|
-
}),
|
|
1943
|
+
blobs: wr(t)
|
|
1944
|
+
}), wr = (e) => {
|
|
1885
1945
|
let t = /* @__PURE__ */ new Set();
|
|
1886
1946
|
return {
|
|
1887
1947
|
has: async (e) => t.has(e),
|
|
1888
1948
|
write: async (n, r, i) => {
|
|
1889
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1890
|
-
await
|
|
1891
|
-
let l =
|
|
1949
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = B(e, "assets");
|
|
1950
|
+
await I(c, { recursive: !0 });
|
|
1951
|
+
let l = ae("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1892
1952
|
l.update(e), u += e.length, n(null, e);
|
|
1893
|
-
} }), f =
|
|
1894
|
-
await o(r, d,
|
|
1953
|
+
} }), f = B(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1954
|
+
await o(r, d, O(f));
|
|
1895
1955
|
let p = l.digest("hex");
|
|
1896
|
-
await a.rename(f,
|
|
1956
|
+
await a.rename(f, B(c, p)), await a.appendFile(B(c, "index.ndjson"), `${JSON.stringify({
|
|
1897
1957
|
key: n,
|
|
1898
1958
|
sha256: p,
|
|
1899
1959
|
size: u,
|
|
@@ -1901,33 +1961,33 @@ var _r = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vr
|
|
|
1901
1961
|
})}\n`), t.add(n);
|
|
1902
1962
|
}
|
|
1903
1963
|
};
|
|
1904
|
-
},
|
|
1964
|
+
}, Tr = xe(le), Er = Buffer.from("VENC1\0\0\0"), Dr = 32, Or = 16, kr = 7, Ar = 16, jr = 262144, Mr = {
|
|
1905
1965
|
N: 32768,
|
|
1906
1966
|
r: 8,
|
|
1907
1967
|
p: 1
|
|
1908
|
-
},
|
|
1968
|
+
}, Nr = 134217728, Pr = 43, Fr = (e, t, n) => {
|
|
1909
1969
|
let r = Buffer.alloc(12);
|
|
1910
|
-
return e.copy(r, 0, 0,
|
|
1970
|
+
return e.copy(r, 0, 0, kr), r.writeUInt32BE(t >>> 0, kr), r.writeUInt8(+!!n, 11), r;
|
|
1911
1971
|
};
|
|
1912
|
-
async function*
|
|
1972
|
+
async function* Ir(e, t) {
|
|
1913
1973
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1914
|
-
let n =
|
|
1915
|
-
...
|
|
1916
|
-
maxmem:
|
|
1917
|
-
}), a = Buffer.alloc(
|
|
1918
|
-
|
|
1974
|
+
let n = se(Or), r = se(kr), i = await Tr(t, n, Dr, {
|
|
1975
|
+
...Mr,
|
|
1976
|
+
maxmem: Nr
|
|
1977
|
+
}), a = Buffer.alloc(Pr);
|
|
1978
|
+
Er.copy(a, 0), a.writeUInt32BE(Mr.N, 8), a.writeUInt32BE(Mr.r, 12), a.writeUInt32BE(Mr.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1919
1979
|
let o = 0, s = (e, t) => {
|
|
1920
|
-
let n =
|
|
1980
|
+
let n = E("aes-256-gcm", i, Fr(r, o, t));
|
|
1921
1981
|
o === 0 && n.setAAD(a);
|
|
1922
1982
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1923
1983
|
o++;
|
|
1924
|
-
let l = Buffer.alloc(5 + s.length +
|
|
1925
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
1984
|
+
let l = Buffer.alloc(5 + s.length + Ar);
|
|
1985
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + Ar, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1926
1986
|
}, c = Buffer.alloc(0);
|
|
1927
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1987
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > jr;) yield s(c.subarray(0, jr), !1), c = c.subarray(jr);
|
|
1928
1988
|
yield s(c, !0);
|
|
1929
1989
|
}
|
|
1930
|
-
async function*
|
|
1990
|
+
async function* Lr(e, t) {
|
|
1931
1991
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1932
1992
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1933
1993
|
let { value: e, done: t } = await n.next();
|
|
@@ -1936,16 +1996,16 @@ async function* Fr(e, t) {
|
|
|
1936
1996
|
for (; r.length < e && !i && await a(););
|
|
1937
1997
|
return r.length >= e;
|
|
1938
1998
|
};
|
|
1939
|
-
if (!await o(
|
|
1940
|
-
let s = r.subarray(0,
|
|
1941
|
-
if (!s.subarray(0, 8).equals(
|
|
1942
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36,
|
|
1999
|
+
if (!await o(Pr)) throw Error("encrypted bundle truncated (no header)");
|
|
2000
|
+
let s = r.subarray(0, Pr);
|
|
2001
|
+
if (!s.subarray(0, 8).equals(Er)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
2002
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Pr)), p = await Tr(t, d, Dr, {
|
|
1943
2003
|
N: c,
|
|
1944
2004
|
r: l,
|
|
1945
2005
|
p: u,
|
|
1946
|
-
maxmem:
|
|
2006
|
+
maxmem: Nr
|
|
1947
2007
|
});
|
|
1948
|
-
r = r.subarray(
|
|
2008
|
+
r = r.subarray(Pr);
|
|
1949
2009
|
let m = 0, h = !1;
|
|
1950
2010
|
for (;;) {
|
|
1951
2011
|
if (!await o(5)) {
|
|
@@ -1954,11 +2014,11 @@ async function* Fr(e, t) {
|
|
|
1954
2014
|
}
|
|
1955
2015
|
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
1956
2016
|
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
1957
|
-
if (t <
|
|
2017
|
+
if (t < Ar) throw Error("encrypted bundle corrupt (short frame)");
|
|
1958
2018
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1959
|
-
let n = r.subarray(5, 5 + t -
|
|
2019
|
+
let n = r.subarray(5, 5 + t - Ar), i = r.subarray(5 + t - Ar, 5 + t);
|
|
1960
2020
|
r = r.subarray(5 + t);
|
|
1961
|
-
let a =
|
|
2021
|
+
let a = ie("aes-256-gcm", p, Fr(f, m, e));
|
|
1962
2022
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1963
2023
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1964
2024
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1966,7 +2026,7 @@ async function* Fr(e, t) {
|
|
|
1966
2026
|
}
|
|
1967
2027
|
//#endregion
|
|
1968
2028
|
//#region src/blobStorage.ts
|
|
1969
|
-
var
|
|
2029
|
+
var Rr = (e, t) => ({
|
|
1970
2030
|
list: () => {
|
|
1971
2031
|
let e = t();
|
|
1972
2032
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1979,32 +2039,32 @@ var Ir = (e, t) => ({
|
|
|
1979
2039
|
} };
|
|
1980
2040
|
},
|
|
1981
2041
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1982
|
-
let r = await n.runPromise(
|
|
2042
|
+
let r = await n.runPromise(j(e, t)), i = await M(r.stream);
|
|
1983
2043
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1984
2044
|
} })
|
|
1985
|
-
}),
|
|
1986
|
-
table:
|
|
2045
|
+
}), zr = (e, t) => Rr(e, () => w(t, {
|
|
2046
|
+
table: A,
|
|
1987
2047
|
chunkSize: 500
|
|
1988
2048
|
}).pipe(l.map((e) => ({
|
|
1989
2049
|
key: String(e.key),
|
|
1990
2050
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1991
2051
|
size: Number(e.size ?? 0)
|
|
1992
|
-
})))),
|
|
2052
|
+
})))), Br = (e) => ({
|
|
1993
2053
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1994
2054
|
write: async (t, r, i) => {
|
|
1995
|
-
let a =
|
|
1996
|
-
await n.runPromise(
|
|
2055
|
+
let a = me("storage", () => ge.from(r));
|
|
2056
|
+
await n.runPromise(he(e, t, a, {
|
|
1997
2057
|
contentType: i.contentType,
|
|
1998
2058
|
totalSize: i.size
|
|
1999
2059
|
}));
|
|
2000
2060
|
}
|
|
2001
|
-
}),
|
|
2061
|
+
}), Vr = (e, t) => {
|
|
2002
2062
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
2003
|
-
return e === "mariadb" ?
|
|
2063
|
+
return e === "mariadb" ? Hr(n) ? { tool: n } : {
|
|
2004
2064
|
tool: r,
|
|
2005
2065
|
hint: `'${n}' is not on PATH, so this ran '${r}' instead. If that binary came from Oracle's MySQL client it queries information_schema.COLUMN_STATISTICS, which MariaDB does not have, and the run dies with error 1109 after the first table. Install the MariaDB client package (which ships '${n}') and re-run. Do NOT reach for --column-statistics=0: that flag does not exist on '${n}', so it fixes the wrong client and breaks the right one. Note: a MariaDB 12.x client requires TLS by default, so running '${n}' BY HAND against a server without it fails with "TLS/SSL error: SSL is required" (2026) and needs --skip-ssl. This command is not affected.`
|
|
2006
2066
|
} : { tool: r };
|
|
2007
|
-
},
|
|
2067
|
+
}, Hr = (e) => (process.env.PATH ?? "").split(z).filter((e) => e !== "").some((t) => de(B(t, e))), Ur = (e) => {
|
|
2008
2068
|
switch (e) {
|
|
2009
2069
|
case "postgres": return "db.dump";
|
|
2010
2070
|
case "mysql":
|
|
@@ -2014,7 +2074,7 @@ var Ir = (e, t) => ({
|
|
|
2014
2074
|
case "mssql": return "db.bacpac";
|
|
2015
2075
|
default: return "db.dump";
|
|
2016
2076
|
}
|
|
2017
|
-
},
|
|
2077
|
+
}, Wr = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
2018
2078
|
if (e.host || e.database) return {
|
|
2019
2079
|
host: e.host ?? "localhost",
|
|
2020
2080
|
port: e.port ?? 3306,
|
|
@@ -2030,7 +2090,7 @@ var Ir = (e, t) => ({
|
|
|
2030
2090
|
password: decodeURIComponent(t.password) || "",
|
|
2031
2091
|
database: t.pathname.replace(/^\//, "") || ""
|
|
2032
2092
|
};
|
|
2033
|
-
},
|
|
2093
|
+
}, Gr = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Kr = (e, t, n) => {
|
|
2034
2094
|
switch (e) {
|
|
2035
2095
|
case "postgres": {
|
|
2036
2096
|
if (t.url) return {
|
|
@@ -2040,7 +2100,7 @@ var Ir = (e, t) => ({
|
|
|
2040
2100
|
"--format=custom",
|
|
2041
2101
|
"--no-owner",
|
|
2042
2102
|
"--no-privileges",
|
|
2043
|
-
...
|
|
2103
|
+
...Jr,
|
|
2044
2104
|
"--dbname",
|
|
2045
2105
|
t.url,
|
|
2046
2106
|
"--file",
|
|
@@ -2059,7 +2119,7 @@ var Ir = (e, t) => ({
|
|
|
2059
2119
|
"--format=custom",
|
|
2060
2120
|
"--no-owner",
|
|
2061
2121
|
"--no-privileges",
|
|
2062
|
-
...
|
|
2122
|
+
...Jr,
|
|
2063
2123
|
"-h",
|
|
2064
2124
|
e.host,
|
|
2065
2125
|
"-p",
|
|
@@ -2079,14 +2139,14 @@ var Ir = (e, t) => ({
|
|
|
2079
2139
|
let r = $(t);
|
|
2080
2140
|
return {
|
|
2081
2141
|
kind: "spawn",
|
|
2082
|
-
...
|
|
2142
|
+
...Vr(e, "dump"),
|
|
2083
2143
|
args: [
|
|
2084
2144
|
"--single-transaction",
|
|
2085
2145
|
"--routines",
|
|
2086
2146
|
"--triggers",
|
|
2087
2147
|
"--no-tablespaces",
|
|
2088
2148
|
"--protocol=TCP",
|
|
2089
|
-
...
|
|
2149
|
+
...qr.map((e) => `--ignore-table=${r.database}.${e}`),
|
|
2090
2150
|
"-h",
|
|
2091
2151
|
r.host,
|
|
2092
2152
|
"-P",
|
|
@@ -2102,7 +2162,7 @@ var Ir = (e, t) => ({
|
|
|
2102
2162
|
case "sqlite":
|
|
2103
2163
|
case "turso": return {
|
|
2104
2164
|
kind: "copy",
|
|
2105
|
-
from:
|
|
2165
|
+
from: Wr(t),
|
|
2106
2166
|
to: n
|
|
2107
2167
|
};
|
|
2108
2168
|
case "mssql": return {
|
|
@@ -2110,7 +2170,7 @@ var Ir = (e, t) => ({
|
|
|
2110
2170
|
tool: "sqlpackage",
|
|
2111
2171
|
args: [
|
|
2112
2172
|
"/Action:Export",
|
|
2113
|
-
`/SourceConnectionString:${
|
|
2173
|
+
`/SourceConnectionString:${Gr($({
|
|
2114
2174
|
...t,
|
|
2115
2175
|
port: t.port ?? 1433
|
|
2116
2176
|
}))}`,
|
|
@@ -2118,12 +2178,12 @@ var Ir = (e, t) => ({
|
|
|
2118
2178
|
],
|
|
2119
2179
|
env: {}
|
|
2120
2180
|
};
|
|
2121
|
-
default: throw new
|
|
2181
|
+
default: throw new K({
|
|
2122
2182
|
tool: "backup",
|
|
2123
2183
|
reason: `no native backup for dialect '${e}'`
|
|
2124
2184
|
});
|
|
2125
2185
|
}
|
|
2126
|
-
},
|
|
2186
|
+
}, qr = [d, u], Jr = qr.flatMap((e) => ["--exclude-table", e]), Yr = (e) => e === "postgres" || e === "mysql" || e === "mariadb" ? qr : [], Xr = (e, t, n) => {
|
|
2127
2187
|
switch (e) {
|
|
2128
2188
|
case "postgres": {
|
|
2129
2189
|
if (t.url) return {
|
|
@@ -2168,7 +2228,7 @@ var Ir = (e, t) => ({
|
|
|
2168
2228
|
let r = $(t);
|
|
2169
2229
|
return {
|
|
2170
2230
|
kind: "spawn",
|
|
2171
|
-
...
|
|
2231
|
+
...Vr(e, "client"),
|
|
2172
2232
|
args: [
|
|
2173
2233
|
"--protocol=TCP",
|
|
2174
2234
|
"-h",
|
|
@@ -2187,14 +2247,14 @@ var Ir = (e, t) => ({
|
|
|
2187
2247
|
case "turso": return {
|
|
2188
2248
|
kind: "copy",
|
|
2189
2249
|
from: n,
|
|
2190
|
-
to:
|
|
2250
|
+
to: Wr(t)
|
|
2191
2251
|
};
|
|
2192
2252
|
case "mssql": return {
|
|
2193
2253
|
kind: "spawn",
|
|
2194
2254
|
tool: "sqlpackage",
|
|
2195
2255
|
args: [
|
|
2196
2256
|
"/Action:Import",
|
|
2197
|
-
`/TargetConnectionString:${
|
|
2257
|
+
`/TargetConnectionString:${Gr($({
|
|
2198
2258
|
...t,
|
|
2199
2259
|
port: t.port ?? 1433
|
|
2200
2260
|
}))}`,
|
|
@@ -2202,27 +2262,27 @@ var Ir = (e, t) => ({
|
|
|
2202
2262
|
],
|
|
2203
2263
|
env: {}
|
|
2204
2264
|
};
|
|
2205
|
-
default: throw new
|
|
2265
|
+
default: throw new K({
|
|
2206
2266
|
tool: "restore",
|
|
2207
2267
|
reason: `no native restore for dialect '${e}'`
|
|
2208
2268
|
});
|
|
2209
2269
|
}
|
|
2210
|
-
},
|
|
2211
|
-
let r = `${e.to}.restoring-${
|
|
2270
|
+
}, Zr = (e) => e.kind === "copy" ? n.async((t) => {
|
|
2271
|
+
let r = `${e.to}.restoring-${ce()}`, i = (e) => {
|
|
2212
2272
|
try {
|
|
2213
|
-
|
|
2273
|
+
pe(r);
|
|
2214
2274
|
} catch {}
|
|
2215
|
-
t(n.fail(new
|
|
2275
|
+
t(n.fail(new K({
|
|
2216
2276
|
tool: "copy",
|
|
2217
2277
|
reason: e
|
|
2218
2278
|
})));
|
|
2219
2279
|
};
|
|
2220
|
-
|
|
2280
|
+
D(e.from, r, (a) => {
|
|
2221
2281
|
if (a) {
|
|
2222
2282
|
i(`copy ${e.from} → ${e.to}: ${a.message}`);
|
|
2223
2283
|
return;
|
|
2224
2284
|
}
|
|
2225
|
-
|
|
2285
|
+
fe(r, e.to, (r) => {
|
|
2226
2286
|
if (r) {
|
|
2227
2287
|
i(`copy ${e.from} → ${e.to}: ${r.message}`);
|
|
2228
2288
|
return;
|
|
@@ -2231,7 +2291,7 @@ var Ir = (e, t) => ({
|
|
|
2231
2291
|
});
|
|
2232
2292
|
});
|
|
2233
2293
|
}) : n.async((t) => {
|
|
2234
|
-
let r =
|
|
2294
|
+
let r = Se(e.tool, [...e.args], {
|
|
2235
2295
|
env: {
|
|
2236
2296
|
...process.env,
|
|
2237
2297
|
...e.env
|
|
@@ -2251,27 +2311,27 @@ var Ir = (e, t) => ({
|
|
|
2251
2311
|
s ??= e;
|
|
2252
2312
|
};
|
|
2253
2313
|
if (e.stdoutFile && r.stdout) {
|
|
2254
|
-
let t =
|
|
2255
|
-
t.on("error", (t) => c(`writing ${
|
|
2314
|
+
let t = O(e.stdoutFile);
|
|
2315
|
+
t.on("error", (t) => c(`writing ${R(e.stdoutFile)}: ${t.message}`)), r.stdout.on("error", (e) => c(`reading the tool's output: ${e.message}`)), r.stdout.pipe(t);
|
|
2256
2316
|
}
|
|
2257
2317
|
let l = 0;
|
|
2258
2318
|
if (e.stdinFile && r.stdin) {
|
|
2259
|
-
let t =
|
|
2319
|
+
let t = ue(e.stdinFile);
|
|
2260
2320
|
t.on("data", (e) => {
|
|
2261
2321
|
l += e.length;
|
|
2262
2322
|
}), t.on("error", (t) => {
|
|
2263
|
-
c(`reading ${
|
|
2323
|
+
c(`reading ${R(e.stdinFile)}: ${t.message}`), r.stdin?.end();
|
|
2264
2324
|
}), r.stdin.on("error", (t) => {
|
|
2265
|
-
t.code !== "EPIPE" && c(`feeding ${
|
|
2325
|
+
t.code !== "EPIPE" && c(`feeding ${R(e.stdinFile)} to ${e.tool}: ${t.message}`);
|
|
2266
2326
|
}), t.pipe(r.stdin);
|
|
2267
2327
|
}
|
|
2268
2328
|
let u = () => {
|
|
2269
2329
|
if (e.stdoutFile !== void 0) try {
|
|
2270
|
-
|
|
2330
|
+
pe(e.stdoutFile);
|
|
2271
2331
|
} catch {}
|
|
2272
2332
|
};
|
|
2273
2333
|
r.on("error", (t) => {
|
|
2274
|
-
u(), o(n.fail(new
|
|
2334
|
+
u(), o(n.fail(new K({
|
|
2275
2335
|
tool: e.tool,
|
|
2276
2336
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${t.message}`,
|
|
2277
2337
|
...e.hint === void 0 ? {} : { hint: e.hint }
|
|
@@ -2282,25 +2342,25 @@ var Ir = (e, t) => ({
|
|
|
2282
2342
|
return;
|
|
2283
2343
|
}
|
|
2284
2344
|
u();
|
|
2285
|
-
let r = e.stdinFile === void 0 ? "" : ` — ${l} of ${
|
|
2286
|
-
o(n.fail(new
|
|
2345
|
+
let r = e.stdinFile === void 0 ? "" : ` — ${l} of ${Qr(e.stdinFile)} byte(s) of ${R(e.stdinFile)} had been fed to it`;
|
|
2346
|
+
o(n.fail(new K({
|
|
2287
2347
|
tool: e.tool,
|
|
2288
|
-
reason: (t === 0 ? `exited cleanly, but ${s}` : `exited with code ${t}${s === void 0 ? "" : ` (${s})`}`) + r + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2348
|
+
reason: (t === 0 ? `exited cleanly, but ${s}` : `exited with code ${t}${s === void 0 ? "" : ` (${s})`}`) + r + (e.stdoutFile === void 0 ? "" : ` — the partial ${R(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
2289
2349
|
stderr: i.trim() === "" ? `(${e.tool} printed nothing to stderr)` : i.slice(-2e3),
|
|
2290
2350
|
...e.hint === void 0 ? {} : { hint: e.hint }
|
|
2291
2351
|
})));
|
|
2292
2352
|
});
|
|
2293
|
-
}),
|
|
2353
|
+
}), Qr = (e) => {
|
|
2294
2354
|
try {
|
|
2295
|
-
return String(
|
|
2355
|
+
return String(k(e).size);
|
|
2296
2356
|
} catch {
|
|
2297
2357
|
return "?";
|
|
2298
2358
|
}
|
|
2299
|
-
},
|
|
2300
|
-
let t = yield*
|
|
2359
|
+
}, $r = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), ei = (e) => n.gen(function* () {
|
|
2360
|
+
let t = yield* st(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
2301
2361
|
store: e.store,
|
|
2302
2362
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
2303
|
-
}), r =
|
|
2363
|
+
}), r = an(t.tables, e.snapshot), i = e.classification ?? Te(e.snapshot), a = e.sampleSize ?? 20, o = ze(r.map((t) => ({
|
|
2304
2364
|
name: t,
|
|
2305
2365
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
2306
2366
|
name: e.name,
|
|
@@ -2308,19 +2368,19 @@ var Ir = (e, t) => ({
|
|
|
2308
2368
|
}))
|
|
2309
2369
|
})), i, e.masking), s = [], c = [];
|
|
2310
2370
|
for (let i of r) {
|
|
2311
|
-
let r = t.predicateFor(i), u = yield*
|
|
2371
|
+
let r = t.predicateFor(i), u = yield* w(e.store, {
|
|
2312
2372
|
table: i,
|
|
2313
2373
|
chunkSize: a,
|
|
2314
2374
|
...r === void 0 ? {} : { where: r }
|
|
2315
|
-
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
2316
|
-
c.push(...
|
|
2375
|
+
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) => He(t, i, o.actions, e.masking.seed));
|
|
2376
|
+
c.push(...Ve(d, i, o.actions));
|
|
2317
2377
|
let f = u[0], p = d[0], m = [];
|
|
2318
2378
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
2319
2379
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
2320
2380
|
let n = e.slice(i.length + 1);
|
|
2321
2381
|
m.push({
|
|
2322
2382
|
column: n,
|
|
2323
|
-
action:
|
|
2383
|
+
action: $r(t),
|
|
2324
2384
|
before: f[n],
|
|
2325
2385
|
after: p[n]
|
|
2326
2386
|
});
|
|
@@ -2335,7 +2395,7 @@ var Ir = (e, t) => ({
|
|
|
2335
2395
|
leaks: c,
|
|
2336
2396
|
unclassified: o.unclassified
|
|
2337
2397
|
};
|
|
2338
|
-
}),
|
|
2398
|
+
}), ti = (e, t) => {
|
|
2339
2399
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
2340
2400
|
for (let t of e) {
|
|
2341
2401
|
let e = n.get(t.name);
|
|
@@ -2390,7 +2450,7 @@ var Ir = (e, t) => ({
|
|
|
2390
2450
|
skipColumns: i,
|
|
2391
2451
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
2392
2452
|
};
|
|
2393
|
-
},
|
|
2453
|
+
}, ni = (e) => {
|
|
2394
2454
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
2395
2455
|
let t = {
|
|
2396
2456
|
safe: "✓",
|
|
@@ -2401,4 +2461,4 @@ var Ir = (e, t) => ({
|
|
|
2401
2461
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
2402
2462
|
};
|
|
2403
2463
|
//#endregion
|
|
2404
|
-
export {
|
|
2464
|
+
export { Mt as ASSETS_DIR, Nt as ASSET_INDEX_FILE, G as BundleError, qe as CodecError, Ye as CompressionError, tt as CrossDialectError, jt as DATA_DIR, we as DEFAULT_CLASS_ACTIONS, Ue as ENVIRONMENT_LOCAL_TABLES, We as ENVIRONMENT_PORTABLE_TABLES, qr as EXCLUDED_FROM_DUMP, St as FORMAT_VERSION, et as ImportModeError, Je as IntegrityError, Pt as LEDGER_FILE, kt as MANIFEST_FILE, $e as MaskingError, K as NativeToolError, Ze as RowsRefusedError, rr as SAVEPOINT_BATCH_SIZE, jn as STAGING_PREFIX, Qe as SchemaDriftError, Xe as ScopeError, U as applyAction, ar as asImportError, ur as assessLiveImport, Wt as assetDone, Ur as backupArtifactName, Kr as backupCommand, Mn as canStage, yn as canonicalType, mt as captureAssetsToCas, Sn as checkPortability, er as checkSchemaDrift, Te as classificationFromSnapshot, ti as classifyImportDrift, yt as compressionExt, it as computeSubsetIds, qn as decideStaging, Ot as decodeManifest, en as decodeRow, Qt as decodeValue, Lr as decryptStream, lr as defineDataProfile, $n as diffSnapshots, wr as dirBlobSink, In as dropStagingSql, Yr as dumpExcludedTables, Vt as emptyLedger, Dt as encodeManifest, $t as encodeRow, Xt as encodeValue, Ir as encryptStream, Ke as environmentLocalNotice, Ge as environmentLocalReason, Hn as execStatements, ni as formatImportDrift, Cn as formatIssue, _t as hasZstd, W as isEnvironmentLocal, bt as makeCompressor, xt as makeDecompressor, He as maskRow, rn as ndjsonToRows, ft as noAssetProgress, vn as outsideReferenceRefusal, _n as outsideReferencesInto, vr as packBundle, Sr as peekBundle, vt as pickCompression, ze as planMasking, ei as previewMasking, ht as readAssetIndex, Ht as readLedger, Ee as resolveAction, st as resolveScope, dt as restoreAssetFromCas, gt as restoreAssetsFromCas, Xr as restoreCommand, nn as rowsToNdjson, gn as runExport, cr as runImport, Zr as runNativeStep, Ve as scanForLeaks, X as stagingNameFor, lt as storageAssetSink, ct as storageAssetSource, Br as storageBlobSink, Rr as storageBlobSource, zr as storageRefsBlobSource, at as subsetToScope, Ut as tableDone, an as topoSortTables, xr as unpackBundle, Cr as unpackBundleToDir, ut as writeAssetToCas, J as writeLedger };
|