@voltro/data-transfer 0.53.0 → 0.55.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +335 -0
- package/dist/index.d.ts +51 -10
- package/dist/index.js +494 -468
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Runtime as o, Schedule as s, Schema as c, Stream as l } from "effect";
|
|
2
|
-
import { DATA_TRANSFERS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d,
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { copyFile as
|
|
6
|
-
import { STORAGE_REFS_TABLE as
|
|
2
|
+
import { DATA_TRANSFERS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d, STAGING_TABLE_PREFIX as f, _voltroDataTransfersTable as p, classifyConstraintViolation as m, derivedRowId as h, ensureTableRegistered as g, eq as _, extractDbCause as v, getTable as y, inSet as b, isNotNull as x, or as S, recordsTable as ee, registerTable as te, streamTable as C, suspendWriteRecorders as ne, suspendWriteRecordersEffect as re, unregisterTable as ie } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as w } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as ae, createDecipheriv as oe, createHash as T, createHmac as se, randomBytes as ce, randomUUID as le, scrypt as ue } from "node:crypto";
|
|
5
|
+
import { copyFile as E, createReadStream as de, createWriteStream as D, existsSync as fe, rename as pe, statSync as O, unlinkSync as me } from "node:fs";
|
|
6
|
+
import { STORAGE_REFS_TABLE as k, fromNodeReadable as he, getObjectStream as A, putObjectStream as ge, toNodeReadable as j } from "@voltro/plugin-storage";
|
|
7
7
|
import * as M from "node:zlib";
|
|
8
|
-
import { PassThrough as N, Readable as
|
|
9
|
-
import { SqlClient as
|
|
10
|
-
import { mkdir as
|
|
11
|
-
import { basename as
|
|
12
|
-
import { promisify as
|
|
13
|
-
import { spawn as
|
|
8
|
+
import { PassThrough as N, Readable as _e } from "node:stream";
|
|
9
|
+
import { SqlClient as P } from "@effect/sql";
|
|
10
|
+
import { mkdir as F, readdir as I, stat as ve } from "node:fs/promises";
|
|
11
|
+
import { basename as L, delimiter as ye, dirname as be, join as R, resolve as xe, sep as Se } from "node:path";
|
|
12
|
+
import { promisify as Ce } from "node:util";
|
|
13
|
+
import { spawn as we } from "node:child_process";
|
|
14
14
|
//#region src/consistency.ts
|
|
15
|
-
var
|
|
15
|
+
var Te = (e) => {
|
|
16
16
|
switch (e) {
|
|
17
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
18
18
|
case "mysql":
|
|
@@ -22,7 +22,7 @@ var we = (e) => {
|
|
|
22
22
|
case "turso": return [];
|
|
23
23
|
default: return [];
|
|
24
24
|
}
|
|
25
|
-
},
|
|
25
|
+
}, Ee = {
|
|
26
26
|
email: { fake: "email" },
|
|
27
27
|
fullName: { fake: "fullName" },
|
|
28
28
|
firstName: { fake: "firstName" },
|
|
@@ -37,7 +37,7 @@ var we = (e) => {
|
|
|
37
37
|
date: "dateShift",
|
|
38
38
|
secret: "null",
|
|
39
39
|
freeText: "redact"
|
|
40
|
-
},
|
|
40
|
+
}, De = (e) => {
|
|
41
41
|
let t = {};
|
|
42
42
|
for (let n of e.tables) {
|
|
43
43
|
let e = {};
|
|
@@ -49,15 +49,15 @@ var we = (e) => {
|
|
|
49
49
|
t[n.name] = e;
|
|
50
50
|
}
|
|
51
51
|
return t;
|
|
52
|
-
},
|
|
52
|
+
}, Oe = (e, t, n, r) => {
|
|
53
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
54
54
|
if (i) return i;
|
|
55
55
|
if (n?.sensitive) {
|
|
56
56
|
let e = n.sensitive.class;
|
|
57
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? Ee[e] ?? "redact";
|
|
58
58
|
}
|
|
59
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
60
|
-
},
|
|
60
|
+
}, ke = (e, t) => se("sha256", e).update(t == null ? "∅" : String(t)).digest(), z = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), Ae = [
|
|
61
61
|
"Ada",
|
|
62
62
|
"Alan",
|
|
63
63
|
"Grace",
|
|
@@ -70,7 +70,7 @@ var we = (e) => {
|
|
|
70
70
|
"Guido",
|
|
71
71
|
"Margaret",
|
|
72
72
|
"Bjarne"
|
|
73
|
-
],
|
|
73
|
+
], je = [
|
|
74
74
|
"Lovelace",
|
|
75
75
|
"Turing",
|
|
76
76
|
"Hopper",
|
|
@@ -83,7 +83,7 @@ var we = (e) => {
|
|
|
83
83
|
"Rossum",
|
|
84
84
|
"Hamilton",
|
|
85
85
|
"Stroustrup"
|
|
86
|
-
],
|
|
86
|
+
], Me = [
|
|
87
87
|
"Maple",
|
|
88
88
|
"Oak",
|
|
89
89
|
"Cedar",
|
|
@@ -92,7 +92,7 @@ var we = (e) => {
|
|
|
92
92
|
"Birch",
|
|
93
93
|
"Willow",
|
|
94
94
|
"Ash"
|
|
95
|
-
],
|
|
95
|
+
], Ne = [
|
|
96
96
|
"Initech",
|
|
97
97
|
"Hooli",
|
|
98
98
|
"Umbrella",
|
|
@@ -101,23 +101,23 @@ var we = (e) => {
|
|
|
101
101
|
"Soylent",
|
|
102
102
|
"Stark",
|
|
103
103
|
"Wayne"
|
|
104
|
-
], B = (e, t, n) => e[z(t, n) % e.length],
|
|
105
|
-
let r =
|
|
104
|
+
], B = (e, t, n) => e[z(t, n) % e.length], Pe = (e, t, n) => {
|
|
105
|
+
let r = ke(n, `${e}:${String(t)}`), i = z(r, 1) % 1e4;
|
|
106
106
|
switch (e) {
|
|
107
|
-
case "email": return `${B(
|
|
108
|
-
case "fullName": return `${B(
|
|
109
|
-
case "firstName": return B(
|
|
110
|
-
case "lastName": return B(
|
|
111
|
-
case "username": return `${B(
|
|
107
|
+
case "email": return `${B(Ae, r, 0).toLowerCase()}.${B(je, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${B(Ae, r, 0)} ${B(je, r, 1)}`;
|
|
109
|
+
case "firstName": return B(Ae, r, 0);
|
|
110
|
+
case "lastName": return B(je, r, 1);
|
|
111
|
+
case "username": return `${B(Ae, r, 0).toLowerCase()}${i}`;
|
|
112
112
|
case "phone": return `+1${String(z(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
-
case "address": return `${z(r, 0) % 9899 + 100} ${B(
|
|
114
|
-
case "company": return B(
|
|
113
|
+
case "address": return `${z(r, 0) % 9899 + 100} ${B(Me, r, 1)} St`;
|
|
114
|
+
case "company": return B(Ne, r, 0);
|
|
115
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
116
116
|
case "ip": return `${z(r, 0) % 256}.${z(r, 1) % 256}.${z(r, 2) % 256}.${z(r, 3) % 254 + 1}`;
|
|
117
|
-
case "creditCard": return
|
|
117
|
+
case "creditCard": return Fe(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
118
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
119
119
|
}
|
|
120
|
-
},
|
|
120
|
+
}, Fe = (e) => {
|
|
121
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
122
122
|
t.forEach((e, t) => {
|
|
123
123
|
let r = e;
|
|
@@ -125,31 +125,31 @@ var we = (e) => {
|
|
|
125
125
|
});
|
|
126
126
|
let r = (10 - n % 10) % 10;
|
|
127
127
|
return e.slice(0, 15) + String(r);
|
|
128
|
-
},
|
|
128
|
+
}, Ie = 864e5, Le = (e, t) => {
|
|
129
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
130
130
|
if (Number.isNaN(n)) return e;
|
|
131
|
-
let r = z(
|
|
131
|
+
let r = z(ke(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Ie);
|
|
132
132
|
return e instanceof Date ? i : i.toISOString();
|
|
133
|
-
},
|
|
133
|
+
}, Re = (e, t) => {
|
|
134
134
|
if (t.value === null || t.value === void 0) return null;
|
|
135
135
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
136
136
|
if (e === "keep") return t.value;
|
|
137
137
|
if (e === "null") return null;
|
|
138
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
139
|
-
if (e === "hash") return
|
|
140
|
-
if (e === "dateShift") return
|
|
141
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return ke(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Le(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return Pe(e.fake, t.value, t.seed);
|
|
142
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
143
143
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
144
|
-
},
|
|
144
|
+
}, ze = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Be = (e, t, n) => {
|
|
145
145
|
let r = {}, i = [], a = [];
|
|
146
146
|
for (let o of e) for (let e of o.columns) {
|
|
147
|
-
let s = `${o.name}.${e.name}`, c =
|
|
147
|
+
let s = `${o.name}.${e.name}`, c = Oe(o.name, e.name, t[o.name]?.[e.name], n);
|
|
148
148
|
if (c === "error") {
|
|
149
149
|
i.push(s);
|
|
150
150
|
continue;
|
|
151
151
|
}
|
|
152
|
-
if (!
|
|
152
|
+
if (!ze(c)) {
|
|
153
153
|
a.push({
|
|
154
154
|
column: s,
|
|
155
155
|
action: c
|
|
@@ -163,7 +163,7 @@ var we = (e) => {
|
|
|
163
163
|
unclassified: i,
|
|
164
164
|
invalid: a
|
|
165
165
|
};
|
|
166
|
-
},
|
|
166
|
+
}, Ve = [
|
|
167
167
|
{
|
|
168
168
|
name: "email",
|
|
169
169
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -180,7 +180,7 @@ var we = (e) => {
|
|
|
180
180
|
name: "creditCard",
|
|
181
181
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
182
182
|
}
|
|
183
|
-
],
|
|
183
|
+
], He = (e, t, n) => {
|
|
184
184
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
185
185
|
for (let [a, o] of Object.entries(n)) {
|
|
186
186
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -188,7 +188,7 @@ var we = (e) => {
|
|
|
188
188
|
for (let a of e) {
|
|
189
189
|
let e = a[n];
|
|
190
190
|
if (typeof e != "string" || e.length === 0) continue;
|
|
191
|
-
let o =
|
|
191
|
+
let o = Ve.find((t) => t.re.test(e));
|
|
192
192
|
if (o && !i.has(n)) {
|
|
193
193
|
i.add(n), r.push({
|
|
194
194
|
table: t,
|
|
@@ -201,9 +201,9 @@ var we = (e) => {
|
|
|
201
201
|
}
|
|
202
202
|
}
|
|
203
203
|
return r;
|
|
204
|
-
},
|
|
204
|
+
}, Ue = (e, t, n, r, i) => {
|
|
205
205
|
let a = {};
|
|
206
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
206
|
+
for (let [o, s] of Object.entries(e)) a[o] = Re(n[`${t}.${o}`] ?? "keep", {
|
|
207
207
|
value: s,
|
|
208
208
|
table: t,
|
|
209
209
|
column: o,
|
|
@@ -211,7 +211,7 @@ var we = (e) => {
|
|
|
211
211
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
212
212
|
});
|
|
213
213
|
return a;
|
|
214
|
-
},
|
|
214
|
+
}, We = /* @__PURE__ */ new Map([
|
|
215
215
|
["_voltro_migration_plans", "the schema fingerprint THIS deployment applied — a foreign row makes the next boot refuse"],
|
|
216
216
|
["_voltro_migrations", "which file-based migrations ran HERE — a foreign row marks unrun migrations as done"],
|
|
217
217
|
["_voltro_seeds", "which seeds ran HERE — a foreign row suppresses seeding the target still needs"],
|
|
@@ -244,7 +244,7 @@ var we = (e) => {
|
|
|
244
244
|
["_voltro_notification_held", "notifications waiting to be SENT from here — a foreign row makes the target deliver something queued elsewhere"],
|
|
245
245
|
["_voltro_webhook_event_stats", "per-deployment delivery counters a backoff decision reads — foreign counts steer the target's retries"],
|
|
246
246
|
["_voltro_webhook_rate_windows", "the live rate-limit windows this deployment is inside — a foreign window throttles or releases wrongly"]
|
|
247
|
-
]),
|
|
247
|
+
]), Ge = /* @__PURE__ */ new Map([
|
|
248
248
|
["_voltro_agent_threads", "conversations the app's users had — app data"],
|
|
249
249
|
["_voltro_agent_messages", "the messages in those conversations — app data"],
|
|
250
250
|
["_voltro_prompts", "prompt definitions the app authored — app data"],
|
|
@@ -279,21 +279,21 @@ var we = (e) => {
|
|
|
279
279
|
["_voltro_comment_threads", "the threads those comments hang on — app data"],
|
|
280
280
|
["_voltro_comment_reactions", "the reactions the app's users left — app data"],
|
|
281
281
|
["_voltro_comment_reads", "which comments each user has read — app data, and losing it re-marks everything unread"]
|
|
282
|
-
]), V = (e) =>
|
|
282
|
+
]), V = (e) => We.has(e), Ke = (e) => We.get(e), qe = (e, t) => {
|
|
283
283
|
if (e.length === 0) return;
|
|
284
|
-
let n = e.map((e) => ` ${e} — ${
|
|
284
|
+
let n = e.map((e) => ` ${e} — ${Ke(e) ?? "describes this deployment"}`);
|
|
285
285
|
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
286
286
|
}, H = class extends c.TaggedError()("BundleError", {
|
|
287
287
|
reason: c.String,
|
|
288
288
|
path: c.optional(c.String)
|
|
289
|
-
}) {},
|
|
289
|
+
}) {}, Je = class extends c.TaggedError()("CodecError", {
|
|
290
290
|
table: c.String,
|
|
291
291
|
reason: c.String
|
|
292
|
-
}) {},
|
|
292
|
+
}) {}, Ye = class extends c.TaggedError()("IntegrityError", {
|
|
293
293
|
what: c.String,
|
|
294
294
|
expected: c.String,
|
|
295
295
|
actual: c.String
|
|
296
|
-
}) {},
|
|
296
|
+
}) {}, Xe = class extends t.TaggedError("CompressionError") {}, U = class extends c.TaggedError()("ScopeError", { reason: c.String }) {}, Ze = class extends c.TaggedError()("RowsRefusedError", {
|
|
297
297
|
rows: c.Array(c.Struct({
|
|
298
298
|
table: c.String,
|
|
299
299
|
id: c.String,
|
|
@@ -307,7 +307,7 @@ var we = (e) => {
|
|
|
307
307
|
refused: c.Number,
|
|
308
308
|
primary: c.Number
|
|
309
309
|
}))
|
|
310
|
-
}) {},
|
|
310
|
+
}) {}, Qe = class extends c.TaggedError()("SchemaDriftError", {
|
|
311
311
|
bundleFingerprint: c.String,
|
|
312
312
|
targetFingerprint: c.String,
|
|
313
313
|
diff: c.Array(c.String)
|
|
@@ -316,20 +316,20 @@ var we = (e) => {
|
|
|
316
316
|
let e = this.stderr?.trim(), t = this.hint?.trim();
|
|
317
317
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}${t ? `\n\nhint: ${t}` : ""}`;
|
|
318
318
|
}
|
|
319
|
-
},
|
|
319
|
+
}, $e = class extends c.TaggedError()("MaskingError", {
|
|
320
320
|
unclassified: c.Array(c.String),
|
|
321
321
|
invalidActions: c.optional(c.Array(c.String))
|
|
322
|
-
}) {},
|
|
322
|
+
}) {}, et = class extends c.TaggedError()("ImportModeError", {
|
|
323
323
|
mode: c.String,
|
|
324
324
|
reason: c.String
|
|
325
|
-
}) {},
|
|
325
|
+
}) {}, tt = class extends c.TaggedError()("CrossDialectError", {
|
|
326
326
|
sourceDialect: c.String,
|
|
327
327
|
targetDialect: c.String,
|
|
328
328
|
blocking: c.Array(c.String)
|
|
329
|
-
}) {}, G = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id",
|
|
329
|
+
}) {}, G = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", nt = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
330
330
|
col: e.name,
|
|
331
331
|
refTable: e.references.table
|
|
332
|
-
})),
|
|
332
|
+
})), rt = (e, t, r) => n.gen(function* () {
|
|
333
333
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), u = (e) => {
|
|
334
334
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
335
335
|
return s.set(e, t), t;
|
|
@@ -347,7 +347,7 @@ var we = (e) => {
|
|
|
347
347
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
348
348
|
};
|
|
349
349
|
}, p = (r, i) => {
|
|
350
|
-
let a = f(r, G(t, r),
|
|
350
|
+
let a = f(r, G(t, r), nt(t, r));
|
|
351
351
|
return l.runForEach(C(e, {
|
|
352
352
|
table: r,
|
|
353
353
|
pk: G(t, r),
|
|
@@ -359,7 +359,7 @@ var we = (e) => {
|
|
|
359
359
|
c.clear();
|
|
360
360
|
for (let [n, r] of e) {
|
|
361
361
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
362
|
-
e.length !== 0 && (yield* p(n,
|
|
362
|
+
e.length !== 0 && (yield* p(n, b(G(t, n), e)));
|
|
363
363
|
}
|
|
364
364
|
}
|
|
365
365
|
});
|
|
@@ -374,21 +374,21 @@ var we = (e) => {
|
|
|
374
374
|
refTable: e.references?.table
|
|
375
375
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
376
376
|
if (a.length === 0) continue;
|
|
377
|
-
let o = s.get(t)?.size ?? 0, c = a.map((e) =>
|
|
378
|
-
yield* p(t, c.length === 1 ? c[0] :
|
|
377
|
+
let o = s.get(t)?.size ?? 0, c = a.map((e) => b(e.col, [...s.get(e.refTable)]));
|
|
378
|
+
yield* p(t, c.length === 1 ? c[0] : S(...c)), (s.get(t)?.size ?? 0) > o && !e.has(t) && r.push(t), (s.get(t)?.size ?? 0) > o && (n = !0);
|
|
379
379
|
}
|
|
380
380
|
for (let t of r) e.add(t);
|
|
381
381
|
if (yield* m, !n) break;
|
|
382
382
|
}
|
|
383
383
|
}
|
|
384
384
|
return o;
|
|
385
|
-
}),
|
|
385
|
+
}), it = (e, t) => ({
|
|
386
386
|
tables: [...e.keys()],
|
|
387
387
|
predicateFor: (n) => {
|
|
388
388
|
let r = e.get(n);
|
|
389
|
-
return r ?
|
|
389
|
+
return r ? b(G(t, n), [...r]) : void 0;
|
|
390
390
|
}
|
|
391
|
-
}),
|
|
391
|
+
}), at = "tenantId", ot = (e, t, r) => n.gen(function* () {
|
|
392
392
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
393
393
|
if (e.kind === "all") {
|
|
394
394
|
let t = new Set(e.exclude ?? []);
|
|
@@ -408,29 +408,29 @@ var we = (e) => {
|
|
|
408
408
|
}
|
|
409
409
|
let o = e.tenantId;
|
|
410
410
|
if (o === void 0 || o === "") return yield* n.fail(new U({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
411
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
412
|
-
if (s.length === 0) return yield* n.fail(new U({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${
|
|
413
|
-
let c = yield*
|
|
414
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
411
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === at) ?? !1);
|
|
412
|
+
if (s.length === 0) return yield* n.fail(new U({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${at}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
413
|
+
let c = yield* rt(r.store, t, {
|
|
414
|
+
seeds: Object.fromEntries(s.map((e) => [e, _(at, o)])),
|
|
415
415
|
pinned: s,
|
|
416
416
|
children: { roots: s }
|
|
417
417
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
418
418
|
return {
|
|
419
419
|
tables: [...s, ...l],
|
|
420
|
-
predicateFor: (e) => u.has(e) ?
|
|
420
|
+
predicateFor: (e) => u.has(e) ? _(at, o) : c.has(e) ? b(G(t, e), [...c.get(e)]) : void 0
|
|
421
421
|
};
|
|
422
|
-
}),
|
|
422
|
+
}), st = (e, t) => ({
|
|
423
423
|
list: t,
|
|
424
424
|
open: (t) => A(e, t)
|
|
425
|
-
}),
|
|
426
|
-
write: (t, n, r) =>
|
|
425
|
+
}), ct = (e) => ({
|
|
426
|
+
write: (t, n, r) => ge(e, t, n, r),
|
|
427
427
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
428
|
-
}),
|
|
428
|
+
}), lt = (e, t) => n.tryPromise({
|
|
429
429
|
try: async () => {
|
|
430
430
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
431
431
|
u.update(e), d += e.length, n(null, e);
|
|
432
432
|
} });
|
|
433
|
-
await o(await
|
|
433
|
+
await o(await j(e.stream), f, n(l));
|
|
434
434
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
435
435
|
try {
|
|
436
436
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -447,26 +447,26 @@ var we = (e) => {
|
|
|
447
447
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
448
448
|
path: t
|
|
449
449
|
})
|
|
450
|
-
}),
|
|
451
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u =
|
|
450
|
+
}), ut = (e, t, r) => n.gen(function* () {
|
|
451
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u = he("bundle", () => a(s)).pipe(l.tap((e) => n.sync(() => c.update(e))));
|
|
452
452
|
yield* e.write(r.key, u, {
|
|
453
453
|
contentType: r.contentType,
|
|
454
454
|
totalSize: r.size
|
|
455
455
|
});
|
|
456
456
|
let d = c.digest("hex");
|
|
457
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
457
|
+
if (d !== r.sha256) return yield* n.fail(new Ye({
|
|
458
458
|
what: `asset ${r.key}`,
|
|
459
459
|
expected: r.sha256,
|
|
460
460
|
actual: d
|
|
461
461
|
}));
|
|
462
|
-
}),
|
|
462
|
+
}), dt = {
|
|
463
463
|
done: () => !1,
|
|
464
464
|
mark: () => n.void
|
|
465
|
-
},
|
|
466
|
-
let t = e.progress ??
|
|
467
|
-
return yield* l.runForEach(e.source.list(), (l) => (a++, t.done(l.key) || c.has(l.key) ? n.void : (c.add(l.key), e.source.open(l.key).pipe(n.map((e) => e), n.catchIf(
|
|
465
|
+
}, ft = (e) => typeof e == "object" && !!e && e.status === 404, pt = (e) => n.gen(function* () {
|
|
466
|
+
let t = e.progress ?? dt, r = 0, i = 0, a = 0, o = 0, s = /* @__PURE__ */ new Set(), c = /* @__PURE__ */ new Set(), u = [];
|
|
467
|
+
return yield* l.runForEach(e.source.list(), (l) => (a++, t.done(l.key) || c.has(l.key) ? n.void : (c.add(l.key), e.source.open(l.key).pipe(n.map((e) => e), n.catchIf(ft, () => n.sync(() => {
|
|
468
468
|
u.push(l.key);
|
|
469
|
-
})), n.flatMap((a) => a === void 0 ? n.void :
|
|
469
|
+
})), n.flatMap((a) => a === void 0 ? n.void : lt(a, e.assetsDir).pipe(n.flatMap((e) => {
|
|
470
470
|
let n = {
|
|
471
471
|
key: l.key,
|
|
472
472
|
sha256: e.sha256,
|
|
@@ -482,18 +482,18 @@ var we = (e) => {
|
|
|
482
482
|
objectBytes: o,
|
|
483
483
|
missing: u
|
|
484
484
|
};
|
|
485
|
-
}),
|
|
485
|
+
}), mt = (e) => he("bundle", () => de(e)).pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.map((e) => JSON.parse(e)), l.mapError((t) => t instanceof H ? t : new H({
|
|
486
486
|
reason: `asset index unreadable: ${String(t)}`,
|
|
487
487
|
path: e
|
|
488
|
-
}))),
|
|
489
|
-
let t = e.progress ??
|
|
490
|
-
return yield* l.runForEach(
|
|
488
|
+
}))), ht = (e) => n.gen(function* () {
|
|
489
|
+
let t = e.progress ?? dt, r = 0, i = /* @__PURE__ */ new Set(), a = /* @__PURE__ */ new Set();
|
|
490
|
+
return yield* l.runForEach(mt(e.indexPath), (o) => t.done(o.key) || a.has(o.key) ? n.void : (a.add(o.key), ut(e.sink, e.assetsDir, o)).pipe(n.andThen(n.sync(() => {
|
|
491
491
|
r++, i.add(o.sha256);
|
|
492
492
|
})), n.andThen(t.mark(o.key, o)))), {
|
|
493
493
|
count: r,
|
|
494
494
|
objects: i.size
|
|
495
495
|
};
|
|
496
|
-
}),
|
|
496
|
+
}), gt = () => typeof M.createZstdCompress == "function", _t = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : gt() ? "zstd" : "gzip", vt = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", yt = (e) => e === "zstd" ? M.createZstdCompress() : e === "gzip" ? M.createGzip() : new N(), bt = (e) => e === "zstd" ? M.createZstdDecompress() : e === "gzip" ? M.createGunzip() : new N(), xt = 1, St = c.Struct({
|
|
497
497
|
name: c.String,
|
|
498
498
|
file: c.String,
|
|
499
499
|
rowCount: c.Number,
|
|
@@ -503,12 +503,12 @@ var we = (e) => {
|
|
|
503
503
|
value: c.String
|
|
504
504
|
}),
|
|
505
505
|
primaryKey: c.String
|
|
506
|
-
}),
|
|
506
|
+
}), Ct = c.Struct({
|
|
507
507
|
kind: c.Literal("all", "tenant", "tables"),
|
|
508
508
|
tenantId: c.optional(c.String),
|
|
509
509
|
tables: c.optional(c.Array(c.String)),
|
|
510
510
|
exclude: c.optional(c.Array(c.String))
|
|
511
|
-
}),
|
|
511
|
+
}), wt = c.Struct({
|
|
512
512
|
count: c.Number,
|
|
513
513
|
totalBytes: c.Number,
|
|
514
514
|
index: c.optional(c.String)
|
|
@@ -519,23 +519,23 @@ c.Struct({
|
|
|
519
519
|
size: c.Number,
|
|
520
520
|
contentType: c.String
|
|
521
521
|
});
|
|
522
|
-
var
|
|
522
|
+
var Tt = c.Struct({
|
|
523
523
|
formatVersion: c.Number,
|
|
524
524
|
createdAt: c.String,
|
|
525
525
|
source: c.Struct({
|
|
526
526
|
dialect: c.String,
|
|
527
527
|
schemaFingerprint: c.String
|
|
528
528
|
}),
|
|
529
|
-
scope:
|
|
529
|
+
scope: Ct,
|
|
530
530
|
consistency: c.Literal("live", "snapshot"),
|
|
531
531
|
compression: c.Literal("zstd", "gzip", "none"),
|
|
532
|
-
tables: c.Array(
|
|
533
|
-
assets:
|
|
532
|
+
tables: c.Array(St),
|
|
533
|
+
assets: wt,
|
|
534
534
|
masking: c.optional(c.Struct({
|
|
535
535
|
policyId: c.optional(c.String),
|
|
536
536
|
transformed: c.Array(c.String)
|
|
537
537
|
}))
|
|
538
|
-
}),
|
|
538
|
+
}), Et = c.encodeSync(c.parseJson(Tt)), Dt = c.decodeUnknownSync(c.parseJson(Tt)), K = "manifest.json", Ot = "README.md", kt = "data", At = "assets", jt = "assets/index.ndjson", Mt = ".ledger.json", Nt = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", Pt = (e, t) => {
|
|
539
539
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
540
540
|
return `# Voltro data export
|
|
541
541
|
|
|
@@ -549,7 +549,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
549
549
|
| **Format version** | ${e.formatVersion} |
|
|
550
550
|
| **Consistency** | ${e.consistency} |
|
|
551
551
|
| **Compression** | ${e.compression} |
|
|
552
|
-
| **Scope** | ${
|
|
552
|
+
| **Scope** | ${Nt(e.scope)} |
|
|
553
553
|
| **Tables** | ${e.tables.length} |
|
|
554
554
|
| **Rows (total)** | ${n} |
|
|
555
555
|
| **Assets** | ${a} |
|
|
@@ -574,26 +574,26 @@ voltro data import <this-directory>
|
|
|
574
574
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
575
575
|
\`\`\`
|
|
576
576
|
`;
|
|
577
|
-
},
|
|
577
|
+
}, Ft = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), It = a.exponential({
|
|
578
578
|
start: .01,
|
|
579
579
|
factor: 2,
|
|
580
580
|
count: 16
|
|
581
|
-
}),
|
|
581
|
+
}), Lt = i.histogram("voltro_data_transfer_table_seconds", It, "Per-table export/import duration in seconds, tagged by phase."), q = (e, t) => {
|
|
582
582
|
if (e) try {
|
|
583
583
|
e(t);
|
|
584
584
|
} catch {}
|
|
585
|
-
},
|
|
585
|
+
}, Rt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(Lt.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), Ft.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
586
586
|
table: t,
|
|
587
587
|
index: r,
|
|
588
588
|
tableCount: a,
|
|
589
589
|
phase: e
|
|
590
|
-
} })),
|
|
590
|
+
} })), zt = () => ({
|
|
591
591
|
tables: {},
|
|
592
592
|
assets: {}
|
|
593
|
-
}),
|
|
593
|
+
}), Bt = (e) => n.tryPromise(async () => {
|
|
594
594
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
595
595
|
return JSON.parse(t);
|
|
596
|
-
}).pipe(n.orElseSucceed(
|
|
596
|
+
}).pipe(n.orElseSucceed(zt)), J = (e, t) => n.tryPromise({
|
|
597
597
|
try: async () => {
|
|
598
598
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
599
599
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
@@ -602,11 +602,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
602
602
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
603
603
|
path: e
|
|
604
604
|
})
|
|
605
|
-
}),
|
|
605
|
+
}), Vt = (e, t) => t in e.tables, Ht = (e, t) => t in e.assets, Ut = /* @__PURE__ */ new Set([
|
|
606
606
|
"json",
|
|
607
607
|
"array",
|
|
608
608
|
"vector"
|
|
609
|
-
]),
|
|
609
|
+
]), Wt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", Gt = (e) => Buffer.from(e).toString("base64"), Kt = (e) => new Uint8Array(Buffer.from(e, "base64")), qt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), Jt = (e, t) => e == null ? null : t !== void 0 && Ut.has(t) ? e : t === "bytes" ? Gt(e) : t === "timestamp" || t === "date" ? qt(e) : typeof e == "bigint" ? {
|
|
610
610
|
$v: "bigint",
|
|
611
611
|
d: e.toString()
|
|
612
612
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -617,30 +617,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
617
617
|
d: e.toISOString()
|
|
618
618
|
} : e instanceof Uint8Array ? {
|
|
619
619
|
$v: "bytes",
|
|
620
|
-
d:
|
|
621
|
-
} : e,
|
|
620
|
+
d: Gt(e)
|
|
621
|
+
} : e, Yt = (e) => {
|
|
622
622
|
switch (e.$v) {
|
|
623
623
|
case "bigint": return BigInt(e.d);
|
|
624
624
|
case "number": return Number(e.d);
|
|
625
625
|
case "date": return new Date(e.d);
|
|
626
|
-
case "bytes": return
|
|
626
|
+
case "bytes": return Kt(e.d);
|
|
627
627
|
default: return e;
|
|
628
628
|
}
|
|
629
|
-
},
|
|
629
|
+
}, Xt = (e, t) => e == null ? null : t !== void 0 && Ut.has(t) ? e : Wt(e) ? Yt(e) : t === "bytes" ? Kt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Zt = (e, t) => {
|
|
630
630
|
let n = {};
|
|
631
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
631
|
+
for (let [r, i] of Object.entries(e)) n[r] = Jt(i, t[r]);
|
|
632
632
|
return n;
|
|
633
|
-
},
|
|
633
|
+
}, Qt = (e, t) => {
|
|
634
634
|
let n = {};
|
|
635
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
635
|
+
for (let [r, i] of Object.entries(e)) n[r] = Xt(i, t[r]);
|
|
636
636
|
return n;
|
|
637
|
-
},
|
|
638
|
-
try: () =>
|
|
639
|
-
catch: (e) => new
|
|
637
|
+
}, $t = new TextEncoder(), en = (e, t) => e.pipe(l.map((e) => $t.encode(`${JSON.stringify(Zt(e, t))}\n`))), tn = (e, t, r) => e.pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.mapEffect((e) => n.try({
|
|
638
|
+
try: () => Qt(JSON.parse(e), r),
|
|
639
|
+
catch: (e) => new Je({
|
|
640
640
|
table: t,
|
|
641
641
|
reason: String(e?.message ?? e)
|
|
642
642
|
})
|
|
643
|
-
}))),
|
|
643
|
+
}))), nn = (e, t) => {
|
|
644
644
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
645
645
|
for (let e of t.tables) {
|
|
646
646
|
if (!n.has(e.name)) continue;
|
|
@@ -666,33 +666,33 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
666
666
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
667
667
|
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(Y(t.cause)) : n.push(r), n.length === 0) {
|
|
668
668
|
let t = String(e ?? "").trim();
|
|
669
|
-
return t === "" || t === "[object Object]" ?
|
|
669
|
+
return t === "" || t === "[object Object]" ? rn(e) : t;
|
|
670
670
|
}
|
|
671
671
|
return n.join(": ");
|
|
672
|
-
},
|
|
672
|
+
}, rn = (e) => {
|
|
673
673
|
try {
|
|
674
674
|
let t = JSON.stringify(e);
|
|
675
675
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
676
676
|
} catch {
|
|
677
677
|
return "unknown error (not serialisable)";
|
|
678
678
|
}
|
|
679
|
-
},
|
|
679
|
+
}, an = (e, t) => {
|
|
680
680
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
681
681
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
682
682
|
return r;
|
|
683
|
-
},
|
|
683
|
+
}, on = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), sn = (e, t) => {
|
|
684
684
|
if (t.length === 0) return e;
|
|
685
685
|
let n = { ...e };
|
|
686
686
|
for (let e of t) delete n[e];
|
|
687
687
|
return n;
|
|
688
|
-
},
|
|
688
|
+
}, cn = (e, t) => {
|
|
689
689
|
let n = e.tables.find((e) => e.name === t);
|
|
690
690
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
691
691
|
let r = n.columns.find((e) => e.type === "id");
|
|
692
692
|
if (r !== void 0) return { column: r.name };
|
|
693
693
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
694
694
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
695
|
-
},
|
|
695
|
+
}, ln = (e, t, r) => n.tryPromise({
|
|
696
696
|
try: async () => {
|
|
697
697
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
698
698
|
d.update(e);
|
|
@@ -701,7 +701,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
701
701
|
n(null, e);
|
|
702
702
|
} }), m = c.fromWeb(l.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
703
703
|
try {
|
|
704
|
-
await s(m, p,
|
|
704
|
+
await s(m, p, yt(r), n(h)), await i.rename(h, t);
|
|
705
705
|
} catch (e) {
|
|
706
706
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
707
707
|
}
|
|
@@ -714,7 +714,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
714
714
|
reason: `write table failed: ${Y(e)}`,
|
|
715
715
|
path: t
|
|
716
716
|
})
|
|
717
|
-
}),
|
|
717
|
+
}), un = (e, t) => n.tryPromise({
|
|
718
718
|
try: async () => {
|
|
719
719
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
720
720
|
},
|
|
@@ -722,13 +722,13 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
722
722
|
reason: `append failed: ${Y(t)}`,
|
|
723
723
|
path: e
|
|
724
724
|
})
|
|
725
|
-
}),
|
|
725
|
+
}), dn = (e, t) => n.gen(function* () {
|
|
726
726
|
let r = [], i = t.ordered.length;
|
|
727
727
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
728
|
-
let o = t.ordered[a], s =
|
|
728
|
+
let o = t.ordered[a], s = an(t.snapshot, o), c = cn(t.snapshot, o);
|
|
729
729
|
if ("refusal" in c) return yield* n.fail(new U({ reason: `export: ${c.refusal}` }));
|
|
730
|
-
let u = c.column, d = `${
|
|
731
|
-
if (
|
|
730
|
+
let u = c.column, d = `${kt}/${o}.ndjson${vt(t.compression)}`;
|
|
731
|
+
if (Vt(t.ledger, o)) {
|
|
732
732
|
let e = t.ledger.tables[o];
|
|
733
733
|
q(t.onProgress, {
|
|
734
734
|
phase: "export",
|
|
@@ -762,7 +762,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
762
762
|
chunkSize: t.chunkSize,
|
|
763
763
|
retry: t.retry,
|
|
764
764
|
...f === void 0 ? {} : { where: f }
|
|
765
|
-
}), m =
|
|
765
|
+
}), m = on(t.snapshot, o), h = m.length === 0 ? p : p.pipe(l.map((e) => sn(e, m))), g = t.mask ? h.pipe(l.map((e) => Ue(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* Rt("export", o, a, i, (e) => e.rowCount, ln(en(g, s), t.bundleFile(d), t.compression));
|
|
766
766
|
t.ledger.tables[o] = {
|
|
767
767
|
rowCount: _,
|
|
768
768
|
checksum: v
|
|
@@ -784,43 +784,43 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
784
784
|
});
|
|
785
785
|
}
|
|
786
786
|
return r;
|
|
787
|
-
}),
|
|
787
|
+
}), fn = (e) => ({
|
|
788
788
|
_tag: "RawSqlFragment",
|
|
789
789
|
strings: [e],
|
|
790
790
|
values: []
|
|
791
|
-
}),
|
|
791
|
+
}), pn = (t, i, a) => n.tryPromise({
|
|
792
792
|
try: () => t.transactional(async (t) => {
|
|
793
|
-
if (t.raw) for (let e of
|
|
793
|
+
if (t.raw) for (let e of Te(i)) await t.raw(fn(e));
|
|
794
794
|
let o = await n.runPromiseExit(a(t));
|
|
795
795
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
796
796
|
return o.value;
|
|
797
797
|
}),
|
|
798
798
|
catch: (e) => e instanceof H ? e : new H({ reason: `snapshot export failed: ${Y(e)}` })
|
|
799
|
-
}),
|
|
800
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
799
|
+
}), mn = (e) => n.gen(function* () {
|
|
800
|
+
let t = e.scope ?? { kind: "all" }, r = _t(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, o = s.exponential("50 millis").pipe(s.jittered, s.intersect(s.recurs(a))), c = yield* n.promise(() => import("node:path")), l = (t) => c.join(e.outDir, t);
|
|
801
801
|
yield* n.tryPromise({
|
|
802
802
|
try: async () => {
|
|
803
803
|
let t = await import("node:fs/promises");
|
|
804
|
-
await t.mkdir(l(
|
|
804
|
+
await t.mkdir(l(kt), { recursive: !0 }), e.assets && await t.mkdir(l(At), { recursive: !0 });
|
|
805
805
|
},
|
|
806
806
|
catch: (t) => new H({
|
|
807
807
|
reason: `mkdir failed: ${Y(t)}`,
|
|
808
808
|
path: e.outDir
|
|
809
809
|
})
|
|
810
810
|
});
|
|
811
|
-
let u = l(
|
|
811
|
+
let u = l(Mt), d = yield* Bt(u), f = e.subset ? it(yield* rt(e.store, e.snapshot, e.subset), e.snapshot) : yield* ot(t, e.snapshot, {
|
|
812
812
|
store: e.store,
|
|
813
813
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
814
|
-
}), p =
|
|
814
|
+
}), p = nn(f.tables, e.snapshot), m = e.consistency ?? "live", h, g;
|
|
815
815
|
if (e.masking) {
|
|
816
|
-
let t = e.classification ??
|
|
816
|
+
let t = e.classification ?? De(e.snapshot), r = Be(p.map((t) => ({
|
|
817
817
|
name: t,
|
|
818
818
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
819
819
|
name: e.name,
|
|
820
820
|
type: e.type
|
|
821
821
|
}))
|
|
822
822
|
})), t, e.masking);
|
|
823
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
823
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new $e({
|
|
824
824
|
unclassified: r.unclassified,
|
|
825
825
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
826
826
|
}));
|
|
@@ -846,14 +846,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
846
846
|
bundleFile: l,
|
|
847
847
|
...g ? { mask: g } : {},
|
|
848
848
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
849
|
-
}, v = m === "snapshot" ? yield*
|
|
849
|
+
}, v = m === "snapshot" ? yield* pn(e.store, e.dialect, (e) => dn(e, _)) : yield* dn(e.store, _), y = 0, b = 0;
|
|
850
850
|
if (e.assets) {
|
|
851
|
-
let t = l(
|
|
851
|
+
let t = l(jt), r = yield* pt({
|
|
852
852
|
source: e.assets,
|
|
853
|
-
assetsDir: l(
|
|
853
|
+
assetsDir: l(At),
|
|
854
854
|
progress: {
|
|
855
855
|
done: (e) => e in d.assets,
|
|
856
|
-
mark: (e, r) => (d.assets[e] = !0,
|
|
856
|
+
mark: (e, r) => (d.assets[e] = !0, un(t, JSON.stringify(r)).pipe(n.andThen(J(u, d))))
|
|
857
857
|
}
|
|
858
858
|
});
|
|
859
859
|
y = r.count, b = r.totalBytes, r.missing.length > 0 && (yield* n.logWarning(`${r.missing.length} of ${r.missing.length + r.count} blob reference(s) point at objects the storage provider does not have; they are NOT in this bundle and no import can bring them back. e.g. ${r.missing.slice(0, 3).map((e) => `'${e}'`).join(", ")}` + (r.missing.length > 3 ? `, and ${r.missing.length - 3} more` : "")));
|
|
@@ -863,7 +863,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
863
863
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
864
864
|
source: {
|
|
865
865
|
dialect: e.dialect,
|
|
866
|
-
schemaFingerprint:
|
|
866
|
+
schemaFingerprint: w(e.snapshot)
|
|
867
867
|
},
|
|
868
868
|
scope: t,
|
|
869
869
|
consistency: m,
|
|
@@ -872,7 +872,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
872
872
|
assets: y > 0 ? {
|
|
873
873
|
count: y,
|
|
874
874
|
totalBytes: b,
|
|
875
|
-
index:
|
|
875
|
+
index: jt
|
|
876
876
|
} : {
|
|
877
877
|
count: 0,
|
|
878
878
|
totalBytes: 0
|
|
@@ -881,7 +881,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
881
881
|
};
|
|
882
882
|
return yield* n.tryPromise({
|
|
883
883
|
try: async () => {
|
|
884
|
-
await (await import("node:fs/promises")).writeFile(l(K),
|
|
884
|
+
await (await import("node:fs/promises")).writeFile(l(K), Et(x));
|
|
885
885
|
},
|
|
886
886
|
catch: (e) => new H({
|
|
887
887
|
reason: `manifest write failed: ${Y(e)}`,
|
|
@@ -890,14 +890,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
890
890
|
}), yield* n.tryPromise({
|
|
891
891
|
try: async () => {
|
|
892
892
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
893
|
-
await e.writeFile(l(
|
|
893
|
+
await e.writeFile(l(Ot), Pt(x, { hostname: t.hostname() }));
|
|
894
894
|
},
|
|
895
895
|
catch: (e) => new H({
|
|
896
896
|
reason: `readme write failed: ${Y(e)}`,
|
|
897
|
-
path:
|
|
897
|
+
path: Ot
|
|
898
898
|
})
|
|
899
899
|
}), x;
|
|
900
|
-
}),
|
|
900
|
+
}), hn = (e, t) => {
|
|
901
901
|
let n = new Set(t), r = [];
|
|
902
902
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
903
903
|
let i = e.references?.table;
|
|
@@ -908,7 +908,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
908
908
|
});
|
|
909
909
|
}
|
|
910
910
|
return r;
|
|
911
|
-
},
|
|
911
|
+
}, gn = (e) => {
|
|
912
912
|
let t = /* @__PURE__ */ new Map();
|
|
913
913
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
914
914
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -916,7 +916,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
916
916
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
917
917
|
});
|
|
918
918
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
919
|
-
},
|
|
919
|
+
}, _n = (e) => {
|
|
920
920
|
switch (e) {
|
|
921
921
|
case "id":
|
|
922
922
|
case "text":
|
|
@@ -935,12 +935,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
935
935
|
case "raw": return "raw";
|
|
936
936
|
default: return "string";
|
|
937
937
|
}
|
|
938
|
-
},
|
|
938
|
+
}, vn = {
|
|
939
939
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
940
940
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
941
941
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
942
|
-
},
|
|
943
|
-
let i =
|
|
942
|
+
}, yn = (e, t, n, r) => {
|
|
943
|
+
let i = _n(n);
|
|
944
944
|
if (i === "raw") return {
|
|
945
945
|
table: e,
|
|
946
946
|
column: t,
|
|
@@ -948,7 +948,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
948
948
|
severity: "block",
|
|
949
949
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
950
950
|
};
|
|
951
|
-
let a =
|
|
951
|
+
let a = vn[i];
|
|
952
952
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
953
953
|
table: e,
|
|
954
954
|
column: t,
|
|
@@ -962,15 +962,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
962
962
|
severity: "block",
|
|
963
963
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
964
964
|
};
|
|
965
|
-
},
|
|
965
|
+
}, bn = (e, t, n) => {
|
|
966
966
|
if (t === n) return [];
|
|
967
967
|
let r = [];
|
|
968
968
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
969
|
-
let a =
|
|
969
|
+
let a = yn(t.name, e, i, n);
|
|
970
970
|
a && r.push(a);
|
|
971
971
|
}
|
|
972
972
|
return r;
|
|
973
|
-
},
|
|
973
|
+
}, xn = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, Sn = (e) => {
|
|
974
974
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
975
975
|
switch (e.kind) {
|
|
976
976
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -979,35 +979,35 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
979
979
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
980
980
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
981
981
|
}
|
|
982
|
-
},
|
|
982
|
+
}, Cn = (e) => {
|
|
983
983
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
984
984
|
return t.length > 0 ? t.join("/") : void 0;
|
|
985
|
-
},
|
|
985
|
+
}, wn = "Failed to execute statement", Tn = (e) => {
|
|
986
986
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
987
987
|
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
988
|
-
return t === `SqlError: ${
|
|
989
|
-
},
|
|
988
|
+
return t === `SqlError: ${wn}` ? wn : t;
|
|
989
|
+
}, En = (e) => {
|
|
990
990
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
991
991
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
992
992
|
unclassified: t,
|
|
993
993
|
raw: n
|
|
994
|
-
}), i =
|
|
995
|
-
if (a !== void 0) return r(`${
|
|
996
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u =
|
|
997
|
-
return r(d ===
|
|
998
|
-
},
|
|
999
|
-
let t =
|
|
994
|
+
}), i = v(e), a = m(i), o = Cn(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
995
|
+
if (a !== void 0) return r(`${Sn(a)}${s}`, !1);
|
|
996
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = Tn(typeof l == "string" && l.length > 0 ? l : String(e)), d = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, f = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, p = f !== void 0 && /[←‖]/.test(f) ? ` (via ${f})` : "";
|
|
997
|
+
return r(d === wn || d.length === 0 ? `the database refused the write and the driver gave no detail${s}${p}` : `${d}${s}`, d === wn || d.length === 0);
|
|
998
|
+
}, Dn = (e) => En(e).reason, On = (e) => {
|
|
999
|
+
let t = v(e), n = m(t);
|
|
1000
1000
|
if (n?.kind === "foreignKeyInUse") {
|
|
1001
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
1001
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = Cn(t);
|
|
1002
1002
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
1003
1003
|
}
|
|
1004
|
-
return
|
|
1005
|
-
},
|
|
1004
|
+
return Dn(e);
|
|
1005
|
+
}, kn = f, X = (e) => `${kn}${e}`, An = (e) => typeof e.run == "function", jn = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
1006
1006
|
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
1007
|
-
written before the first delete is what covers that case.`,
|
|
1008
|
-
if (!
|
|
1007
|
+
written before the first delete is what covers that case.`, Mn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, Z = (e, t) => {
|
|
1008
|
+
if (!Mn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
1009
1009
|
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
1010
|
-
},
|
|
1010
|
+
}, Nn = (e, t) => {
|
|
1011
1011
|
let n = Z(e, t), r = Z(X(e), t);
|
|
1012
1012
|
switch (t) {
|
|
1013
1013
|
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
@@ -1016,10 +1016,10 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1016
1016
|
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
1017
1017
|
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
1018
1018
|
}
|
|
1019
|
-
},
|
|
1019
|
+
}, Pn = (e, t) => {
|
|
1020
1020
|
let n = Z(X(e), t);
|
|
1021
1021
|
return t === "mssql" ? `IF OBJECT_ID('${X(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
1022
|
-
},
|
|
1022
|
+
}, Fn = (e, t) => {
|
|
1023
1023
|
let n = t === "mysql" || t === "mariadb", r = [];
|
|
1024
1024
|
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
1025
1025
|
for (let n of [...e].reverse()) r.push(`DELETE FROM ${Z(n.name, t)}`);
|
|
@@ -1029,7 +1029,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1029
1029
|
r.push(`INSERT INTO ${Z(n.name, t)} (${e}) SELECT ${e} FROM ${Z(X(n.name), t)}`);
|
|
1030
1030
|
}
|
|
1031
1031
|
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
1032
|
-
},
|
|
1032
|
+
}, In = (e, t) => {
|
|
1033
1033
|
let n = [], r = () => {
|
|
1034
1034
|
for (let e of n) t.unregisterTable(e);
|
|
1035
1035
|
};
|
|
@@ -1051,7 +1051,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1051
1051
|
staged: n,
|
|
1052
1052
|
undo: r
|
|
1053
1053
|
};
|
|
1054
|
-
},
|
|
1054
|
+
}, Ln = (e, t) => {
|
|
1055
1055
|
let n = new Set(t), r = [];
|
|
1056
1056
|
for (let t of e.tables) {
|
|
1057
1057
|
if (!n.has(t.name)) continue;
|
|
@@ -1068,22 +1068,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1068
1068
|
}
|
|
1069
1069
|
}
|
|
1070
1070
|
return r;
|
|
1071
|
-
},
|
|
1071
|
+
}, Rn = (e, t) => {
|
|
1072
1072
|
let n = Z(X(e.from), t), r = Z(X(e.to), t), i = Z(e.column, t), a = Z(e.toColumn, t);
|
|
1073
1073
|
return `SELECT ${Z(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
1074
|
-
},
|
|
1074
|
+
}, zn = (e) => {
|
|
1075
1075
|
if (e.length === 0) return;
|
|
1076
1076
|
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
1077
1077
|
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
1078
|
-
},
|
|
1079
|
-
t.length !== 0 && await e.run(
|
|
1080
|
-
}, Bn = async (e, t, r) => {
|
|
1081
|
-
let i = Pn(t, r);
|
|
1082
|
-
i.length !== 0 && await e.run(_e.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
1078
|
+
}, Bn = async (e, t) => {
|
|
1079
|
+
t.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
1083
1080
|
}, Vn = async (e, t, r) => {
|
|
1081
|
+
let i = Fn(t, r);
|
|
1082
|
+
i.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
1083
|
+
}, Hn = async (e, t, r) => {
|
|
1084
1084
|
let i = [];
|
|
1085
1085
|
for (let a of t) {
|
|
1086
|
-
let t = await e.run(
|
|
1086
|
+
let t = await e.run(P.SqlClient.pipe(n.flatMap((e) => e.unsafe(Rn(a, r)))));
|
|
1087
1087
|
for (let e of t) i.push({
|
|
1088
1088
|
table: a.from,
|
|
1089
1089
|
primaryKey: String(e.pk ?? ""),
|
|
@@ -1093,14 +1093,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1093
1093
|
});
|
|
1094
1094
|
}
|
|
1095
1095
|
return i;
|
|
1096
|
-
},
|
|
1096
|
+
}, Un = (e, t) => {
|
|
1097
1097
|
let n = new Map(t.map((e, t) => [e, t]));
|
|
1098
1098
|
return e.filter((e) => {
|
|
1099
1099
|
if (e.from === e.to) return !1;
|
|
1100
1100
|
let t = n.get(e.from), r = n.get(e.to);
|
|
1101
1101
|
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1102
1102
|
});
|
|
1103
|
-
},
|
|
1103
|
+
}, Wn = (e) => e === "mysql" || e === "mariadb", Gn = (e) => {
|
|
1104
1104
|
if (e.mode !== "replace") return {
|
|
1105
1105
|
active: !1,
|
|
1106
1106
|
tables: [],
|
|
@@ -1150,7 +1150,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1150
1150
|
tables: [],
|
|
1151
1151
|
blocker: "every table in the bundle is one that describes a deployment, so there is nothing for a staged swap to move."
|
|
1152
1152
|
};
|
|
1153
|
-
let s = o.map((e) => e.name), c =
|
|
1153
|
+
let s = o.map((e) => e.name), c = Wn(t) ? [] : Un(Ln(n, s), s);
|
|
1154
1154
|
return c.length > 0 ? {
|
|
1155
1155
|
active: !1,
|
|
1156
1156
|
tables: [],
|
|
@@ -1159,7 +1159,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1159
1159
|
active: !0,
|
|
1160
1160
|
tables: o
|
|
1161
1161
|
};
|
|
1162
|
-
},
|
|
1162
|
+
}, Kn = (e) => n.gen(function* () {
|
|
1163
1163
|
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1164
1164
|
if (e.snapshot === void 0) return yield* n.fail(new H({
|
|
1165
1165
|
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
@@ -1172,7 +1172,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1172
1172
|
if (t.tables.length === 0) return;
|
|
1173
1173
|
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1174
1174
|
so it survives a run whose process does not.`);
|
|
1175
|
-
let r = yield*
|
|
1175
|
+
let r = yield* mn({
|
|
1176
1176
|
store: e.store,
|
|
1177
1177
|
snapshot: t,
|
|
1178
1178
|
dialect: e.dialect ?? "unknown",
|
|
@@ -1188,18 +1188,18 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1188
1188
|
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1189
1189
|
tables: r.tables.length
|
|
1190
1190
|
};
|
|
1191
|
-
}),
|
|
1191
|
+
}), qn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, Jn = (e, t) => e.tables.find((e) => e.name === t), Yn = (e, t) => new Map((Jn(e, t)?.columns ?? []).map((e) => [e.name, e])), Xn = /__dropped_\d+$/, Zn = (e, t) => {
|
|
1192
1192
|
let n = [];
|
|
1193
1193
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
1194
|
-
if (
|
|
1194
|
+
if (Jn(t, r.name) === void 0) {
|
|
1195
1195
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
1196
1196
|
continue;
|
|
1197
1197
|
}
|
|
1198
|
-
let e =
|
|
1198
|
+
let e = Yn(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
1199
1199
|
for (let [t, a] of i) {
|
|
1200
1200
|
let i = e.get(t);
|
|
1201
1201
|
if (i === void 0) {
|
|
1202
|
-
n.push(
|
|
1202
|
+
n.push(Xn.test(t) ? `${r.name}.${t}: a soft-dropped column, present only in the SOURCE — reclaim it there (voltro db gc-snapshots --before <date>) or re-export without it; the target is not missing anything` : `${r.name}.${t}: in the bundle, MISSING from the target — the value has nowhere to go`);
|
|
1203
1203
|
continue;
|
|
1204
1204
|
}
|
|
1205
1205
|
i.type !== a.type && n.push(`${r.name}.${t}: type '${a.type}' (bundle) vs '${i.type}' (target)`);
|
|
@@ -1207,22 +1207,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1207
1207
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
1208
1208
|
}
|
|
1209
1209
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
1210
|
-
},
|
|
1211
|
-
let r =
|
|
1210
|
+
}, Qn = (e, t, n) => {
|
|
1211
|
+
let r = w(t);
|
|
1212
1212
|
if (r === e || n === void 0) return {
|
|
1213
1213
|
drifted: !1,
|
|
1214
1214
|
bundleFingerprint: e,
|
|
1215
1215
|
targetFingerprint: r,
|
|
1216
1216
|
diff: []
|
|
1217
1217
|
};
|
|
1218
|
-
let i =
|
|
1218
|
+
let i = Zn(n, t);
|
|
1219
1219
|
return {
|
|
1220
1220
|
drifted: i.length > 0,
|
|
1221
1221
|
bundleFingerprint: e,
|
|
1222
1222
|
targetFingerprint: r,
|
|
1223
1223
|
diff: i
|
|
1224
1224
|
};
|
|
1225
|
-
},
|
|
1225
|
+
}, $n = 2e3, er = 2e3, tr = 200, nr = /* @__PURE__ */ new Set([
|
|
1226
1226
|
"BundleError",
|
|
1227
1227
|
"CodecError",
|
|
1228
1228
|
"IntegrityError",
|
|
@@ -1231,29 +1231,29 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1231
1231
|
"RowsRefusedError",
|
|
1232
1232
|
"SchemaDriftError",
|
|
1233
1233
|
"StorageError"
|
|
1234
|
-
]),
|
|
1234
|
+
]), rr = (t) => {
|
|
1235
1235
|
let n = o.isFiberFailure(t) ? [...e.failures(t[o.FiberFailureCauseId]), ...e.defects(t[o.FiberFailureCauseId])] : [t];
|
|
1236
1236
|
for (let e of n) {
|
|
1237
1237
|
let t = e?._tag;
|
|
1238
|
-
if (typeof t == "string" &&
|
|
1238
|
+
if (typeof t == "string" && nr.has(t)) return e;
|
|
1239
1239
|
if (o.isFiberFailure(e)) {
|
|
1240
|
-
let t =
|
|
1240
|
+
let t = rr(e);
|
|
1241
1241
|
if (t) return t;
|
|
1242
1242
|
}
|
|
1243
1243
|
}
|
|
1244
|
-
},
|
|
1244
|
+
}, ir = (t) => [...e.failures(t)][0] ?? Error(e.pretty(t)), ar = (e) => n.tryPromise({
|
|
1245
1245
|
try: async () => {
|
|
1246
1246
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
1247
|
-
return
|
|
1247
|
+
return Dt(await t.readFile(n.join(e, K), "utf8"));
|
|
1248
1248
|
},
|
|
1249
1249
|
catch: (e) => new H({
|
|
1250
1250
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
1251
1251
|
path: K
|
|
1252
1252
|
})
|
|
1253
|
-
}),
|
|
1253
|
+
}), or = (e) => {
|
|
1254
1254
|
let t;
|
|
1255
1255
|
return n.gen(function* () {
|
|
1256
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
1256
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* ar(e.bundleDir);
|
|
1257
1257
|
if (a.formatVersion !== 1) return yield* n.fail(new H({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
1258
1258
|
if (e.tables !== void 0) {
|
|
1259
1259
|
let t = new Set(a.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
@@ -1274,9 +1274,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1274
1274
|
unique: !1,
|
|
1275
1275
|
hasDefault: !1
|
|
1276
1276
|
}))
|
|
1277
|
-
})) }, r =
|
|
1277
|
+
})) }, r = Qn(s.source.schemaFingerprint, e.targetSnapshot, t);
|
|
1278
1278
|
if (r.drifted) {
|
|
1279
|
-
if (!e.force) return yield* n.fail(new
|
|
1279
|
+
if (!e.force) return yield* n.fail(new Qe({
|
|
1280
1280
|
bundleFingerprint: r.bundleFingerprint,
|
|
1281
1281
|
targetFingerprint: r.targetFingerprint,
|
|
1282
1282
|
diff: r.diff
|
|
@@ -1285,28 +1285,28 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1285
1285
|
}
|
|
1286
1286
|
}
|
|
1287
1287
|
if (e.targetDialect && e.targetDialect !== s.source.dialect) {
|
|
1288
|
-
let t =
|
|
1289
|
-
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new
|
|
1288
|
+
let t = bn(s.tables, s.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1289
|
+
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new tt({
|
|
1290
1290
|
sourceDialect: s.source.dialect,
|
|
1291
1291
|
targetDialect: e.targetDialect,
|
|
1292
|
-
blocking: t.map(
|
|
1292
|
+
blocking: t.map(xn)
|
|
1293
1293
|
}));
|
|
1294
1294
|
}
|
|
1295
|
-
let c = e.verify ?? !0,
|
|
1296
|
-
if (
|
|
1297
|
-
mode:
|
|
1295
|
+
let c = e.verify ?? !0, f = e.mode ?? "upsert", m = e.atomic ?? f === "replace", _ = e.onConflict ?? "skip", v = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), b = yield* Bt(v);
|
|
1296
|
+
if (f === "replace" && s.scope.kind !== "all") return yield* n.fail(new et({
|
|
1297
|
+
mode: f,
|
|
1298
1298
|
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope. To leave specific tables out of a FULL export, use \`--exclude a,b\`: it keeps the scope \`all\` and records what it left out.`
|
|
1299
1299
|
}));
|
|
1300
|
-
if (
|
|
1300
|
+
if (f === "replace" && (s.scope.exclude?.length ?? 0) > 0) {
|
|
1301
1301
|
let t = s.scope.exclude;
|
|
1302
1302
|
e.onWarn?.(`this bundle was exported with ${t.length} table(s) excluded: ${t.join(", ")}.\n A replace empties and refills the tables the bundle CARRIES, so those keep whatever rows the
|
|
1303
1303
|
target already holds. That is not "everything the bundle says" — check it is what you meant.`);
|
|
1304
1304
|
}
|
|
1305
|
-
if (
|
|
1305
|
+
if (f === "replace") {
|
|
1306
1306
|
let t = s.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1307
1307
|
try: () => e.store.incomingForeignKeys(t),
|
|
1308
|
-
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${
|
|
1309
|
-
}) : e.targetSnapshot === void 0 ? [] :
|
|
1308
|
+
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${On(e)}` })
|
|
1309
|
+
}) : e.targetSnapshot === void 0 ? [] : hn(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1310
1310
|
for (let t of r) {
|
|
1311
1311
|
if (!a.has(t.from)) {
|
|
1312
1312
|
let r = yield* n.tryPromise({
|
|
@@ -1328,9 +1328,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1328
1328
|
rowCount: r
|
|
1329
1329
|
});
|
|
1330
1330
|
}
|
|
1331
|
-
if (i.length > 0) return yield* n.fail(new
|
|
1332
|
-
mode:
|
|
1333
|
-
reason:
|
|
1331
|
+
if (i.length > 0) return yield* n.fail(new et({
|
|
1332
|
+
mode: f,
|
|
1333
|
+
reason: gn(i)
|
|
1334
1334
|
}));
|
|
1335
1335
|
}
|
|
1336
1336
|
if (e.dryRun === !0) return e.traceId !== void 0 && (yield* n.promise(() => e.store.update(u, e.traceId, {
|
|
@@ -1346,32 +1346,32 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1346
1346
|
rowsSkipped: 0,
|
|
1347
1347
|
fullyResumed: !1
|
|
1348
1348
|
};
|
|
1349
|
-
let
|
|
1349
|
+
let S = /* @__PURE__ */ new Map();
|
|
1350
1350
|
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1351
1351
|
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1352
|
-
e.length > 0 &&
|
|
1352
|
+
e.length > 0 && S.set(t.name, e);
|
|
1353
1353
|
}
|
|
1354
|
-
let
|
|
1355
|
-
let n =
|
|
1354
|
+
let C = (e, t) => {
|
|
1355
|
+
let n = S.get(e);
|
|
1356
1356
|
if (n === void 0) return t;
|
|
1357
1357
|
let r;
|
|
1358
1358
|
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1359
1359
|
return r ?? t;
|
|
1360
1360
|
}, w = (e, t, n, r = !1) => {
|
|
1361
|
-
let i =
|
|
1362
|
-
return
|
|
1363
|
-
},
|
|
1364
|
-
r.unclassified && (
|
|
1365
|
-
},
|
|
1361
|
+
let i = O ? X(t.name) : t.name, a = t.primaryKey, o = C(t.name, n);
|
|
1362
|
+
return f === "append" ? _ === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : f === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1363
|
+
}, ae = (e, t) => e.savepoint ? e.savepoint(t) : t(), oe = 0, T = (t, n, r) => {
|
|
1364
|
+
r.unclassified && (oe++, !(oe > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (oe === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1365
|
+
}, se = (e, t, r = !0) => n.tryPromise({
|
|
1366
1366
|
try: e,
|
|
1367
|
-
catch: (e) =>
|
|
1368
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason,
|
|
1367
|
+
catch: (e) => En(e)
|
|
1368
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, T(t.entry, t.row, e)), !1)))), ce = (e) => {
|
|
1369
1369
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1370
1370
|
if (t.length !== 0) return {
|
|
1371
1371
|
...e.row,
|
|
1372
1372
|
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1373
1373
|
};
|
|
1374
|
-
},
|
|
1374
|
+
}, le = (e) => {
|
|
1375
1375
|
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
1376
1376
|
if (r !== "reference") return !1;
|
|
1377
1377
|
let i = e.row[n];
|
|
@@ -1389,7 +1389,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1389
1389
|
};
|
|
1390
1390
|
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1391
1391
|
}
|
|
1392
|
-
return new
|
|
1392
|
+
return new Ze({
|
|
1393
1393
|
rows: i.slice(0, 20),
|
|
1394
1394
|
primaryCount: r.filter((e) => !e.derived).length,
|
|
1395
1395
|
totalCount: e.length,
|
|
@@ -1399,104 +1399,104 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1399
1399
|
primary: t.primary
|
|
1400
1400
|
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1401
1401
|
});
|
|
1402
|
-
},
|
|
1402
|
+
}, ue = (e, t) => n.gen(function* () {
|
|
1403
1403
|
let r = t, i = [], a = !0;
|
|
1404
1404
|
for (; r.length > 0 && a;) {
|
|
1405
1405
|
a = !1;
|
|
1406
1406
|
let t = [];
|
|
1407
|
-
for (let n of r) (yield*
|
|
1407
|
+
for (let n of r) (yield* se(() => ae(e, () => w(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1408
1408
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1409
|
-
let r = t[n], o =
|
|
1410
|
-
o !== void 0 && (yield*
|
|
1409
|
+
let r = t[n], o = ce(r);
|
|
1410
|
+
o !== void 0 && (yield* se(() => ae(e, () => w(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1411
1411
|
}
|
|
1412
1412
|
r = t;
|
|
1413
1413
|
}
|
|
1414
|
-
for (let t of i) (yield*
|
|
1415
|
-
if (r.length > 0) return yield* n.fail(
|
|
1416
|
-
}),
|
|
1414
|
+
for (let t of i) (yield* se(() => ae(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1415
|
+
if (r.length > 0) return yield* n.fail(le(r));
|
|
1416
|
+
}), E = (() => {
|
|
1417
1417
|
let t = e.targetDialect;
|
|
1418
1418
|
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1419
|
-
})(),
|
|
1420
|
-
mode:
|
|
1419
|
+
})(), D = Gn({
|
|
1420
|
+
mode: f,
|
|
1421
1421
|
...e.targetDialect === void 0 ? {} : { targetDialect: e.targetDialect },
|
|
1422
1422
|
...e.targetSnapshot === void 0 ? {} : { targetSnapshot: e.targetSnapshot },
|
|
1423
1423
|
bundleTables: s.tables.map((e) => e.name),
|
|
1424
1424
|
isSkipped: V,
|
|
1425
|
-
canRunSql:
|
|
1426
|
-
}),
|
|
1427
|
-
if (
|
|
1425
|
+
canRunSql: An(e.store)
|
|
1426
|
+
}), fe = D.tables, pe = s.tables.filter((e) => !V(e.name) && ee(e.name)).map((e) => e.name), O = D.active, me = !1, k = 0, A = 0, ge, j, M = 0, N = 0, _e = (t, r) => n.gen(function* () {
|
|
1427
|
+
if (f === "replace" && !O && !(r && b.truncated)) {
|
|
1428
1428
|
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1429
1429
|
try: () => t.insert(d, {
|
|
1430
|
-
id:
|
|
1431
|
-
startedAt:
|
|
1430
|
+
id: h("repl", P),
|
|
1431
|
+
startedAt: F,
|
|
1432
1432
|
tables: String(s.tables.length),
|
|
1433
|
-
capture:
|
|
1433
|
+
capture: I?.dir ?? null,
|
|
1434
1434
|
via: e.via ?? "direct"
|
|
1435
1435
|
}),
|
|
1436
1436
|
catch: (e) => new H({ reason: `could not record that this replace started: ${String(e?.message ?? e)}\n The marker is what makes an interrupted run visible instead of silent, so the run does
|
|
1437
1437
|
not proceed without it.` })
|
|
1438
1438
|
}));
|
|
1439
1439
|
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !V(e)), a = !r, o = t.emptyTables ? () => t.emptyTables(i) : a ? async () => {
|
|
1440
|
-
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where:
|
|
1440
|
+
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where: x(e.primaryKey) });
|
|
1441
1441
|
} : () => t.transactional(async (e) => {
|
|
1442
|
-
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where:
|
|
1442
|
+
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: x(t.primaryKey) });
|
|
1443
1443
|
});
|
|
1444
1444
|
yield* n.tryPromise({
|
|
1445
1445
|
try: o,
|
|
1446
|
-
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${
|
|
1447
|
-
}), r && (
|
|
1446
|
+
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${On(e)}` })
|
|
1447
|
+
}), r && (b.truncated = !0, yield* J(v, b));
|
|
1448
1448
|
}
|
|
1449
|
-
let a = [], o = [], u = [],
|
|
1449
|
+
let a = [], o = [], u = [], p = [], g = s.tables.length;
|
|
1450
1450
|
for (let d = 0; d < s.tables.length; d++) {
|
|
1451
|
-
let
|
|
1452
|
-
if (V(
|
|
1453
|
-
|
|
1451
|
+
let h = s.tables[d];
|
|
1452
|
+
if (V(h.name)) {
|
|
1453
|
+
p.push(h.name);
|
|
1454
1454
|
continue;
|
|
1455
1455
|
}
|
|
1456
|
-
if (r &&
|
|
1457
|
-
u.push(
|
|
1456
|
+
if (r && h.name in b.tables) {
|
|
1457
|
+
u.push(h.name), N++, M += b.tables[h.name].rowCount, q(e.onProgress, {
|
|
1458
1458
|
phase: "import",
|
|
1459
|
-
table:
|
|
1459
|
+
table: h.name,
|
|
1460
1460
|
index: d,
|
|
1461
|
-
tableCount:
|
|
1461
|
+
tableCount: g,
|
|
1462
1462
|
event: "done",
|
|
1463
|
-
rowsDone:
|
|
1464
|
-
total:
|
|
1463
|
+
rowsDone: b.tables[h.name].rowCount,
|
|
1464
|
+
total: h.rowCount
|
|
1465
1465
|
});
|
|
1466
1466
|
continue;
|
|
1467
1467
|
}
|
|
1468
|
-
let _ = i.join(e.bundleDir,
|
|
1468
|
+
let _ = i.join(e.bundleDir, h.file);
|
|
1469
1469
|
q(e.onProgress, {
|
|
1470
1470
|
phase: "import",
|
|
1471
|
-
table:
|
|
1471
|
+
table: h.name,
|
|
1472
1472
|
index: d,
|
|
1473
|
-
tableCount:
|
|
1473
|
+
tableCount: g,
|
|
1474
1474
|
event: "start",
|
|
1475
1475
|
rowsDone: 0,
|
|
1476
|
-
total:
|
|
1476
|
+
total: h.rowCount
|
|
1477
1477
|
});
|
|
1478
|
-
let
|
|
1479
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c =
|
|
1478
|
+
let y = a.length, x = n.gen(function* () {
|
|
1479
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c = he("bundle", () => de(_).pipe(bt(s.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), u = async (e) => {
|
|
1480
1480
|
let n = {
|
|
1481
|
-
entry:
|
|
1481
|
+
entry: h,
|
|
1482
1482
|
row: e,
|
|
1483
1483
|
reason: ""
|
|
1484
1484
|
};
|
|
1485
1485
|
try {
|
|
1486
|
-
await
|
|
1486
|
+
await ae(t, () => w(t, h, e));
|
|
1487
1487
|
} catch (t) {
|
|
1488
|
-
let r =
|
|
1489
|
-
n.reason = r.reason,
|
|
1488
|
+
let r = En(t);
|
|
1489
|
+
n.reason = r.reason, T(h, e, r), a.push(n);
|
|
1490
1490
|
}
|
|
1491
1491
|
}, d = yield* n.promise(async () => {
|
|
1492
1492
|
let n = e.copyLoader;
|
|
1493
1493
|
if (!n || n.dialect !== "postgres") return !1;
|
|
1494
|
-
if (
|
|
1495
|
-
if (
|
|
1496
|
-
if (
|
|
1494
|
+
if (m) return me || (me = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1495
|
+
if (f === "replace") return !0;
|
|
1496
|
+
if (f !== "upsert") return !1;
|
|
1497
1497
|
try {
|
|
1498
1498
|
return (await t.query({
|
|
1499
|
-
table:
|
|
1499
|
+
table: h.name,
|
|
1500
1500
|
predicate: void 0,
|
|
1501
1501
|
order: [],
|
|
1502
1502
|
take: 1,
|
|
@@ -1506,19 +1506,19 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1506
1506
|
} catch {
|
|
1507
1507
|
return !1;
|
|
1508
1508
|
}
|
|
1509
|
-
}),
|
|
1509
|
+
}), p = Object.keys(h.columnTypes), g = Math.max(1, e.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
1510
1510
|
if (v.length === 0) return;
|
|
1511
1511
|
let t = v;
|
|
1512
1512
|
v = [];
|
|
1513
1513
|
try {
|
|
1514
1514
|
await e.copyLoader.copyInto({
|
|
1515
|
-
table:
|
|
1516
|
-
columns:
|
|
1517
|
-
columnTypes:
|
|
1515
|
+
table: h.name,
|
|
1516
|
+
columns: p,
|
|
1517
|
+
columnTypes: h.columnTypes,
|
|
1518
1518
|
rows: t
|
|
1519
1519
|
});
|
|
1520
1520
|
} catch (n) {
|
|
1521
|
-
d = !1, e.onWarn?.(`bulk COPY into ${
|
|
1521
|
+
d = !1, e.onWarn?.(`bulk COPY into ${h.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1522
1522
|
for (let e of t) await u(e);
|
|
1523
1523
|
}
|
|
1524
1524
|
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof t.savepoint == "function", S = [], ee = async () => {
|
|
@@ -1527,15 +1527,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1527
1527
|
S = [];
|
|
1528
1528
|
try {
|
|
1529
1529
|
await t.savepoint(async () => {
|
|
1530
|
-
for (let n of e) await w(t,
|
|
1530
|
+
for (let n of e) await w(t, h, n);
|
|
1531
1531
|
});
|
|
1532
1532
|
} catch {
|
|
1533
1533
|
for (let t of e) await u(t);
|
|
1534
1534
|
}
|
|
1535
1535
|
};
|
|
1536
|
-
return yield* l.runForEach(
|
|
1537
|
-
if (o++, d) {
|
|
1538
|
-
v.push(e), v.length >=
|
|
1536
|
+
return yield* l.runForEach(tn(c, h.name, h.columnTypes), (e) => n.promise(async () => {
|
|
1537
|
+
if (o++, j?.(), d) {
|
|
1538
|
+
v.push(e), v.length >= g && await y();
|
|
1539
1539
|
return;
|
|
1540
1540
|
}
|
|
1541
1541
|
if (x) {
|
|
@@ -1547,181 +1547,207 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1547
1547
|
rowCount: o,
|
|
1548
1548
|
checksum: i.digest("hex")
|
|
1549
1549
|
};
|
|
1550
|
-
}), { rowCount: S, checksum: ee } = yield*
|
|
1550
|
+
}), { rowCount: S, checksum: ee } = yield* Rt("import", h.name, d, g, (e) => e.rowCount, x);
|
|
1551
1551
|
if (c) {
|
|
1552
|
-
if (ee !==
|
|
1553
|
-
what: `table ${
|
|
1554
|
-
expected:
|
|
1552
|
+
if (ee !== h.checksum) return yield* n.fail(new Ye({
|
|
1553
|
+
what: `table ${h.name} checksum`,
|
|
1554
|
+
expected: h.checksum,
|
|
1555
1555
|
actual: ee
|
|
1556
1556
|
}));
|
|
1557
|
-
if (S !==
|
|
1558
|
-
what: `table ${
|
|
1559
|
-
expected: String(
|
|
1557
|
+
if (S !== h.rowCount) return yield* n.fail(new Ye({
|
|
1558
|
+
what: `table ${h.name} rowCount`,
|
|
1559
|
+
expected: String(h.rowCount),
|
|
1560
1560
|
actual: String(S)
|
|
1561
1561
|
}));
|
|
1562
1562
|
}
|
|
1563
1563
|
q(e.onProgress, {
|
|
1564
1564
|
phase: "import",
|
|
1565
|
-
table:
|
|
1565
|
+
table: h.name,
|
|
1566
1566
|
index: d,
|
|
1567
|
-
tableCount:
|
|
1567
|
+
tableCount: g,
|
|
1568
1568
|
event: "done",
|
|
1569
1569
|
rowsDone: S,
|
|
1570
|
-
total:
|
|
1571
|
-
}),
|
|
1572
|
-
name:
|
|
1570
|
+
total: h.rowCount
|
|
1571
|
+
}), A++, k += S, ge?.(), j?.(), r && (a.length > y ? o.push({
|
|
1572
|
+
name: h.name,
|
|
1573
1573
|
rowCount: S
|
|
1574
|
-
}) : (
|
|
1574
|
+
}) : (b.tables[h.name] = {
|
|
1575
1575
|
rowCount: S,
|
|
1576
|
-
checksum:
|
|
1577
|
-
}, yield* J(
|
|
1576
|
+
checksum: h.checksum
|
|
1577
|
+
}, yield* J(v, b)));
|
|
1578
1578
|
}
|
|
1579
|
-
if (a.length > 0 && (yield*
|
|
1579
|
+
if (a.length > 0 && (yield* ue(t, a)), f === "replace" && (yield* n.promise(() => t.deleteMany(d, { where: {
|
|
1580
1580
|
column: "startedAt",
|
|
1581
1581
|
op: "lte",
|
|
1582
|
-
value:
|
|
1582
|
+
value: F
|
|
1583
1583
|
} }).then(() => void 0, () => void 0))), r && o.length > 0) {
|
|
1584
1584
|
for (let e of o) {
|
|
1585
1585
|
let t = s.tables.find((t) => t.name === e.name);
|
|
1586
|
-
|
|
1586
|
+
b.tables[e.name] = {
|
|
1587
1587
|
rowCount: e.rowCount,
|
|
1588
1588
|
checksum: t.checksum
|
|
1589
1589
|
};
|
|
1590
1590
|
}
|
|
1591
|
-
yield* J(
|
|
1591
|
+
yield* J(v, b);
|
|
1592
1592
|
}
|
|
1593
1593
|
{
|
|
1594
|
-
let t =
|
|
1594
|
+
let t = qe(p, "skipped");
|
|
1595
1595
|
t !== void 0 && e.onWarn?.(t);
|
|
1596
1596
|
}
|
|
1597
1597
|
if (u.length > 0) {
|
|
1598
1598
|
let t = u.length === s.tables.length;
|
|
1599
|
-
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${
|
|
1599
|
+
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${v} to force a full re-import.`);
|
|
1600
1600
|
}
|
|
1601
|
-
}),
|
|
1601
|
+
}), P = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, F = /* @__PURE__ */ new Date(), I = yield* Kn({
|
|
1602
1602
|
store: e.store,
|
|
1603
|
-
mode:
|
|
1603
|
+
mode: f,
|
|
1604
1604
|
dir: e.rollbackDir,
|
|
1605
1605
|
snapshot: e.targetSnapshot,
|
|
1606
1606
|
dialect: e.targetDialect,
|
|
1607
1607
|
tables: s.tables.map((e) => e.name).filter((e) => !V(e)),
|
|
1608
1608
|
onWarn: e.onWarn
|
|
1609
1609
|
});
|
|
1610
|
-
|
|
1611
|
-
try: () => e.onRollbackCaptured(
|
|
1610
|
+
I !== void 0 && (e.onRollbackCaptured !== void 0 && (yield* n.tryPromise({
|
|
1611
|
+
try: () => e.onRollbackCaptured(I),
|
|
1612
1612
|
catch: (e) => new H({
|
|
1613
1613
|
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1614
|
-
path:
|
|
1614
|
+
path: I.dir
|
|
1615
1615
|
})
|
|
1616
|
-
})), e.onWarn?.(
|
|
1617
|
-
let ve = (e) =>
|
|
1618
|
-
R &&
|
|
1619
|
-
id:
|
|
1620
|
-
mode:
|
|
1616
|
+
})), e.onWarn?.(qn(I)));
|
|
1617
|
+
let ve = (e) => f === "replace" ? ne(e) : e(), L = (e) => f === "replace" ? re(e) : e, ye = e.traceId ?? h("dimp", `${Date.now()}-${s.source.schemaFingerprint ?? ""}`), be = e.traceId !== void 0, R = (e.targetSnapshot?.tables.some((e) => e.name === u) ?? !1) || y(u) !== void 0;
|
|
1618
|
+
R && g(p), R && e.traceId === void 0 && (yield* n.promise(() => e.store.insert(u, {
|
|
1619
|
+
id: ye,
|
|
1620
|
+
mode: f,
|
|
1621
1621
|
via: e.via ?? "direct",
|
|
1622
1622
|
direction: "import",
|
|
1623
1623
|
bundle: e.bundleDir,
|
|
1624
1624
|
sourceFingerprint: s.source.schemaFingerprint ?? null,
|
|
1625
1625
|
tablesWritten: 0,
|
|
1626
1626
|
rowsWritten: 0,
|
|
1627
|
-
staged:
|
|
1627
|
+
staged: f === "replace" ? O : null
|
|
1628
1628
|
}).then(() => {
|
|
1629
|
-
|
|
1629
|
+
be = !0;
|
|
1630
1630
|
}, (t) => {
|
|
1631
1631
|
e.onWarn?.(`the import trace could not be written (${String(t?.message ?? t)}).\n The import continues — this is a record of what happened, not a safety check. Run
|
|
1632
1632
|
\`voltro db apply\` so the next one is recorded.`);
|
|
1633
|
-
}))), e.traceId !== void 0 && R && (yield* n.promise(() => e.store.update(u,
|
|
1634
|
-
staged:
|
|
1633
|
+
}))), e.traceId !== void 0 && R && (yield* n.promise(() => e.store.update(u, ye, {
|
|
1634
|
+
staged: f === "replace" ? O : null,
|
|
1635
1635
|
sourceFingerprint: s.source.schemaFingerprint ?? null
|
|
1636
1636
|
}).then(() => void 0, () => void 0)));
|
|
1637
|
-
let
|
|
1638
|
-
|
|
1639
|
-
if (!
|
|
1637
|
+
let xe = 0, Se = !1;
|
|
1638
|
+
ge = () => {
|
|
1639
|
+
if (!be || Se) return;
|
|
1640
1640
|
let t = Date.now();
|
|
1641
|
-
t -
|
|
1642
|
-
tablesWritten:
|
|
1643
|
-
rowsWritten:
|
|
1641
|
+
t - xe < $n || (xe = t, e.store.update(u, ye, {
|
|
1642
|
+
tablesWritten: A,
|
|
1643
|
+
rowsWritten: k
|
|
1644
1644
|
}).then(() => void 0, () => {
|
|
1645
|
-
|
|
1645
|
+
Se = !0;
|
|
1646
1646
|
}));
|
|
1647
1647
|
};
|
|
1648
|
-
let
|
|
1648
|
+
let Ce = (t) => be ? e.store.update(u, ye, {
|
|
1649
1649
|
finishedAt: t === null ? /* @__PURE__ */ new Date() : null,
|
|
1650
|
-
tablesWritten:
|
|
1651
|
-
rowsWritten:
|
|
1650
|
+
tablesWritten: A,
|
|
1651
|
+
rowsWritten: k,
|
|
1652
1652
|
failure: t
|
|
1653
1653
|
}).then(() => void 0, () => void 0) : Promise.resolve();
|
|
1654
|
-
if (t = (e) =>
|
|
1654
|
+
if (t = (e) => Ce(e), f === "replace" && !O && D.blocker !== void 0 && e.onWarn?.(jn(D.blocker)), f === "replace" && pe.length > 0 && e.onWarn?.(`write recorders are SUSPENDED for this replace (${pe.length} table(s) have one).\n A replace sets a state rather than changing rows, so a per-row history entry would describe
|
|
1655
1655
|
something that did not happen — and this path already writes through the raw store, without
|
|
1656
|
-
tenant scoping or \`audit()\` stamping, for the same reason.`),
|
|
1657
|
-
e.onWarn?.(`staging ${
|
|
1658
|
-
let t = e.store, r =
|
|
1659
|
-
|
|
1656
|
+
tenant scoping or \`audit()\` stamping, for the same reason.`), O) {
|
|
1657
|
+
e.onWarn?.(`staging ${fe.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1658
|
+
let t = e.store, r = fe.map((e) => e.name), i = h("repl", `staging-${P}`), a = !1;
|
|
1659
|
+
if (e.recordInterruption === !0) {
|
|
1660
|
+
yield* n.promise(() => e.store.insert(d, {
|
|
1661
|
+
id: i,
|
|
1662
|
+
startedAt: F,
|
|
1663
|
+
kind: "staging",
|
|
1664
|
+
tables: String(fe.length),
|
|
1665
|
+
capture: I?.dir ?? null,
|
|
1666
|
+
via: e.via ?? "direct",
|
|
1667
|
+
staging: r.map(X).join(","),
|
|
1668
|
+
heartbeatAt: /* @__PURE__ */ new Date(),
|
|
1669
|
+
resumable: !m
|
|
1670
|
+
}).then(() => {
|
|
1671
|
+
a = !0;
|
|
1672
|
+
}, (t) => {
|
|
1673
|
+
e.onWarn?.(`the staging tables could not be recorded (${String(t?.message ?? t)}).\n The import continues and the target is unaffected — but if this run dies before the swap,
|
|
1674
|
+
nothing at boot will mention the scratch tables it leaves. \`voltro data clear-staging\` finds
|
|
1675
|
+
them by introspection.`);
|
|
1676
|
+
}));
|
|
1677
|
+
let t = Date.now();
|
|
1678
|
+
j = () => {
|
|
1679
|
+
if (!a) return;
|
|
1680
|
+
let n = Date.now();
|
|
1681
|
+
n - t < er || (t = n, e.store.update(d, i, { heartbeatAt: /* @__PURE__ */ new Date() }).then(() => void 0, () => void 0));
|
|
1682
|
+
};
|
|
1683
|
+
}
|
|
1684
|
+
let o = () => a ? e.store.delete(d, i).then(() => void 0, () => void 0) : Promise.resolve(), s = In(r, {
|
|
1685
|
+
getTable: (e) => y(e),
|
|
1660
1686
|
registerTable: (e) => {
|
|
1661
|
-
|
|
1687
|
+
te(e);
|
|
1662
1688
|
},
|
|
1663
|
-
unregisterTable:
|
|
1664
|
-
}),
|
|
1689
|
+
unregisterTable: ie
|
|
1690
|
+
}), c = !1;
|
|
1665
1691
|
try {
|
|
1666
|
-
let i =
|
|
1692
|
+
let i = m ? r : r.filter((e) => !Vt(b, e));
|
|
1667
1693
|
yield* n.tryPromise({
|
|
1668
1694
|
try: async () => {
|
|
1669
|
-
await
|
|
1695
|
+
await Bn(t, i.map((e) => Pn(e, E))), await Bn(t, i.map((e) => Nn(e, E)));
|
|
1670
1696
|
},
|
|
1671
1697
|
catch: (e) => new H({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1672
|
-
}), yield*
|
|
1673
|
-
try: () =>
|
|
1674
|
-
|
|
1698
|
+
}), yield* L(_e(e.store, !m)), !m && b.truncated === !0 ? c = !0 : yield* n.tryPromise({
|
|
1699
|
+
try: () => Vn(t, fe, E).then(() => {
|
|
1700
|
+
c = !0;
|
|
1675
1701
|
}),
|
|
1676
1702
|
catch: (e) => new H({ reason: String(e?.message ?? e) })
|
|
1677
1703
|
}).pipe(n.catchAll((i) => n.gen(function* () {
|
|
1678
|
-
let a =
|
|
1704
|
+
let a = zn(yield* n.promise(() => Hn(t, Ln(e.targetSnapshot, r), E).then((e) => e, () => [])));
|
|
1679
1705
|
return yield* n.fail(new H({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1680
|
-
}))), !
|
|
1706
|
+
}))), !m && c && b.truncated !== !0 && (b.truncated = !0, yield* J(v, b));
|
|
1681
1707
|
} finally {
|
|
1682
|
-
|
|
1708
|
+
s.undo(), c || m ? (j = void 0, yield* n.promise(() => Bn(t, r.map((e) => Pn(e, E))).then(() => void 0, () => void 0)), yield* n.promise(o)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1683
1709
|
}
|
|
1684
|
-
} else
|
|
1710
|
+
} else m ? yield* n.tryPromise({
|
|
1685
1711
|
try: () => e.store.transactional(async (e) => {
|
|
1686
|
-
let t = await ve(() => n.runPromiseExit(
|
|
1687
|
-
if (r.isFailure(t)) throw
|
|
1712
|
+
let t = await ve(() => n.runPromiseExit(_e(e, !1)));
|
|
1713
|
+
if (r.isFailure(t)) throw ir(t.cause);
|
|
1688
1714
|
}),
|
|
1689
|
-
catch: (e) =>
|
|
1690
|
-
}) : yield*
|
|
1715
|
+
catch: (e) => rr(e) ?? new H({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1716
|
+
}) : yield* L(_e(e.store, !0));
|
|
1691
1717
|
if (e.assets && s.assets.index && !e.assetsAlreadyRestored) {
|
|
1692
1718
|
let t = e.assets;
|
|
1693
|
-
yield*
|
|
1719
|
+
yield* ht({
|
|
1694
1720
|
sink: t,
|
|
1695
|
-
assetsDir: i.join(e.bundleDir,
|
|
1721
|
+
assetsDir: i.join(e.bundleDir, At),
|
|
1696
1722
|
indexPath: i.join(e.bundleDir, s.assets.index ?? "assets/index.ndjson"),
|
|
1697
1723
|
progress: {
|
|
1698
|
-
done: (e) => e in
|
|
1699
|
-
mark: (e) => (
|
|
1724
|
+
done: (e) => e in b.assets,
|
|
1725
|
+
mark: (e) => (b.assets[e] = !0, J(v, b))
|
|
1700
1726
|
}
|
|
1701
1727
|
});
|
|
1702
1728
|
}
|
|
1703
|
-
return yield* n.promise(() =>
|
|
1729
|
+
return yield* n.promise(() => Ce(null)), {
|
|
1704
1730
|
manifest: s,
|
|
1705
|
-
tablesWritten:
|
|
1706
|
-
rowsWritten:
|
|
1731
|
+
tablesWritten: A,
|
|
1732
|
+
rowsWritten: k,
|
|
1707
1733
|
tablesSkipped: N,
|
|
1708
1734
|
rowsSkipped: M,
|
|
1709
|
-
fullyResumed:
|
|
1710
|
-
...
|
|
1735
|
+
fullyResumed: A === 0 && N > 0,
|
|
1736
|
+
...I === void 0 ? {} : { rollback: I }
|
|
1711
1737
|
};
|
|
1712
1738
|
}).pipe(n.tapError((e) => n.promise(() => t?.(e?.reason ?? e?._tag ?? "the import did not finish") ?? Promise.resolve())));
|
|
1713
|
-
},
|
|
1739
|
+
}, sr = (e) => e, cr = (e) => {
|
|
1714
1740
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1715
1741
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1716
1742
|
return {
|
|
1717
1743
|
refuse: !0,
|
|
1718
1744
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1719
1745
|
};
|
|
1720
|
-
}, Q = Buffer.from("VBUNDLE2"),
|
|
1746
|
+
}, Q = Buffer.from("VBUNDLE2"), lr = 0, ur = 1, dr = 2, fr = 32, pr = async (e) => {
|
|
1721
1747
|
let t = [], n = async (r) => {
|
|
1722
1748
|
let i;
|
|
1723
1749
|
try {
|
|
1724
|
-
i = await
|
|
1750
|
+
i = await I(R(e, r), { withFileTypes: !0 });
|
|
1725
1751
|
} catch {
|
|
1726
1752
|
return;
|
|
1727
1753
|
}
|
|
@@ -1731,22 +1757,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1731
1757
|
}
|
|
1732
1758
|
};
|
|
1733
1759
|
return await n(""), t.sort();
|
|
1734
|
-
},
|
|
1760
|
+
}, mr = (e, t) => {
|
|
1735
1761
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1736
|
-
return r.writeUInt8(
|
|
1737
|
-
},
|
|
1762
|
+
return r.writeUInt8(ur, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1763
|
+
}, hr = (e, t, n) => {
|
|
1738
1764
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1739
|
-
return a.writeUInt8(
|
|
1765
|
+
return a.writeUInt8(dr, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1740
1766
|
};
|
|
1741
|
-
async function*
|
|
1767
|
+
async function* gr(e) {
|
|
1742
1768
|
yield new Uint8Array(Q);
|
|
1743
|
-
for (let t of await
|
|
1769
|
+
for (let t of await pr(e.dir)) {
|
|
1744
1770
|
let n = R(e.dir, t), r = await ve(n);
|
|
1745
|
-
if (yield new Uint8Array(
|
|
1771
|
+
if (yield new Uint8Array(mr(t, r.size)), r.size > 0) for await (let e of de(n)) yield new Uint8Array(e);
|
|
1746
1772
|
}
|
|
1747
1773
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1748
|
-
yield new Uint8Array(
|
|
1749
|
-
let n =
|
|
1774
|
+
yield new Uint8Array(hr(t.key, t.contentType, t.size));
|
|
1775
|
+
let n = T("sha256"), r = 0;
|
|
1750
1776
|
for await (let i of e.blobs.open(t.key)) {
|
|
1751
1777
|
let e = Buffer.from(i);
|
|
1752
1778
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1754,11 +1780,11 @@ async function* mr(e) {
|
|
|
1754
1780
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1755
1781
|
yield new Uint8Array(n.digest());
|
|
1756
1782
|
}
|
|
1757
|
-
yield new Uint8Array(Buffer.from([
|
|
1783
|
+
yield new Uint8Array(Buffer.from([lr]));
|
|
1758
1784
|
}
|
|
1759
|
-
var
|
|
1760
|
-
let n =
|
|
1761
|
-
await
|
|
1785
|
+
var _r = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vr = (e) => new Promise((t) => e.end(t)), yr = async (e, t) => {
|
|
1786
|
+
let n = xe(t.dir);
|
|
1787
|
+
await F(n, { recursive: !0 });
|
|
1762
1788
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1763
1789
|
let { value: e, done: t } = await i.next();
|
|
1764
1790
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1773,24 +1799,24 @@ var hr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), gr
|
|
|
1773
1799
|
for (;;) {
|
|
1774
1800
|
await s(1);
|
|
1775
1801
|
let e = c(1).readUInt8(0);
|
|
1776
|
-
if (e ===
|
|
1777
|
-
if (e ===
|
|
1802
|
+
if (e === lr) return;
|
|
1803
|
+
if (e === ur) {
|
|
1778
1804
|
await s(4);
|
|
1779
1805
|
let e = c(4).readUInt32BE(0);
|
|
1780
1806
|
await s(e);
|
|
1781
1807
|
let t = c(e).toString("utf8");
|
|
1782
1808
|
await s(8);
|
|
1783
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1784
|
-
if (a !== n && !a.startsWith(n +
|
|
1785
|
-
await
|
|
1786
|
-
let o =
|
|
1809
|
+
let i = c(8).readBigUInt64BE(0), a = xe(n, t);
|
|
1810
|
+
if (a !== n && !a.startsWith(n + Se)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1811
|
+
await F(be(a), { recursive: !0 });
|
|
1812
|
+
let o = D(a);
|
|
1787
1813
|
for (; i > 0n;) {
|
|
1788
1814
|
r.length === 0 && await s(1);
|
|
1789
1815
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1790
|
-
await
|
|
1816
|
+
await _r(o, c(e)), i -= BigInt(e);
|
|
1791
1817
|
}
|
|
1792
|
-
await
|
|
1793
|
-
} else if (e ===
|
|
1818
|
+
await vr(o);
|
|
1819
|
+
} else if (e === dr) {
|
|
1794
1820
|
await s(4);
|
|
1795
1821
|
let e = c(4).readUInt32BE(0);
|
|
1796
1822
|
await s(e);
|
|
@@ -1800,24 +1826,24 @@ var hr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), gr
|
|
|
1800
1826
|
await s(i);
|
|
1801
1827
|
let a = c(i).toString("utf8");
|
|
1802
1828
|
await s(8);
|
|
1803
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1829
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = T("sha256"), d = new N(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1804
1830
|
contentType: a,
|
|
1805
1831
|
size: Number(o)
|
|
1806
1832
|
}), p = o;
|
|
1807
1833
|
for (; p > 0n;) {
|
|
1808
1834
|
r.length === 0 && await s(1);
|
|
1809
1835
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1810
|
-
u.update(t), l || await
|
|
1836
|
+
u.update(t), l || await _r(d, t), p -= BigInt(e);
|
|
1811
1837
|
}
|
|
1812
|
-
l ? d.destroy() : await
|
|
1813
|
-
let m = c(
|
|
1838
|
+
l ? d.destroy() : await vr(d), await f, await s(fr);
|
|
1839
|
+
let m = c(fr);
|
|
1814
1840
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1815
1841
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1816
1842
|
}
|
|
1817
1843
|
} finally {
|
|
1818
1844
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1819
1845
|
}
|
|
1820
|
-
},
|
|
1846
|
+
}, br = async (e, t) => {
|
|
1821
1847
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1822
1848
|
let { value: e, done: t } = await a.next();
|
|
1823
1849
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1839,8 +1865,8 @@ var hr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), gr
|
|
|
1839
1865
|
if (!await s(Q.length) || !c(Q.length).equals(Q)) throw Error("not a bundle archive (bad magic)");
|
|
1840
1866
|
for (; await s(1);) {
|
|
1841
1867
|
let e = c(1).readUInt8(0);
|
|
1842
|
-
if (e ===
|
|
1843
|
-
if (e !==
|
|
1868
|
+
if (e === lr || e === dr) break;
|
|
1869
|
+
if (e !== ur) throw Error(`unknown bundle entry kind ${e}`);
|
|
1844
1870
|
if (!await s(4)) break;
|
|
1845
1871
|
let t = c(4).readUInt32BE(0);
|
|
1846
1872
|
if (!await s(t)) break;
|
|
@@ -1850,20 +1876,20 @@ var hr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), gr
|
|
|
1850
1876
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1851
1877
|
}
|
|
1852
1878
|
return r;
|
|
1853
|
-
},
|
|
1879
|
+
}, xr = (e, t) => yr(e, {
|
|
1854
1880
|
dir: t,
|
|
1855
|
-
blobs:
|
|
1856
|
-
}),
|
|
1881
|
+
blobs: Sr(t)
|
|
1882
|
+
}), Sr = (e) => {
|
|
1857
1883
|
let t = /* @__PURE__ */ new Set();
|
|
1858
1884
|
return {
|
|
1859
1885
|
has: async (e) => t.has(e),
|
|
1860
1886
|
write: async (n, r, i) => {
|
|
1861
1887
|
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = R(e, "assets");
|
|
1862
|
-
await
|
|
1863
|
-
let l =
|
|
1888
|
+
await F(c, { recursive: !0 });
|
|
1889
|
+
let l = T("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1864
1890
|
l.update(e), u += e.length, n(null, e);
|
|
1865
1891
|
} }), f = R(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1866
|
-
await o(r, d,
|
|
1892
|
+
await o(r, d, D(f));
|
|
1867
1893
|
let p = l.digest("hex");
|
|
1868
1894
|
await a.rename(f, R(c, p)), await a.appendFile(R(c, "index.ndjson"), `${JSON.stringify({
|
|
1869
1895
|
key: n,
|
|
@@ -1873,33 +1899,33 @@ var hr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), gr
|
|
|
1873
1899
|
})}\n`), t.add(n);
|
|
1874
1900
|
}
|
|
1875
1901
|
};
|
|
1876
|
-
},
|
|
1902
|
+
}, Cr = Ce(ue), wr = Buffer.from("VENC1\0\0\0"), Tr = 32, Er = 16, Dr = 7, Or = 16, kr = 262144, Ar = {
|
|
1877
1903
|
N: 32768,
|
|
1878
1904
|
r: 8,
|
|
1879
1905
|
p: 1
|
|
1880
|
-
},
|
|
1906
|
+
}, jr = 134217728, Mr = 43, Nr = (e, t, n) => {
|
|
1881
1907
|
let r = Buffer.alloc(12);
|
|
1882
|
-
return e.copy(r, 0, 0,
|
|
1908
|
+
return e.copy(r, 0, 0, Dr), r.writeUInt32BE(t >>> 0, Dr), r.writeUInt8(+!!n, 11), r;
|
|
1883
1909
|
};
|
|
1884
|
-
async function*
|
|
1910
|
+
async function* Pr(e, t) {
|
|
1885
1911
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1886
|
-
let n =
|
|
1887
|
-
...
|
|
1888
|
-
maxmem:
|
|
1889
|
-
}), a = Buffer.alloc(
|
|
1890
|
-
|
|
1912
|
+
let n = ce(Er), r = ce(Dr), i = await Cr(t, n, Tr, {
|
|
1913
|
+
...Ar,
|
|
1914
|
+
maxmem: jr
|
|
1915
|
+
}), a = Buffer.alloc(Mr);
|
|
1916
|
+
wr.copy(a, 0), a.writeUInt32BE(Ar.N, 8), a.writeUInt32BE(Ar.r, 12), a.writeUInt32BE(Ar.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1891
1917
|
let o = 0, s = (e, t) => {
|
|
1892
|
-
let n =
|
|
1918
|
+
let n = ae("aes-256-gcm", i, Nr(r, o, t));
|
|
1893
1919
|
o === 0 && n.setAAD(a);
|
|
1894
1920
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1895
1921
|
o++;
|
|
1896
|
-
let l = Buffer.alloc(5 + s.length +
|
|
1897
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
1922
|
+
let l = Buffer.alloc(5 + s.length + Or);
|
|
1923
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + Or, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1898
1924
|
}, c = Buffer.alloc(0);
|
|
1899
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1925
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > kr;) yield s(c.subarray(0, kr), !1), c = c.subarray(kr);
|
|
1900
1926
|
yield s(c, !0);
|
|
1901
1927
|
}
|
|
1902
|
-
async function*
|
|
1928
|
+
async function* Fr(e, t) {
|
|
1903
1929
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1904
1930
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1905
1931
|
let { value: e, done: t } = await n.next();
|
|
@@ -1908,16 +1934,16 @@ async function* Nr(e, t) {
|
|
|
1908
1934
|
for (; r.length < e && !i && await a(););
|
|
1909
1935
|
return r.length >= e;
|
|
1910
1936
|
};
|
|
1911
|
-
if (!await o(
|
|
1912
|
-
let s = r.subarray(0,
|
|
1913
|
-
if (!s.subarray(0, 8).equals(
|
|
1914
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36,
|
|
1937
|
+
if (!await o(Mr)) throw Error("encrypted bundle truncated (no header)");
|
|
1938
|
+
let s = r.subarray(0, Mr);
|
|
1939
|
+
if (!s.subarray(0, 8).equals(wr)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1940
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Mr)), p = await Cr(t, d, Tr, {
|
|
1915
1941
|
N: c,
|
|
1916
1942
|
r: l,
|
|
1917
1943
|
p: u,
|
|
1918
|
-
maxmem:
|
|
1944
|
+
maxmem: jr
|
|
1919
1945
|
});
|
|
1920
|
-
r = r.subarray(
|
|
1946
|
+
r = r.subarray(Mr);
|
|
1921
1947
|
let m = 0, h = !1;
|
|
1922
1948
|
for (;;) {
|
|
1923
1949
|
if (!await o(5)) {
|
|
@@ -1926,11 +1952,11 @@ async function* Nr(e, t) {
|
|
|
1926
1952
|
}
|
|
1927
1953
|
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
1928
1954
|
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
1929
|
-
if (t <
|
|
1955
|
+
if (t < Or) throw Error("encrypted bundle corrupt (short frame)");
|
|
1930
1956
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1931
|
-
let n = r.subarray(5, 5 + t -
|
|
1957
|
+
let n = r.subarray(5, 5 + t - Or), i = r.subarray(5 + t - Or, 5 + t);
|
|
1932
1958
|
r = r.subarray(5 + t);
|
|
1933
|
-
let a =
|
|
1959
|
+
let a = oe("aes-256-gcm", p, Nr(f, m, e));
|
|
1934
1960
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1935
1961
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1936
1962
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1938,7 +1964,7 @@ async function* Nr(e, t) {
|
|
|
1938
1964
|
}
|
|
1939
1965
|
//#endregion
|
|
1940
1966
|
//#region src/blobStorage.ts
|
|
1941
|
-
var
|
|
1967
|
+
var Ir = (e, t) => ({
|
|
1942
1968
|
list: () => {
|
|
1943
1969
|
let e = t();
|
|
1944
1970
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1951,32 +1977,32 @@ var Pr = (e, t) => ({
|
|
|
1951
1977
|
} };
|
|
1952
1978
|
},
|
|
1953
1979
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1954
|
-
let r = await n.runPromise(A(e, t)), i = await
|
|
1980
|
+
let r = await n.runPromise(A(e, t)), i = await j(r.stream);
|
|
1955
1981
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1956
1982
|
} })
|
|
1957
|
-
}),
|
|
1958
|
-
table:
|
|
1983
|
+
}), Lr = (e, t) => Ir(e, () => C(t, {
|
|
1984
|
+
table: k,
|
|
1959
1985
|
chunkSize: 500
|
|
1960
1986
|
}).pipe(l.map((e) => ({
|
|
1961
1987
|
key: String(e.key),
|
|
1962
1988
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1963
1989
|
size: Number(e.size ?? 0)
|
|
1964
|
-
})))),
|
|
1990
|
+
})))), Rr = (e) => ({
|
|
1965
1991
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1966
1992
|
write: async (t, r, i) => {
|
|
1967
|
-
let a =
|
|
1968
|
-
await n.runPromise(
|
|
1993
|
+
let a = he("storage", () => _e.from(r));
|
|
1994
|
+
await n.runPromise(ge(e, t, a, {
|
|
1969
1995
|
contentType: i.contentType,
|
|
1970
1996
|
totalSize: i.size
|
|
1971
1997
|
}));
|
|
1972
1998
|
}
|
|
1973
|
-
}),
|
|
1999
|
+
}), zr = (e, t) => {
|
|
1974
2000
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1975
|
-
return e === "mariadb" ?
|
|
2001
|
+
return e === "mariadb" ? Br(n) ? { tool: n } : {
|
|
1976
2002
|
tool: r,
|
|
1977
2003
|
hint: `'${n}' is not on PATH, so this ran '${r}' instead. If that binary came from Oracle's MySQL client it queries information_schema.COLUMN_STATISTICS, which MariaDB does not have, and the run dies with error 1109 after the first table. Install the MariaDB client package (which ships '${n}') and re-run. Do NOT reach for --column-statistics=0: that flag does not exist on '${n}', so it fixes the wrong client and breaks the right one. Note: a MariaDB 12.x client requires TLS by default, so running '${n}' BY HAND against a server without it fails with "TLS/SSL error: SSL is required" (2026) and needs --skip-ssl. This command is not affected.`
|
|
1978
2004
|
} : { tool: r };
|
|
1979
|
-
},
|
|
2005
|
+
}, Br = (e) => (process.env.PATH ?? "").split(ye).filter((e) => e !== "").some((t) => fe(R(t, e))), Vr = (e) => {
|
|
1980
2006
|
switch (e) {
|
|
1981
2007
|
case "postgres": return "db.dump";
|
|
1982
2008
|
case "mysql":
|
|
@@ -1986,7 +2012,7 @@ var Pr = (e, t) => ({
|
|
|
1986
2012
|
case "mssql": return "db.bacpac";
|
|
1987
2013
|
default: return "db.dump";
|
|
1988
2014
|
}
|
|
1989
|
-
},
|
|
2015
|
+
}, Hr = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1990
2016
|
if (e.host || e.database) return {
|
|
1991
2017
|
host: e.host ?? "localhost",
|
|
1992
2018
|
port: e.port ?? 3306,
|
|
@@ -2002,7 +2028,7 @@ var Pr = (e, t) => ({
|
|
|
2002
2028
|
password: decodeURIComponent(t.password) || "",
|
|
2003
2029
|
database: t.pathname.replace(/^\//, "") || ""
|
|
2004
2030
|
};
|
|
2005
|
-
},
|
|
2031
|
+
}, Ur = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Wr = (e, t, n) => {
|
|
2006
2032
|
switch (e) {
|
|
2007
2033
|
case "postgres": {
|
|
2008
2034
|
if (t.url) return {
|
|
@@ -2012,7 +2038,7 @@ var Pr = (e, t) => ({
|
|
|
2012
2038
|
"--format=custom",
|
|
2013
2039
|
"--no-owner",
|
|
2014
2040
|
"--no-privileges",
|
|
2015
|
-
...
|
|
2041
|
+
...Kr,
|
|
2016
2042
|
"--dbname",
|
|
2017
2043
|
t.url,
|
|
2018
2044
|
"--file",
|
|
@@ -2031,7 +2057,7 @@ var Pr = (e, t) => ({
|
|
|
2031
2057
|
"--format=custom",
|
|
2032
2058
|
"--no-owner",
|
|
2033
2059
|
"--no-privileges",
|
|
2034
|
-
...
|
|
2060
|
+
...Kr,
|
|
2035
2061
|
"-h",
|
|
2036
2062
|
e.host,
|
|
2037
2063
|
"-p",
|
|
@@ -2051,14 +2077,14 @@ var Pr = (e, t) => ({
|
|
|
2051
2077
|
let r = $(t);
|
|
2052
2078
|
return {
|
|
2053
2079
|
kind: "spawn",
|
|
2054
|
-
...
|
|
2080
|
+
...zr(e, "dump"),
|
|
2055
2081
|
args: [
|
|
2056
2082
|
"--single-transaction",
|
|
2057
2083
|
"--routines",
|
|
2058
2084
|
"--triggers",
|
|
2059
2085
|
"--no-tablespaces",
|
|
2060
2086
|
"--protocol=TCP",
|
|
2061
|
-
...
|
|
2087
|
+
...Gr.map((e) => `--ignore-table=${r.database}.${e}`),
|
|
2062
2088
|
"-h",
|
|
2063
2089
|
r.host,
|
|
2064
2090
|
"-P",
|
|
@@ -2074,7 +2100,7 @@ var Pr = (e, t) => ({
|
|
|
2074
2100
|
case "sqlite":
|
|
2075
2101
|
case "turso": return {
|
|
2076
2102
|
kind: "copy",
|
|
2077
|
-
from:
|
|
2103
|
+
from: Hr(t),
|
|
2078
2104
|
to: n
|
|
2079
2105
|
};
|
|
2080
2106
|
case "mssql": return {
|
|
@@ -2082,7 +2108,7 @@ var Pr = (e, t) => ({
|
|
|
2082
2108
|
tool: "sqlpackage",
|
|
2083
2109
|
args: [
|
|
2084
2110
|
"/Action:Export",
|
|
2085
|
-
`/SourceConnectionString:${
|
|
2111
|
+
`/SourceConnectionString:${Ur($({
|
|
2086
2112
|
...t,
|
|
2087
2113
|
port: t.port ?? 1433
|
|
2088
2114
|
}))}`,
|
|
@@ -2095,7 +2121,7 @@ var Pr = (e, t) => ({
|
|
|
2095
2121
|
reason: `no native backup for dialect '${e}'`
|
|
2096
2122
|
});
|
|
2097
2123
|
}
|
|
2098
|
-
},
|
|
2124
|
+
}, Gr = [d, u], Kr = Gr.flatMap((e) => ["--exclude-table", e]), qr = (e) => e === "postgres" || e === "mysql" || e === "mariadb" ? Gr : [], Jr = (e, t, n) => {
|
|
2099
2125
|
switch (e) {
|
|
2100
2126
|
case "postgres": {
|
|
2101
2127
|
if (t.url) return {
|
|
@@ -2140,7 +2166,7 @@ var Pr = (e, t) => ({
|
|
|
2140
2166
|
let r = $(t);
|
|
2141
2167
|
return {
|
|
2142
2168
|
kind: "spawn",
|
|
2143
|
-
...
|
|
2169
|
+
...zr(e, "client"),
|
|
2144
2170
|
args: [
|
|
2145
2171
|
"--protocol=TCP",
|
|
2146
2172
|
"-h",
|
|
@@ -2159,14 +2185,14 @@ var Pr = (e, t) => ({
|
|
|
2159
2185
|
case "turso": return {
|
|
2160
2186
|
kind: "copy",
|
|
2161
2187
|
from: n,
|
|
2162
|
-
to:
|
|
2188
|
+
to: Hr(t)
|
|
2163
2189
|
};
|
|
2164
2190
|
case "mssql": return {
|
|
2165
2191
|
kind: "spawn",
|
|
2166
2192
|
tool: "sqlpackage",
|
|
2167
2193
|
args: [
|
|
2168
2194
|
"/Action:Import",
|
|
2169
|
-
`/TargetConnectionString:${
|
|
2195
|
+
`/TargetConnectionString:${Ur($({
|
|
2170
2196
|
...t,
|
|
2171
2197
|
port: t.port ?? 1433
|
|
2172
2198
|
}))}`,
|
|
@@ -2179,22 +2205,22 @@ var Pr = (e, t) => ({
|
|
|
2179
2205
|
reason: `no native restore for dialect '${e}'`
|
|
2180
2206
|
});
|
|
2181
2207
|
}
|
|
2182
|
-
},
|
|
2183
|
-
let r = `${e.to}.restoring-${
|
|
2208
|
+
}, Yr = (e) => e.kind === "copy" ? n.async((t) => {
|
|
2209
|
+
let r = `${e.to}.restoring-${le()}`, i = (e) => {
|
|
2184
2210
|
try {
|
|
2185
|
-
|
|
2211
|
+
me(r);
|
|
2186
2212
|
} catch {}
|
|
2187
2213
|
t(n.fail(new W({
|
|
2188
2214
|
tool: "copy",
|
|
2189
2215
|
reason: e
|
|
2190
2216
|
})));
|
|
2191
2217
|
};
|
|
2192
|
-
|
|
2218
|
+
E(e.from, r, (a) => {
|
|
2193
2219
|
if (a) {
|
|
2194
2220
|
i(`copy ${e.from} → ${e.to}: ${a.message}`);
|
|
2195
2221
|
return;
|
|
2196
2222
|
}
|
|
2197
|
-
|
|
2223
|
+
pe(r, e.to, (r) => {
|
|
2198
2224
|
if (r) {
|
|
2199
2225
|
i(`copy ${e.from} → ${e.to}: ${r.message}`);
|
|
2200
2226
|
return;
|
|
@@ -2203,7 +2229,7 @@ var Pr = (e, t) => ({
|
|
|
2203
2229
|
});
|
|
2204
2230
|
});
|
|
2205
2231
|
}) : n.async((t) => {
|
|
2206
|
-
let r =
|
|
2232
|
+
let r = we(e.tool, [...e.args], {
|
|
2207
2233
|
env: {
|
|
2208
2234
|
...process.env,
|
|
2209
2235
|
...e.env
|
|
@@ -2223,23 +2249,23 @@ var Pr = (e, t) => ({
|
|
|
2223
2249
|
s ??= e;
|
|
2224
2250
|
};
|
|
2225
2251
|
if (e.stdoutFile && r.stdout) {
|
|
2226
|
-
let t =
|
|
2227
|
-
t.on("error", (t) => c(`writing ${
|
|
2252
|
+
let t = D(e.stdoutFile);
|
|
2253
|
+
t.on("error", (t) => c(`writing ${L(e.stdoutFile)}: ${t.message}`)), r.stdout.on("error", (e) => c(`reading the tool's output: ${e.message}`)), r.stdout.pipe(t);
|
|
2228
2254
|
}
|
|
2229
2255
|
let l = 0;
|
|
2230
2256
|
if (e.stdinFile && r.stdin) {
|
|
2231
|
-
let t =
|
|
2257
|
+
let t = de(e.stdinFile);
|
|
2232
2258
|
t.on("data", (e) => {
|
|
2233
2259
|
l += e.length;
|
|
2234
2260
|
}), t.on("error", (t) => {
|
|
2235
|
-
c(`reading ${
|
|
2261
|
+
c(`reading ${L(e.stdinFile)}: ${t.message}`), r.stdin?.end();
|
|
2236
2262
|
}), r.stdin.on("error", (t) => {
|
|
2237
|
-
t.code !== "EPIPE" && c(`feeding ${
|
|
2263
|
+
t.code !== "EPIPE" && c(`feeding ${L(e.stdinFile)} to ${e.tool}: ${t.message}`);
|
|
2238
2264
|
}), t.pipe(r.stdin);
|
|
2239
2265
|
}
|
|
2240
2266
|
let u = () => {
|
|
2241
2267
|
if (e.stdoutFile !== void 0) try {
|
|
2242
|
-
|
|
2268
|
+
me(e.stdoutFile);
|
|
2243
2269
|
} catch {}
|
|
2244
2270
|
};
|
|
2245
2271
|
r.on("error", (t) => {
|
|
@@ -2254,25 +2280,25 @@ var Pr = (e, t) => ({
|
|
|
2254
2280
|
return;
|
|
2255
2281
|
}
|
|
2256
2282
|
u();
|
|
2257
|
-
let r = e.stdinFile === void 0 ? "" : ` — ${l} of ${
|
|
2283
|
+
let r = e.stdinFile === void 0 ? "" : ` — ${l} of ${Xr(e.stdinFile)} byte(s) of ${L(e.stdinFile)} had been fed to it`;
|
|
2258
2284
|
o(n.fail(new W({
|
|
2259
2285
|
tool: e.tool,
|
|
2260
|
-
reason: (t === 0 ? `exited cleanly, but ${s}` : `exited with code ${t}${s === void 0 ? "" : ` (${s})`}`) + r + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2286
|
+
reason: (t === 0 ? `exited cleanly, but ${s}` : `exited with code ${t}${s === void 0 ? "" : ` (${s})`}`) + r + (e.stdoutFile === void 0 ? "" : ` — the partial ${L(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
2261
2287
|
stderr: i.trim() === "" ? `(${e.tool} printed nothing to stderr)` : i.slice(-2e3),
|
|
2262
2288
|
...e.hint === void 0 ? {} : { hint: e.hint }
|
|
2263
2289
|
})));
|
|
2264
2290
|
});
|
|
2265
|
-
}),
|
|
2291
|
+
}), Xr = (e) => {
|
|
2266
2292
|
try {
|
|
2267
|
-
return String(
|
|
2293
|
+
return String(O(e).size);
|
|
2268
2294
|
} catch {
|
|
2269
2295
|
return "?";
|
|
2270
2296
|
}
|
|
2271
|
-
},
|
|
2272
|
-
let t = yield*
|
|
2297
|
+
}, Zr = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Qr = (e) => n.gen(function* () {
|
|
2298
|
+
let t = yield* ot(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
2273
2299
|
store: e.store,
|
|
2274
2300
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
2275
|
-
}), r =
|
|
2301
|
+
}), r = nn(t.tables, e.snapshot), i = e.classification ?? De(e.snapshot), a = e.sampleSize ?? 20, o = Be(r.map((t) => ({
|
|
2276
2302
|
name: t,
|
|
2277
2303
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
2278
2304
|
name: e.name,
|
|
@@ -2284,15 +2310,15 @@ var Pr = (e, t) => ({
|
|
|
2284
2310
|
table: i,
|
|
2285
2311
|
chunkSize: a,
|
|
2286
2312
|
...r === void 0 ? {} : { where: r }
|
|
2287
|
-
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
2288
|
-
c.push(...
|
|
2313
|
+
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) => Ue(t, i, o.actions, e.masking.seed));
|
|
2314
|
+
c.push(...He(d, i, o.actions));
|
|
2289
2315
|
let f = u[0], p = d[0], m = [];
|
|
2290
2316
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
2291
2317
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
2292
2318
|
let n = e.slice(i.length + 1);
|
|
2293
2319
|
m.push({
|
|
2294
2320
|
column: n,
|
|
2295
|
-
action:
|
|
2321
|
+
action: Zr(t),
|
|
2296
2322
|
before: f[n],
|
|
2297
2323
|
after: p[n]
|
|
2298
2324
|
});
|
|
@@ -2307,7 +2333,7 @@ var Pr = (e, t) => ({
|
|
|
2307
2333
|
leaks: c,
|
|
2308
2334
|
unclassified: o.unclassified
|
|
2309
2335
|
};
|
|
2310
|
-
}),
|
|
2336
|
+
}), $r = (e, t) => {
|
|
2311
2337
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
2312
2338
|
for (let t of e) {
|
|
2313
2339
|
let e = n.get(t.name);
|
|
@@ -2362,7 +2388,7 @@ var Pr = (e, t) => ({
|
|
|
2362
2388
|
skipColumns: i,
|
|
2363
2389
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
2364
2390
|
};
|
|
2365
|
-
},
|
|
2391
|
+
}, ei = (e) => {
|
|
2366
2392
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
2367
2393
|
let t = {
|
|
2368
2394
|
safe: "✓",
|
|
@@ -2373,4 +2399,4 @@ var Pr = (e, t) => ({
|
|
|
2373
2399
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
2374
2400
|
};
|
|
2375
2401
|
//#endregion
|
|
2376
|
-
export {
|
|
2402
|
+
export { At as ASSETS_DIR, jt as ASSET_INDEX_FILE, H as BundleError, Je as CodecError, Xe as CompressionError, tt as CrossDialectError, kt as DATA_DIR, Ee as DEFAULT_CLASS_ACTIONS, We as ENVIRONMENT_LOCAL_TABLES, Ge as ENVIRONMENT_PORTABLE_TABLES, Gr as EXCLUDED_FROM_DUMP, xt as FORMAT_VERSION, et as ImportModeError, Ye as IntegrityError, Mt as LEDGER_FILE, K as MANIFEST_FILE, $e as MaskingError, W as NativeToolError, Ze as RowsRefusedError, tr as SAVEPOINT_BATCH_SIZE, kn as STAGING_PREFIX, Qe as SchemaDriftError, U as ScopeError, Re as applyAction, rr as asImportError, cr as assessLiveImport, Ht as assetDone, Vr as backupArtifactName, Wr as backupCommand, An as canStage, _n as canonicalType, pt as captureAssetsToCas, bn as checkPortability, Qn as checkSchemaDrift, De as classificationFromSnapshot, $r as classifyImportDrift, vt as compressionExt, rt as computeSubsetIds, Gn as decideStaging, Dt as decodeManifest, Qt as decodeRow, Xt as decodeValue, Fr as decryptStream, sr as defineDataProfile, Zn as diffSnapshots, Sr as dirBlobSink, Pn as dropStagingSql, qr as dumpExcludedTables, zt as emptyLedger, Et as encodeManifest, Zt as encodeRow, Jt as encodeValue, Pr as encryptStream, qe as environmentLocalNotice, Ke as environmentLocalReason, Bn as execStatements, ei as formatImportDrift, xn as formatIssue, gt as hasZstd, V as isEnvironmentLocal, yt as makeCompressor, bt as makeDecompressor, Ue as maskRow, tn as ndjsonToRows, dt as noAssetProgress, gn as outsideReferenceRefusal, hn as outsideReferencesInto, gr as packBundle, br as peekBundle, _t as pickCompression, Be as planMasking, Qr as previewMasking, mt as readAssetIndex, Bt as readLedger, Oe as resolveAction, ot as resolveScope, ut as restoreAssetFromCas, ht as restoreAssetsFromCas, Jr as restoreCommand, en as rowsToNdjson, mn as runExport, or as runImport, Yr as runNativeStep, He as scanForLeaks, X as stagingNameFor, ct as storageAssetSink, st as storageAssetSource, Rr as storageBlobSink, Ir as storageBlobSource, Lr as storageRefsBlobSource, it as subsetToScope, Vt as tableDone, nn as topoSortTables, yr as unpackBundle, xr as unpackBundleToDir, lt as writeAssetToCas, J as writeLedger };
|