@voltro/data-transfer 0.54.0 → 0.56.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +639 -2
- package/dist/index.d.ts +51 -10
- package/dist/index.js +496 -468
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Runtime as o, Schedule as s, Schema as c, Stream as l } from "effect";
|
|
2
|
-
import { DATA_TRANSFERS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d,
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { copyFile as
|
|
6
|
-
import { STORAGE_REFS_TABLE as
|
|
2
|
+
import { DATA_TRANSFERS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d, STAGING_TABLE_PREFIX as f, _voltroDataTransfersTable as p, classifyConstraintViolation as m, derivedRowId as h, ensureTableRegistered as g, eq as _, extractDbCause as v, getTable as y, inSet as b, isNotNull as x, or as S, recordsTable as ee, registerTable as te, streamTable as C, suspendWriteRecorders as ne, suspendWriteRecordersEffect as re, unregisterTable as ie } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as w } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as ae, createDecipheriv as oe, createHash as T, createHmac as se, randomBytes as ce, randomUUID as le, scrypt as ue } from "node:crypto";
|
|
5
|
+
import { copyFile as E, createReadStream as de, createWriteStream as D, existsSync as fe, rename as pe, statSync as O, unlinkSync as me } from "node:fs";
|
|
6
|
+
import { STORAGE_REFS_TABLE as k, fromNodeReadable as he, getObjectStream as A, putObjectStream as ge, toNodeReadable as j } from "@voltro/plugin-storage";
|
|
7
7
|
import * as M from "node:zlib";
|
|
8
|
-
import { PassThrough as N, Readable as
|
|
9
|
-
import { SqlClient as
|
|
10
|
-
import { mkdir as
|
|
11
|
-
import { basename as
|
|
12
|
-
import { promisify as
|
|
13
|
-
import { spawn as
|
|
8
|
+
import { PassThrough as N, Readable as _e } from "node:stream";
|
|
9
|
+
import { SqlClient as P } from "@effect/sql";
|
|
10
|
+
import { mkdir as F, readdir as I, stat as ve } from "node:fs/promises";
|
|
11
|
+
import { basename as L, delimiter as ye, dirname as be, join as R, resolve as xe, sep as Se } from "node:path";
|
|
12
|
+
import { promisify as Ce } from "node:util";
|
|
13
|
+
import { spawn as we } from "node:child_process";
|
|
14
14
|
//#region src/consistency.ts
|
|
15
|
-
var
|
|
15
|
+
var Te = (e) => {
|
|
16
16
|
switch (e) {
|
|
17
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
18
18
|
case "mysql":
|
|
@@ -22,7 +22,7 @@ var we = (e) => {
|
|
|
22
22
|
case "turso": return [];
|
|
23
23
|
default: return [];
|
|
24
24
|
}
|
|
25
|
-
},
|
|
25
|
+
}, Ee = {
|
|
26
26
|
email: { fake: "email" },
|
|
27
27
|
fullName: { fake: "fullName" },
|
|
28
28
|
firstName: { fake: "firstName" },
|
|
@@ -37,7 +37,7 @@ var we = (e) => {
|
|
|
37
37
|
date: "dateShift",
|
|
38
38
|
secret: "null",
|
|
39
39
|
freeText: "redact"
|
|
40
|
-
},
|
|
40
|
+
}, De = (e) => {
|
|
41
41
|
let t = {};
|
|
42
42
|
for (let n of e.tables) {
|
|
43
43
|
let e = {};
|
|
@@ -49,15 +49,15 @@ var we = (e) => {
|
|
|
49
49
|
t[n.name] = e;
|
|
50
50
|
}
|
|
51
51
|
return t;
|
|
52
|
-
},
|
|
52
|
+
}, Oe = (e, t, n, r) => {
|
|
53
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
54
54
|
if (i) return i;
|
|
55
55
|
if (n?.sensitive) {
|
|
56
56
|
let e = n.sensitive.class;
|
|
57
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? Ee[e] ?? "redact";
|
|
58
58
|
}
|
|
59
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
60
|
-
},
|
|
60
|
+
}, ke = (e, t) => se("sha256", e).update(t == null ? "∅" : String(t)).digest(), z = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), Ae = [
|
|
61
61
|
"Ada",
|
|
62
62
|
"Alan",
|
|
63
63
|
"Grace",
|
|
@@ -70,7 +70,7 @@ var we = (e) => {
|
|
|
70
70
|
"Guido",
|
|
71
71
|
"Margaret",
|
|
72
72
|
"Bjarne"
|
|
73
|
-
],
|
|
73
|
+
], je = [
|
|
74
74
|
"Lovelace",
|
|
75
75
|
"Turing",
|
|
76
76
|
"Hopper",
|
|
@@ -83,7 +83,7 @@ var we = (e) => {
|
|
|
83
83
|
"Rossum",
|
|
84
84
|
"Hamilton",
|
|
85
85
|
"Stroustrup"
|
|
86
|
-
],
|
|
86
|
+
], Me = [
|
|
87
87
|
"Maple",
|
|
88
88
|
"Oak",
|
|
89
89
|
"Cedar",
|
|
@@ -92,7 +92,7 @@ var we = (e) => {
|
|
|
92
92
|
"Birch",
|
|
93
93
|
"Willow",
|
|
94
94
|
"Ash"
|
|
95
|
-
],
|
|
95
|
+
], Ne = [
|
|
96
96
|
"Initech",
|
|
97
97
|
"Hooli",
|
|
98
98
|
"Umbrella",
|
|
@@ -101,23 +101,23 @@ var we = (e) => {
|
|
|
101
101
|
"Soylent",
|
|
102
102
|
"Stark",
|
|
103
103
|
"Wayne"
|
|
104
|
-
], B = (e, t, n) => e[z(t, n) % e.length],
|
|
105
|
-
let r =
|
|
104
|
+
], B = (e, t, n) => e[z(t, n) % e.length], Pe = (e, t, n) => {
|
|
105
|
+
let r = ke(n, `${e}:${String(t)}`), i = z(r, 1) % 1e4;
|
|
106
106
|
switch (e) {
|
|
107
|
-
case "email": return `${B(
|
|
108
|
-
case "fullName": return `${B(
|
|
109
|
-
case "firstName": return B(
|
|
110
|
-
case "lastName": return B(
|
|
111
|
-
case "username": return `${B(
|
|
107
|
+
case "email": return `${B(Ae, r, 0).toLowerCase()}.${B(je, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${B(Ae, r, 0)} ${B(je, r, 1)}`;
|
|
109
|
+
case "firstName": return B(Ae, r, 0);
|
|
110
|
+
case "lastName": return B(je, r, 1);
|
|
111
|
+
case "username": return `${B(Ae, r, 0).toLowerCase()}${i}`;
|
|
112
112
|
case "phone": return `+1${String(z(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
-
case "address": return `${z(r, 0) % 9899 + 100} ${B(
|
|
114
|
-
case "company": return B(
|
|
113
|
+
case "address": return `${z(r, 0) % 9899 + 100} ${B(Me, r, 1)} St`;
|
|
114
|
+
case "company": return B(Ne, r, 0);
|
|
115
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
116
116
|
case "ip": return `${z(r, 0) % 256}.${z(r, 1) % 256}.${z(r, 2) % 256}.${z(r, 3) % 254 + 1}`;
|
|
117
|
-
case "creditCard": return
|
|
117
|
+
case "creditCard": return Fe(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
118
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
119
119
|
}
|
|
120
|
-
},
|
|
120
|
+
}, Fe = (e) => {
|
|
121
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
122
122
|
t.forEach((e, t) => {
|
|
123
123
|
let r = e;
|
|
@@ -125,31 +125,31 @@ var we = (e) => {
|
|
|
125
125
|
});
|
|
126
126
|
let r = (10 - n % 10) % 10;
|
|
127
127
|
return e.slice(0, 15) + String(r);
|
|
128
|
-
},
|
|
128
|
+
}, Ie = 864e5, Le = (e, t) => {
|
|
129
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
130
130
|
if (Number.isNaN(n)) return e;
|
|
131
|
-
let r = z(
|
|
131
|
+
let r = z(ke(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Ie);
|
|
132
132
|
return e instanceof Date ? i : i.toISOString();
|
|
133
|
-
},
|
|
133
|
+
}, Re = (e, t) => {
|
|
134
134
|
if (t.value === null || t.value === void 0) return null;
|
|
135
135
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
136
136
|
if (e === "keep") return t.value;
|
|
137
137
|
if (e === "null") return null;
|
|
138
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
139
|
-
if (e === "hash") return
|
|
140
|
-
if (e === "dateShift") return
|
|
141
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return ke(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Le(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return Pe(e.fake, t.value, t.seed);
|
|
142
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
143
143
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
144
|
-
},
|
|
144
|
+
}, ze = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Be = (e, t, n) => {
|
|
145
145
|
let r = {}, i = [], a = [];
|
|
146
146
|
for (let o of e) for (let e of o.columns) {
|
|
147
|
-
let s = `${o.name}.${e.name}`, c =
|
|
147
|
+
let s = `${o.name}.${e.name}`, c = Oe(o.name, e.name, t[o.name]?.[e.name], n);
|
|
148
148
|
if (c === "error") {
|
|
149
149
|
i.push(s);
|
|
150
150
|
continue;
|
|
151
151
|
}
|
|
152
|
-
if (!
|
|
152
|
+
if (!ze(c)) {
|
|
153
153
|
a.push({
|
|
154
154
|
column: s,
|
|
155
155
|
action: c
|
|
@@ -163,7 +163,7 @@ var we = (e) => {
|
|
|
163
163
|
unclassified: i,
|
|
164
164
|
invalid: a
|
|
165
165
|
};
|
|
166
|
-
},
|
|
166
|
+
}, Ve = [
|
|
167
167
|
{
|
|
168
168
|
name: "email",
|
|
169
169
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -180,7 +180,7 @@ var we = (e) => {
|
|
|
180
180
|
name: "creditCard",
|
|
181
181
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
182
182
|
}
|
|
183
|
-
],
|
|
183
|
+
], He = (e, t, n) => {
|
|
184
184
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
185
185
|
for (let [a, o] of Object.entries(n)) {
|
|
186
186
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -188,7 +188,7 @@ var we = (e) => {
|
|
|
188
188
|
for (let a of e) {
|
|
189
189
|
let e = a[n];
|
|
190
190
|
if (typeof e != "string" || e.length === 0) continue;
|
|
191
|
-
let o =
|
|
191
|
+
let o = Ve.find((t) => t.re.test(e));
|
|
192
192
|
if (o && !i.has(n)) {
|
|
193
193
|
i.add(n), r.push({
|
|
194
194
|
table: t,
|
|
@@ -201,9 +201,9 @@ var we = (e) => {
|
|
|
201
201
|
}
|
|
202
202
|
}
|
|
203
203
|
return r;
|
|
204
|
-
},
|
|
204
|
+
}, Ue = (e, t, n, r, i) => {
|
|
205
205
|
let a = {};
|
|
206
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
206
|
+
for (let [o, s] of Object.entries(e)) a[o] = Re(n[`${t}.${o}`] ?? "keep", {
|
|
207
207
|
value: s,
|
|
208
208
|
table: t,
|
|
209
209
|
column: o,
|
|
@@ -211,13 +211,15 @@ var we = (e) => {
|
|
|
211
211
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
212
212
|
});
|
|
213
213
|
return a;
|
|
214
|
-
},
|
|
214
|
+
}, We = /* @__PURE__ */ new Map([
|
|
215
215
|
["_voltro_migration_plans", "the schema fingerprint THIS deployment applied — a foreign row makes the next boot refuse"],
|
|
216
216
|
["_voltro_migrations", "which file-based migrations ran HERE — a foreign row marks unrun migrations as done"],
|
|
217
217
|
["_voltro_seeds", "which seeds ran HERE — a foreign row suppresses seeding the target still needs"],
|
|
218
218
|
["_voltro_cdc_offsets", "binlog/WAL positions of the SOURCE server — meaningless and unsafe against another one"],
|
|
219
219
|
["_voltro_notification_push_subscriptions", "browser push endpoints bound to THIS deployment's VAPID key — the browser's applicationServerKey is derived from `VOLTRO_VAPID_PRIVATE_KEY`, so a push service refuses a delivery signed by another one. A foreign row makes the target attempt a delivery that must fail, and the auto-prune then deletes a subscription that was valid where it came from"],
|
|
220
220
|
["_voltro_schedule_claims", "live leases held by THIS deployment's replicas — a foreign row hands out a lease nobody holds"],
|
|
221
|
+
["_voltro_change_claims", "the record that ONE of THIS deployment's replicas already handled a given change — a foreign row asserts that an effect fired here when it fired somewhere else, so the subscriber or reaction it gates stands down for a notification nobody sent"],
|
|
222
|
+
["_voltro_replica_observations", "what THIS deployment's replicas last published about themselves — a foreign row invents a replica that does not exist here, and a fleet answer would then report it as present or, once it ages, as a replica that has stopped answering"],
|
|
221
223
|
["_voltro_data_transfers", "which bundles moved through HERE, in or out — a foreign row claims a transfer this deployment never ran"],
|
|
222
224
|
["_voltro_replace_in_progress", "the marker a destructive import HERE left behind — a foreign row refuses a boot over a healthy database"],
|
|
223
225
|
["_voltro_wakeups", "pending wakeups addressed to THIS deployment's runners"],
|
|
@@ -244,7 +246,7 @@ var we = (e) => {
|
|
|
244
246
|
["_voltro_notification_held", "notifications waiting to be SENT from here — a foreign row makes the target deliver something queued elsewhere"],
|
|
245
247
|
["_voltro_webhook_event_stats", "per-deployment delivery counters a backoff decision reads — foreign counts steer the target's retries"],
|
|
246
248
|
["_voltro_webhook_rate_windows", "the live rate-limit windows this deployment is inside — a foreign window throttles or releases wrongly"]
|
|
247
|
-
]),
|
|
249
|
+
]), Ge = /* @__PURE__ */ new Map([
|
|
248
250
|
["_voltro_agent_threads", "conversations the app's users had — app data"],
|
|
249
251
|
["_voltro_agent_messages", "the messages in those conversations — app data"],
|
|
250
252
|
["_voltro_prompts", "prompt definitions the app authored — app data"],
|
|
@@ -279,21 +281,21 @@ var we = (e) => {
|
|
|
279
281
|
["_voltro_comment_threads", "the threads those comments hang on — app data"],
|
|
280
282
|
["_voltro_comment_reactions", "the reactions the app's users left — app data"],
|
|
281
283
|
["_voltro_comment_reads", "which comments each user has read — app data, and losing it re-marks everything unread"]
|
|
282
|
-
]), V = (e) =>
|
|
284
|
+
]), V = (e) => We.has(e), Ke = (e) => We.get(e), qe = (e, t) => {
|
|
283
285
|
if (e.length === 0) return;
|
|
284
|
-
let n = e.map((e) => ` ${e} — ${
|
|
286
|
+
let n = e.map((e) => ` ${e} — ${Ke(e) ?? "describes this deployment"}`);
|
|
285
287
|
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
286
288
|
}, H = class extends c.TaggedError()("BundleError", {
|
|
287
289
|
reason: c.String,
|
|
288
290
|
path: c.optional(c.String)
|
|
289
|
-
}) {},
|
|
291
|
+
}) {}, Je = class extends c.TaggedError()("CodecError", {
|
|
290
292
|
table: c.String,
|
|
291
293
|
reason: c.String
|
|
292
|
-
}) {},
|
|
294
|
+
}) {}, Ye = class extends c.TaggedError()("IntegrityError", {
|
|
293
295
|
what: c.String,
|
|
294
296
|
expected: c.String,
|
|
295
297
|
actual: c.String
|
|
296
|
-
}) {},
|
|
298
|
+
}) {}, Xe = class extends t.TaggedError("CompressionError") {}, U = class extends c.TaggedError()("ScopeError", { reason: c.String }) {}, Ze = class extends c.TaggedError()("RowsRefusedError", {
|
|
297
299
|
rows: c.Array(c.Struct({
|
|
298
300
|
table: c.String,
|
|
299
301
|
id: c.String,
|
|
@@ -307,7 +309,7 @@ var we = (e) => {
|
|
|
307
309
|
refused: c.Number,
|
|
308
310
|
primary: c.Number
|
|
309
311
|
}))
|
|
310
|
-
}) {},
|
|
312
|
+
}) {}, Qe = class extends c.TaggedError()("SchemaDriftError", {
|
|
311
313
|
bundleFingerprint: c.String,
|
|
312
314
|
targetFingerprint: c.String,
|
|
313
315
|
diff: c.Array(c.String)
|
|
@@ -316,20 +318,20 @@ var we = (e) => {
|
|
|
316
318
|
let e = this.stderr?.trim(), t = this.hint?.trim();
|
|
317
319
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}${t ? `\n\nhint: ${t}` : ""}`;
|
|
318
320
|
}
|
|
319
|
-
},
|
|
321
|
+
}, $e = class extends c.TaggedError()("MaskingError", {
|
|
320
322
|
unclassified: c.Array(c.String),
|
|
321
323
|
invalidActions: c.optional(c.Array(c.String))
|
|
322
|
-
}) {},
|
|
324
|
+
}) {}, et = class extends c.TaggedError()("ImportModeError", {
|
|
323
325
|
mode: c.String,
|
|
324
326
|
reason: c.String
|
|
325
|
-
}) {},
|
|
327
|
+
}) {}, tt = class extends c.TaggedError()("CrossDialectError", {
|
|
326
328
|
sourceDialect: c.String,
|
|
327
329
|
targetDialect: c.String,
|
|
328
330
|
blocking: c.Array(c.String)
|
|
329
|
-
}) {}, G = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id",
|
|
331
|
+
}) {}, G = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", nt = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
330
332
|
col: e.name,
|
|
331
333
|
refTable: e.references.table
|
|
332
|
-
})),
|
|
334
|
+
})), rt = (e, t, r) => n.gen(function* () {
|
|
333
335
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), u = (e) => {
|
|
334
336
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
335
337
|
return s.set(e, t), t;
|
|
@@ -347,7 +349,7 @@ var we = (e) => {
|
|
|
347
349
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
348
350
|
};
|
|
349
351
|
}, p = (r, i) => {
|
|
350
|
-
let a = f(r, G(t, r),
|
|
352
|
+
let a = f(r, G(t, r), nt(t, r));
|
|
351
353
|
return l.runForEach(C(e, {
|
|
352
354
|
table: r,
|
|
353
355
|
pk: G(t, r),
|
|
@@ -359,7 +361,7 @@ var we = (e) => {
|
|
|
359
361
|
c.clear();
|
|
360
362
|
for (let [n, r] of e) {
|
|
361
363
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
362
|
-
e.length !== 0 && (yield* p(n,
|
|
364
|
+
e.length !== 0 && (yield* p(n, b(G(t, n), e)));
|
|
363
365
|
}
|
|
364
366
|
}
|
|
365
367
|
});
|
|
@@ -374,21 +376,21 @@ var we = (e) => {
|
|
|
374
376
|
refTable: e.references?.table
|
|
375
377
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
376
378
|
if (a.length === 0) continue;
|
|
377
|
-
let o = s.get(t)?.size ?? 0, c = a.map((e) =>
|
|
378
|
-
yield* p(t, c.length === 1 ? c[0] :
|
|
379
|
+
let o = s.get(t)?.size ?? 0, c = a.map((e) => b(e.col, [...s.get(e.refTable)]));
|
|
380
|
+
yield* p(t, c.length === 1 ? c[0] : S(...c)), (s.get(t)?.size ?? 0) > o && !e.has(t) && r.push(t), (s.get(t)?.size ?? 0) > o && (n = !0);
|
|
379
381
|
}
|
|
380
382
|
for (let t of r) e.add(t);
|
|
381
383
|
if (yield* m, !n) break;
|
|
382
384
|
}
|
|
383
385
|
}
|
|
384
386
|
return o;
|
|
385
|
-
}),
|
|
387
|
+
}), it = (e, t) => ({
|
|
386
388
|
tables: [...e.keys()],
|
|
387
389
|
predicateFor: (n) => {
|
|
388
390
|
let r = e.get(n);
|
|
389
|
-
return r ?
|
|
391
|
+
return r ? b(G(t, n), [...r]) : void 0;
|
|
390
392
|
}
|
|
391
|
-
}),
|
|
393
|
+
}), at = "tenantId", ot = (e, t, r) => n.gen(function* () {
|
|
392
394
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
393
395
|
if (e.kind === "all") {
|
|
394
396
|
let t = new Set(e.exclude ?? []);
|
|
@@ -408,29 +410,29 @@ var we = (e) => {
|
|
|
408
410
|
}
|
|
409
411
|
let o = e.tenantId;
|
|
410
412
|
if (o === void 0 || o === "") return yield* n.fail(new U({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
411
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
412
|
-
if (s.length === 0) return yield* n.fail(new U({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${
|
|
413
|
-
let c = yield*
|
|
414
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
413
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === at) ?? !1);
|
|
414
|
+
if (s.length === 0) return yield* n.fail(new U({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${at}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
415
|
+
let c = yield* rt(r.store, t, {
|
|
416
|
+
seeds: Object.fromEntries(s.map((e) => [e, _(at, o)])),
|
|
415
417
|
pinned: s,
|
|
416
418
|
children: { roots: s }
|
|
417
419
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
418
420
|
return {
|
|
419
421
|
tables: [...s, ...l],
|
|
420
|
-
predicateFor: (e) => u.has(e) ?
|
|
422
|
+
predicateFor: (e) => u.has(e) ? _(at, o) : c.has(e) ? b(G(t, e), [...c.get(e)]) : void 0
|
|
421
423
|
};
|
|
422
|
-
}),
|
|
424
|
+
}), st = (e, t) => ({
|
|
423
425
|
list: t,
|
|
424
426
|
open: (t) => A(e, t)
|
|
425
|
-
}),
|
|
426
|
-
write: (t, n, r) =>
|
|
427
|
+
}), ct = (e) => ({
|
|
428
|
+
write: (t, n, r) => ge(e, t, n, r),
|
|
427
429
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
428
|
-
}),
|
|
430
|
+
}), lt = (e, t) => n.tryPromise({
|
|
429
431
|
try: async () => {
|
|
430
432
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
431
433
|
u.update(e), d += e.length, n(null, e);
|
|
432
434
|
} });
|
|
433
|
-
await o(await
|
|
435
|
+
await o(await j(e.stream), f, n(l));
|
|
434
436
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
435
437
|
try {
|
|
436
438
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -447,26 +449,26 @@ var we = (e) => {
|
|
|
447
449
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
448
450
|
path: t
|
|
449
451
|
})
|
|
450
|
-
}),
|
|
451
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u =
|
|
452
|
+
}), ut = (e, t, r) => n.gen(function* () {
|
|
453
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u = he("bundle", () => a(s)).pipe(l.tap((e) => n.sync(() => c.update(e))));
|
|
452
454
|
yield* e.write(r.key, u, {
|
|
453
455
|
contentType: r.contentType,
|
|
454
456
|
totalSize: r.size
|
|
455
457
|
});
|
|
456
458
|
let d = c.digest("hex");
|
|
457
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
459
|
+
if (d !== r.sha256) return yield* n.fail(new Ye({
|
|
458
460
|
what: `asset ${r.key}`,
|
|
459
461
|
expected: r.sha256,
|
|
460
462
|
actual: d
|
|
461
463
|
}));
|
|
462
|
-
}),
|
|
464
|
+
}), dt = {
|
|
463
465
|
done: () => !1,
|
|
464
466
|
mark: () => n.void
|
|
465
|
-
},
|
|
466
|
-
let t = e.progress ??
|
|
467
|
-
return yield* l.runForEach(e.source.list(), (l) => (a++, t.done(l.key) || c.has(l.key) ? n.void : (c.add(l.key), e.source.open(l.key).pipe(n.map((e) => e), n.catchIf(
|
|
467
|
+
}, ft = (e) => typeof e == "object" && !!e && e.status === 404, pt = (e) => n.gen(function* () {
|
|
468
|
+
let t = e.progress ?? dt, r = 0, i = 0, a = 0, o = 0, s = /* @__PURE__ */ new Set(), c = /* @__PURE__ */ new Set(), u = [];
|
|
469
|
+
return yield* l.runForEach(e.source.list(), (l) => (a++, t.done(l.key) || c.has(l.key) ? n.void : (c.add(l.key), e.source.open(l.key).pipe(n.map((e) => e), n.catchIf(ft, () => n.sync(() => {
|
|
468
470
|
u.push(l.key);
|
|
469
|
-
})), n.flatMap((a) => a === void 0 ? n.void :
|
|
471
|
+
})), n.flatMap((a) => a === void 0 ? n.void : lt(a, e.assetsDir).pipe(n.flatMap((e) => {
|
|
470
472
|
let n = {
|
|
471
473
|
key: l.key,
|
|
472
474
|
sha256: e.sha256,
|
|
@@ -482,18 +484,18 @@ var we = (e) => {
|
|
|
482
484
|
objectBytes: o,
|
|
483
485
|
missing: u
|
|
484
486
|
};
|
|
485
|
-
}),
|
|
487
|
+
}), mt = (e) => he("bundle", () => de(e)).pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.map((e) => JSON.parse(e)), l.mapError((t) => t instanceof H ? t : new H({
|
|
486
488
|
reason: `asset index unreadable: ${String(t)}`,
|
|
487
489
|
path: e
|
|
488
|
-
}))),
|
|
489
|
-
let t = e.progress ??
|
|
490
|
-
return yield* l.runForEach(
|
|
490
|
+
}))), ht = (e) => n.gen(function* () {
|
|
491
|
+
let t = e.progress ?? dt, r = 0, i = /* @__PURE__ */ new Set(), a = /* @__PURE__ */ new Set();
|
|
492
|
+
return yield* l.runForEach(mt(e.indexPath), (o) => t.done(o.key) || a.has(o.key) ? n.void : (a.add(o.key), ut(e.sink, e.assetsDir, o)).pipe(n.andThen(n.sync(() => {
|
|
491
493
|
r++, i.add(o.sha256);
|
|
492
494
|
})), n.andThen(t.mark(o.key, o)))), {
|
|
493
495
|
count: r,
|
|
494
496
|
objects: i.size
|
|
495
497
|
};
|
|
496
|
-
}),
|
|
498
|
+
}), gt = () => typeof M.createZstdCompress == "function", _t = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : gt() ? "zstd" : "gzip", vt = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", yt = (e) => e === "zstd" ? M.createZstdCompress() : e === "gzip" ? M.createGzip() : new N(), bt = (e) => e === "zstd" ? M.createZstdDecompress() : e === "gzip" ? M.createGunzip() : new N(), xt = 1, St = c.Struct({
|
|
497
499
|
name: c.String,
|
|
498
500
|
file: c.String,
|
|
499
501
|
rowCount: c.Number,
|
|
@@ -503,12 +505,12 @@ var we = (e) => {
|
|
|
503
505
|
value: c.String
|
|
504
506
|
}),
|
|
505
507
|
primaryKey: c.String
|
|
506
|
-
}),
|
|
508
|
+
}), Ct = c.Struct({
|
|
507
509
|
kind: c.Literal("all", "tenant", "tables"),
|
|
508
510
|
tenantId: c.optional(c.String),
|
|
509
511
|
tables: c.optional(c.Array(c.String)),
|
|
510
512
|
exclude: c.optional(c.Array(c.String))
|
|
511
|
-
}),
|
|
513
|
+
}), wt = c.Struct({
|
|
512
514
|
count: c.Number,
|
|
513
515
|
totalBytes: c.Number,
|
|
514
516
|
index: c.optional(c.String)
|
|
@@ -519,23 +521,23 @@ c.Struct({
|
|
|
519
521
|
size: c.Number,
|
|
520
522
|
contentType: c.String
|
|
521
523
|
});
|
|
522
|
-
var
|
|
524
|
+
var Tt = c.Struct({
|
|
523
525
|
formatVersion: c.Number,
|
|
524
526
|
createdAt: c.String,
|
|
525
527
|
source: c.Struct({
|
|
526
528
|
dialect: c.String,
|
|
527
529
|
schemaFingerprint: c.String
|
|
528
530
|
}),
|
|
529
|
-
scope:
|
|
531
|
+
scope: Ct,
|
|
530
532
|
consistency: c.Literal("live", "snapshot"),
|
|
531
533
|
compression: c.Literal("zstd", "gzip", "none"),
|
|
532
|
-
tables: c.Array(
|
|
533
|
-
assets:
|
|
534
|
+
tables: c.Array(St),
|
|
535
|
+
assets: wt,
|
|
534
536
|
masking: c.optional(c.Struct({
|
|
535
537
|
policyId: c.optional(c.String),
|
|
536
538
|
transformed: c.Array(c.String)
|
|
537
539
|
}))
|
|
538
|
-
}),
|
|
540
|
+
}), Et = c.encodeSync(c.parseJson(Tt)), Dt = c.decodeUnknownSync(c.parseJson(Tt)), K = "manifest.json", Ot = "README.md", kt = "data", At = "assets", jt = "assets/index.ndjson", Mt = ".ledger.json", Nt = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", Pt = (e, t) => {
|
|
539
541
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
540
542
|
return `# Voltro data export
|
|
541
543
|
|
|
@@ -549,7 +551,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
549
551
|
| **Format version** | ${e.formatVersion} |
|
|
550
552
|
| **Consistency** | ${e.consistency} |
|
|
551
553
|
| **Compression** | ${e.compression} |
|
|
552
|
-
| **Scope** | ${
|
|
554
|
+
| **Scope** | ${Nt(e.scope)} |
|
|
553
555
|
| **Tables** | ${e.tables.length} |
|
|
554
556
|
| **Rows (total)** | ${n} |
|
|
555
557
|
| **Assets** | ${a} |
|
|
@@ -574,26 +576,26 @@ voltro data import <this-directory>
|
|
|
574
576
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
575
577
|
\`\`\`
|
|
576
578
|
`;
|
|
577
|
-
},
|
|
579
|
+
}, Ft = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), It = a.exponential({
|
|
578
580
|
start: .01,
|
|
579
581
|
factor: 2,
|
|
580
582
|
count: 16
|
|
581
|
-
}),
|
|
583
|
+
}), Lt = i.histogram("voltro_data_transfer_table_seconds", It, "Per-table export/import duration in seconds, tagged by phase."), q = (e, t) => {
|
|
582
584
|
if (e) try {
|
|
583
585
|
e(t);
|
|
584
586
|
} catch {}
|
|
585
|
-
},
|
|
587
|
+
}, Rt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(Lt.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), Ft.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
586
588
|
table: t,
|
|
587
589
|
index: r,
|
|
588
590
|
tableCount: a,
|
|
589
591
|
phase: e
|
|
590
|
-
} })),
|
|
592
|
+
} })), zt = () => ({
|
|
591
593
|
tables: {},
|
|
592
594
|
assets: {}
|
|
593
|
-
}),
|
|
595
|
+
}), Bt = (e) => n.tryPromise(async () => {
|
|
594
596
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
595
597
|
return JSON.parse(t);
|
|
596
|
-
}).pipe(n.orElseSucceed(
|
|
598
|
+
}).pipe(n.orElseSucceed(zt)), J = (e, t) => n.tryPromise({
|
|
597
599
|
try: async () => {
|
|
598
600
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
599
601
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
@@ -602,11 +604,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
602
604
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
603
605
|
path: e
|
|
604
606
|
})
|
|
605
|
-
}),
|
|
607
|
+
}), Vt = (e, t) => t in e.tables, Ht = (e, t) => t in e.assets, Ut = /* @__PURE__ */ new Set([
|
|
606
608
|
"json",
|
|
607
609
|
"array",
|
|
608
610
|
"vector"
|
|
609
|
-
]),
|
|
611
|
+
]), Wt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", Gt = (e) => Buffer.from(e).toString("base64"), Kt = (e) => new Uint8Array(Buffer.from(e, "base64")), qt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), Jt = (e, t) => e == null ? null : t !== void 0 && Ut.has(t) ? e : t === "bytes" ? Gt(e) : t === "timestamp" || t === "date" ? qt(e) : typeof e == "bigint" ? {
|
|
610
612
|
$v: "bigint",
|
|
611
613
|
d: e.toString()
|
|
612
614
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -617,30 +619,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
617
619
|
d: e.toISOString()
|
|
618
620
|
} : e instanceof Uint8Array ? {
|
|
619
621
|
$v: "bytes",
|
|
620
|
-
d:
|
|
621
|
-
} : e,
|
|
622
|
+
d: Gt(e)
|
|
623
|
+
} : e, Yt = (e) => {
|
|
622
624
|
switch (e.$v) {
|
|
623
625
|
case "bigint": return BigInt(e.d);
|
|
624
626
|
case "number": return Number(e.d);
|
|
625
627
|
case "date": return new Date(e.d);
|
|
626
|
-
case "bytes": return
|
|
628
|
+
case "bytes": return Kt(e.d);
|
|
627
629
|
default: return e;
|
|
628
630
|
}
|
|
629
|
-
},
|
|
631
|
+
}, Xt = (e, t) => e == null ? null : t !== void 0 && Ut.has(t) ? e : Wt(e) ? Yt(e) : t === "bytes" ? Kt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Zt = (e, t) => {
|
|
630
632
|
let n = {};
|
|
631
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
633
|
+
for (let [r, i] of Object.entries(e)) n[r] = Jt(i, t[r]);
|
|
632
634
|
return n;
|
|
633
|
-
},
|
|
635
|
+
}, Qt = (e, t) => {
|
|
634
636
|
let n = {};
|
|
635
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
637
|
+
for (let [r, i] of Object.entries(e)) n[r] = Xt(i, t[r]);
|
|
636
638
|
return n;
|
|
637
|
-
},
|
|
638
|
-
try: () =>
|
|
639
|
-
catch: (e) => new
|
|
639
|
+
}, $t = new TextEncoder(), en = (e, t) => e.pipe(l.map((e) => $t.encode(`${JSON.stringify(Zt(e, t))}\n`))), tn = (e, t, r) => e.pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.mapEffect((e) => n.try({
|
|
640
|
+
try: () => Qt(JSON.parse(e), r),
|
|
641
|
+
catch: (e) => new Je({
|
|
640
642
|
table: t,
|
|
641
643
|
reason: String(e?.message ?? e)
|
|
642
644
|
})
|
|
643
|
-
}))),
|
|
645
|
+
}))), nn = (e, t) => {
|
|
644
646
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
645
647
|
for (let e of t.tables) {
|
|
646
648
|
if (!n.has(e.name)) continue;
|
|
@@ -666,33 +668,33 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
666
668
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
667
669
|
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(Y(t.cause)) : n.push(r), n.length === 0) {
|
|
668
670
|
let t = String(e ?? "").trim();
|
|
669
|
-
return t === "" || t === "[object Object]" ?
|
|
671
|
+
return t === "" || t === "[object Object]" ? rn(e) : t;
|
|
670
672
|
}
|
|
671
673
|
return n.join(": ");
|
|
672
|
-
},
|
|
674
|
+
}, rn = (e) => {
|
|
673
675
|
try {
|
|
674
676
|
let t = JSON.stringify(e);
|
|
675
677
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
676
678
|
} catch {
|
|
677
679
|
return "unknown error (not serialisable)";
|
|
678
680
|
}
|
|
679
|
-
},
|
|
681
|
+
}, an = (e, t) => {
|
|
680
682
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
681
683
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
682
684
|
return r;
|
|
683
|
-
},
|
|
685
|
+
}, on = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), sn = (e, t) => {
|
|
684
686
|
if (t.length === 0) return e;
|
|
685
687
|
let n = { ...e };
|
|
686
688
|
for (let e of t) delete n[e];
|
|
687
689
|
return n;
|
|
688
|
-
},
|
|
690
|
+
}, cn = (e, t) => {
|
|
689
691
|
let n = e.tables.find((e) => e.name === t);
|
|
690
692
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
691
693
|
let r = n.columns.find((e) => e.type === "id");
|
|
692
694
|
if (r !== void 0) return { column: r.name };
|
|
693
695
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
694
696
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
695
|
-
},
|
|
697
|
+
}, ln = (e, t, r) => n.tryPromise({
|
|
696
698
|
try: async () => {
|
|
697
699
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
698
700
|
d.update(e);
|
|
@@ -701,7 +703,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
701
703
|
n(null, e);
|
|
702
704
|
} }), m = c.fromWeb(l.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
703
705
|
try {
|
|
704
|
-
await s(m, p,
|
|
706
|
+
await s(m, p, yt(r), n(h)), await i.rename(h, t);
|
|
705
707
|
} catch (e) {
|
|
706
708
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
707
709
|
}
|
|
@@ -714,7 +716,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
714
716
|
reason: `write table failed: ${Y(e)}`,
|
|
715
717
|
path: t
|
|
716
718
|
})
|
|
717
|
-
}),
|
|
719
|
+
}), un = (e, t) => n.tryPromise({
|
|
718
720
|
try: async () => {
|
|
719
721
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
720
722
|
},
|
|
@@ -722,13 +724,13 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
722
724
|
reason: `append failed: ${Y(t)}`,
|
|
723
725
|
path: e
|
|
724
726
|
})
|
|
725
|
-
}),
|
|
727
|
+
}), dn = (e, t) => n.gen(function* () {
|
|
726
728
|
let r = [], i = t.ordered.length;
|
|
727
729
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
728
|
-
let o = t.ordered[a], s =
|
|
730
|
+
let o = t.ordered[a], s = an(t.snapshot, o), c = cn(t.snapshot, o);
|
|
729
731
|
if ("refusal" in c) return yield* n.fail(new U({ reason: `export: ${c.refusal}` }));
|
|
730
|
-
let u = c.column, d = `${
|
|
731
|
-
if (
|
|
732
|
+
let u = c.column, d = `${kt}/${o}.ndjson${vt(t.compression)}`;
|
|
733
|
+
if (Vt(t.ledger, o)) {
|
|
732
734
|
let e = t.ledger.tables[o];
|
|
733
735
|
q(t.onProgress, {
|
|
734
736
|
phase: "export",
|
|
@@ -762,7 +764,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
762
764
|
chunkSize: t.chunkSize,
|
|
763
765
|
retry: t.retry,
|
|
764
766
|
...f === void 0 ? {} : { where: f }
|
|
765
|
-
}), m =
|
|
767
|
+
}), m = on(t.snapshot, o), h = m.length === 0 ? p : p.pipe(l.map((e) => sn(e, m))), g = t.mask ? h.pipe(l.map((e) => Ue(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* Rt("export", o, a, i, (e) => e.rowCount, ln(en(g, s), t.bundleFile(d), t.compression));
|
|
766
768
|
t.ledger.tables[o] = {
|
|
767
769
|
rowCount: _,
|
|
768
770
|
checksum: v
|
|
@@ -784,43 +786,43 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
784
786
|
});
|
|
785
787
|
}
|
|
786
788
|
return r;
|
|
787
|
-
}),
|
|
789
|
+
}), fn = (e) => ({
|
|
788
790
|
_tag: "RawSqlFragment",
|
|
789
791
|
strings: [e],
|
|
790
792
|
values: []
|
|
791
|
-
}),
|
|
793
|
+
}), pn = (t, i, a) => n.tryPromise({
|
|
792
794
|
try: () => t.transactional(async (t) => {
|
|
793
|
-
if (t.raw) for (let e of
|
|
795
|
+
if (t.raw) for (let e of Te(i)) await t.raw(fn(e));
|
|
794
796
|
let o = await n.runPromiseExit(a(t));
|
|
795
797
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
796
798
|
return o.value;
|
|
797
799
|
}),
|
|
798
800
|
catch: (e) => e instanceof H ? e : new H({ reason: `snapshot export failed: ${Y(e)}` })
|
|
799
|
-
}),
|
|
800
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
801
|
+
}), mn = (e) => n.gen(function* () {
|
|
802
|
+
let t = e.scope ?? { kind: "all" }, r = _t(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, o = s.exponential("50 millis").pipe(s.jittered, s.intersect(s.recurs(a))), c = yield* n.promise(() => import("node:path")), l = (t) => c.join(e.outDir, t);
|
|
801
803
|
yield* n.tryPromise({
|
|
802
804
|
try: async () => {
|
|
803
805
|
let t = await import("node:fs/promises");
|
|
804
|
-
await t.mkdir(l(
|
|
806
|
+
await t.mkdir(l(kt), { recursive: !0 }), e.assets && await t.mkdir(l(At), { recursive: !0 });
|
|
805
807
|
},
|
|
806
808
|
catch: (t) => new H({
|
|
807
809
|
reason: `mkdir failed: ${Y(t)}`,
|
|
808
810
|
path: e.outDir
|
|
809
811
|
})
|
|
810
812
|
});
|
|
811
|
-
let u = l(
|
|
813
|
+
let u = l(Mt), d = yield* Bt(u), f = e.subset ? it(yield* rt(e.store, e.snapshot, e.subset), e.snapshot) : yield* ot(t, e.snapshot, {
|
|
812
814
|
store: e.store,
|
|
813
815
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
814
|
-
}), p =
|
|
816
|
+
}), p = nn(f.tables, e.snapshot), m = e.consistency ?? "live", h, g;
|
|
815
817
|
if (e.masking) {
|
|
816
|
-
let t = e.classification ??
|
|
818
|
+
let t = e.classification ?? De(e.snapshot), r = Be(p.map((t) => ({
|
|
817
819
|
name: t,
|
|
818
820
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
819
821
|
name: e.name,
|
|
820
822
|
type: e.type
|
|
821
823
|
}))
|
|
822
824
|
})), t, e.masking);
|
|
823
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
825
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new $e({
|
|
824
826
|
unclassified: r.unclassified,
|
|
825
827
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
826
828
|
}));
|
|
@@ -846,14 +848,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
846
848
|
bundleFile: l,
|
|
847
849
|
...g ? { mask: g } : {},
|
|
848
850
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
849
|
-
}, v = m === "snapshot" ? yield*
|
|
851
|
+
}, v = m === "snapshot" ? yield* pn(e.store, e.dialect, (e) => dn(e, _)) : yield* dn(e.store, _), y = 0, b = 0;
|
|
850
852
|
if (e.assets) {
|
|
851
|
-
let t = l(
|
|
853
|
+
let t = l(jt), r = yield* pt({
|
|
852
854
|
source: e.assets,
|
|
853
|
-
assetsDir: l(
|
|
855
|
+
assetsDir: l(At),
|
|
854
856
|
progress: {
|
|
855
857
|
done: (e) => e in d.assets,
|
|
856
|
-
mark: (e, r) => (d.assets[e] = !0,
|
|
858
|
+
mark: (e, r) => (d.assets[e] = !0, un(t, JSON.stringify(r)).pipe(n.andThen(J(u, d))))
|
|
857
859
|
}
|
|
858
860
|
});
|
|
859
861
|
y = r.count, b = r.totalBytes, r.missing.length > 0 && (yield* n.logWarning(`${r.missing.length} of ${r.missing.length + r.count} blob reference(s) point at objects the storage provider does not have; they are NOT in this bundle and no import can bring them back. e.g. ${r.missing.slice(0, 3).map((e) => `'${e}'`).join(", ")}` + (r.missing.length > 3 ? `, and ${r.missing.length - 3} more` : "")));
|
|
@@ -863,7 +865,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
863
865
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
864
866
|
source: {
|
|
865
867
|
dialect: e.dialect,
|
|
866
|
-
schemaFingerprint:
|
|
868
|
+
schemaFingerprint: w(e.snapshot)
|
|
867
869
|
},
|
|
868
870
|
scope: t,
|
|
869
871
|
consistency: m,
|
|
@@ -872,7 +874,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
872
874
|
assets: y > 0 ? {
|
|
873
875
|
count: y,
|
|
874
876
|
totalBytes: b,
|
|
875
|
-
index:
|
|
877
|
+
index: jt
|
|
876
878
|
} : {
|
|
877
879
|
count: 0,
|
|
878
880
|
totalBytes: 0
|
|
@@ -881,7 +883,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
881
883
|
};
|
|
882
884
|
return yield* n.tryPromise({
|
|
883
885
|
try: async () => {
|
|
884
|
-
await (await import("node:fs/promises")).writeFile(l(K),
|
|
886
|
+
await (await import("node:fs/promises")).writeFile(l(K), Et(x));
|
|
885
887
|
},
|
|
886
888
|
catch: (e) => new H({
|
|
887
889
|
reason: `manifest write failed: ${Y(e)}`,
|
|
@@ -890,14 +892,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
890
892
|
}), yield* n.tryPromise({
|
|
891
893
|
try: async () => {
|
|
892
894
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
893
|
-
await e.writeFile(l(
|
|
895
|
+
await e.writeFile(l(Ot), Pt(x, { hostname: t.hostname() }));
|
|
894
896
|
},
|
|
895
897
|
catch: (e) => new H({
|
|
896
898
|
reason: `readme write failed: ${Y(e)}`,
|
|
897
|
-
path:
|
|
899
|
+
path: Ot
|
|
898
900
|
})
|
|
899
901
|
}), x;
|
|
900
|
-
}),
|
|
902
|
+
}), hn = (e, t) => {
|
|
901
903
|
let n = new Set(t), r = [];
|
|
902
904
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
903
905
|
let i = e.references?.table;
|
|
@@ -908,7 +910,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
908
910
|
});
|
|
909
911
|
}
|
|
910
912
|
return r;
|
|
911
|
-
},
|
|
913
|
+
}, gn = (e) => {
|
|
912
914
|
let t = /* @__PURE__ */ new Map();
|
|
913
915
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
914
916
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -916,7 +918,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
916
918
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
917
919
|
});
|
|
918
920
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
919
|
-
},
|
|
921
|
+
}, _n = (e) => {
|
|
920
922
|
switch (e) {
|
|
921
923
|
case "id":
|
|
922
924
|
case "text":
|
|
@@ -935,12 +937,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
935
937
|
case "raw": return "raw";
|
|
936
938
|
default: return "string";
|
|
937
939
|
}
|
|
938
|
-
},
|
|
940
|
+
}, vn = {
|
|
939
941
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
940
942
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
941
943
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
942
|
-
},
|
|
943
|
-
let i =
|
|
944
|
+
}, yn = (e, t, n, r) => {
|
|
945
|
+
let i = _n(n);
|
|
944
946
|
if (i === "raw") return {
|
|
945
947
|
table: e,
|
|
946
948
|
column: t,
|
|
@@ -948,7 +950,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
948
950
|
severity: "block",
|
|
949
951
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
950
952
|
};
|
|
951
|
-
let a =
|
|
953
|
+
let a = vn[i];
|
|
952
954
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
953
955
|
table: e,
|
|
954
956
|
column: t,
|
|
@@ -962,15 +964,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
962
964
|
severity: "block",
|
|
963
965
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
964
966
|
};
|
|
965
|
-
},
|
|
967
|
+
}, bn = (e, t, n) => {
|
|
966
968
|
if (t === n) return [];
|
|
967
969
|
let r = [];
|
|
968
970
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
969
|
-
let a =
|
|
971
|
+
let a = yn(t.name, e, i, n);
|
|
970
972
|
a && r.push(a);
|
|
971
973
|
}
|
|
972
974
|
return r;
|
|
973
|
-
},
|
|
975
|
+
}, xn = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, Sn = (e) => {
|
|
974
976
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
975
977
|
switch (e.kind) {
|
|
976
978
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -979,35 +981,35 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
979
981
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
980
982
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
981
983
|
}
|
|
982
|
-
},
|
|
984
|
+
}, Cn = (e) => {
|
|
983
985
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
984
986
|
return t.length > 0 ? t.join("/") : void 0;
|
|
985
|
-
},
|
|
987
|
+
}, wn = "Failed to execute statement", Tn = (e) => {
|
|
986
988
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
987
989
|
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
988
|
-
return t === `SqlError: ${
|
|
989
|
-
},
|
|
990
|
+
return t === `SqlError: ${wn}` ? wn : t;
|
|
991
|
+
}, En = (e) => {
|
|
990
992
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
991
993
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
992
994
|
unclassified: t,
|
|
993
995
|
raw: n
|
|
994
|
-
}), i =
|
|
995
|
-
if (a !== void 0) return r(`${
|
|
996
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u =
|
|
997
|
-
return r(d ===
|
|
998
|
-
},
|
|
999
|
-
let t =
|
|
996
|
+
}), i = v(e), a = m(i), o = Cn(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
997
|
+
if (a !== void 0) return r(`${Sn(a)}${s}`, !1);
|
|
998
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = Tn(typeof l == "string" && l.length > 0 ? l : String(e)), d = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, f = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, p = f !== void 0 && /[←‖]/.test(f) ? ` (via ${f})` : "";
|
|
999
|
+
return r(d === wn || d.length === 0 ? `the database refused the write and the driver gave no detail${s}${p}` : `${d}${s}`, d === wn || d.length === 0);
|
|
1000
|
+
}, Dn = (e) => En(e).reason, On = (e) => {
|
|
1001
|
+
let t = v(e), n = m(t);
|
|
1000
1002
|
if (n?.kind === "foreignKeyInUse") {
|
|
1001
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
1003
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = Cn(t);
|
|
1002
1004
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
1003
1005
|
}
|
|
1004
|
-
return
|
|
1005
|
-
},
|
|
1006
|
+
return Dn(e);
|
|
1007
|
+
}, kn = f, X = (e) => `${kn}${e}`, An = (e) => typeof e.run == "function", jn = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
1006
1008
|
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
1007
|
-
written before the first delete is what covers that case.`,
|
|
1008
|
-
if (!
|
|
1009
|
+
written before the first delete is what covers that case.`, Mn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, Z = (e, t) => {
|
|
1010
|
+
if (!Mn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
1009
1011
|
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
1010
|
-
},
|
|
1012
|
+
}, Nn = (e, t) => {
|
|
1011
1013
|
let n = Z(e, t), r = Z(X(e), t);
|
|
1012
1014
|
switch (t) {
|
|
1013
1015
|
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
@@ -1016,10 +1018,10 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1016
1018
|
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
1017
1019
|
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
1018
1020
|
}
|
|
1019
|
-
},
|
|
1021
|
+
}, Pn = (e, t) => {
|
|
1020
1022
|
let n = Z(X(e), t);
|
|
1021
1023
|
return t === "mssql" ? `IF OBJECT_ID('${X(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
1022
|
-
},
|
|
1024
|
+
}, Fn = (e, t) => {
|
|
1023
1025
|
let n = t === "mysql" || t === "mariadb", r = [];
|
|
1024
1026
|
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
1025
1027
|
for (let n of [...e].reverse()) r.push(`DELETE FROM ${Z(n.name, t)}`);
|
|
@@ -1029,7 +1031,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1029
1031
|
r.push(`INSERT INTO ${Z(n.name, t)} (${e}) SELECT ${e} FROM ${Z(X(n.name), t)}`);
|
|
1030
1032
|
}
|
|
1031
1033
|
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
1032
|
-
},
|
|
1034
|
+
}, In = (e, t) => {
|
|
1033
1035
|
let n = [], r = () => {
|
|
1034
1036
|
for (let e of n) t.unregisterTable(e);
|
|
1035
1037
|
};
|
|
@@ -1051,7 +1053,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1051
1053
|
staged: n,
|
|
1052
1054
|
undo: r
|
|
1053
1055
|
};
|
|
1054
|
-
},
|
|
1056
|
+
}, Ln = (e, t) => {
|
|
1055
1057
|
let n = new Set(t), r = [];
|
|
1056
1058
|
for (let t of e.tables) {
|
|
1057
1059
|
if (!n.has(t.name)) continue;
|
|
@@ -1068,22 +1070,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1068
1070
|
}
|
|
1069
1071
|
}
|
|
1070
1072
|
return r;
|
|
1071
|
-
},
|
|
1073
|
+
}, Rn = (e, t) => {
|
|
1072
1074
|
let n = Z(X(e.from), t), r = Z(X(e.to), t), i = Z(e.column, t), a = Z(e.toColumn, t);
|
|
1073
1075
|
return `SELECT ${Z(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
1074
|
-
},
|
|
1076
|
+
}, zn = (e) => {
|
|
1075
1077
|
if (e.length === 0) return;
|
|
1076
1078
|
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
1077
1079
|
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
1078
|
-
},
|
|
1079
|
-
t.length !== 0 && await e.run(
|
|
1080
|
-
}, Bn = async (e, t, r) => {
|
|
1081
|
-
let i = Pn(t, r);
|
|
1082
|
-
i.length !== 0 && await e.run(_e.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
1080
|
+
}, Bn = async (e, t) => {
|
|
1081
|
+
t.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
1083
1082
|
}, Vn = async (e, t, r) => {
|
|
1083
|
+
let i = Fn(t, r);
|
|
1084
|
+
i.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
1085
|
+
}, Hn = async (e, t, r) => {
|
|
1084
1086
|
let i = [];
|
|
1085
1087
|
for (let a of t) {
|
|
1086
|
-
let t = await e.run(
|
|
1088
|
+
let t = await e.run(P.SqlClient.pipe(n.flatMap((e) => e.unsafe(Rn(a, r)))));
|
|
1087
1089
|
for (let e of t) i.push({
|
|
1088
1090
|
table: a.from,
|
|
1089
1091
|
primaryKey: String(e.pk ?? ""),
|
|
@@ -1093,14 +1095,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1093
1095
|
});
|
|
1094
1096
|
}
|
|
1095
1097
|
return i;
|
|
1096
|
-
},
|
|
1098
|
+
}, Un = (e, t) => {
|
|
1097
1099
|
let n = new Map(t.map((e, t) => [e, t]));
|
|
1098
1100
|
return e.filter((e) => {
|
|
1099
1101
|
if (e.from === e.to) return !1;
|
|
1100
1102
|
let t = n.get(e.from), r = n.get(e.to);
|
|
1101
1103
|
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1102
1104
|
});
|
|
1103
|
-
},
|
|
1105
|
+
}, Wn = (e) => e === "mysql" || e === "mariadb", Gn = (e) => {
|
|
1104
1106
|
if (e.mode !== "replace") return {
|
|
1105
1107
|
active: !1,
|
|
1106
1108
|
tables: [],
|
|
@@ -1150,7 +1152,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1150
1152
|
tables: [],
|
|
1151
1153
|
blocker: "every table in the bundle is one that describes a deployment, so there is nothing for a staged swap to move."
|
|
1152
1154
|
};
|
|
1153
|
-
let s = o.map((e) => e.name), c =
|
|
1155
|
+
let s = o.map((e) => e.name), c = Wn(t) ? [] : Un(Ln(n, s), s);
|
|
1154
1156
|
return c.length > 0 ? {
|
|
1155
1157
|
active: !1,
|
|
1156
1158
|
tables: [],
|
|
@@ -1159,7 +1161,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1159
1161
|
active: !0,
|
|
1160
1162
|
tables: o
|
|
1161
1163
|
};
|
|
1162
|
-
},
|
|
1164
|
+
}, Kn = (e) => n.gen(function* () {
|
|
1163
1165
|
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1164
1166
|
if (e.snapshot === void 0) return yield* n.fail(new H({
|
|
1165
1167
|
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
@@ -1172,7 +1174,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1172
1174
|
if (t.tables.length === 0) return;
|
|
1173
1175
|
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1174
1176
|
so it survives a run whose process does not.`);
|
|
1175
|
-
let r = yield*
|
|
1177
|
+
let r = yield* mn({
|
|
1176
1178
|
store: e.store,
|
|
1177
1179
|
snapshot: t,
|
|
1178
1180
|
dialect: e.dialect ?? "unknown",
|
|
@@ -1188,18 +1190,18 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1188
1190
|
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1189
1191
|
tables: r.tables.length
|
|
1190
1192
|
};
|
|
1191
|
-
}),
|
|
1193
|
+
}), qn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, Jn = (e, t) => e.tables.find((e) => e.name === t), Yn = (e, t) => new Map((Jn(e, t)?.columns ?? []).map((e) => [e.name, e])), Xn = /__dropped_\d+$/, Zn = (e, t) => {
|
|
1192
1194
|
let n = [];
|
|
1193
1195
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
1194
|
-
if (
|
|
1196
|
+
if (Jn(t, r.name) === void 0) {
|
|
1195
1197
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
1196
1198
|
continue;
|
|
1197
1199
|
}
|
|
1198
|
-
let e =
|
|
1200
|
+
let e = Yn(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
1199
1201
|
for (let [t, a] of i) {
|
|
1200
1202
|
let i = e.get(t);
|
|
1201
1203
|
if (i === void 0) {
|
|
1202
|
-
n.push(
|
|
1204
|
+
n.push(Xn.test(t) ? `${r.name}.${t}: a soft-dropped column, present only in the SOURCE — reclaim it there (voltro db gc-snapshots --before <date>) or re-export without it; the target is not missing anything` : `${r.name}.${t}: in the bundle, MISSING from the target — the value has nowhere to go`);
|
|
1203
1205
|
continue;
|
|
1204
1206
|
}
|
|
1205
1207
|
i.type !== a.type && n.push(`${r.name}.${t}: type '${a.type}' (bundle) vs '${i.type}' (target)`);
|
|
@@ -1207,22 +1209,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1207
1209
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
1208
1210
|
}
|
|
1209
1211
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
1210
|
-
},
|
|
1211
|
-
let r =
|
|
1212
|
+
}, Qn = (e, t, n) => {
|
|
1213
|
+
let r = w(t);
|
|
1212
1214
|
if (r === e || n === void 0) return {
|
|
1213
1215
|
drifted: !1,
|
|
1214
1216
|
bundleFingerprint: e,
|
|
1215
1217
|
targetFingerprint: r,
|
|
1216
1218
|
diff: []
|
|
1217
1219
|
};
|
|
1218
|
-
let i =
|
|
1220
|
+
let i = Zn(n, t);
|
|
1219
1221
|
return {
|
|
1220
1222
|
drifted: i.length > 0,
|
|
1221
1223
|
bundleFingerprint: e,
|
|
1222
1224
|
targetFingerprint: r,
|
|
1223
1225
|
diff: i
|
|
1224
1226
|
};
|
|
1225
|
-
},
|
|
1227
|
+
}, $n = 2e3, er = 2e3, tr = 200, nr = /* @__PURE__ */ new Set([
|
|
1226
1228
|
"BundleError",
|
|
1227
1229
|
"CodecError",
|
|
1228
1230
|
"IntegrityError",
|
|
@@ -1231,29 +1233,29 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1231
1233
|
"RowsRefusedError",
|
|
1232
1234
|
"SchemaDriftError",
|
|
1233
1235
|
"StorageError"
|
|
1234
|
-
]),
|
|
1236
|
+
]), rr = (t) => {
|
|
1235
1237
|
let n = o.isFiberFailure(t) ? [...e.failures(t[o.FiberFailureCauseId]), ...e.defects(t[o.FiberFailureCauseId])] : [t];
|
|
1236
1238
|
for (let e of n) {
|
|
1237
1239
|
let t = e?._tag;
|
|
1238
|
-
if (typeof t == "string" &&
|
|
1240
|
+
if (typeof t == "string" && nr.has(t)) return e;
|
|
1239
1241
|
if (o.isFiberFailure(e)) {
|
|
1240
|
-
let t =
|
|
1242
|
+
let t = rr(e);
|
|
1241
1243
|
if (t) return t;
|
|
1242
1244
|
}
|
|
1243
1245
|
}
|
|
1244
|
-
},
|
|
1246
|
+
}, ir = (t) => [...e.failures(t)][0] ?? Error(e.pretty(t)), ar = (e) => n.tryPromise({
|
|
1245
1247
|
try: async () => {
|
|
1246
1248
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
1247
|
-
return
|
|
1249
|
+
return Dt(await t.readFile(n.join(e, K), "utf8"));
|
|
1248
1250
|
},
|
|
1249
1251
|
catch: (e) => new H({
|
|
1250
1252
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
1251
1253
|
path: K
|
|
1252
1254
|
})
|
|
1253
|
-
}),
|
|
1255
|
+
}), or = (e) => {
|
|
1254
1256
|
let t;
|
|
1255
1257
|
return n.gen(function* () {
|
|
1256
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
1258
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* ar(e.bundleDir);
|
|
1257
1259
|
if (a.formatVersion !== 1) return yield* n.fail(new H({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
1258
1260
|
if (e.tables !== void 0) {
|
|
1259
1261
|
let t = new Set(a.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
@@ -1274,9 +1276,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1274
1276
|
unique: !1,
|
|
1275
1277
|
hasDefault: !1
|
|
1276
1278
|
}))
|
|
1277
|
-
})) }, r =
|
|
1279
|
+
})) }, r = Qn(s.source.schemaFingerprint, e.targetSnapshot, t);
|
|
1278
1280
|
if (r.drifted) {
|
|
1279
|
-
if (!e.force) return yield* n.fail(new
|
|
1281
|
+
if (!e.force) return yield* n.fail(new Qe({
|
|
1280
1282
|
bundleFingerprint: r.bundleFingerprint,
|
|
1281
1283
|
targetFingerprint: r.targetFingerprint,
|
|
1282
1284
|
diff: r.diff
|
|
@@ -1285,28 +1287,28 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1285
1287
|
}
|
|
1286
1288
|
}
|
|
1287
1289
|
if (e.targetDialect && e.targetDialect !== s.source.dialect) {
|
|
1288
|
-
let t =
|
|
1289
|
-
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new
|
|
1290
|
+
let t = bn(s.tables, s.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1291
|
+
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new tt({
|
|
1290
1292
|
sourceDialect: s.source.dialect,
|
|
1291
1293
|
targetDialect: e.targetDialect,
|
|
1292
|
-
blocking: t.map(
|
|
1294
|
+
blocking: t.map(xn)
|
|
1293
1295
|
}));
|
|
1294
1296
|
}
|
|
1295
|
-
let c = e.verify ?? !0,
|
|
1296
|
-
if (
|
|
1297
|
-
mode:
|
|
1297
|
+
let c = e.verify ?? !0, f = e.mode ?? "upsert", m = e.atomic ?? f === "replace", _ = e.onConflict ?? "skip", v = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), b = yield* Bt(v);
|
|
1298
|
+
if (f === "replace" && s.scope.kind !== "all") return yield* n.fail(new et({
|
|
1299
|
+
mode: f,
|
|
1298
1300
|
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope. To leave specific tables out of a FULL export, use \`--exclude a,b\`: it keeps the scope \`all\` and records what it left out.`
|
|
1299
1301
|
}));
|
|
1300
|
-
if (
|
|
1302
|
+
if (f === "replace" && (s.scope.exclude?.length ?? 0) > 0) {
|
|
1301
1303
|
let t = s.scope.exclude;
|
|
1302
1304
|
e.onWarn?.(`this bundle was exported with ${t.length} table(s) excluded: ${t.join(", ")}.\n A replace empties and refills the tables the bundle CARRIES, so those keep whatever rows the
|
|
1303
1305
|
target already holds. That is not "everything the bundle says" — check it is what you meant.`);
|
|
1304
1306
|
}
|
|
1305
|
-
if (
|
|
1307
|
+
if (f === "replace") {
|
|
1306
1308
|
let t = s.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1307
1309
|
try: () => e.store.incomingForeignKeys(t),
|
|
1308
|
-
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${
|
|
1309
|
-
}) : e.targetSnapshot === void 0 ? [] :
|
|
1310
|
+
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${On(e)}` })
|
|
1311
|
+
}) : e.targetSnapshot === void 0 ? [] : hn(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1310
1312
|
for (let t of r) {
|
|
1311
1313
|
if (!a.has(t.from)) {
|
|
1312
1314
|
let r = yield* n.tryPromise({
|
|
@@ -1328,9 +1330,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1328
1330
|
rowCount: r
|
|
1329
1331
|
});
|
|
1330
1332
|
}
|
|
1331
|
-
if (i.length > 0) return yield* n.fail(new
|
|
1332
|
-
mode:
|
|
1333
|
-
reason:
|
|
1333
|
+
if (i.length > 0) return yield* n.fail(new et({
|
|
1334
|
+
mode: f,
|
|
1335
|
+
reason: gn(i)
|
|
1334
1336
|
}));
|
|
1335
1337
|
}
|
|
1336
1338
|
if (e.dryRun === !0) return e.traceId !== void 0 && (yield* n.promise(() => e.store.update(u, e.traceId, {
|
|
@@ -1346,32 +1348,32 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1346
1348
|
rowsSkipped: 0,
|
|
1347
1349
|
fullyResumed: !1
|
|
1348
1350
|
};
|
|
1349
|
-
let
|
|
1351
|
+
let S = /* @__PURE__ */ new Map();
|
|
1350
1352
|
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1351
1353
|
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1352
|
-
e.length > 0 &&
|
|
1354
|
+
e.length > 0 && S.set(t.name, e);
|
|
1353
1355
|
}
|
|
1354
|
-
let
|
|
1355
|
-
let n =
|
|
1356
|
+
let C = (e, t) => {
|
|
1357
|
+
let n = S.get(e);
|
|
1356
1358
|
if (n === void 0) return t;
|
|
1357
1359
|
let r;
|
|
1358
1360
|
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1359
1361
|
return r ?? t;
|
|
1360
1362
|
}, w = (e, t, n, r = !1) => {
|
|
1361
|
-
let i =
|
|
1362
|
-
return
|
|
1363
|
-
},
|
|
1364
|
-
r.unclassified && (
|
|
1365
|
-
},
|
|
1363
|
+
let i = O ? X(t.name) : t.name, a = t.primaryKey, o = C(t.name, n);
|
|
1364
|
+
return f === "append" ? _ === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : f === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1365
|
+
}, ae = (e, t) => e.savepoint ? e.savepoint(t) : t(), oe = 0, T = (t, n, r) => {
|
|
1366
|
+
r.unclassified && (oe++, !(oe > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (oe === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1367
|
+
}, se = (e, t, r = !0) => n.tryPromise({
|
|
1366
1368
|
try: e,
|
|
1367
|
-
catch: (e) =>
|
|
1368
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason,
|
|
1369
|
+
catch: (e) => En(e)
|
|
1370
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, T(t.entry, t.row, e)), !1)))), ce = (e) => {
|
|
1369
1371
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1370
1372
|
if (t.length !== 0) return {
|
|
1371
1373
|
...e.row,
|
|
1372
1374
|
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1373
1375
|
};
|
|
1374
|
-
},
|
|
1376
|
+
}, le = (e) => {
|
|
1375
1377
|
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
1376
1378
|
if (r !== "reference") return !1;
|
|
1377
1379
|
let i = e.row[n];
|
|
@@ -1389,7 +1391,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1389
1391
|
};
|
|
1390
1392
|
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1391
1393
|
}
|
|
1392
|
-
return new
|
|
1394
|
+
return new Ze({
|
|
1393
1395
|
rows: i.slice(0, 20),
|
|
1394
1396
|
primaryCount: r.filter((e) => !e.derived).length,
|
|
1395
1397
|
totalCount: e.length,
|
|
@@ -1399,104 +1401,104 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1399
1401
|
primary: t.primary
|
|
1400
1402
|
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1401
1403
|
});
|
|
1402
|
-
},
|
|
1404
|
+
}, ue = (e, t) => n.gen(function* () {
|
|
1403
1405
|
let r = t, i = [], a = !0;
|
|
1404
1406
|
for (; r.length > 0 && a;) {
|
|
1405
1407
|
a = !1;
|
|
1406
1408
|
let t = [];
|
|
1407
|
-
for (let n of r) (yield*
|
|
1409
|
+
for (let n of r) (yield* se(() => ae(e, () => w(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1408
1410
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1409
|
-
let r = t[n], o =
|
|
1410
|
-
o !== void 0 && (yield*
|
|
1411
|
+
let r = t[n], o = ce(r);
|
|
1412
|
+
o !== void 0 && (yield* se(() => ae(e, () => w(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1411
1413
|
}
|
|
1412
1414
|
r = t;
|
|
1413
1415
|
}
|
|
1414
|
-
for (let t of i) (yield*
|
|
1415
|
-
if (r.length > 0) return yield* n.fail(
|
|
1416
|
-
}),
|
|
1416
|
+
for (let t of i) (yield* se(() => ae(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1417
|
+
if (r.length > 0) return yield* n.fail(le(r));
|
|
1418
|
+
}), E = (() => {
|
|
1417
1419
|
let t = e.targetDialect;
|
|
1418
1420
|
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1419
|
-
})(),
|
|
1420
|
-
mode:
|
|
1421
|
+
})(), D = Gn({
|
|
1422
|
+
mode: f,
|
|
1421
1423
|
...e.targetDialect === void 0 ? {} : { targetDialect: e.targetDialect },
|
|
1422
1424
|
...e.targetSnapshot === void 0 ? {} : { targetSnapshot: e.targetSnapshot },
|
|
1423
1425
|
bundleTables: s.tables.map((e) => e.name),
|
|
1424
1426
|
isSkipped: V,
|
|
1425
|
-
canRunSql:
|
|
1426
|
-
}),
|
|
1427
|
-
if (
|
|
1427
|
+
canRunSql: An(e.store)
|
|
1428
|
+
}), fe = D.tables, pe = s.tables.filter((e) => !V(e.name) && ee(e.name)).map((e) => e.name), O = D.active, me = !1, k = 0, A = 0, ge, j, M = 0, N = 0, _e = (t, r) => n.gen(function* () {
|
|
1429
|
+
if (f === "replace" && !O && !(r && b.truncated)) {
|
|
1428
1430
|
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1429
1431
|
try: () => t.insert(d, {
|
|
1430
|
-
id:
|
|
1431
|
-
startedAt:
|
|
1432
|
+
id: h("repl", P),
|
|
1433
|
+
startedAt: F,
|
|
1432
1434
|
tables: String(s.tables.length),
|
|
1433
|
-
capture:
|
|
1435
|
+
capture: I?.dir ?? null,
|
|
1434
1436
|
via: e.via ?? "direct"
|
|
1435
1437
|
}),
|
|
1436
1438
|
catch: (e) => new H({ reason: `could not record that this replace started: ${String(e?.message ?? e)}\n The marker is what makes an interrupted run visible instead of silent, so the run does
|
|
1437
1439
|
not proceed without it.` })
|
|
1438
1440
|
}));
|
|
1439
1441
|
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !V(e)), a = !r, o = t.emptyTables ? () => t.emptyTables(i) : a ? async () => {
|
|
1440
|
-
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where:
|
|
1442
|
+
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where: x(e.primaryKey) });
|
|
1441
1443
|
} : () => t.transactional(async (e) => {
|
|
1442
|
-
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where:
|
|
1444
|
+
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: x(t.primaryKey) });
|
|
1443
1445
|
});
|
|
1444
1446
|
yield* n.tryPromise({
|
|
1445
1447
|
try: o,
|
|
1446
|
-
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${
|
|
1447
|
-
}), r && (
|
|
1448
|
+
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${On(e)}` })
|
|
1449
|
+
}), r && (b.truncated = !0, yield* J(v, b));
|
|
1448
1450
|
}
|
|
1449
|
-
let a = [], o = [], u = [],
|
|
1451
|
+
let a = [], o = [], u = [], p = [], g = s.tables.length;
|
|
1450
1452
|
for (let d = 0; d < s.tables.length; d++) {
|
|
1451
|
-
let
|
|
1452
|
-
if (V(
|
|
1453
|
-
|
|
1453
|
+
let h = s.tables[d];
|
|
1454
|
+
if (V(h.name)) {
|
|
1455
|
+
p.push(h.name);
|
|
1454
1456
|
continue;
|
|
1455
1457
|
}
|
|
1456
|
-
if (r &&
|
|
1457
|
-
u.push(
|
|
1458
|
+
if (r && h.name in b.tables) {
|
|
1459
|
+
u.push(h.name), N++, M += b.tables[h.name].rowCount, q(e.onProgress, {
|
|
1458
1460
|
phase: "import",
|
|
1459
|
-
table:
|
|
1461
|
+
table: h.name,
|
|
1460
1462
|
index: d,
|
|
1461
|
-
tableCount:
|
|
1463
|
+
tableCount: g,
|
|
1462
1464
|
event: "done",
|
|
1463
|
-
rowsDone:
|
|
1464
|
-
total:
|
|
1465
|
+
rowsDone: b.tables[h.name].rowCount,
|
|
1466
|
+
total: h.rowCount
|
|
1465
1467
|
});
|
|
1466
1468
|
continue;
|
|
1467
1469
|
}
|
|
1468
|
-
let _ = i.join(e.bundleDir,
|
|
1470
|
+
let _ = i.join(e.bundleDir, h.file);
|
|
1469
1471
|
q(e.onProgress, {
|
|
1470
1472
|
phase: "import",
|
|
1471
|
-
table:
|
|
1473
|
+
table: h.name,
|
|
1472
1474
|
index: d,
|
|
1473
|
-
tableCount:
|
|
1475
|
+
tableCount: g,
|
|
1474
1476
|
event: "start",
|
|
1475
1477
|
rowsDone: 0,
|
|
1476
|
-
total:
|
|
1478
|
+
total: h.rowCount
|
|
1477
1479
|
});
|
|
1478
|
-
let
|
|
1479
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c =
|
|
1480
|
+
let y = a.length, x = n.gen(function* () {
|
|
1481
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c = he("bundle", () => de(_).pipe(bt(s.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), u = async (e) => {
|
|
1480
1482
|
let n = {
|
|
1481
|
-
entry:
|
|
1483
|
+
entry: h,
|
|
1482
1484
|
row: e,
|
|
1483
1485
|
reason: ""
|
|
1484
1486
|
};
|
|
1485
1487
|
try {
|
|
1486
|
-
await
|
|
1488
|
+
await ae(t, () => w(t, h, e));
|
|
1487
1489
|
} catch (t) {
|
|
1488
|
-
let r =
|
|
1489
|
-
n.reason = r.reason,
|
|
1490
|
+
let r = En(t);
|
|
1491
|
+
n.reason = r.reason, T(h, e, r), a.push(n);
|
|
1490
1492
|
}
|
|
1491
1493
|
}, d = yield* n.promise(async () => {
|
|
1492
1494
|
let n = e.copyLoader;
|
|
1493
1495
|
if (!n || n.dialect !== "postgres") return !1;
|
|
1494
|
-
if (
|
|
1495
|
-
if (
|
|
1496
|
-
if (
|
|
1496
|
+
if (m) return me || (me = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1497
|
+
if (f === "replace") return !0;
|
|
1498
|
+
if (f !== "upsert") return !1;
|
|
1497
1499
|
try {
|
|
1498
1500
|
return (await t.query({
|
|
1499
|
-
table:
|
|
1501
|
+
table: h.name,
|
|
1500
1502
|
predicate: void 0,
|
|
1501
1503
|
order: [],
|
|
1502
1504
|
take: 1,
|
|
@@ -1506,19 +1508,19 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1506
1508
|
} catch {
|
|
1507
1509
|
return !1;
|
|
1508
1510
|
}
|
|
1509
|
-
}),
|
|
1511
|
+
}), p = Object.keys(h.columnTypes), g = Math.max(1, e.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
1510
1512
|
if (v.length === 0) return;
|
|
1511
1513
|
let t = v;
|
|
1512
1514
|
v = [];
|
|
1513
1515
|
try {
|
|
1514
1516
|
await e.copyLoader.copyInto({
|
|
1515
|
-
table:
|
|
1516
|
-
columns:
|
|
1517
|
-
columnTypes:
|
|
1517
|
+
table: h.name,
|
|
1518
|
+
columns: p,
|
|
1519
|
+
columnTypes: h.columnTypes,
|
|
1518
1520
|
rows: t
|
|
1519
1521
|
});
|
|
1520
1522
|
} catch (n) {
|
|
1521
|
-
d = !1, e.onWarn?.(`bulk COPY into ${
|
|
1523
|
+
d = !1, e.onWarn?.(`bulk COPY into ${h.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1522
1524
|
for (let e of t) await u(e);
|
|
1523
1525
|
}
|
|
1524
1526
|
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof t.savepoint == "function", S = [], ee = async () => {
|
|
@@ -1527,15 +1529,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1527
1529
|
S = [];
|
|
1528
1530
|
try {
|
|
1529
1531
|
await t.savepoint(async () => {
|
|
1530
|
-
for (let n of e) await w(t,
|
|
1532
|
+
for (let n of e) await w(t, h, n);
|
|
1531
1533
|
});
|
|
1532
1534
|
} catch {
|
|
1533
1535
|
for (let t of e) await u(t);
|
|
1534
1536
|
}
|
|
1535
1537
|
};
|
|
1536
|
-
return yield* l.runForEach(
|
|
1537
|
-
if (o++, d) {
|
|
1538
|
-
v.push(e), v.length >=
|
|
1538
|
+
return yield* l.runForEach(tn(c, h.name, h.columnTypes), (e) => n.promise(async () => {
|
|
1539
|
+
if (o++, j?.(), d) {
|
|
1540
|
+
v.push(e), v.length >= g && await y();
|
|
1539
1541
|
return;
|
|
1540
1542
|
}
|
|
1541
1543
|
if (x) {
|
|
@@ -1547,181 +1549,207 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1547
1549
|
rowCount: o,
|
|
1548
1550
|
checksum: i.digest("hex")
|
|
1549
1551
|
};
|
|
1550
|
-
}), { rowCount: S, checksum: ee } = yield*
|
|
1552
|
+
}), { rowCount: S, checksum: ee } = yield* Rt("import", h.name, d, g, (e) => e.rowCount, x);
|
|
1551
1553
|
if (c) {
|
|
1552
|
-
if (ee !==
|
|
1553
|
-
what: `table ${
|
|
1554
|
-
expected:
|
|
1554
|
+
if (ee !== h.checksum) return yield* n.fail(new Ye({
|
|
1555
|
+
what: `table ${h.name} checksum`,
|
|
1556
|
+
expected: h.checksum,
|
|
1555
1557
|
actual: ee
|
|
1556
1558
|
}));
|
|
1557
|
-
if (S !==
|
|
1558
|
-
what: `table ${
|
|
1559
|
-
expected: String(
|
|
1559
|
+
if (S !== h.rowCount) return yield* n.fail(new Ye({
|
|
1560
|
+
what: `table ${h.name} rowCount`,
|
|
1561
|
+
expected: String(h.rowCount),
|
|
1560
1562
|
actual: String(S)
|
|
1561
1563
|
}));
|
|
1562
1564
|
}
|
|
1563
1565
|
q(e.onProgress, {
|
|
1564
1566
|
phase: "import",
|
|
1565
|
-
table:
|
|
1567
|
+
table: h.name,
|
|
1566
1568
|
index: d,
|
|
1567
|
-
tableCount:
|
|
1569
|
+
tableCount: g,
|
|
1568
1570
|
event: "done",
|
|
1569
1571
|
rowsDone: S,
|
|
1570
|
-
total:
|
|
1571
|
-
}),
|
|
1572
|
-
name:
|
|
1572
|
+
total: h.rowCount
|
|
1573
|
+
}), A++, k += S, ge?.(), j?.(), r && (a.length > y ? o.push({
|
|
1574
|
+
name: h.name,
|
|
1573
1575
|
rowCount: S
|
|
1574
|
-
}) : (
|
|
1576
|
+
}) : (b.tables[h.name] = {
|
|
1575
1577
|
rowCount: S,
|
|
1576
|
-
checksum:
|
|
1577
|
-
}, yield* J(
|
|
1578
|
+
checksum: h.checksum
|
|
1579
|
+
}, yield* J(v, b)));
|
|
1578
1580
|
}
|
|
1579
|
-
if (a.length > 0 && (yield*
|
|
1581
|
+
if (a.length > 0 && (yield* ue(t, a)), f === "replace" && (yield* n.promise(() => t.deleteMany(d, { where: {
|
|
1580
1582
|
column: "startedAt",
|
|
1581
1583
|
op: "lte",
|
|
1582
|
-
value:
|
|
1584
|
+
value: F
|
|
1583
1585
|
} }).then(() => void 0, () => void 0))), r && o.length > 0) {
|
|
1584
1586
|
for (let e of o) {
|
|
1585
1587
|
let t = s.tables.find((t) => t.name === e.name);
|
|
1586
|
-
|
|
1588
|
+
b.tables[e.name] = {
|
|
1587
1589
|
rowCount: e.rowCount,
|
|
1588
1590
|
checksum: t.checksum
|
|
1589
1591
|
};
|
|
1590
1592
|
}
|
|
1591
|
-
yield* J(
|
|
1593
|
+
yield* J(v, b);
|
|
1592
1594
|
}
|
|
1593
1595
|
{
|
|
1594
|
-
let t =
|
|
1596
|
+
let t = qe(p, "skipped");
|
|
1595
1597
|
t !== void 0 && e.onWarn?.(t);
|
|
1596
1598
|
}
|
|
1597
1599
|
if (u.length > 0) {
|
|
1598
1600
|
let t = u.length === s.tables.length;
|
|
1599
|
-
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${
|
|
1601
|
+
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${v} to force a full re-import.`);
|
|
1600
1602
|
}
|
|
1601
|
-
}),
|
|
1603
|
+
}), P = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, F = /* @__PURE__ */ new Date(), I = yield* Kn({
|
|
1602
1604
|
store: e.store,
|
|
1603
|
-
mode:
|
|
1605
|
+
mode: f,
|
|
1604
1606
|
dir: e.rollbackDir,
|
|
1605
1607
|
snapshot: e.targetSnapshot,
|
|
1606
1608
|
dialect: e.targetDialect,
|
|
1607
1609
|
tables: s.tables.map((e) => e.name).filter((e) => !V(e)),
|
|
1608
1610
|
onWarn: e.onWarn
|
|
1609
1611
|
});
|
|
1610
|
-
|
|
1611
|
-
try: () => e.onRollbackCaptured(
|
|
1612
|
+
I !== void 0 && (e.onRollbackCaptured !== void 0 && (yield* n.tryPromise({
|
|
1613
|
+
try: () => e.onRollbackCaptured(I),
|
|
1612
1614
|
catch: (e) => new H({
|
|
1613
1615
|
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1614
|
-
path:
|
|
1616
|
+
path: I.dir
|
|
1615
1617
|
})
|
|
1616
|
-
})), e.onWarn?.(
|
|
1617
|
-
let ve = (e) =>
|
|
1618
|
-
R &&
|
|
1619
|
-
id:
|
|
1620
|
-
mode:
|
|
1618
|
+
})), e.onWarn?.(qn(I)));
|
|
1619
|
+
let ve = (e) => f === "replace" ? ne(e) : e(), L = (e) => f === "replace" ? re(e) : e, ye = e.traceId ?? h("dimp", `${Date.now()}-${s.source.schemaFingerprint ?? ""}`), be = e.traceId !== void 0, R = (e.targetSnapshot?.tables.some((e) => e.name === u) ?? !1) || y(u) !== void 0;
|
|
1620
|
+
R && g(p), R && e.traceId === void 0 && (yield* n.promise(() => e.store.insert(u, {
|
|
1621
|
+
id: ye,
|
|
1622
|
+
mode: f,
|
|
1621
1623
|
via: e.via ?? "direct",
|
|
1622
1624
|
direction: "import",
|
|
1623
1625
|
bundle: e.bundleDir,
|
|
1624
1626
|
sourceFingerprint: s.source.schemaFingerprint ?? null,
|
|
1625
1627
|
tablesWritten: 0,
|
|
1626
1628
|
rowsWritten: 0,
|
|
1627
|
-
staged:
|
|
1629
|
+
staged: f === "replace" ? O : null
|
|
1628
1630
|
}).then(() => {
|
|
1629
|
-
|
|
1631
|
+
be = !0;
|
|
1630
1632
|
}, (t) => {
|
|
1631
1633
|
e.onWarn?.(`the import trace could not be written (${String(t?.message ?? t)}).\n The import continues — this is a record of what happened, not a safety check. Run
|
|
1632
1634
|
\`voltro db apply\` so the next one is recorded.`);
|
|
1633
|
-
}))), e.traceId !== void 0 && R && (yield* n.promise(() => e.store.update(u,
|
|
1634
|
-
staged:
|
|
1635
|
+
}))), e.traceId !== void 0 && R && (yield* n.promise(() => e.store.update(u, ye, {
|
|
1636
|
+
staged: f === "replace" ? O : null,
|
|
1635
1637
|
sourceFingerprint: s.source.schemaFingerprint ?? null
|
|
1636
1638
|
}).then(() => void 0, () => void 0)));
|
|
1637
|
-
let
|
|
1638
|
-
|
|
1639
|
-
if (!
|
|
1639
|
+
let xe = 0, Se = !1;
|
|
1640
|
+
ge = () => {
|
|
1641
|
+
if (!be || Se) return;
|
|
1640
1642
|
let t = Date.now();
|
|
1641
|
-
t -
|
|
1642
|
-
tablesWritten:
|
|
1643
|
-
rowsWritten:
|
|
1643
|
+
t - xe < $n || (xe = t, e.store.update(u, ye, {
|
|
1644
|
+
tablesWritten: A,
|
|
1645
|
+
rowsWritten: k
|
|
1644
1646
|
}).then(() => void 0, () => {
|
|
1645
|
-
|
|
1647
|
+
Se = !0;
|
|
1646
1648
|
}));
|
|
1647
1649
|
};
|
|
1648
|
-
let
|
|
1650
|
+
let Ce = (t) => be ? e.store.update(u, ye, {
|
|
1649
1651
|
finishedAt: t === null ? /* @__PURE__ */ new Date() : null,
|
|
1650
|
-
tablesWritten:
|
|
1651
|
-
rowsWritten:
|
|
1652
|
+
tablesWritten: A,
|
|
1653
|
+
rowsWritten: k,
|
|
1652
1654
|
failure: t
|
|
1653
1655
|
}).then(() => void 0, () => void 0) : Promise.resolve();
|
|
1654
|
-
if (t = (e) =>
|
|
1656
|
+
if (t = (e) => Ce(e), f === "replace" && !O && D.blocker !== void 0 && e.onWarn?.(jn(D.blocker)), f === "replace" && pe.length > 0 && e.onWarn?.(`write recorders are SUSPENDED for this replace (${pe.length} table(s) have one).\n A replace sets a state rather than changing rows, so a per-row history entry would describe
|
|
1655
1657
|
something that did not happen — and this path already writes through the raw store, without
|
|
1656
|
-
tenant scoping or \`audit()\` stamping, for the same reason.`),
|
|
1657
|
-
e.onWarn?.(`staging ${
|
|
1658
|
-
let t = e.store, r =
|
|
1659
|
-
|
|
1658
|
+
tenant scoping or \`audit()\` stamping, for the same reason.`), O) {
|
|
1659
|
+
e.onWarn?.(`staging ${fe.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1660
|
+
let t = e.store, r = fe.map((e) => e.name), i = h("repl", `staging-${P}`), a = !1;
|
|
1661
|
+
if (e.recordInterruption === !0) {
|
|
1662
|
+
yield* n.promise(() => e.store.insert(d, {
|
|
1663
|
+
id: i,
|
|
1664
|
+
startedAt: F,
|
|
1665
|
+
kind: "staging",
|
|
1666
|
+
tables: String(fe.length),
|
|
1667
|
+
capture: I?.dir ?? null,
|
|
1668
|
+
via: e.via ?? "direct",
|
|
1669
|
+
staging: r.map(X).join(","),
|
|
1670
|
+
heartbeatAt: /* @__PURE__ */ new Date(),
|
|
1671
|
+
resumable: !m
|
|
1672
|
+
}).then(() => {
|
|
1673
|
+
a = !0;
|
|
1674
|
+
}, (t) => {
|
|
1675
|
+
e.onWarn?.(`the staging tables could not be recorded (${String(t?.message ?? t)}).\n The import continues and the target is unaffected — but if this run dies before the swap,
|
|
1676
|
+
nothing at boot will mention the scratch tables it leaves. \`voltro data clear-staging\` finds
|
|
1677
|
+
them by introspection.`);
|
|
1678
|
+
}));
|
|
1679
|
+
let t = Date.now();
|
|
1680
|
+
j = () => {
|
|
1681
|
+
if (!a) return;
|
|
1682
|
+
let n = Date.now();
|
|
1683
|
+
n - t < er || (t = n, e.store.update(d, i, { heartbeatAt: /* @__PURE__ */ new Date() }).then(() => void 0, () => void 0));
|
|
1684
|
+
};
|
|
1685
|
+
}
|
|
1686
|
+
let o = () => a ? e.store.delete(d, i).then(() => void 0, () => void 0) : Promise.resolve(), s = In(r, {
|
|
1687
|
+
getTable: (e) => y(e),
|
|
1660
1688
|
registerTable: (e) => {
|
|
1661
|
-
|
|
1689
|
+
te(e);
|
|
1662
1690
|
},
|
|
1663
|
-
unregisterTable:
|
|
1664
|
-
}),
|
|
1691
|
+
unregisterTable: ie
|
|
1692
|
+
}), c = !1;
|
|
1665
1693
|
try {
|
|
1666
|
-
let i =
|
|
1694
|
+
let i = m ? r : r.filter((e) => !Vt(b, e));
|
|
1667
1695
|
yield* n.tryPromise({
|
|
1668
1696
|
try: async () => {
|
|
1669
|
-
await
|
|
1697
|
+
await Bn(t, i.map((e) => Pn(e, E))), await Bn(t, i.map((e) => Nn(e, E)));
|
|
1670
1698
|
},
|
|
1671
1699
|
catch: (e) => new H({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1672
|
-
}), yield*
|
|
1673
|
-
try: () =>
|
|
1674
|
-
|
|
1700
|
+
}), yield* L(_e(e.store, !m)), !m && b.truncated === !0 ? c = !0 : yield* n.tryPromise({
|
|
1701
|
+
try: () => Vn(t, fe, E).then(() => {
|
|
1702
|
+
c = !0;
|
|
1675
1703
|
}),
|
|
1676
1704
|
catch: (e) => new H({ reason: String(e?.message ?? e) })
|
|
1677
1705
|
}).pipe(n.catchAll((i) => n.gen(function* () {
|
|
1678
|
-
let a =
|
|
1706
|
+
let a = zn(yield* n.promise(() => Hn(t, Ln(e.targetSnapshot, r), E).then((e) => e, () => [])));
|
|
1679
1707
|
return yield* n.fail(new H({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1680
|
-
}))), !
|
|
1708
|
+
}))), !m && c && b.truncated !== !0 && (b.truncated = !0, yield* J(v, b));
|
|
1681
1709
|
} finally {
|
|
1682
|
-
|
|
1710
|
+
s.undo(), c || m ? (j = void 0, yield* n.promise(() => Bn(t, r.map((e) => Pn(e, E))).then(() => void 0, () => void 0)), yield* n.promise(o)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1683
1711
|
}
|
|
1684
|
-
} else
|
|
1712
|
+
} else m ? yield* n.tryPromise({
|
|
1685
1713
|
try: () => e.store.transactional(async (e) => {
|
|
1686
|
-
let t = await ve(() => n.runPromiseExit(
|
|
1687
|
-
if (r.isFailure(t)) throw
|
|
1714
|
+
let t = await ve(() => n.runPromiseExit(_e(e, !1)));
|
|
1715
|
+
if (r.isFailure(t)) throw ir(t.cause);
|
|
1688
1716
|
}),
|
|
1689
|
-
catch: (e) =>
|
|
1690
|
-
}) : yield*
|
|
1717
|
+
catch: (e) => rr(e) ?? new H({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1718
|
+
}) : yield* L(_e(e.store, !0));
|
|
1691
1719
|
if (e.assets && s.assets.index && !e.assetsAlreadyRestored) {
|
|
1692
1720
|
let t = e.assets;
|
|
1693
|
-
yield*
|
|
1721
|
+
yield* ht({
|
|
1694
1722
|
sink: t,
|
|
1695
|
-
assetsDir: i.join(e.bundleDir,
|
|
1723
|
+
assetsDir: i.join(e.bundleDir, At),
|
|
1696
1724
|
indexPath: i.join(e.bundleDir, s.assets.index ?? "assets/index.ndjson"),
|
|
1697
1725
|
progress: {
|
|
1698
|
-
done: (e) => e in
|
|
1699
|
-
mark: (e) => (
|
|
1726
|
+
done: (e) => e in b.assets,
|
|
1727
|
+
mark: (e) => (b.assets[e] = !0, J(v, b))
|
|
1700
1728
|
}
|
|
1701
1729
|
});
|
|
1702
1730
|
}
|
|
1703
|
-
return yield* n.promise(() =>
|
|
1731
|
+
return yield* n.promise(() => Ce(null)), {
|
|
1704
1732
|
manifest: s,
|
|
1705
|
-
tablesWritten:
|
|
1706
|
-
rowsWritten:
|
|
1733
|
+
tablesWritten: A,
|
|
1734
|
+
rowsWritten: k,
|
|
1707
1735
|
tablesSkipped: N,
|
|
1708
1736
|
rowsSkipped: M,
|
|
1709
|
-
fullyResumed:
|
|
1710
|
-
...
|
|
1737
|
+
fullyResumed: A === 0 && N > 0,
|
|
1738
|
+
...I === void 0 ? {} : { rollback: I }
|
|
1711
1739
|
};
|
|
1712
1740
|
}).pipe(n.tapError((e) => n.promise(() => t?.(e?.reason ?? e?._tag ?? "the import did not finish") ?? Promise.resolve())));
|
|
1713
|
-
},
|
|
1741
|
+
}, sr = (e) => e, cr = (e) => {
|
|
1714
1742
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1715
1743
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1716
1744
|
return {
|
|
1717
1745
|
refuse: !0,
|
|
1718
1746
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1719
1747
|
};
|
|
1720
|
-
}, Q = Buffer.from("VBUNDLE2"),
|
|
1748
|
+
}, Q = Buffer.from("VBUNDLE2"), lr = 0, ur = 1, dr = 2, fr = 32, pr = async (e) => {
|
|
1721
1749
|
let t = [], n = async (r) => {
|
|
1722
1750
|
let i;
|
|
1723
1751
|
try {
|
|
1724
|
-
i = await
|
|
1752
|
+
i = await I(R(e, r), { withFileTypes: !0 });
|
|
1725
1753
|
} catch {
|
|
1726
1754
|
return;
|
|
1727
1755
|
}
|
|
@@ -1731,22 +1759,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1731
1759
|
}
|
|
1732
1760
|
};
|
|
1733
1761
|
return await n(""), t.sort();
|
|
1734
|
-
},
|
|
1762
|
+
}, mr = (e, t) => {
|
|
1735
1763
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1736
|
-
return r.writeUInt8(
|
|
1737
|
-
},
|
|
1764
|
+
return r.writeUInt8(ur, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1765
|
+
}, hr = (e, t, n) => {
|
|
1738
1766
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1739
|
-
return a.writeUInt8(
|
|
1767
|
+
return a.writeUInt8(dr, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1740
1768
|
};
|
|
1741
|
-
async function*
|
|
1769
|
+
async function* gr(e) {
|
|
1742
1770
|
yield new Uint8Array(Q);
|
|
1743
|
-
for (let t of await
|
|
1771
|
+
for (let t of await pr(e.dir)) {
|
|
1744
1772
|
let n = R(e.dir, t), r = await ve(n);
|
|
1745
|
-
if (yield new Uint8Array(
|
|
1773
|
+
if (yield new Uint8Array(mr(t, r.size)), r.size > 0) for await (let e of de(n)) yield new Uint8Array(e);
|
|
1746
1774
|
}
|
|
1747
1775
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1748
|
-
yield new Uint8Array(
|
|
1749
|
-
let n =
|
|
1776
|
+
yield new Uint8Array(hr(t.key, t.contentType, t.size));
|
|
1777
|
+
let n = T("sha256"), r = 0;
|
|
1750
1778
|
for await (let i of e.blobs.open(t.key)) {
|
|
1751
1779
|
let e = Buffer.from(i);
|
|
1752
1780
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1754,11 +1782,11 @@ async function* mr(e) {
|
|
|
1754
1782
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1755
1783
|
yield new Uint8Array(n.digest());
|
|
1756
1784
|
}
|
|
1757
|
-
yield new Uint8Array(Buffer.from([
|
|
1785
|
+
yield new Uint8Array(Buffer.from([lr]));
|
|
1758
1786
|
}
|
|
1759
|
-
var
|
|
1760
|
-
let n =
|
|
1761
|
-
await
|
|
1787
|
+
var _r = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), vr = (e) => new Promise((t) => e.end(t)), yr = async (e, t) => {
|
|
1788
|
+
let n = xe(t.dir);
|
|
1789
|
+
await F(n, { recursive: !0 });
|
|
1762
1790
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1763
1791
|
let { value: e, done: t } = await i.next();
|
|
1764
1792
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1773,24 +1801,24 @@ var hr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), gr
|
|
|
1773
1801
|
for (;;) {
|
|
1774
1802
|
await s(1);
|
|
1775
1803
|
let e = c(1).readUInt8(0);
|
|
1776
|
-
if (e ===
|
|
1777
|
-
if (e ===
|
|
1804
|
+
if (e === lr) return;
|
|
1805
|
+
if (e === ur) {
|
|
1778
1806
|
await s(4);
|
|
1779
1807
|
let e = c(4).readUInt32BE(0);
|
|
1780
1808
|
await s(e);
|
|
1781
1809
|
let t = c(e).toString("utf8");
|
|
1782
1810
|
await s(8);
|
|
1783
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1784
|
-
if (a !== n && !a.startsWith(n +
|
|
1785
|
-
await
|
|
1786
|
-
let o =
|
|
1811
|
+
let i = c(8).readBigUInt64BE(0), a = xe(n, t);
|
|
1812
|
+
if (a !== n && !a.startsWith(n + Se)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1813
|
+
await F(be(a), { recursive: !0 });
|
|
1814
|
+
let o = D(a);
|
|
1787
1815
|
for (; i > 0n;) {
|
|
1788
1816
|
r.length === 0 && await s(1);
|
|
1789
1817
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1790
|
-
await
|
|
1818
|
+
await _r(o, c(e)), i -= BigInt(e);
|
|
1791
1819
|
}
|
|
1792
|
-
await
|
|
1793
|
-
} else if (e ===
|
|
1820
|
+
await vr(o);
|
|
1821
|
+
} else if (e === dr) {
|
|
1794
1822
|
await s(4);
|
|
1795
1823
|
let e = c(4).readUInt32BE(0);
|
|
1796
1824
|
await s(e);
|
|
@@ -1800,24 +1828,24 @@ var hr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), gr
|
|
|
1800
1828
|
await s(i);
|
|
1801
1829
|
let a = c(i).toString("utf8");
|
|
1802
1830
|
await s(8);
|
|
1803
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1831
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = T("sha256"), d = new N(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1804
1832
|
contentType: a,
|
|
1805
1833
|
size: Number(o)
|
|
1806
1834
|
}), p = o;
|
|
1807
1835
|
for (; p > 0n;) {
|
|
1808
1836
|
r.length === 0 && await s(1);
|
|
1809
1837
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1810
|
-
u.update(t), l || await
|
|
1838
|
+
u.update(t), l || await _r(d, t), p -= BigInt(e);
|
|
1811
1839
|
}
|
|
1812
|
-
l ? d.destroy() : await
|
|
1813
|
-
let m = c(
|
|
1840
|
+
l ? d.destroy() : await vr(d), await f, await s(fr);
|
|
1841
|
+
let m = c(fr);
|
|
1814
1842
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1815
1843
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1816
1844
|
}
|
|
1817
1845
|
} finally {
|
|
1818
1846
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1819
1847
|
}
|
|
1820
|
-
},
|
|
1848
|
+
}, br = async (e, t) => {
|
|
1821
1849
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1822
1850
|
let { value: e, done: t } = await a.next();
|
|
1823
1851
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1839,8 +1867,8 @@ var hr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), gr
|
|
|
1839
1867
|
if (!await s(Q.length) || !c(Q.length).equals(Q)) throw Error("not a bundle archive (bad magic)");
|
|
1840
1868
|
for (; await s(1);) {
|
|
1841
1869
|
let e = c(1).readUInt8(0);
|
|
1842
|
-
if (e ===
|
|
1843
|
-
if (e !==
|
|
1870
|
+
if (e === lr || e === dr) break;
|
|
1871
|
+
if (e !== ur) throw Error(`unknown bundle entry kind ${e}`);
|
|
1844
1872
|
if (!await s(4)) break;
|
|
1845
1873
|
let t = c(4).readUInt32BE(0);
|
|
1846
1874
|
if (!await s(t)) break;
|
|
@@ -1850,20 +1878,20 @@ var hr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), gr
|
|
|
1850
1878
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1851
1879
|
}
|
|
1852
1880
|
return r;
|
|
1853
|
-
},
|
|
1881
|
+
}, xr = (e, t) => yr(e, {
|
|
1854
1882
|
dir: t,
|
|
1855
|
-
blobs:
|
|
1856
|
-
}),
|
|
1883
|
+
blobs: Sr(t)
|
|
1884
|
+
}), Sr = (e) => {
|
|
1857
1885
|
let t = /* @__PURE__ */ new Set();
|
|
1858
1886
|
return {
|
|
1859
1887
|
has: async (e) => t.has(e),
|
|
1860
1888
|
write: async (n, r, i) => {
|
|
1861
1889
|
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = R(e, "assets");
|
|
1862
|
-
await
|
|
1863
|
-
let l =
|
|
1890
|
+
await F(c, { recursive: !0 });
|
|
1891
|
+
let l = T("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1864
1892
|
l.update(e), u += e.length, n(null, e);
|
|
1865
1893
|
} }), f = R(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1866
|
-
await o(r, d,
|
|
1894
|
+
await o(r, d, D(f));
|
|
1867
1895
|
let p = l.digest("hex");
|
|
1868
1896
|
await a.rename(f, R(c, p)), await a.appendFile(R(c, "index.ndjson"), `${JSON.stringify({
|
|
1869
1897
|
key: n,
|
|
@@ -1873,33 +1901,33 @@ var hr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), gr
|
|
|
1873
1901
|
})}\n`), t.add(n);
|
|
1874
1902
|
}
|
|
1875
1903
|
};
|
|
1876
|
-
},
|
|
1904
|
+
}, Cr = Ce(ue), wr = Buffer.from("VENC1\0\0\0"), Tr = 32, Er = 16, Dr = 7, Or = 16, kr = 262144, Ar = {
|
|
1877
1905
|
N: 32768,
|
|
1878
1906
|
r: 8,
|
|
1879
1907
|
p: 1
|
|
1880
|
-
},
|
|
1908
|
+
}, jr = 134217728, Mr = 43, Nr = (e, t, n) => {
|
|
1881
1909
|
let r = Buffer.alloc(12);
|
|
1882
|
-
return e.copy(r, 0, 0,
|
|
1910
|
+
return e.copy(r, 0, 0, Dr), r.writeUInt32BE(t >>> 0, Dr), r.writeUInt8(+!!n, 11), r;
|
|
1883
1911
|
};
|
|
1884
|
-
async function*
|
|
1912
|
+
async function* Pr(e, t) {
|
|
1885
1913
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1886
|
-
let n =
|
|
1887
|
-
...
|
|
1888
|
-
maxmem:
|
|
1889
|
-
}), a = Buffer.alloc(
|
|
1890
|
-
|
|
1914
|
+
let n = ce(Er), r = ce(Dr), i = await Cr(t, n, Tr, {
|
|
1915
|
+
...Ar,
|
|
1916
|
+
maxmem: jr
|
|
1917
|
+
}), a = Buffer.alloc(Mr);
|
|
1918
|
+
wr.copy(a, 0), a.writeUInt32BE(Ar.N, 8), a.writeUInt32BE(Ar.r, 12), a.writeUInt32BE(Ar.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1891
1919
|
let o = 0, s = (e, t) => {
|
|
1892
|
-
let n =
|
|
1920
|
+
let n = ae("aes-256-gcm", i, Nr(r, o, t));
|
|
1893
1921
|
o === 0 && n.setAAD(a);
|
|
1894
1922
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1895
1923
|
o++;
|
|
1896
|
-
let l = Buffer.alloc(5 + s.length +
|
|
1897
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
1924
|
+
let l = Buffer.alloc(5 + s.length + Or);
|
|
1925
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + Or, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1898
1926
|
}, c = Buffer.alloc(0);
|
|
1899
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1927
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > kr;) yield s(c.subarray(0, kr), !1), c = c.subarray(kr);
|
|
1900
1928
|
yield s(c, !0);
|
|
1901
1929
|
}
|
|
1902
|
-
async function*
|
|
1930
|
+
async function* Fr(e, t) {
|
|
1903
1931
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1904
1932
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1905
1933
|
let { value: e, done: t } = await n.next();
|
|
@@ -1908,16 +1936,16 @@ async function* Nr(e, t) {
|
|
|
1908
1936
|
for (; r.length < e && !i && await a(););
|
|
1909
1937
|
return r.length >= e;
|
|
1910
1938
|
};
|
|
1911
|
-
if (!await o(
|
|
1912
|
-
let s = r.subarray(0,
|
|
1913
|
-
if (!s.subarray(0, 8).equals(
|
|
1914
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36,
|
|
1939
|
+
if (!await o(Mr)) throw Error("encrypted bundle truncated (no header)");
|
|
1940
|
+
let s = r.subarray(0, Mr);
|
|
1941
|
+
if (!s.subarray(0, 8).equals(wr)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1942
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Mr)), p = await Cr(t, d, Tr, {
|
|
1915
1943
|
N: c,
|
|
1916
1944
|
r: l,
|
|
1917
1945
|
p: u,
|
|
1918
|
-
maxmem:
|
|
1946
|
+
maxmem: jr
|
|
1919
1947
|
});
|
|
1920
|
-
r = r.subarray(
|
|
1948
|
+
r = r.subarray(Mr);
|
|
1921
1949
|
let m = 0, h = !1;
|
|
1922
1950
|
for (;;) {
|
|
1923
1951
|
if (!await o(5)) {
|
|
@@ -1926,11 +1954,11 @@ async function* Nr(e, t) {
|
|
|
1926
1954
|
}
|
|
1927
1955
|
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
1928
1956
|
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
1929
|
-
if (t <
|
|
1957
|
+
if (t < Or) throw Error("encrypted bundle corrupt (short frame)");
|
|
1930
1958
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1931
|
-
let n = r.subarray(5, 5 + t -
|
|
1959
|
+
let n = r.subarray(5, 5 + t - Or), i = r.subarray(5 + t - Or, 5 + t);
|
|
1932
1960
|
r = r.subarray(5 + t);
|
|
1933
|
-
let a =
|
|
1961
|
+
let a = oe("aes-256-gcm", p, Nr(f, m, e));
|
|
1934
1962
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1935
1963
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1936
1964
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1938,7 +1966,7 @@ async function* Nr(e, t) {
|
|
|
1938
1966
|
}
|
|
1939
1967
|
//#endregion
|
|
1940
1968
|
//#region src/blobStorage.ts
|
|
1941
|
-
var
|
|
1969
|
+
var Ir = (e, t) => ({
|
|
1942
1970
|
list: () => {
|
|
1943
1971
|
let e = t();
|
|
1944
1972
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1951,32 +1979,32 @@ var Pr = (e, t) => ({
|
|
|
1951
1979
|
} };
|
|
1952
1980
|
},
|
|
1953
1981
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1954
|
-
let r = await n.runPromise(A(e, t)), i = await
|
|
1982
|
+
let r = await n.runPromise(A(e, t)), i = await j(r.stream);
|
|
1955
1983
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1956
1984
|
} })
|
|
1957
|
-
}),
|
|
1958
|
-
table:
|
|
1985
|
+
}), Lr = (e, t) => Ir(e, () => C(t, {
|
|
1986
|
+
table: k,
|
|
1959
1987
|
chunkSize: 500
|
|
1960
1988
|
}).pipe(l.map((e) => ({
|
|
1961
1989
|
key: String(e.key),
|
|
1962
1990
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1963
1991
|
size: Number(e.size ?? 0)
|
|
1964
|
-
})))),
|
|
1992
|
+
})))), Rr = (e) => ({
|
|
1965
1993
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1966
1994
|
write: async (t, r, i) => {
|
|
1967
|
-
let a =
|
|
1968
|
-
await n.runPromise(
|
|
1995
|
+
let a = he("storage", () => _e.from(r));
|
|
1996
|
+
await n.runPromise(ge(e, t, a, {
|
|
1969
1997
|
contentType: i.contentType,
|
|
1970
1998
|
totalSize: i.size
|
|
1971
1999
|
}));
|
|
1972
2000
|
}
|
|
1973
|
-
}),
|
|
2001
|
+
}), zr = (e, t) => {
|
|
1974
2002
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1975
|
-
return e === "mariadb" ?
|
|
2003
|
+
return e === "mariadb" ? Br(n) ? { tool: n } : {
|
|
1976
2004
|
tool: r,
|
|
1977
2005
|
hint: `'${n}' is not on PATH, so this ran '${r}' instead. If that binary came from Oracle's MySQL client it queries information_schema.COLUMN_STATISTICS, which MariaDB does not have, and the run dies with error 1109 after the first table. Install the MariaDB client package (which ships '${n}') and re-run. Do NOT reach for --column-statistics=0: that flag does not exist on '${n}', so it fixes the wrong client and breaks the right one. Note: a MariaDB 12.x client requires TLS by default, so running '${n}' BY HAND against a server without it fails with "TLS/SSL error: SSL is required" (2026) and needs --skip-ssl. This command is not affected.`
|
|
1978
2006
|
} : { tool: r };
|
|
1979
|
-
},
|
|
2007
|
+
}, Br = (e) => (process.env.PATH ?? "").split(ye).filter((e) => e !== "").some((t) => fe(R(t, e))), Vr = (e) => {
|
|
1980
2008
|
switch (e) {
|
|
1981
2009
|
case "postgres": return "db.dump";
|
|
1982
2010
|
case "mysql":
|
|
@@ -1986,7 +2014,7 @@ var Pr = (e, t) => ({
|
|
|
1986
2014
|
case "mssql": return "db.bacpac";
|
|
1987
2015
|
default: return "db.dump";
|
|
1988
2016
|
}
|
|
1989
|
-
},
|
|
2017
|
+
}, Hr = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1990
2018
|
if (e.host || e.database) return {
|
|
1991
2019
|
host: e.host ?? "localhost",
|
|
1992
2020
|
port: e.port ?? 3306,
|
|
@@ -2002,7 +2030,7 @@ var Pr = (e, t) => ({
|
|
|
2002
2030
|
password: decodeURIComponent(t.password) || "",
|
|
2003
2031
|
database: t.pathname.replace(/^\//, "") || ""
|
|
2004
2032
|
};
|
|
2005
|
-
},
|
|
2033
|
+
}, Ur = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Wr = (e, t, n) => {
|
|
2006
2034
|
switch (e) {
|
|
2007
2035
|
case "postgres": {
|
|
2008
2036
|
if (t.url) return {
|
|
@@ -2012,7 +2040,7 @@ var Pr = (e, t) => ({
|
|
|
2012
2040
|
"--format=custom",
|
|
2013
2041
|
"--no-owner",
|
|
2014
2042
|
"--no-privileges",
|
|
2015
|
-
...
|
|
2043
|
+
...Kr,
|
|
2016
2044
|
"--dbname",
|
|
2017
2045
|
t.url,
|
|
2018
2046
|
"--file",
|
|
@@ -2031,7 +2059,7 @@ var Pr = (e, t) => ({
|
|
|
2031
2059
|
"--format=custom",
|
|
2032
2060
|
"--no-owner",
|
|
2033
2061
|
"--no-privileges",
|
|
2034
|
-
...
|
|
2062
|
+
...Kr,
|
|
2035
2063
|
"-h",
|
|
2036
2064
|
e.host,
|
|
2037
2065
|
"-p",
|
|
@@ -2051,14 +2079,14 @@ var Pr = (e, t) => ({
|
|
|
2051
2079
|
let r = $(t);
|
|
2052
2080
|
return {
|
|
2053
2081
|
kind: "spawn",
|
|
2054
|
-
...
|
|
2082
|
+
...zr(e, "dump"),
|
|
2055
2083
|
args: [
|
|
2056
2084
|
"--single-transaction",
|
|
2057
2085
|
"--routines",
|
|
2058
2086
|
"--triggers",
|
|
2059
2087
|
"--no-tablespaces",
|
|
2060
2088
|
"--protocol=TCP",
|
|
2061
|
-
...
|
|
2089
|
+
...Gr.map((e) => `--ignore-table=${r.database}.${e}`),
|
|
2062
2090
|
"-h",
|
|
2063
2091
|
r.host,
|
|
2064
2092
|
"-P",
|
|
@@ -2074,7 +2102,7 @@ var Pr = (e, t) => ({
|
|
|
2074
2102
|
case "sqlite":
|
|
2075
2103
|
case "turso": return {
|
|
2076
2104
|
kind: "copy",
|
|
2077
|
-
from:
|
|
2105
|
+
from: Hr(t),
|
|
2078
2106
|
to: n
|
|
2079
2107
|
};
|
|
2080
2108
|
case "mssql": return {
|
|
@@ -2082,7 +2110,7 @@ var Pr = (e, t) => ({
|
|
|
2082
2110
|
tool: "sqlpackage",
|
|
2083
2111
|
args: [
|
|
2084
2112
|
"/Action:Export",
|
|
2085
|
-
`/SourceConnectionString:${
|
|
2113
|
+
`/SourceConnectionString:${Ur($({
|
|
2086
2114
|
...t,
|
|
2087
2115
|
port: t.port ?? 1433
|
|
2088
2116
|
}))}`,
|
|
@@ -2095,7 +2123,7 @@ var Pr = (e, t) => ({
|
|
|
2095
2123
|
reason: `no native backup for dialect '${e}'`
|
|
2096
2124
|
});
|
|
2097
2125
|
}
|
|
2098
|
-
},
|
|
2126
|
+
}, Gr = [d, u], Kr = Gr.flatMap((e) => ["--exclude-table", e]), qr = (e) => e === "postgres" || e === "mysql" || e === "mariadb" ? Gr : [], Jr = (e, t, n) => {
|
|
2099
2127
|
switch (e) {
|
|
2100
2128
|
case "postgres": {
|
|
2101
2129
|
if (t.url) return {
|
|
@@ -2140,7 +2168,7 @@ var Pr = (e, t) => ({
|
|
|
2140
2168
|
let r = $(t);
|
|
2141
2169
|
return {
|
|
2142
2170
|
kind: "spawn",
|
|
2143
|
-
...
|
|
2171
|
+
...zr(e, "client"),
|
|
2144
2172
|
args: [
|
|
2145
2173
|
"--protocol=TCP",
|
|
2146
2174
|
"-h",
|
|
@@ -2159,14 +2187,14 @@ var Pr = (e, t) => ({
|
|
|
2159
2187
|
case "turso": return {
|
|
2160
2188
|
kind: "copy",
|
|
2161
2189
|
from: n,
|
|
2162
|
-
to:
|
|
2190
|
+
to: Hr(t)
|
|
2163
2191
|
};
|
|
2164
2192
|
case "mssql": return {
|
|
2165
2193
|
kind: "spawn",
|
|
2166
2194
|
tool: "sqlpackage",
|
|
2167
2195
|
args: [
|
|
2168
2196
|
"/Action:Import",
|
|
2169
|
-
`/TargetConnectionString:${
|
|
2197
|
+
`/TargetConnectionString:${Ur($({
|
|
2170
2198
|
...t,
|
|
2171
2199
|
port: t.port ?? 1433
|
|
2172
2200
|
}))}`,
|
|
@@ -2179,22 +2207,22 @@ var Pr = (e, t) => ({
|
|
|
2179
2207
|
reason: `no native restore for dialect '${e}'`
|
|
2180
2208
|
});
|
|
2181
2209
|
}
|
|
2182
|
-
},
|
|
2183
|
-
let r = `${e.to}.restoring-${
|
|
2210
|
+
}, Yr = (e) => e.kind === "copy" ? n.async((t) => {
|
|
2211
|
+
let r = `${e.to}.restoring-${le()}`, i = (e) => {
|
|
2184
2212
|
try {
|
|
2185
|
-
|
|
2213
|
+
me(r);
|
|
2186
2214
|
} catch {}
|
|
2187
2215
|
t(n.fail(new W({
|
|
2188
2216
|
tool: "copy",
|
|
2189
2217
|
reason: e
|
|
2190
2218
|
})));
|
|
2191
2219
|
};
|
|
2192
|
-
|
|
2220
|
+
E(e.from, r, (a) => {
|
|
2193
2221
|
if (a) {
|
|
2194
2222
|
i(`copy ${e.from} → ${e.to}: ${a.message}`);
|
|
2195
2223
|
return;
|
|
2196
2224
|
}
|
|
2197
|
-
|
|
2225
|
+
pe(r, e.to, (r) => {
|
|
2198
2226
|
if (r) {
|
|
2199
2227
|
i(`copy ${e.from} → ${e.to}: ${r.message}`);
|
|
2200
2228
|
return;
|
|
@@ -2203,7 +2231,7 @@ var Pr = (e, t) => ({
|
|
|
2203
2231
|
});
|
|
2204
2232
|
});
|
|
2205
2233
|
}) : n.async((t) => {
|
|
2206
|
-
let r =
|
|
2234
|
+
let r = we(e.tool, [...e.args], {
|
|
2207
2235
|
env: {
|
|
2208
2236
|
...process.env,
|
|
2209
2237
|
...e.env
|
|
@@ -2223,23 +2251,23 @@ var Pr = (e, t) => ({
|
|
|
2223
2251
|
s ??= e;
|
|
2224
2252
|
};
|
|
2225
2253
|
if (e.stdoutFile && r.stdout) {
|
|
2226
|
-
let t =
|
|
2227
|
-
t.on("error", (t) => c(`writing ${
|
|
2254
|
+
let t = D(e.stdoutFile);
|
|
2255
|
+
t.on("error", (t) => c(`writing ${L(e.stdoutFile)}: ${t.message}`)), r.stdout.on("error", (e) => c(`reading the tool's output: ${e.message}`)), r.stdout.pipe(t);
|
|
2228
2256
|
}
|
|
2229
2257
|
let l = 0;
|
|
2230
2258
|
if (e.stdinFile && r.stdin) {
|
|
2231
|
-
let t =
|
|
2259
|
+
let t = de(e.stdinFile);
|
|
2232
2260
|
t.on("data", (e) => {
|
|
2233
2261
|
l += e.length;
|
|
2234
2262
|
}), t.on("error", (t) => {
|
|
2235
|
-
c(`reading ${
|
|
2263
|
+
c(`reading ${L(e.stdinFile)}: ${t.message}`), r.stdin?.end();
|
|
2236
2264
|
}), r.stdin.on("error", (t) => {
|
|
2237
|
-
t.code !== "EPIPE" && c(`feeding ${
|
|
2265
|
+
t.code !== "EPIPE" && c(`feeding ${L(e.stdinFile)} to ${e.tool}: ${t.message}`);
|
|
2238
2266
|
}), t.pipe(r.stdin);
|
|
2239
2267
|
}
|
|
2240
2268
|
let u = () => {
|
|
2241
2269
|
if (e.stdoutFile !== void 0) try {
|
|
2242
|
-
|
|
2270
|
+
me(e.stdoutFile);
|
|
2243
2271
|
} catch {}
|
|
2244
2272
|
};
|
|
2245
2273
|
r.on("error", (t) => {
|
|
@@ -2254,25 +2282,25 @@ var Pr = (e, t) => ({
|
|
|
2254
2282
|
return;
|
|
2255
2283
|
}
|
|
2256
2284
|
u();
|
|
2257
|
-
let r = e.stdinFile === void 0 ? "" : ` — ${l} of ${
|
|
2285
|
+
let r = e.stdinFile === void 0 ? "" : ` — ${l} of ${Xr(e.stdinFile)} byte(s) of ${L(e.stdinFile)} had been fed to it`;
|
|
2258
2286
|
o(n.fail(new W({
|
|
2259
2287
|
tool: e.tool,
|
|
2260
|
-
reason: (t === 0 ? `exited cleanly, but ${s}` : `exited with code ${t}${s === void 0 ? "" : ` (${s})`}`) + r + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2288
|
+
reason: (t === 0 ? `exited cleanly, but ${s}` : `exited with code ${t}${s === void 0 ? "" : ` (${s})`}`) + r + (e.stdoutFile === void 0 ? "" : ` — the partial ${L(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
2261
2289
|
stderr: i.trim() === "" ? `(${e.tool} printed nothing to stderr)` : i.slice(-2e3),
|
|
2262
2290
|
...e.hint === void 0 ? {} : { hint: e.hint }
|
|
2263
2291
|
})));
|
|
2264
2292
|
});
|
|
2265
|
-
}),
|
|
2293
|
+
}), Xr = (e) => {
|
|
2266
2294
|
try {
|
|
2267
|
-
return String(
|
|
2295
|
+
return String(O(e).size);
|
|
2268
2296
|
} catch {
|
|
2269
2297
|
return "?";
|
|
2270
2298
|
}
|
|
2271
|
-
},
|
|
2272
|
-
let t = yield*
|
|
2299
|
+
}, Zr = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Qr = (e) => n.gen(function* () {
|
|
2300
|
+
let t = yield* ot(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
2273
2301
|
store: e.store,
|
|
2274
2302
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
2275
|
-
}), r =
|
|
2303
|
+
}), r = nn(t.tables, e.snapshot), i = e.classification ?? De(e.snapshot), a = e.sampleSize ?? 20, o = Be(r.map((t) => ({
|
|
2276
2304
|
name: t,
|
|
2277
2305
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
2278
2306
|
name: e.name,
|
|
@@ -2284,15 +2312,15 @@ var Pr = (e, t) => ({
|
|
|
2284
2312
|
table: i,
|
|
2285
2313
|
chunkSize: a,
|
|
2286
2314
|
...r === void 0 ? {} : { where: r }
|
|
2287
|
-
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
2288
|
-
c.push(...
|
|
2315
|
+
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) => Ue(t, i, o.actions, e.masking.seed));
|
|
2316
|
+
c.push(...He(d, i, o.actions));
|
|
2289
2317
|
let f = u[0], p = d[0], m = [];
|
|
2290
2318
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
2291
2319
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
2292
2320
|
let n = e.slice(i.length + 1);
|
|
2293
2321
|
m.push({
|
|
2294
2322
|
column: n,
|
|
2295
|
-
action:
|
|
2323
|
+
action: Zr(t),
|
|
2296
2324
|
before: f[n],
|
|
2297
2325
|
after: p[n]
|
|
2298
2326
|
});
|
|
@@ -2307,7 +2335,7 @@ var Pr = (e, t) => ({
|
|
|
2307
2335
|
leaks: c,
|
|
2308
2336
|
unclassified: o.unclassified
|
|
2309
2337
|
};
|
|
2310
|
-
}),
|
|
2338
|
+
}), $r = (e, t) => {
|
|
2311
2339
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
2312
2340
|
for (let t of e) {
|
|
2313
2341
|
let e = n.get(t.name);
|
|
@@ -2362,7 +2390,7 @@ var Pr = (e, t) => ({
|
|
|
2362
2390
|
skipColumns: i,
|
|
2363
2391
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
2364
2392
|
};
|
|
2365
|
-
},
|
|
2393
|
+
}, ei = (e) => {
|
|
2366
2394
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
2367
2395
|
let t = {
|
|
2368
2396
|
safe: "✓",
|
|
@@ -2373,4 +2401,4 @@ var Pr = (e, t) => ({
|
|
|
2373
2401
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
2374
2402
|
};
|
|
2375
2403
|
//#endregion
|
|
2376
|
-
export {
|
|
2404
|
+
export { At as ASSETS_DIR, jt as ASSET_INDEX_FILE, H as BundleError, Je as CodecError, Xe as CompressionError, tt as CrossDialectError, kt as DATA_DIR, Ee as DEFAULT_CLASS_ACTIONS, We as ENVIRONMENT_LOCAL_TABLES, Ge as ENVIRONMENT_PORTABLE_TABLES, Gr as EXCLUDED_FROM_DUMP, xt as FORMAT_VERSION, et as ImportModeError, Ye as IntegrityError, Mt as LEDGER_FILE, K as MANIFEST_FILE, $e as MaskingError, W as NativeToolError, Ze as RowsRefusedError, tr as SAVEPOINT_BATCH_SIZE, kn as STAGING_PREFIX, Qe as SchemaDriftError, U as ScopeError, Re as applyAction, rr as asImportError, cr as assessLiveImport, Ht as assetDone, Vr as backupArtifactName, Wr as backupCommand, An as canStage, _n as canonicalType, pt as captureAssetsToCas, bn as checkPortability, Qn as checkSchemaDrift, De as classificationFromSnapshot, $r as classifyImportDrift, vt as compressionExt, rt as computeSubsetIds, Gn as decideStaging, Dt as decodeManifest, Qt as decodeRow, Xt as decodeValue, Fr as decryptStream, sr as defineDataProfile, Zn as diffSnapshots, Sr as dirBlobSink, Pn as dropStagingSql, qr as dumpExcludedTables, zt as emptyLedger, Et as encodeManifest, Zt as encodeRow, Jt as encodeValue, Pr as encryptStream, qe as environmentLocalNotice, Ke as environmentLocalReason, Bn as execStatements, ei as formatImportDrift, xn as formatIssue, gt as hasZstd, V as isEnvironmentLocal, yt as makeCompressor, bt as makeDecompressor, Ue as maskRow, tn as ndjsonToRows, dt as noAssetProgress, gn as outsideReferenceRefusal, hn as outsideReferencesInto, gr as packBundle, br as peekBundle, _t as pickCompression, Be as planMasking, Qr as previewMasking, mt as readAssetIndex, Bt as readLedger, Oe as resolveAction, ot as resolveScope, ut as restoreAssetFromCas, ht as restoreAssetsFromCas, Jr as restoreCommand, en as rowsToNdjson, mn as runExport, or as runImport, Yr as runNativeStep, He as scanForLeaks, X as stagingNameFor, ct as storageAssetSink, st as storageAssetSource, Rr as storageBlobSink, Ir as storageBlobSource, Lr as storageRefsBlobSource, it as subsetToScope, Vt as tableDone, nn as topoSortTables, yr as unpackBundle, xr as unpackBundleToDir, lt as writeAssetToCas, J as writeLedger };
|