@voltro/data-transfer 0.49.0 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/dist/index.d.ts +74 -0
- package/dist/index.js +447 -394
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Runtime as o, Schedule as s, Schema as c, Stream as l } from "effect";
|
|
2
2
|
import { DATA_TRANSFERS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d, _voltroDataTransfersTable as f, classifyConstraintViolation as p, derivedRowId as m, ensureTableRegistered as h, eq as g, extractDbCause as _, getTable as v, inSet as y, isNotNull as b, or as x, recordsTable as S, registerTable as C, streamTable as w, suspendWriteRecorders as ee, suspendWriteRecordersEffect as te, unregisterTable as ne } from "@voltro/database";
|
|
3
3
|
import { fingerprintSchema as re } from "@voltro/database/sql";
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import {
|
|
6
|
-
import
|
|
7
|
-
import
|
|
8
|
-
import {
|
|
9
|
-
import { SqlClient as
|
|
10
|
-
import { mkdir as
|
|
11
|
-
import { basename as
|
|
12
|
-
import { promisify as
|
|
13
|
-
import { spawn as
|
|
4
|
+
import { createCipheriv as ie, createDecipheriv as T, createHash as E, createHmac as ae, randomBytes as D, randomUUID as oe, scrypt as se } from "node:crypto";
|
|
5
|
+
import { copyFile as ce, createReadStream as le, createWriteStream as O, existsSync as k, rename as ue, unlinkSync as de } from "node:fs";
|
|
6
|
+
import { STORAGE_REFS_TABLE as A, fromNodeReadable as fe, getObjectStream as pe, putObjectStream as j, toNodeReadable as M } from "@voltro/plugin-storage";
|
|
7
|
+
import * as N from "node:zlib";
|
|
8
|
+
import { PassThrough as P, Readable as me } from "node:stream";
|
|
9
|
+
import { SqlClient as F } from "@effect/sql";
|
|
10
|
+
import { mkdir as he, readdir as ge, stat as I } from "node:fs/promises";
|
|
11
|
+
import { basename as _e, delimiter as ve, dirname as L, join as R, resolve as ye, sep as be } from "node:path";
|
|
12
|
+
import { promisify as xe } from "node:util";
|
|
13
|
+
import { spawn as Se } from "node:child_process";
|
|
14
14
|
//#region src/consistency.ts
|
|
15
|
-
var
|
|
15
|
+
var Ce = (e) => {
|
|
16
16
|
switch (e) {
|
|
17
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
18
18
|
case "mysql":
|
|
@@ -22,7 +22,7 @@ var xe = (e) => {
|
|
|
22
22
|
case "turso": return [];
|
|
23
23
|
default: return [];
|
|
24
24
|
}
|
|
25
|
-
},
|
|
25
|
+
}, we = {
|
|
26
26
|
email: { fake: "email" },
|
|
27
27
|
fullName: { fake: "fullName" },
|
|
28
28
|
firstName: { fake: "firstName" },
|
|
@@ -37,7 +37,7 @@ var xe = (e) => {
|
|
|
37
37
|
date: "dateShift",
|
|
38
38
|
secret: "null",
|
|
39
39
|
freeText: "redact"
|
|
40
|
-
},
|
|
40
|
+
}, Te = (e) => {
|
|
41
41
|
let t = {};
|
|
42
42
|
for (let n of e.tables) {
|
|
43
43
|
let e = {};
|
|
@@ -49,15 +49,15 @@ var xe = (e) => {
|
|
|
49
49
|
t[n.name] = e;
|
|
50
50
|
}
|
|
51
51
|
return t;
|
|
52
|
-
},
|
|
52
|
+
}, Ee = (e, t, n, r) => {
|
|
53
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
54
54
|
if (i) return i;
|
|
55
55
|
if (n?.sensitive) {
|
|
56
56
|
let e = n.sensitive.class;
|
|
57
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? we[e] ?? "redact";
|
|
58
58
|
}
|
|
59
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
60
|
-
},
|
|
60
|
+
}, De = (e, t) => ae("sha256", e).update(t == null ? "∅" : String(t)).digest(), z = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), Oe = [
|
|
61
61
|
"Ada",
|
|
62
62
|
"Alan",
|
|
63
63
|
"Grace",
|
|
@@ -70,7 +70,7 @@ var xe = (e) => {
|
|
|
70
70
|
"Guido",
|
|
71
71
|
"Margaret",
|
|
72
72
|
"Bjarne"
|
|
73
|
-
],
|
|
73
|
+
], ke = [
|
|
74
74
|
"Lovelace",
|
|
75
75
|
"Turing",
|
|
76
76
|
"Hopper",
|
|
@@ -83,7 +83,7 @@ var xe = (e) => {
|
|
|
83
83
|
"Rossum",
|
|
84
84
|
"Hamilton",
|
|
85
85
|
"Stroustrup"
|
|
86
|
-
],
|
|
86
|
+
], Ae = [
|
|
87
87
|
"Maple",
|
|
88
88
|
"Oak",
|
|
89
89
|
"Cedar",
|
|
@@ -92,7 +92,7 @@ var xe = (e) => {
|
|
|
92
92
|
"Birch",
|
|
93
93
|
"Willow",
|
|
94
94
|
"Ash"
|
|
95
|
-
],
|
|
95
|
+
], je = [
|
|
96
96
|
"Initech",
|
|
97
97
|
"Hooli",
|
|
98
98
|
"Umbrella",
|
|
@@ -101,23 +101,23 @@ var xe = (e) => {
|
|
|
101
101
|
"Soylent",
|
|
102
102
|
"Stark",
|
|
103
103
|
"Wayne"
|
|
104
|
-
], B = (e, t, n) => e[z(t, n) % e.length],
|
|
105
|
-
let r =
|
|
104
|
+
], B = (e, t, n) => e[z(t, n) % e.length], Me = (e, t, n) => {
|
|
105
|
+
let r = De(n, `${e}:${String(t)}`), i = z(r, 1) % 1e4;
|
|
106
106
|
switch (e) {
|
|
107
|
-
case "email": return `${B(
|
|
108
|
-
case "fullName": return `${B(
|
|
109
|
-
case "firstName": return B(
|
|
110
|
-
case "lastName": return B(
|
|
111
|
-
case "username": return `${B(
|
|
107
|
+
case "email": return `${B(Oe, r, 0).toLowerCase()}.${B(ke, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${B(Oe, r, 0)} ${B(ke, r, 1)}`;
|
|
109
|
+
case "firstName": return B(Oe, r, 0);
|
|
110
|
+
case "lastName": return B(ke, r, 1);
|
|
111
|
+
case "username": return `${B(Oe, r, 0).toLowerCase()}${i}`;
|
|
112
112
|
case "phone": return `+1${String(z(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
-
case "address": return `${z(r, 0) % 9899 + 100} ${B(
|
|
114
|
-
case "company": return B(
|
|
113
|
+
case "address": return `${z(r, 0) % 9899 + 100} ${B(Ae, r, 1)} St`;
|
|
114
|
+
case "company": return B(je, r, 0);
|
|
115
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
116
116
|
case "ip": return `${z(r, 0) % 256}.${z(r, 1) % 256}.${z(r, 2) % 256}.${z(r, 3) % 254 + 1}`;
|
|
117
|
-
case "creditCard": return
|
|
117
|
+
case "creditCard": return Ne(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
118
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
119
119
|
}
|
|
120
|
-
},
|
|
120
|
+
}, Ne = (e) => {
|
|
121
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
122
122
|
t.forEach((e, t) => {
|
|
123
123
|
let r = e;
|
|
@@ -125,31 +125,31 @@ var xe = (e) => {
|
|
|
125
125
|
});
|
|
126
126
|
let r = (10 - n % 10) % 10;
|
|
127
127
|
return e.slice(0, 15) + String(r);
|
|
128
|
-
},
|
|
128
|
+
}, Pe = 864e5, Fe = (e, t) => {
|
|
129
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
130
130
|
if (Number.isNaN(n)) return e;
|
|
131
|
-
let r = z(
|
|
131
|
+
let r = z(De(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Pe);
|
|
132
132
|
return e instanceof Date ? i : i.toISOString();
|
|
133
|
-
},
|
|
133
|
+
}, Ie = (e, t) => {
|
|
134
134
|
if (t.value === null || t.value === void 0) return null;
|
|
135
135
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
136
136
|
if (e === "keep") return t.value;
|
|
137
137
|
if (e === "null") return null;
|
|
138
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
139
|
-
if (e === "hash") return
|
|
140
|
-
if (e === "dateShift") return
|
|
141
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return De(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Fe(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return Me(e.fake, t.value, t.seed);
|
|
142
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
143
143
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
144
|
-
},
|
|
144
|
+
}, Le = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Re = (e, t, n) => {
|
|
145
145
|
let r = {}, i = [], a = [];
|
|
146
146
|
for (let o of e) for (let e of o.columns) {
|
|
147
|
-
let s = `${o.name}.${e.name}`, c =
|
|
147
|
+
let s = `${o.name}.${e.name}`, c = Ee(o.name, e.name, t[o.name]?.[e.name], n);
|
|
148
148
|
if (c === "error") {
|
|
149
149
|
i.push(s);
|
|
150
150
|
continue;
|
|
151
151
|
}
|
|
152
|
-
if (!
|
|
152
|
+
if (!Le(c)) {
|
|
153
153
|
a.push({
|
|
154
154
|
column: s,
|
|
155
155
|
action: c
|
|
@@ -163,7 +163,7 @@ var xe = (e) => {
|
|
|
163
163
|
unclassified: i,
|
|
164
164
|
invalid: a
|
|
165
165
|
};
|
|
166
|
-
},
|
|
166
|
+
}, ze = [
|
|
167
167
|
{
|
|
168
168
|
name: "email",
|
|
169
169
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -180,7 +180,7 @@ var xe = (e) => {
|
|
|
180
180
|
name: "creditCard",
|
|
181
181
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
182
182
|
}
|
|
183
|
-
],
|
|
183
|
+
], Be = (e, t, n) => {
|
|
184
184
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
185
185
|
for (let [a, o] of Object.entries(n)) {
|
|
186
186
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -188,7 +188,7 @@ var xe = (e) => {
|
|
|
188
188
|
for (let a of e) {
|
|
189
189
|
let e = a[n];
|
|
190
190
|
if (typeof e != "string" || e.length === 0) continue;
|
|
191
|
-
let o =
|
|
191
|
+
let o = ze.find((t) => t.re.test(e));
|
|
192
192
|
if (o && !i.has(n)) {
|
|
193
193
|
i.add(n), r.push({
|
|
194
194
|
table: t,
|
|
@@ -201,9 +201,9 @@ var xe = (e) => {
|
|
|
201
201
|
}
|
|
202
202
|
}
|
|
203
203
|
return r;
|
|
204
|
-
},
|
|
204
|
+
}, Ve = (e, t, n, r, i) => {
|
|
205
205
|
let a = {};
|
|
206
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
206
|
+
for (let [o, s] of Object.entries(e)) a[o] = Ie(n[`${t}.${o}`] ?? "keep", {
|
|
207
207
|
value: s,
|
|
208
208
|
table: t,
|
|
209
209
|
column: o,
|
|
@@ -211,7 +211,7 @@ var xe = (e) => {
|
|
|
211
211
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
212
212
|
});
|
|
213
213
|
return a;
|
|
214
|
-
},
|
|
214
|
+
}, He = /* @__PURE__ */ new Map([
|
|
215
215
|
["_voltro_migration_plans", "the schema fingerprint THIS deployment applied — a foreign row makes the next boot refuse"],
|
|
216
216
|
["_voltro_migrations", "which file-based migrations ran HERE — a foreign row marks unrun migrations as done"],
|
|
217
217
|
["_voltro_seeds", "which seeds ran HERE — a foreign row suppresses seeding the target still needs"],
|
|
@@ -243,7 +243,7 @@ var xe = (e) => {
|
|
|
243
243
|
["_voltro_notification_held", "notifications waiting to be SENT from here — a foreign row makes the target deliver something queued elsewhere"],
|
|
244
244
|
["_voltro_webhook_event_stats", "per-deployment delivery counters a backoff decision reads — foreign counts steer the target's retries"],
|
|
245
245
|
["_voltro_webhook_rate_windows", "the live rate-limit windows this deployment is inside — a foreign window throttles or releases wrongly"]
|
|
246
|
-
]),
|
|
246
|
+
]), Ue = /* @__PURE__ */ new Map([
|
|
247
247
|
["_voltro_agent_threads", "conversations the app's users had — app data"],
|
|
248
248
|
["_voltro_agent_messages", "the messages in those conversations — app data"],
|
|
249
249
|
["_voltro_prompts", "prompt definitions the app authored — app data"],
|
|
@@ -274,21 +274,21 @@ var xe = (e) => {
|
|
|
274
274
|
["_voltro_audit_log", "the app's audit trail — compliance data the app owns, not deployment bookkeeping"],
|
|
275
275
|
["_voltro_feature_flags", "the flags the app defined — app configuration"],
|
|
276
276
|
["_voltro_feature_flag_audit", "who changed which flag — belongs with the flags"]
|
|
277
|
-
]), V = (e) =>
|
|
277
|
+
]), V = (e) => He.has(e), We = (e) => He.get(e), Ge = (e, t) => {
|
|
278
278
|
if (e.length === 0) return;
|
|
279
|
-
let n = e.map((e) => ` ${e} — ${
|
|
279
|
+
let n = e.map((e) => ` ${e} — ${We(e) ?? "describes this deployment"}`);
|
|
280
280
|
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
281
281
|
}, H = class extends c.TaggedError()("BundleError", {
|
|
282
282
|
reason: c.String,
|
|
283
283
|
path: c.optional(c.String)
|
|
284
|
-
}) {},
|
|
284
|
+
}) {}, Ke = class extends c.TaggedError()("CodecError", {
|
|
285
285
|
table: c.String,
|
|
286
286
|
reason: c.String
|
|
287
|
-
}) {},
|
|
287
|
+
}) {}, qe = class extends c.TaggedError()("IntegrityError", {
|
|
288
288
|
what: c.String,
|
|
289
289
|
expected: c.String,
|
|
290
290
|
actual: c.String
|
|
291
|
-
}) {},
|
|
291
|
+
}) {}, Je = class extends t.TaggedError("CompressionError") {}, U = class extends c.TaggedError()("ScopeError", { reason: c.String }) {}, Ye = class extends c.TaggedError()("RowsRefusedError", {
|
|
292
292
|
rows: c.Array(c.Struct({
|
|
293
293
|
table: c.String,
|
|
294
294
|
id: c.String,
|
|
@@ -302,29 +302,29 @@ var xe = (e) => {
|
|
|
302
302
|
refused: c.Number,
|
|
303
303
|
primary: c.Number
|
|
304
304
|
}))
|
|
305
|
-
}) {},
|
|
305
|
+
}) {}, Xe = class extends c.TaggedError()("SchemaDriftError", {
|
|
306
306
|
bundleFingerprint: c.String,
|
|
307
307
|
targetFingerprint: c.String,
|
|
308
308
|
diff: c.Array(c.String)
|
|
309
309
|
}) {}, W = class extends t.TaggedError("NativeToolError") {
|
|
310
310
|
get message() {
|
|
311
|
-
let e = this.stderr?.trim();
|
|
312
|
-
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
311
|
+
let e = this.stderr?.trim(), t = this.hint?.trim();
|
|
312
|
+
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}${t ? `\n\nhint: ${t}` : ""}`;
|
|
313
313
|
}
|
|
314
|
-
},
|
|
314
|
+
}, Ze = class extends c.TaggedError()("MaskingError", {
|
|
315
315
|
unclassified: c.Array(c.String),
|
|
316
316
|
invalidActions: c.optional(c.Array(c.String))
|
|
317
|
-
}) {},
|
|
317
|
+
}) {}, Qe = class extends c.TaggedError()("ImportModeError", {
|
|
318
318
|
mode: c.String,
|
|
319
319
|
reason: c.String
|
|
320
|
-
}) {},
|
|
320
|
+
}) {}, $e = class extends c.TaggedError()("CrossDialectError", {
|
|
321
321
|
sourceDialect: c.String,
|
|
322
322
|
targetDialect: c.String,
|
|
323
323
|
blocking: c.Array(c.String)
|
|
324
|
-
}) {}, G = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id",
|
|
324
|
+
}) {}, G = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", et = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
325
325
|
col: e.name,
|
|
326
326
|
refTable: e.references.table
|
|
327
|
-
})),
|
|
327
|
+
})), tt = (e, t, r) => n.gen(function* () {
|
|
328
328
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), u = (e) => {
|
|
329
329
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
330
330
|
return s.set(e, t), t;
|
|
@@ -342,7 +342,7 @@ var xe = (e) => {
|
|
|
342
342
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
343
343
|
};
|
|
344
344
|
}, p = (r, i) => {
|
|
345
|
-
let a = f(r, G(t, r),
|
|
345
|
+
let a = f(r, G(t, r), et(t, r));
|
|
346
346
|
return l.runForEach(w(e, {
|
|
347
347
|
table: r,
|
|
348
348
|
pk: G(t, r),
|
|
@@ -377,13 +377,13 @@ var xe = (e) => {
|
|
|
377
377
|
}
|
|
378
378
|
}
|
|
379
379
|
return o;
|
|
380
|
-
}),
|
|
380
|
+
}), nt = (e, t) => ({
|
|
381
381
|
tables: [...e.keys()],
|
|
382
382
|
predicateFor: (n) => {
|
|
383
383
|
let r = e.get(n);
|
|
384
384
|
return r ? y(G(t, n), [...r]) : void 0;
|
|
385
385
|
}
|
|
386
|
-
}),
|
|
386
|
+
}), rt = "tenantId", it = (e, t, r) => n.gen(function* () {
|
|
387
387
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
388
388
|
if (e.kind === "all") {
|
|
389
389
|
let t = new Set(e.exclude ?? []);
|
|
@@ -403,29 +403,29 @@ var xe = (e) => {
|
|
|
403
403
|
}
|
|
404
404
|
let o = e.tenantId;
|
|
405
405
|
if (o === void 0 || o === "") return yield* n.fail(new U({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
406
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
407
|
-
if (s.length === 0) return yield* n.fail(new U({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${
|
|
408
|
-
let c = yield*
|
|
409
|
-
seeds: Object.fromEntries(s.map((e) => [e, g(
|
|
406
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === rt) ?? !1);
|
|
407
|
+
if (s.length === 0) return yield* n.fail(new U({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${rt}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
408
|
+
let c = yield* tt(r.store, t, {
|
|
409
|
+
seeds: Object.fromEntries(s.map((e) => [e, g(rt, o)])),
|
|
410
410
|
pinned: s,
|
|
411
411
|
children: { roots: s }
|
|
412
412
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
413
413
|
return {
|
|
414
414
|
tables: [...s, ...l],
|
|
415
|
-
predicateFor: (e) => u.has(e) ? g(
|
|
415
|
+
predicateFor: (e) => u.has(e) ? g(rt, o) : c.has(e) ? y(G(t, e), [...c.get(e)]) : void 0
|
|
416
416
|
};
|
|
417
|
-
}),
|
|
417
|
+
}), at = (e, t) => ({
|
|
418
418
|
list: t,
|
|
419
|
-
open: (t) =>
|
|
420
|
-
}),
|
|
421
|
-
write: (t, n, r) =>
|
|
419
|
+
open: (t) => pe(e, t)
|
|
420
|
+
}), ot = (e) => ({
|
|
421
|
+
write: (t, n, r) => j(e, t, n, r),
|
|
422
422
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
423
|
-
}),
|
|
423
|
+
}), st = (e, t) => n.tryPromise({
|
|
424
424
|
try: async () => {
|
|
425
425
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
426
426
|
u.update(e), d += e.length, n(null, e);
|
|
427
427
|
} });
|
|
428
|
-
await o(await
|
|
428
|
+
await o(await M(e.stream), f, n(l));
|
|
429
429
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
430
430
|
try {
|
|
431
431
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -442,19 +442,44 @@ var xe = (e) => {
|
|
|
442
442
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
443
443
|
path: t
|
|
444
444
|
})
|
|
445
|
-
}),
|
|
446
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u =
|
|
445
|
+
}), ct = (e, t, r) => n.gen(function* () {
|
|
446
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u = fe("bundle", () => a(s)).pipe(l.tap((e) => n.sync(() => c.update(e))));
|
|
447
447
|
yield* e.write(r.key, u, {
|
|
448
448
|
contentType: r.contentType,
|
|
449
449
|
totalSize: r.size
|
|
450
450
|
});
|
|
451
451
|
let d = c.digest("hex");
|
|
452
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
452
|
+
if (d !== r.sha256) return yield* n.fail(new qe({
|
|
453
453
|
what: `asset ${r.key}`,
|
|
454
454
|
expected: r.sha256,
|
|
455
455
|
actual: d
|
|
456
456
|
}));
|
|
457
|
-
}),
|
|
457
|
+
}), lt = {
|
|
458
|
+
done: () => !1,
|
|
459
|
+
mark: () => n.void
|
|
460
|
+
}, ut = (e) => n.gen(function* () {
|
|
461
|
+
let t = e.progress ?? lt, r = 0, i = 0;
|
|
462
|
+
return yield* l.runForEach(e.source.list(), (a) => t.done(a.key) ? n.void : e.source.open(a.key).pipe(n.flatMap((t) => st(t, e.assetsDir)), n.flatMap((e) => {
|
|
463
|
+
let n = {
|
|
464
|
+
key: a.key,
|
|
465
|
+
sha256: e.sha256,
|
|
466
|
+
size: e.size,
|
|
467
|
+
contentType: e.contentType
|
|
468
|
+
};
|
|
469
|
+
return r++, i += e.size, t.mark(a.key, n);
|
|
470
|
+
}))).pipe(n.mapError((e) => e instanceof H ? e : new H({ reason: `asset export failed: ${String(e)}` }))), {
|
|
471
|
+
count: r,
|
|
472
|
+
totalBytes: i
|
|
473
|
+
};
|
|
474
|
+
}), dt = (e) => fe("bundle", () => le(e)).pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.map((e) => JSON.parse(e)), l.mapError((t) => t instanceof H ? t : new H({
|
|
475
|
+
reason: `asset index unreadable: ${String(t)}`,
|
|
476
|
+
path: e
|
|
477
|
+
}))), ft = (e) => n.gen(function* () {
|
|
478
|
+
let t = e.progress ?? lt, r = 0;
|
|
479
|
+
return yield* l.runForEach(dt(e.indexPath), (i) => t.done(i.key) ? n.void : ct(e.sink, e.assetsDir, i).pipe(n.andThen(n.sync(() => {
|
|
480
|
+
r++;
|
|
481
|
+
})), n.andThen(t.mark(i.key, i)))), r;
|
|
482
|
+
}), pt = () => typeof N.createZstdCompress == "function", mt = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : pt() ? "zstd" : "gzip", ht = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", gt = (e) => e === "zstd" ? N.createZstdCompress() : e === "gzip" ? N.createGzip() : new P(), _t = (e) => e === "zstd" ? N.createZstdDecompress() : e === "gzip" ? N.createGunzip() : new P(), vt = 1, yt = c.Struct({
|
|
458
483
|
name: c.String,
|
|
459
484
|
file: c.String,
|
|
460
485
|
rowCount: c.Number,
|
|
@@ -464,12 +489,12 @@ var xe = (e) => {
|
|
|
464
489
|
value: c.String
|
|
465
490
|
}),
|
|
466
491
|
primaryKey: c.String
|
|
467
|
-
}),
|
|
492
|
+
}), bt = c.Struct({
|
|
468
493
|
kind: c.Literal("all", "tenant", "tables"),
|
|
469
494
|
tenantId: c.optional(c.String),
|
|
470
495
|
tables: c.optional(c.Array(c.String)),
|
|
471
496
|
exclude: c.optional(c.Array(c.String))
|
|
472
|
-
}),
|
|
497
|
+
}), xt = c.Struct({
|
|
473
498
|
count: c.Number,
|
|
474
499
|
totalBytes: c.Number,
|
|
475
500
|
index: c.optional(c.String)
|
|
@@ -480,23 +505,23 @@ c.Struct({
|
|
|
480
505
|
size: c.Number,
|
|
481
506
|
contentType: c.String
|
|
482
507
|
});
|
|
483
|
-
var
|
|
508
|
+
var St = c.Struct({
|
|
484
509
|
formatVersion: c.Number,
|
|
485
510
|
createdAt: c.String,
|
|
486
511
|
source: c.Struct({
|
|
487
512
|
dialect: c.String,
|
|
488
513
|
schemaFingerprint: c.String
|
|
489
514
|
}),
|
|
490
|
-
scope:
|
|
515
|
+
scope: bt,
|
|
491
516
|
consistency: c.Literal("live", "snapshot"),
|
|
492
517
|
compression: c.Literal("zstd", "gzip", "none"),
|
|
493
|
-
tables: c.Array(
|
|
494
|
-
assets:
|
|
518
|
+
tables: c.Array(yt),
|
|
519
|
+
assets: xt,
|
|
495
520
|
masking: c.optional(c.Struct({
|
|
496
521
|
policyId: c.optional(c.String),
|
|
497
522
|
transformed: c.Array(c.String)
|
|
498
523
|
}))
|
|
499
|
-
}),
|
|
524
|
+
}), Ct = c.encodeSync(c.parseJson(St)), wt = c.decodeUnknownSync(c.parseJson(St)), K = "manifest.json", Tt = "README.md", Et = "data", Dt = "assets", Ot = "assets/index.ndjson", kt = ".ledger.json", At = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", jt = (e, t) => {
|
|
500
525
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
501
526
|
return `# Voltro data export
|
|
502
527
|
|
|
@@ -510,7 +535,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
510
535
|
| **Format version** | ${e.formatVersion} |
|
|
511
536
|
| **Consistency** | ${e.consistency} |
|
|
512
537
|
| **Compression** | ${e.compression} |
|
|
513
|
-
| **Scope** | ${
|
|
538
|
+
| **Scope** | ${At(e.scope)} |
|
|
514
539
|
| **Tables** | ${e.tables.length} |
|
|
515
540
|
| **Rows (total)** | ${n} |
|
|
516
541
|
| **Assets** | ${a} |
|
|
@@ -535,26 +560,26 @@ voltro data import <this-directory>
|
|
|
535
560
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
536
561
|
\`\`\`
|
|
537
562
|
`;
|
|
538
|
-
},
|
|
563
|
+
}, Mt = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), Nt = a.exponential({
|
|
539
564
|
start: .01,
|
|
540
565
|
factor: 2,
|
|
541
566
|
count: 16
|
|
542
|
-
}),
|
|
567
|
+
}), Pt = i.histogram("voltro_data_transfer_table_seconds", Nt, "Per-table export/import duration in seconds, tagged by phase."), q = (e, t) => {
|
|
543
568
|
if (e) try {
|
|
544
569
|
e(t);
|
|
545
570
|
} catch {}
|
|
546
|
-
},
|
|
571
|
+
}, Ft = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(Pt.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), Mt.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
547
572
|
table: t,
|
|
548
573
|
index: r,
|
|
549
574
|
tableCount: a,
|
|
550
575
|
phase: e
|
|
551
|
-
} })),
|
|
576
|
+
} })), It = () => ({
|
|
552
577
|
tables: {},
|
|
553
578
|
assets: {}
|
|
554
|
-
}),
|
|
579
|
+
}), Lt = (e) => n.tryPromise(async () => {
|
|
555
580
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
556
581
|
return JSON.parse(t);
|
|
557
|
-
}).pipe(n.orElseSucceed(
|
|
582
|
+
}).pipe(n.orElseSucceed(It)), J = (e, t) => n.tryPromise({
|
|
558
583
|
try: async () => {
|
|
559
584
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
560
585
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
@@ -563,11 +588,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
563
588
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
564
589
|
path: e
|
|
565
590
|
})
|
|
566
|
-
}),
|
|
591
|
+
}), Rt = (e, t) => t in e.tables, zt = (e, t) => t in e.assets, Bt = /* @__PURE__ */ new Set([
|
|
567
592
|
"json",
|
|
568
593
|
"array",
|
|
569
594
|
"vector"
|
|
570
|
-
]),
|
|
595
|
+
]), Vt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", Ht = (e) => Buffer.from(e).toString("base64"), Ut = (e) => new Uint8Array(Buffer.from(e, "base64")), Wt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), Gt = (e, t) => e == null ? null : t !== void 0 && Bt.has(t) ? e : t === "bytes" ? Ht(e) : t === "timestamp" || t === "date" ? Wt(e) : typeof e == "bigint" ? {
|
|
571
596
|
$v: "bigint",
|
|
572
597
|
d: e.toString()
|
|
573
598
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -578,30 +603,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
578
603
|
d: e.toISOString()
|
|
579
604
|
} : e instanceof Uint8Array ? {
|
|
580
605
|
$v: "bytes",
|
|
581
|
-
d:
|
|
582
|
-
} : e,
|
|
606
|
+
d: Ht(e)
|
|
607
|
+
} : e, Kt = (e) => {
|
|
583
608
|
switch (e.$v) {
|
|
584
609
|
case "bigint": return BigInt(e.d);
|
|
585
610
|
case "number": return Number(e.d);
|
|
586
611
|
case "date": return new Date(e.d);
|
|
587
|
-
case "bytes": return
|
|
612
|
+
case "bytes": return Ut(e.d);
|
|
588
613
|
default: return e;
|
|
589
614
|
}
|
|
590
|
-
},
|
|
615
|
+
}, qt = (e, t) => e == null ? null : t !== void 0 && Bt.has(t) ? e : Vt(e) ? Kt(e) : t === "bytes" ? Ut(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Jt = (e, t) => {
|
|
591
616
|
let n = {};
|
|
592
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
617
|
+
for (let [r, i] of Object.entries(e)) n[r] = Gt(i, t[r]);
|
|
593
618
|
return n;
|
|
594
|
-
},
|
|
619
|
+
}, Yt = (e, t) => {
|
|
595
620
|
let n = {};
|
|
596
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
621
|
+
for (let [r, i] of Object.entries(e)) n[r] = qt(i, t[r]);
|
|
597
622
|
return n;
|
|
598
|
-
},
|
|
599
|
-
try: () =>
|
|
600
|
-
catch: (e) => new
|
|
623
|
+
}, Xt = new TextEncoder(), Zt = (e, t) => e.pipe(l.map((e) => Xt.encode(`${JSON.stringify(Jt(e, t))}\n`))), Qt = (e, t, r) => e.pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.mapEffect((e) => n.try({
|
|
624
|
+
try: () => Yt(JSON.parse(e), r),
|
|
625
|
+
catch: (e) => new Ke({
|
|
601
626
|
table: t,
|
|
602
627
|
reason: String(e?.message ?? e)
|
|
603
628
|
})
|
|
604
|
-
}))),
|
|
629
|
+
}))), $t = (e, t) => {
|
|
605
630
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
606
631
|
for (let e of t.tables) {
|
|
607
632
|
if (!n.has(e.name)) continue;
|
|
@@ -627,33 +652,33 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
627
652
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
628
653
|
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(Y(t.cause)) : n.push(r), n.length === 0) {
|
|
629
654
|
let t = String(e ?? "").trim();
|
|
630
|
-
return t === "" || t === "[object Object]" ?
|
|
655
|
+
return t === "" || t === "[object Object]" ? en(e) : t;
|
|
631
656
|
}
|
|
632
657
|
return n.join(": ");
|
|
633
|
-
},
|
|
658
|
+
}, en = (e) => {
|
|
634
659
|
try {
|
|
635
660
|
let t = JSON.stringify(e);
|
|
636
661
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
637
662
|
} catch {
|
|
638
663
|
return "unknown error (not serialisable)";
|
|
639
664
|
}
|
|
640
|
-
},
|
|
665
|
+
}, tn = (e, t) => {
|
|
641
666
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
642
667
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
643
668
|
return r;
|
|
644
|
-
},
|
|
669
|
+
}, nn = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), rn = (e, t) => {
|
|
645
670
|
if (t.length === 0) return e;
|
|
646
671
|
let n = { ...e };
|
|
647
672
|
for (let e of t) delete n[e];
|
|
648
673
|
return n;
|
|
649
|
-
},
|
|
674
|
+
}, an = (e, t) => {
|
|
650
675
|
let n = e.tables.find((e) => e.name === t);
|
|
651
676
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
652
677
|
let r = n.columns.find((e) => e.type === "id");
|
|
653
678
|
if (r !== void 0) return { column: r.name };
|
|
654
679
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
655
680
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
656
|
-
},
|
|
681
|
+
}, on = (e, t, r) => n.tryPromise({
|
|
657
682
|
try: async () => {
|
|
658
683
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
659
684
|
d.update(e);
|
|
@@ -662,7 +687,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
662
687
|
n(null, e);
|
|
663
688
|
} }), m = c.fromWeb(l.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
664
689
|
try {
|
|
665
|
-
await s(m, p,
|
|
690
|
+
await s(m, p, gt(r), n(h)), await i.rename(h, t);
|
|
666
691
|
} catch (e) {
|
|
667
692
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
668
693
|
}
|
|
@@ -675,7 +700,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
675
700
|
reason: `write table failed: ${Y(e)}`,
|
|
676
701
|
path: t
|
|
677
702
|
})
|
|
678
|
-
}),
|
|
703
|
+
}), sn = (e, t) => n.tryPromise({
|
|
679
704
|
try: async () => {
|
|
680
705
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
681
706
|
},
|
|
@@ -683,13 +708,13 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
683
708
|
reason: `append failed: ${Y(t)}`,
|
|
684
709
|
path: e
|
|
685
710
|
})
|
|
686
|
-
}),
|
|
711
|
+
}), cn = (e, t) => n.gen(function* () {
|
|
687
712
|
let r = [], i = t.ordered.length;
|
|
688
713
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
689
|
-
let o = t.ordered[a], s =
|
|
714
|
+
let o = t.ordered[a], s = tn(t.snapshot, o), c = an(t.snapshot, o);
|
|
690
715
|
if ("refusal" in c) return yield* n.fail(new U({ reason: `export: ${c.refusal}` }));
|
|
691
|
-
let u = c.column, d = `${
|
|
692
|
-
if (
|
|
716
|
+
let u = c.column, d = `${Et}/${o}.ndjson${ht(t.compression)}`;
|
|
717
|
+
if (Rt(t.ledger, o)) {
|
|
693
718
|
let e = t.ledger.tables[o];
|
|
694
719
|
q(t.onProgress, {
|
|
695
720
|
phase: "export",
|
|
@@ -723,7 +748,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
723
748
|
chunkSize: t.chunkSize,
|
|
724
749
|
retry: t.retry,
|
|
725
750
|
...f === void 0 ? {} : { where: f }
|
|
726
|
-
}), m =
|
|
751
|
+
}), m = nn(t.snapshot, o), h = m.length === 0 ? p : p.pipe(l.map((e) => rn(e, m))), g = t.mask ? h.pipe(l.map((e) => Ve(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* Ft("export", o, a, i, (e) => e.rowCount, on(Zt(g, s), t.bundleFile(d), t.compression));
|
|
727
752
|
t.ledger.tables[o] = {
|
|
728
753
|
rowCount: _,
|
|
729
754
|
checksum: v
|
|
@@ -745,82 +770,81 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
745
770
|
});
|
|
746
771
|
}
|
|
747
772
|
return r;
|
|
748
|
-
}),
|
|
773
|
+
}), ln = (e) => ({
|
|
749
774
|
_tag: "RawSqlFragment",
|
|
750
775
|
strings: [e],
|
|
751
776
|
values: []
|
|
752
|
-
}),
|
|
777
|
+
}), un = (t, i, a) => n.tryPromise({
|
|
753
778
|
try: () => t.transactional(async (t) => {
|
|
754
|
-
if (t.raw) for (let e of
|
|
779
|
+
if (t.raw) for (let e of Ce(i)) await t.raw(ln(e));
|
|
755
780
|
let o = await n.runPromiseExit(a(t));
|
|
756
781
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
757
782
|
return o.value;
|
|
758
783
|
}),
|
|
759
784
|
catch: (e) => e instanceof H ? e : new H({ reason: `snapshot export failed: ${Y(e)}` })
|
|
760
|
-
}),
|
|
761
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
785
|
+
}), dn = (e) => n.gen(function* () {
|
|
786
|
+
let t = e.scope ?? { kind: "all" }, r = mt(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, o = s.exponential("50 millis").pipe(s.jittered, s.intersect(s.recurs(a))), c = yield* n.promise(() => import("node:path")), l = (t) => c.join(e.outDir, t);
|
|
762
787
|
yield* n.tryPromise({
|
|
763
788
|
try: async () => {
|
|
764
789
|
let t = await import("node:fs/promises");
|
|
765
|
-
await t.mkdir(
|
|
790
|
+
await t.mkdir(l(Et), { recursive: !0 }), e.assets && await t.mkdir(l(Dt), { recursive: !0 });
|
|
766
791
|
},
|
|
767
792
|
catch: (t) => new H({
|
|
768
793
|
reason: `mkdir failed: ${Y(t)}`,
|
|
769
794
|
path: e.outDir
|
|
770
795
|
})
|
|
771
796
|
});
|
|
772
|
-
let
|
|
797
|
+
let u = l(kt), d = yield* Lt(u), f = e.subset ? nt(yield* tt(e.store, e.snapshot, e.subset), e.snapshot) : yield* it(t, e.snapshot, {
|
|
773
798
|
store: e.store,
|
|
774
799
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
775
|
-
}),
|
|
800
|
+
}), p = $t(f.tables, e.snapshot), m = e.consistency ?? "live", h, g;
|
|
776
801
|
if (e.masking) {
|
|
777
|
-
let t = e.classification ??
|
|
802
|
+
let t = e.classification ?? Te(e.snapshot), r = Re(p.map((t) => ({
|
|
778
803
|
name: t,
|
|
779
804
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
780
805
|
name: e.name,
|
|
781
806
|
type: e.type
|
|
782
807
|
}))
|
|
783
808
|
})), t, e.masking);
|
|
784
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
809
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new Ze({
|
|
785
810
|
unclassified: r.unclassified,
|
|
786
811
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
787
812
|
}));
|
|
788
|
-
|
|
813
|
+
g = {
|
|
789
814
|
actions: r.actions,
|
|
790
815
|
seed: e.masking.seed
|
|
791
816
|
};
|
|
792
817
|
let i = Object.entries(r.actions).filter(([, e]) => e !== "keep").map(([e]) => e);
|
|
793
|
-
|
|
818
|
+
h = {
|
|
794
819
|
...e.masking.id ? { policyId: e.masking.id } : {},
|
|
795
820
|
transformed: i
|
|
796
821
|
};
|
|
797
822
|
}
|
|
798
|
-
let
|
|
823
|
+
let _ = {
|
|
799
824
|
snapshot: e.snapshot,
|
|
800
|
-
ordered:
|
|
801
|
-
resolved:
|
|
825
|
+
ordered: p,
|
|
826
|
+
resolved: f,
|
|
802
827
|
compression: r,
|
|
803
828
|
chunkSize: i,
|
|
804
829
|
retry: o,
|
|
805
|
-
ledger:
|
|
806
|
-
ledgerPath:
|
|
807
|
-
bundleFile:
|
|
808
|
-
...
|
|
830
|
+
ledger: d,
|
|
831
|
+
ledgerPath: u,
|
|
832
|
+
bundleFile: l,
|
|
833
|
+
...g ? { mask: g } : {},
|
|
809
834
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
810
|
-
},
|
|
835
|
+
}, v = m === "snapshot" ? yield* un(e.store, e.dialect, (e) => cn(e, _)) : yield* cn(e.store, _), y = 0, b = 0;
|
|
811
836
|
if (e.assets) {
|
|
812
|
-
let t =
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
}))).pipe(n.mapError((e) => e instanceof H ? e : new H({ reason: `asset export failed: ${String(e)}` })));
|
|
837
|
+
let t = l(Ot), r = yield* ut({
|
|
838
|
+
source: e.assets,
|
|
839
|
+
assetsDir: l(Dt),
|
|
840
|
+
progress: {
|
|
841
|
+
done: (e) => e in d.assets,
|
|
842
|
+
mark: (e, r) => (d.assets[e] = !0, sn(t, JSON.stringify(r)).pipe(n.andThen(J(u, d))))
|
|
843
|
+
}
|
|
844
|
+
});
|
|
845
|
+
y = r.count, b = r.totalBytes;
|
|
822
846
|
}
|
|
823
|
-
let
|
|
847
|
+
let x = {
|
|
824
848
|
formatVersion: 1,
|
|
825
849
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
826
850
|
source: {
|
|
@@ -828,22 +852,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
828
852
|
schemaFingerprint: re(e.snapshot)
|
|
829
853
|
},
|
|
830
854
|
scope: t,
|
|
831
|
-
consistency:
|
|
855
|
+
consistency: m,
|
|
832
856
|
compression: r,
|
|
833
|
-
tables:
|
|
834
|
-
assets:
|
|
835
|
-
count:
|
|
836
|
-
totalBytes:
|
|
837
|
-
index:
|
|
857
|
+
tables: v,
|
|
858
|
+
assets: y > 0 ? {
|
|
859
|
+
count: y,
|
|
860
|
+
totalBytes: b,
|
|
861
|
+
index: Ot
|
|
838
862
|
} : {
|
|
839
863
|
count: 0,
|
|
840
864
|
totalBytes: 0
|
|
841
865
|
},
|
|
842
|
-
...
|
|
866
|
+
...h ? { masking: h } : {}
|
|
843
867
|
};
|
|
844
868
|
return yield* n.tryPromise({
|
|
845
869
|
try: async () => {
|
|
846
|
-
await (await import("node:fs/promises")).writeFile(
|
|
870
|
+
await (await import("node:fs/promises")).writeFile(l(K), Ct(x));
|
|
847
871
|
},
|
|
848
872
|
catch: (e) => new H({
|
|
849
873
|
reason: `manifest write failed: ${Y(e)}`,
|
|
@@ -852,14 +876,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
852
876
|
}), yield* n.tryPromise({
|
|
853
877
|
try: async () => {
|
|
854
878
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
855
|
-
await e.writeFile(
|
|
879
|
+
await e.writeFile(l(Tt), jt(x, { hostname: t.hostname() }));
|
|
856
880
|
},
|
|
857
881
|
catch: (e) => new H({
|
|
858
882
|
reason: `readme write failed: ${Y(e)}`,
|
|
859
|
-
path:
|
|
883
|
+
path: Tt
|
|
860
884
|
})
|
|
861
|
-
}),
|
|
862
|
-
}),
|
|
885
|
+
}), x;
|
|
886
|
+
}), fn = (e, t) => {
|
|
863
887
|
let n = new Set(t), r = [];
|
|
864
888
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
865
889
|
let i = e.references?.table;
|
|
@@ -870,7 +894,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
870
894
|
});
|
|
871
895
|
}
|
|
872
896
|
return r;
|
|
873
|
-
},
|
|
897
|
+
}, pn = (e) => {
|
|
874
898
|
let t = /* @__PURE__ */ new Map();
|
|
875
899
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
876
900
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -878,7 +902,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
878
902
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
879
903
|
});
|
|
880
904
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
881
|
-
},
|
|
905
|
+
}, mn = (e) => {
|
|
882
906
|
switch (e) {
|
|
883
907
|
case "id":
|
|
884
908
|
case "text":
|
|
@@ -897,12 +921,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
897
921
|
case "raw": return "raw";
|
|
898
922
|
default: return "string";
|
|
899
923
|
}
|
|
900
|
-
},
|
|
924
|
+
}, hn = {
|
|
901
925
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
902
926
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
903
927
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
904
|
-
},
|
|
905
|
-
let i =
|
|
928
|
+
}, gn = (e, t, n, r) => {
|
|
929
|
+
let i = mn(n);
|
|
906
930
|
if (i === "raw") return {
|
|
907
931
|
table: e,
|
|
908
932
|
column: t,
|
|
@@ -910,7 +934,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
910
934
|
severity: "block",
|
|
911
935
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
912
936
|
};
|
|
913
|
-
let a =
|
|
937
|
+
let a = hn[i];
|
|
914
938
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
915
939
|
table: e,
|
|
916
940
|
column: t,
|
|
@@ -924,15 +948,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
924
948
|
severity: "block",
|
|
925
949
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
926
950
|
};
|
|
927
|
-
},
|
|
951
|
+
}, _n = (e, t, n) => {
|
|
928
952
|
if (t === n) return [];
|
|
929
953
|
let r = [];
|
|
930
954
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
931
|
-
let a =
|
|
955
|
+
let a = gn(t.name, e, i, n);
|
|
932
956
|
a && r.push(a);
|
|
933
957
|
}
|
|
934
958
|
return r;
|
|
935
|
-
},
|
|
959
|
+
}, vn = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, yn = (e) => {
|
|
936
960
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
937
961
|
switch (e.kind) {
|
|
938
962
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -941,35 +965,35 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
941
965
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
942
966
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
943
967
|
}
|
|
944
|
-
},
|
|
968
|
+
}, bn = (e) => {
|
|
945
969
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
946
970
|
return t.length > 0 ? t.join("/") : void 0;
|
|
947
|
-
},
|
|
971
|
+
}, xn = "Failed to execute statement", Sn = (e) => {
|
|
948
972
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
949
973
|
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
950
|
-
return t === `SqlError: ${
|
|
951
|
-
},
|
|
974
|
+
return t === `SqlError: ${xn}` ? xn : t;
|
|
975
|
+
}, Cn = (e) => {
|
|
952
976
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
953
977
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
954
978
|
unclassified: t,
|
|
955
979
|
raw: n
|
|
956
|
-
}), i = _(e), a = p(i), o =
|
|
957
|
-
if (a !== void 0) return r(`${
|
|
958
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u =
|
|
959
|
-
return r(d ===
|
|
960
|
-
},
|
|
980
|
+
}), i = _(e), a = p(i), o = bn(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
981
|
+
if (a !== void 0) return r(`${yn(a)}${s}`, !1);
|
|
982
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = Sn(typeof l == "string" && l.length > 0 ? l : String(e)), d = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, f = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, m = f !== void 0 && /[←‖]/.test(f) ? ` (via ${f})` : "";
|
|
983
|
+
return r(d === xn || d.length === 0 ? `the database refused the write and the driver gave no detail${s}${m}` : `${d}${s}`, d === xn || d.length === 0);
|
|
984
|
+
}, wn = (e) => Cn(e).reason, Tn = (e) => {
|
|
961
985
|
let t = _(e), n = p(t);
|
|
962
986
|
if (n?.kind === "foreignKeyInUse") {
|
|
963
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
987
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = bn(t);
|
|
964
988
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
965
989
|
}
|
|
966
|
-
return
|
|
967
|
-
},
|
|
990
|
+
return wn(e);
|
|
991
|
+
}, En = "_voltro_staging_", X = (e) => `${En}${e}`, Dn = (e) => typeof e.run == "function", On = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
968
992
|
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
969
|
-
written before the first delete is what covers that case.`,
|
|
970
|
-
if (!
|
|
993
|
+
written before the first delete is what covers that case.`, kn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, Z = (e, t) => {
|
|
994
|
+
if (!kn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
971
995
|
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
972
|
-
},
|
|
996
|
+
}, An = (e, t) => {
|
|
973
997
|
let n = Z(e, t), r = Z(X(e), t);
|
|
974
998
|
switch (t) {
|
|
975
999
|
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
@@ -978,10 +1002,10 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
978
1002
|
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
979
1003
|
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
980
1004
|
}
|
|
981
|
-
},
|
|
1005
|
+
}, jn = (e, t) => {
|
|
982
1006
|
let n = Z(X(e), t);
|
|
983
1007
|
return t === "mssql" ? `IF OBJECT_ID('${X(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
984
|
-
},
|
|
1008
|
+
}, Mn = (e, t) => {
|
|
985
1009
|
let n = t === "mysql" || t === "mariadb", r = [];
|
|
986
1010
|
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
987
1011
|
for (let n of [...e].reverse()) r.push(`DELETE FROM ${Z(n.name, t)}`);
|
|
@@ -991,7 +1015,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
991
1015
|
r.push(`INSERT INTO ${Z(n.name, t)} (${e}) SELECT ${e} FROM ${Z(X(n.name), t)}`);
|
|
992
1016
|
}
|
|
993
1017
|
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
994
|
-
},
|
|
1018
|
+
}, Nn = (e, t) => {
|
|
995
1019
|
let n = [], r = () => {
|
|
996
1020
|
for (let e of n) t.unregisterTable(e);
|
|
997
1021
|
};
|
|
@@ -1013,7 +1037,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1013
1037
|
staged: n,
|
|
1014
1038
|
undo: r
|
|
1015
1039
|
};
|
|
1016
|
-
},
|
|
1040
|
+
}, Pn = (e, t) => {
|
|
1017
1041
|
let n = new Set(t), r = [];
|
|
1018
1042
|
for (let t of e.tables) {
|
|
1019
1043
|
if (!n.has(t.name)) continue;
|
|
@@ -1030,22 +1054,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1030
1054
|
}
|
|
1031
1055
|
}
|
|
1032
1056
|
return r;
|
|
1033
|
-
},
|
|
1057
|
+
}, Fn = (e, t) => {
|
|
1034
1058
|
let n = Z(X(e.from), t), r = Z(X(e.to), t), i = Z(e.column, t), a = Z(e.toColumn, t);
|
|
1035
1059
|
return `SELECT ${Z(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
1036
|
-
},
|
|
1060
|
+
}, In = (e) => {
|
|
1037
1061
|
if (e.length === 0) return;
|
|
1038
1062
|
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
1039
1063
|
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
1040
|
-
},
|
|
1041
|
-
t.length !== 0 && await e.run(
|
|
1042
|
-
},
|
|
1043
|
-
let i =
|
|
1044
|
-
i.length !== 0 && await e.run(
|
|
1045
|
-
},
|
|
1064
|
+
}, Ln = async (e, t) => {
|
|
1065
|
+
t.length !== 0 && await e.run(F.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
1066
|
+
}, Rn = async (e, t, r) => {
|
|
1067
|
+
let i = Mn(t, r);
|
|
1068
|
+
i.length !== 0 && await e.run(F.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
1069
|
+
}, zn = async (e, t, r) => {
|
|
1046
1070
|
let i = [];
|
|
1047
1071
|
for (let a of t) {
|
|
1048
|
-
let t = await e.run(
|
|
1072
|
+
let t = await e.run(F.SqlClient.pipe(n.flatMap((e) => e.unsafe(Fn(a, r)))));
|
|
1049
1073
|
for (let e of t) i.push({
|
|
1050
1074
|
table: a.from,
|
|
1051
1075
|
primaryKey: String(e.pk ?? ""),
|
|
@@ -1055,14 +1079,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1055
1079
|
});
|
|
1056
1080
|
}
|
|
1057
1081
|
return i;
|
|
1058
|
-
},
|
|
1082
|
+
}, Bn = (e, t) => {
|
|
1059
1083
|
let n = new Map(t.map((e, t) => [e, t]));
|
|
1060
1084
|
return e.filter((e) => {
|
|
1061
1085
|
if (e.from === e.to) return !1;
|
|
1062
1086
|
let t = n.get(e.from), r = n.get(e.to);
|
|
1063
1087
|
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1064
1088
|
});
|
|
1065
|
-
},
|
|
1089
|
+
}, Vn = (e) => e === "mysql" || e === "mariadb", Hn = (e) => {
|
|
1066
1090
|
if (e.mode !== "replace") return {
|
|
1067
1091
|
active: !1,
|
|
1068
1092
|
tables: [],
|
|
@@ -1112,7 +1136,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1112
1136
|
tables: [],
|
|
1113
1137
|
blocker: "every table in the bundle is one that describes a deployment, so there is nothing for a staged swap to move."
|
|
1114
1138
|
};
|
|
1115
|
-
let s = o.map((e) => e.name), c =
|
|
1139
|
+
let s = o.map((e) => e.name), c = Vn(t) ? [] : Bn(Pn(n, s), s);
|
|
1116
1140
|
return c.length > 0 ? {
|
|
1117
1141
|
active: !1,
|
|
1118
1142
|
tables: [],
|
|
@@ -1121,7 +1145,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1121
1145
|
active: !0,
|
|
1122
1146
|
tables: o
|
|
1123
1147
|
};
|
|
1124
|
-
},
|
|
1148
|
+
}, Un = (e) => n.gen(function* () {
|
|
1125
1149
|
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1126
1150
|
if (e.snapshot === void 0) return yield* n.fail(new H({
|
|
1127
1151
|
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
@@ -1134,7 +1158,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1134
1158
|
if (t.tables.length === 0) return;
|
|
1135
1159
|
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1136
1160
|
so it survives a run whose process does not.`);
|
|
1137
|
-
let r = yield*
|
|
1161
|
+
let r = yield* dn({
|
|
1138
1162
|
store: e.store,
|
|
1139
1163
|
snapshot: t,
|
|
1140
1164
|
dialect: e.dialect ?? "unknown",
|
|
@@ -1150,18 +1174,18 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1150
1174
|
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1151
1175
|
tables: r.tables.length
|
|
1152
1176
|
};
|
|
1153
|
-
}),
|
|
1177
|
+
}), Wn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, Gn = (e, t) => e.tables.find((e) => e.name === t), Kn = (e, t) => new Map((Gn(e, t)?.columns ?? []).map((e) => [e.name, e])), qn = /__dropped_\d+$/, Jn = (e, t) => {
|
|
1154
1178
|
let n = [];
|
|
1155
1179
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
1156
|
-
if (
|
|
1180
|
+
if (Gn(t, r.name) === void 0) {
|
|
1157
1181
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
1158
1182
|
continue;
|
|
1159
1183
|
}
|
|
1160
|
-
let e =
|
|
1184
|
+
let e = Kn(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
1161
1185
|
for (let [t, a] of i) {
|
|
1162
1186
|
let i = e.get(t);
|
|
1163
1187
|
if (i === void 0) {
|
|
1164
|
-
n.push(
|
|
1188
|
+
n.push(qn.test(t) ? `${r.name}.${t}: a soft-dropped column, present only in the SOURCE — reclaim it there (voltro db gc-snapshots --before <date>) or re-export without it; the target is not missing anything` : `${r.name}.${t}: in the bundle, MISSING from the target — the value has nowhere to go`);
|
|
1165
1189
|
continue;
|
|
1166
1190
|
}
|
|
1167
1191
|
i.type !== a.type && n.push(`${r.name}.${t}: type '${a.type}' (bundle) vs '${i.type}' (target)`);
|
|
@@ -1169,7 +1193,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1169
1193
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
1170
1194
|
}
|
|
1171
1195
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
1172
|
-
},
|
|
1196
|
+
}, Yn = (e, t, n) => {
|
|
1173
1197
|
let r = re(t);
|
|
1174
1198
|
if (r === e || n === void 0) return {
|
|
1175
1199
|
drifted: !1,
|
|
@@ -1177,14 +1201,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1177
1201
|
targetFingerprint: r,
|
|
1178
1202
|
diff: []
|
|
1179
1203
|
};
|
|
1180
|
-
let i =
|
|
1204
|
+
let i = Jn(n, t);
|
|
1181
1205
|
return {
|
|
1182
1206
|
drifted: i.length > 0,
|
|
1183
1207
|
bundleFingerprint: e,
|
|
1184
1208
|
targetFingerprint: r,
|
|
1185
1209
|
diff: i
|
|
1186
1210
|
};
|
|
1187
|
-
},
|
|
1211
|
+
}, Xn = 2e3, Zn = 200, Qn = /* @__PURE__ */ new Set([
|
|
1188
1212
|
"BundleError",
|
|
1189
1213
|
"CodecError",
|
|
1190
1214
|
"IntegrityError",
|
|
@@ -1193,29 +1217,29 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1193
1217
|
"RowsRefusedError",
|
|
1194
1218
|
"SchemaDriftError",
|
|
1195
1219
|
"StorageError"
|
|
1196
|
-
]),
|
|
1220
|
+
]), $n = (t) => {
|
|
1197
1221
|
let n = o.isFiberFailure(t) ? [...e.failures(t[o.FiberFailureCauseId]), ...e.defects(t[o.FiberFailureCauseId])] : [t];
|
|
1198
1222
|
for (let e of n) {
|
|
1199
1223
|
let t = e?._tag;
|
|
1200
|
-
if (typeof t == "string" &&
|
|
1224
|
+
if (typeof t == "string" && Qn.has(t)) return e;
|
|
1201
1225
|
if (o.isFiberFailure(e)) {
|
|
1202
|
-
let t =
|
|
1226
|
+
let t = $n(e);
|
|
1203
1227
|
if (t) return t;
|
|
1204
1228
|
}
|
|
1205
1229
|
}
|
|
1206
|
-
},
|
|
1230
|
+
}, er = (t) => [...e.failures(t)][0] ?? Error(e.pretty(t)), tr = (e) => n.tryPromise({
|
|
1207
1231
|
try: async () => {
|
|
1208
1232
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
1209
|
-
return
|
|
1233
|
+
return wt(await t.readFile(n.join(e, K), "utf8"));
|
|
1210
1234
|
},
|
|
1211
1235
|
catch: (e) => new H({
|
|
1212
1236
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
1213
1237
|
path: K
|
|
1214
1238
|
})
|
|
1215
|
-
}),
|
|
1239
|
+
}), nr = (e) => {
|
|
1216
1240
|
let t;
|
|
1217
1241
|
return n.gen(function* () {
|
|
1218
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
1242
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* tr(e.bundleDir);
|
|
1219
1243
|
if (a.formatVersion !== 1) return yield* n.fail(new H({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
1220
1244
|
if (e.tables !== void 0) {
|
|
1221
1245
|
let t = new Set(a.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
@@ -1236,9 +1260,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1236
1260
|
unique: !1,
|
|
1237
1261
|
hasDefault: !1
|
|
1238
1262
|
}))
|
|
1239
|
-
})) }, r =
|
|
1263
|
+
})) }, r = Yn(s.source.schemaFingerprint, e.targetSnapshot, t);
|
|
1240
1264
|
if (r.drifted) {
|
|
1241
|
-
if (!e.force) return yield* n.fail(new
|
|
1265
|
+
if (!e.force) return yield* n.fail(new Xe({
|
|
1242
1266
|
bundleFingerprint: r.bundleFingerprint,
|
|
1243
1267
|
targetFingerprint: r.targetFingerprint,
|
|
1244
1268
|
diff: r.diff
|
|
@@ -1247,15 +1271,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1247
1271
|
}
|
|
1248
1272
|
}
|
|
1249
1273
|
if (e.targetDialect && e.targetDialect !== s.source.dialect) {
|
|
1250
|
-
let t =
|
|
1251
|
-
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new
|
|
1274
|
+
let t = _n(s.tables, s.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1275
|
+
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new $e({
|
|
1252
1276
|
sourceDialect: s.source.dialect,
|
|
1253
1277
|
targetDialect: e.targetDialect,
|
|
1254
|
-
blocking: t.map(
|
|
1278
|
+
blocking: t.map(vn)
|
|
1255
1279
|
}));
|
|
1256
1280
|
}
|
|
1257
|
-
let c = e.verify ?? !0, p = e.mode ?? "upsert", g = e.atomic ?? p === "replace", _ = e.onConflict ?? "skip", y = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), x = yield*
|
|
1258
|
-
if (p === "replace" && s.scope.kind !== "all") return yield* n.fail(new
|
|
1281
|
+
let c = e.verify ?? !0, p = e.mode ?? "upsert", g = e.atomic ?? p === "replace", _ = e.onConflict ?? "skip", y = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), x = yield* Lt(y);
|
|
1282
|
+
if (p === "replace" && s.scope.kind !== "all") return yield* n.fail(new Qe({
|
|
1259
1283
|
mode: p,
|
|
1260
1284
|
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope. To leave specific tables out of a FULL export, use \`--exclude a,b\`: it keeps the scope \`all\` and records what it left out.`
|
|
1261
1285
|
}));
|
|
@@ -1267,8 +1291,8 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1267
1291
|
if (p === "replace") {
|
|
1268
1292
|
let t = s.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1269
1293
|
try: () => e.store.incomingForeignKeys(t),
|
|
1270
|
-
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${
|
|
1271
|
-
}) : e.targetSnapshot === void 0 ? [] :
|
|
1294
|
+
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${Tn(e)}` })
|
|
1295
|
+
}) : e.targetSnapshot === void 0 ? [] : fn(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1272
1296
|
for (let t of r) {
|
|
1273
1297
|
if (!a.has(t.from)) {
|
|
1274
1298
|
let r = yield* n.tryPromise({
|
|
@@ -1290,9 +1314,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1290
1314
|
rowCount: r
|
|
1291
1315
|
});
|
|
1292
1316
|
}
|
|
1293
|
-
if (i.length > 0) return yield* n.fail(new
|
|
1317
|
+
if (i.length > 0) return yield* n.fail(new Qe({
|
|
1294
1318
|
mode: p,
|
|
1295
|
-
reason:
|
|
1319
|
+
reason: pn(i)
|
|
1296
1320
|
}));
|
|
1297
1321
|
}
|
|
1298
1322
|
if (e.dryRun === !0) return e.traceId !== void 0 && (yield* n.promise(() => e.store.update(u, e.traceId, {
|
|
@@ -1319,14 +1343,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1319
1343
|
let r;
|
|
1320
1344
|
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1321
1345
|
return r ?? t;
|
|
1322
|
-
},
|
|
1323
|
-
let i =
|
|
1346
|
+
}, ie = (e, t, n, r = !1) => {
|
|
1347
|
+
let i = A ? X(t.name) : t.name, a = t.primaryKey, o = re(t.name, n);
|
|
1324
1348
|
return p === "append" ? _ === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : p === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1325
|
-
},
|
|
1349
|
+
}, T = (e, t) => e.savepoint ? e.savepoint(t) : t(), E = 0, ae = (t, n, r) => {
|
|
1326
1350
|
r.unclassified && (E++, !(E > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (E === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1327
1351
|
}, D = (e, t, r = !0) => n.tryPromise({
|
|
1328
1352
|
try: e,
|
|
1329
|
-
catch: (e) =>
|
|
1353
|
+
catch: (e) => Cn(e)
|
|
1330
1354
|
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, ae(t.entry, t.row, e)), !1)))), oe = (e) => {
|
|
1331
1355
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1332
1356
|
if (t.length !== 0) return {
|
|
@@ -1351,7 +1375,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1351
1375
|
};
|
|
1352
1376
|
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1353
1377
|
}
|
|
1354
|
-
return new
|
|
1378
|
+
return new Ye({
|
|
1355
1379
|
rows: i.slice(0, 20),
|
|
1356
1380
|
primaryCount: r.filter((e) => !e.derived).length,
|
|
1357
1381
|
totalCount: e.length,
|
|
@@ -1361,32 +1385,32 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1361
1385
|
primary: t.primary
|
|
1362
1386
|
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1363
1387
|
});
|
|
1364
|
-
},
|
|
1388
|
+
}, ce = (e, t) => n.gen(function* () {
|
|
1365
1389
|
let r = t, i = [], a = !0;
|
|
1366
1390
|
for (; r.length > 0 && a;) {
|
|
1367
1391
|
a = !1;
|
|
1368
1392
|
let t = [];
|
|
1369
|
-
for (let n of r) (yield* D(() =>
|
|
1393
|
+
for (let n of r) (yield* D(() => T(e, () => ie(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1370
1394
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1371
1395
|
let r = t[n], o = oe(r);
|
|
1372
|
-
o !== void 0 && (yield* D(() =>
|
|
1396
|
+
o !== void 0 && (yield* D(() => T(e, () => ie(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1373
1397
|
}
|
|
1374
1398
|
r = t;
|
|
1375
1399
|
}
|
|
1376
|
-
for (let t of i) (yield* D(() =>
|
|
1400
|
+
for (let t of i) (yield* D(() => T(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1377
1401
|
if (r.length > 0) return yield* n.fail(se(r));
|
|
1378
1402
|
}), O = (() => {
|
|
1379
1403
|
let t = e.targetDialect;
|
|
1380
1404
|
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1381
|
-
})(), k =
|
|
1405
|
+
})(), k = Hn({
|
|
1382
1406
|
mode: p,
|
|
1383
1407
|
...e.targetDialect === void 0 ? {} : { targetDialect: e.targetDialect },
|
|
1384
1408
|
...e.targetSnapshot === void 0 ? {} : { targetSnapshot: e.targetSnapshot },
|
|
1385
1409
|
bundleTables: s.tables.map((e) => e.name),
|
|
1386
1410
|
isSkipped: V,
|
|
1387
|
-
canRunSql:
|
|
1388
|
-
}),
|
|
1389
|
-
if (p === "replace" && !
|
|
1411
|
+
canRunSql: Dn(e.store)
|
|
1412
|
+
}), ue = k.tables, de = s.tables.filter((e) => !V(e.name) && S(e.name)).map((e) => e.name), A = k.active, pe = !1, j = 0, M = 0, N, P = 0, me = 0, F = (t, r) => n.gen(function* () {
|
|
1413
|
+
if (p === "replace" && !A && !(r && x.truncated)) {
|
|
1390
1414
|
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1391
1415
|
try: () => t.insert(d, {
|
|
1392
1416
|
id: m("repl", he),
|
|
@@ -1405,7 +1429,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1405
1429
|
});
|
|
1406
1430
|
yield* n.tryPromise({
|
|
1407
1431
|
try: o,
|
|
1408
|
-
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${
|
|
1432
|
+
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${Tn(e)}` })
|
|
1409
1433
|
}), r && (x.truncated = !0, yield* J(y, x));
|
|
1410
1434
|
}
|
|
1411
1435
|
let a = [], o = [], u = [], f = [], h = s.tables.length;
|
|
@@ -1416,7 +1440,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1416
1440
|
continue;
|
|
1417
1441
|
}
|
|
1418
1442
|
if (r && m.name in x.tables) {
|
|
1419
|
-
u.push(m.name),
|
|
1443
|
+
u.push(m.name), me++, P += x.tables[m.name].rowCount, q(e.onProgress, {
|
|
1420
1444
|
phase: "import",
|
|
1421
1445
|
table: m.name,
|
|
1422
1446
|
index: d,
|
|
@@ -1438,22 +1462,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1438
1462
|
total: m.rowCount
|
|
1439
1463
|
});
|
|
1440
1464
|
let v = a.length, b = n.gen(function* () {
|
|
1441
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c =
|
|
1465
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c = fe("bundle", () => le(_).pipe(_t(s.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), u = async (e) => {
|
|
1442
1466
|
let n = {
|
|
1443
1467
|
entry: m,
|
|
1444
1468
|
row: e,
|
|
1445
1469
|
reason: ""
|
|
1446
1470
|
};
|
|
1447
1471
|
try {
|
|
1448
|
-
await
|
|
1472
|
+
await T(t, () => ie(t, m, e));
|
|
1449
1473
|
} catch (t) {
|
|
1450
|
-
let r =
|
|
1474
|
+
let r = Cn(t);
|
|
1451
1475
|
n.reason = r.reason, ae(m, e, r), a.push(n);
|
|
1452
1476
|
}
|
|
1453
1477
|
}, d = yield* n.promise(async () => {
|
|
1454
1478
|
let n = e.copyLoader;
|
|
1455
1479
|
if (!n || n.dialect !== "postgres") return !1;
|
|
1456
|
-
if (g) return
|
|
1480
|
+
if (g) return pe || (pe = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1457
1481
|
if (p === "replace") return !0;
|
|
1458
1482
|
if (p !== "upsert") return !1;
|
|
1459
1483
|
try {
|
|
@@ -1489,13 +1513,13 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1489
1513
|
S = [];
|
|
1490
1514
|
try {
|
|
1491
1515
|
await t.savepoint(async () => {
|
|
1492
|
-
for (let n of e) await
|
|
1516
|
+
for (let n of e) await ie(t, m, n);
|
|
1493
1517
|
});
|
|
1494
1518
|
} catch {
|
|
1495
1519
|
for (let t of e) await u(t);
|
|
1496
1520
|
}
|
|
1497
1521
|
};
|
|
1498
|
-
return yield* l.runForEach(
|
|
1522
|
+
return yield* l.runForEach(Qt(c, m.name, m.columnTypes), (e) => n.promise(async () => {
|
|
1499
1523
|
if (o++, d) {
|
|
1500
1524
|
v.push(e), v.length >= h && await y();
|
|
1501
1525
|
return;
|
|
@@ -1509,14 +1533,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1509
1533
|
rowCount: o,
|
|
1510
1534
|
checksum: i.digest("hex")
|
|
1511
1535
|
};
|
|
1512
|
-
}), { rowCount: S, checksum: C } = yield*
|
|
1536
|
+
}), { rowCount: S, checksum: C } = yield* Ft("import", m.name, d, h, (e) => e.rowCount, b);
|
|
1513
1537
|
if (c) {
|
|
1514
|
-
if (C !== m.checksum) return yield* n.fail(new
|
|
1538
|
+
if (C !== m.checksum) return yield* n.fail(new qe({
|
|
1515
1539
|
what: `table ${m.name} checksum`,
|
|
1516
1540
|
expected: m.checksum,
|
|
1517
1541
|
actual: C
|
|
1518
1542
|
}));
|
|
1519
|
-
if (S !== m.rowCount) return yield* n.fail(new
|
|
1543
|
+
if (S !== m.rowCount) return yield* n.fail(new qe({
|
|
1520
1544
|
what: `table ${m.name} rowCount`,
|
|
1521
1545
|
expected: String(m.rowCount),
|
|
1522
1546
|
actual: String(S)
|
|
@@ -1530,7 +1554,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1530
1554
|
event: "done",
|
|
1531
1555
|
rowsDone: S,
|
|
1532
1556
|
total: m.rowCount
|
|
1533
|
-
}),
|
|
1557
|
+
}), M++, j += S, N?.(), r && (a.length > v ? o.push({
|
|
1534
1558
|
name: m.name,
|
|
1535
1559
|
rowCount: S
|
|
1536
1560
|
}) : (x.tables[m.name] = {
|
|
@@ -1538,7 +1562,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1538
1562
|
checksum: m.checksum
|
|
1539
1563
|
}, yield* J(y, x)));
|
|
1540
1564
|
}
|
|
1541
|
-
if (a.length > 0 && (yield*
|
|
1565
|
+
if (a.length > 0 && (yield* ce(t, a)), p === "replace" && (yield* n.promise(() => t.deleteMany(d, { where: {
|
|
1542
1566
|
column: "startedAt",
|
|
1543
1567
|
op: "lte",
|
|
1544
1568
|
value: ge
|
|
@@ -1553,14 +1577,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1553
1577
|
yield* J(y, x);
|
|
1554
1578
|
}
|
|
1555
1579
|
{
|
|
1556
|
-
let t =
|
|
1580
|
+
let t = Ge(f, "skipped");
|
|
1557
1581
|
t !== void 0 && e.onWarn?.(t);
|
|
1558
1582
|
}
|
|
1559
1583
|
if (u.length > 0) {
|
|
1560
1584
|
let t = u.length === s.tables.length;
|
|
1561
1585
|
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${y} to force a full re-import.`);
|
|
1562
1586
|
}
|
|
1563
|
-
}), he = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, ge = /* @__PURE__ */ new Date(), I = yield*
|
|
1587
|
+
}), he = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, ge = /* @__PURE__ */ new Date(), I = yield* Un({
|
|
1564
1588
|
store: e.store,
|
|
1565
1589
|
mode: p,
|
|
1566
1590
|
dir: e.rollbackDir,
|
|
@@ -1575,10 +1599,10 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1575
1599
|
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1576
1600
|
path: I.dir
|
|
1577
1601
|
})
|
|
1578
|
-
})), e.onWarn?.(
|
|
1579
|
-
let _e = (e) => p === "replace" ? ee(e) : e(),
|
|
1602
|
+
})), e.onWarn?.(Wn(I)));
|
|
1603
|
+
let _e = (e) => p === "replace" ? ee(e) : e(), ve = (e) => p === "replace" ? te(e) : e, L = e.traceId ?? m("dimp", `${Date.now()}-${s.source.schemaFingerprint ?? ""}`), R = e.traceId !== void 0, ye = (e.targetSnapshot?.tables.some((e) => e.name === u) ?? !1) || v(u) !== void 0;
|
|
1580
1604
|
ye && h(f), ye && e.traceId === void 0 && (yield* n.promise(() => e.store.insert(u, {
|
|
1581
|
-
id:
|
|
1605
|
+
id: L,
|
|
1582
1606
|
mode: p,
|
|
1583
1607
|
via: e.via ?? "direct",
|
|
1584
1608
|
direction: "import",
|
|
@@ -1586,38 +1610,38 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1586
1610
|
sourceFingerprint: s.source.schemaFingerprint ?? null,
|
|
1587
1611
|
tablesWritten: 0,
|
|
1588
1612
|
rowsWritten: 0,
|
|
1589
|
-
staged: p === "replace" ?
|
|
1613
|
+
staged: p === "replace" ? A : null
|
|
1590
1614
|
}).then(() => {
|
|
1591
|
-
|
|
1615
|
+
R = !0;
|
|
1592
1616
|
}, (t) => {
|
|
1593
1617
|
e.onWarn?.(`the import trace could not be written (${String(t?.message ?? t)}).\n The import continues — this is a record of what happened, not a safety check. Run
|
|
1594
1618
|
\`voltro db apply\` so the next one is recorded.`);
|
|
1595
|
-
}))), e.traceId !== void 0 && ye && (yield* n.promise(() => e.store.update(u,
|
|
1596
|
-
staged: p === "replace" ?
|
|
1619
|
+
}))), e.traceId !== void 0 && ye && (yield* n.promise(() => e.store.update(u, L, {
|
|
1620
|
+
staged: p === "replace" ? A : null,
|
|
1597
1621
|
sourceFingerprint: s.source.schemaFingerprint ?? null
|
|
1598
1622
|
}).then(() => void 0, () => void 0)));
|
|
1599
1623
|
let be = 0, xe = !1;
|
|
1600
|
-
|
|
1601
|
-
if (!
|
|
1624
|
+
N = () => {
|
|
1625
|
+
if (!R || xe) return;
|
|
1602
1626
|
let t = Date.now();
|
|
1603
|
-
t - be <
|
|
1604
|
-
tablesWritten:
|
|
1605
|
-
rowsWritten:
|
|
1627
|
+
t - be < Xn || (be = t, e.store.update(u, L, {
|
|
1628
|
+
tablesWritten: M,
|
|
1629
|
+
rowsWritten: j
|
|
1606
1630
|
}).then(() => void 0, () => {
|
|
1607
1631
|
xe = !0;
|
|
1608
1632
|
}));
|
|
1609
1633
|
};
|
|
1610
|
-
let Se = (t) =>
|
|
1634
|
+
let Se = (t) => R ? e.store.update(u, L, {
|
|
1611
1635
|
finishedAt: t === null ? /* @__PURE__ */ new Date() : null,
|
|
1612
|
-
tablesWritten:
|
|
1613
|
-
rowsWritten:
|
|
1636
|
+
tablesWritten: M,
|
|
1637
|
+
rowsWritten: j,
|
|
1614
1638
|
failure: t
|
|
1615
1639
|
}).then(() => void 0, () => void 0) : Promise.resolve();
|
|
1616
|
-
if (t = (e) => Se(e), p === "replace" && !
|
|
1640
|
+
if (t = (e) => Se(e), p === "replace" && !A && k.blocker !== void 0 && e.onWarn?.(On(k.blocker)), p === "replace" && de.length > 0 && e.onWarn?.(`write recorders are SUSPENDED for this replace (${de.length} table(s) have one).\n A replace sets a state rather than changing rows, so a per-row history entry would describe
|
|
1617
1641
|
something that did not happen — and this path already writes through the raw store, without
|
|
1618
|
-
tenant scoping or \`audit()\` stamping, for the same reason.`),
|
|
1619
|
-
e.onWarn?.(`staging ${
|
|
1620
|
-
let t = e.store, r =
|
|
1642
|
+
tenant scoping or \`audit()\` stamping, for the same reason.`), A) {
|
|
1643
|
+
e.onWarn?.(`staging ${ue.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1644
|
+
let t = e.store, r = ue.map((e) => e.name), i = Nn(r, {
|
|
1621
1645
|
getTable: (e) => v(e),
|
|
1622
1646
|
registerTable: (e) => {
|
|
1623
1647
|
C(e);
|
|
@@ -1625,59 +1649,65 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1625
1649
|
unregisterTable: ne
|
|
1626
1650
|
}), a = !1;
|
|
1627
1651
|
try {
|
|
1628
|
-
let i = g ? r : r.filter((e) => !
|
|
1652
|
+
let i = g ? r : r.filter((e) => !Rt(x, e));
|
|
1629
1653
|
yield* n.tryPromise({
|
|
1630
1654
|
try: async () => {
|
|
1631
|
-
await
|
|
1655
|
+
await Ln(t, i.map((e) => jn(e, O))), await Ln(t, i.map((e) => An(e, O)));
|
|
1632
1656
|
},
|
|
1633
1657
|
catch: (e) => new H({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1634
|
-
}), yield*
|
|
1635
|
-
try: () =>
|
|
1658
|
+
}), yield* ve(F(e.store, !g)), !g && x.truncated === !0 ? a = !0 : yield* n.tryPromise({
|
|
1659
|
+
try: () => Rn(t, ue, O).then(() => {
|
|
1636
1660
|
a = !0;
|
|
1637
1661
|
}),
|
|
1638
1662
|
catch: (e) => new H({ reason: String(e?.message ?? e) })
|
|
1639
1663
|
}).pipe(n.catchAll((i) => n.gen(function* () {
|
|
1640
|
-
let a =
|
|
1664
|
+
let a = In(yield* n.promise(() => zn(t, Pn(e.targetSnapshot, r), O).then((e) => e, () => [])));
|
|
1641
1665
|
return yield* n.fail(new H({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1642
1666
|
}))), !g && a && x.truncated !== !0 && (x.truncated = !0, yield* J(y, x));
|
|
1643
1667
|
} finally {
|
|
1644
|
-
i.undo(), a || g ? yield* n.promise(() =>
|
|
1668
|
+
i.undo(), a || g ? yield* n.promise(() => Ln(t, r.map((e) => jn(e, O))).then(() => void 0, () => void 0)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1645
1669
|
}
|
|
1646
1670
|
} else g ? yield* n.tryPromise({
|
|
1647
1671
|
try: () => e.store.transactional(async (e) => {
|
|
1648
|
-
let t = await _e(() => n.runPromiseExit(
|
|
1649
|
-
if (r.isFailure(t)) throw
|
|
1672
|
+
let t = await _e(() => n.runPromiseExit(F(e, !1)));
|
|
1673
|
+
if (r.isFailure(t)) throw er(t.cause);
|
|
1650
1674
|
}),
|
|
1651
|
-
catch: (e) =>
|
|
1652
|
-
}) : yield*
|
|
1675
|
+
catch: (e) => $n(e) ?? new H({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1676
|
+
}) : yield* ve(F(e.store, !0));
|
|
1653
1677
|
if (e.assets && s.assets.index && !e.assetsAlreadyRestored) {
|
|
1654
|
-
let t = e.assets
|
|
1655
|
-
yield*
|
|
1656
|
-
|
|
1657
|
-
|
|
1678
|
+
let t = e.assets;
|
|
1679
|
+
yield* ft({
|
|
1680
|
+
sink: t,
|
|
1681
|
+
assetsDir: i.join(e.bundleDir, Dt),
|
|
1682
|
+
indexPath: i.join(e.bundleDir, s.assets.index ?? "assets/index.ndjson"),
|
|
1683
|
+
progress: {
|
|
1684
|
+
done: (e) => e in x.assets,
|
|
1685
|
+
mark: (e) => (x.assets[e] = !0, J(y, x))
|
|
1686
|
+
}
|
|
1687
|
+
});
|
|
1658
1688
|
}
|
|
1659
1689
|
return yield* n.promise(() => Se(null)), {
|
|
1660
1690
|
manifest: s,
|
|
1661
|
-
tablesWritten:
|
|
1662
|
-
rowsWritten:
|
|
1663
|
-
tablesSkipped:
|
|
1691
|
+
tablesWritten: M,
|
|
1692
|
+
rowsWritten: j,
|
|
1693
|
+
tablesSkipped: me,
|
|
1664
1694
|
rowsSkipped: P,
|
|
1665
|
-
fullyResumed:
|
|
1695
|
+
fullyResumed: M === 0 && me > 0,
|
|
1666
1696
|
...I === void 0 ? {} : { rollback: I }
|
|
1667
1697
|
};
|
|
1668
1698
|
}).pipe(n.tapError((e) => n.promise(() => t?.(e?.reason ?? e?._tag ?? "the import did not finish") ?? Promise.resolve())));
|
|
1669
|
-
},
|
|
1699
|
+
}, rr = (e) => e, ir = (e) => {
|
|
1670
1700
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1671
1701
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1672
1702
|
return {
|
|
1673
1703
|
refuse: !0,
|
|
1674
1704
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1675
1705
|
};
|
|
1676
|
-
}, Q = Buffer.from("VBUNDLE2"),
|
|
1706
|
+
}, Q = Buffer.from("VBUNDLE2"), ar = 0, or = 1, sr = 2, cr = 32, lr = async (e) => {
|
|
1677
1707
|
let t = [], n = async (r) => {
|
|
1678
1708
|
let i;
|
|
1679
1709
|
try {
|
|
1680
|
-
i = await
|
|
1710
|
+
i = await ge(R(e, r), { withFileTypes: !0 });
|
|
1681
1711
|
} catch {
|
|
1682
1712
|
return;
|
|
1683
1713
|
}
|
|
@@ -1687,21 +1717,21 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1687
1717
|
}
|
|
1688
1718
|
};
|
|
1689
1719
|
return await n(""), t.sort();
|
|
1690
|
-
},
|
|
1720
|
+
}, ur = (e, t) => {
|
|
1691
1721
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1692
|
-
return r.writeUInt8(
|
|
1693
|
-
},
|
|
1722
|
+
return r.writeUInt8(or, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1723
|
+
}, dr = (e, t, n) => {
|
|
1694
1724
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1695
|
-
return a.writeUInt8(
|
|
1725
|
+
return a.writeUInt8(sr, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1696
1726
|
};
|
|
1697
|
-
async function*
|
|
1727
|
+
async function* fr(e) {
|
|
1698
1728
|
yield new Uint8Array(Q);
|
|
1699
|
-
for (let t of await
|
|
1700
|
-
let n =
|
|
1701
|
-
if (yield new Uint8Array(
|
|
1729
|
+
for (let t of await lr(e.dir)) {
|
|
1730
|
+
let n = R(e.dir, t), r = await I(n);
|
|
1731
|
+
if (yield new Uint8Array(ur(t, r.size)), r.size > 0) for await (let e of le(n)) yield new Uint8Array(e);
|
|
1702
1732
|
}
|
|
1703
1733
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1704
|
-
yield new Uint8Array(
|
|
1734
|
+
yield new Uint8Array(dr(t.key, t.contentType, t.size));
|
|
1705
1735
|
let n = E("sha256"), r = 0;
|
|
1706
1736
|
for await (let i of e.blobs.open(t.key)) {
|
|
1707
1737
|
let e = Buffer.from(i);
|
|
@@ -1710,11 +1740,11 @@ async function* or(e) {
|
|
|
1710
1740
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1711
1741
|
yield new Uint8Array(n.digest());
|
|
1712
1742
|
}
|
|
1713
|
-
yield new Uint8Array(Buffer.from([
|
|
1743
|
+
yield new Uint8Array(Buffer.from([ar]));
|
|
1714
1744
|
}
|
|
1715
|
-
var
|
|
1716
|
-
let n =
|
|
1717
|
-
await
|
|
1745
|
+
var pr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), mr = (e) => new Promise((t) => e.end(t)), hr = async (e, t) => {
|
|
1746
|
+
let n = ye(t.dir);
|
|
1747
|
+
await he(n, { recursive: !0 });
|
|
1718
1748
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1719
1749
|
let { value: e, done: t } = await i.next();
|
|
1720
1750
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1729,24 +1759,24 @@ var sr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), cr
|
|
|
1729
1759
|
for (;;) {
|
|
1730
1760
|
await s(1);
|
|
1731
1761
|
let e = c(1).readUInt8(0);
|
|
1732
|
-
if (e ===
|
|
1733
|
-
if (e ===
|
|
1762
|
+
if (e === ar) return;
|
|
1763
|
+
if (e === or) {
|
|
1734
1764
|
await s(4);
|
|
1735
1765
|
let e = c(4).readUInt32BE(0);
|
|
1736
1766
|
await s(e);
|
|
1737
1767
|
let t = c(e).toString("utf8");
|
|
1738
1768
|
await s(8);
|
|
1739
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1740
|
-
if (a !== n && !a.startsWith(n +
|
|
1741
|
-
await
|
|
1742
|
-
let o =
|
|
1769
|
+
let i = c(8).readBigUInt64BE(0), a = ye(n, t);
|
|
1770
|
+
if (a !== n && !a.startsWith(n + be)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1771
|
+
await he(L(a), { recursive: !0 });
|
|
1772
|
+
let o = O(a);
|
|
1743
1773
|
for (; i > 0n;) {
|
|
1744
1774
|
r.length === 0 && await s(1);
|
|
1745
1775
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1746
|
-
await
|
|
1776
|
+
await pr(o, c(e)), i -= BigInt(e);
|
|
1747
1777
|
}
|
|
1748
|
-
await
|
|
1749
|
-
} else if (e ===
|
|
1778
|
+
await mr(o);
|
|
1779
|
+
} else if (e === sr) {
|
|
1750
1780
|
await s(4);
|
|
1751
1781
|
let e = c(4).readUInt32BE(0);
|
|
1752
1782
|
await s(e);
|
|
@@ -1756,24 +1786,24 @@ var sr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), cr
|
|
|
1756
1786
|
await s(i);
|
|
1757
1787
|
let a = c(i).toString("utf8");
|
|
1758
1788
|
await s(8);
|
|
1759
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = E("sha256"), d = new
|
|
1789
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = E("sha256"), d = new P(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1760
1790
|
contentType: a,
|
|
1761
1791
|
size: Number(o)
|
|
1762
1792
|
}), p = o;
|
|
1763
1793
|
for (; p > 0n;) {
|
|
1764
1794
|
r.length === 0 && await s(1);
|
|
1765
1795
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1766
|
-
u.update(t), l || await
|
|
1796
|
+
u.update(t), l || await pr(d, t), p -= BigInt(e);
|
|
1767
1797
|
}
|
|
1768
|
-
l ? d.destroy() : await
|
|
1769
|
-
let m = c(
|
|
1798
|
+
l ? d.destroy() : await mr(d), await f, await s(cr);
|
|
1799
|
+
let m = c(cr);
|
|
1770
1800
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1771
1801
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1772
1802
|
}
|
|
1773
1803
|
} finally {
|
|
1774
1804
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1775
1805
|
}
|
|
1776
|
-
},
|
|
1806
|
+
}, gr = async (e, t) => {
|
|
1777
1807
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1778
1808
|
let { value: e, done: t } = await a.next();
|
|
1779
1809
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1795,8 +1825,8 @@ var sr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), cr
|
|
|
1795
1825
|
if (!await s(Q.length) || !c(Q.length).equals(Q)) throw Error("not a bundle archive (bad magic)");
|
|
1796
1826
|
for (; await s(1);) {
|
|
1797
1827
|
let e = c(1).readUInt8(0);
|
|
1798
|
-
if (e ===
|
|
1799
|
-
if (e !==
|
|
1828
|
+
if (e === ar || e === sr) break;
|
|
1829
|
+
if (e !== or) throw Error(`unknown bundle entry kind ${e}`);
|
|
1800
1830
|
if (!await s(4)) break;
|
|
1801
1831
|
let t = c(4).readUInt32BE(0);
|
|
1802
1832
|
if (!await s(t)) break;
|
|
@@ -1806,22 +1836,22 @@ var sr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), cr
|
|
|
1806
1836
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1807
1837
|
}
|
|
1808
1838
|
return r;
|
|
1809
|
-
},
|
|
1839
|
+
}, _r = (e, t) => hr(e, {
|
|
1810
1840
|
dir: t,
|
|
1811
|
-
blobs:
|
|
1812
|
-
}),
|
|
1841
|
+
blobs: vr(t)
|
|
1842
|
+
}), vr = (e) => {
|
|
1813
1843
|
let t = /* @__PURE__ */ new Set();
|
|
1814
1844
|
return {
|
|
1815
1845
|
has: async (e) => t.has(e),
|
|
1816
1846
|
write: async (n, r, i) => {
|
|
1817
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1818
|
-
await
|
|
1847
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = R(e, "assets");
|
|
1848
|
+
await he(c, { recursive: !0 });
|
|
1819
1849
|
let l = E("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1820
1850
|
l.update(e), u += e.length, n(null, e);
|
|
1821
|
-
} }), f =
|
|
1822
|
-
await o(r, d,
|
|
1851
|
+
} }), f = R(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1852
|
+
await o(r, d, O(f));
|
|
1823
1853
|
let p = l.digest("hex");
|
|
1824
|
-
await a.rename(f,
|
|
1854
|
+
await a.rename(f, R(c, p)), await a.appendFile(R(c, "index.ndjson"), `${JSON.stringify({
|
|
1825
1855
|
key: n,
|
|
1826
1856
|
sha256: p,
|
|
1827
1857
|
size: u,
|
|
@@ -1829,33 +1859,33 @@ var sr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), cr
|
|
|
1829
1859
|
})}\n`), t.add(n);
|
|
1830
1860
|
}
|
|
1831
1861
|
};
|
|
1832
|
-
},
|
|
1862
|
+
}, yr = xe(se), br = Buffer.from("VENC1\0\0\0"), xr = 32, Sr = 16, Cr = 7, wr = 16, Tr = 262144, Er = {
|
|
1833
1863
|
N: 32768,
|
|
1834
1864
|
r: 8,
|
|
1835
1865
|
p: 1
|
|
1836
|
-
},
|
|
1866
|
+
}, Dr = 134217728, Or = 43, kr = (e, t, n) => {
|
|
1837
1867
|
let r = Buffer.alloc(12);
|
|
1838
|
-
return e.copy(r, 0, 0,
|
|
1868
|
+
return e.copy(r, 0, 0, Cr), r.writeUInt32BE(t >>> 0, Cr), r.writeUInt8(+!!n, 11), r;
|
|
1839
1869
|
};
|
|
1840
|
-
async function*
|
|
1870
|
+
async function* Ar(e, t) {
|
|
1841
1871
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1842
|
-
let n = D(
|
|
1843
|
-
...
|
|
1844
|
-
maxmem:
|
|
1845
|
-
}), a = Buffer.alloc(
|
|
1846
|
-
|
|
1872
|
+
let n = D(Sr), r = D(Cr), i = await yr(t, n, xr, {
|
|
1873
|
+
...Er,
|
|
1874
|
+
maxmem: Dr
|
|
1875
|
+
}), a = Buffer.alloc(Or);
|
|
1876
|
+
br.copy(a, 0), a.writeUInt32BE(Er.N, 8), a.writeUInt32BE(Er.r, 12), a.writeUInt32BE(Er.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1847
1877
|
let o = 0, s = (e, t) => {
|
|
1848
|
-
let n =
|
|
1878
|
+
let n = ie("aes-256-gcm", i, kr(r, o, t));
|
|
1849
1879
|
o === 0 && n.setAAD(a);
|
|
1850
1880
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1851
1881
|
o++;
|
|
1852
|
-
let l = Buffer.alloc(5 + s.length +
|
|
1853
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
1882
|
+
let l = Buffer.alloc(5 + s.length + wr);
|
|
1883
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + wr, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1854
1884
|
}, c = Buffer.alloc(0);
|
|
1855
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1885
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > Tr;) yield s(c.subarray(0, Tr), !1), c = c.subarray(Tr);
|
|
1856
1886
|
yield s(c, !0);
|
|
1857
1887
|
}
|
|
1858
|
-
async function*
|
|
1888
|
+
async function* jr(e, t) {
|
|
1859
1889
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1860
1890
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1861
1891
|
let { value: e, done: t } = await n.next();
|
|
@@ -1864,16 +1894,16 @@ async function* Tr(e, t) {
|
|
|
1864
1894
|
for (; r.length < e && !i && await a(););
|
|
1865
1895
|
return r.length >= e;
|
|
1866
1896
|
};
|
|
1867
|
-
if (!await o(
|
|
1868
|
-
let s = r.subarray(0,
|
|
1869
|
-
if (!s.subarray(0, 8).equals(
|
|
1870
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36,
|
|
1897
|
+
if (!await o(Or)) throw Error("encrypted bundle truncated (no header)");
|
|
1898
|
+
let s = r.subarray(0, Or);
|
|
1899
|
+
if (!s.subarray(0, 8).equals(br)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1900
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Or)), p = await yr(t, d, xr, {
|
|
1871
1901
|
N: c,
|
|
1872
1902
|
r: l,
|
|
1873
1903
|
p: u,
|
|
1874
|
-
maxmem:
|
|
1904
|
+
maxmem: Dr
|
|
1875
1905
|
});
|
|
1876
|
-
r = r.subarray(
|
|
1906
|
+
r = r.subarray(Or);
|
|
1877
1907
|
let m = 0, h = !1;
|
|
1878
1908
|
for (;;) {
|
|
1879
1909
|
if (!await o(5)) {
|
|
@@ -1882,11 +1912,11 @@ async function* Tr(e, t) {
|
|
|
1882
1912
|
}
|
|
1883
1913
|
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
1884
1914
|
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
1885
|
-
if (t <
|
|
1915
|
+
if (t < wr) throw Error("encrypted bundle corrupt (short frame)");
|
|
1886
1916
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1887
|
-
let n = r.subarray(5, 5 + t -
|
|
1917
|
+
let n = r.subarray(5, 5 + t - wr), i = r.subarray(5 + t - wr, 5 + t);
|
|
1888
1918
|
r = r.subarray(5 + t);
|
|
1889
|
-
let a =
|
|
1919
|
+
let a = T("aes-256-gcm", p, kr(f, m, e));
|
|
1890
1920
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1891
1921
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1892
1922
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1894,7 +1924,7 @@ async function* Tr(e, t) {
|
|
|
1894
1924
|
}
|
|
1895
1925
|
//#endregion
|
|
1896
1926
|
//#region src/blobStorage.ts
|
|
1897
|
-
var
|
|
1927
|
+
var Mr = (e, t) => ({
|
|
1898
1928
|
list: () => {
|
|
1899
1929
|
let e = t();
|
|
1900
1930
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1907,29 +1937,32 @@ var Er = (e, t) => ({
|
|
|
1907
1937
|
} };
|
|
1908
1938
|
},
|
|
1909
1939
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1910
|
-
let r = await n.runPromise(
|
|
1940
|
+
let r = await n.runPromise(pe(e, t)), i = await M(r.stream);
|
|
1911
1941
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1912
1942
|
} })
|
|
1913
|
-
}),
|
|
1914
|
-
table:
|
|
1943
|
+
}), Nr = (e, t) => Mr(e, () => w(t, {
|
|
1944
|
+
table: A,
|
|
1915
1945
|
chunkSize: 500
|
|
1916
1946
|
}).pipe(l.map((e) => ({
|
|
1917
1947
|
key: String(e.key),
|
|
1918
1948
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1919
1949
|
size: Number(e.size ?? 0)
|
|
1920
|
-
})))),
|
|
1950
|
+
})))), Pr = (e) => ({
|
|
1921
1951
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1922
1952
|
write: async (t, r, i) => {
|
|
1923
|
-
let a =
|
|
1924
|
-
await n.runPromise(
|
|
1953
|
+
let a = fe("storage", () => me.from(r));
|
|
1954
|
+
await n.runPromise(j(e, t, a, {
|
|
1925
1955
|
contentType: i.contentType,
|
|
1926
1956
|
totalSize: i.size
|
|
1927
1957
|
}));
|
|
1928
1958
|
}
|
|
1929
|
-
}),
|
|
1959
|
+
}), Fr = (e, t) => {
|
|
1930
1960
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1931
|
-
return e === "mariadb"
|
|
1932
|
-
|
|
1961
|
+
return e === "mariadb" ? Ir(n) ? { tool: n } : {
|
|
1962
|
+
tool: r,
|
|
1963
|
+
hint: `'${n}' is not on PATH, so this ran '${r}' instead. If that binary came from Oracle's MySQL client it queries information_schema.COLUMN_STATISTICS, which MariaDB does not have, and the run dies with error 1109 after the first table. Install the MariaDB client package (which ships '${n}') and re-run. Do NOT reach for --column-statistics=0: that flag does not exist on '${n}', so it fixes the wrong client and breaks the right one.`
|
|
1964
|
+
} : { tool: r };
|
|
1965
|
+
}, Ir = (e) => (process.env.PATH ?? "").split(ve).filter((e) => e !== "").some((t) => k(R(t, e))), Lr = (e) => {
|
|
1933
1966
|
switch (e) {
|
|
1934
1967
|
case "postgres": return "db.dump";
|
|
1935
1968
|
case "mysql":
|
|
@@ -1939,7 +1972,7 @@ var Er = (e, t) => ({
|
|
|
1939
1972
|
case "mssql": return "db.bacpac";
|
|
1940
1973
|
default: return "db.dump";
|
|
1941
1974
|
}
|
|
1942
|
-
},
|
|
1975
|
+
}, Rr = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1943
1976
|
if (e.host || e.database) return {
|
|
1944
1977
|
host: e.host ?? "localhost",
|
|
1945
1978
|
port: e.port ?? 3306,
|
|
@@ -1955,7 +1988,7 @@ var Er = (e, t) => ({
|
|
|
1955
1988
|
password: decodeURIComponent(t.password) || "",
|
|
1956
1989
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1957
1990
|
};
|
|
1958
|
-
},
|
|
1991
|
+
}, zr = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Br = (e, t, n) => {
|
|
1959
1992
|
switch (e) {
|
|
1960
1993
|
case "postgres": {
|
|
1961
1994
|
if (t.url) return {
|
|
@@ -2002,7 +2035,7 @@ var Er = (e, t) => ({
|
|
|
2002
2035
|
let r = $(t);
|
|
2003
2036
|
return {
|
|
2004
2037
|
kind: "spawn",
|
|
2005
|
-
|
|
2038
|
+
...Fr(e, "dump"),
|
|
2006
2039
|
args: [
|
|
2007
2040
|
"--single-transaction",
|
|
2008
2041
|
"--routines",
|
|
@@ -2024,7 +2057,7 @@ var Er = (e, t) => ({
|
|
|
2024
2057
|
case "sqlite":
|
|
2025
2058
|
case "turso": return {
|
|
2026
2059
|
kind: "copy",
|
|
2027
|
-
from:
|
|
2060
|
+
from: Rr(t),
|
|
2028
2061
|
to: n
|
|
2029
2062
|
};
|
|
2030
2063
|
case "mssql": return {
|
|
@@ -2032,7 +2065,7 @@ var Er = (e, t) => ({
|
|
|
2032
2065
|
tool: "sqlpackage",
|
|
2033
2066
|
args: [
|
|
2034
2067
|
"/Action:Export",
|
|
2035
|
-
`/SourceConnectionString:${
|
|
2068
|
+
`/SourceConnectionString:${zr($({
|
|
2036
2069
|
...t,
|
|
2037
2070
|
port: t.port ?? 1433
|
|
2038
2071
|
}))}`,
|
|
@@ -2045,7 +2078,7 @@ var Er = (e, t) => ({
|
|
|
2045
2078
|
reason: `no native backup for dialect '${e}'`
|
|
2046
2079
|
});
|
|
2047
2080
|
}
|
|
2048
|
-
},
|
|
2081
|
+
}, Vr = (e, t, n) => {
|
|
2049
2082
|
switch (e) {
|
|
2050
2083
|
case "postgres": {
|
|
2051
2084
|
if (t.url) return {
|
|
@@ -2090,7 +2123,7 @@ var Er = (e, t) => ({
|
|
|
2090
2123
|
let r = $(t);
|
|
2091
2124
|
return {
|
|
2092
2125
|
kind: "spawn",
|
|
2093
|
-
|
|
2126
|
+
...Fr(e, "client"),
|
|
2094
2127
|
args: [
|
|
2095
2128
|
"--protocol=TCP",
|
|
2096
2129
|
"-h",
|
|
@@ -2109,14 +2142,14 @@ var Er = (e, t) => ({
|
|
|
2109
2142
|
case "turso": return {
|
|
2110
2143
|
kind: "copy",
|
|
2111
2144
|
from: n,
|
|
2112
|
-
to:
|
|
2145
|
+
to: Rr(t)
|
|
2113
2146
|
};
|
|
2114
2147
|
case "mssql": return {
|
|
2115
2148
|
kind: "spawn",
|
|
2116
2149
|
tool: "sqlpackage",
|
|
2117
2150
|
args: [
|
|
2118
2151
|
"/Action:Import",
|
|
2119
|
-
`/TargetConnectionString:${
|
|
2152
|
+
`/TargetConnectionString:${zr($({
|
|
2120
2153
|
...t,
|
|
2121
2154
|
port: t.port ?? 1433
|
|
2122
2155
|
}))}`,
|
|
@@ -2129,13 +2162,31 @@ var Er = (e, t) => ({
|
|
|
2129
2162
|
reason: `no native restore for dialect '${e}'`
|
|
2130
2163
|
});
|
|
2131
2164
|
}
|
|
2132
|
-
},
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2165
|
+
}, Hr = (e) => e.kind === "copy" ? n.async((t) => {
|
|
2166
|
+
let r = `${e.to}.restoring-${oe()}`, i = (e) => {
|
|
2167
|
+
try {
|
|
2168
|
+
de(r);
|
|
2169
|
+
} catch {}
|
|
2170
|
+
t(n.fail(new W({
|
|
2171
|
+
tool: "copy",
|
|
2172
|
+
reason: e
|
|
2173
|
+
})));
|
|
2174
|
+
};
|
|
2175
|
+
ce(e.from, r, (a) => {
|
|
2176
|
+
if (a) {
|
|
2177
|
+
i(`copy ${e.from} → ${e.to}: ${a.message}`);
|
|
2178
|
+
return;
|
|
2179
|
+
}
|
|
2180
|
+
ue(r, e.to, (r) => {
|
|
2181
|
+
if (r) {
|
|
2182
|
+
i(`copy ${e.from} → ${e.to}: ${r.message}`);
|
|
2183
|
+
return;
|
|
2184
|
+
}
|
|
2185
|
+
t(n.void);
|
|
2186
|
+
});
|
|
2187
|
+
});
|
|
2137
2188
|
}) : n.async((t) => {
|
|
2138
|
-
let r =
|
|
2189
|
+
let r = Se(e.tool, [...e.args], {
|
|
2139
2190
|
env: {
|
|
2140
2191
|
...process.env,
|
|
2141
2192
|
...e.env
|
|
@@ -2148,16 +2199,17 @@ var Er = (e, t) => ({
|
|
|
2148
2199
|
}), i = "";
|
|
2149
2200
|
r.stderr?.on("data", (e) => {
|
|
2150
2201
|
i += String(e);
|
|
2151
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(
|
|
2202
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(O(e.stdoutFile)), e.stdinFile && r.stdin && le(e.stdinFile).pipe(r.stdin);
|
|
2152
2203
|
let a = () => {
|
|
2153
2204
|
if (e.stdoutFile !== void 0) try {
|
|
2154
|
-
|
|
2205
|
+
de(e.stdoutFile);
|
|
2155
2206
|
} catch {}
|
|
2156
2207
|
};
|
|
2157
2208
|
r.on("error", (r) => {
|
|
2158
2209
|
a(), t(n.fail(new W({
|
|
2159
2210
|
tool: e.tool,
|
|
2160
|
-
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}
|
|
2211
|
+
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`,
|
|
2212
|
+
...e.hint === void 0 ? {} : { hint: e.hint }
|
|
2161
2213
|
})));
|
|
2162
2214
|
}), r.on("close", (r) => {
|
|
2163
2215
|
if (r === 0) {
|
|
@@ -2166,15 +2218,16 @@ var Er = (e, t) => ({
|
|
|
2166
2218
|
}
|
|
2167
2219
|
a(), t(n.fail(new W({
|
|
2168
2220
|
tool: e.tool,
|
|
2169
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2170
|
-
stderr: i.slice(-2e3)
|
|
2221
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${_e(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
2222
|
+
stderr: i.slice(-2e3),
|
|
2223
|
+
...e.hint === void 0 ? {} : { hint: e.hint }
|
|
2171
2224
|
})));
|
|
2172
2225
|
});
|
|
2173
|
-
}),
|
|
2174
|
-
let t = yield*
|
|
2226
|
+
}), Ur = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Wr = (e) => n.gen(function* () {
|
|
2227
|
+
let t = yield* it(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
2175
2228
|
store: e.store,
|
|
2176
2229
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
2177
|
-
}), r =
|
|
2230
|
+
}), r = $t(t.tables, e.snapshot), i = e.classification ?? Te(e.snapshot), a = e.sampleSize ?? 20, o = Re(r.map((t) => ({
|
|
2178
2231
|
name: t,
|
|
2179
2232
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
2180
2233
|
name: e.name,
|
|
@@ -2186,15 +2239,15 @@ var Er = (e, t) => ({
|
|
|
2186
2239
|
table: i,
|
|
2187
2240
|
chunkSize: a,
|
|
2188
2241
|
...r === void 0 ? {} : { where: r }
|
|
2189
|
-
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
2190
|
-
c.push(...
|
|
2242
|
+
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) => Ve(t, i, o.actions, e.masking.seed));
|
|
2243
|
+
c.push(...Be(d, i, o.actions));
|
|
2191
2244
|
let f = u[0], p = d[0], m = [];
|
|
2192
2245
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
2193
2246
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
2194
2247
|
let n = e.slice(i.length + 1);
|
|
2195
2248
|
m.push({
|
|
2196
2249
|
column: n,
|
|
2197
|
-
action:
|
|
2250
|
+
action: Ur(t),
|
|
2198
2251
|
before: f[n],
|
|
2199
2252
|
after: p[n]
|
|
2200
2253
|
});
|
|
@@ -2209,7 +2262,7 @@ var Er = (e, t) => ({
|
|
|
2209
2262
|
leaks: c,
|
|
2210
2263
|
unclassified: o.unclassified
|
|
2211
2264
|
};
|
|
2212
|
-
}),
|
|
2265
|
+
}), Gr = (e, t) => {
|
|
2213
2266
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
2214
2267
|
for (let t of e) {
|
|
2215
2268
|
let e = n.get(t.name);
|
|
@@ -2264,7 +2317,7 @@ var Er = (e, t) => ({
|
|
|
2264
2317
|
skipColumns: i,
|
|
2265
2318
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
2266
2319
|
};
|
|
2267
|
-
},
|
|
2320
|
+
}, Kr = (e) => {
|
|
2268
2321
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
2269
2322
|
let t = {
|
|
2270
2323
|
safe: "✓",
|
|
@@ -2275,4 +2328,4 @@ var Er = (e, t) => ({
|
|
|
2275
2328
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
2276
2329
|
};
|
|
2277
2330
|
//#endregion
|
|
2278
|
-
export {
|
|
2331
|
+
export { Dt as ASSETS_DIR, Ot as ASSET_INDEX_FILE, H as BundleError, Ke as CodecError, Je as CompressionError, $e as CrossDialectError, Et as DATA_DIR, we as DEFAULT_CLASS_ACTIONS, He as ENVIRONMENT_LOCAL_TABLES, Ue as ENVIRONMENT_PORTABLE_TABLES, vt as FORMAT_VERSION, Qe as ImportModeError, qe as IntegrityError, kt as LEDGER_FILE, K as MANIFEST_FILE, Ze as MaskingError, W as NativeToolError, Ye as RowsRefusedError, Zn as SAVEPOINT_BATCH_SIZE, En as STAGING_PREFIX, Xe as SchemaDriftError, U as ScopeError, Ie as applyAction, $n as asImportError, ir as assessLiveImport, zt as assetDone, Lr as backupArtifactName, Br as backupCommand, mn as canonicalType, ut as captureAssetsToCas, _n as checkPortability, Yn as checkSchemaDrift, Te as classificationFromSnapshot, Gr as classifyImportDrift, ht as compressionExt, tt as computeSubsetIds, Hn as decideStaging, wt as decodeManifest, Yt as decodeRow, qt as decodeValue, jr as decryptStream, rr as defineDataProfile, Jn as diffSnapshots, vr as dirBlobSink, jn as dropStagingSql, It as emptyLedger, Ct as encodeManifest, Jt as encodeRow, Gt as encodeValue, Ar as encryptStream, Ge as environmentLocalNotice, We as environmentLocalReason, Ln as execStatements, Kr as formatImportDrift, vn as formatIssue, pt as hasZstd, V as isEnvironmentLocal, gt as makeCompressor, _t as makeDecompressor, Ve as maskRow, Qt as ndjsonToRows, lt as noAssetProgress, pn as outsideReferenceRefusal, fn as outsideReferencesInto, fr as packBundle, gr as peekBundle, mt as pickCompression, Re as planMasking, Wr as previewMasking, dt as readAssetIndex, Lt as readLedger, Ee as resolveAction, it as resolveScope, ct as restoreAssetFromCas, ft as restoreAssetsFromCas, Vr as restoreCommand, Zt as rowsToNdjson, dn as runExport, nr as runImport, Hr as runNativeStep, Be as scanForLeaks, X as stagingNameFor, ot as storageAssetSink, at as storageAssetSource, Pr as storageBlobSink, Mr as storageBlobSource, Nr as storageRefsBlobSource, nt as subsetToScope, Rt as tableDone, $t as topoSortTables, hr as unpackBundle, _r as unpackBundleToDir, st as writeAssetToCas, J as writeLedger };
|