@voltro/data-transfer 0.46.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/index.js +757 -729
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Runtime as o, Schedule as s, Schema as c, Stream as l } from "effect";
|
|
2
|
-
import {
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { STORAGE_REFS_TABLE as
|
|
6
|
-
import * as
|
|
7
|
-
import { PassThrough as
|
|
8
|
-
import { copyFile as
|
|
9
|
-
import { SqlClient as
|
|
10
|
-
import { mkdir as
|
|
11
|
-
import { basename as
|
|
12
|
-
import { promisify as
|
|
13
|
-
import { spawn as
|
|
2
|
+
import { IMPORT_RUNS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d, classifyConstraintViolation as f, derivedRowId as p, eq as m, extractDbCause as h, getTable as g, inSet as _, isNotNull as v, or as y, recordsTable as b, registerTable as ee, streamTable as x, suspendWriteRecorders as S, suspendWriteRecordersEffect as te, unregisterTable as ne } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as re } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as C, createDecipheriv as w, createHash as T, createHmac as ie, randomBytes as E, scrypt as ae } from "node:crypto";
|
|
5
|
+
import { STORAGE_REFS_TABLE as oe, fromNodeReadable as se, getObjectStream as ce, putObjectStream as D, toNodeReadable as O } from "@voltro/plugin-storage";
|
|
6
|
+
import * as k from "node:zlib";
|
|
7
|
+
import { PassThrough as A, Readable as j } from "node:stream";
|
|
8
|
+
import { copyFile as le, createReadStream as ue, createWriteStream as M, existsSync as N, unlinkSync as de } from "node:fs";
|
|
9
|
+
import { SqlClient as P } from "@effect/sql";
|
|
10
|
+
import { mkdir as F, readdir as fe, stat as pe } from "node:fs/promises";
|
|
11
|
+
import { basename as I, delimiter as me, dirname as he, join as L, resolve as ge, sep as _e } from "node:path";
|
|
12
|
+
import { promisify as ve } from "node:util";
|
|
13
|
+
import { spawn as ye } from "node:child_process";
|
|
14
14
|
//#region src/consistency.ts
|
|
15
|
-
var
|
|
15
|
+
var be = (e) => {
|
|
16
16
|
switch (e) {
|
|
17
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
18
18
|
case "mysql":
|
|
@@ -22,7 +22,7 @@ var ye = (e) => {
|
|
|
22
22
|
case "turso": return [];
|
|
23
23
|
default: return [];
|
|
24
24
|
}
|
|
25
|
-
},
|
|
25
|
+
}, xe = {
|
|
26
26
|
email: { fake: "email" },
|
|
27
27
|
fullName: { fake: "fullName" },
|
|
28
28
|
firstName: { fake: "firstName" },
|
|
@@ -37,7 +37,7 @@ var ye = (e) => {
|
|
|
37
37
|
date: "dateShift",
|
|
38
38
|
secret: "null",
|
|
39
39
|
freeText: "redact"
|
|
40
|
-
},
|
|
40
|
+
}, Se = (e) => {
|
|
41
41
|
let t = {};
|
|
42
42
|
for (let n of e.tables) {
|
|
43
43
|
let e = {};
|
|
@@ -49,15 +49,15 @@ var ye = (e) => {
|
|
|
49
49
|
t[n.name] = e;
|
|
50
50
|
}
|
|
51
51
|
return t;
|
|
52
|
-
},
|
|
52
|
+
}, Ce = (e, t, n, r) => {
|
|
53
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
54
54
|
if (i) return i;
|
|
55
55
|
if (n?.sensitive) {
|
|
56
56
|
let e = n.sensitive.class;
|
|
57
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? xe[e] ?? "redact";
|
|
58
58
|
}
|
|
59
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
60
|
-
},
|
|
60
|
+
}, we = (e, t) => ie("sha256", e).update(t == null ? "∅" : String(t)).digest(), R = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), Te = [
|
|
61
61
|
"Ada",
|
|
62
62
|
"Alan",
|
|
63
63
|
"Grace",
|
|
@@ -70,7 +70,7 @@ var ye = (e) => {
|
|
|
70
70
|
"Guido",
|
|
71
71
|
"Margaret",
|
|
72
72
|
"Bjarne"
|
|
73
|
-
],
|
|
73
|
+
], Ee = [
|
|
74
74
|
"Lovelace",
|
|
75
75
|
"Turing",
|
|
76
76
|
"Hopper",
|
|
@@ -83,7 +83,7 @@ var ye = (e) => {
|
|
|
83
83
|
"Rossum",
|
|
84
84
|
"Hamilton",
|
|
85
85
|
"Stroustrup"
|
|
86
|
-
],
|
|
86
|
+
], De = [
|
|
87
87
|
"Maple",
|
|
88
88
|
"Oak",
|
|
89
89
|
"Cedar",
|
|
@@ -92,7 +92,7 @@ var ye = (e) => {
|
|
|
92
92
|
"Birch",
|
|
93
93
|
"Willow",
|
|
94
94
|
"Ash"
|
|
95
|
-
],
|
|
95
|
+
], Oe = [
|
|
96
96
|
"Initech",
|
|
97
97
|
"Hooli",
|
|
98
98
|
"Umbrella",
|
|
@@ -101,23 +101,23 @@ var ye = (e) => {
|
|
|
101
101
|
"Soylent",
|
|
102
102
|
"Stark",
|
|
103
103
|
"Wayne"
|
|
104
|
-
],
|
|
105
|
-
let r =
|
|
104
|
+
], z = (e, t, n) => e[R(t, n) % e.length], ke = (e, t, n) => {
|
|
105
|
+
let r = we(n, `${e}:${String(t)}`), i = R(r, 1) % 1e4;
|
|
106
106
|
switch (e) {
|
|
107
|
-
case "email": return `${
|
|
108
|
-
case "fullName": return `${
|
|
109
|
-
case "firstName": return
|
|
110
|
-
case "lastName": return
|
|
111
|
-
case "username": return `${
|
|
112
|
-
case "phone": return `+1${String(
|
|
113
|
-
case "address": return `${
|
|
114
|
-
case "company": return
|
|
107
|
+
case "email": return `${z(Te, r, 0).toLowerCase()}.${z(Ee, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${z(Te, r, 0)} ${z(Ee, r, 1)}`;
|
|
109
|
+
case "firstName": return z(Te, r, 0);
|
|
110
|
+
case "lastName": return z(Ee, r, 1);
|
|
111
|
+
case "username": return `${z(Te, r, 0).toLowerCase()}${i}`;
|
|
112
|
+
case "phone": return `+1${String(R(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
+
case "address": return `${R(r, 0) % 9899 + 100} ${z(De, r, 1)} St`;
|
|
114
|
+
case "company": return z(Oe, r, 0);
|
|
115
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
116
|
-
case "ip": return `${
|
|
117
|
-
case "creditCard": return
|
|
116
|
+
case "ip": return `${R(r, 0) % 256}.${R(r, 1) % 256}.${R(r, 2) % 256}.${R(r, 3) % 254 + 1}`;
|
|
117
|
+
case "creditCard": return Ae(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
118
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
119
119
|
}
|
|
120
|
-
},
|
|
120
|
+
}, Ae = (e) => {
|
|
121
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
122
122
|
t.forEach((e, t) => {
|
|
123
123
|
let r = e;
|
|
@@ -125,31 +125,31 @@ var ye = (e) => {
|
|
|
125
125
|
});
|
|
126
126
|
let r = (10 - n % 10) % 10;
|
|
127
127
|
return e.slice(0, 15) + String(r);
|
|
128
|
-
},
|
|
128
|
+
}, je = 864e5, Me = (e, t) => {
|
|
129
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
130
130
|
if (Number.isNaN(n)) return e;
|
|
131
|
-
let r =
|
|
131
|
+
let r = R(we(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * je);
|
|
132
132
|
return e instanceof Date ? i : i.toISOString();
|
|
133
|
-
},
|
|
133
|
+
}, Ne = (e, t) => {
|
|
134
134
|
if (t.value === null || t.value === void 0) return null;
|
|
135
135
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
136
136
|
if (e === "keep") return t.value;
|
|
137
137
|
if (e === "null") return null;
|
|
138
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
139
|
-
if (e === "hash") return
|
|
140
|
-
if (e === "dateShift") return
|
|
141
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return we(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Me(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return ke(e.fake, t.value, t.seed);
|
|
142
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
143
143
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
144
|
-
},
|
|
144
|
+
}, Pe = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Fe = (e, t, n) => {
|
|
145
145
|
let r = {}, i = [], a = [];
|
|
146
146
|
for (let o of e) for (let e of o.columns) {
|
|
147
|
-
let s = `${o.name}.${e.name}`, c =
|
|
147
|
+
let s = `${o.name}.${e.name}`, c = Ce(o.name, e.name, t[o.name]?.[e.name], n);
|
|
148
148
|
if (c === "error") {
|
|
149
149
|
i.push(s);
|
|
150
150
|
continue;
|
|
151
151
|
}
|
|
152
|
-
if (!
|
|
152
|
+
if (!Pe(c)) {
|
|
153
153
|
a.push({
|
|
154
154
|
column: s,
|
|
155
155
|
action: c
|
|
@@ -163,7 +163,7 @@ var ye = (e) => {
|
|
|
163
163
|
unclassified: i,
|
|
164
164
|
invalid: a
|
|
165
165
|
};
|
|
166
|
-
},
|
|
166
|
+
}, Ie = [
|
|
167
167
|
{
|
|
168
168
|
name: "email",
|
|
169
169
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -180,7 +180,7 @@ var ye = (e) => {
|
|
|
180
180
|
name: "creditCard",
|
|
181
181
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
182
182
|
}
|
|
183
|
-
],
|
|
183
|
+
], Le = (e, t, n) => {
|
|
184
184
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
185
185
|
for (let [a, o] of Object.entries(n)) {
|
|
186
186
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -188,7 +188,7 @@ var ye = (e) => {
|
|
|
188
188
|
for (let a of e) {
|
|
189
189
|
let e = a[n];
|
|
190
190
|
if (typeof e != "string" || e.length === 0) continue;
|
|
191
|
-
let o =
|
|
191
|
+
let o = Ie.find((t) => t.re.test(e));
|
|
192
192
|
if (o && !i.has(n)) {
|
|
193
193
|
i.add(n), r.push({
|
|
194
194
|
table: t,
|
|
@@ -201,9 +201,9 @@ var ye = (e) => {
|
|
|
201
201
|
}
|
|
202
202
|
}
|
|
203
203
|
return r;
|
|
204
|
-
},
|
|
204
|
+
}, Re = (e, t, n, r, i) => {
|
|
205
205
|
let a = {};
|
|
206
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
206
|
+
for (let [o, s] of Object.entries(e)) a[o] = Ne(n[`${t}.${o}`] ?? "keep", {
|
|
207
207
|
value: s,
|
|
208
208
|
table: t,
|
|
209
209
|
column: o,
|
|
@@ -211,33 +211,34 @@ var ye = (e) => {
|
|
|
211
211
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
212
212
|
});
|
|
213
213
|
return a;
|
|
214
|
-
},
|
|
214
|
+
}, ze = /* @__PURE__ */ new Map([
|
|
215
215
|
["_voltro_migration_plans", "the schema fingerprint THIS deployment applied — a foreign row makes the next boot refuse"],
|
|
216
216
|
["_voltro_migrations", "which file-based migrations ran HERE — a foreign row marks unrun migrations as done"],
|
|
217
217
|
["_voltro_seeds", "which seeds ran HERE — a foreign row suppresses seeding the target still needs"],
|
|
218
218
|
["_voltro_cdc_offsets", "binlog/WAL positions of the SOURCE server — meaningless and unsafe against another one"],
|
|
219
219
|
["_voltro_schedule_claims", "live leases held by THIS deployment's replicas — a foreign row hands out a lease nobody holds"],
|
|
220
|
+
["_voltro_data_imports", "which bundles were imported HERE — a foreign row claims an import this deployment never ran"],
|
|
220
221
|
["_voltro_replace_in_progress", "the marker a destructive import HERE left behind — a foreign row refuses a boot over a healthy database"],
|
|
221
222
|
["_voltro_wakeups", "pending wakeups addressed to THIS deployment's runners"],
|
|
222
223
|
["_voltro_workflow_watermarks", "how far a consumer has read THIS deployment's event journal — a foreign position skips events"],
|
|
223
224
|
["_voltro_workflow_pending", "starts waiting to be drained HERE — a foreign row makes the target RUN a workflow somebody queued elsewhere"],
|
|
224
225
|
["_voltro_workflow_admissions", "the in-flight accounting a concurrency gate decides on — a foreign row makes the target refuse or admit wrongly"],
|
|
225
226
|
["_voltro_workflow_pauses", "which workflows an operator paused HERE — a foreign row silently pauses them in the target"]
|
|
226
|
-
]),
|
|
227
|
+
]), B = (e) => ze.has(e), Be = (e) => ze.get(e), Ve = (e, t) => {
|
|
227
228
|
if (e.length === 0) return;
|
|
228
|
-
let n = e.map((e) => ` ${e} — ${
|
|
229
|
+
let n = e.map((e) => ` ${e} — ${Be(e) ?? "describes this deployment"}`);
|
|
229
230
|
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
230
|
-
},
|
|
231
|
+
}, V = class extends c.TaggedError()("BundleError", {
|
|
231
232
|
reason: c.String,
|
|
232
233
|
path: c.optional(c.String)
|
|
233
|
-
}) {},
|
|
234
|
+
}) {}, He = class extends c.TaggedError()("CodecError", {
|
|
234
235
|
table: c.String,
|
|
235
236
|
reason: c.String
|
|
236
|
-
}) {},
|
|
237
|
+
}) {}, Ue = class extends c.TaggedError()("IntegrityError", {
|
|
237
238
|
what: c.String,
|
|
238
239
|
expected: c.String,
|
|
239
240
|
actual: c.String
|
|
240
|
-
}) {},
|
|
241
|
+
}) {}, We = class extends t.TaggedError("CompressionError") {}, H = class extends c.TaggedError()("ScopeError", { reason: c.String }) {}, Ge = class extends c.TaggedError()("RowsRefusedError", {
|
|
241
242
|
rows: c.Array(c.Struct({
|
|
242
243
|
table: c.String,
|
|
243
244
|
id: c.String,
|
|
@@ -251,29 +252,29 @@ var ye = (e) => {
|
|
|
251
252
|
refused: c.Number,
|
|
252
253
|
primary: c.Number
|
|
253
254
|
}))
|
|
254
|
-
}) {},
|
|
255
|
+
}) {}, Ke = class extends c.TaggedError()("SchemaDriftError", {
|
|
255
256
|
bundleFingerprint: c.String,
|
|
256
257
|
targetFingerprint: c.String,
|
|
257
258
|
diff: c.Array(c.String)
|
|
258
|
-
}) {},
|
|
259
|
+
}) {}, U = class extends t.TaggedError("NativeToolError") {
|
|
259
260
|
get message() {
|
|
260
261
|
let e = this.stderr?.trim();
|
|
261
262
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
262
263
|
}
|
|
263
|
-
},
|
|
264
|
+
}, qe = class extends c.TaggedError()("MaskingError", {
|
|
264
265
|
unclassified: c.Array(c.String),
|
|
265
266
|
invalidActions: c.optional(c.Array(c.String))
|
|
266
|
-
}) {},
|
|
267
|
+
}) {}, Je = class extends c.TaggedError()("ImportModeError", {
|
|
267
268
|
mode: c.String,
|
|
268
269
|
reason: c.String
|
|
269
|
-
}) {},
|
|
270
|
+
}) {}, Ye = class extends c.TaggedError()("CrossDialectError", {
|
|
270
271
|
sourceDialect: c.String,
|
|
271
272
|
targetDialect: c.String,
|
|
272
273
|
blocking: c.Array(c.String)
|
|
273
|
-
}) {},
|
|
274
|
+
}) {}, W = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Xe = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
274
275
|
col: e.name,
|
|
275
276
|
refTable: e.references.table
|
|
276
|
-
})),
|
|
277
|
+
})), Ze = (e, t, r) => n.gen(function* () {
|
|
277
278
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), u = (e) => {
|
|
278
279
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
279
280
|
return s.set(e, t), t;
|
|
@@ -291,10 +292,10 @@ var ye = (e) => {
|
|
|
291
292
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
292
293
|
};
|
|
293
294
|
}, p = (r, i) => {
|
|
294
|
-
let a = f(r,
|
|
295
|
+
let a = f(r, W(t, r), Xe(t, r));
|
|
295
296
|
return l.runForEach(x(e, {
|
|
296
297
|
table: r,
|
|
297
|
-
pk:
|
|
298
|
+
pk: W(t, r),
|
|
298
299
|
...i === void 0 ? {} : { where: i }
|
|
299
300
|
}), (e) => n.sync(() => a(e)));
|
|
300
301
|
}, m = n.gen(function* () {
|
|
@@ -303,7 +304,7 @@ var ye = (e) => {
|
|
|
303
304
|
c.clear();
|
|
304
305
|
for (let [n, r] of e) {
|
|
305
306
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
306
|
-
e.length !== 0 && (yield* p(n,
|
|
307
|
+
e.length !== 0 && (yield* p(n, _(W(t, n), e)));
|
|
307
308
|
}
|
|
308
309
|
}
|
|
309
310
|
});
|
|
@@ -318,29 +319,29 @@ var ye = (e) => {
|
|
|
318
319
|
refTable: e.references?.table
|
|
319
320
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
320
321
|
if (a.length === 0) continue;
|
|
321
|
-
let o = s.get(t)?.size ?? 0, c = a.map((e) =>
|
|
322
|
-
yield* p(t, c.length === 1 ? c[0] :
|
|
322
|
+
let o = s.get(t)?.size ?? 0, c = a.map((e) => _(e.col, [...s.get(e.refTable)]));
|
|
323
|
+
yield* p(t, c.length === 1 ? c[0] : y(...c)), (s.get(t)?.size ?? 0) > o && !e.has(t) && r.push(t), (s.get(t)?.size ?? 0) > o && (n = !0);
|
|
323
324
|
}
|
|
324
325
|
for (let t of r) e.add(t);
|
|
325
326
|
if (yield* m, !n) break;
|
|
326
327
|
}
|
|
327
328
|
}
|
|
328
329
|
return o;
|
|
329
|
-
}),
|
|
330
|
+
}), Qe = (e, t) => ({
|
|
330
331
|
tables: [...e.keys()],
|
|
331
332
|
predicateFor: (n) => {
|
|
332
333
|
let r = e.get(n);
|
|
333
|
-
return r ?
|
|
334
|
+
return r ? _(W(t, n), [...r]) : void 0;
|
|
334
335
|
}
|
|
335
|
-
}),
|
|
336
|
+
}), $e = "tenantId", et = (e, t, r) => n.gen(function* () {
|
|
336
337
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
337
338
|
if (e.kind === "all") return {
|
|
338
|
-
tables: i.filter((e) => !
|
|
339
|
+
tables: i.filter((e) => !B(e)),
|
|
339
340
|
predicateFor: () => void 0
|
|
340
341
|
};
|
|
341
342
|
if (e.kind === "tables") {
|
|
342
343
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
343
|
-
if (a.length > 0) return yield* n.fail(new
|
|
344
|
+
if (a.length > 0) return yield* n.fail(new H({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
344
345
|
let o = new Set(t);
|
|
345
346
|
return {
|
|
346
347
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -348,30 +349,30 @@ var ye = (e) => {
|
|
|
348
349
|
};
|
|
349
350
|
}
|
|
350
351
|
let o = e.tenantId;
|
|
351
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
352
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
353
|
-
if (s.length === 0) return yield* n.fail(new
|
|
354
|
-
let c = yield*
|
|
355
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
352
|
+
if (o === void 0 || o === "") return yield* n.fail(new H({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
353
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === $e) ?? !1);
|
|
354
|
+
if (s.length === 0) return yield* n.fail(new H({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${$e}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
355
|
+
let c = yield* Ze(r.store, t, {
|
|
356
|
+
seeds: Object.fromEntries(s.map((e) => [e, m($e, o)])),
|
|
356
357
|
pinned: s,
|
|
357
358
|
children: { roots: s }
|
|
358
359
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
359
360
|
return {
|
|
360
361
|
tables: [...s, ...l],
|
|
361
|
-
predicateFor: (e) => u.has(e) ?
|
|
362
|
+
predicateFor: (e) => u.has(e) ? m($e, o) : c.has(e) ? _(W(t, e), [...c.get(e)]) : void 0
|
|
362
363
|
};
|
|
363
|
-
}),
|
|
364
|
+
}), tt = (e, t) => ({
|
|
364
365
|
list: t,
|
|
365
|
-
open: (t) =>
|
|
366
|
-
}),
|
|
367
|
-
write: (t, n, r) =>
|
|
366
|
+
open: (t) => ce(e, t)
|
|
367
|
+
}), nt = (e) => ({
|
|
368
|
+
write: (t, n, r) => D(e, t, n, r),
|
|
368
369
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
369
|
-
}),
|
|
370
|
+
}), rt = (e, t) => n.tryPromise({
|
|
370
371
|
try: async () => {
|
|
371
372
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
372
373
|
u.update(e), d += e.length, n(null, e);
|
|
373
374
|
} });
|
|
374
|
-
await o(await
|
|
375
|
+
await o(await O(e.stream), f, n(l));
|
|
375
376
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
376
377
|
try {
|
|
377
378
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -384,23 +385,23 @@ var ye = (e) => {
|
|
|
384
385
|
contentType: e.contentType
|
|
385
386
|
};
|
|
386
387
|
},
|
|
387
|
-
catch: (e) => new
|
|
388
|
+
catch: (e) => new V({
|
|
388
389
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
389
390
|
path: t
|
|
390
391
|
})
|
|
391
|
-
}),
|
|
392
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u =
|
|
392
|
+
}), it = (e, t, r) => n.gen(function* () {
|
|
393
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u = se("bundle", () => a(s)).pipe(l.tap((e) => n.sync(() => c.update(e))));
|
|
393
394
|
yield* e.write(r.key, u, {
|
|
394
395
|
contentType: r.contentType,
|
|
395
396
|
totalSize: r.size
|
|
396
397
|
});
|
|
397
398
|
let d = c.digest("hex");
|
|
398
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
399
|
+
if (d !== r.sha256) return yield* n.fail(new Ue({
|
|
399
400
|
what: `asset ${r.key}`,
|
|
400
401
|
expected: r.sha256,
|
|
401
402
|
actual: d
|
|
402
403
|
}));
|
|
403
|
-
}),
|
|
404
|
+
}), at = () => typeof k.createZstdCompress == "function", ot = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : at() ? "zstd" : "gzip", st = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", ct = (e) => e === "zstd" ? k.createZstdCompress() : e === "gzip" ? k.createGzip() : new A(), lt = (e) => e === "zstd" ? k.createZstdDecompress() : e === "gzip" ? k.createGunzip() : new A(), ut = 1, dt = c.Struct({
|
|
404
405
|
name: c.String,
|
|
405
406
|
file: c.String,
|
|
406
407
|
rowCount: c.Number,
|
|
@@ -410,11 +411,11 @@ var ye = (e) => {
|
|
|
410
411
|
value: c.String
|
|
411
412
|
}),
|
|
412
413
|
primaryKey: c.String
|
|
413
|
-
}),
|
|
414
|
+
}), ft = c.Struct({
|
|
414
415
|
kind: c.Literal("all", "tenant", "tables"),
|
|
415
416
|
tenantId: c.optional(c.String),
|
|
416
417
|
tables: c.optional(c.Array(c.String))
|
|
417
|
-
}),
|
|
418
|
+
}), pt = c.Struct({
|
|
418
419
|
count: c.Number,
|
|
419
420
|
totalBytes: c.Number,
|
|
420
421
|
index: c.optional(c.String)
|
|
@@ -425,23 +426,23 @@ c.Struct({
|
|
|
425
426
|
size: c.Number,
|
|
426
427
|
contentType: c.String
|
|
427
428
|
});
|
|
428
|
-
var
|
|
429
|
+
var mt = c.Struct({
|
|
429
430
|
formatVersion: c.Number,
|
|
430
431
|
createdAt: c.String,
|
|
431
432
|
source: c.Struct({
|
|
432
433
|
dialect: c.String,
|
|
433
434
|
schemaFingerprint: c.String
|
|
434
435
|
}),
|
|
435
|
-
scope:
|
|
436
|
+
scope: ft,
|
|
436
437
|
consistency: c.Literal("live", "snapshot"),
|
|
437
438
|
compression: c.Literal("zstd", "gzip", "none"),
|
|
438
|
-
tables: c.Array(
|
|
439
|
-
assets:
|
|
439
|
+
tables: c.Array(dt),
|
|
440
|
+
assets: pt,
|
|
440
441
|
masking: c.optional(c.Struct({
|
|
441
442
|
policyId: c.optional(c.String),
|
|
442
443
|
transformed: c.Array(c.String)
|
|
443
444
|
}))
|
|
444
|
-
}),
|
|
445
|
+
}), ht = c.encodeSync(c.parseJson(mt)), gt = c.decodeUnknownSync(c.parseJson(mt)), G = "manifest.json", _t = "README.md", vt = "data", yt = "assets", bt = "assets/index.ndjson", xt = ".ledger.json", St = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", Ct = (e, t) => {
|
|
445
446
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
446
447
|
return `# Voltro data export
|
|
447
448
|
|
|
@@ -455,7 +456,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
455
456
|
| **Format version** | ${e.formatVersion} |
|
|
456
457
|
| **Consistency** | ${e.consistency} |
|
|
457
458
|
| **Compression** | ${e.compression} |
|
|
458
|
-
| **Scope** | ${
|
|
459
|
+
| **Scope** | ${St(e.scope)} |
|
|
459
460
|
| **Tables** | ${e.tables.length} |
|
|
460
461
|
| **Rows (total)** | ${n} |
|
|
461
462
|
| **Assets** | ${a} |
|
|
@@ -480,39 +481,39 @@ voltro data import <this-directory>
|
|
|
480
481
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
481
482
|
\`\`\`
|
|
482
483
|
`;
|
|
483
|
-
},
|
|
484
|
+
}, wt = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), Tt = a.exponential({
|
|
484
485
|
start: .01,
|
|
485
486
|
factor: 2,
|
|
486
487
|
count: 16
|
|
487
|
-
}),
|
|
488
|
+
}), Et = i.histogram("voltro_data_transfer_table_seconds", Tt, "Per-table export/import duration in seconds, tagged by phase."), K = (e, t) => {
|
|
488
489
|
if (e) try {
|
|
489
490
|
e(t);
|
|
490
491
|
} catch {}
|
|
491
|
-
},
|
|
492
|
+
}, Dt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(Et.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), wt.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
492
493
|
table: t,
|
|
493
494
|
index: r,
|
|
494
495
|
tableCount: a,
|
|
495
496
|
phase: e
|
|
496
|
-
} })),
|
|
497
|
+
} })), Ot = () => ({
|
|
497
498
|
tables: {},
|
|
498
499
|
assets: {}
|
|
499
|
-
}),
|
|
500
|
+
}), kt = (e) => n.tryPromise(async () => {
|
|
500
501
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
501
502
|
return JSON.parse(t);
|
|
502
|
-
}).pipe(n.orElseSucceed(
|
|
503
|
+
}).pipe(n.orElseSucceed(Ot)), q = (e, t) => n.tryPromise({
|
|
503
504
|
try: async () => {
|
|
504
505
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
505
506
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
506
507
|
},
|
|
507
|
-
catch: (t) => new
|
|
508
|
+
catch: (t) => new V({
|
|
508
509
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
509
510
|
path: e
|
|
510
511
|
})
|
|
511
|
-
}),
|
|
512
|
+
}), At = (e, t) => t in e.tables, jt = (e, t) => t in e.assets, Mt = /* @__PURE__ */ new Set([
|
|
512
513
|
"json",
|
|
513
514
|
"array",
|
|
514
515
|
"vector"
|
|
515
|
-
]),
|
|
516
|
+
]), Nt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", Pt = (e) => Buffer.from(e).toString("base64"), Ft = (e) => new Uint8Array(Buffer.from(e, "base64")), It = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), Lt = (e, t) => e == null ? null : t !== void 0 && Mt.has(t) ? e : t === "bytes" ? Pt(e) : t === "timestamp" || t === "date" ? It(e) : typeof e == "bigint" ? {
|
|
516
517
|
$v: "bigint",
|
|
517
518
|
d: e.toString()
|
|
518
519
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -523,30 +524,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
523
524
|
d: e.toISOString()
|
|
524
525
|
} : e instanceof Uint8Array ? {
|
|
525
526
|
$v: "bytes",
|
|
526
|
-
d:
|
|
527
|
-
} : e,
|
|
527
|
+
d: Pt(e)
|
|
528
|
+
} : e, Rt = (e) => {
|
|
528
529
|
switch (e.$v) {
|
|
529
530
|
case "bigint": return BigInt(e.d);
|
|
530
531
|
case "number": return Number(e.d);
|
|
531
532
|
case "date": return new Date(e.d);
|
|
532
|
-
case "bytes": return
|
|
533
|
+
case "bytes": return Ft(e.d);
|
|
533
534
|
default: return e;
|
|
534
535
|
}
|
|
535
|
-
},
|
|
536
|
+
}, zt = (e, t) => e == null ? null : t !== void 0 && Mt.has(t) ? e : Nt(e) ? Rt(e) : t === "bytes" ? Ft(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Bt = (e, t) => {
|
|
536
537
|
let n = {};
|
|
537
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
538
|
+
for (let [r, i] of Object.entries(e)) n[r] = Lt(i, t[r]);
|
|
538
539
|
return n;
|
|
539
|
-
},
|
|
540
|
+
}, Vt = (e, t) => {
|
|
540
541
|
let n = {};
|
|
541
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
542
|
+
for (let [r, i] of Object.entries(e)) n[r] = zt(i, t[r]);
|
|
542
543
|
return n;
|
|
543
|
-
},
|
|
544
|
-
try: () =>
|
|
545
|
-
catch: (e) => new
|
|
544
|
+
}, Ht = new TextEncoder(), Ut = (e, t) => e.pipe(l.map((e) => Ht.encode(`${JSON.stringify(Bt(e, t))}\n`))), Wt = (e, t, r) => e.pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.mapEffect((e) => n.try({
|
|
545
|
+
try: () => Vt(JSON.parse(e), r),
|
|
546
|
+
catch: (e) => new He({
|
|
546
547
|
table: t,
|
|
547
548
|
reason: String(e?.message ?? e)
|
|
548
549
|
})
|
|
549
|
-
}))),
|
|
550
|
+
}))), Gt = (e, t) => {
|
|
550
551
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
551
552
|
for (let e of t.tables) {
|
|
552
553
|
if (!n.has(e.name)) continue;
|
|
@@ -566,39 +567,39 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
566
567
|
};
|
|
567
568
|
for (let t of e) n.has(t) && s(t);
|
|
568
569
|
return i;
|
|
569
|
-
},
|
|
570
|
+
}, J = (e) => {
|
|
570
571
|
let t = e, n = [];
|
|
571
572
|
typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
|
|
572
573
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
573
|
-
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(
|
|
574
|
+
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(J(t.cause)) : n.push(r), n.length === 0) {
|
|
574
575
|
let t = String(e ?? "").trim();
|
|
575
|
-
return t === "" || t === "[object Object]" ?
|
|
576
|
+
return t === "" || t === "[object Object]" ? Kt(e) : t;
|
|
576
577
|
}
|
|
577
578
|
return n.join(": ");
|
|
578
|
-
},
|
|
579
|
+
}, Kt = (e) => {
|
|
579
580
|
try {
|
|
580
581
|
let t = JSON.stringify(e);
|
|
581
582
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
582
583
|
} catch {
|
|
583
584
|
return "unknown error (not serialisable)";
|
|
584
585
|
}
|
|
585
|
-
},
|
|
586
|
+
}, qt = (e, t) => {
|
|
586
587
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
587
588
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
588
589
|
return r;
|
|
589
|
-
},
|
|
590
|
+
}, Jt = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), Yt = (e, t) => {
|
|
590
591
|
if (t.length === 0) return e;
|
|
591
592
|
let n = { ...e };
|
|
592
593
|
for (let e of t) delete n[e];
|
|
593
594
|
return n;
|
|
594
|
-
},
|
|
595
|
+
}, Xt = (e, t) => {
|
|
595
596
|
let n = e.tables.find((e) => e.name === t);
|
|
596
597
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
597
598
|
let r = n.columns.find((e) => e.type === "id");
|
|
598
599
|
if (r !== void 0) return { column: r.name };
|
|
599
600
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
600
601
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
601
|
-
},
|
|
602
|
+
}, Zt = (e, t, r) => n.tryPromise({
|
|
602
603
|
try: async () => {
|
|
603
604
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
604
605
|
d.update(e);
|
|
@@ -607,7 +608,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
607
608
|
n(null, e);
|
|
608
609
|
} }), m = c.fromWeb(l.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
609
610
|
try {
|
|
610
|
-
await s(m, p,
|
|
611
|
+
await s(m, p, ct(r), n(h)), await i.rename(h, t);
|
|
611
612
|
} catch (e) {
|
|
612
613
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
613
614
|
}
|
|
@@ -616,27 +617,27 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
616
617
|
checksum: d.digest("hex")
|
|
617
618
|
};
|
|
618
619
|
},
|
|
619
|
-
catch: (e) => new
|
|
620
|
-
reason: `write table failed: ${
|
|
620
|
+
catch: (e) => new V({
|
|
621
|
+
reason: `write table failed: ${J(e)}`,
|
|
621
622
|
path: t
|
|
622
623
|
})
|
|
623
|
-
}),
|
|
624
|
+
}), Qt = (e, t) => n.tryPromise({
|
|
624
625
|
try: async () => {
|
|
625
626
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
626
627
|
},
|
|
627
|
-
catch: (t) => new
|
|
628
|
-
reason: `append failed: ${
|
|
628
|
+
catch: (t) => new V({
|
|
629
|
+
reason: `append failed: ${J(t)}`,
|
|
629
630
|
path: e
|
|
630
631
|
})
|
|
631
|
-
}),
|
|
632
|
+
}), $t = (e, t) => n.gen(function* () {
|
|
632
633
|
let r = [], i = t.ordered.length;
|
|
633
634
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
634
|
-
let o = t.ordered[a], s =
|
|
635
|
-
if ("refusal" in c) return yield* n.fail(new
|
|
636
|
-
let u = c.column, d = `${
|
|
637
|
-
if (
|
|
635
|
+
let o = t.ordered[a], s = qt(t.snapshot, o), c = Xt(t.snapshot, o);
|
|
636
|
+
if ("refusal" in c) return yield* n.fail(new H({ reason: `export: ${c.refusal}` }));
|
|
637
|
+
let u = c.column, d = `${vt}/${o}.ndjson${st(t.compression)}`;
|
|
638
|
+
if (At(t.ledger, o)) {
|
|
638
639
|
let e = t.ledger.tables[o];
|
|
639
|
-
|
|
640
|
+
K(t.onProgress, {
|
|
640
641
|
phase: "export",
|
|
641
642
|
table: o,
|
|
642
643
|
index: a,
|
|
@@ -654,7 +655,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
654
655
|
});
|
|
655
656
|
continue;
|
|
656
657
|
}
|
|
657
|
-
|
|
658
|
+
K(t.onProgress, {
|
|
658
659
|
phase: "export",
|
|
659
660
|
table: o,
|
|
660
661
|
index: a,
|
|
@@ -668,11 +669,11 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
668
669
|
chunkSize: t.chunkSize,
|
|
669
670
|
retry: t.retry,
|
|
670
671
|
...f === void 0 ? {} : { where: f }
|
|
671
|
-
}), m =
|
|
672
|
+
}), m = Jt(t.snapshot, o), h = m.length === 0 ? p : p.pipe(l.map((e) => Yt(e, m))), g = t.mask ? h.pipe(l.map((e) => Re(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* Dt("export", o, a, i, (e) => e.rowCount, Zt(Ut(g, s), t.bundleFile(d), t.compression));
|
|
672
673
|
t.ledger.tables[o] = {
|
|
673
674
|
rowCount: _,
|
|
674
675
|
checksum: v
|
|
675
|
-
}, yield*
|
|
676
|
+
}, yield* q(t.ledgerPath, t.ledger), K(t.onProgress, {
|
|
676
677
|
phase: "export",
|
|
677
678
|
table: o,
|
|
678
679
|
index: a,
|
|
@@ -690,43 +691,43 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
690
691
|
});
|
|
691
692
|
}
|
|
692
693
|
return r;
|
|
693
|
-
}),
|
|
694
|
+
}), en = (e) => ({
|
|
694
695
|
_tag: "RawSqlFragment",
|
|
695
696
|
strings: [e],
|
|
696
697
|
values: []
|
|
697
|
-
}),
|
|
698
|
+
}), tn = (t, i, a) => n.tryPromise({
|
|
698
699
|
try: () => t.transactional(async (t) => {
|
|
699
|
-
if (t.raw) for (let e of
|
|
700
|
+
if (t.raw) for (let e of be(i)) await t.raw(en(e));
|
|
700
701
|
let o = await n.runPromiseExit(a(t));
|
|
701
702
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
702
703
|
return o.value;
|
|
703
704
|
}),
|
|
704
|
-
catch: (e) => e instanceof
|
|
705
|
-
}),
|
|
706
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
705
|
+
catch: (e) => e instanceof V ? e : new V({ reason: `snapshot export failed: ${J(e)}` })
|
|
706
|
+
}), nn = (e) => n.gen(function* () {
|
|
707
|
+
let t = e.scope ?? { kind: "all" }, r = ot(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, o = s.exponential("50 millis").pipe(s.jittered, s.intersect(s.recurs(a))), c = yield* n.promise(() => import("node:path")), u = (t) => c.join(e.outDir, t);
|
|
707
708
|
yield* n.tryPromise({
|
|
708
709
|
try: async () => {
|
|
709
710
|
let t = await import("node:fs/promises");
|
|
710
|
-
await t.mkdir(u(
|
|
711
|
+
await t.mkdir(u(vt), { recursive: !0 }), e.assets && await t.mkdir(u(yt), { recursive: !0 });
|
|
711
712
|
},
|
|
712
|
-
catch: (t) => new
|
|
713
|
-
reason: `mkdir failed: ${
|
|
713
|
+
catch: (t) => new V({
|
|
714
|
+
reason: `mkdir failed: ${J(t)}`,
|
|
714
715
|
path: e.outDir
|
|
715
716
|
})
|
|
716
717
|
});
|
|
717
|
-
let d = u(
|
|
718
|
+
let d = u(xt), f = yield* kt(d), p = e.subset ? Qe(yield* Ze(e.store, e.snapshot, e.subset), e.snapshot) : yield* et(t, e.snapshot, {
|
|
718
719
|
store: e.store,
|
|
719
720
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
720
|
-
}), m =
|
|
721
|
+
}), m = Gt(p.tables, e.snapshot), h = e.consistency ?? "live", g, _;
|
|
721
722
|
if (e.masking) {
|
|
722
|
-
let t = e.classification ??
|
|
723
|
+
let t = e.classification ?? Se(e.snapshot), r = Fe(m.map((t) => ({
|
|
723
724
|
name: t,
|
|
724
725
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
725
726
|
name: e.name,
|
|
726
727
|
type: e.type
|
|
727
728
|
}))
|
|
728
729
|
})), t, e.masking);
|
|
729
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
730
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new qe({
|
|
730
731
|
unclassified: r.unclassified,
|
|
731
732
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
732
733
|
}));
|
|
@@ -752,25 +753,25 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
752
753
|
bundleFile: u,
|
|
753
754
|
..._ ? { mask: _ } : {},
|
|
754
755
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
755
|
-
}, y = h === "snapshot" ? yield*
|
|
756
|
+
}, y = h === "snapshot" ? yield* tn(e.store, e.dialect, (e) => $t(e, v)) : yield* $t(e.store, v), b = 0, ee = 0;
|
|
756
757
|
if (e.assets) {
|
|
757
|
-
let t = u(
|
|
758
|
-
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
758
|
+
let t = u(bt);
|
|
759
|
+
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => rt(e, u(yt))), n.flatMap((e) => {
|
|
759
760
|
let i = {
|
|
760
761
|
key: r.key,
|
|
761
762
|
sha256: e.sha256,
|
|
762
763
|
size: e.size,
|
|
763
764
|
contentType: e.contentType
|
|
764
765
|
};
|
|
765
|
-
return b++,
|
|
766
|
-
}))).pipe(n.mapError((e) => e instanceof
|
|
766
|
+
return b++, ee += e.size, f.assets[r.key] = !0, Qt(t, JSON.stringify(i)).pipe(n.andThen(q(d, f)));
|
|
767
|
+
}))).pipe(n.mapError((e) => e instanceof V ? e : new V({ reason: `asset export failed: ${String(e)}` })));
|
|
767
768
|
}
|
|
768
|
-
let
|
|
769
|
+
let x = {
|
|
769
770
|
formatVersion: 1,
|
|
770
771
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
771
772
|
source: {
|
|
772
773
|
dialect: e.dialect,
|
|
773
|
-
schemaFingerprint:
|
|
774
|
+
schemaFingerprint: re(e.snapshot)
|
|
774
775
|
},
|
|
775
776
|
scope: t,
|
|
776
777
|
consistency: h,
|
|
@@ -778,8 +779,8 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
778
779
|
tables: y,
|
|
779
780
|
assets: b > 0 ? {
|
|
780
781
|
count: b,
|
|
781
|
-
totalBytes:
|
|
782
|
-
index:
|
|
782
|
+
totalBytes: ee,
|
|
783
|
+
index: bt
|
|
783
784
|
} : {
|
|
784
785
|
count: 0,
|
|
785
786
|
totalBytes: 0
|
|
@@ -788,23 +789,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
788
789
|
};
|
|
789
790
|
return yield* n.tryPromise({
|
|
790
791
|
try: async () => {
|
|
791
|
-
await (await import("node:fs/promises")).writeFile(u(
|
|
792
|
+
await (await import("node:fs/promises")).writeFile(u(G), ht(x));
|
|
792
793
|
},
|
|
793
|
-
catch: (e) => new
|
|
794
|
-
reason: `manifest write failed: ${
|
|
795
|
-
path:
|
|
794
|
+
catch: (e) => new V({
|
|
795
|
+
reason: `manifest write failed: ${J(e)}`,
|
|
796
|
+
path: G
|
|
796
797
|
})
|
|
797
798
|
}), yield* n.tryPromise({
|
|
798
799
|
try: async () => {
|
|
799
800
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
800
|
-
await e.writeFile(u(
|
|
801
|
+
await e.writeFile(u(_t), Ct(x, { hostname: t.hostname() }));
|
|
801
802
|
},
|
|
802
|
-
catch: (e) => new
|
|
803
|
-
reason: `readme write failed: ${
|
|
804
|
-
path:
|
|
803
|
+
catch: (e) => new V({
|
|
804
|
+
reason: `readme write failed: ${J(e)}`,
|
|
805
|
+
path: _t
|
|
805
806
|
})
|
|
806
|
-
}),
|
|
807
|
-
}),
|
|
807
|
+
}), x;
|
|
808
|
+
}), rn = (e, t) => {
|
|
808
809
|
let n = new Set(t), r = [];
|
|
809
810
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
810
811
|
let i = e.references?.table;
|
|
@@ -815,7 +816,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
815
816
|
});
|
|
816
817
|
}
|
|
817
818
|
return r;
|
|
818
|
-
},
|
|
819
|
+
}, an = (e) => {
|
|
819
820
|
let t = /* @__PURE__ */ new Map();
|
|
820
821
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
821
822
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -823,7 +824,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
823
824
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
824
825
|
});
|
|
825
826
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
826
|
-
},
|
|
827
|
+
}, on = (e) => {
|
|
827
828
|
switch (e) {
|
|
828
829
|
case "id":
|
|
829
830
|
case "text":
|
|
@@ -842,12 +843,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
842
843
|
case "raw": return "raw";
|
|
843
844
|
default: return "string";
|
|
844
845
|
}
|
|
845
|
-
},
|
|
846
|
+
}, sn = {
|
|
846
847
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
847
848
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
848
849
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
849
|
-
},
|
|
850
|
-
let i =
|
|
850
|
+
}, cn = (e, t, n, r) => {
|
|
851
|
+
let i = on(n);
|
|
851
852
|
if (i === "raw") return {
|
|
852
853
|
table: e,
|
|
853
854
|
column: t,
|
|
@@ -855,7 +856,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
855
856
|
severity: "block",
|
|
856
857
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
857
858
|
};
|
|
858
|
-
let a =
|
|
859
|
+
let a = sn[i];
|
|
859
860
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
860
861
|
table: e,
|
|
861
862
|
column: t,
|
|
@@ -869,15 +870,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
869
870
|
severity: "block",
|
|
870
871
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
871
872
|
};
|
|
872
|
-
},
|
|
873
|
+
}, ln = (e, t, n) => {
|
|
873
874
|
if (t === n) return [];
|
|
874
875
|
let r = [];
|
|
875
876
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
876
|
-
let a =
|
|
877
|
+
let a = cn(t.name, e, i, n);
|
|
877
878
|
a && r.push(a);
|
|
878
879
|
}
|
|
879
880
|
return r;
|
|
880
|
-
},
|
|
881
|
+
}, un = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, dn = (e) => {
|
|
881
882
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
882
883
|
switch (e.kind) {
|
|
883
884
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -886,36 +887,36 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
886
887
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
887
888
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
888
889
|
}
|
|
889
|
-
},
|
|
890
|
+
}, fn = (e) => {
|
|
890
891
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
891
892
|
return t.length > 0 ? t.join("/") : void 0;
|
|
892
|
-
},
|
|
893
|
+
}, pn = "Failed to execute statement", mn = (e) => {
|
|
893
894
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
894
895
|
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
895
|
-
return t === `SqlError: ${
|
|
896
|
-
},
|
|
896
|
+
return t === `SqlError: ${pn}` ? pn : t;
|
|
897
|
+
}, hn = (e) => {
|
|
897
898
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
898
899
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
899
900
|
unclassified: t,
|
|
900
901
|
raw: n
|
|
901
|
-
}), i =
|
|
902
|
-
if (a !== void 0) return r(`${
|
|
903
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u =
|
|
904
|
-
return r(
|
|
905
|
-
},
|
|
906
|
-
let t =
|
|
902
|
+
}), i = h(e), a = f(i), o = fn(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
903
|
+
if (a !== void 0) return r(`${dn(a)}${s}`, !1);
|
|
904
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = mn(typeof l == "string" && l.length > 0 ? l : String(e)), d = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, p = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, m = p !== void 0 && /[←‖]/.test(p) ? ` (via ${p})` : "";
|
|
905
|
+
return r(d === pn || d.length === 0 ? `the database refused the write and the driver gave no detail${s}${m}` : `${d}${s}`, d === pn || d.length === 0);
|
|
906
|
+
}, gn = (e) => hn(e).reason, _n = (e) => {
|
|
907
|
+
let t = h(e), n = f(t);
|
|
907
908
|
if (n?.kind === "foreignKeyInUse") {
|
|
908
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
909
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = fn(t);
|
|
909
910
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
910
911
|
}
|
|
911
|
-
return
|
|
912
|
-
},
|
|
912
|
+
return gn(e);
|
|
913
|
+
}, vn = "_voltro_staging_", Y = (e) => `${vn}${e}`, yn = (e) => typeof e.run == "function", bn = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
913
914
|
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
914
|
-
written before the first delete is what covers that case.`,
|
|
915
|
-
if (!
|
|
915
|
+
written before the first delete is what covers that case.`, xn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, X = (e, t) => {
|
|
916
|
+
if (!xn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
916
917
|
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
917
|
-
},
|
|
918
|
-
let n =
|
|
918
|
+
}, Sn = (e, t) => {
|
|
919
|
+
let n = X(e, t), r = X(Y(e), t);
|
|
919
920
|
switch (t) {
|
|
920
921
|
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
921
922
|
case "mysql":
|
|
@@ -923,20 +924,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
923
924
|
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
924
925
|
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
925
926
|
}
|
|
926
|
-
},
|
|
927
|
-
let n = Y(
|
|
928
|
-
return t === "mssql" ? `IF OBJECT_ID('${
|
|
929
|
-
},
|
|
927
|
+
}, Cn = (e, t) => {
|
|
928
|
+
let n = X(Y(e), t);
|
|
929
|
+
return t === "mssql" ? `IF OBJECT_ID('${Y(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
930
|
+
}, wn = (e, t) => {
|
|
930
931
|
let n = t === "mysql" || t === "mariadb", r = [];
|
|
931
932
|
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
932
|
-
for (let n of [...e].reverse()) r.push(`DELETE FROM ${
|
|
933
|
+
for (let n of [...e].reverse()) r.push(`DELETE FROM ${X(n.name, t)}`);
|
|
933
934
|
for (let n of e) {
|
|
934
935
|
if (n.columns.length === 0) throw Error(`refusing to build a swap for '${n.name}' with no columns: an empty list would have to mean \`SELECT *\`, which is the shape that fails on a generated column.`);
|
|
935
|
-
let e = n.columns.map((e) =>
|
|
936
|
-
r.push(`INSERT INTO ${
|
|
936
|
+
let e = n.columns.map((e) => X(e, t)).join(", ");
|
|
937
|
+
r.push(`INSERT INTO ${X(n.name, t)} (${e}) SELECT ${e} FROM ${X(Y(n.name), t)}`);
|
|
937
938
|
}
|
|
938
939
|
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
939
|
-
},
|
|
940
|
+
}, Tn = (e, t) => {
|
|
940
941
|
let n = [], r = () => {
|
|
941
942
|
for (let e of n) t.unregisterTable(e);
|
|
942
943
|
};
|
|
@@ -944,7 +945,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
944
945
|
for (let r of e) {
|
|
945
946
|
let e = t.getTable(r);
|
|
946
947
|
if (e === void 0) continue;
|
|
947
|
-
let i =
|
|
948
|
+
let i = Y(r);
|
|
948
949
|
t.registerTable({
|
|
949
950
|
...e,
|
|
950
951
|
tableName: i,
|
|
@@ -958,7 +959,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
958
959
|
staged: n,
|
|
959
960
|
undo: r
|
|
960
961
|
};
|
|
961
|
-
},
|
|
962
|
+
}, En = (e, t) => {
|
|
962
963
|
let n = new Set(t), r = [];
|
|
963
964
|
for (let t of e.tables) {
|
|
964
965
|
if (!n.has(t.name)) continue;
|
|
@@ -975,22 +976,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
975
976
|
}
|
|
976
977
|
}
|
|
977
978
|
return r;
|
|
978
|
-
},
|
|
979
|
-
let n = Y(
|
|
980
|
-
return `SELECT ${
|
|
981
|
-
},
|
|
979
|
+
}, Dn = (e, t) => {
|
|
980
|
+
let n = X(Y(e.from), t), r = X(Y(e.to), t), i = X(e.column, t), a = X(e.toColumn, t);
|
|
981
|
+
return `SELECT ${X(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
982
|
+
}, On = (e) => {
|
|
982
983
|
if (e.length === 0) return;
|
|
983
984
|
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
984
985
|
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
985
|
-
},
|
|
986
|
-
t.length !== 0 && await e.run(
|
|
987
|
-
},
|
|
988
|
-
let i =
|
|
989
|
-
i.length !== 0 && await e.run(
|
|
990
|
-
},
|
|
986
|
+
}, kn = async (e, t) => {
|
|
987
|
+
t.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
988
|
+
}, An = async (e, t, r) => {
|
|
989
|
+
let i = wn(t, r);
|
|
990
|
+
i.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
991
|
+
}, jn = async (e, t, r) => {
|
|
991
992
|
let i = [];
|
|
992
993
|
for (let a of t) {
|
|
993
|
-
let t = await e.run(
|
|
994
|
+
let t = await e.run(P.SqlClient.pipe(n.flatMap((e) => e.unsafe(Dn(a, r)))));
|
|
994
995
|
for (let e of t) i.push({
|
|
995
996
|
table: a.from,
|
|
996
997
|
primaryKey: String(e.pk ?? ""),
|
|
@@ -1000,16 +1001,16 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1000
1001
|
});
|
|
1001
1002
|
}
|
|
1002
1003
|
return i;
|
|
1003
|
-
},
|
|
1004
|
+
}, Mn = (e, t) => {
|
|
1004
1005
|
let n = new Map(t.map((e, t) => [e, t]));
|
|
1005
1006
|
return e.filter((e) => {
|
|
1006
1007
|
if (e.from === e.to) return !1;
|
|
1007
1008
|
let t = n.get(e.from), r = n.get(e.to);
|
|
1008
1009
|
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1009
1010
|
});
|
|
1010
|
-
},
|
|
1011
|
+
}, Nn = (e) => e === "mysql" || e === "mariadb", Pn = (e) => n.gen(function* () {
|
|
1011
1012
|
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1012
|
-
if (e.snapshot === void 0) return yield* n.fail(new
|
|
1013
|
+
if (e.snapshot === void 0) return yield* n.fail(new V({
|
|
1013
1014
|
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
1014
1015
|
path: e.dir
|
|
1015
1016
|
}));
|
|
@@ -1020,14 +1021,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1020
1021
|
if (t.tables.length === 0) return;
|
|
1021
1022
|
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1022
1023
|
so it survives a run whose process does not.`);
|
|
1023
|
-
let r = yield*
|
|
1024
|
+
let r = yield* nn({
|
|
1024
1025
|
store: e.store,
|
|
1025
1026
|
snapshot: t,
|
|
1026
1027
|
dialect: e.dialect ?? "unknown",
|
|
1027
1028
|
outDir: e.dir,
|
|
1028
1029
|
scope: { kind: "all" },
|
|
1029
1030
|
compression: "none"
|
|
1030
|
-
}).pipe(n.mapError((t) => new
|
|
1031
|
+
}).pipe(n.mapError((t) => new V({
|
|
1031
1032
|
reason: `rollback capture failed, so the import did not start and the target is untouched: ${t.reason ?? String(t)}\n Fix the capture, or re-run with \`--no-rollback\` to proceed without one.`,
|
|
1032
1033
|
path: e.dir
|
|
1033
1034
|
})));
|
|
@@ -1036,14 +1037,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1036
1037
|
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1037
1038
|
tables: r.tables.length
|
|
1038
1039
|
};
|
|
1039
|
-
}),
|
|
1040
|
+
}), Fn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, In = (e, t) => e.tables.find((e) => e.name === t), Ln = (e, t) => new Map((In(e, t)?.columns ?? []).map((e) => [e.name, e])), Rn = (e, t) => {
|
|
1040
1041
|
let n = [];
|
|
1041
1042
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
1042
|
-
if (
|
|
1043
|
+
if (In(t, r.name) === void 0) {
|
|
1043
1044
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
1044
1045
|
continue;
|
|
1045
1046
|
}
|
|
1046
|
-
let e =
|
|
1047
|
+
let e = Ln(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
1047
1048
|
for (let [t, a] of i) {
|
|
1048
1049
|
let i = e.get(t);
|
|
1049
1050
|
if (i === void 0) {
|
|
@@ -1055,22 +1056,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1055
1056
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
1056
1057
|
}
|
|
1057
1058
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
1058
|
-
},
|
|
1059
|
-
let r =
|
|
1059
|
+
}, zn = (e, t, n) => {
|
|
1060
|
+
let r = re(t);
|
|
1060
1061
|
if (r === e || n === void 0) return {
|
|
1061
1062
|
drifted: !1,
|
|
1062
1063
|
bundleFingerprint: e,
|
|
1063
1064
|
targetFingerprint: r,
|
|
1064
1065
|
diff: []
|
|
1065
1066
|
};
|
|
1066
|
-
let i =
|
|
1067
|
+
let i = Rn(n, t);
|
|
1067
1068
|
return {
|
|
1068
1069
|
drifted: i.length > 0,
|
|
1069
1070
|
bundleFingerprint: e,
|
|
1070
1071
|
targetFingerprint: r,
|
|
1071
1072
|
diff: i
|
|
1072
1073
|
};
|
|
1073
|
-
},
|
|
1074
|
+
}, Bn = 200, Vn = /* @__PURE__ */ new Set([
|
|
1074
1075
|
"BundleError",
|
|
1075
1076
|
"CodecError",
|
|
1076
1077
|
"IntegrityError",
|
|
@@ -1079,447 +1080,474 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1079
1080
|
"RowsRefusedError",
|
|
1080
1081
|
"SchemaDriftError",
|
|
1081
1082
|
"StorageError"
|
|
1082
|
-
]),
|
|
1083
|
+
]), Hn = (t) => {
|
|
1083
1084
|
let n = o.isFiberFailure(t) ? [...e.failures(t[o.FiberFailureCauseId]), ...e.defects(t[o.FiberFailureCauseId])] : [t];
|
|
1084
1085
|
for (let e of n) {
|
|
1085
1086
|
let t = e?._tag;
|
|
1086
|
-
if (typeof t == "string" &&
|
|
1087
|
+
if (typeof t == "string" && Vn.has(t)) return e;
|
|
1087
1088
|
if (o.isFiberFailure(e)) {
|
|
1088
|
-
let t =
|
|
1089
|
+
let t = Hn(e);
|
|
1089
1090
|
if (t) return t;
|
|
1090
1091
|
}
|
|
1091
1092
|
}
|
|
1092
|
-
},
|
|
1093
|
+
}, Un = (t) => [...e.failures(t)][0] ?? Error(e.pretty(t)), Wn = (e) => n.tryPromise({
|
|
1093
1094
|
try: async () => {
|
|
1094
1095
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
1095
|
-
return
|
|
1096
|
+
return gt(await t.readFile(n.join(e, G), "utf8"));
|
|
1096
1097
|
},
|
|
1097
|
-
catch: (e) => new
|
|
1098
|
+
catch: (e) => new V({
|
|
1098
1099
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
1099
|
-
path:
|
|
1100
|
+
path: G
|
|
1100
1101
|
})
|
|
1101
|
-
}),
|
|
1102
|
-
let t
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
if (
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
tables:
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1102
|
+
}), Gn = (e) => {
|
|
1103
|
+
let t;
|
|
1104
|
+
return n.gen(function* () {
|
|
1105
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* Wn(e.bundleDir);
|
|
1106
|
+
if (a.formatVersion !== 1) return yield* n.fail(new V({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
1107
|
+
if (e.tables !== void 0) {
|
|
1108
|
+
let t = new Set(a.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
1109
|
+
if (r.length > 0) return yield* n.fail(new V({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...t].sort().join(", ")}.` }));
|
|
1110
|
+
}
|
|
1111
|
+
let o = e.tables === void 0 ? void 0 : new Set(e.tables), s = o === void 0 ? a : {
|
|
1112
|
+
...a,
|
|
1113
|
+
tables: a.tables.filter((e) => o.has(e.name))
|
|
1114
|
+
};
|
|
1115
|
+
if (e.targetSnapshot) {
|
|
1116
|
+
let t = { tables: s.tables.map((e) => ({
|
|
1117
|
+
name: e.name,
|
|
1118
|
+
indexes: [],
|
|
1119
|
+
columns: Object.entries(e.columnTypes).map(([e, t]) => ({
|
|
1120
|
+
name: e,
|
|
1121
|
+
type: t,
|
|
1122
|
+
nullable: !0,
|
|
1123
|
+
unique: !1,
|
|
1124
|
+
hasDefault: !1
|
|
1125
|
+
}))
|
|
1126
|
+
})) }, r = zn(s.source.schemaFingerprint, e.targetSnapshot, t);
|
|
1127
|
+
if (r.drifted) {
|
|
1128
|
+
if (!e.force) return yield* n.fail(new Ke({
|
|
1129
|
+
bundleFingerprint: r.bundleFingerprint,
|
|
1130
|
+
targetFingerprint: r.targetFingerprint,
|
|
1131
|
+
diff: r.diff
|
|
1132
|
+
}));
|
|
1133
|
+
e.onWarn?.(`schema drift: the target schema (fingerprint ${r.targetFingerprint.slice(0, 16)}) differs from the bundle's source (${r.bundleFingerprint.slice(0, 16)}); importing anyway (--force). ${r.diff.join("; ")}`);
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
if (e.targetDialect && e.targetDialect !== s.source.dialect) {
|
|
1137
|
+
let t = ln(s.tables, s.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1138
|
+
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new Ye({
|
|
1139
|
+
sourceDialect: s.source.dialect,
|
|
1140
|
+
targetDialect: e.targetDialect,
|
|
1141
|
+
blocking: t.map(un)
|
|
1129
1142
|
}));
|
|
1130
|
-
e.onWarn?.(`schema drift: the target schema (fingerprint ${r.targetFingerprint.slice(0, 16)}) differs from the bundle's source (${r.bundleFingerprint.slice(0, 16)}); importing anyway (--force). ${r.diff.join("; ")}`);
|
|
1131
1143
|
}
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
sourceDialect: o.source.dialect,
|
|
1137
|
-
targetDialect: e.targetDialect,
|
|
1138
|
-
blocking: t.map(cn)
|
|
1144
|
+
let c = e.verify ?? !0, f = e.mode ?? "upsert", m = e.atomic ?? f === "replace", h = e.onConflict ?? "skip", _ = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), y = yield* kt(_);
|
|
1145
|
+
if (f === "replace" && s.scope.kind !== "all") return yield* n.fail(new Je({
|
|
1146
|
+
mode: f,
|
|
1147
|
+
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
1139
1148
|
}));
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1149
|
+
if (f === "replace") {
|
|
1150
|
+
let t = s.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1151
|
+
try: () => e.store.incomingForeignKeys(t),
|
|
1152
|
+
catch: (e) => new V({ reason: `foreign-key pre-flight failed: ${_n(e)}` })
|
|
1153
|
+
}) : e.targetSnapshot === void 0 ? [] : rn(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1154
|
+
for (let t of r) {
|
|
1155
|
+
if (!a.has(t.from)) {
|
|
1156
|
+
let r = yield* n.tryPromise({
|
|
1157
|
+
try: () => e.store.query({
|
|
1158
|
+
table: t.from,
|
|
1159
|
+
predicate: void 0,
|
|
1160
|
+
order: [],
|
|
1161
|
+
take: void 0,
|
|
1162
|
+
skip: void 0,
|
|
1163
|
+
projection: []
|
|
1164
|
+
}),
|
|
1165
|
+
catch: () => new V({ reason: `unreadable: ${t.from}` })
|
|
1166
|
+
}).pipe(n.map((e) => e.length), n.orElseSucceed(() => 0));
|
|
1167
|
+
a.set(t.from, r);
|
|
1168
|
+
}
|
|
1169
|
+
let r = a.get(t.from) ?? 0;
|
|
1170
|
+
r > 0 && i.push({
|
|
1171
|
+
...t,
|
|
1172
|
+
rowCount: r
|
|
1173
|
+
});
|
|
1165
1174
|
}
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
});
|
|
1175
|
+
if (i.length > 0) return yield* n.fail(new Je({
|
|
1176
|
+
mode: f,
|
|
1177
|
+
reason: an(i)
|
|
1178
|
+
}));
|
|
1171
1179
|
}
|
|
1172
|
-
if (
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
tablesWritten: 0,
|
|
1180
|
-
rowsWritten: 0,
|
|
1181
|
-
tablesSkipped: 0,
|
|
1182
|
-
rowsSkipped: 0,
|
|
1183
|
-
fullyResumed: !1
|
|
1184
|
-
};
|
|
1185
|
-
let v = /* @__PURE__ */ new Map();
|
|
1186
|
-
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1187
|
-
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1188
|
-
e.length > 0 && v.set(t.name, e);
|
|
1189
|
-
}
|
|
1190
|
-
let x = (e, t) => {
|
|
1191
|
-
let n = v.get(e);
|
|
1192
|
-
if (n === void 0) return t;
|
|
1193
|
-
let r;
|
|
1194
|
-
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1195
|
-
return r ?? t;
|
|
1196
|
-
}, C = (e, t, n, r = !1) => {
|
|
1197
|
-
let i = j ? J(t.name) : t.name, a = t.primaryKey, o = x(t.name, n);
|
|
1198
|
-
return c === "append" ? p === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : c === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1199
|
-
}, w = (e, t) => e.savepoint ? e.savepoint(t) : t(), T = 0, E = (t, n, r) => {
|
|
1200
|
-
r.unclassified && (T++, !(T > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (T === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1201
|
-
}, ee = (e, t, r = !0) => n.tryPromise({
|
|
1202
|
-
try: e,
|
|
1203
|
-
catch: (e) => pn(e)
|
|
1204
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, E(t.entry, t.row, e)), !1)))), te = (e) => {
|
|
1205
|
-
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1206
|
-
if (t.length !== 0) return {
|
|
1207
|
-
...e.row,
|
|
1208
|
-
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1180
|
+
if (e.dryRun === !0) return {
|
|
1181
|
+
manifest: s,
|
|
1182
|
+
tablesWritten: 0,
|
|
1183
|
+
rowsWritten: 0,
|
|
1184
|
+
tablesSkipped: 0,
|
|
1185
|
+
rowsSkipped: 0,
|
|
1186
|
+
fullyResumed: !1
|
|
1209
1187
|
};
|
|
1210
|
-
|
|
1211
|
-
let t
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
return i != null && t.has(String(i));
|
|
1215
|
-
}), r = e.map((e) => ({
|
|
1216
|
-
table: e.entry.name,
|
|
1217
|
-
id: String(e.row[e.entry.primaryKey]),
|
|
1218
|
-
reason: e.reason,
|
|
1219
|
-
derived: n(e)
|
|
1220
|
-
})), i = [...r.filter((e) => !e.derived), ...r.filter((e) => e.derived)], a = /* @__PURE__ */ new Map();
|
|
1221
|
-
for (let e of r) {
|
|
1222
|
-
let t = a.get(e.table) ?? {
|
|
1223
|
-
refused: 0,
|
|
1224
|
-
primary: 0
|
|
1225
|
-
};
|
|
1226
|
-
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1188
|
+
let x = /* @__PURE__ */ new Map();
|
|
1189
|
+
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1190
|
+
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1191
|
+
e.length > 0 && x.set(t.name, e);
|
|
1227
1192
|
}
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
})
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
if (
|
|
1245
|
-
|
|
1246
|
-
|
|
1193
|
+
let re = (e, t) => {
|
|
1194
|
+
let n = x.get(e);
|
|
1195
|
+
if (n === void 0) return t;
|
|
1196
|
+
let r;
|
|
1197
|
+
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1198
|
+
return r ?? t;
|
|
1199
|
+
}, C = (e, t, n, r = !1) => {
|
|
1200
|
+
let i = j ? Y(t.name) : t.name, a = t.primaryKey, o = re(t.name, n);
|
|
1201
|
+
return f === "append" ? h === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : f === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1202
|
+
}, w = (e, t) => e.savepoint ? e.savepoint(t) : t(), T = 0, ie = (t, n, r) => {
|
|
1203
|
+
r.unclassified && (T++, !(T > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (T === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1204
|
+
}, E = (e, t, r = !0) => n.tryPromise({
|
|
1205
|
+
try: e,
|
|
1206
|
+
catch: (e) => hn(e)
|
|
1207
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, ie(t.entry, t.row, e)), !1)))), ae = (e) => {
|
|
1208
|
+
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1209
|
+
if (t.length !== 0) return {
|
|
1210
|
+
...e.row,
|
|
1211
|
+
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1212
|
+
};
|
|
1213
|
+
}, oe = (e) => {
|
|
1214
|
+
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
1215
|
+
if (r !== "reference") return !1;
|
|
1216
|
+
let i = e.row[n];
|
|
1217
|
+
return i != null && t.has(String(i));
|
|
1218
|
+
}), r = e.map((e) => ({
|
|
1219
|
+
table: e.entry.name,
|
|
1220
|
+
id: String(e.row[e.entry.primaryKey]),
|
|
1221
|
+
reason: e.reason,
|
|
1222
|
+
derived: n(e)
|
|
1223
|
+
})), i = [...r.filter((e) => !e.derived), ...r.filter((e) => e.derived)], a = /* @__PURE__ */ new Map();
|
|
1224
|
+
for (let e of r) {
|
|
1225
|
+
let t = a.get(e.table) ?? {
|
|
1226
|
+
refused: 0,
|
|
1227
|
+
primary: 0
|
|
1228
|
+
};
|
|
1229
|
+
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1247
1230
|
}
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
if (t === void 0) return [];
|
|
1258
|
-
let n = new Map(t.tables.map((e) => [e.name, e])), r = [];
|
|
1259
|
-
for (let e of o.tables) {
|
|
1260
|
-
if (z(e.name)) continue;
|
|
1261
|
-
let t = n.get(e.name);
|
|
1262
|
-
if (t === void 0) return [];
|
|
1263
|
-
let i = t.columns.filter((e) => e.generatedAs === void 0).map((e) => e.name);
|
|
1264
|
-
if (i.length === 0) return [];
|
|
1265
|
-
r.push({
|
|
1266
|
-
name: e.name,
|
|
1267
|
-
columns: i
|
|
1231
|
+
return new Ge({
|
|
1232
|
+
rows: i.slice(0, 20),
|
|
1233
|
+
primaryCount: r.filter((e) => !e.derived).length,
|
|
1234
|
+
totalCount: e.length,
|
|
1235
|
+
byTable: [...a.entries()].map(([e, t]) => ({
|
|
1236
|
+
table: e,
|
|
1237
|
+
refused: t.refused,
|
|
1238
|
+
primary: t.primary
|
|
1239
|
+
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1268
1240
|
});
|
|
1269
|
-
}
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1241
|
+
}, ce = (e, t) => n.gen(function* () {
|
|
1242
|
+
let r = t, i = [], a = !0;
|
|
1243
|
+
for (; r.length > 0 && a;) {
|
|
1244
|
+
a = !1;
|
|
1245
|
+
let t = [];
|
|
1246
|
+
for (let n of r) (yield* E(() => w(e, () => C(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1247
|
+
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1248
|
+
let r = t[n], o = ae(r);
|
|
1249
|
+
o !== void 0 && (yield* E(() => w(e, () => C(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1250
|
+
}
|
|
1251
|
+
r = t;
|
|
1252
|
+
}
|
|
1253
|
+
for (let t of i) (yield* E(() => w(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1254
|
+
if (r.length > 0) return yield* n.fail(oe(r));
|
|
1255
|
+
}), D = (() => {
|
|
1256
|
+
let t = e.targetDialect;
|
|
1257
|
+
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1258
|
+
})(), O = (() => {
|
|
1259
|
+
let t = e.targetSnapshot;
|
|
1260
|
+
if (t === void 0) return [];
|
|
1261
|
+
let n = new Map(t.tables.map((e) => [e.name, e])), r = [];
|
|
1262
|
+
for (let e of s.tables) {
|
|
1263
|
+
if (B(e.name)) continue;
|
|
1264
|
+
let t = n.get(e.name);
|
|
1265
|
+
if (t === void 0) return [];
|
|
1266
|
+
let i = t.columns.filter((e) => e.generatedAs === void 0).map((e) => e.name);
|
|
1267
|
+
if (i.length === 0) return [];
|
|
1268
|
+
r.push({
|
|
1269
|
+
name: e.name,
|
|
1270
|
+
columns: i
|
|
1271
|
+
});
|
|
1272
|
+
}
|
|
1273
|
+
return r;
|
|
1274
|
+
})(), k = s.tables.filter((e) => !B(e.name) && b(e.name)).map((e) => e.name), A = D === void 0 || Nn(D) ? [] : Mn(e.targetSnapshot === void 0 ? [] : En(e.targetSnapshot, O.map((e) => e.name)), O.map((e) => e.name)), j = f === "replace" && D !== void 0 && O.length > 0 && A.length === 0 && yn(e.store), le = !1, M = 0, N = 0, de = 0, P = 0, F = (t, r) => n.gen(function* () {
|
|
1275
|
+
if (f === "replace" && !j && !(r && y.truncated)) {
|
|
1276
|
+
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1277
|
+
try: () => t.insert(d, {
|
|
1278
|
+
id: p("repl", fe),
|
|
1279
|
+
startedAt: pe,
|
|
1280
|
+
tables: String(s.tables.length),
|
|
1281
|
+
capture: I?.dir ?? null,
|
|
1282
|
+
via: e.via ?? "direct"
|
|
1283
|
+
}),
|
|
1284
|
+
catch: (e) => new V({ reason: `could not record that this replace started: ${String(e?.message ?? e)}\n The marker is what makes an interrupted run visible instead of silent, so the run does
|
|
1282
1285
|
not proceed without it.` })
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
}
|
|
1294
|
-
let a = [], p = [], h = [], v = [], y = o.tables.length;
|
|
1295
|
-
for (let u = 0; u < o.tables.length; u++) {
|
|
1296
|
-
let f = o.tables[u];
|
|
1297
|
-
if (z(f.name)) {
|
|
1298
|
-
v.push(f.name);
|
|
1299
|
-
continue;
|
|
1286
|
+
}));
|
|
1287
|
+
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !B(e)), a = !r, o = t.emptyTables ? () => t.emptyTables(i) : a ? async () => {
|
|
1288
|
+
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where: v(e.primaryKey) });
|
|
1289
|
+
} : () => t.transactional(async (e) => {
|
|
1290
|
+
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: v(t.primaryKey) });
|
|
1291
|
+
});
|
|
1292
|
+
yield* n.tryPromise({
|
|
1293
|
+
try: o,
|
|
1294
|
+
catch: (e) => new V({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${_n(e)}` })
|
|
1295
|
+
}), r && (y.truncated = !0, yield* q(_, y));
|
|
1300
1296
|
}
|
|
1301
|
-
|
|
1302
|
-
|
|
1297
|
+
let a = [], o = [], u = [], h = [], g = s.tables.length;
|
|
1298
|
+
for (let d = 0; d < s.tables.length; d++) {
|
|
1299
|
+
let p = s.tables[d];
|
|
1300
|
+
if (B(p.name)) {
|
|
1301
|
+
h.push(p.name);
|
|
1302
|
+
continue;
|
|
1303
|
+
}
|
|
1304
|
+
if (r && p.name in y.tables) {
|
|
1305
|
+
u.push(p.name), P++, de += y.tables[p.name].rowCount, K(e.onProgress, {
|
|
1306
|
+
phase: "import",
|
|
1307
|
+
table: p.name,
|
|
1308
|
+
index: d,
|
|
1309
|
+
tableCount: g,
|
|
1310
|
+
event: "done",
|
|
1311
|
+
rowsDone: y.tables[p.name].rowCount,
|
|
1312
|
+
total: p.rowCount
|
|
1313
|
+
});
|
|
1314
|
+
continue;
|
|
1315
|
+
}
|
|
1316
|
+
let v = i.join(e.bundleDir, p.file);
|
|
1317
|
+
K(e.onProgress, {
|
|
1303
1318
|
phase: "import",
|
|
1304
|
-
table:
|
|
1305
|
-
index:
|
|
1306
|
-
tableCount:
|
|
1307
|
-
event: "
|
|
1308
|
-
rowsDone:
|
|
1309
|
-
total:
|
|
1319
|
+
table: p.name,
|
|
1320
|
+
index: d,
|
|
1321
|
+
tableCount: g,
|
|
1322
|
+
event: "start",
|
|
1323
|
+
rowsDone: 0,
|
|
1324
|
+
total: p.rowCount
|
|
1310
1325
|
});
|
|
1311
|
-
|
|
1326
|
+
let b = a.length, ee = n.gen(function* () {
|
|
1327
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c = se("bundle", () => ue(v).pipe(lt(s.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), u = async (e) => {
|
|
1328
|
+
let n = {
|
|
1329
|
+
entry: p,
|
|
1330
|
+
row: e,
|
|
1331
|
+
reason: ""
|
|
1332
|
+
};
|
|
1333
|
+
try {
|
|
1334
|
+
await w(t, () => C(t, p, e));
|
|
1335
|
+
} catch (t) {
|
|
1336
|
+
let r = hn(t);
|
|
1337
|
+
n.reason = r.reason, ie(p, e, r), a.push(n);
|
|
1338
|
+
}
|
|
1339
|
+
}, d = yield* n.promise(async () => {
|
|
1340
|
+
let n = e.copyLoader;
|
|
1341
|
+
if (!n || n.dialect !== "postgres") return !1;
|
|
1342
|
+
if (m) return le || (le = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1343
|
+
if (f === "replace") return !0;
|
|
1344
|
+
if (f !== "upsert") return !1;
|
|
1345
|
+
try {
|
|
1346
|
+
return (await t.query({
|
|
1347
|
+
table: p.name,
|
|
1348
|
+
predicate: void 0,
|
|
1349
|
+
order: [],
|
|
1350
|
+
take: 1,
|
|
1351
|
+
skip: void 0,
|
|
1352
|
+
projection: void 0
|
|
1353
|
+
})).length === 0;
|
|
1354
|
+
} catch {
|
|
1355
|
+
return !1;
|
|
1356
|
+
}
|
|
1357
|
+
}), h = Object.keys(p.columnTypes), g = Math.max(1, e.copyBatchSize ?? 5e3), _ = [], y = async () => {
|
|
1358
|
+
if (_.length === 0) return;
|
|
1359
|
+
let t = _;
|
|
1360
|
+
_ = [];
|
|
1361
|
+
try {
|
|
1362
|
+
await e.copyLoader.copyInto({
|
|
1363
|
+
table: p.name,
|
|
1364
|
+
columns: h,
|
|
1365
|
+
columnTypes: p.columnTypes,
|
|
1366
|
+
rows: t
|
|
1367
|
+
});
|
|
1368
|
+
} catch (n) {
|
|
1369
|
+
d = !1, e.onWarn?.(`bulk COPY into ${p.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1370
|
+
for (let e of t) await u(e);
|
|
1371
|
+
}
|
|
1372
|
+
}, b = Math.max(1, e.savepointBatchSize ?? 200), ee = typeof t.savepoint == "function", x = [], S = async () => {
|
|
1373
|
+
if (x.length === 0) return;
|
|
1374
|
+
let e = x;
|
|
1375
|
+
x = [];
|
|
1376
|
+
try {
|
|
1377
|
+
await t.savepoint(async () => {
|
|
1378
|
+
for (let n of e) await C(t, p, n);
|
|
1379
|
+
});
|
|
1380
|
+
} catch {
|
|
1381
|
+
for (let t of e) await u(t);
|
|
1382
|
+
}
|
|
1383
|
+
};
|
|
1384
|
+
return yield* l.runForEach(Wt(c, p.name, p.columnTypes), (e) => n.promise(async () => {
|
|
1385
|
+
if (o++, d) {
|
|
1386
|
+
_.push(e), _.length >= g && await y();
|
|
1387
|
+
return;
|
|
1388
|
+
}
|
|
1389
|
+
if (ee) {
|
|
1390
|
+
x.push(e), x.length >= b && await S();
|
|
1391
|
+
return;
|
|
1392
|
+
}
|
|
1393
|
+
await u(e);
|
|
1394
|
+
})), yield* n.promise(S), yield* n.promise(y), {
|
|
1395
|
+
rowCount: o,
|
|
1396
|
+
checksum: i.digest("hex")
|
|
1397
|
+
};
|
|
1398
|
+
}), { rowCount: x, checksum: S } = yield* Dt("import", p.name, d, g, (e) => e.rowCount, ee);
|
|
1399
|
+
if (c) {
|
|
1400
|
+
if (S !== p.checksum) return yield* n.fail(new Ue({
|
|
1401
|
+
what: `table ${p.name} checksum`,
|
|
1402
|
+
expected: p.checksum,
|
|
1403
|
+
actual: S
|
|
1404
|
+
}));
|
|
1405
|
+
if (x !== p.rowCount) return yield* n.fail(new Ue({
|
|
1406
|
+
what: `table ${p.name} rowCount`,
|
|
1407
|
+
expected: String(p.rowCount),
|
|
1408
|
+
actual: String(x)
|
|
1409
|
+
}));
|
|
1410
|
+
}
|
|
1411
|
+
K(e.onProgress, {
|
|
1412
|
+
phase: "import",
|
|
1413
|
+
table: p.name,
|
|
1414
|
+
index: d,
|
|
1415
|
+
tableCount: g,
|
|
1416
|
+
event: "done",
|
|
1417
|
+
rowsDone: x,
|
|
1418
|
+
total: p.rowCount
|
|
1419
|
+
}), N++, M += x, r && (a.length > b ? o.push({
|
|
1420
|
+
name: p.name,
|
|
1421
|
+
rowCount: x
|
|
1422
|
+
}) : (y.tables[p.name] = {
|
|
1423
|
+
rowCount: x,
|
|
1424
|
+
checksum: p.checksum
|
|
1425
|
+
}, yield* q(_, y)));
|
|
1312
1426
|
}
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
let b = a.length, x = n.gen(function* () {
|
|
1324
|
-
let { createHash: t } = yield* n.promise(() => import("node:crypto")), i = t("sha256"), s = 0, u = ie("bundle", () => se(_).pipe(st(o.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), p = async (e) => {
|
|
1325
|
-
let t = {
|
|
1326
|
-
entry: f,
|
|
1327
|
-
row: e,
|
|
1328
|
-
reason: ""
|
|
1427
|
+
if (a.length > 0 && (yield* ce(t, a)), f === "replace" && !(r && y.truncated) && (yield* n.promise(() => t.deleteMany(d, { where: {
|
|
1428
|
+
column: "startedAt",
|
|
1429
|
+
op: "lte",
|
|
1430
|
+
value: pe
|
|
1431
|
+
} }).then(() => void 0, () => void 0))), r && o.length > 0) {
|
|
1432
|
+
for (let e of o) {
|
|
1433
|
+
let t = s.tables.find((t) => t.name === e.name);
|
|
1434
|
+
y.tables[e.name] = {
|
|
1435
|
+
rowCount: e.rowCount,
|
|
1436
|
+
checksum: t.checksum
|
|
1329
1437
|
};
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
} catch (n) {
|
|
1333
|
-
let r = pn(n);
|
|
1334
|
-
t.reason = r.reason, E(f, e, r), a.push(t);
|
|
1335
|
-
}
|
|
1336
|
-
}, m = yield* n.promise(async () => {
|
|
1337
|
-
let t = e.copyLoader;
|
|
1338
|
-
if (!t || t.dialect !== "postgres") return !1;
|
|
1339
|
-
if (d) return ae || (ae = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1340
|
-
if (c === "replace") return !0;
|
|
1341
|
-
if (c !== "upsert") return !1;
|
|
1342
|
-
try {
|
|
1343
|
-
return (await r.query({
|
|
1344
|
-
table: f.name,
|
|
1345
|
-
predicate: void 0,
|
|
1346
|
-
order: [],
|
|
1347
|
-
take: 1,
|
|
1348
|
-
skip: void 0,
|
|
1349
|
-
projection: void 0
|
|
1350
|
-
})).length === 0;
|
|
1351
|
-
} catch {
|
|
1352
|
-
return !1;
|
|
1353
|
-
}
|
|
1354
|
-
}), h = Object.keys(f.columnTypes), g = Math.max(1, e.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
1355
|
-
if (v.length === 0) return;
|
|
1356
|
-
let t = v;
|
|
1357
|
-
v = [];
|
|
1358
|
-
try {
|
|
1359
|
-
await e.copyLoader.copyInto({
|
|
1360
|
-
table: f.name,
|
|
1361
|
-
columns: h,
|
|
1362
|
-
columnTypes: f.columnTypes,
|
|
1363
|
-
rows: t
|
|
1364
|
-
});
|
|
1365
|
-
} catch (n) {
|
|
1366
|
-
m = !1, e.onWarn?.(`bulk COPY into ${f.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1367
|
-
for (let e of t) await p(e);
|
|
1368
|
-
}
|
|
1369
|
-
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof r.savepoint == "function", S = [], T = async () => {
|
|
1370
|
-
if (S.length === 0) return;
|
|
1371
|
-
let e = S;
|
|
1372
|
-
S = [];
|
|
1373
|
-
try {
|
|
1374
|
-
await r.savepoint(async () => {
|
|
1375
|
-
for (let t of e) await C(r, f, t);
|
|
1376
|
-
});
|
|
1377
|
-
} catch {
|
|
1378
|
-
for (let t of e) await p(t);
|
|
1379
|
-
}
|
|
1380
|
-
};
|
|
1381
|
-
return yield* l.runForEach(Ht(u, f.name, f.columnTypes), (e) => n.promise(async () => {
|
|
1382
|
-
if (s++, m) {
|
|
1383
|
-
v.push(e), v.length >= g && await y();
|
|
1384
|
-
return;
|
|
1385
|
-
}
|
|
1386
|
-
if (x) {
|
|
1387
|
-
S.push(e), S.length >= b && await T();
|
|
1388
|
-
return;
|
|
1389
|
-
}
|
|
1390
|
-
await p(e);
|
|
1391
|
-
})), yield* n.promise(T), yield* n.promise(y), {
|
|
1392
|
-
rowCount: s,
|
|
1393
|
-
checksum: i.digest("hex")
|
|
1394
|
-
};
|
|
1395
|
-
}), { rowCount: S, checksum: T } = yield* Tt("import", f.name, u, y, (e) => e.rowCount, x);
|
|
1396
|
-
if (s) {
|
|
1397
|
-
if (T !== f.checksum) return yield* n.fail(new Ve({
|
|
1398
|
-
what: `table ${f.name} checksum`,
|
|
1399
|
-
expected: f.checksum,
|
|
1400
|
-
actual: T
|
|
1401
|
-
}));
|
|
1402
|
-
if (S !== f.rowCount) return yield* n.fail(new Ve({
|
|
1403
|
-
what: `table ${f.name} rowCount`,
|
|
1404
|
-
expected: String(f.rowCount),
|
|
1405
|
-
actual: String(S)
|
|
1406
|
-
}));
|
|
1438
|
+
}
|
|
1439
|
+
yield* q(_, y);
|
|
1407
1440
|
}
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
index: u,
|
|
1412
|
-
tableCount: y,
|
|
1413
|
-
event: "done",
|
|
1414
|
-
rowsDone: S,
|
|
1415
|
-
total: f.rowCount
|
|
1416
|
-
}), M++, oe += S, i && (a.length > b ? p.push({
|
|
1417
|
-
name: f.name,
|
|
1418
|
-
rowCount: S
|
|
1419
|
-
}) : (g.tables[f.name] = {
|
|
1420
|
-
rowCount: S,
|
|
1421
|
-
checksum: f.checksum
|
|
1422
|
-
}, yield* K(m, g)));
|
|
1423
|
-
}
|
|
1424
|
-
if (a.length > 0 && (yield* re(r, a)), c === "replace" && !(i && g.truncated) && (yield* n.promise(() => r.deleteMany(u, { where: {
|
|
1425
|
-
column: "startedAt",
|
|
1426
|
-
op: "lte",
|
|
1427
|
-
value: de
|
|
1428
|
-
} }).then(() => void 0, () => void 0))), i && p.length > 0) {
|
|
1429
|
-
for (let e of p) {
|
|
1430
|
-
let t = o.tables.find((t) => t.name === e.name);
|
|
1431
|
-
g.tables[e.name] = {
|
|
1432
|
-
rowCount: e.rowCount,
|
|
1433
|
-
checksum: t.checksum
|
|
1434
|
-
};
|
|
1441
|
+
{
|
|
1442
|
+
let t = Ve(h, "skipped");
|
|
1443
|
+
t !== void 0 && e.onWarn?.(t);
|
|
1435
1444
|
}
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1445
|
+
if (u.length > 0) {
|
|
1446
|
+
let t = u.length === s.tables.length;
|
|
1447
|
+
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${_} to force a full re-import.`);
|
|
1448
|
+
}
|
|
1449
|
+
}), fe = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, pe = /* @__PURE__ */ new Date(), I = yield* Pn({
|
|
1450
|
+
store: e.store,
|
|
1451
|
+
mode: f,
|
|
1452
|
+
dir: e.rollbackDir,
|
|
1453
|
+
snapshot: e.targetSnapshot,
|
|
1454
|
+
dialect: e.targetDialect,
|
|
1455
|
+
tables: s.tables.map((e) => e.name).filter((e) => !B(e)),
|
|
1456
|
+
onWarn: e.onWarn
|
|
1457
|
+
});
|
|
1458
|
+
I !== void 0 && (e.onRollbackCaptured !== void 0 && (yield* n.tryPromise({
|
|
1459
|
+
try: () => e.onRollbackCaptured(I),
|
|
1460
|
+
catch: (e) => new V({
|
|
1461
|
+
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1462
|
+
path: I.dir
|
|
1463
|
+
})
|
|
1464
|
+
})), e.onWarn?.(Fn(I)));
|
|
1465
|
+
let me = (e) => f === "replace" ? S(e) : e(), he = (e) => f === "replace" ? te(e) : e, L = p("dimp", `${Date.now()}-${s.source.schemaFingerprint ?? ""}`), ge = !1;
|
|
1466
|
+
g(u) !== void 0 && (yield* n.promise(() => e.store.insert(u, {
|
|
1467
|
+
id: L,
|
|
1468
|
+
mode: f,
|
|
1469
|
+
via: e.via ?? "direct",
|
|
1470
|
+
bundle: e.bundleDir,
|
|
1471
|
+
sourceFingerprint: s.source.schemaFingerprint ?? null,
|
|
1472
|
+
tablesWritten: 0,
|
|
1473
|
+
rowsWritten: 0
|
|
1474
|
+
}).then(() => {
|
|
1475
|
+
ge = !0;
|
|
1476
|
+
}, (t) => {
|
|
1477
|
+
e.onWarn?.(`the import trace could not be written (${String(t?.message ?? t)}).\n The import continues — this is a record of what happened, not a safety check. Run
|
|
1478
|
+
\`voltro db apply\` so the next one is recorded.`);
|
|
1479
|
+
})));
|
|
1480
|
+
let _e = (t) => ge ? e.store.update(u, L, {
|
|
1481
|
+
finishedAt: t === null ? /* @__PURE__ */ new Date() : null,
|
|
1482
|
+
tablesWritten: N,
|
|
1483
|
+
rowsWritten: M,
|
|
1484
|
+
failure: t
|
|
1485
|
+
}).then(() => void 0, () => void 0) : Promise.resolve();
|
|
1486
|
+
if (t = (e) => _e(e), !j && f === "replace" && m && A.length > 0 && e.onWarn?.(bn(`the bundle's tables reference each other in a cycle (${A.map((e) => `${e.from}.${e.column} → ${e.to}`).join(", ")}), and a bulk copy cannot satisfy both directions on this dialect.`)), f === "replace" && k.length > 0 && e.onWarn?.(`write recorders are SUSPENDED for this replace (${k.length} table(s) have one).\n A replace sets a state rather than changing rows, so a per-row history entry would describe
|
|
1487
|
+
something that did not happen — and this path already writes through the raw store, without
|
|
1488
|
+
tenant scoping or \`audit()\` stamping, for the same reason.`), j) {
|
|
1489
|
+
e.onWarn?.(`staging ${O.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1490
|
+
let t = e.store, r = O.map((e) => e.name), i = Tn(r, {
|
|
1491
|
+
getTable: (e) => g(e),
|
|
1492
|
+
registerTable: (e) => {
|
|
1493
|
+
ee(e);
|
|
1475
1494
|
},
|
|
1476
|
-
|
|
1477
|
-
}),
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1495
|
+
unregisterTable: ne
|
|
1496
|
+
}), a = !1;
|
|
1497
|
+
try {
|
|
1498
|
+
let i = m ? r : r.filter((e) => !At(y, e));
|
|
1499
|
+
yield* n.tryPromise({
|
|
1500
|
+
try: async () => {
|
|
1501
|
+
await kn(t, i.map((e) => Cn(e, D))), await kn(t, i.map((e) => Sn(e, D)));
|
|
1502
|
+
},
|
|
1503
|
+
catch: (e) => new V({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1504
|
+
}), yield* he(F(e.store, !m)), !m && y.truncated === !0 ? a = !0 : yield* n.tryPromise({
|
|
1505
|
+
try: () => An(t, O, D).then(() => {
|
|
1506
|
+
a = !0;
|
|
1507
|
+
}),
|
|
1508
|
+
catch: (e) => new V({ reason: String(e?.message ?? e) })
|
|
1509
|
+
}).pipe(n.catchAll((i) => n.gen(function* () {
|
|
1510
|
+
let a = On(yield* n.promise(() => jn(t, En(e.targetSnapshot, r), D).then((e) => e, () => [])));
|
|
1511
|
+
return yield* n.fail(new V({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1512
|
+
}))), !m && a && y.truncated !== !0 && (y.truncated = !0, yield* q(_, y));
|
|
1513
|
+
} finally {
|
|
1514
|
+
i.undo(), a || m ? yield* n.promise(() => kn(t, r.map((e) => Cn(e, D))).then(() => void 0, () => void 0)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1515
|
+
}
|
|
1516
|
+
} else m ? yield* n.tryPromise({
|
|
1517
|
+
try: () => e.store.transactional(async (e) => {
|
|
1518
|
+
let t = await me(() => n.runPromiseExit(F(e, !1)));
|
|
1519
|
+
if (r.isFailure(t)) throw Un(t.cause);
|
|
1520
|
+
}),
|
|
1521
|
+
catch: (e) => Hn(e) ?? new V({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1522
|
+
}) : yield* he(F(e.store, !0));
|
|
1523
|
+
if (e.assets && s.assets.index && !e.assetsAlreadyRestored) {
|
|
1524
|
+
let t = e.assets, r = i.join(e.bundleDir, yt), a = i.join(e.bundleDir, s.assets.index ?? "assets/index.ndjson"), o = se("bundle", () => ue(a)).pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.map((e) => JSON.parse(e)));
|
|
1525
|
+
yield* l.runForEach(o, (e) => e.key in y.assets ? n.void : it(t, r, e).pipe(n.andThen(n.sync(() => {
|
|
1526
|
+
y.assets[e.key] = !0;
|
|
1527
|
+
})), n.andThen(q(_, y))));
|
|
1488
1528
|
}
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
})), n.andThen(K(m, g))));
|
|
1501
|
-
}
|
|
1502
|
-
return {
|
|
1503
|
-
manifest: o,
|
|
1504
|
-
tablesWritten: M,
|
|
1505
|
-
rowsWritten: oe,
|
|
1506
|
-
tablesSkipped: le,
|
|
1507
|
-
rowsSkipped: ce,
|
|
1508
|
-
fullyResumed: M === 0 && le > 0,
|
|
1509
|
-
...P === void 0 ? {} : { rollback: P }
|
|
1510
|
-
};
|
|
1511
|
-
}), Wn = (e) => e, Gn = (e) => {
|
|
1529
|
+
return yield* n.promise(() => _e(null)), {
|
|
1530
|
+
manifest: s,
|
|
1531
|
+
tablesWritten: N,
|
|
1532
|
+
rowsWritten: M,
|
|
1533
|
+
tablesSkipped: P,
|
|
1534
|
+
rowsSkipped: de,
|
|
1535
|
+
fullyResumed: N === 0 && P > 0,
|
|
1536
|
+
...I === void 0 ? {} : { rollback: I }
|
|
1537
|
+
};
|
|
1538
|
+
}).pipe(n.tapError((e) => n.promise(() => t?.(e?.reason ?? e?._tag ?? "the import did not finish") ?? Promise.resolve())));
|
|
1539
|
+
}, Kn = (e) => e, qn = (e) => {
|
|
1512
1540
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1513
1541
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1514
1542
|
return {
|
|
1515
1543
|
refuse: !0,
|
|
1516
1544
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1517
1545
|
};
|
|
1518
|
-
},
|
|
1546
|
+
}, Z = Buffer.from("VBUNDLE2"), Jn = 0, Yn = 1, Xn = 2, Zn = 32, Qn = async (e) => {
|
|
1519
1547
|
let t = [], n = async (r) => {
|
|
1520
1548
|
let i;
|
|
1521
1549
|
try {
|
|
1522
|
-
i = await
|
|
1550
|
+
i = await fe(L(e, r), { withFileTypes: !0 });
|
|
1523
1551
|
} catch {
|
|
1524
1552
|
return;
|
|
1525
1553
|
}
|
|
@@ -1529,22 +1557,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1529
1557
|
}
|
|
1530
1558
|
};
|
|
1531
1559
|
return await n(""), t.sort();
|
|
1532
|
-
},
|
|
1560
|
+
}, $n = (e, t) => {
|
|
1533
1561
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1534
|
-
return r.writeUInt8(
|
|
1535
|
-
},
|
|
1562
|
+
return r.writeUInt8(Yn, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1563
|
+
}, er = (e, t, n) => {
|
|
1536
1564
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1537
|
-
return a.writeUInt8(
|
|
1565
|
+
return a.writeUInt8(Xn, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1538
1566
|
};
|
|
1539
|
-
async function*
|
|
1540
|
-
yield new Uint8Array(
|
|
1541
|
-
for (let t of await
|
|
1542
|
-
let n =
|
|
1543
|
-
if (yield new Uint8Array(
|
|
1567
|
+
async function* tr(e) {
|
|
1568
|
+
yield new Uint8Array(Z);
|
|
1569
|
+
for (let t of await Qn(e.dir)) {
|
|
1570
|
+
let n = L(e.dir, t), r = await pe(n);
|
|
1571
|
+
if (yield new Uint8Array($n(t, r.size)), r.size > 0) for await (let e of ue(n)) yield new Uint8Array(e);
|
|
1544
1572
|
}
|
|
1545
1573
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1546
|
-
yield new Uint8Array(
|
|
1547
|
-
let n =
|
|
1574
|
+
yield new Uint8Array(er(t.key, t.contentType, t.size));
|
|
1575
|
+
let n = T("sha256"), r = 0;
|
|
1548
1576
|
for await (let i of e.blobs.open(t.key)) {
|
|
1549
1577
|
let e = Buffer.from(i);
|
|
1550
1578
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1552,11 +1580,11 @@ async function* $n(e) {
|
|
|
1552
1580
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1553
1581
|
yield new Uint8Array(n.digest());
|
|
1554
1582
|
}
|
|
1555
|
-
yield new Uint8Array(Buffer.from([
|
|
1583
|
+
yield new Uint8Array(Buffer.from([Jn]));
|
|
1556
1584
|
}
|
|
1557
|
-
var
|
|
1558
|
-
let n =
|
|
1559
|
-
await
|
|
1585
|
+
var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr = (e) => new Promise((t) => e.end(t)), ir = async (e, t) => {
|
|
1586
|
+
let n = ge(t.dir);
|
|
1587
|
+
await F(n, { recursive: !0 });
|
|
1560
1588
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1561
1589
|
let { value: e, done: t } = await i.next();
|
|
1562
1590
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1567,28 +1595,28 @@ var er = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tr
|
|
|
1567
1595
|
return r = r.subarray(e), t;
|
|
1568
1596
|
};
|
|
1569
1597
|
try {
|
|
1570
|
-
if (await s(
|
|
1598
|
+
if (await s(Z.length), !c(Z.length).equals(Z)) throw Error("not a bundle archive (bad magic)");
|
|
1571
1599
|
for (;;) {
|
|
1572
1600
|
await s(1);
|
|
1573
1601
|
let e = c(1).readUInt8(0);
|
|
1574
|
-
if (e ===
|
|
1575
|
-
if (e ===
|
|
1602
|
+
if (e === Jn) return;
|
|
1603
|
+
if (e === Yn) {
|
|
1576
1604
|
await s(4);
|
|
1577
1605
|
let e = c(4).readUInt32BE(0);
|
|
1578
1606
|
await s(e);
|
|
1579
1607
|
let t = c(e).toString("utf8");
|
|
1580
1608
|
await s(8);
|
|
1581
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1582
|
-
if (a !== n && !a.startsWith(n +
|
|
1583
|
-
await
|
|
1609
|
+
let i = c(8).readBigUInt64BE(0), a = ge(n, t);
|
|
1610
|
+
if (a !== n && !a.startsWith(n + _e)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1611
|
+
await F(he(a), { recursive: !0 });
|
|
1584
1612
|
let o = M(a);
|
|
1585
1613
|
for (; i > 0n;) {
|
|
1586
1614
|
r.length === 0 && await s(1);
|
|
1587
1615
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1588
|
-
await
|
|
1616
|
+
await nr(o, c(e)), i -= BigInt(e);
|
|
1589
1617
|
}
|
|
1590
|
-
await
|
|
1591
|
-
} else if (e ===
|
|
1618
|
+
await rr(o);
|
|
1619
|
+
} else if (e === Xn) {
|
|
1592
1620
|
await s(4);
|
|
1593
1621
|
let e = c(4).readUInt32BE(0);
|
|
1594
1622
|
await s(e);
|
|
@@ -1598,24 +1626,24 @@ var er = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tr
|
|
|
1598
1626
|
await s(i);
|
|
1599
1627
|
let a = c(i).toString("utf8");
|
|
1600
1628
|
await s(8);
|
|
1601
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1629
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = T("sha256"), d = new A(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1602
1630
|
contentType: a,
|
|
1603
1631
|
size: Number(o)
|
|
1604
1632
|
}), p = o;
|
|
1605
1633
|
for (; p > 0n;) {
|
|
1606
1634
|
r.length === 0 && await s(1);
|
|
1607
1635
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1608
|
-
u.update(t), l || await
|
|
1636
|
+
u.update(t), l || await nr(d, t), p -= BigInt(e);
|
|
1609
1637
|
}
|
|
1610
|
-
l ? d.destroy() : await
|
|
1611
|
-
let m = c(
|
|
1638
|
+
l ? d.destroy() : await rr(d), await f, await s(Zn);
|
|
1639
|
+
let m = c(Zn);
|
|
1612
1640
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1613
1641
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1614
1642
|
}
|
|
1615
1643
|
} finally {
|
|
1616
1644
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1617
1645
|
}
|
|
1618
|
-
},
|
|
1646
|
+
}, ar = async (e, t) => {
|
|
1619
1647
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1620
1648
|
let { value: e, done: t } = await a.next();
|
|
1621
1649
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1634,11 +1662,11 @@ var er = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tr
|
|
|
1634
1662
|
}
|
|
1635
1663
|
return Buffer.concat(t);
|
|
1636
1664
|
};
|
|
1637
|
-
if (!await s(
|
|
1665
|
+
if (!await s(Z.length) || !c(Z.length).equals(Z)) throw Error("not a bundle archive (bad magic)");
|
|
1638
1666
|
for (; await s(1);) {
|
|
1639
1667
|
let e = c(1).readUInt8(0);
|
|
1640
|
-
if (e ===
|
|
1641
|
-
if (e !==
|
|
1668
|
+
if (e === Jn || e === Xn) break;
|
|
1669
|
+
if (e !== Yn) throw Error(`unknown bundle entry kind ${e}`);
|
|
1642
1670
|
if (!await s(4)) break;
|
|
1643
1671
|
let t = c(4).readUInt32BE(0);
|
|
1644
1672
|
if (!await s(t)) break;
|
|
@@ -1648,22 +1676,22 @@ var er = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tr
|
|
|
1648
1676
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1649
1677
|
}
|
|
1650
1678
|
return r;
|
|
1651
|
-
},
|
|
1679
|
+
}, or = (e, t) => ir(e, {
|
|
1652
1680
|
dir: t,
|
|
1653
|
-
blobs:
|
|
1654
|
-
}),
|
|
1681
|
+
blobs: sr(t)
|
|
1682
|
+
}), sr = (e) => {
|
|
1655
1683
|
let t = /* @__PURE__ */ new Set();
|
|
1656
1684
|
return {
|
|
1657
1685
|
has: async (e) => t.has(e),
|
|
1658
1686
|
write: async (n, r, i) => {
|
|
1659
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1660
|
-
await
|
|
1661
|
-
let l =
|
|
1687
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = L(e, "assets");
|
|
1688
|
+
await F(c, { recursive: !0 });
|
|
1689
|
+
let l = T("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1662
1690
|
l.update(e), u += e.length, n(null, e);
|
|
1663
|
-
} }), f =
|
|
1691
|
+
} }), f = L(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1664
1692
|
await o(r, d, M(f));
|
|
1665
1693
|
let p = l.digest("hex");
|
|
1666
|
-
await a.rename(f,
|
|
1694
|
+
await a.rename(f, L(c, p)), await a.appendFile(L(c, "index.ndjson"), `${JSON.stringify({
|
|
1667
1695
|
key: n,
|
|
1668
1696
|
sha256: p,
|
|
1669
1697
|
size: u,
|
|
@@ -1671,33 +1699,33 @@ var er = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tr
|
|
|
1671
1699
|
})}\n`), t.add(n);
|
|
1672
1700
|
}
|
|
1673
1701
|
};
|
|
1674
|
-
},
|
|
1702
|
+
}, cr = ve(ae), lr = Buffer.from("VENC1\0\0\0"), ur = 32, dr = 16, fr = 7, pr = 16, mr = 262144, hr = {
|
|
1675
1703
|
N: 32768,
|
|
1676
1704
|
r: 8,
|
|
1677
1705
|
p: 1
|
|
1678
|
-
},
|
|
1706
|
+
}, gr = 134217728, Q = 43, _r = (e, t, n) => {
|
|
1679
1707
|
let r = Buffer.alloc(12);
|
|
1680
|
-
return e.copy(r, 0, 0,
|
|
1708
|
+
return e.copy(r, 0, 0, fr), r.writeUInt32BE(t >>> 0, fr), r.writeUInt8(+!!n, 11), r;
|
|
1681
1709
|
};
|
|
1682
|
-
async function*
|
|
1710
|
+
async function* vr(e, t) {
|
|
1683
1711
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1684
|
-
let n =
|
|
1685
|
-
...
|
|
1686
|
-
maxmem:
|
|
1712
|
+
let n = E(dr), r = E(fr), i = await cr(t, n, ur, {
|
|
1713
|
+
...hr,
|
|
1714
|
+
maxmem: gr
|
|
1687
1715
|
}), a = Buffer.alloc(Q);
|
|
1688
|
-
|
|
1716
|
+
lr.copy(a, 0), a.writeUInt32BE(hr.N, 8), a.writeUInt32BE(hr.r, 12), a.writeUInt32BE(hr.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1689
1717
|
let o = 0, s = (e, t) => {
|
|
1690
|
-
let n =
|
|
1718
|
+
let n = C("aes-256-gcm", i, _r(r, o, t));
|
|
1691
1719
|
o === 0 && n.setAAD(a);
|
|
1692
1720
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1693
1721
|
o++;
|
|
1694
|
-
let l = Buffer.alloc(5 + s.length +
|
|
1695
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
1722
|
+
let l = Buffer.alloc(5 + s.length + pr);
|
|
1723
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + pr, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1696
1724
|
}, c = Buffer.alloc(0);
|
|
1697
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1725
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > mr;) yield s(c.subarray(0, mr), !1), c = c.subarray(mr);
|
|
1698
1726
|
yield s(c, !0);
|
|
1699
1727
|
}
|
|
1700
|
-
async function*
|
|
1728
|
+
async function* yr(e, t) {
|
|
1701
1729
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1702
1730
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1703
1731
|
let { value: e, done: t } = await n.next();
|
|
@@ -1708,12 +1736,12 @@ async function* gr(e, t) {
|
|
|
1708
1736
|
};
|
|
1709
1737
|
if (!await o(Q)) throw Error("encrypted bundle truncated (no header)");
|
|
1710
1738
|
let s = r.subarray(0, Q);
|
|
1711
|
-
if (!s.subarray(0, 8).equals(
|
|
1712
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await
|
|
1739
|
+
if (!s.subarray(0, 8).equals(lr)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1740
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await cr(t, d, ur, {
|
|
1713
1741
|
N: c,
|
|
1714
1742
|
r: l,
|
|
1715
1743
|
p: u,
|
|
1716
|
-
maxmem:
|
|
1744
|
+
maxmem: gr
|
|
1717
1745
|
});
|
|
1718
1746
|
r = r.subarray(Q);
|
|
1719
1747
|
let m = 0, h = !1;
|
|
@@ -1724,11 +1752,11 @@ async function* gr(e, t) {
|
|
|
1724
1752
|
}
|
|
1725
1753
|
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
1726
1754
|
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
1727
|
-
if (t <
|
|
1755
|
+
if (t < pr) throw Error("encrypted bundle corrupt (short frame)");
|
|
1728
1756
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1729
|
-
let n = r.subarray(5, 5 + t -
|
|
1757
|
+
let n = r.subarray(5, 5 + t - pr), i = r.subarray(5 + t - pr, 5 + t);
|
|
1730
1758
|
r = r.subarray(5 + t);
|
|
1731
|
-
let a =
|
|
1759
|
+
let a = w("aes-256-gcm", p, _r(f, m, e));
|
|
1732
1760
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1733
1761
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1734
1762
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1736,7 +1764,7 @@ async function* gr(e, t) {
|
|
|
1736
1764
|
}
|
|
1737
1765
|
//#endregion
|
|
1738
1766
|
//#region src/blobStorage.ts
|
|
1739
|
-
var
|
|
1767
|
+
var br = (e, t) => ({
|
|
1740
1768
|
list: () => {
|
|
1741
1769
|
let e = t();
|
|
1742
1770
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1749,29 +1777,29 @@ var _r = (e, t) => ({
|
|
|
1749
1777
|
} };
|
|
1750
1778
|
},
|
|
1751
1779
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1752
|
-
let r = await n.runPromise(
|
|
1780
|
+
let r = await n.runPromise(ce(e, t)), i = await O(r.stream);
|
|
1753
1781
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1754
1782
|
} })
|
|
1755
|
-
}),
|
|
1756
|
-
table:
|
|
1783
|
+
}), xr = (e, t) => br(e, () => x(t, {
|
|
1784
|
+
table: oe,
|
|
1757
1785
|
chunkSize: 500
|
|
1758
1786
|
}).pipe(l.map((e) => ({
|
|
1759
1787
|
key: String(e.key),
|
|
1760
1788
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1761
1789
|
size: Number(e.size ?? 0)
|
|
1762
|
-
})))),
|
|
1790
|
+
})))), Sr = (e) => ({
|
|
1763
1791
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1764
1792
|
write: async (t, r, i) => {
|
|
1765
|
-
let a =
|
|
1766
|
-
await n.runPromise(
|
|
1793
|
+
let a = se("storage", () => j.from(r));
|
|
1794
|
+
await n.runPromise(D(e, t, a, {
|
|
1767
1795
|
contentType: i.contentType,
|
|
1768
1796
|
totalSize: i.size
|
|
1769
1797
|
}));
|
|
1770
1798
|
}
|
|
1771
|
-
}),
|
|
1799
|
+
}), Cr = (e, t) => {
|
|
1772
1800
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1773
|
-
return e === "mariadb" &&
|
|
1774
|
-
},
|
|
1801
|
+
return e === "mariadb" && wr(n) ? n : r;
|
|
1802
|
+
}, wr = (e) => (process.env.PATH ?? "").split(me).filter((e) => e !== "").some((t) => N(L(t, e))), Tr = (e) => {
|
|
1775
1803
|
switch (e) {
|
|
1776
1804
|
case "postgres": return "db.dump";
|
|
1777
1805
|
case "mysql":
|
|
@@ -1781,7 +1809,7 @@ var _r = (e, t) => ({
|
|
|
1781
1809
|
case "mssql": return "db.bacpac";
|
|
1782
1810
|
default: return "db.dump";
|
|
1783
1811
|
}
|
|
1784
|
-
},
|
|
1812
|
+
}, Er = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1785
1813
|
if (e.host || e.database) return {
|
|
1786
1814
|
host: e.host ?? "localhost",
|
|
1787
1815
|
port: e.port ?? 3306,
|
|
@@ -1797,7 +1825,7 @@ var _r = (e, t) => ({
|
|
|
1797
1825
|
password: decodeURIComponent(t.password) || "",
|
|
1798
1826
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1799
1827
|
};
|
|
1800
|
-
},
|
|
1828
|
+
}, Dr = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Or = (e, t, n) => {
|
|
1801
1829
|
switch (e) {
|
|
1802
1830
|
case "postgres": {
|
|
1803
1831
|
if (t.url) return {
|
|
@@ -1844,7 +1872,7 @@ var _r = (e, t) => ({
|
|
|
1844
1872
|
let r = $(t);
|
|
1845
1873
|
return {
|
|
1846
1874
|
kind: "spawn",
|
|
1847
|
-
tool:
|
|
1875
|
+
tool: Cr(e, "dump"),
|
|
1848
1876
|
args: [
|
|
1849
1877
|
"--single-transaction",
|
|
1850
1878
|
"--routines",
|
|
@@ -1866,7 +1894,7 @@ var _r = (e, t) => ({
|
|
|
1866
1894
|
case "sqlite":
|
|
1867
1895
|
case "turso": return {
|
|
1868
1896
|
kind: "copy",
|
|
1869
|
-
from:
|
|
1897
|
+
from: Er(t),
|
|
1870
1898
|
to: n
|
|
1871
1899
|
};
|
|
1872
1900
|
case "mssql": return {
|
|
@@ -1874,7 +1902,7 @@ var _r = (e, t) => ({
|
|
|
1874
1902
|
tool: "sqlpackage",
|
|
1875
1903
|
args: [
|
|
1876
1904
|
"/Action:Export",
|
|
1877
|
-
`/SourceConnectionString:${
|
|
1905
|
+
`/SourceConnectionString:${Dr($({
|
|
1878
1906
|
...t,
|
|
1879
1907
|
port: t.port ?? 1433
|
|
1880
1908
|
}))}`,
|
|
@@ -1882,12 +1910,12 @@ var _r = (e, t) => ({
|
|
|
1882
1910
|
],
|
|
1883
1911
|
env: {}
|
|
1884
1912
|
};
|
|
1885
|
-
default: throw new
|
|
1913
|
+
default: throw new U({
|
|
1886
1914
|
tool: "backup",
|
|
1887
1915
|
reason: `no native backup for dialect '${e}'`
|
|
1888
1916
|
});
|
|
1889
1917
|
}
|
|
1890
|
-
},
|
|
1918
|
+
}, kr = (e, t, n) => {
|
|
1891
1919
|
switch (e) {
|
|
1892
1920
|
case "postgres": {
|
|
1893
1921
|
if (t.url) return {
|
|
@@ -1932,7 +1960,7 @@ var _r = (e, t) => ({
|
|
|
1932
1960
|
let r = $(t);
|
|
1933
1961
|
return {
|
|
1934
1962
|
kind: "spawn",
|
|
1935
|
-
tool:
|
|
1963
|
+
tool: Cr(e, "client"),
|
|
1936
1964
|
args: [
|
|
1937
1965
|
"--protocol=TCP",
|
|
1938
1966
|
"-h",
|
|
@@ -1951,14 +1979,14 @@ var _r = (e, t) => ({
|
|
|
1951
1979
|
case "turso": return {
|
|
1952
1980
|
kind: "copy",
|
|
1953
1981
|
from: n,
|
|
1954
|
-
to:
|
|
1982
|
+
to: Er(t)
|
|
1955
1983
|
};
|
|
1956
1984
|
case "mssql": return {
|
|
1957
1985
|
kind: "spawn",
|
|
1958
1986
|
tool: "sqlpackage",
|
|
1959
1987
|
args: [
|
|
1960
1988
|
"/Action:Import",
|
|
1961
|
-
`/TargetConnectionString:${
|
|
1989
|
+
`/TargetConnectionString:${Dr($({
|
|
1962
1990
|
...t,
|
|
1963
1991
|
port: t.port ?? 1433
|
|
1964
1992
|
}))}`,
|
|
@@ -1966,18 +1994,18 @@ var _r = (e, t) => ({
|
|
|
1966
1994
|
],
|
|
1967
1995
|
env: {}
|
|
1968
1996
|
};
|
|
1969
|
-
default: throw new
|
|
1997
|
+
default: throw new U({
|
|
1970
1998
|
tool: "restore",
|
|
1971
1999
|
reason: `no native restore for dialect '${e}'`
|
|
1972
2000
|
});
|
|
1973
2001
|
}
|
|
1974
|
-
},
|
|
1975
|
-
|
|
2002
|
+
}, Ar = (e) => e.kind === "copy" ? n.async((t) => {
|
|
2003
|
+
le(e.from, e.to, (r) => t(r ? n.fail(new U({
|
|
1976
2004
|
tool: "copy",
|
|
1977
2005
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1978
2006
|
})) : n.void));
|
|
1979
2007
|
}) : n.async((t) => {
|
|
1980
|
-
let r =
|
|
2008
|
+
let r = ye(e.tool, [...e.args], {
|
|
1981
2009
|
env: {
|
|
1982
2010
|
...process.env,
|
|
1983
2011
|
...e.env
|
|
@@ -1990,14 +2018,14 @@ var _r = (e, t) => ({
|
|
|
1990
2018
|
}), i = "";
|
|
1991
2019
|
r.stderr?.on("data", (e) => {
|
|
1992
2020
|
i += String(e);
|
|
1993
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin &&
|
|
2021
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin && ue(e.stdinFile).pipe(r.stdin);
|
|
1994
2022
|
let a = () => {
|
|
1995
2023
|
if (e.stdoutFile !== void 0) try {
|
|
1996
|
-
|
|
2024
|
+
de(e.stdoutFile);
|
|
1997
2025
|
} catch {}
|
|
1998
2026
|
};
|
|
1999
2027
|
r.on("error", (r) => {
|
|
2000
|
-
a(), t(n.fail(new
|
|
2028
|
+
a(), t(n.fail(new U({
|
|
2001
2029
|
tool: e.tool,
|
|
2002
2030
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`
|
|
2003
2031
|
})));
|
|
@@ -2006,17 +2034,17 @@ var _r = (e, t) => ({
|
|
|
2006
2034
|
t(n.void);
|
|
2007
2035
|
return;
|
|
2008
2036
|
}
|
|
2009
|
-
a(), t(n.fail(new
|
|
2037
|
+
a(), t(n.fail(new U({
|
|
2010
2038
|
tool: e.tool,
|
|
2011
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2039
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${I(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
2012
2040
|
stderr: i.slice(-2e3)
|
|
2013
2041
|
})));
|
|
2014
2042
|
});
|
|
2015
|
-
}),
|
|
2016
|
-
let t = yield*
|
|
2043
|
+
}), jr = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Mr = (e) => n.gen(function* () {
|
|
2044
|
+
let t = yield* et(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
2017
2045
|
store: e.store,
|
|
2018
2046
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
2019
|
-
}), r =
|
|
2047
|
+
}), r = Gt(t.tables, e.snapshot), i = e.classification ?? Se(e.snapshot), a = e.sampleSize ?? 20, o = Fe(r.map((t) => ({
|
|
2020
2048
|
name: t,
|
|
2021
2049
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
2022
2050
|
name: e.name,
|
|
@@ -2028,15 +2056,15 @@ var _r = (e, t) => ({
|
|
|
2028
2056
|
table: i,
|
|
2029
2057
|
chunkSize: a,
|
|
2030
2058
|
...r === void 0 ? {} : { where: r }
|
|
2031
|
-
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
2032
|
-
c.push(...
|
|
2059
|
+
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) => Re(t, i, o.actions, e.masking.seed));
|
|
2060
|
+
c.push(...Le(d, i, o.actions));
|
|
2033
2061
|
let f = u[0], p = d[0], m = [];
|
|
2034
2062
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
2035
2063
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
2036
2064
|
let n = e.slice(i.length + 1);
|
|
2037
2065
|
m.push({
|
|
2038
2066
|
column: n,
|
|
2039
|
-
action:
|
|
2067
|
+
action: jr(t),
|
|
2040
2068
|
before: f[n],
|
|
2041
2069
|
after: p[n]
|
|
2042
2070
|
});
|
|
@@ -2051,7 +2079,7 @@ var _r = (e, t) => ({
|
|
|
2051
2079
|
leaks: c,
|
|
2052
2080
|
unclassified: o.unclassified
|
|
2053
2081
|
};
|
|
2054
|
-
}),
|
|
2082
|
+
}), Nr = (e, t) => {
|
|
2055
2083
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
2056
2084
|
for (let t of e) {
|
|
2057
2085
|
let e = n.get(t.name);
|
|
@@ -2106,7 +2134,7 @@ var _r = (e, t) => ({
|
|
|
2106
2134
|
skipColumns: i,
|
|
2107
2135
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
2108
2136
|
};
|
|
2109
|
-
},
|
|
2137
|
+
}, Pr = (e) => {
|
|
2110
2138
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
2111
2139
|
let t = {
|
|
2112
2140
|
safe: "✓",
|
|
@@ -2117,4 +2145,4 @@ var _r = (e, t) => ({
|
|
|
2117
2145
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
2118
2146
|
};
|
|
2119
2147
|
//#endregion
|
|
2120
|
-
export {
|
|
2148
|
+
export { yt as ASSETS_DIR, bt as ASSET_INDEX_FILE, V as BundleError, He as CodecError, We as CompressionError, Ye as CrossDialectError, vt as DATA_DIR, xe as DEFAULT_CLASS_ACTIONS, ut as FORMAT_VERSION, Je as ImportModeError, Ue as IntegrityError, xt as LEDGER_FILE, G as MANIFEST_FILE, qe as MaskingError, U as NativeToolError, Ge as RowsRefusedError, Bn as SAVEPOINT_BATCH_SIZE, vn as STAGING_PREFIX, Ke as SchemaDriftError, H as ScopeError, Ne as applyAction, Hn as asImportError, qn as assessLiveImport, jt as assetDone, Tr as backupArtifactName, Or as backupCommand, on as canonicalType, ln as checkPortability, zn as checkSchemaDrift, Se as classificationFromSnapshot, Nr as classifyImportDrift, st as compressionExt, Ze as computeSubsetIds, gt as decodeManifest, Vt as decodeRow, zt as decodeValue, yr as decryptStream, Kn as defineDataProfile, Rn as diffSnapshots, sr as dirBlobSink, Cn as dropStagingSql, Ot as emptyLedger, ht as encodeManifest, Bt as encodeRow, Lt as encodeValue, vr as encryptStream, kn as execStatements, Pr as formatImportDrift, un as formatIssue, at as hasZstd, ct as makeCompressor, lt as makeDecompressor, Re as maskRow, Wt as ndjsonToRows, an as outsideReferenceRefusal, rn as outsideReferencesInto, tr as packBundle, ar as peekBundle, ot as pickCompression, Fe as planMasking, Mr as previewMasking, kt as readLedger, Ce as resolveAction, et as resolveScope, it as restoreAssetFromCas, kr as restoreCommand, Ut as rowsToNdjson, nn as runExport, Gn as runImport, Ar as runNativeStep, Le as scanForLeaks, Y as stagingNameFor, nt as storageAssetSink, tt as storageAssetSource, Sr as storageBlobSink, br as storageBlobSource, xr as storageRefsBlobSource, Qe as subsetToScope, At as tableDone, Gt as topoSortTables, ir as unpackBundle, or as unpackBundleToDir, rt as writeAssetToCas, q as writeLedger };
|