@voltro/data-transfer 0.44.1 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +280 -1
- package/dist/index.d.ts +95 -0
- package/dist/index.js +813 -564
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
|
-
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a,
|
|
2
|
-
import { classifyConstraintViolation as
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { STORAGE_REFS_TABLE as
|
|
6
|
-
import * as
|
|
7
|
-
import { PassThrough as
|
|
8
|
-
import { copyFile as
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
12
|
-
import {
|
|
1
|
+
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Runtime as o, Schedule as s, Schema as c, Stream as l } from "effect";
|
|
2
|
+
import { REPLACE_IN_PROGRESS_TABLE as u, classifyConstraintViolation as d, derivedRowId as f, eq as p, extractDbCause as m, getTable as h, inSet as g, isNotNull as _, or as v, recordsTable as y, registerTable as b, streamTable as x, unregisterTable as S } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as C } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as w, createDecipheriv as T, createHash as E, createHmac as ee, randomBytes as te, scrypt as ne } from "node:crypto";
|
|
5
|
+
import { STORAGE_REFS_TABLE as re, fromNodeReadable as ie, getObjectStream as D, putObjectStream as O, toNodeReadable as k } from "@voltro/plugin-storage";
|
|
6
|
+
import * as A from "node:zlib";
|
|
7
|
+
import { PassThrough as j, Readable as ae } from "node:stream";
|
|
8
|
+
import { copyFile as oe, createReadStream as se, createWriteStream as M, existsSync as ce, unlinkSync as le } from "node:fs";
|
|
9
|
+
import { SqlClient as N } from "@effect/sql";
|
|
10
|
+
import { mkdir as ue, readdir as de, stat as P } from "node:fs/promises";
|
|
11
|
+
import { basename as fe, delimiter as pe, dirname as me, join as F, resolve as he, sep as ge } from "node:path";
|
|
12
|
+
import { promisify as _e } from "node:util";
|
|
13
|
+
import { spawn as ve } from "node:child_process";
|
|
13
14
|
//#region src/consistency.ts
|
|
14
|
-
var
|
|
15
|
+
var ye = (e) => {
|
|
15
16
|
switch (e) {
|
|
16
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
17
18
|
case "mysql":
|
|
@@ -21,7 +22,7 @@ var pe = (e) => {
|
|
|
21
22
|
case "turso": return [];
|
|
22
23
|
default: return [];
|
|
23
24
|
}
|
|
24
|
-
},
|
|
25
|
+
}, be = {
|
|
25
26
|
email: { fake: "email" },
|
|
26
27
|
fullName: { fake: "fullName" },
|
|
27
28
|
firstName: { fake: "firstName" },
|
|
@@ -36,7 +37,7 @@ var pe = (e) => {
|
|
|
36
37
|
date: "dateShift",
|
|
37
38
|
secret: "null",
|
|
38
39
|
freeText: "redact"
|
|
39
|
-
},
|
|
40
|
+
}, xe = (e) => {
|
|
40
41
|
let t = {};
|
|
41
42
|
for (let n of e.tables) {
|
|
42
43
|
let e = {};
|
|
@@ -48,15 +49,15 @@ var pe = (e) => {
|
|
|
48
49
|
t[n.name] = e;
|
|
49
50
|
}
|
|
50
51
|
return t;
|
|
51
|
-
},
|
|
52
|
+
}, Se = (e, t, n, r) => {
|
|
52
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
53
54
|
if (i) return i;
|
|
54
55
|
if (n?.sensitive) {
|
|
55
56
|
let e = n.sensitive.class;
|
|
56
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? be[e] ?? "redact";
|
|
57
58
|
}
|
|
58
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
59
|
-
},
|
|
60
|
+
}, Ce = (e, t) => ee("sha256", e).update(t == null ? "∅" : String(t)).digest(), I = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), L = [
|
|
60
61
|
"Ada",
|
|
61
62
|
"Alan",
|
|
62
63
|
"Grace",
|
|
@@ -69,7 +70,7 @@ var pe = (e) => {
|
|
|
69
70
|
"Guido",
|
|
70
71
|
"Margaret",
|
|
71
72
|
"Bjarne"
|
|
72
|
-
],
|
|
73
|
+
], we = [
|
|
73
74
|
"Lovelace",
|
|
74
75
|
"Turing",
|
|
75
76
|
"Hopper",
|
|
@@ -82,7 +83,7 @@ var pe = (e) => {
|
|
|
82
83
|
"Rossum",
|
|
83
84
|
"Hamilton",
|
|
84
85
|
"Stroustrup"
|
|
85
|
-
],
|
|
86
|
+
], Te = [
|
|
86
87
|
"Maple",
|
|
87
88
|
"Oak",
|
|
88
89
|
"Cedar",
|
|
@@ -91,7 +92,7 @@ var pe = (e) => {
|
|
|
91
92
|
"Birch",
|
|
92
93
|
"Willow",
|
|
93
94
|
"Ash"
|
|
94
|
-
],
|
|
95
|
+
], Ee = [
|
|
95
96
|
"Initech",
|
|
96
97
|
"Hooli",
|
|
97
98
|
"Umbrella",
|
|
@@ -100,23 +101,23 @@ var pe = (e) => {
|
|
|
100
101
|
"Soylent",
|
|
101
102
|
"Stark",
|
|
102
103
|
"Wayne"
|
|
103
|
-
],
|
|
104
|
-
let r =
|
|
104
|
+
], R = (e, t, n) => e[I(t, n) % e.length], De = (e, t, n) => {
|
|
105
|
+
let r = Ce(n, `${e}:${String(t)}`), i = I(r, 1) % 1e4;
|
|
105
106
|
switch (e) {
|
|
106
|
-
case "email": return `${
|
|
107
|
-
case "fullName": return `${
|
|
108
|
-
case "firstName": return
|
|
109
|
-
case "lastName": return
|
|
110
|
-
case "username": return `${
|
|
111
|
-
case "phone": return `+1${String(
|
|
112
|
-
case "address": return `${
|
|
113
|
-
case "company": return
|
|
107
|
+
case "email": return `${R(L, r, 0).toLowerCase()}.${R(we, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${R(L, r, 0)} ${R(we, r, 1)}`;
|
|
109
|
+
case "firstName": return R(L, r, 0);
|
|
110
|
+
case "lastName": return R(we, r, 1);
|
|
111
|
+
case "username": return `${R(L, r, 0).toLowerCase()}${i}`;
|
|
112
|
+
case "phone": return `+1${String(I(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
+
case "address": return `${I(r, 0) % 9899 + 100} ${R(Te, r, 1)} St`;
|
|
114
|
+
case "company": return R(Ee, r, 0);
|
|
114
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
115
|
-
case "ip": return `${
|
|
116
|
-
case "creditCard": return
|
|
116
|
+
case "ip": return `${I(r, 0) % 256}.${I(r, 1) % 256}.${I(r, 2) % 256}.${I(r, 3) % 254 + 1}`;
|
|
117
|
+
case "creditCard": return Oe(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
117
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
118
119
|
}
|
|
119
|
-
},
|
|
120
|
+
}, Oe = (e) => {
|
|
120
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
121
122
|
t.forEach((e, t) => {
|
|
122
123
|
let r = e;
|
|
@@ -124,31 +125,31 @@ var pe = (e) => {
|
|
|
124
125
|
});
|
|
125
126
|
let r = (10 - n % 10) % 10;
|
|
126
127
|
return e.slice(0, 15) + String(r);
|
|
127
|
-
},
|
|
128
|
+
}, ke = 864e5, Ae = (e, t) => {
|
|
128
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
129
130
|
if (Number.isNaN(n)) return e;
|
|
130
|
-
let r =
|
|
131
|
+
let r = I(Ce(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * ke);
|
|
131
132
|
return e instanceof Date ? i : i.toISOString();
|
|
132
|
-
},
|
|
133
|
+
}, je = (e, t) => {
|
|
133
134
|
if (t.value === null || t.value === void 0) return null;
|
|
134
135
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
135
136
|
if (e === "keep") return t.value;
|
|
136
137
|
if (e === "null") return null;
|
|
137
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
138
|
-
if (e === "hash") return
|
|
139
|
-
if (e === "dateShift") return
|
|
140
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return Ce(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Ae(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return De(e.fake, t.value, t.seed);
|
|
141
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
142
143
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
143
|
-
},
|
|
144
|
+
}, Me = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Ne = (e, t, n) => {
|
|
144
145
|
let r = {}, i = [], a = [];
|
|
145
146
|
for (let o of e) for (let e of o.columns) {
|
|
146
|
-
let s = `${o.name}.${e.name}`, c =
|
|
147
|
+
let s = `${o.name}.${e.name}`, c = Se(o.name, e.name, t[o.name]?.[e.name], n);
|
|
147
148
|
if (c === "error") {
|
|
148
149
|
i.push(s);
|
|
149
150
|
continue;
|
|
150
151
|
}
|
|
151
|
-
if (!
|
|
152
|
+
if (!Me(c)) {
|
|
152
153
|
a.push({
|
|
153
154
|
column: s,
|
|
154
155
|
action: c
|
|
@@ -162,7 +163,7 @@ var pe = (e) => {
|
|
|
162
163
|
unclassified: i,
|
|
163
164
|
invalid: a
|
|
164
165
|
};
|
|
165
|
-
},
|
|
166
|
+
}, Pe = [
|
|
166
167
|
{
|
|
167
168
|
name: "email",
|
|
168
169
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -179,7 +180,7 @@ var pe = (e) => {
|
|
|
179
180
|
name: "creditCard",
|
|
180
181
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
181
182
|
}
|
|
182
|
-
],
|
|
183
|
+
], Fe = (e, t, n) => {
|
|
183
184
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
184
185
|
for (let [a, o] of Object.entries(n)) {
|
|
185
186
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -187,7 +188,7 @@ var pe = (e) => {
|
|
|
187
188
|
for (let a of e) {
|
|
188
189
|
let e = a[n];
|
|
189
190
|
if (typeof e != "string" || e.length === 0) continue;
|
|
190
|
-
let o =
|
|
191
|
+
let o = Pe.find((t) => t.re.test(e));
|
|
191
192
|
if (o && !i.has(n)) {
|
|
192
193
|
i.add(n), r.push({
|
|
193
194
|
table: t,
|
|
@@ -200,9 +201,9 @@ var pe = (e) => {
|
|
|
200
201
|
}
|
|
201
202
|
}
|
|
202
203
|
return r;
|
|
203
|
-
},
|
|
204
|
+
}, Ie = (e, t, n, r, i) => {
|
|
204
205
|
let a = {};
|
|
205
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
206
|
+
for (let [o, s] of Object.entries(e)) a[o] = je(n[`${t}.${o}`] ?? "keep", {
|
|
206
207
|
value: s,
|
|
207
208
|
table: t,
|
|
208
209
|
column: o,
|
|
@@ -210,61 +211,77 @@ var pe = (e) => {
|
|
|
210
211
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
211
212
|
});
|
|
212
213
|
return a;
|
|
213
|
-
},
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
214
|
+
}, Le = /* @__PURE__ */ new Map([
|
|
215
|
+
["_voltro_migration_plans", "the schema fingerprint THIS deployment applied — a foreign row makes the next boot refuse"],
|
|
216
|
+
["_voltro_migrations", "which file-based migrations ran HERE — a foreign row marks unrun migrations as done"],
|
|
217
|
+
["_voltro_seeds", "which seeds ran HERE — a foreign row suppresses seeding the target still needs"],
|
|
218
|
+
["_voltro_cdc_offsets", "binlog/WAL positions of the SOURCE server — meaningless and unsafe against another one"],
|
|
219
|
+
["_voltro_schedule_claims", "live leases held by THIS deployment's replicas — a foreign row hands out a lease nobody holds"],
|
|
220
|
+
["_voltro_replace_in_progress", "the marker a destructive import HERE left behind — a foreign row refuses a boot over a healthy database"],
|
|
221
|
+
["_voltro_wakeups", "pending wakeups addressed to THIS deployment's runners"],
|
|
222
|
+
["_voltro_workflow_watermarks", "how far a consumer has read THIS deployment's event journal — a foreign position skips events"],
|
|
223
|
+
["_voltro_workflow_pending", "starts waiting to be drained HERE — a foreign row makes the target RUN a workflow somebody queued elsewhere"],
|
|
224
|
+
["_voltro_workflow_admissions", "the in-flight accounting a concurrency gate decides on — a foreign row makes the target refuse or admit wrongly"],
|
|
225
|
+
["_voltro_workflow_pauses", "which workflows an operator paused HERE — a foreign row silently pauses them in the target"]
|
|
226
|
+
]), z = (e) => Le.has(e), Re = (e) => Le.get(e), ze = (e, t) => {
|
|
227
|
+
if (e.length === 0) return;
|
|
228
|
+
let n = e.map((e) => ` ${e} — ${Re(e) ?? "describes this deployment"}`);
|
|
229
|
+
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
230
|
+
}, B = class extends c.TaggedError()("BundleError", {
|
|
231
|
+
reason: c.String,
|
|
232
|
+
path: c.optional(c.String)
|
|
233
|
+
}) {}, Be = class extends c.TaggedError()("CodecError", {
|
|
234
|
+
table: c.String,
|
|
235
|
+
reason: c.String
|
|
236
|
+
}) {}, Ve = class extends c.TaggedError()("IntegrityError", {
|
|
237
|
+
what: c.String,
|
|
238
|
+
expected: c.String,
|
|
239
|
+
actual: c.String
|
|
240
|
+
}) {}, He = class extends t.TaggedError("CompressionError") {}, V = class extends c.TaggedError()("ScopeError", { reason: c.String }) {}, Ue = class extends c.TaggedError()("RowsRefusedError", {
|
|
241
|
+
rows: c.Array(c.Struct({
|
|
242
|
+
table: c.String,
|
|
243
|
+
id: c.String,
|
|
244
|
+
reason: c.String,
|
|
245
|
+
derived: c.Boolean
|
|
229
246
|
})),
|
|
230
|
-
primaryCount:
|
|
231
|
-
totalCount:
|
|
232
|
-
byTable:
|
|
233
|
-
table:
|
|
234
|
-
refused:
|
|
235
|
-
primary:
|
|
247
|
+
primaryCount: c.Number,
|
|
248
|
+
totalCount: c.Number,
|
|
249
|
+
byTable: c.Array(c.Struct({
|
|
250
|
+
table: c.String,
|
|
251
|
+
refused: c.Number,
|
|
252
|
+
primary: c.Number
|
|
236
253
|
}))
|
|
237
|
-
}) {},
|
|
238
|
-
bundleFingerprint:
|
|
239
|
-
targetFingerprint:
|
|
240
|
-
diff:
|
|
241
|
-
}) {},
|
|
254
|
+
}) {}, We = class extends c.TaggedError()("SchemaDriftError", {
|
|
255
|
+
bundleFingerprint: c.String,
|
|
256
|
+
targetFingerprint: c.String,
|
|
257
|
+
diff: c.Array(c.String)
|
|
258
|
+
}) {}, H = class extends t.TaggedError("NativeToolError") {
|
|
242
259
|
get message() {
|
|
243
260
|
let e = this.stderr?.trim();
|
|
244
261
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
245
262
|
}
|
|
246
|
-
},
|
|
247
|
-
unclassified:
|
|
248
|
-
invalidActions:
|
|
249
|
-
}) {},
|
|
250
|
-
mode:
|
|
251
|
-
reason:
|
|
252
|
-
}) {},
|
|
253
|
-
sourceDialect:
|
|
254
|
-
targetDialect:
|
|
255
|
-
blocking:
|
|
256
|
-
}) {},
|
|
263
|
+
}, Ge = class extends c.TaggedError()("MaskingError", {
|
|
264
|
+
unclassified: c.Array(c.String),
|
|
265
|
+
invalidActions: c.optional(c.Array(c.String))
|
|
266
|
+
}) {}, Ke = class extends c.TaggedError()("ImportModeError", {
|
|
267
|
+
mode: c.String,
|
|
268
|
+
reason: c.String
|
|
269
|
+
}) {}, qe = class extends c.TaggedError()("CrossDialectError", {
|
|
270
|
+
sourceDialect: c.String,
|
|
271
|
+
targetDialect: c.String,
|
|
272
|
+
blocking: c.Array(c.String)
|
|
273
|
+
}) {}, U = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Je = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
257
274
|
col: e.name,
|
|
258
275
|
refTable: e.references.table
|
|
259
|
-
})),
|
|
260
|
-
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(),
|
|
276
|
+
})), Ye = (e, t, r) => n.gen(function* () {
|
|
277
|
+
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), u = (e) => {
|
|
261
278
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
262
279
|
return s.set(e, t), t;
|
|
263
280
|
}, d = (e, t) => {
|
|
264
281
|
if (t == null || a.has(e) || s.get(e)?.has(t)) return;
|
|
265
|
-
let n =
|
|
266
|
-
n.add(t),
|
|
267
|
-
},
|
|
282
|
+
let n = c.get(e) ?? /* @__PURE__ */ new Set();
|
|
283
|
+
n.add(t), c.set(e, n);
|
|
284
|
+
}, f = (e, t, n) => {
|
|
268
285
|
let r = u(e), i = a.has(e) ? void 0 : o.get(e) ?? (() => {
|
|
269
286
|
let t = /* @__PURE__ */ new Set();
|
|
270
287
|
return o.set(e, t), t;
|
|
@@ -273,25 +290,25 @@ var pe = (e) => {
|
|
|
273
290
|
r.add(e[t]), i?.add(e[t]);
|
|
274
291
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
275
292
|
};
|
|
276
|
-
},
|
|
277
|
-
let a =
|
|
278
|
-
return
|
|
293
|
+
}, p = (r, i) => {
|
|
294
|
+
let a = f(r, U(t, r), Je(t, r));
|
|
295
|
+
return l.runForEach(x(e, {
|
|
279
296
|
table: r,
|
|
280
|
-
pk:
|
|
297
|
+
pk: U(t, r),
|
|
281
298
|
...i === void 0 ? {} : { where: i }
|
|
282
299
|
}), (e) => n.sync(() => a(e)));
|
|
283
|
-
},
|
|
284
|
-
for (let e = 0; e < i &&
|
|
285
|
-
let e = [...
|
|
286
|
-
|
|
300
|
+
}, m = n.gen(function* () {
|
|
301
|
+
for (let e = 0; e < i && c.size > 0; e++) {
|
|
302
|
+
let e = [...c.entries()];
|
|
303
|
+
c.clear();
|
|
287
304
|
for (let [n, r] of e) {
|
|
288
305
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
289
|
-
e.length !== 0 && (yield*
|
|
306
|
+
e.length !== 0 && (yield* p(n, g(U(t, n), e)));
|
|
290
307
|
}
|
|
291
308
|
}
|
|
292
309
|
});
|
|
293
|
-
for (let [e, t] of Object.entries(r.seeds)) yield*
|
|
294
|
-
if (yield*
|
|
310
|
+
for (let [e, t] of Object.entries(r.seeds)) yield* p(e, t);
|
|
311
|
+
if (yield* m, r.children) {
|
|
295
312
|
let e = new Set(r.children === !0 ? [...s.keys()] : r.children.roots.filter((e) => s.has(e)));
|
|
296
313
|
for (let n = 0; n < i; n++) {
|
|
297
314
|
let n = !1, r = [];
|
|
@@ -301,29 +318,29 @@ var pe = (e) => {
|
|
|
301
318
|
refTable: e.references?.table
|
|
302
319
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
303
320
|
if (a.length === 0) continue;
|
|
304
|
-
let o = s.get(t)?.size ?? 0, c = a.map((e) =>
|
|
305
|
-
yield*
|
|
321
|
+
let o = s.get(t)?.size ?? 0, c = a.map((e) => g(e.col, [...s.get(e.refTable)]));
|
|
322
|
+
yield* p(t, c.length === 1 ? c[0] : v(...c)), (s.get(t)?.size ?? 0) > o && !e.has(t) && r.push(t), (s.get(t)?.size ?? 0) > o && (n = !0);
|
|
306
323
|
}
|
|
307
324
|
for (let t of r) e.add(t);
|
|
308
|
-
if (yield*
|
|
325
|
+
if (yield* m, !n) break;
|
|
309
326
|
}
|
|
310
327
|
}
|
|
311
328
|
return o;
|
|
312
|
-
}),
|
|
329
|
+
}), Xe = (e, t) => ({
|
|
313
330
|
tables: [...e.keys()],
|
|
314
331
|
predicateFor: (n) => {
|
|
315
332
|
let r = e.get(n);
|
|
316
|
-
return r ?
|
|
333
|
+
return r ? g(U(t, n), [...r]) : void 0;
|
|
317
334
|
}
|
|
318
|
-
}),
|
|
335
|
+
}), Ze = "tenantId", Qe = (e, t, r) => n.gen(function* () {
|
|
319
336
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
320
337
|
if (e.kind === "all") return {
|
|
321
|
-
tables: i,
|
|
338
|
+
tables: i.filter((e) => !z(e)),
|
|
322
339
|
predicateFor: () => void 0
|
|
323
340
|
};
|
|
324
341
|
if (e.kind === "tables") {
|
|
325
342
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
326
|
-
if (a.length > 0) return yield* n.fail(new
|
|
343
|
+
if (a.length > 0) return yield* n.fail(new V({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
327
344
|
let o = new Set(t);
|
|
328
345
|
return {
|
|
329
346
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -331,30 +348,30 @@ var pe = (e) => {
|
|
|
331
348
|
};
|
|
332
349
|
}
|
|
333
350
|
let o = e.tenantId;
|
|
334
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
335
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
336
|
-
if (s.length === 0) return yield* n.fail(new
|
|
337
|
-
let c = yield*
|
|
338
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
351
|
+
if (o === void 0 || o === "") return yield* n.fail(new V({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
352
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === Ze) ?? !1);
|
|
353
|
+
if (s.length === 0) return yield* n.fail(new V({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${Ze}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
354
|
+
let c = yield* Ye(r.store, t, {
|
|
355
|
+
seeds: Object.fromEntries(s.map((e) => [e, p(Ze, o)])),
|
|
339
356
|
pinned: s,
|
|
340
357
|
children: { roots: s }
|
|
341
|
-
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e),
|
|
358
|
+
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
342
359
|
return {
|
|
343
360
|
tables: [...s, ...l],
|
|
344
|
-
predicateFor: (e) =>
|
|
361
|
+
predicateFor: (e) => u.has(e) ? p(Ze, o) : c.has(e) ? g(U(t, e), [...c.get(e)]) : void 0
|
|
345
362
|
};
|
|
346
|
-
}),
|
|
363
|
+
}), $e = (e, t) => ({
|
|
347
364
|
list: t,
|
|
348
|
-
open: (t) =>
|
|
349
|
-
}),
|
|
350
|
-
write: (t, n, r) =>
|
|
365
|
+
open: (t) => D(e, t)
|
|
366
|
+
}), et = (e) => ({
|
|
367
|
+
write: (t, n, r) => O(e, t, n, r),
|
|
351
368
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
352
|
-
}),
|
|
369
|
+
}), tt = (e, t) => n.tryPromise({
|
|
353
370
|
try: async () => {
|
|
354
371
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
355
372
|
u.update(e), d += e.length, n(null, e);
|
|
356
373
|
} });
|
|
357
|
-
await o(await
|
|
374
|
+
await o(await k(e.stream), f, n(l));
|
|
358
375
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
359
376
|
try {
|
|
360
377
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -367,64 +384,64 @@ var pe = (e) => {
|
|
|
367
384
|
contentType: e.contentType
|
|
368
385
|
};
|
|
369
386
|
},
|
|
370
|
-
catch: (e) => new
|
|
387
|
+
catch: (e) => new B({
|
|
371
388
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
372
389
|
path: t
|
|
373
390
|
})
|
|
374
|
-
}),
|
|
375
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256),
|
|
391
|
+
}), nt = (e, t, r) => n.gen(function* () {
|
|
392
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u = ie("bundle", () => a(s)).pipe(l.tap((e) => n.sync(() => c.update(e))));
|
|
376
393
|
yield* e.write(r.key, u, {
|
|
377
394
|
contentType: r.contentType,
|
|
378
395
|
totalSize: r.size
|
|
379
396
|
});
|
|
380
|
-
let d =
|
|
381
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
397
|
+
let d = c.digest("hex");
|
|
398
|
+
if (d !== r.sha256) return yield* n.fail(new Ve({
|
|
382
399
|
what: `asset ${r.key}`,
|
|
383
400
|
expected: r.sha256,
|
|
384
401
|
actual: d
|
|
385
402
|
}));
|
|
386
|
-
}),
|
|
387
|
-
name:
|
|
388
|
-
file:
|
|
389
|
-
rowCount:
|
|
390
|
-
checksum:
|
|
391
|
-
columnTypes:
|
|
392
|
-
key:
|
|
393
|
-
value:
|
|
403
|
+
}), rt = () => typeof A.createZstdCompress == "function", it = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : rt() ? "zstd" : "gzip", at = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", ot = (e) => e === "zstd" ? A.createZstdCompress() : e === "gzip" ? A.createGzip() : new j(), st = (e) => e === "zstd" ? A.createZstdDecompress() : e === "gzip" ? A.createGunzip() : new j(), ct = 1, lt = c.Struct({
|
|
404
|
+
name: c.String,
|
|
405
|
+
file: c.String,
|
|
406
|
+
rowCount: c.Number,
|
|
407
|
+
checksum: c.String,
|
|
408
|
+
columnTypes: c.Record({
|
|
409
|
+
key: c.String,
|
|
410
|
+
value: c.String
|
|
394
411
|
}),
|
|
395
|
-
primaryKey:
|
|
396
|
-
}),
|
|
397
|
-
kind:
|
|
398
|
-
tenantId:
|
|
399
|
-
tables:
|
|
400
|
-
}),
|
|
401
|
-
count:
|
|
402
|
-
totalBytes:
|
|
403
|
-
index:
|
|
412
|
+
primaryKey: c.String
|
|
413
|
+
}), ut = c.Struct({
|
|
414
|
+
kind: c.Literal("all", "tenant", "tables"),
|
|
415
|
+
tenantId: c.optional(c.String),
|
|
416
|
+
tables: c.optional(c.Array(c.String))
|
|
417
|
+
}), dt = c.Struct({
|
|
418
|
+
count: c.Number,
|
|
419
|
+
totalBytes: c.Number,
|
|
420
|
+
index: c.optional(c.String)
|
|
404
421
|
});
|
|
405
|
-
|
|
406
|
-
key:
|
|
407
|
-
sha256:
|
|
408
|
-
size:
|
|
409
|
-
contentType:
|
|
422
|
+
c.Struct({
|
|
423
|
+
key: c.String,
|
|
424
|
+
sha256: c.String,
|
|
425
|
+
size: c.Number,
|
|
426
|
+
contentType: c.String
|
|
410
427
|
});
|
|
411
|
-
var
|
|
412
|
-
formatVersion:
|
|
413
|
-
createdAt:
|
|
414
|
-
source:
|
|
415
|
-
dialect:
|
|
416
|
-
schemaFingerprint:
|
|
428
|
+
var ft = c.Struct({
|
|
429
|
+
formatVersion: c.Number,
|
|
430
|
+
createdAt: c.String,
|
|
431
|
+
source: c.Struct({
|
|
432
|
+
dialect: c.String,
|
|
433
|
+
schemaFingerprint: c.String
|
|
417
434
|
}),
|
|
418
|
-
scope:
|
|
419
|
-
consistency:
|
|
420
|
-
compression:
|
|
421
|
-
tables:
|
|
422
|
-
assets:
|
|
423
|
-
masking:
|
|
424
|
-
policyId:
|
|
425
|
-
transformed:
|
|
435
|
+
scope: ut,
|
|
436
|
+
consistency: c.Literal("live", "snapshot"),
|
|
437
|
+
compression: c.Literal("zstd", "gzip", "none"),
|
|
438
|
+
tables: c.Array(lt),
|
|
439
|
+
assets: dt,
|
|
440
|
+
masking: c.optional(c.Struct({
|
|
441
|
+
policyId: c.optional(c.String),
|
|
442
|
+
transformed: c.Array(c.String)
|
|
426
443
|
}))
|
|
427
|
-
}),
|
|
444
|
+
}), pt = c.encodeSync(c.parseJson(ft)), mt = c.decodeUnknownSync(c.parseJson(ft)), W = "manifest.json", ht = "README.md", gt = "data", _t = "assets", vt = "assets/index.ndjson", yt = ".ledger.json", bt = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", xt = (e, t) => {
|
|
428
445
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
429
446
|
return `# Voltro data export
|
|
430
447
|
|
|
@@ -438,7 +455,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
438
455
|
| **Format version** | ${e.formatVersion} |
|
|
439
456
|
| **Consistency** | ${e.consistency} |
|
|
440
457
|
| **Compression** | ${e.compression} |
|
|
441
|
-
| **Scope** | ${
|
|
458
|
+
| **Scope** | ${bt(e.scope)} |
|
|
442
459
|
| **Tables** | ${e.tables.length} |
|
|
443
460
|
| **Rows (total)** | ${n} |
|
|
444
461
|
| **Assets** | ${a} |
|
|
@@ -463,39 +480,39 @@ voltro data import <this-directory>
|
|
|
463
480
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
464
481
|
\`\`\`
|
|
465
482
|
`;
|
|
466
|
-
},
|
|
483
|
+
}, St = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), Ct = a.exponential({
|
|
467
484
|
start: .01,
|
|
468
485
|
factor: 2,
|
|
469
486
|
count: 16
|
|
470
|
-
}),
|
|
487
|
+
}), wt = i.histogram("voltro_data_transfer_table_seconds", Ct, "Per-table export/import duration in seconds, tagged by phase."), G = (e, t) => {
|
|
471
488
|
if (e) try {
|
|
472
489
|
e(t);
|
|
473
490
|
} catch {}
|
|
474
|
-
},
|
|
491
|
+
}, Tt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(wt.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), St.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
475
492
|
table: t,
|
|
476
493
|
index: r,
|
|
477
494
|
tableCount: a,
|
|
478
495
|
phase: e
|
|
479
|
-
} })),
|
|
496
|
+
} })), Et = () => ({
|
|
480
497
|
tables: {},
|
|
481
498
|
assets: {}
|
|
482
|
-
}),
|
|
499
|
+
}), Dt = (e) => n.tryPromise(async () => {
|
|
483
500
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
484
501
|
return JSON.parse(t);
|
|
485
|
-
}).pipe(n.orElseSucceed(
|
|
502
|
+
}).pipe(n.orElseSucceed(Et)), K = (e, t) => n.tryPromise({
|
|
486
503
|
try: async () => {
|
|
487
504
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
488
505
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
489
506
|
},
|
|
490
|
-
catch: (t) => new
|
|
507
|
+
catch: (t) => new B({
|
|
491
508
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
492
509
|
path: e
|
|
493
510
|
})
|
|
494
|
-
}),
|
|
511
|
+
}), Ot = (e, t) => t in e.tables, kt = (e, t) => t in e.assets, At = /* @__PURE__ */ new Set([
|
|
495
512
|
"json",
|
|
496
513
|
"array",
|
|
497
514
|
"vector"
|
|
498
|
-
]),
|
|
515
|
+
]), jt = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", Mt = (e) => Buffer.from(e).toString("base64"), Nt = (e) => new Uint8Array(Buffer.from(e, "base64")), Pt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), Ft = (e, t) => e == null ? null : t !== void 0 && At.has(t) ? e : t === "bytes" ? Mt(e) : t === "timestamp" || t === "date" ? Pt(e) : typeof e == "bigint" ? {
|
|
499
516
|
$v: "bigint",
|
|
500
517
|
d: e.toString()
|
|
501
518
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -506,30 +523,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
506
523
|
d: e.toISOString()
|
|
507
524
|
} : e instanceof Uint8Array ? {
|
|
508
525
|
$v: "bytes",
|
|
509
|
-
d:
|
|
510
|
-
} : e,
|
|
526
|
+
d: Mt(e)
|
|
527
|
+
} : e, It = (e) => {
|
|
511
528
|
switch (e.$v) {
|
|
512
529
|
case "bigint": return BigInt(e.d);
|
|
513
530
|
case "number": return Number(e.d);
|
|
514
531
|
case "date": return new Date(e.d);
|
|
515
|
-
case "bytes": return
|
|
532
|
+
case "bytes": return Nt(e.d);
|
|
516
533
|
default: return e;
|
|
517
534
|
}
|
|
518
|
-
},
|
|
535
|
+
}, Lt = (e, t) => e == null ? null : t !== void 0 && At.has(t) ? e : jt(e) ? It(e) : t === "bytes" ? Nt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Rt = (e, t) => {
|
|
519
536
|
let n = {};
|
|
520
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
537
|
+
for (let [r, i] of Object.entries(e)) n[r] = Ft(i, t[r]);
|
|
521
538
|
return n;
|
|
522
|
-
},
|
|
539
|
+
}, zt = (e, t) => {
|
|
523
540
|
let n = {};
|
|
524
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
541
|
+
for (let [r, i] of Object.entries(e)) n[r] = Lt(i, t[r]);
|
|
525
542
|
return n;
|
|
526
|
-
},
|
|
527
|
-
try: () =>
|
|
528
|
-
catch: (e) => new
|
|
543
|
+
}, Bt = new TextEncoder(), Vt = (e, t) => e.pipe(l.map((e) => Bt.encode(`${JSON.stringify(Rt(e, t))}\n`))), Ht = (e, t, r) => e.pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.mapEffect((e) => n.try({
|
|
544
|
+
try: () => zt(JSON.parse(e), r),
|
|
545
|
+
catch: (e) => new Be({
|
|
529
546
|
table: t,
|
|
530
547
|
reason: String(e?.message ?? e)
|
|
531
548
|
})
|
|
532
|
-
}))),
|
|
549
|
+
}))), Ut = (e, t) => {
|
|
533
550
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
534
551
|
for (let e of t.tables) {
|
|
535
552
|
if (!n.has(e.name)) continue;
|
|
@@ -555,42 +572,42 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
555
572
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
556
573
|
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(q(t.cause)) : n.push(r), n.length === 0) {
|
|
557
574
|
let t = String(e ?? "").trim();
|
|
558
|
-
return t === "" || t === "[object Object]" ?
|
|
575
|
+
return t === "" || t === "[object Object]" ? Wt(e) : t;
|
|
559
576
|
}
|
|
560
577
|
return n.join(": ");
|
|
561
|
-
},
|
|
578
|
+
}, Wt = (e) => {
|
|
562
579
|
try {
|
|
563
580
|
let t = JSON.stringify(e);
|
|
564
581
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
565
582
|
} catch {
|
|
566
583
|
return "unknown error (not serialisable)";
|
|
567
584
|
}
|
|
568
|
-
},
|
|
585
|
+
}, Gt = (e, t) => {
|
|
569
586
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
570
587
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
571
588
|
return r;
|
|
572
|
-
},
|
|
589
|
+
}, Kt = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), qt = (e, t) => {
|
|
573
590
|
if (t.length === 0) return e;
|
|
574
591
|
let n = { ...e };
|
|
575
592
|
for (let e of t) delete n[e];
|
|
576
593
|
return n;
|
|
577
|
-
},
|
|
594
|
+
}, Jt = (e, t) => {
|
|
578
595
|
let n = e.tables.find((e) => e.name === t);
|
|
579
596
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
580
597
|
let r = n.columns.find((e) => e.type === "id");
|
|
581
598
|
if (r !== void 0) return { column: r.name };
|
|
582
599
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
583
600
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
584
|
-
},
|
|
601
|
+
}, Yt = (e, t, r) => n.tryPromise({
|
|
585
602
|
try: async () => {
|
|
586
|
-
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable:
|
|
603
|
+
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
587
604
|
d.update(e);
|
|
588
605
|
let r = e;
|
|
589
606
|
for (let e = 0; e < r.length; e++) r[e] === 10 && f++;
|
|
590
607
|
n(null, e);
|
|
591
|
-
} }), m =
|
|
608
|
+
} }), m = c.fromWeb(l.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
592
609
|
try {
|
|
593
|
-
await s(m, p,
|
|
610
|
+
await s(m, p, ot(r), n(h)), await i.rename(h, t);
|
|
594
611
|
} catch (e) {
|
|
595
612
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
596
613
|
}
|
|
@@ -599,25 +616,25 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
599
616
|
checksum: d.digest("hex")
|
|
600
617
|
};
|
|
601
618
|
},
|
|
602
|
-
catch: (e) => new
|
|
619
|
+
catch: (e) => new B({
|
|
603
620
|
reason: `write table failed: ${q(e)}`,
|
|
604
621
|
path: t
|
|
605
622
|
})
|
|
606
|
-
}),
|
|
623
|
+
}), Xt = (e, t) => n.tryPromise({
|
|
607
624
|
try: async () => {
|
|
608
625
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
609
626
|
},
|
|
610
|
-
catch: (t) => new
|
|
627
|
+
catch: (t) => new B({
|
|
611
628
|
reason: `append failed: ${q(t)}`,
|
|
612
629
|
path: e
|
|
613
630
|
})
|
|
614
|
-
}),
|
|
631
|
+
}), Zt = (e, t) => n.gen(function* () {
|
|
615
632
|
let r = [], i = t.ordered.length;
|
|
616
633
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
617
|
-
let o = t.ordered[a], s =
|
|
618
|
-
if ("refusal" in
|
|
619
|
-
let u =
|
|
620
|
-
if (
|
|
634
|
+
let o = t.ordered[a], s = Gt(t.snapshot, o), c = Jt(t.snapshot, o);
|
|
635
|
+
if ("refusal" in c) return yield* n.fail(new V({ reason: `export: ${c.refusal}` }));
|
|
636
|
+
let u = c.column, d = `${gt}/${o}.ndjson${at(t.compression)}`;
|
|
637
|
+
if (Ot(t.ledger, o)) {
|
|
621
638
|
let e = t.ledger.tables[o];
|
|
622
639
|
G(t.onProgress, {
|
|
623
640
|
phase: "export",
|
|
@@ -645,149 +662,149 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
645
662
|
event: "start",
|
|
646
663
|
rowsDone: 0
|
|
647
664
|
});
|
|
648
|
-
let f = t.resolved.predicateFor(o), p =
|
|
665
|
+
let f = t.resolved.predicateFor(o), p = x(e, {
|
|
649
666
|
table: o,
|
|
650
667
|
pk: u,
|
|
651
668
|
chunkSize: t.chunkSize,
|
|
652
669
|
retry: t.retry,
|
|
653
670
|
...f === void 0 ? {} : { where: f }
|
|
654
|
-
}), m =
|
|
671
|
+
}), m = Kt(t.snapshot, o), h = m.length === 0 ? p : p.pipe(l.map((e) => qt(e, m))), g = t.mask ? h.pipe(l.map((e) => Ie(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* Tt("export", o, a, i, (e) => e.rowCount, Yt(Vt(g, s), t.bundleFile(d), t.compression));
|
|
655
672
|
t.ledger.tables[o] = {
|
|
656
|
-
rowCount:
|
|
657
|
-
checksum:
|
|
673
|
+
rowCount: _,
|
|
674
|
+
checksum: v
|
|
658
675
|
}, yield* K(t.ledgerPath, t.ledger), G(t.onProgress, {
|
|
659
676
|
phase: "export",
|
|
660
677
|
table: o,
|
|
661
678
|
index: a,
|
|
662
679
|
tableCount: i,
|
|
663
680
|
event: "done",
|
|
664
|
-
rowsDone:
|
|
665
|
-
total:
|
|
681
|
+
rowsDone: _,
|
|
682
|
+
total: _
|
|
666
683
|
}), r.push({
|
|
667
684
|
name: o,
|
|
668
685
|
file: d,
|
|
669
|
-
rowCount:
|
|
670
|
-
checksum:
|
|
686
|
+
rowCount: _,
|
|
687
|
+
checksum: v,
|
|
671
688
|
columnTypes: s,
|
|
672
689
|
primaryKey: u
|
|
673
690
|
});
|
|
674
691
|
}
|
|
675
692
|
return r;
|
|
676
|
-
}),
|
|
693
|
+
}), Qt = (e) => ({
|
|
677
694
|
_tag: "RawSqlFragment",
|
|
678
695
|
strings: [e],
|
|
679
696
|
values: []
|
|
680
|
-
}),
|
|
697
|
+
}), $t = (t, i, a) => n.tryPromise({
|
|
681
698
|
try: () => t.transactional(async (t) => {
|
|
682
|
-
if (t.raw) for (let e of
|
|
699
|
+
if (t.raw) for (let e of ye(i)) await t.raw(Qt(e));
|
|
683
700
|
let o = await n.runPromiseExit(a(t));
|
|
684
701
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
685
702
|
return o.value;
|
|
686
703
|
}),
|
|
687
|
-
catch: (e) => e instanceof
|
|
688
|
-
}),
|
|
689
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
704
|
+
catch: (e) => e instanceof B ? e : new B({ reason: `snapshot export failed: ${q(e)}` })
|
|
705
|
+
}), en = (e) => n.gen(function* () {
|
|
706
|
+
let t = e.scope ?? { kind: "all" }, r = it(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, o = s.exponential("50 millis").pipe(s.jittered, s.intersect(s.recurs(a))), c = yield* n.promise(() => import("node:path")), u = (t) => c.join(e.outDir, t);
|
|
690
707
|
yield* n.tryPromise({
|
|
691
708
|
try: async () => {
|
|
692
709
|
let t = await import("node:fs/promises");
|
|
693
|
-
await t.mkdir(u(
|
|
710
|
+
await t.mkdir(u(gt), { recursive: !0 }), e.assets && await t.mkdir(u(_t), { recursive: !0 });
|
|
694
711
|
},
|
|
695
|
-
catch: (t) => new
|
|
712
|
+
catch: (t) => new B({
|
|
696
713
|
reason: `mkdir failed: ${q(t)}`,
|
|
697
714
|
path: e.outDir
|
|
698
715
|
})
|
|
699
716
|
});
|
|
700
|
-
let d = u(
|
|
717
|
+
let d = u(yt), f = yield* Dt(d), p = e.subset ? Xe(yield* Ye(e.store, e.snapshot, e.subset), e.snapshot) : yield* Qe(t, e.snapshot, {
|
|
701
718
|
store: e.store,
|
|
702
719
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
703
|
-
}), m =
|
|
720
|
+
}), m = Ut(p.tables, e.snapshot), h = e.consistency ?? "live", g, _;
|
|
704
721
|
if (e.masking) {
|
|
705
|
-
let t = e.classification ??
|
|
722
|
+
let t = e.classification ?? xe(e.snapshot), r = Ne(m.map((t) => ({
|
|
706
723
|
name: t,
|
|
707
724
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
708
725
|
name: e.name,
|
|
709
726
|
type: e.type
|
|
710
727
|
}))
|
|
711
728
|
})), t, e.masking);
|
|
712
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
729
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new Ge({
|
|
713
730
|
unclassified: r.unclassified,
|
|
714
731
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
715
732
|
}));
|
|
716
|
-
|
|
733
|
+
_ = {
|
|
717
734
|
actions: r.actions,
|
|
718
735
|
seed: e.masking.seed
|
|
719
736
|
};
|
|
720
737
|
let i = Object.entries(r.actions).filter(([, e]) => e !== "keep").map(([e]) => e);
|
|
721
|
-
|
|
738
|
+
g = {
|
|
722
739
|
...e.masking.id ? { policyId: e.masking.id } : {},
|
|
723
740
|
transformed: i
|
|
724
741
|
};
|
|
725
742
|
}
|
|
726
|
-
let
|
|
743
|
+
let v = {
|
|
727
744
|
snapshot: e.snapshot,
|
|
728
745
|
ordered: m,
|
|
729
746
|
resolved: p,
|
|
730
747
|
compression: r,
|
|
731
748
|
chunkSize: i,
|
|
732
|
-
retry:
|
|
749
|
+
retry: o,
|
|
733
750
|
ledger: f,
|
|
734
751
|
ledgerPath: d,
|
|
735
752
|
bundleFile: u,
|
|
736
|
-
...
|
|
753
|
+
..._ ? { mask: _ } : {},
|
|
737
754
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
738
|
-
},
|
|
755
|
+
}, y = h === "snapshot" ? yield* $t(e.store, e.dialect, (e) => Zt(e, v)) : yield* Zt(e.store, v), b = 0, x = 0;
|
|
739
756
|
if (e.assets) {
|
|
740
|
-
let t = u(
|
|
741
|
-
yield*
|
|
757
|
+
let t = u(vt);
|
|
758
|
+
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => tt(e, u(_t))), n.flatMap((e) => {
|
|
742
759
|
let i = {
|
|
743
760
|
key: r.key,
|
|
744
761
|
sha256: e.sha256,
|
|
745
762
|
size: e.size,
|
|
746
763
|
contentType: e.contentType
|
|
747
764
|
};
|
|
748
|
-
return
|
|
749
|
-
}))).pipe(n.mapError((e) => e instanceof
|
|
765
|
+
return b++, x += e.size, f.assets[r.key] = !0, Xt(t, JSON.stringify(i)).pipe(n.andThen(K(d, f)));
|
|
766
|
+
}))).pipe(n.mapError((e) => e instanceof B ? e : new B({ reason: `asset export failed: ${String(e)}` })));
|
|
750
767
|
}
|
|
751
|
-
let
|
|
768
|
+
let S = {
|
|
752
769
|
formatVersion: 1,
|
|
753
770
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
754
771
|
source: {
|
|
755
772
|
dialect: e.dialect,
|
|
756
|
-
schemaFingerprint:
|
|
773
|
+
schemaFingerprint: C(e.snapshot)
|
|
757
774
|
},
|
|
758
775
|
scope: t,
|
|
759
776
|
consistency: h,
|
|
760
777
|
compression: r,
|
|
761
|
-
tables:
|
|
762
|
-
assets:
|
|
763
|
-
count:
|
|
764
|
-
totalBytes:
|
|
765
|
-
index:
|
|
778
|
+
tables: y,
|
|
779
|
+
assets: b > 0 ? {
|
|
780
|
+
count: b,
|
|
781
|
+
totalBytes: x,
|
|
782
|
+
index: vt
|
|
766
783
|
} : {
|
|
767
784
|
count: 0,
|
|
768
785
|
totalBytes: 0
|
|
769
786
|
},
|
|
770
|
-
...
|
|
787
|
+
...g ? { masking: g } : {}
|
|
771
788
|
};
|
|
772
789
|
return yield* n.tryPromise({
|
|
773
790
|
try: async () => {
|
|
774
|
-
await (await import("node:fs/promises")).writeFile(u(
|
|
791
|
+
await (await import("node:fs/promises")).writeFile(u(W), pt(S));
|
|
775
792
|
},
|
|
776
|
-
catch: (e) => new
|
|
793
|
+
catch: (e) => new B({
|
|
777
794
|
reason: `manifest write failed: ${q(e)}`,
|
|
778
|
-
path:
|
|
795
|
+
path: W
|
|
779
796
|
})
|
|
780
797
|
}), yield* n.tryPromise({
|
|
781
798
|
try: async () => {
|
|
782
799
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
783
|
-
await e.writeFile(u(
|
|
800
|
+
await e.writeFile(u(ht), xt(S, { hostname: t.hostname() }));
|
|
784
801
|
},
|
|
785
|
-
catch: (e) => new
|
|
802
|
+
catch: (e) => new B({
|
|
786
803
|
reason: `readme write failed: ${q(e)}`,
|
|
787
|
-
path:
|
|
804
|
+
path: ht
|
|
788
805
|
})
|
|
789
|
-
}),
|
|
790
|
-
}),
|
|
806
|
+
}), S;
|
|
807
|
+
}), tn = (e, t) => {
|
|
791
808
|
let n = new Set(t), r = [];
|
|
792
809
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
793
810
|
let i = e.references?.table;
|
|
@@ -798,7 +815,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
798
815
|
});
|
|
799
816
|
}
|
|
800
817
|
return r;
|
|
801
|
-
},
|
|
818
|
+
}, nn = (e) => {
|
|
802
819
|
let t = /* @__PURE__ */ new Map();
|
|
803
820
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
804
821
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -806,7 +823,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
806
823
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
807
824
|
});
|
|
808
825
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
809
|
-
},
|
|
826
|
+
}, rn = (e) => {
|
|
810
827
|
switch (e) {
|
|
811
828
|
case "id":
|
|
812
829
|
case "text":
|
|
@@ -825,12 +842,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
825
842
|
case "raw": return "raw";
|
|
826
843
|
default: return "string";
|
|
827
844
|
}
|
|
828
|
-
},
|
|
845
|
+
}, an = {
|
|
829
846
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
830
847
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
831
848
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
832
|
-
},
|
|
833
|
-
let i =
|
|
849
|
+
}, on = (e, t, n, r) => {
|
|
850
|
+
let i = rn(n);
|
|
834
851
|
if (i === "raw") return {
|
|
835
852
|
table: e,
|
|
836
853
|
column: t,
|
|
@@ -838,7 +855,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
838
855
|
severity: "block",
|
|
839
856
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
840
857
|
};
|
|
841
|
-
let a =
|
|
858
|
+
let a = an[i];
|
|
842
859
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
843
860
|
table: e,
|
|
844
861
|
column: t,
|
|
@@ -852,15 +869,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
852
869
|
severity: "block",
|
|
853
870
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
854
871
|
};
|
|
855
|
-
},
|
|
872
|
+
}, sn = (e, t, n) => {
|
|
856
873
|
if (t === n) return [];
|
|
857
874
|
let r = [];
|
|
858
875
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
859
|
-
let a =
|
|
876
|
+
let a = on(t.name, e, i, n);
|
|
860
877
|
a && r.push(a);
|
|
861
878
|
}
|
|
862
879
|
return r;
|
|
863
|
-
},
|
|
880
|
+
}, cn = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, ln = (e) => {
|
|
864
881
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
865
882
|
switch (e.kind) {
|
|
866
883
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -869,37 +886,164 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
869
886
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
870
887
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
871
888
|
}
|
|
872
|
-
},
|
|
889
|
+
}, un = (e) => {
|
|
873
890
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
874
891
|
return t.length > 0 ? t.join("/") : void 0;
|
|
875
|
-
},
|
|
892
|
+
}, dn = "Failed to execute statement", fn = (e) => {
|
|
876
893
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
877
894
|
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
878
|
-
return t === `SqlError: ${
|
|
879
|
-
},
|
|
895
|
+
return t === `SqlError: ${dn}` ? dn : t;
|
|
896
|
+
}, pn = (e) => {
|
|
880
897
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
881
898
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
882
899
|
unclassified: t,
|
|
883
900
|
raw: n
|
|
884
|
-
}), i =
|
|
885
|
-
if (a !== void 0) return r(`${
|
|
886
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0,
|
|
887
|
-
return r(
|
|
888
|
-
},
|
|
889
|
-
let t =
|
|
901
|
+
}), i = m(e), a = d(i), o = un(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
902
|
+
if (a !== void 0) return r(`${ln(a)}${s}`, !1);
|
|
903
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = fn(typeof l == "string" && l.length > 0 ? l : String(e)), f = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, p = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, h = p !== void 0 && /[←‖]/.test(p) ? ` (via ${p})` : "";
|
|
904
|
+
return r(f === dn || f.length === 0 ? `the database refused the write and the driver gave no detail${s}${h}` : `${f}${s}`, f === dn || f.length === 0);
|
|
905
|
+
}, mn = (e) => pn(e).reason, hn = (e) => {
|
|
906
|
+
let t = m(e), n = d(t);
|
|
890
907
|
if (n?.kind === "foreignKeyInUse") {
|
|
891
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
908
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = un(t);
|
|
892
909
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
893
910
|
}
|
|
894
|
-
return
|
|
895
|
-
},
|
|
911
|
+
return mn(e);
|
|
912
|
+
}, gn = "_voltro_staging_", J = (e) => `${gn}${e}`, _n = (e) => typeof e.run == "function", vn = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
913
|
+
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
914
|
+
written before the first delete is what covers that case.`, yn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, Y = (e, t) => {
|
|
915
|
+
if (!yn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
916
|
+
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
917
|
+
}, bn = (e, t) => {
|
|
918
|
+
let n = Y(e, t), r = Y(J(e), t);
|
|
919
|
+
switch (t) {
|
|
920
|
+
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
921
|
+
case "mysql":
|
|
922
|
+
case "mariadb": return `CREATE TABLE ${r} LIKE ${n}`;
|
|
923
|
+
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
924
|
+
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
925
|
+
}
|
|
926
|
+
}, xn = (e, t) => {
|
|
927
|
+
let n = Y(J(e), t);
|
|
928
|
+
return t === "mssql" ? `IF OBJECT_ID('${J(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
929
|
+
}, Sn = (e, t) => {
|
|
930
|
+
let n = t === "mysql" || t === "mariadb", r = [];
|
|
931
|
+
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
932
|
+
for (let n of [...e].reverse()) r.push(`DELETE FROM ${Y(n.name, t)}`);
|
|
933
|
+
for (let n of e) {
|
|
934
|
+
if (n.columns.length === 0) throw Error(`refusing to build a swap for '${n.name}' with no columns: an empty list would have to mean \`SELECT *\`, which is the shape that fails on a generated column.`);
|
|
935
|
+
let e = n.columns.map((e) => Y(e, t)).join(", ");
|
|
936
|
+
r.push(`INSERT INTO ${Y(n.name, t)} (${e}) SELECT ${e} FROM ${Y(J(n.name), t)}`);
|
|
937
|
+
}
|
|
938
|
+
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
939
|
+
}, Cn = (e, t) => {
|
|
940
|
+
let n = [], r = () => {
|
|
941
|
+
for (let e of n) t.unregisterTable(e);
|
|
942
|
+
};
|
|
943
|
+
try {
|
|
944
|
+
for (let r of e) {
|
|
945
|
+
let e = t.getTable(r);
|
|
946
|
+
if (e === void 0) continue;
|
|
947
|
+
let i = J(r);
|
|
948
|
+
t.registerTable({
|
|
949
|
+
...e,
|
|
950
|
+
tableName: i,
|
|
951
|
+
isReactive: !1
|
|
952
|
+
}), n.push(i);
|
|
953
|
+
}
|
|
954
|
+
} catch (e) {
|
|
955
|
+
throw r(), e;
|
|
956
|
+
}
|
|
957
|
+
return {
|
|
958
|
+
staged: n,
|
|
959
|
+
undo: r
|
|
960
|
+
};
|
|
961
|
+
}, wn = (e, t) => {
|
|
962
|
+
let n = new Set(t), r = [];
|
|
963
|
+
for (let t of e.tables) {
|
|
964
|
+
if (!n.has(t.name)) continue;
|
|
965
|
+
let e = t.primaryKey?.[0] ?? "id";
|
|
966
|
+
for (let i of t.columns) {
|
|
967
|
+
let a = i.references;
|
|
968
|
+
a === void 0 || !n.has(a.table) || r.push({
|
|
969
|
+
from: t.name,
|
|
970
|
+
column: i.name,
|
|
971
|
+
to: a.table,
|
|
972
|
+
toColumn: a.column,
|
|
973
|
+
primaryKey: e
|
|
974
|
+
});
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
return r;
|
|
978
|
+
}, Tn = (e, t) => {
|
|
979
|
+
let n = Y(J(e.from), t), r = Y(J(e.to), t), i = Y(e.column, t), a = Y(e.toColumn, t);
|
|
980
|
+
return `SELECT ${Y(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
981
|
+
}, En = (e) => {
|
|
982
|
+
if (e.length === 0) return;
|
|
983
|
+
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
984
|
+
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
985
|
+
}, Dn = async (e, t) => {
|
|
986
|
+
t.length !== 0 && await e.run(N.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
987
|
+
}, On = async (e, t, r) => {
|
|
988
|
+
let i = Sn(t, r);
|
|
989
|
+
i.length !== 0 && await e.run(N.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
990
|
+
}, kn = async (e, t, r) => {
|
|
991
|
+
let i = [];
|
|
992
|
+
for (let a of t) {
|
|
993
|
+
let t = await e.run(N.SqlClient.pipe(n.flatMap((e) => e.unsafe(Tn(a, r)))));
|
|
994
|
+
for (let e of t) i.push({
|
|
995
|
+
table: a.from,
|
|
996
|
+
primaryKey: String(e.pk ?? ""),
|
|
997
|
+
column: a.column,
|
|
998
|
+
value: e.value,
|
|
999
|
+
references: a.to
|
|
1000
|
+
});
|
|
1001
|
+
}
|
|
1002
|
+
return i;
|
|
1003
|
+
}, An = (e, t) => {
|
|
1004
|
+
let n = new Map(t.map((e, t) => [e, t]));
|
|
1005
|
+
return e.filter((e) => {
|
|
1006
|
+
if (e.from === e.to) return !1;
|
|
1007
|
+
let t = n.get(e.from), r = n.get(e.to);
|
|
1008
|
+
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1009
|
+
});
|
|
1010
|
+
}, jn = (e) => e === "mysql" || e === "mariadb", Mn = (e) => n.gen(function* () {
|
|
1011
|
+
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1012
|
+
if (e.snapshot === void 0) return yield* n.fail(new B({
|
|
1013
|
+
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
1014
|
+
path: e.dir
|
|
1015
|
+
}));
|
|
1016
|
+
let t = {
|
|
1017
|
+
...e.snapshot,
|
|
1018
|
+
tables: e.snapshot.tables.filter((t) => e.tables.includes(t.name))
|
|
1019
|
+
};
|
|
1020
|
+
if (t.tables.length === 0) return;
|
|
1021
|
+
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1022
|
+
so it survives a run whose process does not.`);
|
|
1023
|
+
let r = yield* en({
|
|
1024
|
+
store: e.store,
|
|
1025
|
+
snapshot: t,
|
|
1026
|
+
dialect: e.dialect ?? "unknown",
|
|
1027
|
+
outDir: e.dir,
|
|
1028
|
+
scope: { kind: "all" },
|
|
1029
|
+
compression: "none"
|
|
1030
|
+
}).pipe(n.mapError((t) => new B({
|
|
1031
|
+
reason: `rollback capture failed, so the import did not start and the target is untouched: ${t.reason ?? String(t)}\n Fix the capture, or re-run with \`--no-rollback\` to proceed without one.`,
|
|
1032
|
+
path: e.dir
|
|
1033
|
+
})));
|
|
1034
|
+
return {
|
|
1035
|
+
dir: e.dir,
|
|
1036
|
+
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1037
|
+
tables: r.tables.length
|
|
1038
|
+
};
|
|
1039
|
+
}), Nn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, Pn = (e, t) => e.tables.find((e) => e.name === t), Fn = (e, t) => new Map((Pn(e, t)?.columns ?? []).map((e) => [e.name, e])), In = (e, t) => {
|
|
896
1040
|
let n = [];
|
|
897
1041
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
898
|
-
if (
|
|
1042
|
+
if (Pn(t, r.name) === void 0) {
|
|
899
1043
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
900
1044
|
continue;
|
|
901
1045
|
}
|
|
902
|
-
let e =
|
|
1046
|
+
let e = Fn(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
903
1047
|
for (let [t, a] of i) {
|
|
904
1048
|
let i = e.get(t);
|
|
905
1049
|
if (i === void 0) {
|
|
@@ -911,43 +1055,62 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
911
1055
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
912
1056
|
}
|
|
913
1057
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
914
|
-
},
|
|
915
|
-
let r =
|
|
1058
|
+
}, Ln = (e, t, n) => {
|
|
1059
|
+
let r = C(t);
|
|
916
1060
|
if (r === e || n === void 0) return {
|
|
917
1061
|
drifted: !1,
|
|
918
1062
|
bundleFingerprint: e,
|
|
919
1063
|
targetFingerprint: r,
|
|
920
1064
|
diff: []
|
|
921
1065
|
};
|
|
922
|
-
let i =
|
|
1066
|
+
let i = In(n, t);
|
|
923
1067
|
return {
|
|
924
1068
|
drifted: i.length > 0,
|
|
925
1069
|
bundleFingerprint: e,
|
|
926
1070
|
targetFingerprint: r,
|
|
927
1071
|
diff: i
|
|
928
1072
|
};
|
|
929
|
-
},
|
|
1073
|
+
}, Rn = 200, zn = /* @__PURE__ */ new Set([
|
|
1074
|
+
"BundleError",
|
|
1075
|
+
"CodecError",
|
|
1076
|
+
"IntegrityError",
|
|
1077
|
+
"CrossDialectError",
|
|
1078
|
+
"ImportModeError",
|
|
1079
|
+
"RowsRefusedError",
|
|
1080
|
+
"SchemaDriftError",
|
|
1081
|
+
"StorageError"
|
|
1082
|
+
]), Bn = (t) => {
|
|
1083
|
+
let n = o.isFiberFailure(t) ? [...e.failures(t[o.FiberFailureCauseId]), ...e.defects(t[o.FiberFailureCauseId])] : [t];
|
|
1084
|
+
for (let e of n) {
|
|
1085
|
+
let t = e?._tag;
|
|
1086
|
+
if (typeof t == "string" && zn.has(t)) return e;
|
|
1087
|
+
if (o.isFiberFailure(e)) {
|
|
1088
|
+
let t = Bn(e);
|
|
1089
|
+
if (t) return t;
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
}, Vn = (t) => [...e.failures(t)][0] ?? Error(e.pretty(t)), Hn = (e) => n.tryPromise({
|
|
930
1093
|
try: async () => {
|
|
931
1094
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
932
|
-
return
|
|
1095
|
+
return mt(await t.readFile(n.join(e, W), "utf8"));
|
|
933
1096
|
},
|
|
934
|
-
catch: (e) => new
|
|
1097
|
+
catch: (e) => new B({
|
|
935
1098
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
936
|
-
path:
|
|
1099
|
+
path: W
|
|
937
1100
|
})
|
|
938
|
-
}),
|
|
939
|
-
let
|
|
940
|
-
if (
|
|
941
|
-
if (
|
|
942
|
-
let
|
|
943
|
-
if (r.length > 0) return yield* n.fail(new
|
|
1101
|
+
}), Un = (e) => n.gen(function* () {
|
|
1102
|
+
let t = yield* n.promise(() => import("node:path")), i = yield* Hn(e.bundleDir);
|
|
1103
|
+
if (i.formatVersion !== 1) return yield* n.fail(new B({ reason: `unsupported bundle formatVersion ${i.formatVersion} (expected 1)` }));
|
|
1104
|
+
if (e.tables !== void 0) {
|
|
1105
|
+
let t = new Set(i.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
1106
|
+
if (r.length > 0) return yield* n.fail(new B({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...t].sort().join(", ")}.` }));
|
|
944
1107
|
}
|
|
945
|
-
let
|
|
946
|
-
...
|
|
947
|
-
tables:
|
|
1108
|
+
let a = e.tables === void 0 ? void 0 : new Set(e.tables), o = a === void 0 ? i : {
|
|
1109
|
+
...i,
|
|
1110
|
+
tables: i.tables.filter((e) => a.has(e.name))
|
|
948
1111
|
};
|
|
949
|
-
if (
|
|
950
|
-
let
|
|
1112
|
+
if (e.targetSnapshot) {
|
|
1113
|
+
let t = { tables: o.tables.map((e) => ({
|
|
951
1114
|
name: e.name,
|
|
952
1115
|
indexes: [],
|
|
953
1116
|
columns: Object.entries(e.columnTypes).map(([e, t]) => ({
|
|
@@ -957,94 +1120,94 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
957
1120
|
unique: !1,
|
|
958
1121
|
hasDefault: !1
|
|
959
1122
|
}))
|
|
960
|
-
})) }, r =
|
|
1123
|
+
})) }, r = Ln(o.source.schemaFingerprint, e.targetSnapshot, t);
|
|
961
1124
|
if (r.drifted) {
|
|
962
|
-
if (!
|
|
1125
|
+
if (!e.force) return yield* n.fail(new We({
|
|
963
1126
|
bundleFingerprint: r.bundleFingerprint,
|
|
964
1127
|
targetFingerprint: r.targetFingerprint,
|
|
965
1128
|
diff: r.diff
|
|
966
1129
|
}));
|
|
967
|
-
|
|
1130
|
+
e.onWarn?.(`schema drift: the target schema (fingerprint ${r.targetFingerprint.slice(0, 16)}) differs from the bundle's source (${r.bundleFingerprint.slice(0, 16)}); importing anyway (--force). ${r.diff.join("; ")}`);
|
|
968
1131
|
}
|
|
969
1132
|
}
|
|
970
|
-
if (
|
|
971
|
-
let
|
|
972
|
-
if (
|
|
973
|
-
sourceDialect:
|
|
974
|
-
targetDialect:
|
|
975
|
-
blocking:
|
|
1133
|
+
if (e.targetDialect && e.targetDialect !== o.source.dialect) {
|
|
1134
|
+
let t = sn(o.tables, o.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1135
|
+
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new qe({
|
|
1136
|
+
sourceDialect: o.source.dialect,
|
|
1137
|
+
targetDialect: e.targetDialect,
|
|
1138
|
+
blocking: t.map(cn)
|
|
976
1139
|
}));
|
|
977
1140
|
}
|
|
978
|
-
let
|
|
979
|
-
if (
|
|
980
|
-
mode:
|
|
981
|
-
reason: `replace refuses a partial bundle (scope='${
|
|
1141
|
+
let s = e.verify ?? !0, c = e.mode ?? "upsert", d = e.atomic ?? c === "replace", p = e.onConflict ?? "skip", m = e.ledgerPath ?? t.join(e.bundleDir, ".import..ledger.json"), g = yield* Dt(m);
|
|
1142
|
+
if (c === "replace" && o.scope.kind !== "all") return yield* n.fail(new Ke({
|
|
1143
|
+
mode: c,
|
|
1144
|
+
reason: `replace refuses a partial bundle (scope='${o.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
982
1145
|
}));
|
|
983
|
-
if (
|
|
984
|
-
let
|
|
985
|
-
try: () =>
|
|
986
|
-
catch: (e) => new
|
|
987
|
-
}) :
|
|
988
|
-
for (let
|
|
989
|
-
if (!a.has(
|
|
1146
|
+
if (c === "replace") {
|
|
1147
|
+
let t = o.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1148
|
+
try: () => e.store.incomingForeignKeys(t),
|
|
1149
|
+
catch: (e) => new B({ reason: `foreign-key pre-flight failed: ${hn(e)}` })
|
|
1150
|
+
}) : e.targetSnapshot === void 0 ? [] : tn(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1151
|
+
for (let t of r) {
|
|
1152
|
+
if (!a.has(t.from)) {
|
|
990
1153
|
let r = yield* n.tryPromise({
|
|
991
|
-
try: () =>
|
|
992
|
-
table:
|
|
1154
|
+
try: () => e.store.query({
|
|
1155
|
+
table: t.from,
|
|
993
1156
|
predicate: void 0,
|
|
994
1157
|
order: [],
|
|
995
1158
|
take: void 0,
|
|
996
1159
|
skip: void 0,
|
|
997
1160
|
projection: []
|
|
998
1161
|
}),
|
|
999
|
-
catch: () => new
|
|
1162
|
+
catch: () => new B({ reason: `unreadable: ${t.from}` })
|
|
1000
1163
|
}).pipe(n.map((e) => e.length), n.orElseSucceed(() => 0));
|
|
1001
|
-
a.set(
|
|
1164
|
+
a.set(t.from, r);
|
|
1002
1165
|
}
|
|
1003
|
-
let r = a.get(
|
|
1166
|
+
let r = a.get(t.from) ?? 0;
|
|
1004
1167
|
r > 0 && i.push({
|
|
1005
|
-
...
|
|
1168
|
+
...t,
|
|
1006
1169
|
rowCount: r
|
|
1007
1170
|
});
|
|
1008
1171
|
}
|
|
1009
|
-
if (i.length > 0) return yield* n.fail(new
|
|
1010
|
-
mode:
|
|
1011
|
-
reason:
|
|
1172
|
+
if (i.length > 0) return yield* n.fail(new Ke({
|
|
1173
|
+
mode: c,
|
|
1174
|
+
reason: nn(i)
|
|
1012
1175
|
}));
|
|
1013
1176
|
}
|
|
1014
|
-
if (
|
|
1015
|
-
manifest:
|
|
1177
|
+
if (e.dryRun === !0) return {
|
|
1178
|
+
manifest: o,
|
|
1016
1179
|
tablesWritten: 0,
|
|
1017
1180
|
rowsWritten: 0,
|
|
1018
1181
|
tablesSkipped: 0,
|
|
1019
1182
|
rowsSkipped: 0,
|
|
1020
1183
|
fullyResumed: !1
|
|
1021
1184
|
};
|
|
1022
|
-
let
|
|
1023
|
-
for (let
|
|
1024
|
-
let
|
|
1025
|
-
|
|
1185
|
+
let v = /* @__PURE__ */ new Map();
|
|
1186
|
+
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1187
|
+
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1188
|
+
e.length > 0 && v.set(t.name, e);
|
|
1026
1189
|
}
|
|
1027
|
-
let
|
|
1028
|
-
let n =
|
|
1190
|
+
let x = (e, t) => {
|
|
1191
|
+
let n = v.get(e);
|
|
1029
1192
|
if (n === void 0) return t;
|
|
1030
1193
|
let r;
|
|
1031
1194
|
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1032
1195
|
return r ?? t;
|
|
1033
|
-
},
|
|
1034
|
-
let i = t.name, a = t.primaryKey, o =
|
|
1035
|
-
return
|
|
1036
|
-
},
|
|
1037
|
-
r.unclassified && (
|
|
1038
|
-
},
|
|
1196
|
+
}, C = (e, t, n, r = !1) => {
|
|
1197
|
+
let i = j ? J(t.name) : t.name, a = t.primaryKey, o = x(t.name, n);
|
|
1198
|
+
return c === "append" ? p === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : c === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1199
|
+
}, w = (e, t) => e.savepoint ? e.savepoint(t) : t(), T = 0, E = (t, n, r) => {
|
|
1200
|
+
r.unclassified && (T++, !(T > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (T === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1201
|
+
}, ee = (e, t, r = !0) => n.tryPromise({
|
|
1039
1202
|
try: e,
|
|
1040
|
-
catch: (e) =>
|
|
1041
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason,
|
|
1203
|
+
catch: (e) => pn(e)
|
|
1204
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, E(t.entry, t.row, e)), !1)))), te = (e) => {
|
|
1042
1205
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1043
1206
|
if (t.length !== 0) return {
|
|
1044
1207
|
...e.row,
|
|
1045
1208
|
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1046
1209
|
};
|
|
1047
|
-
},
|
|
1210
|
+
}, ne = (e) => {
|
|
1048
1211
|
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
1049
1212
|
if (r !== "reference") return !1;
|
|
1050
1213
|
let i = e.row[n];
|
|
@@ -1062,7 +1225,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1062
1225
|
};
|
|
1063
1226
|
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1064
1227
|
}
|
|
1065
|
-
return new
|
|
1228
|
+
return new Ue({
|
|
1066
1229
|
rows: i.slice(0, 20),
|
|
1067
1230
|
primaryCount: r.filter((e) => !e.derived).length,
|
|
1068
1231
|
totalCount: e.length,
|
|
@@ -1072,79 +1235,113 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1072
1235
|
primary: t.primary
|
|
1073
1236
|
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1074
1237
|
});
|
|
1075
|
-
},
|
|
1238
|
+
}, re = (e, t) => n.gen(function* () {
|
|
1076
1239
|
let r = t, i = [], a = !0;
|
|
1077
1240
|
for (; r.length > 0 && a;) {
|
|
1078
1241
|
a = !1;
|
|
1079
1242
|
let t = [];
|
|
1080
|
-
for (let n of r) (yield*
|
|
1243
|
+
for (let n of r) (yield* ee(() => w(e, () => C(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1081
1244
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1082
|
-
let r = t[n], o =
|
|
1083
|
-
o !== void 0 && (yield*
|
|
1245
|
+
let r = t[n], o = te(r);
|
|
1246
|
+
o !== void 0 && (yield* ee(() => w(e, () => C(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1084
1247
|
}
|
|
1085
1248
|
r = t;
|
|
1086
1249
|
}
|
|
1087
|
-
for (let t of i) (yield*
|
|
1088
|
-
if (r.length > 0) return yield* n.fail(
|
|
1089
|
-
}), D =
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1250
|
+
for (let t of i) (yield* ee(() => w(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1251
|
+
if (r.length > 0) return yield* n.fail(ne(r));
|
|
1252
|
+
}), D = (() => {
|
|
1253
|
+
let t = e.targetDialect;
|
|
1254
|
+
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1255
|
+
})(), O = (() => {
|
|
1256
|
+
let t = e.targetSnapshot;
|
|
1257
|
+
if (t === void 0) return [];
|
|
1258
|
+
let n = new Map(t.tables.map((e) => [e.name, e])), r = [];
|
|
1259
|
+
for (let e of o.tables) {
|
|
1260
|
+
if (z(e.name)) continue;
|
|
1261
|
+
let t = n.get(e.name);
|
|
1262
|
+
if (t === void 0) return [];
|
|
1263
|
+
let i = t.columns.filter((e) => e.generatedAs === void 0).map((e) => e.name);
|
|
1264
|
+
if (i.length === 0) return [];
|
|
1265
|
+
r.push({
|
|
1266
|
+
name: e.name,
|
|
1267
|
+
columns: i
|
|
1268
|
+
});
|
|
1269
|
+
}
|
|
1270
|
+
return r;
|
|
1271
|
+
})(), k = O.filter((e) => y(e.name)).map((e) => e.name), A = D === void 0 || jn(D) ? [] : An(e.targetSnapshot === void 0 ? [] : wn(e.targetSnapshot, O.map((e) => e.name)), O.map((e) => e.name)), j = c === "replace" && D !== void 0 && O.length > 0 && k.length === 0 && A.length === 0 && _n(e.store), ae = !1, oe = 0, M = 0, ce = 0, le = 0, N = (r, i) => n.gen(function* () {
|
|
1272
|
+
if (c === "replace" && !j && !(i && g.truncated)) {
|
|
1273
|
+
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1274
|
+
try: () => r.insert(u, {
|
|
1275
|
+
id: f("repl", ue),
|
|
1276
|
+
startedAt: de,
|
|
1277
|
+
tables: String(o.tables.length),
|
|
1278
|
+
capture: P?.dir ?? null,
|
|
1279
|
+
via: e.via ?? "direct"
|
|
1280
|
+
}),
|
|
1281
|
+
catch: (e) => new B({ reason: `could not record that this replace started: ${String(e?.message ?? e)}\n The marker is what makes an interrupted run visible instead of silent, so the run does
|
|
1282
|
+
not proceed without it.` })
|
|
1283
|
+
}));
|
|
1284
|
+
let t = [...o.tables].reverse().map((e) => e.name).filter((e) => !z(e)), a = !i, s = r.emptyTables ? () => r.emptyTables(t) : a ? async () => {
|
|
1285
|
+
for (let e of [...o.tables].reverse()) await r.deleteMany(e.name, { where: _(e.primaryKey) });
|
|
1286
|
+
} : () => r.transactional(async (e) => {
|
|
1287
|
+
for (let t of [...o.tables].reverse()) await e.deleteMany(t.name, { where: _(t.primaryKey) });
|
|
1095
1288
|
});
|
|
1096
1289
|
yield* n.tryPromise({
|
|
1097
|
-
try:
|
|
1098
|
-
catch: (e) => new
|
|
1099
|
-
}),
|
|
1290
|
+
try: s,
|
|
1291
|
+
catch: (e) => new B({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${hn(e)}` })
|
|
1292
|
+
}), i && (g.truncated = !0, yield* K(m, g));
|
|
1100
1293
|
}
|
|
1101
|
-
let a = [],
|
|
1102
|
-
for (let
|
|
1103
|
-
let
|
|
1104
|
-
if (
|
|
1105
|
-
|
|
1294
|
+
let a = [], p = [], h = [], v = [], y = o.tables.length;
|
|
1295
|
+
for (let u = 0; u < o.tables.length; u++) {
|
|
1296
|
+
let f = o.tables[u];
|
|
1297
|
+
if (z(f.name)) {
|
|
1298
|
+
v.push(f.name);
|
|
1299
|
+
continue;
|
|
1300
|
+
}
|
|
1301
|
+
if (i && f.name in g.tables) {
|
|
1302
|
+
h.push(f.name), le++, ce += g.tables[f.name].rowCount, G(e.onProgress, {
|
|
1106
1303
|
phase: "import",
|
|
1107
|
-
table:
|
|
1108
|
-
index:
|
|
1109
|
-
tableCount:
|
|
1304
|
+
table: f.name,
|
|
1305
|
+
index: u,
|
|
1306
|
+
tableCount: y,
|
|
1110
1307
|
event: "done",
|
|
1111
|
-
rowsDone:
|
|
1112
|
-
total:
|
|
1308
|
+
rowsDone: g.tables[f.name].rowCount,
|
|
1309
|
+
total: f.rowCount
|
|
1113
1310
|
});
|
|
1114
1311
|
continue;
|
|
1115
1312
|
}
|
|
1116
|
-
let
|
|
1117
|
-
G(
|
|
1313
|
+
let _ = t.join(e.bundleDir, f.file);
|
|
1314
|
+
G(e.onProgress, {
|
|
1118
1315
|
phase: "import",
|
|
1119
|
-
table:
|
|
1120
|
-
index:
|
|
1121
|
-
tableCount:
|
|
1316
|
+
table: f.name,
|
|
1317
|
+
index: u,
|
|
1318
|
+
tableCount: y,
|
|
1122
1319
|
event: "start",
|
|
1123
1320
|
rowsDone: 0,
|
|
1124
|
-
total:
|
|
1321
|
+
total: f.rowCount
|
|
1125
1322
|
});
|
|
1126
|
-
let
|
|
1127
|
-
let { createHash:
|
|
1128
|
-
let
|
|
1129
|
-
entry:
|
|
1130
|
-
row:
|
|
1323
|
+
let b = a.length, x = n.gen(function* () {
|
|
1324
|
+
let { createHash: t } = yield* n.promise(() => import("node:crypto")), i = t("sha256"), s = 0, u = ie("bundle", () => se(_).pipe(st(o.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), p = async (e) => {
|
|
1325
|
+
let t = {
|
|
1326
|
+
entry: f,
|
|
1327
|
+
row: e,
|
|
1131
1328
|
reason: ""
|
|
1132
1329
|
};
|
|
1133
1330
|
try {
|
|
1134
|
-
await
|
|
1135
|
-
} catch (
|
|
1136
|
-
let r =
|
|
1137
|
-
|
|
1331
|
+
await w(r, () => C(r, f, e));
|
|
1332
|
+
} catch (n) {
|
|
1333
|
+
let r = pn(n);
|
|
1334
|
+
t.reason = r.reason, E(f, e, r), a.push(t);
|
|
1138
1335
|
}
|
|
1139
|
-
},
|
|
1140
|
-
let
|
|
1141
|
-
if (!
|
|
1142
|
-
if (d) return
|
|
1143
|
-
if (
|
|
1144
|
-
if (
|
|
1336
|
+
}, m = yield* n.promise(async () => {
|
|
1337
|
+
let t = e.copyLoader;
|
|
1338
|
+
if (!t || t.dialect !== "postgres") return !1;
|
|
1339
|
+
if (d) return ae || (ae = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1340
|
+
if (c === "replace") return !0;
|
|
1341
|
+
if (c !== "upsert") return !1;
|
|
1145
1342
|
try {
|
|
1146
|
-
return (await
|
|
1147
|
-
table:
|
|
1343
|
+
return (await r.query({
|
|
1344
|
+
table: f.name,
|
|
1148
1345
|
predicate: void 0,
|
|
1149
1346
|
order: [],
|
|
1150
1347
|
take: 1,
|
|
@@ -1154,123 +1351,175 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1154
1351
|
} catch {
|
|
1155
1352
|
return !1;
|
|
1156
1353
|
}
|
|
1157
|
-
}),
|
|
1158
|
-
if (
|
|
1159
|
-
let
|
|
1160
|
-
|
|
1354
|
+
}), h = Object.keys(f.columnTypes), g = Math.max(1, e.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
1355
|
+
if (v.length === 0) return;
|
|
1356
|
+
let t = v;
|
|
1357
|
+
v = [];
|
|
1161
1358
|
try {
|
|
1162
|
-
await
|
|
1163
|
-
table:
|
|
1164
|
-
columns:
|
|
1165
|
-
columnTypes:
|
|
1166
|
-
rows:
|
|
1359
|
+
await e.copyLoader.copyInto({
|
|
1360
|
+
table: f.name,
|
|
1361
|
+
columns: h,
|
|
1362
|
+
columnTypes: f.columnTypes,
|
|
1363
|
+
rows: t
|
|
1167
1364
|
});
|
|
1168
1365
|
} catch (n) {
|
|
1169
|
-
|
|
1170
|
-
for (let
|
|
1366
|
+
m = !1, e.onWarn?.(`bulk COPY into ${f.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1367
|
+
for (let e of t) await p(e);
|
|
1171
1368
|
}
|
|
1172
|
-
},
|
|
1173
|
-
if (
|
|
1174
|
-
let
|
|
1175
|
-
|
|
1369
|
+
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof r.savepoint == "function", S = [], T = async () => {
|
|
1370
|
+
if (S.length === 0) return;
|
|
1371
|
+
let e = S;
|
|
1372
|
+
S = [];
|
|
1176
1373
|
try {
|
|
1177
|
-
await
|
|
1178
|
-
for (let
|
|
1374
|
+
await r.savepoint(async () => {
|
|
1375
|
+
for (let t of e) await C(r, f, t);
|
|
1179
1376
|
});
|
|
1180
1377
|
} catch {
|
|
1181
|
-
for (let
|
|
1378
|
+
for (let t of e) await p(t);
|
|
1182
1379
|
}
|
|
1183
1380
|
};
|
|
1184
|
-
return yield*
|
|
1185
|
-
if (
|
|
1186
|
-
|
|
1381
|
+
return yield* l.runForEach(Ht(u, f.name, f.columnTypes), (e) => n.promise(async () => {
|
|
1382
|
+
if (s++, m) {
|
|
1383
|
+
v.push(e), v.length >= g && await y();
|
|
1187
1384
|
return;
|
|
1188
1385
|
}
|
|
1189
|
-
if (
|
|
1190
|
-
|
|
1386
|
+
if (x) {
|
|
1387
|
+
S.push(e), S.length >= b && await T();
|
|
1191
1388
|
return;
|
|
1192
1389
|
}
|
|
1193
|
-
await
|
|
1194
|
-
})), yield* n.promise(
|
|
1195
|
-
rowCount:
|
|
1390
|
+
await p(e);
|
|
1391
|
+
})), yield* n.promise(T), yield* n.promise(y), {
|
|
1392
|
+
rowCount: s,
|
|
1196
1393
|
checksum: i.digest("hex")
|
|
1197
1394
|
};
|
|
1198
|
-
}), { rowCount:
|
|
1199
|
-
if (
|
|
1200
|
-
if (
|
|
1201
|
-
what: `table ${
|
|
1202
|
-
expected:
|
|
1203
|
-
actual:
|
|
1395
|
+
}), { rowCount: S, checksum: T } = yield* Tt("import", f.name, u, y, (e) => e.rowCount, x);
|
|
1396
|
+
if (s) {
|
|
1397
|
+
if (T !== f.checksum) return yield* n.fail(new Ve({
|
|
1398
|
+
what: `table ${f.name} checksum`,
|
|
1399
|
+
expected: f.checksum,
|
|
1400
|
+
actual: T
|
|
1204
1401
|
}));
|
|
1205
|
-
if (
|
|
1206
|
-
what: `table ${
|
|
1207
|
-
expected: String(
|
|
1208
|
-
actual: String(
|
|
1402
|
+
if (S !== f.rowCount) return yield* n.fail(new Ve({
|
|
1403
|
+
what: `table ${f.name} rowCount`,
|
|
1404
|
+
expected: String(f.rowCount),
|
|
1405
|
+
actual: String(S)
|
|
1209
1406
|
}));
|
|
1210
1407
|
}
|
|
1211
|
-
G(
|
|
1408
|
+
G(e.onProgress, {
|
|
1212
1409
|
phase: "import",
|
|
1213
|
-
table:
|
|
1214
|
-
index:
|
|
1215
|
-
tableCount:
|
|
1410
|
+
table: f.name,
|
|
1411
|
+
index: u,
|
|
1412
|
+
tableCount: y,
|
|
1216
1413
|
event: "done",
|
|
1217
|
-
rowsDone:
|
|
1218
|
-
total:
|
|
1219
|
-
}),
|
|
1220
|
-
name:
|
|
1221
|
-
rowCount:
|
|
1222
|
-
}) : (
|
|
1223
|
-
rowCount:
|
|
1224
|
-
checksum:
|
|
1225
|
-
}, yield* K(m,
|
|
1414
|
+
rowsDone: S,
|
|
1415
|
+
total: f.rowCount
|
|
1416
|
+
}), M++, oe += S, i && (a.length > b ? p.push({
|
|
1417
|
+
name: f.name,
|
|
1418
|
+
rowCount: S
|
|
1419
|
+
}) : (g.tables[f.name] = {
|
|
1420
|
+
rowCount: S,
|
|
1421
|
+
checksum: f.checksum
|
|
1422
|
+
}, yield* K(m, g)));
|
|
1226
1423
|
}
|
|
1227
|
-
if (a.length > 0 && (yield*
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1424
|
+
if (a.length > 0 && (yield* re(r, a)), c === "replace" && !(i && g.truncated) && (yield* n.promise(() => r.deleteMany(u, { where: {
|
|
1425
|
+
column: "startedAt",
|
|
1426
|
+
op: "lte",
|
|
1427
|
+
value: de
|
|
1428
|
+
} }).then(() => void 0, () => void 0))), i && p.length > 0) {
|
|
1429
|
+
for (let e of p) {
|
|
1430
|
+
let t = o.tables.find((t) => t.name === e.name);
|
|
1431
|
+
g.tables[e.name] = {
|
|
1231
1432
|
rowCount: e.rowCount,
|
|
1232
1433
|
checksum: t.checksum
|
|
1233
1434
|
};
|
|
1234
1435
|
}
|
|
1235
|
-
yield* K(m,
|
|
1436
|
+
yield* K(m, g);
|
|
1236
1437
|
}
|
|
1237
|
-
|
|
1238
|
-
let
|
|
1239
|
-
t
|
|
1438
|
+
{
|
|
1439
|
+
let t = ze(v, "skipped");
|
|
1440
|
+
t !== void 0 && e.onWarn?.(t);
|
|
1240
1441
|
}
|
|
1442
|
+
if (h.length > 0) {
|
|
1443
|
+
let t = h.length === o.tables.length;
|
|
1444
|
+
e.onWarn?.(`resume: ${h.length} of ${o.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${h.join(", ")}. Delete ${m} to force a full re-import.`);
|
|
1445
|
+
}
|
|
1446
|
+
}), ue = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, de = /* @__PURE__ */ new Date(), P = yield* Mn({
|
|
1447
|
+
store: e.store,
|
|
1448
|
+
mode: c,
|
|
1449
|
+
dir: e.rollbackDir,
|
|
1450
|
+
snapshot: e.targetSnapshot,
|
|
1451
|
+
dialect: e.targetDialect,
|
|
1452
|
+
tables: o.tables.map((e) => e.name).filter((e) => !z(e)),
|
|
1453
|
+
onWarn: e.onWarn
|
|
1241
1454
|
});
|
|
1242
|
-
if (
|
|
1243
|
-
try: () =>
|
|
1244
|
-
|
|
1245
|
-
|
|
1455
|
+
if (P !== void 0 && (e.onRollbackCaptured !== void 0 && (yield* n.tryPromise({
|
|
1456
|
+
try: () => e.onRollbackCaptured(P),
|
|
1457
|
+
catch: (e) => new B({
|
|
1458
|
+
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1459
|
+
path: P.dir
|
|
1460
|
+
})
|
|
1461
|
+
})), e.onWarn?.(Nn(P))), !j && c === "replace" && d && A.length > 0 && e.onWarn?.(vn(`the bundle's tables reference each other in a cycle (${A.map((e) => `${e.from}.${e.column} → ${e.to}`).join(", ")}), and a bulk copy cannot satisfy both directions on this dialect.`)), !j && c === "replace" && d && k.length > 0 && e.onWarn?.(vn(`a write recorder is registered for ${k.join(", ")}, and a recorder is keyed by table name — a staged write would find none and the recorder would silently not run.`)), j) {
|
|
1462
|
+
e.onWarn?.(`staging ${O.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1463
|
+
let t = e.store, r = O.map((e) => e.name), i = Cn(r, {
|
|
1464
|
+
getTable: (e) => h(e),
|
|
1465
|
+
registerTable: (e) => {
|
|
1466
|
+
b(e);
|
|
1467
|
+
},
|
|
1468
|
+
unregisterTable: S
|
|
1469
|
+
}), a = !1;
|
|
1470
|
+
try {
|
|
1471
|
+
let i = d ? r : r.filter((e) => !Ot(g, e));
|
|
1472
|
+
yield* n.tryPromise({
|
|
1473
|
+
try: async () => {
|
|
1474
|
+
await Dn(t, i.map((e) => xn(e, D))), await Dn(t, i.map((e) => bn(e, D)));
|
|
1475
|
+
},
|
|
1476
|
+
catch: (e) => new B({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1477
|
+
}), yield* N(e.store, !d), !d && g.truncated === !0 ? a = !0 : yield* n.tryPromise({
|
|
1478
|
+
try: () => On(t, O, D).then(() => {
|
|
1479
|
+
a = !0;
|
|
1480
|
+
}),
|
|
1481
|
+
catch: (e) => new B({ reason: String(e?.message ?? e) })
|
|
1482
|
+
}).pipe(n.catchAll((i) => n.gen(function* () {
|
|
1483
|
+
let a = En(yield* n.promise(() => kn(t, wn(e.targetSnapshot, r), D).then((e) => e, () => [])));
|
|
1484
|
+
return yield* n.fail(new B({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1485
|
+
}))), !d && a && g.truncated !== !0 && (g.truncated = !0, yield* K(m, g));
|
|
1486
|
+
} finally {
|
|
1487
|
+
i.undo(), a || d ? yield* n.promise(() => Dn(t, r.map((e) => xn(e, D))).then(() => void 0, () => void 0)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1488
|
+
}
|
|
1489
|
+
} else d ? yield* n.tryPromise({
|
|
1490
|
+
try: () => e.store.transactional(async (e) => {
|
|
1491
|
+
let t = await n.runPromiseExit(N(e, !1));
|
|
1492
|
+
if (r.isFailure(t)) throw Vn(t.cause);
|
|
1246
1493
|
}),
|
|
1247
|
-
catch: (e) => new
|
|
1248
|
-
}) : yield*
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1494
|
+
catch: (e) => Bn(e) ?? new B({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1495
|
+
}) : yield* N(e.store, !0);
|
|
1496
|
+
if (e.assets && o.assets.index && !e.assetsAlreadyRestored) {
|
|
1497
|
+
let r = e.assets, i = t.join(e.bundleDir, _t), a = t.join(e.bundleDir, o.assets.index ?? "assets/index.ndjson"), s = ie("bundle", () => se(a)).pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.map((e) => JSON.parse(e)));
|
|
1498
|
+
yield* l.runForEach(s, (e) => e.key in g.assets ? n.void : nt(r, i, e).pipe(n.andThen(n.sync(() => {
|
|
1499
|
+
g.assets[e.key] = !0;
|
|
1500
|
+
})), n.andThen(K(m, g))));
|
|
1253
1501
|
}
|
|
1254
1502
|
return {
|
|
1255
|
-
manifest:
|
|
1256
|
-
tablesWritten:
|
|
1257
|
-
rowsWritten:
|
|
1258
|
-
tablesSkipped:
|
|
1259
|
-
rowsSkipped:
|
|
1260
|
-
fullyResumed:
|
|
1503
|
+
manifest: o,
|
|
1504
|
+
tablesWritten: M,
|
|
1505
|
+
rowsWritten: oe,
|
|
1506
|
+
tablesSkipped: le,
|
|
1507
|
+
rowsSkipped: ce,
|
|
1508
|
+
fullyResumed: M === 0 && le > 0,
|
|
1509
|
+
...P === void 0 ? {} : { rollback: P }
|
|
1261
1510
|
};
|
|
1262
|
-
}),
|
|
1511
|
+
}), Wn = (e) => e, Gn = (e) => {
|
|
1263
1512
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1264
1513
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1265
1514
|
return {
|
|
1266
1515
|
refuse: !0,
|
|
1267
1516
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1268
1517
|
};
|
|
1269
|
-
},
|
|
1518
|
+
}, X = Buffer.from("VBUNDLE2"), Kn = 0, qn = 1, Jn = 2, Yn = 32, Xn = async (e) => {
|
|
1270
1519
|
let t = [], n = async (r) => {
|
|
1271
1520
|
let i;
|
|
1272
1521
|
try {
|
|
1273
|
-
i = await
|
|
1522
|
+
i = await de(F(e, r), { withFileTypes: !0 });
|
|
1274
1523
|
} catch {
|
|
1275
1524
|
return;
|
|
1276
1525
|
}
|
|
@@ -1280,22 +1529,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1280
1529
|
}
|
|
1281
1530
|
};
|
|
1282
1531
|
return await n(""), t.sort();
|
|
1283
|
-
},
|
|
1532
|
+
}, Zn = (e, t) => {
|
|
1284
1533
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1285
|
-
return r.writeUInt8(
|
|
1286
|
-
},
|
|
1534
|
+
return r.writeUInt8(qn, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1535
|
+
}, Qn = (e, t, n) => {
|
|
1287
1536
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1288
|
-
return a.writeUInt8(
|
|
1537
|
+
return a.writeUInt8(Jn, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1289
1538
|
};
|
|
1290
|
-
async function*
|
|
1291
|
-
yield new Uint8Array(
|
|
1292
|
-
for (let t of await
|
|
1293
|
-
let n =
|
|
1294
|
-
if (yield new Uint8Array(
|
|
1539
|
+
async function* $n(e) {
|
|
1540
|
+
yield new Uint8Array(X);
|
|
1541
|
+
for (let t of await Xn(e.dir)) {
|
|
1542
|
+
let n = F(e.dir, t), r = await P(n);
|
|
1543
|
+
if (yield new Uint8Array(Zn(t, r.size)), r.size > 0) for await (let e of se(n)) yield new Uint8Array(e);
|
|
1295
1544
|
}
|
|
1296
1545
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1297
|
-
yield new Uint8Array(
|
|
1298
|
-
let n =
|
|
1546
|
+
yield new Uint8Array(Qn(t.key, t.contentType, t.size));
|
|
1547
|
+
let n = E("sha256"), r = 0;
|
|
1299
1548
|
for await (let i of e.blobs.open(t.key)) {
|
|
1300
1549
|
let e = Buffer.from(i);
|
|
1301
1550
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1303,11 +1552,11 @@ async function* yn(e) {
|
|
|
1303
1552
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1304
1553
|
yield new Uint8Array(n.digest());
|
|
1305
1554
|
}
|
|
1306
|
-
yield new Uint8Array(Buffer.from([
|
|
1555
|
+
yield new Uint8Array(Buffer.from([Kn]));
|
|
1307
1556
|
}
|
|
1308
|
-
var
|
|
1309
|
-
let n =
|
|
1310
|
-
await
|
|
1557
|
+
var er = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tr = (e) => new Promise((t) => e.end(t)), nr = async (e, t) => {
|
|
1558
|
+
let n = he(t.dir);
|
|
1559
|
+
await ue(n, { recursive: !0 });
|
|
1311
1560
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1312
1561
|
let { value: e, done: t } = await i.next();
|
|
1313
1562
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1318,28 +1567,28 @@ var bn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), xn
|
|
|
1318
1567
|
return r = r.subarray(e), t;
|
|
1319
1568
|
};
|
|
1320
1569
|
try {
|
|
1321
|
-
if (await s(
|
|
1570
|
+
if (await s(X.length), !c(X.length).equals(X)) throw Error("not a bundle archive (bad magic)");
|
|
1322
1571
|
for (;;) {
|
|
1323
1572
|
await s(1);
|
|
1324
1573
|
let e = c(1).readUInt8(0);
|
|
1325
|
-
if (e ===
|
|
1326
|
-
if (e ===
|
|
1574
|
+
if (e === Kn) return;
|
|
1575
|
+
if (e === qn) {
|
|
1327
1576
|
await s(4);
|
|
1328
1577
|
let e = c(4).readUInt32BE(0);
|
|
1329
1578
|
await s(e);
|
|
1330
1579
|
let t = c(e).toString("utf8");
|
|
1331
1580
|
await s(8);
|
|
1332
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1333
|
-
if (a !== n && !a.startsWith(n +
|
|
1334
|
-
await
|
|
1581
|
+
let i = c(8).readBigUInt64BE(0), a = he(n, t);
|
|
1582
|
+
if (a !== n && !a.startsWith(n + ge)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1583
|
+
await ue(me(a), { recursive: !0 });
|
|
1335
1584
|
let o = M(a);
|
|
1336
1585
|
for (; i > 0n;) {
|
|
1337
1586
|
r.length === 0 && await s(1);
|
|
1338
1587
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1339
|
-
await
|
|
1588
|
+
await er(o, c(e)), i -= BigInt(e);
|
|
1340
1589
|
}
|
|
1341
|
-
await
|
|
1342
|
-
} else if (e ===
|
|
1590
|
+
await tr(o);
|
|
1591
|
+
} else if (e === Jn) {
|
|
1343
1592
|
await s(4);
|
|
1344
1593
|
let e = c(4).readUInt32BE(0);
|
|
1345
1594
|
await s(e);
|
|
@@ -1349,24 +1598,24 @@ var bn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), xn
|
|
|
1349
1598
|
await s(i);
|
|
1350
1599
|
let a = c(i).toString("utf8");
|
|
1351
1600
|
await s(8);
|
|
1352
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1601
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = E("sha256"), d = new j(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1353
1602
|
contentType: a,
|
|
1354
1603
|
size: Number(o)
|
|
1355
1604
|
}), p = o;
|
|
1356
1605
|
for (; p > 0n;) {
|
|
1357
1606
|
r.length === 0 && await s(1);
|
|
1358
1607
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1359
|
-
u.update(t), l || await
|
|
1608
|
+
u.update(t), l || await er(d, t), p -= BigInt(e);
|
|
1360
1609
|
}
|
|
1361
|
-
l ? d.destroy() : await
|
|
1362
|
-
let m = c(
|
|
1610
|
+
l ? d.destroy() : await tr(d), await f, await s(Yn);
|
|
1611
|
+
let m = c(Yn);
|
|
1363
1612
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1364
1613
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1365
1614
|
}
|
|
1366
1615
|
} finally {
|
|
1367
1616
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1368
1617
|
}
|
|
1369
|
-
},
|
|
1618
|
+
}, rr = async (e, t) => {
|
|
1370
1619
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1371
1620
|
let { value: e, done: t } = await a.next();
|
|
1372
1621
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1385,11 +1634,11 @@ var bn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), xn
|
|
|
1385
1634
|
}
|
|
1386
1635
|
return Buffer.concat(t);
|
|
1387
1636
|
};
|
|
1388
|
-
if (!await s(
|
|
1637
|
+
if (!await s(X.length) || !c(X.length).equals(X)) throw Error("not a bundle archive (bad magic)");
|
|
1389
1638
|
for (; await s(1);) {
|
|
1390
1639
|
let e = c(1).readUInt8(0);
|
|
1391
|
-
if (e ===
|
|
1392
|
-
if (e !==
|
|
1640
|
+
if (e === Kn || e === Jn) break;
|
|
1641
|
+
if (e !== qn) throw Error(`unknown bundle entry kind ${e}`);
|
|
1393
1642
|
if (!await s(4)) break;
|
|
1394
1643
|
let t = c(4).readUInt32BE(0);
|
|
1395
1644
|
if (!await s(t)) break;
|
|
@@ -1399,22 +1648,22 @@ var bn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), xn
|
|
|
1399
1648
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1400
1649
|
}
|
|
1401
1650
|
return r;
|
|
1402
|
-
},
|
|
1651
|
+
}, ir = (e, t) => nr(e, {
|
|
1403
1652
|
dir: t,
|
|
1404
|
-
blobs:
|
|
1405
|
-
}),
|
|
1653
|
+
blobs: ar(t)
|
|
1654
|
+
}), ar = (e) => {
|
|
1406
1655
|
let t = /* @__PURE__ */ new Set();
|
|
1407
1656
|
return {
|
|
1408
1657
|
has: async (e) => t.has(e),
|
|
1409
1658
|
write: async (n, r, i) => {
|
|
1410
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1411
|
-
await
|
|
1412
|
-
let l =
|
|
1659
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = F(e, "assets");
|
|
1660
|
+
await ue(c, { recursive: !0 });
|
|
1661
|
+
let l = E("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1413
1662
|
l.update(e), u += e.length, n(null, e);
|
|
1414
|
-
} }), f =
|
|
1663
|
+
} }), f = F(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1415
1664
|
await o(r, d, M(f));
|
|
1416
1665
|
let p = l.digest("hex");
|
|
1417
|
-
await a.rename(f,
|
|
1666
|
+
await a.rename(f, F(c, p)), await a.appendFile(F(c, "index.ndjson"), `${JSON.stringify({
|
|
1418
1667
|
key: n,
|
|
1419
1668
|
sha256: p,
|
|
1420
1669
|
size: u,
|
|
@@ -1422,33 +1671,33 @@ var bn = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), xn
|
|
|
1422
1671
|
})}\n`), t.add(n);
|
|
1423
1672
|
}
|
|
1424
1673
|
};
|
|
1425
|
-
},
|
|
1674
|
+
}, or = _e(ne), sr = Buffer.from("VENC1\0\0\0"), cr = 32, lr = 16, ur = 7, Z = 16, dr = 262144, fr = {
|
|
1426
1675
|
N: 32768,
|
|
1427
1676
|
r: 8,
|
|
1428
1677
|
p: 1
|
|
1429
|
-
},
|
|
1678
|
+
}, pr = 134217728, Q = 43, mr = (e, t, n) => {
|
|
1430
1679
|
let r = Buffer.alloc(12);
|
|
1431
|
-
return e.copy(r, 0, 0,
|
|
1680
|
+
return e.copy(r, 0, 0, ur), r.writeUInt32BE(t >>> 0, ur), r.writeUInt8(+!!n, 11), r;
|
|
1432
1681
|
};
|
|
1433
|
-
async function*
|
|
1682
|
+
async function* hr(e, t) {
|
|
1434
1683
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1435
|
-
let n =
|
|
1436
|
-
...
|
|
1437
|
-
maxmem:
|
|
1684
|
+
let n = te(lr), r = te(ur), i = await or(t, n, cr, {
|
|
1685
|
+
...fr,
|
|
1686
|
+
maxmem: pr
|
|
1438
1687
|
}), a = Buffer.alloc(Q);
|
|
1439
|
-
|
|
1688
|
+
sr.copy(a, 0), a.writeUInt32BE(fr.N, 8), a.writeUInt32BE(fr.r, 12), a.writeUInt32BE(fr.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1440
1689
|
let o = 0, s = (e, t) => {
|
|
1441
|
-
let n =
|
|
1690
|
+
let n = w("aes-256-gcm", i, mr(r, o, t));
|
|
1442
1691
|
o === 0 && n.setAAD(a);
|
|
1443
1692
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1444
1693
|
o++;
|
|
1445
|
-
let l = Buffer.alloc(5 + s.length +
|
|
1446
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
1694
|
+
let l = Buffer.alloc(5 + s.length + Z);
|
|
1695
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + Z, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1447
1696
|
}, c = Buffer.alloc(0);
|
|
1448
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1697
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > dr;) yield s(c.subarray(0, dr), !1), c = c.subarray(dr);
|
|
1449
1698
|
yield s(c, !0);
|
|
1450
1699
|
}
|
|
1451
|
-
async function*
|
|
1700
|
+
async function* gr(e, t) {
|
|
1452
1701
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1453
1702
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1454
1703
|
let { value: e, done: t } = await n.next();
|
|
@@ -1459,12 +1708,12 @@ async function* Fn(e, t) {
|
|
|
1459
1708
|
};
|
|
1460
1709
|
if (!await o(Q)) throw Error("encrypted bundle truncated (no header)");
|
|
1461
1710
|
let s = r.subarray(0, Q);
|
|
1462
|
-
if (!s.subarray(0, 8).equals(
|
|
1463
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await
|
|
1711
|
+
if (!s.subarray(0, 8).equals(sr)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1712
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Q)), p = await or(t, d, cr, {
|
|
1464
1713
|
N: c,
|
|
1465
1714
|
r: l,
|
|
1466
1715
|
p: u,
|
|
1467
|
-
maxmem:
|
|
1716
|
+
maxmem: pr
|
|
1468
1717
|
});
|
|
1469
1718
|
r = r.subarray(Q);
|
|
1470
1719
|
let m = 0, h = !1;
|
|
@@ -1475,11 +1724,11 @@ async function* Fn(e, t) {
|
|
|
1475
1724
|
}
|
|
1476
1725
|
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
1477
1726
|
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
1478
|
-
if (t <
|
|
1727
|
+
if (t < Z) throw Error("encrypted bundle corrupt (short frame)");
|
|
1479
1728
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1480
|
-
let n = r.subarray(5, 5 + t -
|
|
1729
|
+
let n = r.subarray(5, 5 + t - Z), i = r.subarray(5 + t - Z, 5 + t);
|
|
1481
1730
|
r = r.subarray(5 + t);
|
|
1482
|
-
let a =
|
|
1731
|
+
let a = T("aes-256-gcm", p, mr(f, m, e));
|
|
1483
1732
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1484
1733
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1485
1734
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1487,11 +1736,11 @@ async function* Fn(e, t) {
|
|
|
1487
1736
|
}
|
|
1488
1737
|
//#endregion
|
|
1489
1738
|
//#region src/blobStorage.ts
|
|
1490
|
-
var
|
|
1739
|
+
var _r = (e, t) => ({
|
|
1491
1740
|
list: () => {
|
|
1492
1741
|
let e = t();
|
|
1493
1742
|
return { async *[Symbol.asyncIterator]() {
|
|
1494
|
-
let t = await n.runPromise(
|
|
1743
|
+
let t = await n.runPromise(l.runCollect(e));
|
|
1495
1744
|
for (let e of t) yield {
|
|
1496
1745
|
key: e.key,
|
|
1497
1746
|
contentType: e.contentType,
|
|
@@ -1500,29 +1749,29 @@ var In = (e, t) => ({
|
|
|
1500
1749
|
} };
|
|
1501
1750
|
},
|
|
1502
1751
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1503
|
-
let r = await n.runPromise(
|
|
1752
|
+
let r = await n.runPromise(D(e, t)), i = await k(r.stream);
|
|
1504
1753
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1505
1754
|
} })
|
|
1506
|
-
}),
|
|
1507
|
-
table:
|
|
1755
|
+
}), vr = (e, t) => _r(e, () => x(t, {
|
|
1756
|
+
table: re,
|
|
1508
1757
|
chunkSize: 500
|
|
1509
|
-
}).pipe(
|
|
1758
|
+
}).pipe(l.map((e) => ({
|
|
1510
1759
|
key: String(e.key),
|
|
1511
1760
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1512
1761
|
size: Number(e.size ?? 0)
|
|
1513
|
-
})))),
|
|
1762
|
+
})))), yr = (e) => ({
|
|
1514
1763
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1515
1764
|
write: async (t, r, i) => {
|
|
1516
|
-
let a =
|
|
1517
|
-
await n.runPromise(
|
|
1765
|
+
let a = ie("storage", () => ae.from(r));
|
|
1766
|
+
await n.runPromise(O(e, t, a, {
|
|
1518
1767
|
contentType: i.contentType,
|
|
1519
1768
|
totalSize: i.size
|
|
1520
1769
|
}));
|
|
1521
1770
|
}
|
|
1522
|
-
}),
|
|
1771
|
+
}), br = (e, t) => {
|
|
1523
1772
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1524
|
-
return e === "mariadb" &&
|
|
1525
|
-
},
|
|
1773
|
+
return e === "mariadb" && xr(n) ? n : r;
|
|
1774
|
+
}, xr = (e) => (process.env.PATH ?? "").split(pe).filter((e) => e !== "").some((t) => ce(F(t, e))), Sr = (e) => {
|
|
1526
1775
|
switch (e) {
|
|
1527
1776
|
case "postgres": return "db.dump";
|
|
1528
1777
|
case "mysql":
|
|
@@ -1532,7 +1781,7 @@ var In = (e, t) => ({
|
|
|
1532
1781
|
case "mssql": return "db.bacpac";
|
|
1533
1782
|
default: return "db.dump";
|
|
1534
1783
|
}
|
|
1535
|
-
},
|
|
1784
|
+
}, Cr = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1536
1785
|
if (e.host || e.database) return {
|
|
1537
1786
|
host: e.host ?? "localhost",
|
|
1538
1787
|
port: e.port ?? 3306,
|
|
@@ -1548,7 +1797,7 @@ var In = (e, t) => ({
|
|
|
1548
1797
|
password: decodeURIComponent(t.password) || "",
|
|
1549
1798
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1550
1799
|
};
|
|
1551
|
-
},
|
|
1800
|
+
}, wr = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Tr = (e, t, n) => {
|
|
1552
1801
|
switch (e) {
|
|
1553
1802
|
case "postgres": {
|
|
1554
1803
|
if (t.url) return {
|
|
@@ -1595,7 +1844,7 @@ var In = (e, t) => ({
|
|
|
1595
1844
|
let r = $(t);
|
|
1596
1845
|
return {
|
|
1597
1846
|
kind: "spawn",
|
|
1598
|
-
tool:
|
|
1847
|
+
tool: br(e, "dump"),
|
|
1599
1848
|
args: [
|
|
1600
1849
|
"--single-transaction",
|
|
1601
1850
|
"--routines",
|
|
@@ -1617,7 +1866,7 @@ var In = (e, t) => ({
|
|
|
1617
1866
|
case "sqlite":
|
|
1618
1867
|
case "turso": return {
|
|
1619
1868
|
kind: "copy",
|
|
1620
|
-
from:
|
|
1869
|
+
from: Cr(t),
|
|
1621
1870
|
to: n
|
|
1622
1871
|
};
|
|
1623
1872
|
case "mssql": return {
|
|
@@ -1625,7 +1874,7 @@ var In = (e, t) => ({
|
|
|
1625
1874
|
tool: "sqlpackage",
|
|
1626
1875
|
args: [
|
|
1627
1876
|
"/Action:Export",
|
|
1628
|
-
`/SourceConnectionString:${
|
|
1877
|
+
`/SourceConnectionString:${wr($({
|
|
1629
1878
|
...t,
|
|
1630
1879
|
port: t.port ?? 1433
|
|
1631
1880
|
}))}`,
|
|
@@ -1633,12 +1882,12 @@ var In = (e, t) => ({
|
|
|
1633
1882
|
],
|
|
1634
1883
|
env: {}
|
|
1635
1884
|
};
|
|
1636
|
-
default: throw new
|
|
1885
|
+
default: throw new H({
|
|
1637
1886
|
tool: "backup",
|
|
1638
1887
|
reason: `no native backup for dialect '${e}'`
|
|
1639
1888
|
});
|
|
1640
1889
|
}
|
|
1641
|
-
},
|
|
1890
|
+
}, Er = (e, t, n) => {
|
|
1642
1891
|
switch (e) {
|
|
1643
1892
|
case "postgres": {
|
|
1644
1893
|
if (t.url) return {
|
|
@@ -1683,7 +1932,7 @@ var In = (e, t) => ({
|
|
|
1683
1932
|
let r = $(t);
|
|
1684
1933
|
return {
|
|
1685
1934
|
kind: "spawn",
|
|
1686
|
-
tool:
|
|
1935
|
+
tool: br(e, "client"),
|
|
1687
1936
|
args: [
|
|
1688
1937
|
"--protocol=TCP",
|
|
1689
1938
|
"-h",
|
|
@@ -1702,14 +1951,14 @@ var In = (e, t) => ({
|
|
|
1702
1951
|
case "turso": return {
|
|
1703
1952
|
kind: "copy",
|
|
1704
1953
|
from: n,
|
|
1705
|
-
to:
|
|
1954
|
+
to: Cr(t)
|
|
1706
1955
|
};
|
|
1707
1956
|
case "mssql": return {
|
|
1708
1957
|
kind: "spawn",
|
|
1709
1958
|
tool: "sqlpackage",
|
|
1710
1959
|
args: [
|
|
1711
1960
|
"/Action:Import",
|
|
1712
|
-
`/TargetConnectionString:${
|
|
1961
|
+
`/TargetConnectionString:${wr($({
|
|
1713
1962
|
...t,
|
|
1714
1963
|
port: t.port ?? 1433
|
|
1715
1964
|
}))}`,
|
|
@@ -1717,18 +1966,18 @@ var In = (e, t) => ({
|
|
|
1717
1966
|
],
|
|
1718
1967
|
env: {}
|
|
1719
1968
|
};
|
|
1720
|
-
default: throw new
|
|
1969
|
+
default: throw new H({
|
|
1721
1970
|
tool: "restore",
|
|
1722
1971
|
reason: `no native restore for dialect '${e}'`
|
|
1723
1972
|
});
|
|
1724
1973
|
}
|
|
1725
|
-
},
|
|
1726
|
-
|
|
1974
|
+
}, Dr = (e) => e.kind === "copy" ? n.async((t) => {
|
|
1975
|
+
oe(e.from, e.to, (r) => t(r ? n.fail(new H({
|
|
1727
1976
|
tool: "copy",
|
|
1728
1977
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1729
1978
|
})) : n.void));
|
|
1730
1979
|
}) : n.async((t) => {
|
|
1731
|
-
let r =
|
|
1980
|
+
let r = ve(e.tool, [...e.args], {
|
|
1732
1981
|
env: {
|
|
1733
1982
|
...process.env,
|
|
1734
1983
|
...e.env
|
|
@@ -1741,14 +1990,14 @@ var In = (e, t) => ({
|
|
|
1741
1990
|
}), i = "";
|
|
1742
1991
|
r.stderr?.on("data", (e) => {
|
|
1743
1992
|
i += String(e);
|
|
1744
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin &&
|
|
1993
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin && se(e.stdinFile).pipe(r.stdin);
|
|
1745
1994
|
let a = () => {
|
|
1746
1995
|
if (e.stdoutFile !== void 0) try {
|
|
1747
|
-
|
|
1996
|
+
le(e.stdoutFile);
|
|
1748
1997
|
} catch {}
|
|
1749
1998
|
};
|
|
1750
1999
|
r.on("error", (r) => {
|
|
1751
|
-
a(), t(n.fail(new
|
|
2000
|
+
a(), t(n.fail(new H({
|
|
1752
2001
|
tool: e.tool,
|
|
1753
2002
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`
|
|
1754
2003
|
})));
|
|
@@ -1757,37 +2006,37 @@ var In = (e, t) => ({
|
|
|
1757
2006
|
t(n.void);
|
|
1758
2007
|
return;
|
|
1759
2008
|
}
|
|
1760
|
-
a(), t(n.fail(new
|
|
2009
|
+
a(), t(n.fail(new H({
|
|
1761
2010
|
tool: e.tool,
|
|
1762
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2011
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${fe(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
1763
2012
|
stderr: i.slice(-2e3)
|
|
1764
2013
|
})));
|
|
1765
2014
|
});
|
|
1766
|
-
}),
|
|
1767
|
-
let t = yield*
|
|
2015
|
+
}), Or = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), kr = (e) => n.gen(function* () {
|
|
2016
|
+
let t = yield* Qe(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
1768
2017
|
store: e.store,
|
|
1769
2018
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
1770
|
-
}), r =
|
|
2019
|
+
}), r = Ut(t.tables, e.snapshot), i = e.classification ?? xe(e.snapshot), a = e.sampleSize ?? 20, o = Ne(r.map((t) => ({
|
|
1771
2020
|
name: t,
|
|
1772
2021
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
1773
2022
|
name: e.name,
|
|
1774
2023
|
type: e.type
|
|
1775
2024
|
}))
|
|
1776
|
-
})), i, e.masking), s = [],
|
|
2025
|
+
})), i, e.masking), s = [], c = [];
|
|
1777
2026
|
for (let i of r) {
|
|
1778
|
-
let r = t.predicateFor(i), u = yield*
|
|
2027
|
+
let r = t.predicateFor(i), u = yield* x(e.store, {
|
|
1779
2028
|
table: i,
|
|
1780
2029
|
chunkSize: a,
|
|
1781
2030
|
...r === void 0 ? {} : { where: r }
|
|
1782
|
-
}).pipe(
|
|
1783
|
-
|
|
2031
|
+
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) => Ie(t, i, o.actions, e.masking.seed));
|
|
2032
|
+
c.push(...Fe(d, i, o.actions));
|
|
1784
2033
|
let f = u[0], p = d[0], m = [];
|
|
1785
2034
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
1786
2035
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
1787
2036
|
let n = e.slice(i.length + 1);
|
|
1788
2037
|
m.push({
|
|
1789
2038
|
column: n,
|
|
1790
|
-
action:
|
|
2039
|
+
action: Or(t),
|
|
1791
2040
|
before: f[n],
|
|
1792
2041
|
after: p[n]
|
|
1793
2042
|
});
|
|
@@ -1799,10 +2048,10 @@ var In = (e, t) => ({
|
|
|
1799
2048
|
}
|
|
1800
2049
|
return {
|
|
1801
2050
|
tables: s,
|
|
1802
|
-
leaks:
|
|
2051
|
+
leaks: c,
|
|
1803
2052
|
unclassified: o.unclassified
|
|
1804
2053
|
};
|
|
1805
|
-
}),
|
|
2054
|
+
}), Ar = (e, t) => {
|
|
1806
2055
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
1807
2056
|
for (let t of e) {
|
|
1808
2057
|
let e = n.get(t.name);
|
|
@@ -1857,7 +2106,7 @@ var In = (e, t) => ({
|
|
|
1857
2106
|
skipColumns: i,
|
|
1858
2107
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
1859
2108
|
};
|
|
1860
|
-
},
|
|
2109
|
+
}, jr = (e) => {
|
|
1861
2110
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
1862
2111
|
let t = {
|
|
1863
2112
|
safe: "✓",
|
|
@@ -1868,4 +2117,4 @@ var In = (e, t) => ({
|
|
|
1868
2117
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
1869
2118
|
};
|
|
1870
2119
|
//#endregion
|
|
1871
|
-
export {
|
|
2120
|
+
export { _t as ASSETS_DIR, vt as ASSET_INDEX_FILE, B as BundleError, Be as CodecError, He as CompressionError, qe as CrossDialectError, gt as DATA_DIR, be as DEFAULT_CLASS_ACTIONS, ct as FORMAT_VERSION, Ke as ImportModeError, Ve as IntegrityError, yt as LEDGER_FILE, W as MANIFEST_FILE, Ge as MaskingError, H as NativeToolError, Ue as RowsRefusedError, Rn as SAVEPOINT_BATCH_SIZE, gn as STAGING_PREFIX, We as SchemaDriftError, V as ScopeError, je as applyAction, Bn as asImportError, Gn as assessLiveImport, kt as assetDone, Sr as backupArtifactName, Tr as backupCommand, rn as canonicalType, sn as checkPortability, Ln as checkSchemaDrift, xe as classificationFromSnapshot, Ar as classifyImportDrift, at as compressionExt, Ye as computeSubsetIds, mt as decodeManifest, zt as decodeRow, Lt as decodeValue, gr as decryptStream, Wn as defineDataProfile, In as diffSnapshots, ar as dirBlobSink, xn as dropStagingSql, Et as emptyLedger, pt as encodeManifest, Rt as encodeRow, Ft as encodeValue, hr as encryptStream, Dn as execStatements, jr as formatImportDrift, cn as formatIssue, rt as hasZstd, ot as makeCompressor, st as makeDecompressor, Ie as maskRow, Ht as ndjsonToRows, nn as outsideReferenceRefusal, tn as outsideReferencesInto, $n as packBundle, rr as peekBundle, it as pickCompression, Ne as planMasking, kr as previewMasking, Dt as readLedger, Se as resolveAction, Qe as resolveScope, nt as restoreAssetFromCas, Er as restoreCommand, Vt as rowsToNdjson, en as runExport, Un as runImport, Dr as runNativeStep, Fe as scanForLeaks, J as stagingNameFor, et as storageAssetSink, $e as storageAssetSource, yr as storageBlobSink, _r as storageBlobSource, vr as storageRefsBlobSource, Xe as subsetToScope, Ot as tableDone, Ut as topoSortTables, nr as unpackBundle, ir as unpackBundleToDir, tt as writeAssetToCas, K as writeLedger };
|