@voltro/data-transfer 0.85.0 → 0.87.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5987 -0
- package/THIRD-PARTY-NOTICES.md +860 -249
- package/dist/assetAdmission.js +92 -0
- package/dist/assets.js +68 -0
- package/dist/blobStorage.js +64 -0
- package/dist/bundleArchive.js +310 -0
- package/dist/bundleCrypto.js +106 -0
- package/dist/bundlePaths.js +4 -0
- package/dist/casAssetPhase.js +87 -0
- package/dist/compress.js +6 -0
- package/dist/consistency.js +14 -0
- package/dist/crossDialect.js +58 -0
- package/dist/driftClassification.js +58 -0
- package/dist/environmentLocal.js +386 -0
- package/dist/errors.js +89 -0
- package/dist/exporter.js +271 -0
- package/dist/filesystemAdmission.js +33 -0
- package/dist/importer.js +576 -0
- package/dist/index.d.ts +202 -46
- package/dist/index.js +30 -2673
- package/dist/ledger.js +21 -0
- package/dist/liveGuard.js +11 -0
- package/dist/manifest.js +56 -0
- package/dist/masking.js +246 -0
- package/dist/maskingPreview.js +48 -0
- package/dist/nativeDump.js +418 -0
- package/dist/ndjson.js +13 -0
- package/dist/physicalRequest.js +10 -0
- package/dist/profile.js +4 -0
- package/dist/progress.js +17 -0
- package/dist/readme.js +43 -0
- package/dist/replacePreflight.js +23 -0
- package/dist/rollbackCapture.js +36 -0
- package/dist/rowFailureReason.js +36 -0
- package/dist/schemaDrift.js +40 -0
- package/dist/scope.js +40 -0
- package/dist/sealedKeys.js +40 -0
- package/dist/stagedSwap.js +164 -0
- package/dist/storageAssetMetadata.js +47 -0
- package/dist/subset.js +90 -0
- package/dist/topo.js +24 -0
- package/dist/typedCodec.js +36 -0
- package/package.json +3 -3
|
@@ -0,0 +1,418 @@
|
|
|
1
|
+
import { NativeToolError as e } from "./errors.js";
|
|
2
|
+
import { Effect as t } from "effect";
|
|
3
|
+
import { DATA_TRANSFERS_TABLE as n, REPLACE_IN_PROGRESS_TABLE as r } from "@voltro/database";
|
|
4
|
+
import { randomUUID as i } from "node:crypto";
|
|
5
|
+
import { copyFile as a, createReadStream as o, createWriteStream as s, existsSync as c, rename as l, statSync as u, unlinkSync as d } from "node:fs";
|
|
6
|
+
import { basename as f, delimiter as p, join as m } from "node:path";
|
|
7
|
+
import { spawn as h } from "node:child_process";
|
|
8
|
+
//#region src/nativeDump.ts
|
|
9
|
+
var g = (e, t) => {
|
|
10
|
+
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
11
|
+
return e === "mariadb" ? _(n) ? { tool: n } : {
|
|
12
|
+
tool: r,
|
|
13
|
+
hint: `'${n}' is not on PATH, so this ran '${r}' instead. If that binary came from Oracle's MySQL client it queries information_schema.COLUMN_STATISTICS, which MariaDB does not have, and the run dies with error 1109 after the first table. Install the MariaDB client package (which ships '${n}') and re-run. Do NOT reach for --column-statistics=0: that flag does not exist on '${n}', so it fixes the wrong client and breaks the right one. Note: a MariaDB 12.x client requires TLS by default, so running '${n}' BY HAND against a server without it fails with "TLS/SSL error: SSL is required" (2026) and needs --skip-ssl. This command is not affected.`
|
|
14
|
+
} : { tool: r };
|
|
15
|
+
}, _ = (e) => (process.env.PATH ?? "").split(p).filter((e) => e !== "").some((t) => c(m(t, e))), v = (e) => {
|
|
16
|
+
switch (e) {
|
|
17
|
+
case "postgres": return "db.dump";
|
|
18
|
+
case "mysql":
|
|
19
|
+
case "mariadb": return "db.sql";
|
|
20
|
+
case "sqlite":
|
|
21
|
+
case "turso": return "db.sqlite";
|
|
22
|
+
case "mssql": return "db.bacpac";
|
|
23
|
+
default: return "db.dump";
|
|
24
|
+
}
|
|
25
|
+
}, y = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), b = (e, t = 3306) => {
|
|
26
|
+
if (e.host || e.database) return {
|
|
27
|
+
host: e.host ?? "localhost",
|
|
28
|
+
port: e.port ?? t,
|
|
29
|
+
username: e.username ?? "root",
|
|
30
|
+
password: e.password ?? "",
|
|
31
|
+
database: e.database ?? ""
|
|
32
|
+
};
|
|
33
|
+
let n = new URL(e.url ?? "");
|
|
34
|
+
return {
|
|
35
|
+
host: n.hostname || "localhost",
|
|
36
|
+
port: e.port ?? (n.port ? Number(n.port) : t),
|
|
37
|
+
username: decodeURIComponent(n.username) || "root",
|
|
38
|
+
password: decodeURIComponent(n.password) || "",
|
|
39
|
+
database: n.pathname.replace(/^\//, "") || ""
|
|
40
|
+
};
|
|
41
|
+
}, x = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, S = (t, n, r) => {
|
|
42
|
+
switch (t) {
|
|
43
|
+
case "postgres": {
|
|
44
|
+
if (n.url) return {
|
|
45
|
+
kind: "spawn",
|
|
46
|
+
tool: "pg_dump",
|
|
47
|
+
args: [
|
|
48
|
+
"--format=custom",
|
|
49
|
+
"--no-owner",
|
|
50
|
+
"--no-privileges",
|
|
51
|
+
...w,
|
|
52
|
+
"--dbname",
|
|
53
|
+
n.url,
|
|
54
|
+
"--file",
|
|
55
|
+
r
|
|
56
|
+
],
|
|
57
|
+
env: {}
|
|
58
|
+
};
|
|
59
|
+
let e = b(n, 5432);
|
|
60
|
+
return {
|
|
61
|
+
kind: "spawn",
|
|
62
|
+
tool: "pg_dump",
|
|
63
|
+
args: [
|
|
64
|
+
"--format=custom",
|
|
65
|
+
"--no-owner",
|
|
66
|
+
"--no-privileges",
|
|
67
|
+
...w,
|
|
68
|
+
"-h",
|
|
69
|
+
e.host,
|
|
70
|
+
"-p",
|
|
71
|
+
String(e.port),
|
|
72
|
+
"-U",
|
|
73
|
+
e.username,
|
|
74
|
+
"-d",
|
|
75
|
+
e.database,
|
|
76
|
+
"--file",
|
|
77
|
+
r
|
|
78
|
+
],
|
|
79
|
+
env: { PGPASSWORD: e.password }
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
case "mysql":
|
|
83
|
+
case "mariadb": {
|
|
84
|
+
let e = b(n);
|
|
85
|
+
return {
|
|
86
|
+
kind: "spawn",
|
|
87
|
+
...g(t, "dump"),
|
|
88
|
+
args: [
|
|
89
|
+
"--single-transaction",
|
|
90
|
+
"--routines",
|
|
91
|
+
"--triggers",
|
|
92
|
+
"--no-tablespaces",
|
|
93
|
+
"--protocol=TCP",
|
|
94
|
+
...C.map((t) => `--ignore-table=${e.database}.${t}`),
|
|
95
|
+
"-h",
|
|
96
|
+
e.host,
|
|
97
|
+
"-P",
|
|
98
|
+
String(e.port),
|
|
99
|
+
"-u",
|
|
100
|
+
e.username,
|
|
101
|
+
e.database
|
|
102
|
+
],
|
|
103
|
+
env: { MYSQL_PWD: e.password },
|
|
104
|
+
stdoutFile: r
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
case "sqlite": return {
|
|
108
|
+
kind: "sqlite-backup",
|
|
109
|
+
from: y(n),
|
|
110
|
+
to: r
|
|
111
|
+
};
|
|
112
|
+
case "turso": return {
|
|
113
|
+
kind: "vacuum-into",
|
|
114
|
+
conn: n,
|
|
115
|
+
to: r
|
|
116
|
+
};
|
|
117
|
+
case "mssql": return {
|
|
118
|
+
kind: "spawn",
|
|
119
|
+
tool: "sqlpackage",
|
|
120
|
+
args: [
|
|
121
|
+
"/Action:Export",
|
|
122
|
+
`/SourceConnectionString:${x(b(n, 1433))}`,
|
|
123
|
+
`/TargetFile:${r}`
|
|
124
|
+
],
|
|
125
|
+
env: {}
|
|
126
|
+
};
|
|
127
|
+
default: throw new e({
|
|
128
|
+
tool: "backup",
|
|
129
|
+
reason: `no native backup for dialect '${t}'`
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
}, C = [r, n], w = C.flatMap((e) => ["--exclude-table", e]), T = (e) => e === "postgres" || e === "mysql" || e === "mariadb" ? C : [], E = (t, n, r) => {
|
|
133
|
+
switch (t) {
|
|
134
|
+
case "postgres": {
|
|
135
|
+
if (n.url) return {
|
|
136
|
+
kind: "spawn",
|
|
137
|
+
tool: "pg_restore",
|
|
138
|
+
args: [
|
|
139
|
+
"--clean",
|
|
140
|
+
"--if-exists",
|
|
141
|
+
"--no-owner",
|
|
142
|
+
"--dbname",
|
|
143
|
+
n.url,
|
|
144
|
+
r
|
|
145
|
+
],
|
|
146
|
+
env: {}
|
|
147
|
+
};
|
|
148
|
+
let e = b(n, 5432);
|
|
149
|
+
return {
|
|
150
|
+
kind: "spawn",
|
|
151
|
+
tool: "pg_restore",
|
|
152
|
+
args: [
|
|
153
|
+
"--clean",
|
|
154
|
+
"--if-exists",
|
|
155
|
+
"--no-owner",
|
|
156
|
+
"-h",
|
|
157
|
+
e.host,
|
|
158
|
+
"-p",
|
|
159
|
+
String(e.port),
|
|
160
|
+
"-U",
|
|
161
|
+
e.username,
|
|
162
|
+
"-d",
|
|
163
|
+
e.database,
|
|
164
|
+
r
|
|
165
|
+
],
|
|
166
|
+
env: { PGPASSWORD: e.password }
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
case "mysql":
|
|
170
|
+
case "mariadb": {
|
|
171
|
+
let e = b(n);
|
|
172
|
+
return {
|
|
173
|
+
kind: "spawn",
|
|
174
|
+
...g(t, "client"),
|
|
175
|
+
args: [
|
|
176
|
+
"--protocol=TCP",
|
|
177
|
+
"-h",
|
|
178
|
+
e.host,
|
|
179
|
+
"-P",
|
|
180
|
+
String(e.port),
|
|
181
|
+
"-u",
|
|
182
|
+
e.username,
|
|
183
|
+
e.database
|
|
184
|
+
],
|
|
185
|
+
env: { MYSQL_PWD: e.password },
|
|
186
|
+
stdinFile: r
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
case "sqlite":
|
|
190
|
+
case "turso": return {
|
|
191
|
+
kind: "copy",
|
|
192
|
+
from: r,
|
|
193
|
+
to: y(n)
|
|
194
|
+
};
|
|
195
|
+
case "mssql": {
|
|
196
|
+
let e = b(n, 1433), t = A(e.database);
|
|
197
|
+
return {
|
|
198
|
+
kind: "mssql-replace",
|
|
199
|
+
load: {
|
|
200
|
+
kind: "spawn",
|
|
201
|
+
tool: "sqlpackage",
|
|
202
|
+
args: [
|
|
203
|
+
"/Action:Import",
|
|
204
|
+
`/TargetConnectionString:${x({
|
|
205
|
+
...e,
|
|
206
|
+
database: t
|
|
207
|
+
})}`,
|
|
208
|
+
`/SourceFile:${r}`
|
|
209
|
+
],
|
|
210
|
+
env: {}
|
|
211
|
+
},
|
|
212
|
+
server: {
|
|
213
|
+
host: e.host,
|
|
214
|
+
port: e.port,
|
|
215
|
+
username: e.username,
|
|
216
|
+
password: e.password,
|
|
217
|
+
database: "master"
|
|
218
|
+
},
|
|
219
|
+
live: e.database,
|
|
220
|
+
scratch: t,
|
|
221
|
+
carry: C
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
default: throw new e({
|
|
225
|
+
tool: "restore",
|
|
226
|
+
reason: `no native restore for dialect '${t}'`
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
}, D = (e) => `[${e.replaceAll("]", "]]")}]`, O = (e) => `N'${e.replaceAll("'", "''")}'`, k = "__voltro_restore_", A = (e, t = i().slice(0, 8)) => `${e}${k}${t}`, j = (e) => e.replace(k, "__voltro_replaced_"), M = (e) => `IF DB_ID(${O(e)}) IS NOT NULL BEGIN ALTER DATABASE ${D(e)} SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE ${D(e)} END`, N = (e) => `DECLARE @leftover sysname, @drop nvarchar(max);
|
|
230
|
+
DECLARE leftovers CURSOR LOCAL FAST_FORWARD FOR SELECT name FROM sys.databases WHERE name LIKE ${O(`${`${e}${k}`.replace(/[\\%_[]/g, (e) => `\\${e}`)}%`)} ESCAPE N'\\';
|
|
231
|
+
OPEN leftovers; FETCH NEXT FROM leftovers INTO @leftover;
|
|
232
|
+
WHILE @@FETCH_STATUS = 0 BEGIN
|
|
233
|
+
SET @drop = N'ALTER DATABASE ' + QUOTENAME(@leftover) + N' SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE ' + QUOTENAME(@leftover);
|
|
234
|
+
EXEC(@drop);
|
|
235
|
+
FETCH NEXT FROM leftovers INTO @leftover;
|
|
236
|
+
END`, P = (e) => {
|
|
237
|
+
let t = D(e.live), n = D(e.scratch), r = j(e.scratch);
|
|
238
|
+
return {
|
|
239
|
+
prepare: [N(e.live)],
|
|
240
|
+
carry: e.carry.map((e) => {
|
|
241
|
+
let r = O(`${n}.dbo.${D(e)}`), i = O(`${t}.dbo.${D(e)}`);
|
|
242
|
+
return `IF OBJECT_ID(${r}) IS NOT NULL BEGIN
|
|
243
|
+
DELETE FROM ${n}.dbo.${D(e)};
|
|
244
|
+
IF OBJECT_ID(${i}) IS NOT NULL BEGIN
|
|
245
|
+
DECLARE @columns nvarchar(max) = (SELECT STRING_AGG(QUOTENAME(s.name), ',') FROM ${n}.sys.columns s
|
|
246
|
+
JOIN ${t}.sys.columns l ON l.name = s.name AND l.object_id = OBJECT_ID(${i})
|
|
247
|
+
WHERE s.object_id = OBJECT_ID(${r}) AND s.is_computed = 0);
|
|
248
|
+
EXEC(N'INSERT INTO ${n.replaceAll("'", "''")}.dbo.${D(e).replaceAll("'", "''")} (' + @columns + N') SELECT ' + @columns + N' FROM ${t.replaceAll("'", "''")}.dbo.${D(e).replaceAll("'", "''")}');
|
|
249
|
+
END
|
|
250
|
+
END`;
|
|
251
|
+
}),
|
|
252
|
+
swap: [
|
|
253
|
+
`ALTER DATABASE ${t} SET SINGLE_USER WITH ROLLBACK IMMEDIATE`,
|
|
254
|
+
`ALTER DATABASE ${t} MODIFY NAME = ${D(r)}`,
|
|
255
|
+
`ALTER DATABASE ${n} MODIFY NAME = ${t}`,
|
|
256
|
+
M(r)
|
|
257
|
+
],
|
|
258
|
+
undoSwap: [`IF DB_ID(${O(e.live)}) IS NULL AND DB_ID(${O(r)}) IS NOT NULL ALTER DATABASE ${D(r)} MODIFY NAME = ${t}`, `IF DB_ID(${O(e.live)}) IS NOT NULL ALTER DATABASE ${t} SET MULTI_USER`],
|
|
259
|
+
cleanup: [M(e.scratch)]
|
|
260
|
+
};
|
|
261
|
+
}, F = (e) => {
|
|
262
|
+
let t = [];
|
|
263
|
+
for (let n = e, r = 0; n != null && r < 4; r++) {
|
|
264
|
+
let e = n instanceof Error ? n.message : typeof n == "object" && "message" in n ? String(n.message) : String(n);
|
|
265
|
+
e && !t.includes(e) && t.push(e), n = typeof n == "object" && "cause" in n ? n.cause : void 0;
|
|
266
|
+
}
|
|
267
|
+
return t.join(": ");
|
|
268
|
+
}, I = (n, r) => {
|
|
269
|
+
if (r === void 0) return t.fail(new e({
|
|
270
|
+
tool: "sqlpackage",
|
|
271
|
+
reason: "an mssql restore swaps databases and needs a SQL connection to the server; none was provided"
|
|
272
|
+
}));
|
|
273
|
+
let i = P(n), a = (i, a) => r(n.server, a).pipe(t.mapError((t) => new e({
|
|
274
|
+
tool: "sqlpackage",
|
|
275
|
+
reason: `${i}: ${F(t)}`
|
|
276
|
+
}))), o = a("dropping the scratch database", i.cleanup).pipe(t.ignore);
|
|
277
|
+
return t.gen(function* () {
|
|
278
|
+
yield* a(`preparing ${n.scratch}`, i.prepare), yield* L(n.load).pipe(t.tapError(() => o)), yield* a(`carrying ${n.carry.join(", ")} into the imported copy`, i.carry).pipe(t.tapError(() => o)), yield* a(`swapping ${n.scratch} in as ${n.live}`, i.swap).pipe(t.tapError(() => a("putting the original database back", i.undoSwap).pipe(t.ignore)));
|
|
279
|
+
});
|
|
280
|
+
}, L = (n, r) => {
|
|
281
|
+
if (n.kind === "mssql-replace") return I(n, r);
|
|
282
|
+
if (n.kind === "vacuum-into") {
|
|
283
|
+
if (r === void 0) return t.fail(new e({
|
|
284
|
+
tool: "vacuum-into",
|
|
285
|
+
reason: "a VACUUM INTO backup needs a SQL connection to the database"
|
|
286
|
+
}));
|
|
287
|
+
let a = `${n.to}.backing-up-${i()}`;
|
|
288
|
+
return r(n.conn, [`VACUUM INTO '${a.replaceAll("'", "''")}'`]).pipe(t.flatMap(() => t.callback((e) => {
|
|
289
|
+
l(a, n.to, (n) => e(n ? t.fail(n) : t.void));
|
|
290
|
+
})), t.mapError((t) => {
|
|
291
|
+
try {
|
|
292
|
+
d(a);
|
|
293
|
+
} catch {}
|
|
294
|
+
return new e({
|
|
295
|
+
tool: "vacuum-into",
|
|
296
|
+
reason: `VACUUM INTO ${n.to}: ${t instanceof Error ? t.message : String(t)}`
|
|
297
|
+
});
|
|
298
|
+
}));
|
|
299
|
+
}
|
|
300
|
+
return n.kind === "sqlite-backup" ? t.tryPromise({
|
|
301
|
+
try: async () => {
|
|
302
|
+
let { backup: e, DatabaseSync: t } = await import("node:sqlite"), r = `${n.to}.backing-up-${i()}`, a = new t(n.from, { readOnly: !0 });
|
|
303
|
+
try {
|
|
304
|
+
await e(a, r);
|
|
305
|
+
} catch (e) {
|
|
306
|
+
try {
|
|
307
|
+
d(r);
|
|
308
|
+
} catch {}
|
|
309
|
+
throw e;
|
|
310
|
+
} finally {
|
|
311
|
+
a.close();
|
|
312
|
+
}
|
|
313
|
+
await new Promise((e, t) => {
|
|
314
|
+
l(r, n.to, (n) => {
|
|
315
|
+
n ? t(n) : e();
|
|
316
|
+
});
|
|
317
|
+
});
|
|
318
|
+
},
|
|
319
|
+
catch: (t) => new e({
|
|
320
|
+
tool: "sqlite-backup",
|
|
321
|
+
reason: `backup ${n.from} → ${n.to}: ${t instanceof Error ? t.message : String(t)}`
|
|
322
|
+
})
|
|
323
|
+
}) : n.kind === "copy" ? t.callback((r) => {
|
|
324
|
+
let o = `${n.to}.restoring-${i()}`, s = (n) => {
|
|
325
|
+
try {
|
|
326
|
+
d(o);
|
|
327
|
+
} catch {}
|
|
328
|
+
r(t.fail(new e({
|
|
329
|
+
tool: "copy",
|
|
330
|
+
reason: n
|
|
331
|
+
})));
|
|
332
|
+
};
|
|
333
|
+
a(n.from, o, (e) => {
|
|
334
|
+
if (e) s(`copy ${n.from} → ${n.to}: ${e.message}`);
|
|
335
|
+
else {
|
|
336
|
+
for (let e of [
|
|
337
|
+
`${n.to}-wal`,
|
|
338
|
+
`${n.to}-shm`,
|
|
339
|
+
`${n.to}-journal`,
|
|
340
|
+
`${n.to}-log`
|
|
341
|
+
]) try {
|
|
342
|
+
d(e);
|
|
343
|
+
} catch {}
|
|
344
|
+
l(o, n.to, (e) => {
|
|
345
|
+
e ? s(`copy ${n.from} → ${n.to}: ${e.message}`) : r(t.void);
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
});
|
|
349
|
+
}) : t.callback((r) => {
|
|
350
|
+
let i = h(n.tool, [...n.args], {
|
|
351
|
+
env: {
|
|
352
|
+
...process.env,
|
|
353
|
+
...n.env
|
|
354
|
+
},
|
|
355
|
+
stdio: [
|
|
356
|
+
"pipe",
|
|
357
|
+
n.stdoutFile ? "pipe" : "inherit",
|
|
358
|
+
"pipe"
|
|
359
|
+
]
|
|
360
|
+
}), a = "";
|
|
361
|
+
i.stderr?.on("data", (e) => {
|
|
362
|
+
a += String(e);
|
|
363
|
+
});
|
|
364
|
+
let c = !1, l = (e) => {
|
|
365
|
+
c || (c = !0, r(e));
|
|
366
|
+
}, u, p = (e) => {
|
|
367
|
+
u ??= e;
|
|
368
|
+
};
|
|
369
|
+
if (n.stdoutFile && i.stdout) {
|
|
370
|
+
let e = s(n.stdoutFile);
|
|
371
|
+
e.on("error", (e) => p(`writing ${f(n.stdoutFile)}: ${e.message}`)), i.stdout.on("error", (e) => p(`reading the tool's output: ${e.message}`)), i.stdout.pipe(e);
|
|
372
|
+
}
|
|
373
|
+
let m = 0;
|
|
374
|
+
if (n.stdinFile && i.stdin) {
|
|
375
|
+
let e = o(n.stdinFile);
|
|
376
|
+
e.on("data", (e) => {
|
|
377
|
+
m += e.length;
|
|
378
|
+
}), e.on("error", (e) => {
|
|
379
|
+
p(`reading ${f(n.stdinFile)}: ${e.message}`), i.stdin?.end();
|
|
380
|
+
}), i.stdin.on("error", (e) => {
|
|
381
|
+
e.code !== "EPIPE" && p(`feeding ${f(n.stdinFile)} to ${n.tool}: ${e.message}`);
|
|
382
|
+
}), e.pipe(i.stdin);
|
|
383
|
+
}
|
|
384
|
+
let g = () => {
|
|
385
|
+
if (n.stdoutFile !== void 0) try {
|
|
386
|
+
d(n.stdoutFile);
|
|
387
|
+
} catch {}
|
|
388
|
+
};
|
|
389
|
+
i.on("error", (r) => {
|
|
390
|
+
g(), l(t.fail(new e({
|
|
391
|
+
tool: n.tool,
|
|
392
|
+
reason: `failed to launch (is '${n.tool}' installed + on PATH?): ${r.message}`,
|
|
393
|
+
...n.hint === void 0 ? {} : { hint: n.hint }
|
|
394
|
+
})));
|
|
395
|
+
}), i.on("close", (r) => {
|
|
396
|
+
if (r === 0 && u === void 0) {
|
|
397
|
+
l(t.void);
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
g();
|
|
401
|
+
let i = n.stdinFile === void 0 ? "" : ` — ${m} of ${R(n.stdinFile)} byte(s) of ${f(n.stdinFile)} had been fed to it`;
|
|
402
|
+
l(t.fail(new e({
|
|
403
|
+
tool: n.tool,
|
|
404
|
+
reason: (r === 0 ? `exited cleanly, but ${u}` : `exited with code ${r}${u === void 0 ? "" : ` (${u})`}`) + i + (n.stdoutFile === void 0 ? "" : ` — the partial ${f(n.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
405
|
+
stderr: a.trim() === "" ? `(${n.tool} printed nothing to stderr)` : a.slice(-2e3),
|
|
406
|
+
...n.hint === void 0 ? {} : { hint: n.hint }
|
|
407
|
+
})));
|
|
408
|
+
});
|
|
409
|
+
});
|
|
410
|
+
}, R = (e) => {
|
|
411
|
+
try {
|
|
412
|
+
return String(u(e).size);
|
|
413
|
+
} catch {
|
|
414
|
+
return "?";
|
|
415
|
+
}
|
|
416
|
+
};
|
|
417
|
+
//#endregion
|
|
418
|
+
export { C as EXCLUDED_FROM_DUMP, v as backupArtifactName, S as backupCommand, T as dumpExcludedTables, P as mssqlReplaceStatements, A as mssqlScratchName, E as restoreCommand, L as runNativeStep };
|
package/dist/ndjson.js
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { CodecError as e } from "./errors.js";
|
|
2
|
+
import { decodeRow as t, encodeRow as n } from "./typedCodec.js";
|
|
3
|
+
import { Effect as r, Stream as i } from "effect";
|
|
4
|
+
//#region src/ndjson.ts
|
|
5
|
+
var a = new TextEncoder(), o = (e, t) => e.pipe(i.map((e) => a.encode(`${JSON.stringify(n(e, t))}\n`))), s = (n, a, o) => n.pipe(i.decodeText(), i.splitLines, i.filter((e) => e.trim().length > 0), i.mapEffect((n) => r.try({
|
|
6
|
+
try: () => t(JSON.parse(n), o),
|
|
7
|
+
catch: (t) => new e({
|
|
8
|
+
table: a,
|
|
9
|
+
reason: String(t?.message ?? t)
|
|
10
|
+
})
|
|
11
|
+
})));
|
|
12
|
+
//#endregion
|
|
13
|
+
export { s as ndjsonToRows, o as rowsToNdjson };
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { Effect as e } from "effect";
|
|
2
|
+
//#region src/physicalRequest.ts
|
|
3
|
+
var t = (t, n) => e.callback((r) => {
|
|
4
|
+
let i = new AbortController(), a = Promise.resolve().then(() => t(i.signal));
|
|
5
|
+
return a.then((t) => r(e.succeed(t)), (t) => r(e.fail(n(t)))), e.sync(() => {
|
|
6
|
+
i.abort();
|
|
7
|
+
}).pipe(e.andThen(e.promise(() => a.then(() => void 0, () => void 0))));
|
|
8
|
+
});
|
|
9
|
+
//#endregion
|
|
10
|
+
export { t as physicalRequest };
|
package/dist/profile.js
ADDED
package/dist/progress.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Effect as e, Metric as t } from "effect";
|
|
2
|
+
//#region src/progress.ts
|
|
3
|
+
var n = t.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), r = Array.from({ length: 16 }, (e, t) => .01 * 2 ** t), i = t.histogram("voltro_data_transfer_table_seconds", {
|
|
4
|
+
boundaries: r,
|
|
5
|
+
description: "Per-table export/import duration in seconds, tagged by phase."
|
|
6
|
+
}), a = (e, t) => {
|
|
7
|
+
if (e) try {
|
|
8
|
+
e(t);
|
|
9
|
+
} catch {}
|
|
10
|
+
}, o = (r, a, o, s, c, l) => e.Do.pipe(e.bind("start", () => e.clockWith((e) => e.currentTimeMillis)), e.bind("result", () => l), e.tap(({ start: o, result: s }) => e.clockWith((e) => e.currentTimeMillis).pipe(e.flatMap((l) => e.andThen(i.pipe(t.withAttributes({ phase: r }), t.update((l - o) / 1e3)), n.pipe(t.withAttributes({ phase: r }), t.withAttributes({ table: a }), t.update(c(s))))))), e.map(({ result: e }) => e), e.withSpan(`data-transfer.${r}.table`, { attributes: {
|
|
11
|
+
table: a,
|
|
12
|
+
index: o,
|
|
13
|
+
tableCount: s,
|
|
14
|
+
phase: r
|
|
15
|
+
} }));
|
|
16
|
+
//#endregion
|
|
17
|
+
export { a as reportProgress, o as withTableSpan };
|
package/dist/readme.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
//#region src/readme.ts
|
|
2
|
+
var e = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", t = (t, n) => {
|
|
3
|
+
let r = t.tables.reduce((e, t) => e + t.rowCount, 0), i = t.masking ? `✅ **Masked at the source** — ${t.masking.transformed.length} column(s) transformed${t.masking.policyId ? ` (policy \`${t.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", a = t.masking && t.masking.transformed.length > 0 ? `\n\nMasked columns: ${t.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", o = t.assets.count > 0 ? `${t.assets.count} object(s), ${t.assets.totalBytes} bytes` : "none", s = t.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
4
|
+
return `# Voltro data export
|
|
5
|
+
|
|
6
|
+
The machine-readable manifest is \`manifest.json\`; this file is a human summary.
|
|
7
|
+
|
|
8
|
+
| | |
|
|
9
|
+
|---|---|
|
|
10
|
+
| **Generated** | ${t.createdAt} |
|
|
11
|
+
| **Exported from** | \`${n.hostname}\` |
|
|
12
|
+
| **Source dialect** | \`${t.source.dialect}\` |
|
|
13
|
+
| **Format version** | ${t.formatVersion} |
|
|
14
|
+
| **Consistency** | ${t.consistency} |
|
|
15
|
+
| **Compression** | ${t.compression} |
|
|
16
|
+
| **Scope** | ${e(t.scope)} |
|
|
17
|
+
| **Tables** | ${t.tables.length} |
|
|
18
|
+
| **Rows (total)** | ${r} |
|
|
19
|
+
| **Assets** | ${o} |
|
|
20
|
+
|
|
21
|
+
## Data sensitivity
|
|
22
|
+
|
|
23
|
+
${i}${a}
|
|
24
|
+
|
|
25
|
+
## Tables
|
|
26
|
+
|
|
27
|
+
| Table | Rows |
|
|
28
|
+
|---|---|
|
|
29
|
+
${s}
|
|
30
|
+
|
|
31
|
+
## Restore
|
|
32
|
+
|
|
33
|
+
\`\`\`bash
|
|
34
|
+
# into a stopped/replica DB directly:
|
|
35
|
+
voltro data import <this-directory>
|
|
36
|
+
|
|
37
|
+
# or into a running instance (needs the data-transfer secret):
|
|
38
|
+
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
39
|
+
\`\`\`
|
|
40
|
+
`;
|
|
41
|
+
};
|
|
42
|
+
//#endregion
|
|
43
|
+
export { t as buildReadme };
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
//#region src/replacePreflight.ts
|
|
2
|
+
var e = (e, t) => {
|
|
3
|
+
let n = new Set(t), r = [];
|
|
4
|
+
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
5
|
+
let i = e.references?.table;
|
|
6
|
+
i !== void 0 && n.has(i) && r.push({
|
|
7
|
+
from: t.name,
|
|
8
|
+
column: e.name,
|
|
9
|
+
to: i
|
|
10
|
+
});
|
|
11
|
+
}
|
|
12
|
+
return r;
|
|
13
|
+
}, t = (e) => {
|
|
14
|
+
let t = /* @__PURE__ */ new Map();
|
|
15
|
+
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
16
|
+
let n = [...t.entries()].map(([e, t]) => {
|
|
17
|
+
let n = t[0].rowCount, r = t.map((e) => `${e.column} → ${e.to}`).join(", ");
|
|
18
|
+
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
19
|
+
});
|
|
20
|
+
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
21
|
+
};
|
|
22
|
+
//#endregion
|
|
23
|
+
export { t as outsideReferenceRefusal, e as outsideReferencesInto };
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { BundleError as e } from "./errors.js";
|
|
2
|
+
import { runExport as t } from "./exporter.js";
|
|
3
|
+
import { Effect as n } from "effect";
|
|
4
|
+
//#region src/rollbackCapture.ts
|
|
5
|
+
var r = (r) => n.gen(function* () {
|
|
6
|
+
if (r.mode !== "replace" || r.dir === void 0 || r.tables.length === 0) return;
|
|
7
|
+
if (r.snapshot === void 0) return yield* n.fail(new e({
|
|
8
|
+
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
9
|
+
path: r.dir
|
|
10
|
+
}));
|
|
11
|
+
let i = {
|
|
12
|
+
...r.snapshot,
|
|
13
|
+
tables: r.snapshot.tables.filter((e) => r.tables.includes(e.name))
|
|
14
|
+
};
|
|
15
|
+
if (i.tables.length === 0) return;
|
|
16
|
+
r.onWarn?.(`capturing the target's current rows before emptying it → ${r.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
17
|
+
so it survives a run whose process does not.`);
|
|
18
|
+
let a = yield* t({
|
|
19
|
+
store: r.store,
|
|
20
|
+
snapshot: i,
|
|
21
|
+
dialect: r.dialect ?? "unknown",
|
|
22
|
+
outDir: r.dir,
|
|
23
|
+
scope: { kind: "all" },
|
|
24
|
+
compression: "none"
|
|
25
|
+
}).pipe(n.mapError((t) => new e({
|
|
26
|
+
reason: `rollback capture failed, so the import did not start and the target is untouched: ${t.reason ?? String(t)}\n Fix the capture, or re-run with \`--no-rollback\` to proceed without one.`,
|
|
27
|
+
path: r.dir
|
|
28
|
+
})));
|
|
29
|
+
return {
|
|
30
|
+
dir: r.dir,
|
|
31
|
+
rows: a.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
32
|
+
tables: a.tables.length
|
|
33
|
+
};
|
|
34
|
+
}), i = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`;
|
|
35
|
+
//#endregion
|
|
36
|
+
export { r as captureRollback, i as rollbackNotice };
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { classifyConstraintViolation as e, extractDbCause as t } from "@voltro/database";
|
|
2
|
+
//#region src/rowFailureReason.ts
|
|
3
|
+
var n = (e) => {
|
|
4
|
+
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
5
|
+
switch (e.kind) {
|
|
6
|
+
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
7
|
+
case "foreignKeyInUse": return `foreign key${n}: the row is still referenced by other rows`;
|
|
8
|
+
case "unique": return `unique constraint${n}: a row with this value already exists`;
|
|
9
|
+
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
10
|
+
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
11
|
+
}
|
|
12
|
+
}, r = (e) => {
|
|
13
|
+
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
14
|
+
return t.length > 0 ? t.join("/") : void 0;
|
|
15
|
+
}, i = "Failed to execute statement", a = (e) => {
|
|
16
|
+
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
17
|
+
return t === `SqlError: ${i}` ? i : t;
|
|
18
|
+
}, o = (o) => {
|
|
19
|
+
let s = String(o?.stack ?? o), c = s.length <= 1200 ? s : `${s.slice(0, 1200)}…`, l = (e, t) => ({
|
|
20
|
+
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
21
|
+
unclassified: t,
|
|
22
|
+
raw: c
|
|
23
|
+
}), u = t(o), d = e(u), f = r(u), p = f === void 0 ? "" : ` [${f}]`;
|
|
24
|
+
if (d !== void 0) return l(`${n(d)}${p}`, !1);
|
|
25
|
+
let m = typeof u.dbMessage == "string" ? u.dbMessage : void 0, h = o?.message, g = a(typeof h == "string" && h.length > 0 ? h : String(o)), _ = m !== void 0 && m !== g && !m.startsWith("SqlError:") ? m : g, v = typeof u.dbCauseChain == "string" ? u.dbCauseChain : void 0, y = v !== void 0 && /[←‖]/.test(v) ? ` (via ${v})` : "";
|
|
26
|
+
return l(_ === i || _.length === 0 ? `the database refused the write and the driver gave no detail${p}${y}` : `${_}${p}`, _ === i || _.length === 0);
|
|
27
|
+
}, s = (e) => o(e).reason, c = (n) => {
|
|
28
|
+
let i = t(n), a = e(i);
|
|
29
|
+
if (a?.kind === "foreignKeyInUse") {
|
|
30
|
+
let e = a.constraint ?? a.column, t = e === void 0 ? "" : ` ${e}`, n = r(i);
|
|
31
|
+
return `foreign key${t}: rows elsewhere still reference this table${n === void 0 ? "" : ` [${n}]`}`;
|
|
32
|
+
}
|
|
33
|
+
return s(n);
|
|
34
|
+
};
|
|
35
|
+
//#endregion
|
|
36
|
+
export { o as rowFailureDiagnosis, s as rowFailureReason, c as tableFailureReason };
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { fingerprintSchema as e } from "@voltro/database/sql";
|
|
2
|
+
//#region src/schemaDrift.ts
|
|
3
|
+
var t = /__dropped_\d+$/, n = (e, n) => {
|
|
4
|
+
let r = [], i = /* @__PURE__ */ new Map();
|
|
5
|
+
for (let e of n.tables) {
|
|
6
|
+
let t = e.name;
|
|
7
|
+
i.has(t) || i.set(t, e);
|
|
8
|
+
}
|
|
9
|
+
for (let n of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
10
|
+
let e = i.get(n.name);
|
|
11
|
+
if (e === void 0) {
|
|
12
|
+
r.push(`table '${n.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
13
|
+
continue;
|
|
14
|
+
}
|
|
15
|
+
let a = new Map(e.columns.map((e) => [e.name, e])), o = new Map(n.columns.map((e) => [e.name, e]));
|
|
16
|
+
for (let [e, i] of o) {
|
|
17
|
+
let o = a.get(e);
|
|
18
|
+
o === void 0 ? r.push(t.test(e) ? `${n.name}.${e}: a soft-dropped column, present only in the SOURCE — reclaim it there (voltro db gc-snapshots --before <date>) or re-export without it; the target is not missing anything` : `${n.name}.${e}: in the bundle, MISSING from the target — the value has nowhere to go`) : o.type !== i.type && r.push(`${n.name}.${e}: type '${i.type}' (bundle) vs '${o.type}' (target)`);
|
|
19
|
+
}
|
|
20
|
+
for (let [e, t] of a) o.has(e) || !t.nullable && !t.hasDefault && r.push(`${n.name}.${e}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
21
|
+
}
|
|
22
|
+
return r.length > 30 ? [...r.slice(0, 30), `… and ${r.length - 30} more difference(s)`] : r;
|
|
23
|
+
}, r = (t, r, i) => {
|
|
24
|
+
let a = e(r);
|
|
25
|
+
if (a === t || i === void 0) return {
|
|
26
|
+
drifted: !1,
|
|
27
|
+
bundleFingerprint: t,
|
|
28
|
+
targetFingerprint: a,
|
|
29
|
+
diff: []
|
|
30
|
+
};
|
|
31
|
+
let o = n(i, r);
|
|
32
|
+
return {
|
|
33
|
+
drifted: o.length > 0,
|
|
34
|
+
bundleFingerprint: t,
|
|
35
|
+
targetFingerprint: a,
|
|
36
|
+
diff: o
|
|
37
|
+
};
|
|
38
|
+
};
|
|
39
|
+
//#endregion
|
|
40
|
+
export { r as checkSchemaDrift, n as diffSnapshots };
|