@voltro/data-transfer 0.47.0 → 0.49.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/dist/index.d.ts +164 -9
- package/dist/index.js +649 -519
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Runtime as o, Schedule as s, Schema as c, Stream as l } from "effect";
|
|
2
|
-
import {
|
|
2
|
+
import { DATA_TRANSFERS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d, _voltroDataTransfersTable as f, classifyConstraintViolation as p, derivedRowId as m, ensureTableRegistered as h, eq as g, extractDbCause as _, getTable as v, inSet as y, isNotNull as b, or as x, recordsTable as S, registerTable as C, streamTable as w, suspendWriteRecorders as ee, suspendWriteRecordersEffect as te, unregisterTable as ne } from "@voltro/database";
|
|
3
3
|
import { fingerprintSchema as re } from "@voltro/database/sql";
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { STORAGE_REFS_TABLE as
|
|
6
|
-
import * as
|
|
7
|
-
import { PassThrough as
|
|
8
|
-
import { copyFile as
|
|
4
|
+
import { createCipheriv as T, createDecipheriv as ie, createHash as E, createHmac as ae, randomBytes as D, scrypt as oe } from "node:crypto";
|
|
5
|
+
import { STORAGE_REFS_TABLE as se, fromNodeReadable as ce, getObjectStream as le, putObjectStream as O, toNodeReadable as k } from "@voltro/plugin-storage";
|
|
6
|
+
import * as A from "node:zlib";
|
|
7
|
+
import { PassThrough as ue, Readable as j } from "node:stream";
|
|
8
|
+
import { copyFile as de, createReadStream as fe, createWriteStream as M, existsSync as N, unlinkSync as pe } from "node:fs";
|
|
9
9
|
import { SqlClient as P } from "@effect/sql";
|
|
10
|
-
import { mkdir as F, readdir as
|
|
11
|
-
import { basename as
|
|
12
|
-
import { promisify as
|
|
13
|
-
import { spawn as
|
|
10
|
+
import { mkdir as F, readdir as me, stat as he } from "node:fs/promises";
|
|
11
|
+
import { basename as ge, delimiter as I, dirname as _e, join as L, resolve as R, sep as ve } from "node:path";
|
|
12
|
+
import { promisify as ye } from "node:util";
|
|
13
|
+
import { spawn as be } from "node:child_process";
|
|
14
14
|
//#region src/consistency.ts
|
|
15
|
-
var
|
|
15
|
+
var xe = (e) => {
|
|
16
16
|
switch (e) {
|
|
17
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
18
18
|
case "mysql":
|
|
@@ -22,7 +22,7 @@ var be = (e) => {
|
|
|
22
22
|
case "turso": return [];
|
|
23
23
|
default: return [];
|
|
24
24
|
}
|
|
25
|
-
},
|
|
25
|
+
}, Se = {
|
|
26
26
|
email: { fake: "email" },
|
|
27
27
|
fullName: { fake: "fullName" },
|
|
28
28
|
firstName: { fake: "firstName" },
|
|
@@ -37,7 +37,7 @@ var be = (e) => {
|
|
|
37
37
|
date: "dateShift",
|
|
38
38
|
secret: "null",
|
|
39
39
|
freeText: "redact"
|
|
40
|
-
},
|
|
40
|
+
}, Ce = (e) => {
|
|
41
41
|
let t = {};
|
|
42
42
|
for (let n of e.tables) {
|
|
43
43
|
let e = {};
|
|
@@ -49,15 +49,15 @@ var be = (e) => {
|
|
|
49
49
|
t[n.name] = e;
|
|
50
50
|
}
|
|
51
51
|
return t;
|
|
52
|
-
},
|
|
52
|
+
}, we = (e, t, n, r) => {
|
|
53
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
54
54
|
if (i) return i;
|
|
55
55
|
if (n?.sensitive) {
|
|
56
56
|
let e = n.sensitive.class;
|
|
57
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? Se[e] ?? "redact";
|
|
58
58
|
}
|
|
59
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
60
|
-
},
|
|
60
|
+
}, Te = (e, t) => ae("sha256", e).update(t == null ? "∅" : String(t)).digest(), z = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), Ee = [
|
|
61
61
|
"Ada",
|
|
62
62
|
"Alan",
|
|
63
63
|
"Grace",
|
|
@@ -70,7 +70,7 @@ var be = (e) => {
|
|
|
70
70
|
"Guido",
|
|
71
71
|
"Margaret",
|
|
72
72
|
"Bjarne"
|
|
73
|
-
],
|
|
73
|
+
], De = [
|
|
74
74
|
"Lovelace",
|
|
75
75
|
"Turing",
|
|
76
76
|
"Hopper",
|
|
@@ -83,7 +83,7 @@ var be = (e) => {
|
|
|
83
83
|
"Rossum",
|
|
84
84
|
"Hamilton",
|
|
85
85
|
"Stroustrup"
|
|
86
|
-
],
|
|
86
|
+
], Oe = [
|
|
87
87
|
"Maple",
|
|
88
88
|
"Oak",
|
|
89
89
|
"Cedar",
|
|
@@ -92,7 +92,7 @@ var be = (e) => {
|
|
|
92
92
|
"Birch",
|
|
93
93
|
"Willow",
|
|
94
94
|
"Ash"
|
|
95
|
-
],
|
|
95
|
+
], ke = [
|
|
96
96
|
"Initech",
|
|
97
97
|
"Hooli",
|
|
98
98
|
"Umbrella",
|
|
@@ -101,23 +101,23 @@ var be = (e) => {
|
|
|
101
101
|
"Soylent",
|
|
102
102
|
"Stark",
|
|
103
103
|
"Wayne"
|
|
104
|
-
],
|
|
105
|
-
let r =
|
|
104
|
+
], B = (e, t, n) => e[z(t, n) % e.length], Ae = (e, t, n) => {
|
|
105
|
+
let r = Te(n, `${e}:${String(t)}`), i = z(r, 1) % 1e4;
|
|
106
106
|
switch (e) {
|
|
107
|
-
case "email": return `${
|
|
108
|
-
case "fullName": return `${
|
|
109
|
-
case "firstName": return
|
|
110
|
-
case "lastName": return
|
|
111
|
-
case "username": return `${
|
|
112
|
-
case "phone": return `+1${String(
|
|
113
|
-
case "address": return `${
|
|
114
|
-
case "company": return
|
|
107
|
+
case "email": return `${B(Ee, r, 0).toLowerCase()}.${B(De, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${B(Ee, r, 0)} ${B(De, r, 1)}`;
|
|
109
|
+
case "firstName": return B(Ee, r, 0);
|
|
110
|
+
case "lastName": return B(De, r, 1);
|
|
111
|
+
case "username": return `${B(Ee, r, 0).toLowerCase()}${i}`;
|
|
112
|
+
case "phone": return `+1${String(z(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
+
case "address": return `${z(r, 0) % 9899 + 100} ${B(Oe, r, 1)} St`;
|
|
114
|
+
case "company": return B(ke, r, 0);
|
|
115
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
116
|
-
case "ip": return `${
|
|
117
|
-
case "creditCard": return
|
|
116
|
+
case "ip": return `${z(r, 0) % 256}.${z(r, 1) % 256}.${z(r, 2) % 256}.${z(r, 3) % 254 + 1}`;
|
|
117
|
+
case "creditCard": return je(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
118
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
119
119
|
}
|
|
120
|
-
},
|
|
120
|
+
}, je = (e) => {
|
|
121
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
122
122
|
t.forEach((e, t) => {
|
|
123
123
|
let r = e;
|
|
@@ -125,31 +125,31 @@ var be = (e) => {
|
|
|
125
125
|
});
|
|
126
126
|
let r = (10 - n % 10) % 10;
|
|
127
127
|
return e.slice(0, 15) + String(r);
|
|
128
|
-
},
|
|
128
|
+
}, Me = 864e5, Ne = (e, t) => {
|
|
129
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
130
130
|
if (Number.isNaN(n)) return e;
|
|
131
|
-
let r =
|
|
131
|
+
let r = z(Te(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Me);
|
|
132
132
|
return e instanceof Date ? i : i.toISOString();
|
|
133
|
-
},
|
|
133
|
+
}, Pe = (e, t) => {
|
|
134
134
|
if (t.value === null || t.value === void 0) return null;
|
|
135
135
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
136
136
|
if (e === "keep") return t.value;
|
|
137
137
|
if (e === "null") return null;
|
|
138
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
139
|
-
if (e === "hash") return
|
|
140
|
-
if (e === "dateShift") return
|
|
141
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return Te(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Ne(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return Ae(e.fake, t.value, t.seed);
|
|
142
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
143
143
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
144
|
-
},
|
|
144
|
+
}, Fe = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Ie = (e, t, n) => {
|
|
145
145
|
let r = {}, i = [], a = [];
|
|
146
146
|
for (let o of e) for (let e of o.columns) {
|
|
147
|
-
let s = `${o.name}.${e.name}`, c =
|
|
147
|
+
let s = `${o.name}.${e.name}`, c = we(o.name, e.name, t[o.name]?.[e.name], n);
|
|
148
148
|
if (c === "error") {
|
|
149
149
|
i.push(s);
|
|
150
150
|
continue;
|
|
151
151
|
}
|
|
152
|
-
if (!
|
|
152
|
+
if (!Fe(c)) {
|
|
153
153
|
a.push({
|
|
154
154
|
column: s,
|
|
155
155
|
action: c
|
|
@@ -163,7 +163,7 @@ var be = (e) => {
|
|
|
163
163
|
unclassified: i,
|
|
164
164
|
invalid: a
|
|
165
165
|
};
|
|
166
|
-
},
|
|
166
|
+
}, Le = [
|
|
167
167
|
{
|
|
168
168
|
name: "email",
|
|
169
169
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -180,7 +180,7 @@ var be = (e) => {
|
|
|
180
180
|
name: "creditCard",
|
|
181
181
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
182
182
|
}
|
|
183
|
-
],
|
|
183
|
+
], Re = (e, t, n) => {
|
|
184
184
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
185
185
|
for (let [a, o] of Object.entries(n)) {
|
|
186
186
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -188,7 +188,7 @@ var be = (e) => {
|
|
|
188
188
|
for (let a of e) {
|
|
189
189
|
let e = a[n];
|
|
190
190
|
if (typeof e != "string" || e.length === 0) continue;
|
|
191
|
-
let o =
|
|
191
|
+
let o = Le.find((t) => t.re.test(e));
|
|
192
192
|
if (o && !i.has(n)) {
|
|
193
193
|
i.add(n), r.push({
|
|
194
194
|
table: t,
|
|
@@ -201,9 +201,9 @@ var be = (e) => {
|
|
|
201
201
|
}
|
|
202
202
|
}
|
|
203
203
|
return r;
|
|
204
|
-
},
|
|
204
|
+
}, ze = (e, t, n, r, i) => {
|
|
205
205
|
let a = {};
|
|
206
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
206
|
+
for (let [o, s] of Object.entries(e)) a[o] = Pe(n[`${t}.${o}`] ?? "keep", {
|
|
207
207
|
value: s,
|
|
208
208
|
table: t,
|
|
209
209
|
column: o,
|
|
@@ -211,34 +211,84 @@ var be = (e) => {
|
|
|
211
211
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
212
212
|
});
|
|
213
213
|
return a;
|
|
214
|
-
},
|
|
214
|
+
}, Be = /* @__PURE__ */ new Map([
|
|
215
215
|
["_voltro_migration_plans", "the schema fingerprint THIS deployment applied — a foreign row makes the next boot refuse"],
|
|
216
216
|
["_voltro_migrations", "which file-based migrations ran HERE — a foreign row marks unrun migrations as done"],
|
|
217
217
|
["_voltro_seeds", "which seeds ran HERE — a foreign row suppresses seeding the target still needs"],
|
|
218
218
|
["_voltro_cdc_offsets", "binlog/WAL positions of the SOURCE server — meaningless and unsafe against another one"],
|
|
219
219
|
["_voltro_schedule_claims", "live leases held by THIS deployment's replicas — a foreign row hands out a lease nobody holds"],
|
|
220
|
-
["
|
|
220
|
+
["_voltro_data_transfers", "which bundles moved through HERE, in or out — a foreign row claims a transfer this deployment never ran"],
|
|
221
221
|
["_voltro_replace_in_progress", "the marker a destructive import HERE left behind — a foreign row refuses a boot over a healthy database"],
|
|
222
222
|
["_voltro_wakeups", "pending wakeups addressed to THIS deployment's runners"],
|
|
223
223
|
["_voltro_workflow_watermarks", "how far a consumer has read THIS deployment's event journal — a foreign position skips events"],
|
|
224
224
|
["_voltro_workflow_pending", "starts waiting to be drained HERE — a foreign row makes the target RUN a workflow somebody queued elsewhere"],
|
|
225
225
|
["_voltro_workflow_admissions", "the in-flight accounting a concurrency gate decides on — a foreign row makes the target refuse or admit wrongly"],
|
|
226
|
-
["_voltro_workflow_pauses", "which workflows an operator paused HERE — a foreign row silently pauses them in the target"]
|
|
227
|
-
|
|
226
|
+
["_voltro_workflow_pauses", "which workflows an operator paused HERE — a foreign row silently pauses them in the target"],
|
|
227
|
+
["_voltro_workflow_event_deliveries", "which (event, trigger) pairs were already delivered HERE — a foreign row SUPPRESSES a delivery the target owes"],
|
|
228
|
+
["_voltro_traces", "spans recorded HERE — a foreign span claims a request this deployment never served"],
|
|
229
|
+
["_voltro_undo_log", "the before-images of mutations made HERE — a foreign row offers to undo a write into rows the target does not have"],
|
|
230
|
+
["_voltro_outbox", "messages waiting to be SENT by this deployment — a foreign row makes the target deliver something queued elsewhere"],
|
|
231
|
+
["_voltro_outbox_attempts", "the delivery attempts this deployment made — they belong to its outbox rows and to no other"],
|
|
232
|
+
["_voltro_ai_inferences", "offloaded inference work queued for THIS deployment's runners — a foreign row makes the target run somebody else's"],
|
|
233
|
+
["_voltro_ai_budget", "the live spend RESERVATION counter a budget refuses against — a foreign row makes the target refuse calls it should allow"],
|
|
234
|
+
["_voltro_ai_usage", "what this deployment spent on model calls — a foreign row overstates a bill nobody here ran up"],
|
|
235
|
+
["_voltro_budget_holds", "in-flight holds against a running deployment's budget — the same accounting `_voltro_workflow_admissions` keeps"],
|
|
236
|
+
["_voltro_storage_grants", "short-lived upload/download grants issued HERE — a foreign grant authorises nothing in the target"],
|
|
237
|
+
["_voltro_storage_usage", "how much storage this deployment accounts for — a foreign row makes a quota decision on somebody else's bytes"],
|
|
238
|
+
["_voltro_idempotency", "the request keys this deployment has already answered — a foreign key SUPPRESSES a request the target never served"],
|
|
239
|
+
["_voltro_webhook_deliveries", "the delivery attempts this deployment made to a subscriber — a foreign row claims a call the target never placed"],
|
|
240
|
+
["_voltro_schedule_runs", "every schedule firing that happened HERE, replica ids included — a foreign row is history the target did not live"],
|
|
241
|
+
["_voltro_cdc_log", "the change rows this deployment captured from its own database — a foreign row replays a change that never happened here"],
|
|
242
|
+
["_voltro_notification_deliveries", "the notification attempts this deployment made — a foreign row claims a send the target never performed"],
|
|
243
|
+
["_voltro_notification_held", "notifications waiting to be SENT from here — a foreign row makes the target deliver something queued elsewhere"],
|
|
244
|
+
["_voltro_webhook_event_stats", "per-deployment delivery counters a backoff decision reads — foreign counts steer the target's retries"],
|
|
245
|
+
["_voltro_webhook_rate_windows", "the live rate-limit windows this deployment is inside — a foreign window throttles or releases wrongly"]
|
|
246
|
+
]), Ve = /* @__PURE__ */ new Map([
|
|
247
|
+
["_voltro_agent_threads", "conversations the app's users had — app data"],
|
|
248
|
+
["_voltro_agent_messages", "the messages in those conversations — app data"],
|
|
249
|
+
["_voltro_prompts", "prompt definitions the app authored — app data"],
|
|
250
|
+
["_voltro_api_keys", "keys the app issued (hashed) — app data, and a restore without them locks callers out"],
|
|
251
|
+
["_voltro_kv", "the app's own key/value rows — app data by construction"],
|
|
252
|
+
["_voltro_approvals", "human decisions on workflow runs — they belong with the runs"],
|
|
253
|
+
["_voltro_connections", "third-party connections the app's users authorised. Portable as ROWS; note the secrets are encrypted with the project's key, so a target without that key holds ciphertext it cannot read"],
|
|
254
|
+
["_voltro_connection_grants", "who may use which connection — belongs with the connections"],
|
|
255
|
+
["_voltro_storage_blobs", "the stored objects' metadata — the assets travel with the bundle"],
|
|
256
|
+
["_voltro_storage_refs", "which rows reference which blob — belongs with the blobs"],
|
|
257
|
+
["_voltro_webhook_targets", "the endpoints subscribers registered through the app — app data"],
|
|
258
|
+
["_voltro_workflow_runs", "what the app's workflows did — history the app owns"],
|
|
259
|
+
["_voltro_workflow_run_steps", "the steps of those runs — belongs with the runs"],
|
|
260
|
+
["_voltro_workflow_run_events", "the events those runs emitted — belongs with the runs"],
|
|
261
|
+
["_voltro_workflow_events", "the app's event journal — app data (the target keeps its OWN watermarks, which are local)"],
|
|
262
|
+
["_voltro_workflow_start_contexts", "the context a start was made with — belongs with the runs"],
|
|
263
|
+
["_voltro_events", "the app's domain events — app data"],
|
|
264
|
+
["_voltro_ai_flows", "the flow definitions the app authored — app data"],
|
|
265
|
+
["_voltro_ai_flow_runs", "what those flows did — history the app owns, like a workflow run"],
|
|
266
|
+
["_voltro_notification_inbox", "what the app's users received — app data, and a restore without it loses their inbox"],
|
|
267
|
+
["_voltro_notification_preferences", "the per-user delivery settings the app's users chose — app data"],
|
|
268
|
+
["_voltro_notification_quiet_hours", "the per-user quiet hours the app's users chose — app data"],
|
|
269
|
+
["_voltro_notification_topic_subscriptions", "which topics the app's users subscribed to — app data"],
|
|
270
|
+
["_voltro_rebac_tuples", "the relationship tuples the app's authorisation decisions are made from — app data"],
|
|
271
|
+
["_voltro_scim_users", "identities an external directory provisioned into the app — app data"],
|
|
272
|
+
["_voltro_scim_groups", "groups an external directory provisioned into the app — app data"],
|
|
273
|
+
["_voltro_row_history", "the app's own row-version history — the point of keeping versions is that they survive a move"],
|
|
274
|
+
["_voltro_audit_log", "the app's audit trail — compliance data the app owns, not deployment bookkeeping"],
|
|
275
|
+
["_voltro_feature_flags", "the flags the app defined — app configuration"],
|
|
276
|
+
["_voltro_feature_flag_audit", "who changed which flag — belongs with the flags"]
|
|
277
|
+
]), V = (e) => Be.has(e), He = (e) => Be.get(e), Ue = (e, t) => {
|
|
228
278
|
if (e.length === 0) return;
|
|
229
|
-
let n = e.map((e) => ` ${e} — ${
|
|
279
|
+
let n = e.map((e) => ` ${e} — ${He(e) ?? "describes this deployment"}`);
|
|
230
280
|
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
231
|
-
},
|
|
281
|
+
}, H = class extends c.TaggedError()("BundleError", {
|
|
232
282
|
reason: c.String,
|
|
233
283
|
path: c.optional(c.String)
|
|
234
|
-
}) {},
|
|
284
|
+
}) {}, We = class extends c.TaggedError()("CodecError", {
|
|
235
285
|
table: c.String,
|
|
236
286
|
reason: c.String
|
|
237
|
-
}) {},
|
|
287
|
+
}) {}, Ge = class extends c.TaggedError()("IntegrityError", {
|
|
238
288
|
what: c.String,
|
|
239
289
|
expected: c.String,
|
|
240
290
|
actual: c.String
|
|
241
|
-
}) {},
|
|
291
|
+
}) {}, Ke = class extends t.TaggedError("CompressionError") {}, U = class extends c.TaggedError()("ScopeError", { reason: c.String }) {}, qe = class extends c.TaggedError()("RowsRefusedError", {
|
|
242
292
|
rows: c.Array(c.Struct({
|
|
243
293
|
table: c.String,
|
|
244
294
|
id: c.String,
|
|
@@ -252,29 +302,29 @@ var be = (e) => {
|
|
|
252
302
|
refused: c.Number,
|
|
253
303
|
primary: c.Number
|
|
254
304
|
}))
|
|
255
|
-
}) {},
|
|
305
|
+
}) {}, Je = class extends c.TaggedError()("SchemaDriftError", {
|
|
256
306
|
bundleFingerprint: c.String,
|
|
257
307
|
targetFingerprint: c.String,
|
|
258
308
|
diff: c.Array(c.String)
|
|
259
|
-
}) {},
|
|
309
|
+
}) {}, W = class extends t.TaggedError("NativeToolError") {
|
|
260
310
|
get message() {
|
|
261
311
|
let e = this.stderr?.trim();
|
|
262
312
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
263
313
|
}
|
|
264
|
-
},
|
|
314
|
+
}, Ye = class extends c.TaggedError()("MaskingError", {
|
|
265
315
|
unclassified: c.Array(c.String),
|
|
266
316
|
invalidActions: c.optional(c.Array(c.String))
|
|
267
|
-
}) {},
|
|
317
|
+
}) {}, Xe = class extends c.TaggedError()("ImportModeError", {
|
|
268
318
|
mode: c.String,
|
|
269
319
|
reason: c.String
|
|
270
|
-
}) {},
|
|
320
|
+
}) {}, Ze = class extends c.TaggedError()("CrossDialectError", {
|
|
271
321
|
sourceDialect: c.String,
|
|
272
322
|
targetDialect: c.String,
|
|
273
323
|
blocking: c.Array(c.String)
|
|
274
|
-
}) {},
|
|
324
|
+
}) {}, G = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Qe = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
275
325
|
col: e.name,
|
|
276
326
|
refTable: e.references.table
|
|
277
|
-
})),
|
|
327
|
+
})), $e = (e, t, r) => n.gen(function* () {
|
|
278
328
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), u = (e) => {
|
|
279
329
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
280
330
|
return s.set(e, t), t;
|
|
@@ -292,10 +342,10 @@ var be = (e) => {
|
|
|
292
342
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
293
343
|
};
|
|
294
344
|
}, p = (r, i) => {
|
|
295
|
-
let a = f(r,
|
|
296
|
-
return l.runForEach(
|
|
345
|
+
let a = f(r, G(t, r), Qe(t, r));
|
|
346
|
+
return l.runForEach(w(e, {
|
|
297
347
|
table: r,
|
|
298
|
-
pk:
|
|
348
|
+
pk: G(t, r),
|
|
299
349
|
...i === void 0 ? {} : { where: i }
|
|
300
350
|
}), (e) => n.sync(() => a(e)));
|
|
301
351
|
}, m = n.gen(function* () {
|
|
@@ -304,7 +354,7 @@ var be = (e) => {
|
|
|
304
354
|
c.clear();
|
|
305
355
|
for (let [n, r] of e) {
|
|
306
356
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
307
|
-
e.length !== 0 && (yield* p(n,
|
|
357
|
+
e.length !== 0 && (yield* p(n, y(G(t, n), e)));
|
|
308
358
|
}
|
|
309
359
|
}
|
|
310
360
|
});
|
|
@@ -319,29 +369,32 @@ var be = (e) => {
|
|
|
319
369
|
refTable: e.references?.table
|
|
320
370
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
321
371
|
if (a.length === 0) continue;
|
|
322
|
-
let o = s.get(t)?.size ?? 0, c = a.map((e) =>
|
|
323
|
-
yield* p(t, c.length === 1 ? c[0] :
|
|
372
|
+
let o = s.get(t)?.size ?? 0, c = a.map((e) => y(e.col, [...s.get(e.refTable)]));
|
|
373
|
+
yield* p(t, c.length === 1 ? c[0] : x(...c)), (s.get(t)?.size ?? 0) > o && !e.has(t) && r.push(t), (s.get(t)?.size ?? 0) > o && (n = !0);
|
|
324
374
|
}
|
|
325
375
|
for (let t of r) e.add(t);
|
|
326
376
|
if (yield* m, !n) break;
|
|
327
377
|
}
|
|
328
378
|
}
|
|
329
379
|
return o;
|
|
330
|
-
}),
|
|
380
|
+
}), et = (e, t) => ({
|
|
331
381
|
tables: [...e.keys()],
|
|
332
382
|
predicateFor: (n) => {
|
|
333
383
|
let r = e.get(n);
|
|
334
|
-
return r ?
|
|
384
|
+
return r ? y(G(t, n), [...r]) : void 0;
|
|
335
385
|
}
|
|
336
|
-
}),
|
|
386
|
+
}), tt = "tenantId", nt = (e, t, r) => n.gen(function* () {
|
|
337
387
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
338
|
-
if (e.kind === "all")
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
388
|
+
if (e.kind === "all") {
|
|
389
|
+
let t = new Set(e.exclude ?? []);
|
|
390
|
+
return {
|
|
391
|
+
tables: i.filter((e) => !V(e) && !t.has(e)),
|
|
392
|
+
predicateFor: () => void 0
|
|
393
|
+
};
|
|
394
|
+
}
|
|
342
395
|
if (e.kind === "tables") {
|
|
343
396
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
344
|
-
if (a.length > 0) return yield* n.fail(new
|
|
397
|
+
if (a.length > 0) return yield* n.fail(new U({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
345
398
|
let o = new Set(t);
|
|
346
399
|
return {
|
|
347
400
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -349,30 +402,30 @@ var be = (e) => {
|
|
|
349
402
|
};
|
|
350
403
|
}
|
|
351
404
|
let o = e.tenantId;
|
|
352
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
353
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
354
|
-
if (s.length === 0) return yield* n.fail(new
|
|
355
|
-
let c = yield*
|
|
356
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
405
|
+
if (o === void 0 || o === "") return yield* n.fail(new U({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
406
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === tt) ?? !1);
|
|
407
|
+
if (s.length === 0) return yield* n.fail(new U({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${tt}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
408
|
+
let c = yield* $e(r.store, t, {
|
|
409
|
+
seeds: Object.fromEntries(s.map((e) => [e, g(tt, o)])),
|
|
357
410
|
pinned: s,
|
|
358
411
|
children: { roots: s }
|
|
359
412
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
360
413
|
return {
|
|
361
414
|
tables: [...s, ...l],
|
|
362
|
-
predicateFor: (e) => u.has(e) ?
|
|
415
|
+
predicateFor: (e) => u.has(e) ? g(tt, o) : c.has(e) ? y(G(t, e), [...c.get(e)]) : void 0
|
|
363
416
|
};
|
|
364
|
-
}),
|
|
417
|
+
}), rt = (e, t) => ({
|
|
365
418
|
list: t,
|
|
366
|
-
open: (t) =>
|
|
367
|
-
}),
|
|
368
|
-
write: (t, n, r) =>
|
|
419
|
+
open: (t) => le(e, t)
|
|
420
|
+
}), it = (e) => ({
|
|
421
|
+
write: (t, n, r) => O(e, t, n, r),
|
|
369
422
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
370
|
-
}),
|
|
423
|
+
}), at = (e, t) => n.tryPromise({
|
|
371
424
|
try: async () => {
|
|
372
425
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
373
426
|
u.update(e), d += e.length, n(null, e);
|
|
374
427
|
} });
|
|
375
|
-
await o(await
|
|
428
|
+
await o(await k(e.stream), f, n(l));
|
|
376
429
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
377
430
|
try {
|
|
378
431
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -385,23 +438,23 @@ var be = (e) => {
|
|
|
385
438
|
contentType: e.contentType
|
|
386
439
|
};
|
|
387
440
|
},
|
|
388
|
-
catch: (e) => new
|
|
441
|
+
catch: (e) => new H({
|
|
389
442
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
390
443
|
path: t
|
|
391
444
|
})
|
|
392
|
-
}),
|
|
393
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u =
|
|
445
|
+
}), ot = (e, t, r) => n.gen(function* () {
|
|
446
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u = ce("bundle", () => a(s)).pipe(l.tap((e) => n.sync(() => c.update(e))));
|
|
394
447
|
yield* e.write(r.key, u, {
|
|
395
448
|
contentType: r.contentType,
|
|
396
449
|
totalSize: r.size
|
|
397
450
|
});
|
|
398
451
|
let d = c.digest("hex");
|
|
399
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
452
|
+
if (d !== r.sha256) return yield* n.fail(new Ge({
|
|
400
453
|
what: `asset ${r.key}`,
|
|
401
454
|
expected: r.sha256,
|
|
402
455
|
actual: d
|
|
403
456
|
}));
|
|
404
|
-
}),
|
|
457
|
+
}), st = () => typeof A.createZstdCompress == "function", ct = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : st() ? "zstd" : "gzip", lt = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", ut = (e) => e === "zstd" ? A.createZstdCompress() : e === "gzip" ? A.createGzip() : new ue(), dt = (e) => e === "zstd" ? A.createZstdDecompress() : e === "gzip" ? A.createGunzip() : new ue(), ft = 1, pt = c.Struct({
|
|
405
458
|
name: c.String,
|
|
406
459
|
file: c.String,
|
|
407
460
|
rowCount: c.Number,
|
|
@@ -411,11 +464,12 @@ var be = (e) => {
|
|
|
411
464
|
value: c.String
|
|
412
465
|
}),
|
|
413
466
|
primaryKey: c.String
|
|
414
|
-
}),
|
|
467
|
+
}), mt = c.Struct({
|
|
415
468
|
kind: c.Literal("all", "tenant", "tables"),
|
|
416
469
|
tenantId: c.optional(c.String),
|
|
417
|
-
tables: c.optional(c.Array(c.String))
|
|
418
|
-
|
|
470
|
+
tables: c.optional(c.Array(c.String)),
|
|
471
|
+
exclude: c.optional(c.Array(c.String))
|
|
472
|
+
}), ht = c.Struct({
|
|
419
473
|
count: c.Number,
|
|
420
474
|
totalBytes: c.Number,
|
|
421
475
|
index: c.optional(c.String)
|
|
@@ -426,23 +480,23 @@ c.Struct({
|
|
|
426
480
|
size: c.Number,
|
|
427
481
|
contentType: c.String
|
|
428
482
|
});
|
|
429
|
-
var
|
|
483
|
+
var gt = c.Struct({
|
|
430
484
|
formatVersion: c.Number,
|
|
431
485
|
createdAt: c.String,
|
|
432
486
|
source: c.Struct({
|
|
433
487
|
dialect: c.String,
|
|
434
488
|
schemaFingerprint: c.String
|
|
435
489
|
}),
|
|
436
|
-
scope:
|
|
490
|
+
scope: mt,
|
|
437
491
|
consistency: c.Literal("live", "snapshot"),
|
|
438
492
|
compression: c.Literal("zstd", "gzip", "none"),
|
|
439
|
-
tables: c.Array(
|
|
440
|
-
assets:
|
|
493
|
+
tables: c.Array(pt),
|
|
494
|
+
assets: ht,
|
|
441
495
|
masking: c.optional(c.Struct({
|
|
442
496
|
policyId: c.optional(c.String),
|
|
443
497
|
transformed: c.Array(c.String)
|
|
444
498
|
}))
|
|
445
|
-
}),
|
|
499
|
+
}), _t = c.encodeSync(c.parseJson(gt)), vt = c.decodeUnknownSync(c.parseJson(gt)), K = "manifest.json", yt = "README.md", bt = "data", xt = "assets", St = "assets/index.ndjson", Ct = ".ledger.json", wt = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", Tt = (e, t) => {
|
|
446
500
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
447
501
|
return `# Voltro data export
|
|
448
502
|
|
|
@@ -456,7 +510,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
456
510
|
| **Format version** | ${e.formatVersion} |
|
|
457
511
|
| **Consistency** | ${e.consistency} |
|
|
458
512
|
| **Compression** | ${e.compression} |
|
|
459
|
-
| **Scope** | ${
|
|
513
|
+
| **Scope** | ${wt(e.scope)} |
|
|
460
514
|
| **Tables** | ${e.tables.length} |
|
|
461
515
|
| **Rows (total)** | ${n} |
|
|
462
516
|
| **Assets** | ${a} |
|
|
@@ -481,39 +535,39 @@ voltro data import <this-directory>
|
|
|
481
535
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
482
536
|
\`\`\`
|
|
483
537
|
`;
|
|
484
|
-
},
|
|
538
|
+
}, Et = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), Dt = a.exponential({
|
|
485
539
|
start: .01,
|
|
486
540
|
factor: 2,
|
|
487
541
|
count: 16
|
|
488
|
-
}),
|
|
542
|
+
}), Ot = i.histogram("voltro_data_transfer_table_seconds", Dt, "Per-table export/import duration in seconds, tagged by phase."), q = (e, t) => {
|
|
489
543
|
if (e) try {
|
|
490
544
|
e(t);
|
|
491
545
|
} catch {}
|
|
492
|
-
},
|
|
546
|
+
}, kt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(Ot.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), Et.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
493
547
|
table: t,
|
|
494
548
|
index: r,
|
|
495
549
|
tableCount: a,
|
|
496
550
|
phase: e
|
|
497
|
-
} })),
|
|
551
|
+
} })), At = () => ({
|
|
498
552
|
tables: {},
|
|
499
553
|
assets: {}
|
|
500
|
-
}),
|
|
554
|
+
}), jt = (e) => n.tryPromise(async () => {
|
|
501
555
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
502
556
|
return JSON.parse(t);
|
|
503
|
-
}).pipe(n.orElseSucceed(
|
|
557
|
+
}).pipe(n.orElseSucceed(At)), J = (e, t) => n.tryPromise({
|
|
504
558
|
try: async () => {
|
|
505
559
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
506
560
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
507
561
|
},
|
|
508
|
-
catch: (t) => new
|
|
562
|
+
catch: (t) => new H({
|
|
509
563
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
510
564
|
path: e
|
|
511
565
|
})
|
|
512
|
-
}),
|
|
566
|
+
}), Mt = (e, t) => t in e.tables, Nt = (e, t) => t in e.assets, Pt = /* @__PURE__ */ new Set([
|
|
513
567
|
"json",
|
|
514
568
|
"array",
|
|
515
569
|
"vector"
|
|
516
|
-
]),
|
|
570
|
+
]), Ft = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", It = (e) => Buffer.from(e).toString("base64"), Lt = (e) => new Uint8Array(Buffer.from(e, "base64")), Rt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), zt = (e, t) => e == null ? null : t !== void 0 && Pt.has(t) ? e : t === "bytes" ? It(e) : t === "timestamp" || t === "date" ? Rt(e) : typeof e == "bigint" ? {
|
|
517
571
|
$v: "bigint",
|
|
518
572
|
d: e.toString()
|
|
519
573
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -524,30 +578,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
524
578
|
d: e.toISOString()
|
|
525
579
|
} : e instanceof Uint8Array ? {
|
|
526
580
|
$v: "bytes",
|
|
527
|
-
d:
|
|
528
|
-
} : e,
|
|
581
|
+
d: It(e)
|
|
582
|
+
} : e, Bt = (e) => {
|
|
529
583
|
switch (e.$v) {
|
|
530
584
|
case "bigint": return BigInt(e.d);
|
|
531
585
|
case "number": return Number(e.d);
|
|
532
586
|
case "date": return new Date(e.d);
|
|
533
|
-
case "bytes": return
|
|
587
|
+
case "bytes": return Lt(e.d);
|
|
534
588
|
default: return e;
|
|
535
589
|
}
|
|
536
|
-
},
|
|
590
|
+
}, Vt = (e, t) => e == null ? null : t !== void 0 && Pt.has(t) ? e : Ft(e) ? Bt(e) : t === "bytes" ? Lt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Ht = (e, t) => {
|
|
537
591
|
let n = {};
|
|
538
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
592
|
+
for (let [r, i] of Object.entries(e)) n[r] = zt(i, t[r]);
|
|
539
593
|
return n;
|
|
540
|
-
},
|
|
594
|
+
}, Ut = (e, t) => {
|
|
541
595
|
let n = {};
|
|
542
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
596
|
+
for (let [r, i] of Object.entries(e)) n[r] = Vt(i, t[r]);
|
|
543
597
|
return n;
|
|
544
|
-
},
|
|
545
|
-
try: () =>
|
|
546
|
-
catch: (e) => new
|
|
598
|
+
}, Wt = new TextEncoder(), Gt = (e, t) => e.pipe(l.map((e) => Wt.encode(`${JSON.stringify(Ht(e, t))}\n`))), Kt = (e, t, r) => e.pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.mapEffect((e) => n.try({
|
|
599
|
+
try: () => Ut(JSON.parse(e), r),
|
|
600
|
+
catch: (e) => new We({
|
|
547
601
|
table: t,
|
|
548
602
|
reason: String(e?.message ?? e)
|
|
549
603
|
})
|
|
550
|
-
}))),
|
|
604
|
+
}))), qt = (e, t) => {
|
|
551
605
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
552
606
|
for (let e of t.tables) {
|
|
553
607
|
if (!n.has(e.name)) continue;
|
|
@@ -567,39 +621,39 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
567
621
|
};
|
|
568
622
|
for (let t of e) n.has(t) && s(t);
|
|
569
623
|
return i;
|
|
570
|
-
},
|
|
624
|
+
}, Y = (e) => {
|
|
571
625
|
let t = e, n = [];
|
|
572
626
|
typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
|
|
573
627
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
574
|
-
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(
|
|
628
|
+
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(Y(t.cause)) : n.push(r), n.length === 0) {
|
|
575
629
|
let t = String(e ?? "").trim();
|
|
576
|
-
return t === "" || t === "[object Object]" ?
|
|
630
|
+
return t === "" || t === "[object Object]" ? Jt(e) : t;
|
|
577
631
|
}
|
|
578
632
|
return n.join(": ");
|
|
579
|
-
},
|
|
633
|
+
}, Jt = (e) => {
|
|
580
634
|
try {
|
|
581
635
|
let t = JSON.stringify(e);
|
|
582
636
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
583
637
|
} catch {
|
|
584
638
|
return "unknown error (not serialisable)";
|
|
585
639
|
}
|
|
586
|
-
},
|
|
640
|
+
}, Yt = (e, t) => {
|
|
587
641
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
588
642
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
589
643
|
return r;
|
|
590
|
-
},
|
|
644
|
+
}, Xt = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), Zt = (e, t) => {
|
|
591
645
|
if (t.length === 0) return e;
|
|
592
646
|
let n = { ...e };
|
|
593
647
|
for (let e of t) delete n[e];
|
|
594
648
|
return n;
|
|
595
|
-
},
|
|
649
|
+
}, Qt = (e, t) => {
|
|
596
650
|
let n = e.tables.find((e) => e.name === t);
|
|
597
651
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
598
652
|
let r = n.columns.find((e) => e.type === "id");
|
|
599
653
|
if (r !== void 0) return { column: r.name };
|
|
600
654
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
601
655
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
602
|
-
},
|
|
656
|
+
}, $t = (e, t, r) => n.tryPromise({
|
|
603
657
|
try: async () => {
|
|
604
658
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
605
659
|
d.update(e);
|
|
@@ -608,7 +662,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
608
662
|
n(null, e);
|
|
609
663
|
} }), m = c.fromWeb(l.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
610
664
|
try {
|
|
611
|
-
await s(m, p,
|
|
665
|
+
await s(m, p, ut(r), n(h)), await i.rename(h, t);
|
|
612
666
|
} catch (e) {
|
|
613
667
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
614
668
|
}
|
|
@@ -617,27 +671,27 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
617
671
|
checksum: d.digest("hex")
|
|
618
672
|
};
|
|
619
673
|
},
|
|
620
|
-
catch: (e) => new
|
|
621
|
-
reason: `write table failed: ${
|
|
674
|
+
catch: (e) => new H({
|
|
675
|
+
reason: `write table failed: ${Y(e)}`,
|
|
622
676
|
path: t
|
|
623
677
|
})
|
|
624
|
-
}),
|
|
678
|
+
}), en = (e, t) => n.tryPromise({
|
|
625
679
|
try: async () => {
|
|
626
680
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
627
681
|
},
|
|
628
|
-
catch: (t) => new
|
|
629
|
-
reason: `append failed: ${
|
|
682
|
+
catch: (t) => new H({
|
|
683
|
+
reason: `append failed: ${Y(t)}`,
|
|
630
684
|
path: e
|
|
631
685
|
})
|
|
632
|
-
}),
|
|
686
|
+
}), tn = (e, t) => n.gen(function* () {
|
|
633
687
|
let r = [], i = t.ordered.length;
|
|
634
688
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
635
|
-
let o = t.ordered[a], s =
|
|
636
|
-
if ("refusal" in c) return yield* n.fail(new
|
|
637
|
-
let u = c.column, d = `${
|
|
638
|
-
if (
|
|
689
|
+
let o = t.ordered[a], s = Yt(t.snapshot, o), c = Qt(t.snapshot, o);
|
|
690
|
+
if ("refusal" in c) return yield* n.fail(new U({ reason: `export: ${c.refusal}` }));
|
|
691
|
+
let u = c.column, d = `${bt}/${o}.ndjson${lt(t.compression)}`;
|
|
692
|
+
if (Mt(t.ledger, o)) {
|
|
639
693
|
let e = t.ledger.tables[o];
|
|
640
|
-
|
|
694
|
+
q(t.onProgress, {
|
|
641
695
|
phase: "export",
|
|
642
696
|
table: o,
|
|
643
697
|
index: a,
|
|
@@ -655,7 +709,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
655
709
|
});
|
|
656
710
|
continue;
|
|
657
711
|
}
|
|
658
|
-
|
|
712
|
+
q(t.onProgress, {
|
|
659
713
|
phase: "export",
|
|
660
714
|
table: o,
|
|
661
715
|
index: a,
|
|
@@ -663,17 +717,17 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
663
717
|
event: "start",
|
|
664
718
|
rowsDone: 0
|
|
665
719
|
});
|
|
666
|
-
let f = t.resolved.predicateFor(o), p =
|
|
720
|
+
let f = t.resolved.predicateFor(o), p = w(e, {
|
|
667
721
|
table: o,
|
|
668
722
|
pk: u,
|
|
669
723
|
chunkSize: t.chunkSize,
|
|
670
724
|
retry: t.retry,
|
|
671
725
|
...f === void 0 ? {} : { where: f }
|
|
672
|
-
}), m =
|
|
726
|
+
}), m = Xt(t.snapshot, o), h = m.length === 0 ? p : p.pipe(l.map((e) => Zt(e, m))), g = t.mask ? h.pipe(l.map((e) => ze(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* kt("export", o, a, i, (e) => e.rowCount, $t(Gt(g, s), t.bundleFile(d), t.compression));
|
|
673
727
|
t.ledger.tables[o] = {
|
|
674
728
|
rowCount: _,
|
|
675
729
|
checksum: v
|
|
676
|
-
}, yield*
|
|
730
|
+
}, yield* J(t.ledgerPath, t.ledger), q(t.onProgress, {
|
|
677
731
|
phase: "export",
|
|
678
732
|
table: o,
|
|
679
733
|
index: a,
|
|
@@ -691,43 +745,43 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
691
745
|
});
|
|
692
746
|
}
|
|
693
747
|
return r;
|
|
694
|
-
}),
|
|
748
|
+
}), nn = (e) => ({
|
|
695
749
|
_tag: "RawSqlFragment",
|
|
696
750
|
strings: [e],
|
|
697
751
|
values: []
|
|
698
|
-
}),
|
|
752
|
+
}), rn = (t, i, a) => n.tryPromise({
|
|
699
753
|
try: () => t.transactional(async (t) => {
|
|
700
|
-
if (t.raw) for (let e of
|
|
754
|
+
if (t.raw) for (let e of xe(i)) await t.raw(nn(e));
|
|
701
755
|
let o = await n.runPromiseExit(a(t));
|
|
702
756
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
703
757
|
return o.value;
|
|
704
758
|
}),
|
|
705
|
-
catch: (e) => e instanceof
|
|
706
|
-
}),
|
|
707
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
759
|
+
catch: (e) => e instanceof H ? e : new H({ reason: `snapshot export failed: ${Y(e)}` })
|
|
760
|
+
}), an = (e) => n.gen(function* () {
|
|
761
|
+
let t = e.scope ?? { kind: "all" }, r = ct(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, o = s.exponential("50 millis").pipe(s.jittered, s.intersect(s.recurs(a))), c = yield* n.promise(() => import("node:path")), u = (t) => c.join(e.outDir, t);
|
|
708
762
|
yield* n.tryPromise({
|
|
709
763
|
try: async () => {
|
|
710
764
|
let t = await import("node:fs/promises");
|
|
711
|
-
await t.mkdir(u(
|
|
765
|
+
await t.mkdir(u(bt), { recursive: !0 }), e.assets && await t.mkdir(u(xt), { recursive: !0 });
|
|
712
766
|
},
|
|
713
|
-
catch: (t) => new
|
|
714
|
-
reason: `mkdir failed: ${
|
|
767
|
+
catch: (t) => new H({
|
|
768
|
+
reason: `mkdir failed: ${Y(t)}`,
|
|
715
769
|
path: e.outDir
|
|
716
770
|
})
|
|
717
771
|
});
|
|
718
|
-
let d = u(
|
|
772
|
+
let d = u(Ct), f = yield* jt(d), p = e.subset ? et(yield* $e(e.store, e.snapshot, e.subset), e.snapshot) : yield* nt(t, e.snapshot, {
|
|
719
773
|
store: e.store,
|
|
720
774
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
721
|
-
}), m =
|
|
775
|
+
}), m = qt(p.tables, e.snapshot), h = e.consistency ?? "live", g, _;
|
|
722
776
|
if (e.masking) {
|
|
723
|
-
let t = e.classification ??
|
|
777
|
+
let t = e.classification ?? Ce(e.snapshot), r = Ie(m.map((t) => ({
|
|
724
778
|
name: t,
|
|
725
779
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
726
780
|
name: e.name,
|
|
727
781
|
type: e.type
|
|
728
782
|
}))
|
|
729
783
|
})), t, e.masking);
|
|
730
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
784
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new Ye({
|
|
731
785
|
unclassified: r.unclassified,
|
|
732
786
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
733
787
|
}));
|
|
@@ -753,20 +807,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
753
807
|
bundleFile: u,
|
|
754
808
|
..._ ? { mask: _ } : {},
|
|
755
809
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
756
|
-
}, y = h === "snapshot" ? yield*
|
|
810
|
+
}, y = h === "snapshot" ? yield* rn(e.store, e.dialect, (e) => tn(e, v)) : yield* tn(e.store, v), b = 0, x = 0;
|
|
757
811
|
if (e.assets) {
|
|
758
|
-
let t = u(
|
|
759
|
-
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
812
|
+
let t = u(St);
|
|
813
|
+
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => at(e, u(xt))), n.flatMap((e) => {
|
|
760
814
|
let i = {
|
|
761
815
|
key: r.key,
|
|
762
816
|
sha256: e.sha256,
|
|
763
817
|
size: e.size,
|
|
764
818
|
contentType: e.contentType
|
|
765
819
|
};
|
|
766
|
-
return b++,
|
|
767
|
-
}))).pipe(n.mapError((e) => e instanceof
|
|
820
|
+
return b++, x += e.size, f.assets[r.key] = !0, en(t, JSON.stringify(i)).pipe(n.andThen(J(d, f)));
|
|
821
|
+
}))).pipe(n.mapError((e) => e instanceof H ? e : new H({ reason: `asset export failed: ${String(e)}` })));
|
|
768
822
|
}
|
|
769
|
-
let
|
|
823
|
+
let S = {
|
|
770
824
|
formatVersion: 1,
|
|
771
825
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
772
826
|
source: {
|
|
@@ -779,8 +833,8 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
779
833
|
tables: y,
|
|
780
834
|
assets: b > 0 ? {
|
|
781
835
|
count: b,
|
|
782
|
-
totalBytes:
|
|
783
|
-
index:
|
|
836
|
+
totalBytes: x,
|
|
837
|
+
index: St
|
|
784
838
|
} : {
|
|
785
839
|
count: 0,
|
|
786
840
|
totalBytes: 0
|
|
@@ -789,23 +843,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
789
843
|
};
|
|
790
844
|
return yield* n.tryPromise({
|
|
791
845
|
try: async () => {
|
|
792
|
-
await (await import("node:fs/promises")).writeFile(u(
|
|
846
|
+
await (await import("node:fs/promises")).writeFile(u(K), _t(S));
|
|
793
847
|
},
|
|
794
|
-
catch: (e) => new
|
|
795
|
-
reason: `manifest write failed: ${
|
|
796
|
-
path:
|
|
848
|
+
catch: (e) => new H({
|
|
849
|
+
reason: `manifest write failed: ${Y(e)}`,
|
|
850
|
+
path: K
|
|
797
851
|
})
|
|
798
852
|
}), yield* n.tryPromise({
|
|
799
853
|
try: async () => {
|
|
800
854
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
801
|
-
await e.writeFile(u(
|
|
855
|
+
await e.writeFile(u(yt), Tt(S, { hostname: t.hostname() }));
|
|
802
856
|
},
|
|
803
|
-
catch: (e) => new
|
|
804
|
-
reason: `readme write failed: ${
|
|
805
|
-
path:
|
|
857
|
+
catch: (e) => new H({
|
|
858
|
+
reason: `readme write failed: ${Y(e)}`,
|
|
859
|
+
path: yt
|
|
806
860
|
})
|
|
807
|
-
}),
|
|
808
|
-
}),
|
|
861
|
+
}), S;
|
|
862
|
+
}), on = (e, t) => {
|
|
809
863
|
let n = new Set(t), r = [];
|
|
810
864
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
811
865
|
let i = e.references?.table;
|
|
@@ -816,7 +870,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
816
870
|
});
|
|
817
871
|
}
|
|
818
872
|
return r;
|
|
819
|
-
},
|
|
873
|
+
}, sn = (e) => {
|
|
820
874
|
let t = /* @__PURE__ */ new Map();
|
|
821
875
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
822
876
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -824,7 +878,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
824
878
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
825
879
|
});
|
|
826
880
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
827
|
-
},
|
|
881
|
+
}, cn = (e) => {
|
|
828
882
|
switch (e) {
|
|
829
883
|
case "id":
|
|
830
884
|
case "text":
|
|
@@ -843,12 +897,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
843
897
|
case "raw": return "raw";
|
|
844
898
|
default: return "string";
|
|
845
899
|
}
|
|
846
|
-
},
|
|
900
|
+
}, ln = {
|
|
847
901
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
848
902
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
849
903
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
850
|
-
},
|
|
851
|
-
let i =
|
|
904
|
+
}, un = (e, t, n, r) => {
|
|
905
|
+
let i = cn(n);
|
|
852
906
|
if (i === "raw") return {
|
|
853
907
|
table: e,
|
|
854
908
|
column: t,
|
|
@@ -856,7 +910,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
856
910
|
severity: "block",
|
|
857
911
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
858
912
|
};
|
|
859
|
-
let a =
|
|
913
|
+
let a = ln[i];
|
|
860
914
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
861
915
|
table: e,
|
|
862
916
|
column: t,
|
|
@@ -870,15 +924,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
870
924
|
severity: "block",
|
|
871
925
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
872
926
|
};
|
|
873
|
-
},
|
|
927
|
+
}, dn = (e, t, n) => {
|
|
874
928
|
if (t === n) return [];
|
|
875
929
|
let r = [];
|
|
876
930
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
877
|
-
let a =
|
|
931
|
+
let a = un(t.name, e, i, n);
|
|
878
932
|
a && r.push(a);
|
|
879
933
|
}
|
|
880
934
|
return r;
|
|
881
|
-
},
|
|
935
|
+
}, fn = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, pn = (e) => {
|
|
882
936
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
883
937
|
switch (e.kind) {
|
|
884
938
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -887,36 +941,36 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
887
941
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
888
942
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
889
943
|
}
|
|
890
|
-
},
|
|
944
|
+
}, mn = (e) => {
|
|
891
945
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
892
946
|
return t.length > 0 ? t.join("/") : void 0;
|
|
893
|
-
},
|
|
947
|
+
}, hn = "Failed to execute statement", gn = (e) => {
|
|
894
948
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
895
949
|
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
896
|
-
return t === `SqlError: ${
|
|
897
|
-
},
|
|
950
|
+
return t === `SqlError: ${hn}` ? hn : t;
|
|
951
|
+
}, _n = (e) => {
|
|
898
952
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
899
953
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
900
954
|
unclassified: t,
|
|
901
955
|
raw: n
|
|
902
|
-
}), i =
|
|
903
|
-
if (a !== void 0) return r(`${
|
|
904
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u =
|
|
905
|
-
return r(d ===
|
|
906
|
-
},
|
|
907
|
-
let t =
|
|
956
|
+
}), i = _(e), a = p(i), o = mn(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
957
|
+
if (a !== void 0) return r(`${pn(a)}${s}`, !1);
|
|
958
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = gn(typeof l == "string" && l.length > 0 ? l : String(e)), d = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, f = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, m = f !== void 0 && /[←‖]/.test(f) ? ` (via ${f})` : "";
|
|
959
|
+
return r(d === hn || d.length === 0 ? `the database refused the write and the driver gave no detail${s}${m}` : `${d}${s}`, d === hn || d.length === 0);
|
|
960
|
+
}, vn = (e) => _n(e).reason, yn = (e) => {
|
|
961
|
+
let t = _(e), n = p(t);
|
|
908
962
|
if (n?.kind === "foreignKeyInUse") {
|
|
909
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
963
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = mn(t);
|
|
910
964
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
911
965
|
}
|
|
912
|
-
return
|
|
913
|
-
},
|
|
966
|
+
return vn(e);
|
|
967
|
+
}, bn = "_voltro_staging_", X = (e) => `${bn}${e}`, xn = (e) => typeof e.run == "function", Sn = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
914
968
|
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
915
|
-
written before the first delete is what covers that case.`,
|
|
916
|
-
if (!
|
|
969
|
+
written before the first delete is what covers that case.`, Cn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, Z = (e, t) => {
|
|
970
|
+
if (!Cn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
917
971
|
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
918
|
-
},
|
|
919
|
-
let n =
|
|
972
|
+
}, wn = (e, t) => {
|
|
973
|
+
let n = Z(e, t), r = Z(X(e), t);
|
|
920
974
|
switch (t) {
|
|
921
975
|
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
922
976
|
case "mysql":
|
|
@@ -924,20 +978,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
924
978
|
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
925
979
|
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
926
980
|
}
|
|
927
|
-
},
|
|
928
|
-
let n = X(
|
|
929
|
-
return t === "mssql" ? `IF OBJECT_ID('${
|
|
930
|
-
},
|
|
981
|
+
}, Tn = (e, t) => {
|
|
982
|
+
let n = Z(X(e), t);
|
|
983
|
+
return t === "mssql" ? `IF OBJECT_ID('${X(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
984
|
+
}, En = (e, t) => {
|
|
931
985
|
let n = t === "mysql" || t === "mariadb", r = [];
|
|
932
986
|
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
933
|
-
for (let n of [...e].reverse()) r.push(`DELETE FROM ${
|
|
987
|
+
for (let n of [...e].reverse()) r.push(`DELETE FROM ${Z(n.name, t)}`);
|
|
934
988
|
for (let n of e) {
|
|
935
989
|
if (n.columns.length === 0) throw Error(`refusing to build a swap for '${n.name}' with no columns: an empty list would have to mean \`SELECT *\`, which is the shape that fails on a generated column.`);
|
|
936
|
-
let e = n.columns.map((e) =>
|
|
937
|
-
r.push(`INSERT INTO ${
|
|
990
|
+
let e = n.columns.map((e) => Z(e, t)).join(", ");
|
|
991
|
+
r.push(`INSERT INTO ${Z(n.name, t)} (${e}) SELECT ${e} FROM ${Z(X(n.name), t)}`);
|
|
938
992
|
}
|
|
939
993
|
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
940
|
-
},
|
|
994
|
+
}, Dn = (e, t) => {
|
|
941
995
|
let n = [], r = () => {
|
|
942
996
|
for (let e of n) t.unregisterTable(e);
|
|
943
997
|
};
|
|
@@ -945,7 +999,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
945
999
|
for (let r of e) {
|
|
946
1000
|
let e = t.getTable(r);
|
|
947
1001
|
if (e === void 0) continue;
|
|
948
|
-
let i =
|
|
1002
|
+
let i = X(r);
|
|
949
1003
|
t.registerTable({
|
|
950
1004
|
...e,
|
|
951
1005
|
tableName: i,
|
|
@@ -959,7 +1013,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
959
1013
|
staged: n,
|
|
960
1014
|
undo: r
|
|
961
1015
|
};
|
|
962
|
-
},
|
|
1016
|
+
}, On = (e, t) => {
|
|
963
1017
|
let n = new Set(t), r = [];
|
|
964
1018
|
for (let t of e.tables) {
|
|
965
1019
|
if (!n.has(t.name)) continue;
|
|
@@ -976,22 +1030,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
976
1030
|
}
|
|
977
1031
|
}
|
|
978
1032
|
return r;
|
|
979
|
-
},
|
|
980
|
-
let n = X(
|
|
981
|
-
return `SELECT ${
|
|
982
|
-
},
|
|
1033
|
+
}, kn = (e, t) => {
|
|
1034
|
+
let n = Z(X(e.from), t), r = Z(X(e.to), t), i = Z(e.column, t), a = Z(e.toColumn, t);
|
|
1035
|
+
return `SELECT ${Z(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
1036
|
+
}, An = (e) => {
|
|
983
1037
|
if (e.length === 0) return;
|
|
984
1038
|
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
985
1039
|
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
986
|
-
},
|
|
1040
|
+
}, jn = async (e, t) => {
|
|
987
1041
|
t.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
988
|
-
},
|
|
989
|
-
let i =
|
|
1042
|
+
}, Mn = async (e, t, r) => {
|
|
1043
|
+
let i = En(t, r);
|
|
990
1044
|
i.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
991
|
-
},
|
|
1045
|
+
}, Nn = async (e, t, r) => {
|
|
992
1046
|
let i = [];
|
|
993
1047
|
for (let a of t) {
|
|
994
|
-
let t = await e.run(P.SqlClient.pipe(n.flatMap((e) => e.unsafe(
|
|
1048
|
+
let t = await e.run(P.SqlClient.pipe(n.flatMap((e) => e.unsafe(kn(a, r)))));
|
|
995
1049
|
for (let e of t) i.push({
|
|
996
1050
|
table: a.from,
|
|
997
1051
|
primaryKey: String(e.pk ?? ""),
|
|
@@ -1001,16 +1055,75 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1001
1055
|
});
|
|
1002
1056
|
}
|
|
1003
1057
|
return i;
|
|
1004
|
-
},
|
|
1058
|
+
}, Pn = (e, t) => {
|
|
1005
1059
|
let n = new Map(t.map((e, t) => [e, t]));
|
|
1006
1060
|
return e.filter((e) => {
|
|
1007
1061
|
if (e.from === e.to) return !1;
|
|
1008
1062
|
let t = n.get(e.from), r = n.get(e.to);
|
|
1009
1063
|
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1010
1064
|
});
|
|
1011
|
-
},
|
|
1065
|
+
}, Fn = (e) => e === "mysql" || e === "mariadb", In = (e) => {
|
|
1066
|
+
if (e.mode !== "replace") return {
|
|
1067
|
+
active: !1,
|
|
1068
|
+
tables: [],
|
|
1069
|
+
blocker: `staging applies to \`replace\` only (this run is \`${e.mode}\`).`
|
|
1070
|
+
};
|
|
1071
|
+
let t = (() => {
|
|
1072
|
+
let t = e.targetDialect;
|
|
1073
|
+
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1074
|
+
})();
|
|
1075
|
+
if (t === void 0) return {
|
|
1076
|
+
active: !1,
|
|
1077
|
+
tables: [],
|
|
1078
|
+
blocker: `the target dialect (${e.targetDialect ?? "unknown"}) is not one this build can stage on.`
|
|
1079
|
+
};
|
|
1080
|
+
if (!e.canRunSql) return {
|
|
1081
|
+
active: !1,
|
|
1082
|
+
tables: [],
|
|
1083
|
+
blocker: "this store cannot send DDL (no raw-SQL channel), which a staged swap needs."
|
|
1084
|
+
};
|
|
1085
|
+
let n = e.targetSnapshot;
|
|
1086
|
+
if (n === void 0) return {
|
|
1087
|
+
active: !1,
|
|
1088
|
+
tables: [],
|
|
1089
|
+
blocker: "this caller did not supply a snapshot of the target schema, so the staging tables cannot be given the target's column shape."
|
|
1090
|
+
};
|
|
1091
|
+
let r = new Map(n.tables.map((e) => [e.name, e])), i = e.bundleTables.filter((t) => !e.isSkipped(t)), a = i.filter((e) => !r.has(e));
|
|
1092
|
+
if (a.length > 0) return {
|
|
1093
|
+
active: !1,
|
|
1094
|
+
tables: [],
|
|
1095
|
+
blocker: `the bundle carries ${a.length} table(s) the target does not have (${a.join(", ")}). Staging copies the TARGET's column shape, and there is none for those.`
|
|
1096
|
+
};
|
|
1097
|
+
let o = [];
|
|
1098
|
+
for (let e of i) {
|
|
1099
|
+
let t = r.get(e).columns.filter((e) => e.generatedAs === void 0).map((e) => e.name);
|
|
1100
|
+
if (t.length === 0) return {
|
|
1101
|
+
active: !1,
|
|
1102
|
+
tables: [],
|
|
1103
|
+
blocker: `\`${e}\` has no ordinary columns in the target (every column is generated), so a staged copy would have nothing to write.`
|
|
1104
|
+
};
|
|
1105
|
+
o.push({
|
|
1106
|
+
name: e,
|
|
1107
|
+
columns: t
|
|
1108
|
+
});
|
|
1109
|
+
}
|
|
1110
|
+
if (o.length === 0) return {
|
|
1111
|
+
active: !1,
|
|
1112
|
+
tables: [],
|
|
1113
|
+
blocker: "every table in the bundle is one that describes a deployment, so there is nothing for a staged swap to move."
|
|
1114
|
+
};
|
|
1115
|
+
let s = o.map((e) => e.name), c = Fn(t) ? [] : Pn(On(n, s), s);
|
|
1116
|
+
return c.length > 0 ? {
|
|
1117
|
+
active: !1,
|
|
1118
|
+
tables: [],
|
|
1119
|
+
blocker: `the bundle's tables reference each other in a cycle (${c.map((e) => `${e.from}.${e.column} → ${e.to}`).join(", ")}), and a bulk copy cannot satisfy both directions on this dialect.`
|
|
1120
|
+
} : {
|
|
1121
|
+
active: !0,
|
|
1122
|
+
tables: o
|
|
1123
|
+
};
|
|
1124
|
+
}, Ln = (e) => n.gen(function* () {
|
|
1012
1125
|
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1013
|
-
if (e.snapshot === void 0) return yield* n.fail(new
|
|
1126
|
+
if (e.snapshot === void 0) return yield* n.fail(new H({
|
|
1014
1127
|
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
1015
1128
|
path: e.dir
|
|
1016
1129
|
}));
|
|
@@ -1021,14 +1134,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1021
1134
|
if (t.tables.length === 0) return;
|
|
1022
1135
|
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1023
1136
|
so it survives a run whose process does not.`);
|
|
1024
|
-
let r = yield*
|
|
1137
|
+
let r = yield* an({
|
|
1025
1138
|
store: e.store,
|
|
1026
1139
|
snapshot: t,
|
|
1027
1140
|
dialect: e.dialect ?? "unknown",
|
|
1028
1141
|
outDir: e.dir,
|
|
1029
1142
|
scope: { kind: "all" },
|
|
1030
1143
|
compression: "none"
|
|
1031
|
-
}).pipe(n.mapError((t) => new
|
|
1144
|
+
}).pipe(n.mapError((t) => new H({
|
|
1032
1145
|
reason: `rollback capture failed, so the import did not start and the target is untouched: ${t.reason ?? String(t)}\n Fix the capture, or re-run with \`--no-rollback\` to proceed without one.`,
|
|
1033
1146
|
path: e.dir
|
|
1034
1147
|
})));
|
|
@@ -1037,18 +1150,18 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1037
1150
|
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1038
1151
|
tables: r.tables.length
|
|
1039
1152
|
};
|
|
1040
|
-
}),
|
|
1153
|
+
}), Rn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, zn = (e, t) => e.tables.find((e) => e.name === t), Bn = (e, t) => new Map((zn(e, t)?.columns ?? []).map((e) => [e.name, e])), Vn = /__dropped_\d+$/, Hn = (e, t) => {
|
|
1041
1154
|
let n = [];
|
|
1042
1155
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
1043
|
-
if (
|
|
1156
|
+
if (zn(t, r.name) === void 0) {
|
|
1044
1157
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
1045
1158
|
continue;
|
|
1046
1159
|
}
|
|
1047
|
-
let e =
|
|
1160
|
+
let e = Bn(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
1048
1161
|
for (let [t, a] of i) {
|
|
1049
1162
|
let i = e.get(t);
|
|
1050
1163
|
if (i === void 0) {
|
|
1051
|
-
n.push(`${r.name}.${t}: in the bundle, MISSING from the target — the value has nowhere to go`);
|
|
1164
|
+
n.push(Vn.test(t) ? `${r.name}.${t}: a soft-dropped column, present only in the SOURCE — reclaim it there (voltro db gc-snapshots --before <date>) or re-export without it; the target is not missing anything` : `${r.name}.${t}: in the bundle, MISSING from the target — the value has nowhere to go`);
|
|
1052
1165
|
continue;
|
|
1053
1166
|
}
|
|
1054
1167
|
i.type !== a.type && n.push(`${r.name}.${t}: type '${a.type}' (bundle) vs '${i.type}' (target)`);
|
|
@@ -1056,7 +1169,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1056
1169
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
1057
1170
|
}
|
|
1058
1171
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
1059
|
-
},
|
|
1172
|
+
}, Un = (e, t, n) => {
|
|
1060
1173
|
let r = re(t);
|
|
1061
1174
|
if (r === e || n === void 0) return {
|
|
1062
1175
|
drifted: !1,
|
|
@@ -1064,14 +1177,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1064
1177
|
targetFingerprint: r,
|
|
1065
1178
|
diff: []
|
|
1066
1179
|
};
|
|
1067
|
-
let i =
|
|
1180
|
+
let i = Hn(n, t);
|
|
1068
1181
|
return {
|
|
1069
1182
|
drifted: i.length > 0,
|
|
1070
1183
|
bundleFingerprint: e,
|
|
1071
1184
|
targetFingerprint: r,
|
|
1072
1185
|
diff: i
|
|
1073
1186
|
};
|
|
1074
|
-
},
|
|
1187
|
+
}, Wn = 2e3, Gn = 200, Kn = /* @__PURE__ */ new Set([
|
|
1075
1188
|
"BundleError",
|
|
1076
1189
|
"CodecError",
|
|
1077
1190
|
"IntegrityError",
|
|
@@ -1080,33 +1193,33 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1080
1193
|
"RowsRefusedError",
|
|
1081
1194
|
"SchemaDriftError",
|
|
1082
1195
|
"StorageError"
|
|
1083
|
-
]),
|
|
1196
|
+
]), qn = (t) => {
|
|
1084
1197
|
let n = o.isFiberFailure(t) ? [...e.failures(t[o.FiberFailureCauseId]), ...e.defects(t[o.FiberFailureCauseId])] : [t];
|
|
1085
1198
|
for (let e of n) {
|
|
1086
1199
|
let t = e?._tag;
|
|
1087
|
-
if (typeof t == "string" &&
|
|
1200
|
+
if (typeof t == "string" && Kn.has(t)) return e;
|
|
1088
1201
|
if (o.isFiberFailure(e)) {
|
|
1089
|
-
let t =
|
|
1202
|
+
let t = qn(e);
|
|
1090
1203
|
if (t) return t;
|
|
1091
1204
|
}
|
|
1092
1205
|
}
|
|
1093
|
-
},
|
|
1206
|
+
}, Jn = (t) => [...e.failures(t)][0] ?? Error(e.pretty(t)), Yn = (e) => n.tryPromise({
|
|
1094
1207
|
try: async () => {
|
|
1095
1208
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
1096
|
-
return
|
|
1209
|
+
return vt(await t.readFile(n.join(e, K), "utf8"));
|
|
1097
1210
|
},
|
|
1098
|
-
catch: (e) => new
|
|
1211
|
+
catch: (e) => new H({
|
|
1099
1212
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
1100
|
-
path:
|
|
1213
|
+
path: K
|
|
1101
1214
|
})
|
|
1102
|
-
}),
|
|
1215
|
+
}), Xn = (e) => {
|
|
1103
1216
|
let t;
|
|
1104
1217
|
return n.gen(function* () {
|
|
1105
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
1106
|
-
if (a.formatVersion !== 1) return yield* n.fail(new
|
|
1218
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* Yn(e.bundleDir);
|
|
1219
|
+
if (a.formatVersion !== 1) return yield* n.fail(new H({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
1107
1220
|
if (e.tables !== void 0) {
|
|
1108
1221
|
let t = new Set(a.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
1109
|
-
if (r.length > 0) return yield* n.fail(new
|
|
1222
|
+
if (r.length > 0) return yield* n.fail(new H({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...t].sort().join(", ")}.` }));
|
|
1110
1223
|
}
|
|
1111
1224
|
let o = e.tables === void 0 ? void 0 : new Set(e.tables), s = o === void 0 ? a : {
|
|
1112
1225
|
...a,
|
|
@@ -1123,9 +1236,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1123
1236
|
unique: !1,
|
|
1124
1237
|
hasDefault: !1
|
|
1125
1238
|
}))
|
|
1126
|
-
})) }, r =
|
|
1239
|
+
})) }, r = Un(s.source.schemaFingerprint, e.targetSnapshot, t);
|
|
1127
1240
|
if (r.drifted) {
|
|
1128
|
-
if (!e.force) return yield* n.fail(new
|
|
1241
|
+
if (!e.force) return yield* n.fail(new Je({
|
|
1129
1242
|
bundleFingerprint: r.bundleFingerprint,
|
|
1130
1243
|
targetFingerprint: r.targetFingerprint,
|
|
1131
1244
|
diff: r.diff
|
|
@@ -1134,23 +1247,28 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1134
1247
|
}
|
|
1135
1248
|
}
|
|
1136
1249
|
if (e.targetDialect && e.targetDialect !== s.source.dialect) {
|
|
1137
|
-
let t =
|
|
1138
|
-
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new
|
|
1250
|
+
let t = dn(s.tables, s.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1251
|
+
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new Ze({
|
|
1139
1252
|
sourceDialect: s.source.dialect,
|
|
1140
1253
|
targetDialect: e.targetDialect,
|
|
1141
|
-
blocking: t.map(
|
|
1254
|
+
blocking: t.map(fn)
|
|
1142
1255
|
}));
|
|
1143
1256
|
}
|
|
1144
|
-
let c = e.verify ?? !0,
|
|
1145
|
-
if (
|
|
1146
|
-
mode:
|
|
1147
|
-
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
1257
|
+
let c = e.verify ?? !0, p = e.mode ?? "upsert", g = e.atomic ?? p === "replace", _ = e.onConflict ?? "skip", y = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), x = yield* jt(y);
|
|
1258
|
+
if (p === "replace" && s.scope.kind !== "all") return yield* n.fail(new Xe({
|
|
1259
|
+
mode: p,
|
|
1260
|
+
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope. To leave specific tables out of a FULL export, use \`--exclude a,b\`: it keeps the scope \`all\` and records what it left out.`
|
|
1148
1261
|
}));
|
|
1149
|
-
if (
|
|
1262
|
+
if (p === "replace" && (s.scope.exclude?.length ?? 0) > 0) {
|
|
1263
|
+
let t = s.scope.exclude;
|
|
1264
|
+
e.onWarn?.(`this bundle was exported with ${t.length} table(s) excluded: ${t.join(", ")}.\n A replace empties and refills the tables the bundle CARRIES, so those keep whatever rows the
|
|
1265
|
+
target already holds. That is not "everything the bundle says" — check it is what you meant.`);
|
|
1266
|
+
}
|
|
1267
|
+
if (p === "replace") {
|
|
1150
1268
|
let t = s.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1151
1269
|
try: () => e.store.incomingForeignKeys(t),
|
|
1152
|
-
catch: (e) => new
|
|
1153
|
-
}) : e.targetSnapshot === void 0 ? [] :
|
|
1270
|
+
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${yn(e)}` })
|
|
1271
|
+
}) : e.targetSnapshot === void 0 ? [] : on(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1154
1272
|
for (let t of r) {
|
|
1155
1273
|
if (!a.has(t.from)) {
|
|
1156
1274
|
let r = yield* n.tryPromise({
|
|
@@ -1162,7 +1280,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1162
1280
|
skip: void 0,
|
|
1163
1281
|
projection: []
|
|
1164
1282
|
}),
|
|
1165
|
-
catch: () => new
|
|
1283
|
+
catch: () => new H({ reason: `unreadable: ${t.from}` })
|
|
1166
1284
|
}).pipe(n.map((e) => e.length), n.orElseSucceed(() => 0));
|
|
1167
1285
|
a.set(t.from, r);
|
|
1168
1286
|
}
|
|
@@ -1172,12 +1290,17 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1172
1290
|
rowCount: r
|
|
1173
1291
|
});
|
|
1174
1292
|
}
|
|
1175
|
-
if (i.length > 0) return yield* n.fail(new
|
|
1176
|
-
mode:
|
|
1177
|
-
reason:
|
|
1293
|
+
if (i.length > 0) return yield* n.fail(new Xe({
|
|
1294
|
+
mode: p,
|
|
1295
|
+
reason: sn(i)
|
|
1178
1296
|
}));
|
|
1179
1297
|
}
|
|
1180
|
-
if (e.dryRun === !0) return {
|
|
1298
|
+
if (e.dryRun === !0) return e.traceId !== void 0 && (yield* n.promise(() => e.store.update(u, e.traceId, {
|
|
1299
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
1300
|
+
tablesWritten: 0,
|
|
1301
|
+
rowsWritten: 0,
|
|
1302
|
+
failure: null
|
|
1303
|
+
}).then(() => void 0, () => void 0))), {
|
|
1181
1304
|
manifest: s,
|
|
1182
1305
|
tablesWritten: 0,
|
|
1183
1306
|
rowsWritten: 0,
|
|
@@ -1185,32 +1308,32 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1185
1308
|
rowsSkipped: 0,
|
|
1186
1309
|
fullyResumed: !1
|
|
1187
1310
|
};
|
|
1188
|
-
let
|
|
1311
|
+
let w = /* @__PURE__ */ new Map();
|
|
1189
1312
|
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1190
1313
|
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1191
|
-
e.length > 0 &&
|
|
1314
|
+
e.length > 0 && w.set(t.name, e);
|
|
1192
1315
|
}
|
|
1193
1316
|
let re = (e, t) => {
|
|
1194
|
-
let n =
|
|
1317
|
+
let n = w.get(e);
|
|
1195
1318
|
if (n === void 0) return t;
|
|
1196
1319
|
let r;
|
|
1197
1320
|
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1198
1321
|
return r ?? t;
|
|
1199
|
-
},
|
|
1200
|
-
let i = j ?
|
|
1201
|
-
return
|
|
1202
|
-
},
|
|
1203
|
-
r.unclassified && (
|
|
1204
|
-
},
|
|
1322
|
+
}, T = (e, t, n, r = !1) => {
|
|
1323
|
+
let i = j ? X(t.name) : t.name, a = t.primaryKey, o = re(t.name, n);
|
|
1324
|
+
return p === "append" ? _ === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : p === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1325
|
+
}, ie = (e, t) => e.savepoint ? e.savepoint(t) : t(), E = 0, ae = (t, n, r) => {
|
|
1326
|
+
r.unclassified && (E++, !(E > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (E === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1327
|
+
}, D = (e, t, r = !0) => n.tryPromise({
|
|
1205
1328
|
try: e,
|
|
1206
|
-
catch: (e) =>
|
|
1207
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason,
|
|
1329
|
+
catch: (e) => _n(e)
|
|
1330
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, ae(t.entry, t.row, e)), !1)))), oe = (e) => {
|
|
1208
1331
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1209
1332
|
if (t.length !== 0) return {
|
|
1210
1333
|
...e.row,
|
|
1211
1334
|
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1212
1335
|
};
|
|
1213
|
-
},
|
|
1336
|
+
}, se = (e) => {
|
|
1214
1337
|
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
1215
1338
|
if (r !== "reference") return !1;
|
|
1216
1339
|
let i = e.row[n];
|
|
@@ -1228,7 +1351,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1228
1351
|
};
|
|
1229
1352
|
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1230
1353
|
}
|
|
1231
|
-
return new
|
|
1354
|
+
return new qe({
|
|
1232
1355
|
rows: i.slice(0, 20),
|
|
1233
1356
|
primaryCount: r.filter((e) => !e.derived).length,
|
|
1234
1357
|
totalCount: e.length,
|
|
@@ -1238,113 +1361,104 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1238
1361
|
primary: t.primary
|
|
1239
1362
|
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1240
1363
|
});
|
|
1241
|
-
},
|
|
1364
|
+
}, le = (e, t) => n.gen(function* () {
|
|
1242
1365
|
let r = t, i = [], a = !0;
|
|
1243
1366
|
for (; r.length > 0 && a;) {
|
|
1244
1367
|
a = !1;
|
|
1245
1368
|
let t = [];
|
|
1246
|
-
for (let n of r) (yield*
|
|
1369
|
+
for (let n of r) (yield* D(() => ie(e, () => T(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1247
1370
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1248
|
-
let r = t[n], o =
|
|
1249
|
-
o !== void 0 && (yield*
|
|
1371
|
+
let r = t[n], o = oe(r);
|
|
1372
|
+
o !== void 0 && (yield* D(() => ie(e, () => T(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1250
1373
|
}
|
|
1251
1374
|
r = t;
|
|
1252
1375
|
}
|
|
1253
|
-
for (let t of i) (yield*
|
|
1254
|
-
if (r.length > 0) return yield* n.fail(
|
|
1255
|
-
}),
|
|
1376
|
+
for (let t of i) (yield* D(() => ie(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1377
|
+
if (r.length > 0) return yield* n.fail(se(r));
|
|
1378
|
+
}), O = (() => {
|
|
1256
1379
|
let t = e.targetDialect;
|
|
1257
1380
|
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1258
|
-
})(),
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
if (i.length === 0) return [];
|
|
1268
|
-
r.push({
|
|
1269
|
-
name: e.name,
|
|
1270
|
-
columns: i
|
|
1271
|
-
});
|
|
1272
|
-
}
|
|
1273
|
-
return r;
|
|
1274
|
-
})(), k = s.tables.filter((e) => !B(e.name) && b(e.name)).map((e) => e.name), A = D === void 0 || Nn(D) ? [] : Mn(e.targetSnapshot === void 0 ? [] : En(e.targetSnapshot, O.map((e) => e.name)), O.map((e) => e.name)), j = f === "replace" && D !== void 0 && O.length > 0 && A.length === 0 && yn(e.store), le = !1, M = 0, N = 0, de = 0, P = 0, F = (t, r) => n.gen(function* () {
|
|
1275
|
-
if (f === "replace" && !j && !(r && y.truncated)) {
|
|
1381
|
+
})(), k = In({
|
|
1382
|
+
mode: p,
|
|
1383
|
+
...e.targetDialect === void 0 ? {} : { targetDialect: e.targetDialect },
|
|
1384
|
+
...e.targetSnapshot === void 0 ? {} : { targetSnapshot: e.targetSnapshot },
|
|
1385
|
+
bundleTables: s.tables.map((e) => e.name),
|
|
1386
|
+
isSkipped: V,
|
|
1387
|
+
canRunSql: xn(e.store)
|
|
1388
|
+
}), A = k.tables, ue = s.tables.filter((e) => !V(e.name) && S(e.name)).map((e) => e.name), j = k.active, de = !1, M = 0, N = 0, pe, P = 0, F = 0, me = (t, r) => n.gen(function* () {
|
|
1389
|
+
if (p === "replace" && !j && !(r && x.truncated)) {
|
|
1276
1390
|
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1277
1391
|
try: () => t.insert(d, {
|
|
1278
|
-
id:
|
|
1279
|
-
startedAt:
|
|
1392
|
+
id: m("repl", he),
|
|
1393
|
+
startedAt: ge,
|
|
1280
1394
|
tables: String(s.tables.length),
|
|
1281
1395
|
capture: I?.dir ?? null,
|
|
1282
1396
|
via: e.via ?? "direct"
|
|
1283
1397
|
}),
|
|
1284
|
-
catch: (e) => new
|
|
1398
|
+
catch: (e) => new H({ reason: `could not record that this replace started: ${String(e?.message ?? e)}\n The marker is what makes an interrupted run visible instead of silent, so the run does
|
|
1285
1399
|
not proceed without it.` })
|
|
1286
1400
|
}));
|
|
1287
|
-
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !
|
|
1288
|
-
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where:
|
|
1401
|
+
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !V(e)), a = !r, o = t.emptyTables ? () => t.emptyTables(i) : a ? async () => {
|
|
1402
|
+
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where: b(e.primaryKey) });
|
|
1289
1403
|
} : () => t.transactional(async (e) => {
|
|
1290
|
-
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where:
|
|
1404
|
+
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: b(t.primaryKey) });
|
|
1291
1405
|
});
|
|
1292
1406
|
yield* n.tryPromise({
|
|
1293
1407
|
try: o,
|
|
1294
|
-
catch: (e) => new
|
|
1295
|
-
}), r && (
|
|
1408
|
+
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${yn(e)}` })
|
|
1409
|
+
}), r && (x.truncated = !0, yield* J(y, x));
|
|
1296
1410
|
}
|
|
1297
|
-
let a = [], o = [], u = [],
|
|
1411
|
+
let a = [], o = [], u = [], f = [], h = s.tables.length;
|
|
1298
1412
|
for (let d = 0; d < s.tables.length; d++) {
|
|
1299
|
-
let
|
|
1300
|
-
if (
|
|
1301
|
-
|
|
1413
|
+
let m = s.tables[d];
|
|
1414
|
+
if (V(m.name)) {
|
|
1415
|
+
f.push(m.name);
|
|
1302
1416
|
continue;
|
|
1303
1417
|
}
|
|
1304
|
-
if (r &&
|
|
1305
|
-
u.push(
|
|
1418
|
+
if (r && m.name in x.tables) {
|
|
1419
|
+
u.push(m.name), F++, P += x.tables[m.name].rowCount, q(e.onProgress, {
|
|
1306
1420
|
phase: "import",
|
|
1307
|
-
table:
|
|
1421
|
+
table: m.name,
|
|
1308
1422
|
index: d,
|
|
1309
|
-
tableCount:
|
|
1423
|
+
tableCount: h,
|
|
1310
1424
|
event: "done",
|
|
1311
|
-
rowsDone:
|
|
1312
|
-
total:
|
|
1425
|
+
rowsDone: x.tables[m.name].rowCount,
|
|
1426
|
+
total: m.rowCount
|
|
1313
1427
|
});
|
|
1314
1428
|
continue;
|
|
1315
1429
|
}
|
|
1316
|
-
let
|
|
1317
|
-
|
|
1430
|
+
let _ = i.join(e.bundleDir, m.file);
|
|
1431
|
+
q(e.onProgress, {
|
|
1318
1432
|
phase: "import",
|
|
1319
|
-
table:
|
|
1433
|
+
table: m.name,
|
|
1320
1434
|
index: d,
|
|
1321
|
-
tableCount:
|
|
1435
|
+
tableCount: h,
|
|
1322
1436
|
event: "start",
|
|
1323
1437
|
rowsDone: 0,
|
|
1324
|
-
total:
|
|
1438
|
+
total: m.rowCount
|
|
1325
1439
|
});
|
|
1326
|
-
let
|
|
1327
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c =
|
|
1440
|
+
let v = a.length, b = n.gen(function* () {
|
|
1441
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c = ce("bundle", () => fe(_).pipe(dt(s.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), u = async (e) => {
|
|
1328
1442
|
let n = {
|
|
1329
|
-
entry:
|
|
1443
|
+
entry: m,
|
|
1330
1444
|
row: e,
|
|
1331
1445
|
reason: ""
|
|
1332
1446
|
};
|
|
1333
1447
|
try {
|
|
1334
|
-
await
|
|
1448
|
+
await ie(t, () => T(t, m, e));
|
|
1335
1449
|
} catch (t) {
|
|
1336
|
-
let r =
|
|
1337
|
-
n.reason = r.reason,
|
|
1450
|
+
let r = _n(t);
|
|
1451
|
+
n.reason = r.reason, ae(m, e, r), a.push(n);
|
|
1338
1452
|
}
|
|
1339
1453
|
}, d = yield* n.promise(async () => {
|
|
1340
1454
|
let n = e.copyLoader;
|
|
1341
1455
|
if (!n || n.dialect !== "postgres") return !1;
|
|
1342
|
-
if (
|
|
1343
|
-
if (
|
|
1344
|
-
if (
|
|
1456
|
+
if (g) return de || (de = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1457
|
+
if (p === "replace") return !0;
|
|
1458
|
+
if (p !== "upsert") return !1;
|
|
1345
1459
|
try {
|
|
1346
1460
|
return (await t.query({
|
|
1347
|
-
table:
|
|
1461
|
+
table: m.name,
|
|
1348
1462
|
predicate: void 0,
|
|
1349
1463
|
order: [],
|
|
1350
1464
|
take: 1,
|
|
@@ -1354,200 +1468,216 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1354
1468
|
} catch {
|
|
1355
1469
|
return !1;
|
|
1356
1470
|
}
|
|
1357
|
-
}),
|
|
1358
|
-
if (
|
|
1359
|
-
let t =
|
|
1360
|
-
|
|
1471
|
+
}), f = Object.keys(m.columnTypes), h = Math.max(1, e.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
1472
|
+
if (v.length === 0) return;
|
|
1473
|
+
let t = v;
|
|
1474
|
+
v = [];
|
|
1361
1475
|
try {
|
|
1362
1476
|
await e.copyLoader.copyInto({
|
|
1363
|
-
table:
|
|
1364
|
-
columns:
|
|
1365
|
-
columnTypes:
|
|
1477
|
+
table: m.name,
|
|
1478
|
+
columns: f,
|
|
1479
|
+
columnTypes: m.columnTypes,
|
|
1366
1480
|
rows: t
|
|
1367
1481
|
});
|
|
1368
1482
|
} catch (n) {
|
|
1369
|
-
d = !1, e.onWarn?.(`bulk COPY into ${
|
|
1483
|
+
d = !1, e.onWarn?.(`bulk COPY into ${m.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1370
1484
|
for (let e of t) await u(e);
|
|
1371
1485
|
}
|
|
1372
|
-
}, b = Math.max(1, e.savepointBatchSize ?? 200),
|
|
1373
|
-
if (
|
|
1374
|
-
let e =
|
|
1375
|
-
|
|
1486
|
+
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof t.savepoint == "function", S = [], C = async () => {
|
|
1487
|
+
if (S.length === 0) return;
|
|
1488
|
+
let e = S;
|
|
1489
|
+
S = [];
|
|
1376
1490
|
try {
|
|
1377
1491
|
await t.savepoint(async () => {
|
|
1378
|
-
for (let n of e) await
|
|
1492
|
+
for (let n of e) await T(t, m, n);
|
|
1379
1493
|
});
|
|
1380
1494
|
} catch {
|
|
1381
1495
|
for (let t of e) await u(t);
|
|
1382
1496
|
}
|
|
1383
1497
|
};
|
|
1384
|
-
return yield* l.runForEach(
|
|
1498
|
+
return yield* l.runForEach(Kt(c, m.name, m.columnTypes), (e) => n.promise(async () => {
|
|
1385
1499
|
if (o++, d) {
|
|
1386
|
-
|
|
1500
|
+
v.push(e), v.length >= h && await y();
|
|
1387
1501
|
return;
|
|
1388
1502
|
}
|
|
1389
|
-
if (
|
|
1390
|
-
|
|
1503
|
+
if (x) {
|
|
1504
|
+
S.push(e), S.length >= b && await C();
|
|
1391
1505
|
return;
|
|
1392
1506
|
}
|
|
1393
1507
|
await u(e);
|
|
1394
|
-
})), yield* n.promise(
|
|
1508
|
+
})), yield* n.promise(C), yield* n.promise(y), {
|
|
1395
1509
|
rowCount: o,
|
|
1396
1510
|
checksum: i.digest("hex")
|
|
1397
1511
|
};
|
|
1398
|
-
}), { rowCount:
|
|
1512
|
+
}), { rowCount: S, checksum: C } = yield* kt("import", m.name, d, h, (e) => e.rowCount, b);
|
|
1399
1513
|
if (c) {
|
|
1400
|
-
if (
|
|
1401
|
-
what: `table ${
|
|
1402
|
-
expected:
|
|
1403
|
-
actual:
|
|
1514
|
+
if (C !== m.checksum) return yield* n.fail(new Ge({
|
|
1515
|
+
what: `table ${m.name} checksum`,
|
|
1516
|
+
expected: m.checksum,
|
|
1517
|
+
actual: C
|
|
1404
1518
|
}));
|
|
1405
|
-
if (
|
|
1406
|
-
what: `table ${
|
|
1407
|
-
expected: String(
|
|
1408
|
-
actual: String(
|
|
1519
|
+
if (S !== m.rowCount) return yield* n.fail(new Ge({
|
|
1520
|
+
what: `table ${m.name} rowCount`,
|
|
1521
|
+
expected: String(m.rowCount),
|
|
1522
|
+
actual: String(S)
|
|
1409
1523
|
}));
|
|
1410
1524
|
}
|
|
1411
|
-
|
|
1525
|
+
q(e.onProgress, {
|
|
1412
1526
|
phase: "import",
|
|
1413
|
-
table:
|
|
1527
|
+
table: m.name,
|
|
1414
1528
|
index: d,
|
|
1415
|
-
tableCount:
|
|
1529
|
+
tableCount: h,
|
|
1416
1530
|
event: "done",
|
|
1417
|
-
rowsDone:
|
|
1418
|
-
total:
|
|
1419
|
-
}), N++, M +=
|
|
1420
|
-
name:
|
|
1421
|
-
rowCount:
|
|
1422
|
-
}) : (
|
|
1423
|
-
rowCount:
|
|
1424
|
-
checksum:
|
|
1425
|
-
}, yield*
|
|
1531
|
+
rowsDone: S,
|
|
1532
|
+
total: m.rowCount
|
|
1533
|
+
}), N++, M += S, pe?.(), r && (a.length > v ? o.push({
|
|
1534
|
+
name: m.name,
|
|
1535
|
+
rowCount: S
|
|
1536
|
+
}) : (x.tables[m.name] = {
|
|
1537
|
+
rowCount: S,
|
|
1538
|
+
checksum: m.checksum
|
|
1539
|
+
}, yield* J(y, x)));
|
|
1426
1540
|
}
|
|
1427
|
-
if (a.length > 0 && (yield*
|
|
1541
|
+
if (a.length > 0 && (yield* le(t, a)), p === "replace" && (yield* n.promise(() => t.deleteMany(d, { where: {
|
|
1428
1542
|
column: "startedAt",
|
|
1429
1543
|
op: "lte",
|
|
1430
|
-
value:
|
|
1544
|
+
value: ge
|
|
1431
1545
|
} }).then(() => void 0, () => void 0))), r && o.length > 0) {
|
|
1432
1546
|
for (let e of o) {
|
|
1433
1547
|
let t = s.tables.find((t) => t.name === e.name);
|
|
1434
|
-
|
|
1548
|
+
x.tables[e.name] = {
|
|
1435
1549
|
rowCount: e.rowCount,
|
|
1436
1550
|
checksum: t.checksum
|
|
1437
1551
|
};
|
|
1438
1552
|
}
|
|
1439
|
-
yield*
|
|
1553
|
+
yield* J(y, x);
|
|
1440
1554
|
}
|
|
1441
1555
|
{
|
|
1442
|
-
let t =
|
|
1556
|
+
let t = Ue(f, "skipped");
|
|
1443
1557
|
t !== void 0 && e.onWarn?.(t);
|
|
1444
1558
|
}
|
|
1445
1559
|
if (u.length > 0) {
|
|
1446
1560
|
let t = u.length === s.tables.length;
|
|
1447
|
-
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${
|
|
1561
|
+
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${y} to force a full re-import.`);
|
|
1448
1562
|
}
|
|
1449
|
-
}),
|
|
1563
|
+
}), he = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, ge = /* @__PURE__ */ new Date(), I = yield* Ln({
|
|
1450
1564
|
store: e.store,
|
|
1451
|
-
mode:
|
|
1565
|
+
mode: p,
|
|
1452
1566
|
dir: e.rollbackDir,
|
|
1453
1567
|
snapshot: e.targetSnapshot,
|
|
1454
1568
|
dialect: e.targetDialect,
|
|
1455
|
-
tables: s.tables.map((e) => e.name).filter((e) => !
|
|
1569
|
+
tables: s.tables.map((e) => e.name).filter((e) => !V(e)),
|
|
1456
1570
|
onWarn: e.onWarn
|
|
1457
1571
|
});
|
|
1458
1572
|
I !== void 0 && (e.onRollbackCaptured !== void 0 && (yield* n.tryPromise({
|
|
1459
1573
|
try: () => e.onRollbackCaptured(I),
|
|
1460
|
-
catch: (e) => new
|
|
1574
|
+
catch: (e) => new H({
|
|
1461
1575
|
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1462
1576
|
path: I.dir
|
|
1463
1577
|
})
|
|
1464
|
-
})), e.onWarn?.(
|
|
1465
|
-
let
|
|
1466
|
-
|
|
1467
|
-
id:
|
|
1468
|
-
mode:
|
|
1578
|
+
})), e.onWarn?.(Rn(I)));
|
|
1579
|
+
let _e = (e) => p === "replace" ? ee(e) : e(), L = (e) => p === "replace" ? te(e) : e, R = e.traceId ?? m("dimp", `${Date.now()}-${s.source.schemaFingerprint ?? ""}`), ve = e.traceId !== void 0, ye = (e.targetSnapshot?.tables.some((e) => e.name === u) ?? !1) || v(u) !== void 0;
|
|
1580
|
+
ye && h(f), ye && e.traceId === void 0 && (yield* n.promise(() => e.store.insert(u, {
|
|
1581
|
+
id: R,
|
|
1582
|
+
mode: p,
|
|
1469
1583
|
via: e.via ?? "direct",
|
|
1584
|
+
direction: "import",
|
|
1470
1585
|
bundle: e.bundleDir,
|
|
1471
1586
|
sourceFingerprint: s.source.schemaFingerprint ?? null,
|
|
1472
1587
|
tablesWritten: 0,
|
|
1473
|
-
rowsWritten: 0
|
|
1588
|
+
rowsWritten: 0,
|
|
1589
|
+
staged: p === "replace" ? j : null
|
|
1474
1590
|
}).then(() => {
|
|
1475
|
-
|
|
1591
|
+
ve = !0;
|
|
1476
1592
|
}, (t) => {
|
|
1477
1593
|
e.onWarn?.(`the import trace could not be written (${String(t?.message ?? t)}).\n The import continues — this is a record of what happened, not a safety check. Run
|
|
1478
1594
|
\`voltro db apply\` so the next one is recorded.`);
|
|
1479
|
-
})))
|
|
1480
|
-
|
|
1595
|
+
}))), e.traceId !== void 0 && ye && (yield* n.promise(() => e.store.update(u, R, {
|
|
1596
|
+
staged: p === "replace" ? j : null,
|
|
1597
|
+
sourceFingerprint: s.source.schemaFingerprint ?? null
|
|
1598
|
+
}).then(() => void 0, () => void 0)));
|
|
1599
|
+
let be = 0, xe = !1;
|
|
1600
|
+
pe = () => {
|
|
1601
|
+
if (!ve || xe) return;
|
|
1602
|
+
let t = Date.now();
|
|
1603
|
+
t - be < Wn || (be = t, e.store.update(u, R, {
|
|
1604
|
+
tablesWritten: N,
|
|
1605
|
+
rowsWritten: M
|
|
1606
|
+
}).then(() => void 0, () => {
|
|
1607
|
+
xe = !0;
|
|
1608
|
+
}));
|
|
1609
|
+
};
|
|
1610
|
+
let Se = (t) => ve ? e.store.update(u, R, {
|
|
1481
1611
|
finishedAt: t === null ? /* @__PURE__ */ new Date() : null,
|
|
1482
1612
|
tablesWritten: N,
|
|
1483
1613
|
rowsWritten: M,
|
|
1484
1614
|
failure: t
|
|
1485
1615
|
}).then(() => void 0, () => void 0) : Promise.resolve();
|
|
1486
|
-
if (t = (e) =>
|
|
1616
|
+
if (t = (e) => Se(e), p === "replace" && !j && k.blocker !== void 0 && e.onWarn?.(Sn(k.blocker)), p === "replace" && ue.length > 0 && e.onWarn?.(`write recorders are SUSPENDED for this replace (${ue.length} table(s) have one).\n A replace sets a state rather than changing rows, so a per-row history entry would describe
|
|
1487
1617
|
something that did not happen — and this path already writes through the raw store, without
|
|
1488
1618
|
tenant scoping or \`audit()\` stamping, for the same reason.`), j) {
|
|
1489
|
-
e.onWarn?.(`staging ${
|
|
1490
|
-
let t = e.store, r =
|
|
1491
|
-
getTable: (e) =>
|
|
1619
|
+
e.onWarn?.(`staging ${A.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1620
|
+
let t = e.store, r = A.map((e) => e.name), i = Dn(r, {
|
|
1621
|
+
getTable: (e) => v(e),
|
|
1492
1622
|
registerTable: (e) => {
|
|
1493
|
-
|
|
1623
|
+
C(e);
|
|
1494
1624
|
},
|
|
1495
1625
|
unregisterTable: ne
|
|
1496
1626
|
}), a = !1;
|
|
1497
1627
|
try {
|
|
1498
|
-
let i =
|
|
1628
|
+
let i = g ? r : r.filter((e) => !Mt(x, e));
|
|
1499
1629
|
yield* n.tryPromise({
|
|
1500
1630
|
try: async () => {
|
|
1501
|
-
await
|
|
1631
|
+
await jn(t, i.map((e) => Tn(e, O))), await jn(t, i.map((e) => wn(e, O)));
|
|
1502
1632
|
},
|
|
1503
|
-
catch: (e) => new
|
|
1504
|
-
}), yield*
|
|
1505
|
-
try: () =>
|
|
1633
|
+
catch: (e) => new H({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1634
|
+
}), yield* L(me(e.store, !g)), !g && x.truncated === !0 ? a = !0 : yield* n.tryPromise({
|
|
1635
|
+
try: () => Mn(t, A, O).then(() => {
|
|
1506
1636
|
a = !0;
|
|
1507
1637
|
}),
|
|
1508
|
-
catch: (e) => new
|
|
1638
|
+
catch: (e) => new H({ reason: String(e?.message ?? e) })
|
|
1509
1639
|
}).pipe(n.catchAll((i) => n.gen(function* () {
|
|
1510
|
-
let a =
|
|
1511
|
-
return yield* n.fail(new
|
|
1512
|
-
}))), !
|
|
1640
|
+
let a = An(yield* n.promise(() => Nn(t, On(e.targetSnapshot, r), O).then((e) => e, () => [])));
|
|
1641
|
+
return yield* n.fail(new H({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1642
|
+
}))), !g && a && x.truncated !== !0 && (x.truncated = !0, yield* J(y, x));
|
|
1513
1643
|
} finally {
|
|
1514
|
-
i.undo(), a ||
|
|
1644
|
+
i.undo(), a || g ? yield* n.promise(() => jn(t, r.map((e) => Tn(e, O))).then(() => void 0, () => void 0)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1515
1645
|
}
|
|
1516
|
-
} else
|
|
1646
|
+
} else g ? yield* n.tryPromise({
|
|
1517
1647
|
try: () => e.store.transactional(async (e) => {
|
|
1518
|
-
let t = await
|
|
1519
|
-
if (r.isFailure(t)) throw
|
|
1648
|
+
let t = await _e(() => n.runPromiseExit(me(e, !1)));
|
|
1649
|
+
if (r.isFailure(t)) throw Jn(t.cause);
|
|
1520
1650
|
}),
|
|
1521
|
-
catch: (e) =>
|
|
1522
|
-
}) : yield*
|
|
1651
|
+
catch: (e) => qn(e) ?? new H({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1652
|
+
}) : yield* L(me(e.store, !0));
|
|
1523
1653
|
if (e.assets && s.assets.index && !e.assetsAlreadyRestored) {
|
|
1524
|
-
let t = e.assets, r = i.join(e.bundleDir,
|
|
1525
|
-
yield* l.runForEach(o, (e) => e.key in
|
|
1526
|
-
|
|
1527
|
-
})), n.andThen(
|
|
1654
|
+
let t = e.assets, r = i.join(e.bundleDir, xt), a = i.join(e.bundleDir, s.assets.index ?? "assets/index.ndjson"), o = ce("bundle", () => fe(a)).pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.map((e) => JSON.parse(e)));
|
|
1655
|
+
yield* l.runForEach(o, (e) => e.key in x.assets ? n.void : ot(t, r, e).pipe(n.andThen(n.sync(() => {
|
|
1656
|
+
x.assets[e.key] = !0;
|
|
1657
|
+
})), n.andThen(J(y, x))));
|
|
1528
1658
|
}
|
|
1529
|
-
return yield* n.promise(() =>
|
|
1659
|
+
return yield* n.promise(() => Se(null)), {
|
|
1530
1660
|
manifest: s,
|
|
1531
1661
|
tablesWritten: N,
|
|
1532
1662
|
rowsWritten: M,
|
|
1533
|
-
tablesSkipped:
|
|
1534
|
-
rowsSkipped:
|
|
1535
|
-
fullyResumed: N === 0 &&
|
|
1663
|
+
tablesSkipped: F,
|
|
1664
|
+
rowsSkipped: P,
|
|
1665
|
+
fullyResumed: N === 0 && F > 0,
|
|
1536
1666
|
...I === void 0 ? {} : { rollback: I }
|
|
1537
1667
|
};
|
|
1538
1668
|
}).pipe(n.tapError((e) => n.promise(() => t?.(e?.reason ?? e?._tag ?? "the import did not finish") ?? Promise.resolve())));
|
|
1539
|
-
},
|
|
1669
|
+
}, Zn = (e) => e, Qn = (e) => {
|
|
1540
1670
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1541
1671
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1542
1672
|
return {
|
|
1543
1673
|
refuse: !0,
|
|
1544
1674
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1545
1675
|
};
|
|
1546
|
-
},
|
|
1676
|
+
}, Q = Buffer.from("VBUNDLE2"), $n = 0, er = 1, tr = 2, nr = 32, rr = async (e) => {
|
|
1547
1677
|
let t = [], n = async (r) => {
|
|
1548
1678
|
let i;
|
|
1549
1679
|
try {
|
|
1550
|
-
i = await
|
|
1680
|
+
i = await me(L(e, r), { withFileTypes: !0 });
|
|
1551
1681
|
} catch {
|
|
1552
1682
|
return;
|
|
1553
1683
|
}
|
|
@@ -1557,22 +1687,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1557
1687
|
}
|
|
1558
1688
|
};
|
|
1559
1689
|
return await n(""), t.sort();
|
|
1560
|
-
},
|
|
1690
|
+
}, ir = (e, t) => {
|
|
1561
1691
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1562
|
-
return r.writeUInt8(
|
|
1563
|
-
},
|
|
1692
|
+
return r.writeUInt8(er, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1693
|
+
}, ar = (e, t, n) => {
|
|
1564
1694
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1565
|
-
return a.writeUInt8(
|
|
1695
|
+
return a.writeUInt8(tr, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1566
1696
|
};
|
|
1567
|
-
async function*
|
|
1568
|
-
yield new Uint8Array(
|
|
1569
|
-
for (let t of await
|
|
1570
|
-
let n = L(e.dir, t), r = await
|
|
1571
|
-
if (yield new Uint8Array(
|
|
1697
|
+
async function* or(e) {
|
|
1698
|
+
yield new Uint8Array(Q);
|
|
1699
|
+
for (let t of await rr(e.dir)) {
|
|
1700
|
+
let n = L(e.dir, t), r = await he(n);
|
|
1701
|
+
if (yield new Uint8Array(ir(t, r.size)), r.size > 0) for await (let e of fe(n)) yield new Uint8Array(e);
|
|
1572
1702
|
}
|
|
1573
1703
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1574
|
-
yield new Uint8Array(
|
|
1575
|
-
let n =
|
|
1704
|
+
yield new Uint8Array(ar(t.key, t.contentType, t.size));
|
|
1705
|
+
let n = E("sha256"), r = 0;
|
|
1576
1706
|
for await (let i of e.blobs.open(t.key)) {
|
|
1577
1707
|
let e = Buffer.from(i);
|
|
1578
1708
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1580,10 +1710,10 @@ async function* tr(e) {
|
|
|
1580
1710
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1581
1711
|
yield new Uint8Array(n.digest());
|
|
1582
1712
|
}
|
|
1583
|
-
yield new Uint8Array(Buffer.from([
|
|
1713
|
+
yield new Uint8Array(Buffer.from([$n]));
|
|
1584
1714
|
}
|
|
1585
|
-
var
|
|
1586
|
-
let n =
|
|
1715
|
+
var sr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), cr = (e) => new Promise((t) => e.end(t)), lr = async (e, t) => {
|
|
1716
|
+
let n = R(t.dir);
|
|
1587
1717
|
await F(n, { recursive: !0 });
|
|
1588
1718
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1589
1719
|
let { value: e, done: t } = await i.next();
|
|
@@ -1595,28 +1725,28 @@ var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr
|
|
|
1595
1725
|
return r = r.subarray(e), t;
|
|
1596
1726
|
};
|
|
1597
1727
|
try {
|
|
1598
|
-
if (await s(
|
|
1728
|
+
if (await s(Q.length), !c(Q.length).equals(Q)) throw Error("not a bundle archive (bad magic)");
|
|
1599
1729
|
for (;;) {
|
|
1600
1730
|
await s(1);
|
|
1601
1731
|
let e = c(1).readUInt8(0);
|
|
1602
|
-
if (e ===
|
|
1603
|
-
if (e ===
|
|
1732
|
+
if (e === $n) return;
|
|
1733
|
+
if (e === er) {
|
|
1604
1734
|
await s(4);
|
|
1605
1735
|
let e = c(4).readUInt32BE(0);
|
|
1606
1736
|
await s(e);
|
|
1607
1737
|
let t = c(e).toString("utf8");
|
|
1608
1738
|
await s(8);
|
|
1609
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1610
|
-
if (a !== n && !a.startsWith(n +
|
|
1611
|
-
await F(
|
|
1739
|
+
let i = c(8).readBigUInt64BE(0), a = R(n, t);
|
|
1740
|
+
if (a !== n && !a.startsWith(n + ve)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1741
|
+
await F(_e(a), { recursive: !0 });
|
|
1612
1742
|
let o = M(a);
|
|
1613
1743
|
for (; i > 0n;) {
|
|
1614
1744
|
r.length === 0 && await s(1);
|
|
1615
1745
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1616
|
-
await
|
|
1746
|
+
await sr(o, c(e)), i -= BigInt(e);
|
|
1617
1747
|
}
|
|
1618
|
-
await
|
|
1619
|
-
} else if (e ===
|
|
1748
|
+
await cr(o);
|
|
1749
|
+
} else if (e === tr) {
|
|
1620
1750
|
await s(4);
|
|
1621
1751
|
let e = c(4).readUInt32BE(0);
|
|
1622
1752
|
await s(e);
|
|
@@ -1626,24 +1756,24 @@ var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr
|
|
|
1626
1756
|
await s(i);
|
|
1627
1757
|
let a = c(i).toString("utf8");
|
|
1628
1758
|
await s(8);
|
|
1629
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1759
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = E("sha256"), d = new ue(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1630
1760
|
contentType: a,
|
|
1631
1761
|
size: Number(o)
|
|
1632
1762
|
}), p = o;
|
|
1633
1763
|
for (; p > 0n;) {
|
|
1634
1764
|
r.length === 0 && await s(1);
|
|
1635
1765
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1636
|
-
u.update(t), l || await
|
|
1766
|
+
u.update(t), l || await sr(d, t), p -= BigInt(e);
|
|
1637
1767
|
}
|
|
1638
|
-
l ? d.destroy() : await
|
|
1639
|
-
let m = c(
|
|
1768
|
+
l ? d.destroy() : await cr(d), await f, await s(nr);
|
|
1769
|
+
let m = c(nr);
|
|
1640
1770
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1641
1771
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1642
1772
|
}
|
|
1643
1773
|
} finally {
|
|
1644
1774
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1645
1775
|
}
|
|
1646
|
-
},
|
|
1776
|
+
}, ur = async (e, t) => {
|
|
1647
1777
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1648
1778
|
let { value: e, done: t } = await a.next();
|
|
1649
1779
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1662,11 +1792,11 @@ var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr
|
|
|
1662
1792
|
}
|
|
1663
1793
|
return Buffer.concat(t);
|
|
1664
1794
|
};
|
|
1665
|
-
if (!await s(
|
|
1795
|
+
if (!await s(Q.length) || !c(Q.length).equals(Q)) throw Error("not a bundle archive (bad magic)");
|
|
1666
1796
|
for (; await s(1);) {
|
|
1667
1797
|
let e = c(1).readUInt8(0);
|
|
1668
|
-
if (e ===
|
|
1669
|
-
if (e !==
|
|
1798
|
+
if (e === $n || e === tr) break;
|
|
1799
|
+
if (e !== er) throw Error(`unknown bundle entry kind ${e}`);
|
|
1670
1800
|
if (!await s(4)) break;
|
|
1671
1801
|
let t = c(4).readUInt32BE(0);
|
|
1672
1802
|
if (!await s(t)) break;
|
|
@@ -1676,17 +1806,17 @@ var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr
|
|
|
1676
1806
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1677
1807
|
}
|
|
1678
1808
|
return r;
|
|
1679
|
-
},
|
|
1809
|
+
}, dr = (e, t) => lr(e, {
|
|
1680
1810
|
dir: t,
|
|
1681
|
-
blobs:
|
|
1682
|
-
}),
|
|
1811
|
+
blobs: fr(t)
|
|
1812
|
+
}), fr = (e) => {
|
|
1683
1813
|
let t = /* @__PURE__ */ new Set();
|
|
1684
1814
|
return {
|
|
1685
1815
|
has: async (e) => t.has(e),
|
|
1686
1816
|
write: async (n, r, i) => {
|
|
1687
1817
|
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = L(e, "assets");
|
|
1688
1818
|
await F(c, { recursive: !0 });
|
|
1689
|
-
let l =
|
|
1819
|
+
let l = E("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1690
1820
|
l.update(e), u += e.length, n(null, e);
|
|
1691
1821
|
} }), f = L(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1692
1822
|
await o(r, d, M(f));
|
|
@@ -1699,33 +1829,33 @@ var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr
|
|
|
1699
1829
|
})}\n`), t.add(n);
|
|
1700
1830
|
}
|
|
1701
1831
|
};
|
|
1702
|
-
},
|
|
1832
|
+
}, pr = ye(oe), mr = Buffer.from("VENC1\0\0\0"), hr = 32, gr = 16, _r = 7, vr = 16, yr = 262144, br = {
|
|
1703
1833
|
N: 32768,
|
|
1704
1834
|
r: 8,
|
|
1705
1835
|
p: 1
|
|
1706
|
-
},
|
|
1836
|
+
}, xr = 134217728, Sr = 43, Cr = (e, t, n) => {
|
|
1707
1837
|
let r = Buffer.alloc(12);
|
|
1708
|
-
return e.copy(r, 0, 0,
|
|
1838
|
+
return e.copy(r, 0, 0, _r), r.writeUInt32BE(t >>> 0, _r), r.writeUInt8(+!!n, 11), r;
|
|
1709
1839
|
};
|
|
1710
|
-
async function*
|
|
1840
|
+
async function* wr(e, t) {
|
|
1711
1841
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1712
|
-
let n =
|
|
1713
|
-
...
|
|
1714
|
-
maxmem:
|
|
1715
|
-
}), a = Buffer.alloc(
|
|
1716
|
-
|
|
1842
|
+
let n = D(gr), r = D(_r), i = await pr(t, n, hr, {
|
|
1843
|
+
...br,
|
|
1844
|
+
maxmem: xr
|
|
1845
|
+
}), a = Buffer.alloc(Sr);
|
|
1846
|
+
mr.copy(a, 0), a.writeUInt32BE(br.N, 8), a.writeUInt32BE(br.r, 12), a.writeUInt32BE(br.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1717
1847
|
let o = 0, s = (e, t) => {
|
|
1718
|
-
let n =
|
|
1848
|
+
let n = T("aes-256-gcm", i, Cr(r, o, t));
|
|
1719
1849
|
o === 0 && n.setAAD(a);
|
|
1720
1850
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1721
1851
|
o++;
|
|
1722
|
-
let l = Buffer.alloc(5 + s.length +
|
|
1723
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
1852
|
+
let l = Buffer.alloc(5 + s.length + vr);
|
|
1853
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + vr, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1724
1854
|
}, c = Buffer.alloc(0);
|
|
1725
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1855
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > yr;) yield s(c.subarray(0, yr), !1), c = c.subarray(yr);
|
|
1726
1856
|
yield s(c, !0);
|
|
1727
1857
|
}
|
|
1728
|
-
async function*
|
|
1858
|
+
async function* Tr(e, t) {
|
|
1729
1859
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1730
1860
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1731
1861
|
let { value: e, done: t } = await n.next();
|
|
@@ -1734,16 +1864,16 @@ async function* yr(e, t) {
|
|
|
1734
1864
|
for (; r.length < e && !i && await a(););
|
|
1735
1865
|
return r.length >= e;
|
|
1736
1866
|
};
|
|
1737
|
-
if (!await o(
|
|
1738
|
-
let s = r.subarray(0,
|
|
1739
|
-
if (!s.subarray(0, 8).equals(
|
|
1740
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36,
|
|
1867
|
+
if (!await o(Sr)) throw Error("encrypted bundle truncated (no header)");
|
|
1868
|
+
let s = r.subarray(0, Sr);
|
|
1869
|
+
if (!s.subarray(0, 8).equals(mr)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1870
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, Sr)), p = await pr(t, d, hr, {
|
|
1741
1871
|
N: c,
|
|
1742
1872
|
r: l,
|
|
1743
1873
|
p: u,
|
|
1744
|
-
maxmem:
|
|
1874
|
+
maxmem: xr
|
|
1745
1875
|
});
|
|
1746
|
-
r = r.subarray(
|
|
1876
|
+
r = r.subarray(Sr);
|
|
1747
1877
|
let m = 0, h = !1;
|
|
1748
1878
|
for (;;) {
|
|
1749
1879
|
if (!await o(5)) {
|
|
@@ -1752,11 +1882,11 @@ async function* yr(e, t) {
|
|
|
1752
1882
|
}
|
|
1753
1883
|
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
1754
1884
|
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
1755
|
-
if (t <
|
|
1885
|
+
if (t < vr) throw Error("encrypted bundle corrupt (short frame)");
|
|
1756
1886
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1757
|
-
let n = r.subarray(5, 5 + t -
|
|
1887
|
+
let n = r.subarray(5, 5 + t - vr), i = r.subarray(5 + t - vr, 5 + t);
|
|
1758
1888
|
r = r.subarray(5 + t);
|
|
1759
|
-
let a =
|
|
1889
|
+
let a = ie("aes-256-gcm", p, Cr(f, m, e));
|
|
1760
1890
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1761
1891
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1762
1892
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1764,7 +1894,7 @@ async function* yr(e, t) {
|
|
|
1764
1894
|
}
|
|
1765
1895
|
//#endregion
|
|
1766
1896
|
//#region src/blobStorage.ts
|
|
1767
|
-
var
|
|
1897
|
+
var Er = (e, t) => ({
|
|
1768
1898
|
list: () => {
|
|
1769
1899
|
let e = t();
|
|
1770
1900
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1777,29 +1907,29 @@ var br = (e, t) => ({
|
|
|
1777
1907
|
} };
|
|
1778
1908
|
},
|
|
1779
1909
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1780
|
-
let r = await n.runPromise(
|
|
1910
|
+
let r = await n.runPromise(le(e, t)), i = await k(r.stream);
|
|
1781
1911
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1782
1912
|
} })
|
|
1783
|
-
}),
|
|
1784
|
-
table:
|
|
1913
|
+
}), Dr = (e, t) => Er(e, () => w(t, {
|
|
1914
|
+
table: se,
|
|
1785
1915
|
chunkSize: 500
|
|
1786
1916
|
}).pipe(l.map((e) => ({
|
|
1787
1917
|
key: String(e.key),
|
|
1788
1918
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1789
1919
|
size: Number(e.size ?? 0)
|
|
1790
|
-
})))),
|
|
1920
|
+
})))), Or = (e) => ({
|
|
1791
1921
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1792
1922
|
write: async (t, r, i) => {
|
|
1793
|
-
let a =
|
|
1794
|
-
await n.runPromise(
|
|
1923
|
+
let a = ce("storage", () => j.from(r));
|
|
1924
|
+
await n.runPromise(O(e, t, a, {
|
|
1795
1925
|
contentType: i.contentType,
|
|
1796
1926
|
totalSize: i.size
|
|
1797
1927
|
}));
|
|
1798
1928
|
}
|
|
1799
|
-
}),
|
|
1929
|
+
}), kr = (e, t) => {
|
|
1800
1930
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1801
|
-
return e === "mariadb" &&
|
|
1802
|
-
},
|
|
1931
|
+
return e === "mariadb" && Ar(n) ? n : r;
|
|
1932
|
+
}, Ar = (e) => (process.env.PATH ?? "").split(I).filter((e) => e !== "").some((t) => N(L(t, e))), jr = (e) => {
|
|
1803
1933
|
switch (e) {
|
|
1804
1934
|
case "postgres": return "db.dump";
|
|
1805
1935
|
case "mysql":
|
|
@@ -1809,7 +1939,7 @@ var br = (e, t) => ({
|
|
|
1809
1939
|
case "mssql": return "db.bacpac";
|
|
1810
1940
|
default: return "db.dump";
|
|
1811
1941
|
}
|
|
1812
|
-
},
|
|
1942
|
+
}, Mr = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1813
1943
|
if (e.host || e.database) return {
|
|
1814
1944
|
host: e.host ?? "localhost",
|
|
1815
1945
|
port: e.port ?? 3306,
|
|
@@ -1825,7 +1955,7 @@ var br = (e, t) => ({
|
|
|
1825
1955
|
password: decodeURIComponent(t.password) || "",
|
|
1826
1956
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1827
1957
|
};
|
|
1828
|
-
},
|
|
1958
|
+
}, Nr = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Pr = (e, t, n) => {
|
|
1829
1959
|
switch (e) {
|
|
1830
1960
|
case "postgres": {
|
|
1831
1961
|
if (t.url) return {
|
|
@@ -1872,7 +2002,7 @@ var br = (e, t) => ({
|
|
|
1872
2002
|
let r = $(t);
|
|
1873
2003
|
return {
|
|
1874
2004
|
kind: "spawn",
|
|
1875
|
-
tool:
|
|
2005
|
+
tool: kr(e, "dump"),
|
|
1876
2006
|
args: [
|
|
1877
2007
|
"--single-transaction",
|
|
1878
2008
|
"--routines",
|
|
@@ -1894,7 +2024,7 @@ var br = (e, t) => ({
|
|
|
1894
2024
|
case "sqlite":
|
|
1895
2025
|
case "turso": return {
|
|
1896
2026
|
kind: "copy",
|
|
1897
|
-
from:
|
|
2027
|
+
from: Mr(t),
|
|
1898
2028
|
to: n
|
|
1899
2029
|
};
|
|
1900
2030
|
case "mssql": return {
|
|
@@ -1902,7 +2032,7 @@ var br = (e, t) => ({
|
|
|
1902
2032
|
tool: "sqlpackage",
|
|
1903
2033
|
args: [
|
|
1904
2034
|
"/Action:Export",
|
|
1905
|
-
`/SourceConnectionString:${
|
|
2035
|
+
`/SourceConnectionString:${Nr($({
|
|
1906
2036
|
...t,
|
|
1907
2037
|
port: t.port ?? 1433
|
|
1908
2038
|
}))}`,
|
|
@@ -1910,12 +2040,12 @@ var br = (e, t) => ({
|
|
|
1910
2040
|
],
|
|
1911
2041
|
env: {}
|
|
1912
2042
|
};
|
|
1913
|
-
default: throw new
|
|
2043
|
+
default: throw new W({
|
|
1914
2044
|
tool: "backup",
|
|
1915
2045
|
reason: `no native backup for dialect '${e}'`
|
|
1916
2046
|
});
|
|
1917
2047
|
}
|
|
1918
|
-
},
|
|
2048
|
+
}, Fr = (e, t, n) => {
|
|
1919
2049
|
switch (e) {
|
|
1920
2050
|
case "postgres": {
|
|
1921
2051
|
if (t.url) return {
|
|
@@ -1960,7 +2090,7 @@ var br = (e, t) => ({
|
|
|
1960
2090
|
let r = $(t);
|
|
1961
2091
|
return {
|
|
1962
2092
|
kind: "spawn",
|
|
1963
|
-
tool:
|
|
2093
|
+
tool: kr(e, "client"),
|
|
1964
2094
|
args: [
|
|
1965
2095
|
"--protocol=TCP",
|
|
1966
2096
|
"-h",
|
|
@@ -1979,14 +2109,14 @@ var br = (e, t) => ({
|
|
|
1979
2109
|
case "turso": return {
|
|
1980
2110
|
kind: "copy",
|
|
1981
2111
|
from: n,
|
|
1982
|
-
to:
|
|
2112
|
+
to: Mr(t)
|
|
1983
2113
|
};
|
|
1984
2114
|
case "mssql": return {
|
|
1985
2115
|
kind: "spawn",
|
|
1986
2116
|
tool: "sqlpackage",
|
|
1987
2117
|
args: [
|
|
1988
2118
|
"/Action:Import",
|
|
1989
|
-
`/TargetConnectionString:${
|
|
2119
|
+
`/TargetConnectionString:${Nr($({
|
|
1990
2120
|
...t,
|
|
1991
2121
|
port: t.port ?? 1433
|
|
1992
2122
|
}))}`,
|
|
@@ -1994,18 +2124,18 @@ var br = (e, t) => ({
|
|
|
1994
2124
|
],
|
|
1995
2125
|
env: {}
|
|
1996
2126
|
};
|
|
1997
|
-
default: throw new
|
|
2127
|
+
default: throw new W({
|
|
1998
2128
|
tool: "restore",
|
|
1999
2129
|
reason: `no native restore for dialect '${e}'`
|
|
2000
2130
|
});
|
|
2001
2131
|
}
|
|
2002
|
-
},
|
|
2003
|
-
|
|
2132
|
+
}, Ir = (e) => e.kind === "copy" ? n.async((t) => {
|
|
2133
|
+
de(e.from, e.to, (r) => t(r ? n.fail(new W({
|
|
2004
2134
|
tool: "copy",
|
|
2005
2135
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
2006
2136
|
})) : n.void));
|
|
2007
2137
|
}) : n.async((t) => {
|
|
2008
|
-
let r =
|
|
2138
|
+
let r = be(e.tool, [...e.args], {
|
|
2009
2139
|
env: {
|
|
2010
2140
|
...process.env,
|
|
2011
2141
|
...e.env
|
|
@@ -2018,14 +2148,14 @@ var br = (e, t) => ({
|
|
|
2018
2148
|
}), i = "";
|
|
2019
2149
|
r.stderr?.on("data", (e) => {
|
|
2020
2150
|
i += String(e);
|
|
2021
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin &&
|
|
2151
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin && fe(e.stdinFile).pipe(r.stdin);
|
|
2022
2152
|
let a = () => {
|
|
2023
2153
|
if (e.stdoutFile !== void 0) try {
|
|
2024
|
-
|
|
2154
|
+
pe(e.stdoutFile);
|
|
2025
2155
|
} catch {}
|
|
2026
2156
|
};
|
|
2027
2157
|
r.on("error", (r) => {
|
|
2028
|
-
a(), t(n.fail(new
|
|
2158
|
+
a(), t(n.fail(new W({
|
|
2029
2159
|
tool: e.tool,
|
|
2030
2160
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`
|
|
2031
2161
|
})));
|
|
@@ -2034,17 +2164,17 @@ var br = (e, t) => ({
|
|
|
2034
2164
|
t(n.void);
|
|
2035
2165
|
return;
|
|
2036
2166
|
}
|
|
2037
|
-
a(), t(n.fail(new
|
|
2167
|
+
a(), t(n.fail(new W({
|
|
2038
2168
|
tool: e.tool,
|
|
2039
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2169
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${ge(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
2040
2170
|
stderr: i.slice(-2e3)
|
|
2041
2171
|
})));
|
|
2042
2172
|
});
|
|
2043
|
-
}),
|
|
2044
|
-
let t = yield*
|
|
2173
|
+
}), Lr = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Rr = (e) => n.gen(function* () {
|
|
2174
|
+
let t = yield* nt(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
2045
2175
|
store: e.store,
|
|
2046
2176
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
2047
|
-
}), r =
|
|
2177
|
+
}), r = qt(t.tables, e.snapshot), i = e.classification ?? Ce(e.snapshot), a = e.sampleSize ?? 20, o = Ie(r.map((t) => ({
|
|
2048
2178
|
name: t,
|
|
2049
2179
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
2050
2180
|
name: e.name,
|
|
@@ -2052,19 +2182,19 @@ var br = (e, t) => ({
|
|
|
2052
2182
|
}))
|
|
2053
2183
|
})), i, e.masking), s = [], c = [];
|
|
2054
2184
|
for (let i of r) {
|
|
2055
|
-
let r = t.predicateFor(i), u = yield*
|
|
2185
|
+
let r = t.predicateFor(i), u = yield* w(e.store, {
|
|
2056
2186
|
table: i,
|
|
2057
2187
|
chunkSize: a,
|
|
2058
2188
|
...r === void 0 ? {} : { where: r }
|
|
2059
|
-
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
2060
|
-
c.push(...
|
|
2189
|
+
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) => ze(t, i, o.actions, e.masking.seed));
|
|
2190
|
+
c.push(...Re(d, i, o.actions));
|
|
2061
2191
|
let f = u[0], p = d[0], m = [];
|
|
2062
2192
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
2063
2193
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
2064
2194
|
let n = e.slice(i.length + 1);
|
|
2065
2195
|
m.push({
|
|
2066
2196
|
column: n,
|
|
2067
|
-
action:
|
|
2197
|
+
action: Lr(t),
|
|
2068
2198
|
before: f[n],
|
|
2069
2199
|
after: p[n]
|
|
2070
2200
|
});
|
|
@@ -2079,7 +2209,7 @@ var br = (e, t) => ({
|
|
|
2079
2209
|
leaks: c,
|
|
2080
2210
|
unclassified: o.unclassified
|
|
2081
2211
|
};
|
|
2082
|
-
}),
|
|
2212
|
+
}), zr = (e, t) => {
|
|
2083
2213
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
2084
2214
|
for (let t of e) {
|
|
2085
2215
|
let e = n.get(t.name);
|
|
@@ -2134,7 +2264,7 @@ var br = (e, t) => ({
|
|
|
2134
2264
|
skipColumns: i,
|
|
2135
2265
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
2136
2266
|
};
|
|
2137
|
-
},
|
|
2267
|
+
}, Br = (e) => {
|
|
2138
2268
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
2139
2269
|
let t = {
|
|
2140
2270
|
safe: "✓",
|
|
@@ -2145,4 +2275,4 @@ var br = (e, t) => ({
|
|
|
2145
2275
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
2146
2276
|
};
|
|
2147
2277
|
//#endregion
|
|
2148
|
-
export {
|
|
2278
|
+
export { xt as ASSETS_DIR, St as ASSET_INDEX_FILE, H as BundleError, We as CodecError, Ke as CompressionError, Ze as CrossDialectError, bt as DATA_DIR, Se as DEFAULT_CLASS_ACTIONS, Be as ENVIRONMENT_LOCAL_TABLES, Ve as ENVIRONMENT_PORTABLE_TABLES, ft as FORMAT_VERSION, Xe as ImportModeError, Ge as IntegrityError, Ct as LEDGER_FILE, K as MANIFEST_FILE, Ye as MaskingError, W as NativeToolError, qe as RowsRefusedError, Gn as SAVEPOINT_BATCH_SIZE, bn as STAGING_PREFIX, Je as SchemaDriftError, U as ScopeError, Pe as applyAction, qn as asImportError, Qn as assessLiveImport, Nt as assetDone, jr as backupArtifactName, Pr as backupCommand, cn as canonicalType, dn as checkPortability, Un as checkSchemaDrift, Ce as classificationFromSnapshot, zr as classifyImportDrift, lt as compressionExt, $e as computeSubsetIds, In as decideStaging, vt as decodeManifest, Ut as decodeRow, Vt as decodeValue, Tr as decryptStream, Zn as defineDataProfile, Hn as diffSnapshots, fr as dirBlobSink, Tn as dropStagingSql, At as emptyLedger, _t as encodeManifest, Ht as encodeRow, zt as encodeValue, wr as encryptStream, Ue as environmentLocalNotice, He as environmentLocalReason, jn as execStatements, Br as formatImportDrift, fn as formatIssue, st as hasZstd, V as isEnvironmentLocal, ut as makeCompressor, dt as makeDecompressor, ze as maskRow, Kt as ndjsonToRows, sn as outsideReferenceRefusal, on as outsideReferencesInto, or as packBundle, ur as peekBundle, ct as pickCompression, Ie as planMasking, Rr as previewMasking, jt as readLedger, we as resolveAction, nt as resolveScope, ot as restoreAssetFromCas, Fr as restoreCommand, Gt as rowsToNdjson, an as runExport, Xn as runImport, Ir as runNativeStep, Re as scanForLeaks, X as stagingNameFor, it as storageAssetSink, rt as storageAssetSource, Or as storageBlobSink, Er as storageBlobSource, Dr as storageRefsBlobSource, et as subsetToScope, Mt as tableDone, qt as topoSortTables, lr as unpackBundle, dr as unpackBundleToDir, at as writeAssetToCas, J as writeLedger };
|