@voltro/data-transfer 0.47.0 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -0
- package/dist/index.d.ts +151 -0
- package/dist/index.js +628 -508
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Runtime as o, Schedule as s, Schema as c, Stream as l } from "effect";
|
|
2
|
-
import { IMPORT_RUNS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d,
|
|
2
|
+
import { IMPORT_RUNS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d, _voltroImportRunsTable as f, classifyConstraintViolation as p, derivedRowId as m, ensureTableRegistered as h, eq as g, extractDbCause as _, getTable as v, inSet as y, isNotNull as b, or as x, recordsTable as S, registerTable as C, streamTable as w, suspendWriteRecorders as ee, suspendWriteRecordersEffect as te, unregisterTable as ne } from "@voltro/database";
|
|
3
3
|
import { fingerprintSchema as re } from "@voltro/database/sql";
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { STORAGE_REFS_TABLE as
|
|
4
|
+
import { createCipheriv as T, createDecipheriv as ie, createHash as E, createHmac as ae, randomBytes as oe, scrypt as se } from "node:crypto";
|
|
5
|
+
import { STORAGE_REFS_TABLE as ce, fromNodeReadable as le, getObjectStream as ue, putObjectStream as D, toNodeReadable as O } from "@voltro/plugin-storage";
|
|
6
6
|
import * as k from "node:zlib";
|
|
7
7
|
import { PassThrough as A, Readable as j } from "node:stream";
|
|
8
|
-
import { copyFile as
|
|
8
|
+
import { copyFile as de, createReadStream as fe, createWriteStream as M, existsSync as N, unlinkSync as pe } from "node:fs";
|
|
9
9
|
import { SqlClient as P } from "@effect/sql";
|
|
10
|
-
import { mkdir as F, readdir as
|
|
11
|
-
import { basename as
|
|
12
|
-
import { promisify as
|
|
13
|
-
import { spawn as
|
|
10
|
+
import { mkdir as F, readdir as me, stat as he } from "node:fs/promises";
|
|
11
|
+
import { basename as ge, delimiter as I, dirname as _e, join as L, resolve as R, sep as ve } from "node:path";
|
|
12
|
+
import { promisify as ye } from "node:util";
|
|
13
|
+
import { spawn as be } from "node:child_process";
|
|
14
14
|
//#region src/consistency.ts
|
|
15
|
-
var
|
|
15
|
+
var xe = (e) => {
|
|
16
16
|
switch (e) {
|
|
17
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
18
18
|
case "mysql":
|
|
@@ -22,7 +22,7 @@ var be = (e) => {
|
|
|
22
22
|
case "turso": return [];
|
|
23
23
|
default: return [];
|
|
24
24
|
}
|
|
25
|
-
},
|
|
25
|
+
}, Se = {
|
|
26
26
|
email: { fake: "email" },
|
|
27
27
|
fullName: { fake: "fullName" },
|
|
28
28
|
firstName: { fake: "firstName" },
|
|
@@ -37,7 +37,7 @@ var be = (e) => {
|
|
|
37
37
|
date: "dateShift",
|
|
38
38
|
secret: "null",
|
|
39
39
|
freeText: "redact"
|
|
40
|
-
},
|
|
40
|
+
}, Ce = (e) => {
|
|
41
41
|
let t = {};
|
|
42
42
|
for (let n of e.tables) {
|
|
43
43
|
let e = {};
|
|
@@ -49,15 +49,15 @@ var be = (e) => {
|
|
|
49
49
|
t[n.name] = e;
|
|
50
50
|
}
|
|
51
51
|
return t;
|
|
52
|
-
},
|
|
52
|
+
}, we = (e, t, n, r) => {
|
|
53
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
54
54
|
if (i) return i;
|
|
55
55
|
if (n?.sensitive) {
|
|
56
56
|
let e = n.sensitive.class;
|
|
57
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? Se[e] ?? "redact";
|
|
58
58
|
}
|
|
59
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
60
|
-
},
|
|
60
|
+
}, Te = (e, t) => ae("sha256", e).update(t == null ? "∅" : String(t)).digest(), z = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), Ee = [
|
|
61
61
|
"Ada",
|
|
62
62
|
"Alan",
|
|
63
63
|
"Grace",
|
|
@@ -70,7 +70,7 @@ var be = (e) => {
|
|
|
70
70
|
"Guido",
|
|
71
71
|
"Margaret",
|
|
72
72
|
"Bjarne"
|
|
73
|
-
],
|
|
73
|
+
], De = [
|
|
74
74
|
"Lovelace",
|
|
75
75
|
"Turing",
|
|
76
76
|
"Hopper",
|
|
@@ -83,7 +83,7 @@ var be = (e) => {
|
|
|
83
83
|
"Rossum",
|
|
84
84
|
"Hamilton",
|
|
85
85
|
"Stroustrup"
|
|
86
|
-
],
|
|
86
|
+
], Oe = [
|
|
87
87
|
"Maple",
|
|
88
88
|
"Oak",
|
|
89
89
|
"Cedar",
|
|
@@ -92,7 +92,7 @@ var be = (e) => {
|
|
|
92
92
|
"Birch",
|
|
93
93
|
"Willow",
|
|
94
94
|
"Ash"
|
|
95
|
-
],
|
|
95
|
+
], ke = [
|
|
96
96
|
"Initech",
|
|
97
97
|
"Hooli",
|
|
98
98
|
"Umbrella",
|
|
@@ -101,23 +101,23 @@ var be = (e) => {
|
|
|
101
101
|
"Soylent",
|
|
102
102
|
"Stark",
|
|
103
103
|
"Wayne"
|
|
104
|
-
],
|
|
105
|
-
let r =
|
|
104
|
+
], B = (e, t, n) => e[z(t, n) % e.length], Ae = (e, t, n) => {
|
|
105
|
+
let r = Te(n, `${e}:${String(t)}`), i = z(r, 1) % 1e4;
|
|
106
106
|
switch (e) {
|
|
107
|
-
case "email": return `${
|
|
108
|
-
case "fullName": return `${
|
|
109
|
-
case "firstName": return
|
|
110
|
-
case "lastName": return
|
|
111
|
-
case "username": return `${
|
|
112
|
-
case "phone": return `+1${String(
|
|
113
|
-
case "address": return `${
|
|
114
|
-
case "company": return
|
|
107
|
+
case "email": return `${B(Ee, r, 0).toLowerCase()}.${B(De, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${B(Ee, r, 0)} ${B(De, r, 1)}`;
|
|
109
|
+
case "firstName": return B(Ee, r, 0);
|
|
110
|
+
case "lastName": return B(De, r, 1);
|
|
111
|
+
case "username": return `${B(Ee, r, 0).toLowerCase()}${i}`;
|
|
112
|
+
case "phone": return `+1${String(z(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
+
case "address": return `${z(r, 0) % 9899 + 100} ${B(Oe, r, 1)} St`;
|
|
114
|
+
case "company": return B(ke, r, 0);
|
|
115
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
116
|
-
case "ip": return `${
|
|
117
|
-
case "creditCard": return
|
|
116
|
+
case "ip": return `${z(r, 0) % 256}.${z(r, 1) % 256}.${z(r, 2) % 256}.${z(r, 3) % 254 + 1}`;
|
|
117
|
+
case "creditCard": return je(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
118
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
119
119
|
}
|
|
120
|
-
},
|
|
120
|
+
}, je = (e) => {
|
|
121
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
122
122
|
t.forEach((e, t) => {
|
|
123
123
|
let r = e;
|
|
@@ -125,31 +125,31 @@ var be = (e) => {
|
|
|
125
125
|
});
|
|
126
126
|
let r = (10 - n % 10) % 10;
|
|
127
127
|
return e.slice(0, 15) + String(r);
|
|
128
|
-
},
|
|
128
|
+
}, Me = 864e5, Ne = (e, t) => {
|
|
129
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
130
130
|
if (Number.isNaN(n)) return e;
|
|
131
|
-
let r =
|
|
131
|
+
let r = z(Te(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Me);
|
|
132
132
|
return e instanceof Date ? i : i.toISOString();
|
|
133
|
-
},
|
|
133
|
+
}, Pe = (e, t) => {
|
|
134
134
|
if (t.value === null || t.value === void 0) return null;
|
|
135
135
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
136
136
|
if (e === "keep") return t.value;
|
|
137
137
|
if (e === "null") return null;
|
|
138
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
139
|
-
if (e === "hash") return
|
|
140
|
-
if (e === "dateShift") return
|
|
141
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return Te(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Ne(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return Ae(e.fake, t.value, t.seed);
|
|
142
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
143
143
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
144
|
-
},
|
|
144
|
+
}, Fe = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Ie = (e, t, n) => {
|
|
145
145
|
let r = {}, i = [], a = [];
|
|
146
146
|
for (let o of e) for (let e of o.columns) {
|
|
147
|
-
let s = `${o.name}.${e.name}`, c =
|
|
147
|
+
let s = `${o.name}.${e.name}`, c = we(o.name, e.name, t[o.name]?.[e.name], n);
|
|
148
148
|
if (c === "error") {
|
|
149
149
|
i.push(s);
|
|
150
150
|
continue;
|
|
151
151
|
}
|
|
152
|
-
if (!
|
|
152
|
+
if (!Fe(c)) {
|
|
153
153
|
a.push({
|
|
154
154
|
column: s,
|
|
155
155
|
action: c
|
|
@@ -163,7 +163,7 @@ var be = (e) => {
|
|
|
163
163
|
unclassified: i,
|
|
164
164
|
invalid: a
|
|
165
165
|
};
|
|
166
|
-
},
|
|
166
|
+
}, Le = [
|
|
167
167
|
{
|
|
168
168
|
name: "email",
|
|
169
169
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -180,7 +180,7 @@ var be = (e) => {
|
|
|
180
180
|
name: "creditCard",
|
|
181
181
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
182
182
|
}
|
|
183
|
-
],
|
|
183
|
+
], Re = (e, t, n) => {
|
|
184
184
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
185
185
|
for (let [a, o] of Object.entries(n)) {
|
|
186
186
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -188,7 +188,7 @@ var be = (e) => {
|
|
|
188
188
|
for (let a of e) {
|
|
189
189
|
let e = a[n];
|
|
190
190
|
if (typeof e != "string" || e.length === 0) continue;
|
|
191
|
-
let o =
|
|
191
|
+
let o = Le.find((t) => t.re.test(e));
|
|
192
192
|
if (o && !i.has(n)) {
|
|
193
193
|
i.add(n), r.push({
|
|
194
194
|
table: t,
|
|
@@ -201,9 +201,9 @@ var be = (e) => {
|
|
|
201
201
|
}
|
|
202
202
|
}
|
|
203
203
|
return r;
|
|
204
|
-
},
|
|
204
|
+
}, ze = (e, t, n, r, i) => {
|
|
205
205
|
let a = {};
|
|
206
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
206
|
+
for (let [o, s] of Object.entries(e)) a[o] = Pe(n[`${t}.${o}`] ?? "keep", {
|
|
207
207
|
value: s,
|
|
208
208
|
table: t,
|
|
209
209
|
column: o,
|
|
@@ -211,7 +211,7 @@ var be = (e) => {
|
|
|
211
211
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
212
212
|
});
|
|
213
213
|
return a;
|
|
214
|
-
},
|
|
214
|
+
}, Be = /* @__PURE__ */ new Map([
|
|
215
215
|
["_voltro_migration_plans", "the schema fingerprint THIS deployment applied — a foreign row makes the next boot refuse"],
|
|
216
216
|
["_voltro_migrations", "which file-based migrations ran HERE — a foreign row marks unrun migrations as done"],
|
|
217
217
|
["_voltro_seeds", "which seeds ran HERE — a foreign row suppresses seeding the target still needs"],
|
|
@@ -223,22 +223,72 @@ var be = (e) => {
|
|
|
223
223
|
["_voltro_workflow_watermarks", "how far a consumer has read THIS deployment's event journal — a foreign position skips events"],
|
|
224
224
|
["_voltro_workflow_pending", "starts waiting to be drained HERE — a foreign row makes the target RUN a workflow somebody queued elsewhere"],
|
|
225
225
|
["_voltro_workflow_admissions", "the in-flight accounting a concurrency gate decides on — a foreign row makes the target refuse or admit wrongly"],
|
|
226
|
-
["_voltro_workflow_pauses", "which workflows an operator paused HERE — a foreign row silently pauses them in the target"]
|
|
227
|
-
|
|
226
|
+
["_voltro_workflow_pauses", "which workflows an operator paused HERE — a foreign row silently pauses them in the target"],
|
|
227
|
+
["_voltro_workflow_event_deliveries", "which (event, trigger) pairs were already delivered HERE — a foreign row SUPPRESSES a delivery the target owes"],
|
|
228
|
+
["_voltro_traces", "spans recorded HERE — a foreign span claims a request this deployment never served"],
|
|
229
|
+
["_voltro_undo_log", "the before-images of mutations made HERE — a foreign row offers to undo a write into rows the target does not have"],
|
|
230
|
+
["_voltro_outbox", "messages waiting to be SENT by this deployment — a foreign row makes the target deliver something queued elsewhere"],
|
|
231
|
+
["_voltro_outbox_attempts", "the delivery attempts this deployment made — they belong to its outbox rows and to no other"],
|
|
232
|
+
["_voltro_ai_inferences", "offloaded inference work queued for THIS deployment's runners — a foreign row makes the target run somebody else's"],
|
|
233
|
+
["_voltro_ai_budget", "the live spend RESERVATION counter a budget refuses against — a foreign row makes the target refuse calls it should allow"],
|
|
234
|
+
["_voltro_ai_usage", "what this deployment spent on model calls — a foreign row overstates a bill nobody here ran up"],
|
|
235
|
+
["_voltro_budget_holds", "in-flight holds against a running deployment's budget — the same accounting `_voltro_workflow_admissions` keeps"],
|
|
236
|
+
["_voltro_storage_grants", "short-lived upload/download grants issued HERE — a foreign grant authorises nothing in the target"],
|
|
237
|
+
["_voltro_storage_usage", "how much storage this deployment accounts for — a foreign row makes a quota decision on somebody else's bytes"],
|
|
238
|
+
["_voltro_idempotency", "the request keys this deployment has already answered — a foreign key SUPPRESSES a request the target never served"],
|
|
239
|
+
["_voltro_webhook_deliveries", "the delivery attempts this deployment made to a subscriber — a foreign row claims a call the target never placed"],
|
|
240
|
+
["_voltro_schedule_runs", "every schedule firing that happened HERE, replica ids included — a foreign row is history the target did not live"],
|
|
241
|
+
["_voltro_cdc_log", "the change rows this deployment captured from its own database — a foreign row replays a change that never happened here"],
|
|
242
|
+
["_voltro_notification_deliveries", "the notification attempts this deployment made — a foreign row claims a send the target never performed"],
|
|
243
|
+
["_voltro_notification_held", "notifications waiting to be SENT from here — a foreign row makes the target deliver something queued elsewhere"],
|
|
244
|
+
["_voltro_webhook_event_stats", "per-deployment delivery counters a backoff decision reads — foreign counts steer the target's retries"],
|
|
245
|
+
["_voltro_webhook_rate_windows", "the live rate-limit windows this deployment is inside — a foreign window throttles or releases wrongly"]
|
|
246
|
+
]), Ve = /* @__PURE__ */ new Map([
|
|
247
|
+
["_voltro_agent_threads", "conversations the app's users had — app data"],
|
|
248
|
+
["_voltro_agent_messages", "the messages in those conversations — app data"],
|
|
249
|
+
["_voltro_prompts", "prompt definitions the app authored — app data"],
|
|
250
|
+
["_voltro_api_keys", "keys the app issued (hashed) — app data, and a restore without them locks callers out"],
|
|
251
|
+
["_voltro_kv", "the app's own key/value rows — app data by construction"],
|
|
252
|
+
["_voltro_approvals", "human decisions on workflow runs — they belong with the runs"],
|
|
253
|
+
["_voltro_connections", "third-party connections the app's users authorised. Portable as ROWS; note the secrets are encrypted with the project's key, so a target without that key holds ciphertext it cannot read"],
|
|
254
|
+
["_voltro_connection_grants", "who may use which connection — belongs with the connections"],
|
|
255
|
+
["_voltro_storage_blobs", "the stored objects' metadata — the assets travel with the bundle"],
|
|
256
|
+
["_voltro_storage_refs", "which rows reference which blob — belongs with the blobs"],
|
|
257
|
+
["_voltro_webhook_targets", "the endpoints subscribers registered through the app — app data"],
|
|
258
|
+
["_voltro_workflow_runs", "what the app's workflows did — history the app owns"],
|
|
259
|
+
["_voltro_workflow_run_steps", "the steps of those runs — belongs with the runs"],
|
|
260
|
+
["_voltro_workflow_run_events", "the events those runs emitted — belongs with the runs"],
|
|
261
|
+
["_voltro_workflow_events", "the app's event journal — app data (the target keeps its OWN watermarks, which are local)"],
|
|
262
|
+
["_voltro_workflow_start_contexts", "the context a start was made with — belongs with the runs"],
|
|
263
|
+
["_voltro_events", "the app's domain events — app data"],
|
|
264
|
+
["_voltro_ai_flows", "the flow definitions the app authored — app data"],
|
|
265
|
+
["_voltro_ai_flow_runs", "what those flows did — history the app owns, like a workflow run"],
|
|
266
|
+
["_voltro_notification_inbox", "what the app's users received — app data, and a restore without it loses their inbox"],
|
|
267
|
+
["_voltro_notification_preferences", "the per-user delivery settings the app's users chose — app data"],
|
|
268
|
+
["_voltro_notification_quiet_hours", "the per-user quiet hours the app's users chose — app data"],
|
|
269
|
+
["_voltro_notification_topic_subscriptions", "which topics the app's users subscribed to — app data"],
|
|
270
|
+
["_voltro_rebac_tuples", "the relationship tuples the app's authorisation decisions are made from — app data"],
|
|
271
|
+
["_voltro_scim_users", "identities an external directory provisioned into the app — app data"],
|
|
272
|
+
["_voltro_scim_groups", "groups an external directory provisioned into the app — app data"],
|
|
273
|
+
["_voltro_row_history", "the app's own row-version history — the point of keeping versions is that they survive a move"],
|
|
274
|
+
["_voltro_audit_log", "the app's audit trail — compliance data the app owns, not deployment bookkeeping"],
|
|
275
|
+
["_voltro_feature_flags", "the flags the app defined — app configuration"],
|
|
276
|
+
["_voltro_feature_flag_audit", "who changed which flag — belongs with the flags"]
|
|
277
|
+
]), V = (e) => Be.has(e), He = (e) => Be.get(e), Ue = (e, t) => {
|
|
228
278
|
if (e.length === 0) return;
|
|
229
|
-
let n = e.map((e) => ` ${e} — ${
|
|
279
|
+
let n = e.map((e) => ` ${e} — ${He(e) ?? "describes this deployment"}`);
|
|
230
280
|
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
231
|
-
},
|
|
281
|
+
}, H = class extends c.TaggedError()("BundleError", {
|
|
232
282
|
reason: c.String,
|
|
233
283
|
path: c.optional(c.String)
|
|
234
|
-
}) {},
|
|
284
|
+
}) {}, We = class extends c.TaggedError()("CodecError", {
|
|
235
285
|
table: c.String,
|
|
236
286
|
reason: c.String
|
|
237
|
-
}) {},
|
|
287
|
+
}) {}, Ge = class extends c.TaggedError()("IntegrityError", {
|
|
238
288
|
what: c.String,
|
|
239
289
|
expected: c.String,
|
|
240
290
|
actual: c.String
|
|
241
|
-
}) {},
|
|
291
|
+
}) {}, Ke = class extends t.TaggedError("CompressionError") {}, U = class extends c.TaggedError()("ScopeError", { reason: c.String }) {}, qe = class extends c.TaggedError()("RowsRefusedError", {
|
|
242
292
|
rows: c.Array(c.Struct({
|
|
243
293
|
table: c.String,
|
|
244
294
|
id: c.String,
|
|
@@ -252,29 +302,29 @@ var be = (e) => {
|
|
|
252
302
|
refused: c.Number,
|
|
253
303
|
primary: c.Number
|
|
254
304
|
}))
|
|
255
|
-
}) {},
|
|
305
|
+
}) {}, Je = class extends c.TaggedError()("SchemaDriftError", {
|
|
256
306
|
bundleFingerprint: c.String,
|
|
257
307
|
targetFingerprint: c.String,
|
|
258
308
|
diff: c.Array(c.String)
|
|
259
|
-
}) {},
|
|
309
|
+
}) {}, W = class extends t.TaggedError("NativeToolError") {
|
|
260
310
|
get message() {
|
|
261
311
|
let e = this.stderr?.trim();
|
|
262
312
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
263
313
|
}
|
|
264
|
-
},
|
|
314
|
+
}, Ye = class extends c.TaggedError()("MaskingError", {
|
|
265
315
|
unclassified: c.Array(c.String),
|
|
266
316
|
invalidActions: c.optional(c.Array(c.String))
|
|
267
|
-
}) {},
|
|
317
|
+
}) {}, Xe = class extends c.TaggedError()("ImportModeError", {
|
|
268
318
|
mode: c.String,
|
|
269
319
|
reason: c.String
|
|
270
|
-
}) {},
|
|
320
|
+
}) {}, Ze = class extends c.TaggedError()("CrossDialectError", {
|
|
271
321
|
sourceDialect: c.String,
|
|
272
322
|
targetDialect: c.String,
|
|
273
323
|
blocking: c.Array(c.String)
|
|
274
|
-
}) {},
|
|
324
|
+
}) {}, G = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Qe = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
275
325
|
col: e.name,
|
|
276
326
|
refTable: e.references.table
|
|
277
|
-
})),
|
|
327
|
+
})), $e = (e, t, r) => n.gen(function* () {
|
|
278
328
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), u = (e) => {
|
|
279
329
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
280
330
|
return s.set(e, t), t;
|
|
@@ -292,10 +342,10 @@ var be = (e) => {
|
|
|
292
342
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
293
343
|
};
|
|
294
344
|
}, p = (r, i) => {
|
|
295
|
-
let a = f(r,
|
|
296
|
-
return l.runForEach(
|
|
345
|
+
let a = f(r, G(t, r), Qe(t, r));
|
|
346
|
+
return l.runForEach(w(e, {
|
|
297
347
|
table: r,
|
|
298
|
-
pk:
|
|
348
|
+
pk: G(t, r),
|
|
299
349
|
...i === void 0 ? {} : { where: i }
|
|
300
350
|
}), (e) => n.sync(() => a(e)));
|
|
301
351
|
}, m = n.gen(function* () {
|
|
@@ -304,7 +354,7 @@ var be = (e) => {
|
|
|
304
354
|
c.clear();
|
|
305
355
|
for (let [n, r] of e) {
|
|
306
356
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
307
|
-
e.length !== 0 && (yield* p(n,
|
|
357
|
+
e.length !== 0 && (yield* p(n, y(G(t, n), e)));
|
|
308
358
|
}
|
|
309
359
|
}
|
|
310
360
|
});
|
|
@@ -319,29 +369,32 @@ var be = (e) => {
|
|
|
319
369
|
refTable: e.references?.table
|
|
320
370
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
321
371
|
if (a.length === 0) continue;
|
|
322
|
-
let o = s.get(t)?.size ?? 0, c = a.map((e) =>
|
|
323
|
-
yield* p(t, c.length === 1 ? c[0] :
|
|
372
|
+
let o = s.get(t)?.size ?? 0, c = a.map((e) => y(e.col, [...s.get(e.refTable)]));
|
|
373
|
+
yield* p(t, c.length === 1 ? c[0] : x(...c)), (s.get(t)?.size ?? 0) > o && !e.has(t) && r.push(t), (s.get(t)?.size ?? 0) > o && (n = !0);
|
|
324
374
|
}
|
|
325
375
|
for (let t of r) e.add(t);
|
|
326
376
|
if (yield* m, !n) break;
|
|
327
377
|
}
|
|
328
378
|
}
|
|
329
379
|
return o;
|
|
330
|
-
}),
|
|
380
|
+
}), et = (e, t) => ({
|
|
331
381
|
tables: [...e.keys()],
|
|
332
382
|
predicateFor: (n) => {
|
|
333
383
|
let r = e.get(n);
|
|
334
|
-
return r ?
|
|
384
|
+
return r ? y(G(t, n), [...r]) : void 0;
|
|
335
385
|
}
|
|
336
|
-
}),
|
|
386
|
+
}), tt = "tenantId", nt = (e, t, r) => n.gen(function* () {
|
|
337
387
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
338
|
-
if (e.kind === "all")
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
388
|
+
if (e.kind === "all") {
|
|
389
|
+
let t = new Set(e.exclude ?? []);
|
|
390
|
+
return {
|
|
391
|
+
tables: i.filter((e) => !V(e) && !t.has(e)),
|
|
392
|
+
predicateFor: () => void 0
|
|
393
|
+
};
|
|
394
|
+
}
|
|
342
395
|
if (e.kind === "tables") {
|
|
343
396
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
344
|
-
if (a.length > 0) return yield* n.fail(new
|
|
397
|
+
if (a.length > 0) return yield* n.fail(new U({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
345
398
|
let o = new Set(t);
|
|
346
399
|
return {
|
|
347
400
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -349,25 +402,25 @@ var be = (e) => {
|
|
|
349
402
|
};
|
|
350
403
|
}
|
|
351
404
|
let o = e.tenantId;
|
|
352
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
353
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
354
|
-
if (s.length === 0) return yield* n.fail(new
|
|
355
|
-
let c = yield*
|
|
356
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
405
|
+
if (o === void 0 || o === "") return yield* n.fail(new U({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
406
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === tt) ?? !1);
|
|
407
|
+
if (s.length === 0) return yield* n.fail(new U({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${tt}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
408
|
+
let c = yield* $e(r.store, t, {
|
|
409
|
+
seeds: Object.fromEntries(s.map((e) => [e, g(tt, o)])),
|
|
357
410
|
pinned: s,
|
|
358
411
|
children: { roots: s }
|
|
359
412
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
360
413
|
return {
|
|
361
414
|
tables: [...s, ...l],
|
|
362
|
-
predicateFor: (e) => u.has(e) ?
|
|
415
|
+
predicateFor: (e) => u.has(e) ? g(tt, o) : c.has(e) ? y(G(t, e), [...c.get(e)]) : void 0
|
|
363
416
|
};
|
|
364
|
-
}),
|
|
417
|
+
}), rt = (e, t) => ({
|
|
365
418
|
list: t,
|
|
366
|
-
open: (t) =>
|
|
367
|
-
}),
|
|
419
|
+
open: (t) => ue(e, t)
|
|
420
|
+
}), it = (e) => ({
|
|
368
421
|
write: (t, n, r) => D(e, t, n, r),
|
|
369
422
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
370
|
-
}),
|
|
423
|
+
}), at = (e, t) => n.tryPromise({
|
|
371
424
|
try: async () => {
|
|
372
425
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
373
426
|
u.update(e), d += e.length, n(null, e);
|
|
@@ -385,23 +438,23 @@ var be = (e) => {
|
|
|
385
438
|
contentType: e.contentType
|
|
386
439
|
};
|
|
387
440
|
},
|
|
388
|
-
catch: (e) => new
|
|
441
|
+
catch: (e) => new H({
|
|
389
442
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
390
443
|
path: t
|
|
391
444
|
})
|
|
392
|
-
}),
|
|
393
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u =
|
|
445
|
+
}), ot = (e, t, r) => n.gen(function* () {
|
|
446
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u = le("bundle", () => a(s)).pipe(l.tap((e) => n.sync(() => c.update(e))));
|
|
394
447
|
yield* e.write(r.key, u, {
|
|
395
448
|
contentType: r.contentType,
|
|
396
449
|
totalSize: r.size
|
|
397
450
|
});
|
|
398
451
|
let d = c.digest("hex");
|
|
399
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
452
|
+
if (d !== r.sha256) return yield* n.fail(new Ge({
|
|
400
453
|
what: `asset ${r.key}`,
|
|
401
454
|
expected: r.sha256,
|
|
402
455
|
actual: d
|
|
403
456
|
}));
|
|
404
|
-
}),
|
|
457
|
+
}), st = () => typeof k.createZstdCompress == "function", ct = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : st() ? "zstd" : "gzip", lt = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", ut = (e) => e === "zstd" ? k.createZstdCompress() : e === "gzip" ? k.createGzip() : new A(), dt = (e) => e === "zstd" ? k.createZstdDecompress() : e === "gzip" ? k.createGunzip() : new A(), ft = 1, pt = c.Struct({
|
|
405
458
|
name: c.String,
|
|
406
459
|
file: c.String,
|
|
407
460
|
rowCount: c.Number,
|
|
@@ -411,11 +464,12 @@ var be = (e) => {
|
|
|
411
464
|
value: c.String
|
|
412
465
|
}),
|
|
413
466
|
primaryKey: c.String
|
|
414
|
-
}),
|
|
467
|
+
}), mt = c.Struct({
|
|
415
468
|
kind: c.Literal("all", "tenant", "tables"),
|
|
416
469
|
tenantId: c.optional(c.String),
|
|
417
|
-
tables: c.optional(c.Array(c.String))
|
|
418
|
-
|
|
470
|
+
tables: c.optional(c.Array(c.String)),
|
|
471
|
+
exclude: c.optional(c.Array(c.String))
|
|
472
|
+
}), ht = c.Struct({
|
|
419
473
|
count: c.Number,
|
|
420
474
|
totalBytes: c.Number,
|
|
421
475
|
index: c.optional(c.String)
|
|
@@ -426,23 +480,23 @@ c.Struct({
|
|
|
426
480
|
size: c.Number,
|
|
427
481
|
contentType: c.String
|
|
428
482
|
});
|
|
429
|
-
var
|
|
483
|
+
var gt = c.Struct({
|
|
430
484
|
formatVersion: c.Number,
|
|
431
485
|
createdAt: c.String,
|
|
432
486
|
source: c.Struct({
|
|
433
487
|
dialect: c.String,
|
|
434
488
|
schemaFingerprint: c.String
|
|
435
489
|
}),
|
|
436
|
-
scope:
|
|
490
|
+
scope: mt,
|
|
437
491
|
consistency: c.Literal("live", "snapshot"),
|
|
438
492
|
compression: c.Literal("zstd", "gzip", "none"),
|
|
439
|
-
tables: c.Array(
|
|
440
|
-
assets:
|
|
493
|
+
tables: c.Array(pt),
|
|
494
|
+
assets: ht,
|
|
441
495
|
masking: c.optional(c.Struct({
|
|
442
496
|
policyId: c.optional(c.String),
|
|
443
497
|
transformed: c.Array(c.String)
|
|
444
498
|
}))
|
|
445
|
-
}),
|
|
499
|
+
}), _t = c.encodeSync(c.parseJson(gt)), vt = c.decodeUnknownSync(c.parseJson(gt)), K = "manifest.json", yt = "README.md", bt = "data", xt = "assets", St = "assets/index.ndjson", Ct = ".ledger.json", wt = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", Tt = (e, t) => {
|
|
446
500
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
447
501
|
return `# Voltro data export
|
|
448
502
|
|
|
@@ -456,7 +510,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
456
510
|
| **Format version** | ${e.formatVersion} |
|
|
457
511
|
| **Consistency** | ${e.consistency} |
|
|
458
512
|
| **Compression** | ${e.compression} |
|
|
459
|
-
| **Scope** | ${
|
|
513
|
+
| **Scope** | ${wt(e.scope)} |
|
|
460
514
|
| **Tables** | ${e.tables.length} |
|
|
461
515
|
| **Rows (total)** | ${n} |
|
|
462
516
|
| **Assets** | ${a} |
|
|
@@ -481,39 +535,39 @@ voltro data import <this-directory>
|
|
|
481
535
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
482
536
|
\`\`\`
|
|
483
537
|
`;
|
|
484
|
-
},
|
|
538
|
+
}, Et = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), Dt = a.exponential({
|
|
485
539
|
start: .01,
|
|
486
540
|
factor: 2,
|
|
487
541
|
count: 16
|
|
488
|
-
}),
|
|
542
|
+
}), Ot = i.histogram("voltro_data_transfer_table_seconds", Dt, "Per-table export/import duration in seconds, tagged by phase."), q = (e, t) => {
|
|
489
543
|
if (e) try {
|
|
490
544
|
e(t);
|
|
491
545
|
} catch {}
|
|
492
|
-
},
|
|
546
|
+
}, kt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(Ot.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), Et.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
493
547
|
table: t,
|
|
494
548
|
index: r,
|
|
495
549
|
tableCount: a,
|
|
496
550
|
phase: e
|
|
497
|
-
} })),
|
|
551
|
+
} })), At = () => ({
|
|
498
552
|
tables: {},
|
|
499
553
|
assets: {}
|
|
500
|
-
}),
|
|
554
|
+
}), jt = (e) => n.tryPromise(async () => {
|
|
501
555
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
502
556
|
return JSON.parse(t);
|
|
503
|
-
}).pipe(n.orElseSucceed(
|
|
557
|
+
}).pipe(n.orElseSucceed(At)), J = (e, t) => n.tryPromise({
|
|
504
558
|
try: async () => {
|
|
505
559
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
506
560
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
507
561
|
},
|
|
508
|
-
catch: (t) => new
|
|
562
|
+
catch: (t) => new H({
|
|
509
563
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
510
564
|
path: e
|
|
511
565
|
})
|
|
512
|
-
}),
|
|
566
|
+
}), Mt = (e, t) => t in e.tables, Nt = (e, t) => t in e.assets, Pt = /* @__PURE__ */ new Set([
|
|
513
567
|
"json",
|
|
514
568
|
"array",
|
|
515
569
|
"vector"
|
|
516
|
-
]),
|
|
570
|
+
]), Ft = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", It = (e) => Buffer.from(e).toString("base64"), Lt = (e) => new Uint8Array(Buffer.from(e, "base64")), Rt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), zt = (e, t) => e == null ? null : t !== void 0 && Pt.has(t) ? e : t === "bytes" ? It(e) : t === "timestamp" || t === "date" ? Rt(e) : typeof e == "bigint" ? {
|
|
517
571
|
$v: "bigint",
|
|
518
572
|
d: e.toString()
|
|
519
573
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -524,30 +578,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
524
578
|
d: e.toISOString()
|
|
525
579
|
} : e instanceof Uint8Array ? {
|
|
526
580
|
$v: "bytes",
|
|
527
|
-
d:
|
|
528
|
-
} : e,
|
|
581
|
+
d: It(e)
|
|
582
|
+
} : e, Bt = (e) => {
|
|
529
583
|
switch (e.$v) {
|
|
530
584
|
case "bigint": return BigInt(e.d);
|
|
531
585
|
case "number": return Number(e.d);
|
|
532
586
|
case "date": return new Date(e.d);
|
|
533
|
-
case "bytes": return
|
|
587
|
+
case "bytes": return Lt(e.d);
|
|
534
588
|
default: return e;
|
|
535
589
|
}
|
|
536
|
-
},
|
|
590
|
+
}, Vt = (e, t) => e == null ? null : t !== void 0 && Pt.has(t) ? e : Ft(e) ? Bt(e) : t === "bytes" ? Lt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Ht = (e, t) => {
|
|
537
591
|
let n = {};
|
|
538
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
592
|
+
for (let [r, i] of Object.entries(e)) n[r] = zt(i, t[r]);
|
|
539
593
|
return n;
|
|
540
|
-
},
|
|
594
|
+
}, Ut = (e, t) => {
|
|
541
595
|
let n = {};
|
|
542
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
596
|
+
for (let [r, i] of Object.entries(e)) n[r] = Vt(i, t[r]);
|
|
543
597
|
return n;
|
|
544
|
-
},
|
|
545
|
-
try: () =>
|
|
546
|
-
catch: (e) => new
|
|
598
|
+
}, Wt = new TextEncoder(), Gt = (e, t) => e.pipe(l.map((e) => Wt.encode(`${JSON.stringify(Ht(e, t))}\n`))), Kt = (e, t, r) => e.pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.mapEffect((e) => n.try({
|
|
599
|
+
try: () => Ut(JSON.parse(e), r),
|
|
600
|
+
catch: (e) => new We({
|
|
547
601
|
table: t,
|
|
548
602
|
reason: String(e?.message ?? e)
|
|
549
603
|
})
|
|
550
|
-
}))),
|
|
604
|
+
}))), qt = (e, t) => {
|
|
551
605
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
552
606
|
for (let e of t.tables) {
|
|
553
607
|
if (!n.has(e.name)) continue;
|
|
@@ -567,39 +621,39 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
567
621
|
};
|
|
568
622
|
for (let t of e) n.has(t) && s(t);
|
|
569
623
|
return i;
|
|
570
|
-
},
|
|
624
|
+
}, Y = (e) => {
|
|
571
625
|
let t = e, n = [];
|
|
572
626
|
typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
|
|
573
627
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
574
|
-
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(
|
|
628
|
+
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(Y(t.cause)) : n.push(r), n.length === 0) {
|
|
575
629
|
let t = String(e ?? "").trim();
|
|
576
|
-
return t === "" || t === "[object Object]" ?
|
|
630
|
+
return t === "" || t === "[object Object]" ? Jt(e) : t;
|
|
577
631
|
}
|
|
578
632
|
return n.join(": ");
|
|
579
|
-
},
|
|
633
|
+
}, Jt = (e) => {
|
|
580
634
|
try {
|
|
581
635
|
let t = JSON.stringify(e);
|
|
582
636
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
583
637
|
} catch {
|
|
584
638
|
return "unknown error (not serialisable)";
|
|
585
639
|
}
|
|
586
|
-
},
|
|
640
|
+
}, Yt = (e, t) => {
|
|
587
641
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
588
642
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
589
643
|
return r;
|
|
590
|
-
},
|
|
644
|
+
}, Xt = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), Zt = (e, t) => {
|
|
591
645
|
if (t.length === 0) return e;
|
|
592
646
|
let n = { ...e };
|
|
593
647
|
for (let e of t) delete n[e];
|
|
594
648
|
return n;
|
|
595
|
-
},
|
|
649
|
+
}, Qt = (e, t) => {
|
|
596
650
|
let n = e.tables.find((e) => e.name === t);
|
|
597
651
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
598
652
|
let r = n.columns.find((e) => e.type === "id");
|
|
599
653
|
if (r !== void 0) return { column: r.name };
|
|
600
654
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
601
655
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
602
|
-
},
|
|
656
|
+
}, $t = (e, t, r) => n.tryPromise({
|
|
603
657
|
try: async () => {
|
|
604
658
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
605
659
|
d.update(e);
|
|
@@ -608,7 +662,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
608
662
|
n(null, e);
|
|
609
663
|
} }), m = c.fromWeb(l.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
610
664
|
try {
|
|
611
|
-
await s(m, p,
|
|
665
|
+
await s(m, p, ut(r), n(h)), await i.rename(h, t);
|
|
612
666
|
} catch (e) {
|
|
613
667
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
614
668
|
}
|
|
@@ -617,27 +671,27 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
617
671
|
checksum: d.digest("hex")
|
|
618
672
|
};
|
|
619
673
|
},
|
|
620
|
-
catch: (e) => new
|
|
621
|
-
reason: `write table failed: ${
|
|
674
|
+
catch: (e) => new H({
|
|
675
|
+
reason: `write table failed: ${Y(e)}`,
|
|
622
676
|
path: t
|
|
623
677
|
})
|
|
624
|
-
}),
|
|
678
|
+
}), en = (e, t) => n.tryPromise({
|
|
625
679
|
try: async () => {
|
|
626
680
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
627
681
|
},
|
|
628
|
-
catch: (t) => new
|
|
629
|
-
reason: `append failed: ${
|
|
682
|
+
catch: (t) => new H({
|
|
683
|
+
reason: `append failed: ${Y(t)}`,
|
|
630
684
|
path: e
|
|
631
685
|
})
|
|
632
|
-
}),
|
|
686
|
+
}), tn = (e, t) => n.gen(function* () {
|
|
633
687
|
let r = [], i = t.ordered.length;
|
|
634
688
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
635
|
-
let o = t.ordered[a], s =
|
|
636
|
-
if ("refusal" in c) return yield* n.fail(new
|
|
637
|
-
let u = c.column, d = `${
|
|
638
|
-
if (
|
|
689
|
+
let o = t.ordered[a], s = Yt(t.snapshot, o), c = Qt(t.snapshot, o);
|
|
690
|
+
if ("refusal" in c) return yield* n.fail(new U({ reason: `export: ${c.refusal}` }));
|
|
691
|
+
let u = c.column, d = `${bt}/${o}.ndjson${lt(t.compression)}`;
|
|
692
|
+
if (Mt(t.ledger, o)) {
|
|
639
693
|
let e = t.ledger.tables[o];
|
|
640
|
-
|
|
694
|
+
q(t.onProgress, {
|
|
641
695
|
phase: "export",
|
|
642
696
|
table: o,
|
|
643
697
|
index: a,
|
|
@@ -655,7 +709,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
655
709
|
});
|
|
656
710
|
continue;
|
|
657
711
|
}
|
|
658
|
-
|
|
712
|
+
q(t.onProgress, {
|
|
659
713
|
phase: "export",
|
|
660
714
|
table: o,
|
|
661
715
|
index: a,
|
|
@@ -663,17 +717,17 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
663
717
|
event: "start",
|
|
664
718
|
rowsDone: 0
|
|
665
719
|
});
|
|
666
|
-
let f = t.resolved.predicateFor(o), p =
|
|
720
|
+
let f = t.resolved.predicateFor(o), p = w(e, {
|
|
667
721
|
table: o,
|
|
668
722
|
pk: u,
|
|
669
723
|
chunkSize: t.chunkSize,
|
|
670
724
|
retry: t.retry,
|
|
671
725
|
...f === void 0 ? {} : { where: f }
|
|
672
|
-
}), m =
|
|
726
|
+
}), m = Xt(t.snapshot, o), h = m.length === 0 ? p : p.pipe(l.map((e) => Zt(e, m))), g = t.mask ? h.pipe(l.map((e) => ze(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* kt("export", o, a, i, (e) => e.rowCount, $t(Gt(g, s), t.bundleFile(d), t.compression));
|
|
673
727
|
t.ledger.tables[o] = {
|
|
674
728
|
rowCount: _,
|
|
675
729
|
checksum: v
|
|
676
|
-
}, yield*
|
|
730
|
+
}, yield* J(t.ledgerPath, t.ledger), q(t.onProgress, {
|
|
677
731
|
phase: "export",
|
|
678
732
|
table: o,
|
|
679
733
|
index: a,
|
|
@@ -691,43 +745,43 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
691
745
|
});
|
|
692
746
|
}
|
|
693
747
|
return r;
|
|
694
|
-
}),
|
|
748
|
+
}), nn = (e) => ({
|
|
695
749
|
_tag: "RawSqlFragment",
|
|
696
750
|
strings: [e],
|
|
697
751
|
values: []
|
|
698
|
-
}),
|
|
752
|
+
}), rn = (t, i, a) => n.tryPromise({
|
|
699
753
|
try: () => t.transactional(async (t) => {
|
|
700
|
-
if (t.raw) for (let e of
|
|
754
|
+
if (t.raw) for (let e of xe(i)) await t.raw(nn(e));
|
|
701
755
|
let o = await n.runPromiseExit(a(t));
|
|
702
756
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
703
757
|
return o.value;
|
|
704
758
|
}),
|
|
705
|
-
catch: (e) => e instanceof
|
|
706
|
-
}),
|
|
707
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
759
|
+
catch: (e) => e instanceof H ? e : new H({ reason: `snapshot export failed: ${Y(e)}` })
|
|
760
|
+
}), an = (e) => n.gen(function* () {
|
|
761
|
+
let t = e.scope ?? { kind: "all" }, r = ct(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, o = s.exponential("50 millis").pipe(s.jittered, s.intersect(s.recurs(a))), c = yield* n.promise(() => import("node:path")), u = (t) => c.join(e.outDir, t);
|
|
708
762
|
yield* n.tryPromise({
|
|
709
763
|
try: async () => {
|
|
710
764
|
let t = await import("node:fs/promises");
|
|
711
|
-
await t.mkdir(u(
|
|
765
|
+
await t.mkdir(u(bt), { recursive: !0 }), e.assets && await t.mkdir(u(xt), { recursive: !0 });
|
|
712
766
|
},
|
|
713
|
-
catch: (t) => new
|
|
714
|
-
reason: `mkdir failed: ${
|
|
767
|
+
catch: (t) => new H({
|
|
768
|
+
reason: `mkdir failed: ${Y(t)}`,
|
|
715
769
|
path: e.outDir
|
|
716
770
|
})
|
|
717
771
|
});
|
|
718
|
-
let d = u(
|
|
772
|
+
let d = u(Ct), f = yield* jt(d), p = e.subset ? et(yield* $e(e.store, e.snapshot, e.subset), e.snapshot) : yield* nt(t, e.snapshot, {
|
|
719
773
|
store: e.store,
|
|
720
774
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
721
|
-
}), m =
|
|
775
|
+
}), m = qt(p.tables, e.snapshot), h = e.consistency ?? "live", g, _;
|
|
722
776
|
if (e.masking) {
|
|
723
|
-
let t = e.classification ??
|
|
777
|
+
let t = e.classification ?? Ce(e.snapshot), r = Ie(m.map((t) => ({
|
|
724
778
|
name: t,
|
|
725
779
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
726
780
|
name: e.name,
|
|
727
781
|
type: e.type
|
|
728
782
|
}))
|
|
729
783
|
})), t, e.masking);
|
|
730
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
784
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new Ye({
|
|
731
785
|
unclassified: r.unclassified,
|
|
732
786
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
733
787
|
}));
|
|
@@ -753,20 +807,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
753
807
|
bundleFile: u,
|
|
754
808
|
..._ ? { mask: _ } : {},
|
|
755
809
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
756
|
-
}, y = h === "snapshot" ? yield*
|
|
810
|
+
}, y = h === "snapshot" ? yield* rn(e.store, e.dialect, (e) => tn(e, v)) : yield* tn(e.store, v), b = 0, x = 0;
|
|
757
811
|
if (e.assets) {
|
|
758
|
-
let t = u(
|
|
759
|
-
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
812
|
+
let t = u(St);
|
|
813
|
+
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => at(e, u(xt))), n.flatMap((e) => {
|
|
760
814
|
let i = {
|
|
761
815
|
key: r.key,
|
|
762
816
|
sha256: e.sha256,
|
|
763
817
|
size: e.size,
|
|
764
818
|
contentType: e.contentType
|
|
765
819
|
};
|
|
766
|
-
return b++,
|
|
767
|
-
}))).pipe(n.mapError((e) => e instanceof
|
|
820
|
+
return b++, x += e.size, f.assets[r.key] = !0, en(t, JSON.stringify(i)).pipe(n.andThen(J(d, f)));
|
|
821
|
+
}))).pipe(n.mapError((e) => e instanceof H ? e : new H({ reason: `asset export failed: ${String(e)}` })));
|
|
768
822
|
}
|
|
769
|
-
let
|
|
823
|
+
let S = {
|
|
770
824
|
formatVersion: 1,
|
|
771
825
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
772
826
|
source: {
|
|
@@ -779,8 +833,8 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
779
833
|
tables: y,
|
|
780
834
|
assets: b > 0 ? {
|
|
781
835
|
count: b,
|
|
782
|
-
totalBytes:
|
|
783
|
-
index:
|
|
836
|
+
totalBytes: x,
|
|
837
|
+
index: St
|
|
784
838
|
} : {
|
|
785
839
|
count: 0,
|
|
786
840
|
totalBytes: 0
|
|
@@ -789,23 +843,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
789
843
|
};
|
|
790
844
|
return yield* n.tryPromise({
|
|
791
845
|
try: async () => {
|
|
792
|
-
await (await import("node:fs/promises")).writeFile(u(
|
|
846
|
+
await (await import("node:fs/promises")).writeFile(u(K), _t(S));
|
|
793
847
|
},
|
|
794
|
-
catch: (e) => new
|
|
795
|
-
reason: `manifest write failed: ${
|
|
796
|
-
path:
|
|
848
|
+
catch: (e) => new H({
|
|
849
|
+
reason: `manifest write failed: ${Y(e)}`,
|
|
850
|
+
path: K
|
|
797
851
|
})
|
|
798
852
|
}), yield* n.tryPromise({
|
|
799
853
|
try: async () => {
|
|
800
854
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
801
|
-
await e.writeFile(u(
|
|
855
|
+
await e.writeFile(u(yt), Tt(S, { hostname: t.hostname() }));
|
|
802
856
|
},
|
|
803
|
-
catch: (e) => new
|
|
804
|
-
reason: `readme write failed: ${
|
|
805
|
-
path:
|
|
857
|
+
catch: (e) => new H({
|
|
858
|
+
reason: `readme write failed: ${Y(e)}`,
|
|
859
|
+
path: yt
|
|
806
860
|
})
|
|
807
|
-
}),
|
|
808
|
-
}),
|
|
861
|
+
}), S;
|
|
862
|
+
}), on = (e, t) => {
|
|
809
863
|
let n = new Set(t), r = [];
|
|
810
864
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
811
865
|
let i = e.references?.table;
|
|
@@ -816,7 +870,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
816
870
|
});
|
|
817
871
|
}
|
|
818
872
|
return r;
|
|
819
|
-
},
|
|
873
|
+
}, sn = (e) => {
|
|
820
874
|
let t = /* @__PURE__ */ new Map();
|
|
821
875
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
822
876
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -824,7 +878,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
824
878
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
825
879
|
});
|
|
826
880
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
827
|
-
},
|
|
881
|
+
}, cn = (e) => {
|
|
828
882
|
switch (e) {
|
|
829
883
|
case "id":
|
|
830
884
|
case "text":
|
|
@@ -843,12 +897,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
843
897
|
case "raw": return "raw";
|
|
844
898
|
default: return "string";
|
|
845
899
|
}
|
|
846
|
-
},
|
|
900
|
+
}, ln = {
|
|
847
901
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
848
902
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
849
903
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
850
|
-
},
|
|
851
|
-
let i =
|
|
904
|
+
}, un = (e, t, n, r) => {
|
|
905
|
+
let i = cn(n);
|
|
852
906
|
if (i === "raw") return {
|
|
853
907
|
table: e,
|
|
854
908
|
column: t,
|
|
@@ -856,7 +910,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
856
910
|
severity: "block",
|
|
857
911
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
858
912
|
};
|
|
859
|
-
let a =
|
|
913
|
+
let a = ln[i];
|
|
860
914
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
861
915
|
table: e,
|
|
862
916
|
column: t,
|
|
@@ -870,15 +924,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
870
924
|
severity: "block",
|
|
871
925
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
872
926
|
};
|
|
873
|
-
},
|
|
927
|
+
}, dn = (e, t, n) => {
|
|
874
928
|
if (t === n) return [];
|
|
875
929
|
let r = [];
|
|
876
930
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
877
|
-
let a =
|
|
931
|
+
let a = un(t.name, e, i, n);
|
|
878
932
|
a && r.push(a);
|
|
879
933
|
}
|
|
880
934
|
return r;
|
|
881
|
-
},
|
|
935
|
+
}, fn = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, pn = (e) => {
|
|
882
936
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
883
937
|
switch (e.kind) {
|
|
884
938
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -887,36 +941,36 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
887
941
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
888
942
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
889
943
|
}
|
|
890
|
-
},
|
|
944
|
+
}, mn = (e) => {
|
|
891
945
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
892
946
|
return t.length > 0 ? t.join("/") : void 0;
|
|
893
|
-
},
|
|
947
|
+
}, hn = "Failed to execute statement", gn = (e) => {
|
|
894
948
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
895
949
|
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
896
|
-
return t === `SqlError: ${
|
|
897
|
-
},
|
|
950
|
+
return t === `SqlError: ${hn}` ? hn : t;
|
|
951
|
+
}, _n = (e) => {
|
|
898
952
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
899
953
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
900
954
|
unclassified: t,
|
|
901
955
|
raw: n
|
|
902
|
-
}), i =
|
|
903
|
-
if (a !== void 0) return r(`${
|
|
904
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u =
|
|
905
|
-
return r(d ===
|
|
906
|
-
},
|
|
907
|
-
let t =
|
|
956
|
+
}), i = _(e), a = p(i), o = mn(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
957
|
+
if (a !== void 0) return r(`${pn(a)}${s}`, !1);
|
|
958
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = gn(typeof l == "string" && l.length > 0 ? l : String(e)), d = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, f = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, m = f !== void 0 && /[←‖]/.test(f) ? ` (via ${f})` : "";
|
|
959
|
+
return r(d === hn || d.length === 0 ? `the database refused the write and the driver gave no detail${s}${m}` : `${d}${s}`, d === hn || d.length === 0);
|
|
960
|
+
}, vn = (e) => _n(e).reason, yn = (e) => {
|
|
961
|
+
let t = _(e), n = p(t);
|
|
908
962
|
if (n?.kind === "foreignKeyInUse") {
|
|
909
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
963
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = mn(t);
|
|
910
964
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
911
965
|
}
|
|
912
|
-
return
|
|
913
|
-
},
|
|
966
|
+
return vn(e);
|
|
967
|
+
}, bn = "_voltro_staging_", X = (e) => `${bn}${e}`, xn = (e) => typeof e.run == "function", Sn = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
914
968
|
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
915
|
-
written before the first delete is what covers that case.`,
|
|
916
|
-
if (!
|
|
969
|
+
written before the first delete is what covers that case.`, Cn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, Z = (e, t) => {
|
|
970
|
+
if (!Cn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
917
971
|
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
918
|
-
},
|
|
919
|
-
let n =
|
|
972
|
+
}, wn = (e, t) => {
|
|
973
|
+
let n = Z(e, t), r = Z(X(e), t);
|
|
920
974
|
switch (t) {
|
|
921
975
|
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
922
976
|
case "mysql":
|
|
@@ -924,20 +978,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
924
978
|
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
925
979
|
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
926
980
|
}
|
|
927
|
-
},
|
|
928
|
-
let n = X(
|
|
929
|
-
return t === "mssql" ? `IF OBJECT_ID('${
|
|
930
|
-
},
|
|
981
|
+
}, Tn = (e, t) => {
|
|
982
|
+
let n = Z(X(e), t);
|
|
983
|
+
return t === "mssql" ? `IF OBJECT_ID('${X(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
984
|
+
}, En = (e, t) => {
|
|
931
985
|
let n = t === "mysql" || t === "mariadb", r = [];
|
|
932
986
|
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
933
|
-
for (let n of [...e].reverse()) r.push(`DELETE FROM ${
|
|
987
|
+
for (let n of [...e].reverse()) r.push(`DELETE FROM ${Z(n.name, t)}`);
|
|
934
988
|
for (let n of e) {
|
|
935
989
|
if (n.columns.length === 0) throw Error(`refusing to build a swap for '${n.name}' with no columns: an empty list would have to mean \`SELECT *\`, which is the shape that fails on a generated column.`);
|
|
936
|
-
let e = n.columns.map((e) =>
|
|
937
|
-
r.push(`INSERT INTO ${
|
|
990
|
+
let e = n.columns.map((e) => Z(e, t)).join(", ");
|
|
991
|
+
r.push(`INSERT INTO ${Z(n.name, t)} (${e}) SELECT ${e} FROM ${Z(X(n.name), t)}`);
|
|
938
992
|
}
|
|
939
993
|
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
940
|
-
},
|
|
994
|
+
}, Dn = (e, t) => {
|
|
941
995
|
let n = [], r = () => {
|
|
942
996
|
for (let e of n) t.unregisterTable(e);
|
|
943
997
|
};
|
|
@@ -945,7 +999,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
945
999
|
for (let r of e) {
|
|
946
1000
|
let e = t.getTable(r);
|
|
947
1001
|
if (e === void 0) continue;
|
|
948
|
-
let i =
|
|
1002
|
+
let i = X(r);
|
|
949
1003
|
t.registerTable({
|
|
950
1004
|
...e,
|
|
951
1005
|
tableName: i,
|
|
@@ -959,7 +1013,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
959
1013
|
staged: n,
|
|
960
1014
|
undo: r
|
|
961
1015
|
};
|
|
962
|
-
},
|
|
1016
|
+
}, On = (e, t) => {
|
|
963
1017
|
let n = new Set(t), r = [];
|
|
964
1018
|
for (let t of e.tables) {
|
|
965
1019
|
if (!n.has(t.name)) continue;
|
|
@@ -976,22 +1030,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
976
1030
|
}
|
|
977
1031
|
}
|
|
978
1032
|
return r;
|
|
979
|
-
},
|
|
980
|
-
let n = X(
|
|
981
|
-
return `SELECT ${
|
|
982
|
-
},
|
|
1033
|
+
}, kn = (e, t) => {
|
|
1034
|
+
let n = Z(X(e.from), t), r = Z(X(e.to), t), i = Z(e.column, t), a = Z(e.toColumn, t);
|
|
1035
|
+
return `SELECT ${Z(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
1036
|
+
}, An = (e) => {
|
|
983
1037
|
if (e.length === 0) return;
|
|
984
1038
|
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
985
1039
|
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
986
|
-
},
|
|
1040
|
+
}, jn = async (e, t) => {
|
|
987
1041
|
t.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
988
|
-
},
|
|
989
|
-
let i =
|
|
1042
|
+
}, Mn = async (e, t, r) => {
|
|
1043
|
+
let i = En(t, r);
|
|
990
1044
|
i.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
991
|
-
},
|
|
1045
|
+
}, Nn = async (e, t, r) => {
|
|
992
1046
|
let i = [];
|
|
993
1047
|
for (let a of t) {
|
|
994
|
-
let t = await e.run(P.SqlClient.pipe(n.flatMap((e) => e.unsafe(
|
|
1048
|
+
let t = await e.run(P.SqlClient.pipe(n.flatMap((e) => e.unsafe(kn(a, r)))));
|
|
995
1049
|
for (let e of t) i.push({
|
|
996
1050
|
table: a.from,
|
|
997
1051
|
primaryKey: String(e.pk ?? ""),
|
|
@@ -1001,16 +1055,75 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1001
1055
|
});
|
|
1002
1056
|
}
|
|
1003
1057
|
return i;
|
|
1004
|
-
},
|
|
1058
|
+
}, Pn = (e, t) => {
|
|
1005
1059
|
let n = new Map(t.map((e, t) => [e, t]));
|
|
1006
1060
|
return e.filter((e) => {
|
|
1007
1061
|
if (e.from === e.to) return !1;
|
|
1008
1062
|
let t = n.get(e.from), r = n.get(e.to);
|
|
1009
1063
|
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1010
1064
|
});
|
|
1011
|
-
},
|
|
1065
|
+
}, Fn = (e) => e === "mysql" || e === "mariadb", In = (e) => {
|
|
1066
|
+
if (e.mode !== "replace") return {
|
|
1067
|
+
active: !1,
|
|
1068
|
+
tables: [],
|
|
1069
|
+
blocker: `staging applies to \`replace\` only (this run is \`${e.mode}\`).`
|
|
1070
|
+
};
|
|
1071
|
+
let t = (() => {
|
|
1072
|
+
let t = e.targetDialect;
|
|
1073
|
+
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1074
|
+
})();
|
|
1075
|
+
if (t === void 0) return {
|
|
1076
|
+
active: !1,
|
|
1077
|
+
tables: [],
|
|
1078
|
+
blocker: `the target dialect (${e.targetDialect ?? "unknown"}) is not one this build can stage on.`
|
|
1079
|
+
};
|
|
1080
|
+
if (!e.canRunSql) return {
|
|
1081
|
+
active: !1,
|
|
1082
|
+
tables: [],
|
|
1083
|
+
blocker: "this store cannot send DDL (no raw-SQL channel), which a staged swap needs."
|
|
1084
|
+
};
|
|
1085
|
+
let n = e.targetSnapshot;
|
|
1086
|
+
if (n === void 0) return {
|
|
1087
|
+
active: !1,
|
|
1088
|
+
tables: [],
|
|
1089
|
+
blocker: "this caller did not supply a snapshot of the target schema, so the staging tables cannot be given the target's column shape."
|
|
1090
|
+
};
|
|
1091
|
+
let r = new Map(n.tables.map((e) => [e.name, e])), i = e.bundleTables.filter((t) => !e.isSkipped(t)), a = i.filter((e) => !r.has(e));
|
|
1092
|
+
if (a.length > 0) return {
|
|
1093
|
+
active: !1,
|
|
1094
|
+
tables: [],
|
|
1095
|
+
blocker: `the bundle carries ${a.length} table(s) the target does not have (${a.join(", ")}). Staging copies the TARGET's column shape, and there is none for those.`
|
|
1096
|
+
};
|
|
1097
|
+
let o = [];
|
|
1098
|
+
for (let e of i) {
|
|
1099
|
+
let t = r.get(e).columns.filter((e) => e.generatedAs === void 0).map((e) => e.name);
|
|
1100
|
+
if (t.length === 0) return {
|
|
1101
|
+
active: !1,
|
|
1102
|
+
tables: [],
|
|
1103
|
+
blocker: `\`${e}\` has no ordinary columns in the target (every column is generated), so a staged copy would have nothing to write.`
|
|
1104
|
+
};
|
|
1105
|
+
o.push({
|
|
1106
|
+
name: e,
|
|
1107
|
+
columns: t
|
|
1108
|
+
});
|
|
1109
|
+
}
|
|
1110
|
+
if (o.length === 0) return {
|
|
1111
|
+
active: !1,
|
|
1112
|
+
tables: [],
|
|
1113
|
+
blocker: "every table in the bundle is one that describes a deployment, so there is nothing for a staged swap to move."
|
|
1114
|
+
};
|
|
1115
|
+
let s = o.map((e) => e.name), c = Fn(t) ? [] : Pn(On(n, s), s);
|
|
1116
|
+
return c.length > 0 ? {
|
|
1117
|
+
active: !1,
|
|
1118
|
+
tables: [],
|
|
1119
|
+
blocker: `the bundle's tables reference each other in a cycle (${c.map((e) => `${e.from}.${e.column} → ${e.to}`).join(", ")}), and a bulk copy cannot satisfy both directions on this dialect.`
|
|
1120
|
+
} : {
|
|
1121
|
+
active: !0,
|
|
1122
|
+
tables: o
|
|
1123
|
+
};
|
|
1124
|
+
}, Ln = (e) => n.gen(function* () {
|
|
1012
1125
|
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1013
|
-
if (e.snapshot === void 0) return yield* n.fail(new
|
|
1126
|
+
if (e.snapshot === void 0) return yield* n.fail(new H({
|
|
1014
1127
|
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
1015
1128
|
path: e.dir
|
|
1016
1129
|
}));
|
|
@@ -1021,14 +1134,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1021
1134
|
if (t.tables.length === 0) return;
|
|
1022
1135
|
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1023
1136
|
so it survives a run whose process does not.`);
|
|
1024
|
-
let r = yield*
|
|
1137
|
+
let r = yield* an({
|
|
1025
1138
|
store: e.store,
|
|
1026
1139
|
snapshot: t,
|
|
1027
1140
|
dialect: e.dialect ?? "unknown",
|
|
1028
1141
|
outDir: e.dir,
|
|
1029
1142
|
scope: { kind: "all" },
|
|
1030
1143
|
compression: "none"
|
|
1031
|
-
}).pipe(n.mapError((t) => new
|
|
1144
|
+
}).pipe(n.mapError((t) => new H({
|
|
1032
1145
|
reason: `rollback capture failed, so the import did not start and the target is untouched: ${t.reason ?? String(t)}\n Fix the capture, or re-run with \`--no-rollback\` to proceed without one.`,
|
|
1033
1146
|
path: e.dir
|
|
1034
1147
|
})));
|
|
@@ -1037,14 +1150,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1037
1150
|
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1038
1151
|
tables: r.tables.length
|
|
1039
1152
|
};
|
|
1040
|
-
}),
|
|
1153
|
+
}), Rn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, zn = (e, t) => e.tables.find((e) => e.name === t), Bn = (e, t) => new Map((zn(e, t)?.columns ?? []).map((e) => [e.name, e])), Vn = (e, t) => {
|
|
1041
1154
|
let n = [];
|
|
1042
1155
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
1043
|
-
if (
|
|
1156
|
+
if (zn(t, r.name) === void 0) {
|
|
1044
1157
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
1045
1158
|
continue;
|
|
1046
1159
|
}
|
|
1047
|
-
let e =
|
|
1160
|
+
let e = Bn(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
1048
1161
|
for (let [t, a] of i) {
|
|
1049
1162
|
let i = e.get(t);
|
|
1050
1163
|
if (i === void 0) {
|
|
@@ -1056,7 +1169,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1056
1169
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
1057
1170
|
}
|
|
1058
1171
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
1059
|
-
},
|
|
1172
|
+
}, Hn = (e, t, n) => {
|
|
1060
1173
|
let r = re(t);
|
|
1061
1174
|
if (r === e || n === void 0) return {
|
|
1062
1175
|
drifted: !1,
|
|
@@ -1064,14 +1177,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1064
1177
|
targetFingerprint: r,
|
|
1065
1178
|
diff: []
|
|
1066
1179
|
};
|
|
1067
|
-
let i =
|
|
1180
|
+
let i = Vn(n, t);
|
|
1068
1181
|
return {
|
|
1069
1182
|
drifted: i.length > 0,
|
|
1070
1183
|
bundleFingerprint: e,
|
|
1071
1184
|
targetFingerprint: r,
|
|
1072
1185
|
diff: i
|
|
1073
1186
|
};
|
|
1074
|
-
},
|
|
1187
|
+
}, Un = 2e3, Wn = 200, Gn = /* @__PURE__ */ new Set([
|
|
1075
1188
|
"BundleError",
|
|
1076
1189
|
"CodecError",
|
|
1077
1190
|
"IntegrityError",
|
|
@@ -1080,33 +1193,33 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1080
1193
|
"RowsRefusedError",
|
|
1081
1194
|
"SchemaDriftError",
|
|
1082
1195
|
"StorageError"
|
|
1083
|
-
]),
|
|
1196
|
+
]), Kn = (t) => {
|
|
1084
1197
|
let n = o.isFiberFailure(t) ? [...e.failures(t[o.FiberFailureCauseId]), ...e.defects(t[o.FiberFailureCauseId])] : [t];
|
|
1085
1198
|
for (let e of n) {
|
|
1086
1199
|
let t = e?._tag;
|
|
1087
|
-
if (typeof t == "string" &&
|
|
1200
|
+
if (typeof t == "string" && Gn.has(t)) return e;
|
|
1088
1201
|
if (o.isFiberFailure(e)) {
|
|
1089
|
-
let t =
|
|
1202
|
+
let t = Kn(e);
|
|
1090
1203
|
if (t) return t;
|
|
1091
1204
|
}
|
|
1092
1205
|
}
|
|
1093
|
-
},
|
|
1206
|
+
}, qn = (t) => [...e.failures(t)][0] ?? Error(e.pretty(t)), Jn = (e) => n.tryPromise({
|
|
1094
1207
|
try: async () => {
|
|
1095
1208
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
1096
|
-
return
|
|
1209
|
+
return vt(await t.readFile(n.join(e, K), "utf8"));
|
|
1097
1210
|
},
|
|
1098
|
-
catch: (e) => new
|
|
1211
|
+
catch: (e) => new H({
|
|
1099
1212
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
1100
|
-
path:
|
|
1213
|
+
path: K
|
|
1101
1214
|
})
|
|
1102
|
-
}),
|
|
1215
|
+
}), Yn = (e) => {
|
|
1103
1216
|
let t;
|
|
1104
1217
|
return n.gen(function* () {
|
|
1105
|
-
let i = yield* n.promise(() => import("node:path")), a = yield*
|
|
1106
|
-
if (a.formatVersion !== 1) return yield* n.fail(new
|
|
1218
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* Jn(e.bundleDir);
|
|
1219
|
+
if (a.formatVersion !== 1) return yield* n.fail(new H({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
1107
1220
|
if (e.tables !== void 0) {
|
|
1108
1221
|
let t = new Set(a.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
1109
|
-
if (r.length > 0) return yield* n.fail(new
|
|
1222
|
+
if (r.length > 0) return yield* n.fail(new H({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...t].sort().join(", ")}.` }));
|
|
1110
1223
|
}
|
|
1111
1224
|
let o = e.tables === void 0 ? void 0 : new Set(e.tables), s = o === void 0 ? a : {
|
|
1112
1225
|
...a,
|
|
@@ -1123,9 +1236,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1123
1236
|
unique: !1,
|
|
1124
1237
|
hasDefault: !1
|
|
1125
1238
|
}))
|
|
1126
|
-
})) }, r =
|
|
1239
|
+
})) }, r = Hn(s.source.schemaFingerprint, e.targetSnapshot, t);
|
|
1127
1240
|
if (r.drifted) {
|
|
1128
|
-
if (!e.force) return yield* n.fail(new
|
|
1241
|
+
if (!e.force) return yield* n.fail(new Je({
|
|
1129
1242
|
bundleFingerprint: r.bundleFingerprint,
|
|
1130
1243
|
targetFingerprint: r.targetFingerprint,
|
|
1131
1244
|
diff: r.diff
|
|
@@ -1134,23 +1247,28 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1134
1247
|
}
|
|
1135
1248
|
}
|
|
1136
1249
|
if (e.targetDialect && e.targetDialect !== s.source.dialect) {
|
|
1137
|
-
let t =
|
|
1138
|
-
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new
|
|
1250
|
+
let t = dn(s.tables, s.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1251
|
+
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new Ze({
|
|
1139
1252
|
sourceDialect: s.source.dialect,
|
|
1140
1253
|
targetDialect: e.targetDialect,
|
|
1141
|
-
blocking: t.map(
|
|
1254
|
+
blocking: t.map(fn)
|
|
1142
1255
|
}));
|
|
1143
1256
|
}
|
|
1144
|
-
let c = e.verify ?? !0,
|
|
1145
|
-
if (
|
|
1146
|
-
mode:
|
|
1147
|
-
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope.`
|
|
1257
|
+
let c = e.verify ?? !0, p = e.mode ?? "upsert", g = e.atomic ?? p === "replace", _ = e.onConflict ?? "skip", y = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), x = yield* jt(y);
|
|
1258
|
+
if (p === "replace" && s.scope.kind !== "all") return yield* n.fail(new Xe({
|
|
1259
|
+
mode: p,
|
|
1260
|
+
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope. To leave specific tables out of a FULL export, use \`--exclude a,b\`: it keeps the scope \`all\` and records what it left out.`
|
|
1148
1261
|
}));
|
|
1149
|
-
if (
|
|
1262
|
+
if (p === "replace" && (s.scope.exclude?.length ?? 0) > 0) {
|
|
1263
|
+
let t = s.scope.exclude;
|
|
1264
|
+
e.onWarn?.(`this bundle was exported with ${t.length} table(s) excluded: ${t.join(", ")}.\n A replace empties and refills the tables the bundle CARRIES, so those keep whatever rows the
|
|
1265
|
+
target already holds. That is not "everything the bundle says" — check it is what you meant.`);
|
|
1266
|
+
}
|
|
1267
|
+
if (p === "replace") {
|
|
1150
1268
|
let t = s.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1151
1269
|
try: () => e.store.incomingForeignKeys(t),
|
|
1152
|
-
catch: (e) => new
|
|
1153
|
-
}) : e.targetSnapshot === void 0 ? [] :
|
|
1270
|
+
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${yn(e)}` })
|
|
1271
|
+
}) : e.targetSnapshot === void 0 ? [] : on(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1154
1272
|
for (let t of r) {
|
|
1155
1273
|
if (!a.has(t.from)) {
|
|
1156
1274
|
let r = yield* n.tryPromise({
|
|
@@ -1162,7 +1280,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1162
1280
|
skip: void 0,
|
|
1163
1281
|
projection: []
|
|
1164
1282
|
}),
|
|
1165
|
-
catch: () => new
|
|
1283
|
+
catch: () => new H({ reason: `unreadable: ${t.from}` })
|
|
1166
1284
|
}).pipe(n.map((e) => e.length), n.orElseSucceed(() => 0));
|
|
1167
1285
|
a.set(t.from, r);
|
|
1168
1286
|
}
|
|
@@ -1172,9 +1290,9 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1172
1290
|
rowCount: r
|
|
1173
1291
|
});
|
|
1174
1292
|
}
|
|
1175
|
-
if (i.length > 0) return yield* n.fail(new
|
|
1176
|
-
mode:
|
|
1177
|
-
reason:
|
|
1293
|
+
if (i.length > 0) return yield* n.fail(new Xe({
|
|
1294
|
+
mode: p,
|
|
1295
|
+
reason: sn(i)
|
|
1178
1296
|
}));
|
|
1179
1297
|
}
|
|
1180
1298
|
if (e.dryRun === !0) return {
|
|
@@ -1185,32 +1303,32 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1185
1303
|
rowsSkipped: 0,
|
|
1186
1304
|
fullyResumed: !1
|
|
1187
1305
|
};
|
|
1188
|
-
let
|
|
1306
|
+
let w = /* @__PURE__ */ new Map();
|
|
1189
1307
|
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1190
1308
|
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1191
|
-
e.length > 0 &&
|
|
1309
|
+
e.length > 0 && w.set(t.name, e);
|
|
1192
1310
|
}
|
|
1193
1311
|
let re = (e, t) => {
|
|
1194
|
-
let n =
|
|
1312
|
+
let n = w.get(e);
|
|
1195
1313
|
if (n === void 0) return t;
|
|
1196
1314
|
let r;
|
|
1197
1315
|
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1198
1316
|
return r ?? t;
|
|
1199
|
-
},
|
|
1200
|
-
let i = j ?
|
|
1201
|
-
return
|
|
1202
|
-
},
|
|
1203
|
-
r.unclassified && (
|
|
1204
|
-
},
|
|
1317
|
+
}, T = (e, t, n, r = !1) => {
|
|
1318
|
+
let i = j ? X(t.name) : t.name, a = t.primaryKey, o = re(t.name, n);
|
|
1319
|
+
return p === "append" ? _ === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : p === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1320
|
+
}, ie = (e, t) => e.savepoint ? e.savepoint(t) : t(), E = 0, ae = (t, n, r) => {
|
|
1321
|
+
r.unclassified && (E++, !(E > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (E === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1322
|
+
}, oe = (e, t, r = !0) => n.tryPromise({
|
|
1205
1323
|
try: e,
|
|
1206
|
-
catch: (e) =>
|
|
1207
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason,
|
|
1324
|
+
catch: (e) => _n(e)
|
|
1325
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, ae(t.entry, t.row, e)), !1)))), se = (e) => {
|
|
1208
1326
|
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1209
1327
|
if (t.length !== 0) return {
|
|
1210
1328
|
...e.row,
|
|
1211
1329
|
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1212
1330
|
};
|
|
1213
|
-
},
|
|
1331
|
+
}, ce = (e) => {
|
|
1214
1332
|
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
1215
1333
|
if (r !== "reference") return !1;
|
|
1216
1334
|
let i = e.row[n];
|
|
@@ -1228,7 +1346,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1228
1346
|
};
|
|
1229
1347
|
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1230
1348
|
}
|
|
1231
|
-
return new
|
|
1349
|
+
return new qe({
|
|
1232
1350
|
rows: i.slice(0, 20),
|
|
1233
1351
|
primaryCount: r.filter((e) => !e.derived).length,
|
|
1234
1352
|
totalCount: e.length,
|
|
@@ -1238,113 +1356,104 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1238
1356
|
primary: t.primary
|
|
1239
1357
|
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1240
1358
|
});
|
|
1241
|
-
},
|
|
1359
|
+
}, ue = (e, t) => n.gen(function* () {
|
|
1242
1360
|
let r = t, i = [], a = !0;
|
|
1243
1361
|
for (; r.length > 0 && a;) {
|
|
1244
1362
|
a = !1;
|
|
1245
1363
|
let t = [];
|
|
1246
|
-
for (let n of r) (yield*
|
|
1364
|
+
for (let n of r) (yield* oe(() => ie(e, () => T(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1247
1365
|
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1248
|
-
let r = t[n], o =
|
|
1249
|
-
o !== void 0 && (yield*
|
|
1366
|
+
let r = t[n], o = se(r);
|
|
1367
|
+
o !== void 0 && (yield* oe(() => ie(e, () => T(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1250
1368
|
}
|
|
1251
1369
|
r = t;
|
|
1252
1370
|
}
|
|
1253
|
-
for (let t of i) (yield*
|
|
1254
|
-
if (r.length > 0) return yield* n.fail(
|
|
1371
|
+
for (let t of i) (yield* oe(() => ie(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1372
|
+
if (r.length > 0) return yield* n.fail(ce(r));
|
|
1255
1373
|
}), D = (() => {
|
|
1256
1374
|
let t = e.targetDialect;
|
|
1257
1375
|
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1258
|
-
})(), O = (
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
if (i.length === 0) return [];
|
|
1268
|
-
r.push({
|
|
1269
|
-
name: e.name,
|
|
1270
|
-
columns: i
|
|
1271
|
-
});
|
|
1272
|
-
}
|
|
1273
|
-
return r;
|
|
1274
|
-
})(), k = s.tables.filter((e) => !B(e.name) && b(e.name)).map((e) => e.name), A = D === void 0 || Nn(D) ? [] : Mn(e.targetSnapshot === void 0 ? [] : En(e.targetSnapshot, O.map((e) => e.name)), O.map((e) => e.name)), j = f === "replace" && D !== void 0 && O.length > 0 && A.length === 0 && yn(e.store), le = !1, M = 0, N = 0, de = 0, P = 0, F = (t, r) => n.gen(function* () {
|
|
1275
|
-
if (f === "replace" && !j && !(r && y.truncated)) {
|
|
1376
|
+
})(), O = In({
|
|
1377
|
+
mode: p,
|
|
1378
|
+
...e.targetDialect === void 0 ? {} : { targetDialect: e.targetDialect },
|
|
1379
|
+
...e.targetSnapshot === void 0 ? {} : { targetSnapshot: e.targetSnapshot },
|
|
1380
|
+
bundleTables: s.tables.map((e) => e.name),
|
|
1381
|
+
isSkipped: V,
|
|
1382
|
+
canRunSql: xn(e.store)
|
|
1383
|
+
}), k = O.tables, A = s.tables.filter((e) => !V(e.name) && S(e.name)).map((e) => e.name), j = O.active, de = !1, M = 0, N = 0, pe, P = 0, F = 0, me = (t, r) => n.gen(function* () {
|
|
1384
|
+
if (p === "replace" && !j && !(r && x.truncated)) {
|
|
1276
1385
|
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1277
1386
|
try: () => t.insert(d, {
|
|
1278
|
-
id:
|
|
1279
|
-
startedAt:
|
|
1387
|
+
id: m("repl", he),
|
|
1388
|
+
startedAt: ge,
|
|
1280
1389
|
tables: String(s.tables.length),
|
|
1281
1390
|
capture: I?.dir ?? null,
|
|
1282
1391
|
via: e.via ?? "direct"
|
|
1283
1392
|
}),
|
|
1284
|
-
catch: (e) => new
|
|
1393
|
+
catch: (e) => new H({ reason: `could not record that this replace started: ${String(e?.message ?? e)}\n The marker is what makes an interrupted run visible instead of silent, so the run does
|
|
1285
1394
|
not proceed without it.` })
|
|
1286
1395
|
}));
|
|
1287
|
-
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !
|
|
1288
|
-
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where:
|
|
1396
|
+
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !V(e)), a = !r, o = t.emptyTables ? () => t.emptyTables(i) : a ? async () => {
|
|
1397
|
+
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where: b(e.primaryKey) });
|
|
1289
1398
|
} : () => t.transactional(async (e) => {
|
|
1290
|
-
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where:
|
|
1399
|
+
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: b(t.primaryKey) });
|
|
1291
1400
|
});
|
|
1292
1401
|
yield* n.tryPromise({
|
|
1293
1402
|
try: o,
|
|
1294
|
-
catch: (e) => new
|
|
1295
|
-
}), r && (
|
|
1403
|
+
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${yn(e)}` })
|
|
1404
|
+
}), r && (x.truncated = !0, yield* J(y, x));
|
|
1296
1405
|
}
|
|
1297
|
-
let a = [], o = [], u = [],
|
|
1406
|
+
let a = [], o = [], u = [], f = [], h = s.tables.length;
|
|
1298
1407
|
for (let d = 0; d < s.tables.length; d++) {
|
|
1299
|
-
let
|
|
1300
|
-
if (
|
|
1301
|
-
|
|
1408
|
+
let m = s.tables[d];
|
|
1409
|
+
if (V(m.name)) {
|
|
1410
|
+
f.push(m.name);
|
|
1302
1411
|
continue;
|
|
1303
1412
|
}
|
|
1304
|
-
if (r &&
|
|
1305
|
-
u.push(
|
|
1413
|
+
if (r && m.name in x.tables) {
|
|
1414
|
+
u.push(m.name), F++, P += x.tables[m.name].rowCount, q(e.onProgress, {
|
|
1306
1415
|
phase: "import",
|
|
1307
|
-
table:
|
|
1416
|
+
table: m.name,
|
|
1308
1417
|
index: d,
|
|
1309
|
-
tableCount:
|
|
1418
|
+
tableCount: h,
|
|
1310
1419
|
event: "done",
|
|
1311
|
-
rowsDone:
|
|
1312
|
-
total:
|
|
1420
|
+
rowsDone: x.tables[m.name].rowCount,
|
|
1421
|
+
total: m.rowCount
|
|
1313
1422
|
});
|
|
1314
1423
|
continue;
|
|
1315
1424
|
}
|
|
1316
|
-
let
|
|
1317
|
-
|
|
1425
|
+
let _ = i.join(e.bundleDir, m.file);
|
|
1426
|
+
q(e.onProgress, {
|
|
1318
1427
|
phase: "import",
|
|
1319
|
-
table:
|
|
1428
|
+
table: m.name,
|
|
1320
1429
|
index: d,
|
|
1321
|
-
tableCount:
|
|
1430
|
+
tableCount: h,
|
|
1322
1431
|
event: "start",
|
|
1323
1432
|
rowsDone: 0,
|
|
1324
|
-
total:
|
|
1433
|
+
total: m.rowCount
|
|
1325
1434
|
});
|
|
1326
|
-
let
|
|
1327
|
-
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c =
|
|
1435
|
+
let v = a.length, b = n.gen(function* () {
|
|
1436
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c = le("bundle", () => fe(_).pipe(dt(s.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), u = async (e) => {
|
|
1328
1437
|
let n = {
|
|
1329
|
-
entry:
|
|
1438
|
+
entry: m,
|
|
1330
1439
|
row: e,
|
|
1331
1440
|
reason: ""
|
|
1332
1441
|
};
|
|
1333
1442
|
try {
|
|
1334
|
-
await
|
|
1443
|
+
await ie(t, () => T(t, m, e));
|
|
1335
1444
|
} catch (t) {
|
|
1336
|
-
let r =
|
|
1337
|
-
n.reason = r.reason,
|
|
1445
|
+
let r = _n(t);
|
|
1446
|
+
n.reason = r.reason, ae(m, e, r), a.push(n);
|
|
1338
1447
|
}
|
|
1339
1448
|
}, d = yield* n.promise(async () => {
|
|
1340
1449
|
let n = e.copyLoader;
|
|
1341
1450
|
if (!n || n.dialect !== "postgres") return !1;
|
|
1342
|
-
if (
|
|
1343
|
-
if (
|
|
1344
|
-
if (
|
|
1451
|
+
if (g) return de || (de = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1452
|
+
if (p === "replace") return !0;
|
|
1453
|
+
if (p !== "upsert") return !1;
|
|
1345
1454
|
try {
|
|
1346
1455
|
return (await t.query({
|
|
1347
|
-
table:
|
|
1456
|
+
table: m.name,
|
|
1348
1457
|
predicate: void 0,
|
|
1349
1458
|
order: [],
|
|
1350
1459
|
take: 1,
|
|
@@ -1354,200 +1463,211 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1354
1463
|
} catch {
|
|
1355
1464
|
return !1;
|
|
1356
1465
|
}
|
|
1357
|
-
}),
|
|
1358
|
-
if (
|
|
1359
|
-
let t =
|
|
1360
|
-
|
|
1466
|
+
}), f = Object.keys(m.columnTypes), h = Math.max(1, e.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
1467
|
+
if (v.length === 0) return;
|
|
1468
|
+
let t = v;
|
|
1469
|
+
v = [];
|
|
1361
1470
|
try {
|
|
1362
1471
|
await e.copyLoader.copyInto({
|
|
1363
|
-
table:
|
|
1364
|
-
columns:
|
|
1365
|
-
columnTypes:
|
|
1472
|
+
table: m.name,
|
|
1473
|
+
columns: f,
|
|
1474
|
+
columnTypes: m.columnTypes,
|
|
1366
1475
|
rows: t
|
|
1367
1476
|
});
|
|
1368
1477
|
} catch (n) {
|
|
1369
|
-
d = !1, e.onWarn?.(`bulk COPY into ${
|
|
1478
|
+
d = !1, e.onWarn?.(`bulk COPY into ${m.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1370
1479
|
for (let e of t) await u(e);
|
|
1371
1480
|
}
|
|
1372
|
-
}, b = Math.max(1, e.savepointBatchSize ?? 200),
|
|
1373
|
-
if (
|
|
1374
|
-
let e =
|
|
1375
|
-
|
|
1481
|
+
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof t.savepoint == "function", S = [], C = async () => {
|
|
1482
|
+
if (S.length === 0) return;
|
|
1483
|
+
let e = S;
|
|
1484
|
+
S = [];
|
|
1376
1485
|
try {
|
|
1377
1486
|
await t.savepoint(async () => {
|
|
1378
|
-
for (let n of e) await
|
|
1487
|
+
for (let n of e) await T(t, m, n);
|
|
1379
1488
|
});
|
|
1380
1489
|
} catch {
|
|
1381
1490
|
for (let t of e) await u(t);
|
|
1382
1491
|
}
|
|
1383
1492
|
};
|
|
1384
|
-
return yield* l.runForEach(
|
|
1493
|
+
return yield* l.runForEach(Kt(c, m.name, m.columnTypes), (e) => n.promise(async () => {
|
|
1385
1494
|
if (o++, d) {
|
|
1386
|
-
|
|
1495
|
+
v.push(e), v.length >= h && await y();
|
|
1387
1496
|
return;
|
|
1388
1497
|
}
|
|
1389
|
-
if (
|
|
1390
|
-
|
|
1498
|
+
if (x) {
|
|
1499
|
+
S.push(e), S.length >= b && await C();
|
|
1391
1500
|
return;
|
|
1392
1501
|
}
|
|
1393
1502
|
await u(e);
|
|
1394
|
-
})), yield* n.promise(
|
|
1503
|
+
})), yield* n.promise(C), yield* n.promise(y), {
|
|
1395
1504
|
rowCount: o,
|
|
1396
1505
|
checksum: i.digest("hex")
|
|
1397
1506
|
};
|
|
1398
|
-
}), { rowCount:
|
|
1507
|
+
}), { rowCount: S, checksum: C } = yield* kt("import", m.name, d, h, (e) => e.rowCount, b);
|
|
1399
1508
|
if (c) {
|
|
1400
|
-
if (
|
|
1401
|
-
what: `table ${
|
|
1402
|
-
expected:
|
|
1403
|
-
actual:
|
|
1509
|
+
if (C !== m.checksum) return yield* n.fail(new Ge({
|
|
1510
|
+
what: `table ${m.name} checksum`,
|
|
1511
|
+
expected: m.checksum,
|
|
1512
|
+
actual: C
|
|
1404
1513
|
}));
|
|
1405
|
-
if (
|
|
1406
|
-
what: `table ${
|
|
1407
|
-
expected: String(
|
|
1408
|
-
actual: String(
|
|
1514
|
+
if (S !== m.rowCount) return yield* n.fail(new Ge({
|
|
1515
|
+
what: `table ${m.name} rowCount`,
|
|
1516
|
+
expected: String(m.rowCount),
|
|
1517
|
+
actual: String(S)
|
|
1409
1518
|
}));
|
|
1410
1519
|
}
|
|
1411
|
-
|
|
1520
|
+
q(e.onProgress, {
|
|
1412
1521
|
phase: "import",
|
|
1413
|
-
table:
|
|
1522
|
+
table: m.name,
|
|
1414
1523
|
index: d,
|
|
1415
|
-
tableCount:
|
|
1524
|
+
tableCount: h,
|
|
1416
1525
|
event: "done",
|
|
1417
|
-
rowsDone:
|
|
1418
|
-
total:
|
|
1419
|
-
}), N++, M +=
|
|
1420
|
-
name:
|
|
1421
|
-
rowCount:
|
|
1422
|
-
}) : (
|
|
1423
|
-
rowCount:
|
|
1424
|
-
checksum:
|
|
1425
|
-
}, yield*
|
|
1526
|
+
rowsDone: S,
|
|
1527
|
+
total: m.rowCount
|
|
1528
|
+
}), N++, M += S, pe?.(), r && (a.length > v ? o.push({
|
|
1529
|
+
name: m.name,
|
|
1530
|
+
rowCount: S
|
|
1531
|
+
}) : (x.tables[m.name] = {
|
|
1532
|
+
rowCount: S,
|
|
1533
|
+
checksum: m.checksum
|
|
1534
|
+
}, yield* J(y, x)));
|
|
1426
1535
|
}
|
|
1427
|
-
if (a.length > 0 && (yield*
|
|
1536
|
+
if (a.length > 0 && (yield* ue(t, a)), p === "replace" && !(r && x.truncated) && (yield* n.promise(() => t.deleteMany(d, { where: {
|
|
1428
1537
|
column: "startedAt",
|
|
1429
1538
|
op: "lte",
|
|
1430
|
-
value:
|
|
1539
|
+
value: ge
|
|
1431
1540
|
} }).then(() => void 0, () => void 0))), r && o.length > 0) {
|
|
1432
1541
|
for (let e of o) {
|
|
1433
1542
|
let t = s.tables.find((t) => t.name === e.name);
|
|
1434
|
-
|
|
1543
|
+
x.tables[e.name] = {
|
|
1435
1544
|
rowCount: e.rowCount,
|
|
1436
1545
|
checksum: t.checksum
|
|
1437
1546
|
};
|
|
1438
1547
|
}
|
|
1439
|
-
yield*
|
|
1548
|
+
yield* J(y, x);
|
|
1440
1549
|
}
|
|
1441
1550
|
{
|
|
1442
|
-
let t =
|
|
1551
|
+
let t = Ue(f, "skipped");
|
|
1443
1552
|
t !== void 0 && e.onWarn?.(t);
|
|
1444
1553
|
}
|
|
1445
1554
|
if (u.length > 0) {
|
|
1446
1555
|
let t = u.length === s.tables.length;
|
|
1447
|
-
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${
|
|
1556
|
+
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${y} to force a full re-import.`);
|
|
1448
1557
|
}
|
|
1449
|
-
}),
|
|
1558
|
+
}), he = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, ge = /* @__PURE__ */ new Date(), I = yield* Ln({
|
|
1450
1559
|
store: e.store,
|
|
1451
|
-
mode:
|
|
1560
|
+
mode: p,
|
|
1452
1561
|
dir: e.rollbackDir,
|
|
1453
1562
|
snapshot: e.targetSnapshot,
|
|
1454
1563
|
dialect: e.targetDialect,
|
|
1455
|
-
tables: s.tables.map((e) => e.name).filter((e) => !
|
|
1564
|
+
tables: s.tables.map((e) => e.name).filter((e) => !V(e)),
|
|
1456
1565
|
onWarn: e.onWarn
|
|
1457
1566
|
});
|
|
1458
1567
|
I !== void 0 && (e.onRollbackCaptured !== void 0 && (yield* n.tryPromise({
|
|
1459
1568
|
try: () => e.onRollbackCaptured(I),
|
|
1460
|
-
catch: (e) => new
|
|
1569
|
+
catch: (e) => new H({
|
|
1461
1570
|
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1462
1571
|
path: I.dir
|
|
1463
1572
|
})
|
|
1464
|
-
})), e.onWarn?.(
|
|
1465
|
-
let
|
|
1466
|
-
|
|
1467
|
-
id:
|
|
1468
|
-
mode:
|
|
1573
|
+
})), e.onWarn?.(Rn(I)));
|
|
1574
|
+
let _e = (e) => p === "replace" ? ee(e) : e(), L = (e) => p === "replace" ? te(e) : e, R = m("dimp", `${Date.now()}-${s.source.schemaFingerprint ?? ""}`), ve = !1, ye = (e.targetSnapshot?.tables.some((e) => e.name === u) ?? !1) || v(u) !== void 0;
|
|
1575
|
+
ye && h(f), ye && (yield* n.promise(() => e.store.insert(u, {
|
|
1576
|
+
id: R,
|
|
1577
|
+
mode: p,
|
|
1469
1578
|
via: e.via ?? "direct",
|
|
1470
1579
|
bundle: e.bundleDir,
|
|
1471
1580
|
sourceFingerprint: s.source.schemaFingerprint ?? null,
|
|
1472
1581
|
tablesWritten: 0,
|
|
1473
1582
|
rowsWritten: 0
|
|
1474
1583
|
}).then(() => {
|
|
1475
|
-
|
|
1584
|
+
ve = !0;
|
|
1476
1585
|
}, (t) => {
|
|
1477
1586
|
e.onWarn?.(`the import trace could not be written (${String(t?.message ?? t)}).\n The import continues — this is a record of what happened, not a safety check. Run
|
|
1478
1587
|
\`voltro db apply\` so the next one is recorded.`);
|
|
1479
1588
|
})));
|
|
1480
|
-
let
|
|
1589
|
+
let be = 0, xe = !1;
|
|
1590
|
+
pe = () => {
|
|
1591
|
+
if (!ve || xe) return;
|
|
1592
|
+
let t = Date.now();
|
|
1593
|
+
t - be < Un || (be = t, e.store.update(u, R, {
|
|
1594
|
+
tablesWritten: N,
|
|
1595
|
+
rowsWritten: M
|
|
1596
|
+
}).then(() => void 0, () => {
|
|
1597
|
+
xe = !0;
|
|
1598
|
+
}));
|
|
1599
|
+
};
|
|
1600
|
+
let Se = (t) => ve ? e.store.update(u, R, {
|
|
1481
1601
|
finishedAt: t === null ? /* @__PURE__ */ new Date() : null,
|
|
1482
1602
|
tablesWritten: N,
|
|
1483
1603
|
rowsWritten: M,
|
|
1484
1604
|
failure: t
|
|
1485
1605
|
}).then(() => void 0, () => void 0) : Promise.resolve();
|
|
1486
|
-
if (t = (e) =>
|
|
1606
|
+
if (t = (e) => Se(e), p === "replace" && !j && O.blocker !== void 0 && e.onWarn?.(Sn(O.blocker)), p === "replace" && A.length > 0 && e.onWarn?.(`write recorders are SUSPENDED for this replace (${A.length} table(s) have one).\n A replace sets a state rather than changing rows, so a per-row history entry would describe
|
|
1487
1607
|
something that did not happen — and this path already writes through the raw store, without
|
|
1488
1608
|
tenant scoping or \`audit()\` stamping, for the same reason.`), j) {
|
|
1489
|
-
e.onWarn?.(`staging ${
|
|
1490
|
-
let t = e.store, r =
|
|
1491
|
-
getTable: (e) =>
|
|
1609
|
+
e.onWarn?.(`staging ${k.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1610
|
+
let t = e.store, r = k.map((e) => e.name), i = Dn(r, {
|
|
1611
|
+
getTable: (e) => v(e),
|
|
1492
1612
|
registerTable: (e) => {
|
|
1493
|
-
|
|
1613
|
+
C(e);
|
|
1494
1614
|
},
|
|
1495
1615
|
unregisterTable: ne
|
|
1496
1616
|
}), a = !1;
|
|
1497
1617
|
try {
|
|
1498
|
-
let i =
|
|
1618
|
+
let i = g ? r : r.filter((e) => !Mt(x, e));
|
|
1499
1619
|
yield* n.tryPromise({
|
|
1500
1620
|
try: async () => {
|
|
1501
|
-
await
|
|
1621
|
+
await jn(t, i.map((e) => Tn(e, D))), await jn(t, i.map((e) => wn(e, D)));
|
|
1502
1622
|
},
|
|
1503
|
-
catch: (e) => new
|
|
1504
|
-
}), yield*
|
|
1505
|
-
try: () =>
|
|
1623
|
+
catch: (e) => new H({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1624
|
+
}), yield* L(me(e.store, !g)), !g && x.truncated === !0 ? a = !0 : yield* n.tryPromise({
|
|
1625
|
+
try: () => Mn(t, k, D).then(() => {
|
|
1506
1626
|
a = !0;
|
|
1507
1627
|
}),
|
|
1508
|
-
catch: (e) => new
|
|
1628
|
+
catch: (e) => new H({ reason: String(e?.message ?? e) })
|
|
1509
1629
|
}).pipe(n.catchAll((i) => n.gen(function* () {
|
|
1510
|
-
let a =
|
|
1511
|
-
return yield* n.fail(new
|
|
1512
|
-
}))), !
|
|
1630
|
+
let a = An(yield* n.promise(() => Nn(t, On(e.targetSnapshot, r), D).then((e) => e, () => [])));
|
|
1631
|
+
return yield* n.fail(new H({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1632
|
+
}))), !g && a && x.truncated !== !0 && (x.truncated = !0, yield* J(y, x));
|
|
1513
1633
|
} finally {
|
|
1514
|
-
i.undo(), a ||
|
|
1634
|
+
i.undo(), a || g ? yield* n.promise(() => jn(t, r.map((e) => Tn(e, D))).then(() => void 0, () => void 0)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1515
1635
|
}
|
|
1516
|
-
} else
|
|
1636
|
+
} else g ? yield* n.tryPromise({
|
|
1517
1637
|
try: () => e.store.transactional(async (e) => {
|
|
1518
|
-
let t = await
|
|
1519
|
-
if (r.isFailure(t)) throw
|
|
1638
|
+
let t = await _e(() => n.runPromiseExit(me(e, !1)));
|
|
1639
|
+
if (r.isFailure(t)) throw qn(t.cause);
|
|
1520
1640
|
}),
|
|
1521
|
-
catch: (e) =>
|
|
1522
|
-
}) : yield*
|
|
1641
|
+
catch: (e) => Kn(e) ?? new H({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1642
|
+
}) : yield* L(me(e.store, !0));
|
|
1523
1643
|
if (e.assets && s.assets.index && !e.assetsAlreadyRestored) {
|
|
1524
|
-
let t = e.assets, r = i.join(e.bundleDir,
|
|
1525
|
-
yield* l.runForEach(o, (e) => e.key in
|
|
1526
|
-
|
|
1527
|
-
})), n.andThen(
|
|
1644
|
+
let t = e.assets, r = i.join(e.bundleDir, xt), a = i.join(e.bundleDir, s.assets.index ?? "assets/index.ndjson"), o = le("bundle", () => fe(a)).pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.map((e) => JSON.parse(e)));
|
|
1645
|
+
yield* l.runForEach(o, (e) => e.key in x.assets ? n.void : ot(t, r, e).pipe(n.andThen(n.sync(() => {
|
|
1646
|
+
x.assets[e.key] = !0;
|
|
1647
|
+
})), n.andThen(J(y, x))));
|
|
1528
1648
|
}
|
|
1529
|
-
return yield* n.promise(() =>
|
|
1649
|
+
return yield* n.promise(() => Se(null)), {
|
|
1530
1650
|
manifest: s,
|
|
1531
1651
|
tablesWritten: N,
|
|
1532
1652
|
rowsWritten: M,
|
|
1533
|
-
tablesSkipped:
|
|
1534
|
-
rowsSkipped:
|
|
1535
|
-
fullyResumed: N === 0 &&
|
|
1653
|
+
tablesSkipped: F,
|
|
1654
|
+
rowsSkipped: P,
|
|
1655
|
+
fullyResumed: N === 0 && F > 0,
|
|
1536
1656
|
...I === void 0 ? {} : { rollback: I }
|
|
1537
1657
|
};
|
|
1538
1658
|
}).pipe(n.tapError((e) => n.promise(() => t?.(e?.reason ?? e?._tag ?? "the import did not finish") ?? Promise.resolve())));
|
|
1539
|
-
},
|
|
1659
|
+
}, Xn = (e) => e, Zn = (e) => {
|
|
1540
1660
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1541
1661
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1542
1662
|
return {
|
|
1543
1663
|
refuse: !0,
|
|
1544
1664
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1545
1665
|
};
|
|
1546
|
-
},
|
|
1666
|
+
}, Q = Buffer.from("VBUNDLE2"), Qn = 0, $n = 1, er = 2, tr = 32, nr = async (e) => {
|
|
1547
1667
|
let t = [], n = async (r) => {
|
|
1548
1668
|
let i;
|
|
1549
1669
|
try {
|
|
1550
|
-
i = await
|
|
1670
|
+
i = await me(L(e, r), { withFileTypes: !0 });
|
|
1551
1671
|
} catch {
|
|
1552
1672
|
return;
|
|
1553
1673
|
}
|
|
@@ -1557,22 +1677,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1557
1677
|
}
|
|
1558
1678
|
};
|
|
1559
1679
|
return await n(""), t.sort();
|
|
1560
|
-
},
|
|
1680
|
+
}, rr = (e, t) => {
|
|
1561
1681
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1562
|
-
return r.writeUInt8(
|
|
1563
|
-
},
|
|
1682
|
+
return r.writeUInt8($n, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1683
|
+
}, ir = (e, t, n) => {
|
|
1564
1684
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1565
|
-
return a.writeUInt8(
|
|
1685
|
+
return a.writeUInt8(er, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1566
1686
|
};
|
|
1567
|
-
async function*
|
|
1568
|
-
yield new Uint8Array(
|
|
1569
|
-
for (let t of await
|
|
1570
|
-
let n = L(e.dir, t), r = await
|
|
1571
|
-
if (yield new Uint8Array(
|
|
1687
|
+
async function* ar(e) {
|
|
1688
|
+
yield new Uint8Array(Q);
|
|
1689
|
+
for (let t of await nr(e.dir)) {
|
|
1690
|
+
let n = L(e.dir, t), r = await he(n);
|
|
1691
|
+
if (yield new Uint8Array(rr(t, r.size)), r.size > 0) for await (let e of fe(n)) yield new Uint8Array(e);
|
|
1572
1692
|
}
|
|
1573
1693
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1574
|
-
yield new Uint8Array(
|
|
1575
|
-
let n =
|
|
1694
|
+
yield new Uint8Array(ir(t.key, t.contentType, t.size));
|
|
1695
|
+
let n = E("sha256"), r = 0;
|
|
1576
1696
|
for await (let i of e.blobs.open(t.key)) {
|
|
1577
1697
|
let e = Buffer.from(i);
|
|
1578
1698
|
n.update(e), r += e.length, yield new Uint8Array(e);
|
|
@@ -1580,10 +1700,10 @@ async function* tr(e) {
|
|
|
1580
1700
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1581
1701
|
yield new Uint8Array(n.digest());
|
|
1582
1702
|
}
|
|
1583
|
-
yield new Uint8Array(Buffer.from([
|
|
1703
|
+
yield new Uint8Array(Buffer.from([Qn]));
|
|
1584
1704
|
}
|
|
1585
|
-
var
|
|
1586
|
-
let n =
|
|
1705
|
+
var or = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), sr = (e) => new Promise((t) => e.end(t)), cr = async (e, t) => {
|
|
1706
|
+
let n = R(t.dir);
|
|
1587
1707
|
await F(n, { recursive: !0 });
|
|
1588
1708
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1589
1709
|
let { value: e, done: t } = await i.next();
|
|
@@ -1595,28 +1715,28 @@ var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr
|
|
|
1595
1715
|
return r = r.subarray(e), t;
|
|
1596
1716
|
};
|
|
1597
1717
|
try {
|
|
1598
|
-
if (await s(
|
|
1718
|
+
if (await s(Q.length), !c(Q.length).equals(Q)) throw Error("not a bundle archive (bad magic)");
|
|
1599
1719
|
for (;;) {
|
|
1600
1720
|
await s(1);
|
|
1601
1721
|
let e = c(1).readUInt8(0);
|
|
1602
|
-
if (e ===
|
|
1603
|
-
if (e ===
|
|
1722
|
+
if (e === Qn) return;
|
|
1723
|
+
if (e === $n) {
|
|
1604
1724
|
await s(4);
|
|
1605
1725
|
let e = c(4).readUInt32BE(0);
|
|
1606
1726
|
await s(e);
|
|
1607
1727
|
let t = c(e).toString("utf8");
|
|
1608
1728
|
await s(8);
|
|
1609
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1610
|
-
if (a !== n && !a.startsWith(n +
|
|
1611
|
-
await F(
|
|
1729
|
+
let i = c(8).readBigUInt64BE(0), a = R(n, t);
|
|
1730
|
+
if (a !== n && !a.startsWith(n + ve)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1731
|
+
await F(_e(a), { recursive: !0 });
|
|
1612
1732
|
let o = M(a);
|
|
1613
1733
|
for (; i > 0n;) {
|
|
1614
1734
|
r.length === 0 && await s(1);
|
|
1615
1735
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1616
|
-
await
|
|
1736
|
+
await or(o, c(e)), i -= BigInt(e);
|
|
1617
1737
|
}
|
|
1618
|
-
await
|
|
1619
|
-
} else if (e ===
|
|
1738
|
+
await sr(o);
|
|
1739
|
+
} else if (e === er) {
|
|
1620
1740
|
await s(4);
|
|
1621
1741
|
let e = c(4).readUInt32BE(0);
|
|
1622
1742
|
await s(e);
|
|
@@ -1626,24 +1746,24 @@ var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr
|
|
|
1626
1746
|
await s(i);
|
|
1627
1747
|
let a = c(i).toString("utf8");
|
|
1628
1748
|
await s(8);
|
|
1629
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u =
|
|
1749
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = E("sha256"), d = new A(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1630
1750
|
contentType: a,
|
|
1631
1751
|
size: Number(o)
|
|
1632
1752
|
}), p = o;
|
|
1633
1753
|
for (; p > 0n;) {
|
|
1634
1754
|
r.length === 0 && await s(1);
|
|
1635
1755
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1636
|
-
u.update(t), l || await
|
|
1756
|
+
u.update(t), l || await or(d, t), p -= BigInt(e);
|
|
1637
1757
|
}
|
|
1638
|
-
l ? d.destroy() : await
|
|
1639
|
-
let m = c(
|
|
1758
|
+
l ? d.destroy() : await sr(d), await f, await s(tr);
|
|
1759
|
+
let m = c(tr);
|
|
1640
1760
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1641
1761
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1642
1762
|
}
|
|
1643
1763
|
} finally {
|
|
1644
1764
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1645
1765
|
}
|
|
1646
|
-
},
|
|
1766
|
+
}, lr = async (e, t) => {
|
|
1647
1767
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1648
1768
|
let { value: e, done: t } = await a.next();
|
|
1649
1769
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1662,11 +1782,11 @@ var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr
|
|
|
1662
1782
|
}
|
|
1663
1783
|
return Buffer.concat(t);
|
|
1664
1784
|
};
|
|
1665
|
-
if (!await s(
|
|
1785
|
+
if (!await s(Q.length) || !c(Q.length).equals(Q)) throw Error("not a bundle archive (bad magic)");
|
|
1666
1786
|
for (; await s(1);) {
|
|
1667
1787
|
let e = c(1).readUInt8(0);
|
|
1668
|
-
if (e ===
|
|
1669
|
-
if (e !==
|
|
1788
|
+
if (e === Qn || e === er) break;
|
|
1789
|
+
if (e !== $n) throw Error(`unknown bundle entry kind ${e}`);
|
|
1670
1790
|
if (!await s(4)) break;
|
|
1671
1791
|
let t = c(4).readUInt32BE(0);
|
|
1672
1792
|
if (!await s(t)) break;
|
|
@@ -1676,17 +1796,17 @@ var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr
|
|
|
1676
1796
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1677
1797
|
}
|
|
1678
1798
|
return r;
|
|
1679
|
-
},
|
|
1799
|
+
}, ur = (e, t) => cr(e, {
|
|
1680
1800
|
dir: t,
|
|
1681
|
-
blobs:
|
|
1682
|
-
}),
|
|
1801
|
+
blobs: dr(t)
|
|
1802
|
+
}), dr = (e) => {
|
|
1683
1803
|
let t = /* @__PURE__ */ new Set();
|
|
1684
1804
|
return {
|
|
1685
1805
|
has: async (e) => t.has(e),
|
|
1686
1806
|
write: async (n, r, i) => {
|
|
1687
1807
|
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = L(e, "assets");
|
|
1688
1808
|
await F(c, { recursive: !0 });
|
|
1689
|
-
let l =
|
|
1809
|
+
let l = E("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1690
1810
|
l.update(e), u += e.length, n(null, e);
|
|
1691
1811
|
} }), f = L(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1692
1812
|
await o(r, d, M(f));
|
|
@@ -1699,33 +1819,33 @@ var nr = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), rr
|
|
|
1699
1819
|
})}\n`), t.add(n);
|
|
1700
1820
|
}
|
|
1701
1821
|
};
|
|
1702
|
-
},
|
|
1822
|
+
}, fr = ye(se), pr = Buffer.from("VENC1\0\0\0"), mr = 32, hr = 16, gr = 7, _r = 16, vr = 262144, yr = {
|
|
1703
1823
|
N: 32768,
|
|
1704
1824
|
r: 8,
|
|
1705
1825
|
p: 1
|
|
1706
|
-
},
|
|
1826
|
+
}, br = 134217728, xr = 43, Sr = (e, t, n) => {
|
|
1707
1827
|
let r = Buffer.alloc(12);
|
|
1708
|
-
return e.copy(r, 0, 0,
|
|
1828
|
+
return e.copy(r, 0, 0, gr), r.writeUInt32BE(t >>> 0, gr), r.writeUInt8(+!!n, 11), r;
|
|
1709
1829
|
};
|
|
1710
|
-
async function*
|
|
1830
|
+
async function* Cr(e, t) {
|
|
1711
1831
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1712
|
-
let n =
|
|
1713
|
-
...
|
|
1714
|
-
maxmem:
|
|
1715
|
-
}), a = Buffer.alloc(
|
|
1716
|
-
|
|
1832
|
+
let n = oe(hr), r = oe(gr), i = await fr(t, n, mr, {
|
|
1833
|
+
...yr,
|
|
1834
|
+
maxmem: br
|
|
1835
|
+
}), a = Buffer.alloc(xr);
|
|
1836
|
+
pr.copy(a, 0), a.writeUInt32BE(yr.N, 8), a.writeUInt32BE(yr.r, 12), a.writeUInt32BE(yr.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1717
1837
|
let o = 0, s = (e, t) => {
|
|
1718
|
-
let n =
|
|
1838
|
+
let n = T("aes-256-gcm", i, Sr(r, o, t));
|
|
1719
1839
|
o === 0 && n.setAAD(a);
|
|
1720
1840
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1721
1841
|
o++;
|
|
1722
|
-
let l = Buffer.alloc(5 + s.length +
|
|
1723
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
1842
|
+
let l = Buffer.alloc(5 + s.length + _r);
|
|
1843
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + _r, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1724
1844
|
}, c = Buffer.alloc(0);
|
|
1725
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1845
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > vr;) yield s(c.subarray(0, vr), !1), c = c.subarray(vr);
|
|
1726
1846
|
yield s(c, !0);
|
|
1727
1847
|
}
|
|
1728
|
-
async function*
|
|
1848
|
+
async function* wr(e, t) {
|
|
1729
1849
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1730
1850
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1731
1851
|
let { value: e, done: t } = await n.next();
|
|
@@ -1734,16 +1854,16 @@ async function* yr(e, t) {
|
|
|
1734
1854
|
for (; r.length < e && !i && await a(););
|
|
1735
1855
|
return r.length >= e;
|
|
1736
1856
|
};
|
|
1737
|
-
if (!await o(
|
|
1738
|
-
let s = r.subarray(0,
|
|
1739
|
-
if (!s.subarray(0, 8).equals(
|
|
1740
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36,
|
|
1857
|
+
if (!await o(xr)) throw Error("encrypted bundle truncated (no header)");
|
|
1858
|
+
let s = r.subarray(0, xr);
|
|
1859
|
+
if (!s.subarray(0, 8).equals(pr)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1860
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, xr)), p = await fr(t, d, mr, {
|
|
1741
1861
|
N: c,
|
|
1742
1862
|
r: l,
|
|
1743
1863
|
p: u,
|
|
1744
|
-
maxmem:
|
|
1864
|
+
maxmem: br
|
|
1745
1865
|
});
|
|
1746
|
-
r = r.subarray(
|
|
1866
|
+
r = r.subarray(xr);
|
|
1747
1867
|
let m = 0, h = !1;
|
|
1748
1868
|
for (;;) {
|
|
1749
1869
|
if (!await o(5)) {
|
|
@@ -1752,11 +1872,11 @@ async function* yr(e, t) {
|
|
|
1752
1872
|
}
|
|
1753
1873
|
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
1754
1874
|
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
1755
|
-
if (t <
|
|
1875
|
+
if (t < _r) throw Error("encrypted bundle corrupt (short frame)");
|
|
1756
1876
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1757
|
-
let n = r.subarray(5, 5 + t -
|
|
1877
|
+
let n = r.subarray(5, 5 + t - _r), i = r.subarray(5 + t - _r, 5 + t);
|
|
1758
1878
|
r = r.subarray(5 + t);
|
|
1759
|
-
let a =
|
|
1879
|
+
let a = ie("aes-256-gcm", p, Sr(f, m, e));
|
|
1760
1880
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1761
1881
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1762
1882
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1764,7 +1884,7 @@ async function* yr(e, t) {
|
|
|
1764
1884
|
}
|
|
1765
1885
|
//#endregion
|
|
1766
1886
|
//#region src/blobStorage.ts
|
|
1767
|
-
var
|
|
1887
|
+
var Tr = (e, t) => ({
|
|
1768
1888
|
list: () => {
|
|
1769
1889
|
let e = t();
|
|
1770
1890
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1777,29 +1897,29 @@ var br = (e, t) => ({
|
|
|
1777
1897
|
} };
|
|
1778
1898
|
},
|
|
1779
1899
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1780
|
-
let r = await n.runPromise(
|
|
1900
|
+
let r = await n.runPromise(ue(e, t)), i = await O(r.stream);
|
|
1781
1901
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1782
1902
|
} })
|
|
1783
|
-
}),
|
|
1784
|
-
table:
|
|
1903
|
+
}), Er = (e, t) => Tr(e, () => w(t, {
|
|
1904
|
+
table: ce,
|
|
1785
1905
|
chunkSize: 500
|
|
1786
1906
|
}).pipe(l.map((e) => ({
|
|
1787
1907
|
key: String(e.key),
|
|
1788
1908
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1789
1909
|
size: Number(e.size ?? 0)
|
|
1790
|
-
})))),
|
|
1910
|
+
})))), Dr = (e) => ({
|
|
1791
1911
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1792
1912
|
write: async (t, r, i) => {
|
|
1793
|
-
let a =
|
|
1913
|
+
let a = le("storage", () => j.from(r));
|
|
1794
1914
|
await n.runPromise(D(e, t, a, {
|
|
1795
1915
|
contentType: i.contentType,
|
|
1796
1916
|
totalSize: i.size
|
|
1797
1917
|
}));
|
|
1798
1918
|
}
|
|
1799
|
-
}),
|
|
1919
|
+
}), Or = (e, t) => {
|
|
1800
1920
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1801
|
-
return e === "mariadb" &&
|
|
1802
|
-
},
|
|
1921
|
+
return e === "mariadb" && kr(n) ? n : r;
|
|
1922
|
+
}, kr = (e) => (process.env.PATH ?? "").split(I).filter((e) => e !== "").some((t) => N(L(t, e))), Ar = (e) => {
|
|
1803
1923
|
switch (e) {
|
|
1804
1924
|
case "postgres": return "db.dump";
|
|
1805
1925
|
case "mysql":
|
|
@@ -1809,7 +1929,7 @@ var br = (e, t) => ({
|
|
|
1809
1929
|
case "mssql": return "db.bacpac";
|
|
1810
1930
|
default: return "db.dump";
|
|
1811
1931
|
}
|
|
1812
|
-
},
|
|
1932
|
+
}, jr = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1813
1933
|
if (e.host || e.database) return {
|
|
1814
1934
|
host: e.host ?? "localhost",
|
|
1815
1935
|
port: e.port ?? 3306,
|
|
@@ -1825,7 +1945,7 @@ var br = (e, t) => ({
|
|
|
1825
1945
|
password: decodeURIComponent(t.password) || "",
|
|
1826
1946
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1827
1947
|
};
|
|
1828
|
-
},
|
|
1948
|
+
}, Mr = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Nr = (e, t, n) => {
|
|
1829
1949
|
switch (e) {
|
|
1830
1950
|
case "postgres": {
|
|
1831
1951
|
if (t.url) return {
|
|
@@ -1872,7 +1992,7 @@ var br = (e, t) => ({
|
|
|
1872
1992
|
let r = $(t);
|
|
1873
1993
|
return {
|
|
1874
1994
|
kind: "spawn",
|
|
1875
|
-
tool:
|
|
1995
|
+
tool: Or(e, "dump"),
|
|
1876
1996
|
args: [
|
|
1877
1997
|
"--single-transaction",
|
|
1878
1998
|
"--routines",
|
|
@@ -1894,7 +2014,7 @@ var br = (e, t) => ({
|
|
|
1894
2014
|
case "sqlite":
|
|
1895
2015
|
case "turso": return {
|
|
1896
2016
|
kind: "copy",
|
|
1897
|
-
from:
|
|
2017
|
+
from: jr(t),
|
|
1898
2018
|
to: n
|
|
1899
2019
|
};
|
|
1900
2020
|
case "mssql": return {
|
|
@@ -1902,7 +2022,7 @@ var br = (e, t) => ({
|
|
|
1902
2022
|
tool: "sqlpackage",
|
|
1903
2023
|
args: [
|
|
1904
2024
|
"/Action:Export",
|
|
1905
|
-
`/SourceConnectionString:${
|
|
2025
|
+
`/SourceConnectionString:${Mr($({
|
|
1906
2026
|
...t,
|
|
1907
2027
|
port: t.port ?? 1433
|
|
1908
2028
|
}))}`,
|
|
@@ -1910,12 +2030,12 @@ var br = (e, t) => ({
|
|
|
1910
2030
|
],
|
|
1911
2031
|
env: {}
|
|
1912
2032
|
};
|
|
1913
|
-
default: throw new
|
|
2033
|
+
default: throw new W({
|
|
1914
2034
|
tool: "backup",
|
|
1915
2035
|
reason: `no native backup for dialect '${e}'`
|
|
1916
2036
|
});
|
|
1917
2037
|
}
|
|
1918
|
-
},
|
|
2038
|
+
}, Pr = (e, t, n) => {
|
|
1919
2039
|
switch (e) {
|
|
1920
2040
|
case "postgres": {
|
|
1921
2041
|
if (t.url) return {
|
|
@@ -1960,7 +2080,7 @@ var br = (e, t) => ({
|
|
|
1960
2080
|
let r = $(t);
|
|
1961
2081
|
return {
|
|
1962
2082
|
kind: "spawn",
|
|
1963
|
-
tool:
|
|
2083
|
+
tool: Or(e, "client"),
|
|
1964
2084
|
args: [
|
|
1965
2085
|
"--protocol=TCP",
|
|
1966
2086
|
"-h",
|
|
@@ -1979,14 +2099,14 @@ var br = (e, t) => ({
|
|
|
1979
2099
|
case "turso": return {
|
|
1980
2100
|
kind: "copy",
|
|
1981
2101
|
from: n,
|
|
1982
|
-
to:
|
|
2102
|
+
to: jr(t)
|
|
1983
2103
|
};
|
|
1984
2104
|
case "mssql": return {
|
|
1985
2105
|
kind: "spawn",
|
|
1986
2106
|
tool: "sqlpackage",
|
|
1987
2107
|
args: [
|
|
1988
2108
|
"/Action:Import",
|
|
1989
|
-
`/TargetConnectionString:${
|
|
2109
|
+
`/TargetConnectionString:${Mr($({
|
|
1990
2110
|
...t,
|
|
1991
2111
|
port: t.port ?? 1433
|
|
1992
2112
|
}))}`,
|
|
@@ -1994,18 +2114,18 @@ var br = (e, t) => ({
|
|
|
1994
2114
|
],
|
|
1995
2115
|
env: {}
|
|
1996
2116
|
};
|
|
1997
|
-
default: throw new
|
|
2117
|
+
default: throw new W({
|
|
1998
2118
|
tool: "restore",
|
|
1999
2119
|
reason: `no native restore for dialect '${e}'`
|
|
2000
2120
|
});
|
|
2001
2121
|
}
|
|
2002
|
-
},
|
|
2003
|
-
|
|
2122
|
+
}, Fr = (e) => e.kind === "copy" ? n.async((t) => {
|
|
2123
|
+
de(e.from, e.to, (r) => t(r ? n.fail(new W({
|
|
2004
2124
|
tool: "copy",
|
|
2005
2125
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
2006
2126
|
})) : n.void));
|
|
2007
2127
|
}) : n.async((t) => {
|
|
2008
|
-
let r =
|
|
2128
|
+
let r = be(e.tool, [...e.args], {
|
|
2009
2129
|
env: {
|
|
2010
2130
|
...process.env,
|
|
2011
2131
|
...e.env
|
|
@@ -2018,14 +2138,14 @@ var br = (e, t) => ({
|
|
|
2018
2138
|
}), i = "";
|
|
2019
2139
|
r.stderr?.on("data", (e) => {
|
|
2020
2140
|
i += String(e);
|
|
2021
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin &&
|
|
2141
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin && fe(e.stdinFile).pipe(r.stdin);
|
|
2022
2142
|
let a = () => {
|
|
2023
2143
|
if (e.stdoutFile !== void 0) try {
|
|
2024
|
-
|
|
2144
|
+
pe(e.stdoutFile);
|
|
2025
2145
|
} catch {}
|
|
2026
2146
|
};
|
|
2027
2147
|
r.on("error", (r) => {
|
|
2028
|
-
a(), t(n.fail(new
|
|
2148
|
+
a(), t(n.fail(new W({
|
|
2029
2149
|
tool: e.tool,
|
|
2030
2150
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`
|
|
2031
2151
|
})));
|
|
@@ -2034,17 +2154,17 @@ var br = (e, t) => ({
|
|
|
2034
2154
|
t(n.void);
|
|
2035
2155
|
return;
|
|
2036
2156
|
}
|
|
2037
|
-
a(), t(n.fail(new
|
|
2157
|
+
a(), t(n.fail(new W({
|
|
2038
2158
|
tool: e.tool,
|
|
2039
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2159
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${ge(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
2040
2160
|
stderr: i.slice(-2e3)
|
|
2041
2161
|
})));
|
|
2042
2162
|
});
|
|
2043
|
-
}),
|
|
2044
|
-
let t = yield*
|
|
2163
|
+
}), Ir = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Lr = (e) => n.gen(function* () {
|
|
2164
|
+
let t = yield* nt(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
2045
2165
|
store: e.store,
|
|
2046
2166
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
2047
|
-
}), r =
|
|
2167
|
+
}), r = qt(t.tables, e.snapshot), i = e.classification ?? Ce(e.snapshot), a = e.sampleSize ?? 20, o = Ie(r.map((t) => ({
|
|
2048
2168
|
name: t,
|
|
2049
2169
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
2050
2170
|
name: e.name,
|
|
@@ -2052,19 +2172,19 @@ var br = (e, t) => ({
|
|
|
2052
2172
|
}))
|
|
2053
2173
|
})), i, e.masking), s = [], c = [];
|
|
2054
2174
|
for (let i of r) {
|
|
2055
|
-
let r = t.predicateFor(i), u = yield*
|
|
2175
|
+
let r = t.predicateFor(i), u = yield* w(e.store, {
|
|
2056
2176
|
table: i,
|
|
2057
2177
|
chunkSize: a,
|
|
2058
2178
|
...r === void 0 ? {} : { where: r }
|
|
2059
|
-
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
2060
|
-
c.push(...
|
|
2179
|
+
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) => ze(t, i, o.actions, e.masking.seed));
|
|
2180
|
+
c.push(...Re(d, i, o.actions));
|
|
2061
2181
|
let f = u[0], p = d[0], m = [];
|
|
2062
2182
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
2063
2183
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
2064
2184
|
let n = e.slice(i.length + 1);
|
|
2065
2185
|
m.push({
|
|
2066
2186
|
column: n,
|
|
2067
|
-
action:
|
|
2187
|
+
action: Ir(t),
|
|
2068
2188
|
before: f[n],
|
|
2069
2189
|
after: p[n]
|
|
2070
2190
|
});
|
|
@@ -2079,7 +2199,7 @@ var br = (e, t) => ({
|
|
|
2079
2199
|
leaks: c,
|
|
2080
2200
|
unclassified: o.unclassified
|
|
2081
2201
|
};
|
|
2082
|
-
}),
|
|
2202
|
+
}), Rr = (e, t) => {
|
|
2083
2203
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
2084
2204
|
for (let t of e) {
|
|
2085
2205
|
let e = n.get(t.name);
|
|
@@ -2134,7 +2254,7 @@ var br = (e, t) => ({
|
|
|
2134
2254
|
skipColumns: i,
|
|
2135
2255
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
2136
2256
|
};
|
|
2137
|
-
},
|
|
2257
|
+
}, zr = (e) => {
|
|
2138
2258
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
2139
2259
|
let t = {
|
|
2140
2260
|
safe: "✓",
|
|
@@ -2145,4 +2265,4 @@ var br = (e, t) => ({
|
|
|
2145
2265
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
2146
2266
|
};
|
|
2147
2267
|
//#endregion
|
|
2148
|
-
export {
|
|
2268
|
+
export { xt as ASSETS_DIR, St as ASSET_INDEX_FILE, H as BundleError, We as CodecError, Ke as CompressionError, Ze as CrossDialectError, bt as DATA_DIR, Se as DEFAULT_CLASS_ACTIONS, Be as ENVIRONMENT_LOCAL_TABLES, Ve as ENVIRONMENT_PORTABLE_TABLES, ft as FORMAT_VERSION, Xe as ImportModeError, Ge as IntegrityError, Ct as LEDGER_FILE, K as MANIFEST_FILE, Ye as MaskingError, W as NativeToolError, qe as RowsRefusedError, Wn as SAVEPOINT_BATCH_SIZE, bn as STAGING_PREFIX, Je as SchemaDriftError, U as ScopeError, Pe as applyAction, Kn as asImportError, Zn as assessLiveImport, Nt as assetDone, Ar as backupArtifactName, Nr as backupCommand, cn as canonicalType, dn as checkPortability, Hn as checkSchemaDrift, Ce as classificationFromSnapshot, Rr as classifyImportDrift, lt as compressionExt, $e as computeSubsetIds, In as decideStaging, vt as decodeManifest, Ut as decodeRow, Vt as decodeValue, wr as decryptStream, Xn as defineDataProfile, Vn as diffSnapshots, dr as dirBlobSink, Tn as dropStagingSql, At as emptyLedger, _t as encodeManifest, Ht as encodeRow, zt as encodeValue, Cr as encryptStream, Ue as environmentLocalNotice, He as environmentLocalReason, jn as execStatements, zr as formatImportDrift, fn as formatIssue, st as hasZstd, V as isEnvironmentLocal, ut as makeCompressor, dt as makeDecompressor, ze as maskRow, Kt as ndjsonToRows, sn as outsideReferenceRefusal, on as outsideReferencesInto, ar as packBundle, lr as peekBundle, ct as pickCompression, Ie as planMasking, Lr as previewMasking, jt as readLedger, we as resolveAction, nt as resolveScope, ot as restoreAssetFromCas, Pr as restoreCommand, Gt as rowsToNdjson, an as runExport, Yn as runImport, Fr as runNativeStep, Re as scanForLeaks, X as stagingNameFor, it as storageAssetSink, rt as storageAssetSource, Dr as storageBlobSink, Tr as storageBlobSource, Er as storageRefsBlobSource, et as subsetToScope, Mt as tableDone, qt as topoSortTables, cr as unpackBundle, ur as unpackBundleToDir, at as writeAssetToCas, J as writeLedger };
|