@voltro/data-transfer 0.46.0 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +110 -0
- package/dist/index.d.ts +151 -0
- package/dist/index.js +884 -736
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
import { Cause as e, Data as t, Effect as n, Exit as r, Metric as i, MetricBoundaries as a, Runtime as o, Schedule as s, Schema as c, Stream as l } from "effect";
|
|
2
|
-
import {
|
|
3
|
-
import { fingerprintSchema as
|
|
4
|
-
import { createCipheriv as
|
|
5
|
-
import { STORAGE_REFS_TABLE as
|
|
6
|
-
import * as
|
|
7
|
-
import { PassThrough as
|
|
8
|
-
import { copyFile as
|
|
9
|
-
import { SqlClient as
|
|
10
|
-
import { mkdir as
|
|
11
|
-
import { basename as
|
|
12
|
-
import { promisify as
|
|
13
|
-
import { spawn as
|
|
2
|
+
import { IMPORT_RUNS_TABLE as u, REPLACE_IN_PROGRESS_TABLE as d, _voltroImportRunsTable as f, classifyConstraintViolation as p, derivedRowId as m, ensureTableRegistered as h, eq as g, extractDbCause as _, getTable as v, inSet as y, isNotNull as b, or as x, recordsTable as S, registerTable as C, streamTable as w, suspendWriteRecorders as ee, suspendWriteRecordersEffect as te, unregisterTable as ne } from "@voltro/database";
|
|
3
|
+
import { fingerprintSchema as re } from "@voltro/database/sql";
|
|
4
|
+
import { createCipheriv as T, createDecipheriv as ie, createHash as E, createHmac as ae, randomBytes as oe, scrypt as se } from "node:crypto";
|
|
5
|
+
import { STORAGE_REFS_TABLE as ce, fromNodeReadable as le, getObjectStream as ue, putObjectStream as D, toNodeReadable as O } from "@voltro/plugin-storage";
|
|
6
|
+
import * as k from "node:zlib";
|
|
7
|
+
import { PassThrough as A, Readable as j } from "node:stream";
|
|
8
|
+
import { copyFile as de, createReadStream as fe, createWriteStream as M, existsSync as N, unlinkSync as pe } from "node:fs";
|
|
9
|
+
import { SqlClient as P } from "@effect/sql";
|
|
10
|
+
import { mkdir as F, readdir as me, stat as he } from "node:fs/promises";
|
|
11
|
+
import { basename as ge, delimiter as I, dirname as _e, join as L, resolve as R, sep as ve } from "node:path";
|
|
12
|
+
import { promisify as ye } from "node:util";
|
|
13
|
+
import { spawn as be } from "node:child_process";
|
|
14
14
|
//#region src/consistency.ts
|
|
15
|
-
var
|
|
15
|
+
var xe = (e) => {
|
|
16
16
|
switch (e) {
|
|
17
17
|
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
18
18
|
case "mysql":
|
|
@@ -22,7 +22,7 @@ var ye = (e) => {
|
|
|
22
22
|
case "turso": return [];
|
|
23
23
|
default: return [];
|
|
24
24
|
}
|
|
25
|
-
},
|
|
25
|
+
}, Se = {
|
|
26
26
|
email: { fake: "email" },
|
|
27
27
|
fullName: { fake: "fullName" },
|
|
28
28
|
firstName: { fake: "firstName" },
|
|
@@ -37,7 +37,7 @@ var ye = (e) => {
|
|
|
37
37
|
date: "dateShift",
|
|
38
38
|
secret: "null",
|
|
39
39
|
freeText: "redact"
|
|
40
|
-
},
|
|
40
|
+
}, Ce = (e) => {
|
|
41
41
|
let t = {};
|
|
42
42
|
for (let n of e.tables) {
|
|
43
43
|
let e = {};
|
|
@@ -49,15 +49,15 @@ var ye = (e) => {
|
|
|
49
49
|
t[n.name] = e;
|
|
50
50
|
}
|
|
51
51
|
return t;
|
|
52
|
-
},
|
|
52
|
+
}, we = (e, t, n, r) => {
|
|
53
53
|
let i = r.columns?.[`${e}.${t}`];
|
|
54
54
|
if (i) return i;
|
|
55
55
|
if (n?.sensitive) {
|
|
56
56
|
let e = n.sensitive.class;
|
|
57
|
-
return r.classes?.[e] ??
|
|
57
|
+
return r.classes?.[e] ?? Se[e] ?? "redact";
|
|
58
58
|
}
|
|
59
59
|
return n?.safe ? r.onSafe ?? "keep" : n?.columnType === "id" || n?.columnType === "reference" ? "keep" : r.onUnclassified ?? "error";
|
|
60
|
-
},
|
|
60
|
+
}, Te = (e, t) => ae("sha256", e).update(t == null ? "∅" : String(t)).digest(), z = (e, t) => e.readUInt32BE(t * 4 % (e.length - 4)), Ee = [
|
|
61
61
|
"Ada",
|
|
62
62
|
"Alan",
|
|
63
63
|
"Grace",
|
|
@@ -70,7 +70,7 @@ var ye = (e) => {
|
|
|
70
70
|
"Guido",
|
|
71
71
|
"Margaret",
|
|
72
72
|
"Bjarne"
|
|
73
|
-
],
|
|
73
|
+
], De = [
|
|
74
74
|
"Lovelace",
|
|
75
75
|
"Turing",
|
|
76
76
|
"Hopper",
|
|
@@ -83,7 +83,7 @@ var ye = (e) => {
|
|
|
83
83
|
"Rossum",
|
|
84
84
|
"Hamilton",
|
|
85
85
|
"Stroustrup"
|
|
86
|
-
],
|
|
86
|
+
], Oe = [
|
|
87
87
|
"Maple",
|
|
88
88
|
"Oak",
|
|
89
89
|
"Cedar",
|
|
@@ -92,7 +92,7 @@ var ye = (e) => {
|
|
|
92
92
|
"Birch",
|
|
93
93
|
"Willow",
|
|
94
94
|
"Ash"
|
|
95
|
-
],
|
|
95
|
+
], ke = [
|
|
96
96
|
"Initech",
|
|
97
97
|
"Hooli",
|
|
98
98
|
"Umbrella",
|
|
@@ -101,23 +101,23 @@ var ye = (e) => {
|
|
|
101
101
|
"Soylent",
|
|
102
102
|
"Stark",
|
|
103
103
|
"Wayne"
|
|
104
|
-
],
|
|
105
|
-
let r =
|
|
104
|
+
], B = (e, t, n) => e[z(t, n) % e.length], Ae = (e, t, n) => {
|
|
105
|
+
let r = Te(n, `${e}:${String(t)}`), i = z(r, 1) % 1e4;
|
|
106
106
|
switch (e) {
|
|
107
|
-
case "email": return `${
|
|
108
|
-
case "fullName": return `${
|
|
109
|
-
case "firstName": return
|
|
110
|
-
case "lastName": return
|
|
111
|
-
case "username": return `${
|
|
112
|
-
case "phone": return `+1${String(
|
|
113
|
-
case "address": return `${
|
|
114
|
-
case "company": return
|
|
107
|
+
case "email": return `${B(Ee, r, 0).toLowerCase()}.${B(De, r, 2).toLowerCase()}${i}@example.com`;
|
|
108
|
+
case "fullName": return `${B(Ee, r, 0)} ${B(De, r, 1)}`;
|
|
109
|
+
case "firstName": return B(Ee, r, 0);
|
|
110
|
+
case "lastName": return B(De, r, 1);
|
|
111
|
+
case "username": return `${B(Ee, r, 0).toLowerCase()}${i}`;
|
|
112
|
+
case "phone": return `+1${String(z(r, 0) % 1e10).padStart(10, "0")}`;
|
|
113
|
+
case "address": return `${z(r, 0) % 9899 + 100} ${B(Oe, r, 1)} St`;
|
|
114
|
+
case "company": return B(ke, r, 0);
|
|
115
115
|
case "url": return `https://example.com/${r.toString("hex").slice(0, 12)}`;
|
|
116
|
-
case "ip": return `${
|
|
117
|
-
case "creditCard": return
|
|
116
|
+
case "ip": return `${z(r, 0) % 256}.${z(r, 1) % 256}.${z(r, 2) % 256}.${z(r, 3) % 254 + 1}`;
|
|
117
|
+
case "creditCard": return je(`4${r.toString("hex").replace(/\D/g, "").padEnd(14, "0").slice(0, 14)}`);
|
|
118
118
|
default: return `${e}_${r.toString("hex").slice(0, 8)}`;
|
|
119
119
|
}
|
|
120
|
-
},
|
|
120
|
+
}, je = (e) => {
|
|
121
121
|
let t = e.slice(0, 15).split("").map(Number), n = 0;
|
|
122
122
|
t.forEach((e, t) => {
|
|
123
123
|
let r = e;
|
|
@@ -125,31 +125,31 @@ var ye = (e) => {
|
|
|
125
125
|
});
|
|
126
126
|
let r = (10 - n % 10) % 10;
|
|
127
127
|
return e.slice(0, 15) + String(r);
|
|
128
|
-
},
|
|
128
|
+
}, Me = 864e5, Ne = (e, t) => {
|
|
129
129
|
let n = e instanceof Date ? e.getTime() : Date.parse(String(e));
|
|
130
130
|
if (Number.isNaN(n)) return e;
|
|
131
|
-
let r =
|
|
131
|
+
let r = z(Te(t, "date-shift-offset"), 0) % 730 - 365, i = new Date(n + r * Me);
|
|
132
132
|
return e instanceof Date ? i : i.toISOString();
|
|
133
|
-
},
|
|
133
|
+
}, Pe = (e, t) => {
|
|
134
134
|
if (t.value === null || t.value === void 0) return null;
|
|
135
135
|
if (e === "error") throw Error(`masking: unresolved 'error' action for ${t.table}.${t.column}`);
|
|
136
136
|
if (e === "keep") return t.value;
|
|
137
137
|
if (e === "null") return null;
|
|
138
138
|
if (e === "redact") return t.columnType === "text" || typeof t.value == "string" ? "[redacted]" : null;
|
|
139
|
-
if (e === "hash") return
|
|
140
|
-
if (e === "dateShift") return
|
|
141
|
-
if (typeof e == "object" && "fake" in e) return
|
|
139
|
+
if (e === "hash") return Te(t.seed, t.value).toString("hex").slice(0, 16);
|
|
140
|
+
if (e === "dateShift") return Ne(t.value, t.seed);
|
|
141
|
+
if (typeof e == "object" && "fake" in e) return Ae(e.fake, t.value, t.seed);
|
|
142
142
|
if (typeof e == "object" && "custom" in e) return e.custom(t);
|
|
143
143
|
throw Error(`masking: '${t.table}.${t.column}' resolved to an action this build does not understand (${JSON.stringify(e)}). Refusing to copy the value through. Valid actions: 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | { fake: '<class>' } | { custom: fn }.`);
|
|
144
|
-
},
|
|
144
|
+
}, Fe = (e) => typeof e == "string" ? e === "keep" || e === "null" || e === "redact" || e === "hash" || e === "dateShift" || e === "error" : typeof e != "object" || !e ? !1 : "fake" in e ? typeof e.fake == "string" : "custom" in e && typeof e.custom == "function", Ie = (e, t, n) => {
|
|
145
145
|
let r = {}, i = [], a = [];
|
|
146
146
|
for (let o of e) for (let e of o.columns) {
|
|
147
|
-
let s = `${o.name}.${e.name}`, c =
|
|
147
|
+
let s = `${o.name}.${e.name}`, c = we(o.name, e.name, t[o.name]?.[e.name], n);
|
|
148
148
|
if (c === "error") {
|
|
149
149
|
i.push(s);
|
|
150
150
|
continue;
|
|
151
151
|
}
|
|
152
|
-
if (!
|
|
152
|
+
if (!Fe(c)) {
|
|
153
153
|
a.push({
|
|
154
154
|
column: s,
|
|
155
155
|
action: c
|
|
@@ -163,7 +163,7 @@ var ye = (e) => {
|
|
|
163
163
|
unclassified: i,
|
|
164
164
|
invalid: a
|
|
165
165
|
};
|
|
166
|
-
},
|
|
166
|
+
}, Le = [
|
|
167
167
|
{
|
|
168
168
|
name: "email",
|
|
169
169
|
re: /[\w.+-]+@[\w-]+\.[\w.-]+/
|
|
@@ -180,7 +180,7 @@ var ye = (e) => {
|
|
|
180
180
|
name: "creditCard",
|
|
181
181
|
re: /\b(?:\d[ -]?){13,16}\b/
|
|
182
182
|
}
|
|
183
|
-
],
|
|
183
|
+
], Re = (e, t, n) => {
|
|
184
184
|
let r = [], i = /* @__PURE__ */ new Set();
|
|
185
185
|
for (let [a, o] of Object.entries(n)) {
|
|
186
186
|
if (!a.startsWith(`${t}.`) || o !== "keep") continue;
|
|
@@ -188,7 +188,7 @@ var ye = (e) => {
|
|
|
188
188
|
for (let a of e) {
|
|
189
189
|
let e = a[n];
|
|
190
190
|
if (typeof e != "string" || e.length === 0) continue;
|
|
191
|
-
let o =
|
|
191
|
+
let o = Le.find((t) => t.re.test(e));
|
|
192
192
|
if (o && !i.has(n)) {
|
|
193
193
|
i.add(n), r.push({
|
|
194
194
|
table: t,
|
|
@@ -201,9 +201,9 @@ var ye = (e) => {
|
|
|
201
201
|
}
|
|
202
202
|
}
|
|
203
203
|
return r;
|
|
204
|
-
},
|
|
204
|
+
}, ze = (e, t, n, r, i) => {
|
|
205
205
|
let a = {};
|
|
206
|
-
for (let [o, s] of Object.entries(e)) a[o] =
|
|
206
|
+
for (let [o, s] of Object.entries(e)) a[o] = Pe(n[`${t}.${o}`] ?? "keep", {
|
|
207
207
|
value: s,
|
|
208
208
|
table: t,
|
|
209
209
|
column: o,
|
|
@@ -211,33 +211,84 @@ var ye = (e) => {
|
|
|
211
211
|
...i?.[o] ? { columnType: i[o] } : {}
|
|
212
212
|
});
|
|
213
213
|
return a;
|
|
214
|
-
},
|
|
214
|
+
}, Be = /* @__PURE__ */ new Map([
|
|
215
215
|
["_voltro_migration_plans", "the schema fingerprint THIS deployment applied — a foreign row makes the next boot refuse"],
|
|
216
216
|
["_voltro_migrations", "which file-based migrations ran HERE — a foreign row marks unrun migrations as done"],
|
|
217
217
|
["_voltro_seeds", "which seeds ran HERE — a foreign row suppresses seeding the target still needs"],
|
|
218
218
|
["_voltro_cdc_offsets", "binlog/WAL positions of the SOURCE server — meaningless and unsafe against another one"],
|
|
219
219
|
["_voltro_schedule_claims", "live leases held by THIS deployment's replicas — a foreign row hands out a lease nobody holds"],
|
|
220
|
+
["_voltro_data_imports", "which bundles were imported HERE — a foreign row claims an import this deployment never ran"],
|
|
220
221
|
["_voltro_replace_in_progress", "the marker a destructive import HERE left behind — a foreign row refuses a boot over a healthy database"],
|
|
221
222
|
["_voltro_wakeups", "pending wakeups addressed to THIS deployment's runners"],
|
|
222
223
|
["_voltro_workflow_watermarks", "how far a consumer has read THIS deployment's event journal — a foreign position skips events"],
|
|
223
224
|
["_voltro_workflow_pending", "starts waiting to be drained HERE — a foreign row makes the target RUN a workflow somebody queued elsewhere"],
|
|
224
225
|
["_voltro_workflow_admissions", "the in-flight accounting a concurrency gate decides on — a foreign row makes the target refuse or admit wrongly"],
|
|
225
|
-
["_voltro_workflow_pauses", "which workflows an operator paused HERE — a foreign row silently pauses them in the target"]
|
|
226
|
-
|
|
226
|
+
["_voltro_workflow_pauses", "which workflows an operator paused HERE — a foreign row silently pauses them in the target"],
|
|
227
|
+
["_voltro_workflow_event_deliveries", "which (event, trigger) pairs were already delivered HERE — a foreign row SUPPRESSES a delivery the target owes"],
|
|
228
|
+
["_voltro_traces", "spans recorded HERE — a foreign span claims a request this deployment never served"],
|
|
229
|
+
["_voltro_undo_log", "the before-images of mutations made HERE — a foreign row offers to undo a write into rows the target does not have"],
|
|
230
|
+
["_voltro_outbox", "messages waiting to be SENT by this deployment — a foreign row makes the target deliver something queued elsewhere"],
|
|
231
|
+
["_voltro_outbox_attempts", "the delivery attempts this deployment made — they belong to its outbox rows and to no other"],
|
|
232
|
+
["_voltro_ai_inferences", "offloaded inference work queued for THIS deployment's runners — a foreign row makes the target run somebody else's"],
|
|
233
|
+
["_voltro_ai_budget", "the live spend RESERVATION counter a budget refuses against — a foreign row makes the target refuse calls it should allow"],
|
|
234
|
+
["_voltro_ai_usage", "what this deployment spent on model calls — a foreign row overstates a bill nobody here ran up"],
|
|
235
|
+
["_voltro_budget_holds", "in-flight holds against a running deployment's budget — the same accounting `_voltro_workflow_admissions` keeps"],
|
|
236
|
+
["_voltro_storage_grants", "short-lived upload/download grants issued HERE — a foreign grant authorises nothing in the target"],
|
|
237
|
+
["_voltro_storage_usage", "how much storage this deployment accounts for — a foreign row makes a quota decision on somebody else's bytes"],
|
|
238
|
+
["_voltro_idempotency", "the request keys this deployment has already answered — a foreign key SUPPRESSES a request the target never served"],
|
|
239
|
+
["_voltro_webhook_deliveries", "the delivery attempts this deployment made to a subscriber — a foreign row claims a call the target never placed"],
|
|
240
|
+
["_voltro_schedule_runs", "every schedule firing that happened HERE, replica ids included — a foreign row is history the target did not live"],
|
|
241
|
+
["_voltro_cdc_log", "the change rows this deployment captured from its own database — a foreign row replays a change that never happened here"],
|
|
242
|
+
["_voltro_notification_deliveries", "the notification attempts this deployment made — a foreign row claims a send the target never performed"],
|
|
243
|
+
["_voltro_notification_held", "notifications waiting to be SENT from here — a foreign row makes the target deliver something queued elsewhere"],
|
|
244
|
+
["_voltro_webhook_event_stats", "per-deployment delivery counters a backoff decision reads — foreign counts steer the target's retries"],
|
|
245
|
+
["_voltro_webhook_rate_windows", "the live rate-limit windows this deployment is inside — a foreign window throttles or releases wrongly"]
|
|
246
|
+
]), Ve = /* @__PURE__ */ new Map([
|
|
247
|
+
["_voltro_agent_threads", "conversations the app's users had — app data"],
|
|
248
|
+
["_voltro_agent_messages", "the messages in those conversations — app data"],
|
|
249
|
+
["_voltro_prompts", "prompt definitions the app authored — app data"],
|
|
250
|
+
["_voltro_api_keys", "keys the app issued (hashed) — app data, and a restore without them locks callers out"],
|
|
251
|
+
["_voltro_kv", "the app's own key/value rows — app data by construction"],
|
|
252
|
+
["_voltro_approvals", "human decisions on workflow runs — they belong with the runs"],
|
|
253
|
+
["_voltro_connections", "third-party connections the app's users authorised. Portable as ROWS; note the secrets are encrypted with the project's key, so a target without that key holds ciphertext it cannot read"],
|
|
254
|
+
["_voltro_connection_grants", "who may use which connection — belongs with the connections"],
|
|
255
|
+
["_voltro_storage_blobs", "the stored objects' metadata — the assets travel with the bundle"],
|
|
256
|
+
["_voltro_storage_refs", "which rows reference which blob — belongs with the blobs"],
|
|
257
|
+
["_voltro_webhook_targets", "the endpoints subscribers registered through the app — app data"],
|
|
258
|
+
["_voltro_workflow_runs", "what the app's workflows did — history the app owns"],
|
|
259
|
+
["_voltro_workflow_run_steps", "the steps of those runs — belongs with the runs"],
|
|
260
|
+
["_voltro_workflow_run_events", "the events those runs emitted — belongs with the runs"],
|
|
261
|
+
["_voltro_workflow_events", "the app's event journal — app data (the target keeps its OWN watermarks, which are local)"],
|
|
262
|
+
["_voltro_workflow_start_contexts", "the context a start was made with — belongs with the runs"],
|
|
263
|
+
["_voltro_events", "the app's domain events — app data"],
|
|
264
|
+
["_voltro_ai_flows", "the flow definitions the app authored — app data"],
|
|
265
|
+
["_voltro_ai_flow_runs", "what those flows did — history the app owns, like a workflow run"],
|
|
266
|
+
["_voltro_notification_inbox", "what the app's users received — app data, and a restore without it loses their inbox"],
|
|
267
|
+
["_voltro_notification_preferences", "the per-user delivery settings the app's users chose — app data"],
|
|
268
|
+
["_voltro_notification_quiet_hours", "the per-user quiet hours the app's users chose — app data"],
|
|
269
|
+
["_voltro_notification_topic_subscriptions", "which topics the app's users subscribed to — app data"],
|
|
270
|
+
["_voltro_rebac_tuples", "the relationship tuples the app's authorisation decisions are made from — app data"],
|
|
271
|
+
["_voltro_scim_users", "identities an external directory provisioned into the app — app data"],
|
|
272
|
+
["_voltro_scim_groups", "groups an external directory provisioned into the app — app data"],
|
|
273
|
+
["_voltro_row_history", "the app's own row-version history — the point of keeping versions is that they survive a move"],
|
|
274
|
+
["_voltro_audit_log", "the app's audit trail — compliance data the app owns, not deployment bookkeeping"],
|
|
275
|
+
["_voltro_feature_flags", "the flags the app defined — app configuration"],
|
|
276
|
+
["_voltro_feature_flag_audit", "who changed which flag — belongs with the flags"]
|
|
277
|
+
]), V = (e) => Be.has(e), He = (e) => Be.get(e), Ue = (e, t) => {
|
|
227
278
|
if (e.length === 0) return;
|
|
228
|
-
let n = e.map((e) => ` ${e} — ${
|
|
279
|
+
let n = e.map((e) => ` ${e} — ${He(e) ?? "describes this deployment"}`);
|
|
229
280
|
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
230
|
-
},
|
|
281
|
+
}, H = class extends c.TaggedError()("BundleError", {
|
|
231
282
|
reason: c.String,
|
|
232
283
|
path: c.optional(c.String)
|
|
233
|
-
}) {},
|
|
284
|
+
}) {}, We = class extends c.TaggedError()("CodecError", {
|
|
234
285
|
table: c.String,
|
|
235
286
|
reason: c.String
|
|
236
|
-
}) {},
|
|
287
|
+
}) {}, Ge = class extends c.TaggedError()("IntegrityError", {
|
|
237
288
|
what: c.String,
|
|
238
289
|
expected: c.String,
|
|
239
290
|
actual: c.String
|
|
240
|
-
}) {},
|
|
291
|
+
}) {}, Ke = class extends t.TaggedError("CompressionError") {}, U = class extends c.TaggedError()("ScopeError", { reason: c.String }) {}, qe = class extends c.TaggedError()("RowsRefusedError", {
|
|
241
292
|
rows: c.Array(c.Struct({
|
|
242
293
|
table: c.String,
|
|
243
294
|
id: c.String,
|
|
@@ -251,29 +302,29 @@ var ye = (e) => {
|
|
|
251
302
|
refused: c.Number,
|
|
252
303
|
primary: c.Number
|
|
253
304
|
}))
|
|
254
|
-
}) {},
|
|
305
|
+
}) {}, Je = class extends c.TaggedError()("SchemaDriftError", {
|
|
255
306
|
bundleFingerprint: c.String,
|
|
256
307
|
targetFingerprint: c.String,
|
|
257
308
|
diff: c.Array(c.String)
|
|
258
|
-
}) {},
|
|
309
|
+
}) {}, W = class extends t.TaggedError("NativeToolError") {
|
|
259
310
|
get message() {
|
|
260
311
|
let e = this.stderr?.trim();
|
|
261
312
|
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}`;
|
|
262
313
|
}
|
|
263
|
-
},
|
|
314
|
+
}, Ye = class extends c.TaggedError()("MaskingError", {
|
|
264
315
|
unclassified: c.Array(c.String),
|
|
265
316
|
invalidActions: c.optional(c.Array(c.String))
|
|
266
|
-
}) {},
|
|
317
|
+
}) {}, Xe = class extends c.TaggedError()("ImportModeError", {
|
|
267
318
|
mode: c.String,
|
|
268
319
|
reason: c.String
|
|
269
|
-
}) {},
|
|
320
|
+
}) {}, Ze = class extends c.TaggedError()("CrossDialectError", {
|
|
270
321
|
sourceDialect: c.String,
|
|
271
322
|
targetDialect: c.String,
|
|
272
323
|
blocking: c.Array(c.String)
|
|
273
|
-
}) {},
|
|
324
|
+
}) {}, G = (e, t) => e.tables.find((e) => e.name === t)?.columns.find((e) => e.type === "id")?.name ?? "id", Qe = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.references).map((e) => ({
|
|
274
325
|
col: e.name,
|
|
275
326
|
refTable: e.references.table
|
|
276
|
-
})),
|
|
327
|
+
})), $e = (e, t, r) => n.gen(function* () {
|
|
277
328
|
let i = r.maxDepth ?? 12, a = new Set(r.pinned ?? []), o = /* @__PURE__ */ new Map(), s = /* @__PURE__ */ new Map(), c = /* @__PURE__ */ new Map(), u = (e) => {
|
|
278
329
|
let t = s.get(e) ?? /* @__PURE__ */ new Set();
|
|
279
330
|
return s.set(e, t), t;
|
|
@@ -291,10 +342,10 @@ var ye = (e) => {
|
|
|
291
342
|
for (let t of n) d(t.refTable, e[t.col]);
|
|
292
343
|
};
|
|
293
344
|
}, p = (r, i) => {
|
|
294
|
-
let a = f(r,
|
|
295
|
-
return l.runForEach(
|
|
345
|
+
let a = f(r, G(t, r), Qe(t, r));
|
|
346
|
+
return l.runForEach(w(e, {
|
|
296
347
|
table: r,
|
|
297
|
-
pk:
|
|
348
|
+
pk: G(t, r),
|
|
298
349
|
...i === void 0 ? {} : { where: i }
|
|
299
350
|
}), (e) => n.sync(() => a(e)));
|
|
300
351
|
}, m = n.gen(function* () {
|
|
@@ -303,7 +354,7 @@ var ye = (e) => {
|
|
|
303
354
|
c.clear();
|
|
304
355
|
for (let [n, r] of e) {
|
|
305
356
|
let e = [...r].filter((e) => !s.get(n)?.has(e));
|
|
306
|
-
e.length !== 0 && (yield* p(n,
|
|
357
|
+
e.length !== 0 && (yield* p(n, y(G(t, n), e)));
|
|
307
358
|
}
|
|
308
359
|
}
|
|
309
360
|
});
|
|
@@ -318,29 +369,32 @@ var ye = (e) => {
|
|
|
318
369
|
refTable: e.references?.table
|
|
319
370
|
})).filter((t) => !!t.refTable && e.has(t.refTable) && (s.get(t.refTable)?.size ?? 0) > 0);
|
|
320
371
|
if (a.length === 0) continue;
|
|
321
|
-
let o = s.get(t)?.size ?? 0, c = a.map((e) =>
|
|
322
|
-
yield* p(t, c.length === 1 ? c[0] :
|
|
372
|
+
let o = s.get(t)?.size ?? 0, c = a.map((e) => y(e.col, [...s.get(e.refTable)]));
|
|
373
|
+
yield* p(t, c.length === 1 ? c[0] : x(...c)), (s.get(t)?.size ?? 0) > o && !e.has(t) && r.push(t), (s.get(t)?.size ?? 0) > o && (n = !0);
|
|
323
374
|
}
|
|
324
375
|
for (let t of r) e.add(t);
|
|
325
376
|
if (yield* m, !n) break;
|
|
326
377
|
}
|
|
327
378
|
}
|
|
328
379
|
return o;
|
|
329
|
-
}),
|
|
380
|
+
}), et = (e, t) => ({
|
|
330
381
|
tables: [...e.keys()],
|
|
331
382
|
predicateFor: (n) => {
|
|
332
383
|
let r = e.get(n);
|
|
333
|
-
return r ?
|
|
384
|
+
return r ? y(G(t, n), [...r]) : void 0;
|
|
334
385
|
}
|
|
335
|
-
}),
|
|
386
|
+
}), tt = "tenantId", nt = (e, t, r) => n.gen(function* () {
|
|
336
387
|
let i = t.tables.map((e) => e.name), a = new Map(t.tables.map((e) => [e.name, e]));
|
|
337
|
-
if (e.kind === "all")
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
388
|
+
if (e.kind === "all") {
|
|
389
|
+
let t = new Set(e.exclude ?? []);
|
|
390
|
+
return {
|
|
391
|
+
tables: i.filter((e) => !V(e) && !t.has(e)),
|
|
392
|
+
predicateFor: () => void 0
|
|
393
|
+
};
|
|
394
|
+
}
|
|
341
395
|
if (e.kind === "tables") {
|
|
342
396
|
let t = e.tables ?? [], r = new Set(i), a = t.filter((e) => !r.has(e));
|
|
343
|
-
if (a.length > 0) return yield* n.fail(new
|
|
397
|
+
if (a.length > 0) return yield* n.fail(new U({ reason: `scope { kind: 'tables' } names ${a.length} table(s) that are not in the schema: ${a.join(", ")}. Refusing to write a bundle that silently omits them. Known tables: ${i.length}.` }));
|
|
344
398
|
let o = new Set(t);
|
|
345
399
|
return {
|
|
346
400
|
tables: i.filter((e) => o.has(e)),
|
|
@@ -348,30 +402,30 @@ var ye = (e) => {
|
|
|
348
402
|
};
|
|
349
403
|
}
|
|
350
404
|
let o = e.tenantId;
|
|
351
|
-
if (o === void 0 || o === "") return yield* n.fail(new
|
|
352
|
-
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name ===
|
|
353
|
-
if (s.length === 0) return yield* n.fail(new
|
|
354
|
-
let c = yield*
|
|
355
|
-
seeds: Object.fromEntries(s.map((e) => [e,
|
|
405
|
+
if (o === void 0 || o === "") return yield* n.fail(new U({ reason: "scope { kind: 'tenant' } requires a tenantId — pass --tenant <id> (CLI) or scope.tenantId (API). Refusing to fall back to an unfiltered export." }));
|
|
406
|
+
let s = r.tenantTables ? r.tenantTables.filter((e) => a.has(e)) : i.filter((e) => a.get(e)?.columns.some((e) => e.name === tt) ?? !1);
|
|
407
|
+
if (s.length === 0) return yield* n.fail(new U({ reason: r.tenantTables ? "tenant scope: none of the provided tenantTables exist in the schema — nothing to filter." : `tenant scope: no table carries a '${tt}' column — nothing to filter. A tenant export of a schema without tenant-scoped tables would be the full database; refusing.` }));
|
|
408
|
+
let c = yield* $e(r.store, t, {
|
|
409
|
+
seeds: Object.fromEntries(s.map((e) => [e, g(tt, o)])),
|
|
356
410
|
pinned: s,
|
|
357
411
|
children: { roots: s }
|
|
358
412
|
}), l = [...c.entries()].filter(([, e]) => e.size > 0).map(([e]) => e), u = new Set(s);
|
|
359
413
|
return {
|
|
360
414
|
tables: [...s, ...l],
|
|
361
|
-
predicateFor: (e) => u.has(e) ?
|
|
415
|
+
predicateFor: (e) => u.has(e) ? g(tt, o) : c.has(e) ? y(G(t, e), [...c.get(e)]) : void 0
|
|
362
416
|
};
|
|
363
|
-
}),
|
|
417
|
+
}), rt = (e, t) => ({
|
|
364
418
|
list: t,
|
|
365
|
-
open: (t) =>
|
|
366
|
-
}),
|
|
367
|
-
write: (t, n, r) =>
|
|
419
|
+
open: (t) => ue(e, t)
|
|
420
|
+
}), it = (e) => ({
|
|
421
|
+
write: (t, n, r) => D(e, t, n, r),
|
|
368
422
|
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
369
|
-
}),
|
|
423
|
+
}), at = (e, t) => n.tryPromise({
|
|
370
424
|
try: async () => {
|
|
371
425
|
let { createWriteStream: n } = await import("node:fs"), r = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = await import("node:path"), l = c.join(t, `.tmp-${a()}`), u = i("sha256"), d = 0, f = new s({ transform(e, t, n) {
|
|
372
426
|
u.update(e), d += e.length, n(null, e);
|
|
373
427
|
} });
|
|
374
|
-
await o(await
|
|
428
|
+
await o(await O(e.stream), f, n(l));
|
|
375
429
|
let p = u.digest("hex"), m = c.join(t, p);
|
|
376
430
|
try {
|
|
377
431
|
await r.access(m), await r.rm(l, { force: !0 });
|
|
@@ -384,23 +438,23 @@ var ye = (e) => {
|
|
|
384
438
|
contentType: e.contentType
|
|
385
439
|
};
|
|
386
440
|
},
|
|
387
|
-
catch: (e) => new
|
|
441
|
+
catch: (e) => new H({
|
|
388
442
|
reason: `asset write failed: ${String(e?.message ?? e)}`,
|
|
389
443
|
path: t
|
|
390
444
|
})
|
|
391
|
-
}),
|
|
392
|
-
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u =
|
|
445
|
+
}), ot = (e, t, r) => n.gen(function* () {
|
|
446
|
+
let i = yield* n.promise(() => import("node:path")), { createReadStream: a } = yield* n.promise(() => import("node:fs")), { createHash: o } = yield* n.promise(() => import("node:crypto")), s = i.join(t, r.sha256), c = o("sha256"), u = le("bundle", () => a(s)).pipe(l.tap((e) => n.sync(() => c.update(e))));
|
|
393
447
|
yield* e.write(r.key, u, {
|
|
394
448
|
contentType: r.contentType,
|
|
395
449
|
totalSize: r.size
|
|
396
450
|
});
|
|
397
451
|
let d = c.digest("hex");
|
|
398
|
-
if (d !== r.sha256) return yield* n.fail(new
|
|
452
|
+
if (d !== r.sha256) return yield* n.fail(new Ge({
|
|
399
453
|
what: `asset ${r.key}`,
|
|
400
454
|
expected: r.sha256,
|
|
401
455
|
actual: d
|
|
402
456
|
}));
|
|
403
|
-
}),
|
|
457
|
+
}), st = () => typeof k.createZstdCompress == "function", ct = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : st() ? "zstd" : "gzip", lt = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", ut = (e) => e === "zstd" ? k.createZstdCompress() : e === "gzip" ? k.createGzip() : new A(), dt = (e) => e === "zstd" ? k.createZstdDecompress() : e === "gzip" ? k.createGunzip() : new A(), ft = 1, pt = c.Struct({
|
|
404
458
|
name: c.String,
|
|
405
459
|
file: c.String,
|
|
406
460
|
rowCount: c.Number,
|
|
@@ -410,11 +464,12 @@ var ye = (e) => {
|
|
|
410
464
|
value: c.String
|
|
411
465
|
}),
|
|
412
466
|
primaryKey: c.String
|
|
413
|
-
}),
|
|
467
|
+
}), mt = c.Struct({
|
|
414
468
|
kind: c.Literal("all", "tenant", "tables"),
|
|
415
469
|
tenantId: c.optional(c.String),
|
|
416
|
-
tables: c.optional(c.Array(c.String))
|
|
417
|
-
|
|
470
|
+
tables: c.optional(c.Array(c.String)),
|
|
471
|
+
exclude: c.optional(c.Array(c.String))
|
|
472
|
+
}), ht = c.Struct({
|
|
418
473
|
count: c.Number,
|
|
419
474
|
totalBytes: c.Number,
|
|
420
475
|
index: c.optional(c.String)
|
|
@@ -425,23 +480,23 @@ c.Struct({
|
|
|
425
480
|
size: c.Number,
|
|
426
481
|
contentType: c.String
|
|
427
482
|
});
|
|
428
|
-
var
|
|
483
|
+
var gt = c.Struct({
|
|
429
484
|
formatVersion: c.Number,
|
|
430
485
|
createdAt: c.String,
|
|
431
486
|
source: c.Struct({
|
|
432
487
|
dialect: c.String,
|
|
433
488
|
schemaFingerprint: c.String
|
|
434
489
|
}),
|
|
435
|
-
scope:
|
|
490
|
+
scope: mt,
|
|
436
491
|
consistency: c.Literal("live", "snapshot"),
|
|
437
492
|
compression: c.Literal("zstd", "gzip", "none"),
|
|
438
|
-
tables: c.Array(
|
|
439
|
-
assets:
|
|
493
|
+
tables: c.Array(pt),
|
|
494
|
+
assets: ht,
|
|
440
495
|
masking: c.optional(c.Struct({
|
|
441
496
|
policyId: c.optional(c.String),
|
|
442
497
|
transformed: c.Array(c.String)
|
|
443
498
|
}))
|
|
444
|
-
}),
|
|
499
|
+
}), _t = c.encodeSync(c.parseJson(gt)), vt = c.decodeUnknownSync(c.parseJson(gt)), K = "manifest.json", yt = "README.md", bt = "data", xt = "assets", St = "assets/index.ndjson", Ct = ".ledger.json", wt = (e) => e.kind === "tenant" ? `tenant \`${e.tenantId}\`` : e.kind === "tables" ? `tables \`${(e.tables ?? []).join(", ")}\`` : "all tables", Tt = (e, t) => {
|
|
445
500
|
let n = e.tables.reduce((e, t) => e + t.rowCount, 0), r = e.masking ? `✅ **Masked at the source** — ${e.masking.transformed.length} column(s) transformed${e.masking.policyId ? ` (policy \`${e.masking.policyId}\`)` : ""}. Personal data was replaced with realistic, referentially-consistent fakes before any byte was written.` : "⚠️ **NOT masked — this bundle contains REAL data (PII included).** Treat it as production data: do not copy it into a lower environment or share it without review. To produce a safe copy, re-export under a masking profile (`--profile`).", i = e.masking && e.masking.transformed.length > 0 ? `\n\nMasked columns: ${e.masking.transformed.map((e) => `\`${e}\``).join(", ")}.` : "", a = e.assets.count > 0 ? `${e.assets.count} object(s), ${e.assets.totalBytes} bytes` : "none", o = e.tables.map((e) => `| \`${e.name}\` | ${e.rowCount} |`).join("\n");
|
|
446
501
|
return `# Voltro data export
|
|
447
502
|
|
|
@@ -455,7 +510,7 @@ The machine-readable manifest is \`manifest.json\`; this file is a human summary
|
|
|
455
510
|
| **Format version** | ${e.formatVersion} |
|
|
456
511
|
| **Consistency** | ${e.consistency} |
|
|
457
512
|
| **Compression** | ${e.compression} |
|
|
458
|
-
| **Scope** | ${
|
|
513
|
+
| **Scope** | ${wt(e.scope)} |
|
|
459
514
|
| **Tables** | ${e.tables.length} |
|
|
460
515
|
| **Rows (total)** | ${n} |
|
|
461
516
|
| **Assets** | ${a} |
|
|
@@ -480,39 +535,39 @@ voltro data import <this-directory>
|
|
|
480
535
|
voltro data import <this-directory> --target api --api-url <url> --token <secret>
|
|
481
536
|
\`\`\`
|
|
482
537
|
`;
|
|
483
|
-
},
|
|
538
|
+
}, Et = i.counter("voltro_data_transfer_rows", { description: "Rows exported/imported by @voltro/data-transfer, tagged by phase + table." }), Dt = a.exponential({
|
|
484
539
|
start: .01,
|
|
485
540
|
factor: 2,
|
|
486
541
|
count: 16
|
|
487
|
-
}),
|
|
542
|
+
}), Ot = i.histogram("voltro_data_transfer_table_seconds", Dt, "Per-table export/import duration in seconds, tagged by phase."), q = (e, t) => {
|
|
488
543
|
if (e) try {
|
|
489
544
|
e(t);
|
|
490
545
|
} catch {}
|
|
491
|
-
},
|
|
546
|
+
}, kt = (e, t, r, a, o, s) => n.Do.pipe(n.bind("start", () => n.clockWith((e) => e.currentTimeMillis)), n.bind("result", () => s), n.tap(({ start: r, result: a }) => n.clockWith((e) => e.currentTimeMillis).pipe(n.flatMap((s) => n.zipRight(Ot.pipe(i.tagged("phase", e), i.update((s - r) / 1e3)), Et.pipe(i.tagged("phase", e), i.tagged("table", t), i.update(o(a))))))), n.map(({ result: e }) => e), n.withSpan(`data-transfer.${e}.table`, { attributes: {
|
|
492
547
|
table: t,
|
|
493
548
|
index: r,
|
|
494
549
|
tableCount: a,
|
|
495
550
|
phase: e
|
|
496
|
-
} })),
|
|
551
|
+
} })), At = () => ({
|
|
497
552
|
tables: {},
|
|
498
553
|
assets: {}
|
|
499
|
-
}),
|
|
554
|
+
}), jt = (e) => n.tryPromise(async () => {
|
|
500
555
|
let t = await (await import("node:fs/promises")).readFile(e, "utf8");
|
|
501
556
|
return JSON.parse(t);
|
|
502
|
-
}).pipe(n.orElseSucceed(
|
|
557
|
+
}).pipe(n.orElseSucceed(At)), J = (e, t) => n.tryPromise({
|
|
503
558
|
try: async () => {
|
|
504
559
|
let n = await import("node:fs/promises"), { randomUUID: r } = await import("node:crypto"), i = `${e}.tmp-${r()}`;
|
|
505
560
|
await n.writeFile(i, JSON.stringify(t)), await n.rename(i, e);
|
|
506
561
|
},
|
|
507
|
-
catch: (t) => new
|
|
562
|
+
catch: (t) => new H({
|
|
508
563
|
reason: `ledger write failed: ${String(t?.message ?? t)}`,
|
|
509
564
|
path: e
|
|
510
565
|
})
|
|
511
|
-
}),
|
|
566
|
+
}), Mt = (e, t) => t in e.tables, Nt = (e, t) => t in e.assets, Pt = /* @__PURE__ */ new Set([
|
|
512
567
|
"json",
|
|
513
568
|
"array",
|
|
514
569
|
"vector"
|
|
515
|
-
]),
|
|
570
|
+
]), Ft = (e) => typeof e == "object" && !!e && !Array.isArray(e) && typeof e.$v == "string", It = (e) => Buffer.from(e).toString("base64"), Lt = (e) => new Uint8Array(Buffer.from(e, "base64")), Rt = (e) => e instanceof Date ? e.toISOString() : typeof e == "number" ? new Date(e).toISOString() : String(e), zt = (e, t) => e == null ? null : t !== void 0 && Pt.has(t) ? e : t === "bytes" ? It(e) : t === "timestamp" || t === "date" ? Rt(e) : typeof e == "bigint" ? {
|
|
516
571
|
$v: "bigint",
|
|
517
572
|
d: e.toString()
|
|
518
573
|
} : typeof e == "number" && !Number.isFinite(e) ? {
|
|
@@ -523,30 +578,30 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
523
578
|
d: e.toISOString()
|
|
524
579
|
} : e instanceof Uint8Array ? {
|
|
525
580
|
$v: "bytes",
|
|
526
|
-
d:
|
|
527
|
-
} : e,
|
|
581
|
+
d: It(e)
|
|
582
|
+
} : e, Bt = (e) => {
|
|
528
583
|
switch (e.$v) {
|
|
529
584
|
case "bigint": return BigInt(e.d);
|
|
530
585
|
case "number": return Number(e.d);
|
|
531
586
|
case "date": return new Date(e.d);
|
|
532
|
-
case "bytes": return
|
|
587
|
+
case "bytes": return Lt(e.d);
|
|
533
588
|
default: return e;
|
|
534
589
|
}
|
|
535
|
-
},
|
|
590
|
+
}, Vt = (e, t) => e == null ? null : t !== void 0 && Pt.has(t) ? e : Ft(e) ? Bt(e) : t === "bytes" ? Lt(e) : t === "timestamp" || t === "date" ? new Date(e) : e, Ht = (e, t) => {
|
|
536
591
|
let n = {};
|
|
537
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
592
|
+
for (let [r, i] of Object.entries(e)) n[r] = zt(i, t[r]);
|
|
538
593
|
return n;
|
|
539
|
-
},
|
|
594
|
+
}, Ut = (e, t) => {
|
|
540
595
|
let n = {};
|
|
541
|
-
for (let [r, i] of Object.entries(e)) n[r] =
|
|
596
|
+
for (let [r, i] of Object.entries(e)) n[r] = Vt(i, t[r]);
|
|
542
597
|
return n;
|
|
543
|
-
},
|
|
544
|
-
try: () =>
|
|
545
|
-
catch: (e) => new
|
|
598
|
+
}, Wt = new TextEncoder(), Gt = (e, t) => e.pipe(l.map((e) => Wt.encode(`${JSON.stringify(Ht(e, t))}\n`))), Kt = (e, t, r) => e.pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.mapEffect((e) => n.try({
|
|
599
|
+
try: () => Ut(JSON.parse(e), r),
|
|
600
|
+
catch: (e) => new We({
|
|
546
601
|
table: t,
|
|
547
602
|
reason: String(e?.message ?? e)
|
|
548
603
|
})
|
|
549
|
-
}))),
|
|
604
|
+
}))), qt = (e, t) => {
|
|
550
605
|
let n = new Set(e), r = /* @__PURE__ */ new Map();
|
|
551
606
|
for (let e of t.tables) {
|
|
552
607
|
if (!n.has(e.name)) continue;
|
|
@@ -566,39 +621,39 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
566
621
|
};
|
|
567
622
|
for (let t of e) n.has(t) && s(t);
|
|
568
623
|
return i;
|
|
569
|
-
},
|
|
624
|
+
}, Y = (e) => {
|
|
570
625
|
let t = e, n = [];
|
|
571
626
|
typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
|
|
572
627
|
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
573
|
-
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(
|
|
628
|
+
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(Y(t.cause)) : n.push(r), n.length === 0) {
|
|
574
629
|
let t = String(e ?? "").trim();
|
|
575
|
-
return t === "" || t === "[object Object]" ?
|
|
630
|
+
return t === "" || t === "[object Object]" ? Jt(e) : t;
|
|
576
631
|
}
|
|
577
632
|
return n.join(": ");
|
|
578
|
-
},
|
|
633
|
+
}, Jt = (e) => {
|
|
579
634
|
try {
|
|
580
635
|
let t = JSON.stringify(e);
|
|
581
636
|
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
582
637
|
} catch {
|
|
583
638
|
return "unknown error (not serialisable)";
|
|
584
639
|
}
|
|
585
|
-
},
|
|
640
|
+
}, Yt = (e, t) => {
|
|
586
641
|
let n = e.tables.find((e) => e.name === t), r = {};
|
|
587
642
|
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
588
643
|
return r;
|
|
589
|
-
},
|
|
644
|
+
}, Xt = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), Zt = (e, t) => {
|
|
590
645
|
if (t.length === 0) return e;
|
|
591
646
|
let n = { ...e };
|
|
592
647
|
for (let e of t) delete n[e];
|
|
593
648
|
return n;
|
|
594
|
-
},
|
|
649
|
+
}, Qt = (e, t) => {
|
|
595
650
|
let n = e.tables.find((e) => e.name === t);
|
|
596
651
|
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
597
652
|
let r = n.columns.find((e) => e.type === "id");
|
|
598
653
|
if (r !== void 0) return { column: r.name };
|
|
599
654
|
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
600
655
|
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
601
|
-
},
|
|
656
|
+
}, $t = (e, t, r) => n.tryPromise({
|
|
602
657
|
try: async () => {
|
|
603
658
|
let { createWriteStream: n } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: u } = await import("node:stream"), d = a("sha256"), f = 0, p = new u({ transform(e, t, n) {
|
|
604
659
|
d.update(e);
|
|
@@ -607,7 +662,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
607
662
|
n(null, e);
|
|
608
663
|
} }), m = c.fromWeb(l.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
609
664
|
try {
|
|
610
|
-
await s(m, p,
|
|
665
|
+
await s(m, p, ut(r), n(h)), await i.rename(h, t);
|
|
611
666
|
} catch (e) {
|
|
612
667
|
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
613
668
|
}
|
|
@@ -616,27 +671,27 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
616
671
|
checksum: d.digest("hex")
|
|
617
672
|
};
|
|
618
673
|
},
|
|
619
|
-
catch: (e) => new
|
|
620
|
-
reason: `write table failed: ${
|
|
674
|
+
catch: (e) => new H({
|
|
675
|
+
reason: `write table failed: ${Y(e)}`,
|
|
621
676
|
path: t
|
|
622
677
|
})
|
|
623
|
-
}),
|
|
678
|
+
}), en = (e, t) => n.tryPromise({
|
|
624
679
|
try: async () => {
|
|
625
680
|
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
626
681
|
},
|
|
627
|
-
catch: (t) => new
|
|
628
|
-
reason: `append failed: ${
|
|
682
|
+
catch: (t) => new H({
|
|
683
|
+
reason: `append failed: ${Y(t)}`,
|
|
629
684
|
path: e
|
|
630
685
|
})
|
|
631
|
-
}),
|
|
686
|
+
}), tn = (e, t) => n.gen(function* () {
|
|
632
687
|
let r = [], i = t.ordered.length;
|
|
633
688
|
for (let a = 0; a < t.ordered.length; a++) {
|
|
634
|
-
let o = t.ordered[a], s =
|
|
635
|
-
if ("refusal" in c) return yield* n.fail(new
|
|
636
|
-
let u = c.column, d = `${
|
|
637
|
-
if (
|
|
689
|
+
let o = t.ordered[a], s = Yt(t.snapshot, o), c = Qt(t.snapshot, o);
|
|
690
|
+
if ("refusal" in c) return yield* n.fail(new U({ reason: `export: ${c.refusal}` }));
|
|
691
|
+
let u = c.column, d = `${bt}/${o}.ndjson${lt(t.compression)}`;
|
|
692
|
+
if (Mt(t.ledger, o)) {
|
|
638
693
|
let e = t.ledger.tables[o];
|
|
639
|
-
|
|
694
|
+
q(t.onProgress, {
|
|
640
695
|
phase: "export",
|
|
641
696
|
table: o,
|
|
642
697
|
index: a,
|
|
@@ -654,7 +709,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
654
709
|
});
|
|
655
710
|
continue;
|
|
656
711
|
}
|
|
657
|
-
|
|
712
|
+
q(t.onProgress, {
|
|
658
713
|
phase: "export",
|
|
659
714
|
table: o,
|
|
660
715
|
index: a,
|
|
@@ -662,17 +717,17 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
662
717
|
event: "start",
|
|
663
718
|
rowsDone: 0
|
|
664
719
|
});
|
|
665
|
-
let f = t.resolved.predicateFor(o), p =
|
|
720
|
+
let f = t.resolved.predicateFor(o), p = w(e, {
|
|
666
721
|
table: o,
|
|
667
722
|
pk: u,
|
|
668
723
|
chunkSize: t.chunkSize,
|
|
669
724
|
retry: t.retry,
|
|
670
725
|
...f === void 0 ? {} : { where: f }
|
|
671
|
-
}), m =
|
|
726
|
+
}), m = Xt(t.snapshot, o), h = m.length === 0 ? p : p.pipe(l.map((e) => Zt(e, m))), g = t.mask ? h.pipe(l.map((e) => ze(e, o, t.mask.actions, t.mask.seed, s))) : h, { rowCount: _, checksum: v } = yield* kt("export", o, a, i, (e) => e.rowCount, $t(Gt(g, s), t.bundleFile(d), t.compression));
|
|
672
727
|
t.ledger.tables[o] = {
|
|
673
728
|
rowCount: _,
|
|
674
729
|
checksum: v
|
|
675
|
-
}, yield*
|
|
730
|
+
}, yield* J(t.ledgerPath, t.ledger), q(t.onProgress, {
|
|
676
731
|
phase: "export",
|
|
677
732
|
table: o,
|
|
678
733
|
index: a,
|
|
@@ -690,43 +745,43 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
690
745
|
});
|
|
691
746
|
}
|
|
692
747
|
return r;
|
|
693
|
-
}),
|
|
748
|
+
}), nn = (e) => ({
|
|
694
749
|
_tag: "RawSqlFragment",
|
|
695
750
|
strings: [e],
|
|
696
751
|
values: []
|
|
697
|
-
}),
|
|
752
|
+
}), rn = (t, i, a) => n.tryPromise({
|
|
698
753
|
try: () => t.transactional(async (t) => {
|
|
699
|
-
if (t.raw) for (let e of
|
|
754
|
+
if (t.raw) for (let e of xe(i)) await t.raw(nn(e));
|
|
700
755
|
let o = await n.runPromiseExit(a(t));
|
|
701
756
|
if (r.isFailure(o)) throw Error(e.pretty(o.cause));
|
|
702
757
|
return o.value;
|
|
703
758
|
}),
|
|
704
|
-
catch: (e) => e instanceof
|
|
705
|
-
}),
|
|
706
|
-
let t = e.scope ?? { kind: "all" }, r =
|
|
759
|
+
catch: (e) => e instanceof H ? e : new H({ reason: `snapshot export failed: ${Y(e)}` })
|
|
760
|
+
}), an = (e) => n.gen(function* () {
|
|
761
|
+
let t = e.scope ?? { kind: "all" }, r = ct(e.compression), i = e.chunkSize ?? 1e3, a = e.retryTimes ?? 5, o = s.exponential("50 millis").pipe(s.jittered, s.intersect(s.recurs(a))), c = yield* n.promise(() => import("node:path")), u = (t) => c.join(e.outDir, t);
|
|
707
762
|
yield* n.tryPromise({
|
|
708
763
|
try: async () => {
|
|
709
764
|
let t = await import("node:fs/promises");
|
|
710
|
-
await t.mkdir(u(
|
|
765
|
+
await t.mkdir(u(bt), { recursive: !0 }), e.assets && await t.mkdir(u(xt), { recursive: !0 });
|
|
711
766
|
},
|
|
712
|
-
catch: (t) => new
|
|
713
|
-
reason: `mkdir failed: ${
|
|
767
|
+
catch: (t) => new H({
|
|
768
|
+
reason: `mkdir failed: ${Y(t)}`,
|
|
714
769
|
path: e.outDir
|
|
715
770
|
})
|
|
716
771
|
});
|
|
717
|
-
let d = u(
|
|
772
|
+
let d = u(Ct), f = yield* jt(d), p = e.subset ? et(yield* $e(e.store, e.snapshot, e.subset), e.snapshot) : yield* nt(t, e.snapshot, {
|
|
718
773
|
store: e.store,
|
|
719
774
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
720
|
-
}), m =
|
|
775
|
+
}), m = qt(p.tables, e.snapshot), h = e.consistency ?? "live", g, _;
|
|
721
776
|
if (e.masking) {
|
|
722
|
-
let t = e.classification ??
|
|
777
|
+
let t = e.classification ?? Ce(e.snapshot), r = Ie(m.map((t) => ({
|
|
723
778
|
name: t,
|
|
724
779
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
725
780
|
name: e.name,
|
|
726
781
|
type: e.type
|
|
727
782
|
}))
|
|
728
783
|
})), t, e.masking);
|
|
729
|
-
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new
|
|
784
|
+
if (r.unclassified.length > 0 || r.invalid.length > 0) return yield* n.fail(new Ye({
|
|
730
785
|
unclassified: r.unclassified,
|
|
731
786
|
...r.invalid.length > 0 ? { invalidActions: r.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
732
787
|
}));
|
|
@@ -752,25 +807,25 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
752
807
|
bundleFile: u,
|
|
753
808
|
..._ ? { mask: _ } : {},
|
|
754
809
|
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
755
|
-
}, y = h === "snapshot" ? yield*
|
|
810
|
+
}, y = h === "snapshot" ? yield* rn(e.store, e.dialect, (e) => tn(e, v)) : yield* tn(e.store, v), b = 0, x = 0;
|
|
756
811
|
if (e.assets) {
|
|
757
|
-
let t = u(
|
|
758
|
-
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) =>
|
|
812
|
+
let t = u(St);
|
|
813
|
+
yield* l.runForEach(e.assets.list(), (r) => r.key in f.assets ? n.void : e.assets.open(r.key).pipe(n.flatMap((e) => at(e, u(xt))), n.flatMap((e) => {
|
|
759
814
|
let i = {
|
|
760
815
|
key: r.key,
|
|
761
816
|
sha256: e.sha256,
|
|
762
817
|
size: e.size,
|
|
763
818
|
contentType: e.contentType
|
|
764
819
|
};
|
|
765
|
-
return b++, x += e.size, f.assets[r.key] = !0,
|
|
766
|
-
}))).pipe(n.mapError((e) => e instanceof
|
|
820
|
+
return b++, x += e.size, f.assets[r.key] = !0, en(t, JSON.stringify(i)).pipe(n.andThen(J(d, f)));
|
|
821
|
+
}))).pipe(n.mapError((e) => e instanceof H ? e : new H({ reason: `asset export failed: ${String(e)}` })));
|
|
767
822
|
}
|
|
768
823
|
let S = {
|
|
769
824
|
formatVersion: 1,
|
|
770
825
|
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
771
826
|
source: {
|
|
772
827
|
dialect: e.dialect,
|
|
773
|
-
schemaFingerprint:
|
|
828
|
+
schemaFingerprint: re(e.snapshot)
|
|
774
829
|
},
|
|
775
830
|
scope: t,
|
|
776
831
|
consistency: h,
|
|
@@ -779,7 +834,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
779
834
|
assets: b > 0 ? {
|
|
780
835
|
count: b,
|
|
781
836
|
totalBytes: x,
|
|
782
|
-
index:
|
|
837
|
+
index: St
|
|
783
838
|
} : {
|
|
784
839
|
count: 0,
|
|
785
840
|
totalBytes: 0
|
|
@@ -788,23 +843,23 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
788
843
|
};
|
|
789
844
|
return yield* n.tryPromise({
|
|
790
845
|
try: async () => {
|
|
791
|
-
await (await import("node:fs/promises")).writeFile(u(
|
|
846
|
+
await (await import("node:fs/promises")).writeFile(u(K), _t(S));
|
|
792
847
|
},
|
|
793
|
-
catch: (e) => new
|
|
794
|
-
reason: `manifest write failed: ${
|
|
795
|
-
path:
|
|
848
|
+
catch: (e) => new H({
|
|
849
|
+
reason: `manifest write failed: ${Y(e)}`,
|
|
850
|
+
path: K
|
|
796
851
|
})
|
|
797
852
|
}), yield* n.tryPromise({
|
|
798
853
|
try: async () => {
|
|
799
854
|
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
800
|
-
await e.writeFile(u(
|
|
855
|
+
await e.writeFile(u(yt), Tt(S, { hostname: t.hostname() }));
|
|
801
856
|
},
|
|
802
|
-
catch: (e) => new
|
|
803
|
-
reason: `readme write failed: ${
|
|
804
|
-
path:
|
|
857
|
+
catch: (e) => new H({
|
|
858
|
+
reason: `readme write failed: ${Y(e)}`,
|
|
859
|
+
path: yt
|
|
805
860
|
})
|
|
806
861
|
}), S;
|
|
807
|
-
}),
|
|
862
|
+
}), on = (e, t) => {
|
|
808
863
|
let n = new Set(t), r = [];
|
|
809
864
|
for (let t of e.tables) if (!n.has(t.name)) for (let e of t.columns) {
|
|
810
865
|
let i = e.references?.table;
|
|
@@ -815,7 +870,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
815
870
|
});
|
|
816
871
|
}
|
|
817
872
|
return r;
|
|
818
|
-
},
|
|
873
|
+
}, sn = (e) => {
|
|
819
874
|
let t = /* @__PURE__ */ new Map();
|
|
820
875
|
for (let n of e) t.set(n.from, [...t.get(n.from) ?? [], n]);
|
|
821
876
|
let n = [...t.entries()].map(([e, t]) => {
|
|
@@ -823,7 +878,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
823
878
|
return ` ${e} (${n} row${n === 1 ? "" : "s"}): ${r}`;
|
|
824
879
|
});
|
|
825
880
|
return `replace cannot empty this target: ${t.size} table(s) OUTSIDE the bundle hold rows that reference tables INSIDE it. Emptying the bundle's tables would leave those rows pointing at nothing, and the bundle cannot restore them.\n${n.join("\n")}\nNothing has been deleted. Either re-export with a scope that includes those tables, or use --mode upsert if those rows are meant to survive.`;
|
|
826
|
-
},
|
|
881
|
+
}, cn = (e) => {
|
|
827
882
|
switch (e) {
|
|
828
883
|
case "id":
|
|
829
884
|
case "text":
|
|
@@ -842,12 +897,12 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
842
897
|
case "raw": return "raw";
|
|
843
898
|
default: return "string";
|
|
844
899
|
}
|
|
845
|
-
},
|
|
900
|
+
}, ln = {
|
|
846
901
|
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
847
902
|
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
848
903
|
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
849
|
-
},
|
|
850
|
-
let i =
|
|
904
|
+
}, un = (e, t, n, r) => {
|
|
905
|
+
let i = cn(n);
|
|
851
906
|
if (i === "raw") return {
|
|
852
907
|
table: e,
|
|
853
908
|
column: t,
|
|
@@ -855,7 +910,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
855
910
|
severity: "block",
|
|
856
911
|
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${r}`
|
|
857
912
|
};
|
|
858
|
-
let a =
|
|
913
|
+
let a = ln[i];
|
|
859
914
|
return !a || a.has(r) ? null : i === "array" || i === "interval" ? {
|
|
860
915
|
table: e,
|
|
861
916
|
column: t,
|
|
@@ -869,15 +924,15 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
869
924
|
severity: "block",
|
|
870
925
|
reason: `${i} columns require a ${[...a].join("/")} target; ${r} has no compatible column type`
|
|
871
926
|
};
|
|
872
|
-
},
|
|
927
|
+
}, dn = (e, t, n) => {
|
|
873
928
|
if (t === n) return [];
|
|
874
929
|
let r = [];
|
|
875
930
|
for (let t of e) for (let [e, i] of Object.entries(t.columnTypes)) {
|
|
876
|
-
let a =
|
|
931
|
+
let a = un(t.name, e, i, n);
|
|
877
932
|
a && r.push(a);
|
|
878
933
|
}
|
|
879
934
|
return r;
|
|
880
|
-
},
|
|
935
|
+
}, fn = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`, pn = (e) => {
|
|
881
936
|
let t = e.constraint ?? e.column, n = t === void 0 ? "" : ` ${t}`;
|
|
882
937
|
switch (e.kind) {
|
|
883
938
|
case "foreignKey": return `foreign key${n}: the referenced row does not exist (import it first, or check the bundle's table order)`;
|
|
@@ -886,36 +941,36 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
886
941
|
case "notNull": return `not-null constraint${n}: the column requires a value`;
|
|
887
942
|
case "check": return `check constraint${n}: the row does not satisfy it`;
|
|
888
943
|
}
|
|
889
|
-
},
|
|
944
|
+
}, mn = (e) => {
|
|
890
945
|
let t = [e.dbCode, e.dbErrno ?? e.dbNumber].filter((e) => e !== void 0).map(String);
|
|
891
946
|
return t.length > 0 ? t.join("/") : void 0;
|
|
892
|
-
},
|
|
947
|
+
}, hn = "Failed to execute statement", gn = (e) => {
|
|
893
948
|
let t = (e.split("\n", 1)[0] ?? "").trim();
|
|
894
949
|
for (; t.startsWith("(FiberFailure)");) t = t.slice(14).trim();
|
|
895
|
-
return t === `SqlError: ${
|
|
896
|
-
},
|
|
950
|
+
return t === `SqlError: ${hn}` ? hn : t;
|
|
951
|
+
}, _n = (e) => {
|
|
897
952
|
let t = String(e?.stack ?? e), n = t.length <= 1200 ? t : `${t.slice(0, 1200)}…`, r = (e, t) => ({
|
|
898
953
|
reason: e.length <= 300 ? e : `${e.slice(0, 300)}…`,
|
|
899
954
|
unclassified: t,
|
|
900
955
|
raw: n
|
|
901
|
-
}), i =
|
|
902
|
-
if (a !== void 0) return r(`${
|
|
903
|
-
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u =
|
|
904
|
-
return r(
|
|
905
|
-
},
|
|
906
|
-
let t =
|
|
956
|
+
}), i = _(e), a = p(i), o = mn(i), s = o === void 0 ? "" : ` [${o}]`;
|
|
957
|
+
if (a !== void 0) return r(`${pn(a)}${s}`, !1);
|
|
958
|
+
let c = typeof i.dbMessage == "string" ? i.dbMessage : void 0, l = e?.message, u = gn(typeof l == "string" && l.length > 0 ? l : String(e)), d = c !== void 0 && c !== u && !c.startsWith("SqlError:") ? c : u, f = typeof i.dbCauseChain == "string" ? i.dbCauseChain : void 0, m = f !== void 0 && /[←‖]/.test(f) ? ` (via ${f})` : "";
|
|
959
|
+
return r(d === hn || d.length === 0 ? `the database refused the write and the driver gave no detail${s}${m}` : `${d}${s}`, d === hn || d.length === 0);
|
|
960
|
+
}, vn = (e) => _n(e).reason, yn = (e) => {
|
|
961
|
+
let t = _(e), n = p(t);
|
|
907
962
|
if (n?.kind === "foreignKeyInUse") {
|
|
908
|
-
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i =
|
|
963
|
+
let e = n.constraint ?? n.column, r = e === void 0 ? "" : ` ${e}`, i = mn(t);
|
|
909
964
|
return `foreign key${r}: rows elsewhere still reference this table${i === void 0 ? "" : ` [${i}]`}`;
|
|
910
965
|
}
|
|
911
|
-
return
|
|
912
|
-
},
|
|
966
|
+
return vn(e);
|
|
967
|
+
}, bn = "_voltro_staging_", X = (e) => `${bn}${e}`, xn = (e) => typeof e.run == "function", Sn = (e) => `staging is not available for this run — loading directly into the target instead: ${e}\n The load therefore holds the destructive transaction open for its whole duration, so a
|
|
913
968
|
process that dies mid-load depends on the database rolling it back. The rollback capture
|
|
914
|
-
written before the first delete is what covers that case.`,
|
|
915
|
-
if (!
|
|
969
|
+
written before the first delete is what covers that case.`, Cn = /^[a-zA-Z_][a-zA-Z0-9_]*$/, Z = (e, t) => {
|
|
970
|
+
if (!Cn.test(e)) throw Error(`refusing to build SQL for the table name ${JSON.stringify(e)}: it is outside the identifier class the framework allows. A name from a bundle manifest reaches this concatenation, so it is checked here rather than assumed to still be what the declaration validated.`);
|
|
916
971
|
return t === "mysql" || t === "mariadb" ? `\`${e}\`` : t === "mssql" ? `[${e}]` : `"${e}"`;
|
|
917
|
-
},
|
|
918
|
-
let n =
|
|
972
|
+
}, wn = (e, t) => {
|
|
973
|
+
let n = Z(e, t), r = Z(X(e), t);
|
|
919
974
|
switch (t) {
|
|
920
975
|
case "postgres": return `CREATE TABLE ${r} (LIKE ${n})`;
|
|
921
976
|
case "mysql":
|
|
@@ -923,20 +978,20 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
923
978
|
case "sqlite": return `CREATE TABLE ${r} AS SELECT * FROM ${n} WHERE 0`;
|
|
924
979
|
case "mssql": return `SELECT * INTO ${r} FROM ${n} WHERE 1 = 0`;
|
|
925
980
|
}
|
|
926
|
-
},
|
|
927
|
-
let n =
|
|
928
|
-
return t === "mssql" ? `IF OBJECT_ID('${
|
|
929
|
-
},
|
|
981
|
+
}, Tn = (e, t) => {
|
|
982
|
+
let n = Z(X(e), t);
|
|
983
|
+
return t === "mssql" ? `IF OBJECT_ID('${X(e)}', 'U') IS NOT NULL DROP TABLE ${n}` : `DROP TABLE IF EXISTS ${n}`;
|
|
984
|
+
}, En = (e, t) => {
|
|
930
985
|
let n = t === "mysql" || t === "mariadb", r = [];
|
|
931
986
|
n && r.push("SET FOREIGN_KEY_CHECKS = 0");
|
|
932
|
-
for (let n of [...e].reverse()) r.push(`DELETE FROM ${
|
|
987
|
+
for (let n of [...e].reverse()) r.push(`DELETE FROM ${Z(n.name, t)}`);
|
|
933
988
|
for (let n of e) {
|
|
934
989
|
if (n.columns.length === 0) throw Error(`refusing to build a swap for '${n.name}' with no columns: an empty list would have to mean \`SELECT *\`, which is the shape that fails on a generated column.`);
|
|
935
|
-
let e = n.columns.map((e) =>
|
|
936
|
-
r.push(`INSERT INTO ${
|
|
990
|
+
let e = n.columns.map((e) => Z(e, t)).join(", ");
|
|
991
|
+
r.push(`INSERT INTO ${Z(n.name, t)} (${e}) SELECT ${e} FROM ${Z(X(n.name), t)}`);
|
|
937
992
|
}
|
|
938
993
|
return n && r.push("SET FOREIGN_KEY_CHECKS = 1"), r;
|
|
939
|
-
},
|
|
994
|
+
}, Dn = (e, t) => {
|
|
940
995
|
let n = [], r = () => {
|
|
941
996
|
for (let e of n) t.unregisterTable(e);
|
|
942
997
|
};
|
|
@@ -944,7 +999,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
944
999
|
for (let r of e) {
|
|
945
1000
|
let e = t.getTable(r);
|
|
946
1001
|
if (e === void 0) continue;
|
|
947
|
-
let i =
|
|
1002
|
+
let i = X(r);
|
|
948
1003
|
t.registerTable({
|
|
949
1004
|
...e,
|
|
950
1005
|
tableName: i,
|
|
@@ -958,7 +1013,7 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
958
1013
|
staged: n,
|
|
959
1014
|
undo: r
|
|
960
1015
|
};
|
|
961
|
-
},
|
|
1016
|
+
}, On = (e, t) => {
|
|
962
1017
|
let n = new Set(t), r = [];
|
|
963
1018
|
for (let t of e.tables) {
|
|
964
1019
|
if (!n.has(t.name)) continue;
|
|
@@ -975,22 +1030,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
975
1030
|
}
|
|
976
1031
|
}
|
|
977
1032
|
return r;
|
|
978
|
-
},
|
|
979
|
-
let n =
|
|
980
|
-
return `SELECT ${
|
|
981
|
-
},
|
|
1033
|
+
}, kn = (e, t) => {
|
|
1034
|
+
let n = Z(X(e.from), t), r = Z(X(e.to), t), i = Z(e.column, t), a = Z(e.toColumn, t);
|
|
1035
|
+
return `SELECT ${Z(e.primaryKey, t)} AS pk, ${i} AS value FROM ${n} c WHERE ${i} IS NOT NULL AND NOT EXISTS (SELECT 1 FROM ${r} p WHERE p.${a} = c.${i})`;
|
|
1036
|
+
}, An = (e) => {
|
|
982
1037
|
if (e.length === 0) return;
|
|
983
1038
|
let t = e.slice(0, 20), n = t.map((e) => ` ${e.table}.${e.primaryKey}: ${e.column} = ${JSON.stringify(e.value)} — no such row in ${e.references}`);
|
|
984
1039
|
return `the swap could not run: ${e.length} row(s) in the bundle reference a row the bundle does not carry.\n${n.join("\n")}\n` + (e.length > t.length ? ` … and ${e.length - t.length} more\n` : "") + " The target is UNCHANGED — the swap runs in one transaction and none of it committed.\n Every offending row is listed above rather than only the first, so this can be fixed in one pass.";
|
|
985
|
-
},
|
|
986
|
-
t.length !== 0 && await e.run(
|
|
987
|
-
},
|
|
988
|
-
let i =
|
|
989
|
-
i.length !== 0 && await e.run(
|
|
990
|
-
},
|
|
1040
|
+
}, jn = async (e, t) => {
|
|
1041
|
+
t.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => n.forEach(t, (t) => e.unsafe(t), { discard: !0 }))));
|
|
1042
|
+
}, Mn = async (e, t, r) => {
|
|
1043
|
+
let i = En(t, r);
|
|
1044
|
+
i.length !== 0 && await e.run(P.SqlClient.pipe(n.flatMap((e) => e.withTransaction(n.forEach(i, (t) => e.unsafe(t), { discard: !0 })))));
|
|
1045
|
+
}, Nn = async (e, t, r) => {
|
|
991
1046
|
let i = [];
|
|
992
1047
|
for (let a of t) {
|
|
993
|
-
let t = await e.run(
|
|
1048
|
+
let t = await e.run(P.SqlClient.pipe(n.flatMap((e) => e.unsafe(kn(a, r)))));
|
|
994
1049
|
for (let e of t) i.push({
|
|
995
1050
|
table: a.from,
|
|
996
1051
|
primaryKey: String(e.pk ?? ""),
|
|
@@ -1000,16 +1055,75 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1000
1055
|
});
|
|
1001
1056
|
}
|
|
1002
1057
|
return i;
|
|
1003
|
-
},
|
|
1058
|
+
}, Pn = (e, t) => {
|
|
1004
1059
|
let n = new Map(t.map((e, t) => [e, t]));
|
|
1005
1060
|
return e.filter((e) => {
|
|
1006
1061
|
if (e.from === e.to) return !1;
|
|
1007
1062
|
let t = n.get(e.from), r = n.get(e.to);
|
|
1008
1063
|
return t === void 0 || r === void 0 ? !1 : r > t;
|
|
1009
1064
|
});
|
|
1010
|
-
},
|
|
1065
|
+
}, Fn = (e) => e === "mysql" || e === "mariadb", In = (e) => {
|
|
1066
|
+
if (e.mode !== "replace") return {
|
|
1067
|
+
active: !1,
|
|
1068
|
+
tables: [],
|
|
1069
|
+
blocker: `staging applies to \`replace\` only (this run is \`${e.mode}\`).`
|
|
1070
|
+
};
|
|
1071
|
+
let t = (() => {
|
|
1072
|
+
let t = e.targetDialect;
|
|
1073
|
+
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1074
|
+
})();
|
|
1075
|
+
if (t === void 0) return {
|
|
1076
|
+
active: !1,
|
|
1077
|
+
tables: [],
|
|
1078
|
+
blocker: `the target dialect (${e.targetDialect ?? "unknown"}) is not one this build can stage on.`
|
|
1079
|
+
};
|
|
1080
|
+
if (!e.canRunSql) return {
|
|
1081
|
+
active: !1,
|
|
1082
|
+
tables: [],
|
|
1083
|
+
blocker: "this store cannot send DDL (no raw-SQL channel), which a staged swap needs."
|
|
1084
|
+
};
|
|
1085
|
+
let n = e.targetSnapshot;
|
|
1086
|
+
if (n === void 0) return {
|
|
1087
|
+
active: !1,
|
|
1088
|
+
tables: [],
|
|
1089
|
+
blocker: "this caller did not supply a snapshot of the target schema, so the staging tables cannot be given the target's column shape."
|
|
1090
|
+
};
|
|
1091
|
+
let r = new Map(n.tables.map((e) => [e.name, e])), i = e.bundleTables.filter((t) => !e.isSkipped(t)), a = i.filter((e) => !r.has(e));
|
|
1092
|
+
if (a.length > 0) return {
|
|
1093
|
+
active: !1,
|
|
1094
|
+
tables: [],
|
|
1095
|
+
blocker: `the bundle carries ${a.length} table(s) the target does not have (${a.join(", ")}). Staging copies the TARGET's column shape, and there is none for those.`
|
|
1096
|
+
};
|
|
1097
|
+
let o = [];
|
|
1098
|
+
for (let e of i) {
|
|
1099
|
+
let t = r.get(e).columns.filter((e) => e.generatedAs === void 0).map((e) => e.name);
|
|
1100
|
+
if (t.length === 0) return {
|
|
1101
|
+
active: !1,
|
|
1102
|
+
tables: [],
|
|
1103
|
+
blocker: `\`${e}\` has no ordinary columns in the target (every column is generated), so a staged copy would have nothing to write.`
|
|
1104
|
+
};
|
|
1105
|
+
o.push({
|
|
1106
|
+
name: e,
|
|
1107
|
+
columns: t
|
|
1108
|
+
});
|
|
1109
|
+
}
|
|
1110
|
+
if (o.length === 0) return {
|
|
1111
|
+
active: !1,
|
|
1112
|
+
tables: [],
|
|
1113
|
+
blocker: "every table in the bundle is one that describes a deployment, so there is nothing for a staged swap to move."
|
|
1114
|
+
};
|
|
1115
|
+
let s = o.map((e) => e.name), c = Fn(t) ? [] : Pn(On(n, s), s);
|
|
1116
|
+
return c.length > 0 ? {
|
|
1117
|
+
active: !1,
|
|
1118
|
+
tables: [],
|
|
1119
|
+
blocker: `the bundle's tables reference each other in a cycle (${c.map((e) => `${e.from}.${e.column} → ${e.to}`).join(", ")}), and a bulk copy cannot satisfy both directions on this dialect.`
|
|
1120
|
+
} : {
|
|
1121
|
+
active: !0,
|
|
1122
|
+
tables: o
|
|
1123
|
+
};
|
|
1124
|
+
}, Ln = (e) => n.gen(function* () {
|
|
1011
1125
|
if (e.mode !== "replace" || e.dir === void 0 || e.tables.length === 0) return;
|
|
1012
|
-
if (e.snapshot === void 0) return yield* n.fail(new
|
|
1126
|
+
if (e.snapshot === void 0) return yield* n.fail(new H({
|
|
1013
1127
|
reason: "rollback capture needs the TARGET's schema, and this run has none.\n `--mode replace` empties the target first, so a run without a capture can leave it\n emptied and unfilled if the process does not survive to roll back. Pass the target\n snapshot, or `--no-rollback` to accept that risk deliberately.",
|
|
1014
1128
|
path: e.dir
|
|
1015
1129
|
}));
|
|
@@ -1020,14 +1134,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1020
1134
|
if (t.tables.length === 0) return;
|
|
1021
1135
|
e.onWarn?.(`capturing the target's current rows before emptying it → ${e.dir}\n This is what a failed replace is restored FROM. It is written before the first delete,
|
|
1022
1136
|
so it survives a run whose process does not.`);
|
|
1023
|
-
let r = yield*
|
|
1137
|
+
let r = yield* an({
|
|
1024
1138
|
store: e.store,
|
|
1025
1139
|
snapshot: t,
|
|
1026
1140
|
dialect: e.dialect ?? "unknown",
|
|
1027
1141
|
outDir: e.dir,
|
|
1028
1142
|
scope: { kind: "all" },
|
|
1029
1143
|
compression: "none"
|
|
1030
|
-
}).pipe(n.mapError((t) => new
|
|
1144
|
+
}).pipe(n.mapError((t) => new H({
|
|
1031
1145
|
reason: `rollback capture failed, so the import did not start and the target is untouched: ${t.reason ?? String(t)}\n Fix the capture, or re-run with \`--no-rollback\` to proceed without one.`,
|
|
1032
1146
|
path: e.dir
|
|
1033
1147
|
})));
|
|
@@ -1036,14 +1150,14 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1036
1150
|
rows: r.tables.reduce((e, t) => e + t.rowCount, 0),
|
|
1037
1151
|
tables: r.tables.length
|
|
1038
1152
|
};
|
|
1039
|
-
}),
|
|
1153
|
+
}), Rn = (e) => `rollback capture: ${e.rows} row(s) across ${e.tables} table(s) → ${e.dir}\n If this run does not finish, restore with: voltro data import ${e.dir} --mode replace`, zn = (e, t) => e.tables.find((e) => e.name === t), Bn = (e, t) => new Map((zn(e, t)?.columns ?? []).map((e) => [e.name, e])), Vn = (e, t) => {
|
|
1040
1154
|
let n = [];
|
|
1041
1155
|
for (let r of [...e.tables].sort((e, t) => e.name.localeCompare(t.name))) {
|
|
1042
|
-
if (
|
|
1156
|
+
if (zn(t, r.name) === void 0) {
|
|
1043
1157
|
n.push(`table '${r.name}' is in the bundle but MISSING from the target — its rows have nowhere to go`);
|
|
1044
1158
|
continue;
|
|
1045
1159
|
}
|
|
1046
|
-
let e =
|
|
1160
|
+
let e = Bn(t, r.name), i = new Map(r.columns.map((e) => [e.name, e]));
|
|
1047
1161
|
for (let [t, a] of i) {
|
|
1048
1162
|
let i = e.get(t);
|
|
1049
1163
|
if (i === void 0) {
|
|
@@ -1055,22 +1169,22 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1055
1169
|
for (let [t, a] of e) i.has(t) || !a.nullable && !a.hasDefault && n.push(`${r.name}.${t}: the target requires it (NOT NULL, no default) and the bundle carries no value — every row of this table would fail`);
|
|
1056
1170
|
}
|
|
1057
1171
|
return n.length > 30 ? [...n.slice(0, 30), `… and ${n.length - 30} more difference(s)`] : n;
|
|
1058
|
-
},
|
|
1059
|
-
let r =
|
|
1172
|
+
}, Hn = (e, t, n) => {
|
|
1173
|
+
let r = re(t);
|
|
1060
1174
|
if (r === e || n === void 0) return {
|
|
1061
1175
|
drifted: !1,
|
|
1062
1176
|
bundleFingerprint: e,
|
|
1063
1177
|
targetFingerprint: r,
|
|
1064
1178
|
diff: []
|
|
1065
1179
|
};
|
|
1066
|
-
let i =
|
|
1180
|
+
let i = Vn(n, t);
|
|
1067
1181
|
return {
|
|
1068
1182
|
drifted: i.length > 0,
|
|
1069
1183
|
bundleFingerprint: e,
|
|
1070
1184
|
targetFingerprint: r,
|
|
1071
1185
|
diff: i
|
|
1072
1186
|
};
|
|
1073
|
-
},
|
|
1187
|
+
}, Un = 2e3, Wn = 200, Gn = /* @__PURE__ */ new Set([
|
|
1074
1188
|
"BundleError",
|
|
1075
1189
|
"CodecError",
|
|
1076
1190
|
"IntegrityError",
|
|
@@ -1079,447 +1193,481 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1079
1193
|
"RowsRefusedError",
|
|
1080
1194
|
"SchemaDriftError",
|
|
1081
1195
|
"StorageError"
|
|
1082
|
-
]),
|
|
1196
|
+
]), Kn = (t) => {
|
|
1083
1197
|
let n = o.isFiberFailure(t) ? [...e.failures(t[o.FiberFailureCauseId]), ...e.defects(t[o.FiberFailureCauseId])] : [t];
|
|
1084
1198
|
for (let e of n) {
|
|
1085
1199
|
let t = e?._tag;
|
|
1086
|
-
if (typeof t == "string" &&
|
|
1200
|
+
if (typeof t == "string" && Gn.has(t)) return e;
|
|
1087
1201
|
if (o.isFiberFailure(e)) {
|
|
1088
|
-
let t =
|
|
1202
|
+
let t = Kn(e);
|
|
1089
1203
|
if (t) return t;
|
|
1090
1204
|
}
|
|
1091
1205
|
}
|
|
1092
|
-
},
|
|
1206
|
+
}, qn = (t) => [...e.failures(t)][0] ?? Error(e.pretty(t)), Jn = (e) => n.tryPromise({
|
|
1093
1207
|
try: async () => {
|
|
1094
1208
|
let t = await import("node:fs/promises"), n = await import("node:path");
|
|
1095
|
-
return
|
|
1209
|
+
return vt(await t.readFile(n.join(e, K), "utf8"));
|
|
1096
1210
|
},
|
|
1097
|
-
catch: (e) => new
|
|
1211
|
+
catch: (e) => new H({
|
|
1098
1212
|
reason: `manifest read/parse failed: ${String(e?.message ?? e)}`,
|
|
1099
|
-
path:
|
|
1213
|
+
path: K
|
|
1100
1214
|
})
|
|
1101
|
-
}),
|
|
1102
|
-
let t
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
if (
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
tables:
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1215
|
+
}), Yn = (e) => {
|
|
1216
|
+
let t;
|
|
1217
|
+
return n.gen(function* () {
|
|
1218
|
+
let i = yield* n.promise(() => import("node:path")), a = yield* Jn(e.bundleDir);
|
|
1219
|
+
if (a.formatVersion !== 1) return yield* n.fail(new H({ reason: `unsupported bundle formatVersion ${a.formatVersion} (expected 1)` }));
|
|
1220
|
+
if (e.tables !== void 0) {
|
|
1221
|
+
let t = new Set(a.tables.map((e) => e.name)), r = e.tables.filter((e) => !t.has(e));
|
|
1222
|
+
if (r.length > 0) return yield* n.fail(new H({ reason: `--tables names ${r.length} table(s) this bundle does not carry: ${r.join(", ")}. The bundle carries: ${[...t].sort().join(", ")}.` }));
|
|
1223
|
+
}
|
|
1224
|
+
let o = e.tables === void 0 ? void 0 : new Set(e.tables), s = o === void 0 ? a : {
|
|
1225
|
+
...a,
|
|
1226
|
+
tables: a.tables.filter((e) => o.has(e.name))
|
|
1227
|
+
};
|
|
1228
|
+
if (e.targetSnapshot) {
|
|
1229
|
+
let t = { tables: s.tables.map((e) => ({
|
|
1230
|
+
name: e.name,
|
|
1231
|
+
indexes: [],
|
|
1232
|
+
columns: Object.entries(e.columnTypes).map(([e, t]) => ({
|
|
1233
|
+
name: e,
|
|
1234
|
+
type: t,
|
|
1235
|
+
nullable: !0,
|
|
1236
|
+
unique: !1,
|
|
1237
|
+
hasDefault: !1
|
|
1238
|
+
}))
|
|
1239
|
+
})) }, r = Hn(s.source.schemaFingerprint, e.targetSnapshot, t);
|
|
1240
|
+
if (r.drifted) {
|
|
1241
|
+
if (!e.force) return yield* n.fail(new Je({
|
|
1242
|
+
bundleFingerprint: r.bundleFingerprint,
|
|
1243
|
+
targetFingerprint: r.targetFingerprint,
|
|
1244
|
+
diff: r.diff
|
|
1245
|
+
}));
|
|
1246
|
+
e.onWarn?.(`schema drift: the target schema (fingerprint ${r.targetFingerprint.slice(0, 16)}) differs from the bundle's source (${r.bundleFingerprint.slice(0, 16)}); importing anyway (--force). ${r.diff.join("; ")}`);
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
if (e.targetDialect && e.targetDialect !== s.source.dialect) {
|
|
1250
|
+
let t = dn(s.tables, s.source.dialect, e.targetDialect).filter((e) => e.severity === "block");
|
|
1251
|
+
if (t.length > 0 && !e.allowIncompatible) return yield* n.fail(new Ze({
|
|
1252
|
+
sourceDialect: s.source.dialect,
|
|
1253
|
+
targetDialect: e.targetDialect,
|
|
1254
|
+
blocking: t.map(fn)
|
|
1129
1255
|
}));
|
|
1130
|
-
e.onWarn?.(`schema drift: the target schema (fingerprint ${r.targetFingerprint.slice(0, 16)}) differs from the bundle's source (${r.bundleFingerprint.slice(0, 16)}); importing anyway (--force). ${r.diff.join("; ")}`);
|
|
1131
1256
|
}
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
sourceDialect: o.source.dialect,
|
|
1137
|
-
targetDialect: e.targetDialect,
|
|
1138
|
-
blocking: t.map(cn)
|
|
1257
|
+
let c = e.verify ?? !0, p = e.mode ?? "upsert", g = e.atomic ?? p === "replace", _ = e.onConflict ?? "skip", y = e.ledgerPath ?? i.join(e.bundleDir, ".import..ledger.json"), x = yield* jt(y);
|
|
1258
|
+
if (p === "replace" && s.scope.kind !== "all") return yield* n.fail(new Xe({
|
|
1259
|
+
mode: p,
|
|
1260
|
+
reason: `replace refuses a partial bundle (scope='${s.scope.kind}') — truncating would delete rows not in the bundle. Use --mode upsert, or re-export with full scope. To leave specific tables out of a FULL export, use \`--exclude a,b\`: it keeps the scope \`all\` and records what it left out.`
|
|
1139
1261
|
}));
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1262
|
+
if (p === "replace" && (s.scope.exclude?.length ?? 0) > 0) {
|
|
1263
|
+
let t = s.scope.exclude;
|
|
1264
|
+
e.onWarn?.(`this bundle was exported with ${t.length} table(s) excluded: ${t.join(", ")}.\n A replace empties and refills the tables the bundle CARRIES, so those keep whatever rows the
|
|
1265
|
+
target already holds. That is not "everything the bundle says" — check it is what you meant.`);
|
|
1266
|
+
}
|
|
1267
|
+
if (p === "replace") {
|
|
1268
|
+
let t = s.tables.map((e) => e.name), r = e.store.incomingForeignKeys ? yield* n.tryPromise({
|
|
1269
|
+
try: () => e.store.incomingForeignKeys(t),
|
|
1270
|
+
catch: (e) => new H({ reason: `foreign-key pre-flight failed: ${yn(e)}` })
|
|
1271
|
+
}) : e.targetSnapshot === void 0 ? [] : on(e.targetSnapshot, t), i = [], a = /* @__PURE__ */ new Map();
|
|
1272
|
+
for (let t of r) {
|
|
1273
|
+
if (!a.has(t.from)) {
|
|
1274
|
+
let r = yield* n.tryPromise({
|
|
1275
|
+
try: () => e.store.query({
|
|
1276
|
+
table: t.from,
|
|
1277
|
+
predicate: void 0,
|
|
1278
|
+
order: [],
|
|
1279
|
+
take: void 0,
|
|
1280
|
+
skip: void 0,
|
|
1281
|
+
projection: []
|
|
1282
|
+
}),
|
|
1283
|
+
catch: () => new H({ reason: `unreadable: ${t.from}` })
|
|
1284
|
+
}).pipe(n.map((e) => e.length), n.orElseSucceed(() => 0));
|
|
1285
|
+
a.set(t.from, r);
|
|
1286
|
+
}
|
|
1287
|
+
let r = a.get(t.from) ?? 0;
|
|
1288
|
+
r > 0 && i.push({
|
|
1289
|
+
...t,
|
|
1290
|
+
rowCount: r
|
|
1291
|
+
});
|
|
1165
1292
|
}
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
});
|
|
1293
|
+
if (i.length > 0) return yield* n.fail(new Xe({
|
|
1294
|
+
mode: p,
|
|
1295
|
+
reason: sn(i)
|
|
1296
|
+
}));
|
|
1171
1297
|
}
|
|
1172
|
-
if (
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
tablesWritten: 0,
|
|
1180
|
-
rowsWritten: 0,
|
|
1181
|
-
tablesSkipped: 0,
|
|
1182
|
-
rowsSkipped: 0,
|
|
1183
|
-
fullyResumed: !1
|
|
1184
|
-
};
|
|
1185
|
-
let v = /* @__PURE__ */ new Map();
|
|
1186
|
-
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1187
|
-
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1188
|
-
e.length > 0 && v.set(t.name, e);
|
|
1189
|
-
}
|
|
1190
|
-
let x = (e, t) => {
|
|
1191
|
-
let n = v.get(e);
|
|
1192
|
-
if (n === void 0) return t;
|
|
1193
|
-
let r;
|
|
1194
|
-
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1195
|
-
return r ?? t;
|
|
1196
|
-
}, C = (e, t, n, r = !1) => {
|
|
1197
|
-
let i = j ? J(t.name) : t.name, a = t.primaryKey, o = x(t.name, n);
|
|
1198
|
-
return c === "append" ? p === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : c === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1199
|
-
}, w = (e, t) => e.savepoint ? e.savepoint(t) : t(), T = 0, E = (t, n, r) => {
|
|
1200
|
-
r.unclassified && (T++, !(T > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (T === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1201
|
-
}, ee = (e, t, r = !0) => n.tryPromise({
|
|
1202
|
-
try: e,
|
|
1203
|
-
catch: (e) => pn(e)
|
|
1204
|
-
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, E(t.entry, t.row, e)), !1)))), te = (e) => {
|
|
1205
|
-
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1206
|
-
if (t.length !== 0) return {
|
|
1207
|
-
...e.row,
|
|
1208
|
-
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1298
|
+
if (e.dryRun === !0) return {
|
|
1299
|
+
manifest: s,
|
|
1300
|
+
tablesWritten: 0,
|
|
1301
|
+
rowsWritten: 0,
|
|
1302
|
+
tablesSkipped: 0,
|
|
1303
|
+
rowsSkipped: 0,
|
|
1304
|
+
fullyResumed: !1
|
|
1209
1305
|
};
|
|
1210
|
-
|
|
1211
|
-
let t
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
return i != null && t.has(String(i));
|
|
1215
|
-
}), r = e.map((e) => ({
|
|
1216
|
-
table: e.entry.name,
|
|
1217
|
-
id: String(e.row[e.entry.primaryKey]),
|
|
1218
|
-
reason: e.reason,
|
|
1219
|
-
derived: n(e)
|
|
1220
|
-
})), i = [...r.filter((e) => !e.derived), ...r.filter((e) => e.derived)], a = /* @__PURE__ */ new Map();
|
|
1221
|
-
for (let e of r) {
|
|
1222
|
-
let t = a.get(e.table) ?? {
|
|
1223
|
-
refused: 0,
|
|
1224
|
-
primary: 0
|
|
1225
|
-
};
|
|
1226
|
-
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1306
|
+
let w = /* @__PURE__ */ new Map();
|
|
1307
|
+
for (let t of e.targetSnapshot?.tables ?? []) {
|
|
1308
|
+
let e = t.columns.filter((e) => e.generatedAs !== void 0).map((e) => e.name);
|
|
1309
|
+
e.length > 0 && w.set(t.name, e);
|
|
1227
1310
|
}
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
})
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
if (
|
|
1245
|
-
|
|
1246
|
-
|
|
1311
|
+
let re = (e, t) => {
|
|
1312
|
+
let n = w.get(e);
|
|
1313
|
+
if (n === void 0) return t;
|
|
1314
|
+
let r;
|
|
1315
|
+
for (let e of n) e in t && (r ??= { ...t }, delete r[e]);
|
|
1316
|
+
return r ?? t;
|
|
1317
|
+
}, T = (e, t, n, r = !1) => {
|
|
1318
|
+
let i = j ? X(t.name) : t.name, a = t.primaryKey, o = re(t.name, n);
|
|
1319
|
+
return p === "append" ? _ === "fail" ? e.insert(i, o) : e.insertIgnore(i, o, { conflictColumns: [a] }) : p === "replace" ? r ? e.upsert(i, o, { conflictColumns: [a] }) : e.insert(i, o) : e.upsert(i, o, { conflictColumns: [a] });
|
|
1320
|
+
}, ie = (e, t) => e.savepoint ? e.savepoint(t) : t(), E = 0, ae = (t, n, r) => {
|
|
1321
|
+
r.unclassified && (E++, !(E > 3) && e.onWarn?.(`row refused with no driver detail — table '${t.name}', row '${String(n.id ?? "?")}'. The full rendering follows so the LAYER that refused is on the record; it is logged, never returned. ${r.raw}` + (E === 3 ? " (further unclassified refusals are counted, not logged)" : "")));
|
|
1322
|
+
}, oe = (e, t, r = !0) => n.tryPromise({
|
|
1323
|
+
try: e,
|
|
1324
|
+
catch: (e) => _n(e)
|
|
1325
|
+
}).pipe(n.map(() => !0), n.catchAll((e) => n.sync(() => (r && (t.reason = e.reason, ae(t.entry, t.row, e)), !1)))), se = (e) => {
|
|
1326
|
+
let t = Object.entries(e.entry.columnTypes).filter(([t, n]) => n === "reference" && e.row[t] !== null && e.row[t] !== void 0).map(([e]) => e);
|
|
1327
|
+
if (t.length !== 0) return {
|
|
1328
|
+
...e.row,
|
|
1329
|
+
...Object.fromEntries(t.map((e) => [e, null]))
|
|
1330
|
+
};
|
|
1331
|
+
}, ce = (e) => {
|
|
1332
|
+
let t = new Set(e.map((e) => String(e.row[e.entry.primaryKey]))), n = (e) => Object.entries(e.entry.columnTypes).some(([n, r]) => {
|
|
1333
|
+
if (r !== "reference") return !1;
|
|
1334
|
+
let i = e.row[n];
|
|
1335
|
+
return i != null && t.has(String(i));
|
|
1336
|
+
}), r = e.map((e) => ({
|
|
1337
|
+
table: e.entry.name,
|
|
1338
|
+
id: String(e.row[e.entry.primaryKey]),
|
|
1339
|
+
reason: e.reason,
|
|
1340
|
+
derived: n(e)
|
|
1341
|
+
})), i = [...r.filter((e) => !e.derived), ...r.filter((e) => e.derived)], a = /* @__PURE__ */ new Map();
|
|
1342
|
+
for (let e of r) {
|
|
1343
|
+
let t = a.get(e.table) ?? {
|
|
1344
|
+
refused: 0,
|
|
1345
|
+
primary: 0
|
|
1346
|
+
};
|
|
1347
|
+
t.refused++, e.derived || t.primary++, a.set(e.table, t);
|
|
1247
1348
|
}
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
if (t === void 0) return [];
|
|
1258
|
-
let n = new Map(t.tables.map((e) => [e.name, e])), r = [];
|
|
1259
|
-
for (let e of o.tables) {
|
|
1260
|
-
if (z(e.name)) continue;
|
|
1261
|
-
let t = n.get(e.name);
|
|
1262
|
-
if (t === void 0) return [];
|
|
1263
|
-
let i = t.columns.filter((e) => e.generatedAs === void 0).map((e) => e.name);
|
|
1264
|
-
if (i.length === 0) return [];
|
|
1265
|
-
r.push({
|
|
1266
|
-
name: e.name,
|
|
1267
|
-
columns: i
|
|
1349
|
+
return new qe({
|
|
1350
|
+
rows: i.slice(0, 20),
|
|
1351
|
+
primaryCount: r.filter((e) => !e.derived).length,
|
|
1352
|
+
totalCount: e.length,
|
|
1353
|
+
byTable: [...a.entries()].map(([e, t]) => ({
|
|
1354
|
+
table: e,
|
|
1355
|
+
refused: t.refused,
|
|
1356
|
+
primary: t.primary
|
|
1357
|
+
})).sort((e, t) => t.refused - e.refused || e.table.localeCompare(t.table)).slice(0, 100)
|
|
1268
1358
|
});
|
|
1269
|
-
}
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1359
|
+
}, ue = (e, t) => n.gen(function* () {
|
|
1360
|
+
let r = t, i = [], a = !0;
|
|
1361
|
+
for (; r.length > 0 && a;) {
|
|
1362
|
+
a = !1;
|
|
1363
|
+
let t = [];
|
|
1364
|
+
for (let n of r) (yield* oe(() => ie(e, () => T(e, n.entry, n.row, !0)), n)) ? a = !0 : t.push(n);
|
|
1365
|
+
if (!a && t.length > 0) for (let n = t.length - 1; n >= 0; n--) {
|
|
1366
|
+
let r = t[n], o = se(r);
|
|
1367
|
+
o !== void 0 && (yield* oe(() => ie(e, () => T(e, r.entry, o, !0)), r, !1)) && (i.push(r), t.splice(n, 1), a = !0);
|
|
1368
|
+
}
|
|
1369
|
+
r = t;
|
|
1370
|
+
}
|
|
1371
|
+
for (let t of i) (yield* oe(() => ie(e, () => e.upsert(t.entry.name, t.row, { conflictColumns: [t.entry.primaryKey] })), t)) || (r = [...r, t]);
|
|
1372
|
+
if (r.length > 0) return yield* n.fail(ce(r));
|
|
1373
|
+
}), D = (() => {
|
|
1374
|
+
let t = e.targetDialect;
|
|
1375
|
+
return t === "postgres" || t === "mysql" || t === "mariadb" || t === "sqlite" || t === "mssql" ? t : void 0;
|
|
1376
|
+
})(), O = In({
|
|
1377
|
+
mode: p,
|
|
1378
|
+
...e.targetDialect === void 0 ? {} : { targetDialect: e.targetDialect },
|
|
1379
|
+
...e.targetSnapshot === void 0 ? {} : { targetSnapshot: e.targetSnapshot },
|
|
1380
|
+
bundleTables: s.tables.map((e) => e.name),
|
|
1381
|
+
isSkipped: V,
|
|
1382
|
+
canRunSql: xn(e.store)
|
|
1383
|
+
}), k = O.tables, A = s.tables.filter((e) => !V(e.name) && S(e.name)).map((e) => e.name), j = O.active, de = !1, M = 0, N = 0, pe, P = 0, F = 0, me = (t, r) => n.gen(function* () {
|
|
1384
|
+
if (p === "replace" && !j && !(r && x.truncated)) {
|
|
1385
|
+
e.recordInterruption === !0 && (yield* n.tryPromise({
|
|
1386
|
+
try: () => t.insert(d, {
|
|
1387
|
+
id: m("repl", he),
|
|
1388
|
+
startedAt: ge,
|
|
1389
|
+
tables: String(s.tables.length),
|
|
1390
|
+
capture: I?.dir ?? null,
|
|
1391
|
+
via: e.via ?? "direct"
|
|
1392
|
+
}),
|
|
1393
|
+
catch: (e) => new H({ reason: `could not record that this replace started: ${String(e?.message ?? e)}\n The marker is what makes an interrupted run visible instead of silent, so the run does
|
|
1282
1394
|
not proceed without it.` })
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
}
|
|
1294
|
-
let a = [], p = [], h = [], v = [], y = o.tables.length;
|
|
1295
|
-
for (let u = 0; u < o.tables.length; u++) {
|
|
1296
|
-
let f = o.tables[u];
|
|
1297
|
-
if (z(f.name)) {
|
|
1298
|
-
v.push(f.name);
|
|
1299
|
-
continue;
|
|
1395
|
+
}));
|
|
1396
|
+
let i = [...s.tables].reverse().map((e) => e.name).filter((e) => !V(e)), a = !r, o = t.emptyTables ? () => t.emptyTables(i) : a ? async () => {
|
|
1397
|
+
for (let e of [...s.tables].reverse()) await t.deleteMany(e.name, { where: b(e.primaryKey) });
|
|
1398
|
+
} : () => t.transactional(async (e) => {
|
|
1399
|
+
for (let t of [...s.tables].reverse()) await e.deleteMany(t.name, { where: b(t.primaryKey) });
|
|
1400
|
+
});
|
|
1401
|
+
yield* n.tryPromise({
|
|
1402
|
+
try: o,
|
|
1403
|
+
catch: (e) => new H({ reason: `emptying the target for --mode replace failed (nothing was deleted): ${yn(e)}` })
|
|
1404
|
+
}), r && (x.truncated = !0, yield* J(y, x));
|
|
1300
1405
|
}
|
|
1301
|
-
|
|
1302
|
-
|
|
1406
|
+
let a = [], o = [], u = [], f = [], h = s.tables.length;
|
|
1407
|
+
for (let d = 0; d < s.tables.length; d++) {
|
|
1408
|
+
let m = s.tables[d];
|
|
1409
|
+
if (V(m.name)) {
|
|
1410
|
+
f.push(m.name);
|
|
1411
|
+
continue;
|
|
1412
|
+
}
|
|
1413
|
+
if (r && m.name in x.tables) {
|
|
1414
|
+
u.push(m.name), F++, P += x.tables[m.name].rowCount, q(e.onProgress, {
|
|
1415
|
+
phase: "import",
|
|
1416
|
+
table: m.name,
|
|
1417
|
+
index: d,
|
|
1418
|
+
tableCount: h,
|
|
1419
|
+
event: "done",
|
|
1420
|
+
rowsDone: x.tables[m.name].rowCount,
|
|
1421
|
+
total: m.rowCount
|
|
1422
|
+
});
|
|
1423
|
+
continue;
|
|
1424
|
+
}
|
|
1425
|
+
let _ = i.join(e.bundleDir, m.file);
|
|
1426
|
+
q(e.onProgress, {
|
|
1303
1427
|
phase: "import",
|
|
1304
|
-
table:
|
|
1305
|
-
index:
|
|
1306
|
-
tableCount:
|
|
1307
|
-
event: "
|
|
1308
|
-
rowsDone:
|
|
1309
|
-
total:
|
|
1428
|
+
table: m.name,
|
|
1429
|
+
index: d,
|
|
1430
|
+
tableCount: h,
|
|
1431
|
+
event: "start",
|
|
1432
|
+
rowsDone: 0,
|
|
1433
|
+
total: m.rowCount
|
|
1310
1434
|
});
|
|
1311
|
-
|
|
1435
|
+
let v = a.length, b = n.gen(function* () {
|
|
1436
|
+
let { createHash: r } = yield* n.promise(() => import("node:crypto")), i = r("sha256"), o = 0, c = le("bundle", () => fe(_).pipe(dt(s.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), u = async (e) => {
|
|
1437
|
+
let n = {
|
|
1438
|
+
entry: m,
|
|
1439
|
+
row: e,
|
|
1440
|
+
reason: ""
|
|
1441
|
+
};
|
|
1442
|
+
try {
|
|
1443
|
+
await ie(t, () => T(t, m, e));
|
|
1444
|
+
} catch (t) {
|
|
1445
|
+
let r = _n(t);
|
|
1446
|
+
n.reason = r.reason, ae(m, e, r), a.push(n);
|
|
1447
|
+
}
|
|
1448
|
+
}, d = yield* n.promise(async () => {
|
|
1449
|
+
let n = e.copyLoader;
|
|
1450
|
+
if (!n || n.dialect !== "postgres") return !1;
|
|
1451
|
+
if (g) return de || (de = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1452
|
+
if (p === "replace") return !0;
|
|
1453
|
+
if (p !== "upsert") return !1;
|
|
1454
|
+
try {
|
|
1455
|
+
return (await t.query({
|
|
1456
|
+
table: m.name,
|
|
1457
|
+
predicate: void 0,
|
|
1458
|
+
order: [],
|
|
1459
|
+
take: 1,
|
|
1460
|
+
skip: void 0,
|
|
1461
|
+
projection: void 0
|
|
1462
|
+
})).length === 0;
|
|
1463
|
+
} catch {
|
|
1464
|
+
return !1;
|
|
1465
|
+
}
|
|
1466
|
+
}), f = Object.keys(m.columnTypes), h = Math.max(1, e.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
1467
|
+
if (v.length === 0) return;
|
|
1468
|
+
let t = v;
|
|
1469
|
+
v = [];
|
|
1470
|
+
try {
|
|
1471
|
+
await e.copyLoader.copyInto({
|
|
1472
|
+
table: m.name,
|
|
1473
|
+
columns: f,
|
|
1474
|
+
columnTypes: m.columnTypes,
|
|
1475
|
+
rows: t
|
|
1476
|
+
});
|
|
1477
|
+
} catch (n) {
|
|
1478
|
+
d = !1, e.onWarn?.(`bulk COPY into ${m.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1479
|
+
for (let e of t) await u(e);
|
|
1480
|
+
}
|
|
1481
|
+
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof t.savepoint == "function", S = [], C = async () => {
|
|
1482
|
+
if (S.length === 0) return;
|
|
1483
|
+
let e = S;
|
|
1484
|
+
S = [];
|
|
1485
|
+
try {
|
|
1486
|
+
await t.savepoint(async () => {
|
|
1487
|
+
for (let n of e) await T(t, m, n);
|
|
1488
|
+
});
|
|
1489
|
+
} catch {
|
|
1490
|
+
for (let t of e) await u(t);
|
|
1491
|
+
}
|
|
1492
|
+
};
|
|
1493
|
+
return yield* l.runForEach(Kt(c, m.name, m.columnTypes), (e) => n.promise(async () => {
|
|
1494
|
+
if (o++, d) {
|
|
1495
|
+
v.push(e), v.length >= h && await y();
|
|
1496
|
+
return;
|
|
1497
|
+
}
|
|
1498
|
+
if (x) {
|
|
1499
|
+
S.push(e), S.length >= b && await C();
|
|
1500
|
+
return;
|
|
1501
|
+
}
|
|
1502
|
+
await u(e);
|
|
1503
|
+
})), yield* n.promise(C), yield* n.promise(y), {
|
|
1504
|
+
rowCount: o,
|
|
1505
|
+
checksum: i.digest("hex")
|
|
1506
|
+
};
|
|
1507
|
+
}), { rowCount: S, checksum: C } = yield* kt("import", m.name, d, h, (e) => e.rowCount, b);
|
|
1508
|
+
if (c) {
|
|
1509
|
+
if (C !== m.checksum) return yield* n.fail(new Ge({
|
|
1510
|
+
what: `table ${m.name} checksum`,
|
|
1511
|
+
expected: m.checksum,
|
|
1512
|
+
actual: C
|
|
1513
|
+
}));
|
|
1514
|
+
if (S !== m.rowCount) return yield* n.fail(new Ge({
|
|
1515
|
+
what: `table ${m.name} rowCount`,
|
|
1516
|
+
expected: String(m.rowCount),
|
|
1517
|
+
actual: String(S)
|
|
1518
|
+
}));
|
|
1519
|
+
}
|
|
1520
|
+
q(e.onProgress, {
|
|
1521
|
+
phase: "import",
|
|
1522
|
+
table: m.name,
|
|
1523
|
+
index: d,
|
|
1524
|
+
tableCount: h,
|
|
1525
|
+
event: "done",
|
|
1526
|
+
rowsDone: S,
|
|
1527
|
+
total: m.rowCount
|
|
1528
|
+
}), N++, M += S, pe?.(), r && (a.length > v ? o.push({
|
|
1529
|
+
name: m.name,
|
|
1530
|
+
rowCount: S
|
|
1531
|
+
}) : (x.tables[m.name] = {
|
|
1532
|
+
rowCount: S,
|
|
1533
|
+
checksum: m.checksum
|
|
1534
|
+
}, yield* J(y, x)));
|
|
1312
1535
|
}
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
let b = a.length, x = n.gen(function* () {
|
|
1324
|
-
let { createHash: t } = yield* n.promise(() => import("node:crypto")), i = t("sha256"), s = 0, u = ie("bundle", () => se(_).pipe(st(o.compression))).pipe(l.tap((e) => n.sync(() => i.update(e)))), p = async (e) => {
|
|
1325
|
-
let t = {
|
|
1326
|
-
entry: f,
|
|
1327
|
-
row: e,
|
|
1328
|
-
reason: ""
|
|
1536
|
+
if (a.length > 0 && (yield* ue(t, a)), p === "replace" && !(r && x.truncated) && (yield* n.promise(() => t.deleteMany(d, { where: {
|
|
1537
|
+
column: "startedAt",
|
|
1538
|
+
op: "lte",
|
|
1539
|
+
value: ge
|
|
1540
|
+
} }).then(() => void 0, () => void 0))), r && o.length > 0) {
|
|
1541
|
+
for (let e of o) {
|
|
1542
|
+
let t = s.tables.find((t) => t.name === e.name);
|
|
1543
|
+
x.tables[e.name] = {
|
|
1544
|
+
rowCount: e.rowCount,
|
|
1545
|
+
checksum: t.checksum
|
|
1329
1546
|
};
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
} catch (n) {
|
|
1333
|
-
let r = pn(n);
|
|
1334
|
-
t.reason = r.reason, E(f, e, r), a.push(t);
|
|
1335
|
-
}
|
|
1336
|
-
}, m = yield* n.promise(async () => {
|
|
1337
|
-
let t = e.copyLoader;
|
|
1338
|
-
if (!t || t.dialect !== "postgres") return !1;
|
|
1339
|
-
if (d) return ae || (ae = !0, e.onWarn?.("bulk COPY is off for this run: it needs its own connection and this import runs in ONE transaction (the default for --mode replace, so a failure mid-load cannot leave the target emptied and unfilled). Pass --no-atomic to trade that guarantee back for COPY.")), !1;
|
|
1340
|
-
if (c === "replace") return !0;
|
|
1341
|
-
if (c !== "upsert") return !1;
|
|
1342
|
-
try {
|
|
1343
|
-
return (await r.query({
|
|
1344
|
-
table: f.name,
|
|
1345
|
-
predicate: void 0,
|
|
1346
|
-
order: [],
|
|
1347
|
-
take: 1,
|
|
1348
|
-
skip: void 0,
|
|
1349
|
-
projection: void 0
|
|
1350
|
-
})).length === 0;
|
|
1351
|
-
} catch {
|
|
1352
|
-
return !1;
|
|
1353
|
-
}
|
|
1354
|
-
}), h = Object.keys(f.columnTypes), g = Math.max(1, e.copyBatchSize ?? 5e3), v = [], y = async () => {
|
|
1355
|
-
if (v.length === 0) return;
|
|
1356
|
-
let t = v;
|
|
1357
|
-
v = [];
|
|
1358
|
-
try {
|
|
1359
|
-
await e.copyLoader.copyInto({
|
|
1360
|
-
table: f.name,
|
|
1361
|
-
columns: h,
|
|
1362
|
-
columnTypes: f.columnTypes,
|
|
1363
|
-
rows: t
|
|
1364
|
-
});
|
|
1365
|
-
} catch (n) {
|
|
1366
|
-
m = !1, e.onWarn?.(`bulk COPY into ${f.name} failed (${String(n?.message ?? n)}); falling back to per-row writes for this table`);
|
|
1367
|
-
for (let e of t) await p(e);
|
|
1368
|
-
}
|
|
1369
|
-
}, b = Math.max(1, e.savepointBatchSize ?? 200), x = typeof r.savepoint == "function", S = [], T = async () => {
|
|
1370
|
-
if (S.length === 0) return;
|
|
1371
|
-
let e = S;
|
|
1372
|
-
S = [];
|
|
1373
|
-
try {
|
|
1374
|
-
await r.savepoint(async () => {
|
|
1375
|
-
for (let t of e) await C(r, f, t);
|
|
1376
|
-
});
|
|
1377
|
-
} catch {
|
|
1378
|
-
for (let t of e) await p(t);
|
|
1379
|
-
}
|
|
1380
|
-
};
|
|
1381
|
-
return yield* l.runForEach(Ht(u, f.name, f.columnTypes), (e) => n.promise(async () => {
|
|
1382
|
-
if (s++, m) {
|
|
1383
|
-
v.push(e), v.length >= g && await y();
|
|
1384
|
-
return;
|
|
1385
|
-
}
|
|
1386
|
-
if (x) {
|
|
1387
|
-
S.push(e), S.length >= b && await T();
|
|
1388
|
-
return;
|
|
1389
|
-
}
|
|
1390
|
-
await p(e);
|
|
1391
|
-
})), yield* n.promise(T), yield* n.promise(y), {
|
|
1392
|
-
rowCount: s,
|
|
1393
|
-
checksum: i.digest("hex")
|
|
1394
|
-
};
|
|
1395
|
-
}), { rowCount: S, checksum: T } = yield* Tt("import", f.name, u, y, (e) => e.rowCount, x);
|
|
1396
|
-
if (s) {
|
|
1397
|
-
if (T !== f.checksum) return yield* n.fail(new Ve({
|
|
1398
|
-
what: `table ${f.name} checksum`,
|
|
1399
|
-
expected: f.checksum,
|
|
1400
|
-
actual: T
|
|
1401
|
-
}));
|
|
1402
|
-
if (S !== f.rowCount) return yield* n.fail(new Ve({
|
|
1403
|
-
what: `table ${f.name} rowCount`,
|
|
1404
|
-
expected: String(f.rowCount),
|
|
1405
|
-
actual: String(S)
|
|
1406
|
-
}));
|
|
1547
|
+
}
|
|
1548
|
+
yield* J(y, x);
|
|
1407
1549
|
}
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
index: u,
|
|
1412
|
-
tableCount: y,
|
|
1413
|
-
event: "done",
|
|
1414
|
-
rowsDone: S,
|
|
1415
|
-
total: f.rowCount
|
|
1416
|
-
}), M++, oe += S, i && (a.length > b ? p.push({
|
|
1417
|
-
name: f.name,
|
|
1418
|
-
rowCount: S
|
|
1419
|
-
}) : (g.tables[f.name] = {
|
|
1420
|
-
rowCount: S,
|
|
1421
|
-
checksum: f.checksum
|
|
1422
|
-
}, yield* K(m, g)));
|
|
1423
|
-
}
|
|
1424
|
-
if (a.length > 0 && (yield* re(r, a)), c === "replace" && !(i && g.truncated) && (yield* n.promise(() => r.deleteMany(u, { where: {
|
|
1425
|
-
column: "startedAt",
|
|
1426
|
-
op: "lte",
|
|
1427
|
-
value: de
|
|
1428
|
-
} }).then(() => void 0, () => void 0))), i && p.length > 0) {
|
|
1429
|
-
for (let e of p) {
|
|
1430
|
-
let t = o.tables.find((t) => t.name === e.name);
|
|
1431
|
-
g.tables[e.name] = {
|
|
1432
|
-
rowCount: e.rowCount,
|
|
1433
|
-
checksum: t.checksum
|
|
1434
|
-
};
|
|
1550
|
+
{
|
|
1551
|
+
let t = Ue(f, "skipped");
|
|
1552
|
+
t !== void 0 && e.onWarn?.(t);
|
|
1435
1553
|
}
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1554
|
+
if (u.length > 0) {
|
|
1555
|
+
let t = u.length === s.tables.length;
|
|
1556
|
+
e.onWarn?.(`resume: ${u.length} of ${s.tables.length} table(s) were already applied by an earlier run of THIS bundle directory, so this run wrote NO rows for them${t ? " — that is every table in the bundle, so nothing was written at all" : ""}. Tables: ${u.join(", ")}. Delete ${y} to force a full re-import.`);
|
|
1557
|
+
}
|
|
1558
|
+
}), he = `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`, ge = /* @__PURE__ */ new Date(), I = yield* Ln({
|
|
1559
|
+
store: e.store,
|
|
1560
|
+
mode: p,
|
|
1561
|
+
dir: e.rollbackDir,
|
|
1562
|
+
snapshot: e.targetSnapshot,
|
|
1563
|
+
dialect: e.targetDialect,
|
|
1564
|
+
tables: s.tables.map((e) => e.name).filter((e) => !V(e)),
|
|
1565
|
+
onWarn: e.onWarn
|
|
1566
|
+
});
|
|
1567
|
+
I !== void 0 && (e.onRollbackCaptured !== void 0 && (yield* n.tryPromise({
|
|
1568
|
+
try: () => e.onRollbackCaptured(I),
|
|
1569
|
+
catch: (e) => new H({
|
|
1570
|
+
reason: `the rollback capture could not be stored durably, so the import did not start and the target is untouched: ${String(e?.message ?? e)}\n The capture only helps if it outlives this process — that is the failure it exists for.`,
|
|
1571
|
+
path: I.dir
|
|
1572
|
+
})
|
|
1573
|
+
})), e.onWarn?.(Rn(I)));
|
|
1574
|
+
let _e = (e) => p === "replace" ? ee(e) : e(), L = (e) => p === "replace" ? te(e) : e, R = m("dimp", `${Date.now()}-${s.source.schemaFingerprint ?? ""}`), ve = !1, ye = (e.targetSnapshot?.tables.some((e) => e.name === u) ?? !1) || v(u) !== void 0;
|
|
1575
|
+
ye && h(f), ye && (yield* n.promise(() => e.store.insert(u, {
|
|
1576
|
+
id: R,
|
|
1577
|
+
mode: p,
|
|
1578
|
+
via: e.via ?? "direct",
|
|
1579
|
+
bundle: e.bundleDir,
|
|
1580
|
+
sourceFingerprint: s.source.schemaFingerprint ?? null,
|
|
1581
|
+
tablesWritten: 0,
|
|
1582
|
+
rowsWritten: 0
|
|
1583
|
+
}).then(() => {
|
|
1584
|
+
ve = !0;
|
|
1585
|
+
}, (t) => {
|
|
1586
|
+
e.onWarn?.(`the import trace could not be written (${String(t?.message ?? t)}).\n The import continues — this is a record of what happened, not a safety check. Run
|
|
1587
|
+
\`voltro db apply\` so the next one is recorded.`);
|
|
1588
|
+
})));
|
|
1589
|
+
let be = 0, xe = !1;
|
|
1590
|
+
pe = () => {
|
|
1591
|
+
if (!ve || xe) return;
|
|
1592
|
+
let t = Date.now();
|
|
1593
|
+
t - be < Un || (be = t, e.store.update(u, R, {
|
|
1594
|
+
tablesWritten: N,
|
|
1595
|
+
rowsWritten: M
|
|
1596
|
+
}).then(() => void 0, () => {
|
|
1597
|
+
xe = !0;
|
|
1598
|
+
}));
|
|
1599
|
+
};
|
|
1600
|
+
let Se = (t) => ve ? e.store.update(u, R, {
|
|
1601
|
+
finishedAt: t === null ? /* @__PURE__ */ new Date() : null,
|
|
1602
|
+
tablesWritten: N,
|
|
1603
|
+
rowsWritten: M,
|
|
1604
|
+
failure: t
|
|
1605
|
+
}).then(() => void 0, () => void 0) : Promise.resolve();
|
|
1606
|
+
if (t = (e) => Se(e), p === "replace" && !j && O.blocker !== void 0 && e.onWarn?.(Sn(O.blocker)), p === "replace" && A.length > 0 && e.onWarn?.(`write recorders are SUSPENDED for this replace (${A.length} table(s) have one).\n A replace sets a state rather than changing rows, so a per-row history entry would describe
|
|
1607
|
+
something that did not happen — and this path already writes through the raw store, without
|
|
1608
|
+
tenant scoping or \`audit()\` stamping, for the same reason.`), j) {
|
|
1609
|
+
e.onWarn?.(`staging ${k.length} table(s) before the swap — the target keeps its rows until the load stands.\n The destructive step is one server-side transaction at the end, not the whole load.`);
|
|
1610
|
+
let t = e.store, r = k.map((e) => e.name), i = Dn(r, {
|
|
1611
|
+
getTable: (e) => v(e),
|
|
1612
|
+
registerTable: (e) => {
|
|
1613
|
+
C(e);
|
|
1475
1614
|
},
|
|
1476
|
-
|
|
1477
|
-
}),
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1615
|
+
unregisterTable: ne
|
|
1616
|
+
}), a = !1;
|
|
1617
|
+
try {
|
|
1618
|
+
let i = g ? r : r.filter((e) => !Mt(x, e));
|
|
1619
|
+
yield* n.tryPromise({
|
|
1620
|
+
try: async () => {
|
|
1621
|
+
await jn(t, i.map((e) => Tn(e, D))), await jn(t, i.map((e) => wn(e, D)));
|
|
1622
|
+
},
|
|
1623
|
+
catch: (e) => new H({ reason: `could not prepare the staging tables, so the import did not start and the target is untouched: ${String(e?.message ?? e)}` })
|
|
1624
|
+
}), yield* L(me(e.store, !g)), !g && x.truncated === !0 ? a = !0 : yield* n.tryPromise({
|
|
1625
|
+
try: () => Mn(t, k, D).then(() => {
|
|
1626
|
+
a = !0;
|
|
1627
|
+
}),
|
|
1628
|
+
catch: (e) => new H({ reason: String(e?.message ?? e) })
|
|
1629
|
+
}).pipe(n.catchAll((i) => n.gen(function* () {
|
|
1630
|
+
let a = An(yield* n.promise(() => Nn(t, On(e.targetSnapshot, r), D).then((e) => e, () => [])));
|
|
1631
|
+
return yield* n.fail(new H({ reason: a ?? `the swap failed and the target is UNCHANGED: ${i.reason}` }));
|
|
1632
|
+
}))), !g && a && x.truncated !== !0 && (x.truncated = !0, yield* J(y, x));
|
|
1633
|
+
} finally {
|
|
1634
|
+
i.undo(), a || g ? yield* n.promise(() => jn(t, r.map((e) => Tn(e, D))).then(() => void 0, () => void 0)) : e.onWarn?.("the staged rows are KEPT so a re-run can continue from them rather than reload.\n If you are not going to re-run this bundle, drop them: voltro data clear-staging --yes");
|
|
1635
|
+
}
|
|
1636
|
+
} else g ? yield* n.tryPromise({
|
|
1637
|
+
try: () => e.store.transactional(async (e) => {
|
|
1638
|
+
let t = await _e(() => n.runPromiseExit(me(e, !1)));
|
|
1639
|
+
if (r.isFailure(t)) throw qn(t.cause);
|
|
1640
|
+
}),
|
|
1641
|
+
catch: (e) => Kn(e) ?? new H({ reason: `atomic import failed (rolled back): ${String(e?.message ?? e)}` })
|
|
1642
|
+
}) : yield* L(me(e.store, !0));
|
|
1643
|
+
if (e.assets && s.assets.index && !e.assetsAlreadyRestored) {
|
|
1644
|
+
let t = e.assets, r = i.join(e.bundleDir, xt), a = i.join(e.bundleDir, s.assets.index ?? "assets/index.ndjson"), o = le("bundle", () => fe(a)).pipe(l.decodeText(), l.splitLines, l.filter((e) => e.trim().length > 0), l.map((e) => JSON.parse(e)));
|
|
1645
|
+
yield* l.runForEach(o, (e) => e.key in x.assets ? n.void : ot(t, r, e).pipe(n.andThen(n.sync(() => {
|
|
1646
|
+
x.assets[e.key] = !0;
|
|
1647
|
+
})), n.andThen(J(y, x))));
|
|
1488
1648
|
}
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
})), n.andThen(K(m, g))));
|
|
1501
|
-
}
|
|
1502
|
-
return {
|
|
1503
|
-
manifest: o,
|
|
1504
|
-
tablesWritten: M,
|
|
1505
|
-
rowsWritten: oe,
|
|
1506
|
-
tablesSkipped: le,
|
|
1507
|
-
rowsSkipped: ce,
|
|
1508
|
-
fullyResumed: M === 0 && le > 0,
|
|
1509
|
-
...P === void 0 ? {} : { rollback: P }
|
|
1510
|
-
};
|
|
1511
|
-
}), Wn = (e) => e, Gn = (e) => {
|
|
1649
|
+
return yield* n.promise(() => Se(null)), {
|
|
1650
|
+
manifest: s,
|
|
1651
|
+
tablesWritten: N,
|
|
1652
|
+
rowsWritten: M,
|
|
1653
|
+
tablesSkipped: F,
|
|
1654
|
+
rowsSkipped: P,
|
|
1655
|
+
fullyResumed: N === 0 && F > 0,
|
|
1656
|
+
...I === void 0 ? {} : { rollback: I }
|
|
1657
|
+
};
|
|
1658
|
+
}).pipe(n.tapError((e) => n.promise(() => t?.(e?.reason ?? e?._tag ?? "the import did not finish") ?? Promise.resolve())));
|
|
1659
|
+
}, Xn = (e) => e, Zn = (e) => {
|
|
1512
1660
|
if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
|
|
1513
1661
|
let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
|
|
1514
1662
|
return {
|
|
1515
1663
|
refuse: !0,
|
|
1516
1664
|
message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
|
|
1517
1665
|
};
|
|
1518
|
-
},
|
|
1666
|
+
}, Q = Buffer.from("VBUNDLE2"), Qn = 0, $n = 1, er = 2, tr = 32, nr = async (e) => {
|
|
1519
1667
|
let t = [], n = async (r) => {
|
|
1520
1668
|
let i;
|
|
1521
1669
|
try {
|
|
1522
|
-
i = await
|
|
1670
|
+
i = await me(L(e, r), { withFileTypes: !0 });
|
|
1523
1671
|
} catch {
|
|
1524
1672
|
return;
|
|
1525
1673
|
}
|
|
@@ -1529,21 +1677,21 @@ voltro data import <this-directory> --target api --api-url <url> --token <secret
|
|
|
1529
1677
|
}
|
|
1530
1678
|
};
|
|
1531
1679
|
return await n(""), t.sort();
|
|
1532
|
-
},
|
|
1680
|
+
}, rr = (e, t) => {
|
|
1533
1681
|
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
1534
|
-
return r.writeUInt8(
|
|
1535
|
-
},
|
|
1682
|
+
return r.writeUInt8($n, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
1683
|
+
}, ir = (e, t, n) => {
|
|
1536
1684
|
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
1537
|
-
return a.writeUInt8(
|
|
1685
|
+
return a.writeUInt8(er, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
1538
1686
|
};
|
|
1539
|
-
async function*
|
|
1540
|
-
yield new Uint8Array(
|
|
1541
|
-
for (let t of await
|
|
1542
|
-
let n =
|
|
1543
|
-
if (yield new Uint8Array(
|
|
1687
|
+
async function* ar(e) {
|
|
1688
|
+
yield new Uint8Array(Q);
|
|
1689
|
+
for (let t of await nr(e.dir)) {
|
|
1690
|
+
let n = L(e.dir, t), r = await he(n);
|
|
1691
|
+
if (yield new Uint8Array(rr(t, r.size)), r.size > 0) for await (let e of fe(n)) yield new Uint8Array(e);
|
|
1544
1692
|
}
|
|
1545
1693
|
if (e.blobs) for await (let t of e.blobs.list()) {
|
|
1546
|
-
yield new Uint8Array(
|
|
1694
|
+
yield new Uint8Array(ir(t.key, t.contentType, t.size));
|
|
1547
1695
|
let n = E("sha256"), r = 0;
|
|
1548
1696
|
for await (let i of e.blobs.open(t.key)) {
|
|
1549
1697
|
let e = Buffer.from(i);
|
|
@@ -1552,11 +1700,11 @@ async function* $n(e) {
|
|
|
1552
1700
|
if (r !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${r}`);
|
|
1553
1701
|
yield new Uint8Array(n.digest());
|
|
1554
1702
|
}
|
|
1555
|
-
yield new Uint8Array(Buffer.from([
|
|
1703
|
+
yield new Uint8Array(Buffer.from([Qn]));
|
|
1556
1704
|
}
|
|
1557
|
-
var
|
|
1558
|
-
let n =
|
|
1559
|
-
await
|
|
1705
|
+
var or = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), sr = (e) => new Promise((t) => e.end(t)), cr = async (e, t) => {
|
|
1706
|
+
let n = R(t.dir);
|
|
1707
|
+
await F(n, { recursive: !0 });
|
|
1560
1708
|
let r = Buffer.alloc(0), i = e[Symbol.asyncIterator](), a = !1, o = async () => {
|
|
1561
1709
|
let { value: e, done: t } = await i.next();
|
|
1562
1710
|
return t ? (a = !0, !1) : (r = r.length === 0 ? Buffer.from(e) : Buffer.concat([r, Buffer.from(e)]), !0);
|
|
@@ -1567,28 +1715,28 @@ var er = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tr
|
|
|
1567
1715
|
return r = r.subarray(e), t;
|
|
1568
1716
|
};
|
|
1569
1717
|
try {
|
|
1570
|
-
if (await s(
|
|
1718
|
+
if (await s(Q.length), !c(Q.length).equals(Q)) throw Error("not a bundle archive (bad magic)");
|
|
1571
1719
|
for (;;) {
|
|
1572
1720
|
await s(1);
|
|
1573
1721
|
let e = c(1).readUInt8(0);
|
|
1574
|
-
if (e ===
|
|
1575
|
-
if (e ===
|
|
1722
|
+
if (e === Qn) return;
|
|
1723
|
+
if (e === $n) {
|
|
1576
1724
|
await s(4);
|
|
1577
1725
|
let e = c(4).readUInt32BE(0);
|
|
1578
1726
|
await s(e);
|
|
1579
1727
|
let t = c(e).toString("utf8");
|
|
1580
1728
|
await s(8);
|
|
1581
|
-
let i = c(8).readBigUInt64BE(0), a =
|
|
1582
|
-
if (a !== n && !a.startsWith(n +
|
|
1583
|
-
await
|
|
1729
|
+
let i = c(8).readBigUInt64BE(0), a = R(n, t);
|
|
1730
|
+
if (a !== n && !a.startsWith(n + ve)) throw Error(`archive path '${t}' escapes the destination`);
|
|
1731
|
+
await F(_e(a), { recursive: !0 });
|
|
1584
1732
|
let o = M(a);
|
|
1585
1733
|
for (; i > 0n;) {
|
|
1586
1734
|
r.length === 0 && await s(1);
|
|
1587
1735
|
let e = i < BigInt(r.length) ? Number(i) : r.length;
|
|
1588
|
-
await
|
|
1736
|
+
await or(o, c(e)), i -= BigInt(e);
|
|
1589
1737
|
}
|
|
1590
|
-
await
|
|
1591
|
-
} else if (e ===
|
|
1738
|
+
await sr(o);
|
|
1739
|
+
} else if (e === er) {
|
|
1592
1740
|
await s(4);
|
|
1593
1741
|
let e = c(4).readUInt32BE(0);
|
|
1594
1742
|
await s(e);
|
|
@@ -1598,24 +1746,24 @@ var er = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tr
|
|
|
1598
1746
|
await s(i);
|
|
1599
1747
|
let a = c(i).toString("utf8");
|
|
1600
1748
|
await s(8);
|
|
1601
|
-
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = E("sha256"), d = new
|
|
1749
|
+
let o = c(8).readBigUInt64BE(0), l = await t.blobs.has(n), u = E("sha256"), d = new A(), f = l ? Promise.resolve() : t.blobs.write(n, d, {
|
|
1602
1750
|
contentType: a,
|
|
1603
1751
|
size: Number(o)
|
|
1604
1752
|
}), p = o;
|
|
1605
1753
|
for (; p > 0n;) {
|
|
1606
1754
|
r.length === 0 && await s(1);
|
|
1607
1755
|
let e = p < BigInt(r.length) ? Number(p) : r.length, t = c(e);
|
|
1608
|
-
u.update(t), l || await
|
|
1756
|
+
u.update(t), l || await or(d, t), p -= BigInt(e);
|
|
1609
1757
|
}
|
|
1610
|
-
l ? d.destroy() : await
|
|
1611
|
-
let m = c(
|
|
1758
|
+
l ? d.destroy() : await sr(d), await f, await s(tr);
|
|
1759
|
+
let m = c(tr);
|
|
1612
1760
|
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
1613
1761
|
} else throw Error(`unknown bundle entry kind ${e}`);
|
|
1614
1762
|
}
|
|
1615
1763
|
} finally {
|
|
1616
1764
|
a || await Promise.resolve(i.return?.()).catch(() => void 0);
|
|
1617
1765
|
}
|
|
1618
|
-
},
|
|
1766
|
+
}, lr = async (e, t) => {
|
|
1619
1767
|
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
1620
1768
|
let { value: e, done: t } = await a.next();
|
|
1621
1769
|
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
@@ -1634,11 +1782,11 @@ var er = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tr
|
|
|
1634
1782
|
}
|
|
1635
1783
|
return Buffer.concat(t);
|
|
1636
1784
|
};
|
|
1637
|
-
if (!await s(
|
|
1785
|
+
if (!await s(Q.length) || !c(Q.length).equals(Q)) throw Error("not a bundle archive (bad magic)");
|
|
1638
1786
|
for (; await s(1);) {
|
|
1639
1787
|
let e = c(1).readUInt8(0);
|
|
1640
|
-
if (e ===
|
|
1641
|
-
if (e !==
|
|
1788
|
+
if (e === Qn || e === er) break;
|
|
1789
|
+
if (e !== $n) throw Error(`unknown bundle entry kind ${e}`);
|
|
1642
1790
|
if (!await s(4)) break;
|
|
1643
1791
|
let t = c(4).readUInt32BE(0);
|
|
1644
1792
|
if (!await s(t)) break;
|
|
@@ -1648,22 +1796,22 @@ var er = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tr
|
|
|
1648
1796
|
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
1649
1797
|
}
|
|
1650
1798
|
return r;
|
|
1651
|
-
},
|
|
1799
|
+
}, ur = (e, t) => cr(e, {
|
|
1652
1800
|
dir: t,
|
|
1653
|
-
blobs:
|
|
1654
|
-
}),
|
|
1801
|
+
blobs: dr(t)
|
|
1802
|
+
}), dr = (e) => {
|
|
1655
1803
|
let t = /* @__PURE__ */ new Set();
|
|
1656
1804
|
return {
|
|
1657
1805
|
has: async (e) => t.has(e),
|
|
1658
1806
|
write: async (n, r, i) => {
|
|
1659
|
-
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c =
|
|
1660
|
-
await
|
|
1807
|
+
let a = await import("node:fs/promises"), { pipeline: o } = await import("node:stream/promises"), { Transform: s } = await import("node:stream"), c = L(e, "assets");
|
|
1808
|
+
await F(c, { recursive: !0 });
|
|
1661
1809
|
let l = E("sha256"), u = 0, d = new s({ transform(e, t, n) {
|
|
1662
1810
|
l.update(e), u += e.length, n(null, e);
|
|
1663
|
-
} }), f =
|
|
1811
|
+
} }), f = L(c, `.tmp-${Date.now()}-${t.size}`);
|
|
1664
1812
|
await o(r, d, M(f));
|
|
1665
1813
|
let p = l.digest("hex");
|
|
1666
|
-
await a.rename(f,
|
|
1814
|
+
await a.rename(f, L(c, p)), await a.appendFile(L(c, "index.ndjson"), `${JSON.stringify({
|
|
1667
1815
|
key: n,
|
|
1668
1816
|
sha256: p,
|
|
1669
1817
|
size: u,
|
|
@@ -1671,33 +1819,33 @@ var er = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), tr
|
|
|
1671
1819
|
})}\n`), t.add(n);
|
|
1672
1820
|
}
|
|
1673
1821
|
};
|
|
1674
|
-
},
|
|
1822
|
+
}, fr = ye(se), pr = Buffer.from("VENC1\0\0\0"), mr = 32, hr = 16, gr = 7, _r = 16, vr = 262144, yr = {
|
|
1675
1823
|
N: 32768,
|
|
1676
1824
|
r: 8,
|
|
1677
1825
|
p: 1
|
|
1678
|
-
},
|
|
1826
|
+
}, br = 134217728, xr = 43, Sr = (e, t, n) => {
|
|
1679
1827
|
let r = Buffer.alloc(12);
|
|
1680
|
-
return e.copy(r, 0, 0,
|
|
1828
|
+
return e.copy(r, 0, 0, gr), r.writeUInt32BE(t >>> 0, gr), r.writeUInt8(+!!n, 11), r;
|
|
1681
1829
|
};
|
|
1682
|
-
async function*
|
|
1830
|
+
async function* Cr(e, t) {
|
|
1683
1831
|
if (!t) throw Error("encryptStream: a non-empty passphrase is required");
|
|
1684
|
-
let n =
|
|
1685
|
-
...
|
|
1686
|
-
maxmem:
|
|
1687
|
-
}), a = Buffer.alloc(
|
|
1688
|
-
|
|
1832
|
+
let n = oe(hr), r = oe(gr), i = await fr(t, n, mr, {
|
|
1833
|
+
...yr,
|
|
1834
|
+
maxmem: br
|
|
1835
|
+
}), a = Buffer.alloc(xr);
|
|
1836
|
+
pr.copy(a, 0), a.writeUInt32BE(yr.N, 8), a.writeUInt32BE(yr.r, 12), a.writeUInt32BE(yr.p, 16), n.copy(a, 20), r.copy(a, 36), yield new Uint8Array(a);
|
|
1689
1837
|
let o = 0, s = (e, t) => {
|
|
1690
|
-
let n =
|
|
1838
|
+
let n = T("aes-256-gcm", i, Sr(r, o, t));
|
|
1691
1839
|
o === 0 && n.setAAD(a);
|
|
1692
1840
|
let s = Buffer.concat([n.update(e), n.final()]), c = n.getAuthTag();
|
|
1693
1841
|
o++;
|
|
1694
|
-
let l = Buffer.alloc(5 + s.length +
|
|
1695
|
-
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length +
|
|
1842
|
+
let l = Buffer.alloc(5 + s.length + _r);
|
|
1843
|
+
return l.writeUInt8(+!!t, 0), l.writeUInt32BE(s.length + _r, 1), s.copy(l, 5), c.copy(l, 5 + s.length), new Uint8Array(l);
|
|
1696
1844
|
}, c = Buffer.alloc(0);
|
|
1697
|
-
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length >
|
|
1845
|
+
for await (let t of e) for (c = c.length === 0 ? Buffer.from(t) : Buffer.concat([c, Buffer.from(t)]); c.length > vr;) yield s(c.subarray(0, vr), !1), c = c.subarray(vr);
|
|
1698
1846
|
yield s(c, !0);
|
|
1699
1847
|
}
|
|
1700
|
-
async function*
|
|
1848
|
+
async function* wr(e, t) {
|
|
1701
1849
|
if (!t) throw Error("decryptStream: a non-empty passphrase is required");
|
|
1702
1850
|
let n = e[Symbol.asyncIterator](), r = Buffer.alloc(0), i = !1, a = async () => {
|
|
1703
1851
|
let { value: e, done: t } = await n.next();
|
|
@@ -1706,16 +1854,16 @@ async function* gr(e, t) {
|
|
|
1706
1854
|
for (; r.length < e && !i && await a(););
|
|
1707
1855
|
return r.length >= e;
|
|
1708
1856
|
};
|
|
1709
|
-
if (!await o(
|
|
1710
|
-
let s = r.subarray(0,
|
|
1711
|
-
if (!s.subarray(0, 8).equals(
|
|
1712
|
-
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36,
|
|
1857
|
+
if (!await o(xr)) throw Error("encrypted bundle truncated (no header)");
|
|
1858
|
+
let s = r.subarray(0, xr);
|
|
1859
|
+
if (!s.subarray(0, 8).equals(pr)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
1860
|
+
let c = s.readUInt32BE(8), l = s.readUInt32BE(12), u = s.readUInt32BE(16), d = Buffer.from(s.subarray(20, 36)), f = Buffer.from(s.subarray(36, xr)), p = await fr(t, d, mr, {
|
|
1713
1861
|
N: c,
|
|
1714
1862
|
r: l,
|
|
1715
1863
|
p: u,
|
|
1716
|
-
maxmem:
|
|
1864
|
+
maxmem: br
|
|
1717
1865
|
});
|
|
1718
|
-
r = r.subarray(
|
|
1866
|
+
r = r.subarray(xr);
|
|
1719
1867
|
let m = 0, h = !1;
|
|
1720
1868
|
for (;;) {
|
|
1721
1869
|
if (!await o(5)) {
|
|
@@ -1724,11 +1872,11 @@ async function* gr(e, t) {
|
|
|
1724
1872
|
}
|
|
1725
1873
|
if (h) throw Error("encrypted bundle has trailing data after the final frame");
|
|
1726
1874
|
let e = r.readUInt8(0) === 1, t = r.readUInt32BE(1);
|
|
1727
|
-
if (t <
|
|
1875
|
+
if (t < _r) throw Error("encrypted bundle corrupt (short frame)");
|
|
1728
1876
|
if (!await o(5 + t)) throw Error("encrypted bundle truncated (frame body)");
|
|
1729
|
-
let n = r.subarray(5, 5 + t -
|
|
1877
|
+
let n = r.subarray(5, 5 + t - _r), i = r.subarray(5 + t - _r, 5 + t);
|
|
1730
1878
|
r = r.subarray(5 + t);
|
|
1731
|
-
let a =
|
|
1879
|
+
let a = ie("aes-256-gcm", p, Sr(f, m, e));
|
|
1732
1880
|
m === 0 && a.setAAD(s), a.setAuthTag(i);
|
|
1733
1881
|
let c = Buffer.concat([a.update(n), a.final()]);
|
|
1734
1882
|
m++, e && (h = !0), c.length > 0 && (yield new Uint8Array(c));
|
|
@@ -1736,7 +1884,7 @@ async function* gr(e, t) {
|
|
|
1736
1884
|
}
|
|
1737
1885
|
//#endregion
|
|
1738
1886
|
//#region src/blobStorage.ts
|
|
1739
|
-
var
|
|
1887
|
+
var Tr = (e, t) => ({
|
|
1740
1888
|
list: () => {
|
|
1741
1889
|
let e = t();
|
|
1742
1890
|
return { async *[Symbol.asyncIterator]() {
|
|
@@ -1749,29 +1897,29 @@ var _r = (e, t) => ({
|
|
|
1749
1897
|
} };
|
|
1750
1898
|
},
|
|
1751
1899
|
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
1752
|
-
let r = await n.runPromise(
|
|
1900
|
+
let r = await n.runPromise(ue(e, t)), i = await O(r.stream);
|
|
1753
1901
|
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
1754
1902
|
} })
|
|
1755
|
-
}),
|
|
1756
|
-
table:
|
|
1903
|
+
}), Er = (e, t) => Tr(e, () => w(t, {
|
|
1904
|
+
table: ce,
|
|
1757
1905
|
chunkSize: 500
|
|
1758
1906
|
}).pipe(l.map((e) => ({
|
|
1759
1907
|
key: String(e.key),
|
|
1760
1908
|
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
1761
1909
|
size: Number(e.size ?? 0)
|
|
1762
|
-
})))),
|
|
1910
|
+
})))), Dr = (e) => ({
|
|
1763
1911
|
has: (t) => n.runPromise(e.head(t).pipe(n.map((e) => e !== null))),
|
|
1764
1912
|
write: async (t, r, i) => {
|
|
1765
|
-
let a =
|
|
1766
|
-
await n.runPromise(
|
|
1913
|
+
let a = le("storage", () => j.from(r));
|
|
1914
|
+
await n.runPromise(D(e, t, a, {
|
|
1767
1915
|
contentType: i.contentType,
|
|
1768
1916
|
totalSize: i.size
|
|
1769
1917
|
}));
|
|
1770
1918
|
}
|
|
1771
|
-
}),
|
|
1919
|
+
}), Or = (e, t) => {
|
|
1772
1920
|
let n = t === "dump" ? "mariadb-dump" : "mariadb", r = t === "dump" ? "mysqldump" : "mysql";
|
|
1773
|
-
return e === "mariadb" &&
|
|
1774
|
-
},
|
|
1921
|
+
return e === "mariadb" && kr(n) ? n : r;
|
|
1922
|
+
}, kr = (e) => (process.env.PATH ?? "").split(I).filter((e) => e !== "").some((t) => N(L(t, e))), Ar = (e) => {
|
|
1775
1923
|
switch (e) {
|
|
1776
1924
|
case "postgres": return "db.dump";
|
|
1777
1925
|
case "mysql":
|
|
@@ -1781,7 +1929,7 @@ var _r = (e, t) => ({
|
|
|
1781
1929
|
case "mssql": return "db.bacpac";
|
|
1782
1930
|
default: return "db.dump";
|
|
1783
1931
|
}
|
|
1784
|
-
},
|
|
1932
|
+
}, jr = (e) => e.filename ?? e.database ?? ((e.url ?? "").replace(/^file:/, "") || "app.sqlite"), $ = (e) => {
|
|
1785
1933
|
if (e.host || e.database) return {
|
|
1786
1934
|
host: e.host ?? "localhost",
|
|
1787
1935
|
port: e.port ?? 3306,
|
|
@@ -1797,7 +1945,7 @@ var _r = (e, t) => ({
|
|
|
1797
1945
|
password: decodeURIComponent(t.password) || "",
|
|
1798
1946
|
database: t.pathname.replace(/^\//, "") || ""
|
|
1799
1947
|
};
|
|
1800
|
-
},
|
|
1948
|
+
}, Mr = (e) => `Server=${e.host},${e.port};Database=${e.database};User ID=${e.username};Password=${e.password};TrustServerCertificate=True`, Nr = (e, t, n) => {
|
|
1801
1949
|
switch (e) {
|
|
1802
1950
|
case "postgres": {
|
|
1803
1951
|
if (t.url) return {
|
|
@@ -1844,7 +1992,7 @@ var _r = (e, t) => ({
|
|
|
1844
1992
|
let r = $(t);
|
|
1845
1993
|
return {
|
|
1846
1994
|
kind: "spawn",
|
|
1847
|
-
tool:
|
|
1995
|
+
tool: Or(e, "dump"),
|
|
1848
1996
|
args: [
|
|
1849
1997
|
"--single-transaction",
|
|
1850
1998
|
"--routines",
|
|
@@ -1866,7 +2014,7 @@ var _r = (e, t) => ({
|
|
|
1866
2014
|
case "sqlite":
|
|
1867
2015
|
case "turso": return {
|
|
1868
2016
|
kind: "copy",
|
|
1869
|
-
from:
|
|
2017
|
+
from: jr(t),
|
|
1870
2018
|
to: n
|
|
1871
2019
|
};
|
|
1872
2020
|
case "mssql": return {
|
|
@@ -1874,7 +2022,7 @@ var _r = (e, t) => ({
|
|
|
1874
2022
|
tool: "sqlpackage",
|
|
1875
2023
|
args: [
|
|
1876
2024
|
"/Action:Export",
|
|
1877
|
-
`/SourceConnectionString:${
|
|
2025
|
+
`/SourceConnectionString:${Mr($({
|
|
1878
2026
|
...t,
|
|
1879
2027
|
port: t.port ?? 1433
|
|
1880
2028
|
}))}`,
|
|
@@ -1882,12 +2030,12 @@ var _r = (e, t) => ({
|
|
|
1882
2030
|
],
|
|
1883
2031
|
env: {}
|
|
1884
2032
|
};
|
|
1885
|
-
default: throw new
|
|
2033
|
+
default: throw new W({
|
|
1886
2034
|
tool: "backup",
|
|
1887
2035
|
reason: `no native backup for dialect '${e}'`
|
|
1888
2036
|
});
|
|
1889
2037
|
}
|
|
1890
|
-
},
|
|
2038
|
+
}, Pr = (e, t, n) => {
|
|
1891
2039
|
switch (e) {
|
|
1892
2040
|
case "postgres": {
|
|
1893
2041
|
if (t.url) return {
|
|
@@ -1932,7 +2080,7 @@ var _r = (e, t) => ({
|
|
|
1932
2080
|
let r = $(t);
|
|
1933
2081
|
return {
|
|
1934
2082
|
kind: "spawn",
|
|
1935
|
-
tool:
|
|
2083
|
+
tool: Or(e, "client"),
|
|
1936
2084
|
args: [
|
|
1937
2085
|
"--protocol=TCP",
|
|
1938
2086
|
"-h",
|
|
@@ -1951,14 +2099,14 @@ var _r = (e, t) => ({
|
|
|
1951
2099
|
case "turso": return {
|
|
1952
2100
|
kind: "copy",
|
|
1953
2101
|
from: n,
|
|
1954
|
-
to:
|
|
2102
|
+
to: jr(t)
|
|
1955
2103
|
};
|
|
1956
2104
|
case "mssql": return {
|
|
1957
2105
|
kind: "spawn",
|
|
1958
2106
|
tool: "sqlpackage",
|
|
1959
2107
|
args: [
|
|
1960
2108
|
"/Action:Import",
|
|
1961
|
-
`/TargetConnectionString:${
|
|
2109
|
+
`/TargetConnectionString:${Mr($({
|
|
1962
2110
|
...t,
|
|
1963
2111
|
port: t.port ?? 1433
|
|
1964
2112
|
}))}`,
|
|
@@ -1966,18 +2114,18 @@ var _r = (e, t) => ({
|
|
|
1966
2114
|
],
|
|
1967
2115
|
env: {}
|
|
1968
2116
|
};
|
|
1969
|
-
default: throw new
|
|
2117
|
+
default: throw new W({
|
|
1970
2118
|
tool: "restore",
|
|
1971
2119
|
reason: `no native restore for dialect '${e}'`
|
|
1972
2120
|
});
|
|
1973
2121
|
}
|
|
1974
|
-
},
|
|
1975
|
-
|
|
2122
|
+
}, Fr = (e) => e.kind === "copy" ? n.async((t) => {
|
|
2123
|
+
de(e.from, e.to, (r) => t(r ? n.fail(new W({
|
|
1976
2124
|
tool: "copy",
|
|
1977
2125
|
reason: `copy ${e.from} → ${e.to}: ${r.message}`
|
|
1978
2126
|
})) : n.void));
|
|
1979
2127
|
}) : n.async((t) => {
|
|
1980
|
-
let r =
|
|
2128
|
+
let r = be(e.tool, [...e.args], {
|
|
1981
2129
|
env: {
|
|
1982
2130
|
...process.env,
|
|
1983
2131
|
...e.env
|
|
@@ -1990,14 +2138,14 @@ var _r = (e, t) => ({
|
|
|
1990
2138
|
}), i = "";
|
|
1991
2139
|
r.stderr?.on("data", (e) => {
|
|
1992
2140
|
i += String(e);
|
|
1993
|
-
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin &&
|
|
2141
|
+
}), e.stdoutFile && r.stdout && r.stdout.pipe(M(e.stdoutFile)), e.stdinFile && r.stdin && fe(e.stdinFile).pipe(r.stdin);
|
|
1994
2142
|
let a = () => {
|
|
1995
2143
|
if (e.stdoutFile !== void 0) try {
|
|
1996
|
-
|
|
2144
|
+
pe(e.stdoutFile);
|
|
1997
2145
|
} catch {}
|
|
1998
2146
|
};
|
|
1999
2147
|
r.on("error", (r) => {
|
|
2000
|
-
a(), t(n.fail(new
|
|
2148
|
+
a(), t(n.fail(new W({
|
|
2001
2149
|
tool: e.tool,
|
|
2002
2150
|
reason: `failed to launch (is '${e.tool}' installed + on PATH?): ${r.message}`
|
|
2003
2151
|
})));
|
|
@@ -2006,17 +2154,17 @@ var _r = (e, t) => ({
|
|
|
2006
2154
|
t(n.void);
|
|
2007
2155
|
return;
|
|
2008
2156
|
}
|
|
2009
|
-
a(), t(n.fail(new
|
|
2157
|
+
a(), t(n.fail(new W({
|
|
2010
2158
|
tool: e.tool,
|
|
2011
|
-
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${
|
|
2159
|
+
reason: `exited with code ${r}` + (e.stdoutFile === void 0 ? "" : ` — the partial ${ge(e.stdoutFile)} was removed, so it cannot be mistaken for a backup`),
|
|
2012
2160
|
stderr: i.slice(-2e3)
|
|
2013
2161
|
})));
|
|
2014
2162
|
});
|
|
2015
|
-
}),
|
|
2016
|
-
let t = yield*
|
|
2163
|
+
}), Ir = (e) => typeof e == "object" && e ? "fake" in e ? `fake:${e.fake}` : "custom" : String(e), Lr = (e) => n.gen(function* () {
|
|
2164
|
+
let t = yield* nt(e.scope ?? { kind: "all" }, e.snapshot, {
|
|
2017
2165
|
store: e.store,
|
|
2018
2166
|
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
2019
|
-
}), r =
|
|
2167
|
+
}), r = qt(t.tables, e.snapshot), i = e.classification ?? Ce(e.snapshot), a = e.sampleSize ?? 20, o = Ie(r.map((t) => ({
|
|
2020
2168
|
name: t,
|
|
2021
2169
|
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
2022
2170
|
name: e.name,
|
|
@@ -2024,19 +2172,19 @@ var _r = (e, t) => ({
|
|
|
2024
2172
|
}))
|
|
2025
2173
|
})), i, e.masking), s = [], c = [];
|
|
2026
2174
|
for (let i of r) {
|
|
2027
|
-
let r = t.predicateFor(i), u = yield*
|
|
2175
|
+
let r = t.predicateFor(i), u = yield* w(e.store, {
|
|
2028
2176
|
table: i,
|
|
2029
2177
|
chunkSize: a,
|
|
2030
2178
|
...r === void 0 ? {} : { where: r }
|
|
2031
|
-
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) =>
|
|
2032
|
-
c.push(...
|
|
2179
|
+
}).pipe(l.take(a), l.runCollect, n.map((e) => [...e])), d = u.map((t) => ze(t, i, o.actions, e.masking.seed));
|
|
2180
|
+
c.push(...Re(d, i, o.actions));
|
|
2033
2181
|
let f = u[0], p = d[0], m = [];
|
|
2034
2182
|
if (f && p) for (let [e, t] of Object.entries(o.actions)) {
|
|
2035
2183
|
if (!e.startsWith(`${i}.`) || t === "keep") continue;
|
|
2036
2184
|
let n = e.slice(i.length + 1);
|
|
2037
2185
|
m.push({
|
|
2038
2186
|
column: n,
|
|
2039
|
-
action:
|
|
2187
|
+
action: Ir(t),
|
|
2040
2188
|
before: f[n],
|
|
2041
2189
|
after: p[n]
|
|
2042
2190
|
});
|
|
@@ -2051,7 +2199,7 @@ var _r = (e, t) => ({
|
|
|
2051
2199
|
leaks: c,
|
|
2052
2200
|
unclassified: o.unclassified
|
|
2053
2201
|
};
|
|
2054
|
-
}),
|
|
2202
|
+
}), Rr = (e, t) => {
|
|
2055
2203
|
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
2056
2204
|
for (let t of e) {
|
|
2057
2205
|
let e = n.get(t.name);
|
|
@@ -2106,7 +2254,7 @@ var _r = (e, t) => ({
|
|
|
2106
2254
|
skipColumns: i,
|
|
2107
2255
|
refuse: r.some((e) => e.verdict === "refuse")
|
|
2108
2256
|
};
|
|
2109
|
-
},
|
|
2257
|
+
}, zr = (e) => {
|
|
2110
2258
|
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
2111
2259
|
let t = {
|
|
2112
2260
|
safe: "✓",
|
|
@@ -2117,4 +2265,4 @@ var _r = (e, t) => ({
|
|
|
2117
2265
|
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
2118
2266
|
};
|
|
2119
2267
|
//#endregion
|
|
2120
|
-
export {
|
|
2268
|
+
export { xt as ASSETS_DIR, St as ASSET_INDEX_FILE, H as BundleError, We as CodecError, Ke as CompressionError, Ze as CrossDialectError, bt as DATA_DIR, Se as DEFAULT_CLASS_ACTIONS, Be as ENVIRONMENT_LOCAL_TABLES, Ve as ENVIRONMENT_PORTABLE_TABLES, ft as FORMAT_VERSION, Xe as ImportModeError, Ge as IntegrityError, Ct as LEDGER_FILE, K as MANIFEST_FILE, Ye as MaskingError, W as NativeToolError, qe as RowsRefusedError, Wn as SAVEPOINT_BATCH_SIZE, bn as STAGING_PREFIX, Je as SchemaDriftError, U as ScopeError, Pe as applyAction, Kn as asImportError, Zn as assessLiveImport, Nt as assetDone, Ar as backupArtifactName, Nr as backupCommand, cn as canonicalType, dn as checkPortability, Hn as checkSchemaDrift, Ce as classificationFromSnapshot, Rr as classifyImportDrift, lt as compressionExt, $e as computeSubsetIds, In as decideStaging, vt as decodeManifest, Ut as decodeRow, Vt as decodeValue, wr as decryptStream, Xn as defineDataProfile, Vn as diffSnapshots, dr as dirBlobSink, Tn as dropStagingSql, At as emptyLedger, _t as encodeManifest, Ht as encodeRow, zt as encodeValue, Cr as encryptStream, Ue as environmentLocalNotice, He as environmentLocalReason, jn as execStatements, zr as formatImportDrift, fn as formatIssue, st as hasZstd, V as isEnvironmentLocal, ut as makeCompressor, dt as makeDecompressor, ze as maskRow, Kt as ndjsonToRows, sn as outsideReferenceRefusal, on as outsideReferencesInto, ar as packBundle, lr as peekBundle, ct as pickCompression, Ie as planMasking, Lr as previewMasking, jt as readLedger, we as resolveAction, nt as resolveScope, ot as restoreAssetFromCas, Pr as restoreCommand, Gt as rowsToNdjson, an as runExport, Yn as runImport, Fr as runNativeStep, Re as scanForLeaks, X as stagingNameFor, it as storageAssetSink, rt as storageAssetSource, Dr as storageBlobSink, Tr as storageBlobSource, Er as storageRefsBlobSource, et as subsetToScope, Mt as tableDone, qt as topoSortTables, cr as unpackBundle, ur as unpackBundleToDir, at as writeAssetToCas, J as writeLedger };
|