@voltro/data-transfer 0.85.0 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +666 -0
- package/THIRD-PARTY-NOTICES.md +5 -33
- package/dist/assets.js +47 -0
- package/dist/blobStorage.js +40 -0
- package/dist/bundleArchive.js +165 -0
- package/dist/bundleCrypto.js +106 -0
- package/dist/casAssetPhase.js +43 -0
- package/dist/compress.js +6 -0
- package/dist/consistency.js +14 -0
- package/dist/crossDialect.js +58 -0
- package/dist/driftClassification.js +58 -0
- package/dist/environmentLocal.js +122 -0
- package/dist/errors.js +82 -0
- package/dist/exporter.js +257 -0
- package/dist/importer.js +536 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +28 -2673
- package/dist/ledger.js +21 -0
- package/dist/liveGuard.js +11 -0
- package/dist/manifest.js +52 -0
- package/dist/masking.js +246 -0
- package/dist/maskingPreview.js +48 -0
- package/dist/nativeDump.js +369 -0
- package/dist/ndjson.js +13 -0
- package/dist/profile.js +4 -0
- package/dist/progress.js +17 -0
- package/dist/readme.js +43 -0
- package/dist/replacePreflight.js +23 -0
- package/dist/rollbackCapture.js +36 -0
- package/dist/rowFailureReason.js +36 -0
- package/dist/schemaDrift.js +40 -0
- package/dist/scope.js +40 -0
- package/dist/stagedSwap.js +164 -0
- package/dist/subset.js +90 -0
- package/dist/topo.js +24 -0
- package/dist/typedCodec.js +36 -0
- package/package.json +3 -3
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
//#region src/environmentLocal.ts
|
|
2
|
+
var e = /* @__PURE__ */ new Map([
|
|
3
|
+
["_voltro_storage_reference_catalogue", "the identity of THIS deployment's storage reference catalogue — a foreign marker would claim that independent metadata stores are one catalogue"],
|
|
4
|
+
["_voltro_inspect_create_receipts", "inspect creation commands accepted by THIS deployment — a foreign receipt would replay or suppress a creation the target never accepted"],
|
|
5
|
+
["_voltro_isr_cache", "pages rendered by THIS deployment’s web processes — a foreign row would serve another environment’s HTML here"],
|
|
6
|
+
["_voltro_web_replicas", "heartbeats of the web processes running HERE — a foreign row advertises replicas that do not exist in the target"],
|
|
7
|
+
["_voltro_aggregate_meta", "materialized refresh generations computed HERE — a foreign row would claim that the target refreshed its own source data"],
|
|
8
|
+
["_voltro_aggregate_rows", "materialized rows tied to this deployment’s refresh generation — the target recomputes them from its own data"],
|
|
9
|
+
["_voltro_timeline", "captured source mutations from this deployment — foreign rows invent local timeline history"],
|
|
10
|
+
["_voltro_timeline_meta", "capture epoch and source cursor of this database — foreign metadata claims continuity with another database"],
|
|
11
|
+
["_voltro_timeline_leases", "active capture readers of this database — foreign leases claim readers that do not exist here"],
|
|
12
|
+
["_voltro_timeline_slots", "bounded capture slots belonging to this source epoch — foreign slots invent local mutation history"],
|
|
13
|
+
["_voltro_plugin_migrations", "plugin migration steps applied to this database — a foreign ledger would skip steps that the target has not run"],
|
|
14
|
+
["_voltro_migration_plans", "the schema fingerprint THIS deployment applied — a foreign row makes the next boot refuse"],
|
|
15
|
+
["_voltro_migrations", "which file-based migrations ran HERE — a foreign row marks unrun migrations as done"],
|
|
16
|
+
["_voltro_plugin_migrations", "which plugin migration steps ran HERE — a foreign row marks a plugin's unrun steps as done"],
|
|
17
|
+
["_voltro_schema_owner", "which app owns THIS database — a foreign row hands the target's schema to the source app, so the target's own app refuses to boot on it"],
|
|
18
|
+
["_voltro_deploys", "the deploy THIS database is in the middle of, with the baseline to undo it from — a foreign row makes the next apply undo a deploy the target never started"],
|
|
19
|
+
["_voltro_devices", "devices paired with THIS deployment — token hashes; a foreign row makes a device paired elsewhere authenticate here"],
|
|
20
|
+
["_voltro_seeds", "which seeds ran HERE — a foreign row suppresses seeding the target still needs"],
|
|
21
|
+
["_voltro_cdc_offsets", "binlog/WAL positions of the SOURCE server — meaningless and unsafe against another one"],
|
|
22
|
+
["_voltro_notification_push_subscriptions", "browser push endpoints bound to THIS deployment's VAPID key — the browser's applicationServerKey is derived from `VOLTRO_VAPID_PRIVATE_KEY`, so a push service refuses a delivery signed by another one. A foreign row makes the target attempt a delivery that must fail, and the auto-prune then deletes a subscription that was valid where it came from"],
|
|
23
|
+
["_voltro_schedule_claims", "live leases held by THIS deployment's replicas — a foreign row hands out a lease nobody holds"],
|
|
24
|
+
["_voltro_change_claims", "the record that ONE of THIS deployment's replicas already handled a given change — a foreign row asserts that an effect fired here when it fired somewhere else, so the subscriber or reaction it gates stands down for a notification nobody sent"],
|
|
25
|
+
["_voltro_replica_observations", "what THIS deployment's replicas last published about themselves — a foreign row invents a replica that does not exist here, and a fleet answer would then report it as present or, once it ages, as a replica that has stopped answering"],
|
|
26
|
+
["_voltro_data_transfers", "which bundles moved through HERE, in or out — a foreign row claims a transfer this deployment never ran"],
|
|
27
|
+
["_voltro_replace_in_progress", "the marker a destructive import HERE left behind — a foreign row refuses a boot over a healthy database"],
|
|
28
|
+
["_voltro_wakeups", "pending wakeups addressed to THIS deployment's runners"],
|
|
29
|
+
["_voltro_workflow_watermarks", "how far a consumer has read THIS deployment's event journal — a foreign position skips events"],
|
|
30
|
+
["_voltro_workflow_pending", "starts waiting to be drained HERE — a foreign row makes the target RUN a workflow somebody queued elsewhere"],
|
|
31
|
+
["_voltro_workflow_admissions", "the in-flight accounting a concurrency gate decides on — a foreign row makes the target refuse or admit wrongly"],
|
|
32
|
+
["_voltro_workflow_pauses", "which workflows an operator paused HERE — a foreign row silently pauses them in the target"],
|
|
33
|
+
["_voltro_workflow_event_deliveries", "which (event, trigger) pairs were already delivered HERE — a foreign row SUPPRESSES a delivery the target owes"],
|
|
34
|
+
["_voltro_traces", "spans recorded HERE — a foreign span claims a request this deployment never served"],
|
|
35
|
+
["_voltro_undo_log", "the before-images of mutations made HERE — a foreign row offers to undo a write into rows the target does not have"],
|
|
36
|
+
["_voltro_outbox", "messages waiting to be SENT by this deployment — a foreign row makes the target deliver something queued elsewhere"],
|
|
37
|
+
["_voltro_outbox_attempts", "the delivery attempts this deployment made — they belong to its outbox rows and to no other"],
|
|
38
|
+
["_voltro_ai_inferences", "offloaded inference work queued for THIS deployment's runners — a foreign row makes the target run somebody else's"],
|
|
39
|
+
["_voltro_ai_budget", "the live spend RESERVATION counter a budget refuses against — a foreign row makes the target refuse calls it should allow"],
|
|
40
|
+
["_voltro_ai_usage", "what this deployment spent on model calls — a foreign row overstates a bill nobody here ran up"],
|
|
41
|
+
["_voltro_budget_holds", "in-flight holds against a running deployment's budget — the same accounting `_voltro_workflow_admissions` keeps"],
|
|
42
|
+
["_voltro_ai_budget_reservations", "reservations held against THIS deployment's live spend counter — a foreign row makes the target refuse calls it should allow, exactly as `_voltro_ai_budget` does"],
|
|
43
|
+
["_voltro_storage_grants", "short-lived upload/download grants issued HERE — a foreign grant authorises nothing in the target"],
|
|
44
|
+
["_voltro_storage_usage", "how much storage this deployment accounts for — a foreign row makes a quota decision on somebody else's bytes"],
|
|
45
|
+
["_voltro_storage_operations", "the puts and deletes in flight against THIS provider, by physical key — a foreign row makes the target finish or undo an operation on a key it does not have"],
|
|
46
|
+
["_voltro_storage_object_heads", "which physical generation of each content-addressed object is live HERE — a foreign row points the target at a key it does not own"],
|
|
47
|
+
["_voltro_storage_object_claims", "which callers HERE still hold a shared object — a foreign row keeps the target retaining a key for a claimant it never had, or releases one it does"],
|
|
48
|
+
["_voltro_storage_put_intents", "the ref a reservation HERE will become — a foreign row makes the target confirm a write it never received"],
|
|
49
|
+
["_voltro_storage_operation_locks", "the per-tenant serialisation rows the operation transactions lock — a foreign row is a lock nobody here holds"],
|
|
50
|
+
["_voltro_idempotency", "the request keys this deployment has already answered — a foreign key SUPPRESSES a request the target never served"],
|
|
51
|
+
["_voltro_webhook_deliveries", "the delivery attempts this deployment made to a subscriber — a foreign row claims a call the target never placed"],
|
|
52
|
+
["_voltro_schedule_runs", "every schedule firing that happened HERE, replica ids included — a foreign row is history the target did not live"],
|
|
53
|
+
["_voltro_schedule_backfill_requests", "backfills queued for THIS deployment's runners — a foreign row makes the target re-run a range somebody asked another deployment for"],
|
|
54
|
+
["_voltro_tenant_namespaces", "which physical namespace each tenant lives in HERE — a foreign row points a tenant at a namespace the target does not have"],
|
|
55
|
+
["_voltro_workflow_execution_placements", "which physical store holds each execution HERE — a foreign row sends the target looking in a database it does not own"],
|
|
56
|
+
["_voltro_workflow_pool_permits", "concurrency permits held against a running deployment's budget — the same accounting `_voltro_workflow_admissions` keeps"],
|
|
57
|
+
["_voltro_workflow_admission_locks", "the locks an admission decision is being made under RIGHT NOW — a foreign row blocks or releases a decision the target is not making"],
|
|
58
|
+
["_voltro_workflow_submissions", "starts submitted HERE and not yet admitted — a foreign row makes the target admit a start somebody handed to another deployment"],
|
|
59
|
+
["_voltro_workflow_start_receipts", "the caller-facing identity of a start placed HERE, resolved through this deployment's placements — a foreign receipt names a run the target never accepted"],
|
|
60
|
+
["_voltro_workflow_control_requests", "cancel/resume traffic issued against executions HERE — a foreign row claims an operation nobody asked the target to perform"],
|
|
61
|
+
["_voltro_workflow_queued_cancellations", "cancellations waiting on a run that is queued HERE — a foreign row cancels something the target has not queued"],
|
|
62
|
+
["_voltro_workflow_restart_withdrawals", "withdrawal fences for restart requests issued HERE — a foreign row would block a target restart that its operator never withdrew"],
|
|
63
|
+
["_voltro_workflow_restart_requests", "restarts pending against this deployment's generations — a foreign row restarts a generation the target never entered"],
|
|
64
|
+
["_voltro_workflow_retry_requests", "retry intents an operator raised against runs HERE, keyed by their own request id — a foreign row binds the target to an intent nobody raised against it, and lets a later observation read somebody else's execution"],
|
|
65
|
+
["_voltro_workflow_parent_closures", "which parents are still waiting on children in flight HERE — a foreign row holds a parent open against a child the target is not running"],
|
|
66
|
+
["_voltro_workflow_failure_reports", "failures being handed off to a runner HERE — a foreign row makes the target take delivery of somebody else's failure"],
|
|
67
|
+
["_voltro_cdc_log", "the change rows this deployment captured from its own database — a foreign row replays a change that never happened here"],
|
|
68
|
+
["_voltro_notification_deliveries", "the notification attempts this deployment made — a foreign row claims a send the target never performed"],
|
|
69
|
+
["_voltro_notification_held", "notifications waiting to be SENT from here — a foreign row makes the target deliver something queued elsewhere"],
|
|
70
|
+
["_voltro_webhook_event_stats", "per-deployment delivery counters a backoff decision reads — foreign counts steer the target's retries"],
|
|
71
|
+
["_voltro_webhook_rate_windows", "the live rate-limit windows this deployment is inside — a foreign window throttles or releases wrongly"],
|
|
72
|
+
["cluster_runners", "the workflow runners of THIS deployment and their addresses — a foreign row routes shards to a runner that does not exist here"],
|
|
73
|
+
["cluster_locks", "the shard leases THIS deployment's runners hold — a foreign row holds a shard for nobody, and its work stalls"],
|
|
74
|
+
["cluster_messages", "workflow messages in flight HERE — a foreign row makes the target run work that was started, and may be running, elsewhere"],
|
|
75
|
+
["cluster_replies", "replies to those messages — a foreign row answers a request the target never sent"],
|
|
76
|
+
["cluster_migrations", "which engine migrations ran HERE — a foreign row marks the target's engine tables migrated when they are not"]
|
|
77
|
+
]), t = /* @__PURE__ */ new Map([
|
|
78
|
+
["_voltro_comment_thread_events", "the status history of the app's comment threads — belongs with those threads"],
|
|
79
|
+
["_voltro_agent_threads", "conversations the app's users had — app data"],
|
|
80
|
+
["_voltro_agent_messages", "the messages in those conversations — app data"],
|
|
81
|
+
["_voltro_prompts", "prompt definitions the app authored — app data"],
|
|
82
|
+
["_voltro_api_keys", "keys the app issued (hashed) — app data, and a restore without them locks callers out"],
|
|
83
|
+
["_voltro_tenant_selections", "the tenant each user chose to act in — app data, verified against memberships on every request"],
|
|
84
|
+
["_voltro_kv", "the app's own key/value rows — app data by construction"],
|
|
85
|
+
["_voltro_approvals", "human decisions on workflow runs — they belong with the runs"],
|
|
86
|
+
["_voltro_connections", "third-party connections the app's users authorised. Portable as ROWS; note the secrets are encrypted with the project's key, so a target without that key holds ciphertext it cannot read"],
|
|
87
|
+
["_voltro_connection_grants", "who may use which connection — belongs with the connections"],
|
|
88
|
+
["_voltro_storage_blobs", "the stored objects' metadata — the assets travel with the bundle"],
|
|
89
|
+
["_voltro_storage_refs", "which rows reference which blob — belongs with the blobs"],
|
|
90
|
+
["_voltro_webhook_targets", "the endpoints subscribers registered through the app — app data"],
|
|
91
|
+
["_voltro_workflow_runs", "what the app's workflows did — history the app owns"],
|
|
92
|
+
["_voltro_workflow_run_steps", "the steps of those runs — belongs with the runs"],
|
|
93
|
+
["_voltro_workflow_run_events", "the events those runs emitted — belongs with the runs"],
|
|
94
|
+
["_voltro_workflow_events", "the app's event journal — app data (the target keeps its OWN watermarks, which are local)"],
|
|
95
|
+
["_voltro_workflow_start_contexts", "the context a start was made with — belongs with the runs"],
|
|
96
|
+
["_voltro_events", "the app's domain events — app data"],
|
|
97
|
+
["_voltro_ai_flows", "the flow definitions the app authored — app data"],
|
|
98
|
+
["_voltro_ai_flow_runs", "what those flows did — history the app owns, like a workflow run"],
|
|
99
|
+
["_voltro_notification_inbox", "what the app's users received — app data, and a restore without it loses their inbox"],
|
|
100
|
+
["_voltro_notification_preferences", "the per-user delivery settings the app's users chose — app data"],
|
|
101
|
+
["_voltro_notification_quiet_hours", "the per-user quiet hours the app's users chose — app data"],
|
|
102
|
+
["_voltro_notification_topic_subscriptions", "which topics the app's users subscribed to — app data"],
|
|
103
|
+
["_voltro_rebac_tuples", "the relationship tuples the app's authorisation decisions are made from — app data"],
|
|
104
|
+
["_voltro_scim_users", "identities an external directory provisioned into the app — app data"],
|
|
105
|
+
["_voltro_scim_groups", "groups an external directory provisioned into the app — app data"],
|
|
106
|
+
["_voltro_row_history", "the app's own row-version history — the point of keeping versions is that they survive a move"],
|
|
107
|
+
["_voltro_crdt_versions", "the version history of the app's collaborative documents (named snapshots, archived lineages) — it restores the document rows that travel with it"],
|
|
108
|
+
["_voltro_audit_log", "the app's audit trail — compliance data the app owns, not deployment bookkeeping"],
|
|
109
|
+
["_voltro_feature_flags", "the flags the app defined — app configuration"],
|
|
110
|
+
["_voltro_feature_flag_audit", "who changed which flag — belongs with the flags"],
|
|
111
|
+
["_voltro_comments", "the comments the app's users wrote — app data"],
|
|
112
|
+
["_voltro_comment_threads", "the threads those comments hang on — app data"],
|
|
113
|
+
["_voltro_comment_reactions", "the reactions the app's users left — app data"],
|
|
114
|
+
["_voltro_comment_reads", "which comments each user has read — app data, and losing it re-marks everything unread"],
|
|
115
|
+
["_voltro_comment_thread_events", "the status changes the app's users made on a thread — app data"]
|
|
116
|
+
]), n = (t) => e.has(t), r = (t) => e.get(t), i = (e, t) => {
|
|
117
|
+
if (e.length === 0) return;
|
|
118
|
+
let n = e.map((e) => ` ${e} — ${r(e) ?? "describes this deployment"}`);
|
|
119
|
+
return `${t} ${e.length} table(s) that describe a deployment rather than its data:\n${n.join("\n")}\n A row from another environment is not stale there, it is a false statement about this one.`;
|
|
120
|
+
};
|
|
121
|
+
//#endregion
|
|
122
|
+
export { e as ENVIRONMENT_LOCAL_TABLES, t as ENVIRONMENT_PORTABLE_TABLES, i as environmentLocalNotice, r as environmentLocalReason, n as isEnvironmentLocal };
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { Data as e, Schema as t } from "effect";
|
|
2
|
+
//#region src/errors.ts
|
|
3
|
+
var n = class extends t.TaggedError()("BundleError", {
|
|
4
|
+
reason: t.String,
|
|
5
|
+
path: t.optional(t.String)
|
|
6
|
+
}) {
|
|
7
|
+
get message() {
|
|
8
|
+
return this.path === void 0 ? this.reason : `${this.reason} (${this.path})`;
|
|
9
|
+
}
|
|
10
|
+
}, r = class extends t.TaggedError()("CodecError", {
|
|
11
|
+
table: t.String,
|
|
12
|
+
reason: t.String
|
|
13
|
+
}) {}, i = class extends t.TaggedError()("IntegrityError", {
|
|
14
|
+
what: t.String,
|
|
15
|
+
expected: t.String,
|
|
16
|
+
actual: t.String
|
|
17
|
+
}) {
|
|
18
|
+
get message() {
|
|
19
|
+
return `${this.what}: expected ${this.expected}, got ${this.actual}`;
|
|
20
|
+
}
|
|
21
|
+
}, a = class extends e.TaggedError("CompressionError") {}, o = class extends t.TaggedError()("ScopeError", { reason: t.String }) {
|
|
22
|
+
get message() {
|
|
23
|
+
return this.reason;
|
|
24
|
+
}
|
|
25
|
+
}, s = class extends t.TaggedError()("RowsRefusedError", {
|
|
26
|
+
rows: t.Array(t.Struct({
|
|
27
|
+
table: t.String,
|
|
28
|
+
id: t.String,
|
|
29
|
+
reason: t.String,
|
|
30
|
+
derived: t.Boolean
|
|
31
|
+
})),
|
|
32
|
+
primaryCount: t.Number,
|
|
33
|
+
totalCount: t.Number,
|
|
34
|
+
byTable: t.Array(t.Struct({
|
|
35
|
+
table: t.String,
|
|
36
|
+
refused: t.Number,
|
|
37
|
+
primary: t.Number
|
|
38
|
+
}))
|
|
39
|
+
}) {
|
|
40
|
+
get message() {
|
|
41
|
+
return `RowsRefusedError: ${JSON.stringify({
|
|
42
|
+
rows: this.rows,
|
|
43
|
+
primaryCount: this.primaryCount,
|
|
44
|
+
totalCount: this.totalCount,
|
|
45
|
+
byTable: this.byTable
|
|
46
|
+
})}`;
|
|
47
|
+
}
|
|
48
|
+
}, c = class extends t.TaggedError()("SchemaDriftError", {
|
|
49
|
+
bundleFingerprint: t.String,
|
|
50
|
+
targetFingerprint: t.String,
|
|
51
|
+
diff: t.Array(t.String)
|
|
52
|
+
}) {
|
|
53
|
+
get message() {
|
|
54
|
+
return `bundle schema ${this.bundleFingerprint} does not match target ${this.targetFingerprint}: ${this.diff.join("; ")}`;
|
|
55
|
+
}
|
|
56
|
+
}, l = class extends e.TaggedError("NativeToolError") {
|
|
57
|
+
get message() {
|
|
58
|
+
let e = this.stderr?.trim(), t = this.hint?.trim();
|
|
59
|
+
return `${this.tool}: ${this.reason}${e ? `\n${e}` : ""}${t ? `\n\nhint: ${t}` : ""}`;
|
|
60
|
+
}
|
|
61
|
+
}, u = class extends t.TaggedError()("MaskingError", {
|
|
62
|
+
unclassified: t.Array(t.String),
|
|
63
|
+
invalidActions: t.optional(t.Array(t.String))
|
|
64
|
+
}) {
|
|
65
|
+
get message() {
|
|
66
|
+
let e = [];
|
|
67
|
+
return this.unclassified.length > 0 && e.push(`unclassified columns: ${this.unclassified.join(", ")}`), this.invalidActions !== void 0 && this.invalidActions.length > 0 && e.push(`invalidActions: ${this.invalidActions.join(", ")}`), `masking refused — ${e.join("; ") || "no column may be exported"}`;
|
|
68
|
+
}
|
|
69
|
+
}, d = class extends t.TaggedError()("ImportModeError", {
|
|
70
|
+
mode: t.String,
|
|
71
|
+
reason: t.String
|
|
72
|
+
}) {
|
|
73
|
+
get message() {
|
|
74
|
+
return `import mode '${this.mode}' refused: ${this.reason}`;
|
|
75
|
+
}
|
|
76
|
+
}, f = class extends t.TaggedError()("CrossDialectError", {
|
|
77
|
+
sourceDialect: t.String,
|
|
78
|
+
targetDialect: t.String,
|
|
79
|
+
blocking: t.Array(t.String)
|
|
80
|
+
}) {};
|
|
81
|
+
//#endregion
|
|
82
|
+
export { n as BundleError, r as CodecError, a as CompressionError, f as CrossDialectError, d as ImportModeError, i as IntegrityError, u as MaskingError, l as NativeToolError, s as RowsRefusedError, c as SchemaDriftError, o as ScopeError };
|
package/dist/exporter.js
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import { snapshotPrelude as e } from "./consistency.js";
|
|
2
|
+
import { classificationFromSnapshot as t, maskRow as n, planMasking as r } from "./masking.js";
|
|
3
|
+
import { BundleError as i, MaskingError as a, ScopeError as o } from "./errors.js";
|
|
4
|
+
import { computeSubsetIds as s, subsetToScope as c } from "./subset.js";
|
|
5
|
+
import { resolveScope as l } from "./scope.js";
|
|
6
|
+
import { captureAssetsToCas as u } from "./casAssetPhase.js";
|
|
7
|
+
import { compressionExt as d, makeCompressor as f, pickCompression as p } from "./compress.js";
|
|
8
|
+
import { ASSETS_DIR as m, ASSET_INDEX_FILE as h, DATA_DIR as g, LEDGER_FILE as _, MANIFEST_FILE as v, README_FILE as y, encodeManifest as b } from "./manifest.js";
|
|
9
|
+
import { buildReadme as x } from "./readme.js";
|
|
10
|
+
import { reportProgress as S, withTableSpan as C } from "./progress.js";
|
|
11
|
+
import { readLedger as w, tableDone as T, writeLedger as E } from "./ledger.js";
|
|
12
|
+
import { rowsToNdjson as D } from "./ndjson.js";
|
|
13
|
+
import { topoSortTables as O } from "./topo.js";
|
|
14
|
+
import { Cause as k, Effect as A, Exit as j, Schedule as M, Stream as N } from "effect";
|
|
15
|
+
import { streamTable as P } from "@voltro/database";
|
|
16
|
+
import { fingerprintSchema as F } from "@voltro/database/sql";
|
|
17
|
+
//#region src/exporter.ts
|
|
18
|
+
var I = (e) => {
|
|
19
|
+
let t = e, n = [];
|
|
20
|
+
typeof t?._tag == "string" && t._tag !== "" && n.push(t._tag);
|
|
21
|
+
let r = typeof t?.message == "string" ? t.message.trim() : "";
|
|
22
|
+
if (r === "" ? t?.cause !== void 0 && t.cause !== null && n.push(I(t.cause)) : n.push(r), n.length === 0) {
|
|
23
|
+
let t = String(e ?? "").trim();
|
|
24
|
+
return t === "" || t === "[object Object]" ? L(e) : t;
|
|
25
|
+
}
|
|
26
|
+
return n.join(": ");
|
|
27
|
+
}, L = (e) => {
|
|
28
|
+
try {
|
|
29
|
+
let t = JSON.stringify(e);
|
|
30
|
+
return t === void 0 || t === "{}" ? "unknown error (no message, cause or tag)" : t;
|
|
31
|
+
} catch {
|
|
32
|
+
return "unknown error (not serialisable)";
|
|
33
|
+
}
|
|
34
|
+
}, R = (e, t) => {
|
|
35
|
+
let n = e.tables.find((e) => e.name === t), r = {};
|
|
36
|
+
for (let e of n?.columns ?? []) e.generatedAs === void 0 && (r[e.name] = e.type);
|
|
37
|
+
return r;
|
|
38
|
+
}, z = (e, t) => (e.tables.find((e) => e.name === t)?.columns ?? []).filter((e) => e.generatedAs !== void 0).map((e) => e.name), B = (e, t) => {
|
|
39
|
+
if (t.length === 0) return e;
|
|
40
|
+
let n = { ...e };
|
|
41
|
+
for (let e of t) delete n[e];
|
|
42
|
+
return n;
|
|
43
|
+
}, V = (e, t) => {
|
|
44
|
+
let n = e.tables.find((e) => e.name === t);
|
|
45
|
+
if (n === void 0) return { refusal: `${t}: not in the schema snapshot, so its primary key is unknown.` };
|
|
46
|
+
let r = n.columns.find((e) => e.type === "id");
|
|
47
|
+
if (r !== void 0) return { column: r.name };
|
|
48
|
+
let i = n.indexes.find((e) => e.name === `${t}_pkey`)?.columns ?? n.primaryKey ?? [];
|
|
49
|
+
return i.length === 1 ? { column: i[0] } : i.length > 1 ? { refusal: `${t}: composite primary key (${i.join(", ")}). Keyset export orders by ONE column; ordering by the first alone would split equal values across pages and drop or duplicate rows. Exclude this table from the scope, or give it a single-column key.` } : { refusal: `${t}: no primary key, so there is no stable order to paginate by. A keyset export would return rows in an order the database may change between pages. Exclude this table from the scope.` };
|
|
50
|
+
}, H = (e, t, n) => A.tryPromise({
|
|
51
|
+
try: async () => {
|
|
52
|
+
let { createWriteStream: r } = await import("node:fs"), i = await import("node:fs/promises"), { createHash: a, randomUUID: o } = await import("node:crypto"), { pipeline: s } = await import("node:stream/promises"), { Readable: c, Transform: l } = await import("node:stream"), u = a("sha256"), d = 0, p = new l({ transform(e, t, n) {
|
|
53
|
+
u.update(e);
|
|
54
|
+
let r = e;
|
|
55
|
+
for (let e = 0; e < r.length; e++) r[e] === 10 && d++;
|
|
56
|
+
n(null, e);
|
|
57
|
+
} }), m = c.fromWeb(N.toReadableStream(e)), h = `${t}.tmp-${o()}`;
|
|
58
|
+
try {
|
|
59
|
+
await s(m, p, f(n), r(h)), await i.rename(h, t);
|
|
60
|
+
} catch (e) {
|
|
61
|
+
throw await i.rm(h, { force: !0 }).catch(() => {}), e;
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
rowCount: d,
|
|
65
|
+
checksum: u.digest("hex")
|
|
66
|
+
};
|
|
67
|
+
},
|
|
68
|
+
catch: (e) => new i({
|
|
69
|
+
reason: `write table failed: ${I(e)}`,
|
|
70
|
+
path: t
|
|
71
|
+
})
|
|
72
|
+
}), U = (e, t) => A.tryPromise({
|
|
73
|
+
try: async () => {
|
|
74
|
+
await (await import("node:fs/promises")).appendFile(e, `${t}\n`);
|
|
75
|
+
},
|
|
76
|
+
catch: (t) => new i({
|
|
77
|
+
reason: `append failed: ${I(t)}`,
|
|
78
|
+
path: e
|
|
79
|
+
})
|
|
80
|
+
}), W = (e, t) => A.gen(function* () {
|
|
81
|
+
let r = [], i = t.ordered.length;
|
|
82
|
+
for (let a = 0; a < t.ordered.length; a++) {
|
|
83
|
+
let s = t.ordered[a], c = R(t.snapshot, s), l = V(t.snapshot, s);
|
|
84
|
+
if ("refusal" in l) return yield* A.fail(new o({ reason: `export: ${l.refusal}` }));
|
|
85
|
+
let u = l.column, f = `${g}/${s}.ndjson${d(t.compression)}`;
|
|
86
|
+
if (T(t.ledger, s)) {
|
|
87
|
+
let e = t.ledger.tables[s];
|
|
88
|
+
S(t.onProgress, {
|
|
89
|
+
phase: "export",
|
|
90
|
+
table: s,
|
|
91
|
+
index: a,
|
|
92
|
+
tableCount: i,
|
|
93
|
+
event: "done",
|
|
94
|
+
rowsDone: e.rowCount,
|
|
95
|
+
total: e.rowCount
|
|
96
|
+
}), r.push({
|
|
97
|
+
name: s,
|
|
98
|
+
file: f,
|
|
99
|
+
rowCount: e.rowCount,
|
|
100
|
+
checksum: e.checksum,
|
|
101
|
+
columnTypes: c,
|
|
102
|
+
primaryKey: u
|
|
103
|
+
});
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
S(t.onProgress, {
|
|
107
|
+
phase: "export",
|
|
108
|
+
table: s,
|
|
109
|
+
index: a,
|
|
110
|
+
tableCount: i,
|
|
111
|
+
event: "start",
|
|
112
|
+
rowsDone: 0
|
|
113
|
+
});
|
|
114
|
+
let p = t.resolved.predicateFor(s), m = P(e, {
|
|
115
|
+
table: s,
|
|
116
|
+
pk: u,
|
|
117
|
+
chunkSize: t.chunkSize,
|
|
118
|
+
retry: t.retry,
|
|
119
|
+
...p === void 0 ? {} : { where: p }
|
|
120
|
+
}), h = z(t.snapshot, s), _ = h.length === 0 ? m : m.pipe(N.map((e) => B(e, h))), v = t.mask ? _.pipe(N.map((e) => n(e, s, t.mask.actions, t.mask.seed, c))) : _, { rowCount: y, checksum: b } = yield* C("export", s, a, i, (e) => e.rowCount, H(D(v, c), t.bundleFile(f), t.compression));
|
|
121
|
+
t.ledger.tables[s] = {
|
|
122
|
+
rowCount: y,
|
|
123
|
+
checksum: b
|
|
124
|
+
}, yield* E(t.ledgerPath, t.ledger), S(t.onProgress, {
|
|
125
|
+
phase: "export",
|
|
126
|
+
table: s,
|
|
127
|
+
index: a,
|
|
128
|
+
tableCount: i,
|
|
129
|
+
event: "done",
|
|
130
|
+
rowsDone: y,
|
|
131
|
+
total: y
|
|
132
|
+
}), r.push({
|
|
133
|
+
name: s,
|
|
134
|
+
file: f,
|
|
135
|
+
rowCount: y,
|
|
136
|
+
checksum: b,
|
|
137
|
+
columnTypes: c,
|
|
138
|
+
primaryKey: u
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
return r;
|
|
142
|
+
}), G = (e) => ({
|
|
143
|
+
_tag: "RawSqlFragment",
|
|
144
|
+
strings: [e],
|
|
145
|
+
values: []
|
|
146
|
+
}), K = (t, n, r) => A.tryPromise({
|
|
147
|
+
try: () => t.transactional(async (t) => {
|
|
148
|
+
if (t.raw) for (let r of e(n)) await t.raw(G(r));
|
|
149
|
+
let i = await A.runPromiseExit(r(t));
|
|
150
|
+
if (j.isFailure(i)) throw Error(k.pretty(i.cause));
|
|
151
|
+
return i.value;
|
|
152
|
+
}),
|
|
153
|
+
catch: (e) => e instanceof i ? e : new i({ reason: `snapshot export failed: ${I(e)}` })
|
|
154
|
+
}), q = (e) => A.gen(function* () {
|
|
155
|
+
let n = e.scope ?? { kind: "all" }, o = p(e.compression), d = e.chunkSize ?? 1e3, f = e.retryTimes ?? 5, S = M.max([M.exponential("50 millis").pipe(M.jittered), M.recurs(f)]), C = yield* A.promise(() => import("node:path")), T = (t) => C.join(e.outDir, t);
|
|
156
|
+
yield* A.tryPromise({
|
|
157
|
+
try: async () => {
|
|
158
|
+
let t = await import("node:fs/promises");
|
|
159
|
+
await t.mkdir(T(g), { recursive: !0 }), e.assets && await t.mkdir(T(m), { recursive: !0 });
|
|
160
|
+
},
|
|
161
|
+
catch: (t) => new i({
|
|
162
|
+
reason: `mkdir failed: ${I(t)}`,
|
|
163
|
+
path: e.outDir
|
|
164
|
+
})
|
|
165
|
+
});
|
|
166
|
+
let D = T(_), k = yield* w(D), j = e.subset ? c(yield* s(e.store, e.snapshot, e.subset), e.snapshot) : yield* l(n, e.snapshot, {
|
|
167
|
+
store: e.store,
|
|
168
|
+
...e.tenantTables ? { tenantTables: e.tenantTables } : {}
|
|
169
|
+
}), N = O(j.tables, e.snapshot), P = e.consistency ?? "live", L, R;
|
|
170
|
+
if (e.masking) {
|
|
171
|
+
let n = e.classification ?? t(e.snapshot), i = N.map((t) => ({
|
|
172
|
+
name: t,
|
|
173
|
+
columns: (e.snapshot.tables.find((e) => e.name === t)?.columns ?? []).map((e) => ({
|
|
174
|
+
name: e.name,
|
|
175
|
+
type: e.type
|
|
176
|
+
}))
|
|
177
|
+
})), o = r(i, n, e.masking);
|
|
178
|
+
if (o.unclassified.length > 0 || o.invalid.length > 0) return yield* A.fail(new a({
|
|
179
|
+
unclassified: o.unclassified,
|
|
180
|
+
...o.invalid.length > 0 ? { invalidActions: o.invalid.map((e) => `${e.column}: ${JSON.stringify(e.action)}`) } : {}
|
|
181
|
+
}));
|
|
182
|
+
R = {
|
|
183
|
+
actions: o.actions,
|
|
184
|
+
seed: e.masking.seed
|
|
185
|
+
};
|
|
186
|
+
let s = Object.entries(o.actions).filter(([, e]) => e !== "keep").map(([e]) => e);
|
|
187
|
+
L = {
|
|
188
|
+
...e.masking.id ? { policyId: e.masking.id } : {},
|
|
189
|
+
transformed: s
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
let z = {
|
|
193
|
+
snapshot: e.snapshot,
|
|
194
|
+
ordered: N,
|
|
195
|
+
resolved: j,
|
|
196
|
+
compression: o,
|
|
197
|
+
chunkSize: d,
|
|
198
|
+
retry: S,
|
|
199
|
+
ledger: k,
|
|
200
|
+
ledgerPath: D,
|
|
201
|
+
bundleFile: T,
|
|
202
|
+
...R ? { mask: R } : {},
|
|
203
|
+
...e.onProgress ? { onProgress: e.onProgress } : {}
|
|
204
|
+
}, B = P === "snapshot" ? yield* K(e.store, e.dialect, (e) => W(e, z)) : yield* W(e.store, z), V = 0, H = 0;
|
|
205
|
+
if (e.assets) {
|
|
206
|
+
let t = T(h), n = yield* u({
|
|
207
|
+
source: e.assets,
|
|
208
|
+
assetsDir: T(m),
|
|
209
|
+
progress: {
|
|
210
|
+
done: (e) => e in k.assets,
|
|
211
|
+
mark: (e, n) => (k.assets[e] = !0, U(t, JSON.stringify(n)).pipe(A.andThen(E(D, k))))
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
V = n.count, H = n.totalBytes, n.missing.length > 0 && (yield* A.logWarning(`${n.missing.length} of ${n.missing.length + n.count} blob reference(s) point at objects the storage provider does not have; they are NOT in this bundle and no import can bring them back. e.g. ${n.missing.slice(0, 3).map((e) => `'${e}'`).join(", ")}` + (n.missing.length > 3 ? `, and ${n.missing.length - 3} more` : "")));
|
|
215
|
+
}
|
|
216
|
+
let G = {
|
|
217
|
+
formatVersion: 1,
|
|
218
|
+
createdAt: e.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
219
|
+
source: {
|
|
220
|
+
dialect: e.dialect,
|
|
221
|
+
schemaFingerprint: F(e.snapshot)
|
|
222
|
+
},
|
|
223
|
+
scope: n,
|
|
224
|
+
consistency: P,
|
|
225
|
+
compression: o,
|
|
226
|
+
tables: B,
|
|
227
|
+
assets: V > 0 ? {
|
|
228
|
+
count: V,
|
|
229
|
+
totalBytes: H,
|
|
230
|
+
index: h
|
|
231
|
+
} : {
|
|
232
|
+
count: 0,
|
|
233
|
+
totalBytes: 0
|
|
234
|
+
},
|
|
235
|
+
...L ? { masking: L } : {}
|
|
236
|
+
};
|
|
237
|
+
return yield* A.tryPromise({
|
|
238
|
+
try: async () => {
|
|
239
|
+
await (await import("node:fs/promises")).writeFile(T(v), b(G));
|
|
240
|
+
},
|
|
241
|
+
catch: (e) => new i({
|
|
242
|
+
reason: `manifest write failed: ${I(e)}`,
|
|
243
|
+
path: v
|
|
244
|
+
})
|
|
245
|
+
}), yield* A.tryPromise({
|
|
246
|
+
try: async () => {
|
|
247
|
+
let e = await import("node:fs/promises"), t = await import("node:os");
|
|
248
|
+
await e.writeFile(T(y), x(G, { hostname: t.hostname() }));
|
|
249
|
+
},
|
|
250
|
+
catch: (e) => new i({
|
|
251
|
+
reason: `readme write failed: ${I(e)}`,
|
|
252
|
+
path: y
|
|
253
|
+
})
|
|
254
|
+
}), G;
|
|
255
|
+
});
|
|
256
|
+
//#endregion
|
|
257
|
+
export { q as runExport };
|