@voltro/data-transfer 0.86.0 → 0.88.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +5484 -0
  2. package/THIRD-PARTY-NOTICES.md +2503 -690
  3. package/dist/assetAdmission.js +91 -0
  4. package/dist/assets.js +62 -42
  5. package/dist/blobStorage.js +55 -32
  6. package/dist/bundleArchive.js +249 -105
  7. package/dist/bundleCrypto.js +1 -2
  8. package/dist/bundlePaths.js +3 -0
  9. package/dist/casAssetPhase.js +78 -35
  10. package/dist/compress.js +4 -5
  11. package/dist/consistency.js +1 -2
  12. package/dist/crossDialect.js +1 -2
  13. package/dist/driftClassification.js +1 -2
  14. package/dist/environmentLocal.js +267 -4
  15. package/dist/errors.js +13 -7
  16. package/dist/exporter.js +146 -133
  17. package/dist/filesystemAdmission.js +32 -0
  18. package/dist/importer.js +326 -287
  19. package/dist/index.d.ts +200 -46
  20. package/dist/index.js +28 -26
  21. package/dist/ledger.js +1 -2
  22. package/dist/liveGuard.js +1 -2
  23. package/dist/manifest.js +45 -42
  24. package/dist/masking.js +1 -2
  25. package/dist/maskingPreview.js +1 -2
  26. package/dist/nativeDump.js +134 -86
  27. package/dist/ndjson.js +1 -2
  28. package/dist/physicalRequest.js +9 -0
  29. package/dist/profile.js +1 -2
  30. package/dist/progress.js +1 -2
  31. package/dist/readme.js +1 -2
  32. package/dist/replacePreflight.js +1 -2
  33. package/dist/rollbackCapture.js +1 -2
  34. package/dist/rowFailureReason.js +1 -2
  35. package/dist/schemaDrift.js +1 -2
  36. package/dist/scope.js +1 -2
  37. package/dist/sealedKeys.js +39 -0
  38. package/dist/stagedSwap.js +1 -2
  39. package/dist/storageAssetMetadata.js +46 -0
  40. package/dist/subset.js +1 -2
  41. package/dist/topo.js +1 -2
  42. package/dist/typedCodec.js +1 -2
  43. package/package.json +4 -4
package/dist/index.d.ts CHANGED
@@ -8,14 +8,19 @@ import { Predicate } from '@voltro/database';
8
8
  import { Row } from '@voltro/database';
9
9
  import { Schema } from 'effect';
10
10
  import { SchemaSnapshot } from '@voltro/database/sql';
11
+ import { Scope } from 'effect';
12
+ import { StorageAddress } from '@voltro/plugin-storage';
13
+ import { StorageAddress as StorageAddress_2 } from '@voltro/plugin-storage/errors';
14
+ import { StorageDestinationRouter } from '@voltro/plugin-storage';
11
15
  import { StorageError } from '@voltro/plugin-storage';
12
- import { StorageProvider } from '@voltro/plugin-storage';
13
16
  import { StoredStream } from '@voltro/plugin-storage';
14
17
  import { Stream } from 'effect';
15
18
  import { TableStreamError } from '@voltro/database';
16
19
  import { VoidIfEmpty } from 'effect/Types';
17
20
  import { YieldableError } from 'effect/Cause';
18
21
 
22
+ declare const admitted: unique symbol;
23
+
19
24
  /** Apply one action to one value. A null value is always passed through. */
20
25
  export declare const applyAction: (action: MaskAction, input: MaskInput) => unknown;
21
26
 
@@ -55,23 +60,42 @@ export declare const assessLiveImport: (opts: {
55
60
 
56
61
  export declare const ASSET_INDEX_FILE = "assets/index.ndjson";
57
62
 
63
+ /** The complete logical address, shared with storage ownership. */
64
+ export declare const AssetAddress: Schema.Struct<{
65
+ readonly destination: Schema.String;
66
+ readonly tenantId: Schema.NullOr<Schema.String>;
67
+ readonly key: Schema.String;
68
+ }>;
69
+
70
+ export declare type AssetAddress = StorageAddress_2;
71
+
72
+ /** The exact materialized row selection, after scope, subset and masking. */
73
+ export declare interface AssetBundleContext {
74
+ readonly dir: string;
75
+ readonly tables: ReadonlyArray<TableEntry>;
76
+ readonly compression: Compression;
77
+ }
78
+
79
+ export declare const assetBundleContext: (dir: string) => Effect.Effect<AssetBundleContext, BundleError>;
80
+
58
81
  export declare const assetDone: (ledger: Ledger, sha256: string) => boolean;
59
82
 
60
- /** One content-addressed asset. `path` is `assets/<sha256>`; `key` is the
61
- * original storage key so the importer restores it under the same key. */
62
83
  export declare const AssetEntry: Schema.Struct<{
63
- readonly key: Schema.String;
64
84
  readonly sha256: Schema.String;
65
85
  readonly size: Schema.Number;
66
86
  readonly contentType: Schema.String;
87
+ readonly destination: Schema.String;
88
+ readonly tenantId: Schema.NullOr<Schema.String>;
89
+ readonly key: Schema.String;
67
90
  }>;
68
91
 
69
92
  export declare type AssetEntry = Schema.Schema.Type<typeof AssetEntry>;
70
93
 
71
94
  /** Metadata for one asset to export — enough to enumerate without reading the
72
95
  * bytes. Sourced from the storage refs table by the caller. */
73
- export declare interface AssetInfo {
74
- readonly key: string;
96
+ export declare interface AssetInfo extends StorageAddress {
97
+ /** SQL checksum, when known; capture refuses bytes that contradict it. */
98
+ readonly sha256?: string;
75
99
  readonly contentType: string;
76
100
  readonly size: number;
77
101
  }
@@ -88,7 +112,7 @@ export declare interface AssetInfo {
88
112
  */
89
113
  export declare interface AssetProgress {
90
114
  readonly done: (key: string) => boolean;
91
- /** Record `key` as landed. Called AFTER the bytes are durable, never before —
115
+ /** Record the complete address identity as landed. Called AFTER the bytes are durable, never before —
92
116
  * a ledger written first turns a crash into silent data loss on resume. */
93
117
  readonly mark: (key: string, entry: AssetEntry) => Effect.Effect<void, BundleError>;
94
118
  }
@@ -97,20 +121,22 @@ export declare const ASSETS_DIR = "assets";
97
121
 
98
122
  /** Where import writes assets TO. */
99
123
  export declare interface AssetSink {
100
- readonly write: (key: string, stream: ByteStream, meta: {
124
+ readonly write: (address: StorageAddress, stream: ByteStream, meta: {
101
125
  readonly contentType: string;
102
126
  readonly totalSize?: number;
103
127
  }) => Effect.Effect<unknown, StorageError>;
104
- /** Whether the key already exists at the destination (skip re-upload). */
105
- readonly has: (key: string) => Effect.Effect<boolean, StorageError>;
128
+ /** Resolve the address and probe its destination during admission. */
129
+ readonly has: (address: StorageAddress) => Effect.Effect<boolean, StorageError>;
130
+ /** True only when this address holds the admitted content, never HEAD alone. */
131
+ readonly matches: (entry: AssetEntry) => Effect.Effect<boolean, StorageError>;
106
132
  }
107
133
 
108
134
  /** Where export reads assets FROM. */
109
135
  export declare interface AssetSource {
110
136
  /** Lazily enumerate every asset (metadata only; bytes stream on demand). */
111
- readonly list: () => Stream.Stream<AssetInfo, unknown>;
137
+ readonly list: (bundle?: AssetBundleContext) => Stream.Stream<AssetInfo, unknown>;
112
138
  /** Open one asset as a byte stream. */
113
- readonly open: (key: string) => Effect.Effect<StoredStream, StorageError>;
139
+ readonly open: (address: StorageAddress) => Effect.Effect<StoredStream, StorageError>;
114
140
  }
115
141
 
116
142
  export declare const AssetSummary: Schema.Struct<{
@@ -129,25 +155,42 @@ export declare const backupArtifactName: (dialect: string) => string;
129
155
  /** Build the backup step for a dialect writing to `outFile`. */
130
156
  export declare const backupCommand: (dialect: string, conn: NativeConn, outFile: string) => NativeStep;
131
157
 
132
- /** Where blobs stream TO on import. `has` lets a resumed run skip existing keys. */
158
+ /** Use an archive destination after admission and the table preflight. */
159
+ export declare const blobAssetSink: (sink: BlobSink) => AssetSink;
160
+
161
+ /** Where blobs stream TO on import. Admission resolves every address; matches verifies resume. */
133
162
  export declare interface BlobSink {
134
- has: (key: string) => Promise<boolean>;
135
- write: (key: string, content: AsyncIterable<Uint8Array>, meta: {
163
+ has: (address: AssetAddress, options?: {
164
+ readonly signal: AbortSignal;
165
+ }) => Promise<boolean>;
166
+ matches: (entry: AssetEntry, options?: {
167
+ readonly signal: AbortSignal;
168
+ }) => Promise<boolean>;
169
+ write: (address: AssetAddress, content: AsyncIterable<Uint8Array>, meta: {
136
170
  readonly contentType: string;
137
171
  readonly size: number;
172
+ }, options?: {
173
+ readonly signal: AbortSignal;
138
174
  }) => Promise<void>;
139
175
  }
140
176
 
141
177
  /** Where blobs stream FROM on export (metadata + a byte stream per key). */
142
178
  export declare interface BlobSource {
143
- list: () => AsyncIterable<{
144
- readonly key: string;
179
+ list: (bundleDir?: string) => AsyncIterable<AssetAddress & {
145
180
  readonly contentType: string;
146
181
  readonly size: number;
182
+ readonly sha256?: string;
147
183
  }>;
148
- open: (key: string) => AsyncIterable<Uint8Array>;
184
+ open: (asset: AssetAddress & {
185
+ readonly contentType: string;
186
+ readonly size: number;
187
+ readonly sha256?: string;
188
+ }) => AsyncIterable<Uint8Array>;
149
189
  }
150
190
 
191
+ /** Enumerate the exported refs, never a broader live SQL selection. */
192
+ export declare const bundleAssetInfoStream: (bundle: AssetBundleContext) => Stream.Stream<AssetInfo, unknown>;
193
+
151
194
  /** A bundle is malformed: bad manifest, unknown format version, missing file. */
152
195
  export declare class BundleError extends BundleError_base {
153
196
  /** The text a log line or a run record shows for this refusal. */
@@ -411,13 +454,17 @@ export declare const decodeManifest: (input: unknown, options?: ParseOptions) =>
411
454
  readonly primaryKey: string;
412
455
  }[];
413
456
  readonly consistency: "live" | "snapshot";
414
- readonly compression: "zstd" | "gzip" | "none";
457
+ readonly compression: "none" | "zstd" | "gzip";
415
458
  readonly formatVersion: number;
416
459
  readonly assets: {
417
460
  readonly count: number;
418
461
  readonly totalBytes: number;
419
462
  readonly index?: string | undefined;
420
463
  };
464
+ readonly fieldKeys?: {
465
+ readonly keyIds: readonly string[];
466
+ readonly unnamed: number;
467
+ } | undefined;
421
468
  readonly masking?: {
422
469
  readonly transformed: readonly string[];
423
470
  readonly policyId?: string | undefined;
@@ -535,13 +582,17 @@ export declare const encodeManifest: (input: {
535
582
  readonly primaryKey: string;
536
583
  }[];
537
584
  readonly consistency: "live" | "snapshot";
538
- readonly compression: "zstd" | "gzip" | "none";
585
+ readonly compression: "none" | "zstd" | "gzip";
539
586
  readonly formatVersion: number;
540
587
  readonly assets: {
541
588
  readonly count: number;
542
589
  readonly totalBytes: number;
543
590
  readonly index?: string | undefined;
544
591
  };
592
+ readonly fieldKeys?: {
593
+ readonly keyIds: readonly string[];
594
+ readonly unnamed: number;
595
+ } | undefined;
545
596
  readonly masking?: {
546
597
  readonly transformed: readonly string[];
547
598
  readonly policyId?: string | undefined;
@@ -724,9 +775,26 @@ export declare const ExportScope: Schema.Struct<{
724
775
 
725
776
  export declare type ExportScope = Schema.Schema.Type<typeof ExportScope>;
726
777
 
778
+ /**
779
+ * The bundle's values are sealed under field-encryption keys the target does
780
+ * not hold. Refused before any row is written: importing them anyway writes
781
+ * rows every read of which fails. `--null-sealed` is the explicit choice for
782
+ * another environment — see `ImportFieldKeys`.
783
+ */
784
+ export declare class FieldKeysMissingError extends FieldKeysMissingError_base {
785
+ get message(): string;
786
+ }
787
+
788
+ declare const FieldKeysMissingError_base: Schema.Class<FieldKeysMissingError, Schema.TaggedStruct<"FieldKeysMissingError", {
789
+ /** Key ids the bundle needs that the target does not hold. Never a key. */
790
+ readonly missing: Schema.$Array<Schema.String>;
791
+ /** Key ids the target holds. */
792
+ readonly held: Schema.$Array<Schema.String>;
793
+ }>, YieldableError>;
794
+
727
795
  /** Bump when the on-disk shape changes incompatibly. An importer refuses a
728
796
  * bundle whose `formatVersion` it doesn't understand rather than guessing. */
729
- export declare const FORMAT_VERSION = 1;
797
+ export declare const FORMAT_VERSION = 2;
730
798
 
731
799
  /** Render for the CLI — the same shape `db plan` prints, for the same reason. */
732
800
  export declare const formatImportDrift: (c: DriftClassification) => ReadonlyArray<string>;
@@ -759,7 +827,28 @@ export declare interface ImportCopyLoader {
759
827
  }) => Promise<number>;
760
828
  }
761
829
 
762
- export declare type ImportError = BundleError | CodecError | IntegrityError | CrossDialectError | ImportModeError | RowsRefusedError | SchemaDriftError | StorageError;
830
+ export declare type ImportError = BundleError | CodecError | IntegrityError | CrossDialectError | FieldKeysMissingError | ImportModeError | RowsRefusedError | SchemaDriftError | StorageError;
831
+
832
+ /** What an import is told about the target's keys. */
833
+ export declare interface ImportFieldKeys {
834
+ /** The key ids the target environment holds. */
835
+ readonly held: ReadonlyArray<string>;
836
+ /**
837
+ * Restoring into another environment: write NULL for a whole value no held
838
+ * key opens (the source ciphertext is useless here; users re-enter it), and
839
+ * report sealed copies inside other values, which cannot be nulled.
840
+ */
841
+ readonly nullSealed?: boolean;
842
+ /**
843
+ * Does a held key open this value? Given by a caller that has the keys, for
844
+ * values that name no key. Without it, an unnamed value counts as opened.
845
+ */
846
+ readonly opens?: (value: string, context: {
847
+ readonly table: string;
848
+ readonly column: string;
849
+ readonly row: Readonly<Record<string, unknown>>;
850
+ }) => boolean;
851
+ }
763
852
 
764
853
  /**
765
854
  * How rows are written:
@@ -819,12 +908,6 @@ export declare interface ImportOptions {
819
908
  * `runImport` for why replace cannot sensibly run without it.
820
909
  */
821
910
  readonly atomic?: boolean;
822
- /** The streaming import path (`unpackBundle` with a storage {@link AssetSink})
823
- * restores blobs to dest storage AS the archive streams in — before the table
824
- * phase — so the manifest's asset phase must be skipped here (the blobs are
825
- * already landed + integrity-checked; re-reading them from a temp dir that
826
- * doesn't exist would fail). Set by the streaming importer, not by users. */
827
- readonly assetsAlreadyRestored?: boolean;
828
911
  /** The TARGET app's live schema (from `introspectSchema`). When set, a
829
912
  * fingerprint pre-flight compares it — the SAME way the export computed the
830
913
  * bundle's fingerprint — against `manifest.source.schemaFingerprint`. On a
@@ -935,6 +1018,14 @@ export declare interface ImportOptions {
935
1018
  * run means a clean import.
936
1019
  */
937
1020
  readonly dryRun?: boolean;
1021
+ /**
1022
+ * The target's field-encryption keys. When given, a bundle whose values need
1023
+ * a key the target does not hold is refused before any row is written
1024
+ * ({@link FieldKeysMissingError}) — unless `nullSealed`, which writes NULL
1025
+ * for every whole value no held key opens and reports the sealed copies
1026
+ * inside other values it cannot null.
1027
+ */
1028
+ readonly fieldKeys?: ImportFieldKeys;
938
1029
  /**
939
1030
  * Rows per savepoint when one is needed (inside a transaction, on an engine
940
1031
  * that aborts it on a failed statement). Larger = fewer savepoints on the
@@ -971,6 +1062,8 @@ export declare interface ImportOutcome {
971
1062
  readonly fullyResumed: boolean;
972
1063
  /** Where the target's previous rows were captured, when they were. */
973
1064
  readonly rollback?: RollbackCapture;
1065
+ /** With `fieldKeys.nullSealed`: what was written as NULL, and the sealed copies left unreadable. */
1066
+ readonly sealed?: SealedImportReport;
974
1067
  }
975
1068
 
976
1069
  /** An integrity check failed — a table's content checksum or row count did not
@@ -1021,6 +1114,12 @@ export declare interface Ledger {
1021
1114
  tables: Record<string, {
1022
1115
  readonly rowCount: number;
1023
1116
  readonly checksum: string;
1117
+ /** Export only: the table's envelopes per field-key id — a resumed export
1118
+ * skips the table and still has to name the keys it needs. */
1119
+ readonly sealed?: {
1120
+ readonly byKey: Readonly<Record<string, number>>;
1121
+ readonly unnamed: number;
1122
+ };
1024
1123
  }>;
1025
1124
  /** sha256 → true for every asset that fully landed. */
1026
1125
  assets: Record<string, true>;
@@ -1092,6 +1191,16 @@ export declare const Manifest: Schema.Struct<{
1092
1191
  * assets were exported. Each line is an {@link AssetEntry}. */
1093
1192
  readonly index: Schema.optional<Schema.String>;
1094
1193
  }>;
1194
+ /**
1195
+ * The field-encryption key ids the bundle's values are sealed under, read off
1196
+ * the envelopes (no key needed), and how many name no key. An import checks
1197
+ * the target's keys against it before writing a row. Absent in a bundle from
1198
+ * before this was recorded.
1199
+ */
1200
+ readonly fieldKeys: Schema.optional<Schema.Struct<{
1201
+ readonly keyIds: Schema.$Array<Schema.String>;
1202
+ readonly unnamed: Schema.Number;
1203
+ }>>;
1095
1204
  /** Masking audit — present when the export applied a masking policy. Lists
1096
1205
  * every `table.column` that was transformed (not copied verbatim), so a
1097
1206
  * reviewer can see exactly what was pseudonymised. */
@@ -1291,6 +1400,20 @@ export declare type NativeStep = {
1291
1400
  readonly kind: 'copy';
1292
1401
  readonly from: string;
1293
1402
  readonly to: string;
1403
+ }
1404
+ /** A consistent snapshot of a live SQLite database through its backup API —
1405
+ * write-ahead log included — written beside the target and renamed into place. */
1406
+ | {
1407
+ readonly kind: 'sqlite-backup';
1408
+ readonly from: string;
1409
+ readonly to: string;
1410
+ }
1411
+ /** `VACUUM INTO` through the database's own connection — written beside the
1412
+ * target and renamed into place. Needs the caller's {@link NativeSql}. */
1413
+ | {
1414
+ readonly kind: 'vacuum-into';
1415
+ readonly conn: NativeConn;
1416
+ readonly to: string;
1294
1417
  } | MssqlReplaceStep;
1295
1418
 
1296
1419
  /** A native backup/restore tool failed — missing binary, non-zero exit, or an
@@ -1431,6 +1554,19 @@ export declare interface PortabilityIssue {
1431
1554
  readonly reason: string;
1432
1555
  }
1433
1556
 
1557
+ export declare const prepareAssetsFromCas: (opts: {
1558
+ readonly sink: AssetSink;
1559
+ readonly assetsDir: string;
1560
+ readonly indexPath: string;
1561
+ }) => Effect.Effect<PreparedAssets, BundleError | IntegrityError | StorageError, Scope.Scope>;
1562
+
1563
+ /** Private immutable copies, owned by the calling Effect scope. */
1564
+ export declare interface PreparedAssets {
1565
+ readonly [admitted]: true;
1566
+ readonly assetsDir: string;
1567
+ readonly entries: ReadonlyArray<AssetEntry>;
1568
+ }
1569
+
1434
1570
  export declare const previewMasking: (opts: MaskingPreviewOptions) => Effect.Effect<MaskingPreview, TableStreamError | ScopeError>;
1435
1571
 
1436
1572
  /**
@@ -1469,6 +1605,14 @@ export declare const readAssetIndex: (indexPath: string) => Stream.Stream<AssetE
1469
1605
  * (a missing ledger just means "nothing done yet" — never a hard failure). */
1470
1606
  export declare const readLedger: (path: string) => Effect.Effect<Ledger>;
1471
1607
 
1608
+ /** Reconnect a target after a native database replacement. Re-admit only the
1609
+ * original private immutable snapshot, under the new sink's full checks. */
1610
+ export declare const readmitPreparedAssets: (opts: {
1611
+ readonly prepared: PreparedAssets;
1612
+ readonly previousSink: AssetSink;
1613
+ readonly sink: AssetSink;
1614
+ }) => Effect.Effect<PreparedAssets, BundleError | IntegrityError | StorageError, Scope.Scope>;
1615
+
1472
1616
  /** Resolve the action for one column (column override → class → safe → unclassified). */
1473
1617
  export declare const resolveAction: (table: string, column: string, cls: ColumnClassification | undefined, policy: MaskingPolicy) => MaskAction;
1474
1618
 
@@ -1492,19 +1636,13 @@ export declare interface ResolveScopeDeps {
1492
1636
  }
1493
1637
 
1494
1638
  /**
1495
- * Restore one CAS asset to the sink under `key`, re-hashing the bytes as they
1496
- * stream through and refusing (IntegrityError) if they don't match the expected
1497
- * sha — so a corrupted bundle can never silently re-upload bad bytes.
1639
+ * Admit a private copy of one CAS asset before its bytes reach the sink.
1640
+ * Whole-index consumers use the shared CAS phase to admit all entries first.
1498
1641
  */
1499
- export declare const restoreAssetFromCas: (sink: AssetSink, assetsDir: string, entry: {
1500
- readonly key: string;
1501
- readonly sha256: string;
1502
- readonly size: number;
1503
- readonly contentType: string;
1504
- }) => Effect.Effect<void, BundleError | IntegrityError | StorageError>;
1642
+ export declare const restoreAssetFromCas: (sink: AssetSink, assetsDir: string, entry: AssetEntry) => Effect.Effect<void, BundleError | IntegrityError | StorageError>;
1505
1643
 
1506
1644
  /**
1507
- * Push every asset named in the index back to the sink, re-hashing on the way.
1645
+ * Push every asset from the private, fully verified index back to its admitted sink.
1508
1646
  *
1509
1647
  * A hash mismatch is an {@link IntegrityError} and stops the phase — a corrupted
1510
1648
  * artefact must never silently re-upload bad bytes over good ones, which is the
@@ -1520,6 +1658,7 @@ export declare const restoreAssetsFromCas: (opts: {
1520
1658
  readonly assetsDir: string;
1521
1659
  readonly indexPath: string;
1522
1660
  readonly progress?: AssetProgress;
1661
+ readonly prepared?: PreparedAssets;
1523
1662
  }) => Effect.Effect<RestoredAssets, BundleError | IntegrityError | StorageError>;
1524
1663
 
1525
1664
  /** Build the restore step for a dialect reading `inFile`. */
@@ -1666,6 +1805,19 @@ declare const ScopeError_base: Schema.Class<ScopeError, Schema.TaggedStruct<"Sco
1666
1805
  readonly reason: Schema.String;
1667
1806
  }>, YieldableError>;
1668
1807
 
1808
+ export declare interface SealedImportReport {
1809
+ readonly nulled: ReadonlyArray<{
1810
+ readonly table: string;
1811
+ readonly column: string;
1812
+ readonly count: number;
1813
+ }>;
1814
+ readonly unreadable: ReadonlyArray<{
1815
+ readonly table: string;
1816
+ readonly column: string;
1817
+ readonly count: number;
1818
+ }>;
1819
+ }
1820
+
1669
1821
  /**
1670
1822
  * A store that can run raw SQL for us.
1671
1823
  *
@@ -1754,27 +1906,29 @@ export declare interface StagingSnapshot {
1754
1906
  }>;
1755
1907
  }
1756
1908
 
1909
+ export declare const storageAssetInfo: (row: Row) => AssetInfo;
1910
+
1757
1911
  /** Build an {@link AssetSink} over a storage provider. */
1758
- export declare const storageAssetSink: (provider: StorageProvider) => AssetSink;
1912
+ export declare const storageAssetSink: (router: StorageDestinationRouter) => AssetSink;
1759
1913
 
1760
1914
  /** Build an {@link AssetSource} over a storage provider — `list` is supplied by
1761
1915
  * the caller (typically streaming the `_voltro_storage_refs` table), `open`
1762
1916
  * uses the provider's streaming read (buffered fallback for non-streaming
1763
1917
  * backends). */
1764
- export declare const storageAssetSource: (provider: StorageProvider, list: () => Stream.Stream<AssetInfo, unknown>) => AssetSource;
1918
+ export declare const storageAssetSource: (router: StorageDestinationRouter, list: (bundle?: AssetBundleContext) => Stream.Stream<AssetInfo, unknown>) => AssetSource;
1765
1919
 
1766
1920
  /** Stream blobs TO object storage. `has` powers resume (skip existing keys);
1767
1921
  * `write` streams the archive's blob content straight into the provider. */
1768
- export declare const storageBlobSink: (provider: StorageProvider) => BlobSink;
1922
+ export declare const storageBlobSink: (router: StorageDestinationRouter) => BlobSink;
1769
1923
 
1770
1924
  /** Stream blobs FROM object storage. `list` enumerates metadata (typically the
1771
1925
  * storage refs table); `open` streams each blob's bytes via the provider. */
1772
- export declare const storageBlobSource: (provider: StorageProvider, list: () => Stream.Stream<AssetInfo, unknown>) => BlobSource;
1926
+ export declare const storageBlobSource: (router: StorageDestinationRouter, list: (bundleDir?: string) => Stream.Stream<AssetInfo, unknown>) => BlobSource;
1773
1927
 
1774
1928
  /** A {@link BlobSource} over a provider whose blob list comes from the app's
1775
1929
  * `_voltro_storage_refs` table — the standard export source for a running
1776
1930
  * instance (CLI file export and the prod export endpoint both use it). */
1777
- export declare const storageRefsBlobSource: (provider: StorageProvider, store: DataStore) => BlobSource;
1931
+ export declare const storageRefsBlobSource: (router: StorageDestinationRouter, store: DataStore) => BlobSource;
1778
1932
 
1779
1933
  export declare interface SubsetSpec {
1780
1934
  /** Seed rows per table. `table → predicate` (a `null`/absent predicate seeds
@@ -1854,8 +2008,8 @@ export declare const topoSortTables: (tables: ReadonlyArray<string>, snapshot: S
1854
2008
 
1855
2009
  /**
1856
2010
  * Unpack a framed byte stream. Plain files are written under `dir` (traversal
1857
- * guarded); blob entries are streamed to `blobs` (dest storage) — never to local
1858
- * disk — with the trailing sha verified as they pass through. A blob whose key
2011
+ * guarded); entries are staged and verified through authenticated source EOF
2012
+ * before `blobs` (dest storage) participates. A blob whose key
1859
2013
  * already exists at the sink is skipped (resume).
1860
2014
  */
1861
2015
  export declare const unpackBundle: (source: AsyncIterable<Uint8Array>, opts: {
@@ -1866,7 +2020,7 @@ export declare const unpackBundle: (source: AsyncIterable<Uint8Array>, opts: {
1866
2020
  /** Convenience: unpack a bundle fully to a local directory (plain files as-is,
1867
2021
  * blob entries materialised under `assets/`). For data-only bundles and
1868
2022
  * `voltro data unpack`; the streaming import path passes a storage {@link BlobSink}
1869
- * to {@link unpackBundle} instead so blobs never touch local disk. */
2023
+ * to {@link unpackBundle} for a destination after archive admission. */
1870
2024
  export declare const unpackBundleToDir: (source: AsyncIterable<Uint8Array>, dir: string) => Promise<void>;
1871
2025
 
1872
2026
  /**
package/dist/index.js CHANGED
@@ -1,28 +1,30 @@
1
1
  import { DEFAULT_CLASS_ACTIONS as e, applyAction as t, classificationFromSnapshot as n, maskRow as r, planMasking as i, resolveAction as a, scanForLeaks as o } from "./masking.js";
2
2
  import { ENVIRONMENT_LOCAL_TABLES as s, ENVIRONMENT_PORTABLE_TABLES as c, environmentLocalNotice as l, environmentLocalReason as u, isEnvironmentLocal as d } from "./environmentLocal.js";
3
- import { BundleError as f, CodecError as p, CompressionError as m, CrossDialectError as h, ImportModeError as g, IntegrityError as _, MaskingError as v, NativeToolError as y, RowsRefusedError as b, SchemaDriftError as x, ScopeError as S } from "./errors.js";
4
- import { computeSubsetIds as C, subsetToScope as w } from "./subset.js";
5
- import { resolveScope as T } from "./scope.js";
6
- import { restoreAssetFromCas as E, storageAssetSink as D, storageAssetSource as O, writeAssetToCas as k } from "./assets.js";
7
- import { captureAssetsToCas as A, noAssetProgress as j, readAssetIndex as M, restoreAssetsFromCas as N } from "./casAssetPhase.js";
8
- import { compressionExt as P, hasZstd as F, makeCompressor as I, makeDecompressor as L, pickCompression as R } from "./compress.js";
9
- import { ASSETS_DIR as z, ASSET_INDEX_FILE as B, DATA_DIR as V, FORMAT_VERSION as H, LEDGER_FILE as U, MANIFEST_FILE as W, decodeManifest as G, encodeManifest as K } from "./manifest.js";
10
- import { assetDone as q, emptyLedger as J, readLedger as Y, tableDone as X, writeLedger as Z } from "./ledger.js";
11
- import { decodeRow as Q, decodeValue as $, encodeRow as ee, encodeValue as te } from "./typedCodec.js";
12
- import { ndjsonToRows as ne, rowsToNdjson as re } from "./ndjson.js";
13
- import { topoSortTables as ie } from "./topo.js";
14
- import { runExport as ae } from "./exporter.js";
15
- import { outsideReferenceRefusal as oe, outsideReferencesInto as se } from "./replacePreflight.js";
16
- import { canonicalType as ce, checkPortability as le, formatIssue as ue } from "./crossDialect.js";
17
- import { STAGING_PREFIX as de, canStage as fe, decideStaging as pe, dropStagingSql as me, execStatements as he, stagingNameFor as ge } from "./stagedSwap.js";
18
- import { checkSchemaDrift as _e, diffSnapshots as ve } from "./schemaDrift.js";
19
- import { SAVEPOINT_BATCH_SIZE as ye, asImportError as be, runImport as xe } from "./importer.js";
20
- import { defineDataProfile as Se } from "./profile.js";
21
- import { assessLiveImport as Ce } from "./liveGuard.js";
22
- import { dirBlobSink as we, packBundle as Te, peekBundle as Ee, unpackBundle as De, unpackBundleToDir as Oe } from "./bundleArchive.js";
23
- import { decryptStream as ke, encryptStream as Ae } from "./bundleCrypto.js";
24
- import { storageBlobSink as je, storageBlobSource as Me, storageRefsBlobSource as Ne } from "./blobStorage.js";
25
- import { EXCLUDED_FROM_DUMP as Pe, backupArtifactName as Fe, backupCommand as Ie, dumpExcludedTables as Le, mssqlReplaceStatements as Re, mssqlScratchName as ze, restoreCommand as Be, runNativeStep as Ve } from "./nativeDump.js";
26
- import { previewMasking as He } from "./maskingPreview.js";
27
- import { classifyImportDrift as Ue, formatImportDrift as We } from "./driftClassification.js";
28
- export { z as ASSETS_DIR, B as ASSET_INDEX_FILE, f as BundleError, p as CodecError, m as CompressionError, h as CrossDialectError, V as DATA_DIR, e as DEFAULT_CLASS_ACTIONS, s as ENVIRONMENT_LOCAL_TABLES, c as ENVIRONMENT_PORTABLE_TABLES, Pe as EXCLUDED_FROM_DUMP, H as FORMAT_VERSION, g as ImportModeError, _ as IntegrityError, U as LEDGER_FILE, W as MANIFEST_FILE, v as MaskingError, y as NativeToolError, b as RowsRefusedError, ye as SAVEPOINT_BATCH_SIZE, de as STAGING_PREFIX, x as SchemaDriftError, S as ScopeError, t as applyAction, be as asImportError, Ce as assessLiveImport, q as assetDone, Fe as backupArtifactName, Ie as backupCommand, fe as canStage, ce as canonicalType, A as captureAssetsToCas, le as checkPortability, _e as checkSchemaDrift, n as classificationFromSnapshot, Ue as classifyImportDrift, P as compressionExt, C as computeSubsetIds, pe as decideStaging, G as decodeManifest, Q as decodeRow, $ as decodeValue, ke as decryptStream, Se as defineDataProfile, ve as diffSnapshots, we as dirBlobSink, me as dropStagingSql, Le as dumpExcludedTables, J as emptyLedger, K as encodeManifest, ee as encodeRow, te as encodeValue, Ae as encryptStream, l as environmentLocalNotice, u as environmentLocalReason, he as execStatements, We as formatImportDrift, ue as formatIssue, F as hasZstd, d as isEnvironmentLocal, I as makeCompressor, L as makeDecompressor, r as maskRow, Re as mssqlReplaceStatements, ze as mssqlScratchName, ne as ndjsonToRows, j as noAssetProgress, oe as outsideReferenceRefusal, se as outsideReferencesInto, Te as packBundle, Ee as peekBundle, R as pickCompression, i as planMasking, He as previewMasking, M as readAssetIndex, Y as readLedger, a as resolveAction, T as resolveScope, E as restoreAssetFromCas, N as restoreAssetsFromCas, Be as restoreCommand, re as rowsToNdjson, ae as runExport, xe as runImport, Ve as runNativeStep, o as scanForLeaks, ge as stagingNameFor, D as storageAssetSink, O as storageAssetSource, je as storageBlobSink, Me as storageBlobSource, Ne as storageRefsBlobSource, w as subsetToScope, X as tableDone, ie as topoSortTables, De as unpackBundle, Oe as unpackBundleToDir, k as writeAssetToCas, Z as writeLedger };
3
+ import { BundleError as f, CodecError as p, CompressionError as m, CrossDialectError as h, FieldKeysMissingError as g, ImportModeError as _, IntegrityError as v, MaskingError as y, NativeToolError as b, RowsRefusedError as x, SchemaDriftError as S, ScopeError as C } from "./errors.js";
4
+ import { computeSubsetIds as w, subsetToScope as T } from "./subset.js";
5
+ import { resolveScope as E } from "./scope.js";
6
+ import { ASSETS_DIR as D, ASSET_INDEX_FILE as O, AssetAddress as k, DATA_DIR as A, FORMAT_VERSION as j, LEDGER_FILE as M, MANIFEST_FILE as N, decodeManifest as P, encodeManifest as F } from "./manifest.js";
7
+ import { readmitPreparedAssets as I } from "./assetAdmission.js";
8
+ import { restoreAssetFromCas as L, storageAssetSink as R, storageAssetSource as z, writeAssetToCas as B } from "./assets.js";
9
+ import { captureAssetsToCas as V, noAssetProgress as H, prepareAssetsFromCas as U, readAssetIndex as W, restoreAssetsFromCas as G } from "./casAssetPhase.js";
10
+ import { compressionExt as K, hasZstd as q, makeCompressor as J, makeDecompressor as Y, pickCompression as X } from "./compress.js";
11
+ import { assetDone as Z, emptyLedger as Q, readLedger as $, tableDone as ee, writeLedger as te } from "./ledger.js";
12
+ import { decodeRow as ne, decodeValue as re, encodeRow as ie, encodeValue as ae } from "./typedCodec.js";
13
+ import { ndjsonToRows as oe, rowsToNdjson as se } from "./ndjson.js";
14
+ import { topoSortTables as ce } from "./topo.js";
15
+ import { runExport as le } from "./exporter.js";
16
+ import { outsideReferenceRefusal as ue, outsideReferencesInto as de } from "./replacePreflight.js";
17
+ import { canonicalType as fe, checkPortability as pe, formatIssue as me } from "./crossDialect.js";
18
+ import { STAGING_PREFIX as he, canStage as ge, decideStaging as _e, dropStagingSql as ve, execStatements as ye, stagingNameFor as be } from "./stagedSwap.js";
19
+ import { checkSchemaDrift as xe, diffSnapshots as Se } from "./schemaDrift.js";
20
+ import { SAVEPOINT_BATCH_SIZE as Ce, asImportError as we, runImport as Te } from "./importer.js";
21
+ import { defineDataProfile as Ee } from "./profile.js";
22
+ import { assessLiveImport as De } from "./liveGuard.js";
23
+ import { dirBlobSink as Oe, packBundle as ke, peekBundle as Ae, unpackBundle as je, unpackBundleToDir as Me } from "./bundleArchive.js";
24
+ import { decryptStream as Ne, encryptStream as Pe } from "./bundleCrypto.js";
25
+ import { assetBundleContext as Fe, bundleAssetInfoStream as Ie, storageAssetInfo as Le } from "./storageAssetMetadata.js";
26
+ import { blobAssetSink as Re, storageBlobSink as ze, storageBlobSource as Be, storageRefsBlobSource as Ve } from "./blobStorage.js";
27
+ import { EXCLUDED_FROM_DUMP as He, backupArtifactName as Ue, backupCommand as We, dumpExcludedTables as Ge, mssqlReplaceStatements as Ke, mssqlScratchName as qe, restoreCommand as Je, runNativeStep as Ye } from "./nativeDump.js";
28
+ import { previewMasking as Xe } from "./maskingPreview.js";
29
+ import { classifyImportDrift as Ze, formatImportDrift as Qe } from "./driftClassification.js";
30
+ export { D as ASSETS_DIR, O as ASSET_INDEX_FILE, k as AssetAddress, f as BundleError, p as CodecError, m as CompressionError, h as CrossDialectError, A as DATA_DIR, e as DEFAULT_CLASS_ACTIONS, s as ENVIRONMENT_LOCAL_TABLES, c as ENVIRONMENT_PORTABLE_TABLES, He as EXCLUDED_FROM_DUMP, j as FORMAT_VERSION, g as FieldKeysMissingError, _ as ImportModeError, v as IntegrityError, M as LEDGER_FILE, N as MANIFEST_FILE, y as MaskingError, b as NativeToolError, x as RowsRefusedError, Ce as SAVEPOINT_BATCH_SIZE, he as STAGING_PREFIX, S as SchemaDriftError, C as ScopeError, t as applyAction, we as asImportError, De as assessLiveImport, Fe as assetBundleContext, Z as assetDone, Ue as backupArtifactName, We as backupCommand, Re as blobAssetSink, Ie as bundleAssetInfoStream, ge as canStage, fe as canonicalType, V as captureAssetsToCas, pe as checkPortability, xe as checkSchemaDrift, n as classificationFromSnapshot, Ze as classifyImportDrift, K as compressionExt, w as computeSubsetIds, _e as decideStaging, P as decodeManifest, ne as decodeRow, re as decodeValue, Ne as decryptStream, Ee as defineDataProfile, Se as diffSnapshots, Oe as dirBlobSink, ve as dropStagingSql, Ge as dumpExcludedTables, Q as emptyLedger, F as encodeManifest, ie as encodeRow, ae as encodeValue, Pe as encryptStream, l as environmentLocalNotice, u as environmentLocalReason, ye as execStatements, Qe as formatImportDrift, me as formatIssue, q as hasZstd, d as isEnvironmentLocal, J as makeCompressor, Y as makeDecompressor, r as maskRow, Ke as mssqlReplaceStatements, qe as mssqlScratchName, oe as ndjsonToRows, H as noAssetProgress, ue as outsideReferenceRefusal, de as outsideReferencesInto, ke as packBundle, Ae as peekBundle, X as pickCompression, i as planMasking, U as prepareAssetsFromCas, Xe as previewMasking, W as readAssetIndex, $ as readLedger, I as readmitPreparedAssets, a as resolveAction, E as resolveScope, L as restoreAssetFromCas, G as restoreAssetsFromCas, Je as restoreCommand, se as rowsToNdjson, le as runExport, Te as runImport, Ye as runNativeStep, o as scanForLeaks, be as stagingNameFor, Le as storageAssetInfo, R as storageAssetSink, z as storageAssetSource, ze as storageBlobSink, Be as storageBlobSource, Ve as storageRefsBlobSource, T as subsetToScope, ee as tableDone, ce as topoSortTables, je as unpackBundle, Me as unpackBundleToDir, B as writeAssetToCas, te as writeLedger };
package/dist/ledger.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { BundleError as e } from "./errors.js";
2
2
  import { Effect as t } from "effect";
3
- //#region src/ledger.ts
3
+ // src/ledger.ts
4
4
  var n = () => ({
5
5
  tables: {},
6
6
  assets: {}
@@ -17,5 +17,4 @@ var n = () => ({
17
17
  path: n
18
18
  })
19
19
  }), a = (e, t) => t in e.tables, o = (e, t) => t in e.assets;
20
- //#endregion
21
20
  export { o as assetDone, n as emptyLedger, r as readLedger, a as tableDone, i as writeLedger };
package/dist/liveGuard.js CHANGED
@@ -1,4 +1,4 @@
1
- //#region src/liveGuard.ts
1
+ // src/liveGuard.ts
2
2
  var e = (e) => {
3
3
  if (e.allowLive || e.liveInstance === null || e.sameTarget === "different") return { refuse: !1 };
4
4
  let t = e.targetLabel === void 0 ? "" : ` Target: ${e.targetLabel}.`, n = e.sameTarget === "same" ? " That instance is connected to the SAME database." : " We could not confirm which database that instance is connected to, so this is refused on the conservative reading — an older instance, or one whose connection could not be parsed, reports nothing to compare.";
@@ -7,5 +7,4 @@ var e = (e) => {
7
7
  message: `refusing to write into a target with a LIVE instance (${e.liveInstance}).${t}${n} A direct import is an uncoordinated concurrent writer — readers see partial state, the reactive layer either storms or goes stale, and rows race with live writes. Re-run with --allow-live to override, or use a live-safe strategy (shadow-swap for replace, or the api target).`
8
8
  };
9
9
  };
10
- //#endregion
11
10
  export { e as assessLiveImport };