@voltro/data-transfer 0.86.0 → 0.87.0

package/dist/index.d.ts

@@ -8,14 +8,19 @@ import { Predicate } from '@voltro/database';
 import { Row } from '@voltro/database';
 import { Schema } from 'effect';
 import { SchemaSnapshot } from '@voltro/database/sql';
+import { Scope } from 'effect';
+import { StorageAddress } from '@voltro/plugin-storage';
+import { StorageAddress as StorageAddress_2 } from '@voltro/plugin-storage/errors';
+import { StorageDestinationRouter } from '@voltro/plugin-storage';
 import { StorageError } from '@voltro/plugin-storage';
-import { StorageProvider } from '@voltro/plugin-storage';
 import { StoredStream } from '@voltro/plugin-storage';
 import { Stream } from 'effect';
 import { TableStreamError } from '@voltro/database';
 import { VoidIfEmpty } from 'effect/Types';
 import { YieldableError } from 'effect/Cause';
 
+declare const admitted: unique symbol;
+
 /** Apply one action to one value. A null value is always passed through. */
 export declare const applyAction: (action: MaskAction, input: MaskInput) => unknown;
 
@@ -55,23 +60,42 @@ export declare const assessLiveImport: (opts: {
 
 export declare const ASSET_INDEX_FILE = "assets/index.ndjson";
 
+/** The complete logical address, shared with storage ownership. */
+export declare const AssetAddress: Schema.Struct<{
+    readonly destination: Schema.String;
+    readonly tenantId: Schema.NullOr<Schema.String>;
+    readonly key: Schema.String;
+}>;
+
+export declare type AssetAddress = StorageAddress_2;
+
+/** The exact materialized row selection, after scope, subset and masking. */
+export declare interface AssetBundleContext {
+    readonly dir: string;
+    readonly tables: ReadonlyArray<TableEntry>;
+    readonly compression: Compression;
+}
+
+export declare const assetBundleContext: (dir: string) => Effect.Effect<AssetBundleContext, BundleError>;
+
 export declare const assetDone: (ledger: Ledger, sha256: string) => boolean;
 
-/** One content-addressed asset. `path` is `assets/<sha256>`; `key` is the
- *  original storage key so the importer restores it under the same key. */
 export declare const AssetEntry: Schema.Struct<{
-    readonly key: Schema.String;
     readonly sha256: Schema.String;
     readonly size: Schema.Number;
     readonly contentType: Schema.String;
+    readonly destination: Schema.String;
+    readonly tenantId: Schema.NullOr<Schema.String>;
+    readonly key: Schema.String;
 }>;
 
 export declare type AssetEntry = Schema.Schema.Type<typeof AssetEntry>;
 
 /** Metadata for one asset to export — enough to enumerate without reading the
  *  bytes. Sourced from the storage refs table by the caller. */
-export declare interface AssetInfo {
-    readonly key: string;
+export declare interface AssetInfo extends StorageAddress {
+    /** SQL checksum, when known; capture refuses bytes that contradict it. */
+    readonly sha256?: string;
     readonly contentType: string;
     readonly size: number;
 }
@@ -88,7 +112,7 @@ export declare interface AssetInfo {
  */
 export declare interface AssetProgress {
     readonly done: (key: string) => boolean;
-    /** Record `key` as landed. Called AFTER the bytes are durable, never before —
+    /** Record the complete address identity as landed. Called AFTER the bytes are durable, never before —
      *  a ledger written first turns a crash into silent data loss on resume. */
     readonly mark: (key: string, entry: AssetEntry) => Effect.Effect<void, BundleError>;
 }
@@ -97,20 +121,22 @@ export declare const ASSETS_DIR = "assets";
 
 /** Where import writes assets TO. */
 export declare interface AssetSink {
-    readonly write: (key: string, stream: ByteStream, meta: {
+    readonly write: (address: StorageAddress, stream: ByteStream, meta: {
         readonly contentType: string;
         readonly totalSize?: number;
     }) => Effect.Effect<unknown, StorageError>;
-    /** Whether the key already exists at the destination (skip re-upload). */
-    readonly has: (key: string) => Effect.Effect<boolean, StorageError>;
+    /** Resolve the address and probe its destination during admission. */
+    readonly has: (address: StorageAddress) => Effect.Effect<boolean, StorageError>;
+    /** True only when this address holds the admitted content, never HEAD alone. */
+    readonly matches: (entry: AssetEntry) => Effect.Effect<boolean, StorageError>;
 }
 
 /** Where export reads assets FROM. */
 export declare interface AssetSource {
     /** Lazily enumerate every asset (metadata only; bytes stream on demand). */
-    readonly list: () => Stream.Stream<AssetInfo, unknown>;
+    readonly list: (bundle?: AssetBundleContext) => Stream.Stream<AssetInfo, unknown>;
     /** Open one asset as a byte stream. */
-    readonly open: (key: string) => Effect.Effect<StoredStream, StorageError>;
+    readonly open: (address: StorageAddress) => Effect.Effect<StoredStream, StorageError>;
 }
 
 export declare const AssetSummary: Schema.Struct<{
@@ -129,25 +155,42 @@ export declare const backupArtifactName: (dialect: string) => string;
 /** Build the backup step for a dialect writing to `outFile`. */
 export declare const backupCommand: (dialect: string, conn: NativeConn, outFile: string) => NativeStep;
 
-/** Where blobs stream TO on import. `has` lets a resumed run skip existing keys. */
+/** Use an archive destination after admission and the table preflight. */
+export declare const blobAssetSink: (sink: BlobSink) => AssetSink;
+
+/** Where blobs stream TO on import. Admission resolves every address; matches verifies resume. */
 export declare interface BlobSink {
-    has: (key: string) => Promise<boolean>;
-    write: (key: string, content: AsyncIterable<Uint8Array>, meta: {
+    has: (address: AssetAddress, options?: {
+        readonly signal: AbortSignal;
+    }) => Promise<boolean>;
+    matches: (entry: AssetEntry, options?: {
+        readonly signal: AbortSignal;
+    }) => Promise<boolean>;
+    write: (address: AssetAddress, content: AsyncIterable<Uint8Array>, meta: {
         readonly contentType: string;
         readonly size: number;
+    }, options?: {
+        readonly signal: AbortSignal;
     }) => Promise<void>;
 }
 
 /** Where blobs stream FROM on export (metadata + a byte stream per key). */
 export declare interface BlobSource {
-    list: () => AsyncIterable<{
-        readonly key: string;
+    list: (bundleDir?: string) => AsyncIterable<AssetAddress & {
         readonly contentType: string;
         readonly size: number;
+        readonly sha256?: string;
     }>;
-    open: (key: string) => AsyncIterable<Uint8Array>;
+    open: (asset: AssetAddress & {
+        readonly contentType: string;
+        readonly size: number;
+        readonly sha256?: string;
+    }) => AsyncIterable<Uint8Array>;
 }
 
+/** Enumerate the exported refs, never a broader live SQL selection. */
+export declare const bundleAssetInfoStream: (bundle: AssetBundleContext) => Stream.Stream<AssetInfo, unknown>;
+
 /** A bundle is malformed: bad manifest, unknown format version, missing file. */
 export declare class BundleError extends BundleError_base {
     /** The text a log line or a run record shows for this refusal. */
@@ -411,13 +454,17 @@ export declare const decodeManifest: (input: unknown, options?: ParseOptions) =>
         readonly primaryKey: string;
     }[];
     readonly consistency: "live" | "snapshot";
-    readonly compression: "zstd" | "gzip" | "none";
+    readonly compression: "none" | "zstd" | "gzip";
     readonly formatVersion: number;
     readonly assets: {
         readonly count: number;
         readonly totalBytes: number;
         readonly index?: string | undefined;
     };
+    readonly fieldKeys?: {
+        readonly keyIds: readonly string[];
+        readonly unnamed: number;
+    } | undefined;
     readonly masking?: {
         readonly transformed: readonly string[];
         readonly policyId?: string | undefined;
@@ -535,13 +582,17 @@ export declare const encodeManifest: (input: {
         readonly primaryKey: string;
     }[];
     readonly consistency: "live" | "snapshot";
-    readonly compression: "zstd" | "gzip" | "none";
+    readonly compression: "none" | "zstd" | "gzip";
     readonly formatVersion: number;
     readonly assets: {
         readonly count: number;
         readonly totalBytes: number;
         readonly index?: string | undefined;
     };
+    readonly fieldKeys?: {
+        readonly keyIds: readonly string[];
+        readonly unnamed: number;
+    } | undefined;
     readonly masking?: {
         readonly transformed: readonly string[];
         readonly policyId?: string | undefined;
@@ -724,9 +775,26 @@ export declare const ExportScope: Schema.Struct<{
 
 export declare type ExportScope = Schema.Schema.Type<typeof ExportScope>;
 
+/**
+ * The bundle's values are sealed under field-encryption keys the target does
+ * not hold. Refused before any row is written: importing them anyway writes
+ * rows every read of which fails. `--null-sealed` is the explicit choice for
+ * another environment — see `ImportFieldKeys`.
+ */
+export declare class FieldKeysMissingError extends FieldKeysMissingError_base {
+    get message(): string;
+}
+
+declare const FieldKeysMissingError_base: Schema.Class<FieldKeysMissingError, Schema.TaggedStruct<"FieldKeysMissingError", {
+    /** Key ids the bundle needs that the target does not hold. Never a key. */
+    readonly missing: Schema.$Array<Schema.String>;
+    /** Key ids the target holds. */
+    readonly held: Schema.$Array<Schema.String>;
+}>, YieldableError>;
+
 /** Bump when the on-disk shape changes incompatibly. An importer refuses a
  *  bundle whose `formatVersion` it doesn't understand rather than guessing. */
-export declare const FORMAT_VERSION = 1;
+export declare const FORMAT_VERSION = 2;
 
 /** Render for the CLI — the same shape `db plan` prints, for the same reason. */
 export declare const formatImportDrift: (c: DriftClassification) => ReadonlyArray<string>;
@@ -759,7 +827,28 @@ export declare interface ImportCopyLoader {
     }) => Promise<number>;
 }
 
-export declare type ImportError = BundleError | CodecError | IntegrityError | CrossDialectError | ImportModeError | RowsRefusedError | SchemaDriftError | StorageError;
+export declare type ImportError = BundleError | CodecError | IntegrityError | CrossDialectError | FieldKeysMissingError | ImportModeError | RowsRefusedError | SchemaDriftError | StorageError;
+
+/** What an import is told about the target's keys. */
+export declare interface ImportFieldKeys {
+    /** The key ids the target environment holds. */
+    readonly held: ReadonlyArray<string>;
+    /**
+     * Restoring into another environment: write NULL for a whole value no held
+     * key opens (the source ciphertext is useless here; users re-enter it), and
+     * report sealed copies inside other values, which cannot be nulled.
+     */
+    readonly nullSealed?: boolean;
+    /**
+     * Does a held key open this value? Given by a caller that has the keys, for
+     * values that name no key. Without it, an unnamed value counts as opened.
+     */
+    readonly opens?: (value: string, context: {
+        readonly table: string;
+        readonly column: string;
+        readonly row: Readonly<Record<string, unknown>>;
+    }) => boolean;
+}
 
 /**
  * How rows are written:
@@ -819,12 +908,6 @@ export declare interface ImportOptions {
      * `runImport` for why replace cannot sensibly run without it.
      */
     readonly atomic?: boolean;
-    /** The streaming import path (`unpackBundle` with a storage {@link AssetSink})
-     *  restores blobs to dest storage AS the archive streams in — before the table
-     *  phase — so the manifest's asset phase must be skipped here (the blobs are
-     *  already landed + integrity-checked; re-reading them from a temp dir that
-     *  doesn't exist would fail). Set by the streaming importer, not by users. */
-    readonly assetsAlreadyRestored?: boolean;
     /** The TARGET app's live schema (from `introspectSchema`). When set, a
      *  fingerprint pre-flight compares it — the SAME way the export computed the
      *  bundle's fingerprint — against `manifest.source.schemaFingerprint`. On a
@@ -935,6 +1018,14 @@ export declare interface ImportOptions {
      * run means a clean import.
      */
     readonly dryRun?: boolean;
+    /**
+     * The target's field-encryption keys. When given, a bundle whose values need
+     * a key the target does not hold is refused before any row is written
+     * ({@link FieldKeysMissingError}) — unless `nullSealed`, which writes NULL
+     * for every whole value no held key opens and reports the sealed copies
+     * inside other values it cannot null.
+     */
+    readonly fieldKeys?: ImportFieldKeys;
     /**
      * Rows per savepoint when one is needed (inside a transaction, on an engine
      * that aborts it on a failed statement). Larger = fewer savepoints on the
@@ -971,6 +1062,8 @@ export declare interface ImportOutcome {
     readonly fullyResumed: boolean;
     /** Where the target's previous rows were captured, when they were. */
     readonly rollback?: RollbackCapture;
+    /** With `fieldKeys.nullSealed`: what was written as NULL, and the sealed copies left unreadable. */
+    readonly sealed?: SealedImportReport;
 }
 
 /** An integrity check failed — a table's content checksum or row count did not
@@ -1021,6 +1114,12 @@ export declare interface Ledger {
     tables: Record<string, {
         readonly rowCount: number;
         readonly checksum: string;
+        /** Export only: the table's envelopes per field-key id — a resumed export
+         *  skips the table and still has to name the keys it needs. */
+        readonly sealed?: {
+            readonly byKey: Readonly<Record<string, number>>;
+            readonly unnamed: number;
+        };
     }>;
     /** sha256 → true for every asset that fully landed. */
     assets: Record<string, true>;
@@ -1092,6 +1191,16 @@ export declare const Manifest: Schema.Struct<{
          *  assets were exported. Each line is an {@link AssetEntry}. */
         readonly index: Schema.optional<Schema.String>;
     }>;
+    /**
+     * The field-encryption key ids the bundle's values are sealed under, read off
+     * the envelopes (no key needed), and how many name no key. An import checks
+     * the target's keys against it before writing a row. Absent in a bundle from
+     * before this was recorded.
+     */
+    readonly fieldKeys: Schema.optional<Schema.Struct<{
+        readonly keyIds: Schema.$Array<Schema.String>;
+        readonly unnamed: Schema.Number;
+    }>>;
     /** Masking audit — present when the export applied a masking policy. Lists
      *  every `table.column` that was transformed (not copied verbatim), so a
      *  reviewer can see exactly what was pseudonymised. */
@@ -1291,6 +1400,20 @@ export declare type NativeStep = {
     readonly kind: 'copy';
     readonly from: string;
     readonly to: string;
+}
+/** A consistent snapshot of a live SQLite database through its backup API —
+*  write-ahead log included — written beside the target and renamed into place. */
+| {
+    readonly kind: 'sqlite-backup';
+    readonly from: string;
+    readonly to: string;
+}
+/** `VACUUM INTO` through the database's own connection — written beside the
+*  target and renamed into place. Needs the caller's {@link NativeSql}. */
+| {
+    readonly kind: 'vacuum-into';
+    readonly conn: NativeConn;
+    readonly to: string;
 } | MssqlReplaceStep;
 
 /** A native backup/restore tool failed — missing binary, non-zero exit, or an
@@ -1431,6 +1554,19 @@ export declare interface PortabilityIssue {
     readonly reason: string;
 }
 
+export declare const prepareAssetsFromCas: (opts: {
+    readonly sink: AssetSink;
+    readonly assetsDir: string;
+    readonly indexPath: string;
+}) => Effect.Effect<PreparedAssets, BundleError | IntegrityError | StorageError, Scope.Scope>;
+
+/** Private immutable copies, owned by the calling Effect scope. */
+export declare interface PreparedAssets {
+    readonly [admitted]: true;
+    readonly assetsDir: string;
+    readonly entries: ReadonlyArray<AssetEntry>;
+}
+
 export declare const previewMasking: (opts: MaskingPreviewOptions) => Effect.Effect<MaskingPreview, TableStreamError | ScopeError>;
 
 /**
@@ -1469,6 +1605,14 @@ export declare const readAssetIndex: (indexPath: string) => Stream.Stream<AssetE
  *  (a missing ledger just means "nothing done yet" — never a hard failure). */
 export declare const readLedger: (path: string) => Effect.Effect<Ledger>;
 
+/** Reconnect a target after a native database replacement. Re-admit only the
+ * original private immutable snapshot, under the new sink's full checks. */
+export declare const readmitPreparedAssets: (opts: {
+    readonly prepared: PreparedAssets;
+    readonly previousSink: AssetSink;
+    readonly sink: AssetSink;
+}) => Effect.Effect<PreparedAssets, BundleError | IntegrityError | StorageError, Scope.Scope>;
+
 /** Resolve the action for one column (column override → class → safe → unclassified). */
 export declare const resolveAction: (table: string, column: string, cls: ColumnClassification | undefined, policy: MaskingPolicy) => MaskAction;
 
@@ -1492,19 +1636,13 @@ export declare interface ResolveScopeDeps {
 }
 
 /**
- * Restore one CAS asset to the sink under `key`, re-hashing the bytes as they
- * stream through and refusing (IntegrityError) if they don't match the expected
- * sha — so a corrupted bundle can never silently re-upload bad bytes.
+ * Admit a private copy of one CAS asset before its bytes reach the sink.
+ * Whole-index consumers use the shared CAS phase to admit all entries first.
  */
-export declare const restoreAssetFromCas: (sink: AssetSink, assetsDir: string, entry: {
-    readonly key: string;
-    readonly sha256: string;
-    readonly size: number;
-    readonly contentType: string;
-}) => Effect.Effect<void, BundleError | IntegrityError | StorageError>;
+export declare const restoreAssetFromCas: (sink: AssetSink, assetsDir: string, entry: AssetEntry) => Effect.Effect<void, BundleError | IntegrityError | StorageError>;
 
 /**
- * Push every asset named in the index back to the sink, re-hashing on the way.
+ * Push every asset from the private, fully verified index back to its admitted sink.
  *
  * A hash mismatch is an {@link IntegrityError} and stops the phase — a corrupted
  * artefact must never silently re-upload bad bytes over good ones, which is the
@@ -1520,6 +1658,7 @@ export declare const restoreAssetsFromCas: (opts: {
     readonly assetsDir: string;
     readonly indexPath: string;
     readonly progress?: AssetProgress;
+    readonly prepared?: PreparedAssets;
 }) => Effect.Effect<RestoredAssets, BundleError | IntegrityError | StorageError>;
 
 /** Build the restore step for a dialect reading `inFile`. */
@@ -1666,6 +1805,19 @@ declare const ScopeError_base: Schema.Class<ScopeError, Schema.TaggedStruct<"Sco
     readonly reason: Schema.String;
 }>, YieldableError>;
 
+export declare interface SealedImportReport {
+    readonly nulled: ReadonlyArray<{
+        readonly table: string;
+        readonly column: string;
+        readonly count: number;
+    }>;
+    readonly unreadable: ReadonlyArray<{
+        readonly table: string;
+        readonly column: string;
+        readonly count: number;
+    }>;
+}
+
 /**
  * A store that can run raw SQL for us.
  *
@@ -1754,27 +1906,29 @@ export declare interface StagingSnapshot {
     }>;
 }
 
+export declare const storageAssetInfo: (row: Row) => AssetInfo;
+
 /** Build an {@link AssetSink} over a storage provider. */
-export declare const storageAssetSink: (provider: StorageProvider) => AssetSink;
+export declare const storageAssetSink: (router: StorageDestinationRouter) => AssetSink;
 
 /** Build an {@link AssetSource} over a storage provider — `list` is supplied by
  *  the caller (typically streaming the `_voltro_storage_refs` table), `open`
  *  uses the provider's streaming read (buffered fallback for non-streaming
  *  backends). */
-export declare const storageAssetSource: (provider: StorageProvider, list: () => Stream.Stream<AssetInfo, unknown>) => AssetSource;
+export declare const storageAssetSource: (router: StorageDestinationRouter, list: (bundle?: AssetBundleContext) => Stream.Stream<AssetInfo, unknown>) => AssetSource;
 
 /** Stream blobs TO object storage. `has` powers resume (skip existing keys);
  *  `write` streams the archive's blob content straight into the provider. */
-export declare const storageBlobSink: (provider: StorageProvider) => BlobSink;
+export declare const storageBlobSink: (router: StorageDestinationRouter) => BlobSink;
 
 /** Stream blobs FROM object storage. `list` enumerates metadata (typically the
  *  storage refs table); `open` streams each blob's bytes via the provider. */
-export declare const storageBlobSource: (provider: StorageProvider, list: () => Stream.Stream<AssetInfo, unknown>) => BlobSource;
+export declare const storageBlobSource: (router: StorageDestinationRouter, list: (bundleDir?: string) => Stream.Stream<AssetInfo, unknown>) => BlobSource;
 
 /** A {@link BlobSource} over a provider whose blob list comes from the app's
  *  `_voltro_storage_refs` table — the standard export source for a running
  *  instance (CLI file export and the prod export endpoint both use it). */
-export declare const storageRefsBlobSource: (provider: StorageProvider, store: DataStore) => BlobSource;
+export declare const storageRefsBlobSource: (router: StorageDestinationRouter, store: DataStore) => BlobSource;
 
 export declare interface SubsetSpec {
     /** Seed rows per table. `table → predicate` (a `null`/absent predicate seeds
@@ -1854,8 +2008,8 @@ export declare const topoSortTables: (tables: ReadonlyArray<string>, snapshot: S
 
 /**
  * Unpack a framed byte stream. Plain files are written under `dir` (traversal
- * guarded); blob entries are streamed to `blobs` (dest storage) — never to local
- * disk — with the trailing sha verified as they pass through. A blob whose key
+ * guarded); entries are staged and verified through authenticated source EOF
+ * before `blobs` (dest storage) participates. A blob whose key
  * already exists at the sink is skipped (resume).
  */
 export declare const unpackBundle: (source: AsyncIterable<Uint8Array>, opts: {
@@ -1866,7 +2020,7 @@ export declare const unpackBundle: (source: AsyncIterable<Uint8Array>, opts: {
 /** Convenience: unpack a bundle fully to a local directory (plain files as-is,
  *  blob entries materialised under `assets/`). For data-only bundles and
  *  `voltro data unpack`; the streaming import path passes a storage {@link BlobSink}
- *  to {@link unpackBundle} instead so blobs never touch local disk. */
+ *  to {@link unpackBundle} for a destination after archive admission. */
 export declare const unpackBundleToDir: (source: AsyncIterable<Uint8Array>, dir: string) => Promise<void>;
 
 /**

package/dist/index.js

@@ -1,28 +1,30 @@
 import { DEFAULT_CLASS_ACTIONS as e, applyAction as t, classificationFromSnapshot as n, maskRow as r, planMasking as i, resolveAction as a, scanForLeaks as o } from "./masking.js";
 import { ENVIRONMENT_LOCAL_TABLES as s, ENVIRONMENT_PORTABLE_TABLES as c, environmentLocalNotice as l, environmentLocalReason as u, isEnvironmentLocal as d } from "./environmentLocal.js";
-import { BundleError as f, CodecError as p, CompressionError as m, CrossDialectError as h, ImportModeError as g, IntegrityError as _, MaskingError as v, NativeToolError as y, RowsRefusedError as b, SchemaDriftError as x, ScopeError as S } from "./errors.js";
-import { computeSubsetIds as C, subsetToScope as w } from "./subset.js";
-import { resolveScope as T } from "./scope.js";
-import { restoreAssetFromCas as E, storageAssetSink as D, storageAssetSource as O, writeAssetToCas as k } from "./assets.js";
-import { captureAssetsToCas as A, noAssetProgress as j, readAssetIndex as M, restoreAssetsFromCas as N } from "./casAssetPhase.js";
-import { compressionExt as P, hasZstd as F, makeCompressor as I, makeDecompressor as L, pickCompression as R } from "./compress.js";
-import { ASSETS_DIR as z, ASSET_INDEX_FILE as B, DATA_DIR as V, FORMAT_VERSION as H, LEDGER_FILE as U, MANIFEST_FILE as W, decodeManifest as G, encodeManifest as K } from "./manifest.js";
-import { assetDone as q, emptyLedger as J, readLedger as Y, tableDone as X, writeLedger as Z } from "./ledger.js";
-import { decodeRow as Q, decodeValue as $, encodeRow as ee, encodeValue as te } from "./typedCodec.js";
-import { ndjsonToRows as ne, rowsToNdjson as re } from "./ndjson.js";
-import { topoSortTables as ie } from "./topo.js";
-import { runExport as ae } from "./exporter.js";
-import { outsideReferenceRefusal as oe, outsideReferencesInto as se } from "./replacePreflight.js";
-import { canonicalType as ce, checkPortability as le, formatIssue as ue } from "./crossDialect.js";
-import { STAGING_PREFIX as de, canStage as fe, decideStaging as pe, dropStagingSql as me, execStatements as he, stagingNameFor as ge } from "./stagedSwap.js";
-import { checkSchemaDrift as _e, diffSnapshots as ve } from "./schemaDrift.js";
-import { SAVEPOINT_BATCH_SIZE as ye, asImportError as be, runImport as xe } from "./importer.js";
-import { defineDataProfile as Se } from "./profile.js";
-import { assessLiveImport as Ce } from "./liveGuard.js";
-import { dirBlobSink as we, packBundle as Te, peekBundle as Ee, unpackBundle as De, unpackBundleToDir as Oe } from "./bundleArchive.js";
-import { decryptStream as ke, encryptStream as Ae } from "./bundleCrypto.js";
-import { storageBlobSink as je, storageBlobSource as Me, storageRefsBlobSource as Ne } from "./blobStorage.js";
-import { EXCLUDED_FROM_DUMP as Pe, backupArtifactName as Fe, backupCommand as Ie, dumpExcludedTables as Le, mssqlReplaceStatements as Re, mssqlScratchName as ze, restoreCommand as Be, runNativeStep as Ve } from "./nativeDump.js";
-import { previewMasking as He } from "./maskingPreview.js";
-import { classifyImportDrift as Ue, formatImportDrift as We } from "./driftClassification.js";
-export { z as ASSETS_DIR, B as ASSET_INDEX_FILE, f as BundleError, p as CodecError, m as CompressionError, h as CrossDialectError, V as DATA_DIR, e as DEFAULT_CLASS_ACTIONS, s as ENVIRONMENT_LOCAL_TABLES, c as ENVIRONMENT_PORTABLE_TABLES, Pe as EXCLUDED_FROM_DUMP, H as FORMAT_VERSION, g as ImportModeError, _ as IntegrityError, U as LEDGER_FILE, W as MANIFEST_FILE, v as MaskingError, y as NativeToolError, b as RowsRefusedError, ye as SAVEPOINT_BATCH_SIZE, de as STAGING_PREFIX, x as SchemaDriftError, S as ScopeError, t as applyAction, be as asImportError, Ce as assessLiveImport, q as assetDone, Fe as backupArtifactName, Ie as backupCommand, fe as canStage, ce as canonicalType, A as captureAssetsToCas, le as checkPortability, _e as checkSchemaDrift, n as classificationFromSnapshot, Ue as classifyImportDrift, P as compressionExt, C as computeSubsetIds, pe as decideStaging, G as decodeManifest, Q as decodeRow, $ as decodeValue, ke as decryptStream, Se as defineDataProfile, ve as diffSnapshots, we as dirBlobSink, me as dropStagingSql, Le as dumpExcludedTables, J as emptyLedger, K as encodeManifest, ee as encodeRow, te as encodeValue, Ae as encryptStream, l as environmentLocalNotice, u as environmentLocalReason, he as execStatements, We as formatImportDrift, ue as formatIssue, F as hasZstd, d as isEnvironmentLocal, I as makeCompressor, L as makeDecompressor, r as maskRow, Re as mssqlReplaceStatements, ze as mssqlScratchName, ne as ndjsonToRows, j as noAssetProgress, oe as outsideReferenceRefusal, se as outsideReferencesInto, Te as packBundle, Ee as peekBundle, R as pickCompression, i as planMasking, He as previewMasking, M as readAssetIndex, Y as readLedger, a as resolveAction, T as resolveScope, E as restoreAssetFromCas, N as restoreAssetsFromCas, Be as restoreCommand, re as rowsToNdjson, ae as runExport, xe as runImport, Ve as runNativeStep, o as scanForLeaks, ge as stagingNameFor, D as storageAssetSink, O as storageAssetSource, je as storageBlobSink, Me as storageBlobSource, Ne as storageRefsBlobSource, w as subsetToScope, X as tableDone, ie as topoSortTables, De as unpackBundle, Oe as unpackBundleToDir, k as writeAssetToCas, Z as writeLedger };
+import { BundleError as f, CodecError as p, CompressionError as m, CrossDialectError as h, FieldKeysMissingError as g, ImportModeError as _, IntegrityError as v, MaskingError as y, NativeToolError as b, RowsRefusedError as x, SchemaDriftError as S, ScopeError as C } from "./errors.js";
+import { computeSubsetIds as w, subsetToScope as T } from "./subset.js";
+import { resolveScope as E } from "./scope.js";
+import { ASSETS_DIR as D, ASSET_INDEX_FILE as O, AssetAddress as k, DATA_DIR as A, FORMAT_VERSION as j, LEDGER_FILE as M, MANIFEST_FILE as N, decodeManifest as P, encodeManifest as F } from "./manifest.js";
+import { readmitPreparedAssets as I } from "./assetAdmission.js";
+import { restoreAssetFromCas as L, storageAssetSink as R, storageAssetSource as z, writeAssetToCas as B } from "./assets.js";
+import { captureAssetsToCas as V, noAssetProgress as H, prepareAssetsFromCas as U, readAssetIndex as W, restoreAssetsFromCas as G } from "./casAssetPhase.js";
+import { compressionExt as K, hasZstd as q, makeCompressor as J, makeDecompressor as Y, pickCompression as X } from "./compress.js";
+import { assetDone as Z, emptyLedger as Q, readLedger as $, tableDone as ee, writeLedger as te } from "./ledger.js";
+import { decodeRow as ne, decodeValue as re, encodeRow as ie, encodeValue as ae } from "./typedCodec.js";
+import { ndjsonToRows as oe, rowsToNdjson as se } from "./ndjson.js";
+import { topoSortTables as ce } from "./topo.js";
+import { runExport as le } from "./exporter.js";
+import { outsideReferenceRefusal as ue, outsideReferencesInto as de } from "./replacePreflight.js";
+import { canonicalType as fe, checkPortability as pe, formatIssue as me } from "./crossDialect.js";
+import { STAGING_PREFIX as he, canStage as ge, decideStaging as _e, dropStagingSql as ve, execStatements as ye, stagingNameFor as be } from "./stagedSwap.js";
+import { checkSchemaDrift as xe, diffSnapshots as Se } from "./schemaDrift.js";
+import { SAVEPOINT_BATCH_SIZE as Ce, asImportError as we, runImport as Te } from "./importer.js";
+import { defineDataProfile as Ee } from "./profile.js";
+import { assessLiveImport as De } from "./liveGuard.js";
+import { dirBlobSink as Oe, packBundle as ke, peekBundle as Ae, unpackBundle as je, unpackBundleToDir as Me } from "./bundleArchive.js";
+import { decryptStream as Ne, encryptStream as Pe } from "./bundleCrypto.js";
+import { assetBundleContext as Fe, bundleAssetInfoStream as Ie, storageAssetInfo as Le } from "./storageAssetMetadata.js";
+import { blobAssetSink as Re, storageBlobSink as ze, storageBlobSource as Be, storageRefsBlobSource as Ve } from "./blobStorage.js";
+import { EXCLUDED_FROM_DUMP as He, backupArtifactName as Ue, backupCommand as We, dumpExcludedTables as Ge, mssqlReplaceStatements as Ke, mssqlScratchName as qe, restoreCommand as Je, runNativeStep as Ye } from "./nativeDump.js";
+import { previewMasking as Xe } from "./maskingPreview.js";
+import { classifyImportDrift as Ze, formatImportDrift as Qe } from "./driftClassification.js";
+export { D as ASSETS_DIR, O as ASSET_INDEX_FILE, k as AssetAddress, f as BundleError, p as CodecError, m as CompressionError, h as CrossDialectError, A as DATA_DIR, e as DEFAULT_CLASS_ACTIONS, s as ENVIRONMENT_LOCAL_TABLES, c as ENVIRONMENT_PORTABLE_TABLES, He as EXCLUDED_FROM_DUMP, j as FORMAT_VERSION, g as FieldKeysMissingError, _ as ImportModeError, v as IntegrityError, M as LEDGER_FILE, N as MANIFEST_FILE, y as MaskingError, b as NativeToolError, x as RowsRefusedError, Ce as SAVEPOINT_BATCH_SIZE, he as STAGING_PREFIX, S as SchemaDriftError, C as ScopeError, t as applyAction, we as asImportError, De as assessLiveImport, Fe as assetBundleContext, Z as assetDone, Ue as backupArtifactName, We as backupCommand, Re as blobAssetSink, Ie as bundleAssetInfoStream, ge as canStage, fe as canonicalType, V as captureAssetsToCas, pe as checkPortability, xe as checkSchemaDrift, n as classificationFromSnapshot, Ze as classifyImportDrift, K as compressionExt, w as computeSubsetIds, _e as decideStaging, P as decodeManifest, ne as decodeRow, re as decodeValue, Ne as decryptStream, Ee as defineDataProfile, Se as diffSnapshots, Oe as dirBlobSink, ve as dropStagingSql, Ge as dumpExcludedTables, Q as emptyLedger, F as encodeManifest, ie as encodeRow, ae as encodeValue, Pe as encryptStream, l as environmentLocalNotice, u as environmentLocalReason, ye as execStatements, Qe as formatImportDrift, me as formatIssue, q as hasZstd, d as isEnvironmentLocal, J as makeCompressor, Y as makeDecompressor, r as maskRow, Ke as mssqlReplaceStatements, qe as mssqlScratchName, oe as ndjsonToRows, H as noAssetProgress, ue as outsideReferenceRefusal, de as outsideReferencesInto, ke as packBundle, Ae as peekBundle, X as pickCompression, i as planMasking, U as prepareAssetsFromCas, Xe as previewMasking, W as readAssetIndex, $ as readLedger, I as readmitPreparedAssets, a as resolveAction, E as resolveScope, L as restoreAssetFromCas, G as restoreAssetsFromCas, Je as restoreCommand, se as rowsToNdjson, le as runExport, Te as runImport, Ye as runNativeStep, o as scanForLeaks, be as stagingNameFor, Le as storageAssetInfo, R as storageAssetSink, z as storageAssetSource, ze as storageBlobSink, Be as storageBlobSource, Ve as storageRefsBlobSource, T as subsetToScope, ee as tableDone, ce as topoSortTables, je as unpackBundle, Me as unpackBundleToDir, B as writeAssetToCas, te as writeLedger };

package/dist/manifest.js

@@ -1,52 +1,56 @@
-import { Schema as e } from "effect";
+import { isSafeBundlePath as e } from "./bundlePaths.js";
+import { Schema as t } from "effect";
+import { StorageAddress as n } from "@voltro/plugin-storage/errors";
 //#region src/manifest.ts
-var t = 1, n = e.Struct({
-	name: e.String,
-	file: e.String,
-	rowCount: e.Number,
-	checksum: e.String,
-	columnTypes: e.Record(e.String, e.String),
-	primaryKey: e.String
-}), r = e.Struct({
-	kind: e.Literals([
+var r = 2, i = t.Struct({
+	name: t.String,
+	file: t.String,
+	rowCount: t.Number,
+	checksum: t.String,
+	columnTypes: t.Record(t.String, t.String),
+	primaryKey: t.String
+}), a = t.Struct({
+	kind: t.Literals([
 		"all",
 		"tenant",
 		"tables"
 	]),
-	tenantId: e.optional(e.String),
-	tables: e.optional(e.Array(e.String)),
-	exclude: e.optional(e.Array(e.String))
-}), i = e.Struct({
-	count: e.Number,
-	totalBytes: e.Number,
-	index: e.optional(e.String)
-});
-e.Struct({
-	key: e.String,
-	sha256: e.String,
-	size: e.Number,
-	contentType: e.String
-});
-var a = e.Struct({
-	formatVersion: e.Number,
-	createdAt: e.String,
-	source: e.Struct({
-		dialect: e.String,
-		schemaFingerprint: e.String
+	tenantId: t.optional(t.String),
+	tables: t.optional(t.Array(t.String)),
+	exclude: t.optional(t.Array(t.String))
+}), o = t.Struct({
+	count: t.Number,
+	totalBytes: t.Number,
+	index: t.optional(t.String.pipe(t.check(t.makeFilter(e))))
+}), s = n, c = t.Struct({
+	...s.fields,
+	sha256: t.String.pipe(t.check(t.isPattern(/^[a-f0-9]{64}$/))),
+	size: t.Number.pipe(t.check(t.isInt()), t.check(t.isGreaterThanOrEqualTo(0)), t.check(t.isLessThanOrEqualTo(2 ** 53 - 1))),
+	contentType: t.String.pipe(t.check(t.isMinLength(1)), t.check(t.makeFilter((e) => !/[\uD800-\uDFFF]/u.test(e))))
+}), l = t.Struct({
+	formatVersion: t.Number,
+	createdAt: t.String,
+	source: t.Struct({
+		dialect: t.String,
+		schemaFingerprint: t.String
 	}),
-	scope: r,
-	consistency: e.Literals(["live", "snapshot"]),
-	compression: e.Literals([
+	scope: a,
+	consistency: t.Literals(["live", "snapshot"]),
+	compression: t.Literals([
 		"zstd",
 		"gzip",
 		"none"
 	]),
-	tables: e.Array(n),
-	assets: i,
-	masking: e.optional(e.Struct({
-		policyId: e.optional(e.String),
-		transformed: e.Array(e.String)
+	tables: t.Array(i),
+	assets: o,
+	fieldKeys: t.optional(t.Struct({
+		keyIds: t.Array(t.String),
+		unnamed: t.Number
+	})),
+	masking: t.optional(t.Struct({
+		policyId: t.optional(t.String),
+		transformed: t.Array(t.String)
 	}))
-}), o = e.encodeSync(e.fromJsonString(a)), s = e.decodeUnknownSync(e.fromJsonString(a)), c = "manifest.json", l = "README.md", u = "data", d = "assets", f = "assets/index.ndjson", p = ".ledger.json";
+}), u = t.encodeSync(t.fromJsonString(l)), d = t.decodeUnknownSync(t.fromJsonString(l)), f = "manifest.json", p = "README.md", m = "data", h = "assets", g = "assets/index.ndjson", _ = ".ledger.json";
 //#endregion
-export { d as ASSETS_DIR, f as ASSET_INDEX_FILE, i as AssetSummary, u as DATA_DIR, r as ExportScope, t as FORMAT_VERSION, p as LEDGER_FILE, c as MANIFEST_FILE, a as Manifest, l as README_FILE, n as TableEntry, s as decodeManifest, o as encodeManifest };
+export { h as ASSETS_DIR, g as ASSET_INDEX_FILE, s as AssetAddress, c as AssetEntry, o as AssetSummary, m as DATA_DIR, a as ExportScope, r as FORMAT_VERSION, _ as LEDGER_FILE, f as MANIFEST_FILE, l as Manifest, p as README_FILE, i as TableEntry, d as decodeManifest, u as encodeManifest };