@voltro/data-transfer 0.85.0 → 0.87.0

package/dist/index.d.ts

@@ -8,14 +8,19 @@ import { Predicate } from '@voltro/database';
 import { Row } from '@voltro/database';
 import { Schema } from 'effect';
 import { SchemaSnapshot } from '@voltro/database/sql';
+import { Scope } from 'effect';
+import { StorageAddress } from '@voltro/plugin-storage';
+import { StorageAddress as StorageAddress_2 } from '@voltro/plugin-storage/errors';
+import { StorageDestinationRouter } from '@voltro/plugin-storage';
 import { StorageError } from '@voltro/plugin-storage';
-import { StorageProvider } from '@voltro/plugin-storage';
 import { StoredStream } from '@voltro/plugin-storage';
 import { Stream } from 'effect';
 import { TableStreamError } from '@voltro/database';
 import { VoidIfEmpty } from 'effect/Types';
 import { YieldableError } from 'effect/Cause';
 
+declare const admitted: unique symbol;
+
 /** Apply one action to one value. A null value is always passed through. */
 export declare const applyAction: (action: MaskAction, input: MaskInput) => unknown;
 
@@ -55,23 +60,42 @@ export declare const assessLiveImport: (opts: {
 
 export declare const ASSET_INDEX_FILE = "assets/index.ndjson";
 
+/** The complete logical address, shared with storage ownership. */
+export declare const AssetAddress: Schema.Struct<{
+    readonly destination: Schema.String;
+    readonly tenantId: Schema.NullOr<Schema.String>;
+    readonly key: Schema.String;
+}>;
+
+export declare type AssetAddress = StorageAddress_2;
+
+/** The exact materialized row selection, after scope, subset and masking. */
+export declare interface AssetBundleContext {
+    readonly dir: string;
+    readonly tables: ReadonlyArray<TableEntry>;
+    readonly compression: Compression;
+}
+
+export declare const assetBundleContext: (dir: string) => Effect.Effect<AssetBundleContext, BundleError>;
+
 export declare const assetDone: (ledger: Ledger, sha256: string) => boolean;
 
-/** One content-addressed asset. `path` is `assets/<sha256>`; `key` is the
- *  original storage key so the importer restores it under the same key. */
 export declare const AssetEntry: Schema.Struct<{
-    readonly key: Schema.String;
     readonly sha256: Schema.String;
     readonly size: Schema.Number;
     readonly contentType: Schema.String;
+    readonly destination: Schema.String;
+    readonly tenantId: Schema.NullOr<Schema.String>;
+    readonly key: Schema.String;
 }>;
 
 export declare type AssetEntry = Schema.Schema.Type<typeof AssetEntry>;
 
 /** Metadata for one asset to export — enough to enumerate without reading the
  *  bytes. Sourced from the storage refs table by the caller. */
-export declare interface AssetInfo {
-    readonly key: string;
+export declare interface AssetInfo extends StorageAddress {
+    /** SQL checksum, when known; capture refuses bytes that contradict it. */
+    readonly sha256?: string;
     readonly contentType: string;
     readonly size: number;
 }
@@ -88,7 +112,7 @@ export declare interface AssetInfo {
  */
 export declare interface AssetProgress {
     readonly done: (key: string) => boolean;
-    /** Record `key` as landed. Called AFTER the bytes are durable, never before —
+    /** Record the complete address identity as landed. Called AFTER the bytes are durable, never before —
      *  a ledger written first turns a crash into silent data loss on resume. */
     readonly mark: (key: string, entry: AssetEntry) => Effect.Effect<void, BundleError>;
 }
@@ -97,20 +121,22 @@ export declare const ASSETS_DIR = "assets";
 
 /** Where import writes assets TO. */
 export declare interface AssetSink {
-    readonly write: (key: string, stream: ByteStream, meta: {
+    readonly write: (address: StorageAddress, stream: ByteStream, meta: {
         readonly contentType: string;
         readonly totalSize?: number;
     }) => Effect.Effect<unknown, StorageError>;
-    /** Whether the key already exists at the destination (skip re-upload). */
-    readonly has: (key: string) => Effect.Effect<boolean, StorageError>;
+    /** Resolve the address and probe its destination during admission. */
+    readonly has: (address: StorageAddress) => Effect.Effect<boolean, StorageError>;
+    /** True only when this address holds the admitted content, never HEAD alone. */
+    readonly matches: (entry: AssetEntry) => Effect.Effect<boolean, StorageError>;
 }
 
 /** Where export reads assets FROM. */
 export declare interface AssetSource {
     /** Lazily enumerate every asset (metadata only; bytes stream on demand). */
-    readonly list: () => Stream.Stream<AssetInfo, unknown>;
+    readonly list: (bundle?: AssetBundleContext) => Stream.Stream<AssetInfo, unknown>;
     /** Open one asset as a byte stream. */
-    readonly open: (key: string) => Effect.Effect<StoredStream, StorageError>;
+    readonly open: (address: StorageAddress) => Effect.Effect<StoredStream, StorageError>;
 }
 
 export declare const AssetSummary: Schema.Struct<{
@@ -129,25 +155,42 @@ export declare const backupArtifactName: (dialect: string) => string;
 /** Build the backup step for a dialect writing to `outFile`. */
 export declare const backupCommand: (dialect: string, conn: NativeConn, outFile: string) => NativeStep;
 
-/** Where blobs stream TO on import. `has` lets a resumed run skip existing keys. */
+/** Use an archive destination after admission and the table preflight. */
+export declare const blobAssetSink: (sink: BlobSink) => AssetSink;
+
+/** Where blobs stream TO on import. Admission resolves every address; matches verifies resume. */
 export declare interface BlobSink {
-    has: (key: string) => Promise<boolean>;
-    write: (key: string, content: AsyncIterable<Uint8Array>, meta: {
+    has: (address: AssetAddress, options?: {
+        readonly signal: AbortSignal;
+    }) => Promise<boolean>;
+    matches: (entry: AssetEntry, options?: {
+        readonly signal: AbortSignal;
+    }) => Promise<boolean>;
+    write: (address: AssetAddress, content: AsyncIterable<Uint8Array>, meta: {
         readonly contentType: string;
         readonly size: number;
+    }, options?: {
+        readonly signal: AbortSignal;
     }) => Promise<void>;
 }
 
 /** Where blobs stream FROM on export (metadata + a byte stream per key). */
 export declare interface BlobSource {
-    list: () => AsyncIterable<{
-        readonly key: string;
+    list: (bundleDir?: string) => AsyncIterable<AssetAddress & {
         readonly contentType: string;
         readonly size: number;
+        readonly sha256?: string;
     }>;
-    open: (key: string) => AsyncIterable<Uint8Array>;
+    open: (asset: AssetAddress & {
+        readonly contentType: string;
+        readonly size: number;
+        readonly sha256?: string;
+    }) => AsyncIterable<Uint8Array>;
 }
 
+/** Enumerate the exported refs, never a broader live SQL selection. */
+export declare const bundleAssetInfoStream: (bundle: AssetBundleContext) => Stream.Stream<AssetInfo, unknown>;
+
 /** A bundle is malformed: bad manifest, unknown format version, missing file. */
 export declare class BundleError extends BundleError_base {
     /** The text a log line or a run record shows for this refusal. */
@@ -411,13 +454,17 @@ export declare const decodeManifest: (input: unknown, options?: ParseOptions) =>
         readonly primaryKey: string;
     }[];
     readonly consistency: "live" | "snapshot";
-    readonly compression: "zstd" | "gzip" | "none";
+    readonly compression: "none" | "zstd" | "gzip";
     readonly formatVersion: number;
     readonly assets: {
         readonly count: number;
         readonly totalBytes: number;
         readonly index?: string | undefined;
     };
+    readonly fieldKeys?: {
+        readonly keyIds: readonly string[];
+        readonly unnamed: number;
+    } | undefined;
     readonly masking?: {
         readonly transformed: readonly string[];
         readonly policyId?: string | undefined;
@@ -535,13 +582,17 @@ export declare const encodeManifest: (input: {
         readonly primaryKey: string;
     }[];
     readonly consistency: "live" | "snapshot";
-    readonly compression: "zstd" | "gzip" | "none";
+    readonly compression: "none" | "zstd" | "gzip";
     readonly formatVersion: number;
     readonly assets: {
         readonly count: number;
         readonly totalBytes: number;
         readonly index?: string | undefined;
     };
+    readonly fieldKeys?: {
+        readonly keyIds: readonly string[];
+        readonly unnamed: number;
+    } | undefined;
     readonly masking?: {
         readonly transformed: readonly string[];
         readonly policyId?: string | undefined;
@@ -724,9 +775,26 @@ export declare const ExportScope: Schema.Struct<{
 
 export declare type ExportScope = Schema.Schema.Type<typeof ExportScope>;
 
+/**
+ * The bundle's values are sealed under field-encryption keys the target does
+ * not hold. Refused before any row is written: importing them anyway writes
+ * rows every read of which fails. `--null-sealed` is the explicit choice for
+ * another environment — see `ImportFieldKeys`.
+ */
+export declare class FieldKeysMissingError extends FieldKeysMissingError_base {
+    get message(): string;
+}
+
+declare const FieldKeysMissingError_base: Schema.Class<FieldKeysMissingError, Schema.TaggedStruct<"FieldKeysMissingError", {
+    /** Key ids the bundle needs that the target does not hold. Never a key. */
+    readonly missing: Schema.$Array<Schema.String>;
+    /** Key ids the target holds. */
+    readonly held: Schema.$Array<Schema.String>;
+}>, YieldableError>;
+
 /** Bump when the on-disk shape changes incompatibly. An importer refuses a
  *  bundle whose `formatVersion` it doesn't understand rather than guessing. */
-export declare const FORMAT_VERSION = 1;
+export declare const FORMAT_VERSION = 2;
 
 /** Render for the CLI — the same shape `db plan` prints, for the same reason. */
 export declare const formatImportDrift: (c: DriftClassification) => ReadonlyArray<string>;
@@ -759,7 +827,28 @@ export declare interface ImportCopyLoader {
     }) => Promise<number>;
 }
 
-export declare type ImportError = BundleError | CodecError | IntegrityError | CrossDialectError | ImportModeError | RowsRefusedError | SchemaDriftError | StorageError;
+export declare type ImportError = BundleError | CodecError | IntegrityError | CrossDialectError | FieldKeysMissingError | ImportModeError | RowsRefusedError | SchemaDriftError | StorageError;
+
+/** What an import is told about the target's keys. */
+export declare interface ImportFieldKeys {
+    /** The key ids the target environment holds. */
+    readonly held: ReadonlyArray<string>;
+    /**
+     * Restoring into another environment: write NULL for a whole value no held
+     * key opens (the source ciphertext is useless here; users re-enter it), and
+     * report sealed copies inside other values, which cannot be nulled.
+     */
+    readonly nullSealed?: boolean;
+    /**
+     * Does a held key open this value? Given by a caller that has the keys, for
+     * values that name no key. Without it, an unnamed value counts as opened.
+     */
+    readonly opens?: (value: string, context: {
+        readonly table: string;
+        readonly column: string;
+        readonly row: Readonly<Record<string, unknown>>;
+    }) => boolean;
+}
 
 /**
  * How rows are written:
@@ -819,12 +908,6 @@ export declare interface ImportOptions {
      * `runImport` for why replace cannot sensibly run without it.
      */
     readonly atomic?: boolean;
-    /** The streaming import path (`unpackBundle` with a storage {@link AssetSink})
-     *  restores blobs to dest storage AS the archive streams in — before the table
-     *  phase — so the manifest's asset phase must be skipped here (the blobs are
-     *  already landed + integrity-checked; re-reading them from a temp dir that
-     *  doesn't exist would fail). Set by the streaming importer, not by users. */
-    readonly assetsAlreadyRestored?: boolean;
     /** The TARGET app's live schema (from `introspectSchema`). When set, a
      *  fingerprint pre-flight compares it — the SAME way the export computed the
      *  bundle's fingerprint — against `manifest.source.schemaFingerprint`. On a
@@ -935,6 +1018,14 @@ export declare interface ImportOptions {
      * run means a clean import.
      */
     readonly dryRun?: boolean;
+    /**
+     * The target's field-encryption keys. When given, a bundle whose values need
+     * a key the target does not hold is refused before any row is written
+     * ({@link FieldKeysMissingError}) — unless `nullSealed`, which writes NULL
+     * for every whole value no held key opens and reports the sealed copies
+     * inside other values it cannot null.
+     */
+    readonly fieldKeys?: ImportFieldKeys;
     /**
      * Rows per savepoint when one is needed (inside a transaction, on an engine
      * that aborts it on a failed statement). Larger = fewer savepoints on the
@@ -971,6 +1062,8 @@ export declare interface ImportOutcome {
     readonly fullyResumed: boolean;
     /** Where the target's previous rows were captured, when they were. */
     readonly rollback?: RollbackCapture;
+    /** With `fieldKeys.nullSealed`: what was written as NULL, and the sealed copies left unreadable. */
+    readonly sealed?: SealedImportReport;
 }
 
 /** An integrity check failed — a table's content checksum or row count did not
@@ -1021,6 +1114,12 @@ export declare interface Ledger {
     tables: Record<string, {
         readonly rowCount: number;
         readonly checksum: string;
+        /** Export only: the table's envelopes per field-key id — a resumed export
+         *  skips the table and still has to name the keys it needs. */
+        readonly sealed?: {
+            readonly byKey: Readonly<Record<string, number>>;
+            readonly unnamed: number;
+        };
     }>;
     /** sha256 → true for every asset that fully landed. */
     assets: Record<string, true>;
@@ -1092,6 +1191,16 @@ export declare const Manifest: Schema.Struct<{
          *  assets were exported. Each line is an {@link AssetEntry}. */
         readonly index: Schema.optional<Schema.String>;
     }>;
+    /**
+     * The field-encryption key ids the bundle's values are sealed under, read off
+     * the envelopes (no key needed), and how many name no key. An import checks
+     * the target's keys against it before writing a row. Absent in a bundle from
+     * before this was recorded.
+     */
+    readonly fieldKeys: Schema.optional<Schema.Struct<{
+        readonly keyIds: Schema.$Array<Schema.String>;
+        readonly unnamed: Schema.Number;
+    }>>;
     /** Masking audit — present when the export applied a masking policy. Lists
      *  every `table.column` that was transformed (not copied verbatim), so a
      *  reviewer can see exactly what was pseudonymised. */
@@ -1291,6 +1400,20 @@ export declare type NativeStep = {
     readonly kind: 'copy';
     readonly from: string;
     readonly to: string;
+}
+/** A consistent snapshot of a live SQLite database through its backup API —
+*  write-ahead log included — written beside the target and renamed into place. */
+| {
+    readonly kind: 'sqlite-backup';
+    readonly from: string;
+    readonly to: string;
+}
+/** `VACUUM INTO` through the database's own connection — written beside the
+*  target and renamed into place. Needs the caller's {@link NativeSql}. */
+| {
+    readonly kind: 'vacuum-into';
+    readonly conn: NativeConn;
+    readonly to: string;
 } | MssqlReplaceStep;
 
 /** A native backup/restore tool failed — missing binary, non-zero exit, or an
@@ -1431,6 +1554,19 @@ export declare interface PortabilityIssue {
     readonly reason: string;
 }
 
+export declare const prepareAssetsFromCas: (opts: {
+    readonly sink: AssetSink;
+    readonly assetsDir: string;
+    readonly indexPath: string;
+}) => Effect.Effect<PreparedAssets, BundleError | IntegrityError | StorageError, Scope.Scope>;
+
+/** Private immutable copies, owned by the calling Effect scope. */
+export declare interface PreparedAssets {
+    readonly [admitted]: true;
+    readonly assetsDir: string;
+    readonly entries: ReadonlyArray<AssetEntry>;
+}
+
 export declare const previewMasking: (opts: MaskingPreviewOptions) => Effect.Effect<MaskingPreview, TableStreamError | ScopeError>;
 
 /**
@@ -1469,6 +1605,14 @@ export declare const readAssetIndex: (indexPath: string) => Stream.Stream<AssetE
  *  (a missing ledger just means "nothing done yet" — never a hard failure). */
 export declare const readLedger: (path: string) => Effect.Effect<Ledger>;
 
+/** Reconnect a target after a native database replacement. Re-admit only the
+ * original private immutable snapshot, under the new sink's full checks. */
+export declare const readmitPreparedAssets: (opts: {
+    readonly prepared: PreparedAssets;
+    readonly previousSink: AssetSink;
+    readonly sink: AssetSink;
+}) => Effect.Effect<PreparedAssets, BundleError | IntegrityError | StorageError, Scope.Scope>;
+
 /** Resolve the action for one column (column override → class → safe → unclassified). */
 export declare const resolveAction: (table: string, column: string, cls: ColumnClassification | undefined, policy: MaskingPolicy) => MaskAction;
 
@@ -1492,19 +1636,13 @@ export declare interface ResolveScopeDeps {
 }
 
 /**
- * Restore one CAS asset to the sink under `key`, re-hashing the bytes as they
- * stream through and refusing (IntegrityError) if they don't match the expected
- * sha — so a corrupted bundle can never silently re-upload bad bytes.
+ * Admit a private copy of one CAS asset before its bytes reach the sink.
+ * Whole-index consumers use the shared CAS phase to admit all entries first.
  */
-export declare const restoreAssetFromCas: (sink: AssetSink, assetsDir: string, entry: {
-    readonly key: string;
-    readonly sha256: string;
-    readonly size: number;
-    readonly contentType: string;
-}) => Effect.Effect<void, BundleError | IntegrityError | StorageError>;
+export declare const restoreAssetFromCas: (sink: AssetSink, assetsDir: string, entry: AssetEntry) => Effect.Effect<void, BundleError | IntegrityError | StorageError>;
 
 /**
- * Push every asset named in the index back to the sink, re-hashing on the way.
+ * Push every asset from the private, fully verified index back to its admitted sink.
  *
  * A hash mismatch is an {@link IntegrityError} and stops the phase — a corrupted
  * artefact must never silently re-upload bad bytes over good ones, which is the
@@ -1520,6 +1658,7 @@ export declare const restoreAssetsFromCas: (opts: {
     readonly assetsDir: string;
     readonly indexPath: string;
     readonly progress?: AssetProgress;
+    readonly prepared?: PreparedAssets;
 }) => Effect.Effect<RestoredAssets, BundleError | IntegrityError | StorageError>;
 
 /** Build the restore step for a dialect reading `inFile`. */
@@ -1563,6 +1702,8 @@ declare interface RollbackCapture {
  * list sorted by nothing sees hundreds of lines of which one matters.
  */
 export declare class RowsRefusedError extends RowsRefusedError_base {
+    /** Preserve the bounded refusal report when an error is stored as text. */
+    get message(): string;
 }
 
 declare const RowsRefusedError_base: Schema.Class<RowsRefusedError, Schema.TaggedStruct<"RowsRefusedError", {
@@ -1664,6 +1805,19 @@ declare const ScopeError_base: Schema.Class<ScopeError, Schema.TaggedStruct<"Sco
     readonly reason: Schema.String;
 }>, YieldableError>;
 
+export declare interface SealedImportReport {
+    readonly nulled: ReadonlyArray<{
+        readonly table: string;
+        readonly column: string;
+        readonly count: number;
+    }>;
+    readonly unreadable: ReadonlyArray<{
+        readonly table: string;
+        readonly column: string;
+        readonly count: number;
+    }>;
+}
+
 /**
  * A store that can run raw SQL for us.
  *
@@ -1752,27 +1906,29 @@ export declare interface StagingSnapshot {
     }>;
 }
 
+export declare const storageAssetInfo: (row: Row) => AssetInfo;
+
 /** Build an {@link AssetSink} over a storage provider. */
-export declare const storageAssetSink: (provider: StorageProvider) => AssetSink;
+export declare const storageAssetSink: (router: StorageDestinationRouter) => AssetSink;
 
 /** Build an {@link AssetSource} over a storage provider — `list` is supplied by
  *  the caller (typically streaming the `_voltro_storage_refs` table), `open`
  *  uses the provider's streaming read (buffered fallback for non-streaming
  *  backends). */
-export declare const storageAssetSource: (provider: StorageProvider, list: () => Stream.Stream<AssetInfo, unknown>) => AssetSource;
+export declare const storageAssetSource: (router: StorageDestinationRouter, list: (bundle?: AssetBundleContext) => Stream.Stream<AssetInfo, unknown>) => AssetSource;
 
 /** Stream blobs TO object storage. `has` powers resume (skip existing keys);
  *  `write` streams the archive's blob content straight into the provider. */
-export declare const storageBlobSink: (provider: StorageProvider) => BlobSink;
+export declare const storageBlobSink: (router: StorageDestinationRouter) => BlobSink;
 
 /** Stream blobs FROM object storage. `list` enumerates metadata (typically the
  *  storage refs table); `open` streams each blob's bytes via the provider. */
-export declare const storageBlobSource: (provider: StorageProvider, list: () => Stream.Stream<AssetInfo, unknown>) => BlobSource;
+export declare const storageBlobSource: (router: StorageDestinationRouter, list: (bundleDir?: string) => Stream.Stream<AssetInfo, unknown>) => BlobSource;
 
 /** A {@link BlobSource} over a provider whose blob list comes from the app's
  *  `_voltro_storage_refs` table — the standard export source for a running
  *  instance (CLI file export and the prod export endpoint both use it). */
-export declare const storageRefsBlobSource: (provider: StorageProvider, store: DataStore) => BlobSource;
+export declare const storageRefsBlobSource: (router: StorageDestinationRouter, store: DataStore) => BlobSource;
 
 export declare interface SubsetSpec {
     /** Seed rows per table. `table → predicate` (a `null`/absent predicate seeds
@@ -1852,8 +2008,8 @@ export declare const topoSortTables: (tables: ReadonlyArray<string>, snapshot: S
 
 /**
  * Unpack a framed byte stream. Plain files are written under `dir` (traversal
- * guarded); blob entries are streamed to `blobs` (dest storage) — never to local
- * disk — with the trailing sha verified as they pass through. A blob whose key
+ * guarded); entries are staged and verified through authenticated source EOF
+ * before `blobs` (dest storage) participates. A blob whose key
  * already exists at the sink is skipped (resume).
  */
 export declare const unpackBundle: (source: AsyncIterable<Uint8Array>, opts: {
@@ -1864,7 +2020,7 @@ export declare const unpackBundle: (source: AsyncIterable<Uint8Array>, opts: {
 /** Convenience: unpack a bundle fully to a local directory (plain files as-is,
  *  blob entries materialised under `assets/`). For data-only bundles and
  *  `voltro data unpack`; the streaming import path passes a storage {@link BlobSink}
- *  to {@link unpackBundle} instead so blobs never touch local disk. */
+ *  to {@link unpackBundle} for a destination after archive admission. */
 export declare const unpackBundleToDir: (source: AsyncIterable<Uint8Array>, dir: string) => Promise<void>;
 
 /**