@voltro/data-transfer 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/LICENSE +57 -0
- package/README.md +26 -0
- package/SECURITY.md +56 -0
- package/THIRD-PARTY-NOTICES.md +11075 -0
- package/dist/index.d.ts +1080 -0
- package/dist/index.js +1501 -0
- package/package.json +44 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,1080 @@
|
|
|
1
|
+
import { ByteStream } from '@voltro/plugin-storage';
|
|
2
|
+
import { ColumnType } from '@voltro/database';
|
|
3
|
+
import { DataStore } from '@voltro/database';
|
|
4
|
+
import { Duplex } from 'node:stream';
|
|
5
|
+
import { Effect } from 'effect';
|
|
6
|
+
import { ParseOptions } from 'effect/SchemaAST';
|
|
7
|
+
import { Predicate } from '@voltro/database';
|
|
8
|
+
import { Row } from '@voltro/database';
|
|
9
|
+
import { Schema } from 'effect';
|
|
10
|
+
import { SchemaSnapshot } from '@voltro/database/sql';
|
|
11
|
+
import { StorageError } from '@voltro/plugin-storage';
|
|
12
|
+
import { StorageProvider } from '@voltro/plugin-storage';
|
|
13
|
+
import { StoredStream } from '@voltro/plugin-storage';
|
|
14
|
+
import { Stream } from 'effect';
|
|
15
|
+
import { TableStreamError } from '@voltro/database';
|
|
16
|
+
import { VoidIfEmpty } from 'effect/Types';
|
|
17
|
+
import { YieldableError } from 'effect/Cause';
|
|
18
|
+
|
|
19
|
+
/** Apply one action to one value. A null value is always passed through. */
|
|
20
|
+
export declare const applyAction: (action: MaskAction, input: MaskInput) => unknown;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Refuse a direct/native import when a live instance is detected, unless the
|
|
24
|
+
* operator passed `--allow-live`. `liveInstance` is a human label (url/name) of
|
|
25
|
+
* the detected instance, or `null` when none was found (or detection was
|
|
26
|
+
* inconclusive — the CLI treats "can't tell" as "not detected" so the common
|
|
27
|
+
* import-into-a-stopped-target case isn't blocked).
|
|
28
|
+
*/
|
|
29
|
+
export declare const assessLiveImport: (opts: {
|
|
30
|
+
readonly liveInstance: string | null;
|
|
31
|
+
readonly allowLive: boolean;
|
|
32
|
+
}) => LiveGuardResult;
|
|
33
|
+
|
|
34
|
+
export declare const ASSET_INDEX_FILE = "assets/index.ndjson";
|
|
35
|
+
|
|
36
|
+
export declare const assetDone: (ledger: Ledger, sha256: string) => boolean;
|
|
37
|
+
|
|
38
|
+
/** One content-addressed asset. `path` is `assets/<sha256>`; `key` is the
|
|
39
|
+
* original storage key so the importer restores it under the same key. */
|
|
40
|
+
export declare const AssetEntry: Schema.Struct<{
|
|
41
|
+
key: typeof Schema.String;
|
|
42
|
+
sha256: typeof Schema.String;
|
|
43
|
+
size: typeof Schema.Number;
|
|
44
|
+
contentType: typeof Schema.String;
|
|
45
|
+
}>;
|
|
46
|
+
|
|
47
|
+
export declare type AssetEntry = Schema.Schema.Type<typeof AssetEntry>;
|
|
48
|
+
|
|
49
|
+
/** Metadata for one asset to export — enough to enumerate without reading the
|
|
50
|
+
* bytes. Sourced from the storage refs table by the caller. */
|
|
51
|
+
export declare interface AssetInfo {
|
|
52
|
+
readonly key: string;
|
|
53
|
+
readonly contentType: string;
|
|
54
|
+
readonly size: number;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export declare const ASSETS_DIR = "assets";
|
|
58
|
+
|
|
59
|
+
/** Where import writes assets TO. */
|
|
60
|
+
export declare interface AssetSink {
|
|
61
|
+
readonly write: (key: string, stream: ByteStream, meta: {
|
|
62
|
+
readonly contentType: string;
|
|
63
|
+
readonly totalSize?: number;
|
|
64
|
+
}) => Effect.Effect<unknown, StorageError>;
|
|
65
|
+
/** Whether the key already exists at the destination (skip re-upload). */
|
|
66
|
+
readonly has: (key: string) => Effect.Effect<boolean, StorageError>;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Where export reads assets FROM. */
|
|
70
|
+
export declare interface AssetSource {
|
|
71
|
+
/** Lazily enumerate every asset (metadata only; bytes stream on demand). */
|
|
72
|
+
readonly list: () => Stream.Stream<AssetInfo, unknown>;
|
|
73
|
+
/** Open one asset as a byte stream. */
|
|
74
|
+
readonly open: (key: string) => Effect.Effect<StoredStream, StorageError>;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export declare const AssetSummary: Schema.Struct<{
|
|
78
|
+
count: typeof Schema.Number;
|
|
79
|
+
totalBytes: typeof Schema.Number;
|
|
80
|
+
/** Bundle-relative path of the asset index (`assets/index.ndjson`), when any
|
|
81
|
+
* assets were exported. Each line is an {@link AssetEntry}. */
|
|
82
|
+
index: Schema.optional<typeof Schema.String>;
|
|
83
|
+
}>;
|
|
84
|
+
|
|
85
|
+
export declare type AssetSummary = Schema.Schema.Type<typeof AssetSummary>;
|
|
86
|
+
|
|
87
|
+
/** The artifact file name a backup produces for a dialect. */
|
|
88
|
+
export declare const backupArtifactName: (dialect: string) => string;
|
|
89
|
+
|
|
90
|
+
/** Build the backup step for a dialect writing to `outFile`. */
|
|
91
|
+
export declare const backupCommand: (dialect: string, conn: NativeConn, outFile: string) => NativeStep;
|
|
92
|
+
|
|
93
|
+
/** Where blobs stream TO on import. `has` lets a resumed run skip existing keys. */
|
|
94
|
+
export declare interface BlobSink {
|
|
95
|
+
has: (key: string) => Promise<boolean>;
|
|
96
|
+
write: (key: string, content: AsyncIterable<Uint8Array>, meta: {
|
|
97
|
+
readonly contentType: string;
|
|
98
|
+
readonly size: number;
|
|
99
|
+
}) => Promise<void>;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Where blobs stream FROM on export (metadata + a byte stream per key). */
|
|
103
|
+
export declare interface BlobSource {
|
|
104
|
+
list: () => AsyncIterable<{
|
|
105
|
+
readonly key: string;
|
|
106
|
+
readonly contentType: string;
|
|
107
|
+
readonly size: number;
|
|
108
|
+
}>;
|
|
109
|
+
open: (key: string) => AsyncIterable<Uint8Array>;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** A bundle is malformed: bad manifest, unknown format version, missing file. */
|
|
113
|
+
export declare class BundleError extends BundleError_base {
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
declare const BundleError_base: Schema.TaggedErrorClass<BundleError, "BundleError", {
|
|
117
|
+
readonly _tag: Schema.tag<"BundleError">;
|
|
118
|
+
} & {
|
|
119
|
+
reason: typeof Schema.String;
|
|
120
|
+
path: Schema.optional<typeof Schema.String>;
|
|
121
|
+
}>;
|
|
122
|
+
|
|
123
|
+
/** The portable meaning of a DSL column type. */
|
|
124
|
+
export declare type CanonicalCategory = 'string' | 'integer' | 'float' | 'boolean' | 'datetime' | 'date' | 'json' | 'binary' | 'reference' | 'vector' | 'enum' | 'array' | 'interval' | 'raw';
|
|
125
|
+
|
|
126
|
+
/** Map a DSL `ColumnType` (as a string) to its canonical category. Unknown
|
|
127
|
+
* types (a bundle from a newer framework) fall back to 'string' passthrough. */
|
|
128
|
+
export declare const canonicalType: (type: string) => CanonicalCategory;
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Lint a bundle's column types against a target dialect. Same-dialect import is
|
|
132
|
+
* always clean (returns []). Otherwise returns every column whose type won't
|
|
133
|
+
* port — `block` severity must be resolved (fix the schema or --force);
|
|
134
|
+
* `warn` imports with a representational caveat.
|
|
135
|
+
*/
|
|
136
|
+
export declare const checkPortability: (tables: ReadonlyArray<{
|
|
137
|
+
readonly name: string;
|
|
138
|
+
readonly columnTypes: Readonly<Record<string, string>>;
|
|
139
|
+
}>, sourceDialect: string, targetDialect: string) => ReadonlyArray<PortabilityIssue>;
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Compare the bundle's recorded fingerprint against the target snapshot's
|
|
143
|
+
* fingerprint (computed the SAME way the export did). Returns the verdict + a
|
|
144
|
+
* diff when drifted. Pure — no I/O; the caller introspects the target.
|
|
145
|
+
*/
|
|
146
|
+
export declare const checkSchemaDrift: (bundleFingerprint: string, target: SchemaSnapshot, bundleSchema?: SchemaSnapshot) => DriftCheck;
|
|
147
|
+
|
|
148
|
+
/** table → column → classification (built from the DECLARED schema snapshot). */
|
|
149
|
+
export declare type Classification = Readonly<Record<string, Readonly<Record<string, ColumnClassification>>>>;
|
|
150
|
+
|
|
151
|
+
/** Derive the classification map from a declared schema snapshot. */
|
|
152
|
+
export declare const classificationFromSnapshot: (snapshot: {
|
|
153
|
+
readonly tables: ReadonlyArray<{
|
|
154
|
+
readonly name: string;
|
|
155
|
+
readonly columns: ReadonlyArray<{
|
|
156
|
+
readonly name: string;
|
|
157
|
+
readonly type: string;
|
|
158
|
+
readonly sensitive?: {
|
|
159
|
+
readonly class: string;
|
|
160
|
+
};
|
|
161
|
+
readonly safe?: boolean;
|
|
162
|
+
}>;
|
|
163
|
+
}>;
|
|
164
|
+
}) => Classification;
|
|
165
|
+
|
|
166
|
+
/** A row line failed to parse / encode. Carries the table + line for triage. */
|
|
167
|
+
export declare class CodecError extends CodecError_base {
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
declare const CodecError_base: Schema.TaggedErrorClass<CodecError, "CodecError", {
|
|
171
|
+
readonly _tag: Schema.tag<"CodecError">;
|
|
172
|
+
} & {
|
|
173
|
+
table: typeof Schema.String;
|
|
174
|
+
reason: typeof Schema.String;
|
|
175
|
+
}>;
|
|
176
|
+
|
|
177
|
+
export declare interface ColumnClassification {
|
|
178
|
+
readonly sensitive?: {
|
|
179
|
+
readonly class: string;
|
|
180
|
+
};
|
|
181
|
+
readonly safe?: boolean;
|
|
182
|
+
readonly columnType?: string;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** column name → logical type. Values are the {@link ColumnType} union; typed
|
|
186
|
+
* loosely as string so a manifest from a newer source (unknown type) still
|
|
187
|
+
* decodes (unknown types fall through as scalar passthrough). */
|
|
188
|
+
export declare type ColumnTypes = Readonly<Record<string, ColumnType | string>>;
|
|
189
|
+
|
|
190
|
+
export declare type Compression = 'zstd' | 'gzip' | 'none';
|
|
191
|
+
|
|
192
|
+
/** A compression / decompression step failed. INTERNAL — the pipelines fold a
|
|
193
|
+
* real (de)compression failure into `BundleError` (its `path` locates the
|
|
194
|
+
* file); this tag is the defensive standalone form and never crosses the wire. */
|
|
195
|
+
export declare class CompressionError extends CompressionError_base<{
|
|
196
|
+
readonly reason: string;
|
|
197
|
+
}> {
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
declare const CompressionError_base: new <A extends Record<string, any> = {}>(args: VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => YieldableError & {
|
|
201
|
+
readonly _tag: "CompressionError";
|
|
202
|
+
} & Readonly<A>;
|
|
203
|
+
|
|
204
|
+
/** File-name suffix for a codec — appended after `.ndjson`. */
|
|
205
|
+
export declare const compressionExt: (c: Compression) => string;
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Compute the id set to export per table: the seed rows, every row reached by
|
|
209
|
+
* following foreign keys to their PARENTS (transitively, to a fixpoint), and —
|
|
210
|
+
* when {@link SubsetSpec.children} is set — every row that REFERENCES the
|
|
211
|
+
* exported set (each restricted to the rows whose FK points INTO the set, then
|
|
212
|
+
* re-folded through the parent closure so nothing dangles).
|
|
213
|
+
*
|
|
214
|
+
* Pinned tables (see {@link SubsetSpec.pinned}) never appear in the returned map
|
|
215
|
+
* — their membership stays the seed predicate — but their member ids ARE tracked
|
|
216
|
+
* internally so the child pass can scope rows that reference the pinned rows.
|
|
217
|
+
*/
|
|
218
|
+
export declare const computeSubsetIds: (store: DataStore, snapshot: SchemaSnapshot, spec: SubsetSpec) => Effect.Effect<Map<string, Set<unknown>>, TableStreamError>;
|
|
219
|
+
|
|
220
|
+
/** A cross-dialect import was refused because the source bundle uses column
|
|
221
|
+
* types that don't port cleanly to the target dialect. Lists every blocking
|
|
222
|
+
* incompatibility so the operator can decide (or pass --force). */
|
|
223
|
+
export declare class CrossDialectError extends CrossDialectError_base {
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
declare const CrossDialectError_base: Schema.TaggedErrorClass<CrossDialectError, "CrossDialectError", {
|
|
227
|
+
readonly _tag: Schema.tag<"CrossDialectError">;
|
|
228
|
+
} & {
|
|
229
|
+
sourceDialect: typeof Schema.String;
|
|
230
|
+
targetDialect: typeof Schema.String;
|
|
231
|
+
blocking: Schema.Array$<typeof Schema.String>;
|
|
232
|
+
}>;
|
|
233
|
+
|
|
234
|
+
/** Rows that could not be written even after the importer's deferred-FK
|
|
235
|
+
* resolution (plain retry once every table landed, then FK-shedding + patch).
|
|
236
|
+
* Each entry names the table, the primary-key value, and the store's own
|
|
237
|
+
* reason — a genuinely dangling reference (the parent row is in neither the
|
|
238
|
+
* bundle nor the target) or a NOT NULL FK cycle the dialect can't defer. */
|
|
239
|
+
export declare class DanglingReferenceError extends DanglingReferenceError_base {
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
declare const DanglingReferenceError_base: Schema.TaggedErrorClass<DanglingReferenceError, "DanglingReferenceError", {
|
|
243
|
+
readonly _tag: Schema.tag<"DanglingReferenceError">;
|
|
244
|
+
} & {
|
|
245
|
+
/** The stuck rows (capped — see {@link totalCount}). */
|
|
246
|
+
rows: Schema.Array$<Schema.Struct<{
|
|
247
|
+
table: typeof Schema.String;
|
|
248
|
+
id: typeof Schema.String;
|
|
249
|
+
reason: typeof Schema.String;
|
|
250
|
+
}>>;
|
|
251
|
+
/** Total number of stuck rows (may exceed `rows.length`). */
|
|
252
|
+
totalCount: typeof Schema.Number;
|
|
253
|
+
}>;
|
|
254
|
+
|
|
255
|
+
export declare const DATA_DIR = "data";
|
|
256
|
+
|
|
257
|
+
export declare interface DataTransferProfile {
|
|
258
|
+
readonly scope?: ExportScope;
|
|
259
|
+
readonly subset?: SubsetSpec;
|
|
260
|
+
readonly masking?: MaskingPolicy;
|
|
261
|
+
readonly consistency?: 'live' | 'snapshot';
|
|
262
|
+
readonly compression?: Compression;
|
|
263
|
+
readonly assets?: boolean;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
export declare const decodeManifest: (u: unknown, overrideOptions?: ParseOptions) => {
|
|
267
|
+
readonly source: {
|
|
268
|
+
readonly dialect: string;
|
|
269
|
+
readonly schemaFingerprint: string;
|
|
270
|
+
};
|
|
271
|
+
readonly createdAt: string;
|
|
272
|
+
readonly scope: {
|
|
273
|
+
readonly kind: "tables" | "all" | "tenant";
|
|
274
|
+
readonly tenantId?: string | undefined;
|
|
275
|
+
readonly tables?: readonly string[] | undefined;
|
|
276
|
+
};
|
|
277
|
+
readonly tables: readonly {
|
|
278
|
+
readonly name: string;
|
|
279
|
+
readonly primaryKey: string;
|
|
280
|
+
readonly file: string;
|
|
281
|
+
readonly rowCount: number;
|
|
282
|
+
readonly checksum: string;
|
|
283
|
+
readonly columnTypes: {
|
|
284
|
+
readonly [x: string]: string;
|
|
285
|
+
};
|
|
286
|
+
}[];
|
|
287
|
+
readonly formatVersion: number;
|
|
288
|
+
readonly consistency: "live" | "snapshot";
|
|
289
|
+
readonly compression: "zstd" | "gzip" | "none";
|
|
290
|
+
readonly assets: {
|
|
291
|
+
readonly index?: string | undefined;
|
|
292
|
+
readonly count: number;
|
|
293
|
+
readonly totalBytes: number;
|
|
294
|
+
};
|
|
295
|
+
readonly masking?: {
|
|
296
|
+
readonly policyId?: string | undefined;
|
|
297
|
+
readonly transformed: readonly string[];
|
|
298
|
+
} | undefined;
|
|
299
|
+
};
|
|
300
|
+
|
|
301
|
+
/** Decode a JSON object back into a row with faithful JS types. */
|
|
302
|
+
export declare const decodeRow: (obj: Record<string, unknown>, columnTypes: ColumnTypes) => Row;
|
|
303
|
+
|
|
304
|
+
/** Decode one value for a column of logical type `type`. */
|
|
305
|
+
export declare const decodeValue: (value: unknown, type: string | undefined) => unknown;
|
|
306
|
+
|
|
307
|
+
/** Decrypt a stream produced by {@link encryptStream}. Fails loudly on a wrong
|
|
308
|
+
* passphrase, a tampered byte, or a truncated stream (no authenticated final
|
|
309
|
+
* frame). */
|
|
310
|
+
export declare function decryptStream(source: AsyncIterable<Uint8Array>, passphrase: string): AsyncGenerator<Uint8Array>;
|
|
311
|
+
|
|
312
|
+
/** Built-in class → action defaults. A minimal policy just needs a `seed`. */
|
|
313
|
+
export declare const DEFAULT_CLASS_ACTIONS: Readonly<Record<string, MaskAction>>;
|
|
314
|
+
|
|
315
|
+
/** Identity helper for authoring a profile with full type-checking + inference. */
|
|
316
|
+
export declare const defineDataProfile: (profile: DataTransferProfile) => DataTransferProfile;
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Diff two snapshots into a small list of observable differences. Reports, in a
|
|
320
|
+
* stable order: tables only in the target, tables only in the bundle, and for
|
|
321
|
+
* shared tables the columns added / dropped / type-changed. Capped so a wildly
|
|
322
|
+
* divergent pair still yields an actionable (not enormous) message.
|
|
323
|
+
*/
|
|
324
|
+
export declare const diffSnapshots: (bundle: SchemaSnapshot, target: SchemaSnapshot) => ReadonlyArray<string>;
|
|
325
|
+
|
|
326
|
+
/** A {@link BlobSink} that writes blobs to the LOCAL dir as `assets/<sha>` files
|
|
327
|
+
* + an `assets/index.ndjson` (key→sha→size→contentType) — reproduces a
|
|
328
|
+
* materialised, re-importable bundle directory (for `voltro data unpack`). */
|
|
329
|
+
export declare const dirBlobSink: (dir: string) => BlobSink;
|
|
330
|
+
|
|
331
|
+
export declare interface DriftCheck {
|
|
332
|
+
readonly drifted: boolean;
|
|
333
|
+
readonly bundleFingerprint: string;
|
|
334
|
+
readonly targetFingerprint: string;
|
|
335
|
+
/** Empty when not drifted; a capped, human-readable summary otherwise. */
|
|
336
|
+
readonly diff: ReadonlyArray<string>;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
export declare const emptyLedger: () => Ledger;
|
|
340
|
+
|
|
341
|
+
export declare const encodeManifest: (a: {
|
|
342
|
+
readonly source: {
|
|
343
|
+
readonly dialect: string;
|
|
344
|
+
readonly schemaFingerprint: string;
|
|
345
|
+
};
|
|
346
|
+
readonly createdAt: string;
|
|
347
|
+
readonly scope: {
|
|
348
|
+
readonly kind: "tables" | "all" | "tenant";
|
|
349
|
+
readonly tenantId?: string | undefined;
|
|
350
|
+
readonly tables?: readonly string[] | undefined;
|
|
351
|
+
};
|
|
352
|
+
readonly tables: readonly {
|
|
353
|
+
readonly name: string;
|
|
354
|
+
readonly primaryKey: string;
|
|
355
|
+
readonly file: string;
|
|
356
|
+
readonly rowCount: number;
|
|
357
|
+
readonly checksum: string;
|
|
358
|
+
readonly columnTypes: {
|
|
359
|
+
readonly [x: string]: string;
|
|
360
|
+
};
|
|
361
|
+
}[];
|
|
362
|
+
readonly formatVersion: number;
|
|
363
|
+
readonly consistency: "live" | "snapshot";
|
|
364
|
+
readonly compression: "zstd" | "gzip" | "none";
|
|
365
|
+
readonly assets: {
|
|
366
|
+
readonly index?: string | undefined;
|
|
367
|
+
readonly count: number;
|
|
368
|
+
readonly totalBytes: number;
|
|
369
|
+
};
|
|
370
|
+
readonly masking?: {
|
|
371
|
+
readonly policyId?: string | undefined;
|
|
372
|
+
readonly transformed: readonly string[];
|
|
373
|
+
} | undefined;
|
|
374
|
+
}, overrideOptions?: ParseOptions) => string;
|
|
375
|
+
|
|
376
|
+
/** Encode a whole row to a JSON-safe object. */
|
|
377
|
+
export declare const encodeRow: (row: Row, columnTypes: ColumnTypes) => Record<string, unknown>;
|
|
378
|
+
|
|
379
|
+
/** Encode one value for a column of logical type `type`. */
|
|
380
|
+
export declare const encodeValue: (value: unknown, type: string | undefined) => unknown;
|
|
381
|
+
|
|
382
|
+
/** Encrypt a byte stream with a passphrase. Yields the header then framed,
|
|
383
|
+
* authenticated chunks; the final frame is flagged so truncation is detected. */
|
|
384
|
+
export declare function encryptStream(source: AsyncIterable<Uint8Array>, passphrase: string): AsyncGenerator<Uint8Array>;
|
|
385
|
+
|
|
386
|
+
/** Every error the export pipeline can surface. */
|
|
387
|
+
export declare type ExportError = BundleError | TableStreamError | MaskingError | ScopeError;
|
|
388
|
+
|
|
389
|
+
export declare interface ExportOptions {
|
|
390
|
+
/** Source store — pass the RAW dialect store for a faithful physical read
|
|
391
|
+
* (no tenant/soft-delete scoping); scope filtering is applied explicitly. */
|
|
392
|
+
readonly store: DataStore;
|
|
393
|
+
/** Live schema of the source (from `introspectSchema`). Drives table set,
|
|
394
|
+
* column types, FK order, and the fingerprint stamped in the manifest. */
|
|
395
|
+
readonly snapshot: SchemaSnapshot;
|
|
396
|
+
/** Source dialect id (`postgres` | `mysql` | …) — recorded for import. */
|
|
397
|
+
readonly dialect: string;
|
|
398
|
+
/** Bundle output directory (created if absent). */
|
|
399
|
+
readonly outDir: string;
|
|
400
|
+
/** What to export. Default `{ kind: 'all' }`. */
|
|
401
|
+
readonly scope?: ExportScope;
|
|
402
|
+
/** Names of the tenant-scoped tables, from the schema registry's `tenant()`
|
|
403
|
+
* mixin metadata. Authoritative for the tenant scope when provided — a
|
|
404
|
+
* table can carry a `tenantId` column WITHOUT being tenant-scoped (e.g. a
|
|
405
|
+
* global `users` table whose `tenantId` is the active-org pointer). Omitted
|
|
406
|
+
* → any table with a `tenantId` column counts (documented heuristic). */
|
|
407
|
+
readonly tenantTables?: ReadonlyArray<string>;
|
|
408
|
+
/** Referentially-correct subset: seed predicates per table + FK-parent
|
|
409
|
+
* closure. When set, it REPLACES `scope` (only the seeded tables + their
|
|
410
|
+
* parent closure are exported, filtered to the closure's id sets). */
|
|
411
|
+
readonly subset?: SubsetSpec;
|
|
412
|
+
/** Codec preference; resolved to zstd/gzip by availability. */
|
|
413
|
+
readonly compression?: Compression;
|
|
414
|
+
/** Consistency mode stamped in the manifest — `'live'` reads rows as the
|
|
415
|
+
* keyset stream progresses; `'snapshot'` marks a snapshot-isolated source. */
|
|
416
|
+
readonly consistency?: 'live' | 'snapshot';
|
|
417
|
+
/** Rows per keyset page. Default 1000. */
|
|
418
|
+
readonly chunkSize?: number;
|
|
419
|
+
/** Optional asset source (blobs). Omit to export data only. */
|
|
420
|
+
readonly assets?: AssetSource;
|
|
421
|
+
/** ISO stamp for the manifest. Defaults to now; injectable for tests. */
|
|
422
|
+
readonly createdAt?: string;
|
|
423
|
+
/** Per-page retry attempts on transient read failures. Default 5. */
|
|
424
|
+
readonly retryTimes?: number;
|
|
425
|
+
/** Masking policy — when set, PII columns are pseudonymised/anonymised at the
|
|
426
|
+
* SOURCE before any row is written. Fail-closed: an unclassified column
|
|
427
|
+
* refuses the export (unless the policy's `onUnclassified` opts out). */
|
|
428
|
+
readonly masking?: MaskingPolicy;
|
|
429
|
+
/** Column classification (from the DECLARED schema — `.sensitive()`/`.safe()`).
|
|
430
|
+
* Defaults to deriving it from `snapshot`; pass the declared snapshot's
|
|
431
|
+
* classification explicitly when `snapshot` is the introspected one. */
|
|
432
|
+
readonly classification?: Classification;
|
|
433
|
+
/** Per-table progress sink for long jobs (the CLI wires it to a renderer).
|
|
434
|
+
* Fires a `start` then a `done` event per table — off the per-row hot path.
|
|
435
|
+
* Independent of the always-on `Effect.withSpan` + metrics per table. */
|
|
436
|
+
readonly onProgress?: OnProgress;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/** Export scope — what was selected. `all` = every table; `tenant` = the rows
|
|
440
|
+
* of one tenant (+ its FK closure); `tables` = an explicit set. */
|
|
441
|
+
export declare const ExportScope: Schema.Struct<{
|
|
442
|
+
kind: Schema.Literal<["all", "tenant", "tables"]>;
|
|
443
|
+
tenantId: Schema.optional<typeof Schema.String>;
|
|
444
|
+
tables: Schema.optional<Schema.Array$<typeof Schema.String>>;
|
|
445
|
+
}>;
|
|
446
|
+
|
|
447
|
+
export declare type ExportScope = Schema.Schema.Type<typeof ExportScope>;
|
|
448
|
+
|
|
449
|
+
/** Bump when the on-disk shape changes incompatibly. An importer refuses a
|
|
450
|
+
* bundle whose `formatVersion` it doesn't understand rather than guessing. */
|
|
451
|
+
export declare const FORMAT_VERSION = 1;
|
|
452
|
+
|
|
453
|
+
/** Format one issue for a CLI / error message. */
|
|
454
|
+
export declare const formatIssue: (i: PortabilityIssue) => string;
|
|
455
|
+
|
|
456
|
+
/** True when this Node build exposes the native zstd streams (Node ≥ 22.15). */
|
|
457
|
+
export declare const hasZstd: () => boolean;
|
|
458
|
+
|
|
459
|
+
export declare type ImportError = BundleError | CodecError | IntegrityError | CrossDialectError | ImportModeError | DanglingReferenceError | SchemaDriftError | StorageError;
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* How rows are written:
|
|
463
|
+
* `upsert` — INSERT-or-UPDATE by primary key (default; idempotent re-import).
|
|
464
|
+
* `append` — INSERT only; on a PK conflict `onConflict` decides skip vs fail.
|
|
465
|
+
* `replace` — TRUNCATE the target tables (reverse-FK) then INSERT: the target
|
|
466
|
+
* ends up EXACTLY the bundle. Refused for a partial bundle.
|
|
467
|
+
*/
|
|
468
|
+
export declare type ImportMode = 'upsert' | 'append' | 'replace';
|
|
469
|
+
|
|
470
|
+
/** An import was refused because the requested write-mode is unsafe for this
|
|
471
|
+
* bundle — e.g. `replace` (which truncates) against a PARTIAL bundle (a subset
|
|
472
|
+
* or tenant/table scope), which would delete rows the bundle never carried. */
|
|
473
|
+
export declare class ImportModeError extends ImportModeError_base {
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
declare const ImportModeError_base: Schema.TaggedErrorClass<ImportModeError, "ImportModeError", {
|
|
477
|
+
readonly _tag: Schema.tag<"ImportModeError">;
|
|
478
|
+
} & {
|
|
479
|
+
mode: typeof Schema.String;
|
|
480
|
+
reason: typeof Schema.String;
|
|
481
|
+
}>;
|
|
482
|
+
|
|
483
|
+
export declare interface ImportOptions {
|
|
484
|
+
/** Target store to write into. */
|
|
485
|
+
readonly store: DataStore;
|
|
486
|
+
/** Bundle directory produced by `runExport`. */
|
|
487
|
+
readonly bundleDir: string;
|
|
488
|
+
/** Write semantics. Default `upsert`. */
|
|
489
|
+
readonly mode?: ImportMode;
|
|
490
|
+
/** For `append`: `skip` existing PKs (default) or `fail` on the first conflict. */
|
|
491
|
+
readonly onConflict?: OnConflict;
|
|
492
|
+
/** Optional asset sink (blobs). Omit to import data only. */
|
|
493
|
+
readonly assets?: AssetSink;
|
|
494
|
+
/** Import-side resume journal. Defaults to `<bundleDir>/.import.ledger.json`. */
|
|
495
|
+
readonly ledgerPath?: string;
|
|
496
|
+
/** Verify each table's checksum + row count against the manifest. Default
|
|
497
|
+
* true — turning it off trades safety for a little speed. */
|
|
498
|
+
readonly verify?: boolean;
|
|
499
|
+
/** The target dialect. When set and it differs from the bundle's source
|
|
500
|
+
* dialect, a cross-dialect portability lint runs first; blocking
|
|
501
|
+
* incompatibilities refuse the import (unless {@link allowIncompatible}). */
|
|
502
|
+
readonly targetDialect?: string;
|
|
503
|
+
/** Import despite blocking cross-dialect incompatibilities (the operator has
|
|
504
|
+
* accepted the risk). Warnings never block regardless. */
|
|
505
|
+
readonly allowIncompatible?: boolean;
|
|
506
|
+
/**
|
|
507
|
+
* Run the whole table phase (truncate + load) in ONE transaction, so a LIVE
|
|
508
|
+
* target sees the import all-or-nothing — MVCC readers see the old data until
|
|
509
|
+
* commit, the new data after, never a partial state. The live-safe way to
|
|
510
|
+
* `replace`. Trade-offs: one long-held write transaction (writes to those
|
|
511
|
+
* tables block for the load; readers are unaffected on MVCC dialects), and
|
|
512
|
+
* resume is not per-table (a crash rolls the whole thing back → re-run).
|
|
513
|
+
*/
|
|
514
|
+
readonly atomic?: boolean;
|
|
515
|
+
/** The streaming import path (`unpackBundle` with a storage {@link AssetSink})
|
|
516
|
+
* restores blobs to dest storage AS the archive streams in — before the table
|
|
517
|
+
* phase — so the manifest's asset phase must be skipped here (the blobs are
|
|
518
|
+
* already landed + integrity-checked; re-reading them from a temp dir that
|
|
519
|
+
* doesn't exist would fail). Set by the streaming importer, not by users. */
|
|
520
|
+
readonly assetsAlreadyRestored?: boolean;
|
|
521
|
+
/** The TARGET app's live schema (from `introspectSchema`). When set, a
|
|
522
|
+
* fingerprint pre-flight compares it — the SAME way the export computed the
|
|
523
|
+
* bundle's fingerprint — against `manifest.source.schemaFingerprint`. On a
|
|
524
|
+
* mismatch the import is REFUSED with a {@link SchemaDriftError} naming the
|
|
525
|
+
* drift, before any row is written (unless {@link force}). Omit to skip the
|
|
526
|
+
* pre-flight (the bundle then imports against whatever the target is). */
|
|
527
|
+
readonly targetSnapshot?: SchemaSnapshot;
|
|
528
|
+
/** Import despite schema drift: downgrade the {@link SchemaDriftError} to a
|
|
529
|
+
* loud warning (via {@link onWarn}) and proceed. The operator has accepted
|
|
530
|
+
* that the target shape differs from the bundle's source. */
|
|
531
|
+
readonly force?: boolean;
|
|
532
|
+
/** Sink for non-fatal warnings (the library emits no logs of its own). The
|
|
533
|
+
* CLI wires this to its logger so `--force` drift downgrades are still loud. */
|
|
534
|
+
readonly onWarn?: (message: string) => void;
|
|
535
|
+
/** Per-table progress sink for long jobs (the CLI wires it to a renderer).
|
|
536
|
+
* Fires a `start` (with the manifest's row count as `total`) then a `done`
|
|
537
|
+
* event per table — off the per-row hot path. Independent of the always-on
|
|
538
|
+
* `Effect.withSpan` + metrics per table. */
|
|
539
|
+
readonly onProgress?: OnProgress;
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/** An integrity check failed — a table's content checksum or row count did not
|
|
543
|
+
* match the manifest, or an asset's bytes did not hash to its recorded id. */
|
|
544
|
+
export declare class IntegrityError extends IntegrityError_base {
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
declare const IntegrityError_base: Schema.TaggedErrorClass<IntegrityError, "IntegrityError", {
|
|
548
|
+
readonly _tag: Schema.tag<"IntegrityError">;
|
|
549
|
+
} & {
|
|
550
|
+
what: typeof Schema.String;
|
|
551
|
+
expected: typeof Schema.String;
|
|
552
|
+
actual: typeof Schema.String;
|
|
553
|
+
}>;
|
|
554
|
+
|
|
555
|
+
export declare interface LeakWarning {
|
|
556
|
+
readonly table: string;
|
|
557
|
+
readonly column: string;
|
|
558
|
+
readonly pattern: string;
|
|
559
|
+
readonly sample: string;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
export declare interface Ledger {
|
|
563
|
+
/** table name → completion record (present ⇒ fully written/imported). */
|
|
564
|
+
tables: Record<string, {
|
|
565
|
+
readonly rowCount: number;
|
|
566
|
+
readonly checksum: string;
|
|
567
|
+
}>;
|
|
568
|
+
/** sha256 → true for every asset that fully landed. */
|
|
569
|
+
assets: Record<string, true>;
|
|
570
|
+
/** `replace`-mode import only: set once the target tables have been
|
|
571
|
+
* truncated, so a resumed run doesn't re-truncate the rows it just loaded. */
|
|
572
|
+
truncated?: boolean;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
export declare const LEDGER_FILE = ".ledger.json";
|
|
576
|
+
|
|
577
|
+
export declare interface LiveGuardResult {
|
|
578
|
+
readonly refuse: boolean;
|
|
579
|
+
readonly message?: string;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/** A Duplex that COMPRESSES bytes written to it (identity for `none`). */
|
|
583
|
+
export declare const makeCompressor: (c: Compression) => Duplex;
|
|
584
|
+
|
|
585
|
+
/** A Duplex that DECOMPRESSES bytes written to it (identity for `none`). */
|
|
586
|
+
export declare const makeDecompressor: (c: Compression) => Duplex;
|
|
587
|
+
|
|
588
|
+
export declare const Manifest: Schema.Struct<{
|
|
589
|
+
formatVersion: typeof Schema.Number;
|
|
590
|
+
/** ISO-8601 stamp of when the export started. */
|
|
591
|
+
createdAt: typeof Schema.String;
|
|
592
|
+
source: Schema.Struct<{
|
|
593
|
+
dialect: typeof Schema.String;
|
|
594
|
+
schemaFingerprint: typeof Schema.String;
|
|
595
|
+
}>;
|
|
596
|
+
scope: Schema.Struct<{
|
|
597
|
+
kind: Schema.Literal<["all", "tenant", "tables"]>;
|
|
598
|
+
tenantId: Schema.optional<typeof Schema.String>;
|
|
599
|
+
tables: Schema.optional<Schema.Array$<typeof Schema.String>>;
|
|
600
|
+
}>;
|
|
601
|
+
consistency: Schema.Literal<["live", "snapshot"]>;
|
|
602
|
+
compression: Schema.Literal<["zstd", "gzip", "none"]>;
|
|
603
|
+
tables: Schema.Array$<Schema.Struct<{
|
|
604
|
+
name: typeof Schema.String;
|
|
605
|
+
/** Bundle-relative path, e.g. `data/users.ndjson.zst`. */
|
|
606
|
+
file: typeof Schema.String;
|
|
607
|
+
rowCount: typeof Schema.Number;
|
|
608
|
+
/** sha256 hex of the uncompressed NDJSON bytes. */
|
|
609
|
+
checksum: typeof Schema.String;
|
|
610
|
+
/** column name → logical column type (the `ColumnType` union, as a string). */
|
|
611
|
+
columnTypes: Schema.Record$<typeof Schema.String, typeof Schema.String>;
|
|
612
|
+
/** The unique column keyset-streamed on export (usually `id`). */
|
|
613
|
+
primaryKey: typeof Schema.String;
|
|
614
|
+
}>>;
|
|
615
|
+
assets: Schema.Struct<{
|
|
616
|
+
count: typeof Schema.Number;
|
|
617
|
+
totalBytes: typeof Schema.Number;
|
|
618
|
+
/** Bundle-relative path of the asset index (`assets/index.ndjson`), when any
|
|
619
|
+
* assets were exported. Each line is an {@link AssetEntry}. */
|
|
620
|
+
index: Schema.optional<typeof Schema.String>;
|
|
621
|
+
}>;
|
|
622
|
+
/** Masking audit — present when the export applied a masking policy. Lists
|
|
623
|
+
* every `table.column` that was transformed (not copied verbatim), so a
|
|
624
|
+
* reviewer can see exactly what was pseudonymised. */
|
|
625
|
+
masking: Schema.optional<Schema.Struct<{
|
|
626
|
+
policyId: Schema.optional<typeof Schema.String>;
|
|
627
|
+
transformed: Schema.Array$<typeof Schema.String>;
|
|
628
|
+
}>>;
|
|
629
|
+
}>;
|
|
630
|
+
|
|
631
|
+
export declare type Manifest = Schema.Schema.Type<typeof Manifest>;
|
|
632
|
+
|
|
633
|
+
/** File names inside a bundle directory. */
|
|
634
|
+
export declare const MANIFEST_FILE = "manifest.json";
|
|
635
|
+
|
|
636
|
+
/** What to do with a column's value. */
|
|
637
|
+
export declare type MaskAction = 'keep' | 'null' | 'redact' | 'hash' | 'dateShift' | 'error' | {
|
|
638
|
+
readonly fake: string;
|
|
639
|
+
} | {
|
|
640
|
+
readonly custom: (input: MaskInput) => unknown;
|
|
641
|
+
};
|
|
642
|
+
|
|
643
|
+
/** A masking export was refused because one or more exported columns are
|
|
644
|
+
* neither `.sensitive()` nor `.safe()` (fail-closed). Lists them so the author
|
|
645
|
+
* can classify each — a new column can never silently leak to a lower env. */
|
|
646
|
+
export declare class MaskingError extends MaskingError_base {
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
declare const MaskingError_base: Schema.TaggedErrorClass<MaskingError, "MaskingError", {
|
|
650
|
+
readonly _tag: Schema.tag<"MaskingError">;
|
|
651
|
+
} & {
|
|
652
|
+
unclassified: Schema.Array$<typeof Schema.String>;
|
|
653
|
+
}>;
|
|
654
|
+
|
|
655
|
+
export declare interface MaskingPlan {
|
|
656
|
+
/** `table.column` → resolved action (excludes columns that resolve to error). */
|
|
657
|
+
readonly actions: Readonly<Record<string, MaskAction>>;
|
|
658
|
+
/** Columns that resolved to `error` — export must refuse (or reclassify). */
|
|
659
|
+
readonly unclassified: ReadonlyArray<string>;
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
export declare interface MaskingPolicy {
|
|
663
|
+
/** Recorded in the bundle audit. */
|
|
664
|
+
readonly id?: string;
|
|
665
|
+
/** Secret salt keying every deterministic transform. Keep it stable to keep
|
|
666
|
+
* dev data stable; rotate/discard it to make the pseudonymisation
|
|
667
|
+
* irreversible (anonymisation). REQUIRED — masking without a seed is a bug. */
|
|
668
|
+
readonly seed: string;
|
|
669
|
+
/** Action per sensitivity class (overrides the built-in default for a class). */
|
|
670
|
+
readonly classes?: Readonly<Record<string, MaskAction>>;
|
|
671
|
+
/** Per-column override, keyed `table.column` (wins over class). */
|
|
672
|
+
readonly columns?: Readonly<Record<string, MaskAction>>;
|
|
673
|
+
/** A column that is neither `.sensitive()` nor `.safe()` nor overridden.
|
|
674
|
+
* Default `'error'` (fail-closed): the export refuses until it's classified. */
|
|
675
|
+
readonly onUnclassified?: 'error' | 'keep' | 'null';
|
|
676
|
+
/** What a `.safe()` column gets. Default `'keep'`. */
|
|
677
|
+
readonly onSafe?: 'keep' | 'null';
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
export declare interface MaskingPreview {
|
|
681
|
+
readonly tables: ReadonlyArray<{
|
|
682
|
+
readonly table: string;
|
|
683
|
+
readonly samples: ReadonlyArray<MaskingSample>;
|
|
684
|
+
}>;
|
|
685
|
+
readonly leaks: ReadonlyArray<LeakWarning>;
|
|
686
|
+
/** Columns that WOULD block a real export (fail-closed). Empty = safe to run. */
|
|
687
|
+
readonly unclassified: ReadonlyArray<string>;
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
export declare interface MaskingPreviewOptions {
|
|
691
|
+
readonly store: DataStore;
|
|
692
|
+
readonly snapshot: SchemaSnapshot;
|
|
693
|
+
readonly masking: MaskingPolicy;
|
|
694
|
+
readonly classification?: Classification;
|
|
695
|
+
readonly scope?: ExportScope;
|
|
696
|
+
/** Tenant-scoped table names (mixin metadata) — same contract as
|
|
697
|
+
* `ExportOptions.tenantTables`; only relevant for a tenant scope. */
|
|
698
|
+
readonly tenantTables?: ReadonlyArray<string>;
|
|
699
|
+
/** Rows sampled per table (default 20). */
|
|
700
|
+
readonly sampleSize?: number;
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
export declare interface MaskingSample {
|
|
704
|
+
readonly column: string;
|
|
705
|
+
readonly action: string;
|
|
706
|
+
readonly before: unknown;
|
|
707
|
+
readonly after: unknown;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
export declare interface MaskInput {
|
|
711
|
+
readonly value: unknown;
|
|
712
|
+
readonly table: string;
|
|
713
|
+
readonly column: string;
|
|
714
|
+
readonly columnType?: string;
|
|
715
|
+
readonly seed: string;
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
/** Apply a resolved plan to one row. */
|
|
719
|
+
export declare const maskRow: (row: Row, table: string, actions: Readonly<Record<string, MaskAction>>, seed: string, columnTypes?: Readonly<Record<string, string>>) => Row;
|
|
720
|
+
|
|
721
|
+
export declare interface NativeConn {
|
|
722
|
+
/** Full connection URL (preferred for postgres — the tool consumes it
|
|
723
|
+
* directly). Parsed to flags for mysql. */
|
|
724
|
+
readonly url?: string;
|
|
725
|
+
readonly host?: string;
|
|
726
|
+
readonly port?: number;
|
|
727
|
+
readonly username?: string;
|
|
728
|
+
readonly password?: string;
|
|
729
|
+
readonly database?: string;
|
|
730
|
+
/** sqlite database file path. */
|
|
731
|
+
readonly filename?: string;
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
/** One executable step: spawn a tool (optionally piping a file to stdin, or its
|
|
735
|
+
* stdout to a file), or a pure file copy (sqlite). */
|
|
736
|
+
export declare type NativeStep = {
|
|
737
|
+
readonly kind: 'spawn';
|
|
738
|
+
readonly tool: string;
|
|
739
|
+
readonly args: ReadonlyArray<string>;
|
|
740
|
+
readonly env: Readonly<Record<string, string>>;
|
|
741
|
+
readonly stdoutFile?: string;
|
|
742
|
+
readonly stdinFile?: string;
|
|
743
|
+
} | {
|
|
744
|
+
readonly kind: 'copy';
|
|
745
|
+
readonly from: string;
|
|
746
|
+
readonly to: string;
|
|
747
|
+
};
|
|
748
|
+
|
|
749
|
+
/** A native backup/restore tool failed — missing binary, non-zero exit, or an
|
|
750
|
+
* unsupported dialect. `stderr` carries the tool's own diagnostics. INTERNAL —
|
|
751
|
+
* only the local `voltro backup|restore` path raises it; never on a pipeline
|
|
752
|
+
* error channel, so it never crosses the rpc wire. */
|
|
753
|
+
export declare class NativeToolError extends NativeToolError_base<{
|
|
754
|
+
readonly tool: string;
|
|
755
|
+
readonly reason: string;
|
|
756
|
+
readonly stderr?: string;
|
|
757
|
+
}> {
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
declare const NativeToolError_base: new <A extends Record<string, any> = {}>(args: VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => YieldableError & {
|
|
761
|
+
readonly _tag: "NativeToolError";
|
|
762
|
+
} & Readonly<A>;
|
|
763
|
+
|
|
764
|
+
/** NDJSON byte stream → row stream (split into lines, JSON-parse, typed-decode).
|
|
765
|
+
* A malformed line fails the stream with a {@link CodecError} naming the table
|
|
766
|
+
* rather than a bare SyntaxError. */
|
|
767
|
+
export declare const ndjsonToRows: <E, R>(bytes: Stream.Stream<Uint8Array, E, R>, table: string, columnTypes: ColumnTypes) => Stream.Stream<Row, E | CodecError, R>;
|
|
768
|
+
|
|
769
|
+
export declare type OnConflict = 'skip' | 'fail';
|
|
770
|
+
|
|
771
|
+
/** The caller's progress sink. Plain sync fn — never fails the job. */
|
|
772
|
+
export declare type OnProgress = (event: ProgressEvent) => void;
|
|
773
|
+
|
|
774
|
+
/**
|
|
775
|
+
* Pack a bundle into a framed byte stream. `dir` supplies the small files
|
|
776
|
+
* (tables / manifest / README); `blobs`, when given, streams every content-blob
|
|
777
|
+
* in single-pass (nothing buffered) with a per-blob sha footer.
|
|
778
|
+
*/
|
|
779
|
+
export declare function packBundle(opts: {
|
|
780
|
+
dir: string;
|
|
781
|
+
blobs?: BlobSource;
|
|
782
|
+
}): AsyncGenerator<Uint8Array>;
|
|
783
|
+
|
|
784
|
+
/**
|
|
785
|
+
* Read a FEW small plain files out of a bundle stream and STOP — without
|
|
786
|
+
* materialising the data or blobs. Returns the requested files' bytes (those
|
|
787
|
+
* present before the first blob). `README.md` sits near the front, so peeking it
|
|
788
|
+
* decrypts/reads only the first chunk(s) — cheap even on a 200 GB bundle. Used by
|
|
789
|
+
* `voltro data inspect`.
|
|
790
|
+
*/
|
|
791
|
+
export declare const peekBundle: (source: AsyncIterable<Uint8Array>, names: readonly string[]) => Promise<Map<string, Buffer>>;
|
|
792
|
+
|
|
793
|
+
/**
|
|
794
|
+
* Resolve the effective codec. `undefined`/`'zstd'` → zstd when available, else
|
|
795
|
+
* gzip. `'gzip'` / `'none'` are honoured as-is. Callers record the RESULT in
|
|
796
|
+
* the manifest — never re-derive on import.
|
|
797
|
+
*/
|
|
798
|
+
export declare const pickCompression: (pref?: Compression) => Compression;
|
|
799
|
+
|
|
800
|
+
/**
|
|
801
|
+
* Resolve every column's action ahead of streaming. `unclassified` lists the
|
|
802
|
+
* columns that hit `onUnclassified: 'error'` — the exporter refuses when it's
|
|
803
|
+
* non-empty, so a masking export can never silently copy an unreviewed column.
|
|
804
|
+
*/
|
|
805
|
+
export declare const planMasking: (tables: ReadonlyArray<{
|
|
806
|
+
readonly name: string;
|
|
807
|
+
readonly columns: ReadonlyArray<{
|
|
808
|
+
readonly name: string;
|
|
809
|
+
readonly type?: string;
|
|
810
|
+
}>;
|
|
811
|
+
}>, classification: Classification, policy: MaskingPolicy) => MaskingPlan;
|
|
812
|
+
|
|
813
|
+
export declare interface PortabilityIssue {
|
|
814
|
+
readonly table: string;
|
|
815
|
+
readonly column: string;
|
|
816
|
+
readonly type: string;
|
|
817
|
+
readonly severity: 'block' | 'warn';
|
|
818
|
+
readonly reason: string;
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
export declare const previewMasking: (opts: MaskingPreviewOptions) => Effect.Effect<MaskingPreview, TableStreamError | ScopeError>;
|
|
822
|
+
|
|
823
|
+
/**
|
|
824
|
+
* One progress tick, emitted per table at its phase boundaries. `rowsDone`
|
|
825
|
+
* grows monotonically WITHIN a table's lifetime for a given phase: a `start`
|
|
826
|
+
* carries `rowsDone: 0`, the matching `done` carries the final count. `total`
|
|
827
|
+
* is the table's row count when known (import knows it up front from the
|
|
828
|
+
* manifest; a live export does not until the table drains) — `undefined` when
|
|
829
|
+
* it can't be known ahead of time.
|
|
830
|
+
*/
|
|
831
|
+
export declare interface ProgressEvent {
|
|
832
|
+
readonly phase: ProgressPhase;
|
|
833
|
+
/** Table being processed. */
|
|
834
|
+
readonly table: string;
|
|
835
|
+
/** 0-based index of this table in the ordered set. */
|
|
836
|
+
readonly index: number;
|
|
837
|
+
/** How many tables in total (for "7 / 23"). */
|
|
838
|
+
readonly tableCount: number;
|
|
839
|
+
/** `start` when the table begins, `done` when it finishes (or was resumed —
|
|
840
|
+
* a ledger-skipped table emits a single `done` with its recorded count). */
|
|
841
|
+
readonly event: 'start' | 'done';
|
|
842
|
+
/** Rows processed so far for THIS table. 0 on `start`; final count on `done`. */
|
|
843
|
+
readonly rowsDone: number;
|
|
844
|
+
/** The table's total row count when known ahead of time (import: from the
|
|
845
|
+
* manifest; skipped/resumed export: the ledger count), else `undefined`. */
|
|
846
|
+
readonly total?: number;
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
/** Which pipeline the event belongs to. */
|
|
850
|
+
export declare type ProgressPhase = 'export' | 'import';
|
|
851
|
+
|
|
852
|
+
/** Read the ledger at `path`, or an empty one when it is absent / unreadable
|
|
853
|
+
* (a missing ledger just means "nothing done yet" — never a hard failure). */
|
|
854
|
+
export declare const readLedger: (path: string) => Effect.Effect<Ledger>;
|
|
855
|
+
|
|
856
|
+
/** Resolve the action for one column (column override → class → safe → unclassified). */
|
|
857
|
+
export declare const resolveAction: (table: string, column: string, cls: ColumnClassification | undefined, policy: MaskingPolicy) => MaskAction;
|
|
858
|
+
|
|
859
|
+
export declare interface ResolvedScope {
|
|
860
|
+
/** The unordered set of tables to export (the pipeline topo-sorts them). */
|
|
861
|
+
readonly tables: ReadonlyArray<string>;
|
|
862
|
+
/** Row filter for a table, or undefined for "all rows". */
|
|
863
|
+
readonly predicateFor: (table: string) => Predicate | undefined;
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
export declare const resolveScope: (scope: ExportScope, snapshot: SchemaSnapshot, deps: ResolveScopeDeps) => Effect.Effect<ResolvedScope, ScopeError | TableStreamError>;
|
|
867
|
+
|
|
868
|
+
export declare interface ResolveScopeDeps {
|
|
869
|
+
/** Source store — the tenant scope reads the tenant's rows to compute which
|
|
870
|
+
* untenanted parent rows they reference (row-level subsetting). */
|
|
871
|
+
readonly store: DataStore;
|
|
872
|
+
/** Names of the tenant-scoped tables, from the schema registry's `tenant()`
|
|
873
|
+
* mixin metadata. Authoritative when provided; omitted → the `tenantId`
|
|
874
|
+
* column-name heuristic (documented above). */
|
|
875
|
+
readonly tenantTables?: ReadonlyArray<string>;
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
/**
|
|
879
|
+
* Restore one CAS asset to the sink under `key`, re-hashing the bytes as they
|
|
880
|
+
* stream through and refusing (IntegrityError) if they don't match the expected
|
|
881
|
+
* sha — so a corrupted bundle can never silently re-upload bad bytes.
|
|
882
|
+
*/
|
|
883
|
+
export declare const restoreAssetFromCas: (sink: AssetSink, assetsDir: string, entry: {
|
|
884
|
+
readonly key: string;
|
|
885
|
+
readonly sha256: string;
|
|
886
|
+
readonly size: number;
|
|
887
|
+
readonly contentType: string;
|
|
888
|
+
}) => Effect.Effect<void, BundleError | IntegrityError | StorageError>;
|
|
889
|
+
|
|
890
|
+
/** Build the restore step for a dialect reading `inFile`. */
|
|
891
|
+
export declare const restoreCommand: (dialect: string, conn: NativeConn, inFile: string) => NativeStep;
|
|
892
|
+
|
|
893
|
+
/** Row stream → NDJSON byte stream (each row → typed-encoded JSON + `\n`). The
|
|
894
|
+
* source stream's error + requirement channels pass straight through. */
|
|
895
|
+
export declare const rowsToNdjson: <E, R>(rows: Stream.Stream<Row, E, R>, columnTypes: ColumnTypes) => Stream.Stream<Uint8Array, E, R>;
|
|
896
|
+
|
|
897
|
+
export declare const runExport: (opts: ExportOptions) => Effect.Effect<Manifest, ExportError>;
|
|
898
|
+
|
|
899
|
+
export declare const runImport: (opts: ImportOptions) => Effect.Effect<Manifest, ImportError>;
|
|
900
|
+
|
|
901
|
+
/** Execute a {@link NativeStep} — spawn the tool (wiring stdin/stdout files) or
|
|
902
|
+
* copy the file. Maps a missing binary / non-zero exit to {@link NativeToolError}. */
|
|
903
|
+
export declare const runNativeStep: (step: NativeStep) => Effect.Effect<void, NativeToolError>;
|
|
904
|
+
|
|
905
|
+
/**
|
|
906
|
+
* Scan the KEPT columns of some sample rows for values that LOOK like PII — a
|
|
907
|
+
* cheap tripwire for a misclassified column (`.safe()` on something that isn't,
|
|
908
|
+
* or an unmasked unclassified column under a fail-open policy). Reports at most
|
|
909
|
+
* one warning per column. Does NOT scan transformed columns (they're already
|
|
910
|
+
* fake). Best-effort, never blocks — it surfaces suspicion for a human.
|
|
911
|
+
*/
|
|
912
|
+
export declare const scanForLeaks: (rows: ReadonlyArray<Row>, table: string, actions: Readonly<Record<string, MaskAction>>) => ReadonlyArray<LeakWarning>;
|
|
913
|
+
|
|
914
|
+
/** An import was refused because the TARGET app's live schema has drifted from
|
|
915
|
+
* the schema the bundle was exported against (their fingerprints differ). The
|
|
916
|
+
* bundle's typed codec + FK order assume the source shape; importing into a
|
|
917
|
+
* drifted target fails mid-load with raw DB errors after rows may have landed.
|
|
918
|
+
* Fail-closed: refuse before the table phase. `--force` (CLI) / `force: true`
|
|
919
|
+
* (programmatic) downgrades this to a loud warning and proceeds. `diff` names
|
|
920
|
+
* the observable schema differences (missing/extra tables, per-table column
|
|
921
|
+
* changes) so the operator can see exactly what moved. */
|
|
922
|
+
export declare class SchemaDriftError extends SchemaDriftError_base {
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
declare const SchemaDriftError_base: Schema.TaggedErrorClass<SchemaDriftError, "SchemaDriftError", {
|
|
926
|
+
readonly _tag: Schema.tag<"SchemaDriftError">;
|
|
927
|
+
} & {
|
|
928
|
+
/** The fingerprint stamped in the bundle manifest (the source schema). */
|
|
929
|
+
bundleFingerprint: typeof Schema.String;
|
|
930
|
+
/** The target app's current schema fingerprint. */
|
|
931
|
+
targetFingerprint: typeof Schema.String;
|
|
932
|
+
/** Human-readable summary of the drift (capped), or a note that the shape
|
|
933
|
+
* differs but the specific diff was not enumerated. */
|
|
934
|
+
diff: Schema.Array$<typeof Schema.String>;
|
|
935
|
+
}>;
|
|
936
|
+
|
|
937
|
+
/** An export scope could not be resolved safely — e.g. `{ kind: 'tenant' }`
|
|
938
|
+
* without a `tenantId`. Fail-closed: a scoping ambiguity refuses the export
|
|
939
|
+
* instead of silently widening to more rows than the operator asked for. */
|
|
940
|
+
export declare class ScopeError extends ScopeError_base {
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
declare const ScopeError_base: Schema.TaggedErrorClass<ScopeError, "ScopeError", {
|
|
944
|
+
readonly _tag: Schema.tag<"ScopeError">;
|
|
945
|
+
} & {
|
|
946
|
+
reason: typeof Schema.String;
|
|
947
|
+
}>;
|
|
948
|
+
|
|
949
|
+
/** Build an {@link AssetSink} over a storage provider. */
|
|
950
|
+
export declare const storageAssetSink: (provider: StorageProvider) => AssetSink;
|
|
951
|
+
|
|
952
|
+
/** Build an {@link AssetSource} over a storage provider — `list` is supplied by
|
|
953
|
+
* the caller (typically streaming the `_voltro_storage_refs` table), `open`
|
|
954
|
+
* uses the provider's streaming read (buffered fallback for non-streaming
|
|
955
|
+
* backends). */
|
|
956
|
+
export declare const storageAssetSource: (provider: StorageProvider, list: () => Stream.Stream<AssetInfo, unknown>) => AssetSource;
|
|
957
|
+
|
|
958
|
+
/** Stream blobs TO object storage. `has` powers resume (skip existing keys);
|
|
959
|
+
* `write` streams the archive's blob content straight into the provider. */
|
|
960
|
+
export declare const storageBlobSink: (provider: StorageProvider) => BlobSink;
|
|
961
|
+
|
|
962
|
+
/** Stream blobs FROM object storage. `list` enumerates metadata (typically the
|
|
963
|
+
* storage refs table); `open` streams each blob's bytes via the provider. */
|
|
964
|
+
export declare const storageBlobSource: (provider: StorageProvider, list: () => Stream.Stream<AssetInfo, unknown>) => BlobSource;
|
|
965
|
+
|
|
966
|
+
/** A {@link BlobSource} over a provider whose blob list comes from the app's
|
|
967
|
+
* `_voltro_storage_refs` table — the standard export source for a running
|
|
968
|
+
* instance (CLI file export and the prod export endpoint both use it). */
|
|
969
|
+
export declare const storageRefsBlobSource: (provider: StorageProvider, store: DataStore) => BlobSource;
|
|
970
|
+
|
|
971
|
+
export declare interface SubsetSpec {
|
|
972
|
+
/** Seed rows per table. `table → predicate` (a `null`/absent predicate seeds
|
|
973
|
+
* every row of that table). Only these tables + their FK-parent closure are
|
|
974
|
+
* exported. */
|
|
975
|
+
readonly seeds: Readonly<Record<string, Predicate | undefined>>;
|
|
976
|
+
/** Safety bound on closure depth (default 12). */
|
|
977
|
+
readonly maxDepth?: number;
|
|
978
|
+
/**
|
|
979
|
+
* Tables whose seed predicate is FINAL — the closure never widens them. A
|
|
980
|
+
* reference INTO a pinned table is not followed (the target row is either
|
|
981
|
+
* already covered by that table's own seed predicate, or deliberately out of
|
|
982
|
+
* scope), and pinned tables collect no id sets in the RETURNED map (their
|
|
983
|
+
* rows stay predicate-selected). The tenant scope pins every tenant-scoped
|
|
984
|
+
* table so a closure walk can never pull another tenant's rows into the
|
|
985
|
+
* bundle. NB: even a pinned table's ids are tracked internally so the child
|
|
986
|
+
* pass can scope rows that reference the pinned (tenant) rows.
|
|
987
|
+
*/
|
|
988
|
+
readonly pinned?: ReadonlyArray<string>;
|
|
989
|
+
/**
|
|
990
|
+
* Child-closure: after the parent closure settles, also include rows that
|
|
991
|
+
* REFERENCE the exported set — each restricted to the ids that actually point
|
|
992
|
+
* INTO the set (never the whole child table). The child rows' OWN parents are
|
|
993
|
+
* then folded back through the parent closure, so the result stays
|
|
994
|
+
* referentially complete (fail-closed: a child pulled in never dangles). Off
|
|
995
|
+
* by default — a parents-only subset is the cheaper common case.
|
|
996
|
+
*
|
|
997
|
+
* The walk is anchored on `roots` (a set of "tenant-tight" tables). A child
|
|
998
|
+
* row is pulled ONLY when at least one of its FKs points into a root row —
|
|
999
|
+
* OR into a row of a table ALREADY established as tight in this walk (a child
|
|
1000
|
+
* pulled in earlier). This is the safety valve for the tenant case: `roots`
|
|
1001
|
+
* is the tenant-scoped tables, so a comment on the tenant's project comes
|
|
1002
|
+
* along, but a row that references ONLY a SHARED parent (a global `users`
|
|
1003
|
+
* another tenant also references) is NOT pulled — that would be a
|
|
1004
|
+
* cross-tenant leak. `roots: []` (or omitting the whole option) disables the
|
|
1005
|
+
* child pass. `true` is the escape hatch: anchor on every already-selected
|
|
1006
|
+
* table (org-slice takeout — no tenant boundary to respect).
|
|
1007
|
+
*/
|
|
1008
|
+
readonly children?: {
|
|
1009
|
+
readonly roots: ReadonlyArray<string>;
|
|
1010
|
+
} | true;
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
/** Turn computed id sets into a {@link ResolvedScope} the exporter consumes. */
|
|
1014
|
+
export declare const subsetToScope: (selected: Map<string, Set<unknown>>, snapshot: SchemaSnapshot) => ResolvedScope;
|
|
1015
|
+
|
|
1016
|
+
export declare const tableDone: (ledger: Ledger, name: string) => boolean;
|
|
1017
|
+
|
|
1018
|
+
/** Per-table entry. `columnTypes` maps every exported column to its logical
|
|
1019
|
+
* {@link ColumnType} (as a string) so the importer decodes values without
|
|
1020
|
+
* re-introspecting the source. `checksum` is the sha256 of the UNCOMPRESSED
|
|
1021
|
+
* NDJSON payload — verified during import. */
|
|
1022
|
+
export declare const TableEntry: Schema.Struct<{
|
|
1023
|
+
name: typeof Schema.String;
|
|
1024
|
+
/** Bundle-relative path, e.g. `data/users.ndjson.zst`. */
|
|
1025
|
+
file: typeof Schema.String;
|
|
1026
|
+
rowCount: typeof Schema.Number;
|
|
1027
|
+
/** sha256 hex of the uncompressed NDJSON bytes. */
|
|
1028
|
+
checksum: typeof Schema.String;
|
|
1029
|
+
/** column name → logical column type (the `ColumnType` union, as a string). */
|
|
1030
|
+
columnTypes: Schema.Record$<typeof Schema.String, typeof Schema.String>;
|
|
1031
|
+
/** The unique column keyset-streamed on export (usually `id`). */
|
|
1032
|
+
primaryKey: typeof Schema.String;
|
|
1033
|
+
}>;
|
|
1034
|
+
|
|
1035
|
+
export declare type TableEntry = Schema.Schema.Type<typeof TableEntry>;
|
|
1036
|
+
|
|
1037
|
+
/**
|
|
1038
|
+
* Order `tables` (a subset of the snapshot) so every table comes after the
|
|
1039
|
+
* tables it references. Only FKs WITHIN the selected set constrain the order —
|
|
1040
|
+
* a reference to an unselected table is ignored (the caller scoped it out on
|
|
1041
|
+
* purpose). A cyclic dependency is broken at the edge that would close the
|
|
1042
|
+
* cycle (deterministic for a given snapshot + input order); the importer's
|
|
1043
|
+
* deferred-FK resolution covers the rows that land before their parents.
|
|
1044
|
+
*/
|
|
1045
|
+
export declare const topoSortTables: (tables: ReadonlyArray<string>, snapshot: SchemaSnapshot) => ReadonlyArray<string>;
|
|
1046
|
+
|
|
1047
|
+
/**
|
|
1048
|
+
* Unpack a framed byte stream. Plain files are written under `dir` (traversal
|
|
1049
|
+
* guarded); blob entries are streamed to `blobs` (dest storage) — never to local
|
|
1050
|
+
* disk — with the trailing sha verified as they pass through. A blob whose key
|
|
1051
|
+
* already exists at the sink is skipped (resume).
|
|
1052
|
+
*/
|
|
1053
|
+
export declare const unpackBundle: (source: AsyncIterable<Uint8Array>, opts: {
|
|
1054
|
+
dir: string;
|
|
1055
|
+
blobs: BlobSink;
|
|
1056
|
+
}) => Promise<void>;
|
|
1057
|
+
|
|
1058
|
+
/** Convenience: unpack a bundle fully to a local directory (plain files as-is,
|
|
1059
|
+
* blob entries materialised under `assets/`). For data-only bundles and
|
|
1060
|
+
* `voltro data unpack`; the streaming import path passes a storage {@link BlobSink}
|
|
1061
|
+
* to {@link unpackBundle} instead so blobs never touch local disk. */
|
|
1062
|
+
export declare const unpackBundleToDir: (source: AsyncIterable<Uint8Array>, dir: string) => Promise<void>;
|
|
1063
|
+
|
|
1064
|
+
/**
|
|
1065
|
+
* Stream one asset into the CAS dir, returning its sha256 + size. Hashes while
|
|
1066
|
+
* writing to a temp file, then renames to `assets/<sha>` — or drops the temp if
|
|
1067
|
+
* that hash is already present (dedup). The rename is atomic, so a concurrent
|
|
1068
|
+
* reader never sees a half-written asset.
|
|
1069
|
+
*/
|
|
1070
|
+
export declare const writeAssetToCas: (stored: StoredStream, assetsDir: string) => Effect.Effect<{
|
|
1071
|
+
readonly sha256: string;
|
|
1072
|
+
readonly size: number;
|
|
1073
|
+
readonly contentType: string;
|
|
1074
|
+
}, BundleError>;
|
|
1075
|
+
|
|
1076
|
+
/** Persist the ledger atomically (temp + rename) so a crash mid-write never
|
|
1077
|
+
* corrupts the journal itself. */
|
|
1078
|
+
export declare const writeLedger: (path: string, ledger: Ledger) => Effect.Effect<void, BundleError>;
|
|
1079
|
+
|
|
1080
|
+
export { }
|