@strapi/data-transfer 4.9.0 → 4.10.0-beta.0

package/lib/errors/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./constants"), exports);
+__exportStar(require("./base"), exports);
+//# sourceMappingURL=index.js.map

package/lib/errors/providers.d.ts

@@ -0,0 +1,21 @@
+import { DataTransferError } from './base';
+import { Severity } from './constants';
+declare type ProviderStep = 'initialization' | 'validation' | 'transfer';
+declare type ProviderErrorDetails<P extends ProviderStep = ProviderStep, U = never> = {
+    step: P;
+} & ([U] extends [never] ? unknown : {
+    details?: U;
+});
+export declare class ProviderError<P extends ProviderStep = ProviderStep, U = never, T extends ProviderErrorDetails<P, U> = ProviderErrorDetails<P, U>> extends DataTransferError<T> {
+    constructor(severity: Severity, message?: string, details?: T | null);
+}
+export declare class ProviderInitializationError extends ProviderError<'initialization'> {
+    constructor(message?: string);
+}
+export declare class ProviderValidationError<T = ProviderErrorDetails> extends ProviderError<'validation', T> {
+    constructor(message?: string, details?: T);
+}
+export declare class ProviderTransferError<T = ProviderErrorDetails> extends ProviderError<'transfer', T> {
+    constructor(message?: string, details?: T);
+}
+export {};

package/lib/errors/providers.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProviderTransferError = exports.ProviderValidationError = exports.ProviderInitializationError = exports.ProviderError = void 0;
+const base_1 = require("./base");
+const constants_1 = require("./constants");
+class ProviderError extends base_1.DataTransferError {
+    constructor(severity, message, details) {
+        super('provider', severity, message, details);
+    }
+}
+exports.ProviderError = ProviderError;
+class ProviderInitializationError extends ProviderError {
+    constructor(message) {
+        super(constants_1.SeverityKind.FATAL, message, { step: 'initialization' });
+    }
+}
+exports.ProviderInitializationError = ProviderInitializationError;
+// TODO: these types are not working correctly, ProviderTransferError() is accepting any details object rather than requiring T
+class ProviderValidationError extends ProviderError {
+    constructor(message, details) {
+        super(constants_1.SeverityKind.SILLY, message, { step: 'validation', details });
+    }
+}
+exports.ProviderValidationError = ProviderValidationError;
+// TODO: these types are not working correctly, ProviderTransferError() is accepting any details object rather than requiring T
+class ProviderTransferError extends ProviderError {
+    constructor(message, details) {
+        super(constants_1.SeverityKind.FATAL, message, { step: 'transfer', details });
+    }
+}
+exports.ProviderTransferError = ProviderTransferError;
+//# sourceMappingURL=providers.js.map

package/lib/file/index.d.ts

@@ -0,0 +1 @@
+export * as providers from './providers';

package/lib/file/index.js

@@ -0,0 +1,28 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.providers = void 0;
+exports.providers = __importStar(require("./providers"));
+//# sourceMappingURL=index.js.map

package/lib/file/providers/destination/index.d.ts

@@ -0,0 +1,45 @@
+/// <reference types="node" />
+/// <reference types="stream-chain" />
+import zlib from 'zlib';
+import { Writable } from 'stream';
+import type { IDestinationProvider, IDestinationProviderTransferResults, IMetadata, ProviderType } from '../../../../types';
+export interface ILocalFileDestinationProviderOptions {
+    encryption: {
+        enabled: boolean;
+        key?: string;
+    };
+    compression: {
+        enabled: boolean;
+    };
+    file: {
+        path: string;
+        maxSize?: number;
+        maxSizeJsonl?: number;
+    };
+}
+export interface ILocalFileDestinationProviderTransferResults extends IDestinationProviderTransferResults {
+    file?: {
+        path?: string;
+    };
+}
+export declare const createLocalFileDestinationProvider: (options: ILocalFileDestinationProviderOptions) => LocalFileDestinationProvider;
+declare class LocalFileDestinationProvider implements IDestinationProvider {
+    #private;
+    name: string;
+    type: ProviderType;
+    options: ILocalFileDestinationProviderOptions;
+    results: ILocalFileDestinationProviderTransferResults;
+    constructor(options: ILocalFileDestinationProviderOptions);
+    setMetadata(target: ProviderType, metadata: IMetadata): IDestinationProvider;
+    createGzip(): zlib.Gzip;
+    bootstrap(): void | Promise<void>;
+    close(): Promise<void>;
+    rollback(): Promise<void>;
+    getMetadata(): null;
+    createSchemasWriteStream(): import("stream-chain");
+    createEntitiesWriteStream(): Writable;
+    createLinksWriteStream(): Writable;
+    createConfigurationWriteStream(): Writable;
+    createAssetsWriteStream(): Writable;
+}
+export {};

package/lib/file/providers/destination/index.js

@@ -0,0 +1,174 @@
+"use strict";
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var _LocalFileDestinationProvider_instances, _LocalFileDestinationProvider_providersMetadata, _LocalFileDestinationProvider_archive, _LocalFileDestinationProvider_archivePath_get, _LocalFileDestinationProvider_writeMetadata, _LocalFileDestinationProvider_getMetadataStream;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createLocalFileDestinationProvider = void 0;
+const fs_extra_1 = require("fs-extra");
+const tar_stream_1 = __importDefault(require("tar-stream"));
+const path_1 = __importDefault(require("path"));
+const zlib_1 = __importDefault(require("zlib"));
+const Stringer_1 = require("stream-json/jsonl/Stringer");
+const stream_chain_1 = require("stream-chain");
+const stream_1 = require("stream");
+const encryption_1 = require("../../../utils/encryption");
+const utils_1 = require("./utils");
+const providers_1 = require("../../../errors/providers");
+const createLocalFileDestinationProvider = (options) => {
+    return new LocalFileDestinationProvider(options);
+};
+exports.createLocalFileDestinationProvider = createLocalFileDestinationProvider;
+class LocalFileDestinationProvider {
+    constructor(options) {
+        _LocalFileDestinationProvider_instances.add(this);
+        this.name = 'destination::local-file';
+        this.type = 'destination';
+        this.results = {};
+        _LocalFileDestinationProvider_providersMetadata.set(this, {});
+        _LocalFileDestinationProvider_archive.set(this, {});
+        this.options = options;
+    }
+    setMetadata(target, metadata) {
+        __classPrivateFieldGet(this, _LocalFileDestinationProvider_providersMetadata, "f")[target] = metadata;
+        return this;
+    }
+    createGzip() {
+        return zlib_1.default.createGzip();
+    }
+    bootstrap() {
+        const { compression, encryption } = this.options;
+        if (encryption.enabled && !encryption.key) {
+            throw new Error("Can't encrypt without a key");
+        }
+        __classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream = tar_stream_1.default.pack();
+        const outStream = (0, fs_extra_1.createWriteStream)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_instances, "a", _LocalFileDestinationProvider_archivePath_get));
+        outStream.on('error', (err) => {
+            if (err.code === 'ENOSPC') {
+                throw new providers_1.ProviderTransferError("Your server doesn't have space to proceed with the import.");
+            }
+            throw err;
+        });
+        const archiveTransforms = [];
+        if (compression.enabled) {
+            archiveTransforms.push(this.createGzip());
+        }
+        if (encryption.enabled && encryption.key) {
+            archiveTransforms.push((0, encryption_1.createEncryptionCipher)(encryption.key));
+        }
+        __classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").pipeline = (0, stream_chain_1.chain)([__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, ...archiveTransforms, outStream]);
+        this.results.file = { path: __classPrivateFieldGet(this, _LocalFileDestinationProvider_instances, "a", _LocalFileDestinationProvider_archivePath_get) };
+    }
+    async close() {
+        const { stream, pipeline } = __classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f");
+        if (!stream) {
+            return;
+        }
+        await __classPrivateFieldGet(this, _LocalFileDestinationProvider_instances, "m", _LocalFileDestinationProvider_writeMetadata).call(this);
+        stream.finalize();
+        if (pipeline && !pipeline.closed) {
+            await new Promise((resolve, reject) => {
+                pipeline.on('close', resolve).on('error', reject);
+            });
+        }
+    }
+    async rollback() {
+        await this.close();
+        await (0, fs_extra_1.rm)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_instances, "a", _LocalFileDestinationProvider_archivePath_get), { force: true });
+    }
+    getMetadata() {
+        return null;
+    }
+    createSchemasWriteStream() {
+        if (!__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        const filePathFactory = (0, utils_1.createFilePathFactory)('schemas');
+        const entryStream = (0, utils_1.createTarEntryStream)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return (0, stream_chain_1.chain)([(0, Stringer_1.stringer)(), entryStream]);
+    }
+    createEntitiesWriteStream() {
+        if (!__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        const filePathFactory = (0, utils_1.createFilePathFactory)('entities');
+        const entryStream = (0, utils_1.createTarEntryStream)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return (0, stream_chain_1.chain)([(0, Stringer_1.stringer)(), entryStream]);
+    }
+    createLinksWriteStream() {
+        if (!__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        const filePathFactory = (0, utils_1.createFilePathFactory)('links');
+        const entryStream = (0, utils_1.createTarEntryStream)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return (0, stream_chain_1.chain)([(0, Stringer_1.stringer)(), entryStream]);
+    }
+    createConfigurationWriteStream() {
+        if (!__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        const filePathFactory = (0, utils_1.createFilePathFactory)('configuration');
+        const entryStream = (0, utils_1.createTarEntryStream)(__classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f").stream, filePathFactory, this.options.file.maxSizeJsonl);
+        return (0, stream_chain_1.chain)([(0, Stringer_1.stringer)(), entryStream]);
+    }
+    createAssetsWriteStream() {
+        const { stream: archiveStream } = __classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f");
+        if (!archiveStream) {
+            throw new Error('Archive stream is unavailable');
+        }
+        return new stream_1.Writable({
+            objectMode: true,
+            write(data, _encoding, callback) {
+                const entryPath = path_1.default.join('assets', 'uploads', data.filename);
+                const entry = archiveStream.entry({
+                    name: entryPath,
+                    size: data.stats.size,
+                });
+                if (!entry) {
+                    callback(new Error(`Failed to created a tar entry for ${entryPath}`));
+                    return;
+                }
+                data.stream.pipe(entry);
+                entry
+                    .on('finish', () => {
+                    callback(null);
+                })
+                    .on('error', (error) => {
+                    callback(error);
+                });
+            },
+        });
+    }
+}
+_LocalFileDestinationProvider_providersMetadata = new WeakMap(), _LocalFileDestinationProvider_archive = new WeakMap(), _LocalFileDestinationProvider_instances = new WeakSet(), _LocalFileDestinationProvider_archivePath_get = function _LocalFileDestinationProvider_archivePath_get() {
+    const { encryption, compression, file } = this.options;
+    let filePath = `${file.path}.tar`;
+    if (compression.enabled) {
+        filePath += '.gz';
+    }
+    if (encryption.enabled) {
+        filePath += '.enc';
+    }
+    return filePath;
+}, _LocalFileDestinationProvider_writeMetadata = async function _LocalFileDestinationProvider_writeMetadata() {
+    const metadata = __classPrivateFieldGet(this, _LocalFileDestinationProvider_providersMetadata, "f").source;
+    if (metadata) {
+        await new Promise((resolve) => {
+            const outStream = __classPrivateFieldGet(this, _LocalFileDestinationProvider_instances, "m", _LocalFileDestinationProvider_getMetadataStream).call(this);
+            const data = JSON.stringify(metadata, null, 2);
+            stream_1.Readable.from(data).pipe(outStream).on('close', resolve);
+        });
+    }
+}, _LocalFileDestinationProvider_getMetadataStream = function _LocalFileDestinationProvider_getMetadataStream() {
+    const { stream } = __classPrivateFieldGet(this, _LocalFileDestinationProvider_archive, "f");
+    if (!stream) {
+        throw new Error('Archive stream is unavailable');
+    }
+    return (0, utils_1.createTarEntryStream)(stream, () => 'metadata.json');
+};
+//# sourceMappingURL=index.js.map

package/lib/file/providers/destination/utils.d.ts

@@ -0,0 +1,9 @@
+/// <reference types="node" />
+import { Writable } from 'stream';
+import tar from 'tar-stream';
+/**
+ * Create a file path factory for a given path & prefix.
+ * Upon being called, the factory will return a file path for a given index
+ */
+export declare const createFilePathFactory: (type: string) => (fileIndex?: number) => string;
+export declare const createTarEntryStream: (archive: tar.Pack, pathFactory: (index?: number | undefined) => string, maxSize?: number) => Writable;

package/lib/file/providers/destination/utils.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTarEntryStream = exports.createFilePathFactory = void 0;
+const stream_1 = require("stream");
+const path_1 = require("path");
+/**
+ * Create a file path factory for a given path & prefix.
+ * Upon being called, the factory will return a file path for a given index
+ */
+const createFilePathFactory = (type) => (fileIndex = 0) => {
+    return (0, path_1.join)(
+    // "{type}" directory
+    type, 
+    // "${type}_XXXXX.jsonl" file
+    `${type}_${String(fileIndex).padStart(5, '0')}.jsonl`);
+};
+exports.createFilePathFactory = createFilePathFactory;
+const createTarEntryStream = (archive, pathFactory, maxSize = 2.56e8) => {
+    let fileIndex = 0;
+    let buffer = '';
+    const flush = async () => {
+        if (!buffer) {
+            return;
+        }
+        fileIndex += 1;
+        const name = pathFactory(fileIndex);
+        const size = buffer.length;
+        await new Promise((resolve, reject) => {
+            archive.entry({ name, size }, buffer, (err) => {
+                if (err) {
+                    reject(err);
+                }
+                resolve();
+            });
+        });
+        buffer = '';
+    };
+    const push = (chunk) => {
+        buffer += chunk;
+    };
+    return new stream_1.Writable({
+        async destroy(err, callback) {
+            await flush();
+            callback(err);
+        },
+        async write(chunk, _encoding, callback) {
+            const size = chunk.length;
+            if (chunk.length > maxSize) {
+                callback(new Error(`payload too large: ${chunk.length}>${maxSize}`));
+                return;
+            }
+            if (buffer.length + size > maxSize) {
+                await flush();
+            }
+            push(chunk);
+            callback(null);
+        },
+    });
+};
+exports.createTarEntryStream = createTarEntryStream;
+//# sourceMappingURL=utils.js.map

package/lib/file/providers/index.d.ts

@@ -0,0 +1,2 @@
+export * from './source';
+export * from './destination';

package/lib/file/providers/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./source"), exports);
+__exportStar(require("./destination"), exports);
+//# sourceMappingURL=index.js.map

package/lib/file/providers/source/index.d.ts

@@ -0,0 +1,40 @@
+/// <reference types="lodash" />
+/// <reference types="node" />
+import type { Readable } from 'stream';
+import type { Schema } from '@strapi/strapi';
+import type { IMetadata, ISourceProvider, ProviderType } from '../../../../types';
+/**
+ * Provider options
+ */
+export interface ILocalFileSourceProviderOptions {
+    file: {
+        path: string;
+    };
+    encryption: {
+        enabled: boolean;
+        key?: string;
+    };
+    compression: {
+        enabled: boolean;
+    };
+}
+export declare const createLocalFileSourceProvider: (options: ILocalFileSourceProviderOptions) => LocalFileSourceProvider;
+declare class LocalFileSourceProvider implements ISourceProvider {
+    #private;
+    type: ProviderType;
+    name: string;
+    options: ILocalFileSourceProviderOptions;
+    constructor(options: ILocalFileSourceProviderOptions);
+    /**
+     * Pre flight checks regarding the provided options, making sure that the file can be opened (decrypted, decompressed), etc.
+     */
+    bootstrap(): Promise<void>;
+    getMetadata(): Promise<IMetadata>;
+    getSchemas(): Promise<import("lodash").Dictionary<Schema>>;
+    createEntitiesReadStream(): Readable;
+    createSchemasReadStream(): Readable;
+    createLinksReadStream(): Readable;
+    createConfigurationReadStream(): Readable;
+    createAssetsReadStream(): Readable | Promise<Readable>;
+}
+export {};