@strapi/data-transfer 4.7.0-beta.0 → 4.7.0-exp.117579f4c13806c2cd518e7d7d2f9d0c8a20107d

package/lib/engine/index.js

@@ -230,7 +230,7 @@ class TransferEngine {
         const transform = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_createStageTransformStream).call(this, stage);
         const tracker = __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_progressTracker).call(this, stage, {
             size: (value) => value.stats.size,
-            key: (value) => (0, path_1.extname)(value.filename) ?? 'NA',
+            key: (value) => (0, path_1.extname)(value.filename) || 'No extension',
         });
         await __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_transferStage).call(this, { stage, source, destination, transform, tracker });
     }
@@ -288,7 +288,7 @@ _TransferEngine_metadata = new WeakMap(), _TransferEngine_instances = new WeakSe
     return stream;
 }, _TransferEngine_updateTransferProgress = function _TransferEngine_updateTransferProgress(stage, data, aggregate) {
     if (!this.progress.data[stage]) {
-        this.progress.data[stage] = { count: 0, bytes: 0 };
+        this.progress.data[stage] = { count: 0, bytes: 0, startTime: Date.now() };
     }
     const stageProgress = this.progress.data[stage];
     if (!stageProgress) {
@@ -416,6 +416,12 @@ _TransferEngine_metadata = new WeakMap(), _TransferEngine_instances = new WeakSe
     }
 }, _TransferEngine_transferStage = async function _TransferEngine_transferStage(options) {
     const { stage, source, destination, transform, tracker } = options;
+    const updateEndTime = () => {
+        const stageData = this.progress.data[stage];
+        if (stageData) {
+            stageData.endTime = Date.now();
+        }
+    };
     if (!source || !destination || this.shouldSkipStage(stage)) {
         // Wait until source and destination are closed
         const results = await Promise.allSettled([source, destination].map((stream) => {
@@ -448,11 +454,15 @@ _TransferEngine_metadata = new WeakMap(), _TransferEngine_instances = new WeakSe
         stream
             .pipe(destination)
             .on('error', (e) => {
+            updateEndTime();
             __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_reportError).call(this, e, 'error');
             destination.destroy(e);
             reject(e);
         })
-            .on('close', resolve);
+            .on('close', () => {
+            updateEndTime();
+            resolve();
+        });
     });
     __classPrivateFieldGet(this, _TransferEngine_instances, "m", _TransferEngine_emitStageUpdate).call(this, 'finish', stage);
 }, _TransferEngine_resolveProviderResource = async function _TransferEngine_resolveProviderResource() {

package/lib/errors/constants.d.ts

@@ -1,3 +1,3 @@
 import { ErrorDiagnosticSeverity } from '../engine/diagnostic';
 export declare const SeverityKind: Record<string, ErrorDiagnosticSeverity>;
-export declare type Severity = typeof SeverityKind[keyof typeof SeverityKind];
+export declare type Severity = (typeof SeverityKind)[keyof typeof SeverityKind];

package/lib/strapi/providers/local-destination/index.d.ts

@@ -21,7 +21,7 @@ declare class LocalStrapiDestinationProvider implements IDestinationProvider {
     constructor(options: ILocalStrapiDestinationProviderOptions);
     bootstrap(): Promise<void>;
     close(): Promise<void>;
-    rollback(): void;
+    rollback(): Promise<void>;
     beforeTransfer(): Promise<void>;
     getMetadata(): IMetadata;
     getSchemas(): import("lodash").Dictionary<Partial<import("@strapi/strapi").Schema>>;

package/lib/strapi/providers/local-destination/index.js

@@ -73,8 +73,8 @@ class LocalStrapiDestinationProvider {
             await this.strapi?.destroy();
         }
     }
-    rollback() {
-        this.transaction?.rollback();
+    async rollback() {
+        await this.transaction?.rollback();
     }
     async beforeTransfer() {
         if (!this.strapi) {
@@ -151,7 +151,9 @@ class LocalStrapiDestinationProvider {
                 const writableStream = fse.createWriteStream(entryPath);
                 chunk.stream
                     .pipe(writableStream)
-                    .on('close', callback)
+                    .on('close', () => {
+                    callback(null);
+                })
                     .on('error', async (error) => {
                     const errorMessage = error.code === 'ENOSPC'
                         ? " Your server doesn't have space to proceed with the import. "

package/lib/strapi/providers/remote-destination/index.d.ts

@@ -19,12 +19,14 @@ declare class RemoteStrapiDestinationProvider implements IDestinationProvider {
     options: IRemoteStrapiDestinationProviderOptions;
     ws: WebSocket | null;
     dispatcher: ReturnType<typeof createDispatcher> | null;
+    transferID: string | null;
     constructor(options: IRemoteStrapiDestinationProviderOptions);
     initTransfer(): Promise<string>;
     bootstrap(): Promise<void>;
     close(): Promise<void>;
     getMetadata(): Promise<IMetadata | null> | null;
     beforeTransfer(): Promise<void>;
+    rollback(): Promise<void>;
     getSchemas(): Promise<Strapi.Schemas | null>;
     createEntitiesWriteStream(): Writable;
     createLinksWriteStream(): Writable;

package/lib/strapi/providers/remote-destination/index.js

@@ -4,15 +4,17 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
     if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
     return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
 };
-var _RemoteStrapiDestinationProvider_instances, _RemoteStrapiDestinationProvider_streamStep;
+var _RemoteStrapiDestinationProvider_instances, _RemoteStrapiDestinationProvider_startStepOnce, _RemoteStrapiDestinationProvider_startStep, _RemoteStrapiDestinationProvider_endStep, _RemoteStrapiDestinationProvider_streamStep, _RemoteStrapiDestinationProvider_writeStream;
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createRemoteStrapiDestinationProvider = void 0;
 const ws_1 = require("ws");
 const uuid_1 = require("uuid");
 const stream_1 = require("stream");
+const fp_1 = require("lodash/fp");
 const utils_1 = require("./utils");
 const constants_1 = require("../../remote/constants");
 const providers_1 = require("../../../errors/providers");
+const jsonLength = (obj) => Buffer.byteLength(JSON.stringify(obj));
 class RemoteStrapiDestinationProvider {
     constructor(options) {
         _RemoteStrapiDestinationProvider_instances.add(this);
@@ -21,6 +23,7 @@ class RemoteStrapiDestinationProvider {
         this.options = options;
         this.ws = null;
         this.dispatcher = null;
+        this.transferID = null;
     }
     async initTransfer() {
         const { strategy, restore } = this.options;
@@ -28,17 +31,24 @@ class RemoteStrapiDestinationProvider {
         return new Promise((resolve, reject) => {
             this.ws
                 ?.once('open', async () => {
-                const query = this.dispatcher?.dispatchCommand({
-                    command: 'init',
-                    params: { options: { strategy, restore }, transfer: 'push' },
-                });
-                const res = (await query);
-                if (!res?.transferID) {
-                    return reject(new providers_1.ProviderTransferError('Init failed, invalid response from the server'));
+                try {
+                    const query = this.dispatcher?.dispatchCommand({
+                        command: 'init',
+                        params: { options: { strategy, restore }, transfer: 'push' },
+                    });
+                    const res = (await query);
+                    if (!res?.transferID) {
+                        throw new providers_1.ProviderTransferError('Init failed, invalid response from the server');
+                    }
+                    resolve(res.transferID);
+                }
+                catch (e) {
+                    reject(e);
                 }
-                resolve(res.transferID);
             })
-                .once('error', reject);
+                .once('error', (message) => {
+                reject(message);
+            });
         });
     }
     async bootstrap() {
@@ -76,12 +86,19 @@ class RemoteStrapiDestinationProvider {
         }
         this.ws = ws;
         this.dispatcher = (0, utils_1.createDispatcher)(this.ws);
-        const transferID = await this.initTransfer();
-        this.dispatcher.setTransferProperties({ id: transferID, kind: 'push' });
+        this.transferID = await this.initTransfer();
+        this.dispatcher.setTransferProperties({ id: this.transferID, kind: 'push' });
         await this.dispatcher.dispatchTransferAction('bootstrap');
     }
     async close() {
-        await this.dispatcher?.dispatchTransferAction('close');
+        // Gracefully close the remote transfer process
+        if (this.transferID && this.dispatcher) {
+            await this.dispatcher.dispatchTransferAction('close');
+            await this.dispatcher.dispatchCommand({
+                command: 'end',
+                params: { transferID: this.transferID },
+            });
+        }
         await new Promise((resolve) => {
             const { ws } = this;
             if (!ws || ws.CLOSED) {
@@ -97,6 +114,9 @@ class RemoteStrapiDestinationProvider {
     async beforeTransfer() {
         await this.dispatcher?.dispatchTransferAction('beforeTransfer');
     }
+    async rollback() {
+        await this.dispatcher?.dispatchTransferAction('rollback');
+    }
     getSchemas() {
         if (!this.dispatcher) {
             return Promise.resolve(null);
@@ -104,65 +124,96 @@ class RemoteStrapiDestinationProvider {
         return this.dispatcher.dispatchTransferAction('getSchemas');
     }
     createEntitiesWriteStream() {
-        return new stream_1.Writable({
-            objectMode: true,
-            write: async (entity, _encoding, callback) => {
-                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'entities', entity);
-                callback(e);
-            },
-        });
+        return __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_writeStream).call(this, 'entities');
     }
     createLinksWriteStream() {
-        return new stream_1.Writable({
-            objectMode: true,
-            write: async (link, _encoding, callback) => {
-                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'links', link);
-                callback(e);
-            },
-        });
+        return __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_writeStream).call(this, 'links');
     }
     createConfigurationWriteStream() {
-        return new stream_1.Writable({
-            objectMode: true,
-            write: async (configuration, _encoding, callback) => {
-                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'configuration', configuration);
-                callback(e);
-            },
-        });
+        return __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_writeStream).call(this, 'configuration');
     }
     createAssetsWriteStream() {
+        let batch = [];
+        let hasStarted = false;
+        const batchSize = 1024 * 1024; // 1MB;
+        const batchLength = () => {
+            return batch.reduce((acc, chunk) => (chunk.action === 'stream' ? acc + chunk.data.byteLength : acc), 0);
+        };
+        const startAssetsTransferOnce = __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_startStepOnce).call(this, 'assets');
+        const flush = async () => {
+            await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', batch);
+            batch = [];
+        };
+        const safePush = async (chunk) => {
+            batch.push(chunk);
+            if (batchLength() >= batchSize) {
+                await flush();
+            }
+        };
         return new stream_1.Writable({
             objectMode: true,
             final: async (callback) => {
-                // TODO: replace this stream call by an end call
-                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', null);
-                callback(e);
+                if (batch.length > 0) {
+                    await flush();
+                }
+                if (hasStarted) {
+                    await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', null);
+                    const endStepError = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_endStep).call(this, 'assets');
+                    if (endStepError) {
+                        return callback(endStepError);
+                    }
+                }
+                return callback(null);
             },
-            write: async (asset, _encoding, callback) => {
-                const { filename, filepath, stats, stream } = asset;
+            async write(asset, _encoding, callback) {
+                const startError = await startAssetsTransferOnce();
+                if (startError) {
+                    return callback(startError);
+                }
+                hasStarted = true;
                 const assetID = (0, uuid_1.v4)();
-                await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', {
-                    action: 'start',
-                    assetID,
-                    data: { filename, filepath, stats },
-                });
+                const { filename, filepath, stats, stream } = asset;
+                await safePush({ action: 'start', assetID, data: { filename, filepath, stats } });
                 for await (const chunk of stream) {
-                    await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', {
-                        action: 'stream',
-                        assetID,
-                        data: chunk,
-                    });
+                    await safePush({ action: 'stream', assetID, data: chunk });
                 }
-                await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', {
-                    action: 'end',
-                    assetID,
-                });
+                await safePush({ action: 'end', assetID });
                 callback();
             },
         });
     }
 }
-_RemoteStrapiDestinationProvider_instances = new WeakSet(), _RemoteStrapiDestinationProvider_streamStep = async function _RemoteStrapiDestinationProvider_streamStep(step, data) {
+_RemoteStrapiDestinationProvider_instances = new WeakSet(), _RemoteStrapiDestinationProvider_startStepOnce = function _RemoteStrapiDestinationProvider_startStepOnce(stage) {
+    return (0, fp_1.once)(() => __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_startStep).call(this, stage));
+}, _RemoteStrapiDestinationProvider_startStep = async function _RemoteStrapiDestinationProvider_startStep(step) {
+    try {
+        await this.dispatcher?.dispatchTransferStep({ action: 'start', step });
+    }
+    catch (e) {
+        if (e instanceof Error) {
+            return e;
+        }
+        if (typeof e === 'string') {
+            return new providers_1.ProviderTransferError(e);
+        }
+        return new providers_1.ProviderTransferError('Unexpected error');
+    }
+    return null;
+}, _RemoteStrapiDestinationProvider_endStep = async function _RemoteStrapiDestinationProvider_endStep(step) {
+    try {
+        await this.dispatcher?.dispatchTransferStep({ action: 'end', step });
+    }
+    catch (e) {
+        if (e instanceof Error) {
+            return e;
+        }
+        if (typeof e === 'string') {
+            return new providers_1.ProviderTransferError(e);
+        }
+        return new providers_1.ProviderTransferError('Unexpected error');
+    }
+    return null;
+}, _RemoteStrapiDestinationProvider_streamStep = async function _RemoteStrapiDestinationProvider_streamStep(step, data) {
     try {
         await this.dispatcher?.dispatchTransferStep({ action: 'stream', step, data });
     }
@@ -176,6 +227,40 @@ _RemoteStrapiDestinationProvider_instances = new WeakSet(), _RemoteStrapiDestina
         return new providers_1.ProviderTransferError('Unexpected error');
     }
     return null;
+}, _RemoteStrapiDestinationProvider_writeStream = function _RemoteStrapiDestinationProvider_writeStream(step) {
+    const batchSize = 1024 * 1024; // 1MB;
+    const startTransferOnce = __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_startStepOnce).call(this, step);
+    let batch = [];
+    const batchLength = () => jsonLength(batch);
+    return new stream_1.Writable({
+        objectMode: true,
+        final: async (callback) => {
+            if (batch.length > 0) {
+                const streamError = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, step, batch);
+                batch = [];
+                if (streamError) {
+                    return callback(streamError);
+                }
+            }
+            const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_endStep).call(this, step);
+            callback(e);
+        },
+        write: async (chunk, _encoding, callback) => {
+            const startError = await startTransferOnce();
+            if (startError) {
+                return callback(startError);
+            }
+            batch.push(chunk);
+            if (batchLength() >= batchSize) {
+                const streamError = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, step, batch);
+                batch = [];
+                if (streamError) {
+                    return callback(streamError);
+                }
+            }
+            callback();
+        },
+    });
 };
 const createRemoteStrapiDestinationProvider = (options) => {
     return new RemoteStrapiDestinationProvider(options);

package/lib/strapi/providers/remote-destination/utils.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createDispatcher = void 0;
 const uuid_1 = require("uuid");
+const providers_1 = require("../../../errors/providers");
 const createDispatcher = (ws) => {
     const state = {};
     const dispatch = async (message, options = {}) => {
@@ -24,7 +25,7 @@ const createDispatcher = (ws) => {
                 const response = JSON.parse(raw.toString());
                 if (response.uuid === uuid) {
                     if (response.error) {
-                        return reject(new Error(response.error.message));
+                        return reject(new providers_1.ProviderError('error', response.error.message));
                     }
                     resolve(response.data ?? null);
                 }
@@ -32,7 +33,6 @@ const createDispatcher = (ws) => {
                     ws.once('message', onResponse);
                 }
             };
-            // TODO: What happens if the server sends another message (not a response to this message)
             ws.once('message', onResponse);
         });
     };

package/lib/strapi/remote/controllers/push.d.ts

@@ -13,6 +13,7 @@ export interface IPushController {
         bootstrap(): Promise<void>;
         close(): Promise<void>;
         beforeTransfer(): Promise<void>;
+        rollback(): Promise<void>;
     };
     transfer: {
         [key in TransferPushStep]: <T extends TransferPushMessage>(value: T extends {

package/lib/strapi/remote/controllers/push.js

@@ -34,58 +34,79 @@ const createPushController = (options) => {
             async beforeTransfer() {
                 return provider.beforeTransfer();
             },
+            async rollback() {
+                await provider.rollback();
+            },
         },
         transfer: {
-            async entities(entity) {
+            async entities(entities) {
                 if (!streams.entities) {
                     streams.entities = provider.createEntitiesWriteStream();
                 }
-                await writeAsync(streams.entities, entity);
+                for (const entity of entities) {
+                    if (streams.entities) {
+                        await writeAsync(streams.entities, entity);
+                    }
+                }
             },
-            async links(link) {
+            async links(links) {
                 if (!streams.links) {
                     streams.links = await provider.createLinksWriteStream();
                 }
-                await writeAsync(streams.links, link);
+                for (const link of links) {
+                    if (streams.links) {
+                        await writeAsync(streams.links, link);
+                    }
+                }
             },
-            async configuration(config) {
+            async configuration(configs) {
                 if (!streams.configuration) {
                     streams.configuration = await provider.createConfigurationWriteStream();
                 }
-                await writeAsync(streams.configuration, config);
+                for (const config of configs) {
+                    if (streams.configuration) {
+                        await writeAsync(streams.configuration, config);
+                    }
+                }
             },
-            async assets(payload) {
-                // TODO: close the stream upong receiving an 'end' event instead
-                if (payload === null) {
+            async assets(payloads) {
+                if (payloads === null) {
                     streams.assets?.end();
                     return;
                 }
-                const { action, assetID } = payload;
                 if (!streams.assets) {
                     streams.assets = await provider.createAssetsWriteStream();
                 }
-                if (action === 'start') {
-                    assets[assetID] = { ...payload.data, stream: new stream_chain_1.PassThrough() };
-                    writeAsync(streams.assets, assets[assetID]);
-                }
-                if (action === 'stream') {
-                    // The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array }
-                    // We need to transform it back into a Buffer instance
-                    const rawBuffer = payload.data;
-                    const chunk = Buffer.from(rawBuffer.data);
-                    await writeAsync(assets[assetID].stream, chunk);
-                }
-                if (action === 'end') {
-                    await new Promise((resolve, reject) => {
-                        const { stream } = assets[assetID];
-                        stream
-                            .on('close', () => {
-                            delete assets[assetID];
-                            resolve();
-                        })
-                            .on('error', reject)
-                            .end();
-                    });
+                for (const payload of payloads) {
+                    if (streams.assets.closed) {
+                        return;
+                    }
+                    const { action, assetID } = payload;
+                    if (action === 'start' && streams.assets) {
+                        assets[assetID] = { ...payload.data, stream: new stream_chain_1.PassThrough() };
+                        writeAsync(streams.assets, assets[assetID]);
+                    }
+                    if (action === 'stream') {
+                        // The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array }
+                        // We need to transform it back into a Buffer instance
+                        const rawBuffer = payload.data;
+                        const chunk = Buffer.from(rawBuffer.data);
+                        await writeAsync(assets[assetID].stream, chunk);
+                    }
+                    if (action === 'end') {
+                        await new Promise((resolve, reject) => {
+                            const { stream } = assets[assetID];
+                            stream
+                                .on('close', () => {
+                                delete assets[assetID];
+                                resolve();
+                            })
+                                .on('error', (e) => {
+                                reject(e);
+                            })
+                                .end();
+                        });
+                    }
                 }
             },
         },

package/lib/strapi/remote/flows/default.d.ts

@@ -0,0 +1,3 @@
+import type { Step } from '.';
+declare const _default: readonly Step[];
+export default _default;

package/lib/strapi/remote/flows/default.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = [
+    {
+        kind: 'action',
+        action: 'bootstrap',
+    },
+    {
+        kind: 'action',
+        action: 'init',
+    },
+    {
+        kind: 'action',
+        action: 'beforeTransfer',
+    },
+    {
+        kind: 'transfer',
+        stage: 'schemas',
+    },
+    {
+        kind: 'transfer',
+        stage: 'entities',
+    },
+    {
+        kind: 'transfer',
+        stage: 'assets',
+    },
+    {
+        kind: 'transfer',
+        stage: 'links',
+    },
+    {
+        kind: 'transfer',
+        stage: 'configuration',
+    },
+    {
+        kind: 'action',
+        action: 'close',
+    },
+];
+//# sourceMappingURL=default.js.map

package/lib/strapi/remote/flows/index.d.ts

@@ -0,0 +1,18 @@
+import type { TransferStage } from '../../../../types';
+export declare type Step = {
+    kind: 'action';
+    action: string;
+} | {
+    kind: 'transfer';
+    stage: TransferStage;
+    locked?: boolean;
+};
+export { default as DEFAULT_TRANSFER_FLOW } from './default';
+export declare const createFlow: (flow: readonly Step[]) => {
+    has(step: Step): boolean;
+    can(step: Step): boolean;
+    cannot(step: Step): boolean;
+    set(step: Step): any;
+    get(): Step | null;
+};
+export declare type TransferFlow = ReturnType<typeof createFlow>;

package/lib/strapi/remote/flows/index.js

@@ -0,0 +1,59 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createFlow = exports.DEFAULT_TRANSFER_FLOW = void 0;
+var default_1 = require("./default");
+Object.defineProperty(exports, "DEFAULT_TRANSFER_FLOW", { enumerable: true, get: function () { return __importDefault(default_1).default; } });
+const createFlow = (flow) => {
+    const state = { step: null };
+    /**
+     * Equality check between two steps
+     */
+    const stepEqual = (stepA, stepB) => {
+        if (stepA.kind === 'action' && stepB.kind === 'action') {
+            return stepA.action === stepB.action;
+        }
+        if (stepA.kind === 'transfer' && stepB.kind === 'transfer') {
+            return stepA.stage === stepB.stage;
+        }
+        return false;
+    };
+    /**
+     * Find the index for a given step
+     */
+    const findStepIndex = (step) => flow.findIndex((flowStep) => stepEqual(step, flowStep));
+    return {
+        has(step) {
+            return findStepIndex(step) !== -1;
+        },
+        can(step) {
+            if (state.step === null) {
+                return true;
+            }
+            const indexesDifference = findStepIndex(step) - findStepIndex(state.step);
+            // It's possible to send multiple time the same transfer step in a row
+            if (indexesDifference === 0 && step.kind === 'transfer') {
+                return true;
+            }
+            return indexesDifference > 0;
+        },
+        cannot(step) {
+            return !this.can(step);
+        },
+        set(step) {
+            const canSwitch = this.can(step);
+            if (!canSwitch) {
+                throw new Error('Impossible to proceed to the given step');
+            }
+            state.step = step;
+            return this;
+        },
+        get() {
+            return state.step;
+        },
+    };
+};
+exports.createFlow = createFlow;
+//# sourceMappingURL=index.js.map

package/lib/strapi/remote/handlers.d.ts

@@ -1,7 +1,7 @@
 import type { Context } from 'koa';
 import type { ServerOptions } from 'ws';
 import { TRANSFER_METHODS } from './constants';
-declare type TransferMethod = typeof TRANSFER_METHODS[number];
+declare type TransferMethod = (typeof TRANSFER_METHODS)[number];
 interface IHandlerOptions {
     verify: (ctx: Context, scope?: TransferMethod) => Promise<void>;
     server?: ServerOptions;

package/lib/strapi/remote/handlers.js

@@ -9,6 +9,7 @@ const ws_1 = require("ws");
 const push_1 = __importDefault(require("./controllers/push"));
 const providers_1 = require("../../errors/providers");
 const constants_1 = require("./constants");
+const flows_1 = require("./flows");
 const createTransferHandler = (options) => {
     const { verify, server: serverOptions } = options;
     // Create the websocket server
@@ -24,6 +25,12 @@ const createTransferHandler = (options) => {
                 wss.emit('connection', ws, ctx.req);
                 const state = {};
                 let uuid;
+                function assertValidTransfer(transferState) {
+                    const { transfer, controller } = transferState;
+                    if (!controller || !transfer) {
+                        throw new providers_1.ProviderTransferError('Invalid transfer process');
+                    }
+                }
                 /**
                  * Format error & message to follow the remote transfer protocol
                  */
@@ -68,15 +75,28 @@ const createTransferHandler = (options) => {
                         }
                     }
                 };
-                const teardown = async () => {
-                    await verifyAuth(state.transfer?.kind);
+                const cleanup = () => {
                     delete state.controller;
                     delete state.transfer;
+                };
+                const teardown = async () => {
+                    if (state.controller) {
+                        await state.controller.actions.rollback();
+                    }
+                    cleanup();
+                };
+                const end = async (msg) => {
+                    await verifyAuth(state.transfer?.kind);
+                    if (msg.params.transferID !== state.transfer?.id) {
+                        throw new providers_1.ProviderTransferError('Bad transfer ID provided');
+                    }
+                    cleanup();
                     return { ok: true };
                 };
                 const init = async (msg) => {
-                    // TODO: this only checks for this instance of node: we should consider a database lock
-                    if (state.controller) {
+                    // TODO: For push transfer, we'll probably have to trigger a
+                    // maintenance mode to prevent other transfer at the same time.
+                    if (state.transfer || state.controller) {
                         throw new providers_1.ProviderInitializationError('Transfer already in progres');
                     }
                     const { transfer } = msg.params;
@@ -97,9 +117,27 @@ const createTransferHandler = (options) => {
                             validTransfers: constants_1.TRANSFER_METHODS,
                         });
                     }
-                    state.transfer = { id: (0, crypto_1.randomUUID)(), kind: transfer };
+                    state.transfer = {
+                        id: (0, crypto_1.randomUUID)(),
+                        kind: transfer,
+                        startedAt: Date.now(),
+                        flow: (0, flows_1.createFlow)(flows_1.DEFAULT_TRANSFER_FLOW),
+                    };
                     return { transferID: state.transfer.id };
                 };
+                const status = () => {
+                    if (state.transfer) {
+                        const { transfer } = state;
+                        const elapsed = Date.now() - transfer.startedAt;
+                        return {
+                            active: true,
+                            kind: transfer.kind,
+                            startedAt: transfer.startedAt,
+                            elapsed,
+                        };
+                    }
+                    return { active: false, kind: null, elapsed: null, startedAt: null };
+                };
                 /**
                  * On command message (init, end, status, ...)
                  */
@@ -109,23 +147,24 @@ const createTransferHandler = (options) => {
                         await answer(() => init(msg));
                     }
                     if (command === 'end') {
-                        await answer(teardown);
+                        await answer(() => {
+                            assertValidTransfer(state);
+                            end(msg);
+                        });
                     }
                     if (command === 'status') {
-                        await callback(new providers_1.ProviderTransferError('Command not implemented: "status"', {
-                            command,
-                            validCommands: ['init', 'end', 'status'],
-                        }));
+                        await answer(status);
                     }
                 };
                 const onTransferCommand = async (msg) => {
+                    assertValidTransfer(state);
                     const { transferID, kind } = msg;
-                    const { controller } = state;
-                    await verifyAuth(state.transfer?.kind);
+                    const { controller, transfer } = state;
+                    await verifyAuth(transfer.kind);
                     // TODO: (re)move this check
                     // It shouldn't be possible to start a pull transfer for now, so reaching
                     // this code should be impossible too, but this has been added by security
-                    if (state.transfer?.kind === 'pull') {
+                    if (transfer.kind === 'pull') {
                         return callback(new providers_1.ProviderTransferError('Pull transfer not implemented'));
                     }
                     if (!controller) {
@@ -143,51 +182,98 @@ const createTransferHandler = (options) => {
                                 validActions: Object.keys(controller.actions),
                             }));
                         }
+                        const step = { kind: 'action', action };
+                        const isStepRegistered = transfer.flow.has(step);
+                        if (isStepRegistered) {
+                            if (transfer.flow.cannot(step)) {
+                                return callback(new providers_1.ProviderTransferError(`Invalid action "${action}" found for the current flow `, {
+                                    action,
+                                }));
+                            }
+                            transfer.flow.set(step);
+                        }
                         await answer(() => controller.actions[action]());
                     }
                     // Transfer
                     else if (kind === 'step') {
                         // We can only have push transfer message for the moment
                         const message = msg;
-                        // TODO: lock transfer process
+                        const currentStep = transfer.flow.get();
+                        const step = { kind: 'transfer', stage: message.step };
+                        // Lock the current transfer stage
                         if (message.action === 'start') {
-                            // console.log('Starting transfer for ', message.step);
+                            if (currentStep?.kind === 'transfer' && currentStep.locked) {
+                                return callback(new providers_1.ProviderTransferError(`It's not possible to start a new transfer stage (${message.step}) while another one is in progress (${currentStep.stage})`));
+                            }
+                            if (transfer.flow.cannot(step)) {
+                                return callback(new providers_1.ProviderTransferError(`Invalid stage (${message.step}) provided for the current flow`, { step }));
+                            }
+                            transfer?.flow.set({ ...step, locked: true });
+                            return callback(null, { ok: true });
                         }
-                        // Stream step
-                        else if (message.action === 'stream') {
+                        // Stream operation on the current transfer stage
+                        if (message.action === 'stream') {
+                            if (currentStep?.kind === 'transfer' && !currentStep.locked) {
+                                return callback(new providers_1.ProviderTransferError(`You need to initialize the transfer stage (${message.step}) before starting to stream data`));
+                            }
+                            if (transfer?.flow.cannot(step)) {
+                                return callback(new providers_1.ProviderTransferError(`Invalid stage (${message.step}) provided for the current flow`, { step }));
+                            }
                             await answer(() => controller.transfer[message.step]?.(message.data));
                         }
-                        // TODO: unlock transfer process
-                        else if (message.action === 'end') {
-                            // console.log('Ending transfer for ', message.step);
+                        // Unlock the current transfer stage
+                        if (message.action === 'end') {
+                            // Cannot unlock if not locked (aka: started)
+                            if (currentStep?.kind === 'transfer' && !currentStep.locked) {
+                                return callback(new providers_1.ProviderTransferError(`You need to initialize the transfer stage (${message.step}) before ending it`));
+                            }
+                            // Cannot unlock if invalid step provided
+                            if (transfer?.flow.cannot(step)) {
+                                return callback(new providers_1.ProviderTransferError(`Invalid stage (${message.step}) provided for the current flow`, { step }));
+                            }
+                            transfer?.flow.set({ ...step, locked: false });
+                            return callback(null, { ok: true });
                         }
                     }
                 };
-                ws.on('close', () => {
-                    teardown();
+                ws.on('close', async () => {
+                    await teardown();
                 });
-                ws.on('error', (e) => {
-                    teardown();
+                ws.on('error', async (e) => {
+                    await teardown();
                     strapi.log.error(e);
                 });
                 ws.on('message', async (raw) => {
-                    const msg = JSON.parse(raw.toString());
-                    if (!msg.uuid) {
-                        await callback(new providers_1.ProviderTransferError('Missing uuid in message'));
-                        return;
-                    }
-                    uuid = msg.uuid;
-                    // Regular command message (init, end, status)
-                    if (msg.type === 'command') {
-                        await onCommand(msg);
-                    }
-                    // Transfer message (the transfer must be initialized first)
-                    else if (msg.type === 'transfer') {
-                        await onTransferCommand(msg);
+                    try {
+                        const msg = JSON.parse(raw.toString());
+                        if (!msg.uuid) {
+                            await callback(new providers_1.ProviderTransferError('Missing uuid in message'));
+                            return;
+                        }
+                        uuid = msg.uuid;
+                        // Regular command message (init, end, status)
+                        if (msg.type === 'command') {
+                            await onCommand(msg);
+                        }
+                        // Transfer message (the transfer must be initialized first)
+                        else if (msg.type === 'transfer') {
+                            await onTransferCommand(msg);
+                        }
+                        // Invalid messages
+                        else {
+                            await callback(new providers_1.ProviderTransferError('Bad request'));
+                        }
                     }
-                    // Invalid messages
-                    else {
-                        await callback(new providers_1.ProviderTransferError('Bad request'));
+                    catch (e) {
+                        // Only known errors should be returned to client
+                        if (e instanceof providers_1.ProviderError || e instanceof SyntaxError) {
+                            await callback(e);
+                        }
+                        else {
+                            // TODO: log error to server?
+                            // Unknown errors should not be sent to client
+                            await callback(new providers_1.ProviderTransferError('Unknown transfer error'));
+                        }
                     }
                 });
             });

package/lib/utils/transaction.js

@@ -13,12 +13,26 @@ const createTransaction = (strapi) => {
         resume?.();
     });
     e.on('close', () => {
+        e.removeAllListeners('rollback');
+        e.removeAllListeners('spawn');
         done = true;
         resume?.();
     });
     strapi.db.transaction(async ({ trx, rollback }) => {
-        e.on('rollback', async () => {
-            await rollback();
+        e.once('rollback', async () => {
+            e.removeAllListeners('close');
+            e.removeAllListeners('spawn');
+            try {
+                await rollback();
+                e.emit('rollback_completed');
+            }
+            catch {
+                e.emit('rollback_failed');
+            }
+            finally {
+                done = true;
+                resume?.();
+            }
         });
         while (!done) {
             while (fns.length) {
@@ -62,7 +76,11 @@ const createTransaction = (strapi) => {
             return e.emit('close');
         },
         rollback() {
-            return e.emit('rollback');
+            return new Promise((resolve) => {
+                e.emit('rollback');
+                e.once('rollback_failed', () => resolve(false));
+                e.once('rollback_completed', () => resolve(true));
+            });
         },
     };
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@strapi/data-transfer",
-  "version": "4.7.0-beta.0",
+  "version": "4.7.0-exp.117579f4c13806c2cd518e7d7d2f9d0c8a20107d",
   "description": "Data transfer capabilities for Strapi",
   "keywords": [
     "strapi",
@@ -39,12 +39,12 @@
     "lib": "./lib"
   },
   "dependencies": {
-    "@strapi/logger": "4.7.0-beta.0",
-    "@strapi/strapi": "4.7.0-beta.0",
+    "@strapi/logger": "4.7.0-exp.117579f4c13806c2cd518e7d7d2f9d0c8a20107d",
+    "@strapi/strapi": "4.7.0-exp.117579f4c13806c2cd518e7d7d2f9d0c8a20107d",
     "chalk": "4.1.2",
     "fs-extra": "10.0.0",
     "lodash": "4.17.21",
-    "prettier": "2.7.1",
+    "prettier": "2.8.4",
     "semver": "7.3.8",
     "stream-chain": "2.2.5",
     "stream-json": "1.7.4",
@@ -73,5 +73,5 @@
     "node": ">=14.19.1 <=18.x.x",
     "npm": ">=6.0.0"
   },
-  "gitHead": "880ba7af867ad43c4cc45b47467b76a9b5606b6e"
+  "gitHead": "117579f4c13806c2cd518e7d7d2f9d0c8a20107d"
 }