@strapi/data-transfer 4.9.0-alpha.0 → 4.9.0-beta.2

package/lib/strapi/remote/handlers/utils.js

@@ -0,0 +1,181 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handlerControllerFactory = exports.handleWSUpgrade = exports.isDataTransferMessage = exports.assertValidHeader = exports.transformUpgradeHeader = void 0;
+const crypto_1 = require("crypto");
+const ws_1 = require("ws");
+const providers_1 = require("../../../errors/providers");
+const constants_1 = require("./constants");
+const transformUpgradeHeader = (header = '') => {
+    return header.split(',').map((s) => s.trim().toLowerCase());
+};
+exports.transformUpgradeHeader = transformUpgradeHeader;
+/**
+ * Make sure that the upgrade header is a valid websocket one
+ */
+const assertValidHeader = (ctx) => {
+    const upgradeHeader = (0, exports.transformUpgradeHeader)(ctx.headers.upgrade);
+    if (!upgradeHeader.includes('websocket')) {
+        throw new Error('Invalid Header');
+    }
+};
+exports.assertValidHeader = assertValidHeader;
+const isDataTransferMessage = (message) => {
+    if (!message || typeof message !== 'object') {
+        return false;
+    }
+    const { uuid, type } = message;
+    if (typeof uuid !== 'string' || typeof type !== 'string') {
+        return false;
+    }
+    if (!['command', 'transfer'].includes(type)) {
+        return false;
+    }
+    return true;
+};
+exports.isDataTransferMessage = isDataTransferMessage;
+/**
+ * Handle the upgrade to ws connection
+ */
+const handleWSUpgrade = (wss, ctx, callback) => {
+    (0, exports.assertValidHeader)(ctx);
+    wss.handleUpgrade(ctx.req, ctx.request.socket, Buffer.alloc(0), (client, request) => {
+        // Create a connection between the client & the server
+        wss.emit('connection', client, ctx.req);
+        // Invoke the ws callback
+        callback(client, request);
+    });
+    ctx.respond = false;
+};
+exports.handleWSUpgrade = handleWSUpgrade;
+// Protocol related functions
+const handlerControllerFactory = (implementation) => (options) => {
+    const { verify, server: serverOptions } = options ?? {};
+    const wss = new ws_1.WebSocket.Server({ ...serverOptions, noServer: true });
+    return async (ctx) => {
+        (0, exports.handleWSUpgrade)(wss, ctx, (ws) => {
+            const state = { id: undefined };
+            const prototype = {
+                // Transfer ID
+                get transferID() {
+                    return state.id;
+                },
+                set transferID(id) {
+                    state.id = id;
+                },
+                // Started at
+                get startedAt() {
+                    return state.startedAt;
+                },
+                set startedAt(timestamp) {
+                    state.startedAt = timestamp;
+                },
+                isTransferStarted() {
+                    return this.transferID !== undefined && this.startedAt !== undefined;
+                },
+                assertValidTransfer() {
+                    const isStarted = this.isTransferStarted();
+                    if (!isStarted) {
+                        throw new Error('Invalid Transfer Process');
+                    }
+                },
+                assertValidTransferCommand(command) {
+                    const isDefined = typeof this[command] === 'function';
+                    const isValidTransferCommand = constants_1.VALID_TRANSFER_COMMANDS.includes(command);
+                    if (!isDefined || !isValidTransferCommand) {
+                        throw new Error('Invalid transfer command');
+                    }
+                },
+                respond(uuid, e, data) {
+                    return new Promise((resolve, reject) => {
+                        if (!uuid && !e) {
+                            reject(new Error('Missing uuid for this message'));
+                            return;
+                        }
+                        const payload = JSON.stringify({
+                            uuid,
+                            data: data ?? null,
+                            error: e
+                                ? {
+                                    code: e?.name ?? 'ERR',
+                                    message: e?.message,
+                                }
+                                : null,
+                        });
+                        this.send(payload, (error) => (error ? reject(error) : resolve()));
+                    });
+                },
+                send(message, cb) {
+                    ws.send(message, cb);
+                },
+                confirm(message) {
+                    return new Promise((resolve, reject) => {
+                        const uuid = (0, crypto_1.randomUUID)();
+                        const payload = JSON.stringify({ uuid, data: message });
+                        this.send(payload, (error) => {
+                            if (error) {
+                                reject(error);
+                            }
+                        });
+                        const onResponse = (raw) => {
+                            const response = JSON.parse(raw.toString());
+                            if (response.uuid === uuid) {
+                                if (response.error) {
+                                    return reject(new Error(response.error.message));
+                                }
+                                resolve(response.data ?? null);
+                            }
+                            else {
+                                ws.once('message', onResponse);
+                            }
+                        };
+                        ws.once('message', onResponse);
+                    });
+                },
+                async executeAndRespond(uuid, fn) {
+                    try {
+                        const response = await fn();
+                        this.respond(uuid, null, response);
+                    }
+                    catch (e) {
+                        if (e instanceof Error) {
+                            this.respond(uuid, e);
+                        }
+                        else if (typeof e === 'string') {
+                            this.respond(uuid, new providers_1.ProviderTransferError(e));
+                        }
+                        else {
+                            this.respond(uuid, new providers_1.ProviderTransferError('Unexpected error', {
+                                error: e,
+                            }));
+                        }
+                    }
+                },
+                cleanup() {
+                    this.transferID = undefined;
+                    this.startedAt = undefined;
+                },
+                teardown() {
+                    this.cleanup();
+                },
+                verifyAuth(scope) {
+                    return verify(ctx, scope);
+                },
+                // Transfer commands
+                init() { },
+                end() { },
+                status() { },
+                // Default prototype implementation for events
+                onMessage() { },
+                onError() { },
+                onClose() { },
+            };
+            const handler = Object.assign(Object.create(prototype), implementation(prototype));
+            // Bind ws events to handler methods
+            ws.on('close', (...args) => handler.onClose(...args));
+            ws.on('error', (...args) => handler.onError(...args));
+            ws.on('message', (...args) => handler.onMessage(...args));
+        });
+    };
+};
+exports.handlerControllerFactory = handlerControllerFactory;
+//# sourceMappingURL=utils.js.map

package/lib/strapi/remote/index.d.ts

@@ -1,3 +1,2 @@
-export * as controllers from './controllers';
-export * as routes from './routes';
 export * as constants from './constants';
+export * as handlers from './handlers';

package/lib/strapi/remote/index.js

@@ -23,8 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.constants = exports.routes = exports.controllers = void 0;
-exports.controllers = __importStar(require("./controllers"));
-exports.routes = __importStar(require("./routes"));
+exports.handlers = exports.constants = void 0;
 exports.constants = __importStar(require("./constants"));
+exports.handlers = __importStar(require("./handlers"));
 //# sourceMappingURL=index.js.map

package/lib/utils/transaction.js

@@ -13,12 +13,26 @@ const createTransaction = (strapi) => {
         resume?.();
     });
     e.on('close', () => {
+        e.removeAllListeners('rollback');
+        e.removeAllListeners('spawn');
         done = true;
         resume?.();
     });
     strapi.db.transaction(async ({ trx, rollback }) => {
-        e.on('rollback', async () => {
-            await rollback();
+        e.once('rollback', async () => {
+            e.removeAllListeners('close');
+            e.removeAllListeners('spawn');
+            try {
+                await rollback();
+                e.emit('rollback_completed');
+            }
+            catch {
+                e.emit('rollback_failed');
+            }
+            finally {
+                done = true;
+                resume?.();
+            }
         });
         while (!done) {
             while (fns.length) {
@@ -62,7 +76,11 @@ const createTransaction = (strapi) => {
             return e.emit('close');
         },
         rollback() {
-            return e.emit('rollback');
+            return new Promise((resolve) => {
+                e.emit('rollback');
+                e.once('rollback_failed', () => resolve(false));
+                e.once('rollback_completed', () => resolve(true));
+            });
         },
     };
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@strapi/data-transfer",
-  "version": "4.9.0-alpha.0",
+  "version": "4.9.0-beta.2",
   "description": "Data transfer capabilities for Strapi",
   "keywords": [
     "strapi",
@@ -39,16 +39,15 @@
     "lib": "./lib"
   },
   "dependencies": {
-    "@strapi/logger": "4.9.0-alpha.0",
-    "@strapi/strapi": "4.9.0-alpha.0",
+    "@strapi/logger": "4.9.0-beta.2",
+    "@strapi/strapi": "4.9.0-beta.2",
     "chalk": "4.1.2",
     "fs-extra": "10.0.0",
     "lodash": "4.17.21",
-    "prettier": "2.7.1",
     "semver": "7.3.8",
     "stream-chain": "2.2.5",
     "stream-json": "1.7.4",
-    "tar": "6.1.12",
+    "tar": "6.1.13",
     "tar-stream": "2.2.0",
     "uuid": "9.0.0",
     "ws": "8.11.0"
@@ -58,12 +57,14 @@
     "@types/fs-extra": "9.0.13",
     "@types/jest": "29.2.0",
     "@types/koa": "2.13.4",
+    "@types/lodash": "^4.14.191",
     "@types/semver": "7.3.13",
     "@types/stream-chain": "2.0.1",
     "@types/stream-json": "1.7.3",
-    "@types/tar": "6.1.3",
+    "@types/tar": "6.1.4",
     "@types/tar-stream": "2.2.2",
     "@types/uuid": "9.0.0",
+    "@types/ws": "^8.5.4",
     "knex": "2.4.0",
     "koa": "2.13.4",
     "rimraf": "3.0.2",
@@ -73,5 +74,5 @@
     "node": ">=14.19.1 <=18.x.x",
     "npm": ">=6.0.0"
   },
-  "gitHead": "35f783d0dc07db101e7e62cb4d682f751551f452"
+  "gitHead": "d893ead642592a15a95fefb71c544aaabe4db20b"
 }

package/lib/strapi/register.d.ts

@@ -1,7 +0,0 @@
-/**
- * This is intended to be called on Strapi register phase.
- *
- * It registers a transfer route in the Strapi admin router.
- */
-declare const register: (strapi: Strapi.Strapi) => void;
-export default register;

package/lib/strapi/register.js

@@ -1,13 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const remote_1 = require("./remote");
-/**
- * This is intended to be called on Strapi register phase.
- *
- * It registers a transfer route in the Strapi admin router.
- */
-const register = (strapi) => {
-    remote_1.routes.registerAdminTransferRoute(strapi);
-};
-exports.default = register;
-//# sourceMappingURL=register.js.map

package/lib/strapi/remote/controllers/index.d.ts

@@ -1 +0,0 @@
-export * from './push';

package/lib/strapi/remote/controllers/index.js

@@ -1,18 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./push"), exports);
-//# sourceMappingURL=index.js.map

package/lib/strapi/remote/controllers/push.d.ts

@@ -1,25 +0,0 @@
-/// <reference types="node" />
-import { Writable } from 'stream-chain';
-import type { IMetadata } from '../../../../types';
-import type { TransferPushMessage, TransferPushStep } from '../../../../types/remote/protocol/client';
-import { ILocalStrapiDestinationProviderOptions } from '../../providers';
-export interface IPushController {
-    streams: {
-        [stage in TransferPushStep]?: Writable;
-    };
-    actions: {
-        getMetadata(): Promise<IMetadata>;
-        getSchemas(): Strapi.Schemas;
-        bootstrap(): Promise<void>;
-        close(): Promise<void>;
-        beforeTransfer(): Promise<void>;
-    };
-    transfer: {
-        [key in TransferPushStep]: <T extends TransferPushMessage>(value: T extends {
-            step: key;
-            data: infer U;
-        } ? U : never) => Promise<void>;
-    };
-}
-declare const createPushController: (options: ILocalStrapiDestinationProviderOptions) => IPushController;
-export default createPushController;

package/lib/strapi/remote/controllers/push.js

@@ -1,95 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const stream_chain_1 = require("stream-chain");
-const providers_1 = require("../../providers");
-const createPushController = (options) => {
-    const provider = (0, providers_1.createLocalStrapiDestinationProvider)(options);
-    const streams = {};
-    const assets = {};
-    const writeAsync = (stream, data) => {
-        return new Promise((resolve, reject) => {
-            stream.write(data, (error) => {
-                if (error) {
-                    reject(error);
-                }
-                resolve();
-            });
-        });
-    };
-    return {
-        streams,
-        actions: {
-            async getSchemas() {
-                return provider.getSchemas();
-            },
-            async getMetadata() {
-                return provider.getMetadata();
-            },
-            async bootstrap() {
-                return provider.bootstrap();
-            },
-            async close() {
-                return provider.close();
-            },
-            async beforeTransfer() {
-                return provider.beforeTransfer();
-            },
-        },
-        transfer: {
-            async entities(entity) {
-                if (!streams.entities) {
-                    streams.entities = provider.createEntitiesWriteStream();
-                }
-                await writeAsync(streams.entities, entity);
-            },
-            async links(link) {
-                if (!streams.links) {
-                    streams.links = await provider.createLinksWriteStream();
-                }
-                await writeAsync(streams.links, link);
-            },
-            async configuration(config) {
-                if (!streams.configuration) {
-                    streams.configuration = await provider.createConfigurationWriteStream();
-                }
-                await writeAsync(streams.configuration, config);
-            },
-            async assets(payload) {
-                // TODO: close the stream upong receiving an 'end' event instead
-                if (payload === null) {
-                    streams.assets?.end();
-                    return;
-                }
-                const { action, assetID } = payload;
-                if (!streams.assets) {
-                    streams.assets = await provider.createAssetsWriteStream();
-                }
-                if (action === 'start') {
-                    assets[assetID] = { ...payload.data, stream: new stream_chain_1.PassThrough() };
-                    writeAsync(streams.assets, assets[assetID]);
-                }
-                if (action === 'stream') {
-                    // The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array }
-                    // We need to transform it back into a Buffer instance
-                    const rawBuffer = payload.data;
-                    const chunk = Buffer.from(rawBuffer.data);
-                    await writeAsync(assets[assetID].stream, chunk);
-                }
-                if (action === 'end') {
-                    await new Promise((resolve, reject) => {
-                        const { stream } = assets[assetID];
-                        stream
-                            .on('close', () => {
-                            delete assets[assetID];
-                            resolve();
-                        })
-                            .on('error', reject)
-                            .end();
-                    });
-                }
-            },
-        },
-    };
-};
-exports.default = createPushController;
-//# sourceMappingURL=push.js.map

package/lib/strapi/remote/handlers.d.ts

@@ -1,3 +0,0 @@
-import type { Context } from 'koa';
-import type { ServerOptions } from 'ws';
-export declare const createTransferHandler: (options?: ServerOptions) => (ctx: Context) => Promise<void>;

package/lib/strapi/remote/handlers.js

@@ -1,193 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createTransferHandler = void 0;
-const crypto_1 = require("crypto");
-const ws_1 = require("ws");
-const push_1 = __importDefault(require("./controllers/push"));
-const providers_1 = require("../../errors/providers");
-const constants_1 = require("./constants");
-const createTransferHandler = (options = {}) => async (ctx) => {
-    const upgradeHeader = (ctx.request.headers.upgrade || '')
-        .split(',')
-        .map((s) => s.trim().toLowerCase());
-    // Create the websocket server
-    const wss = new ws_1.WebSocket.Server({ ...options, noServer: true });
-    if (upgradeHeader.includes('websocket')) {
-        wss.handleUpgrade(ctx.req, ctx.request.socket, Buffer.alloc(0), (ws) => {
-            // Create a connection between the client & the server
-            wss.emit('connection', ws, ctx.req);
-            const state = {};
-            let uuid;
-            /**
-             * Format error & message to follow the remote transfer protocol
-             */
-            const callback = (e = null, data) => {
-                return new Promise((resolve, reject) => {
-                    if (!uuid) {
-                        reject(new Error('Missing uuid for this message'));
-                        return;
-                    }
-                    const payload = JSON.stringify({
-                        uuid,
-                        data: data ?? null,
-                        error: e
-                            ? {
-                                code: 'ERR',
-                                message: e?.message,
-                            }
-                            : null,
-                    });
-                    ws.send(payload, (error) => (error ? reject(error) : resolve()));
-                });
-            };
-            /**
-             * Wrap a function call to catch errors and answer the request with the correct format
-             */
-            const answer = async (fn) => {
-                try {
-                    const response = await fn();
-                    callback(null, response);
-                }
-                catch (e) {
-                    if (e instanceof Error) {
-                        callback(e);
-                    }
-                    else if (typeof e === 'string') {
-                        callback(new providers_1.ProviderTransferError(e));
-                    }
-                    else {
-                        callback(new providers_1.ProviderTransferError('Unexpected error', {
-                            error: e,
-                        }));
-                    }
-                }
-            };
-            const teardown = () => {
-                delete state.controller;
-                delete state.transfer;
-                return { ok: true };
-            };
-            const init = (msg) => {
-                // TODO: this only checks for this instance of node: we should consider a database lock
-                if (state.controller) {
-                    throw new providers_1.ProviderInitializationError('Transfer already in progres');
-                }
-                const { transfer } = msg.params;
-                // Push transfer
-                if (transfer === 'push') {
-                    const { options: controllerOptions } = msg.params;
-                    state.controller = (0, push_1.default)({
-                        ...controllerOptions,
-                        autoDestroy: false,
-                        getStrapi: () => strapi,
-                    });
-                }
-                // Pull or any other string
-                else {
-                    throw new providers_1.ProviderTransferError(`Transfer type not implemented: "${transfer}"`, {
-                        transfer,
-                        validTransfers: constants_1.TRANSFER_METHODS,
-                    });
-                }
-                state.transfer = { id: (0, crypto_1.randomUUID)(), kind: transfer };
-                return { transferID: state.transfer.id };
-            };
-            /**
-             * On command message (init, end, status, ...)
-             */
-            const onCommand = async (msg) => {
-                const { command } = msg;
-                if (command === 'init') {
-                    await answer(() => init(msg));
-                }
-                if (command === 'end') {
-                    await answer(teardown);
-                }
-                if (command === 'status') {
-                    await callback(new providers_1.ProviderTransferError('Command not implemented: "status"', {
-                        command,
-                        validCommands: ['init', 'end', 'status'],
-                    }));
-                }
-            };
-            const onTransferCommand = async (msg) => {
-                const { transferID, kind } = msg;
-                const { controller } = state;
-                // TODO: (re)move this check
-                // It shouldn't be possible to start a pull transfer for now, so reaching
-                // this code should be impossible too, but this has been added by security
-                if (state.transfer?.kind === 'pull') {
-                    return callback(new providers_1.ProviderTransferError('Pull transfer not implemented'));
-                }
-                if (!controller) {
-                    return callback(new providers_1.ProviderTransferError("The transfer hasn't been initialized"));
-                }
-                if (!transferID) {
-                    return callback(new providers_1.ProviderTransferError('Missing transfer ID'));
-                }
-                // Action
-                if (kind === 'action') {
-                    const { action } = msg;
-                    if (!(action in controller.actions)) {
-                        return callback(new providers_1.ProviderTransferError(`Invalid action provided: "${action}"`, {
-                            action,
-                            validActions: Object.keys(controller.actions),
-                        }));
-                    }
-                    await answer(() => controller.actions[action]());
-                }
-                // Transfer
-                else if (kind === 'step') {
-                    // We can only have push transfer message for the moment
-                    const message = msg;
-                    // TODO: lock transfer process
-                    if (message.action === 'start') {
-                        // console.log('Starting transfer for ', message.step);
-                    }
-                    // Stream step
-                    else if (message.action === 'stream') {
-                        await answer(() => controller.transfer[message.step]?.(message.data));
-                    }
-                    // TODO: unlock transfer process
-                    else if (message.action === 'end') {
-                        // console.log('Ending transfer for ', message.step);
-                    }
-                }
-            };
-            ws.on('close', () => {
-                teardown();
-            });
-            ws.on('error', (e) => {
-                teardown();
-                // TODO: is logging a console error to the running instance of Strapi ok to do? Should we check for an existing strapi.logger to use?
-                console.error(e);
-            });
-            ws.on('message', async (raw) => {
-                const msg = JSON.parse(raw.toString());
-                if (!msg.uuid) {
-                    await callback(new providers_1.ProviderTransferError('Missing uuid in message'));
-                    return;
-                }
-                uuid = msg.uuid;
-                // Regular command message (init, end, status)
-                if (msg.type === 'command') {
-                    await onCommand(msg);
-                }
-                // Transfer message (the transfer must be initialized first)
-                else if (msg.type === 'transfer') {
-                    await onTransferCommand(msg);
-                }
-                // Invalid messages
-                else {
-                    await callback(new providers_1.ProviderTransferError('Bad request'));
-                }
-            });
-        });
-        ctx.respond = false;
-    }
-};
-exports.createTransferHandler = createTransferHandler;
-//# sourceMappingURL=handlers.js.map

package/lib/strapi/remote/routes.d.ts

@@ -1,21 +0,0 @@
-import type { Context } from 'koa';
-declare module '@strapi/strapi' {
-    interface Strapi {
-        admin: {
-            routes: {
-                method: string;
-                path: string;
-                handler: (ctx: Context) => Promise<void>;
-                config: unknown;
-            }[];
-        };
-    }
-}
-/**
- * Register a transfer route in the Strapi admin router.
- *
- * It exposes a WS server that can be used to run and manage transfer processes.
- *
- * @param strapi - A Strapi instance
- */
-export declare const registerAdminTransferRoute: (strapi: Strapi.Strapi) => void;