@strapi/data-transfer 4.6.0-beta.1 → 4.6.0-beta.2

package/lib/strapi/providers/remote-destination/index.js

@@ -0,0 +1,171 @@
+"use strict";
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var _RemoteStrapiDestinationProvider_instances, _RemoteStrapiDestinationProvider_streamStep;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createRemoteStrapiDestinationProvider = void 0;
+const ws_1 = require("ws");
+const uuid_1 = require("uuid");
+const stream_1 = require("stream");
+const utils_1 = require("./utils");
+const constants_1 = require("../../remote/constants");
+class RemoteStrapiDestinationProvider {
+    constructor(options) {
+        _RemoteStrapiDestinationProvider_instances.add(this);
+        this.name = 'destination::remote-strapi';
+        this.type = 'destination';
+        this.options = options;
+        this.ws = null;
+        this.dispatcher = null;
+    }
+    async initTransfer() {
+        const { strategy, restore } = this.options;
+        // Wait for the connection to be made to the server, then init the transfer
+        return new Promise((resolve, reject) => {
+            this.ws
+                ?.once('open', async () => {
+                const query = this.dispatcher?.dispatchCommand({
+                    command: 'init',
+                    params: { options: { strategy, restore }, transfer: 'push' },
+                });
+                const res = (await query);
+                if (!res?.transferID) {
+                    return reject(new Error('Init failed, invalid response from the server'));
+                }
+                resolve(res.transferID);
+            })
+                .once('error', reject);
+        });
+    }
+    async bootstrap() {
+        const { url, auth } = this.options;
+        let ws;
+        if (!['https:', 'http:'].includes(url.protocol)) {
+            throw new Error(`Invalid protocol "${url.protocol}"`);
+        }
+        const wsProtocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
+        const wsUrl = `${wsProtocol}//${url.host}${url.pathname}${constants_1.TRANSFER_PATH}`;
+        // No auth defined, trying public access for transfer
+        if (!auth) {
+            ws = new ws_1.WebSocket(wsUrl);
+        }
+        // Common token auth, this should be the main auth method
+        else if (auth.type === 'token') {
+            const headers = { Authorization: `Bearer ${auth.token}` };
+            ws = new ws_1.WebSocket(wsUrl, { headers });
+        }
+        // Invalid auth method provided
+        else {
+            throw new Error('Auth method not implemented');
+        }
+        this.ws = ws;
+        this.dispatcher = (0, utils_1.createDispatcher)(this.ws);
+        const transferID = await this.initTransfer();
+        this.dispatcher.setTransferProperties({ id: transferID, kind: 'push' });
+        await this.dispatcher.dispatchTransferAction('bootstrap');
+    }
+    async close() {
+        await this.dispatcher?.dispatchTransferAction('close');
+        await new Promise((resolve) => {
+            const { ws } = this;
+            if (!ws || ws.CLOSED) {
+                resolve();
+                return;
+            }
+            ws.on('close', () => resolve()).close();
+        });
+    }
+    getMetadata() {
+        return this.dispatcher?.dispatchTransferAction('getMetadata') ?? null;
+    }
+    async beforeTransfer() {
+        await this.dispatcher?.dispatchTransferAction('beforeTransfer');
+    }
+    getSchemas() {
+        if (!this.dispatcher) {
+            return Promise.resolve(null);
+        }
+        return this.dispatcher.dispatchTransferAction('getSchemas');
+    }
+    createEntitiesWriteStream() {
+        return new stream_1.Writable({
+            objectMode: true,
+            write: async (entity, _encoding, callback) => {
+                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'entities', entity);
+                callback(e);
+            },
+        });
+    }
+    createLinksWriteStream() {
+        return new stream_1.Writable({
+            objectMode: true,
+            write: async (link, _encoding, callback) => {
+                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'links', link);
+                callback(e);
+            },
+        });
+    }
+    createConfigurationWriteStream() {
+        return new stream_1.Writable({
+            objectMode: true,
+            write: async (configuration, _encoding, callback) => {
+                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'configuration', configuration);
+                callback(e);
+            },
+        });
+    }
+    createAssetsWriteStream() {
+        return new stream_1.Writable({
+            objectMode: true,
+            final: async (callback) => {
+                // TODO: replace this stream call by an end call
+                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', null);
+                callback(e);
+            },
+            write: async (asset, _encoding, callback) => {
+                const { filename, filepath, stats, stream } = asset;
+                const assetID = (0, uuid_1.v4)();
+                await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', {
+                    action: 'start',
+                    assetID,
+                    data: { filename, filepath, stats },
+                });
+                for await (const chunk of stream) {
+                    await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', {
+                        action: 'stream',
+                        assetID,
+                        data: chunk,
+                    });
+                }
+                await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', {
+                    action: 'end',
+                    assetID,
+                });
+                callback();
+            },
+        });
+    }
+}
+_RemoteStrapiDestinationProvider_instances = new WeakSet(), _RemoteStrapiDestinationProvider_streamStep = async function _RemoteStrapiDestinationProvider_streamStep(step, data) {
+    try {
+        await this.dispatcher?.dispatchTransferStep({ action: 'stream', step, data });
+    }
+    catch (e) {
+        if (e instanceof Error) {
+            return e;
+        }
+        if (typeof e === 'string') {
+            return new Error(e);
+        }
+        return new Error('Unexpected error');
+    }
+    return null;
+};
+const createRemoteStrapiDestinationProvider = (options) => {
+    return new RemoteStrapiDestinationProvider(options);
+};
+exports.createRemoteStrapiDestinationProvider = createRemoteStrapiDestinationProvider;
+//# sourceMappingURL=index.js.map

package/lib/strapi/providers/remote-destination/utils.d.ts

@@ -0,0 +1,31 @@
+import { WebSocket } from 'ws';
+import type { client } from '../../../../types/remote/protocol';
+interface IDispatcherState {
+    transfer?: {
+        kind: client.TransferKind;
+        id: string;
+    };
+}
+interface IDispatchOptions {
+    attachTransfer?: boolean;
+}
+declare type Dispatch<T> = Omit<T, 'transferID' | 'uuid'>;
+declare const createDispatcher: (ws: WebSocket) => {
+    readonly transferID: string | undefined;
+    readonly transferKind: "push" | "pull" | undefined;
+    setTransferProperties: (properties: Exclude<IDispatcherState['transfer'], undefined>) => void;
+    dispatch: <U = null>(message: Dispatch<client.Message>, options?: IDispatchOptions) => Promise<U | null>;
+    dispatchCommand: <U_1 extends "end" | "init" | "status">(payload: {
+        command: U_1;
+    } & ([client.GetCommandParams<U_1>] extends [never] ? unknown : {
+        params: client.GetCommandParams<U_1>;
+    })) => Promise<null>;
+    dispatchTransferAction: <T>(action: client.Action['action']) => Promise<T | null>;
+    dispatchTransferStep: <T_1, A extends "stream" | "end" | "start" = "stream" | "end" | "start", S extends "entities" | "links" | "configuration" | "assets" = "entities" | "links" | "configuration" | "assets">(payload: {
+        step: S;
+        action: A;
+    } & (A extends "stream" ? {
+        data: client.GetTransferPushStreamData<S>;
+    } : unknown)) => Promise<T_1 | null>;
+};
+export { createDispatcher };

package/lib/strapi/providers/remote-destination/utils.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDispatcher = void 0;
+const uuid_1 = require("uuid");
+const createDispatcher = (ws) => {
+    const state = {};
+    const dispatch = async (message, options = {}) => {
+        if (!ws) {
+            throw new Error('No websocket connection found');
+        }
+        return new Promise((resolve, reject) => {
+            const uuid = (0, uuid_1.v4)();
+            const payload = { ...message, uuid };
+            if (options.attachTransfer) {
+                Object.assign(payload, { transferID: state.transfer?.id });
+            }
+            const stringifiedPayload = JSON.stringify(payload);
+            ws.send(stringifiedPayload, (error) => {
+                if (error) {
+                    reject(error);
+                }
+            });
+            const onResponse = (raw) => {
+                const response = JSON.parse(raw.toString());
+                if (response.uuid === uuid) {
+                    if (response.error) {
+                        return reject(new Error(response.error.message));
+                    }
+                    resolve(response.data ?? null);
+                }
+                else {
+                    ws.once('message', onResponse);
+                }
+            };
+            // TODO: What happens if the server sends another message (not a response to this message)
+            ws.once('message', onResponse);
+        });
+    };
+    const dispatchCommand = (payload) => {
+        return dispatch({ type: 'command', ...payload });
+    };
+    const dispatchTransferAction = async (action) => {
+        const payload = { type: 'transfer', kind: 'action', action };
+        return dispatch(payload, { attachTransfer: true }) ?? Promise.resolve(null);
+    };
+    const dispatchTransferStep = async (payload) => {
+        const message = {
+            type: 'transfer',
+            kind: 'step',
+            ...payload,
+        };
+        return dispatch(message, { attachTransfer: true }) ?? Promise.resolve(null);
+    };
+    const setTransferProperties = (properties) => {
+        state.transfer = { ...properties };
+    };
+    return {
+        get transferID() {
+            return state.transfer?.id;
+        },
+        get transferKind() {
+            return state.transfer?.kind;
+        },
+        setTransferProperties,
+        dispatch,
+        dispatchCommand,
+        dispatchTransferAction,
+        dispatchTransferStep,
+    };
+};
+exports.createDispatcher = createDispatcher;
+//# sourceMappingURL=utils.js.map

package/{dist/providers/shared/strapi → lib/strapi/queries}/link.d.ts

@@ -1,4 +1,4 @@
-import { ILink } from '../../../../types';
+import { ILink } from '../../../types';
 export declare const createLinkQuery: (strapi: Strapi.Strapi) => () => {
     generateAll: (uid: string) => AsyncGenerator<ILink>;
     generateAllForAttribute: (uid: string, fieldName: string) => AsyncGenerator<ILink>;

package/lib/strapi/register.d.ts

@@ -0,0 +1,7 @@
+/**
+ * This is intended to be called on Strapi register phase.
+ *
+ * It registers a transfer route in the Strapi admin router.
+ */
+declare const register: (strapi: Strapi.Strapi) => void;
+export default register;

package/lib/strapi/register.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const remote_1 = require("./remote");
+/**
+ * This is intended to be called on Strapi register phase.
+ *
+ * It registers a transfer route in the Strapi admin router.
+ */
+const register = (strapi) => {
+    remote_1.routes.registerAdminTransferRoute(strapi);
+};
+exports.default = register;
+//# sourceMappingURL=register.js.map

package/lib/strapi/remote/constants.d.ts

@@ -0,0 +1 @@
+export declare const TRANSFER_PATH = "/transfer";

package/lib/strapi/remote/constants.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TRANSFER_PATH = void 0;
+exports.TRANSFER_PATH = '/transfer';
+//# sourceMappingURL=constants.js.map

package/lib/strapi/remote/controllers/index.d.ts

@@ -0,0 +1 @@
+export * from './push';

package/{dist → lib/strapi/remote/controllers}/index.js

@@ -14,6 +14,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./engine"), exports);
-__exportStar(require("./providers"), exports);
+__exportStar(require("./push"), exports);
 //# sourceMappingURL=index.js.map

package/lib/strapi/remote/controllers/push.d.ts

@@ -0,0 +1,25 @@
+/// <reference types="node" />
+import { Writable } from 'stream-chain';
+import type { IMetadata } from '../../../../types';
+import type { TransferPushMessage, TransferPushStep } from '../../../../types/remote/protocol/client';
+import { ILocalStrapiDestinationProviderOptions } from '../../providers';
+export interface IPushController {
+    streams: {
+        [stage in TransferPushStep]?: Writable;
+    };
+    actions: {
+        getMetadata(): Promise<IMetadata>;
+        getSchemas(): Strapi.Schemas;
+        bootstrap(): Promise<void>;
+        close(): Promise<void>;
+        beforeTransfer(): Promise<void>;
+    };
+    transfer: {
+        [key in TransferPushStep]: <T extends TransferPushMessage>(value: T extends {
+            step: key;
+            data: infer U;
+        } ? U : never) => Promise<void>;
+    };
+}
+declare const createPushController: (options: ILocalStrapiDestinationProviderOptions) => IPushController;
+export default createPushController;

package/lib/strapi/remote/controllers/push.js

@@ -0,0 +1,95 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const stream_chain_1 = require("stream-chain");
+const providers_1 = require("../../providers");
+const createPushController = (options) => {
+    const provider = (0, providers_1.createLocalStrapiDestinationProvider)(options);
+    const streams = {};
+    const assets = {};
+    const writeAsync = (stream, data) => {
+        return new Promise((resolve, reject) => {
+            stream.write(data, (error) => {
+                if (error) {
+                    reject(error);
+                }
+                resolve();
+            });
+        });
+    };
+    return {
+        streams,
+        actions: {
+            async getSchemas() {
+                return provider.getSchemas();
+            },
+            async getMetadata() {
+                return provider.getMetadata();
+            },
+            async bootstrap() {
+                return provider.bootstrap();
+            },
+            async close() {
+                return provider.close();
+            },
+            async beforeTransfer() {
+                return provider.beforeTransfer();
+            },
+        },
+        transfer: {
+            async entities(entity) {
+                if (!streams.entities) {
+                    streams.entities = provider.createEntitiesWriteStream();
+                }
+                await writeAsync(streams.entities, entity);
+            },
+            async links(link) {
+                if (!streams.links) {
+                    streams.links = await provider.createLinksWriteStream();
+                }
+                await writeAsync(streams.links, link);
+            },
+            async configuration(config) {
+                if (!streams.configuration) {
+                    streams.configuration = await provider.createConfigurationWriteStream();
+                }
+                await writeAsync(streams.configuration, config);
+            },
+            async assets(payload) {
+                // TODO: close the stream upong receiving an 'end' event instead
+                if (payload === null) {
+                    streams.assets?.end();
+                    return;
+                }
+                const { action, assetID } = payload;
+                if (!streams.assets) {
+                    streams.assets = await provider.createAssetsWriteStream();
+                }
+                if (action === 'start') {
+                    assets[assetID] = { ...payload.data, stream: new stream_chain_1.PassThrough() };
+                    writeAsync(streams.assets, assets[assetID]);
+                }
+                if (action === 'stream') {
+                    // The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array }
+                    // We need to transform it back into a Buffer instance
+                    const rawBuffer = payload.data;
+                    const chunk = Buffer.from(rawBuffer.data);
+                    await writeAsync(assets[assetID].stream, chunk);
+                }
+                if (action === 'end') {
+                    await new Promise((resolve, reject) => {
+                        const { stream } = assets[assetID];
+                        stream
+                            .on('close', () => {
+                            delete assets[assetID];
+                            resolve();
+                        })
+                            .on('error', reject)
+                            .end();
+                    });
+                }
+            },
+        },
+    };
+};
+exports.default = createPushController;
+//# sourceMappingURL=push.js.map

package/lib/strapi/remote/handlers.d.ts

@@ -0,0 +1,3 @@
+import type { Context } from 'koa';
+import type { ServerOptions } from 'ws';
+export declare const createTransferHandler: (options?: ServerOptions) => (ctx: Context) => Promise<void>;

package/lib/strapi/remote/handlers.js

@@ -0,0 +1,177 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createTransferHandler = void 0;
+const crypto_1 = require("crypto");
+const ws_1 = require("ws");
+const push_1 = __importDefault(require("./controllers/push"));
+const createTransferHandler = (options = {}) => async (ctx) => {
+    const upgradeHeader = (ctx.request.headers.upgrade || '')
+        .split(',')
+        .map((s) => s.trim().toLowerCase());
+    // Create the websocket server
+    const wss = new ws_1.WebSocket.Server({ ...options, noServer: true });
+    if (upgradeHeader.includes('websocket')) {
+        wss.handleUpgrade(ctx.req, ctx.request.socket, Buffer.alloc(0), (ws) => {
+            // Create a connection between the client & the server
+            wss.emit('connection', ws, ctx.req);
+            const state = {};
+            let uuid;
+            /**
+             * Format error & message to follow the remote transfer protocol
+             */
+            const callback = (e = null, data) => {
+                return new Promise((resolve, reject) => {
+                    if (!uuid) {
+                        reject(new Error('Missing uuid for this message'));
+                        return;
+                    }
+                    const payload = JSON.stringify({
+                        uuid,
+                        data: data ?? null,
+                        error: e
+                            ? {
+                                code: 'ERR',
+                                message: e?.message,
+                            }
+                            : null,
+                    });
+                    ws.send(payload, (error) => (error ? reject(error) : resolve()));
+                });
+            };
+            /**
+             * Wrap a function call to catch errors and answer the request with the correct format
+             */
+            const answer = async (fn) => {
+                try {
+                    const response = await fn();
+                    callback(null, response);
+                }
+                catch (e) {
+                    if (e instanceof Error) {
+                        callback(e);
+                    }
+                    else if (typeof e === 'string') {
+                        callback(new Error(e));
+                    }
+                    else {
+                        callback(new Error('Unexpected error'));
+                    }
+                }
+            };
+            const teardown = () => {
+                delete state.controller;
+                delete state.transfer;
+                return { ok: true };
+            };
+            const init = (msg) => {
+                if (state.controller) {
+                    throw new Error('Transfer already in progres');
+                }
+                const { transfer } = msg.params;
+                // Push transfer
+                if (transfer === 'push') {
+                    const { options: controllerOptions } = msg.params;
+                    state.controller = (0, push_1.default)({
+                        ...controllerOptions,
+                        autoDestroy: false,
+                        getStrapi: () => strapi,
+                    });
+                }
+                // Pull or any other string
+                else {
+                    throw new Error(`Transfer not implemented: "${transfer}"`);
+                }
+                state.transfer = { id: (0, crypto_1.randomUUID)(), kind: transfer };
+                return { transferID: state.transfer.id };
+            };
+            /**
+             * On command message (init, end, status, ...)
+             */
+            const onCommand = async (msg) => {
+                const { command } = msg;
+                if (command === 'init') {
+                    await answer(() => init(msg));
+                }
+                if (command === 'end') {
+                    await answer(teardown);
+                }
+                if (command === 'status') {
+                    await callback(new Error('Command not implemented: "status"'));
+                }
+            };
+            const onTransferCommand = async (msg) => {
+                const { transferID, kind } = msg;
+                const { controller } = state;
+                // TODO: (re)move this check
+                // It shouldn't be possible to strart a pull transfer for now, so reaching
+                // this code should be impossible too, but this has been added by security
+                if (state.transfer?.kind === 'pull') {
+                    return callback(new Error('Pull transfer not implemented'));
+                }
+                if (!controller) {
+                    return callback(new Error("The transfer hasn't been initialized"));
+                }
+                if (!transferID) {
+                    return callback(new Error('Missing transfer ID'));
+                }
+                // Action
+                if (kind === 'action') {
+                    const { action } = msg;
+                    if (!(action in controller.actions)) {
+                        return callback(new Error(`Invalid action provided: "${action}"`));
+                    }
+                    await answer(() => controller.actions[action]());
+                }
+                // Transfer
+                else if (kind === 'step') {
+                    // We can only have push transfer message for the moment
+                    const message = msg;
+                    // TODO: lock transfer process
+                    if (message.action === 'start') {
+                        // console.log('Starting transfer for ', message.step);
+                    }
+                    // Stream step
+                    else if (message.action === 'stream') {
+                        await answer(() => controller.transfer[message.step]?.(message.data));
+                    }
+                    // TODO: unlock transfer process
+                    else if (message.action === 'end') {
+                        // console.log('Ending transfer for ', message.step);
+                    }
+                }
+            };
+            ws.on('close', () => {
+                teardown();
+            });
+            ws.on('error', (e) => {
+                teardown();
+                console.error(e);
+            });
+            ws.on('message', async (raw) => {
+                const msg = JSON.parse(raw.toString());
+                if (!msg.uuid) {
+                    throw new Error('Missing uuid in message');
+                }
+                uuid = msg.uuid;
+                // Regular command message (init, end, status)
+                if (msg.type === 'command') {
+                    await onCommand(msg);
+                }
+                // Transfer message (the transfer must be initialized first)
+                else if (msg.type === 'transfer') {
+                    await onTransferCommand(msg);
+                }
+                // Invalid messages
+                else {
+                    await callback(new Error('Bad request'));
+                }
+            });
+        });
+        ctx.respond = false;
+    }
+};
+exports.createTransferHandler = createTransferHandler;
+//# sourceMappingURL=handlers.js.map

package/lib/strapi/remote/index.d.ts

@@ -0,0 +1,3 @@
+export * as controllers from './controllers';
+export * as routes from './routes';
+export * as constants from './constants';