@strapi/data-transfer 4.6.0-beta.1 → 4.6.0

package/lib/strapi/providers/remote-destination/index.d.ts

@@ -0,0 +1,40 @@
+/// <reference types="node" />
+import { WebSocket } from 'ws';
+import { Writable } from 'stream';
+import { createDispatcher } from './utils';
+import type { IDestinationProvider, IMetadata, ProviderType } from '../../../../types';
+import type { ILocalStrapiDestinationProviderOptions } from '../local-destination';
+interface ITokenAuth {
+    type: 'token';
+    token: string;
+}
+interface ICredentialsAuth {
+    type: 'credentials';
+    email: string;
+    password: string;
+}
+export interface IRemoteStrapiDestinationProviderOptions extends Pick<ILocalStrapiDestinationProviderOptions, 'restore' | 'strategy'> {
+    url: URL;
+    auth?: ITokenAuth | ICredentialsAuth;
+}
+declare class RemoteStrapiDestinationProvider implements IDestinationProvider {
+    #private;
+    name: string;
+    type: ProviderType;
+    options: IRemoteStrapiDestinationProviderOptions;
+    ws: WebSocket | null;
+    dispatcher: ReturnType<typeof createDispatcher> | null;
+    constructor(options: IRemoteStrapiDestinationProviderOptions);
+    initTransfer(): Promise<string>;
+    bootstrap(): Promise<void>;
+    close(): Promise<void>;
+    getMetadata(): Promise<IMetadata | null> | null;
+    beforeTransfer(): Promise<void>;
+    getSchemas(): Promise<Strapi.Schemas | null>;
+    createEntitiesWriteStream(): Writable;
+    createLinksWriteStream(): Writable;
+    createConfigurationWriteStream(): Writable;
+    createAssetsWriteStream(): Writable | Promise<Writable>;
+}
+export declare const createRemoteStrapiDestinationProvider: (options: IRemoteStrapiDestinationProviderOptions) => RemoteStrapiDestinationProvider;
+export {};

package/lib/strapi/providers/remote-destination/index.js

@@ -0,0 +1,186 @@
+"use strict";
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var _RemoteStrapiDestinationProvider_instances, _RemoteStrapiDestinationProvider_streamStep;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createRemoteStrapiDestinationProvider = void 0;
+const ws_1 = require("ws");
+const uuid_1 = require("uuid");
+const stream_1 = require("stream");
+const utils_1 = require("./utils");
+const constants_1 = require("../../remote/constants");
+const providers_1 = require("../../../errors/providers");
+class RemoteStrapiDestinationProvider {
+    constructor(options) {
+        _RemoteStrapiDestinationProvider_instances.add(this);
+        this.name = 'destination::remote-strapi';
+        this.type = 'destination';
+        this.options = options;
+        this.ws = null;
+        this.dispatcher = null;
+    }
+    async initTransfer() {
+        const { strategy, restore } = this.options;
+        // Wait for the connection to be made to the server, then init the transfer
+        return new Promise((resolve, reject) => {
+            this.ws
+                ?.once('open', async () => {
+                const query = this.dispatcher?.dispatchCommand({
+                    command: 'init',
+                    params: { options: { strategy, restore }, transfer: 'push' },
+                });
+                const res = (await query);
+                if (!res?.transferID) {
+                    return reject(new providers_1.ProviderTransferError('Init failed, invalid response from the server'));
+                }
+                resolve(res.transferID);
+            })
+                .once('error', reject);
+        });
+    }
+    async bootstrap() {
+        const { url, auth } = this.options;
+        const validProtocols = ['https:', 'http:'];
+        let ws;
+        if (!validProtocols.includes(url.protocol)) {
+            throw new providers_1.ProviderValidationError(`Invalid protocol "${url.protocol}"`, {
+                check: 'url',
+                details: {
+                    protocol: url.protocol,
+                    validProtocols,
+                },
+            });
+        }
+        const wsProtocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
+        const wsUrl = `${wsProtocol}//${url.host}${url.pathname}${constants_1.TRANSFER_PATH}`;
+        const validAuthMethods = ['token'];
+        // No auth defined, trying public access for transfer
+        if (!auth) {
+            ws = new ws_1.WebSocket(wsUrl);
+        }
+        // Common token auth, this should be the main auth method
+        else if (auth.type === 'token') {
+            const headers = { Authorization: `Bearer ${auth.token}` };
+            ws = new ws_1.WebSocket(wsUrl, { headers });
+        }
+        // Invalid auth method provided
+        else {
+            throw new providers_1.ProviderValidationError('Auth method not implemented', {
+                check: 'auth.type',
+                details: {
+                    auth: auth.type,
+                    validAuthMethods,
+                },
+            });
+        }
+        this.ws = ws;
+        this.dispatcher = (0, utils_1.createDispatcher)(this.ws);
+        const transferID = await this.initTransfer();
+        this.dispatcher.setTransferProperties({ id: transferID, kind: 'push' });
+        await this.dispatcher.dispatchTransferAction('bootstrap');
+    }
+    async close() {
+        await this.dispatcher?.dispatchTransferAction('close');
+        await new Promise((resolve) => {
+            const { ws } = this;
+            if (!ws || ws.CLOSED) {
+                resolve();
+                return;
+            }
+            ws.on('close', () => resolve()).close();
+        });
+    }
+    getMetadata() {
+        return this.dispatcher?.dispatchTransferAction('getMetadata') ?? null;
+    }
+    async beforeTransfer() {
+        await this.dispatcher?.dispatchTransferAction('beforeTransfer');
+    }
+    getSchemas() {
+        if (!this.dispatcher) {
+            return Promise.resolve(null);
+        }
+        return this.dispatcher.dispatchTransferAction('getSchemas');
+    }
+    createEntitiesWriteStream() {
+        return new stream_1.Writable({
+            objectMode: true,
+            write: async (entity, _encoding, callback) => {
+                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'entities', entity);
+                callback(e);
+            },
+        });
+    }
+    createLinksWriteStream() {
+        return new stream_1.Writable({
+            objectMode: true,
+            write: async (link, _encoding, callback) => {
+                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'links', link);
+                callback(e);
+            },
+        });
+    }
+    createConfigurationWriteStream() {
+        return new stream_1.Writable({
+            objectMode: true,
+            write: async (configuration, _encoding, callback) => {
+                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'configuration', configuration);
+                callback(e);
+            },
+        });
+    }
+    createAssetsWriteStream() {
+        return new stream_1.Writable({
+            objectMode: true,
+            final: async (callback) => {
+                // TODO: replace this stream call by an end call
+                const e = await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', null);
+                callback(e);
+            },
+            write: async (asset, _encoding, callback) => {
+                const { filename, filepath, stats, stream } = asset;
+                const assetID = (0, uuid_1.v4)();
+                await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', {
+                    action: 'start',
+                    assetID,
+                    data: { filename, filepath, stats },
+                });
+                for await (const chunk of stream) {
+                    await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', {
+                        action: 'stream',
+                        assetID,
+                        data: chunk,
+                    });
+                }
+                await __classPrivateFieldGet(this, _RemoteStrapiDestinationProvider_instances, "m", _RemoteStrapiDestinationProvider_streamStep).call(this, 'assets', {
+                    action: 'end',
+                    assetID,
+                });
+                callback();
+            },
+        });
+    }
+}
+_RemoteStrapiDestinationProvider_instances = new WeakSet(), _RemoteStrapiDestinationProvider_streamStep = async function _RemoteStrapiDestinationProvider_streamStep(step, data) {
+    try {
+        await this.dispatcher?.dispatchTransferStep({ action: 'stream', step, data });
+    }
+    catch (e) {
+        if (e instanceof Error) {
+            return e;
+        }
+        if (typeof e === 'string') {
+            return new providers_1.ProviderTransferError(e);
+        }
+        return new providers_1.ProviderTransferError('Unexpected error');
+    }
+    return null;
+};
+const createRemoteStrapiDestinationProvider = (options) => {
+    return new RemoteStrapiDestinationProvider(options);
+};
+exports.createRemoteStrapiDestinationProvider = createRemoteStrapiDestinationProvider;
+//# sourceMappingURL=index.js.map

package/lib/strapi/providers/remote-destination/utils.d.ts

@@ -0,0 +1,31 @@
+import { WebSocket } from 'ws';
+import type { client } from '../../../../types/remote/protocol';
+interface IDispatcherState {
+    transfer?: {
+        kind: client.TransferKind;
+        id: string;
+    };
+}
+interface IDispatchOptions {
+    attachTransfer?: boolean;
+}
+declare type Dispatch<T> = Omit<T, 'transferID' | 'uuid'>;
+declare const createDispatcher: (ws: WebSocket) => {
+    readonly transferID: string | undefined;
+    readonly transferKind: "push" | "pull" | undefined;
+    setTransferProperties: (properties: Exclude<IDispatcherState['transfer'], undefined>) => void;
+    dispatch: <U = null>(message: Dispatch<client.Message>, options?: IDispatchOptions) => Promise<U | null>;
+    dispatchCommand: <U_1 extends "end" | "init" | "status">(payload: {
+        command: U_1;
+    } & ([client.GetCommandParams<U_1>] extends [never] ? unknown : {
+        params: client.GetCommandParams<U_1>;
+    })) => Promise<null>;
+    dispatchTransferAction: <T>(action: client.Action['action']) => Promise<T | null>;
+    dispatchTransferStep: <T_1, A extends "stream" | "end" | "start" = "stream" | "end" | "start", S extends "entities" | "links" | "configuration" | "assets" = "entities" | "links" | "configuration" | "assets">(payload: {
+        step: S;
+        action: A;
+    } & (A extends "stream" ? {
+        data: client.GetTransferPushStreamData<S>;
+    } : unknown)) => Promise<T_1 | null>;
+};
+export { createDispatcher };

package/lib/strapi/providers/remote-destination/utils.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createDispatcher = void 0;
+const uuid_1 = require("uuid");
+const createDispatcher = (ws) => {
+    const state = {};
+    const dispatch = async (message, options = {}) => {
+        if (!ws) {
+            throw new Error('No websocket connection found');
+        }
+        return new Promise((resolve, reject) => {
+            const uuid = (0, uuid_1.v4)();
+            const payload = { ...message, uuid };
+            if (options.attachTransfer) {
+                Object.assign(payload, { transferID: state.transfer?.id });
+            }
+            const stringifiedPayload = JSON.stringify(payload);
+            ws.send(stringifiedPayload, (error) => {
+                if (error) {
+                    reject(error);
+                }
+            });
+            const onResponse = (raw) => {
+                const response = JSON.parse(raw.toString());
+                if (response.uuid === uuid) {
+                    if (response.error) {
+                        return reject(new Error(response.error.message));
+                    }
+                    resolve(response.data ?? null);
+                }
+                else {
+                    ws.once('message', onResponse);
+                }
+            };
+            // TODO: What happens if the server sends another message (not a response to this message)
+            ws.once('message', onResponse);
+        });
+    };
+    const dispatchCommand = (payload) => {
+        return dispatch({ type: 'command', ...payload });
+    };
+    const dispatchTransferAction = async (action) => {
+        const payload = { type: 'transfer', kind: 'action', action };
+        return dispatch(payload, { attachTransfer: true }) ?? Promise.resolve(null);
+    };
+    const dispatchTransferStep = async (payload) => {
+        const message = {
+            type: 'transfer',
+            kind: 'step',
+            ...payload,
+        };
+        return dispatch(message, { attachTransfer: true }) ?? Promise.resolve(null);
+    };
+    const setTransferProperties = (properties) => {
+        state.transfer = { ...properties };
+    };
+    return {
+        get transferID() {
+            return state.transfer?.id;
+        },
+        get transferKind() {
+            return state.transfer?.kind;
+        },
+        setTransferProperties,
+        dispatch,
+        dispatchCommand,
+        dispatchTransferAction,
+        dispatchTransferStep,
+    };
+};
+exports.createDispatcher = createDispatcher;
+//# sourceMappingURL=utils.js.map

package/{dist/providers/shared/strapi → lib/strapi/queries}/link.d.ts

@@ -1,5 +1,5 @@
-import { ILink } from '../../../../types';
-export declare const createLinkQuery: (strapi: Strapi.Strapi) => () => {
+import { ILink } from '../../../types';
+export declare const createLinkQuery: (strapi: Strapi.Strapi, trx?: any) => () => {
     generateAll: (uid: string) => AsyncGenerator<ILink>;
     generateAllForAttribute: (uid: string, fieldName: string) => AsyncGenerator<ILink>;
     insert: (link: ILink) => Promise<void>;

package/{dist/providers/shared/strapi → lib/strapi/queries}/link.js

@@ -3,7 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.createLinkQuery = void 0;
 const fp_1 = require("lodash/fp");
 // TODO: Remove any types when we'll have types for DB metadata
-const createLinkQuery = (strapi) => {
+const createLinkQuery = (strapi, trx) => {
     const query = () => {
         const { connection } = strapi.db;
         async function* generateAllForAttribute(uid, fieldName) {
@@ -23,6 +23,9 @@ const createLinkQuery = (strapi) => {
             if (attribute.joinColumn) {
                 const joinColumnName = attribute.joinColumn.name;
                 const qb = connection.queryBuilder().select('id', joinColumnName).from(metadata.tableName);
+                if (trx) {
+                    qb.transacting(trx);
+                }
                 // TODO: stream the query to improve performances
                 const entries = await qb;
                 for (const entry of entries) {
@@ -77,6 +80,9 @@ const createLinkQuery = (strapi) => {
                     columns.right.order,
                 ].filter((column) => !(0, fp_1.isNil)(column));
                 qb.select(validColumns);
+                if (trx) {
+                    qb.transacting(trx);
+                }
                 // TODO: stream the query to improve performances
                 const entries = await qb;
                 for (const entry of entries) {
@@ -127,9 +133,13 @@ const createLinkQuery = (strapi) => {
             const payload = {};
             if (attribute.joinColumn) {
                 const joinColumnName = attribute.joinColumn.name;
-                await connection(metadata.tableName)
+                const qb = connection(metadata.tableName)
                     .where('id', left.ref)
                     .update({ [joinColumnName]: right.ref });
+                if (trx) {
+                    qb.transacting(trx);
+                }
+                await qb;
             }
             if (attribute.joinTable) {
                 const { name, joinColumn, inverseJoinColumn, orderColumnName, inverseOrderColumnName, morphColumn, } = attribute.joinTable;
@@ -168,7 +178,11 @@ const createLinkQuery = (strapi) => {
                     assignMorphColumns();
                 }
                 assignOrderColumns();
-                await connection.insert(payload).into(name);
+                const qb = connection.insert(payload).into(name);
+                if (trx) {
+                    qb.transacting(trx);
+                }
+                await qb;
             }
         };
         return { generateAll, generateAllForAttribute, insert };

package/lib/strapi/register.d.ts

@@ -0,0 +1,7 @@
+/**
+ * This is intended to be called on Strapi register phase.
+ *
+ * It registers a transfer route in the Strapi admin router.
+ */
+declare const register: (strapi: Strapi.Strapi) => void;
+export default register;

package/lib/strapi/register.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const remote_1 = require("./remote");
+/**
+ * This is intended to be called on Strapi register phase.
+ *
+ * It registers a transfer route in the Strapi admin router.
+ */
+const register = (strapi) => {
+    remote_1.routes.registerAdminTransferRoute(strapi);
+};
+exports.default = register;
+//# sourceMappingURL=register.js.map

package/lib/strapi/remote/constants.d.ts

@@ -0,0 +1,2 @@
+export declare const TRANSFER_PATH = "/transfer";
+export declare const TRANSFER_METHODS: string[];

package/lib/strapi/remote/constants.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TRANSFER_METHODS = exports.TRANSFER_PATH = void 0;
+exports.TRANSFER_PATH = '/transfer';
+exports.TRANSFER_METHODS = ['push', 'pull'];
+//# sourceMappingURL=constants.js.map

package/lib/strapi/remote/controllers/index.d.ts

@@ -0,0 +1 @@
+export * from './push';

package/lib/strapi/remote/controllers/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./push"), exports);
+//# sourceMappingURL=index.js.map

package/lib/strapi/remote/controllers/push.d.ts

@@ -0,0 +1,25 @@
+/// <reference types="node" />
+import { Writable } from 'stream-chain';
+import type { IMetadata } from '../../../../types';
+import type { TransferPushMessage, TransferPushStep } from '../../../../types/remote/protocol/client';
+import { ILocalStrapiDestinationProviderOptions } from '../../providers';
+export interface IPushController {
+    streams: {
+        [stage in TransferPushStep]?: Writable;
+    };
+    actions: {
+        getMetadata(): Promise<IMetadata>;
+        getSchemas(): Strapi.Schemas;
+        bootstrap(): Promise<void>;
+        close(): Promise<void>;
+        beforeTransfer(): Promise<void>;
+    };
+    transfer: {
+        [key in TransferPushStep]: <T extends TransferPushMessage>(value: T extends {
+            step: key;
+            data: infer U;
+        } ? U : never) => Promise<void>;
+    };
+}
+declare const createPushController: (options: ILocalStrapiDestinationProviderOptions) => IPushController;
+export default createPushController;

package/lib/strapi/remote/controllers/push.js

@@ -0,0 +1,95 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const stream_chain_1 = require("stream-chain");
+const providers_1 = require("../../providers");
+const createPushController = (options) => {
+    const provider = (0, providers_1.createLocalStrapiDestinationProvider)(options);
+    const streams = {};
+    const assets = {};
+    const writeAsync = (stream, data) => {
+        return new Promise((resolve, reject) => {
+            stream.write(data, (error) => {
+                if (error) {
+                    reject(error);
+                }
+                resolve();
+            });
+        });
+    };
+    return {
+        streams,
+        actions: {
+            async getSchemas() {
+                return provider.getSchemas();
+            },
+            async getMetadata() {
+                return provider.getMetadata();
+            },
+            async bootstrap() {
+                return provider.bootstrap();
+            },
+            async close() {
+                return provider.close();
+            },
+            async beforeTransfer() {
+                return provider.beforeTransfer();
+            },
+        },
+        transfer: {
+            async entities(entity) {
+                if (!streams.entities) {
+                    streams.entities = provider.createEntitiesWriteStream();
+                }
+                await writeAsync(streams.entities, entity);
+            },
+            async links(link) {
+                if (!streams.links) {
+                    streams.links = await provider.createLinksWriteStream();
+                }
+                await writeAsync(streams.links, link);
+            },
+            async configuration(config) {
+                if (!streams.configuration) {
+                    streams.configuration = await provider.createConfigurationWriteStream();
+                }
+                await writeAsync(streams.configuration, config);
+            },
+            async assets(payload) {
+                // TODO: close the stream upong receiving an 'end' event instead
+                if (payload === null) {
+                    streams.assets?.end();
+                    return;
+                }
+                const { action, assetID } = payload;
+                if (!streams.assets) {
+                    streams.assets = await provider.createAssetsWriteStream();
+                }
+                if (action === 'start') {
+                    assets[assetID] = { ...payload.data, stream: new stream_chain_1.PassThrough() };
+                    writeAsync(streams.assets, assets[assetID]);
+                }
+                if (action === 'stream') {
+                    // The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array }
+                    // We need to transform it back into a Buffer instance
+                    const rawBuffer = payload.data;
+                    const chunk = Buffer.from(rawBuffer.data);
+                    await writeAsync(assets[assetID].stream, chunk);
+                }
+                if (action === 'end') {
+                    await new Promise((resolve, reject) => {
+                        const { stream } = assets[assetID];
+                        stream
+                            .on('close', () => {
+                            delete assets[assetID];
+                            resolve();
+                        })
+                            .on('error', reject)
+                            .end();
+                    });
+                }
+            },
+        },
+    };
+};
+exports.default = createPushController;
+//# sourceMappingURL=push.js.map

package/lib/strapi/remote/handlers.d.ts

@@ -0,0 +1,3 @@
+import type { Context } from 'koa';
+import type { ServerOptions } from 'ws';
+export declare const createTransferHandler: (options?: ServerOptions) => (ctx: Context) => Promise<void>;