@strapi/data-transfer 4.9.0-alpha.0 → 4.9.0-beta.2

package/lib/strapi/providers/remote-source/index.d.ts

@@ -0,0 +1,36 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+import { WebSocket } from 'ws';
+import type { IMetadata, ISourceProvider, ISourceProviderTransferResults, MaybePromise, ProviderType } from '../../../../types';
+import { ILocalStrapiSourceProviderOptions } from '../local-source';
+import { createDispatcher } from '../utils';
+interface ITransferTokenAuth {
+    type: 'token';
+    token: string;
+}
+export interface IRemoteStrapiSourceProviderOptions extends ILocalStrapiSourceProviderOptions {
+    url: URL;
+    auth?: ITransferTokenAuth;
+}
+declare class RemoteStrapiSourceProvider implements ISourceProvider {
+    #private;
+    name: string;
+    type: ProviderType;
+    options: IRemoteStrapiSourceProviderOptions;
+    ws: WebSocket | null;
+    dispatcher: ReturnType<typeof createDispatcher> | null;
+    constructor(options: IRemoteStrapiSourceProviderOptions);
+    results?: ISourceProviderTransferResults | undefined;
+    createEntitiesReadStream(): MaybePromise<Readable>;
+    createLinksReadStream(): MaybePromise<Readable>;
+    createAssetsReadStream(): Promise<Readable>;
+    createConfigurationReadStream(): MaybePromise<Readable>;
+    getMetadata(): Promise<IMetadata | null>;
+    assertValidProtocol(url: URL): void;
+    initTransfer(): Promise<string>;
+    bootstrap(): Promise<void>;
+    close(): Promise<void>;
+    getSchemas(): Promise<Strapi.Schemas | null>;
+}
+export declare const createRemoteStrapiSourceProvider: (options: IRemoteStrapiSourceProviderOptions) => RemoteStrapiSourceProvider;
+export {};

package/lib/strapi/providers/remote-source/index.js

@@ -0,0 +1,228 @@
+"use strict";
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var _RemoteStrapiSourceProvider_instances, _RemoteStrapiSourceProvider_createStageReadStream, _RemoteStrapiSourceProvider_startStep, _RemoteStrapiSourceProvider_respond, _RemoteStrapiSourceProvider_endStep;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createRemoteStrapiSourceProvider = void 0;
+const stream_1 = require("stream");
+const ws_1 = require("ws");
+const providers_1 = require("../../../errors/providers");
+const constants_1 = require("../../remote/constants");
+const utils_1 = require("../utils");
+class RemoteStrapiSourceProvider {
+    constructor(options) {
+        _RemoteStrapiSourceProvider_instances.add(this);
+        this.name = 'source::remote-strapi';
+        this.type = 'source';
+        this.options = options;
+        this.ws = null;
+        this.dispatcher = null;
+    }
+    createEntitiesReadStream() {
+        return __classPrivateFieldGet(this, _RemoteStrapiSourceProvider_instances, "m", _RemoteStrapiSourceProvider_createStageReadStream).call(this, 'entities');
+    }
+    createLinksReadStream() {
+        return __classPrivateFieldGet(this, _RemoteStrapiSourceProvider_instances, "m", _RemoteStrapiSourceProvider_createStageReadStream).call(this, 'links');
+    }
+    async createAssetsReadStream() {
+        const assets = {};
+        const stream = await __classPrivateFieldGet(this, _RemoteStrapiSourceProvider_instances, "m", _RemoteStrapiSourceProvider_createStageReadStream).call(this, 'assets');
+        const pass = new stream_1.PassThrough({ objectMode: true });
+        stream
+            .on('data', (asset) => {
+            const { chunk, ...rest } = asset;
+            if (!(asset.filename in assets)) {
+                const assetStream = new stream_1.PassThrough();
+                assets[asset.filename] = assetStream;
+                pass.push({ ...rest, stream: assetStream });
+            }
+            if (asset.filename in assets) {
+                // The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array }
+                // We need to transform it back into a Buffer instance
+                assets[asset.filename].push(Buffer.from(chunk.data));
+            }
+        })
+            .on('end', () => {
+            Object.values(assets).forEach((s) => {
+                s.push(null);
+            });
+        })
+            .on('close', () => {
+            pass.end();
+        });
+        return pass;
+    }
+    createConfigurationReadStream() {
+        return __classPrivateFieldGet(this, _RemoteStrapiSourceProvider_instances, "m", _RemoteStrapiSourceProvider_createStageReadStream).call(this, 'configuration');
+    }
+    async getMetadata() {
+        const metadata = await this.dispatcher?.dispatchTransferAction('getMetadata');
+        return metadata ?? null;
+    }
+    assertValidProtocol(url) {
+        const validProtocols = ['https:', 'http:'];
+        if (!validProtocols.includes(url.protocol)) {
+            throw new providers_1.ProviderValidationError(`Invalid protocol "${url.protocol}"`, {
+                check: 'url',
+                details: {
+                    protocol: url.protocol,
+                    validProtocols,
+                },
+            });
+        }
+    }
+    async initTransfer() {
+        return new Promise((resolve, reject) => {
+            this.ws
+                ?.on('unexpected-response', (_req, res) => {
+                if (res.statusCode === 401) {
+                    return reject(new providers_1.ProviderInitializationError('Failed to initialize the connection: Authentication Error'));
+                }
+                if (res.statusCode === 403) {
+                    return reject(new providers_1.ProviderInitializationError('Failed to initialize the connection: Authorization Error'));
+                }
+                if (res.statusCode === 404) {
+                    return reject(new providers_1.ProviderInitializationError('Failed to initialize the connection: Data transfer is not enabled on the remote host'));
+                }
+                return reject(new providers_1.ProviderInitializationError(`Failed to initialize the connection: Unexpected server response ${res.statusCode}`));
+            })
+                ?.once('open', async () => {
+                const query = this.dispatcher?.dispatchCommand({
+                    command: 'init',
+                });
+                const res = (await query);
+                if (!res?.transferID) {
+                    return reject(new providers_1.ProviderTransferError('Init failed, invalid response from the server'));
+                }
+                resolve(res.transferID);
+            })
+                .once('error', reject);
+        });
+    }
+    async bootstrap() {
+        const { url, auth } = this.options;
+        let ws;
+        this.assertValidProtocol(url);
+        const wsProtocol = url.protocol === 'https:' ? 'wss:' : 'ws:';
+        const wsUrl = `${wsProtocol}//${url.host}${url.pathname}${constants_1.TRANSFER_PATH}/pull`;
+        // No auth defined, trying public access for transfer
+        if (!auth) {
+            ws = new ws_1.WebSocket(wsUrl);
+        }
+        // Common token auth, this should be the main auth method
+        else if (auth.type === 'token') {
+            const headers = { Authorization: `Bearer ${auth.token}` };
+            ws = new ws_1.WebSocket(wsUrl, { headers });
+        }
+        // Invalid auth method provided
+        else {
+            throw new providers_1.ProviderValidationError('Auth method not available', {
+                check: 'auth.type',
+                details: {
+                    auth: auth.type,
+                },
+            });
+        }
+        this.ws = ws;
+        this.dispatcher = (0, utils_1.createDispatcher)(this.ws);
+        const transferID = await this.initTransfer();
+        this.dispatcher.setTransferProperties({ id: transferID, kind: 'pull' });
+        await this.dispatcher.dispatchTransferAction('bootstrap');
+    }
+    async close() {
+        await this.dispatcher?.dispatchTransferAction('close');
+        await new Promise((resolve) => {
+            const { ws } = this;
+            if (!ws || ws.CLOSED) {
+                resolve();
+                return;
+            }
+            ws.on('close', () => resolve()).close();
+        });
+    }
+    async getSchemas() {
+        const schemas = (await this.dispatcher?.dispatchTransferAction('getSchemas')) ?? null;
+        return schemas;
+    }
+}
+_RemoteStrapiSourceProvider_instances = new WeakSet(), _RemoteStrapiSourceProvider_createStageReadStream = async function _RemoteStrapiSourceProvider_createStageReadStream(stage) {
+    const startResult = await __classPrivateFieldGet(this, _RemoteStrapiSourceProvider_instances, "m", _RemoteStrapiSourceProvider_startStep).call(this, stage);
+    if (startResult instanceof Error) {
+        throw startResult;
+    }
+    const { id: processID } = startResult;
+    const stream = new stream_1.PassThrough({ objectMode: true });
+    const listener = async (raw) => {
+        const parsed = JSON.parse(raw.toString());
+        // If not a message related to our transfer process, ignore it
+        if (!parsed.uuid || parsed?.data?.type !== 'transfer' || parsed?.data?.id !== processID) {
+            this.ws?.once('message', listener);
+            return;
+        }
+        const { uuid, data: message } = parsed;
+        const { ended, error, data } = message;
+        if (ended) {
+            await __classPrivateFieldGet(this, _RemoteStrapiSourceProvider_instances, "m", _RemoteStrapiSourceProvider_respond).call(this, uuid);
+            await __classPrivateFieldGet(this, _RemoteStrapiSourceProvider_instances, "m", _RemoteStrapiSourceProvider_endStep).call(this, stage);
+            stream.end();
+            return;
+        }
+        if (error) {
+            await __classPrivateFieldGet(this, _RemoteStrapiSourceProvider_instances, "m", _RemoteStrapiSourceProvider_respond).call(this, uuid);
+            await __classPrivateFieldGet(this, _RemoteStrapiSourceProvider_instances, "m", _RemoteStrapiSourceProvider_endStep).call(this, stage);
+            stream.destroy(error);
+            return;
+        }
+        stream.push(data);
+        this.ws?.once('message', listener);
+        await __classPrivateFieldGet(this, _RemoteStrapiSourceProvider_instances, "m", _RemoteStrapiSourceProvider_respond).call(this, uuid);
+    };
+    this.ws?.once('message', listener);
+    return stream;
+}, _RemoteStrapiSourceProvider_startStep = async function _RemoteStrapiSourceProvider_startStep(step) {
+    try {
+        return await this.dispatcher?.dispatchTransferStep({ action: 'start', step });
+    }
+    catch (e) {
+        if (e instanceof Error) {
+            return e;
+        }
+        if (typeof e === 'string') {
+            return new providers_1.ProviderTransferError(e);
+        }
+        return new providers_1.ProviderTransferError('Unexpected error');
+    }
+}, _RemoteStrapiSourceProvider_respond = async function _RemoteStrapiSourceProvider_respond(uuid) {
+    return new Promise((resolve, reject) => {
+        this.ws?.send(JSON.stringify({ uuid }), (e) => {
+            if (e) {
+                reject(e);
+            }
+            else {
+                resolve(e);
+            }
+        });
+    });
+}, _RemoteStrapiSourceProvider_endStep = async function _RemoteStrapiSourceProvider_endStep(step) {
+    try {
+        await this.dispatcher?.dispatchTransferStep({ action: 'end', step });
+    }
+    catch (e) {
+        if (e instanceof Error) {
+            return e;
+        }
+        if (typeof e === 'string') {
+            return new providers_1.ProviderTransferError(e);
+        }
+        return new providers_1.ProviderTransferError('Unexpected error');
+    }
+    return null;
+};
+const createRemoteStrapiSourceProvider = (options) => {
+    return new RemoteStrapiSourceProvider(options);
+};
+exports.createRemoteStrapiSourceProvider = createRemoteStrapiSourceProvider;
+//# sourceMappingURL=index.js.map

package/lib/strapi/providers/{remote-destination/utils.d.ts → utils.d.ts}

@@ -1,5 +1,5 @@
 import { WebSocket } from 'ws';
-import type { client } from '../../../../types/remote/protocol';
+import type { client } from '../../../types/remote/protocol';
 interface IDispatcherState {
     transfer?: {
         kind: client.TransferKind;
@@ -12,13 +12,13 @@ interface IDispatchOptions {
 declare type Dispatch<T> = Omit<T, 'transferID' | 'uuid'>;
 declare const createDispatcher: (ws: WebSocket) => {
     readonly transferID: string | undefined;
-    readonly transferKind: "push" | "pull" | undefined;
+    readonly transferKind: any;
     setTransferProperties: (properties: Exclude<IDispatcherState['transfer'], undefined>) => void;
     dispatch: <U = null>(message: Dispatch<client.Message>, options?: IDispatchOptions) => Promise<U | null>;
     dispatchCommand: <U_1 extends "end" | "init" | "status">(payload: {
         command: U_1;
     } & ([client.GetCommandParams<U_1>] extends [never] ? unknown : {
-        params: client.GetCommandParams<U_1>;
+        params?: client.GetCommandParams<U_1> | undefined;
     })) => Promise<null>;
     dispatchTransferAction: <T>(action: client.Action['action']) => Promise<T | null>;
     dispatchTransferStep: <T_1, A extends "stream" | "end" | "start" = "stream" | "end" | "start", S extends "entities" | "links" | "configuration" | "assets" = "entities" | "links" | "configuration" | "assets">(payload: {

package/lib/strapi/providers/{remote-destination/utils.js → utils.js}

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createDispatcher = void 0;
 const uuid_1 = require("uuid");
+const providers_1 = require("../../errors/providers");
 const createDispatcher = (ws) => {
     const state = {};
     const dispatch = async (message, options = {}) => {
@@ -24,7 +25,7 @@ const createDispatcher = (ws) => {
                 const response = JSON.parse(raw.toString());
                 if (response.uuid === uuid) {
                     if (response.error) {
-                        return reject(new Error(response.error.message));
+                        return reject(new providers_1.ProviderError('error', response.error.message));
                     }
                     resolve(response.data ?? null);
                 }
@@ -32,7 +33,6 @@ const createDispatcher = (ws) => {
                     ws.once('message', onResponse);
                 }
             };
-            // TODO: What happens if the server sends another message (not a response to this message)
             ws.once('message', onResponse);
         });
     };

package/lib/strapi/remote/constants.d.ts

@@ -1,2 +1,4 @@
-export declare const TRANSFER_PATH = "/transfer";
-export declare const TRANSFER_METHODS: string[];
+export declare const TRANSFER_PATH: "/transfer/runner";
+export declare const TRANSFER_METHODS: readonly ["push", "pull"];
+export declare type TransferPath = typeof TRANSFER_PATH;
+export declare type TransferMethod = (typeof TRANSFER_METHODS)[number];

package/lib/strapi/remote/constants.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TRANSFER_METHODS = exports.TRANSFER_PATH = void 0;
-exports.TRANSFER_PATH = '/transfer';
+exports.TRANSFER_PATH = '/transfer/runner';
 exports.TRANSFER_METHODS = ['push', 'pull'];
 //# sourceMappingURL=constants.js.map

package/lib/strapi/remote/flows/default.d.ts

@@ -0,0 +1,3 @@
+import type { Step } from '.';
+declare const _default: readonly Step[];
+export default _default;

package/lib/strapi/remote/flows/default.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = [
+    {
+        kind: 'action',
+        action: 'bootstrap',
+    },
+    {
+        kind: 'action',
+        action: 'init',
+    },
+    {
+        kind: 'action',
+        action: 'beforeTransfer',
+    },
+    {
+        kind: 'transfer',
+        stage: 'schemas',
+    },
+    {
+        kind: 'transfer',
+        stage: 'entities',
+    },
+    {
+        kind: 'transfer',
+        stage: 'assets',
+    },
+    {
+        kind: 'transfer',
+        stage: 'links',
+    },
+    {
+        kind: 'transfer',
+        stage: 'configuration',
+    },
+    {
+        kind: 'action',
+        action: 'close',
+    },
+];
+//# sourceMappingURL=default.js.map

package/lib/strapi/remote/flows/index.d.ts

@@ -0,0 +1,18 @@
+import type { TransferStage } from '../../../../types';
+export declare type Step = {
+    kind: 'action';
+    action: string;
+} | {
+    kind: 'transfer';
+    stage: TransferStage;
+    locked?: boolean;
+};
+export { default as DEFAULT_TRANSFER_FLOW } from './default';
+export interface TransferFlow {
+    has(step: Step): boolean;
+    can(step: Step): boolean;
+    cannot(step: Step): boolean;
+    set(step: Step): this;
+    get(): Step | null;
+}
+export declare const createFlow: (flow: readonly Step[]) => TransferFlow;

package/lib/strapi/remote/flows/index.js

@@ -0,0 +1,59 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createFlow = exports.DEFAULT_TRANSFER_FLOW = void 0;
+var default_1 = require("./default");
+Object.defineProperty(exports, "DEFAULT_TRANSFER_FLOW", { enumerable: true, get: function () { return __importDefault(default_1).default; } });
+const createFlow = (flow) => {
+    const state = { step: null };
+    /**
+     * Equality check between two steps
+     */
+    const stepEqual = (stepA, stepB) => {
+        if (stepA.kind === 'action' && stepB.kind === 'action') {
+            return stepA.action === stepB.action;
+        }
+        if (stepA.kind === 'transfer' && stepB.kind === 'transfer') {
+            return stepA.stage === stepB.stage;
+        }
+        return false;
+    };
+    /**
+     * Find the index for a given step
+     */
+    const findStepIndex = (step) => flow.findIndex((flowStep) => stepEqual(step, flowStep));
+    return {
+        has(step) {
+            return findStepIndex(step) !== -1;
+        },
+        can(step) {
+            if (state.step === null) {
+                return true;
+            }
+            const indexesDifference = findStepIndex(step) - findStepIndex(state.step);
+            // It's possible to send multiple time the same transfer step in a row
+            if (indexesDifference === 0 && step.kind === 'transfer') {
+                return true;
+            }
+            return indexesDifference > 0;
+        },
+        cannot(step) {
+            return !this.can(step);
+        },
+        set(step) {
+            const canSwitch = this.can(step);
+            if (!canSwitch) {
+                throw new Error('Impossible to proceed to the given step');
+            }
+            state.step = step;
+            return this;
+        },
+        get() {
+            return state.step;
+        },
+    };
+};
+exports.createFlow = createFlow;
+//# sourceMappingURL=index.js.map

package/lib/strapi/remote/handlers/abstract.d.ts

@@ -0,0 +1,62 @@
+/// <reference types="node" />
+import type { WebSocket, RawData } from 'ws';
+import type { ValidTransferCommand } from './constants';
+import type { TransferMethod } from '../constants';
+declare type BufferLike = Parameters<WebSocket['send']>[0];
+export interface TransferState {
+    id?: string;
+    startedAt?: number;
+}
+export interface Handler {
+    get transferID(): TransferState['id'];
+    set transferID(id: TransferState['id']);
+    get startedAt(): TransferState['startedAt'];
+    set startedAt(id: TransferState['startedAt']);
+    /**
+     * Returns whether a transfer is currently in progress or not
+     */
+    isTransferStarted(): boolean;
+    /**
+     * Make sure the current transfer is started and initialized
+     */
+    assertValidTransfer(): void;
+    /**
+     * Checks that the given string is a valid transfer command
+     */
+    assertValidTransferCommand(command: string): asserts command is ValidTransferCommand;
+    /**
+     * Respond to a specific message
+     */
+    respond<T = unknown>(uuid?: string, e?: Error | null, data?: T): Promise<void>;
+    /**
+     * It sends a message to the client
+     */
+    send<T extends BufferLike>(message: T, cb?: (err?: Error) => void): void;
+    /**
+     * It sends a message to the client and waits for a confirmation
+     */
+    confirm<T = unknown>(message: T): Promise<void>;
+    /**
+     * Check the current auth has the permission for the given scope
+     */
+    verifyAuth(scope?: TransferMethod): Promise<void>;
+    /**
+     * Invoke a function and return its result to the client
+     */
+    executeAndRespond<T = unknown>(uuid: string, fn: () => T): Promise<void>;
+    /**
+     * Lifecycle called on error or when the ws connection is closed
+     */
+    teardown(): Promise<void> | void;
+    /**
+     * Lifecycle called to cleanup the transfer state
+     */
+    cleanup(): Promise<void> | void;
+    init(...args: unknown[]): unknown;
+    end(...args: unknown[]): unknown;
+    status(...args: unknown[]): unknown;
+    onMessage(message: RawData, isBinary: boolean): Promise<void> | void;
+    onClose(code: number, reason: Buffer): Promise<void> | void;
+    onError(err: Error): Promise<void> | void;
+}
+export {};

package/lib/strapi/remote/handlers/abstract.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=abstract.js.map

package/lib/strapi/remote/handlers/constants.d.ts

@@ -0,0 +1,2 @@
+export declare const VALID_TRANSFER_COMMANDS: readonly ["init", "end", "status"];
+export declare type ValidTransferCommand = (typeof VALID_TRANSFER_COMMANDS)[number];

package/lib/strapi/remote/handlers/constants.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VALID_TRANSFER_COMMANDS = void 0;
+exports.VALID_TRANSFER_COMMANDS = ['init', 'end', 'status'];
+//# sourceMappingURL=constants.js.map

package/lib/strapi/remote/handlers/index.d.ts

@@ -0,0 +1,3 @@
+export { createPushController, PushHandler } from './push';
+export { createPullController, PullHandler } from './pull';
+export { handlerControllerFactory, HandlerOptions } from './utils';

package/lib/strapi/remote/handlers/index.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handlerControllerFactory = exports.createPullController = exports.createPushController = void 0;
+var push_1 = require("./push");
+Object.defineProperty(exports, "createPushController", { enumerable: true, get: function () { return push_1.createPushController; } });
+var pull_1 = require("./pull");
+Object.defineProperty(exports, "createPullController", { enumerable: true, get: function () { return pull_1.createPullController; } });
+var utils_1 = require("./utils");
+Object.defineProperty(exports, "handlerControllerFactory", { enumerable: true, get: function () { return utils_1.handlerControllerFactory; } });
+//# sourceMappingURL=index.js.map

package/lib/strapi/remote/handlers/pull.d.ts

@@ -0,0 +1,22 @@
+/// <reference types="node" />
+/// <reference types="koa" />
+import { Readable } from 'stream';
+import { Handler } from './abstract';
+import { ILocalStrapiSourceProvider } from '../../providers';
+import type { TransferStage, Protocol } from '../../../../types';
+declare const VALID_TRANSFER_ACTIONS: readonly ["bootstrap", "close", "getMetadata", "getSchemas"];
+declare type PullTransferAction = (typeof VALID_TRANSFER_ACTIONS)[number];
+export interface PullHandler extends Handler {
+    provider?: ILocalStrapiSourceProvider;
+    streams?: {
+        [stage in TransferStage]?: Readable;
+    };
+    assertValidTransferAction(action: string): asserts action is PullTransferAction;
+    onTransferMessage(msg: Protocol.client.TransferMessage): Promise<unknown> | unknown;
+    onTransferAction(msg: Protocol.client.Action): Promise<unknown> | unknown;
+    onTransferStep(msg: Protocol.client.TransferPullMessage): Promise<unknown> | unknown;
+    createReadableStreamForStep(step: TransferStage): Promise<void>;
+    flush(stage: TransferStage, id: string): Promise<void> | void;
+}
+export declare const createPullController: (options: import("./utils").HandlerOptions) => (ctx: import("koa").Context) => Promise<void>;
+export {};