@strapi/data-transfer 4.9.0-alpha.0 → 4.9.0-beta.1

package/lib/strapi/remote/handlers/pull.js

@@ -0,0 +1,186 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createPullController = void 0;
+const stream_1 = require("stream");
+const crypto_1 = require("crypto");
+const utils_1 = require("./utils");
+const providers_1 = require("../../providers");
+const providers_2 = require("../../../errors/providers");
+const TRANSFER_KIND = 'pull';
+const VALID_TRANSFER_ACTIONS = ['bootstrap', 'close', 'getMetadata', 'getSchemas'];
+exports.createPullController = (0, utils_1.handlerControllerFactory)((proto) => ({
+    isTransferStarted() {
+        return proto.isTransferStarted.call(this) && this.provider !== undefined;
+    },
+    verifyAuth() {
+        return proto.verifyAuth.call(this, TRANSFER_KIND);
+    },
+    cleanup() {
+        proto.cleanup.call(this);
+        this.streams = {};
+        delete this.provider;
+    },
+    assertValidTransferAction(action) {
+        // Abstract the constant to string[] to allow looser check on the given action
+        const validActions = VALID_TRANSFER_ACTIONS;
+        if (validActions.includes(action)) {
+            return;
+        }
+        throw new providers_2.ProviderTransferError(`Invalid action provided: "${action}"`, {
+            action,
+            validActions: Object.keys(VALID_TRANSFER_ACTIONS),
+        });
+    },
+    async onMessage(raw) {
+        const msg = JSON.parse(raw.toString());
+        if (!(0, utils_1.isDataTransferMessage)(msg)) {
+            return;
+        }
+        if (!msg.uuid) {
+            await this.respond(undefined, new Error('Missing uuid in message'));
+        }
+        const { uuid, type } = msg;
+        // Regular command message (init, end, status)
+        if (type === 'command') {
+            const { command } = msg;
+            await this.executeAndRespond(uuid, () => {
+                this.assertValidTransferCommand(command);
+                // The status command don't have params
+                if (command === 'status') {
+                    return this.status();
+                }
+                return this[command](msg.params);
+            });
+        }
+        // Transfer message (the transfer must be init first)
+        else if (type === 'transfer') {
+            await this.executeAndRespond(uuid, async () => {
+                await this.verifyAuth();
+                this.assertValidTransfer();
+                return this.onTransferMessage(msg);
+            });
+        }
+        // Invalid messages
+        else {
+            await this.respond(uuid, new Error('Bad Request'));
+        }
+    },
+    async onTransferMessage(msg) {
+        const { kind } = msg;
+        if (kind === 'action') {
+            return this.onTransferAction(msg);
+        }
+        if (kind === 'step') {
+            return this.onTransferStep(msg);
+        }
+    },
+    async onTransferAction(msg) {
+        const { action } = msg;
+        this.assertValidTransferAction(action);
+        return this.provider?.[action]();
+    },
+    // TODO: Optimize performances (batching, client packets reconstruction, etc...)
+    async flush(stage, id) {
+        const stream = this.streams?.[stage];
+        if (!stream) {
+            throw new providers_2.ProviderTransferError(`No available stream found for ${stage}`);
+        }
+        try {
+            for await (const chunk of stream) {
+                await this.confirm({ type: 'transfer', data: chunk, ended: false, error: null, id });
+            }
+            await this.confirm({ type: 'transfer', data: null, ended: true, error: null, id });
+        }
+        catch (e) {
+            await this.confirm({ type: 'transfer', data: null, ended: true, error: e, id });
+        }
+    },
+    async onTransferStep(msg) {
+        const { step, action } = msg;
+        if (action === 'start') {
+            if (this.streams?.[step] instanceof stream_1.Readable) {
+                throw new Error('Stream already created, something went wrong');
+            }
+            const flushUUID = (0, crypto_1.randomUUID)();
+            await this.createReadableStreamForStep(step);
+            this.flush(step, flushUUID);
+            return { ok: true, id: flushUUID };
+        }
+        if (action === 'end') {
+            const stream = this.streams?.[step];
+            if (stream?.readableEnded === false) {
+                await new Promise((resolve) => {
+                    stream?.on('close', resolve).destroy();
+                });
+            }
+            delete this.streams?.[step];
+            return { ok: true };
+        }
+    },
+    async createReadableStreamForStep(step) {
+        const mapper = {
+            entities: () => this.provider?.createEntitiesReadStream(),
+            links: () => this.provider?.createLinksReadStream(),
+            configuration: () => this.provider?.createConfigurationReadStream(),
+            assets: () => {
+                const assets = this.provider?.createAssetsReadStream();
+                if (!assets) {
+                    throw new Error('bad');
+                }
+                async function* generator(stream) {
+                    for await (const chunk of stream) {
+                        const { stream: assetStream, ...rest } = chunk;
+                        for await (const assetChunk of assetStream) {
+                            yield { ...rest, chunk: assetChunk };
+                        }
+                    }
+                }
+                return stream_1.Readable.from(generator(assets));
+            },
+        };
+        if (!(step in mapper)) {
+            throw new Error('Invalid transfer step, impossible to create a stream');
+        }
+        if (!this.streams) {
+            throw new Error('Invalid transfer state');
+        }
+        this.streams[step] = await mapper[step]();
+    },
+    // Commands
+    async init() {
+        if (this.transferID || this.provider) {
+            throw new Error('Transfer already in progress');
+        }
+        await this.verifyAuth();
+        this.transferID = (0, crypto_1.randomUUID)();
+        this.startedAt = Date.now();
+        this.streams = {};
+        this.provider = (0, providers_1.createLocalStrapiSourceProvider)({
+            autoDestroy: false,
+            getStrapi: () => strapi,
+        });
+        return { transferID: this.transferID };
+    },
+    async end(params) {
+        await this.verifyAuth();
+        if (this.transferID !== params.transferID) {
+            throw new providers_2.ProviderTransferError('Bad transfer ID provided');
+        }
+        this.cleanup();
+        return { ok: true };
+    },
+    async status() {
+        const isStarted = this.isTransferStarted();
+        if (!isStarted) {
+            const startedAt = this.startedAt;
+            return {
+                active: true,
+                kind: TRANSFER_KIND,
+                startedAt,
+                elapsed: Date.now() - startedAt,
+            };
+        }
+        return { active: false, kind: null, elapsed: null, startedAt: null };
+    },
+}));
+//# sourceMappingURL=pull.js.map

package/lib/strapi/remote/handlers/push.d.ts

@@ -0,0 +1,75 @@
+/// <reference types="node" />
+/// <reference types="koa" />
+import { Writable, PassThrough } from 'stream';
+import type { TransferFlow } from '../flows';
+import type { TransferStage, IAsset, Protocol } from '../../../../types';
+import { createLocalStrapiDestinationProvider } from '../../providers';
+import { Handler } from './abstract';
+declare const VALID_TRANSFER_ACTIONS: readonly ["bootstrap", "close", "rollback", "beforeTransfer", "getMetadata", "getSchemas"];
+declare type PushTransferAction = (typeof VALID_TRANSFER_ACTIONS)[number];
+export interface PushHandler extends Handler {
+    /**
+     * Local Strapi Destination Provider used to write data to the current Strapi instance
+     */
+    provider?: ReturnType<typeof createLocalStrapiDestinationProvider>;
+    /**
+     * Holds all the stages' stream for the current transfer handler (one registry per connection)
+     */
+    streams?: {
+        [stage in TransferStage]?: Writable;
+    };
+    /**
+     * Holds all the transferred assets for the current transfer handler (one registry per connection)
+     */
+    assets: {
+        [filepath: string]: IAsset & {
+            stream: PassThrough;
+        };
+    };
+    /**
+     * Ochestrate and manage the transfer messages' ordering
+     */
+    flow?: TransferFlow;
+    /**
+     * Checks that the given action is a valid push transfer action
+     */
+    assertValidTransferAction(action: string): asserts action is PushTransferAction;
+    /**
+     * Create a new writable stream for the given step in the handler's stream registry
+     */
+    createWritableStreamForStep(step: TransferStage): Promise<void>;
+    /**
+     * Simple override of the auth verification
+     */
+    verifyAuth(): Promise<void>;
+    /**
+     * Callback when receiving a regular transfer message
+     */
+    onTransferMessage(msg: Protocol.client.TransferMessage): Promise<unknown> | unknown;
+    /**
+     * Callback when receiving a transfer action message
+     */
+    onTransferAction(msg: Protocol.client.Action): Promise<unknown> | unknown;
+    /**
+     * Callback when receiving a transfer step message
+     */
+    onTransferStep(msg: Protocol.client.TransferPushMessage): Promise<unknown> | unknown;
+    /**
+     * Start streaming an asset
+     */
+    streamAsset(this: PushHandler, payload: Protocol.client.GetTransferPushStreamData<'assets'>): Promise<void>;
+    /**
+     * Try to move to a specific transfer stage & lock the step
+     */
+    lockTransferStep(stage: TransferStage): void;
+    /**
+     * Try to move to unlock the current step
+     */
+    unlockTransferStep(stage: TransferStage): void;
+    /**
+     * Checks whether it's possible to stream a chunk for the given stage
+     */
+    assertValidStreamTransferStep(stage: TransferStage): void;
+}
+export declare const createPushController: (options: import("./utils").HandlerOptions) => (ctx: import("koa").Context) => Promise<void>;
+export {};

package/lib/strapi/remote/handlers/push.js

@@ -0,0 +1,297 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createPushController = void 0;
+const crypto_1 = require("crypto");
+const stream_1 = require("stream");
+const providers_1 = require("../../../errors/providers");
+const providers_2 = require("../../providers");
+const flows_1 = require("../flows");
+const utils_1 = require("./utils");
+const VALID_TRANSFER_ACTIONS = [
+    'bootstrap',
+    'close',
+    'rollback',
+    'beforeTransfer',
+    'getMetadata',
+    'getSchemas',
+];
+const TRANSFER_KIND = 'push';
+const writeAsync = (stream, data) => {
+    return new Promise((resolve, reject) => {
+        stream.write(data, (error) => {
+            if (error) {
+                reject(error);
+            }
+            resolve();
+        });
+    });
+};
+exports.createPushController = (0, utils_1.handlerControllerFactory)((proto) => ({
+    isTransferStarted() {
+        return proto.isTransferStarted.call(this) && this.provider !== undefined;
+    },
+    verifyAuth() {
+        return proto.verifyAuth.call(this, TRANSFER_KIND);
+    },
+    cleanup() {
+        proto.cleanup.call(this);
+        this.streams = {};
+        this.assets = {};
+        delete this.flow;
+        delete this.provider;
+    },
+    teardown() {
+        if (this.provider) {
+            this.provider.rollback();
+        }
+        proto.teardown.call(this);
+    },
+    assertValidTransfer() {
+        proto.assertValidTransfer.call(this);
+        if (this.provider === undefined) {
+            throw new Error('Invalid Transfer Process');
+        }
+    },
+    assertValidTransferAction(action) {
+        if (VALID_TRANSFER_ACTIONS.includes(action)) {
+            return;
+        }
+        throw new providers_1.ProviderTransferError(`Invalid action provided: "${action}"`, {
+            action,
+            validActions: Object.keys(VALID_TRANSFER_ACTIONS),
+        });
+    },
+    assertValidStreamTransferStep(stage) {
+        const currentStep = this.flow?.get();
+        const nextStep = { kind: 'transfer', stage };
+        if (currentStep?.kind === 'transfer' && !currentStep.locked) {
+            throw new providers_1.ProviderTransferError(`You need to initialize the transfer stage (${nextStep}) before starting to stream data`);
+        }
+        if (this.flow?.cannot(nextStep)) {
+            throw new providers_1.ProviderTransferError(`Invalid stage (${nextStep}) provided for the current flow`, {
+                step: nextStep,
+            });
+        }
+    },
+    async createWritableStreamForStep(step) {
+        const mapper = {
+            entities: () => this.provider?.createEntitiesWriteStream(),
+            links: () => this.provider?.createLinksWriteStream(),
+            configuration: () => this.provider?.createConfigurationWriteStream(),
+            assets: () => this.provider?.createAssetsWriteStream(),
+        };
+        if (!(step in mapper)) {
+            throw new Error('Invalid transfer step, impossible to create a stream');
+        }
+        if (!this.streams) {
+            throw new Error('Invalid transfer state');
+        }
+        this.streams[step] = await mapper[step]();
+    },
+    async onMessage(raw) {
+        const msg = JSON.parse(raw.toString());
+        if (!(0, utils_1.isDataTransferMessage)(msg)) {
+            return;
+        }
+        if (!msg.uuid) {
+            await this.respond(undefined, new Error('Missing uuid in message'));
+        }
+        const { uuid, type } = msg;
+        // Regular command message (init, end, status)
+        if (type === 'command') {
+            const { command } = msg;
+            await this.executeAndRespond(uuid, () => {
+                this.assertValidTransferCommand(command);
+                // The status command don't have params
+                if (command === 'status') {
+                    return this.status();
+                }
+                return this[command](msg.params);
+            });
+        }
+        // Transfer message (the transfer must be init first)
+        else if (type === 'transfer') {
+            await this.executeAndRespond(uuid, async () => {
+                await this.verifyAuth();
+                this.assertValidTransfer();
+                return this.onTransferMessage(msg);
+            });
+        }
+        // Invalid messages
+        else {
+            await this.respond(uuid, new Error('Bad Request'));
+        }
+    },
+    async onTransferMessage(msg) {
+        const { kind } = msg;
+        if (kind === 'action') {
+            return this.onTransferAction(msg);
+        }
+        if (kind === 'step') {
+            return this.onTransferStep(msg);
+        }
+    },
+    lockTransferStep(stage) {
+        const currentStep = this.flow?.get();
+        const nextStep = { kind: 'transfer', stage };
+        if (currentStep?.kind === 'transfer' && currentStep.locked) {
+            throw new providers_1.ProviderTransferError(`It's not possible to start a new transfer stage (${stage}) while another one is in progress (${currentStep.stage})`);
+        }
+        if (this.flow?.cannot(nextStep)) {
+            throw new providers_1.ProviderTransferError(`Invalid stage (${stage}) provided for the current flow`, {
+                step: nextStep,
+            });
+        }
+        this.flow?.set({ ...nextStep, locked: true });
+    },
+    unlockTransferStep(stage) {
+        const currentStep = this.flow?.get();
+        const nextStep = { kind: 'transfer', stage };
+        // Cannot unlock if not locked (aka: started)
+        if (currentStep?.kind === 'transfer' && !currentStep.locked) {
+            throw new providers_1.ProviderTransferError(`You need to initialize the transfer stage (${stage}) before ending it`);
+        }
+        // Cannot unlock if invalid step provided
+        if (this.flow?.cannot(nextStep)) {
+            throw new providers_1.ProviderTransferError(`Invalid stage (${stage}) provided for the current flow`, {
+                step: nextStep,
+            });
+        }
+        this.flow?.set({ ...nextStep, locked: false });
+    },
+    async onTransferStep(msg) {
+        const { step: stage } = msg;
+        if (msg.action === 'start') {
+            this.lockTransferStep(stage);
+            if (this.streams?.[stage] instanceof stream_1.Writable) {
+                throw new Error('Stream already created, something went wrong');
+            }
+            await this.createWritableStreamForStep(stage);
+            return { ok: true };
+        }
+        if (msg.action === 'stream') {
+            this.assertValidStreamTransferStep(stage);
+            // Stream operation on the current transfer stage
+            const stream = this.streams?.[stage];
+            if (!stream) {
+                throw new Error('You need to init first');
+            }
+            // Assets are nested streams
+            if (stage === 'assets') {
+                return this.streamAsset(msg.data);
+            }
+            // For all other steps
+            await Promise.all(msg.data.map((item) => writeAsync(stream, item)));
+        }
+        if (msg.action === 'end') {
+            this.unlockTransferStep(stage);
+            const stream = this.streams?.[stage];
+            if (stream && !stream.closed) {
+                await new Promise((resolve, reject) => {
+                    stream.on('close', resolve).on('error', reject).end();
+                });
+            }
+            delete this.streams?.[stage];
+            return { ok: true };
+        }
+    },
+    async onTransferAction(msg) {
+        const { action } = msg;
+        this.assertValidTransferAction(action);
+        const step = { kind: 'action', action };
+        const isStepRegistered = this.flow?.has(step);
+        if (isStepRegistered) {
+            if (this.flow?.cannot(step)) {
+                throw new providers_1.ProviderTransferError(`Invalid action "${action}" found for the current flow `, {
+                    action,
+                });
+            }
+            this.flow?.set(step);
+        }
+        return this.provider?.[action]();
+    },
+    async streamAsset(payload) {
+        const assetsStream = this.streams?.assets;
+        // TODO: close the stream upong receiving an 'end' event instead
+        if (payload === null) {
+            this.streams?.assets?.end();
+            return;
+        }
+        for (const item of payload) {
+            const { action, assetID } = item;
+            if (!assetsStream) {
+                throw new Error('Stream not defined');
+            }
+            if (action === 'start') {
+                this.assets[assetID] = { ...item.data, stream: new stream_1.PassThrough() };
+                writeAsync(assetsStream, this.assets[assetID]);
+            }
+            if (action === 'stream') {
+                // The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array }
+                // We need to transform it back into a Buffer instance
+                const rawBuffer = item.data;
+                const chunk = Buffer.from(rawBuffer.data);
+                await writeAsync(this.assets[assetID].stream, chunk);
+            }
+            if (action === 'end') {
+                await new Promise((resolve, reject) => {
+                    const { stream: assetStream } = this.assets[assetID];
+                    assetStream
+                        .on('close', () => {
+                        delete this.assets[assetID];
+                        resolve();
+                    })
+                        .on('error', reject)
+                        .end();
+                });
+            }
+        }
+    },
+    onClose() {
+        this.teardown();
+    },
+    onError(err) {
+        this.teardown();
+        strapi.log.error(err);
+    },
+    // Commands
+    async init(params) {
+        if (this.transferID || this.provider) {
+            throw new Error('Transfer already in progress');
+        }
+        await this.verifyAuth();
+        this.transferID = (0, crypto_1.randomUUID)();
+        this.startedAt = Date.now();
+        this.assets = {};
+        this.streams = {};
+        this.flow = (0, flows_1.createFlow)(flows_1.DEFAULT_TRANSFER_FLOW);
+        this.provider = (0, providers_2.createLocalStrapiDestinationProvider)({
+            ...params,
+            autoDestroy: false,
+            getStrapi: () => strapi,
+        });
+        return { transferID: this.transferID };
+    },
+    async status() {
+        const isStarted = this.isTransferStarted();
+        if (isStarted) {
+            const startedAt = this.startedAt;
+            return {
+                active: true,
+                kind: TRANSFER_KIND,
+                startedAt,
+                elapsed: Date.now() - startedAt,
+            };
+        }
+        return { active: false, kind: null, elapsed: null, startedAt: null };
+    },
+    async end(params) {
+        await this.verifyAuth();
+        if (this.transferID !== params.transferID) {
+            throw new providers_1.ProviderTransferError('Bad transfer ID provided');
+        }
+        this.cleanup();
+        return { ok: true };
+    },
+}));
+//# sourceMappingURL=push.js.map

package/lib/strapi/remote/handlers/utils.d.ts

@@ -0,0 +1,25 @@
+/// <reference types="node" />
+import type { Context } from 'koa';
+import type { IncomingMessage } from 'http';
+import type { ServerOptions } from 'ws';
+import { WebSocket, WebSocketServer } from 'ws';
+import type { Handler } from './abstract';
+import type { Protocol } from '../../../../types';
+import { TransferMethod } from '../constants';
+declare type WSCallback = (client: WebSocket, request: IncomingMessage) => void;
+export interface HandlerOptions {
+    verify: (ctx: Context, scope?: TransferMethod) => Promise<void>;
+    server?: ServerOptions;
+}
+export declare const transformUpgradeHeader: (header?: string) => string[];
+/**
+ * Make sure that the upgrade header is a valid websocket one
+ */
+export declare const assertValidHeader: (ctx: Context) => void;
+export declare const isDataTransferMessage: (message: unknown) => message is Protocol.client.Message;
+/**
+ * Handle the upgrade to ws connection
+ */
+export declare const handleWSUpgrade: (wss: WebSocketServer, ctx: Context, callback: WSCallback) => void;
+export declare const handlerControllerFactory: <T extends Partial<Handler>>(implementation: (proto: Handler) => T) => (options: HandlerOptions) => (ctx: Context) => Promise<void>;
+export {};