@strapi/data-transfer 4.9.0 → 4.10.0-beta.0

package/lib/strapi/queries/link.js

@@ -0,0 +1,215 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createLinkQuery = void 0;
+const fp_1 = require("lodash/fp");
+// TODO: Remove any types when we'll have types for DB metadata
+const createLinkQuery = (strapi, trx) => {
+    const query = () => {
+        const { connection } = strapi.db;
+        async function* generateAllForAttribute(uid, fieldName) {
+            const metadata = strapi.db.metadata.get(uid);
+            if (!metadata) {
+                throw new Error(`No metadata found for ${uid}`);
+            }
+            const attributes = filterValidRelationalAttributes(metadata.attributes);
+            if (!(fieldName in attributes)) {
+                throw new Error(`${fieldName} is not a valid relational attribute name`);
+            }
+            const attribute = attributes[fieldName];
+            const kind = getLinkKind(attribute, uid);
+            const { relation, target } = attribute;
+            // The relation is stored in the same table
+            // TODO: handle manyToOne joinColumn
+            if (attribute.joinColumn) {
+                const joinColumnName = attribute.joinColumn.name;
+                const qb = connection.queryBuilder().select('id', joinColumnName).from(metadata.tableName);
+                if (trx) {
+                    qb.transacting(trx);
+                }
+                // TODO: stream the query to improve performances
+                const entries = await qb;
+                for (const entry of entries) {
+                    const ref = entry[joinColumnName];
+                    if (ref !== null) {
+                        yield {
+                            kind,
+                            relation,
+                            left: { type: uid, ref: entry.id, field: fieldName },
+                            right: { type: target, ref },
+                        };
+                    }
+                }
+            }
+            // The relation uses a join table
+            if (attribute.joinTable) {
+                const { name, joinColumn, inverseJoinColumn, orderColumnName, morphColumn, inverseOrderColumnName, } = attribute.joinTable;
+                const qb = connection.queryBuilder().from(name);
+                const columns = {
+                    left: { ref: null },
+                    right: { ref: null },
+                };
+                const left = { type: uid, field: fieldName };
+                const right = {};
+                if (kind === 'relation.basic' || kind === 'relation.circular') {
+                    right.type = attribute.target;
+                    right.field = attribute.inversedBy;
+                    columns.left.ref = joinColumn.name;
+                    columns.right.ref = inverseJoinColumn.name;
+                    if (orderColumnName) {
+                        columns.left.order = orderColumnName;
+                    }
+                    if (inverseOrderColumnName) {
+                        columns.right.order = inverseOrderColumnName;
+                    }
+                }
+                if (kind === 'relation.morph') {
+                    columns.left.ref = joinColumn.name;
+                    columns.right.ref = morphColumn.idColumn.name;
+                    columns.right.type = morphColumn.typeColumn.name;
+                    columns.right.field = 'field';
+                    columns.right.order = 'order';
+                }
+                const validColumns = [
+                    // Left
+                    columns.left.ref,
+                    columns.left.order,
+                    // Right
+                    columns.right.ref,
+                    columns.right.type,
+                    columns.right.field,
+                    columns.right.order,
+                ].filter((column) => !(0, fp_1.isNil)(column));
+                qb.select(validColumns);
+                if (trx) {
+                    qb.transacting(trx);
+                }
+                // TODO: stream the query to improve performances
+                const entries = await qb;
+                for (const entry of entries) {
+                    if (columns.left.ref) {
+                        left.ref = entry[columns.left.ref];
+                    }
+                    if (columns.right.ref) {
+                        right.ref = entry[columns.right.ref];
+                    }
+                    if (columns.left.order) {
+                        left.pos = entry[columns.left.order];
+                    }
+                    if (columns.right.order) {
+                        right.pos = entry[columns.right.order];
+                    }
+                    if (columns.right.type) {
+                        right.type = entry[columns.right.type];
+                    }
+                    if (columns.right.field) {
+                        right.field = entry[columns.right.field];
+                    }
+                    const link = {
+                        kind,
+                        relation,
+                        left: (0, fp_1.clone)(left),
+                        right: (0, fp_1.clone)(right),
+                    };
+                    yield link;
+                }
+            }
+        }
+        async function* generateAll(uid) {
+            const metadata = strapi.db.metadata.get(uid);
+            if (!metadata) {
+                throw new Error(`No metadata found for ${uid}`);
+            }
+            const attributes = filterValidRelationalAttributes(metadata.attributes);
+            for (const fieldName of Object.keys(attributes)) {
+                for await (const link of generateAllForAttribute(uid, fieldName)) {
+                    yield link;
+                }
+            }
+        }
+        const insert = async (link) => {
+            const { kind, left, right } = link;
+            const metadata = strapi.db.metadata.get(left.type);
+            const attribute = metadata.attributes[left.field];
+            const payload = {};
+            if (attribute.joinColumn) {
+                const joinColumnName = attribute.joinColumn.name;
+                const qb = connection(metadata.tableName)
+                    .where('id', left.ref)
+                    .update({ [joinColumnName]: right.ref });
+                if (trx) {
+                    qb.transacting(trx);
+                }
+                await qb;
+            }
+            if (attribute.joinTable) {
+                const { name, joinColumn, inverseJoinColumn, orderColumnName, inverseOrderColumnName, morphColumn, } = attribute.joinTable;
+                if (joinColumn) {
+                    Object.assign(payload, { [joinColumn.name]: left.ref });
+                }
+                const assignInverseColumn = () => {
+                    if (inverseJoinColumn) {
+                        Object.assign(payload, {
+                            [inverseJoinColumn.name]: right.ref,
+                        });
+                    }
+                };
+                const assignOrderColumns = () => {
+                    if (orderColumnName) {
+                        Object.assign(payload, { [orderColumnName]: left.pos ?? null });
+                    }
+                    if (inverseOrderColumnName) {
+                        Object.assign(payload, { [inverseOrderColumnName]: right.pos ?? null });
+                    }
+                };
+                const assignMorphColumns = () => {
+                    const { idColumn, typeColumn } = morphColumn ?? {};
+                    if (idColumn) {
+                        Object.assign(payload, { [idColumn.name]: right.ref });
+                    }
+                    if (typeColumn) {
+                        Object.assign(payload, { [typeColumn.name]: right.type });
+                    }
+                    Object.assign(payload, { order: right.pos ?? null, field: right.field ?? null });
+                };
+                if (kind === 'relation.basic' || kind === 'relation.circular') {
+                    assignInverseColumn();
+                }
+                if (kind === 'relation.morph') {
+                    assignMorphColumns();
+                }
+                assignOrderColumns();
+                const qb = connection.insert(payload).into(name);
+                if (trx) {
+                    qb.transacting(trx);
+                }
+                await qb;
+            }
+        };
+        return { generateAll, generateAllForAttribute, insert };
+    };
+    return query;
+};
+exports.createLinkQuery = createLinkQuery;
+const filterValidRelationalAttributes = (attributes) => {
+    const isOwner = (attribute) => {
+        return attribute.owner || (!attribute.mappedBy && !attribute.morphBy);
+    };
+    const isComponentLike = (attribute) => {
+        return attribute.component || attribute.components;
+    };
+    return Object.entries(attributes)
+        .filter(([, attribute]) => {
+        return attribute.type === 'relation' && isOwner(attribute) && !isComponentLike(attribute);
+    })
+        .reduce((acc, [key, attribute]) => ({ ...acc, [key]: attribute }), {});
+};
+const getLinkKind = (attribute, uid) => {
+    if (attribute.relation.startsWith('morph')) {
+        return 'relation.morph';
+    }
+    if (attribute.target === uid) {
+        return 'relation.circular';
+    }
+    return 'relation.basic';
+};
+//# sourceMappingURL=link.js.map

package/lib/strapi/remote/constants.d.ts

@@ -0,0 +1,2 @@
+export declare const TRANSFER_PATH: "/transfer/runner/connect";
+export declare const TRANSFER_METHODS: readonly ["push", "pull"];

package/lib/strapi/remote/constants.js

@@ -0,0 +1,6 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TRANSFER_METHODS = exports.TRANSFER_PATH = void 0;
+exports.TRANSFER_PATH = '/transfer/runner/connect';
+exports.TRANSFER_METHODS = ['push', 'pull'];
+//# sourceMappingURL=constants.js.map

package/lib/strapi/remote/controllers/index.d.ts

@@ -0,0 +1 @@
+export * from './push';

package/lib/strapi/remote/controllers/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./push"), exports);
+//# sourceMappingURL=index.js.map

package/lib/strapi/remote/controllers/push.d.ts

@@ -0,0 +1,26 @@
+/// <reference types="node" />
+import { Writable } from 'stream-chain';
+import type { IMetadata } from '../../../../types';
+import type { TransferPushMessage, TransferPushStep } from '../../../../types/remote/protocol/client';
+import { ILocalStrapiDestinationProviderOptions } from '../../providers';
+export interface IPushController {
+    streams: {
+        [stage in TransferPushStep]?: Writable;
+    };
+    actions: {
+        getMetadata(): Promise<IMetadata>;
+        getSchemas(): Strapi.Schemas;
+        bootstrap(): Promise<void>;
+        close(): Promise<void>;
+        beforeTransfer(): Promise<void>;
+        rollback(): Promise<void>;
+    };
+    transfer: {
+        [key in TransferPushStep]: <T extends TransferPushMessage>(value: T extends {
+            step: key;
+            data: infer U;
+        } ? U : never) => Promise<void>;
+    };
+}
+declare const createPushController: (options: ILocalStrapiDestinationProviderOptions) => IPushController;
+export default createPushController;

package/lib/strapi/remote/controllers/push.js

@@ -0,0 +1,116 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const stream_chain_1 = require("stream-chain");
+const providers_1 = require("../../providers");
+const createPushController = (options) => {
+    const provider = (0, providers_1.createLocalStrapiDestinationProvider)(options);
+    const streams = {};
+    const assets = {};
+    const writeAsync = (stream, data) => {
+        return new Promise((resolve, reject) => {
+            stream.write(data, (error) => {
+                if (error) {
+                    reject(error);
+                }
+                resolve();
+            });
+        });
+    };
+    return {
+        streams,
+        actions: {
+            async getSchemas() {
+                return provider.getSchemas();
+            },
+            async getMetadata() {
+                return provider.getMetadata();
+            },
+            async bootstrap() {
+                return provider.bootstrap();
+            },
+            async close() {
+                return provider.close();
+            },
+            async beforeTransfer() {
+                return provider.beforeTransfer();
+            },
+            async rollback() {
+                await provider.rollback();
+            },
+        },
+        transfer: {
+            async entities(entities) {
+                if (!streams.entities) {
+                    streams.entities = provider.createEntitiesWriteStream();
+                }
+                for (const entity of entities) {
+                    if (streams.entities) {
+                        await writeAsync(streams.entities, entity);
+                    }
+                }
+            },
+            async links(links) {
+                if (!streams.links) {
+                    streams.links = await provider.createLinksWriteStream();
+                }
+                for (const link of links) {
+                    if (streams.links) {
+                        await writeAsync(streams.links, link);
+                    }
+                }
+            },
+            async configuration(configs) {
+                if (!streams.configuration) {
+                    streams.configuration = await provider.createConfigurationWriteStream();
+                }
+                for (const config of configs) {
+                    if (streams.configuration) {
+                        await writeAsync(streams.configuration, config);
+                    }
+                }
+            },
+            async assets(payloads) {
+                if (payloads === null) {
+                    streams.assets?.end();
+                    return;
+                }
+                if (!streams.assets) {
+                    streams.assets = await provider.createAssetsWriteStream();
+                }
+                for (const payload of payloads) {
+                    if (streams.assets.closed) {
+                        return;
+                    }
+                    const { action, assetID } = payload;
+                    if (action === 'start' && streams.assets) {
+                        assets[assetID] = { ...payload.data, stream: new stream_chain_1.PassThrough() };
+                        writeAsync(streams.assets, assets[assetID]);
+                    }
+                    if (action === 'stream') {
+                        // The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array }
+                        // We need to transform it back into a Buffer instance
+                        const rawBuffer = payload.data;
+                        const chunk = Buffer.from(rawBuffer.data);
+                        await writeAsync(assets[assetID].stream, chunk);
+                    }
+                    if (action === 'end') {
+                        await new Promise((resolve, reject) => {
+                            const { stream } = assets[assetID];
+                            stream
+                                .on('close', () => {
+                                delete assets[assetID];
+                                resolve();
+                            })
+                                .on('error', (e) => {
+                                reject(e);
+                            })
+                                .end();
+                        });
+                    }
+                }
+            },
+        },
+    };
+};
+exports.default = createPushController;
+//# sourceMappingURL=push.js.map

package/lib/strapi/remote/flows/default.d.ts

@@ -0,0 +1,3 @@
+import type { Step } from '.';
+declare const _default: readonly Step[];
+export default _default;

package/lib/strapi/remote/flows/default.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = [
+    {
+        kind: 'action',
+        action: 'bootstrap',
+    },
+    {
+        kind: 'action',
+        action: 'init',
+    },
+    {
+        kind: 'action',
+        action: 'beforeTransfer',
+    },
+    {
+        kind: 'transfer',
+        stage: 'schemas',
+    },
+    {
+        kind: 'transfer',
+        stage: 'entities',
+    },
+    {
+        kind: 'transfer',
+        stage: 'assets',
+    },
+    {
+        kind: 'transfer',
+        stage: 'links',
+    },
+    {
+        kind: 'transfer',
+        stage: 'configuration',
+    },
+    {
+        kind: 'action',
+        action: 'close',
+    },
+];
+//# sourceMappingURL=default.js.map

package/lib/strapi/remote/flows/index.d.ts

@@ -0,0 +1,18 @@
+import type { TransferStage } from '../../../../types';
+export declare type Step = {
+    kind: 'action';
+    action: string;
+} | {
+    kind: 'transfer';
+    stage: TransferStage;
+    locked?: boolean;
+};
+export { default as DEFAULT_TRANSFER_FLOW } from './default';
+export declare const createFlow: (flow: readonly Step[]) => {
+    has(step: Step): boolean;
+    can(step: Step): boolean;
+    cannot(step: Step): boolean;
+    set(step: Step): any;
+    get(): Step | null;
+};
+export declare type TransferFlow = ReturnType<typeof createFlow>;

package/lib/strapi/remote/flows/index.js

@@ -0,0 +1,59 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createFlow = exports.DEFAULT_TRANSFER_FLOW = void 0;
+var default_1 = require("./default");
+Object.defineProperty(exports, "DEFAULT_TRANSFER_FLOW", { enumerable: true, get: function () { return __importDefault(default_1).default; } });
+const createFlow = (flow) => {
+    const state = { step: null };
+    /**
+     * Equality check between two steps
+     */
+    const stepEqual = (stepA, stepB) => {
+        if (stepA.kind === 'action' && stepB.kind === 'action') {
+            return stepA.action === stepB.action;
+        }
+        if (stepA.kind === 'transfer' && stepB.kind === 'transfer') {
+            return stepA.stage === stepB.stage;
+        }
+        return false;
+    };
+    /**
+     * Find the index for a given step
+     */
+    const findStepIndex = (step) => flow.findIndex((flowStep) => stepEqual(step, flowStep));
+    return {
+        has(step) {
+            return findStepIndex(step) !== -1;
+        },
+        can(step) {
+            if (state.step === null) {
+                return true;
+            }
+            const indexesDifference = findStepIndex(step) - findStepIndex(state.step);
+            // It's possible to send multiple time the same transfer step in a row
+            if (indexesDifference === 0 && step.kind === 'transfer') {
+                return true;
+            }
+            return indexesDifference > 0;
+        },
+        cannot(step) {
+            return !this.can(step);
+        },
+        set(step) {
+            const canSwitch = this.can(step);
+            if (!canSwitch) {
+                throw new Error('Impossible to proceed to the given step');
+            }
+            state.step = step;
+            return this;
+        },
+        get() {
+            return state.step;
+        },
+    };
+};
+exports.createFlow = createFlow;
+//# sourceMappingURL=index.js.map

package/lib/strapi/remote/handlers.d.ts

@@ -0,0 +1,10 @@
+import type { Context } from 'koa';
+import type { ServerOptions } from 'ws';
+import { TRANSFER_METHODS } from './constants';
+declare type TransferMethod = (typeof TRANSFER_METHODS)[number];
+interface IHandlerOptions {
+    verify: (ctx: Context, scope?: TransferMethod) => Promise<void>;
+    server?: ServerOptions;
+}
+export declare const createTransferHandler: (options: IHandlerOptions) => (ctx: Context) => Promise<void>;
+export {};