@contentstack/cli-cm-import 1.5.11 → 1.7.0

package/lib/import/module-importer.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+const utils_1 = require("../utils");
+const modules_1 = tslib_1.__importDefault(require("./modules"));
+const modules_js_1 = tslib_1.__importDefault(require("./modules-js"));
+class ModuleImporter {
+    constructor(managementAPIClient, importConfig) {
+        this.managementAPIClient = managementAPIClient;
+        this.stackAPIClient = this.managementAPIClient.stack({
+            api_key: importConfig.apiKey,
+            management_token: importConfig.management_token,
+        });
+        this.importConfig = importConfig;
+    }
+    async start() {
+        if (this.importConfig.branchName) {
+            await (0, utils_1.validateBranch)(this.stackAPIClient, this.importConfig, this.importConfig.branchName);
+        }
+        if (!this.importConfig.master_locale) {
+            let masterLocalResponse = await (0, utils_1.masterLocalDetails)(this.stackAPIClient);
+            this.importConfig['master_locale'] = { code: masterLocalResponse.code };
+            this.importConfig.masterLocale = { code: masterLocalResponse.code };
+        }
+        const backupDir = await (0, utils_1.backupHandler)(this.importConfig);
+        if (backupDir) {
+            this.importConfig.backupDir = backupDir;
+            // To support the old config
+            this.importConfig.data = backupDir;
+        }
+        await (0, utils_1.sanitizeStack)(this.stackAPIClient);
+        return this.import();
+    }
+    async import() {
+        // checks for single module or all modules
+        if (this.importConfig.singleModuleImport) {
+            return this.importByModuleByName(this.importConfig.moduleName);
+        }
+        return this.importAllModules();
+    }
+    async importByModuleByName(moduleName) {
+        (0, utils_1.log)(this.importConfig, `Starting import of ${moduleName} module`, 'info');
+        const basePath = `${this.importConfig.backupDir}/${moduleName}`;
+        // import the modules by name
+        // calls the module runner which inturn calls the module itself
+        // Todo: Implement a mechanism to determine whether module is new or old
+        if (this.importConfig.useNewModuleStructure &&
+            this.importConfig.updatedModules.indexOf(moduleName) !== -1
+        //&& new FsUtility({ basePath }).isNewFsStructure
+        ) {
+            return (0, modules_1.default)({
+                stackAPIClient: this.stackAPIClient,
+                importConfig: this.importConfig,
+                moduleName,
+            });
+        }
+        return (0, modules_js_1.default)({
+            stackAPIClient: this.stackAPIClient,
+            importConfig: this.importConfig,
+            moduleName,
+        });
+    }
+    async importAllModules() {
+        // use the algorithm to determine the parallel and sequential execution of modules
+        for (let moduleName of this.importConfig.modules.types) {
+            await this.importByModuleByName(moduleName);
+        }
+    }
+}
+exports.default = ModuleImporter;

package/lib/import/modules/assets.d.ts

@@ -0,0 +1,63 @@
+import BaseClass, { ApiOptions } from './base-class';
+import { ModuleClassParams } from '../../types';
+export default class ImportAssets extends BaseClass {
+    private fs;
+    private assetsPath;
+    private mapperDirPath;
+    private assetsRootPath;
+    private assetUidMapperPath;
+    private assetUrlMapperPath;
+    private assetFolderUidMapperPath;
+    assetConfig: {
+        dirName: string;
+        assetBatchLimit: number;
+        publishAssets: boolean;
+        fileName: string;
+        importSameStructure: boolean;
+        uploadAssetsConcurrency: number;
+        displayExecutionTime: boolean;
+        importFoldersConcurrency: number;
+        includeVersionedAssets: boolean;
+        host: string;
+        folderValidKeys: string[];
+        validKeys: string[];
+    };
+    private environments;
+    private assetsUidMap;
+    private assetsUrlMap;
+    private assetsFolderMap;
+    constructor({ importConfig, stackAPIClient }: ModuleClassParams);
+    /**
+     * @method start
+     * @returns {Promise<void>} Promise<any>
+     */
+    start(): Promise<void>;
+    /**
+     * @method importFolders
+     * @returns {Promise<any>} Promise<any>
+     */
+    importFolders(): Promise<any>;
+    /**
+     * @method importAssets
+     * @param {boolean} isVersion boolean
+     * @returns {Promise<void>} Promise<void>
+     */
+    importAssets(isVersion?: boolean): Promise<void>;
+    /**
+     * @method serializeAssets
+     * @param {ApiOptions} apiOptions ApiOptions
+     * @returns {ApiOptions} ApiOptions
+     */
+    serializeAssets(apiOptions: ApiOptions): ApiOptions;
+    /**
+     * @method publish
+     * @returns {Promise<void>} Promise<void>
+     */
+    publish(): Promise<void>;
+    /**
+     * @method constructFolderImportOrder
+     * @param {Record<string, any>[]} folders object
+     * @returns {Array<Record<string, any>>} Array<Record<string, any>>
+     */
+    constructFolderImportOrder(folders: any): Array<Record<string, any>>;
+}

package/lib/import/modules/assets.js

@@ -0,0 +1,265 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+const map_1 = tslib_1.__importDefault(require("lodash/map"));
+const values_1 = tslib_1.__importDefault(require("lodash/values"));
+const filter_1 = tslib_1.__importDefault(require("lodash/filter"));
+const unionBy_1 = tslib_1.__importDefault(require("lodash/unionBy"));
+const orderBy_1 = tslib_1.__importDefault(require("lodash/orderBy"));
+const isEmpty_1 = tslib_1.__importDefault(require("lodash/isEmpty"));
+const node_fs_1 = require("node:fs");
+const includes_1 = tslib_1.__importDefault(require("lodash/includes"));
+const node_path_1 = require("node:path");
+const cli_utilities_1 = require("@contentstack/cli-utilities");
+const config_1 = tslib_1.__importDefault(require("../../config"));
+const utils_1 = require("../../utils");
+const base_class_1 = tslib_1.__importDefault(require("./base-class"));
+class ImportAssets extends base_class_1.default {
+    constructor({ importConfig, stackAPIClient }) {
+        super({ importConfig, stackAPIClient });
+        this.assetConfig = config_1.default.modules.assets;
+        this.environments = {};
+        this.assetsUidMap = {};
+        this.assetsUrlMap = {};
+        this.assetsFolderMap = {};
+        this.assetsPath = (0, node_path_1.join)(this.importConfig.backupDir, 'assets');
+        this.mapperDirPath = (0, node_path_1.join)(this.importConfig.backupDir, 'mapper', 'assets');
+        this.assetUidMapperPath = (0, node_path_1.join)(this.mapperDirPath, 'uid-mapping.json');
+        this.assetUrlMapperPath = (0, node_path_1.join)(this.mapperDirPath, 'url-mapping.json');
+        this.assetFolderUidMapperPath = (0, node_path_1.join)(this.mapperDirPath, 'folder-mapping.json');
+        this.assetsRootPath = (0, node_path_1.join)(this.importConfig.backupDir, this.assetConfig.dirName);
+        this.fs = new cli_utilities_1.FsUtility({ basePath: this.mapperDirPath });
+        this.environments = this.fs.readFile((0, node_path_1.join)(this.importConfig.backupDir, 'environments', 'environments.json'), true);
+    }
+    /**
+     * @method start
+     * @returns {Promise<void>} Promise<any>
+     */
+    async start() {
+        // NOTE Step 1: Import folders and create uid mapping file
+        await this.importFolders();
+        // NOTE Step 2: Import versioned assets and create it mapping files (uid, url)
+        if (this.assetConfig.includeVersionedAssets) {
+            if ((0, node_fs_1.existsSync)(`${this.assetsPath}/versions`))
+                await this.importAssets(true);
+            else
+                (0, utils_1.log)(this.importConfig, 'No Versioned assets found to import', 'info');
+        }
+        // NOTE Step 3: Import Assets and create it mapping files (uid, url)
+        await this.importAssets();
+        // NOTE Step 4: Publish assets
+        if (this.assetConfig.publishAssets)
+            await this.publish();
+    }
+    /**
+     * @method importFolders
+     * @returns {Promise<any>} Promise<any>
+     */
+    async importFolders() {
+        const folders = this.fs.readFile((0, node_path_1.resolve)(this.assetsRootPath, 'folders.json'));
+        if ((0, isEmpty_1.default)(folders)) {
+            (0, utils_1.log)(this.importConfig, 'No folders found to import', 'info');
+            return;
+        }
+        const batches = this.constructFolderImportOrder(folders);
+        const onSuccess = ({ response, apiData: { uid, name } = { uid: null, name: '' } }) => {
+            this.assetsFolderMap[uid] = response.uid;
+            (0, utils_1.log)(this.importConfig, `Created folder: '${name}'`, 'success');
+        };
+        const onReject = ({ error, apiData: { name } = { name: '' } }) => {
+            (0, utils_1.log)(this.importConfig, `${name} folder creation failed.!`, 'error');
+            (0, utils_1.log)(this.importConfig, (0, utils_1.formatError)(error), 'error');
+        };
+        const serializeData = (apiOptions) => {
+            if (apiOptions.apiData.parent_uid) {
+                apiOptions.apiData.parent_uid = this.assetsFolderMap[apiOptions.apiData.parent_uid];
+            }
+            return apiOptions;
+        };
+        const batch = (0, map_1.default)((0, unionBy_1.default)(batches, 'parent_uid'), 'parent_uid');
+        for (const parent_uid of batch) {
+            // NOTE create parent folders
+            /* eslint-disable no-await-in-loop */
+            await this.makeConcurrentCall({
+                apiContent: (0, orderBy_1.default)((0, filter_1.default)(batches, { parent_uid }), 'created_at'),
+                processName: 'import assets folders',
+                apiParams: {
+                    serializeData,
+                    reject: onReject,
+                    resolve: onSuccess,
+                    entity: 'create-assets-folder',
+                    includeParamOnCompletion: true,
+                },
+                concurrencyLimit: this.assetConfig.importFoldersConcurrency,
+            }, undefined, false);
+        }
+        if (!(0, isEmpty_1.default)(this.assetsFolderMap)) {
+            this.fs.writeFile(this.assetFolderUidMapperPath, this.assetsFolderMap);
+        }
+    }
+    /**
+     * @method importAssets
+     * @param {boolean} isVersion boolean
+     * @returns {Promise<void>} Promise<void>
+     */
+    async importAssets(isVersion = false) {
+        const processName = isVersion ? 'import versioned assets' : 'import assets';
+        const indexFileName = isVersion ? 'versioned-assets.json' : 'assets.json';
+        const basePath = isVersion ? (0, node_path_1.join)(this.assetsPath, 'versions') : this.assetsPath;
+        const fs = new cli_utilities_1.FsUtility({ basePath, indexFileName });
+        const indexer = fs.indexFileContent;
+        const indexerCount = (0, values_1.default)(indexer).length;
+        const onSuccess = ({ response = {}, apiData: { uid, url, title } = undefined }) => {
+            this.assetsUidMap[uid] = response.uid;
+            this.assetsUrlMap[url] = response.url;
+            (0, utils_1.log)(this.importConfig, `Created asset: '${title}'`, 'info');
+        };
+        const onReject = ({ error, apiData: { title } = undefined }) => {
+            (0, utils_1.log)(this.importConfig, `${title} asset upload failed.!`, 'error');
+            (0, utils_1.log)(this.importConfig, (0, utils_1.formatError)(error), 'error');
+        };
+        /* eslint-disable @typescript-eslint/no-unused-vars, guard-for-in */
+        for (const index in indexer) {
+            const chunk = await fs.readChunkFiles.next().catch((error) => {
+                (0, utils_1.log)(this.importConfig, error, 'error');
+            });
+            if (chunk) {
+                let apiContent = (0, orderBy_1.default)((0, values_1.default)(chunk), '_version');
+                if (isVersion && this.assetConfig.importSameStructure) {
+                    // NOTE to create same structure it must have seed assets/version 1 asset to be created first
+                    await this.makeConcurrentCall({
+                        processName,
+                        indexerCount,
+                        currentIndexer: +index,
+                        apiContent: (0, filter_1.default)(apiContent, ({ _version }) => _version === 1),
+                        apiParams: {
+                            reject: onReject,
+                            resolve: onSuccess,
+                            entity: 'create-assets',
+                            includeParamOnCompletion: true,
+                            serializeData: this.serializeAssets.bind(this),
+                        },
+                        concurrencyLimit: this.assetConfig.uploadAssetsConcurrency,
+                    });
+                    apiContent = (0, filter_1.default)(apiContent, ({ _version }) => _version > 1);
+                }
+                await this.makeConcurrentCall({
+                    apiContent,
+                    processName,
+                    indexerCount,
+                    currentIndexer: +index,
+                    apiParams: {
+                        reject: onReject,
+                        resolve: onSuccess,
+                        entity: 'create-assets',
+                        includeParamOnCompletion: true,
+                        serializeData: this.serializeAssets.bind(this),
+                    },
+                    concurrencyLimit: this.assetConfig.uploadAssetsConcurrency,
+                }, undefined, !isVersion);
+            }
+        }
+        if (!isVersion && (!(0, isEmpty_1.default)(this.assetsUidMap) || !(0, isEmpty_1.default)(this.assetsUrlMap))) {
+            this.fs.writeFile(this.assetUidMapperPath, this.assetsUidMap);
+            this.fs.writeFile(this.assetUrlMapperPath, this.assetsUrlMap);
+        }
+    }
+    /**
+     * @method serializeAssets
+     * @param {ApiOptions} apiOptions ApiOptions
+     * @returns {ApiOptions} ApiOptions
+     */
+    serializeAssets(apiOptions) {
+        const { apiData: asset } = apiOptions;
+        if (!this.assetConfig.importSameStructure &&
+            !this.assetConfig.includeVersionedAssets &&
+            /* eslint-disable @typescript-eslint/no-unused-vars, no-prototype-builtins */
+            this.assetsUidMap.hasOwnProperty(asset.uid)) {
+            (0, utils_1.log)(this.importConfig, `Skipping upload of asset: ${asset.uid}. Its mapped to: ${this.assetsUidMap[asset.uid]}`, 'success');
+            apiOptions.entity = undefined;
+            return apiOptions;
+        }
+        asset.upload = (0, node_path_1.join)(this.assetsPath, 'files', asset.uid, asset.filename);
+        if (asset.parent_uid) {
+            asset.parent_uid = this.assetsFolderMap[asset.parent_uid];
+        }
+        apiOptions.apiData = asset;
+        if (this.assetsUidMap[asset.uid] && this.assetConfig.importSameStructure) {
+            apiOptions.entity = 'replace-assets';
+            apiOptions.uid = this.assetsUidMap[asset.uid];
+        }
+        return apiOptions;
+    }
+    /**
+     * @method publish
+     * @returns {Promise<void>} Promise<void>
+     */
+    async publish() {
+        const fs = new cli_utilities_1.FsUtility({ basePath: this.assetsPath, indexFileName: 'assets.json' });
+        if ((0, isEmpty_1.default)(this.assetsUidMap)) {
+            this.assetsUidMap = fs.readFile(this.assetUidMapperPath, true);
+        }
+        const indexer = fs.indexFileContent;
+        const indexerCount = (0, values_1.default)(indexer).length;
+        const onSuccess = ({ apiData: { uid, title } = undefined }) => {
+            (0, utils_1.log)(this.importConfig, `Asset '${uid}: ${title}' published successfully`, 'success');
+        };
+        const onReject = ({ error, apiData: { uid, title } = undefined }) => {
+            (0, utils_1.log)(this.importConfig, `Asset '${uid}: ${title}' not published`, 'error');
+            (0, utils_1.log)(this.importConfig, (0, utils_1.formatError)(error), 'error');
+        };
+        const serializeData = (apiOptions) => {
+            const { apiData: asset } = apiOptions;
+            const publishDetails = (0, filter_1.default)(asset.publish_details, 'environment');
+            const locales = (0, map_1.default)(publishDetails, 'locale');
+            const environments = (0, map_1.default)(publishDetails, ({ environment }) => this.environments[environment].name);
+            asset.locales = locales;
+            asset.environments = environments;
+            apiOptions.uid = this.assetsUidMap[asset.uid];
+            apiOptions.apiData.publishDetails = { locales, environments };
+            if (!apiOptions.uid)
+                apiOptions.entity = undefined;
+            return apiOptions;
+        };
+        /* eslint-disable @typescript-eslint/no-unused-vars */
+        for (const index in indexer) {
+            const apiContent = (0, filter_1.default)((0, values_1.default)(await fs.readChunkFiles.next()), ({ publish_details }) => !(0, isEmpty_1.default)(publish_details));
+            await this.makeConcurrentCall({
+                apiContent,
+                indexerCount,
+                currentIndexer: +index,
+                processName: 'assets publish',
+                apiParams: {
+                    serializeData,
+                    reject: onReject,
+                    resolve: onSuccess,
+                    entity: 'publish-assets',
+                    includeParamOnCompletion: true,
+                },
+                concurrencyLimit: this.assetConfig.uploadAssetsConcurrency,
+            });
+        }
+    }
+    /**
+     * @method constructFolderImportOrder
+     * @param {Record<string, any>[]} folders object
+     * @returns {Array<Record<string, any>>} Array<Record<string, any>>
+     */
+    constructFolderImportOrder(folders) {
+        let parentUid = [];
+        // NOTE: Read root folder
+        const importOrder = (0, filter_1.default)(folders, { parent_uid: null }).map(({ uid, name, parent_uid, created_at }) => {
+            parentUid.push(uid);
+            return { uid, name, parent_uid, created_at };
+        });
+        while (!(0, isEmpty_1.default)(parentUid)) {
+            // NOTE: Read nested folders every iteration until we find empty folders
+            parentUid = (0, filter_1.default)(folders, ({ parent_uid }) => (0, includes_1.default)(parentUid, parent_uid)).map(({ uid, name, parent_uid, created_at }) => {
+                importOrder.push({ uid, name, parent_uid, created_at });
+                return uid;
+            });
+        }
+        return importOrder;
+    }
+}
+exports.default = ImportAssets;

package/lib/import/modules/base-class.d.ts

@@ -0,0 +1,70 @@
+import { Stack } from '@contentstack/management/types/stack';
+import { ImportConfig, ModuleClassParams } from '../../types';
+export type AdditionalKeys = {
+    backupDir: string;
+};
+export type ApiModuleType = 'create-assets' | 'replace-assets' | 'publish-assets' | 'create-assets-folder' | 'create-extensions' | 'create-locale' | 'update-locale' | 'create-gfs' | 'create-cts' | 'update-cts' | 'update-gfs' | 'create-environments' | 'create-labels' | 'update-labels' | 'create-webhooks' | 'create-workflows' | 'create-custom-role';
+export type ApiOptions = {
+    uid?: string;
+    url?: string;
+    entity: ApiModuleType;
+    apiData?: Record<any, any> | any;
+    resolve: (value: any) => void;
+    reject: (error: any) => void;
+    additionalInfo?: Record<any, any>;
+    includeParamOnCompletion?: boolean;
+    serializeData?: (input: ApiOptions) => any;
+};
+export type EnvType = {
+    processName: string;
+    totalCount?: number;
+    indexerCount?: number;
+    currentIndexer?: number;
+    apiParams?: ApiOptions;
+    concurrencyLimit?: number;
+    apiContent: Record<string, any>[];
+};
+export type CustomPromiseHandlerInput = {
+    index: number;
+    batchIndex: number;
+    element?: Record<string, unknown>;
+    apiParams?: ApiOptions;
+    isLastRequest: boolean;
+};
+export type CustomPromiseHandler = (input: CustomPromiseHandlerInput) => Promise<any>;
+export default abstract class BaseClass {
+    readonly client: Stack;
+    importConfig: ImportConfig;
+    modulesConfig: any;
+    constructor({ importConfig, stackAPIClient }: Omit<ModuleClassParams, 'moduleName'>);
+    get stack(): Stack;
+    /**
+     * @method delay
+     * @param {number} ms number
+     * @returns {Promise} Promise<void>
+     */
+    delay(ms: number): Promise<void>;
+    /**
+     * @method makeConcurrentCall
+     * @param {Record<string, any>} env EnvType
+     * @param {CustomPromiseHandler} promisifyHandler CustomPromiseHandler
+     * @param {boolean} logBatchCompletionMsg boolean
+     * @returns {Promise} Promise<void>
+     */
+    makeConcurrentCall(env: EnvType, promisifyHandler?: CustomPromiseHandler, logBatchCompletionMsg?: boolean): Promise<void>;
+    /**
+     * @method logMsgAndWaitIfRequired
+     * @param {string} processName string
+     * @param {number} start number
+     * @param {number} batchNo - number
+     * @returns {Promise} Promise<void>
+     */
+    logMsgAndWaitIfRequired(processName: string, start: number, totelBatches: number, batchNo: number, logBatchCompletionMsg?: boolean, indexerCount?: number, currentIndexer?: number): Promise<void>;
+    /**
+     * @method makeAPICall
+     * @param {Record<string, any>} apiOptions - Api related params
+     * @param {Record<string, any>} isLastRequest - Boolean
+     * @return {Promise} Promise<void>
+     */
+    makeAPICall(apiOptions: ApiOptions, isLastRequest?: boolean): Promise<void>;
+}