@contentstack/cli-cm-import 1.5.10 → 1.6.0

package/lib/import/modules-js/webhooks.js

@@ -0,0 +1,85 @@
+/*!
+ * Contentstack Import
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
+const mkdirp = require('mkdirp');
+const fs = require('fs');
+const path = require('path');
+const Promise = require('bluebird');
+const chalk = require('chalk');
+const { isEmpty, merge } = require('lodash');
+let { default: config } = require('../../config');
+const { fileHelper, log, formatError } = require('../../utils');
+module.exports = class ImportWebhooks {
+    constructor(importConfig, stackAPIClient) {
+        this.fails = [];
+        this.success = [];
+        this.webUidMapper = {};
+        this.webhooksConfig = config.modules.webhooks;
+        this.reqConcurrency = config.concurrency || config.fetchConcurrency;
+        this.config = merge(config, importConfig);
+        this.stackAPIClient = stackAPIClient;
+    }
+    start() {
+        log(this.config, chalk.white('Migrating webhooks'), 'success');
+        const self = this;
+        let webMapperPath = path.resolve(this.config.data, 'mapper', 'webhooks');
+        let webFailsPath = path.resolve(this.config.data, 'mapper', 'webhooks', 'fails.json');
+        let webSuccessPath = path.resolve(this.config.data, 'mapper', 'webhooks', 'success.json');
+        let webUidMapperPath = path.resolve(this.config.data, 'mapper', 'webhooks', 'uid-mapping.json');
+        let webhooksFolderPath = path.resolve(this.config.data, this.webhooksConfig.dirName);
+        this.webhooks = fileHelper.readFileSync(path.resolve(webhooksFolderPath, this.webhooksConfig.fileName));
+        if (fs.existsSync(webUidMapperPath)) {
+            self.webUidMapper = fileHelper.readFileSync(webUidMapperPath);
+            self.webUidMapper = self.webUidMapper || {};
+        }
+        mkdirp.sync(webMapperPath);
+        return new Promise(function (resolve, reject) {
+            if (self.webhooks == undefined || isEmpty(self.webhooks)) {
+                log(self.config, chalk.white('No Webhooks Found'), 'success');
+                return resolve({ empty: true });
+            }
+            let webUids = Object.keys(self.webhooks);
+            return Promise.map(webUids, function (webUid) {
+                let web = self.webhooks[webUid];
+                if (self.config.importWebhookStatus !== 'current' || self.config.importWebhookStatus === 'disable') {
+                    web.disabled = true;
+                }
+                if (!self.webUidMapper.hasOwnProperty(webUid)) {
+                    let requestOption = { json: { webhook: web } };
+                    return self.stackAPIClient
+                        .webhook()
+                        .create(requestOption.json)
+                        .then(function (response) {
+                        self.success.push(response);
+                        self.webUidMapper[webUid] = response.uid;
+                        fileHelper.writeFileSync(webUidMapperPath, self.webUidMapper);
+                    })
+                        .catch(function (err) {
+                        let error = JSON.parse(err.message);
+                        self.fails.push(web);
+                        log(self.config, `Webhooks '${web.name}' failed to be import.\n ${formatError(error)}`, 'error');
+                    });
+                }
+                else {
+                    // the webhooks has already been created
+                    log(self.config, chalk.white("The Webhooks: '" + web.name + "' already exists. Skipping it to avoid duplicates!"), 'success');
+                }
+                // import 2 webhooks at a time
+            }, { concurrency: self.reqConcurrency })
+                .then(function () {
+                // webhooks have imported successfully
+                fileHelper.writeFileSync(webSuccessPath, self.success);
+                log(self.config, chalk.green('Webhooks have been imported successfully!'), 'success');
+                return resolve();
+            })
+                .catch(function (error) {
+                // error while importing environments
+                fileHelper.writeFileSync(webFailsPath, self.fails);
+                log(self.config, `Webhooks import failed. ${formatError(error)}`, 'error');
+                return reject(error);
+            });
+        });
+    }
+};

package/lib/import/modules-js/workflows.d.ts

@@ -0,0 +1,18 @@
+export = importWorkflows;
+declare class importWorkflows {
+    constructor(importConfig: any, stackAPIClient: any);
+    fails: any[];
+    success: any[];
+    workflowUidMapper: {};
+    workflowConfig: {
+        dirName: string;
+        fileName: string;
+        invalidKeys: string[];
+    };
+    reqConcurrency: number;
+    config: any;
+    stackAPIClient: any;
+    start(): Promise<any>;
+    workflows: any;
+}
+import Promise = require("bluebird");

package/lib/import/modules-js/workflows.js

@@ -0,0 +1,132 @@
+/*!
+ * Contentstack Import
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
+const mkdirp = require('mkdirp');
+const fs = require('fs');
+const path = require('path');
+const Promise = require('bluebird');
+const chalk = require('chalk');
+const { isEmpty, merge } = require('lodash');
+let { default: config } = require('../../config');
+const { fileHelper, log, formatError } = require('../../utils');
+module.exports = class importWorkflows {
+    constructor(importConfig, stackAPIClient) {
+        this.fails = [];
+        this.success = [];
+        this.workflowUidMapper = {};
+        this.workflowConfig = config.modules.workflows;
+        this.reqConcurrency = config.concurrency || config.fetchConcurrency || 1;
+        this.config = merge(config, importConfig);
+        this.stackAPIClient = stackAPIClient;
+    }
+    start() {
+        log(this.config, chalk.white('Migrating workflows'), 'success');
+        let self = this;
+        let workflowMapperPath = path.resolve(this.config.data, 'mapper', 'workflows');
+        let workflowFailsPath = path.resolve(this.config.data, 'workflows', 'fails.json');
+        let workflowSuccessPath = path.resolve(this.config.data, 'workflows', 'success.json');
+        let workflowUidMapperPath = path.resolve(this.config.data, 'mapper', 'workflows', 'uid-mapping.json');
+        let workflowFolderPath = path.resolve(this.config.data, this.workflowConfig.dirName);
+        self.workflows = fileHelper.readFileSync(path.resolve(workflowFolderPath, this.workflowConfig.fileName));
+        if (fs.existsSync(workflowUidMapperPath)) {
+            this.workflowUidMapper = fileHelper.readFileSync(workflowUidMapperPath);
+            this.workflowUidMapper = this.workflowUidMapper || {};
+        }
+        mkdirp.sync(workflowMapperPath);
+        return new Promise(function (resolve, reject) {
+            if (self.workflows == undefined || isEmpty(self.workflows)) {
+                log(self.config, chalk.white('No workflow Found'), 'success');
+                return resolve({ empty: true });
+            }
+            self.workflowsUids = Object.keys(self.workflows);
+            return Promise.map(self.workflowsUids, async function (workflowUid) {
+                let workflow = self.workflows[workflowUid];
+                if (!self.workflowUidMapper.hasOwnProperty(workflowUid)) {
+                    const roleNameMap = {};
+                    const workflowStages = workflow.workflow_stages;
+                    const roles = await self.stackAPIClient.role().fetchAll();
+                    for (const role of roles.items) {
+                        roleNameMap[role.name] = role.uid;
+                    }
+                    for (const stage of workflowStages) {
+                        if (stage.SYS_ACL.users.uids.length && stage.SYS_ACL.users.uids[0] !== '$all') {
+                            stage.SYS_ACL.users.uids = ['$all'];
+                        }
+                        if (stage.SYS_ACL.roles.uids.length) {
+                            try {
+                                for (let i = 0; i < stage.SYS_ACL.roles.uids.length; i++) {
+                                    const roleData = stage.SYS_ACL.roles.uids[i];
+                                    if (!roleNameMap[roleData.name]) {
+                                        // rules.branch is required to create custom roles.
+                                        const branchRuleExists = roleData.rules.find((rule) => rule.module === 'branch');
+                                        if (!branchRuleExists) {
+                                            roleData.rules.push({
+                                                module: 'branch',
+                                                branches: ['main'],
+                                                acl: { read: true },
+                                            });
+                                        }
+                                        const role = await self.stackAPIClient.role().create({ role: roleData });
+                                        stage.SYS_ACL.roles.uids[i] = role.uid;
+                                        roleNameMap[roleData.name] = role.uid;
+                                    }
+                                    else {
+                                        stage.SYS_ACL.roles.uids[i] = roleNameMap[roleData.name];
+                                    }
+                                }
+                            }
+                            catch (error) {
+                                log(self.config, `Error while importing workflows roles. ${formatError(error)}`, 'error');
+                                reject({ message: 'Error while importing workflows roles' });
+                            }
+                        }
+                    }
+                    if (workflow.admin_users !== undefined) {
+                        log(self.config, chalk.yellow('We are skipping import of `Workflow superuser(s)` from workflow'), 'info');
+                        delete workflow.admin_users;
+                    }
+                    // One branch is required to create workflow.
+                    if (!workflow.branches) {
+                        workflow.branches = ['main'];
+                    }
+                    return self.stackAPIClient
+                        .workflow()
+                        .create({ workflow })
+                        .then(function (response) {
+                        self.workflowUidMapper[workflowUid] = response;
+                        fileHelper.writeFileSync(workflowUidMapperPath, self.workflowUidMapper);
+                    })
+                        .catch(function (error) {
+                        self.fails.push(workflow);
+                        if (error.errors.name) {
+                            log(self.config, `workflow ${workflow.name} already exist`, 'error');
+                        }
+                        else if (error.errors['workflow_stages.0.users']) {
+                            log(self.config, "Failed to import Workflows as you've specified certain roles in the Stage transition and access rules section. We currently don't import roles to the stack.", 'error');
+                        }
+                        else {
+                            log(self.config, `workflow ${workflow.name} failed.`, 'error');
+                        }
+                    });
+                }
+                else {
+                    // the workflow has already been created
+                    log(self.config, chalk.white(`The Workflows ${workflow.name} already exists. Skipping it to avoid duplicates!`), 'success');
+                }
+                // import 1 workflows at a time
+            }, { concurrency: self.reqConcurrency })
+                .then(function () {
+                fileHelper.writeFileSync(workflowSuccessPath, self.success);
+                log(self.config, chalk.green('Workflows have been imported successfully!'), 'success');
+                resolve();
+            })
+                .catch(function (error) {
+                fileHelper.writeFileSync(workflowFailsPath, self.fails);
+                log(self.config, `Workflows import failed. ${formatError(error)}`, 'error');
+                return reject(error);
+            });
+        });
+    }
+};

package/lib/types/default-config.d.ts

@@ -0,0 +1,130 @@
+import { Modules } from '.';
+export default interface DefaultConfig {
+    versioning: boolean;
+    host: string;
+    extensionHost: string;
+    developerHubUrls: Record<string, string>;
+    modules: {
+        apiConcurrency: number;
+        types: Modules[];
+        locales: {
+            dirName: string;
+            fileName: string;
+            requiredKeys: string[];
+        };
+        customRoles: {
+            dirName: string;
+            fileName: string;
+            customRolesLocalesFileName: string;
+        };
+        environments: {
+            dirName: string;
+            fileName: string;
+        };
+        labels: {
+            dirName: string;
+            fileName: string;
+        };
+        extensions: {
+            dirName: string;
+            fileName: string;
+        };
+        webhooks: {
+            dirName: string;
+            fileName: string;
+        };
+        releases: {
+            dirName: string;
+            fileName: string;
+            invalidKeys: string[];
+        };
+        workflows: {
+            dirName: string;
+            fileName: string;
+            invalidKeys: string[];
+        };
+        assets: {
+            dirName: string;
+            assetBatchLimit: number;
+            publishAssets: boolean;
+            fileName: string;
+            importSameStructure: boolean;
+            uploadAssetsConcurrency: number;
+            displayExecutionTime: boolean;
+            importFoldersConcurrency: number;
+            includeVersionedAssets: boolean;
+            host: string;
+            folderValidKeys: string[];
+            validKeys: string[];
+        };
+        'assets-old': {
+            dirName: string;
+            fileName: string;
+            limit: number;
+            host: string;
+            validKeys: string[];
+            assetBatchLimit: number;
+            uploadAssetsConcurrency: number;
+            importFoldersConcurrency: number;
+        };
+        content_types: {
+            dirName: string;
+            fileName: string;
+            validKeys: string[];
+            limit: number;
+        };
+        entries: {
+            dirName: string;
+            fileName: string;
+            invalidKeys: string[];
+            limit: number;
+            assetBatchLimit: number;
+        };
+        globalfields: {
+            dirName: string;
+            fileName: string;
+            validKeys: string[];
+            limit: number;
+        };
+        stack: {
+            dirName: string;
+            fileName: string;
+        };
+        marketplace_apps: {
+            dirName: string;
+            fileName: string;
+        };
+        masterLocale: {
+            dirName: string;
+            fileName: string;
+            requiredKeys: string[];
+        };
+    };
+    languagesCode: string[];
+    apis: {
+        userSession: string;
+        locales: string;
+        environments: string;
+        assets: string;
+        content_types: string;
+        entries: string;
+        extensions: string;
+        webhooks: string;
+        globalfields: string;
+        folders: string;
+        stacks: string;
+        labels: string;
+    };
+    updatedModules: string[];
+    rateLimit: number;
+    preserveStackVersion: boolean;
+    entriesPublish: boolean;
+    concurrency: number;
+    importConcurrency: number;
+    fetchConcurrency: number;
+    writeConcurrency: number;
+    developerHubBaseUrl: string;
+    marketplaceAppEncryptionKey: string;
+    getEncryptionKeyMaxRetry: number;
+    useNewModuleStructure: boolean;
+}

package/lib/types/default-config.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/lib/types/import-config.d.ts

@@ -0,0 +1,51 @@
+import { Modules } from '.';
+import DefaultConfig from './default-config';
+export interface ExternalConfig {
+    source_stack?: string;
+    data: string;
+    fetchConcurrency: number;
+    writeConcurrency: number;
+    email?: string;
+    password?: string;
+}
+export default interface ImportConfig extends DefaultConfig, ExternalConfig {
+    contentDir: string;
+    data: string;
+    management_token?: string;
+    apiKey: string;
+    forceStopMarketplaceAppsPrompt: boolean;
+    auth_token?: string;
+    branchName?: string;
+    securedAssets?: boolean;
+    contentTypes?: string[];
+    branches?: branch[];
+    branchEnabled?: boolean;
+    branchDir?: string;
+    moduleName?: Modules;
+    master_locale: masterLocale;
+    headers?: {
+        api_key: string;
+        access_token?: string;
+        authtoken?: string;
+        'X-User-Agent': string;
+    };
+    access_token?: string;
+    isAuthenticated?: boolean;
+    importWebhookStatus?: string;
+    target_stack?: string;
+    singleModuleImport?: boolean;
+    useBackedupDir?: string;
+    masterLocale: masterLocale;
+    backupDir: string;
+    backupConcurrency?: number;
+    authtoken?: string;
+    destinationStackName?: string;
+}
+type branch = {
+    uid: string;
+    source: string;
+};
+type masterLocale = {
+    code: string;
+};
+export {};

package/lib/types/import-config.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/lib/types/index.d.ts

@@ -0,0 +1,30 @@
+import { ContentstackClient } from '@contentstack/cli-utilities';
+import ImportConfig from './import-config';
+export interface AuthOptions {
+    contentstackClient: ContentstackClient;
+}
+export interface ContentStackManagementClient {
+    contentstackClient: object;
+}
+export interface PrintOptions {
+    color?: string;
+}
+export interface InquirePayload {
+    type: string;
+    name: string;
+    message: string;
+    choices?: Array<any>;
+    transformer?: Function;
+}
+export interface User {
+    email: string;
+    authtoken: string;
+}
+export type Modules = 'stack' | 'assets' | 'locales' | 'environments' | 'extensions' | 'webhooks' | 'global-fields' | 'entries' | 'content-types' | 'custom-roles' | 'workflows' | 'labels' | 'marketplace-apps';
+export type ModuleClassParams = {
+    stackAPIClient: ReturnType<ContentstackClient['stack']>;
+    importConfig: ImportConfig;
+    moduleName: Modules;
+};
+export { default as DefaultConfig } from './default-config';
+export { default as ImportConfig } from './import-config';

package/lib/types/index.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/lib/utils/asset-helper.d.ts

@@ -0,0 +1,4 @@
+import Bluebird from 'bluebird';
+import { ImportConfig } from 'src/types';
+export declare const uploadAssetHelper: (config: ImportConfig, req: any, fsPath: string, RETRY?: number) => Bluebird<unknown>;
+export declare const lookupAssets: (data: any, mappedAssetUids: string[], mappedAssetUrls: string[], assetUidMapperPath: string[], installedExtensions: string[]) => any;