@contentstack/cli-cm-import 1.0.1 → 1.2.0

package/src/lib/import/workflows.js

@@ -9,130 +9,170 @@ const fs = require('fs');
 const path = require('path');
 const Promise = require('bluebird');
 const chalk = require('chalk');
-const {isEmpty} = require('lodash');
+const { isEmpty, merge } = require('lodash');
 
 const helper = require('../util/fs');
+const { formatError } = require('../util');
 const { addlogs } = require('../util/log');
 let config = require('../../config/default');
 let stack = require('../util/contentstack-management-sdk');
 
-let reqConcurrency = config.concurrency;
-let workflowConfig = config.modules.workflows;
-let workflowFolderPath;
-let workflowMapperPath;
-let workflowUidMapperPath;
-let workflowSuccessPath;
-let workflowFailsPath;
-let client;
-
-function importWorkflows() {
-  this.fails = [];
-  this.success = [];
-  this.workflowUidMapper = {};
-  this.labelUids = [];
-  if (fs.existsSync(workflowUidMapperPath)) {
-    this.workflowUidMapper = helper.readFile(workflowUidMapperPath);
-    this.workflowUidMapper = this.workflowUidMapper || {};
+module.exports = class importWorkflows {
+  client;
+  fails = [];
+  success = [];
+  workflowUidMapper = {};
+  workflowConfig = config.modules.workflows;
+  reqConcurrency = config.concurrency || config.fetchConcurrency || 1;
+
+  constructor(credentialConfig) {
+    this.config = merge(config, credentialConfig);
+    this.client = stack.Client(this.config);
   }
-}
 
-importWorkflows.prototype = {
-  start: function (credentialConfig) {
+  start() {
+    addlogs(this.config, chalk.white('Migrating workflows'), 'success');
+
     let self = this;
-    config = credentialConfig;
-    client = stack.Client(config);
-    addlogs(config, chalk.white('Migrating workflows'), 'success');
-    workflowFolderPath = path.resolve(config.data, workflowConfig.dirName);
-    self.workflows = helper.readFile(path.resolve(workflowFolderPath, workflowConfig.fileName));
-    workflowMapperPath = path.resolve(config.data, 'mapper', 'workflows');
-    workflowUidMapperPath = path.resolve(config.data, 'mapper', 'workflows', 'uid-mapping.json');
-    workflowSuccessPath = path.resolve(config.data, 'workflows', 'success.json');
-    workflowFailsPath = path.resolve(config.data, 'workflows', 'fails.json');
+    let workflowMapperPath = path.resolve(this.config.data, 'mapper', 'workflows');
+    let workflowFailsPath = path.resolve(this.config.data, 'workflows', 'fails.json');
+    let workflowSuccessPath = path.resolve(this.config.data, 'workflows', 'success.json');
+    let workflowUidMapperPath = path.resolve(this.config.data, 'mapper', 'workflows', 'uid-mapping.json');
+    let workflowFolderPath = path.resolve(this.config.data, this.workflowConfig.dirName);
+
+    self.workflows = helper.readFileSync(path.resolve(workflowFolderPath, this.workflowConfig.fileName));
+
+    if (fs.existsSync(workflowUidMapperPath)) {
+      this.workflowUidMapper = helper.readFileSync(workflowUidMapperPath);
+      this.workflowUidMapper = this.workflowUidMapper || {};
+    }
+
     mkdirp.sync(workflowMapperPath);
+
     return new Promise(function (resolve, reject) {
       if (self.workflows == undefined || isEmpty(self.workflows)) {
-        addlogs(config, chalk.white('No workflow Found'), 'success');
+        addlogs(self.config, chalk.white('No workflow Found'), 'success');
         return resolve({ empty: true });
       }
       self.workflowsUids = Object.keys(self.workflows);
       return Promise.map(
         self.workflowsUids,
-        function (workflowUid) {
+        async function (workflowUid) {
           let workflow = self.workflows[workflowUid];
 
           if (!self.workflowUidMapper.hasOwnProperty(workflowUid)) {
-            const workflowStages = workflow.workflow_stages
+            const roleNameMap = {};
+            const workflowStages = workflow.workflow_stages;
+            const roles = await self.client
+              .stack({ api_key: self.config.target_stack, management_token: self.config.management_token })
+              .role()
+              .fetchAll();
+
+            for (const role of roles.items) {
+              roleNameMap[role.name] = role.uid;
+            }
+
             for (const stage of workflowStages) {
-              if (
-                stage.SYS_ACL.users.uids.length > 0 &&
-                stage.SYS_ACL.users.uids[0] !== '$all'
-              ) {
+              if (stage.SYS_ACL.users.uids.length && stage.SYS_ACL.users.uids[0] !== '$all') {
                 stage.SYS_ACL.users.uids = ['$all'];
               }
 
-              if (stage.SYS_ACL.roles.uids.length > 0) {
-                stage.SYS_ACL.roles.uids = [];
+              if (stage.SYS_ACL.roles.uids.length) {
+                try {
+                  for (let i = 0; i < stage.SYS_ACL.roles.uids.length; i++) {
+                    const roleData = stage.SYS_ACL.roles.uids[i];
+                    if (!roleNameMap[roleData.name]) {
+                      // rules.branch is required to create custom roles.
+                      const branchRuleExists = roleData.rules.find((rule) => rule.module === 'branch');
+                      if (!branchRuleExists) {
+                        roleData.rules.push({
+                          module: 'branch',
+                          branches: ['main'],
+                          acl: { read: true },
+                        });
+                      }
+
+                      const role = await self.client
+                        .stack({ api_key: self.config.target_stack, management_token: self.config.management_token })
+                        .role()
+                        .create({ role: roleData });
+                      stage.SYS_ACL.roles.uids[i] = role.uid;
+                      roleNameMap[roleData.name] = role.uid;
+                    } else {
+                      stage.SYS_ACL.roles.uids[i] = roleNameMap[roleData.name];
+                    }
+                  }
+                } catch (error) {
+                  addlogs(
+                    self.config,
+                    chalk.red('Error while importing workflows roles. ' + formatError(error)),
+                    'error',
+                  );
+                  reject({ message: 'Error while importing workflows roles' });
+                }
               }
             }
 
             if (workflow.admin_users !== undefined) {
-              addlogs(config, chalk.yellow('We are skipping import of `Workflow superuser(s)` from workflow'), 'info');
+              addlogs(
+                self.config,
+                chalk.yellow('We are skipping import of `Workflow superuser(s)` from workflow'),
+                'info',
+              );
               delete workflow.admin_users;
             }
+            // One branch is required to create workflow.
+            if (!workflow.branches) {
+              workflow.branches = ['main'];
+            }
 
-            let requestOption = {
-              workflow,
-            };
-
-            return client
-              .stack({ api_key: config.target_stack, management_token: config.management_token })
+            return self.client
+              .stack({ api_key: self.config.target_stack, management_token: self.config.management_token })
               .workflow()
-              .create(requestOption)
+              .create({ workflow })
               .then(function (response) {
                 self.workflowUidMapper[workflowUid] = response;
-                helper.writeFile(workflowUidMapperPath, self.workflowUidMapper);
+                helper.writeFileSync(workflowUidMapperPath, self.workflowUidMapper);
               })
               .catch(function (error) {
                 self.fails.push(workflow);
                 if (error.errors.name) {
-                  addlogs(config, chalk.red("workflow: '" + workflow.name + "'  already exist"), 'error');
+                  addlogs(self.config, chalk.red("workflow: '" + workflow.name + "'  already exist"), 'error');
                 } else if (error.errors['workflow_stages.0.users']) {
                   addlogs(
-                    config,
+                    self.config,
                     chalk.red(
                       "Failed to import Workflows as you've specified certain roles in the Stage transition and access rules section. We currently don't import roles to the stack.",
                     ),
                     'error',
                   );
                 } else {
-                  addlogs(config, chalk.red("workflow: '" + workflow.name + "'  failed"), 'error');
+                  addlogs(self.config, chalk.red("workflow: '" + workflow.name + "'  failed"), 'error');
                 }
               });
           } else {
             // the workflow has already been created
             addlogs(
-              config,
+              self.config,
               chalk.white("The Workflows: '" + workflow.name + "' already exists. Skipping it to avoid duplicates!"),
               'success',
             );
           }
           // import 1 workflows at a time
         },
-        {
-          concurrency: reqConcurrency,
-        },
+        { concurrency: self.reqConcurrency },
       )
         .then(function () {
-          helper.writeFile(workflowSuccessPath, self.success);
-          addlogs(config, chalk.green('Workflows have been imported successfully!'), 'success');
-          return resolve();
+          helper.writeFileSync(workflowSuccessPath, self.success);
+          addlogs(self.config, chalk.green('Workflows have been imported successfully!'), 'success');
+          resolve();
         })
         .catch(function (error) {
-          helper.writeFile(workflowFailsPath, self.fails);
-          addlogs(config, chalk.red('Workflows import failed'), 'error');
+          helper.writeFileSync(workflowFailsPath, self.fails);
+          addlogs(self.config, chalk.red('Workflows import failed'), 'error');
+          addlogs(self.config, formatError(error), 'error');
           return reject(error);
         });
     });
-  },
+  }
 };
-module.exports = new importWorkflows();

package/src/lib/util/extensionsUidReplace.js

@@ -6,23 +6,25 @@
 
 // eslint-disable-next-line unicorn/filename-case
 let path = require('path');
+const _ = require('lodash');
 let helper = require('./fs');
 let util = require('../util');
 let config = util.getConfig();
 let extensionPath = path.resolve(config.data, 'mapper/extensions', 'uid-mapping.json');
 let globalfieldsPath = path.resolve(config.data, 'mapper/globalfields', 'uid-mapping.json');
+const marketplaceAppPath = path.resolve(config.data, 'marketplace_apps', 'marketplace_apps.json');
 
 // eslint-disable-next-line camelcase
-let extension_uid_Replace = (module.exports = function (schema, preserveStackVersion) {
+let extension_uid_Replace = (module.exports = function (schema, preserveStackVersion, installedExtensions) {
   for (let i in schema) {
     if (schema[i].data_type === 'group') {
-      extension_uid_Replace(schema[i].schema, preserveStackVersion);
+      extension_uid_Replace(schema[i].schema, preserveStackVersion, installedExtensions);
     } else if (schema[i].data_type === 'blocks') {
       for (let block in schema[i].blocks) {
         if (schema[i].blocks[block].hasOwnProperty('reference_to')) {
           delete schema[i].blocks[block].schema;
         } else {
-          extension_uid_Replace(schema[i].blocks[block].schema, preserveStackVersion);
+          extension_uid_Replace(schema[i].blocks[block].schema, preserveStackVersion, installedExtensions);
         }
       }
     } else if (
@@ -37,20 +39,33 @@ let extension_uid_Replace = (module.exports = function (schema, preserveStackVer
       }
     } else if (schema[i].data_type === 'global_field') {
       let global_fields_key_value = schema[i].reference_to;
-      let global_fields_data = helper.readFile(path.join(globalfieldsPath));
+      let global_fields_data = helper.readFileSync(path.join(globalfieldsPath));
       if (global_fields_data && global_fields_data.hasOwnProperty(global_fields_key_value)) {
         schema[i].reference_to = global_fields_data[global_fields_key_value];
       }
     } else if (schema[i].hasOwnProperty('extension_uid')) {
       const extension_key_value = schema[i].extension_uid;
-      const data = helper.readFile(path.join(extensionPath));
+      const data = helper.readFileSync(path.join(extensionPath));
       if (data && data.hasOwnProperty(extension_key_value)) {
         // eslint-disable-next-line camelcase
         schema[i].extension_uid = data[extension_key_value];
+      } else if (schema[i].field_metadata && schema[i].field_metadata.extension) {
+        if (installedExtensions) {
+          const marketplaceApps = helper.readFileSync(marketplaceAppPath);
+          const oldExt = _.find(marketplaceApps, { uid: schema[i].extension_uid });
+
+          if (oldExt) {
+            const ext = _.find(installedExtensions, { type: 'field', title: oldExt.title, app_uid: oldExt.app_uid });
+
+            if (ext) {
+              schema[i].extension_uid = ext.uid;
+            }
+          }
+        }
       }
     } else if (schema[i].data_type === 'json' && schema[i].hasOwnProperty('plugins') && schema[i].plugins.length > 0) {
       const newPluginUidsArray = [];
-      const data = helper.readFile(path.join(extensionPath));
+      const data = helper.readFileSync(path.join(extensionPath));
       schema[i].plugins.forEach((extension_key_value) => {
         if (data && data.hasOwnProperty(extension_key_value)) {
           newPluginUidsArray.push(data[extension_key_value]);
@@ -59,4 +74,4 @@ let extension_uid_Replace = (module.exports = function (schema, preserveStackVer
       schema[i].plugins = newPluginUidsArray;
     }
   }
-})
+});

package/src/lib/util/fs.js

@@ -7,33 +7,112 @@
 var fs = require('fs');
 var path = require('path');
 var mkdirp = require('mkdirp');
+const bigJSON = require('big-json');
 
-exports.readFile = function (filePath, parse) {
-  var data;
+exports.readFileSync = function (filePath, parse) {
+  let data;
   parse = typeof parse === 'undefined' ? true : parse;
   filePath = path.resolve(filePath);
   if (fs.existsSync(filePath)) {
-    data = parse ? JSON.parse(fs.readFileSync(filePath, 'utf-8')) : data;
+    try {
+      data = parse ? JSON.parse(fs.readFileSync(filePath, 'utf-8')) : data;
+    } catch (error) {
+      return data;
+    }
   }
   return data;
 };
 
-exports.writeFile = function (filePath, data) {
+// by default file type is json
+exports.readFile = async (filePath, options = { type: 'json' }) => {
+  return new Promise((resolve, reject) => {
+    filePath = path.resolve(filePath);
+    fs.readFile(filePath, 'utf-8', (error, data) => {
+      if (error) {
+        if (error.code === 'ENOENT') {
+          return resolve();
+        }
+        reject(error);
+      } else {
+        if (options.type !== 'json') {
+          return resolve(data);
+        }
+        resolve(JSON.parse(data));
+      }
+    });
+  });
+};
+
+exports.writeFileSync = function (filePath, data) {
   data = typeof data === 'object' ? JSON.stringify(data) : data || '{}';
   fs.writeFileSync(filePath, data);
 };
 
-exports.makeDirectory = function () {
-  for (var key in arguments) {
-    var dirname = path.resolve(arguments[key]);
-    if (!fs.existsSync(dirname)) {
-      mkdirp.sync(dirname);
-    }
+exports.writeFile = function (filePath, data) {
+  return new Promise((resolve, reject) => {
+    data = typeof data === 'object' ? JSON.stringify(data) : data || '{}';
+    fs.writeFile(filePath, data, (error) => {
+      if (error) {
+        return reject(error);
+      }
+      resolve('done');
+    });
+  });
+};
+
+exports.makeDirectory = async function (path) {
+  if (!path) {
+    throw new Error('Invalid path to create directory');
+  }
+  return mkdirp(path);
+};
+
+exports.readLargeFile = function (filePath, opts = {}) {
+  if (typeof filePath !== 'string') {
+    return;
+  }
+  filePath = path.resolve(filePath);
+  if (fs.existsSync(filePath)) {
+    return new Promise((resolve, reject) => {
+      const readStream = fs.createReadStream(filePath, { encoding: 'utf-8' });
+      const parseStream = bigJSON.createParseStream();
+      parseStream.on('data', function (data) {
+        if (opts.type === 'array') {
+          return resolve(Object.values(data));
+        }
+        resolve(data);
+      });
+      parseStream.on('error', function (error) {
+        console.log('error', error);
+        reject(error);
+      });
+      readStream.pipe(parseStream);
+    });
   }
 };
 
-exports.readdir = function (dirPath) {
-  if (fs.existsSync(path)) {
+exports.writeLargeFile = function (filePath, data) {
+  if (typeof filePath !== 'string' || typeof data !== 'object') {
+    return;
+  }
+  filePath = path.resolve(filePath);
+  return new Promise((resolve, reject) => {
+    const stringifyStream = bigJSON.createStringifyStream({
+      body: data,
+    });
+    var writeStream = fs.createWriteStream(filePath, 'utf-8');
+    stringifyStream.pipe(writeStream);
+    writeStream.on('finish', () => {
+      resolve();
+    });
+    writeStream.on('error', (error) => {
+      reject(error);
+    });
+  });
+};
+
+exports.readdirSync = function (dirPath) {
+  if (fs.existsSync(dirPath)) {
     return fs.readdirSync(dirPath);
   } else {
     return [];

package/src/lib/util/index.js

@@ -13,6 +13,7 @@ var chalk = require('chalk');
 var { addlogs } = require('./log');
 var defaultConfig = require('../../config/default');
 const stack = require('./contentstack-management-sdk');
+const promiseLimit = require('promise-limit');
 var config;
 
 exports.initialization = function (configData) {
@@ -69,7 +70,7 @@ exports.sanitizeStack = function (importConfig) {
           importConfig.modules.stack.fileName,
         );
 
-        const oldStackDetails = fs.readFile(stackFilePath);
+        const oldStackDetails = fs.readFileSync(stackFilePath);
         if (!oldStackDetails || !oldStackDetails.settings || !oldStackDetails.settings.hasOwnProperty('version')) {
           throw new Error(`${JSON.stringify(oldStackDetails)} is invalid!`);
         }
@@ -129,7 +130,7 @@ exports.field_rules_update = function (importConfig, ctPath) {
   return new Promise(function (resolve, reject) {
     let client = stack.Client(importConfig);
 
-    fs.readFile(path.join(ctPath + '/field_rules_uid.json'), async (err, data) => {
+    fs.readFileSync(path.join(ctPath + '/field_rules_uid.json'), async (err, data) => {
       if (err) {
         throw err;
       }
@@ -152,7 +153,7 @@ exports.field_rules_update = function (importConfig, ctPath) {
                   let updatedValue = [];
                   for (let j = 0; j < fieldRulesArray.length; j++) {
                     let splitedFieldRulesValue = fieldRulesArray[j];
-                    let oldUid = helper.readFile(path.join(entryUidMapperPath));
+                    let oldUid = helper.readFileSync(path.join(entryUidMapperPath));
                     if (oldUid.hasOwnProperty(splitedFieldRulesValue)) {
                       updatedValue.push(oldUid[splitedFieldRulesValue]);
                     } else {
@@ -185,3 +186,38 @@ exports.field_rules_update = function (importConfig, ctPath) {
 exports.getConfig = function () {
   return config;
 };
+
+exports.formatError = function (error) {
+  try {
+    if (typeof error === 'string') {
+      error = JSON.parse(error);
+    } else {
+      error = JSON.parse(error.message);
+    }
+  } catch (e) {}
+  let message = error.errorMessage || error.error_message || error.message || error;
+  if (error.errors && Object.keys(error.errors).length > 0) {
+    Object.keys(error.errors).forEach((e) => {
+      let entity = e;
+      if (e === 'authorization') entity = 'Management Token';
+      if (e === 'api_key') entity = 'Stack API key';
+      if (e === 'uid') entity = 'Content Type';
+      if (e === 'access_token') entity = 'Delivery Token';
+      message += ' ' + [entity, error.errors[e]].join(' ');
+    });
+  }
+  return message;
+};
+
+exports.executeTask = function (tasks = [], handler, options) {
+  if (typeof handler !== 'function') {
+    throw new Error('Invalid handler');
+  }
+  const { concurrency = 1 } = options;
+  const limit = promiseLimit(concurrency);
+  return Promise.all(
+    tasks.map((task) => {
+      return limit(() => handler(task));
+    }),
+  );
+};

package/src/lib/util/log.js

@@ -9,11 +9,9 @@ var path = require('path');
 var mkdirp = require('mkdirp');
 var slice = Array.prototype.slice;
 
-
-
 const ansiRegexPattern = [
   '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)',
-'(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))'
+  '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))',
 ].join('|');
 
 function returnString(args) {
@@ -22,14 +20,18 @@ function returnString(args) {
     returnStr = args
       .map(function (item) {
         if (item && typeof item === 'object') {
-          return JSON.stringify(item).replace(/authtoken\":\"blt................/g, 'authtoken":"blt....');
+          try {
+            return JSON.stringify(item).replace(/authtoken\":\"blt................/g, 'authtoken":"blt....');
+          } catch (error) {
+            return item.message;
+          }
         }
         return item;
       })
       .join('  ')
       .trim();
   }
-  returnStr = returnStr.replace(new RegExp(ansiRegexPattern, 'g'), "").trim();
+  returnStr = returnStr.replace(new RegExp(ansiRegexPattern, 'g'), '').trim();
   return returnStr;
 }
 

package/src/lib/util/login.js

@@ -38,7 +38,8 @@ module.exports = function (config) {
       client
         .stack({ api_key: config.target_stack, management_token: config.management_token })
         .fetch()
-        .then(function () {
+        .then(function (stack) {
+          config.destinationStackName = stack.name
           return resolve();
         })
         .catch((error) => {