@contentstack/cli-cm-export 1.0.0 → 1.2.0

package/src/lib/export/stack.js

@@ -4,19 +4,16 @@
  * MIT Licensed
  */
 
-var chalk = require('chalk');
-var mkdirp = require('mkdirp');
-var path = require('path');
-
-var app = require('../../app');
-var helper = require('../util/helper');
-var { addlogs } = require('../util/log');
-const stack = require('../util/contentstack-management-sdk');
+let path = require('path');
+let mkdirp = require('mkdirp');
 
+let helper = require('../util/helper');
+let { addlogs } = require('../util/log');
 let config = require('../../config/default');
+const stack = require('../util/contentstack-management-sdk');
 
-var stackConfig = config.modules.stack;
 let client;
+let stackConfig = config.modules.stack;
 
 function ExportStack() {
   this.requestOption = {
@@ -26,17 +23,33 @@ function ExportStack() {
   };
 }
 
-ExportStack.prototype.start = function (credentialConfig) {
+ExportStack.prototype.start = async function (credentialConfig) {
   config = credentialConfig;
   client = stack.Client(config);
-  const self = this
+  const self = this;
+
+  // NOTE get org uid
+  if (config.auth_token) {
+    const stack = await client
+      .stack({ api_key: config.source_stack, authtoken: config.auth_token })
+      .fetch()
+      .catch((error) => {
+        console.log(error);
+      });
+
+    if (stack && stack.org_uid) {
+      config.org_uid = stack.org_uid;
+      config.sourceStackName = stack.name;
+    }
+  }
+
   if (!config.preserveStackVersion && !config.hasOwnProperty('master_locale')) {
     const apiDetails = {
       limit: 100,
       skip: 0,
       include_count: true,
-    }
-    return self.getLocales(apiDetails)
+    };
+    return self.getLocales(apiDetails);
   } else if (config.preserveStackVersion) {
     addlogs(config, 'Exporting stack details', 'success');
     let stackFolderPath = path.resolve(config.data, stackConfig.dirName);
@@ -59,12 +72,12 @@ ExportStack.prototype.start = function (credentialConfig) {
 };
 
 ExportStack.prototype.getLocales = function (apiDetails) {
-  let self = this
+  let self = this;
   return new Promise((resolve, reject) => {
     const result = client
       .stack({ api_key: config.source_stack, management_token: config.management_token })
       .locale()
-      .query(apiDetails)
+      .query(apiDetails);
 
     result
       .find()
@@ -75,11 +88,11 @@ ExportStack.prototype.getLocales = function (apiDetails) {
           }
         });
         apiDetails.skip += apiDetails.limit;
-        if (masterLocalObj) { return resolve(masterLocalObj); }
-        else if (apiDetails.skip <= response.count) {
-          return resolve(self.getLocales(apiDetails))
-        }
-        else {
+        if (masterLocalObj) {
+          return resolve(masterLocalObj);
+        } else if (apiDetails.skip <= response.count) {
+          return resolve(self.getLocales(apiDetails));
+        } else {
           return reject('Master locale not found');
         }
       })
@@ -87,6 +100,6 @@ ExportStack.prototype.getLocales = function (apiDetails) {
         return reject(error);
       });
   });
-}
+};
 
-module.exports = new ExportStack();
+module.exports = ExportStack;

package/src/lib/export/webhooks.js

@@ -4,72 +4,77 @@
  * MIT Licensed
  */
 
-const mkdirp = require('mkdirp');
 const path = require('path');
 const chalk = require('chalk');
+const mkdirp = require('mkdirp');
+const { merge } = require('lodash');
 
 const helper = require('../util/helper');
+const { formatError } = require('../util');
 const { addlogs } = require('../util/log');
-
-let config = require('../../config/default');
+const config = require('../../config/default');
 const stack = require('../util/contentstack-management-sdk');
-const webhooksConfig = config.modules.webhooks;
-let client;
 
 // Create folder for environments
 
-function ExportWebhooks() {
-  this.requestOptions = {
+module.exports = class ExportWebhooks {
+  config;
+  master = {};
+  webhooks = {};
+  requestOptions = {
     include_count: true,
     asc: 'updated_at',
   };
-  this.master = {};
-  this.webhooks = {};
-}
+  webhooksConfig = config.modules.webhooks;
+
+  constructor(credentialConfig) {
+    this.config = merge(config, credentialConfig);
+  }
 
-ExportWebhooks.prototype.start = function (credentialConfig) {
-  addlogs(config, 'Starting webhooks export', 'success');
-  this.master = {};
-  this.webhooks = {};
-  let self = this;
-  config = credentialConfig;
-  client = stack.Client(config);
-  const webhooksFolderPath = path.resolve(config.data, config.branchName || '', webhooksConfig.dirName);
-  mkdirp.sync(webhooksFolderPath);
-  return new Promise(function (resolve, reject) {
-    client
-      .stack({ api_key: config.source_stack, management_token: config.management_token })
-      .webhook()
-      .fetchAll(self.requestOptions)
-      .then((webhooks) => {
-        if (webhooks.items.length !== 0) {
-          for (var i = 0, total = webhooks.count; i < total; i++) {
-            const webUid = webhooks.items[i].uid;
-            self.master[webUid] = '';
-            self.webhooks[webUid] = webhooks.items[i];
-            delete self.webhooks[webUid].uid;
-            delete self.webhooks[webUid].SYS_ACL;
+  start() {
+    addlogs(this.config, 'Starting webhooks export', 'success');
+
+    const self = this;
+    const client = stack.Client(this.config);
+    const webhooksFolderPath = path.resolve(
+      this.config.data,
+      this.config.branchName || '',
+      self.webhooksConfig.dirName,
+    );
+    mkdirp.sync(webhooksFolderPath);
+    return new Promise((resolve, reject) => {
+      client
+        .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+        .webhook()
+        .fetchAll(self.requestOptions)
+        .then((webhooks) => {
+          if (webhooks.items.length !== 0) {
+            for (let i = 0, total = webhooks.count; i < total; i++) {
+              const webUid = webhooks.items[i].uid;
+              self.master[webUid] = '';
+              self.webhooks[webUid] = webhooks.items[i];
+              delete self.webhooks[webUid].uid;
+              delete self.webhooks[webUid].SYS_ACL;
+            }
+            helper.writeFileSync(path.join(webhooksFolderPath, self.webhooksConfig.fileName), self.webhooks);
+            addlogs(self.config, chalk.green('All the webhooks have been exported successfully'), 'success');
+            return resolve();
+          }
+          addlogs(self.config, 'No webhooks found', 'success');
+          resolve();
+        })
+        .catch(function (error) {
+          if (error.statusCode === 401) {
+            addlogs(
+              self.config,
+              chalk.red('You are not allowed to export webhooks, Unless you provide email and password in config'),
+              'error',
+            );
+            return resolve();
           }
-          helper.writeFile(path.join(webhooksFolderPath, webhooksConfig.fileName), self.webhooks);
-          addlogs(config, chalk.green('All the webhooks have been exported successfully'), 'success');
-          return resolve();
-        }
-        addlogs(config, 'No webhooks found', 'success');
-        return resolve();
-      })
-      .catch(function (error) {
-        if (error.statusCode === 401) {
-          addlogs(
-            config,
-            chalk.red('You are not allowed to export webhooks, Unless you provide email and password in config'),
-            'error',
-          );
-          return resolve();
-        }
-        addlogs(config, error, 'error');
-        return reject();
-      });
-  });
+          addlogs(self.config, formatError(error), 'error');
+          reject('Failed export webhooks');
+        });
+    });
+  }
 };
-
-module.exports = new ExportWebhooks();

package/src/lib/export/workflows.js

@@ -4,67 +4,106 @@
  * MIT Licensed
  */
 
-const mkdirp = require('mkdirp');
 const path = require('path');
 const chalk = require('chalk');
+const mkdirp = require('mkdirp');
+const { merge } = require('lodash');
 
-let helper = require('../util/helper');
-let { addlogs } = require('../util/log');
-
+const helper = require('../util/helper');
+const { addlogs } = require('../util/log');
+const { formatError } = require('../util');
+const config = require('../../config/default');
 const stack = require('../util/contentstack-management-sdk');
-let config = require('../../config/default');
-let workFlowConfig = config.modules.workflows;
-let client;
 
-function ExportWorkFlows() {
-  this.workflows = {};
-}
+module.exports = class ExportWorkFlows {
+  client;
+  config;
+  workflows = {};
+  workFlowConfig = config.modules.workflows;
 
-ExportWorkFlows.prototype.start = function (credentialConfig) {
-  addlogs(config, 'Starting workflow export', 'success');
-  this.workflows = {};
-  let self = this;
-  config = credentialConfig;
-  client = stack.Client(config);
+  constructor(credentialConfig) {
+    this.config = merge(config, credentialConfig);
+  }
 
-  let workflowsFolderPath = path.resolve(config.data, config.branchName || '', workFlowConfig.dirName);
-  mkdirp.sync(workflowsFolderPath);
-  return new Promise(function (resolve) {
-    return client
-      .stack({ api_key: config.source_stack, management_token: config.management_token })
-      .workflow()
-      .fetchAll()
-      .then((response) => {
-        if (response.items.length !== 0) {
-          response.items.forEach(function (workflow) {
-            addlogs(config, workflow.name + ' workflow was exported successfully', 'success');
-            self.workflows[workflow.uid] = workflow;
-            let deleteItems = config.modules.workflows.invalidKeys;
-            deleteItems.forEach((e) => delete workflow[e]);
-          });
-          addlogs(config, chalk.green('All the workflow have been exported successfully'), 'success');
-        }
-        if (response.items.length === 0) {
-          addlogs(config, 'No workflow were found in the Stack', 'success');
-        }
-        helper.writeFile(path.join(workflowsFolderPath, workFlowConfig.fileName), self.workflows);
-        return resolve();
-      })
-      .catch(function (error) {
-        if (error.statusCode === 401) {
-          addlogs(
-            config,
-            chalk.red(
-              'You are not allowed to export workflow, Unless you provide email and password in config',
-              'error',
-            ),
-          );
-          return resolve();
+  start() {
+    addlogs(this.config, 'Starting workflow export', 'success');
+
+    const self = this;
+    this.client = stack.Client(this.config);
+    let workflowsFolderPath = path.resolve(this.config.data, this.config.branchName || '', this.workFlowConfig.dirName);
+
+    mkdirp.sync(workflowsFolderPath);
+
+    return new Promise(function (resolve, reject) {
+      return self.client
+        .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+        .workflow()
+        .fetchAll()
+        .then(async (response) => {
+          try {
+            if (response.items.length) {
+              await self.getWorkflowsData(self, response.items);
+              addlogs(self.config, chalk.green('All the workflow have been exported successfully'), 'success');
+            }
+            if (!response.items.length) {
+              addlogs(self.config, 'No workflow were found in the Stack', 'success');
+            }
+            helper.writeFileSync(path.join(workflowsFolderPath, self.workFlowConfig.fileName), self.workflows);
+            resolve();
+          } catch (error) {
+            addlogs(self.config, formatError(error), 'error');
+            reject(error);
+          }
+        })
+        .catch(function (error) {
+          if (error.statusCode === 401) {
+            addlogs(
+              self.config,
+              chalk.red(
+                'You are not allowed to export workflow, Unless you provide email and password in config',
+                'error',
+              ),
+            );
+            return resolve();
+          }
+          addlogs(self.config, formatError(error), 'error');
+          resolve();
+        });
+    });
+  }
+
+  async getWorkflowRoles(self, workflow) {
+    try {
+      for (const stage of workflow.workflow_stages) {
+        if (stage.SYS_ACL.roles.uids.length) {
+          for (let i = 0; i < stage.SYS_ACL.roles.uids.length; i++) {
+            const roleUid = stage.SYS_ACL.roles.uids[i];
+            const roleData = await self.client
+              .stack({ api_key: config.source_stack, management_token: config.management_token })
+              .role(roleUid)
+              .fetch({ include_rules: true, include_permissions: true });
+            stage.SYS_ACL.roles.uids[i] = roleData;
+          }
         }
-        addlogs(config, error, 'error');
-        return resolve();
-      });
-  });
-};
+      }
+    } catch (error) {
+      console.log('Error getting roles', error && error.message);
+      addlogs(self.config, 'Error fetching roles in export workflows task.', 'error');
+      throw new Error({ message: 'Error fetching roles in export workflows task.' });
+    }
+  }
 
-module.exports = new ExportWorkFlows();
+  async getWorkflowsData(self, workflows) {
+    try {
+      for (const workflow of workflows) {
+        addlogs(self.config, workflow.name + ' workflow was exported successfully', 'success');
+        await self.getWorkflowRoles(self, workflow);
+        self.workflows[workflow.uid] = workflow;
+        let deleteItems = config.modules.workflows.invalidKeys;
+        deleteItems.forEach((e) => delete workflow[e]);
+      }
+    } catch (error) {
+      throw error;
+    }
+  }
+};

package/src/lib/util/export-flags.js

@@ -11,7 +11,7 @@ const helper = require('../util/helper');
 let _ = require('lodash');
 const { cliux } = require('@contentstack/cli-utilities');
 
-exports.configWithMToken = function (config, managementTokens, host, contentTypes, branchName, securedAssets, moduleName) {
+exports.configWithMToken = async function (config, managementTokens, host, contentTypes, branchName, securedAssets, moduleName) {
   let externalConfig = require(config);
   defaultConfig.securedAssets = securedAssets;
   defaultConfig.management_token = managementTokens.token;
@@ -27,7 +27,7 @@ exports.configWithMToken = function (config, managementTokens, host, contentType
     }
   }
   defaultConfig = _.merge(defaultConfig, externalConfig);
-  initial(defaultConfig);
+  await initial(defaultConfig);
 };
 
 exports.parameterWithMToken = async function (
@@ -87,10 +87,10 @@ exports.withoutParameterMToken = async (
   }
   defaultConfig.source_stack = stackUid;
   defaultConfig.data = pathOfExport;
-  initial(defaultConfig);
+  await initial(defaultConfig);
 };
 
-exports.configWithAuthToken = function (config, _authToken, moduleName, host, contentTypes, branchName, securedAssets) {
+exports.configWithAuthToken = async function (config, _authToken, moduleName, host, contentTypes, branchName, securedAssets) {
   let externalConfig = helper.readFile(path.resolve(config));
   defaultConfig.auth_token = _authToken;
   defaultConfig.host = host.cma;
@@ -105,7 +105,7 @@ exports.configWithAuthToken = function (config, _authToken, moduleName, host, co
     }
   }
   defaultConfig = _.merge(defaultConfig, externalConfig);
-  initial(defaultConfig);
+  await initial(defaultConfig);
 };
 
 exports.parametersWithAuthToken = function (
@@ -133,8 +133,7 @@ exports.parametersWithAuthToken = function (
     defaultConfig.cdn = host.cda;
     defaultConfig.data = data;
     defaultConfig.securedAssets = securedAssets;
-    var exportStart = initial(defaultConfig);
-    exportStart
+    await initial(defaultConfig)
       .then(() => {
         return resolve();
       })
@@ -168,5 +167,5 @@ exports.withoutParametersWithAuthToken = async (
   defaultConfig.data = pathOfExport;
   defaultConfig.host = host.cma;
   defaultConfig.cdn = host.cda;
-  initial(defaultConfig);
+  await initial(defaultConfig);
 };

package/src/lib/util/helper.js

@@ -7,8 +7,9 @@
 var fs = require('fs');
 var path = require('path');
 var mkdirp = require('mkdirp');
+var bigJSON = require('big-json');
 
-exports.readFile = function (filePath, parse) {
+exports.readFileSync = function (filePath, parse) {
   var data;
   parse = typeof parse === 'undefined' ? true : parse;
   filePath = path.resolve(filePath);
@@ -18,11 +19,91 @@ exports.readFile = function (filePath, parse) {
   return data;
 };
 
-exports.writeFile = function (filePath, data) {
+// by default file type is json
+exports.readFile = async (filePath, options = { type: 'json' }) => {
+  return new Promise((resolve, reject) => {
+    filePath = path.resolve(filePath);
+    fs.readFile(filePath, 'utf-8', (error, data) => {
+      if (error) {
+        reject(error);
+      } else {
+        if (options.type !== 'json') {
+          return resolve(data);
+        }
+        resolve(JSON.parse(data));
+      }
+    });
+  });
+};
+
+exports.makeDirectory = async function (path) {
+  if (!path) {
+    throw new Error('Invalid path to create directory');
+  }
+  return mkdirp(path);
+};
+
+exports.readLargeFile = function (filePath, opts = {}) {
+  if (typeof filePath !== 'string') {
+    return;
+  }
+  filePath = path.resolve(filePath);
+  if (fs.existsSync(filePath)) {
+    return new Promise((resolve, reject) => {
+      const readStream = fs.createReadStream(filePath, { encoding: 'utf-8' });
+      const parseStream = bigJSON.createParseStream();
+      parseStream.on('data', function (data) {
+        if (opts.type === 'array') {
+          return resolve(Object.values(data));
+        }
+        resolve(data);
+      });
+      parseStream.on('error', function (error) {
+        console.log('error', error);
+        reject(error);
+      });
+      readStream.pipe(parseStream);
+    });
+  }
+};
+
+exports.writeLargeFile = function (filePath, data) {
+  if (typeof filePath !== 'string' || typeof data !== 'object') {
+    return;
+  }
+  filePath = path.resolve(filePath);
+  return new Promise((resolve, reject) => {
+    const stringifyStream = bigJSON.createStringifyStream({
+      body: data,
+    });
+    var writeStream = fs.createWriteStream(filePath, 'utf-8');
+    stringifyStream.pipe(writeStream);
+    writeStream.on('finish', () => {
+      resolve();
+    });
+    writeStream.on('error', (error) => {
+      reject(error);
+    });
+  });
+};
+
+exports.writeFileSync = function (filePath, data) {
   data = typeof data === 'object' ? JSON.stringify(data) : data || '{}';
   fs.writeFileSync(filePath, data);
 };
 
+exports.writeFile = function (filePath, data) {
+  return new Promise((resolve, reject) => {
+    data = typeof data === 'object' ? JSON.stringify(data) : data || '{}';
+    fs.writeFile(filePath, data, (error) => {
+      if (error) {
+        return reject(error);
+      }
+      resolve('done');
+    });
+  });
+};
+
 exports.makeDirectory = function () {
   for (var key in arguments) {
     var dirname = path.resolve(arguments[key]);

package/src/lib/util/index.js

@@ -6,8 +6,8 @@
 
 var _ = require('lodash');
 var defaultConfig = require('../../config/default');
-var { addlogs } = require('../util/log');
 const chalk = require('chalk');
+const promiseLimit = require('promise-limit');
 
 exports.validateConfig = function (config) {
   if (!config.host || !config.cdn) {
@@ -43,3 +43,38 @@ exports.buildAppConfig = function (config) {
   config = _.merge(defaultConfig, config);
   return config;
 };
+
+exports.formatError = function (error) {
+  try {
+    if (typeof error === 'string') {
+      error = JSON.parse(error);
+    } else {
+      error = JSON.parse(error.message);
+    }
+  } catch (e) {}
+  let message = error.errorMessage || error.error_message || error.message || error;
+  if (error.errors && Object.keys(error.errors).length > 0) {
+    Object.keys(error.errors).forEach((e) => {
+      let entity = e;
+      if (e === 'authorization') entity = 'Management Token';
+      if (e === 'api_key') entity = 'Stack API key';
+      if (e === 'uid') entity = 'Content Type';
+      if (e === 'access_token') entity = 'Delivery Token';
+      message += ' ' + [entity, error.errors[e]].join(' ');
+    });
+  }
+  return message;
+};
+
+exports.executeTask = function (tasks = [], handler, options) {
+  if (typeof handler !== 'function') {
+    throw new Error('Invalid handler');
+  }
+  const { concurrency = 1 } = options;
+  const limit = promiseLimit(concurrency);
+  return Promise.all(
+    tasks.map((task) => {
+      return limit(() => handler(task));
+    }),
+  );
+};

package/src/lib/util/log.js

@@ -11,7 +11,7 @@ var slice = Array.prototype.slice;
 
 const ansiRegexPattern = [
   '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)',
-'(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))'
+  '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))',
 ].join('|');
 
 function returnString(args) {
@@ -20,14 +20,17 @@ function returnString(args) {
     returnStr = args
       .map(function (item) {
         if (item && typeof item === 'object') {
-          return JSON.stringify(item).replace(/authtoken\":\"blt................/g, 'authtoken":"blt....');
+          try {
+            return JSON.stringify(item).replace(/authtoken\":\"blt................/g, 'authtoken":"blt....');
+          } catch (error) {}
+          return item;
         }
         return item;
       })
       .join('  ')
       .trim();
   }
-  returnStr = returnStr.replace(new RegExp(ansiRegexPattern, 'g'), "").trim();
+  returnStr = returnStr.replace(new RegExp(ansiRegexPattern, 'g'), '').trim();
   return returnStr;
 }
 var myCustomLevels = {