@contentstack/cli-cm-export 1.1.0 → 1.2.1

package/src/lib/export/workflows.js

@@ -4,98 +4,106 @@
  * MIT Licensed
  */
 
-const mkdirp = require('mkdirp');
 const path = require('path');
 const chalk = require('chalk');
+const mkdirp = require('mkdirp');
+const { merge } = require('lodash');
 
-let helper = require('../util/helper');
-let { addlogs } = require('../util/log');
-
+const helper = require('../util/helper');
+const { addlogs } = require('../util/log');
+const { formatError } = require('../util');
+const config = require('../../config/default');
 const stack = require('../util/contentstack-management-sdk');
-let config = require('../../config/default');
-let workFlowConfig = config.modules.workflows;
-let client;
 
-function ExportWorkFlows() {
-  this.workflows = {};
-}
+module.exports = class ExportWorkFlows {
+  client;
+  config;
+  workflows = {};
+  workFlowConfig = config.modules.workflows;
+
+  constructor(credentialConfig) {
+    this.config = merge(config, credentialConfig);
+  }
 
-ExportWorkFlows.prototype.start = function (credentialConfig) {
-  addlogs(config, 'Starting workflow export', 'success');
-  this.workflows = {};
-  let self = this;
-  config = credentialConfig;
-  client = stack.Client(config);
+  start() {
+    addlogs(this.config, 'Starting workflow export', 'success');
 
-  let workflowsFolderPath = path.resolve(config.data, config.branchName || '', workFlowConfig.dirName);
-  mkdirp.sync(workflowsFolderPath);
-  return new Promise(function (resolve, reject) {
-    return client
-      .stack({ api_key: config.source_stack, management_token: config.management_token })
-      .workflow()
-      .fetchAll()
-      .then(async (response) => {
-        try {
-          if (response.items.length) {
-            await getWorkflowsData(self, response.items);
-            addlogs(config, chalk.green('All the workflow have been exported successfully'), 'success');
+    const self = this;
+    this.client = stack.Client(this.config);
+    let workflowsFolderPath = path.resolve(this.config.data, this.config.branchName || '', this.workFlowConfig.dirName);
+
+    mkdirp.sync(workflowsFolderPath);
+
+    return new Promise(function (resolve, reject) {
+      return self.client
+        .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+        .workflow()
+        .fetchAll()
+        .then(async (response) => {
+          try {
+            if (response.items.length) {
+              await self.getWorkflowsData(self, response.items);
+              addlogs(self.config, chalk.green('All the workflow have been exported successfully'), 'success');
+            }
+            if (!response.items.length) {
+              addlogs(self.config, 'No workflow were found in the Stack', 'success');
+            }
+            helper.writeFileSync(path.join(workflowsFolderPath, self.workFlowConfig.fileName), self.workflows);
+            resolve();
+          } catch (error) {
+            addlogs(self.config, formatError(error), 'error');
+            reject(error);
           }
-          if (!response.items.length) {
-            addlogs(config, 'No workflow were found in the Stack', 'success');
+        })
+        .catch(function (error) {
+          if (error.statusCode === 401) {
+            addlogs(
+              self.config,
+              chalk.red(
+                'You are not allowed to export workflow, Unless you provide email and password in config',
+                'error',
+              ),
+            );
+            return resolve();
           }
-          helper.writeFile(path.join(workflowsFolderPath, workFlowConfig.fileName), self.workflows);
-          return resolve();
-        } catch (error) {
-          return reject(error);
-        }
-      })
-      .catch(function (error) {
-        if (error.statusCode === 401) {
-          addlogs(
-            config,
-            chalk.red(
-              'You are not allowed to export workflow, Unless you provide email and password in config',
-              'error',
-            ),
-          );
-          return resolve();
-        }
-        addlogs(config, error, 'error');
-        return resolve();
-      });
-  });
-};
+          addlogs(self.config, formatError(error), 'error');
+          resolve();
+        });
+    });
+  }
 
-const getWorkflowsData = async (self, workflows) => {
-  try {
-    for (const workflow of workflows) {
-      addlogs(config, workflow.name + ' workflow was exported successfully', 'success');
-      await getWorkflowRoles(workflow);
-      self.workflows[workflow.uid] = workflow;
-      let deleteItems = config.modules.workflows.invalidKeys;
-      deleteItems.forEach((e) => delete workflow[e]);
+  async getWorkflowRoles(self, workflow) {
+    try {
+      for (const stage of workflow.workflow_stages) {
+        if (stage.SYS_ACL.roles.uids.length) {
+          for (let i = 0; i < stage.SYS_ACL.roles.uids.length; i++) {
+            const roleUid = stage.SYS_ACL.roles.uids[i];
+            const roleData = await self.client
+              .stack({ api_key: config.source_stack, management_token: config.management_token })
+              .role(roleUid)
+              .fetch({ include_rules: true, include_permissions: true });
+            stage.SYS_ACL.roles.uids[i] = roleData;
+          }
+        }
+      }
+    } catch (error) {
+      console.log('Error getting roles', error && error.message);
+      addlogs(self.config, 'Error fetching roles in export workflows task.', 'error');
+      throw new Error({ message: 'Error fetching roles in export workflows task.' });
     }
-  } catch (error) {
-    throw error;
   }
-};
 
-const getWorkflowRoles = async (workflow) => {
-  try {
-    for (const stage of workflow.workflow_stages) {
-      if (stage.SYS_ACL.roles.uids.length) {
-        for (let i = 0; i < stage.SYS_ACL.roles.uids.length; i++) {
-          const roleUid = stage.SYS_ACL.roles.uids[i];
-          const roleData = await client.stack({ api_key: config.source_stack, management_token: config.management_token }).role(roleUid).fetch({ include_rules: true, include_permissions: true });
-          stage.SYS_ACL.roles.uids[i] = roleData;
-        }
+  async getWorkflowsData(self, workflows) {
+    try {
+      for (const workflow of workflows) {
+        addlogs(self.config, workflow.name + ' workflow was exported successfully', 'success');
+        await self.getWorkflowRoles(self, workflow);
+        self.workflows[workflow.uid] = workflow;
+        let deleteItems = config.modules.workflows.invalidKeys;
+        deleteItems.forEach((e) => delete workflow[e]);
       }
+    } catch (error) {
+      throw error;
     }
-  } catch (error) {
-    console.log('Error getting roles', error && error.message);
-    addlogs(config, 'Error fetching roles in export workflows task.', 'error');
-    throw { message: 'Error fetching roles in export workflows task.' };
   }
 };
-
-module.exports = new ExportWorkFlows();

package/src/lib/util/export-flags.js

@@ -11,7 +11,7 @@ const helper = require('../util/helper');
 let _ = require('lodash');
 const { cliux } = require('@contentstack/cli-utilities');
 
-exports.configWithMToken = function (config, managementTokens, host, contentTypes, branchName, securedAssets, moduleName) {
+exports.configWithMToken = async function (config, managementTokens, host, contentTypes, branchName, securedAssets, moduleName) {
   let externalConfig = require(config);
   defaultConfig.securedAssets = securedAssets;
   defaultConfig.management_token = managementTokens.token;
@@ -27,7 +27,7 @@ exports.configWithMToken = function (config, managementTokens, host, contentType
     }
   }
   defaultConfig = _.merge(defaultConfig, externalConfig);
-  initial(defaultConfig);
+  await initial(defaultConfig);
 };
 
 exports.parameterWithMToken = async function (
@@ -87,10 +87,10 @@ exports.withoutParameterMToken = async (
   }
   defaultConfig.source_stack = stackUid;
   defaultConfig.data = pathOfExport;
-  initial(defaultConfig);
+  await initial(defaultConfig);
 };
 
-exports.configWithAuthToken = function (config, _authToken, moduleName, host, contentTypes, branchName, securedAssets) {
+exports.configWithAuthToken = async function (config, _authToken, moduleName, host, contentTypes, branchName, securedAssets) {
   let externalConfig = helper.readFile(path.resolve(config));
   defaultConfig.auth_token = _authToken;
   defaultConfig.host = host.cma;
@@ -105,7 +105,7 @@ exports.configWithAuthToken = function (config, _authToken, moduleName, host, co
     }
   }
   defaultConfig = _.merge(defaultConfig, externalConfig);
-  initial(defaultConfig);
+  await initial(defaultConfig);
 };
 
 exports.parametersWithAuthToken = function (
@@ -133,8 +133,7 @@ exports.parametersWithAuthToken = function (
     defaultConfig.cdn = host.cda;
     defaultConfig.data = data;
     defaultConfig.securedAssets = securedAssets;
-    var exportStart = initial(defaultConfig);
-    exportStart
+    await initial(defaultConfig)
       .then(() => {
         return resolve();
       })
@@ -168,5 +167,5 @@ exports.withoutParametersWithAuthToken = async (
   defaultConfig.data = pathOfExport;
   defaultConfig.host = host.cma;
   defaultConfig.cdn = host.cda;
-  initial(defaultConfig);
+  await initial(defaultConfig);
 };

package/src/lib/util/helper.js

@@ -7,8 +7,9 @@
 var fs = require('fs');
 var path = require('path');
 var mkdirp = require('mkdirp');
+var bigJSON = require('big-json');
 
-exports.readFile = function (filePath, parse) {
+exports.readFileSync = function (filePath, parse) {
   var data;
   parse = typeof parse === 'undefined' ? true : parse;
   filePath = path.resolve(filePath);
@@ -18,11 +19,91 @@ exports.readFile = function (filePath, parse) {
   return data;
 };
 
-exports.writeFile = function (filePath, data) {
+// by default file type is json
+exports.readFile = async (filePath, options = { type: 'json' }) => {
+  return new Promise((resolve, reject) => {
+    filePath = path.resolve(filePath);
+    fs.readFile(filePath, 'utf-8', (error, data) => {
+      if (error) {
+        reject(error);
+      } else {
+        if (options.type !== 'json') {
+          return resolve(data);
+        }
+        resolve(JSON.parse(data));
+      }
+    });
+  });
+};
+
+exports.makeDirectory = async function (path) {
+  if (!path) {
+    throw new Error('Invalid path to create directory');
+  }
+  return mkdirp(path);
+};
+
+exports.readLargeFile = function (filePath, opts = {}) {
+  if (typeof filePath !== 'string') {
+    return;
+  }
+  filePath = path.resolve(filePath);
+  if (fs.existsSync(filePath)) {
+    return new Promise((resolve, reject) => {
+      const readStream = fs.createReadStream(filePath, { encoding: 'utf-8' });
+      const parseStream = bigJSON.createParseStream();
+      parseStream.on('data', function (data) {
+        if (opts.type === 'array') {
+          return resolve(Object.values(data));
+        }
+        resolve(data);
+      });
+      parseStream.on('error', function (error) {
+        console.log('error', error);
+        reject(error);
+      });
+      readStream.pipe(parseStream);
+    });
+  }
+};
+
+exports.writeLargeFile = function (filePath, data) {
+  if (typeof filePath !== 'string' || typeof data !== 'object') {
+    return;
+  }
+  filePath = path.resolve(filePath);
+  return new Promise((resolve, reject) => {
+    const stringifyStream = bigJSON.createStringifyStream({
+      body: data,
+    });
+    var writeStream = fs.createWriteStream(filePath, 'utf-8');
+    stringifyStream.pipe(writeStream);
+    writeStream.on('finish', () => {
+      resolve();
+    });
+    writeStream.on('error', (error) => {
+      reject(error);
+    });
+  });
+};
+
+exports.writeFileSync = function (filePath, data) {
   data = typeof data === 'object' ? JSON.stringify(data) : data || '{}';
   fs.writeFileSync(filePath, data);
 };
 
+exports.writeFile = function (filePath, data) {
+  return new Promise((resolve, reject) => {
+    data = typeof data === 'object' ? JSON.stringify(data) : data || '{}';
+    fs.writeFile(filePath, data, (error) => {
+      if (error) {
+        return reject(error);
+      }
+      resolve('done');
+    });
+  });
+};
+
 exports.makeDirectory = function () {
   for (var key in arguments) {
     var dirname = path.resolve(arguments[key]);

package/src/lib/util/index.js

@@ -6,8 +6,8 @@
 
 var _ = require('lodash');
 var defaultConfig = require('../../config/default');
-var { addlogs } = require('../util/log');
 const chalk = require('chalk');
+const promiseLimit = require('promise-limit');
 
 exports.validateConfig = function (config) {
   if (!config.host || !config.cdn) {
@@ -43,3 +43,38 @@ exports.buildAppConfig = function (config) {
   config = _.merge(defaultConfig, config);
   return config;
 };
+
+exports.formatError = function (error) {
+  try {
+    if (typeof error === 'string') {
+      error = JSON.parse(error);
+    } else {
+      error = JSON.parse(error.message);
+    }
+  } catch (e) {}
+  let message = error.errorMessage || error.error_message || error.message || error;
+  if (error.errors && Object.keys(error.errors).length > 0) {
+    Object.keys(error.errors).forEach((e) => {
+      let entity = e;
+      if (e === 'authorization') entity = 'Management Token';
+      if (e === 'api_key') entity = 'Stack API key';
+      if (e === 'uid') entity = 'Content Type';
+      if (e === 'access_token') entity = 'Delivery Token';
+      message += ' ' + [entity, error.errors[e]].join(' ');
+    });
+  }
+  return message;
+};
+
+exports.executeTask = function (tasks = [], handler, options) {
+  if (typeof handler !== 'function') {
+    throw new Error('Invalid handler');
+  }
+  const { concurrency = 1 } = options;
+  const limit = promiseLimit(concurrency);
+  return Promise.all(
+    tasks.map((task) => {
+      return limit(() => handler(task));
+    }),
+  );
+};

package/src/lib/util/log.js

@@ -11,7 +11,7 @@ var slice = Array.prototype.slice;
 
 const ansiRegexPattern = [
   '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)',
-'(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))'
+  '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PR-TZcf-ntqry=><~]))',
 ].join('|');
 
 function returnString(args) {
@@ -20,14 +20,17 @@ function returnString(args) {
     returnStr = args
       .map(function (item) {
         if (item && typeof item === 'object') {
-          return JSON.stringify(item).replace(/authtoken\":\"blt................/g, 'authtoken":"blt....');
+          try {
+            return JSON.stringify(item).replace(/authtoken\":\"blt................/g, 'authtoken":"blt....');
+          } catch (error) {}
+          return item;
         }
         return item;
       })
       .join('  ')
       .trim();
   }
-  returnStr = returnStr.replace(new RegExp(ansiRegexPattern, 'g'), "").trim();
+  returnStr = returnStr.replace(new RegExp(ansiRegexPattern, 'g'), '').trim();
   return returnStr;
 }
 var myCustomLevels = {

package/src/lib/util/marketplace-app-helper.js

@@ -1,5 +1,6 @@
+let config = require('../../config/default');
 const sdk = require('./contentstack-management-sdk');
-const { HttpClient } = require('@contentstack/cli-utilities');
+const { cliux, HttpClient, configHandler } = require('@contentstack/cli-utilities');
 
 const getInstalledExtensions = (config) => {
   const client = sdk.Client(config)
@@ -22,15 +23,13 @@ const getInstalledExtensions = (config) => {
         .then(({ items }) => resolve(items))
         .catch(reject)
     } else if (api_key && auth_token) {
+      const { cma } = configHandler.get('region') || {};
       const headers = {
         api_key,
         authtoken: auth_token
       }
       const httpClient = new HttpClient().headers(headers);
-      const baseUrl = config.host.startsWith('http')
-        ? config.host
-        : `https://${config.host}/v3`;
-      httpClient.get(`${baseUrl}/extensions/?include_marketplace_extensions=true`)
+      httpClient.get(`${cma}/v3/extensions/?include_marketplace_extensions=true`)
         .then(({ data: { extensions } }) => resolve(extensions))
         .catch(reject)
     } else {
@@ -39,4 +38,24 @@ const getInstalledExtensions = (config) => {
   })
 }
 
-module.exports = { getInstalledExtensions }
+const getDeveloperHubUrl = async () => {
+  const { cma, name } = configHandler.get('region') || {};
+  let developerHubBaseUrl = config.developerHubUrls[cma];
+
+  if (!developerHubBaseUrl) {
+    developerHubBaseUrl = await cliux.inquire({
+      type: 'input',
+      name: 'name',
+      validate: (url) => {
+        if (!url) return "Developer-hub URL can't be empty.";
+
+        return true;
+      },
+      message: `Enter the developer-hub base URL for the ${name} region - `,
+    });
+  }
+
+  return developerHubBaseUrl.startsWith('http') ? developerHubBaseUrl : `https://${developerHubBaseUrl}`;
+}
+
+module.exports = { getInstalledExtensions, getDeveloperHubUrl }

package/src/lib/util/setup-branches.js

@@ -8,12 +8,12 @@ const setupBranches = async (config, branch) => {
     throw new Error('Invalid config to setup the branch');
   }
   let branches = [];
-  const headers = { api_key: config.source_stack }
+  const headers = { api_key: config.source_stack };
 
   if (config.auth_token) {
-    headers['authtoken'] = config.auth_token
+    headers['authtoken'] = config.auth_token;
   } else if (config.management_token) {
-    headers['authorization'] = config.management_token
+    headers['authorization'] = config.management_token;
   }
 
   if (typeof branch === 'string') {
@@ -29,14 +29,12 @@ const setupBranches = async (config, branch) => {
     }
   } else {
     try {
-      const result = await HttpClient.create()
-        .headers(headers)
-        .get(`https://${config.host}/v3/stacks/branches`);
+      const result = await HttpClient.create().headers(headers).get(`https://${config.host}/v3/stacks/branches`);
 
       if (result && result.data && Array.isArray(result.data.branches) && result.data.branches.length > 0) {
         branches = result.data.branches;
       } else {
-        branches.push('main');
+        return;
       }
     } catch (error) {
       return;