@contentstack/cli-cm-export 0.1.1-beta.9 → 1.1.0

package/src/lib/export/marketplace-apps.js

@@ -0,0 +1,154 @@
+/*!
+ * Contentstack Export
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
+const _ = require('lodash')
+const path = require('path');
+const chalk = require('chalk');
+const mkdirp = require('mkdirp');
+const eachOf = require('async/eachOf');
+const { cliux, configHandler, HttpClient, NodeCrypto } = require('@contentstack/cli-utilities');
+
+let config = require('../../config/default');
+const { writeFile } = require('../util/helper');
+const { addlogs: log } = require('../util/log');
+let stack = require('../util/contentstack-management-sdk');
+const { getInstalledExtensions } = require('../util/marketplace-app-helper')
+
+let client
+let marketplaceAppConfig = config.modules.marketplace_apps;
+
+function exportMarketplaceApps() {
+  this.marketplaceAppPath = null
+  this.developerHuBaseUrl = null
+  this.start = async (credentialConfig) => {
+    config = credentialConfig;
+    client = stack.Client(config);
+    this.developerHuBaseUrl = await this.getDeveloperHubUrl()
+
+    if (!config.auth_token) {
+      cliux.print('WARNING!!! To export Marketplace apps, you must be logged in. Please check csdx auth:login --help to log in', { color: 'yellow' })
+      return Promise.resolve()
+    }
+
+    log(credentialConfig, 'Starting marketplace app export', 'success');
+    this.marketplaceAppPath = path.resolve(
+      config.data, config.branchName || '',
+      marketplaceAppConfig.dirName
+    );
+    mkdirp.sync(this.marketplaceAppPath);
+
+    return this.exportInstalledExtensions()
+  }
+
+  this.exportInstalledExtensions = () => {
+    const self = this
+
+    return new Promise(async function (resolve, reject) {
+      getInstalledExtensions(config)
+        .then(async (items) => {
+          const installedApps = _.map(
+            _.filter(items, 'app_uid'),
+            ({ uid, title, app_uid, app_installation_uid, type }) => ({ title, uid, app_uid, app_installation_uid, type })
+          )
+          const headers = {
+            authtoken: config.auth_token,
+            organization_uid: config.org_uid
+          }
+          const httpClient = new HttpClient().headers(headers);
+          const nodeCrypto = new NodeCrypto()
+          const developerHubApps = await httpClient.get(`${self.developerHuBaseUrl}/apps`)
+            .then(({ data: { data } }) => data)
+            .catch(err => {
+              console.log(err)
+            }) || []
+
+          eachOf(_.uniqBy(installedApps, 'app_uid'), (app, _key, cb) => {
+            log(config, `Exporting ${app.title} app and it's config.`, 'success')
+            const listOfIndexToBeUpdated = _.map(
+              installedApps,
+              ({ app_uid }, index) => (app_uid === app.app_uid ? index : undefined)
+            ).filter(val => val !== undefined)
+
+            httpClient.get(`${self.developerHuBaseUrl}/installations/${app.app_installation_uid}/installationData`)
+              .then(({ data: result }) => {
+                const { data, error } = result
+                const developerHubApp = _.find(developerHubApps, { uid: app.app_uid })
+
+                _.forEach(listOfIndexToBeUpdated, (index, i) => {
+                  if (developerHubApp) {
+                    installedApps[index]['visibility'] = developerHubApp.visibility
+                    installedApps[index]['manifest'] = _.pick(
+                      developerHubApp,
+                      ['name', 'description', 'icon', 'target_type', 'ui_location', 'webhook', 'oauth'] // NOTE keys can be passed to install new app in the developer hub
+                    )
+                  }
+
+                  if (_.has(data, 'configuration') || _.has(data, 'server_configuration')) {
+                    const { configuration, server_configuration } = data
+  
+                    if (!_.isEmpty(configuration)) {
+                      installedApps[index]['configuration'] = nodeCrypto.encrypt(configuration)
+                    }
+                    if (!_.isEmpty(server_configuration)) {
+                      installedApps[index]['server_configuration'] = nodeCrypto.encrypt(server_configuration)
+                    }
+
+                    if (i === 0) {
+                      log(config, `Exported ${app.title} app and it's config.`, 'success')
+                    }
+                  } else if (error) {
+                    console.log(error)
+                    if (i === 0) {
+                      log(config, `Error on exporting ${app.title} app and it's config.`, 'error')
+                    }
+                  }
+                })
+
+                cb()
+              })
+              .catch(err => {
+                console.log(err)
+                cb()
+              })
+          }, () => {
+            if (!_.isEmpty(installedApps)) {
+              writeFile(
+                path.join(self.marketplaceAppPath, marketplaceAppConfig.fileName),
+                installedApps
+              )
+
+              log(config, chalk.green('All the marketplace apps have been exported successfully'), 'success')
+            } else {
+              log(config, 'No marketplace apps found', 'success')
+            }
+
+            resolve()
+          })
+        }).catch(reject)
+    })
+  }
+
+  this.getDeveloperHubUrl = async () => {
+    const { cma, name } = configHandler.get('region') || {}
+    let developerHubBaseUrl = config.developerHubUrls[cma]
+
+    if (!developerHubBaseUrl) {
+      developerHubBaseUrl = await cliux.inquire({
+        type: 'input',
+        name: 'name',
+        validate: (url) => {
+          if (!url) return 'Developer-hub URL cant be empty.'
+
+          return true
+        },
+        message: `Enter the developer-hub base URL for the ${name} region - `,
+      })
+    }
+
+    return developerHubBaseUrl.startsWith('http') ? developerHubBaseUrl : `https://${developerHubBaseUrl}`
+  }
+}
+
+module.exports = new exportMarketplaceApps();

package/src/lib/export/stack.js

@@ -4,64 +4,104 @@
  * MIT Licensed
  */
 
-var chalk = require('chalk')
-var mkdirp = require('mkdirp')
-var path = require('path')
+var chalk = require('chalk');
+var mkdirp = require('mkdirp');
+var path = require('path');
 
-var app = require('../../app')
-var helper = require('../util/helper')
-var {addlogs} = require('../util/log')
-const stack = require('../util/contentstack-management-sdk')
+var app = require('../../app');
+var helper = require('../util/helper');
+var { addlogs } = require('../util/log');
+const stack = require('../util/contentstack-management-sdk');
 
+let config = require('../../config/default');
 
-let config = require('../../config/default')
+var stackConfig = config.modules.stack;
+let client;
 
-var stackConfig = config.modules.stack
-let client
-
-function ExportStack () {
+function ExportStack() {
   this.requestOption = {
     uri: config.host + config.apis.stacks,
     headers: config.headers,
-    json: true
-  }
+    json: true,
+  };
 }
 
-ExportStack.prototype.start = function (credentialConfig) {
-  config = credentialConfig
-  client = stack.Client(config)
-  if (!config.preserveStackVersion && !config.hasOwnProperty("master_locale")) {
-    return new Promise((resolve, reject) => {
-    var result =  client.stack({ api_key: credentialConfig.source_stack, management_token: credentialConfig.management_token }).locale().query()     
-    result.find()
-      .then(response => {
-        var masterLocalObj = response.items.filter(obj => {
-            if (obj.fallback_locale === null) {
-              return obj
-            }
-            });
-        return resolve(masterLocalObj[0])
-      }).catch(error => {
-        return reject(error)
+ExportStack.prototype.start = async function (credentialConfig) {
+  config = credentialConfig;
+  client = stack.Client(config);
+  const self = this
+
+  // NOTE get org uid
+  if (config.auth_token) {
+    const stack = await client
+      .stack({ api_key: config.source_stack, authtoken: config.auth_token })
+      .fetch()
+      .catch((error) => {
+        console.log(error)
       })
-    })
-  } else if(config.preserveStackVersion) {
-    addlogs(config, 'Exporting stack details', 'success')
-    let stackFolderPath = path.resolve(config.data, stackConfig.dirName)
-    let stackContentsFile = path.resolve(stackFolderPath, stackConfig.fileName)
-  
-    mkdirp.sync(stackFolderPath)
-  
+
+    if (stack && stack.org_uid) {
+      config.org_uid = stack.org_uid
+    }
+  }
+
+  if (!config.preserveStackVersion && !config.hasOwnProperty('master_locale')) {
+    const apiDetails = {
+      limit: 100,
+      skip: 0,
+      include_count: true,
+    }
+    return self.getLocales(apiDetails)
+  } else if (config.preserveStackVersion) {
+    addlogs(config, 'Exporting stack details', 'success');
+    let stackFolderPath = path.resolve(config.data, stackConfig.dirName);
+    let stackContentsFile = path.resolve(stackFolderPath, stackConfig.fileName);
+
+    mkdirp.sync(stackFolderPath);
+
     return new Promise((resolve, reject) => {
-      return client.stack({api_key: config.source_stack}).fetch()
-      .then(response => {
-        helper.writeFile(stackContentsFile, response)
-        addlogs(config, 'Exported stack details successfully!', 'success')
-        return resolve(response)
-      })
-      .catch(reject)
-    })
+      return client
+        .stack({ api_key: config.source_stack })
+        .fetch()
+        .then((response) => {
+          helper.writeFile(stackContentsFile, response);
+          addlogs(config, 'Exported stack details successfully!', 'success');
+          return resolve(response);
+        })
+        .catch(reject);
+    });
   }
+};
+
+ExportStack.prototype.getLocales = function (apiDetails) {
+  let self = this
+  return new Promise((resolve, reject) => {
+    const result = client
+      .stack({ api_key: config.source_stack, management_token: config.management_token })
+      .locale()
+      .query(apiDetails)
+
+    result
+      .find()
+      .then((response) => {
+        const masterLocalObj = response.items.find((obj) => {
+          if (obj.fallback_locale === null) {
+            return obj;
+          }
+        });
+        apiDetails.skip += apiDetails.limit;
+        if (masterLocalObj) { return resolve(masterLocalObj); }
+        else if (apiDetails.skip <= response.count) {
+          return resolve(self.getLocales(apiDetails))
+        }
+        else {
+          return reject('Master locale not found');
+        }
+      })
+      .catch((error) => {
+        return reject(error);
+      });
+  });
 }
 
-module.exports = new ExportStack()
+module.exports = new ExportStack();

package/src/lib/export/webhooks.js

@@ -1,20 +1,20 @@
 /*!
-* Contentstack Export
-* Copyright (c) 2019 Contentstack LLC
-* MIT Licensed
-*/
+ * Contentstack Export
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
 
-const mkdirp = require('mkdirp')
-const path = require('path')
-const chalk = require('chalk')
+const mkdirp = require('mkdirp');
+const path = require('path');
+const chalk = require('chalk');
 
-const helper = require('../util/helper')
-const {addlogs} = require('../util/log')
+const helper = require('../util/helper');
+const { addlogs } = require('../util/log');
 
-let config = require('../../config/default')
-const stack = require('../util/contentstack-management-sdk')
-const webhooksConfig = config.modules.webhooks
-let client
+let config = require('../../config/default');
+const stack = require('../util/contentstack-management-sdk');
+const webhooksConfig = config.modules.webhooks;
+let client;
 
 // Create folder for environments
 
@@ -22,46 +22,54 @@ function ExportWebhooks() {
   this.requestOptions = {
     include_count: true,
     asc: 'updated_at',
-  }
-  this.master = {}
-  this.webhooks = {}
+  };
+  this.master = {};
+  this.webhooks = {};
 }
 
 ExportWebhooks.prototype.start = function (credentialConfig) {
-  addlogs(config, 'Starting webhooks export', 'success')
-  this.master = {}
-  this.webhooks = {}
-  let self = this
-  config = credentialConfig
-  client = stack.Client(config)
-  const webhooksFolderPath = path.resolve(config.data, (config.branchName || ""), webhooksConfig.dirName)
-  mkdirp.sync(webhooksFolderPath)
+  addlogs(config, 'Starting webhooks export', 'success');
+  this.master = {};
+  this.webhooks = {};
+  let self = this;
+  config = credentialConfig;
+  client = stack.Client(config);
+  const webhooksFolderPath = path.resolve(config.data, config.branchName || '', webhooksConfig.dirName);
+  mkdirp.sync(webhooksFolderPath);
   return new Promise(function (resolve, reject) {
-    client.stack({api_key: config.source_stack, management_token: config.management_token}).webhook().fetchAll(self.requestOptions)
-    .then(webhooks => {
-      if (webhooks.items.length !== 0) {
-        for (var i = 0, total = webhooks.count; i < total; i++) {
-          const webUid = webhooks.items[i].uid
-          self.master[webUid] = ''
-          self.webhooks[webUid] = webhooks.items[i]
-          delete self.webhooks[webUid].uid
-          delete self.webhooks[webUid].SYS_ACL
+    client
+      .stack({ api_key: config.source_stack, management_token: config.management_token })
+      .webhook()
+      .fetchAll(self.requestOptions)
+      .then((webhooks) => {
+        if (webhooks.items.length !== 0) {
+          for (var i = 0, total = webhooks.count; i < total; i++) {
+            const webUid = webhooks.items[i].uid;
+            self.master[webUid] = '';
+            self.webhooks[webUid] = webhooks.items[i];
+            delete self.webhooks[webUid].uid;
+            delete self.webhooks[webUid].SYS_ACL;
+          }
+          helper.writeFile(path.join(webhooksFolderPath, webhooksConfig.fileName), self.webhooks);
+          addlogs(config, chalk.green('All the webhooks have been exported successfully'), 'success');
+          return resolve();
         }
-        helper.writeFile(path.join(webhooksFolderPath, webhooksConfig.fileName), self.webhooks)
-        addlogs(config, chalk.green('All the webhooks have been exported successfully'), 'success')
-        return resolve()
-      }
-      addlogs(config, 'No webhooks found', 'success')
-      return resolve()
-    }).catch(function (error) {
-      if (error.statusCode === 401) {
-        addlogs(config, chalk.red('You are not allowed to export webhooks, Unless you provide email and password in config'), 'error')
-        return resolve()
-      }
-      addlogs(config, error, 'error')
-      return reject()
-    })
-  })
-}
+        addlogs(config, 'No webhooks found', 'success');
+        return resolve();
+      })
+      .catch(function (error) {
+        if (error.statusCode === 401) {
+          addlogs(
+            config,
+            chalk.red('You are not allowed to export webhooks, Unless you provide email and password in config'),
+            'error',
+          );
+          return resolve();
+        }
+        addlogs(config, error, 'error');
+        return reject();
+      });
+  });
+};
 
-module.exports = new ExportWebhooks()
+module.exports = new ExportWebhooks();

package/src/lib/export/workflows.js

@@ -4,57 +4,98 @@
  * MIT Licensed
  */
 
-const mkdirp = require('mkdirp')
-const path = require('path')
-const chalk = require('chalk')
+const mkdirp = require('mkdirp');
+const path = require('path');
+const chalk = require('chalk');
 
-let helper = require('../util/helper')
-let {addlogs} = require('../util/log')
+let helper = require('../util/helper');
+let { addlogs } = require('../util/log');
 
-const stack = require('../util/contentstack-management-sdk')
-let config = require('../../config/default')
-let workFlowConfig = config.modules.workflows
-let client
+const stack = require('../util/contentstack-management-sdk');
+let config = require('../../config/default');
+let workFlowConfig = config.modules.workflows;
+let client;
 
 function ExportWorkFlows() {
-  this.workflows = {}
+  this.workflows = {};
 }
 
 ExportWorkFlows.prototype.start = function (credentialConfig) {
-  addlogs(config, 'Starting workflow export', 'success')
-  this.workflows = {}
-  let self = this
-  config = credentialConfig
-  client = stack.Client(config)
-
-  let workflowsFolderPath = path.resolve(config.data, (config.branchName || ""), workFlowConfig.dirName)
-  mkdirp.sync(workflowsFolderPath)
+  addlogs(config, 'Starting workflow export', 'success');
+  this.workflows = {};
+  let self = this;
+  config = credentialConfig;
+  client = stack.Client(config);
+
+  let workflowsFolderPath = path.resolve(config.data, config.branchName || '', workFlowConfig.dirName);
+  mkdirp.sync(workflowsFolderPath);
   return new Promise(function (resolve, reject) {
-    return client.stack({api_key: config.source_stack, management_token: config.management_token}).workflow().fetchAll()
-    .then(response => {
-      if (response.items.length !== 0) {
-        response.items.forEach(function (workflow) {
-          addlogs(config, workflow.name + ' workflow was exported successfully', 'success')
-          self.workflows[workflow.uid] = workflow
-          let deleteItems = config.modules.workflows.invalidKeys
-          deleteItems.forEach(e => delete workflow[e])
-        })
-        addlogs(config, chalk.green('All the workflow have been exported successfully'), 'success')
-      }
-      if (response.items.length === 0) {
-        addlogs(config, 'No workflow were found in the Stack', 'success')
+    return client
+      .stack({ api_key: config.source_stack, management_token: config.management_token })
+      .workflow()
+      .fetchAll()
+      .then(async (response) => {
+        try {
+          if (response.items.length) {
+            await getWorkflowsData(self, response.items);
+            addlogs(config, chalk.green('All the workflow have been exported successfully'), 'success');
+          }
+          if (!response.items.length) {
+            addlogs(config, 'No workflow were found in the Stack', 'success');
+          }
+          helper.writeFile(path.join(workflowsFolderPath, workFlowConfig.fileName), self.workflows);
+          return resolve();
+        } catch (error) {
+          return reject(error);
+        }
+      })
+      .catch(function (error) {
+        if (error.statusCode === 401) {
+          addlogs(
+            config,
+            chalk.red(
+              'You are not allowed to export workflow, Unless you provide email and password in config',
+              'error',
+            ),
+          );
+          return resolve();
+        }
+        addlogs(config, error, 'error');
+        return resolve();
+      });
+  });
+};
+
+const getWorkflowsData = async (self, workflows) => {
+  try {
+    for (const workflow of workflows) {
+      addlogs(config, workflow.name + ' workflow was exported successfully', 'success');
+      await getWorkflowRoles(workflow);
+      self.workflows[workflow.uid] = workflow;
+      let deleteItems = config.modules.workflows.invalidKeys;
+      deleteItems.forEach((e) => delete workflow[e]);
+    }
+  } catch (error) {
+    throw error;
+  }
+};
+
+const getWorkflowRoles = async (workflow) => {
+  try {
+    for (const stage of workflow.workflow_stages) {
+      if (stage.SYS_ACL.roles.uids.length) {
+        for (let i = 0; i < stage.SYS_ACL.roles.uids.length; i++) {
+          const roleUid = stage.SYS_ACL.roles.uids[i];
+          const roleData = await client.stack({ api_key: config.source_stack, management_token: config.management_token }).role(roleUid).fetch({ include_rules: true, include_permissions: true });
+          stage.SYS_ACL.roles.uids[i] = roleData;
+        }
       }
-      helper.writeFile(path.join(workflowsFolderPath, workFlowConfig.fileName), self.workflows)
-      return resolve()
-    }).catch(function (error) {
-      if (error.statusCode === 401) {
-        addlogs(config, chalk.red('You are not allowed to export workflow, Unless you provide email and password in config', 'error'))
-        return resolve()
-      } 
-      addlogs(config, error, 'error')
-      return resolve()
-    })
-  })
-}
+    }
+  } catch (error) {
+    console.log('Error getting roles', error && error.message);
+    addlogs(config, 'Error fetching roles in export workflows task.', 'error');
+    throw { message: 'Error fetching roles in export workflows task.' };
+  }
+};
 
-module.exports = new ExportWorkFlows()
+module.exports = new ExportWorkFlows();

package/src/lib/util/contentstack-management-sdk.js

@@ -1,19 +1,39 @@
-const contentstacksdk = require('@contentstack/management')
+const contentstacksdk = require('@contentstack/management');
+const https = require('https');
+
+const { addlogs } = require('./log');
 
 exports.Client = function (config) {
   const option = {
     host: config.host,
     authtoken: config.auth_token,
     api_key: config.source_stack,
+    maxRequests: 10,
+    retryLimit: 5,
+    timeout: 60000,
+    httpsAgent: new https.Agent({
+      maxSockets: 100,
+      maxFreeSockets: 10,
+      keepAlive: true,
+      timeout: 60000, // active socket keepalive for 60 seconds
+      freeSocketTimeout: 30000, // free socket keepalive for 30 seconds
+    }),
+    retryDelay: Math.floor(Math.random() * (8000 - 3000 + 1) + 3000),
     logHandler: (level, data) => {},
-  }
+    retryCondition: (error) => {
+      if (error.response.status === 408) {
+        addlogs({ data: error.response }, 'Timeout error', 'error');
+      }
+      return false;
+    },
+  };
 
   if (typeof config.branchName === 'string') {
     option.headers = {
       branch: config.branchName,
-    }
+    };
   }
 
-  const client = contentstacksdk.client(option)
-  return client
-}
+  const client = contentstacksdk.client(option);
+  return client;
+};