@contentstack/cli-cm-export 1.2.3 → 1.3.0

package/src/lib/export/entries.js

@@ -37,6 +37,11 @@ class EntriesExport {
       }
       const entryRequestOptions = this.createRequestObjects(locales, contentTypes);
       for (let requestOption of entryRequestOptions) {
+        addlogs(
+          this.exportConfig,
+          `Starting export of entries of content_type - ${requestOption.content_type} locale - ${requestOption.locale}`,
+          'info',
+        );
         await fileHelper.makeDirectory(path.join(this.entriesRootPath, requestOption.content_type));
         const entries = await this.getEntries(requestOption);
         let entriesFilePath = path.join(
@@ -44,7 +49,7 @@ class EntriesExport {
           requestOption.content_type,
           requestOption.locale + '.json',
         );
-        await fileHelper.writeFile(entriesFilePath, entries);
+        await fileHelper.writeLargeFile(entriesFilePath, entries);
         addlogs(
           this.exportConfig,
           `Exported entries of type ${requestOption.content_type} locale ${requestOption.locale}`,

package/src/lib/export/environments.js

@@ -6,11 +6,11 @@
 const path = require('path');
 const chalk = require('chalk');
 const mkdirp = require('mkdirp');
+const { merge } = require('lodash');
 
 const helper = require('../util/helper');
 const { addlogs } = require('../util/log');
 const { formatError } = require('../util');
-let stack = require('../util/contentstack-management-sdk');
 
 module.exports = class ExportEnvironments {
   config = {};
@@ -24,13 +24,13 @@ module.exports = class ExportEnvironments {
     },
   };
 
-  constructor(mergeConfig) {
-    this.config = mergeConfig;
+  constructor(exportConfig, stackAPIClient) {
+    this.stackAPIClient = stackAPIClient;
+    this.config = merge(this.config, exportConfig);
   }
 
   start() {
     const self = this;
-    const client = stack.Client(self.config);
     const environmentConfig = self.config.modules.environments;
     const environmentsFolderPath = path.resolve(
       self.config.data,
@@ -43,15 +43,14 @@ module.exports = class ExportEnvironments {
     addlogs(self.config, 'Starting environment export', 'success');
 
     return new Promise(function (resolve, reject) {
-      client
-        .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+      self.stackAPIClient
         .environment()
         .query(self.requestOptions.qs)
         .find()
         .then((environmentResponse) => {
           if (environmentResponse.items.length !== 0) {
             for (let i = 0, total = environmentResponse.count; i < total; i++) {
-              let envUid = environmentResponse.items[i].uid;
+              const envUid = environmentResponse.items[i].uid;
               self.master[envUid] = '';
               self.environments[envUid] = environmentResponse.items[i];
               delete self.environments[envUid].uid;

package/src/lib/export/extensions.js

@@ -7,12 +7,12 @@
 const mkdirp = require('mkdirp');
 const path = require('path');
 const chalk = require('chalk');
+const { merge } = require('lodash');
 
 const helper = require('../util/helper');
 const { addlogs } = require('../util/log');
 const { formatError } = require('../util');
 let config = require('../../config/default');
-const stack = require('../util/contentstack-management-sdk');
 
 module.exports = class ExportExtensions {
   master = {};
@@ -23,8 +23,9 @@ module.exports = class ExportExtensions {
     include_count: true,
   };
 
-  constructor(mergeConfig) {
-    this.config = mergeConfig;
+  constructor(exportConfig, stackAPIClient) {
+    this.stackAPIClient = stackAPIClient;
+    this.config = merge(config, exportConfig);
   }
 
   start() {
@@ -38,17 +39,15 @@ module.exports = class ExportExtensions {
     );
     // Create folder for extensions
     mkdirp.sync(extensionsFolderPath);
-    let client = stack.Client(this.config);
     return new Promise(function (resolve, reject) {
-      client
-        .stack({ api_key: config.source_stack, management_token: config.management_token })
+      self.stackAPIClient
         .extension()
         .query(self.queryRequestOptions)
         .find()
         .then((extension) => {
           if (extension.items.length !== 0) {
             for (let i = 0, total = extension.count; i < total; i++) {
-              let extUid = extension.items[i].uid;
+              const extUid = extension.items[i].uid;
               self.master[extUid] = '';
               self.extensions[extUid] = extension.items[i];
               delete self.extensions[extUid].uid;

package/src/lib/export/global-fields.js

@@ -7,12 +7,12 @@
 const path = require('path');
 const chalk = require('chalk');
 const mkdirp = require('mkdirp');
+const { merge } = require('lodash');
 
 const helper = require('../util/helper');
 const { addlogs } = require('../util/log');
 const { formatError } = require('../util');
 let config = require('../../config/default');
-const stack = require('../util/contentstack-management-sdk');
 
 module.exports = class ExportGlobalFields {
   limit = 100;
@@ -25,7 +25,7 @@ module.exports = class ExportGlobalFields {
   globalfieldsConfig = config.modules.globalfields;
   validKeys = config.modules.globalfields.validKeys;
 
-  constructor(credentialConfig) {
+  constructor(exportConfig, stackAPIClient) {
     this.requestOptions = {
       qs: {
         skip: 0,
@@ -34,7 +34,8 @@ module.exports = class ExportGlobalFields {
         include_count: true,
       },
     };
-    this.config = { ...config, ...credentialConfig };
+    this.stackAPIClient = stackAPIClient;
+    this.config = merge(config, exportConfig);
     this.globalfieldsFolderPath = path.resolve(
       this.config.data,
       this.config.branchName || '',
@@ -68,16 +69,9 @@ module.exports = class ExportGlobalFields {
 
   getGlobalFields(skip, globalFieldConfig) {
     const self = this;
-    let client = stack.Client(globalFieldConfig);
-
     self.requestOptions.qs.skip = skip;
-
     return new Promise(function (resolve, reject) {
-      client
-        .stack({
-          api_key: globalFieldConfig.source_stack,
-          management_token: globalFieldConfig.management_token,
-        })
+      self.stackAPIClient
         .globalField()
         .query(self.requestOptions.qs)
         .find()
@@ -107,6 +101,9 @@ module.exports = class ExportGlobalFields {
             addlogs(globalFieldConfig, chalk.red(`Failed to export global-fields ${formatError(error)}`), 'error');
             reject(error);
           }
+        })
+        .catch((error) => {
+          reject(error);
         });
     });
   }
@@ -115,7 +112,10 @@ module.exports = class ExportGlobalFields {
     const self = this;
     return new Promise(function (resolve, reject) {
       try {
-        helper.writeFileSync(path.join(self.globalfieldsFolderPath, self.globalfieldsConfig.fileName), self.global_fields);
+        helper.writeFileSync(
+          path.join(self.globalfieldsFolderPath, self.globalfieldsConfig.fileName),
+          self.global_fields,
+        );
         addlogs(self.config, chalk.green('Global Fields export completed successfully'), 'success');
 
         resolve();

package/src/lib/export/labels.js

@@ -9,31 +9,28 @@ const chalk = require('chalk');
 const mkdirp = require('mkdirp');
 const { merge } = require('lodash');
 
-let helper = require('../util/helper');
-let { addlogs } = require('../util/log');
+const helper = require('../util/helper');
+const { addlogs } = require('../util/log');
 const { formatError } = require('../util');
 const config = require('../../config/default');
-const stack = require('../util/contentstack-management-sdk');
 
 module.exports = class ExportLabels {
   labels = {};
   labelConfig = config.modules.labels;
 
-  constructor(credentialConfig) {
-    this.config = merge(config, credentialConfig);
+  constructor(exportConfig, stackAPIClient) {
+    this.stackAPIClient = stackAPIClient;
+    this.config = merge(config, exportConfig);
   }
 
   start() {
     addlogs(this.config, 'Starting labels export', 'success');
-
     const self = this;
-    const client = stack.Client(this.config);
-    let labelsFolderPath = path.resolve(config.data, this.config.branchName || '', self.labelConfig.dirName);
+    const labelsFolderPath = path.resolve(config.data, this.config.branchName || '', self.labelConfig.dirName);
     // Create locale folder
     mkdirp.sync(labelsFolderPath);
     return new Promise(function (resolve, reject) {
-      return client
-        .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+      return self.stackAPIClient
         .label()
         .query()
         .find()
@@ -42,7 +39,7 @@ module.exports = class ExportLabels {
             response.items.forEach(function (label) {
               addlogs(self.config, label.name + ' labels was exported successfully', 'success');
               self.labels[label.uid] = label;
-              let deleteItems = self.config.modules.labels.invalidKeys;
+              const deleteItems = self.config.modules.labels.invalidKeys;
               deleteItems.forEach((e) => delete label[e]);
             });
             addlogs(self.config, chalk.green('All the labels have been exported successfully'), 'success');

package/src/lib/export/marketplace-apps.js

@@ -7,17 +7,18 @@ const _ = require('lodash');
 const path = require('path');
 const chalk = require('chalk');
 const mkdirp = require('mkdirp');
-const eachOf = require('async/eachOf');
-const { cliux, HttpClient, NodeCrypto } = require('@contentstack/cli-utilities');
+const { cliux, HttpClient, NodeCrypto, managementSDKClient } = require('@contentstack/cli-utilities');
 
 const { formatError } = require('../util');
-let config = require('../../config/default');
-const { writeFileSync } = require('../util/helper');
+const config = require('../../config/default');
 const { addlogs: log } = require('../util/log');
-const { getDeveloperHubUrl, getInstalledExtensions } = require('../util/marketplace-app-helper');
+const { writeFileSync } = require('../util/helper');
+const { getDeveloperHubUrl } = require('../util/marketplace-app-helper');
 
 module.exports = class ExportMarketplaceApps {
+  client;
   config;
+  httpClient;
   nodeCrypto;
   marketplaceAppPath = null;
   developerHubBaseUrl = null;
@@ -38,6 +39,13 @@ module.exports = class ExportMarketplaceApps {
       return Promise.resolve();
     }
 
+    await this.getOrgUid();
+
+    this.httpClient = new HttpClient().headers({
+      authtoken: this.config.auth_token,
+      organization_uid: this.config.org_uid,
+    });
+
     log(this.config, 'Starting marketplace app export', 'success');
     this.marketplaceAppPath = path.resolve(
       this.config.data,
@@ -49,6 +57,22 @@ module.exports = class ExportMarketplaceApps {
     return this.exportInstalledExtensions();
   }
 
+  async getOrgUid() {
+    if (this.config.auth_token) {
+      const tempAPIClient = await managementSDKClient({ host: this.config.host });
+      const tempStackData = await tempAPIClient
+        .stack({ api_key: this.config.source_stack })
+        .fetch()
+        .catch((error) => {
+          console.log(error);
+        });
+
+      if (tempStackData && tempStackData.org_uid) {
+        this.config.org_uid = tempStackData.org_uid;
+      }
+    }
+  }
+
   async createNodeCryptoInstance() {
     const cryptoArgs = {};
 
@@ -71,103 +95,79 @@ module.exports = class ExportMarketplaceApps {
     this.nodeCrypto = new NodeCrypto(cryptoArgs);
   }
 
-  exportInstalledExtensions = () => {
-    const self = this;
-    const headers = {
-      authtoken: self.config.auth_token,
-      organization_uid: self.config.org_uid,
-    };
-    const httpClient = new HttpClient().headers(headers);
-
-    return new Promise(async function (resolve, reject) {
-      getInstalledExtensions(self.config)
-        .then(async (items) => {
-          const installedApps = _.map(
-            _.filter(items, 'app_uid'),
-            ({ uid, title, app_uid, app_installation_uid, type }) => ({
-              title,
-              uid,
-              app_uid,
-              app_installation_uid,
-              type,
-            }),
-          );
-          const uniqApps = _.uniqBy(installedApps, 'app_uid');
-
-          const developerHubApps =
-            (await httpClient
-              .get(`${self.developerHubBaseUrl}/apps`)
-              .then(({ data: { data } }) => data)
-              .catch((err) => {
-                log(self.config, err, 'error');
-              })) || [];
-
-          for (const app of uniqApps) {
-            log(self.config, `Exporting ${app.title} app and it's config.`, 'success');
-            const listOfIndexToBeUpdated = _.map(installedApps, ({ app_uid }, index) =>
-              app_uid === app.app_uid ? index : undefined,
-            ).filter((val) => val !== undefined);
-
-            await httpClient
-              .get(`${self.developerHubBaseUrl}/installations/${app.app_installation_uid}/installationData`)
-              .then(async ({ data: result }) => {
-                const { data, error } = result;
-                const developerHubApp = _.find(developerHubApps, { uid: app.app_uid });
-
-                if ((_.has(data, 'configuration') || _.has(data, 'server_configuration')) && !self.nodeCrypto) {
-                  await self.createNodeCryptoInstance();
-                }
-
-                _.forEach(listOfIndexToBeUpdated, (index, i) => {
-                  if (developerHubApp) {
-                    installedApps[index]['visibility'] = developerHubApp.visibility;
-                    installedApps[index]['manifest'] = _.pick(
-                      developerHubApp,
-                      ['name', 'description', 'icon', 'target_type', 'ui_location', 'webhook', 'oauth'], // NOTE keys can be passed to install new app in the developer hub
-                    );
-                  }
-
-                  if (_.has(data, 'configuration') || _.has(data, 'server_configuration')) {
-                    const { configuration, server_configuration } = data;
-
-                    if (!_.isEmpty(configuration)) {
-                      installedApps[index]['configuration'] = self.nodeCrypto.encrypt(configuration);
-                    }
-                    if (!_.isEmpty(server_configuration)) {
-                      installedApps[index]['server_configuration'] = self.nodeCrypto.encrypt(server_configuration);
-                    }
-
-                    if (i === 0) {
-                      log(self.config, `Exported ${app.title} app and it's config.`, 'success');
-                    }
-                  } else if (error) {
-                    console.log(error);
-                    if (i === 0) {
-                      log(self.config, `Error on exporting ${app.title} app and it's config.`, 'error');
-                    }
-                  }
-                });
-              })
-              .catch((err) => {
-                log(self.config, `Failed to export ${app.title} app config ${formatError(error)}`, 'error');
-                console.log(err);
-              });
-          }
+  async exportInstalledExtensions() {
+    const installedApps = (await this.getAllStackSpecificApps()) || [];
 
-          if (!_.isEmpty(installedApps)) {
-            await writeFileSync(path.join(self.marketplaceAppPath, self.marketplaceAppConfig.fileName), installedApps);
+    if (!_.isEmpty(installedApps)) {
+      for (const [index, app] of _.entries(installedApps)) {
+        await this.getAppConfigurations(installedApps, [+index, app]);
+      }
 
-            log(self.config, chalk.green('All the marketplace apps have been exported successfully'), 'success');
-          } else {
-            log(self.config, 'No marketplace apps found', 'success');
+      await writeFileSync(path.join(this.marketplaceAppPath, this.marketplaceAppConfig.fileName), installedApps);
+
+      log(this.config, chalk.green('All the marketplace apps have been exported successfully'), 'success');
+    } else {
+      log(this.config, 'No marketplace apps found', 'success');
+    }
+  }
+
+  getAllStackSpecificApps(listOfApps = [], skip = 0) {
+    return this.httpClient
+      .get(`${this.developerHubBaseUrl}/installations?target_uids=${this.config.source_stack}&skip=${skip}`)
+      .then(async ({ data }) => {
+        const { data: apps, count } = data;
+
+        if (!this.nodeCrypto && _.find(apps, (app) => !_.isEmpty(app.configuration))) {
+          await this.createNodeCryptoInstance();
+        }
+
+        listOfApps.push(
+          ..._.map(apps, (app) => {
+            if (_.has(app, 'configuration')) {
+              app['configuration'] = this.nodeCrypto.encrypt(app.configuration || configuration);
+            }
+
+            return app;
+          }),
+        );
+
+        if (count - (skip + 50) > 0) {
+          return await this.getAllStackSpecificApps(listOfApps, skip + 50);
+        }
+
+        return listOfApps;
+      })
+      .catch((error) => {
+        log(self.config, `Failed to export marketplace-apps ${formatError(error)}`, 'error');
+      });
+  }
+
+  getAppConfigurations(installedApps, [index, app]) {
+    const appName = app.manifest.name;
+    log(this.config, `Exporting ${appName} app and it's config.`, 'success');
+
+    return this.httpClient
+      .get(`${this.developerHubBaseUrl}/installations/${app.uid}/installationData`)
+      .then(async ({ data: result }) => {
+        const { data, error } = result;
+
+        if (_.has(data, 'server_configuration')) {
+          if (!this.nodeCrypto && _.has(data, 'server_configuration')) {
+            await this.createNodeCryptoInstance();
           }
 
-          resolve();
-        })
-        .catch((error) => {
-          log(self.config, `Failed to export marketplace-apps ${formatError(error)}`, 'error');
-          reject(error);
-        });
-    });
-  };
+          if (!_.isEmpty(data.server_configuration)) {
+            installedApps[index]['server_configuration'] = this.nodeCrypto.encrypt(data.server_configuration);
+            log(this.config, `Exported ${appName} app and it's config.`, 'success');
+          } else {
+            log(this.config, `Exported ${appName} app`, 'success');
+          }
+        } else if (error) {
+          log(this.config, `Error on exporting ${appName} app and it's config.`, 'error');
+        }
+      })
+      .catch((err) => {
+        log(this.config, `Failed to export ${appName} app config ${formatError(err)}`, 'error');
+      });
+  }
 };

package/src/lib/export/stack.js

@@ -4,102 +4,96 @@
  * MIT Licensed
  */
 
-let path = require('path');
-let mkdirp = require('mkdirp');
+const path = require('path');
+const mkdirp = require('mkdirp');
+const { merge } = require('lodash');
+const helper = require('../util/helper');
+const { addlogs } = require('../util/log');
+const config = require('../../config/default');
+const { managementSDKClient } = require('@contentstack/cli-utilities');
+module.exports = class ExportStack {
+  stackConfig = config.modules.stack;
 
-let helper = require('../util/helper');
-let { addlogs } = require('../util/log');
-let config = require('../../config/default');
-const stack = require('../util/contentstack-management-sdk');
+  constructor(exportConfig, stackAPIClient, APIClient) {
+    this.config = merge(config, exportConfig);
+    this.stackAPIClient = stackAPIClient;
+    this.APIClient = APIClient;
+    this.requestOption = {
+      uri: this.config.host + this.config.apis.stacks,
+      headers: this.config.headers,
+      json: true,
+    };
+  }
 
-let client;
-let stackConfig = config.modules.stack;
+  async start() {
+    const self = this;
+    if (self.config.auth_token) {
+      const tempAPIClient = await managementSDKClient({ host: config.host });
+      const tempStackData = await tempAPIClient
+        .stack({ api_key: self.config.source_stack })
+        .fetch()
+        .catch((error) => {
+          console.log(error);
+        });
 
-function ExportStack() {
-  this.requestOption = {
-    uri: config.host + config.apis.stacks,
-    headers: config.headers,
-    json: true,
-  };
-}
+      if (tempStackData && tempStackData.org_uid) {
+        self.config.org_uid = tempStackData.org_uid;
+        self.config.sourceStackName = tempStackData.name;
+      }
+    }
 
-ExportStack.prototype.start = async function (credentialConfig) {
-  config = credentialConfig;
-  client = stack.Client(config);
-  const self = this;
+    if (!self.config.preserveStackVersion && !self.config.hasOwnProperty('master_locale')) {
+      const apiDetails = {
+        limit: 100,
+        skip: 0,
+        include_count: true,
+      };
+      return self.getLocales(apiDetails);
+    } else if (self.config.preserveStackVersion) {
+      addlogs(self.config, 'Exporting stack details', 'success');
+      let stackFolderPath = path.resolve(self.config.data, stackConfig.dirName);
+      let stackContentsFile = path.resolve(stackFolderPath, stackConfig.fileName);
 
-  // NOTE get org uid
-  if (config.auth_token) {
-    const stack = await client
-      .stack({ api_key: config.source_stack, authtoken: config.auth_token })
-      .fetch()
-      .catch((error) => {
-        console.log(error);
-      });
+      mkdirp.sync(stackFolderPath);
 
-    if (stack && stack.org_uid) {
-      config.org_uid = stack.org_uid;
-      config.sourceStackName = stack.name;
+      return new Promise((resolve, reject) => {
+        return self.APIClient.stack({ api_key: self.config.source_stack })
+          .fetch()
+          .then((response) => {
+            helper.writeFile(stackContentsFile, response);
+            addlogs(self.config, 'Exported stack details successfully!', 'success');
+            return resolve(response);
+          })
+          .catch(reject);
+      });
     }
   }
 
-  if (!config.preserveStackVersion && !config.hasOwnProperty('master_locale')) {
-    const apiDetails = {
-      limit: 100,
-      skip: 0,
-      include_count: true,
-    };
-    return self.getLocales(apiDetails);
-  } else if (config.preserveStackVersion) {
-    addlogs(config, 'Exporting stack details', 'success');
-    let stackFolderPath = path.resolve(config.data, stackConfig.dirName);
-    let stackContentsFile = path.resolve(stackFolderPath, stackConfig.fileName);
-
-    mkdirp.sync(stackFolderPath);
-
+  getLocales(apiDetails) {
+    const self = this;
     return new Promise((resolve, reject) => {
-      return client
-        .stack({ api_key: config.source_stack })
-        .fetch()
+      const result = self.stackAPIClient.locale().query(apiDetails);
+
+      result
+        .find()
         .then((response) => {
-          helper.writeFile(stackContentsFile, response);
-          addlogs(config, 'Exported stack details successfully!', 'success');
-          return resolve(response);
+          const masterLocalObj = response.items.find((obj) => {
+            if (obj.fallback_locale === null) {
+              return obj;
+            }
+          });
+          apiDetails.skip += apiDetails.limit;
+          if (masterLocalObj) {
+            return resolve(masterLocalObj);
+          } else if (apiDetails.skip <= response.count) {
+            return resolve(self.getLocales(apiDetails));
+          } else {
+            return reject('Master locale not found');
+          }
         })
-        .catch(reject);
+        .catch((error) => {
+          return reject(error);
+        });
     });
   }
 };
-
-ExportStack.prototype.getLocales = function (apiDetails) {
-  let self = this;
-  return new Promise((resolve, reject) => {
-    const result = client
-      .stack({ api_key: config.source_stack, management_token: config.management_token })
-      .locale()
-      .query(apiDetails);
-
-    result
-      .find()
-      .then((response) => {
-        const masterLocalObj = response.items.find((obj) => {
-          if (obj.fallback_locale === null) {
-            return obj;
-          }
-        });
-        apiDetails.skip += apiDetails.limit;
-        if (masterLocalObj) {
-          return resolve(masterLocalObj);
-        } else if (apiDetails.skip <= response.count) {
-          return resolve(self.getLocales(apiDetails));
-        } else {
-          return reject('Master locale not found');
-        }
-      })
-      .catch((error) => {
-        return reject(error);
-      });
-  });
-};
-
-module.exports = ExportStack;