@contentstack/cli-cm-export 1.0.0 → 1.2.0

package/src/lib/export/global-fields.js

@@ -4,77 +4,74 @@
  * MIT Licensed
  */
 
-const mkdirp = require('mkdirp');
 const path = require('path');
 const chalk = require('chalk');
+const mkdirp = require('mkdirp');
 
-const stack = require('../util/contentstack-management-sdk');
 const helper = require('../util/helper');
 const { addlogs } = require('../util/log');
-
+const { formatError } = require('../util');
 let config = require('../../config/default');
-const limit = 100;
-const validKeys = config.modules.globalfields.validKeys;
-let globalfieldsFolderPath;
-const globalfieldsConfig = config.modules.globalfields;
-function ExportGlobalFields() {
-  this.global_fields = [];
-  this.requestOptions = {
-    qs: {
-      include_count: true,
-      asc: 'updated_at',
-      limit: limit,
-    },
-  };
-  this.master = {};
-  this.globalfields = {};
-}
+const stack = require('../util/contentstack-management-sdk');
 
-ExportGlobalFields.prototype = {
-  start: function (credentialConfig) {
-    this.master = {};
-    this.globalfields = {};
-    config = { ...config, ...credentialConfig };
-    globalfieldsFolderPath = path.resolve(config.data, config.branchName || '', globalfieldsConfig.dirName);
-    // Create folder for Global Fields
-    mkdirp.sync(globalfieldsFolderPath);
+module.exports = class ExportGlobalFields {
+  limit = 100;
+  config = {};
+  global_fields = [];
+  master = {};
+  globalfields = {};
+  requestOptions = {};
+  globalfieldsFolderPath;
+  globalfieldsConfig = config.modules.globalfields;
+  validKeys = config.modules.globalfields.validKeys;
+
+  constructor(credentialConfig) {
+    this.requestOptions = {
+      qs: {
+        skip: 0,
+        limit: this.limit,
+        asc: 'updated_at',
+        include_count: true,
+      },
+    };
+    this.config = { ...config, ...credentialConfig };
+    this.globalfieldsFolderPath = path.resolve(
+      this.config.data,
+      this.config.branchName || '',
+      this.globalfieldsConfig.dirName,
+    );
+  }
+
+  start() {
     const self = this;
-    addlogs(config, 'Starting Global Fields export', 'success');
+    // Create folder for Global Fields
+    mkdirp.sync(self.globalfieldsFolderPath);
+    addlogs(self.config, 'Starting Global Fields export', 'success');
+
     return new Promise(function (resolve, reject) {
       try {
         return self
-          .getGlobalFields(null, config)
+          .getGlobalFields(0, self.config)
           .then(function (result) {
             if (!result) {
-              return self
-                .writeGlobalFields()
-                .then(() => {
-                  return resolve();
-                })
-                .catch((error) => {
-                  return reject(error);
-                });
+              return self.writeGlobalFields().then(resolve).catch(reject);
             }
             return resolve();
           })
-          .catch((error) => {
-            return reject(error);
-          });
+          .catch(reject);
       } catch (error) {
+        addlogs(self.config, error, 'error');
         return reject(error);
       }
     });
-  },
-  getGlobalFields: function (skip, globalFieldConfig) {
-    const self = this;
-    if (typeof skip !== 'number') {
-      skip = 0;
-      self.requestOptions.qs.skip = skip;
-    } else {
-      self.requestOptions.qs.skip = skip;
-    }
+  }
 
+  getGlobalFields(skip, globalFieldConfig) {
+    const self = this;
     let client = stack.Client(globalFieldConfig);
+
+    self.requestOptions.qs.skip = skip;
+
     return new Promise(function (resolve, reject) {
       client
         .stack({
@@ -92,14 +89,14 @@ ExportGlobalFields.prototype = {
             }
             globalFieldResponse.items.forEach(function (globalField) {
               for (const key in globalField) {
-                if (validKeys.indexOf(key) === -1) {
+                if (self.validKeys.indexOf(key) === -1) {
                   delete globalField[key];
                 }
               }
               self.global_fields.push(globalField);
             });
 
-            skip += limit;
+            skip += self.limit;
 
             if (skip > globalFieldResponse.count) {
               return resolve();
@@ -107,19 +104,25 @@ ExportGlobalFields.prototype = {
 
             return self.getGlobalFields(skip, globalFieldConfig).then(resolve).catch(reject);
           } catch (error) {
-            return reject(error);
+            addlogs(globalFieldConfig, chalk.red(`Failed to export global-fields ${formatError(error)}`), 'error');
+            reject(error);
           }
         });
     });
-  },
-  writeGlobalFields: function () {
+  }
+
+  writeGlobalFields() {
     const self = this;
-    return new Promise(function (resolve) {
-      helper.writeFile(path.join(globalfieldsFolderPath, globalfieldsConfig.fileName), self.global_fields);
-      addlogs(config, chalk.green('Global Fields export completed successfully'), 'success');
-      return resolve();
+    return new Promise(function (resolve, reject) {
+      try {
+        helper.writeFileSync(path.join(self.globalfieldsFolderPath, self.globalfieldsConfig.fileName), self.global_fields);
+        addlogs(self.config, chalk.green('Global Fields export completed successfully'), 'success');
+
+        resolve();
+      } catch (error) {
+        addlogs(self.config, error, 'error');
+        reject(error);
+      }
     });
-  },
+  }
 };
-
-module.exports = new ExportGlobalFields();

package/src/lib/export/labels.js

@@ -4,69 +4,69 @@
  * MIT Licensed
  */
 
-const mkdirp = require('mkdirp');
 const path = require('path');
 const chalk = require('chalk');
+const mkdirp = require('mkdirp');
+const { merge } = require('lodash');
 
 let helper = require('../util/helper');
 let { addlogs } = require('../util/log');
-
+const { formatError } = require('../util');
+const config = require('../../config/default');
 const stack = require('../util/contentstack-management-sdk');
-let config = require('../../config/default');
-let labelConfig = config.modules.labels;
-let client;
 
-function ExportLabels() {
-  this.requestOptions = {
-    url: config.host + config.apis.labels,
-    headers: config.headers,
-    json: true,
-  };
-  this.labels = {};
-}
+module.exports = class ExportLabels {
+  labels = {};
+  labelConfig = config.modules.labels;
 
-ExportLabels.prototype.start = function (credentialConfig) {
-  addlogs(config, 'Starting labels export', 'success');
-  this.labels = {};
-  let self = this;
-  config = credentialConfig;
-  client = stack.Client(config);
-  let labelsFolderPath = path.resolve(config.data, config.branchName || '', labelConfig.dirName);
-  // Create locale folder
-  mkdirp.sync(labelsFolderPath);
-  return new Promise(function (resolve, reject) {
-    return client
-      .stack({ api_key: config.source_stack, management_token: config.management_token })
-      .label()
-      .query()
-      .find()
-      .then((response) => {
-        if (response.items.length !== 0) {
-          response.items.forEach(function (label) {
-            addlogs(config, label.name + ' labels was exported successfully', 'success');
-            self.labels[label.uid] = label;
-            let deleteItems = config.modules.labels.invalidKeys;
-            deleteItems.forEach((e) => delete label[e]);
-          });
-          addlogs(config, chalk.green('All the labels have been exported successfully'), 'success');
-        } else {
-          addlogs(config, 'No labels found', 'success');
-        }
-        helper.writeFile(path.join(labelsFolderPath, labelConfig.fileName), self.labels);
-        return resolve();
-      })
-      .catch(function (error) {
-        if (error.statusCode === 401) {
-          addlogs(
-            config,
-            chalk.red('You are not allowed to export label, Unless you provide email and password in config', 'error'),
-          );
-          return resolve();
-        }
-        addlogs(config, error, 'error');
-        return reject();
-      });
-  });
-};
+  constructor(credentialConfig) {
+    this.config = merge(config, credentialConfig);
+  }
 
-module.exports = new ExportLabels();
+  start() {
+    addlogs(this.config, 'Starting labels export', 'success');
+
+    const self = this;
+    const client = stack.Client(this.config);
+    let labelsFolderPath = path.resolve(config.data, this.config.branchName || '', self.labelConfig.dirName);
+    // Create locale folder
+    mkdirp.sync(labelsFolderPath);
+    return new Promise(function (resolve, reject) {
+      return client
+        .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+        .label()
+        .query()
+        .find()
+        .then((response) => {
+          if (response.items.length !== 0) {
+            response.items.forEach(function (label) {
+              addlogs(self.config, label.name + ' labels was exported successfully', 'success');
+              self.labels[label.uid] = label;
+              let deleteItems = self.config.modules.labels.invalidKeys;
+              deleteItems.forEach((e) => delete label[e]);
+            });
+            addlogs(self.config, chalk.green('All the labels have been exported successfully'), 'success');
+          } else {
+            addlogs(self.config, 'No labels found', 'success');
+          }
+          helper.writeFileSync(path.join(labelsFolderPath, self.labelConfig.fileName), self.labels);
+          resolve();
+        })
+        .catch(function (error) {
+          if (error.statusCode === 401) {
+            addlogs(
+              self.config,
+              chalk.red(
+                'You are not allowed to export label, Unless you provide email and password in config',
+                'error',
+              ),
+            );
+            return resolve();
+          }
+
+          addlogs(self.config, formatError(error), 'error');
+          reject();
+        });
+    });
+  }
+};

package/src/lib/export/locales.js

@@ -1,103 +1,69 @@
-/*!
- * Contentstack Export
- * Copyright (c) 2019 Contentstack LLC
- * MIT Licensed
- */
-
-const mkdirp = require('mkdirp');
 const path = require('path');
 const chalk = require('chalk');
-
-const helper = require('../util/helper');
+const fileHelper = require('../util/helper');
 const { addlogs } = require('../util/log');
-let config = require('../../config/default');
-let localeConfig = config.modules.locales;
-const masterLocale = config.master_locale;
-let requiredKeys = localeConfig.requiredKeys;
-let stack = require('../util/contentstack-management-sdk');
-let client
-function ExportLocales() {
-  this.qs = {
-    include_count: true,
-    asc: 'updated_at',
-    query: {
-      code: {
-        $nin: [masterLocale.code],
+class LocaleExport {
+  constructor(exportConfig, stackAPIClient) {
+    this.stackAPIClient = stackAPIClient;
+    this.exportConfig = exportConfig;
+    this.localeConfig = exportConfig.modules.locales;
+    this.qs = {
+      include_count: true,
+      asc: 'updated_at',
+      query: {
+        code: {
+          $nin: [exportConfig.master_locale.code],
+        },
       },
-    },
-    only: {
-      BASE: requiredKeys,
-    },
-  };
-  this.locales = {};
-}
+      only: {
+        BASE: this.localeConfig.requiredKeys,
+      },
+    };
 
-ExportLocales.prototype.start = function (credentialConfig) {
-  this.locales = {};
-  addlogs(credentialConfig, 'Starting locale export', 'success');
-  let self = this;
-  config = credentialConfig;
-  self.localesFolderPath = path.resolve(config.data, config.branchName || '', localeConfig.dirName);
-  mkdirp.sync(self.localesFolderPath);
-  client = stack.Client(config);
-  const apiDetails = {
-    limit: 100,
-    skip: 0,
-    include_count: true,
+    this.localesPath = path.resolve(exportConfig.data, exportConfig.branchName || '', this.localeConfig.dirName);
+    this.locales = {};
+    this.fetchConcurrency = this.localeConfig.fetchConcurrency || this.exportConfig.fetchConcurrency;
+    this.writeConcurrency = this.localeConfig.writeConcurrency || this.exportConfig.writeConcurrency;
   }
-  return self.getLocales(apiDetails)
-
-};
-
-ExportLocales.prototype.getLocales = function (apiDetails) {
-  let self = this
-
-  return new Promise(function (resolve, reject) {
-    client
-      .stack({ api_key: config.source_stack, management_token: config.management_token })
-      .locale()
-      .query({ ...self.qs, ...apiDetails })
-      .find()
-      .then((localeResponse) => {
-        if (localeResponse.items.length !== 0) {
-          localeResponse.items.forEach(function (locale) {
-            addlogs(config, locale.name + ' locale was exported successfully', 'success');
-            for (const key in locale) {
-              if (requiredKeys.indexOf(key) === -1) {
-                delete locale[key];
-              }
-            }
-            self.locales[locale.uid] = locale;
-          });
 
-          helper.writeFile(path.join(self.localesFolderPath, localeConfig.fileName), self.locales);
-
-          apiDetails.skip += apiDetails.limit;
-
-          if (apiDetails.skip > localeResponse.count) {
-            addlogs(config, chalk.green('All the locales have been exported successfully'), 'success');
-            return resolve();
-          }
-
-          return self
-            .getLocales(apiDetails)
-            .then(resolve)
-            .catch((error) => {
-              console.log('Get locales errror', error && error.message);
-            });
+  async start() {
+    try {
+      addlogs(this.exportConfig, 'Starting locale export', 'success');
+      await fileHelper.makeDirectory(this.localesPath);
+      await this.getLocales();
+      await fileHelper.writeFile(path.join(this.localesPath, this.localeConfig.fileName), this.locales);
+      addlogs(this.exportConfig, 'Completed locale export', 'success');
+    } catch (error) {
+      addlogs(this.exportConfig, chalk.red(`Failed to export locales ${formatError(error)}`), 'error');
+      throw new Error('Failed to export locales');
+    }
+  }
 
-        } else if (localeResponse.items.length === 0) {
-          addlogs(config, 'No languages found except the master language', 'success');
-          helper.writeFile(path.join(self.localesFolderPath, localeConfig.fileName), self.locales);
-          return resolve();
+  async getLocales(skip = 0) {
+    if (skip) {
+      this.qs.skip = skip;
+    }
+    let localesFetchResponse = await this.stackAPIClient.locale().query(this.qs).find();
+    if (Array.isArray(localesFetchResponse.items) && localesFetchResponse.items.length > 0) {
+      this.sanitizeAttribs(localesFetchResponse.items);
+      skip += this.localeConfig.limit || 100;
+      if (skip > localesFetchResponse.count) {
+        return;
+      }
+      return await this.getLocales(skip);
+    }
+  }
 
+  sanitizeAttribs(locales) {
+    locales.forEach((locale) => {
+      for (let key in locale) {
+        if (this.localeConfig.requiredKeys.indexOf(key) === -1) {
+          delete locale[key];
         }
-      })
-      .catch((error) => {
-        addlogs(config, error, 'error');
-        return reject(error);
-      });
-  });
+      }
+      this.locales[locale.uid] = locale;
+    });
+  }
 }
 
-module.exports = new ExportLocales();
+module.exports = LocaleExport;

package/src/lib/export/marketplace-apps.js

@@ -0,0 +1,153 @@
+/*!
+ * Contentstack Export
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
+const _ = require('lodash');
+const path = require('path');
+const chalk = require('chalk');
+const mkdirp = require('mkdirp');
+const eachOf = require('async/eachOf');
+const { cliux, HttpClient, NodeCrypto } = require('@contentstack/cli-utilities');
+
+const { formatError } = require('../util');
+let config = require('../../config/default');
+const { writeFile } = require('../util/helper');
+const { addlogs: log } = require('../util/log');
+let stack = require('../util/contentstack-management-sdk');
+const { getDeveloperHubUrl, getInstalledExtensions } = require('../util/marketplace-app-helper');
+
+module.exports = class ExportMarketplaceApps {
+  config;
+  marketplaceAppPath = null;
+  developerHubBaseUrl = null;
+  marketplaceAppConfig = config.modules.marketplace_apps;
+
+  constructor(credentialConfig) {
+    this.config = credentialConfig;
+  }
+
+  async start() {
+    this.developerHubBaseUrl = await getDeveloperHubUrl();
+
+    if (!this.config.auth_token) {
+      cliux.print(
+        'WARNING!!! To export Marketplace apps, you must be logged in. Please check csdx auth:login --help to log in',
+        { color: 'yellow' },
+      );
+      return Promise.resolve();
+    }
+
+    log(this.config, 'Starting marketplace app export', 'success');
+    this.marketplaceAppPath = path.resolve(
+      this.config.data,
+      this.config.branchName || '',
+      this.marketplaceAppConfig.dirName,
+    );
+    mkdirp.sync(this.marketplaceAppPath);
+
+    return this.exportInstalledExtensions();
+  }
+
+  exportInstalledExtensions = () => {
+    const self = this;
+
+    return new Promise(async function (resolve, reject) {
+      getInstalledExtensions(self.config)
+        .then(async (items) => {
+          const installedApps = _.map(
+            _.filter(items, 'app_uid'),
+            ({ uid, title, app_uid, app_installation_uid, type }) => ({
+              title,
+              uid,
+              app_uid,
+              app_installation_uid,
+              type,
+            }),
+          );
+          const headers = {
+            authtoken: self.config.auth_token,
+            organization_uid: self.config.org_uid,
+          };
+          const httpClient = new HttpClient().headers(headers);
+          const nodeCrypto = new NodeCrypto();
+          const developerHubApps =
+            (await httpClient
+              .get(`${self.developerHubBaseUrl}/apps`)
+              .then(({ data: { data } }) => data)
+              .catch((err) => {
+                console.log(err);
+              })) || [];
+
+          eachOf(
+            _.uniqBy(installedApps, 'app_uid'),
+            (app, _key, cb) => {
+              log(self.config, `Exporting ${app.title} app and it's config.`, 'success');
+              const listOfIndexToBeUpdated = _.map(installedApps, ({ app_uid }, index) =>
+                app_uid === app.app_uid ? index : undefined,
+              ).filter((val) => val !== undefined);
+
+              httpClient
+                .get(`${self.developerHubBaseUrl}/installations/${app.app_installation_uid}/installationData`)
+                .then(({ data: result }) => {
+                  const { data, error } = result;
+                  const developerHubApp = _.find(developerHubApps, { uid: app.app_uid });
+
+                  _.forEach(listOfIndexToBeUpdated, (index, i) => {
+                    if (developerHubApp) {
+                      installedApps[index]['visibility'] = developerHubApp.visibility;
+                      installedApps[index]['manifest'] = _.pick(
+                        developerHubApp,
+                        ['name', 'description', 'icon', 'target_type', 'ui_location', 'webhook', 'oauth'], // NOTE keys can be passed to install new app in the developer hub
+                      );
+                    }
+
+                    if (_.has(data, 'configuration') || _.has(data, 'server_configuration')) {
+                      const { configuration, server_configuration } = data;
+
+                      if (!_.isEmpty(configuration)) {
+                        installedApps[index]['configuration'] = nodeCrypto.encrypt(configuration);
+                      }
+                      if (!_.isEmpty(server_configuration)) {
+                        installedApps[index]['server_configuration'] = nodeCrypto.encrypt(server_configuration);
+                      }
+
+                      if (i === 0) {
+                        log(self.config, `Exported ${app.title} app and it's config.`, 'success');
+                      }
+                    } else if (error) {
+                      console.log(error);
+                      if (i === 0) {
+                        log(self.config, `Error on exporting ${app.title} app and it's config.`, 'error');
+                      }
+                    }
+                  });
+
+                  cb();
+                })
+                .catch((err) => {
+                  addlogs(self.config, `Failed to export ${app.title} app config ${formatError(error)}`, 'error');
+                  console.log(err);
+                  cb();
+                });
+            },
+            () => {
+              if (!_.isEmpty(installedApps)) {
+                writeFile(path.join(self.marketplaceAppPath, self.marketplaceAppConfig.fileName), installedApps);
+
+                log(self.config, chalk.green('All the marketplace apps have been exported successfully'), 'success');
+              } else {
+                log(self.config, 'No marketplace apps found', 'success');
+              }
+
+              resolve();
+            },
+          );
+        })
+        .catch((error) => {
+          addlogs(self.config, `Failed to export marketplace-apps ${formatError(error)}`, 'error');
+          reject(error);
+        });
+    });
+  };
+};