@contentstack/cli-cm-export 0.1.1-beta.7 → 1.0.0

package/src/lib/export/locales.js

@@ -4,19 +4,18 @@
  * MIT Licensed
  */
 
-const mkdirp = require('mkdirp')
-const path = require('path')
-const chalk = require('chalk')
-
-const helper = require('../util/helper')
-const {addlogs} = require('../util/log')
-let config = require('../../config/default')
-let localeConfig = config.modules.locales
-const masterLocale = config.master_locale
-let requiredKeys = localeConfig.requiredKeys
-let stack = require('../util/contentstack-management-sdk')
-
+const mkdirp = require('mkdirp');
+const path = require('path');
+const chalk = require('chalk');
 
+const helper = require('../util/helper');
+const { addlogs } = require('../util/log');
+let config = require('../../config/default');
+let localeConfig = config.modules.locales;
+const masterLocale = config.master_locale;
+let requiredKeys = localeConfig.requiredKeys;
+let stack = require('../util/contentstack-management-sdk');
+let client
 function ExportLocales() {
   this.qs = {
     include_count: true,
@@ -29,42 +28,76 @@ function ExportLocales() {
     only: {
       BASE: requiredKeys,
     },
-  }
-  this.locales = {}
+  };
+  this.locales = {};
 }
 
 ExportLocales.prototype.start = function (credentialConfig) {
-  addlogs(credentialConfig, 'Starting locale export', 'success')
+  this.locales = {};
+  addlogs(credentialConfig, 'Starting locale export', 'success');
+  let self = this;
+  config = credentialConfig;
+  self.localesFolderPath = path.resolve(config.data, config.branchName || '', localeConfig.dirName);
+  mkdirp.sync(self.localesFolderPath);
+  client = stack.Client(config);
+  const apiDetails = {
+    limit: 100,
+    skip: 0,
+    include_count: true,
+  }
+  return self.getLocales(apiDetails)
+
+};
+
+ExportLocales.prototype.getLocales = function (apiDetails) {
   let self = this
-  config = credentialConfig
-  let localesFolderPath = path.resolve(config.data, localeConfig.dirName)
-  mkdirp.sync(localesFolderPath)
 
-  let client = stack.Client(config)
   return new Promise(function (resolve, reject) {
-    client.stack({api_key: config.source_stack, management_token: config.management_token}).locale().query(self.qs).find()
-    .then(localeResponse => {
-      if (localeResponse.items.length !== 0) {
-        localeResponse.items.forEach(function (locale) {
-          addlogs(credentialConfig, locale.name + ' locale was exported successfully', 'success')
-          for (const key in locale) {
-            if (requiredKeys.indexOf(key) === -1) {
-              delete locale[key]
+    client
+      .stack({ api_key: config.source_stack, management_token: config.management_token })
+      .locale()
+      .query({ ...self.qs, ...apiDetails })
+      .find()
+      .then((localeResponse) => {
+        if (localeResponse.items.length !== 0) {
+          localeResponse.items.forEach(function (locale) {
+            addlogs(config, locale.name + ' locale was exported successfully', 'success');
+            for (const key in locale) {
+              if (requiredKeys.indexOf(key) === -1) {
+                delete locale[key];
+              }
             }
+            self.locales[locale.uid] = locale;
+          });
+
+          helper.writeFile(path.join(self.localesFolderPath, localeConfig.fileName), self.locales);
+
+          apiDetails.skip += apiDetails.limit;
+
+          if (apiDetails.skip > localeResponse.count) {
+            addlogs(config, chalk.green('All the locales have been exported successfully'), 'success');
+            return resolve();
           }
-          self.locales[locale.uid] = locale
-        })
-        addlogs(credentialConfig, chalk.green('All the locales have been exported successfully'), 'success')
-      } else if (localeResponse.items.length === 0) {
-        addlogs(credentialConfig, 'No languages found except the master language', 'success')
-      }
-      helper.writeFile(path.join(localesFolderPath, localeConfig.fileName), self.locales)
-      return resolve()
-    }).catch(error => {
-      addlogs(credentialConfig, error, 'error')
-      return reject()
-    })
-  })
+
+          return self
+            .getLocales(apiDetails)
+            .then(resolve)
+            .catch((error) => {
+              console.log('Get locales errror', error && error.message);
+            });
+
+        } else if (localeResponse.items.length === 0) {
+          addlogs(config, 'No languages found except the master language', 'success');
+          helper.writeFile(path.join(self.localesFolderPath, localeConfig.fileName), self.locales);
+          return resolve();
+
+        }
+      })
+      .catch((error) => {
+        addlogs(config, error, 'error');
+        return reject(error);
+      });
+  });
 }
 
-module.exports = new ExportLocales()
+module.exports = new ExportLocales();

package/src/lib/export/stack.js

@@ -4,64 +4,89 @@
  * MIT Licensed
  */
 
-var chalk = require('chalk')
-var mkdirp = require('mkdirp')
-var path = require('path')
+var chalk = require('chalk');
+var mkdirp = require('mkdirp');
+var path = require('path');
 
-var app = require('../../app')
-var helper = require('../util/helper')
-var {addlogs} = require('../util/log')
-const stack = require('../util/contentstack-management-sdk')
+var app = require('../../app');
+var helper = require('../util/helper');
+var { addlogs } = require('../util/log');
+const stack = require('../util/contentstack-management-sdk');
 
+let config = require('../../config/default');
 
-let config = require('../../config/default')
+var stackConfig = config.modules.stack;
+let client;
 
-var stackConfig = config.modules.stack
-let client
-
-function ExportStack () {
+function ExportStack() {
   this.requestOption = {
     uri: config.host + config.apis.stacks,
     headers: config.headers,
-    json: true
-  }
+    json: true,
+  };
 }
 
 ExportStack.prototype.start = function (credentialConfig) {
-  config = credentialConfig
-  client = stack.Client(config)
-  if (!config.preserveStackVersion && !config.hasOwnProperty("master_locale")) {
-    return new Promise((resolve, reject) => {
-    var result =  client.stack({ api_key: credentialConfig.source_stack, management_token: credentialConfig.management_token }).locale().query()     
-    result.find()
-      .then(response => {
-        var masterLocalObj = response.items.filter(obj => {
-            if (obj.fallback_locale === null) {
-              return obj
-            }
-            });
-        return resolve(masterLocalObj[0])
-      }).catch(error => {
-        return reject(error)
-      })
-    })
-  } else if(config.preserveStackVersion) {
-    addlogs(config, 'Exporting stack details', 'success')
-    let stackFolderPath = path.resolve(config.data, stackConfig.dirName)
-    let stackContentsFile = path.resolve(stackFolderPath, stackConfig.fileName)
-  
-    mkdirp.sync(stackFolderPath)
-  
+  config = credentialConfig;
+  client = stack.Client(config);
+  const self = this
+  if (!config.preserveStackVersion && !config.hasOwnProperty('master_locale')) {
+    const apiDetails = {
+      limit: 100,
+      skip: 0,
+      include_count: true,
+    }
+    return self.getLocales(apiDetails)
+  } else if (config.preserveStackVersion) {
+    addlogs(config, 'Exporting stack details', 'success');
+    let stackFolderPath = path.resolve(config.data, stackConfig.dirName);
+    let stackContentsFile = path.resolve(stackFolderPath, stackConfig.fileName);
+
+    mkdirp.sync(stackFolderPath);
+
     return new Promise((resolve, reject) => {
-      return client.stack({api_key: config.source_stack}).fetch()
-      .then(response => {
-        helper.writeFile(stackContentsFile, response)
-        addlogs(config, 'Exported stack details successfully!', 'success')
-        return resolve(response)
-      })
-      .catch(reject)
-    })
+      return client
+        .stack({ api_key: config.source_stack })
+        .fetch()
+        .then((response) => {
+          helper.writeFile(stackContentsFile, response);
+          addlogs(config, 'Exported stack details successfully!', 'success');
+          return resolve(response);
+        })
+        .catch(reject);
+    });
   }
+};
+
+ExportStack.prototype.getLocales = function (apiDetails) {
+  let self = this
+  return new Promise((resolve, reject) => {
+    const result = client
+      .stack({ api_key: config.source_stack, management_token: config.management_token })
+      .locale()
+      .query(apiDetails)
+
+    result
+      .find()
+      .then((response) => {
+        const masterLocalObj = response.items.find((obj) => {
+          if (obj.fallback_locale === null) {
+            return obj;
+          }
+        });
+        apiDetails.skip += apiDetails.limit;
+        if (masterLocalObj) { return resolve(masterLocalObj); }
+        else if (apiDetails.skip <= response.count) {
+          return resolve(self.getLocales(apiDetails))
+        }
+        else {
+          return reject('Master locale not found');
+        }
+      })
+      .catch((error) => {
+        return reject(error);
+      });
+  });
 }
 
-module.exports = new ExportStack()
+module.exports = new ExportStack();

package/src/lib/export/webhooks.js

@@ -1,20 +1,20 @@
 /*!
-* Contentstack Export
-* Copyright (c) 2019 Contentstack LLC
-* MIT Licensed
-*/
+ * Contentstack Export
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
 
-const mkdirp = require('mkdirp')
-const path = require('path')
-const chalk = require('chalk')
+const mkdirp = require('mkdirp');
+const path = require('path');
+const chalk = require('chalk');
 
-const helper = require('../util/helper')
-const {addlogs} = require('../util/log')
+const helper = require('../util/helper');
+const { addlogs } = require('../util/log');
 
-let config = require('../../config/default')
-const stack = require('../util/contentstack-management-sdk')
-const webhooksConfig = config.modules.webhooks
-let client
+let config = require('../../config/default');
+const stack = require('../util/contentstack-management-sdk');
+const webhooksConfig = config.modules.webhooks;
+let client;
 
 // Create folder for environments
 
@@ -22,44 +22,54 @@ function ExportWebhooks() {
   this.requestOptions = {
     include_count: true,
     asc: 'updated_at',
-  }
-  this.master = {}
-  this.webhooks = {}
+  };
+  this.master = {};
+  this.webhooks = {};
 }
 
 ExportWebhooks.prototype.start = function (credentialConfig) {
-  addlogs(config, 'Starting webhooks export', 'success')
-  let self = this
-  config = credentialConfig
-  client = stack.Client(config)
-  const webhooksFolderPath = path.resolve(config.data, webhooksConfig.dirName)
-  mkdirp.sync(webhooksFolderPath)
+  addlogs(config, 'Starting webhooks export', 'success');
+  this.master = {};
+  this.webhooks = {};
+  let self = this;
+  config = credentialConfig;
+  client = stack.Client(config);
+  const webhooksFolderPath = path.resolve(config.data, config.branchName || '', webhooksConfig.dirName);
+  mkdirp.sync(webhooksFolderPath);
   return new Promise(function (resolve, reject) {
-    client.stack({api_key: config.source_stack, management_token: config.management_token}).webhook().fetchAll(self.requestOptions)
-    .then(webhooks => {
-      if (webhooks.items.length !== 0) {
-        for (var i = 0, total = webhooks.count; i < total; i++) {
-          const webUid = webhooks.items[i].uid
-          self.master[webUid] = ''
-          self.webhooks[webUid] = webhooks.items[i]
-          delete self.webhooks[webUid].uid
-          delete self.webhooks[webUid].SYS_ACL
+    client
+      .stack({ api_key: config.source_stack, management_token: config.management_token })
+      .webhook()
+      .fetchAll(self.requestOptions)
+      .then((webhooks) => {
+        if (webhooks.items.length !== 0) {
+          for (var i = 0, total = webhooks.count; i < total; i++) {
+            const webUid = webhooks.items[i].uid;
+            self.master[webUid] = '';
+            self.webhooks[webUid] = webhooks.items[i];
+            delete self.webhooks[webUid].uid;
+            delete self.webhooks[webUid].SYS_ACL;
+          }
+          helper.writeFile(path.join(webhooksFolderPath, webhooksConfig.fileName), self.webhooks);
+          addlogs(config, chalk.green('All the webhooks have been exported successfully'), 'success');
+          return resolve();
         }
-        helper.writeFile(path.join(webhooksFolderPath, webhooksConfig.fileName), self.webhooks)
-        addlogs(config, chalk.green('All the webhooks have been exported successfully'), 'success')
-        return resolve()
-      }
-      addlogs(config, 'No webhooks found', 'success')
-      return resolve()
-    }).catch(function (error) {
-      if (error.statusCode === 401) {
-        addlogs(config, chalk.red('You are not allowed to export webhooks, Unless you provide email and password in config'), 'error')
-        return resolve()
-      }
-      addlogs(config, error, 'error')
-      return reject()
-    })
-  })
-}
+        addlogs(config, 'No webhooks found', 'success');
+        return resolve();
+      })
+      .catch(function (error) {
+        if (error.statusCode === 401) {
+          addlogs(
+            config,
+            chalk.red('You are not allowed to export webhooks, Unless you provide email and password in config'),
+            'error',
+          );
+          return resolve();
+        }
+        addlogs(config, error, 'error');
+        return reject();
+      });
+  });
+};
 
-module.exports = new ExportWebhooks()
+module.exports = new ExportWebhooks();

package/src/lib/export/workflows.js

@@ -4,56 +4,67 @@
  * MIT Licensed
  */
 
-const mkdirp = require('mkdirp')
-const path = require('path')
-const chalk = require('chalk')
+const mkdirp = require('mkdirp');
+const path = require('path');
+const chalk = require('chalk');
 
-let helper = require('../util/helper')
-let {addlogs} = require('../util/log')
+let helper = require('../util/helper');
+let { addlogs } = require('../util/log');
 
-const stack = require('../util/contentstack-management-sdk')
-let config = require('../../config/default')
-let workFlowConfig = config.modules.workflows
-let client
+const stack = require('../util/contentstack-management-sdk');
+let config = require('../../config/default');
+let workFlowConfig = config.modules.workflows;
+let client;
 
 function ExportWorkFlows() {
-  this.workflows = {}
+  this.workflows = {};
 }
 
 ExportWorkFlows.prototype.start = function (credentialConfig) {
-  addlogs(config, 'Starting workflow export', 'success')
-  let self = this
-  config = credentialConfig
-  client = stack.Client(config)
+  addlogs(config, 'Starting workflow export', 'success');
+  this.workflows = {};
+  let self = this;
+  config = credentialConfig;
+  client = stack.Client(config);
 
-  let workflowsFolderPath = path.resolve(config.data, workFlowConfig.dirName)
-  mkdirp.sync(workflowsFolderPath)
-  return new Promise(function (resolve, reject) {
-    return client.stack({api_key: config.source_stack, management_token: config.management_token}).workflow().fetchAll()
-    .then(response => {
-      if (response.items.length !== 0) {
-        response.items.forEach(function (workflow) {
-          addlogs(config, workflow.name + ' workflow was exported successfully', 'success')
-          self.workflows[workflow.uid] = workflow
-          let deleteItems = config.modules.workflows.invalidKeys
-          deleteItems.forEach(e => delete workflow[e])
-        })
-        addlogs(config, chalk.green('All the workflow have been exported successfully'), 'success')
-      }
-      if (response.items.length === 0) {
-        addlogs(config, 'No workflow were found in the Stack', 'success')
-      }
-      helper.writeFile(path.join(workflowsFolderPath, workFlowConfig.fileName), self.workflows)
-      return resolve()
-    }).catch(function (error) {
-      if (error.statusCode === 401) {
-        addlogs(config, chalk.red('You are not allowed to export workflow, Unless you provide email and password in config', 'error'))
-        return resolve()
-      } 
-      addlogs(config, error, 'error')
-      return resolve()
-    })
-  })
-}
+  let workflowsFolderPath = path.resolve(config.data, config.branchName || '', workFlowConfig.dirName);
+  mkdirp.sync(workflowsFolderPath);
+  return new Promise(function (resolve) {
+    return client
+      .stack({ api_key: config.source_stack, management_token: config.management_token })
+      .workflow()
+      .fetchAll()
+      .then((response) => {
+        if (response.items.length !== 0) {
+          response.items.forEach(function (workflow) {
+            addlogs(config, workflow.name + ' workflow was exported successfully', 'success');
+            self.workflows[workflow.uid] = workflow;
+            let deleteItems = config.modules.workflows.invalidKeys;
+            deleteItems.forEach((e) => delete workflow[e]);
+          });
+          addlogs(config, chalk.green('All the workflow have been exported successfully'), 'success');
+        }
+        if (response.items.length === 0) {
+          addlogs(config, 'No workflow were found in the Stack', 'success');
+        }
+        helper.writeFile(path.join(workflowsFolderPath, workFlowConfig.fileName), self.workflows);
+        return resolve();
+      })
+      .catch(function (error) {
+        if (error.statusCode === 401) {
+          addlogs(
+            config,
+            chalk.red(
+              'You are not allowed to export workflow, Unless you provide email and password in config',
+              'error',
+            ),
+          );
+          return resolve();
+        }
+        addlogs(config, error, 'error');
+        return resolve();
+      });
+  });
+};
 
-module.exports = new ExportWorkFlows()
+module.exports = new ExportWorkFlows();

package/src/lib/util/contentstack-management-sdk.js

@@ -1,13 +1,39 @@
-const contentstacksdk = require('@contentstack/management')
+const contentstacksdk = require('@contentstack/management');
+const https = require('https');
+
+const { addlogs } = require('./log');
 
 exports.Client = function (config) {
   const option = {
     host: config.host,
     authtoken: config.auth_token,
     api_key: config.source_stack,
+    maxRequests: 10,
+    retryLimit: 5,
+    timeout: 60000,
+    httpsAgent: new https.Agent({
+      maxSockets: 100,
+      maxFreeSockets: 10,
+      keepAlive: true,
+      timeout: 60000, // active socket keepalive for 60 seconds
+      freeSocketTimeout: 30000, // free socket keepalive for 30 seconds
+    }),
+    retryDelay: Math.floor(Math.random() * (8000 - 3000 + 1) + 3000),
     logHandler: (level, data) => {},
+    retryCondition: (error) => {
+      if (error.response.status === 408) {
+        addlogs({ data: error.response }, 'Timeout error', 'error');
+      }
+      return false;
+    },
+  };
+
+  if (typeof config.branchName === 'string') {
+    option.headers = {
+      branch: config.branchName,
+    };
   }
 
-  const client = contentstacksdk.client(option)
-  return client
-}
+  const client = contentstacksdk.client(option);
+  return client;
+};