@contentstack/cli-cm-export 1.2.3 → 1.4.0

package/src/lib/export/assets.js

@@ -17,54 +17,52 @@ const helper = require('../util/helper');
 const { addlogs } = require('../util/log');
 
 let config = require('../../config/default');
-const stack = require('../util/contentstack-management-sdk');
 
 module.exports = class ExportAssets {
-  config
-  client
-  bLimit
-  vLimit
-  invalidKeys
-  folderJSONPath
-  folderData = []
-  assetsFolderPath
-  assetContents = {}
-  httpClient = HttpClient.create()
-  assetConfig = config.modules.assets
-
-  constructor(config) {
-    this.config = config
-    this.folderData = []
-    this.assetContents = {}
-    this.assetDownloadRetry = {}
-    this.assetDownloadRetryLimit = 3
-    this.invalidKeys = this.assetConfig.invalidKeys
-    this.bLimit = this.assetConfig.batchLimit || 15
-    this.vLimit = this.assetConfig.downloadLimit || config.fetchConcurrency || 3
+  config;
+  bLimit;
+  vLimit;
+  invalidKeys;
+  folderJSONPath;
+  folderData = [];
+  assetsFolderPath;
+  assetContents = {};
+  httpClient = HttpClient.create();
+  assetConfig = config.modules.assets;
+
+  constructor(exportConfig, stackAPIClient) {
+    this.stackAPIClient = stackAPIClient;
+    this.config = _.merge(config, exportConfig);
+    this.folderData = [];
+    this.assetContents = {};
+    this.assetDownloadRetry = {};
+    this.assetDownloadRetryLimit = 3;
+    this.invalidKeys = this.assetConfig.invalidKeys;
+    this.bLimit = this.assetConfig.batchLimit || 15;
+    this.vLimit = this.assetConfig.downloadLimit || this.config.fetchConcurrency || 3;
   }
 
   start() {
-    const self = this
-    this.assetsFolderPath = path.resolve(this.config.data, this.config.branchName || '', this.assetConfig.dirName)
-    this.assetContentsFile = path.resolve(this.assetsFolderPath, 'assets.json')
-    this.folderJSONPath = path.resolve(this.assetsFolderPath, 'folders.json')
-    self.client = stack.Client(this.config)
+    const self = this;
+    this.assetsFolderPath = path.resolve(this.config.data, this.config.branchName || '', this.assetConfig.dirName);
+    this.assetContentsFile = path.resolve(this.assetsFolderPath, 'assets.json');
+    this.folderJSONPath = path.resolve(this.assetsFolderPath, 'folders.json');
 
-    addlogs(this.config, 'Starting assets export', 'success')
+    addlogs(this.config, 'Starting assets export', 'success');
 
     // Create asset folder
-    mkdirp.sync(this.assetsFolderPath)
+    mkdirp.sync(this.assetsFolderPath);
 
     return new Promise(function (resolve, reject) {
       // TBD: getting all the assets should have optimized
       return self
         .getAssetCount()
         .then(function (count) {
-          const assetBatches = []
+          const assetBatches = [];
 
           if (typeof count !== 'number' || count === 0) {
-            addlogs(self.config, 'No assets found', 'success')
-            return resolve()
+            addlogs(self.config, 'No assets found', 'success');
+            return resolve();
           }
           for (let i = 0; i <= count; i += self.bLimit) {
             assetBatches.push(i);
@@ -84,76 +82,89 @@ module.exports = class ExportAssets {
                           .getVersionedAssetJSON(assetJSON.uid, assetJSON._version)
                           .then(function () {
                             self.assetContents[assetJSON.uid] = assetJSON;
-                          }).catch(function (error) {
+                          })
+                          .catch(function (error) {
                             addlogs(
                               self.config,
                               chalk.red('The following asset failed to download\n' + JSON.stringify(assetJSON)),
-                            )
-                            addlogs(self.config, error, 'error')
-                          })
+                            );
+                            addlogs(self.config, error, 'error');
+                          });
                       } else {
-                        return self.downloadAsset(assetJSON)
+                        return self
+                          .downloadAsset(assetJSON)
                           .then(function () {
-                            self.assetContents[assetJSON.uid] = assetJSON
-                          }).catch((err) => {
-                            addlogs({ errorCode: (err && err.code), uid: assetJSON.uid }, `Asset download failed - ${assetJSON.uid}`, 'error')
-                            return err
+                            self.assetContents[assetJSON.uid] = assetJSON;
                           })
+                          .catch((err) => {
+                            addlogs(
+                              { errorCode: err && err.code, uid: assetJSON.uid },
+                              `Asset download failed - ${assetJSON.uid}`,
+                              'error',
+                            );
+                            return err;
+                          });
                       }
                     },
-                    { concurrency: self.vLimit }
-                  ).then(function () {
-                    addlogs(self.config, 'Batch no ' + (batch + 1) + ' of assets is complete', 'success');
-                    // helper.writeFileSync(this.assetContentsFile, self.assetContents)
-                  }).catch(function (error) {
-                    console.log('Error fetch/download the asset', (error && error.message))
-                    addlogs(self.config, 'Asset batch ' + (batch + 1) + ' failed to download', 'error')
-                    addlogs(self.config, error, 'error')
-                  })
-                }).catch(function (error) {
-                  addlogs(self.config, error, 'error')
-                  reject(error)
+                    { concurrency: self.vLimit },
+                  )
+                    .then(function () {
+                      addlogs(self.config, 'Batch no ' + (batch + 1) + ' of assets is complete', 'success');
+                      // helper.writeFileSync(this.assetContentsFile, self.assetContents)
+                    })
+                    .catch(function (error) {
+                      console.log('Error fetch/download the asset', error && error.message);
+                      addlogs(self.config, 'Asset batch ' + (batch + 1) + ' failed to download', 'error');
+                      addlogs(self.config, error, 'error');
+                    });
                 })
+                .catch(function (error) {
+                  addlogs(self.config, error, 'error');
+                  reject(error);
+                });
             },
-            { concurrency: self.assetConfig.concurrencyLimit || 1 }
-          ).then(function () {
-            helper.writeFileSync(self.assetContentsFile, self.assetContents)
-
-            return self
-              .exportFolders()
-              .then(function () {
-                addlogs(self.config, chalk.green('Asset export completed successfully'), 'success');
-                return resolve();
-              })
-              .catch(function (error) {
-                addlogs(self.config, error, 'success')
-                reject(error)
-              })
-          }).catch(function (error) {
-            helper.writeFileSync(self.assetContentsFile, self.assetContents)
-            addlogs(
-              self.config,
-              chalk.red('Asset export failed due to the following errors ' + JSON.stringify(error), 'error'),
-            )
-            addlogs(self.config, error, 'success')
-            reject(error)
-          })
-        }).catch(function (error) {
-          addlogs(self.config, error, 'success')
-          reject(error)
+            { concurrency: self.assetConfig.concurrencyLimit || 1 },
+          )
+            .then(function () {
+              helper.writeFileSync(self.assetContentsFile, self.assetContents);
+
+              return self
+                .exportFolders()
+                .then(function () {
+                  addlogs(self.config, chalk.green('Asset export completed successfully'), 'success');
+                  return resolve();
+                })
+                .catch(function (error) {
+                  addlogs(self.config, error, 'success');
+                  reject(error);
+                });
+            })
+            .catch(function (error) {
+              helper.writeFileSync(self.assetContentsFile, self.assetContents);
+              addlogs(
+                self.config,
+                chalk.red('Asset export failed due to the following errors ' + JSON.stringify(error), 'error'),
+              );
+              addlogs(self.config, error, 'success');
+              reject(error);
+            });
         })
-    })
+        .catch(function (error) {
+          addlogs(self.config, error, 'success');
+          reject(error);
+        });
+    });
   }
 
   exportFolders() {
-    let self = this;
+    const self = this;
     return new Promise(function (resolve, reject) {
       return self
         .getAssetCount(true)
         .then(function (fCount) {
           if (fCount === 0) {
             addlogs(self.config, 'No folders were found in the stack!', 'success');
-            return resolve()
+            return resolve();
           }
 
           return self
@@ -162,62 +173,61 @@ module.exports = class ExportAssets {
               // asset folders have been successfully exported
               addlogs(self.config, 'Asset-folders have been successfully exported!', 'success');
               return resolve();
-            }).catch(function (error) {
+            })
+            .catch(function (error) {
               addlogs(self.config, chalk.red('Error while exporting asset-folders!'), 'error');
               return reject(error);
-            })
+            });
         })
         .catch(function (error) {
-          addlogs(self.config, error, 'error')
+          addlogs(self.config, error, 'error');
           // error while fetching asset folder count
-          return reject(error)
+          return reject(error);
         });
     });
   }
 
   getFolderJSON(skip, fCount) {
-    let self = this
+    const self = this;
     return new Promise(function (resolve, reject) {
       if (typeof skip !== 'number') {
-        skip = 0
+        skip = 0;
       }
 
       if (skip >= fCount) {
-        helper.writeFileSync(self.folderJSONPath, self.folderData)
-        return resolve()
+        helper.writeFileSync(self.folderJSONPath, self.folderData);
+        return resolve();
       }
 
       const queryRequestObj = {
         skip,
         include_folders: true,
-        query: { is_dir: true }
-      }
+        query: { is_dir: true },
+      };
 
-      self.client
-        .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+      self.stackAPIClient
         .asset()
         .query(queryRequestObj)
         .find()
         .then((response) => {
-          skip += 100
-          self.folderData.push(...response.items)
-          return self.getFolderJSON(skip, fCount).then(resolve).catch(reject)
-        })
-    })
+          skip += 100;
+          self.folderData.push(...response.items);
+          return self.getFolderJSON(skip, fCount).then(resolve).catch(reject);
+        });
+    });
   }
 
   getAssetCount(folder) {
-    const self = this
+    const self = this;
     return new Promise(function (resolve, reject) {
       if (folder && typeof folder === 'boolean') {
-        let queryOptions = {
+        const queryOptions = {
           skip: 99999990,
           include_count: true,
           include_folders: true,
-          query: { is_dir: true }
-        }
-        self.client
-          .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+          query: { is_dir: true },
+        };
+        self.stackAPIClient
           .asset()
           .query(queryOptions)
           .find()
@@ -228,74 +238,71 @@ module.exports = class ExportAssets {
             addlogs(self.config, error, 'error');
           });
       } else {
-        let queryOptions = { skip: 99999990, include_count: true }
-        self.client
-          .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+        const queryOptions = { skip: 99999990, include_count: true };
+        self.stackAPIClient
           .asset()
           .query(queryOptions)
           .find()
           .then(({ count }) => resolve(count))
           .catch((error) => {
-            addlogs(self.config, error, 'error')
-            reject(error)
-          })
+            addlogs(self.config, error, 'error');
+            reject(error);
+          });
       }
-    })
+    });
   }
 
   getAssetJSON(skip) {
-    const self = this
+    const self = this;
     return new Promise(function (resolve, reject) {
       if (typeof skip !== 'number') {
-        skip = 0
+        skip = 0;
       }
       const queryRequestObj = {
         skip: skip,
         limit: self.bLimit,
         include_publish_details: true,
         except: {
-          BASE: self.invalidKeys
-        }
-      }
+          BASE: self.invalidKeys,
+        },
+      };
 
-      self.client
-        .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+      self.stackAPIClient
         .asset()
         .query(queryRequestObj)
         .find()
         .then(({ items }) => resolve(items))
         .catch((error) => {
-          addlogs(self.config, error, 'error')
-          return reject()
-        })
-    })
+          addlogs(self.config, error, 'error');
+          return reject();
+        });
+    });
   }
 
   getVersionedAssetJSON(uid, version, bucket) {
-    let self = this
-    let assetVersionInfo = bucket || []
+    const self = this;
+    const assetVersionInfo = bucket || [];
 
     return new Promise(function (resolve, reject) {
       if (self.assetDownloadRetry[uid + version] > self.assetDownloadRetryLimit) {
-        console.log('Reached max', self.assetDownloadRetry[uid + version])
-        return reject(new Error('Asset Max download retry limit exceeded! ' + uid))
+        console.log('Reached max', self.assetDownloadRetry[uid + version]);
+        return reject(new Error('Asset Max download retry limit exceeded! ' + uid));
       }
 
       if (version <= 0) {
         const assetVersionInfoFile = path.resolve(self.assetsFolderPath, uid, '_contentstack_' + uid + '.json');
-        helper.writeFileSync(assetVersionInfoFile, assetVersionInfo)
-        return resolve()
+        helper.writeFileSync(assetVersionInfoFile, assetVersionInfo);
+        return resolve();
       }
-      let queryrequestOption = {
+      const queryrequestOption = {
         version: version,
         include_publish_details: true,
         except: {
-          BASE: self.invalidKeys
-        }
-      }
+          BASE: self.invalidKeys,
+        },
+      };
 
-      self.client
-        .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+      self.stackAPIClient
         .asset(uid)
         .fetch(queryrequestOption)
         .then((versionedAssetJSONResponse) => {
@@ -305,15 +312,12 @@ module.exports = class ExportAssets {
               assetVersionInfo.splice(0, 0, versionedAssetJSONResponse);
               // Remove duplicates
               assetVersionInfo = _.uniqWith(assetVersionInfo, _.isEqual);
-              self
-                .getVersionedAssetJSON(uid, --version, assetVersionInfo)
-                .then(resolve)
-                .catch(reject)
+              self.getVersionedAssetJSON(uid, --version, assetVersionInfo).then(resolve).catch(reject);
             })
-            .catch(reject)
+            .catch(reject);
         })
         .catch((error) => {
-          addlogs(self.config, error, 'error')
+          addlogs(self.config, error, 'error');
           console.log('Error on  fetch', error && error.message);
 
           if (error.status === 408) {
@@ -322,37 +326,37 @@ module.exports = class ExportAssets {
             self.assetDownloadRetry[uid + version]
               ? ++self.assetDownloadRetry[uid + version]
               : (self.assetDownloadRetry[uid + version] = 1);
-            return self.getVersionedAssetJSON(uid, version, assetVersionInfo)
-              .then(resolve)
-              .catch(reject)
+            return self.getVersionedAssetJSON(uid, version, assetVersionInfo).then(resolve).catch(reject);
           }
 
-          reject(error)
-        })
-    })
+          reject(error);
+        });
+    });
   }
 
   downloadAsset(asset) {
-    let self = this;
+    const self = this;
     return new Promise(async function (resolve, reject) {
-      const assetFolderPath = path.resolve(self.assetsFolderPath, asset.uid)
-      const assetFilePath = path.resolve(assetFolderPath, asset.filename)
+      const assetFolderPath = path.resolve(self.assetsFolderPath, asset.uid);
+      const assetFilePath = path.resolve(assetFolderPath, asset.filename);
 
       if (fs.existsSync(assetFilePath)) {
         addlogs(
           self.config,
           'Skipping download of { title: ' + asset.filename + ', uid: ' + asset.uid + ' }, as they already exist',
-          'success'
-        )
-        return resolve()
+          'success',
+        );
+        return resolve();
       }
       self.assetStream = {
-        url: self.config.securedAssets ? `${asset.url}?authtoken=${self.config.authtoken || self.config.auth_token}` : asset.url,
-      }
-
-      helper.makeDirectory(assetFolderPath)
-      const assetFileStream = fs.createWriteStream(assetFilePath)
-      self.assetStream.url = encodeURI(self.assetStream.url)
+        url: self.config.securedAssets
+          ? `${asset.url}?authtoken=${self.config.authtoken || self.config.auth_token}`
+          : asset.url,
+      };
+
+      await helper.makeDirectory(assetFolderPath);
+      const assetFileStream = fs.createWriteStream(assetFilePath);
+      self.assetStream.url = encodeURI(self.assetStream.url);
       self.httpClient
         .options({ responseType: 'stream' })
         .get(self.assetStream.url)
@@ -361,84 +365,87 @@ module.exports = class ExportAssets {
             const str = progress({
               time: 5000,
               length: assetStreamRequest.headers['content-length'],
-            })
+            });
             str.on('progress', function (progressData) {
               console.log(`${asset.filename}: ${Math.round(progressData.percentage)}%`);
             });
             assetStreamRequest.pipe(str).pipe(assetFileStream);
           }
-          assetStreamRequest.pipe(assetFileStream)
-        }).catch((error) => {
-          addlogs(self.config, error, 'error')
-          reject(error)
+          assetStreamRequest.pipe(assetFileStream);
         })
+        .catch((error) => {
+          addlogs(self.config, error, 'error');
+          reject(error);
+        });
       assetFileStream
         .on('close', function () {
           addlogs(self.config, 'Downloaded ' + asset.filename + ': ' + asset.uid + ' successfully!', 'success');
           return resolve();
         })
         .on('error', (error) => {
-          addlogs(self.config, error, 'error')
-          reject(error)
-        })
+          addlogs(self.config, error, 'error');
+          reject(error);
+        });
     });
   }
 
   getFolders() {
-    let self = this;
+    const self = this;
     return new Promise(function (resolve, reject) {
       return self
         .getAssetCount(true)
         .then(function (count) {
           if (count === 0) {
-            addlogs(self.config, 'No folders were found in the stack', 'success')
-            return resolve()
+            addlogs(self.config, 'No folders were found in the stack', 'success');
+            return resolve();
           }
           return self
             .getFolderDetails(0, count)
             .then(function () {
-              addlogs(self.config, chalk.green('Exported asset-folders successfully!'), 'success')
-              return resolve()
-            }).catch(function (error) {
-              addlogs(self.config, error, 'error')
-              reject(error)
+              addlogs(self.config, chalk.green('Exported asset-folders successfully!'), 'success');
+              return resolve();
             })
-        }).catch(function (error) {
-          addlogs(self.config, error, 'error')
-          reject(error)
+            .catch(function (error) {
+              addlogs(self.config, error, 'error');
+              reject(error);
+            });
+        })
+        .catch(function (error) {
+          addlogs(self.config, error, 'error');
+          reject(error);
         });
     });
   }
 
   getFolderDetails(skip, tCount) {
-    let self = this;
+    const self = this;
     return new Promise(function (resolve, reject) {
       if (typeof skip !== 'number') {
-        skip = 0
+        skip = 0;
       }
       if (skip > tCount) {
-        helper.writeFileSync(self.folderJSONPath, self.folderContents)
-        return resolve()
+        helper.writeFileSync(self.folderJSONPath, self.folderContents);
+        return resolve();
       }
-      let queryRequestObj = {
+      const queryRequestObj = {
         skip: skip,
         include_folders: true,
-        query: { is_dir: true }
-      }
-      self.client
-        .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
+        query: { is_dir: true },
+      };
+      self.stackAPIClient
         .asset()
         .query(queryRequestObj)
         .find()
         .then((folderDetailsResponse) => {
           for (let i in folderDetailsResponse.items) {
-            self.folderContents.push(folderDetailsResponse.items[i])
+            self.folderContents.push(folderDetailsResponse.items[i]);
           }
-          skip += 100
-          return self.getFolderDetails(skip, tCount).then(resolve).catch(reject)
-        }).catch((error) => {
-          addlogs(self.config, error, 'error')
+          skip += 100;
+          return self.getFolderDetails(skip, tCount).then(resolve).catch(reject);
         })
-    })
+        .catch((error) => {
+          addlogs(self.config, error, 'error');
+        });
+    });
   }
-}
+};

package/src/lib/export/custom-roles.js

@@ -9,11 +9,9 @@ const helper = require('../util/helper');
 const { addlogs } = require('../util/log');
 const { formatError } = require('../util');
 const config = require('../../config/default');
-const stack = require('../util/contentstack-management-sdk');
 
 module.exports = class ExportCustomRoles {
   roles = {};
-  client = null;
   customRoles = {};
   EXISTING_ROLES = {
     Admin: 1,
@@ -22,9 +20,9 @@ module.exports = class ExportCustomRoles {
   };
   rolesConfig = config.modules.customRoles;
 
-  constructor(credentialConfig) {
-    this.config = merge(config, credentialConfig);
-    this.client = stack.Client(this.config);
+  constructor(exportConfig, stackAPIClient) {
+    this.stackAPIClient = stackAPIClient;
+    this.config = merge(config, exportConfig);
   }
 
   async start() {
@@ -35,10 +33,7 @@ module.exports = class ExportCustomRoles {
       const rolesFolderPath = path.resolve(this.config.data, this.config.branchName || '', this.rolesConfig.dirName);
       mkdirp.sync(rolesFolderPath);
 
-      const roles = await this.client
-        .stack({ api_key: self.config.source_stack, management_token: self.config.management_token })
-        .role()
-        .fetchAll({ include_rules: true, include_permissions: true });
+      const roles = await this.stackAPIClient.role().fetchAll({ include_rules: true, include_permissions: true });
 
       const customRoles = roles.items.filter((role) => !self.EXISTING_ROLES[role.name]);
 
@@ -50,7 +45,7 @@ module.exports = class ExportCustomRoles {
       await self.getCustomRolesLocales(
         customRoles,
         path.join(rolesFolderPath, self.rolesConfig.customRolesLocalesFileName),
-        this.client,
+        this.stackAPIClient,
         self.config,
       );
       self.customRoles = {};
@@ -75,7 +70,7 @@ module.exports = class ExportCustomRoles {
     }
   }
 
-  async getCustomRolesLocales(customRoles, customRolesLocalesFilepath, client, config) {
+  async getCustomRolesLocales(customRoles, customRolesLocalesFilepath, stackAPIClient, config) {
     const localesMap = {};
     for (const role of customRoles) {
       const rulesLocales = role.rules.find((rule) => rule.module === 'locale');
@@ -87,11 +82,7 @@ module.exports = class ExportCustomRoles {
     }
 
     if (Object.keys(localesMap).length) {
-      const locales = await client
-        .stack({ api_key: config.source_stack, management_token: config.management_token })
-        .locale()
-        .query({})
-        .find();
+      const locales = await stackAPIClient.locale().query({}).find();
       const sourceLocalesMap = {};
       for (const locale of locales.items) {
         sourceLocalesMap[locale.uid] = locale;