@contentstack/cli-cm-import 0.1.1-beta.1 → 0.1.1-beta.12

package/src/lib/import/assets.js

@@ -21,7 +21,7 @@ let util = require('../util')
 let config = require('../../config/default')
 const assetsConfig = config.modules.assets
 let assetBatchLimit = (assetsConfig.hasOwnProperty('batchLimit') && typeof assetBatchLimit === 'number') ?
-    assetsConfig.assetBatchLimit : 2
+  assetsConfig.assetBatchLimit : 2
 let assetsFolderPath
 let mapperDirPath
 let environmentPath
@@ -52,12 +52,12 @@ importAssets.prototype = {
     self.assets = helper.readFile(path.join(assetsFolderPath, assetsConfig.fileName))
     self.environment = helper.readFile(environmentPath)
     if (fs.existsSync(self.uidMapperPath)) {
-    self.uidMapping = helper.readFile(self.uidMapperPath)
+      self.uidMapping = helper.readFile(self.uidMapperPath)
     }
     if (fs.existsSync(self.urlMapperPath)) {
       self.urlMapping = helper.readFile(self.urlMapperPath)
     }
-    
+
     mkdirp.sync(mapperDirPath)
 
     return new Promise(function (resolve, reject) {
@@ -72,7 +72,7 @@ importAssets.prototype = {
       }
 
       return self.importFolders().then(function () {
-        return Promise.map(batches, function (batch, index) {
+        return Promise.map(batches, async function (batch, index) {
           return Promise.map(batch, function (assetUid) {
             if (self.uidMapping.hasOwnProperty(assetUid)) {
               addlogs(config, 'Skipping upload of asset: ' + assetUid + '. Its mapped to: ' + self.uidMapping[
@@ -87,7 +87,7 @@ importAssets.prototype = {
               if (config.versioning) {
                 return self.uploadVersionedAssets(assetUid, currentAssetFolderPath).then(function () {
                 }).catch(function (error) {
-                  addlogs(config, (chalk.red('Asset upload failed to import\n' + error), 'error'))
+                  addlogs(config, (chalk.red('Asset upload failed \n' + error), 'error'))
                 })
               }
               let assetPath = path.resolve(currentAssetFolderPath, self.assets[assetUid].filename)
@@ -101,54 +101,53 @@ importAssets.prototype = {
                 }
               }
 
-              return self.uploadAsset(assetPath, self.assets[assetUid], uidContainer, urlContainer).then(function () {
+              return self.uploadAsset(assetPath, self.assets[assetUid], uidContainer, urlContainer).then(async function () {  
                 self.uidMapping[assetUid] = uidContainer[assetUid]
-                self.urlMapping[self.assets[assetUid].url] = urlContainer[self.assets[
-                assetUid].url]
+                self.urlMapping[self.assets[assetUid].url] = urlContainer[self.assets[assetUid].url]
 
-                if (config.entriesPublish) {
-                  if (self.assets[assetUid].publish_details.length > 0) {
-                    let assetsUid = uidContainer[assetUid]
-                    if(self.assets[assetUid].publish_details.length !== 0 ) {
-                      return self.publish(assetsUid, self.assets[assetUid]).then(function () {
-                        return
-                      }).catch(error => {
-                        return
-                      })
-                    }
+                if (config.entriesPublish && self.assets[assetUid].publish_details.length > 0) {
+                  let assetsUid = uidContainer[assetUid]
+                  try {
+                    return await self.publish(assetsUid, self.assets[assetUid])
+                  } catch (error) {
+                    return error
                   }
                 }
-                return
                 // assetUid has been successfully uploaded
                 // log them onto /mapper/assets/success.json
               }).catch(function (error) {
-                addlogs(config, chalk.red('Asset upload failed to import\n' + error, 'error'))
+                addlogs(config, chalk.red('Asset upload failed \n' + error, 'error'))
+                return error
                 // asset failed to upload
                 // log them onto /mapper/assets/fail.json
               })
             }
             addlogs(config, (currentAssetFolderPath + ' does not exist!'), 'error')
           }, {
-            concurrency: assetBatchLimit,
+            concurrency: 1,
           }).then(function () {
             helper.writeFile(self.uidMapperPath, self.uidMapping)
             helper.writeFile(self.urlMapperPath, self.urlMapping)
             // completed uploading assets
             addlogs(config, 'Completed asset import of batch no: ' + (index + 1), 'success')
+            return index + 1
             // TODO: if there are failures, retry
           })
         }, {
           concurrency: 1,
         }).then(function () {
-          addlogs(config, chalk.green('Asset import completed successfully!'), 'success')
+          let numberOfSuccessfulAssetUploads = Object.keys(self.uidMapping).length
+          if (numberOfSuccessfulAssetUploads > 0) {
+            addlogs(config, chalk.green(numberOfSuccessfulAssetUploads + ' assets uploaded successfully!'), 'success')
+          }
           // TODO: if there are failures, retry
           return resolve()
-        }).catch(function () {
-          return reject()
+        }).catch(function (error) {
+          return reject(error)
         })
-    }).catch(function (error) {
-      return reject()
-    })
+      }).catch(function (error) {
+        return reject(error)
+      })
     })
   },
   uploadVersionedAssets: function (uid, assetFolderPath) {
@@ -187,7 +186,7 @@ importAssets.prototype = {
         return self.updateAsset(assetPath, assetMetadata, filesStreamed, uidContainer, urlContainer)
         .then(function () {
           filesStreamed.push(assetMetadata.filename)
-        }).catch(error => {          
+        }).catch(error => {
         })
       }, {
         concurrency: 1,
@@ -239,6 +238,7 @@ importAssets.prototype = {
         urlContainer[metadata.url] = response.url
         return resolve()
       }).catch(function (error) {
+        return reject(error)
       })
     })
   },
@@ -246,7 +246,7 @@ importAssets.prototype = {
     // let self = this
     return new Promise(function (resolve, reject) {
       let requestOption = {}
-      
+
       if (metadata.hasOwnProperty('parent_uid') && typeof metadata.parent_uid === 'string') {
         requestOption.parent_uid = metadata.parent_uid
       }
@@ -275,7 +275,7 @@ importAssets.prototype = {
   },
 
   importFolders: function () {
-      let self = this
+    let self = this
     return new Promise(function (resolve, reject) {
       let mappedFolderPath = path.resolve(config.data, 'mapper', 'assets', 'folder-mapping.json')
       self.folderDetails = helper.readFile(path.resolve(assetsFolderPath, 'folders.json'))
@@ -299,24 +299,24 @@ importAssets.prototype = {
       let idx = 0
       return Promise.map(self.folderBucket, function () {
         let folder = self.folderBucket[idx]
-        if (createdFolders.hasOwnProperty(folder.json.asset.parent_uid)) { 
+        if (createdFolders.hasOwnProperty(folder.json.asset.parent_uid)) {
           // replace old uid with new
           folder.json.asset.parent_uid = createdFolders[folder.json.asset.parent_uid]
-        }        
+        }
         return client.stack({api_key: config.target_stack, management_token: config.management_token}).asset().folder().create(folder.json)
-        .then(response => {          
+        .then(response => {
           addlogs(config, 'Created folder: \'' + folder.json.asset.name + '\'', 'success')
           // { oldUid: newUid }
           createdFolders[folder.oldUid] = response.uid
           helper.writeFile(mappedFolderPath, createdFolders)
           idx++
-          return;
         }).catch(function (err) {
           let error = JSON.parse(err.message)
           if (error.errors.authorization || error.errors.api_key) {
             addlogs(config, chalk.red('Api_key or management_token is not valid'), 'error')
+            return reject(error)
           }
-          return reject(error)
+          return error
         })
       }, {
         concurrency: 1,
@@ -374,6 +374,7 @@ importAssets.prototype = {
         let parent_uid = coll[j].parent_uid
         if (branch.hasOwnProperty(parent_uid)) {
           branch[parent_uid][coll[j].uid] = {}
+          coll.splice(j, 1);
           self.findBranches(branch[parent_uid], coll)
         }
       }
@@ -402,14 +403,17 @@ importAssets.prototype = {
       })
       requestObject.json.asset.environments = envId
       requestObject.json.asset.locales = locales
-      return client.stack({api_key: config.target_stack, management_token: config.management_token}).asset(assetUid).publish({ publishDetails: requestObject.json.asset})
+      return client.stack({api_key: config.target_stack, management_token: config.management_token}).asset(assetUid).publish({publishDetails: requestObject.json.asset})
       .then(function () {
-        addlogs(config, chalk.green('Asset ' + assetUid + ' published successfully'), 'success')
+        addlogs(config, 'Asset ' + assetUid + ' published successfully', 'success')
         return resolve()
       }).catch(function (err) {
-         let error = JSON.parse(err.message)
-         addlogs(config, chalk.red('Asset ' + assetUid + ' not published, '+ error.errorMessage), 'error')
-         return reject(error)
+        if (err && err.message) {
+          let error = JSON.parse(err.message)
+          addlogs(config, chalk.red('Asset ' + assetUid + ' not published, ' + error.errorMessage), 'error')
+          return reject(err)
+        }
+        return reject(err)
       })
     })
   },

package/src/lib/import/content-types.js

@@ -6,6 +6,7 @@
  */
 let mkdirp = require('mkdirp')
 let fs = require('fs')
+let fsPromises = require('fs').promises
 let path = require('path')
 let _ = require('lodash')
 let Promise = require('bluebird')
@@ -15,24 +16,26 @@ let helper = require('../util/fs')
 let util = require('../util')
 let {addlogs} = require('../util/log')
 let supress = require('../util/extensionsUidReplace')
-let stack = require('../util/contentstack-management-sdk')
+let sdkInstance = require('../util/contentstack-management-sdk')
 
 let config = require('../../config/default')
 let reqConcurrency = config.concurrency
+let requestLimit = config.rateLimit
 let contentTypeConfig = config.modules.content_types
 let globalFieldConfig = config.modules.globalfields
-let globalfieldsFolderPath
+let globalFieldsFolderPath
 let contentTypesFolderPath
 let mapperFolderPath
-let globalFieldMapperFolderpath
+let globalFieldMapperFolderPath
 let globalFieldUpdateFile
-let skipFiles = ['__master.json', '__priority.json', 'schema.json']
+let globalFieldPendingPath
+let skipFiles = ['__master.json', '__priority.json', 'schema.json', '.DS_Store']
 let fileNames
 let field_rules_ct = []
 let client
+let stack = {}
 
 function importContentTypes() {
-  let self = this
   this.contentTypes = []
   this.schemaTemplate = require('../util/schemaTemplate')
   this.requestOptions = {
@@ -45,20 +48,22 @@ importContentTypes.prototype = {
     addlogs(config, 'Migrating contenttypes', 'success')
     let self = this
     config = credentialConfig
-    client = stack.Client(config)
-    globalfieldsFolderPath = path.resolve(config.data, globalFieldConfig.dirName)
+    client = sdkInstance.Client(config)
+    stack = client.stack({api_key: config.target_stack, management_token: config.management_token})
+    globalFieldsFolderPath = path.resolve(config.data, globalFieldConfig.dirName)
     contentTypesFolderPath = path.resolve(config.data, contentTypeConfig.dirName)
     mapperFolderPath = path.join(config.data, 'mapper', 'content_types')
-    globalFieldMapperFolderpath =  helper.readFile(path.join(config.data, 'mapper', 'global_fields', 'success.json'))
+    globalFieldMapperFolderPath =  helper.readFile(path.join(config.data, 'mapper', 'global_fields', 'success.json'))
+    globalFieldPendingPath =  helper.readFile(path.join(config.data, 'mapper', 'global_fields', 'pending_global_fields.js'))
     globalFieldUpdateFile =  path.join(config.data, 'mapper', 'global_fields', 'success.json')
     fileNames = fs.readdirSync(path.join(contentTypesFolderPath))
-    self.globalfields = helper.readFile(path.resolve(globalfieldsFolderPath, globalFieldConfig.fileName))
+    self.globalfields = helper.readFile(path.resolve(globalFieldsFolderPath, globalFieldConfig.fileName))
     for (let index in fileNames) {
       if (skipFiles.indexOf(fileNames[index]) === -1) {
         self.contentTypes.push(helper.readFile(path.join(contentTypesFolderPath, fileNames[index])))
       }
     }
-  
+
     self.contentTypeUids = _.map(self.contentTypes, 'uid')
     self.createdContentTypeUids = []
     if (!fs.existsSync(mapperFolderPath)) {
@@ -69,13 +74,14 @@ importContentTypes.prototype = {
       self.createdContentTypeUids = helper.readFile(path.join(mapperFolderPath, 'success.json')) || []
     }
     self.contentTypeUids = _.difference(self.contentTypeUids, self.createdContentTypeUids)
+    self.uidToTitleMap = self.mapUidToTitle(self.contentTypes)
     // remove contet types, already created
     _.remove(this.contentTypes, function (contentType) {
       return self.contentTypeUids.indexOf(contentType.uid) === -1
     })
     return new Promise(function (resolve, reject) {
       return Promise.map(self.contentTypeUids, function (contentTypeUid) {
-        return self.seedContentTypes(contentTypeUid).then(function () {
+        return self.seedContentTypes(contentTypeUid, self.uidToTitleMap[contentTypeUid]).then(function () {
 
         }).catch(function (error) {
           return reject(error)
@@ -84,84 +90,65 @@ importContentTypes.prototype = {
         // seed 3 content types at a time
         concurrency: reqConcurrency,
       }).then(function () {
-        let batches = []        
-        let lenObj = self.contentTypes        
-        for (let i = 0; i < lenObj.length; i += 7) {
-          batches.push(lenObj.slice(i, i + 7))
+        let batches = []
+        let lenObj = self.contentTypes
+        // var a = Math.round(2.60);
+        for (let i = 0; i < lenObj.length; i += Math.round(requestLimit / 3)) {
+          batches.push(lenObj.slice(i, i + Math.round(requestLimit / 3)))
         }
-        
-        return Promise.map(batches, function (batch) {
-          return Promise.map(batch, function (contentType) {
-          return self.updateContentTypes(contentType).then(function () {
+
+        return Promise.map(batches, async function (batch) {
+          return Promise.map(batch, async function (contentType) {
+            await self.updateContentTypes(contentType)
             addlogs(config, contentType.uid + ' was updated successfully!', 'success')
-            return
-          }).catch(function (err) {
-            return reject()
+          },
+          {
+            concurrency: reqConcurrency,
+          }).then(function () {
+          }).catch(e => {
+            console.log('Something went wrong while migrating content type batch', e)
           })
-        })
-        }).then(function () {
+        }, {
+          concurrency: reqConcurrency,
+        }).then(async function () {
           // eslint-disable-next-line quotes
           if (field_rules_ct.length > 0) {
-            fs.writeFile(contentTypesFolderPath + '/field_rules_uid.json', JSON.stringify(field_rules_ct), function (err) {
-              if (err) throw err
+            // fs.writeFile(contentTypesFolderPath + '/field_rules_uid.json', JSON.stringify(field_rules_ct), function (err) {
+            //   if (err) throw err
+            // })
+
+            await fsPromises.writeFile(contentTypesFolderPath + '/field_rules_uid.json', JSON.stringify(field_rules_ct))
+          }
+
+          if (globalFieldPendingPath && globalFieldPendingPath.length !== 0) {
+            return self.updateGlobalfields().then(function () {
+              addlogs(config, chalk.green('Content types have been imported successfully!'), 'success')
+              return resolve()
+            }).catch(error => {
+              addlogs(config, chalk.green('Error in GlobalFields'), 'success')
+              return reject()
             })
           }
-        
-         if( _globalField_pending.length !== 0 ) {
-          return self.updateGlobalfields().then(function () {
-            addlogs(config, chalk.green('Content types have been imported successfully!'), 'success')
-            return resolve()
-          }).catch(reject)
-         }  else {
-            addlogs(config, chalk.green('Content types have been imported successfully!'), 'success')
-            return resolve()
-         } 
+          addlogs(config, chalk.green('Content types have been imported successfully!'), 'success')
+          return resolve()
         }).catch(error => {
           return reject(error)
-        }) 
-        // content type seeidng completed
-        // return Promise.map(self.contentTypes, function (contentType) {
-        //   return self.updateContentTypes(contentType).then(function () {
-        //     log.success(chalk.white(contentType.uid + ' was updated successfully!'))
-        //     return
-        //   }).catch(function (err) {
-        //     return reject()
-        //   })
-        // }).then(function () {
-        //   // eslint-disable-next-line quotes
-        //   if (field_rules_ct.length > 0) {
-        //     fs.writeFile(contentTypesFolderPath + '/field_rules_uid.json', JSON.stringify(field_rules_ct), function (err) {
-        //       if (err) throw err
-        //     })
-        //   }
-        
-        //  if( _globalField_pending.length !== 0 ) {
-        //   return self.updateGlobalfields().then(function () {
-        //     log.success(chalk.green('Content types have been imported successfully!'))
-        //     return resolve()
-        //   }).catch(reject)
-        //  }  else {
-        //     log.success(chalk.green('Content types have been imported successfully!'))
-        //     return resolve()
-        //  } 
-        // }).catch(error => {
-        //   return reject(error)
-        // })
+        })
       }).catch(error => {
         return reject(error)
       })
     })
   },
-  seedContentTypes: function (uid) {
+  seedContentTypes: function (uid, title) {
     let self = this
     return new Promise(function (resolve, reject) {
       let body = _.cloneDeep(self.schemaTemplate)
       body.content_type.uid = uid
-      body.content_type.title = uid
+      body.content_type.title = title
       let requestObject = _.cloneDeep(self.requestOptions)
       requestObject.json = body
-    return client.stack({api_key: config.target_stack, management_token: config.management_token}).contentType().create(requestObject.json)
-      .then(result => {
+      return stack.contentType().create(requestObject.json)
+      .then(() => {
         return resolve()
       })
       .catch(function (err) {
@@ -177,25 +164,24 @@ importContentTypes.prototype = {
   updateContentTypes: function (contentType) {
     let self = this
     return new Promise(function (resolve, reject) {
-       let requestObject = _.cloneDeep(self.requestOptions)
-      if (contentType.field_rules) {
-        field_rules_ct.push(contentType.uid)
-        delete contentType.field_rules
-      }
-      supress(contentType.schema)
-      requestObject.json.content_type = contentType
-      client.stack({api_key: config.target_stack, management_token: config.management_token}).contentType(contentType.uid).fetch()
-      .then(contentTypeResponse => {
+      setTimeout(function () {
+        let requestObject = _.cloneDeep(self.requestOptions)
+        if (contentType.field_rules) {
+          field_rules_ct.push(contentType.uid)
+          delete contentType.field_rules
+        }
+        supress(contentType.schema)
+        requestObject.json.content_type = contentType
+        let contentTypeResponse = stack.contentType(contentType.uid)
         Object.assign(contentTypeResponse, _.cloneDeep(contentType))
         contentTypeResponse.update()
-      })
-      .then(UpdatedcontentType => {
-        return resolve()
-      }).catch(err => {
-        let error = JSON.parse(err.message)
-        addlogs(config, error, 'error')
-        return reject()
-      })
+        .then(UpdatedcontentType => {
+          return resolve()
+        }).catch(err => {
+          addlogs(config, err, 'error')
+          return reject(err)
+        })
+      }, 1000)
     })
   },
 
@@ -203,23 +189,22 @@ importContentTypes.prototype = {
     let self = this
     return new Promise(function (resolve, reject) {
       // eslint-disable-next-line no-undef
-      return Promise.map(_globalField_pending, function (globalfield) {
-        let lenGlobalField = (self.globalfields).length
-        let Obj = _.find(self.globalfields, { 'uid': globalfield});
-      return client.stack({api_key: config.target_stack, management_token: config.management_token}).globalField(globalfield).fetch()
+      return Promise.map(globalFieldPendingPath, function (globalfield) {
+        let Obj = _.find(self.globalfields, {uid: globalfield})
+        let globalFieldObj = stack.globalField(globalfield)
+        Object.assign(globalFieldObj, _.cloneDeep(Obj))
+        return globalFieldObj.update()
         .then(globalFieldResponse => {
-          globalFieldResponse.schema = Obj.schema
-          globalFieldResponse.update()
-          let updateObjpos = _.findIndex(globalFieldMapperFolderpath, function (successobj) {
+          let updateObjpos = _.findIndex(globalFieldMapperFolderPath, function (successobj) {
             let global_field_uid = globalFieldResponse.uid
             return global_field_uid === successobj
           })
-          globalFieldMapperFolderpath.splice(updateObjpos, 1, Obj)
-          helper.writeFile(globalFieldUpdateFile, globalFieldMapperFolderpath)
+          globalFieldMapperFolderPath.splice(updateObjpos, 1, Obj)
+          helper.writeFile(globalFieldUpdateFile, globalFieldMapperFolderPath)
         }).catch(function (err) {
           let error = JSON.parse(err.message)
           // eslint-disable-next-line no-console
-          addlogs(config, chalk.red('Globalfield failed to update ' + JSON.stringify(error.errors)), 'error')
+          addlogs(config, chalk.red('Global Field failed to update ' + JSON.stringify(error.errors)), 'error')
         })
       }, {
         concurrency: reqConcurrency,
@@ -231,6 +216,14 @@ importContentTypes.prototype = {
       })
     })
   },
+
+  mapUidToTitle: function (contentTypes) {
+    let result = {}
+    contentTypes.forEach(ct => {
+      result[ct.uid] = ct.title
+    })
+    return result
+  },
 }
 
-module.exports = new importContentTypes()
+module.exports = new importContentTypes()