@contentstack/cli-cm-import 0.1.1-beta.15 → 0.1.1-beta.18

package/README.md

@@ -9,6 +9,24 @@ It is Contentstack’s CLI plugin to import content in the stack. To learn how t
 * [Usage](#usage)
 * [Commands](#commands)
 <!-- tocstop -->
+
+For switching to EU region update the hosts at config/default.js
+```js
+{
+  host:'https://eu-api.contentstack.com/v3',
+  cdn: 'https://eu-cdn.contentstack.com/v3',
+ ...
+}
+``` 
+
+For switching to AZURE-NA region update the hosts at config/default.js
+```js
+{
+  host:'https://azure-na-api.contentstack.com/v3',
+  cdn: 'https://azure-na-cdn.contentstack.com/v3'
+ ...
+}
+``` 
 # Usage
 <!-- usage -->
 ```sh-session
@@ -16,7 +34,7 @@ $ npm install -g @contentstack/cli-cm-import
 $ csdx COMMAND
 running command...
 $ csdx (-v|--version|version)
-@contentstack/cli-cm-import/0.1.1-beta.15 linux-x64 node-v16.14.2
+@contentstack/cli-cm-import/0.1.1-beta.18 linux-x64 node-v16.14.2
 $ csdx --help [COMMAND]
 USAGE
   $ csdx COMMAND
@@ -62,5 +80,5 @@ EXAMPLES
   csdx cm:import -A -B <branch name>
 ```
 
-_See code: [src/commands/cm/import.js](https://github.com/contentstack/cli/blob/v0.1.1-beta.15/packages/contentstack-import/src/commands/cm/import.js)_
+_See code: [src/commands/cm/import.js](https://github.com/contentstack/cli/blob/v0.1.1-beta.18/packages/contentstack-import/src/commands/cm/import.js)_
 <!-- commandsstop -->

package/oclif.manifest.json

@@ -1 +1 @@
-{"version":"0.1.1-beta.15","commands":{"cm:import":{"id":"cm:import","description":"Import script for importing the content into new stack\n...\nOnce you export content from the source stack, import it to your destination stack by using the cm:import command.\n","pluginName":"@contentstack/cli-cm-import","pluginType":"core","aliases":[],"examples":["csdx cm:import -A","csdx cm:import -A -s <stack_ApiKey> -d <path/of/export/destination/dir>","csdx cm:import -A -c <path/of/config/dir>","csdx cm:import -A -m <single module name>","csdx cm:import -A -m <single module name> -b <backup dir>","csdx cm:import -a <management_token_alias>","csdx cm:import -a <management_token_alias> -d <path/of/export/destination/dir>","csdx cm:import -a <management_token_alias> -c <path/of/config/file>","csdx cm:import -A -m <single module name>","csdx cm:import -A -B <branch name>"],"flags":{"config":{"name":"config","type":"option","char":"c","description":"[optional] path of config file"},"stack-uid":{"name":"stack-uid","type":"option","char":"s","description":"API key of the target stack"},"data":{"name":"data","type":"option","char":"d","description":"path and location where data is stored"},"management-token-alias":{"name":"management-token-alias","type":"option","char":"a","description":"alias of the management token"},"auth-token":{"name":"auth-token","type":"boolean","char":"A","description":"to use auth token","allowNo":false},"module":{"name":"module","type":"option","char":"m","description":"[optional] specific module name"},"backup-dir":{"name":"backup-dir","type":"option","char":"b","description":"[optional] backup directory name when using specific module"},"branch":{"name":"branch","type":"option","char":"B","description":"[optional] branch name"}},"args":[]}}}
+{"version":"0.1.1-beta.18","commands":{"cm:import":{"id":"cm:import","description":"Import script for importing the content into new stack\n...\nOnce you export content from the source stack, import it to your destination stack by using the cm:import command.\n","pluginName":"@contentstack/cli-cm-import","pluginType":"core","aliases":[],"examples":["csdx cm:import -A","csdx cm:import -A -s <stack_ApiKey> -d <path/of/export/destination/dir>","csdx cm:import -A -c <path/of/config/dir>","csdx cm:import -A -m <single module name>","csdx cm:import -A -m <single module name> -b <backup dir>","csdx cm:import -a <management_token_alias>","csdx cm:import -a <management_token_alias> -d <path/of/export/destination/dir>","csdx cm:import -a <management_token_alias> -c <path/of/config/file>","csdx cm:import -A -m <single module name>","csdx cm:import -A -B <branch name>"],"flags":{"config":{"name":"config","type":"option","char":"c","description":"[optional] path of config file"},"stack-uid":{"name":"stack-uid","type":"option","char":"s","description":"API key of the target stack"},"data":{"name":"data","type":"option","char":"d","description":"path and location where data is stored"},"management-token-alias":{"name":"management-token-alias","type":"option","char":"a","description":"alias of the management token"},"auth-token":{"name":"auth-token","type":"boolean","char":"A","description":"to use auth token","allowNo":false},"module":{"name":"module","type":"option","char":"m","description":"[optional] specific module name"},"backup-dir":{"name":"backup-dir","type":"option","char":"b","description":"[optional] backup directory name when using specific module"},"branch":{"name":"branch","type":"option","char":"B","description":"[optional] branch name"}},"args":[]}}}

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@contentstack/cli-cm-import",
 	"description": "Contentstack CLI plugin to import content into stack",
-	"version": "0.1.1-beta.15",
+	"version": "0.1.1-beta.18",
 	"author": "Contentstack",
 	"bugs": "https://github.com/contentstack/cli/issues",
 	"dependencies": {

package/src/commands/cm/import.js

@@ -38,8 +38,8 @@ class ImportCommand extends Command {
             moduleName,
             host,
             _authToken,
-            branchName,
-            backupdir
+            backupdir,
+            branchName
           )
           .then(() => {
             return resolve()
@@ -51,8 +51,8 @@ class ImportCommand extends Command {
             moduleName,
             host,
             _authToken,
-            branchName,
-            backupdir
+            backupdir,
+            branchName
           )
           .then(() => {
             return resolve()
@@ -63,8 +63,8 @@ class ImportCommand extends Command {
             moduleName,
             host,
             _authToken,
-            branchName,
-            backupdir
+            backupdir,
+            branchName
           )
           .then(() => {
             return resolve()
@@ -80,8 +80,8 @@ class ImportCommand extends Command {
           _authToken,
           moduleName,
           host,
-          branchName,
-          backupdir
+          backupdir,
+          branchName
         )
         .then(() => {
           return resolve()

package/src/config/default.js

@@ -1,5 +1,9 @@
 module.exports = {
   versioning: false,
+  // use below hosts for eu region
+  // host:'https://eu-api.contentstack.com/v3',
+  // use below hosts for azure-na region
+  // host:'https://azure-na-api.contentstack.com/v3',
   // pass locale, only to migrate entries from that locale
   // not passing `locale` will migrate all the locales present
   // locales: ['fr-fr'],

package/src/lib/import/assets.js

@@ -376,6 +376,7 @@ importAssets.prototype = {
           branch[parent_uid][coll[j].uid] = {}
           coll.splice(j, 1);
           self.findBranches(branch[parent_uid], coll)
+          --j
         }
       }
     }

package/src/lib/import/content-types.js

@@ -191,6 +191,7 @@ importContentTypes.prototype = {
       // eslint-disable-next-line no-undef
       return Promise.map(globalFieldPendingPath, function (globalfield) {
         let Obj = _.find(self.globalfields, {uid: globalfield})
+        supress(Obj.schema)
         let globalFieldObj = stack.globalField(globalfield)
         Object.assign(globalFieldObj, _.cloneDeep(Obj))
         return globalFieldObj.update()

package/src/lib/import/entries.js

@@ -60,6 +60,12 @@ function importEntries() {
   this.ctSchemas = {}
   // Array of content type uids, that have reference fields
   this.refSchemas = []
+  // map of content types uids and their json-rte fields
+  this.ctJsonRte = []
+  // map of content types uids and their json-rte fields
+  this.ctJsonRteWithEntryRefs = []
+  // Entry refs that are held back to resolve after all entries have been created
+  this.jsonRteEntryRefs = {}
   // Collection of entries, that were not created, as they already exist on Stack
   this.createdEntriesWOUid = []
   // Collection of entry uids, mapped to the language they exist in
@@ -110,7 +116,7 @@ importEntries.prototype = {
         }
       }
 
-      // Step 1: Removes filed rules from content type
+      // Step 1: Removes field rules from content type
       // This allows to handle cases like self references and circular reference
       // if mandatory reference fields are not filed in entries then avoids the error
       // Also remove field visibility rules
@@ -201,6 +207,18 @@ importEntries.prototype = {
             addlogs(config, `Creating entries for content type ${ctUid} in language ${lang} ...`, 'success')
             for (let eUid in entries) {
               if (eUid) {
+
+                // check ctUid in self.ctJsonRte array, if ct exists there... only then remove entry references for json rte
+                // also with json rte, api creates the json-rte field with the same uid as passed in the payload.
+
+                if (self.ctJsonRte.indexOf(ctUid) > -1) {
+                  entries[eUid] = self.removeUidsFromJsonRteFields(entries[eUid], self.ctSchemas[ctUid].schema)
+                }
+
+                // remove entry references from json-rte fields
+                if (self.ctJsonRteWithEntryRefs.indexOf(ctUid) > -1) {
+                  entries[eUid] = self.removeEntryRefsFromJSONRTE(entries[eUid], self.ctSchemas[ctUid].schema)
+                }
                 // will replace all old asset uid/urls with new ones
                 entries[eUid] = lookupReplaceAssets({
                   content_type: self.ctSchemas[ctUid],
@@ -227,7 +245,7 @@ importEntries.prototype = {
                   }) + ' as it is already created'), 'success')
                   self.success[ctUid] = createdEntries[eUid]
                   // if its a non-master language, i.e. the entry isn't present in the master language
-                  if (lang !== masterLanguage) {
+                  if (lang !== masterLanguage.code) {
                     self.uniqueUids[eUid] = self.uniqueUids[eUid] || {}
                     if (self.uniqueUids[eUid].locales) {
                       self.uniqueUids[eUid].locales.push(lang)
@@ -256,7 +274,7 @@ importEntries.prototype = {
                       self.mappedUids[eUid] = entryResponse.uid
                       createdEntries = entryResponse
                       // if its a non-master language, i.e. the entry isn't present in the master language
-                      if (lang !== masterLanguage) {
+                      if (lang !== masterLanguage.code) {
                         self.uniqueUids[eUid] = self.uniqueUids[eUid] || {}
                         if (self.uniqueUids[eUid].locales) {
                           self.uniqueUids[eUid].locales.push(lang)
@@ -279,7 +297,7 @@ importEntries.prototype = {
                   })
                 }
                 delete requestObject.json.entry.publish_details
-                return client.stack({api_key: config.target_stack, management_token: config.management_token}).contentType(ctUid).entry().create(requestObject.json)
+                return client.stack({api_key: config.target_stack, management_token: config.management_token}).contentType(ctUid).entry().create(requestObject.json, {locale: lang})
                 .then(async entryResponse => {
                   self.success[ctUid] = self.success[ctUid] || []
                   self.success[ctUid].push(entries[eUid])
@@ -287,7 +305,7 @@ importEntries.prototype = {
                     self.mappedUids[eUid] = entryResponse.uid
                     createdEntries = entryResponse
                     // if its a non-master language, i.e. the entry isn't present in the master language
-                    if (lang !== masterLanguage) {
+                    if (lang !== masterLanguage.code) {
                       self.uniqueUids[eUid] = self.uniqueUids[eUid] || {}
                       if (self.uniqueUids[eUid].locales) {
                         self.uniqueUids[eUid].locales.push(lang)
@@ -393,6 +411,8 @@ importEntries.prototype = {
       return Promise.map(self.refSchemas, function (ctUid) {
         let eFolderPath = path.join(entryMapperPath, lang, ctUid)
         let eSuccessFilePath = path.join(eFolderPath, 'success.json')
+        let eFilePath = path.resolve(ePath, ctUid, lang + '.json')
+        let sourceStackEntries = helper.readFile(eFilePath)
 
         if (!fs.existsSync(eSuccessFilePath)) {
           addlogs(config, 'Success file was not found at: ' + eSuccessFilePath, 'success')
@@ -422,9 +442,19 @@ importEntries.prototype = {
         entries = _.map(entries, function (entry) {
           try {
             let uid = entry.uid
+            let updatedEntry
+
+            // restores json rte entry refs if they exist
+            if (self.ctJsonRte.indexOf(ctUid) > -1) {
+              // the entries stored in eSuccessFilePath, have the same uids as the entries from source data
+              updatedEntry = self.restoreJsonRteEntryRefs(entry, sourceStackEntries[entry.uid], schema.schema)
+            } else {
+              updatedEntry = entry
+            }
+            
             let _entry = lookupReplaceEntries({
               content_type: schema,
-              entry: entry,
+              entry: updatedEntry,
             }, _.clone(self.mappedUids), refUidMapperPath)
             // if there's self references, the uid gets replaced
             _entry.uid = uid
@@ -516,7 +546,7 @@ importEntries.prototype = {
       })
     })
   },
-  supressFields: async function () {
+  supressFields: async function () { // it should be spelled as suppressFields
     addlogs(config, chalk.white('Suppressing content type fields...'), 'success')
     let self = this
     return new Promise(function (resolve, reject) {
@@ -529,6 +559,8 @@ importEntries.prototype = {
           let flag = {
             suppressed: false,
             references: false,
+            jsonRte: false,
+            jsonRteEmbeddedEntries: false
           }
           if (contentTypeSchema.field_rules) {
             delete contentTypeSchema.field_rules
@@ -546,6 +578,19 @@ importEntries.prototype = {
             self.refSchemas.push(uid)
           }
 
+          if (flag.jsonRte) {
+            self.ctJsonRte.push(uid)
+            if (flag.jsonRteEmbeddedEntries) {
+              self.ctJsonRteWithEntryRefs.push(uid)
+              // pushing ct uid to refSchemas, because
+              // repostEntries uses refSchemas content types for
+              // reposting entries
+              if (self.refSchemas.indexOf(uid) === -1) {
+                self.refSchemas.push(uid)
+              }
+            }
+          }
+
           // Replace extensions with new UID
           extension_suppress(contentTypeSchema.schema, config.preserveStackVersion)
         }
@@ -851,6 +896,301 @@ importEntries.prototype = {
       })
     })
   },
+  removeEntryRefsFromJSONRTE: function(entry, ctSchema) {
+    for (let i = 0; i < ctSchema.length; i++) {
+      switch(ctSchema[i].data_type) {
+        case 'blocks': {
+          if (entry[ctSchema[i].uid]) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(e => {
+                let key = Object.keys(e).pop()
+                let subBlock = ctSchema[i].blocks.filter(e => e.uid === key).pop()
+                e[key] = this.removeEntryRefsFromJSONRTE(e[key], subBlock.schema)
+                return e
+              })
+            }
+          }
+          break;
+        }
+        case 'global_field':
+        case 'group': {
+          if (entry[ctSchema[i].uid]) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(e => {
+                e = this.removeEntryRefsFromJSONRTE(e, ctSchema[i].schema)
+                return e
+              })
+            } else {
+              entry[ctSchema[i].uid] = this.removeEntryRefsFromJSONRTE(entry[ctSchema[i].uid], ctSchema[i].schema)
+            }
+          }
+          break;
+        }
+        case 'json': {
+          if (entry[ctSchema[i].uid] && ctSchema[i].field_metadata.rich_text_type) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(jsonRteData => {
+                // repeated code from else block, will abstract later
+                let entryReferences = jsonRteData.children.filter(e => this.doEntryReferencesExist(e))
+                if (entryReferences.length > 0) {
+                  jsonRteData.children = jsonRteData.children.filter(e => !this.doEntryReferencesExist(e))
+                  return jsonRteData // return jsonRteData without entry references
+                } else {
+                  return jsonRteData // return jsonRteData as it is, because there are no entry references
+                }
+              })
+            } else {
+              let entryReferences = entry[ctSchema[i].uid].children.filter(e => this.doEntryReferencesExist(e))
+              if (entryReferences.length > 0) {
+                entry[ctSchema[i].uid].children = entry[ctSchema[i].uid].children.filter(e => !this.doEntryReferencesExist(e))
+              }
+            }
+          }
+          break;
+        }
+      }
+    }
+    return entry
+  },
+  doEntryReferencesExist: function(element) { 
+    // checks if the children of p element contain any references
+    // only checking one level deep, not recursive
+
+    if (element.length) {
+      for(let i=0; i < element.length; i++) {
+        if((element[i].type === 'p' || element[i].type === 'a') && element[i].children && element[i].children.length > 0) {
+          return this.doEntryReferencesExist(element[i].children)
+        } else if(this.isEntryRef(element[i])) {
+          return true
+        }
+      }
+    } else {
+      if(this.isEntryRef(element)) {
+        return true
+      }
+
+      if ((element.type === 'p' || element.type === 'a') && element.children && element.children.length > 0) {
+        return this.doEntryReferencesExist(element.children)
+      }
+    }
+    return false
+  },
+  restoreJsonRteEntryRefs: function(entry, sourceStackEntry, ctSchema) {
+    let mappedAssetUids = helper.readFile(mappedAssetUidPath) || {}
+    let mappedAssetUrls = helper.readFile(mappedAssetUrlPath) || {}
+    for (let i = 0; i < ctSchema.length; i++) {
+      switch (ctSchema[i].data_type) {
+        case 'blocks': {
+          if (entry[ctSchema[i].uid]) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map((e, eIndex) => {
+                let key = Object.keys(e).pop()
+                let subBlock = ctSchema[i].blocks.filter(e => e.uid === key).pop()
+                let sourceStackElement = sourceStackEntry[ctSchema[i].uid][eIndex][key]
+                e[key] = this.restoreJsonRteEntryRefs(e[key], sourceStackElement, subBlock.schema)
+                return e
+              })
+            }
+          }
+          break;
+        }
+        case 'global_field':
+        case 'group': {
+          if (entry[ctSchema[i].uid]) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map((e, eIndex) => {
+                let sourceStackElement = sourceStackEntry[ctSchema[i].uid][eIndex]
+                e = this.restoreJsonRteEntryRefs(e, sourceStackElement, ctSchema[i].schema)
+                return e
+              })
+            } else {
+              let sourceStackElement = sourceStackEntry[ctSchema[i].uid]
+              entry[ctSchema[i].uid] = this.restoreJsonRteEntryRefs(entry[ctSchema[i].uid], sourceStackElement, ctSchema[i].schema)
+            }
+          }
+          break;
+        }
+        case 'json': {
+          if (entry[ctSchema[i].uid] && ctSchema[i].field_metadata.rich_text_type) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map((field, index) => {
+
+                // i am facing a Maximum call stack exceeded issue, 
+                // probably because of this loop operation
+
+                let entryRefs = sourceStackEntry[ctSchema[i].uid][index].children
+                .map((e, index) => {
+                  return { index: index, value: e }
+                })
+                .filter(e => this.doEntryReferencesExist(e.value))
+                .map(e => {
+                  // commenting the line below resolved the maximum call stack exceeded issue
+                  // e.value = this.setDirtyTrue(e.value)
+                  this.setDirtyTrue(e.value)
+                  return e
+                })
+                .map(e => {
+                  // commenting the line below resolved the maximum call stack exceeded issue
+                  // e.value = this.resolveAssetRefsInEntryRefsForJsonRte(e, mappedAssetUids, mappedAssetUrls)
+                  this.resolveAssetRefsInEntryRefsForJsonRte(e.value, mappedAssetUids, mappedAssetUrls)
+                  return e
+                })
+
+                if(entryRefs.length > 0) {
+                  entryRefs.forEach(element => {
+                    field.children.splice(element.index, 0, element.value)
+                  })
+                }
+                return field
+              })
+            } else {
+
+              let entryRefs = sourceStackEntry[ctSchema[i].uid].children
+              .map((e, index) => {
+                return { index: index, value: e }
+              })
+              .filter(e => this.doEntryReferencesExist(e.value))
+              .map(e => {
+                this.setDirtyTrue(e.value)
+                return e
+              })
+              .map(e => {
+                this.resolveAssetRefsInEntryRefsForJsonRte(e.value, mappedAssetUids, mappedAssetUrls)
+                return e
+              })
+
+              if(entryRefs.length > 0) {
+                entryRefs.forEach(element => {
+                  entry[ctSchema[i].uid].children.splice(element.index, 0, element.value)                  
+                })
+              }
+            }
+          }
+          break;
+        }
+      }
+    }
+    return entry
+  },
+  isEntryRef: function(element) {
+    return element.type === "reference" && element.attrs.type === "entry"
+  },
+  removeUidsFromJsonRteFields: function(entry, ctSchema) {
+    for (let i = 0; i < ctSchema.length; i++) {
+      switch (ctSchema[i].data_type) {
+        case 'blocks': {
+          if (entry[ctSchema[i].uid]) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(e => {
+                let key = Object.keys(e).pop()
+                let subBlock = ctSchema[i].blocks.filter(e => e.uid === key).pop()
+                e[key] = this.removeUidsFromJsonRteFields(e[key], subBlock.schema)
+                return e
+              })
+            }
+          }
+          break;
+        }
+        case 'global_field':
+        case 'group': {
+          if (entry[ctSchema[i].uid]) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(e => {
+                e = this.removeUidsFromJsonRteFields(e, ctSchema[i].schema)
+                return e
+              })
+            } else {
+              entry[ctSchema[i].uid] = this.removeUidsFromJsonRteFields(entry[ctSchema[i].uid], ctSchema[i].schema)
+            }
+          }
+          break;
+        }
+        case 'json': {
+          if (entry[ctSchema[i].uid] && ctSchema[i].field_metadata.rich_text_type) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(jsonRteData => {
+                delete jsonRteData.uid // remove uid
+                jsonRteData.attrs.dirty = true
+                jsonRteData.children = jsonRteData.children.map(child => this.removeUidsFromChildren(child))
+                return jsonRteData
+              })
+            } else {
+              delete entry[ctSchema[i].uid].uid // remove uid
+              entry[ctSchema[i].uid].attrs.dirty = true
+              entry[ctSchema[i].uid].children = entry[ctSchema[i].uid].children.map(child => this.removeUidsFromChildren(child))
+            }
+          }
+          break;
+        }
+      }
+    }
+    return entry
+  },
+  removeUidsFromChildren: function(children) {
+    if (children.length && children.length > 0) {
+      return children.map(child => {
+        if(child.type && child.type.length > 0) {
+          delete child.uid // remove uid
+          child.attrs.dirty = true
+        }
+        if(child.children && child.children.length > 0) {
+          child.children = this.removeUidsFromChildren(child.children)
+        }
+        return child
+      })
+    } else {
+      if (children.type && children.type.length > 0) {
+        delete children.uid // remove uid
+        children.attrs.dirty = true
+      }
+      if (children.children && children.children.length > 0) {
+        children.children = this.removeUidsFromChildren(children.children)
+      }
+      return children
+    }
+  },
+  setDirtyTrue: function(jsonRteChild) { // also removing uids in this function
+    if (jsonRteChild.type) {
+      jsonRteChild.attrs['dirty'] = true
+      delete jsonRteChild.uid
+
+      if (jsonRteChild.children && jsonRteChild.children.length > 0) {
+        jsonRteChild.children = jsonRteChild.children.map(subElement => this.setDirtyTrue(subElement))
+      }
+    }
+    return jsonRteChild
+  },
+  resolveAssetRefsInEntryRefsForJsonRte: function(jsonRteChild, mappedAssetUids, mappedAssetUrls) {
+    
+      if (jsonRteChild.type) {
+        if (jsonRteChild.attrs.type === 'asset') {
+          let assetUrl
+          if(mappedAssetUids[jsonRteChild.attrs['asset-uid']]) {
+            jsonRteChild.attrs['asset-uid'] = mappedAssetUids[jsonRteChild.attrs['asset-uid']]
+          }
+
+          if (jsonRteChild.attrs['display-type'] !== 'link') {
+            assetUrl = jsonRteChild.attrs['asset-link']
+          } else {
+            assetUrl = jsonRteChild.attrs['href']
+          }
+
+          if(mappedAssetUrls[assetUrl]) {
+            if (jsonRteChild.attrs['display-type'] !== 'link') {
+              jsonRteChild.attrs['asset-link'] = mappedAssetUrls[assetUrl]
+            } else {
+              jsonRteChild.attrs['href'] = mappedAssetUrls[assetUrl]
+            }
+          }
+        }
+
+        if (jsonRteChild.children && jsonRteChild.children.length > 0) {
+          jsonRteChild.children = jsonRteChild.children.map(subElement => this.resolveAssetRefsInEntryRefsForJsonRte(subElement, mappedAssetUids, mappedAssetUrls))
+        }
+      }
+
+      return jsonRteChild
+  }
 }
 
 module.exports = new importEntries()

package/src/lib/import/extensions.js

@@ -76,7 +76,7 @@ importExtensions.prototype = {
           })
         }
         // the extensions has already been created
-        addlogs(config, chalk.white('The extension: \'' + ext.name +
+        addlogs(config, chalk.white('The extension: \'' + ext.title +
             '\' already exists. Skipping it to avoid duplicates!'), 'success')
         // import 2 extensions at a time
       }, {

package/src/lib/util/contentstack-management-sdk.js

@@ -1,4 +1,7 @@
 const contentstacksdk = require("@contentstack/management");
+const https = require("https");
+
+const { addlogs } = require("./log");
 
 exports.Client = function (config) {
   const option = {
@@ -8,6 +11,17 @@ exports.Client = function (config) {
     api_key: config.target_stack,
     maxContentLength: 100000000,
     maxBodyLength: 1000000000,
+    maxRequests: 10,
+    retryLimit: 5,
+    timeout: 60000,
+    httpsAgent: new https.Agent({
+      maxSockets: 100,
+      maxFreeSockets: 10,
+      keepAlive: true,
+      timeout: 60000, // active socket keepalive for 60 seconds
+      freeSocketTimeout: 30000, // free socket keepalive for 30 seconds
+    }),
+    retryDelay: Math.floor(Math.random() * (8000 - 3000 + 1) + 3000),
     logHandler: (level, data) => {},
     retryCondition: (error) => {
       // no async function should be used here
@@ -22,12 +36,12 @@ exports.Client = function (config) {
     retryDelayOptions: {
       base: 1000,
     },
-  }
-  if (typeof config.branchName === 'string') {
+  };
+  if (typeof config.branchName === "string") {
     option.headers = {
       branch: config.branchName,
-    }
+    };
   }
-  const client = contentstacksdk.client(option)
-  return client
-}
+  const client = contentstacksdk.client(option);
+  return client;
+};

package/src/lib/util/lookupReplaceAssets.js

@@ -30,7 +30,7 @@ module.exports = function (data, mappedAssetUids, mappedAssetUrls, assetUidMappe
       if ((schema[i].data_type === 'text' && schema[i].field_metadata && (schema[i].field_metadata.markdown ||
           schema[i].field_metadata.rich_text_type))) {
         parent.push(schema[i].uid);
-        findFileUrls(parent, schema[i], entry, assetUrls);
+        findFileUrls(schema[i], entry, assetUrls);
         parent.pop();
       }
       if (schema[i].data_type === 'group' || schema[i].data_type === 'global_field') {
@@ -49,35 +49,128 @@ module.exports = function (data, mappedAssetUids, mappedAssetUrls, assetUidMappe
           }
         }
       }
+      // added rich_text_type field check because some marketplace extensions also
+      // have data_type has json
+      if (schema[i].data_type === "json" && schema[i].field_metadata.rich_text_type) {
+        parent.push(schema[i].uid)
+        // findFileUrls(schema[i], entry, assetUrls)
+        findAssetIdsFromJsonRte(data.entry, data.content_type.schema)
+        // maybe only one of these checks would be enough
+        parent.pop()
+      }
     }
   };
+
+  function findAssetIdsFromJsonRte(entry, ctSchema) {
+    for (let i = 0; i < ctSchema.length; i++) {
+      switch (ctSchema[i].data_type) {
+        case 'blocks': {
+          if (entry[ctSchema[i].uid]) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(e => {
+                let key = Object.keys(e).pop()
+                let subBlock = ctSchema[i].blocks.filter(e => e.uid === key).pop()
+                findAssetIdsFromJsonRte(e[key], subBlock.schema)
+              })
+            }
+          }
+          break;
+        }
+        case 'global_field':
+        case 'group': {
+          if (entry[ctSchema[i].uid]) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(e => {
+                findAssetIdsFromJsonRte(e, ctSchema[i].schema)
+              })
+            } else {
+              findAssetIdsFromJsonRte(entry[ctSchema[i].uid], ctSchema[i].schema)
+            }
+          }
+          break;
+        }
+        case 'json': {
+          if (entry[ctSchema[i].uid] && ctSchema[i].field_metadata.rich_text_type) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(jsonRteData => {
+                gatherJsonRteAssetIds(jsonRteData)
+              })
+            } else {
+              gatherJsonRteAssetIds(entry[ctSchema[i].uid])
+            }
+          }
+          break;
+        }
+      }
+    }
+  }
   
-  
+  function gatherJsonRteAssetIds(jsonRteData) {
+    jsonRteData.children.forEach(element => {
+      if (element.type) {
+        switch (element.type) {
+          case 'a':
+          case 'p': {
+            if (element.children && element.children.length > 0) {
+              gatherJsonRteAssetIds(element)
+            }
+            break;
+          }
+          case 'reference': {
+            if (Object.keys(element.attrs).length > 0 && element.attrs.type === "asset") {
+              if (assetUids.indexOf(element.attrs['asset-uid']) === -1) {
+                assetUids.push(element.attrs['asset-uid'])
+              }
+              // assets references inserted as link inside entry reference inserted as link did not have asset-link property
+              // instead it had an 'href' property. I haven't seen 'asset-link' and 'href' together yet
+              // writing this condition assuming that this never occurs, need to confirm
+              // (element.attrs['asset-link']) ? assetUrls.push(element.attrs['asset-link']) : assetUrls.push(element.attrs['asset-link'])
+              if (element.attrs['asset-link']) {
+                if (assetUrls.indexOf(element.attrs['asset-link']) === -1) {
+                  assetUrls.push(element.attrs['asset-link'])
+                }
+              } else if (element.attrs['href']) {
+                if (assetUrls.indexOf(element.attrs['href']) === -1) {
+                  assetUrls.push(element.attrs['href'])
+                }
+              }
+            }
+            if (element.children && element.children.length > 0) {
+              gatherJsonRteAssetIds(element)
+            }
+            break;
+          }
+        }
+      }
+    })
+  }
+
   find(data.content_type.schema, data.entry);
   updateFileFields(data.entry, data, null, mappedAssetUids, matchedUids, unmatchedUids);
   assetUids = _.uniq(assetUids);
   assetUrls = _.uniq(assetUrls);
   var entry = JSON.stringify(data.entry);
-  assetUids.forEach(function (assetUid) {
-    var uid = mappedAssetUids[assetUid];
-    if (typeof uid !== 'undefined') {
-      entry = entry.replace(new RegExp(assetUid, 'img'), uid);
-      matchedUids.push(assetUid);
-    } else {
-      unmatchedUids.push(assetUid);
-    }
-  });
 
   assetUrls.forEach(function (assetUrl) {
     var url = mappedAssetUrls[assetUrl];
     if (typeof url !== 'undefined') {
       entry = entry.replace(new RegExp(assetUrl, 'img'), url);
-      unmatchedUrls.push(url);
+      matchedUrls.push(url);
     } else {
       unmatchedUrls.push(assetUrl);
     }
   });
 
+  assetUids.forEach(function (assetUid) {
+    var uid = mappedAssetUids[assetUid];
+    if (typeof uid !== 'undefined') {
+      entry = entry.replace(new RegExp(assetUid, 'img'), uid);
+      matchedUids.push(assetUid);
+    } else {
+      unmatchedUids.push(assetUid);
+    }
+  });
+
   if (matchedUids.length) {
     var matchedAssetUids = helper.readFile(path.join(assetUidMapperPath, 'matched-asset-uids.json'));
     matchedAssetUids = matchedAssetUids || {};
@@ -158,11 +251,11 @@ function findFileUrls (schema, _entry, assetUrls) {
   var text;
   // Regex to detect v3 asset uri patterns
   if (schema && schema.field_metadata && schema.field_metadata.markdown) {
-    text = marked(_entry);
+    text = marked(JSON.stringify(_entry));
   } else {
-    text = _entry;
+    text = JSON.stringify(_entry);
   }
-  markdownRegEx = new RegExp('(https://(assets|images).contentstack.io/v3/assets/(.*?)/(.*?)/(.*?)/(.*)(?="))', 'g');
+  markdownRegEx = new RegExp('(https://(assets|(eu-|azure-na-|stag-)?images).contentstack.(io|com)/v3/assets/(.*?)/(.*?)/(.*?)/(.*?)(?="))', 'g');
   while ((markdownMatch = markdownRegEx.exec(text)) !== null) {
     if (markdownMatch && typeof markdownMatch[0] === 'string') {
       assetUrls.push(markdownMatch[0]);

package/src/lib/util/lookupReplaceEntries.js

@@ -21,6 +21,33 @@ module.exports = function (data, mappedUids, uidMapperPath) {
   var isNewRefFields = false
   var preserveStackVersion = config.preserveStackVersion
 
+  function gatherJsonRteEntryIds(jsonRteData) {
+    jsonRteData.children.forEach(element => {
+      if (element.type) {
+        switch (element.type) {
+          case 'a':
+          case 'p': {
+            if (element.children && element.children.length > 0) {
+              gatherJsonRteEntryIds(element)
+            }
+            break;
+          }
+          case 'reference': {
+            if (Object.keys(element.attrs).length > 0 && element.attrs.type === "entry") {
+              if (uids.indexOf(element.attrs['entry-uid']) === -1) {
+                uids.push(element.attrs['entry-uid'])
+              }
+            }
+            if (element.children && element.children.length > 0) {
+              gatherJsonRteEntryIds(element)
+            }
+            break;
+          }
+        }
+      }
+    })
+  }
+
   var update = function (parent, form_id, entry) {
     var _entry = entry
     var len = parent.length
@@ -83,6 +110,7 @@ module.exports = function (data, mappedUids, uidMapperPath) {
           parent.pop()
         }
         break
+      case 'global_field':
       case 'group':
         parent.push(schema[i].uid)
         find(schema[i].schema, entry)
@@ -97,9 +125,59 @@ module.exports = function (data, mappedUids, uidMapperPath) {
           parent.pop()
         }
         break
+      case 'json':
+        if (schema[i].field_metadata.rich_text_type) {
+          findEntryIdsFromJsonRte(data.entry, data.content_type.schema)
+        }
+        break
       }
     }
   }
+
+  function findEntryIdsFromJsonRte(entry, ctSchema) {
+    for (let i = 0; i < ctSchema.length; i++) {
+      switch (ctSchema[i].data_type) {
+        case 'blocks': {
+          if (entry[ctSchema[i].uid]) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(e => {
+                let key = Object.keys(e).pop()
+                let subBlock = ctSchema[i].blocks.filter(e => e.uid === key).pop()
+                findEntryIdsFromJsonRte(e[key], subBlock.schema)
+              })
+            }
+          }
+          break;
+        }
+        case 'global_field':
+        case 'group': {
+          if (entry[ctSchema[i].uid]) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(e => {
+                findEntryIdsFromJsonRte(e, ctSchema[i].schema)
+              })
+            } else {
+              findEntryIdsFromJsonRte(entry[ctSchema[i].uid], ctSchema[i].schema)
+            }
+          }
+          break;
+        }
+        case 'json': {
+          if (entry[ctSchema[i].uid] && ctSchema[i].field_metadata.rich_text_type) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(jsonRteData => {
+                gatherJsonRteEntryIds(jsonRteData)
+              })
+            } else {
+              gatherJsonRteEntryIds(entry[ctSchema[i].uid])
+            }
+          }
+          break;
+        }
+      }
+    }
+  }
+
   find(data.content_type.schema, data.entry)
   if (isNewRefFields) {
     findUidsInNewRefFields(data.entry, uids)

package/src/lib/util/removeReferenceFields.js

@@ -35,6 +35,13 @@ var removeReferenceFields = module.exports = function (schema, flag) {
           'non_localizable': false
         });
       }
-    }
+    } else if( // handling entry references in json rte
+      schema[i].data_type === 'json' 
+      && schema[i].field_metadata.rich_text_type
+      && schema[i].field_metadata.embed_entry
+      && schema[i].reference_to.length > 1) {
+        flag.supressed = true;
+        schema[i].reference_to = ["sys_assets"]
+      }
   }
 };

package/src/lib/util/supress-mandatory-fields.js

@@ -6,7 +6,7 @@
 
 var supress = module.exports = function (schema, flag) {
   for (var i in schema) {
-    if (schema[i].data_type === 'group') {
+    if (schema[i].data_type === 'group' || schema[i].data_type === 'global_field') {
       supress(schema[i].schema, flag);
     } else if (schema[i].data_type === 'blocks') {
       for (var block in schema[i].blocks) {
@@ -14,6 +14,10 @@ var supress = module.exports = function (schema, flag) {
       }
     } else if (schema[i].data_type === 'reference') {
       flag.references = true;
+    } else if (schema[i].data_type === 'json' && schema[i].field_metadata.rich_text_type) {
+      flag.jsonRte = true
+      if (schema[i].field_metadata.embed_entry === true)
+        flag.jsonRteEmbeddedEntries = true;
     }
 
     if ((schema[i].hasOwnProperty('mandatory') && schema[i].mandatory) || (schema[i].hasOwnProperty('unique') &&