@contentstack/cli-cm-import 0.1.1-beta.13 → 0.1.1-beta.16

package/README.md

@@ -9,6 +9,24 @@ It is Contentstack’s CLI plugin to import content in the stack. To learn how t
 * [Usage](#usage)
 * [Commands](#commands)
 <!-- tocstop -->
+
+For switching to EU region update the hosts at config/default.js
+```js
+{
+  host:'https://eu-api.contentstack.com/v3',
+  cdn: 'https://eu-cdn.contentstack.com/v3',
+ ...
+}
+``` 
+
+For switching to AZURE-NA region update the hosts at config/default.js
+```js
+{
+  host:'https://azure-na-api.contentstack.com/v3',
+  cdn: 'https://azure-na-cdn.contentstack.com/v3'
+ ...
+}
+``` 
 # Usage
 <!-- usage -->
 ```sh-session
@@ -16,7 +34,7 @@ $ npm install -g @contentstack/cli-cm-import
 $ csdx COMMAND
 running command...
 $ csdx (-v|--version|version)
-@contentstack/cli-cm-import/0.1.1-beta.13 linux-x64 node-v12.22.7
+@contentstack/cli-cm-import/0.1.1-beta.16 linux-x64 node-v16.14.2
 $ csdx --help [COMMAND]
 USAGE
   $ csdx COMMAND
@@ -62,5 +80,5 @@ EXAMPLES
   csdx cm:import -A -B <branch name>
 ```
 
-_See code: [src/commands/cm/import.js](https://github.com/contentstack/cli/blob/v0.1.1-beta.13/packages/contentstack-import/src/commands/cm/import.js)_
+_See code: [src/commands/cm/import.js](https://github.com/contentstack/cli/blob/v0.1.1-beta.16/packages/contentstack-import/src/commands/cm/import.js)_
 <!-- commandsstop -->

package/oclif.manifest.json

@@ -1 +1 @@
-{"version":"0.1.1-beta.13","commands":{"cm:import":{"id":"cm:import","description":"Import script for importing the content into new stack\n...\nOnce you export content from the source stack, import it to your destination stack by using the cm:import command.\n","pluginName":"@contentstack/cli-cm-import","pluginType":"core","aliases":[],"examples":["csdx cm:import -A","csdx cm:import -A -s <stack_ApiKey> -d <path/of/export/destination/dir>","csdx cm:import -A -c <path/of/config/dir>","csdx cm:import -A -m <single module name>","csdx cm:import -A -m <single module name> -b <backup dir>","csdx cm:import -a <management_token_alias>","csdx cm:import -a <management_token_alias> -d <path/of/export/destination/dir>","csdx cm:import -a <management_token_alias> -c <path/of/config/file>","csdx cm:import -A -m <single module name>","csdx cm:import -A -B <branch name>"],"flags":{"config":{"name":"config","type":"option","char":"c","description":"[optional] path of config file"},"stack-uid":{"name":"stack-uid","type":"option","char":"s","description":"API key of the target stack"},"data":{"name":"data","type":"option","char":"d","description":"path and location where data is stored"},"management-token-alias":{"name":"management-token-alias","type":"option","char":"a","description":"alias of the management token"},"auth-token":{"name":"auth-token","type":"boolean","char":"A","description":"to use auth token","allowNo":false},"module":{"name":"module","type":"option","char":"m","description":"[optional] specific module name"},"backup-dir":{"name":"backup-dir","type":"option","char":"b","description":"[optional] backup directory name when using specific module"},"branch":{"name":"branch","type":"option","char":"B","description":"[optional] branch name"}},"args":[]}}}
+{"version":"0.1.1-beta.16","commands":{"cm:import":{"id":"cm:import","description":"Import script for importing the content into new stack\n...\nOnce you export content from the source stack, import it to your destination stack by using the cm:import command.\n","pluginName":"@contentstack/cli-cm-import","pluginType":"core","aliases":[],"examples":["csdx cm:import -A","csdx cm:import -A -s <stack_ApiKey> -d <path/of/export/destination/dir>","csdx cm:import -A -c <path/of/config/dir>","csdx cm:import -A -m <single module name>","csdx cm:import -A -m <single module name> -b <backup dir>","csdx cm:import -a <management_token_alias>","csdx cm:import -a <management_token_alias> -d <path/of/export/destination/dir>","csdx cm:import -a <management_token_alias> -c <path/of/config/file>","csdx cm:import -A -m <single module name>","csdx cm:import -A -B <branch name>"],"flags":{"config":{"name":"config","type":"option","char":"c","description":"[optional] path of config file"},"stack-uid":{"name":"stack-uid","type":"option","char":"s","description":"API key of the target stack"},"data":{"name":"data","type":"option","char":"d","description":"path and location where data is stored"},"management-token-alias":{"name":"management-token-alias","type":"option","char":"a","description":"alias of the management token"},"auth-token":{"name":"auth-token","type":"boolean","char":"A","description":"to use auth token","allowNo":false},"module":{"name":"module","type":"option","char":"m","description":"[optional] specific module name"},"backup-dir":{"name":"backup-dir","type":"option","char":"b","description":"[optional] backup directory name when using specific module"},"branch":{"name":"branch","type":"option","char":"B","description":"[optional] branch name"}},"args":[]}}}

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@contentstack/cli-cm-import",
 	"description": "Contentstack CLI plugin to import content into stack",
-	"version": "0.1.1-beta.13",
+	"version": "0.1.1-beta.16",
 	"author": "Contentstack",
 	"bugs": "https://github.com/contentstack/cli/issues",
 	"dependencies": {

package/src/config/default.js

@@ -1,5 +1,9 @@
 module.exports = {
   versioning: false,
+  // use below hosts for eu region
+  // host:'https://eu-api.contentstack.com/v3',
+  // use below hosts for azure-na region
+  // host:'https://azure-na-api.contentstack.com/v3',
   // pass locale, only to migrate entries from that locale
   // not passing `locale` will migrate all the locales present
   // locales: ['fr-fr'],

package/src/lib/import/entries.js

@@ -9,6 +9,7 @@ const path = require('path')
 const _ = require('lodash')
 const mkdirp = require('mkdirp')
 const chalk = require('chalk')
+const crypto = require('crypto')
 
 const helper = require('../util/fs')
 const {addlogs} = require('../util/log')
@@ -60,6 +61,12 @@ function importEntries() {
   this.ctSchemas = {}
   // Array of content type uids, that have reference fields
   this.refSchemas = []
+  // map of content types uids and their json-rte fields
+  this.ctJsonRte = []
+  // map of content types uids and their json-rte fields
+  this.ctJsonRteWithEntryRefs = []
+  // Entry refs that are held back to resolve after all entries have been created
+  this.jsonRteEntryRefs = {}
   // Collection of entries, that were not created, as they already exist on Stack
   this.createdEntriesWOUid = []
   // Collection of entry uids, mapped to the language they exist in
@@ -201,6 +208,23 @@ importEntries.prototype = {
             addlogs(config, `Creating entries for content type ${ctUid} in language ${lang} ...`, 'success')
             for (let eUid in entries) {
               if (eUid) {
+
+                // check ctUid in self.ctJsonRte array, if ct exists there... only then remove entry references for json rte
+                // also with json rte, api creates the json-rte field with the same uid as passed in the payload. So use the node's inbuilt
+                // crypto module to generate a random uid and populate the json rte data
+                // https://www.kindacode.com/article/how-to-easily-generate-a-random-string-in-node-js/
+
+                // while creating entries with json-rte field, the api does not create fresh uids for the json-rte field
+                // and its subsequent children. If the data is passed without a uid, then the fields aren't created. So, I'll
+                // generate the uids for now, and will come up with a better solution later
+                if (self.ctJsonRte.indexOf(ctUid) > -1) {
+                  entries[eUid] = self.generateUidsForJsonRteFields(entries[eUid], self.ctSchemas[ctUid].schema)
+                }
+
+                // remove entry references from json-rte fields
+                if (self.ctJsonRteWithEntryRefs.indexOf(ctUid) > -1) {
+                  entries[eUid] = self.removeEntryRefsFromJSONRTE(entries[eUid], self.ctSchemas[ctUid].schema)
+                }
                 // will replace all old asset uid/urls with new ones
                 entries[eUid] = lookupReplaceAssets({
                   content_type: self.ctSchemas[ctUid],
@@ -393,6 +417,8 @@ importEntries.prototype = {
       return Promise.map(self.refSchemas, function (ctUid) {
         let eFolderPath = path.join(entryMapperPath, lang, ctUid)
         let eSuccessFilePath = path.join(eFolderPath, 'success.json')
+        let eFilePath = path.resolve(ePath, ctUid, lang + '.json')
+        let sourceStackEntries = helper.readFile(eFilePath)
 
         if (!fs.existsSync(eSuccessFilePath)) {
           addlogs(config, 'Success file was not found at: ' + eSuccessFilePath, 'success')
@@ -419,12 +445,22 @@ importEntries.prototype = {
         // map failed reference uids @mapper/language/unmapped-uids.json
         let refUidMapperPath = path.join(entryMapperPath, lang)
 
+        // add entry references to JSON RTE fields
         entries = _.map(entries, function (entry) {
           try {
             let uid = entry.uid
+            let updatedEntry
+
+            // restores json rte entry refs if they exist
+            if (self.ctJsonRte.indexOf(ctUid) > -1) {
+              updatedEntry = self.restoreJsonRteEntryRefs(entry, sourceStackEntries[self.mappedUids[entry.uid]], schema)
+            } else {
+              updatedEntry = entry
+            }
+            
             let _entry = lookupReplaceEntries({
               content_type: schema,
-              entry: entry,
+              entry: updatedEntry,
             }, _.clone(self.mappedUids), refUidMapperPath)
             // if there's self references, the uid gets replaced
             _entry.uid = uid
@@ -516,7 +552,7 @@ importEntries.prototype = {
       })
     })
   },
-  supressFields: async function () {
+  supressFields: async function () { // it should be spelled as suppressFields
     addlogs(config, chalk.white('Suppressing content type fields...'), 'success')
     let self = this
     return new Promise(function (resolve, reject) {
@@ -529,6 +565,8 @@ importEntries.prototype = {
           let flag = {
             suppressed: false,
             references: false,
+            jsonRte: false,
+            jsonRteEmbeddedEntries: false
           }
           if (contentTypeSchema.field_rules) {
             delete contentTypeSchema.field_rules
@@ -546,6 +584,19 @@ importEntries.prototype = {
             self.refSchemas.push(uid)
           }
 
+          if (flag.jsonRte) {
+            self.ctJsonRte.push(uid)
+            if (flag.jsonRteEmbeddedEntries) {
+              self.ctJsonRteWithEntryRefs.push(uid)
+              // pushing ct uid to refSchemas, because
+              // repostEntries uses refSchemas content types for
+              // reposting entries
+              if (self.refSchemas.indexOf(uid) === -1) {
+                self.refSchemas.push(uid)
+              }
+            }
+          }
+
           // Replace extensions with new UID
           extension_suppress(contentTypeSchema.schema, config.preserveStackVersion)
         }
@@ -851,6 +902,233 @@ importEntries.prototype = {
       })
     })
   },
+  removeEntryRefsFromJSONRTE(entry, ctSchema) {
+    for (let i = 0; i < ctSchema.length; i++) {
+      switch(ctSchema[i].data_type) {
+        case 'blocks': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(e => {
+                let key = Object.keys(e).pop()
+                let subBlock = ctSchema[i].blocks.filter(e => e.uid === key).pop()
+                e[key] = this.removeEntryRefsFromJSONRTE(e[key], subBlock.schema)
+                return e
+              })
+            }
+          }
+          break;
+        }
+        case 'global_field':
+        case 'group': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(e => {
+                e = this.removeEntryRefsFromJSONRTE(e, ctSchema[i].schema)
+                return e
+              })
+            } else {
+              entry[ctSchema[i].uid] = this.removeEntryRefsFromJSONRTE(entry[ctSchema[i].uid], ctSchema[i].schema)
+            }
+          }
+          break;
+        }
+        case 'json': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(jsonRteData => {
+                // repeated code from else block, will abstract later
+                let entryReferences = jsonRteData.children.filter(e => this.doEntryReferencesExist(e))
+                if (entryReferences.length > 0) {
+                  jsonRteData.children = jsonRteData.children.filter(e => !this.doEntryReferencesExist(e))
+                  return jsonRteData // return jsonRteData without entry references
+                } else {
+                  return jsonRteData // return jsonRteData as it is, because there are no entry references
+                }
+              })
+            } else {
+              let entryReferences = entry[ctSchema[i].uid].children.filter(e => this.doEntryReferencesExist(e))
+              if (entryReferences.length > 0) {
+                entry[ctSchema[i].uid].children = entry[ctSchema[i].uid].children.filter(e => !this.doEntryReferencesExist(e))
+              }
+            }
+          }
+          break;
+        }
+      }
+    }
+    return entry
+  },
+  doEntryReferencesExist(element) { 
+    // checks if the children of p element contain any references
+    // only checking one level deep, not recursive
+
+    if (element.length) {
+      for(let i=0; i < element.length; i++) {
+        // although most data has only one level of nesting, and this case might never come up
+        // I've handled multiple level of nesting for 'p' elements
+        if(element[i].type === 'p' && element[i].children && element[i].children.length > 0) {
+          return this.doEntryReferencesExist(element[i].children)
+        } else if(this.isEntryRef(element[i])) {
+          return true
+        }
+      }
+    } else {
+      if(this.isEntryRef(element)) {
+        return true
+      }
+
+      if (element.type === "p" && element.children && element.children.length > 0) {
+        return this.doEntryReferencesExist(element.children)
+      }
+    }
+    return false
+  },
+  restoreJsonRteEntryRefs(entry, sourceStackEntry, ctSchema) {
+    for (let i = 0; i < ctSchema.length; i++) {
+      switch (ctSchema[i].data_type) {
+        case 'blocks': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map((e, eIndex) => {
+                let key = Object.keys(e).pop()
+                let subBlock = ctSchema[i].blocks.filter(e => e.uid === key).pop()
+                let sourceStackElement = sourceStackEntry[ctSchema[i].uid][eIndex][key]
+                e[key] = this.restoreJsonRteEntryRefs(e[key], sourceStackElement, subBlock.schema)
+                return e
+              })
+            }
+          }
+          break;
+        }
+        case 'global_field':
+        case 'group': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map((e, eIndex) => {
+                let sourceStackElement = sourceStackEntry[ctSchema[i].uid][eIndex]
+                e = this.restoreJsonRteEntryRefs(e, sourceStackElement, ctSchema[i].schema)
+                return e
+              })
+            } else {
+              let sourceStackElement = sourceStackEntry[ctSchema[i].uid]
+              entry[ctSchema[i].uid] = this.restoreJsonRteEntryRefs(entry[ctSchema[i].uid], sourceStackElement, ctSchema[i].schema)
+            }
+          }
+          break;
+        }
+        case 'json': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map((field, index) => {
+                field.children = [
+                  ...field.children, 
+                  ...sourceStackEntry[ctSchema[i].uid][index].children.filter(e => this.doEntryReferencesExist(e))
+                ]
+                return field
+              })
+            } else {
+              entry[ctSchema[i].uid].children = [
+                ...entry[ctSchema[i].uid].children, 
+                ...sourceStackEntry[ctSchema[i].uid].children.filter(e => this.doEntryReferencesExist(e)),
+              ]
+            }
+          }
+          break;
+        }
+      }
+    }
+    return entry
+    //------------------------------------------------------------------------------------------------------------
+    // if (Object.keys(self.jsonRteEntryRefs).indexOf(entry.uid) > -1) {
+    //   Object.keys(self.jsonRteEntryRefs[entry.uid]).forEach(jsonRteFieldUid => {
+    //     if (self.jsonRteEntryRefs[entry.uid][jsonRteFieldUid].length) { // handles when json_rte is multiple
+    //       entry[jsonRteFieldUid] = entry[jsonRteFieldUid].map((field, index) => {
+    //         field.children = [...field.children, ...self.jsonRteEntryRefs[entry.uid][jsonRteFieldUid][index]]
+    //         return field
+    //       })
+    //     } else {
+    //       entry[jsonRteFieldUid].children = [...entry[jsonRteFieldUid].children, ...self.jsonRteEntryRefs[entry.uid][jsonRteFieldUid].children]
+    //     }
+    //   })
+    // }
+    // return entry
+  },
+  isEntryRef(element) {
+    return element.type === "reference" && element.attrs.type === "entry"
+  },
+  generateUidsForJsonRteFields(entry, ctSchema) {
+    for (let i = 0; i < ctSchema.length; i++) {
+      switch (ctSchema[i].data_type) {
+        case 'blocks': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(e => {
+                let key = Object.keys(e).pop()
+                let subBlock = ctSchema[i].blocks.filter(e => e.uid === key).pop()
+                e[key] = this.generateUidsForJsonRteFields(e[key], subBlock.schema)
+                return e
+              })
+            }
+          }
+          break;
+        }
+        case 'global_field':
+        case 'group': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(e => {
+                e = this.generateUidsForJsonRteFields(e, ctSchema[i].schema)
+                return e
+              })
+            } else {
+              entry[ctSchema[i].uid] = this.generateUidsForJsonRteFields(entry[ctSchema[i].uid], ctSchema[i].schema)
+            }
+          }
+          break;
+        }
+        case 'json': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid] = entry[ctSchema[i].uid].map(jsonRteData => {
+                jsonRteData.uid = this.generateUid()
+                jsonRteData.children = jsonRteData.children.map(child => this.populateChildrenWithUids(child))
+                return jsonRteData
+              })
+            } else {
+              entry[ctSchema[i].uid].uid = this.generateUid()
+              entry[ctSchema[i].uid].children = entry[ctSchema[i].uid].children.map(child => this.populateChildrenWithUids(child))
+            }
+          }
+          break;
+        }
+      }
+    }
+    return entry
+  },
+  populateChildrenWithUids(children) {
+    if (children.length && children.length > 0) {
+      return children.map(child => {
+        if(child.type && child.type.length > 0) {
+          child.uid = this.generateUid()
+        }
+        if(child.children && child.children.length > 0) {
+          child.children = this.populateChildrenWithUids(child.children)
+        }
+        return child
+      })
+    } else {
+      if (children.type && children.type.length > 0) {
+        children.uid = this.generateUid()
+      }
+      if (children.children && children.children.length > 0) {
+        children.children = this.populateChildrenWithUids(children.children)
+      }
+      return children
+    }
+  },
+  generateUid() {
+    return crypto.randomBytes(16).toString('hex')
+  }
 }
 
 module.exports = new importEntries()

package/src/lib/util/extensionsUidReplace.js

@@ -45,6 +45,15 @@ var extension_uid_Replace = module.exports = function (schema, preserveStackVers
         // eslint-disable-next-line camelcase
         schema[i].extension_uid = data[extension_key_value]
       }
+    } else if (schema[i].data_type === 'json' && schema[i].hasOwnProperty('plugins') && schema[i].plugins.length > 0) {
+      const newPluginUidsArray = [];
+      const data = helper.readFile(path.join(extensionPath));
+      schema[i].plugins.forEach((extension_key_value) => {
+        if (data && data.hasOwnProperty(extension_key_value)) {
+          newPluginUidsArray.push(data[extension_key_value]);
+        }
+      });
+      schema[i].plugins = newPluginUidsArray;
     }
   }
 }

package/src/lib/util/log.js

@@ -24,7 +24,6 @@ function returnString (args) {
 
 var myCustomLevels = {
   levels: {
-    error: 0,
     warn: 1,
     info: 2,
     debug: 3
@@ -37,26 +36,53 @@ var myCustomLevels = {
   }
 };
 
-function init (_logPath, logfileName) {
-  var logsDir = path.resolve(_logPath, 'logs', 'import')  
-  // Create dir if doesn't already exist
-  mkdirp.sync(logsDir)
-  var logPath = path.join(logsDir, logfileName + '.log');
+let logger
+let errorLogger
 
-  var transports = [new(winston.transports.File)({
-    filename: logPath,
-    maxFiles: 20,
-    maxsize: 1000000,
-    tailable: true,
-    json: true
-  })];
+let successTransport
+let errorTransport
 
-  transports.push(new(winston.transports.Console)());
+// removed logfileName from arguments
+function init(_logPath) {
+  if (!logger || !errorLogger) {
+    var logsDir = path.resolve(_logPath, 'logs', 'import')
+    // Create dir if doesn't already exist
+    mkdirp.sync(logsDir)
 
-  var logger = new(winston.Logger)({
-    transports: transports,
-    levels: myCustomLevels.levels
-  });
+    successTransport = {
+      filename: path.join(logsDir, 'success.log'),
+      maxFiles: 20,
+      maxsize: 1000000,
+      tailable: true,
+      json: true,
+      level: 'info',
+    }
+
+    errorTransport = {
+      filename: path.join(logsDir, 'error.log'),
+      maxFiles: 20,
+      maxsize: 1000000,
+      tailable: true,
+      json: true,
+      level: 'error',
+    }
+
+    logger = new (winston.Logger)({
+      transports: [
+        new (winston.transports.File)(successTransport),
+        new (winston.transports.Console)()
+      ],
+      levels: myCustomLevels.levels,
+    });
+
+    errorLogger = new (winston.Logger)({
+      transports: [
+        new (winston.transports.File)(errorTransport),
+        new (winston.transports.Console)({level: 'error'})
+      ],
+      levels: { error: 0 }
+    })
+  }
 
   return {
     log: function () {
@@ -77,7 +103,7 @@ function init (_logPath, logfileName) {
       var args = slice.call(arguments);
       var logString = returnString(args);
       if (logString) {
-        logger.log('error', logString);
+        errorLogger.log('error', logString);
       }
     },
     debug: function () {
@@ -93,9 +119,11 @@ function init (_logPath, logfileName) {
 exports.addlogs = async (config, message, type) => {
   var configLogPath
   config.source_stack && config.target_stack ? configLogPath = config.data : configLogPath = config.oldPath
+  // ignoring the type argument, as we are not using it to create a logfile anymore
   if (type !== 'error') {
-    init(configLogPath, type).log(message)
+    // removed type argument from init method
+    init(configLogPath).log(message)
   } else {
-    init(configLogPath, type).error(message)
+    init(configLogPath).error(message)
   }
 }

package/src/lib/util/lookupReplaceAssets.js

@@ -30,7 +30,7 @@ module.exports = function (data, mappedAssetUids, mappedAssetUrls, assetUidMappe
       if ((schema[i].data_type === 'text' && schema[i].field_metadata && (schema[i].field_metadata.markdown ||
           schema[i].field_metadata.rich_text_type))) {
         parent.push(schema[i].uid);
-        findFileUrls(parent, schema[i], entry, assetUrls);
+        findFileUrls(schema[i], entry, assetUrls);
         parent.pop();
       }
       if (schema[i].data_type === 'group' || schema[i].data_type === 'global_field') {
@@ -49,35 +49,112 @@ module.exports = function (data, mappedAssetUids, mappedAssetUrls, assetUidMappe
           }
         }
       }
+      // added rich_text_type field check because some marketplace extensions also
+      // have data_type has json
+      if (schema[i].data_type === "json" && schema[i].field_metadata.rich_text_type) {
+        parent.push(schema[i].uid)
+        // findFileUrls(schema[i], entry, assetUrls)
+        if (assetUids.length === 0) {
+          findAssetIdsFromJsonRte(data.entry, data.content_type.schema)
+        }
+        // maybe only one of these checks would be enough
+        parent.pop()
+      }
     }
   };
-  
-  
+
+  function findAssetIdsFromJsonRte(entry, ctSchema) {
+    for (let i = 0; i < ctSchema.length; i++) {
+      switch (ctSchema[i].data_type) {
+        case 'blocks': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(e => {
+                let key = Object.keys(e).pop()
+                let subBlock = ctSchema[i].blocks.filter(e => e.uid === key).pop()
+                findAssetIdsFromJsonRte(e[key], subBlock.schema)
+              })
+            }
+          }
+          break;
+        }
+        case 'global_field':
+        case 'group': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(e => {
+                findAssetIdsFromJsonRte(e, ctSchema[i].schema)
+              })
+            } else {
+              findAssetIdsFromJsonRte(entry[ctSchema[i].uid], ctSchema[i].schema)
+            }
+          }
+          break;
+        }
+        case 'json': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(jsonRteData => {
+                gatherJsonRteAssetIds(jsonRteData)
+              })
+            } else {
+              gatherJsonRteAssetIds(entry[ctSchema[i].uid])
+            }
+          }
+          break;
+        }
+      }
+    }
+  }
+
+  function gatherJsonRteAssetIds(jsonRteData) {
+    jsonRteData.children.forEach(element => {
+      if (element.type) {
+        switch (element.type) {
+          case 'p': {
+            if (element.children && element.children.length > 0) {
+              gatherJsonRteAssetIds(element)
+            }
+            break;
+          }
+          case 'reference': {
+            if (Object.keys(element.attrs).length > 0 && element.attrs.type === "asset") {
+              assetUids.push(element.attrs['asset-uid'])
+              assetUrls.push(element.attrs['asset-link'])
+            }
+            break;
+          }
+        }
+      }
+    })
+  }
+
   find(data.content_type.schema, data.entry);
   updateFileFields(data.entry, data, null, mappedAssetUids, matchedUids, unmatchedUids);
   assetUids = _.uniq(assetUids);
   assetUrls = _.uniq(assetUrls);
   var entry = JSON.stringify(data.entry);
-  assetUids.forEach(function (assetUid) {
-    var uid = mappedAssetUids[assetUid];
-    if (typeof uid !== 'undefined') {
-      entry = entry.replace(new RegExp(assetUid, 'img'), uid);
-      matchedUids.push(assetUid);
-    } else {
-      unmatchedUids.push(assetUid);
-    }
-  });
 
   assetUrls.forEach(function (assetUrl) {
     var url = mappedAssetUrls[assetUrl];
     if (typeof url !== 'undefined') {
       entry = entry.replace(new RegExp(assetUrl, 'img'), url);
-      unmatchedUrls.push(url);
+      matchedUrls.push(url);
     } else {
       unmatchedUrls.push(assetUrl);
     }
   });
 
+  assetUids.forEach(function (assetUid) {
+    var uid = mappedAssetUids[assetUid];
+    if (typeof uid !== 'undefined') {
+      entry = entry.replace(new RegExp(assetUid, 'img'), uid);
+      matchedUids.push(assetUid);
+    } else {
+      unmatchedUids.push(assetUid);
+    }
+  });
+
   if (matchedUids.length) {
     var matchedAssetUids = helper.readFile(path.join(assetUidMapperPath, 'matched-asset-uids.json'));
     matchedAssetUids = matchedAssetUids || {};
@@ -160,9 +237,9 @@ function findFileUrls (schema, _entry, assetUrls) {
   if (schema && schema.field_metadata && schema.field_metadata.markdown) {
     text = marked(_entry);
   } else {
-    text = _entry;
+    text = JSON.stringify(_entry);
   }
-  markdownRegEx = new RegExp('(https://(assets|images).contentstack.io/v3/assets/(.*?)/(.*?)/(.*?)/(.*)(?="))', 'g');
+  markdownRegEx = new RegExp('(https://(assets|(eu-|azure-na-|stag-)?images).contentstack.(io|com)/v3/assets/(.*?)/(.*?)/(.*?)/(.*?)(?="))', 'g');
   while ((markdownMatch = markdownRegEx.exec(text)) !== null) {
     if (markdownMatch && typeof markdownMatch[0] === 'string') {
       assetUrls.push(markdownMatch[0]);

package/src/lib/util/lookupReplaceEntries.js

@@ -21,6 +21,27 @@ module.exports = function (data, mappedUids, uidMapperPath) {
   var isNewRefFields = false
   var preserveStackVersion = config.preserveStackVersion
 
+  function gatherJsonRteEntryIds(jsonRteData) {
+    jsonRteData.children.forEach(element => {
+      if (element.type) {
+        switch (element.type) {
+          case 'p': {
+            if (element.children && element.children.length > 0) {
+              gatherJsonRteEntryIds(element)
+            }
+            break;
+          }
+          case 'reference': {
+            if (Object.keys(element.attrs).length > 0 && element.attrs.type === "entry") {
+              uids.push(element.attrs['entry-uid'])
+            }
+            break;
+          }
+        }
+      }
+    })
+  }
+
   var update = function (parent, form_id, entry) {
     var _entry = entry
     var len = parent.length
@@ -97,9 +118,61 @@ module.exports = function (data, mappedUids, uidMapperPath) {
           parent.pop()
         }
         break
+      case 'json':
+        if (schema[i].field_metadata.rich_text_type) {
+          if (uids.length === 0) {
+            findEntryIdsFromJsonRte(data.entry, data.content_type.schema)
+          }
+        }
+        break
+      }
+    }
+  }
+
+  function findEntryIdsFromJsonRte(entry, ctSchema) {
+    for (let i = 0; i < ctSchema.length; i++) {
+      switch (ctSchema[i].data_type) {
+        case 'blocks': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(e => {
+                let key = Object.keys(e).pop()
+                let subBlock = ctSchema[i].blocks.filter(e => e.uid === key).pop()
+                findEntryIdsFromJsonRte(e[key], subBlock.schema)
+              })
+            }
+          }
+          break;
+        }
+        case 'global_field':
+        case 'group': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(e => {
+                findEntryIdsFromJsonRte(e, ctSchema[i].schema)
+              })
+            } else {
+              findEntryIdsFromJsonRte(entry[ctSchema[i].uid], ctSchema[i].schema)
+            }
+          }
+          break;
+        }
+        case 'json': {
+          if (entry[ctSchema[i].uid] !== undefined) {
+            if (ctSchema[i].multiple) {
+              entry[ctSchema[i].uid].forEach(jsonRteData => {
+                gatherJsonRteEntryIds(jsonRteData)
+              })
+            } else {
+              gatherJsonRteEntryIds(entry[ctSchema[i].uid])
+            }
+          }
+          break;
+        }
       }
     }
   }
+
   find(data.content_type.schema, data.entry)
   if (isNewRefFields) {
     findUidsInNewRefFields(data.entry, uids)

package/src/lib/util/removeReferenceFields.js

@@ -35,6 +35,13 @@ var removeReferenceFields = module.exports = function (schema, flag) {
           'non_localizable': false
         });
       }
-    }
+    } else if( // handling entry references in json rte
+      schema[i].data_type === 'json' 
+      && schema[i].field_metadata.rich_text_type
+      && schema[i].field_metadata.embed_entry
+      && schema[i].reference_to.length > 1) {
+        flag.supressed = true;
+        schema[i].reference_to = ["sys_assets"]
+      }
   }
 };

package/src/lib/util/supress-mandatory-fields.js

@@ -14,6 +14,10 @@ var supress = module.exports = function (schema, flag) {
       }
     } else if (schema[i].data_type === 'reference') {
       flag.references = true;
+    } else if (schema[i].data_type === 'json' && schema[i].field_metadata.rich_text_type) {
+      flag.jsonRte = true
+      if (schema[i].field_metadata.embed_entry === true)
+        flag.jsonRteEmbeddedEntries = true;
     }
 
     if ((schema[i].hasOwnProperty('mandatory') && schema[i].mandatory) || (schema[i].hasOwnProperty('unique') &&