@contentstack/cli-cm-import 1.0.1 → 1.2.0

package/src/lib/import/entries.js

@@ -10,16 +10,18 @@ const _ = require('lodash');
 const mkdirp = require('mkdirp');
 const chalk = require('chalk');
 
+const util = require('../util');
 const helper = require('../util/fs');
 const { addlogs } = require('../util/log');
-const lookupReplaceAssets = require('../util/lookupReplaceAssets');
-const lookupReplaceEntries = require('../util/lookupReplaceEntries');
 const suppress = require('../util/supress-mandatory-fields');
-const extension_suppress = require('../util/extensionsUidReplace');
-const util = require('../util');
-let config = util.getConfig();
 const stack = require('../util/contentstack-management-sdk');
+const extension_suppress = require('../util/extensionsUidReplace');
+const lookupReplaceAssets = require('../util/lookupReplaceAssets');
+const lookupReplaceEntries = require('../util/lookupReplaceEntries');
+const { getInstalledExtensions } = require('../util/marketplace-app-helper');
+
 let client;
+let config = util.getConfig();
 
 let reqConcurrency = config.concurrency;
 let eConfig = config.modules.entries;
@@ -40,8 +42,10 @@ let masterLanguage;
 
 let skipFiles = ['__master.json', '__priority.json', 'schema.json'];
 let entryBatchLimit = config.rateLimit || 10;
+const importConcurrency = eConfig.importConcurrency || config.importConcurrency;
+const writeConcurrency = eConfig.writeConcurrency || config.writeConcurrency;
 
-function importEntries() {
+function EntriesImport() {
   let self = this;
   mappedAssetUidPath = path.resolve(config.data, 'mapper', 'assets', 'uid-mapping.json');
   mappedAssetUrlPath = path.resolve(config.data, 'mapper', 'assets', 'url-mapping.json');
@@ -56,6 +60,7 @@ function importEntries() {
 
   createdEntriesWOUidPath = path.join(entryMapperPath, 'created-entries-wo-uid.json');
   failedWOPath = path.join(entryMapperPath, 'failedWO.json');
+
   // Object of Schemas, referred to by their content type uid
   this.ctSchemas = {};
   // Array of content type uids, that have reference fields
@@ -76,9 +81,11 @@ function importEntries() {
   this.success = [];
   // Entries that failed to get created OR updated
   this.fails = [];
+  // List of installed extensions to replace uid
+  this.installedExtensions = [];
 
   let files = fs.readdirSync(ctPath);
-  this.environment = helper.readFile(environmentPath);
+  this.environment = helper.readFileSync(environmentPath);
   for (let index in files) {
     if (index) {
       try {
@@ -89,14 +96,14 @@ function importEntries() {
           }
         }
       } catch (error) {
-        console.error(error);
+        addlogs(config, `Failed to read the content types to import entries ${util.formatError(error)}`);
         process.exit(0);
       }
     }
   }
 }
 
-importEntries.prototype = {
+EntriesImport.prototype = {
   /**
    * Start point for entry import
    * @return promise
@@ -107,7 +114,17 @@ importEntries.prototype = {
     client = stack.Client(config);
     masterLanguage = config.master_locale;
     addlogs(config, 'Migrating entries', 'success');
-    let languages = helper.readFile(lPath);
+    let languages = helper.readFileSync(lPath);
+    const appMapperFolderPath = path.join(config.data, 'mapper', 'marketplace_apps');
+
+    if (fs.existsSync(path.join(appMapperFolderPath, 'marketplace-apps.json'))) {
+      self.installedExtensions = helper.readFileSync(path.join(appMapperFolderPath, 'marketplace-apps.json')) || {};
+    }
+
+    if (_.isEmpty(self.installedExtensions)) {
+      self.installedExtensions = await getInstalledExtensions(config);
+    }
+
     return new Promise(function (resolve, reject) {
       let langs = [masterLanguage.code];
       for (let i in languages) {
@@ -123,8 +140,10 @@ importEntries.prototype = {
       return self
         .supressFields()
         .then(async function () {
-          let mappedAssetUids = helper.readFile(mappedAssetUidPath) || {};
-          let mappedAssetUrls = helper.readFile(mappedAssetUrlPath) || {};
+          addlogs(config, 'Completed suppressing content type reference fields', 'success');
+
+          let mappedAssetUids = helper.readFileSync(mappedAssetUidPath) || {};
+          let mappedAssetUrls = helper.readFileSync(mappedAssetUrlPath) || {};
 
           // Step 2: Iterate over available languages to create entries in each.
           let counter = 0;
@@ -136,8 +155,18 @@ importEntries.prototype = {
                 (config.hasOwnProperty('onlylocales') && config.onlylocales.indexOf(lang) !== -1) ||
                 !config.hasOwnProperty('onlylocales')
               ) {
+                addlogs(config, `Starting to create entries ${lang} locale`, 'info');
                 await self.createEntries(lang, mappedAssetUids, mappedAssetUrls);
-                await self.getCreatedEntriesWOUid();
+                addlogs(config, 'Entries created successfully', 'info');
+                try {
+                  await self.getCreatedEntriesWOUid();
+                } catch (error) {
+                  addlogs(
+                    config,
+                    `Failed get the existing entries to update the mapper ${util.formatError(error)}, 'error`,
+                  );
+                }
+                addlogs(config, 'Starting to update entries with references', 'info');
                 await self.repostEntries(lang);
                 addlogs(config, "Successfully imported '" + lang + "' entries!", 'success');
                 counter++;
@@ -151,51 +180,62 @@ importEntries.prototype = {
             },
           ).then(async function () {
             // Step 3: Revert all the changes done in content type in step 1
+            addlogs(config, 'Restoring content type changes', 'info');
             await self.unSuppressFields();
+            addlogs(config, 'Removing entries from master language which got created by default', 'info');
             await self.removeBuggedEntries();
-            let ct_field_visibility_uid = helper.readFile(path.join(ctPath + '/field_rules_uid.json'));
+            addlogs(config, 'Updating the field rules of content type', 'info');
+            let ct_field_visibility_uid = helper.readFileSync(path.join(ctPath + '/field_rules_uid.json'));
             let ct_files = fs.readdirSync(ctPath);
             if (ct_field_visibility_uid && ct_field_visibility_uid != 'undefined') {
               for (const element of ct_field_visibility_uid) {
                 if (ct_files.indexOf(element + '.json') > -1) {
                   let schema = require(path.resolve(ctPath, element));
-                  await self.field_rules_update(schema);
+                  try {
+                    await self.field_rules_update(schema);
+                  } catch (error) {
+                    addlogs(
+                      config,
+                      `Failed to update the field rules for content type ${schema.uid} ${util.formatError(error)}`,
+                    );
+                  }
                 }
               }
             }
             addlogs(config, chalk.green('Entries have been imported successfully!'), 'success');
             if (config.entriesPublish) {
+              addlogs(config, chalk.green('Publishing entries'), 'success');
               return self
                 .publish(langs)
                 .then(function () {
                   addlogs(config, chalk.green('All the entries have been published successfully'), 'success');
                   return resolve();
                 })
-                .catch((errors) => {
-                  addlogs(config, chalk.error('Some entries might have failed to publish.'), 'error');
-                  return reject(errors);
+                .catch((error) => {
+                  addlogs(config, `Error in publishing entries ${util.formatError(error)}`, 'error');
                 });
             }
             return resolve();
           });
         })
         .catch(function (error) {
-          return reject(error);
+          addlogs.log(config, util.formatError(error), 'error');
+          reject('Failed import entries');
         });
     });
   },
 
   createEntries: function (lang, mappedAssetUids, mappedAssetUrls) {
     let self = this;
-    return new Promise(function (resolve, reject) {
+    return new Promise(async function (resolve, reject) {
       let contentTypeUids = Object.keys(self.ctSchemas);
       if (fs.existsSync(entryUidMapperPath)) {
-        self.mappedUids = helper.readFile(entryUidMapperPath);
+        self.mappedUids = await helper.readLargeFile(entryUidMapperPath);
       }
       self.mappedUids = self.mappedUids || {};
       return Promise.map(
         contentTypeUids,
-        function (ctUid) {
+        async function (ctUid) {
           let eLangFolderPath = path.join(entryMapperPath, lang);
           let eLogFolderPath = path.join(entryMapperPath, lang, ctUid);
           mkdirp.sync(eLogFolderPath);
@@ -213,11 +253,11 @@ importEntries.prototype = {
           });
 
           if (fs.existsSync(createdEntriesPath)) {
-            createdEntries = helper.readFile(createdEntriesPath);
+            createdEntries = await helper.readLargeFile(createdEntriesPath);
             createdEntries = createdEntries || {};
           }
           if (fs.existsSync(eFilePath)) {
-            let entries = helper.readFile(eFilePath);
+            let entries = await helper.readLargeFile(eFilePath);
             if (!_.isPlainObject(entries) || _.isEmpty(entries)) {
               addlogs(
                 config,
@@ -228,42 +268,51 @@ importEntries.prototype = {
               addlogs(config, `Creating entries for content type ${ctUid} in language ${lang} ...`, 'success');
               for (let eUid in entries) {
                 if (eUid) {
-                  // check ctUid in self.ctJsonRte array, if ct exists there... only then remove entry references for json rte
-                  // also with json rte, api creates the json-rte field with the same uid as passed in the payload.
+                  try {
+                    // check ctUid in self.ctJsonRte array, if ct exists there... only then remove entry references for json rte
+                    // also with json rte, api creates the json-rte field with the same uid as passed in the payload.
 
-                  if (self.ctJsonRte.indexOf(ctUid) > -1) {
-                    entries[eUid] = self.removeUidsFromJsonRteFields(entries[eUid], self.ctSchemas[ctUid].schema);
-                  }
+                    if (self.ctJsonRte.indexOf(ctUid) > -1) {
+                      entries[eUid] = self.removeUidsFromJsonRteFields(entries[eUid], self.ctSchemas[ctUid].schema);
+                    }
 
-                  // remove entry references from json-rte fields
-                  if (self.ctJsonRteWithEntryRefs.indexOf(ctUid) > -1) {
-                    entries[eUid] = self.removeEntryRefsFromJSONRTE(entries[eUid], self.ctSchemas[ctUid].schema);
+                    // remove entry references from json-rte fields
+                    if (self.ctJsonRteWithEntryRefs.indexOf(ctUid) > -1) {
+                      entries[eUid] = self.removeEntryRefsFromJSONRTE(entries[eUid], self.ctSchemas[ctUid].schema);
+                    }
+                    // will replace all old asset uid/urls with new ones
+                    entries[eUid] = lookupReplaceAssets(
+                      {
+                        content_type: self.ctSchemas[ctUid],
+                        entry: entries[eUid],
+                      },
+                      mappedAssetUids,
+                      mappedAssetUrls,
+                      eLangFolderPath,
+                      self.installedExtensions,
+                    );
+                  } catch (error) {
+                    addlogs(config, 'Failed to update entry while creating entry id ' + eUid);
+                    addlogs(config, util.formatError(error), 'error');
                   }
-                  // will replace all old asset uid/urls with new ones
-                  entries[eUid] = lookupReplaceAssets(
-                    {
-                      content_type: self.ctSchemas[ctUid],
-                      entry: entries[eUid],
-                    },
-                    mappedAssetUids,
-                    mappedAssetUrls,
-                    eLangFolderPath,
-                  );
                 }
               }
               let eUids = Object.keys(entries);
               let batches = [];
 
+              let entryBatchLimit = eConfig.batchLimit || 10;
+              let batchSize = Math.round(entryBatchLimit / 3);
+
               // Run entry creation in batches of ~16~ entries
-              for (let i = 0; i < eUids.length; i += Math.round(entryBatchLimit / 3)) {
-                batches.push(eUids.slice(i, i + Math.round(entryBatchLimit / 3)));
+              for (let i = 0; i < eUids.length; i += batchSize) {
+                batches.push(eUids.slice(i, i + batchSize));
               }
               return Promise.map(
                 batches,
                 async function (batch) {
                   return Promise.map(
                     batch,
-                    async function (eUid) {
+                    async function (eUid, batchIndex) {
                       // if entry is already created
                       if (createdEntries.hasOwnProperty(eUid)) {
                         addlogs(
@@ -322,16 +371,15 @@ importEntries.prototype = {
                               }
                             }
                           })
-                          .catch(function (err) {
-                            let error = JSON.parse(err.message);
-                            addlogs(config, chalk.red('Error updating entry', JSON.stringify(error)), 'error');
+                          .catch(function (error) {
+                            addlogs(config, `Failed to update an entry ${eUid} ${util.formatError(error)}`, 'error');
                             self.fails.push({
                               content_type: ctUid,
                               locale: lang,
                               entry: entries[eUid],
-                              error: error,
+                              error: util.formatError(error),
                             });
-                            return err;
+                            return error;
                           });
                       }
                       delete requestObject.json.entry.publish_details;
@@ -367,11 +415,7 @@ importEntries.prototype = {
                                 'error',
                               );
                             } else {
-                              addlogs(
-                                config,
-                                chalk.red('Error creating entry due to: ' + JSON.stringify(error)),
-                                'error',
-                              );
+                              addlogs(config, `Failed to create an entry ${eUid} ${util.formatError(error)}`, 'error');
                             }
                             self.createdEntriesWOUid.push({
                               content_type: ctUid,
@@ -379,12 +423,12 @@ importEntries.prototype = {
                               entry: entries[eUid],
                               error: error,
                             });
-                            helper.writeFile(createdEntriesWOUidPath, self.createdEntriesWOUid);
+                            helper.writeFileSync(createdEntriesWOUidPath, self.createdEntriesWOUid);
                             return;
                           }
                           // TODO: if status code: 422, check the reason
                           // 429 for rate limit
-                          addlogs(config, chalk.red('Error creating entry', JSON.stringify(error)), 'error');
+                          addlogs(config, `Failed to create an entry ${eUid} ${util.formatError(error)}`, 'error');
                           self.fails.push({
                             content_type: ctUid,
                             locale: lang,
@@ -395,14 +439,14 @@ importEntries.prototype = {
                       // create/update 5 entries at a time
                     },
                     {
-                      concurrency: 1,
+                      concurrency: importConcurrency,
                     },
                   ).then(function () {
-                    helper.writeFile(successEntryLogPath, self.success[ctUid]);
-                    helper.writeFile(failedEntryLogPath, self.fails[ctUid]);
-                    helper.writeFile(entryUidMapperPath, self.mappedUids);
-                    helper.writeFile(uniqueUidMapperPath, self.uniqueUids);
-                    helper.writeFile(createdEntriesPath, createdEntries);
+                    helper.writeFileSync(successEntryLogPath, self.success[ctUid]);
+                    helper.writeFileSync(failedEntryLogPath, self.fails[ctUid]);
+                    helper.writeFileSync(entryUidMapperPath, self.mappedUids);
+                    helper.writeFileSync(uniqueUidMapperPath, self.uniqueUids);
+                    helper.writeFileSync(createdEntriesPath, createdEntries);
                   });
                   // process one batch at a time
                 },
@@ -451,7 +495,7 @@ importEntries.prototype = {
           }
         },
         {
-          concurrency: reqConcurrency,
+          concurrency: 1,
         },
       )
         .then(function () {
@@ -467,7 +511,7 @@ importEntries.prototype = {
   getCreatedEntriesWOUid: function () {
     let self = this;
     return new Promise(function (resolve) {
-      self.createdEntriesWOUid = helper.readFile(createdEntriesWOUidPath);
+      self.createdEntriesWOUid = helper.readFileSync(createdEntriesWOUidPath);
       self.failedWO = [];
       if (_.isArray(self.createdEntriesWOUid) && self.createdEntriesWOUid.length > 0) {
         return Promise.map(
@@ -476,10 +520,10 @@ importEntries.prototype = {
             return self.fetchEntry(entry);
           },
           {
-            concurrency: reqConcurrency,
+            concurrency: importConcurrency,
           },
         ).then(function () {
-          helper.writeFile(failedWOPath, self.failedWO);
+          helper.writeFileSync(failedWOPath, self.failedWO);
           addlogs(config, 'Mapped entries without mapped uid successfully!', 'success');
           return resolve();
         });
@@ -490,25 +534,25 @@ importEntries.prototype = {
   },
   repostEntries: function (lang) {
     let self = this;
-    return new Promise(function (resolve, reject) {
-      let _mapped_ = helper.readFile(path.join(entryMapperPath, 'uid-mapping.json'));
+    return new Promise(async function (resolve, reject) {
+      let _mapped_ = await helper.readLargeFile(path.join(entryMapperPath, 'uid-mapping.json'));
       if (_.isPlainObject(_mapped_)) {
         self.mappedUids = _.merge(_mapped_, self.mappedUids);
       }
       return Promise.map(
         self.refSchemas,
-        function (ctUid) {
+        async function (ctUid) {
           let eFolderPath = path.join(entryMapperPath, lang, ctUid);
           let eSuccessFilePath = path.join(eFolderPath, 'success.json');
           let eFilePath = path.resolve(ePath, ctUid, lang + '.json');
-          let sourceStackEntries = helper.readFile(eFilePath);
+          let sourceStackEntries = await helper.readLargeFile(eFilePath);
 
           if (!fs.existsSync(eSuccessFilePath)) {
             addlogs(config, 'Success file was not found at: ' + eSuccessFilePath, 'success');
             return;
           }
 
-          let entries = helper.readFile(eSuccessFilePath);
+          let entries = await helper.readLargeFile(eSuccessFilePath, { type: 'array' }); // TBD LARGE
           entries = entries || [];
           if (entries.length === 0) {
             addlogs(config, "No entries were created to be updated in '" + lang + "' language!", 'success');
@@ -516,8 +560,8 @@ importEntries.prototype = {
           }
 
           // Keep track of entries that have their references updated
-          let refsUpdatedUids = helper.readFile(path.join(eFolderPath, 'refsUpdatedUids.json'));
-          let refsUpdateFailed = helper.readFile(path.join(eFolderPath, 'refsUpdateFailed.json'));
+          let refsUpdatedUids = helper.readFileSync(path.join(eFolderPath, 'refsUpdatedUids.json'));
+          let refsUpdateFailed = helper.readFileSync(path.join(eFolderPath, 'refsUpdateFailed.json'));
           let schema = self.ctSchemas[ctUid];
 
           let batches = [];
@@ -528,6 +572,8 @@ importEntries.prototype = {
           // map failed reference uids @mapper/language/unmapped-uids.json
           let refUidMapperPath = path.join(entryMapperPath, lang);
 
+          addlogs(config, 'staring to update the entry for reposting');
+
           entries = _.map(entries, function (entry) {
             try {
               let uid = entry.uid;
@@ -553,14 +599,20 @@ importEntries.prototype = {
               _entry.uid = uid;
               return _entry;
             } catch (error) {
-              console.error(error);
-              return error;
+              addlogs(
+                config,
+                `Failed to update the entry ${uid} references while reposting ${util.formatError(error)}`,
+              );
             }
           });
 
-          // Run entry creation in batches of ~16~ entries
-          for (let i = 0; i < entries.length; i += Math.round(entryBatchLimit / 3)) {
-            batches.push(entries.slice(i, i + Math.round(entryBatchLimit / 3)));
+          addlogs(config, 'Starting the reposting process for entries');
+
+          const entryBatchLimit = eConfig.batchLimit || 10;
+          const batchSize = Math.round(entryBatchLimit / 3);
+          // Run entry creation in batches
+          for (let i = 0; i < entries.length; i += batchSize) {
+            batches.push(entries.slice(i, i + batchSize));
           }
           return Promise.map(
             batches,
@@ -594,12 +646,12 @@ importEntries.prototype = {
                     return entryResponse
                       .update({ locale: lang })
                       .then((response) => {
-                        for (let j = 0; j < entries.length; j++) {
-                          if (entries[j].uid === response.uid) {
-                            entries[j] = response;
-                            break;
-                          }
-                        }
+                        // for (let j = 0; j < entries.length; j++) {
+                        //   if (entries[j].uid === response.uid) {
+                        //     entries[j] = response;
+                        //     break;
+                        //   }
+                        // }
                         refsUpdatedUids.push(response.uid);
                         return resolveUpdatedUids();
                       })
@@ -617,7 +669,7 @@ importEntries.prototype = {
                           'error',
                         );
 
-                        addlogs(config, error, 'error');
+                        addlogs(config, util.formatError(error), 'error');
                         refsUpdateFailed.push({
                           content_type: ctUid,
                           entry: entry,
@@ -630,24 +682,25 @@ importEntries.prototype = {
                   await promiseResult;
                 },
                 {
-                  concurrency: reqConcurrency,
+                  concurrency: importConcurrency,
                 },
               )
                 .then(function () {
                   // batch completed successfully
-                  helper.writeFile(path.join(eFolderPath, 'success.json'), entries);
-                  helper.writeFile(path.join(eFolderPath, 'refsUpdatedUids.json'), refsUpdatedUids);
-                  helper.writeFile(path.join(eFolderPath, 'refsUpdateFailed.json'), refsUpdateFailed);
+                  helper.writeFileSync(path.join(eFolderPath, 'success.json'), entries);
+                  helper.writeFileSync(path.join(eFolderPath, 'refsUpdatedUids.json'), refsUpdatedUids);
+                  helper.writeFileSync(path.join(eFolderPath, 'refsUpdateFailed.json'), refsUpdateFailed);
                   addlogs(config, 'Completed re-post entries batch no: ' + (index + 1) + ' successfully!', 'success');
                 })
                 .catch(function (error) {
                   // error while executing entry in batch
                   addlogs(config, chalk.red('Failed re-post entries at batch no: ' + (index + 1)), 'error');
-                  throw error;
+                  addlogs(config, util.formatError(error), 'error');
+                  // throw error;
                 });
             },
             {
-              concurrency: reqConcurrency,
+              concurrency: 1,
             },
           )
             .then(function () {
@@ -660,22 +713,13 @@ importEntries.prototype = {
             })
             .catch(function (error) {
               // error while updating entries with references
-              addlogs(
-                config,
-                chalk.red(
-                  "Failed while importing entries of Content Type: '" +
-                    ctUid +
-                    "' in language: '" +
-                    lang +
-                    "' successfully!",
-                ),
-                'error',
-              );
-              throw error;
+              addlogs(config, chalk.red(`Failed re-post entries of content type ${ctUid} locale ${lang}`, 'error'));
+              addlogs(config, util.formatError(error), 'error');
+              // throw error;
             });
         },
         {
-          concurrency: reqConcurrency,
+          concurrency: 1,
         },
       )
         .then(function () {
@@ -692,9 +736,9 @@ importEntries.prototype = {
   },
   supressFields: async function () {
     // it should be spelled as suppressFields
-    addlogs(config, chalk.white('Suppressing content type fields...'), 'success');
+    addlogs(config, 'Suppressing content type reference fields', 'success');
     let self = this;
-    return new Promise(function (resolve, reject) {
+    return new Promise(async function (resolve, reject) {
       let modifiedSchemas = [];
       let suppressedSchemas = [];
 
@@ -750,12 +794,12 @@ importEntries.prototype = {
           }
 
           // Replace extensions with new UID
-          extension_suppress(contentTypeSchema.schema, config.preserveStackVersion);
+          extension_suppress(contentTypeSchema.schema, config.preserveStackVersion, self.installedExtensions);
         }
       }
 
       // write modified schema in backup file
-      helper.writeFile(modifiedSchemaPath, modifiedSchemas);
+      helper.writeFileSync(modifiedSchemaPath, modifiedSchemas);
 
       return Promise.map(
         suppressedSchemas,
@@ -770,25 +814,22 @@ importEntries.prototype = {
               // empty function
             })
             .catch(function (_error) {
-              addlogs(
-                config,
-                chalk.red("Failed to modify mandatory field of '" + schema.uid + "' content type"),
-                'error',
-              );
+              addlogs(config, util.formatError(error), 'error');
+              reject(`Failed suppress content type ${schema.uid} reference fields`);
             });
           // update 5 content types at a time
         },
         {
           // update reqConcurrency content types at a time
-          concurrency: reqConcurrency,
+          concurrency: importConcurrency,
         },
       )
         .then(function () {
           return resolve();
         })
         .catch(function (error) {
-          addlogs(config, chalk.red('Error while suppressing mandatory field schemas'), 'error');
-          return reject(error);
+          addlogs(config, util.formatError(error), 'error');
+          return reject('Failed to suppress reference fields in content type');
         });
     });
   },
@@ -818,9 +859,9 @@ importEntries.prototype = {
           }
           self.mappedUids[query.entry.uid] = response.body.entries[0].uid;
           let _ePath = path.join(entryMapperPath, query.locale, query.content_type, 'success.json');
-          let entries = helper.readFile(_ePath);
+          let entries = helper.readFileSync(_ePath);
           entries.push(query.entry);
-          helper.writeFile(_ePath, entries);
+          helper.writeFileSync(_ePath, entries);
           addlogs(
             config,
             'Completed mapping entry wo uid: ' + query.entry.uid + ': ' + response.body.entries[0].uid,
@@ -834,8 +875,9 @@ importEntries.prototype = {
     });
   },
   unSuppressFields: function () {
-    return new Promise(function (resolve, reject) {
-      let modifiedSchemas = helper.readFile(modifiedSchemaPath);
+    let self = this;
+    return new Promise(async function (resolve, reject) {
+      let modifiedSchemas = helper.readFileSync(modifiedSchemaPath);
       let modifiedSchemasUids = [];
       let updatedExtensionUidsSchemas = [];
       for (let uid in modifiedSchemas) {
@@ -844,7 +886,8 @@ importEntries.prototype = {
           if (_contentTypeSchema.field_rules) {
             delete _contentTypeSchema.field_rules;
           }
-          extension_suppress(_contentTypeSchema.schema, config.preserveStackVersion);
+
+          extension_suppress(_contentTypeSchema.schema, config.preserveStackVersion, self.installedExtensions);
           updatedExtensionUidsSchemas.push(_contentTypeSchema);
         }
       }
@@ -894,7 +937,7 @@ importEntries.prototype = {
             }
           }
           // re-write, in case some schemas failed to update
-          helper.writeFile(modifiedSchemaPath, _.compact(modifiedSchemas));
+          helper.writeFileSync(modifiedSchemaPath, _.compact(modifiedSchemas));
           addlogs(config, 'Re-modified content type schemas to their original form!', 'success');
           return resolve();
         })
@@ -907,7 +950,7 @@ importEntries.prototype = {
   removeBuggedEntries: function () {
     let self = this;
     return new Promise(function (resolve, reject) {
-      let entries = helper.readFile(uniqueUidMapperPath);
+      let entries = helper.readFileSync(uniqueUidMapperPath);
       let bugged = [];
       let removed = [];
       for (let uid in entries) {
@@ -933,12 +976,11 @@ importEntries.prototype = {
             })
             .catch(function (error) {
               addlogs(config, chalk.red('Failed to remove bugged entry from master language'), 'error');
-              addlogs(config, error, 'error');
-              addlogs(config, JSON.stringify(entry), 'error');
+              addlogs(config, util.formatError(error), 'error');
             });
         },
         {
-          concurrency: reqConcurrency,
+          concurrency: importConcurrency,
         },
       )
         .then(function () {
@@ -949,15 +991,15 @@ importEntries.prototype = {
             }
           }
 
-          helper.writeFile(path.join(entryMapperPath, 'removed-uids.json'), removed);
-          helper.writeFile(path.join(entryMapperPath, 'pending-uids.json'), bugged);
+          helper.writeFileSync(path.join(entryMapperPath, 'removed-uids.json'), removed);
+          helper.writeFileSync(path.join(entryMapperPath, 'pending-uids.json'), bugged);
 
           addlogs(config, chalk.green('The stack has been eradicated from bugged entries!'), 'success');
           return resolve();
         })
         .catch(function (error) {
           // error while removing bugged entries from stack
-          return reject(error);
+          addlogs(config, util.formatError(error), 'error');
         });
     });
   },
@@ -974,7 +1016,7 @@ importEntries.prototype = {
               let updatedValue = [];
               for (const element of fieldRulesArray) {
                 let splitedFieldRulesValue = element;
-                let oldUid = helper.readFile(path.join(entryUidMapperPath));
+                let oldUid = helper.readFileSync(path.join(entryUidMapperPath));
                 if (oldUid.hasOwnProperty(splitedFieldRulesValue)) {
                   updatedValue.push(oldUid[splitedFieldRulesValue]);
                 } else {
@@ -986,7 +1028,7 @@ importEntries.prototype = {
           }
         }
       } else {
-        addlogs(config, 'field_rules is not available...', 'error');
+        addlogs(config, 'field_rules is not available', 'error');
       }
 
       client
@@ -996,13 +1038,13 @@ importEntries.prototype = {
         .then((contentTypeResponse) => {
           // Object.assign(ctObj, _.cloneDeep(schema))
           contentTypeResponse.field_rules = schema.field_rules;
-          contentTypeResponse.update();
+          return contentTypeResponse.update();
         })
         .then(() => {
           return resolve();
         })
         .catch(function (error) {
-          return reject(error);
+          addlogs(config, `failed to update the field rules ${util.formatError(error)}`);
         });
     });
   },
@@ -1013,7 +1055,7 @@ importEntries.prototype = {
     };
 
     let contentTypeUids = Object.keys(self.ctSchemas);
-    let entryMapper = helper.readFile(entryUidMapperPath);
+    let entryMapper = helper.readFileSync(entryUidMapperPath);
 
     return new Promise(function (resolve, reject) {
       return Promise.map(
@@ -1022,16 +1064,20 @@ importEntries.prototype = {
           let lang = langs[counter];
           return Promise.map(
             contentTypeUids,
-            function (ctUid) {
+            async function (ctUid) {
               let eFilePath = path.resolve(ePath, ctUid, lang + '.json');
-              let entries = helper.readFile(eFilePath);
+              let entries = await helper.readLargeFile(eFilePath);
 
               let eUids = Object.keys(entries);
               let batches = [];
+              let batchSize;
 
               if (eUids.length > 0) {
-                for (let i = 0; i < eUids.length; i += entryBatchLimit) {
-                  batches.push(eUids.slice(i, i + entryBatchLimit));
+                let entryBatchLimit = eConfig.batchLimit || 10;
+                batchSize = Math.round(entryBatchLimit / 3);
+                // Run entry creation in batches
+                for (let i = 0; i < eUids.length; i += batchSize) {
+                  batches.push(eUids.slice(i, i + batchSize));
                 }
               } else {
                 return;
@@ -1039,7 +1085,7 @@ importEntries.prototype = {
 
               return Promise.map(
                 batches,
-                async function (batch) {
+                async function (batch, index) {
                   return Promise.map(
                     batch,
                     async function (eUid) {
@@ -1075,16 +1121,20 @@ importEntries.prototype = {
                               // eslint-disable-next-line max-nested-callbacks
                               .then((result) => {
                                 // addlogs(config, 'Entry ' + eUid + ' published successfully in ' + ctUid + ' content type', 'success')
-                                console.log('Entry ' + eUid + ' published successfully in ' + ctUid + ' content type');
+                                addlogs(
+                                  config,
+                                  'Entry ' + eUid + ' published successfully in ' + ctUid + ' content type',
+                                  'success',
+                                );
                                 return resolveEntryPublished(result);
                                 // eslint-disable-next-line max-nested-callbacks
                               })
                               .catch(function (err) {
-                                // addlogs(config, 'Entry ' + eUid + ' not published successfully in ' + ctUid + ' content type', 'error')
-                                console.log(
-                                  'Entry ' + eUid + ' not published successfully in ' + ctUid + ' content type',
+                                addlogs(
+                                  config,
+                                  `failed to publish entry ${eUid} content type ${ctUid} ${util.formatError(err)}`,
                                 );
-                                return rejectEntryPublished(err.errorMessage);
+                                return resolveEntryPublished('');
                               });
                           });
                         }
@@ -1093,7 +1143,7 @@ importEntries.prototype = {
                       }
                     },
                     {
-                      concurrency: reqConcurrency,
+                      concurrency: 1,
                     },
                   )
                     .then(function () {
@@ -1101,8 +1151,7 @@ importEntries.prototype = {
                     })
                     .catch(function (error) {
                       // error while executing entry in batch
-                      addlogs(config, error, 'error');
-                      return error;
+                      addlogs(config, util.formatError(error), 'error');
                     });
                 },
                 {
@@ -1111,15 +1160,10 @@ importEntries.prototype = {
               )
                 .then(function () {
                   // addlogs(config, 'Entries published successfully in ' + ctUid + ' content type', 'success')
-                  console.log('Entries published successfully in ' + ctUid + ' content type');
+                  addlogs('Entries published successfully in ' + ctUid + ' content type');
                 })
                 .catch(function (error) {
-                  addlogs(
-                    config,
-                    'Failed some of the Entry publishing in ' + ctUid + ' content type, go through logs for details.',
-                    'error',
-                  );
-                  return error;
+                  addlogs(config, `failed to publish entry in content type ${ctUid} ${util.formatError(error)}`);
                 });
             },
             {
@@ -1128,9 +1172,10 @@ importEntries.prototype = {
           )
             .then(function () {
               // empty function
+              // addlogs('Published entries successfully in ' +);
             })
             .catch(function (error) {
-              return error;
+              addlogs(`Failed to publish few entries in ${lang} ${util.formatError(error)}`);
             });
         },
         {
@@ -1141,7 +1186,8 @@ importEntries.prototype = {
           return resolve();
         })
         .catch((error) => {
-          return reject(error);
+          addlogs(`Failed to publish entries ${util.formatError(error)}`);
+          // return reject(error);
         });
     });
   },
@@ -1209,11 +1255,7 @@ importEntries.prototype = {
 
     if (element.length) {
       for (const item of element) {
-        if (
-          (item.type === 'p' || item.type === 'a') &&
-          item.children &&
-          item.children.length > 0
-        ) {
+        if ((item.type === 'p' || item.type === 'a') && item.children && item.children.length > 0) {
           return this.doEntryReferencesExist(item.children);
         } else if (this.isEntryRef(item)) {
           return true;
@@ -1231,8 +1273,8 @@ importEntries.prototype = {
     return false;
   },
   restoreJsonRteEntryRefs: function (entry, sourceStackEntry, ctSchema) {
-    let mappedAssetUids = helper.readFile(mappedAssetUidPath) || {};
-    let mappedAssetUrls = helper.readFile(mappedAssetUrlPath) || {};
+    let mappedAssetUids = helper.readFileSync(mappedAssetUidPath) || {};
+    let mappedAssetUrls = helper.readFileSync(mappedAssetUrlPath) || {};
     for (const element of ctSchema) {
       switch (element.data_type) {
         case 'blocks': {
@@ -1314,7 +1356,9 @@ importEntries.prototype = {
 
               if (entryRefs.length > 0) {
                 entryRefs.forEach((entryRef) => {
-                  entry[entryRef.uid].children.splice(entryRef.index, 0, entryRef.value);
+                  if (!_.isEmpty(entry[entryRef.uid]) && entry[entryRef.uid].children) {
+                    entry[entryRef.uid].children.splice(entryRef.index, 0, entryRef.value);
+                  }
                 });
               }
             }
@@ -1363,16 +1407,27 @@ importEntries.prototype = {
             if (element.multiple) {
               entry[element.uid] = entry[element.uid].map((jsonRteData) => {
                 delete jsonRteData.uid; // remove uid
-                jsonRteData.attrs.dirty = true;
-                jsonRteData.children = jsonRteData.children.map((child) => this.removeUidsFromChildren(child));
+
+                if (_.isObject(jsonRteData.attrs)) {
+                  jsonRteData.attrs.dirty = true;
+                }
+
+                if (!_.isEmpty(jsonRteData.children)) {
+                  jsonRteData.children = _.map(jsonRteData.children, (child) => this.removeUidsFromChildren(child));
+                }
+
                 return jsonRteData;
               });
             } else {
               delete entry[element.uid].uid; // remove uid
-              entry[element.uid].attrs.dirty = true;
-              entry[element.uid].children = entry[element.uid].children.map((child) =>
-                this.removeUidsFromChildren(child),
-              );
+              if (entry[element.uid]  && _.isObject(entry[element.uid].attrs)) {
+                entry[element.uid].attrs.dirty = true;
+              }
+              if (entry[element.uid] && !_.isEmpty(entry[element.uid].children)) {
+                entry[element.uid].children = _.map(entry[element.uid].children, (child) =>
+                  this.removeUidsFromChildren(child),
+                );
+              }
             }
           }
           break;
@@ -1386,7 +1441,10 @@ importEntries.prototype = {
       return children.map((child) => {
         if (child.type && child.type.length > 0) {
           delete child.uid; // remove uid
-          child.attrs.dirty = true;
+
+          if (_.isObject(child.attrs)) {
+            child.attrs.dirty = true;
+          }
         }
         if (child.children && child.children.length > 0) {
           child.children = this.removeUidsFromChildren(child.children);
@@ -1396,7 +1454,9 @@ importEntries.prototype = {
     } else {
       if (children.type && children.type.length > 0) {
         delete children.uid; // remove uid
-        children.attrs.dirty = true;
+        if (_.isObject(children.attrs)) {
+          children.attrs.dirty = true;
+        }
       }
       if (children.children && children.children.length > 0) {
         children.children = this.removeUidsFromChildren(children.children);
@@ -1407,7 +1467,9 @@ importEntries.prototype = {
   setDirtyTrue: function (jsonRteChild) {
     // also removing uids in this function
     if (jsonRteChild.type) {
-      jsonRteChild.attrs['dirty'] = true;
+      if (_.isObject(jsonRteChild.attrs)) {
+        jsonRteChild.attrs['dirty'] = true;
+      }
       delete jsonRteChild.uid;
 
       if (jsonRteChild.children && jsonRteChild.children.length > 0) {
@@ -1450,4 +1512,4 @@ importEntries.prototype = {
   },
 };
 
-module.exports = new importEntries();
+module.exports = EntriesImport;