@contentstack/cli-cm-import 1.1.0 → 1.2.0

package/src/lib/import/content-types.js

@@ -1,266 +1,187 @@
-/* eslint-disable no-console */
-/*!
- * Contentstack Import
- * Copyright (c) 2019 Contentstack LLC
- * MIT Licensed
- */
-let mkdirp = require('mkdirp');
-let fs = require('fs');
-let fsPromises = require('fs').promises;
-let path = require('path');
-let _ = require('lodash');
-let Promise = require('bluebird');
-let chalk = require('chalk');
-
-let helper = require('../util/fs');
-let { addlogs } = require('../util/log');
-let config = require('../../config/default');
-let supress = require('../util/extensionsUidReplace');
-let sdkInstance = require('../util/contentstack-management-sdk');
-const { getInstalledExtensions } = require('../util/marketplace-app-helper')
+const fs = require('fs');
+const path = require('path');
+const chalk = require('chalk');
+const fileHelper = require('../util/fs');
+const config = require('../../config/default');
+const supress = require('../util/extensionsUidReplace');
+const { getInstalledExtensions } = require('../util/marketplace-app-helper');
+const { executeTask, formatError } = require('../util');
+const { addlogs } = require('../util/log');
+const { cloneDeep, remove, isEmpty, find, findIndex } = require('lodash');
+const schemaTemplate = require('../util/schemaTemplate');
+
+class ContentTypesImport {
+  constructor(importConfig, stackAPIClient) {
+    this.stackAPIClient = stackAPIClient;
+    this.importConfig = importConfig;
+    this.contentTypeConfig = importConfig.modules.content_types;
+    this.globalFieldConfig = importConfig.modules.globalfields;
+    this.importConcurrency = this.contentTypeConfig.importConcurrency || this.importConfig.importConcurrency;
+    this.writeConcurrency = this.contentTypeConfig.writeConcurrency || this.importConfig.writeConcurrency;
+    this.contentTypesFolderPath = path.join(this.importConfig.data, this.contentTypeConfig.dirName);
+    this.mapperFolderPath = path.join(this.importConfig.data, 'mapper', 'content_types');
+    this.existingContentTypesPath = path.join(this.mapperFolderPath, 'success.json');
+    this.globalFieldsFolderPath = path.resolve(this.importConfig.data, this.globalFieldConfig.dirName);
+    this.globalFieldMapperFolderPath = path.join(importConfig.data, 'mapper', 'global_fields', 'success.json');
+    this.globalFieldPendingPath = path.join(importConfig.data, 'mapper', 'global_fields', 'pending_global_fields.js');
+    this.appMapperFolderPath = path.join(importConfig.data, 'mapper', 'marketplace_apps');
+    this.ignoredFilesInContentTypesFolder = new Map([
+      ['__master.json', 'true'],
+      ['__priority.json', 'true'],
+      ['schema.json', 'true'],
+      ['.DS_Store', 'true'],
+    ]);
+    this.contentTypes = [];
+    this.existingContentTypesUIds;
+    this.titleToUIdMap = new Map();
+    this.requestOptions = {
+      json: {},
+    };
+    this.fieldRules = [];
+    this.installedExtensions = [];
+    this.globalFields = [];
+    this.existingGlobalFields = [];
+    this.pendingGlobalFields = [];
+  }
 
-let reqConcurrency = config.concurrency;
-let requestLimit = config.rateLimit;
-let contentTypeConfig = config.modules.content_types;
-let globalFieldConfig = config.modules.globalfields;
-let globalFieldsFolderPath;
-let contentTypesFolderPath;
-let mapperFolderPath;
-let globalFieldMapperFolderPath;
-let globalFieldUpdateFile;
-let globalFieldPendingPath;
-let skipFiles = ['__master.json', '__priority.json', 'schema.json', '.DS_Store'];
-let fileNames;
-let field_rules_ct = [];
-let client;
-let stack = {};
+  async start() {
+    try {
+      // read content types
+      // remove content types already existing
+      if (fs.existsSync(this.existingContentTypesPath)) {
+        this.existingContentTypesUIds = fileHelper.readFileSync(this.existingContentTypesPath) || [];
+        this.existingContentTypesUIds = new Map(this.existingContentTypesUIds.map((id) => [id, 'true']));
+      }
 
-function importContentTypes() {
-  this.contentTypes = [];
-  this.schemaTemplate = require('../util/schemaTemplate');
-  this.requestOptions = {
-    json: {},
-  };
-  this.installedExtensions = []
-}
+      const contentTypeFiles = fileHelper.readdirSync(this.contentTypesFolderPath);
+      for (let contentTypeName of contentTypeFiles) {
+        if (!this.ignoredFilesInContentTypesFolder.has(contentTypeName)) {
+          const contentTypePath = path.join(this.contentTypesFolderPath, contentTypeName);
+          const contentType = await fileHelper.readFile(contentTypePath);
+          if (!this.existingContentTypesUIds?.has(contentType.uid)) {
+            this.contentTypes.push(await fileHelper.readFile(contentTypePath));
+          }
+        }
+      }
+      // this.mapUidToTitle(this.csontentTypes);
+
+      // seed content type
+      addlogs(this.importConfig, 'Started to seed content types', 'info');
+      await executeTask(this.contentTypes, this.seedContentType.bind(this), { concurrency: this.importConcurrency });
+      addlogs(this.importConfig, 'Created content types', 'success');
+
+      // update content type
+      this.installedExtensions =
+        fileHelper.readFileSync(path.join(this.appMapperFolderPath, 'marketplace-apps.json')) || {};
+      if (isEmpty(this.installedExtensions)) {
+        this.installedExtensions = await getInstalledExtensions(this.importConfig);
+      }
 
-importContentTypes.prototype = {
-  start: async function (credentialConfig) {
-    addlogs(config, 'Migrating contenttypes', 'success');
-    let self = this;
-    config = credentialConfig;
-    client = sdkInstance.Client(config);
-    stack = client.stack({ api_key: config.target_stack, management_token: config.management_token });
-    globalFieldsFolderPath = path.resolve(config.data, globalFieldConfig.dirName);
-    contentTypesFolderPath = path.resolve(config.data, contentTypeConfig.dirName);
-    mapperFolderPath = path.join(config.data, 'mapper', 'content_types');
-    const appMapperFolderPath = path.join(config.data, 'mapper', 'marketplace_apps');
-    globalFieldMapperFolderPath = helper.readFile(path.join(config.data, 'mapper', 'global_fields', 'success.json'));
-    globalFieldPendingPath = helper.readFile(
-      path.join(config.data, 'mapper', 'global_fields', 'pending_global_fields.js'),
-    );
-    globalFieldUpdateFile = path.join(config.data, 'mapper', 'global_fields', 'success.json');
-    fileNames = fs.readdirSync(path.join(contentTypesFolderPath));
-    self.globalfields = helper.readFile(path.resolve(globalFieldsFolderPath, globalFieldConfig.fileName));
+      addlogs(this.importConfig, 'Started to update content types with references', 'info');
+      await executeTask(this.contentTypes, this.updateContentType.bind(this), { concurrency: this.importConcurrency });
+      addlogs(this.importConfig, 'Updated content types with references', 'success');
+
+      // global field update
+      this.pendingGlobalFields = fileHelper.readFileSync(this.globalFieldPendingPath);
+      if (Array.isArray(this.pendingGlobalFields) && this.pendingGlobalFields.length > 0) {
+        this.globalFields = fileHelper.readFileSync(
+          path.resolve(this.globalFieldsFolderPath, this.globalFieldConfig.fileName),
+        );
+        this.existingGlobalFields = fileHelper.readFileSync(this.globalFieldMapperFolderPath);
+        try {
+          addlogs(this.importConfig, 'Started to update pending global field with content type references', 'info');
+          await executeTask(this.pendingGlobalFields, this.updateGlobalFields.bind(this), {
+            concurrency: this.importConcurrency,
+          });
+          addlogs(this.importConfig, 'Updated pending global fields with content type with references', 'success');
+        } catch (error) {
+          addlogs(
+            this.importConfig,
+            `Failed to updates global fields with content type reference ${formatError(error)}`,
+            'error',
+          );
+        }
+      }
 
-    for (let index in fileNames) {
-      if (skipFiles.indexOf(fileNames[index]) === -1) {
-        self.contentTypes.push(helper.readFile(path.join(contentTypesFolderPath, fileNames[index])));
+      // write field rules
+      if (this.fieldRules.length > 0) {
+        try {
+          await fileHelper.writeFile(path.join(this.contentTypesFolderPath, 'field_rules_uid.json', this.fieldRules));
+        } catch (error) {
+          addlogs(this.importConfig, `Failed to write field rules ${formatError(error)}`, 'success');
+        }
       }
-    }
 
-    self.contentTypeUids = _.map(self.contentTypes, 'uid').filter(val => val);
-    self.createdContentTypeUids = [];
-    if (!fs.existsSync(mapperFolderPath)) {
-      mkdirp.sync(mapperFolderPath);
-    }
-    // avoid re-creating content types that already exists in the stack
-    if (fs.existsSync(path.join(mapperFolderPath, 'success.json'))) {
-      self.createdContentTypeUids = helper.readFile(path.join(mapperFolderPath, 'success.json')) || [];
+      addlogs(this.importConfig, chalk.green('Content types imported successfully'), 'success');
+    } catch (error) {
+      addlogs(this.importConfig, formatError(error), 'error');
+      throw new Error('Failed to import content types');
     }
+  }
 
-    if (fs.existsSync(path.join(appMapperFolderPath, 'marketplace-apps.json'))) {
-      self.installedExtensions = helper.readFile(path.join(appMapperFolderPath, 'marketplace-apps.json')) || {};
+  async seedContentType(contentType) {
+    const body = cloneDeep(schemaTemplate);
+    body.content_type.uid = contentType.uid;
+    body.content_type.title = contentType.title;
+    const requestObject = cloneDeep(this.requestOptions);
+    requestObject.json = body;
+
+    try {
+      await this.stackAPIClient.contentType().create(requestObject.json);
+    } catch (error) {
+      if (error.error_code === 115 && (error.errors.uid || error.errors.title)) {
+        // content type uid already exists
+        // _.remove(self.contentTypes, { uid: uid });
+        return true;
+      }
+      throw error;
     }
+  }
 
-    if (_.isEmpty(self.installedExtensions)) {
-      self.installedExtensions = await getInstalledExtensions(config)
+  async updateContentType(contentType) {
+    if (typeof contentType !== 'object') return;
+    const requestObject = cloneDeep(this.requestOptions);
+    if (contentType.field_rules) {
+      this.fieldRules.push(contentType.uid);
+      delete contentType.field_rules;
     }
 
-    self.contentTypeUids = _.difference(self.contentTypeUids, self.createdContentTypeUids);
-    self.uidToTitleMap = self.mapUidToTitle(self.contentTypes);
-    // remove content types, already created
-    _.remove(this.contentTypes, function (contentType) {
-      return self.contentTypeUids.indexOf(contentType.uid) === -1;
-    });
-
-    return new Promise(function (resolve, reject) {
-      if (self.contentTypes === undefined || _.isEmpty(self.contentTypes)) {
-        addlogs(config, chalk.yellow('No Content types found'), 'success');
-        return resolve({ empty: true })
-      }
-      return Promise.map(
-        self.contentTypeUids,
-        function (contentTypeUid) {
-          return self
-            .seedContentTypes(contentTypeUid, self.uidToTitleMap[contentTypeUid])
-            .catch(reject);
-        },
-        {
-          // seed 3 content types at a time
-          concurrency: reqConcurrency,
-        },
-      )
-        .then(function () {
-          let batches = [];
-          let lenObj = self.contentTypes;
-          for (let i = 0; i < lenObj.length; i += Math.round(requestLimit / 3)) {
-            batches.push(lenObj.slice(i, i + Math.round(requestLimit / 3)));
-          }
-
-          return Promise.map(
-            batches,
-            async function (batch) {
-              return Promise.map(
-                batch,
-                async function (contentType) {
-                  await self.updateContentTypes(contentType)
-                  addlogs(config, contentType.uid + ' was updated successfully!', 'success');
-                },
-                {
-                  concurrency: reqConcurrency,
-                },
-              ).catch((e) => {
-                console.log('Something went wrong while migrating content type batch', e);
-              });
-            },
-            {
-              concurrency: reqConcurrency
-            }
-          ).then(async function () {
-            // eslint-disable-next-line quotes
-            if (field_rules_ct.length > 0) {
-              await fsPromises.writeFile(
-                contentTypesFolderPath + '/field_rules_uid.json',
-                JSON.stringify(field_rules_ct),
-              );
-            }
-
-            if (globalFieldPendingPath && globalFieldPendingPath.length !== 0) {
-              return self
-                .updateGlobalfields()
-                .then(function () {
-                  addlogs(config, chalk.green('Content types have been imported successfully!'), 'success');
-                  return resolve();
-                })
-                .catch((_error) => {
-                  addlogs(config, chalk.green('Error in GlobalFields'), 'success');
-                  return reject();
-                });
-            }
-            addlogs(config, chalk.green('Content types have been imported successfully!'), 'success');
-            return resolve();
-          }).catch((error) => {
-            return reject(error);
-          });
-        })
-        .catch((error) => {
-          return reject(error);
-        });
-    });
-  },
-  seedContentTypes: function (uid, title) {
-    let self = this;
-    return new Promise(function (resolve, reject) {
-      let body = _.cloneDeep(self.schemaTemplate);
-      body.content_type.uid = uid;
-      body.content_type.title = title;
-      let requestObject = _.cloneDeep(self.requestOptions);
-      requestObject.json = body;
-
-      return stack
-        .contentType()
-        .create(requestObject.json)
-        .then(resolve)
-        .catch(function (err) {
-          let error = JSON.parse(err.message);
-          if (error.error_code === 115 && (error.errors.uid || error.errors.title)) {
-            // content type uid already exists
-            return resolve();
-          }
-          return reject(error);
-        });
-    });
-  },
-  updateContentTypes: function (contentType) {
-    let self = this;
-    return new Promise(function (resolve, reject) {
-      setTimeout(async function () {
-        let requestObject = _.cloneDeep(self.requestOptions);
-        if (contentType.field_rules) {
-          field_rules_ct.push(contentType.uid);
-          delete contentType.field_rules;
-        }
-
-        supress(contentType.schema, config.preserveStackVersion, self.installedExtensions);
-        requestObject.json.content_type = contentType;
-        let contentTypeResponse = stack.contentType(contentType.uid);
-        Object.assign(contentTypeResponse, _.cloneDeep(contentType));
-        contentTypeResponse
-          .update()
-          .then((_updatedcontentType) => {
-            return resolve();
-          })
-          .catch((err) => {
-            addlogs(config, err, 'error');
-            return reject(err);
-          });
-      }, 1000);
-    });
-  },
-
-  updateGlobalfields: function () {
-    let self = this;
-    return new Promise(function (resolve, reject) {
-      // eslint-disable-next-line no-undef
-      return Promise.map(globalFieldPendingPath, async function (globalfield) {
-        let Obj = _.find(self.globalfields, { uid: globalfield });
+    supress(contentType.schema, this.importConfig.preserveStackVersion, this.installedExtensions);
+    requestObject.json.content_type = contentType;
+    const contentTypeResponse = this.stackAPIClient.contentType(contentType.uid);
+    Object.assign(contentTypeResponse, cloneDeep(contentType));
+    await contentTypeResponse.update();
+    addlogs(this.importConfig, contentType.uid + ' updated with references', 'success');
+  }
 
-        supress(Obj.schema, config.preserveStackVersion, self.installedExtensions);
-        let globalFieldObj = stack.globalField(globalfield);
-        Object.assign(globalFieldObj, _.cloneDeep(Obj));
-        return globalFieldObj
-          .update()
-          .then((globalFieldResponse) => {
-            let updateObjpos = _.findIndex(globalFieldMapperFolderPath, function (successobj) {
-              let global_field_uid = globalFieldResponse.uid;
-              return global_field_uid === successobj;
-            });
-            globalFieldMapperFolderPath.splice(updateObjpos, 1, Obj);
-            helper.writeFile(globalFieldUpdateFile, globalFieldMapperFolderPath);
+  async updateGlobalFields({ uid }) {
+    const globalField = find(this.globalFields, { uid });
+    supress(globalField.schema, this.importConfig.preserveStackVersion, this.installedExtensions);
+    let globalFieldObj = this.stackAPIClient.globalField(globalField);
+    Object.assign(globalFieldObj, cloneDeep(globalField));
+    try {
+      const globalFieldResponse = await globalFieldObj.update();
+      const existingGlobalField = findIndex(this.existingGlobalFields, (existingGlobalFieldUId) => {
+        return globalFieldResponse.uid === existingGlobalFieldUId;
+      });
 
-            resolve(globalFieldResponse)
-          })
-          .catch(function (err) {
-            let error = JSON.parse(err.message);
-            // eslint-disable-next-line no-console
-            addlogs(config, chalk.red('Global Field failed to update ' + JSON.stringify(error.errors)), 'error');
-          })
-          .catch(function (error) {
-            // failed to update modified schemas back to their original form
-            return reject(error);
-          });
+      // Improve write the updated global fields once all updates are completed
+      this.existingGlobalFields.splice(existingGlobalField, 1, globalField);
+      await fileHelper.writeFile(this.globalFieldMapperFolderPath, this.existingGlobalFields).catch((error) => {
+        addlogs(this.importConfig, `failed to write updated the global field ${uid} ${formatError(error)}`);
       });
-    });
-  },
+      addlogs(this.importConfig, `Updated the global field ${uid} with content type references `);
+      return true;
+    } catch (error) {
+      addlogs(this.importConfig, `failed to update the global field ${uid} ${formatError(error)}`);
+    }
+  }
 
-  mapUidToTitle: function (contentTypes) {
-    let result = {};
-    contentTypes.forEach((ct) => {
-      result[ct.uid] = ct.title;
+  async mapUidToTitle() {
+    this.contentTypes.forEach((ct) => {
+      this.titleToUIdMap.set(ct.uid, ct.title);
     });
-    return result;
   }
-};
+}
 
-module.exports = new importContentTypes();
+module.exports = ContentTypesImport;