@contentstack/cli-cm-import 1.0.1 → 1.2.0

package/src/lib/import/content-types.js

@@ -1,251 +1,187 @@
-/* eslint-disable no-console */
-/*!
- * Contentstack Import
- * Copyright (c) 2019 Contentstack LLC
- * MIT Licensed
- */
-let mkdirp = require('mkdirp');
-let fs = require('fs');
-let fsPromises = require('fs').promises;
-let path = require('path');
-let _ = require('lodash');
-let Promise = require('bluebird');
-let chalk = require('chalk');
-
-let helper = require('../util/fs');
-let { addlogs } = require('../util/log');
-let supress = require('../util/extensionsUidReplace');
-let sdkInstance = require('../util/contentstack-management-sdk');
-
-let config = require('../../config/default');
-let reqConcurrency = config.concurrency;
-let requestLimit = config.rateLimit;
-let contentTypeConfig = config.modules.content_types;
-let globalFieldConfig = config.modules.globalfields;
-let globalFieldsFolderPath;
-let contentTypesFolderPath;
-let mapperFolderPath;
-let globalFieldMapperFolderPath;
-let globalFieldUpdateFile;
-let globalFieldPendingPath;
-let skipFiles = ['__master.json', '__priority.json', 'schema.json', '.DS_Store'];
-let fileNames;
-let field_rules_ct = [];
-let client;
-let stack = {};
-
-function importContentTypes() {
-  this.contentTypes = [];
-  this.schemaTemplate = require('../util/schemaTemplate');
-  this.requestOptions = {
-    json: {},
-  };
-}
+const fs = require('fs');
+const path = require('path');
+const chalk = require('chalk');
+const fileHelper = require('../util/fs');
+const config = require('../../config/default');
+const supress = require('../util/extensionsUidReplace');
+const { getInstalledExtensions } = require('../util/marketplace-app-helper');
+const { executeTask, formatError } = require('../util');
+const { addlogs } = require('../util/log');
+const { cloneDeep, remove, isEmpty, find, findIndex } = require('lodash');
+const schemaTemplate = require('../util/schemaTemplate');
+
+class ContentTypesImport {
+  constructor(importConfig, stackAPIClient) {
+    this.stackAPIClient = stackAPIClient;
+    this.importConfig = importConfig;
+    this.contentTypeConfig = importConfig.modules.content_types;
+    this.globalFieldConfig = importConfig.modules.globalfields;
+    this.importConcurrency = this.contentTypeConfig.importConcurrency || this.importConfig.importConcurrency;
+    this.writeConcurrency = this.contentTypeConfig.writeConcurrency || this.importConfig.writeConcurrency;
+    this.contentTypesFolderPath = path.join(this.importConfig.data, this.contentTypeConfig.dirName);
+    this.mapperFolderPath = path.join(this.importConfig.data, 'mapper', 'content_types');
+    this.existingContentTypesPath = path.join(this.mapperFolderPath, 'success.json');
+    this.globalFieldsFolderPath = path.resolve(this.importConfig.data, this.globalFieldConfig.dirName);
+    this.globalFieldMapperFolderPath = path.join(importConfig.data, 'mapper', 'global_fields', 'success.json');
+    this.globalFieldPendingPath = path.join(importConfig.data, 'mapper', 'global_fields', 'pending_global_fields.js');
+    this.appMapperFolderPath = path.join(importConfig.data, 'mapper', 'marketplace_apps');
+    this.ignoredFilesInContentTypesFolder = new Map([
+      ['__master.json', 'true'],
+      ['__priority.json', 'true'],
+      ['schema.json', 'true'],
+      ['.DS_Store', 'true'],
+    ]);
+    this.contentTypes = [];
+    this.existingContentTypesUIds;
+    this.titleToUIdMap = new Map();
+    this.requestOptions = {
+      json: {},
+    };
+    this.fieldRules = [];
+    this.installedExtensions = [];
+    this.globalFields = [];
+    this.existingGlobalFields = [];
+    this.pendingGlobalFields = [];
+  }
 
-importContentTypes.prototype = {
-  start: function (credentialConfig) {
-    addlogs(config, 'Migrating contenttypes', 'success');
-    let self = this;
-    config = credentialConfig;
-    client = sdkInstance.Client(config);
-    stack = client.stack({ api_key: config.target_stack, management_token: config.management_token });
-    globalFieldsFolderPath = path.resolve(config.data, globalFieldConfig.dirName);
-    contentTypesFolderPath = path.resolve(config.data, contentTypeConfig.dirName);
-    mapperFolderPath = path.join(config.data, 'mapper', 'content_types');
-    globalFieldMapperFolderPath = helper.readFile(path.join(config.data, 'mapper', 'global_fields', 'success.json'));
-    globalFieldPendingPath = helper.readFile(
-      path.join(config.data, 'mapper', 'global_fields', 'pending_global_fields.js'),
-    );
-    globalFieldUpdateFile = path.join(config.data, 'mapper', 'global_fields', 'success.json');
-    fileNames = fs.readdirSync(path.join(contentTypesFolderPath));
-    self.globalfields = helper.readFile(path.resolve(globalFieldsFolderPath, globalFieldConfig.fileName));
-    for (let index in fileNames) {
-      if (skipFiles.indexOf(fileNames[index]) === -1) {
-        self.contentTypes.push(helper.readFile(path.join(contentTypesFolderPath, fileNames[index])));
+  async start() {
+    try {
+      // read content types
+      // remove content types already existing
+      if (fs.existsSync(this.existingContentTypesPath)) {
+        this.existingContentTypesUIds = fileHelper.readFileSync(this.existingContentTypesPath) || [];
+        this.existingContentTypesUIds = new Map(this.existingContentTypesUIds.map((id) => [id, 'true']));
       }
-    }
 
-    self.contentTypeUids = _.map(self.contentTypes, 'uid').filter(val => val);
-    self.createdContentTypeUids = [];
-    if (!fs.existsSync(mapperFolderPath)) {
-      mkdirp.sync(mapperFolderPath);
-    }
-    // avoid re-creating content types that already exists in the stack
-    if (fs.existsSync(path.join(mapperFolderPath, 'success.json'))) {
-      self.createdContentTypeUids = helper.readFile(path.join(mapperFolderPath, 'success.json')) || [];
-    }
-    self.contentTypeUids = _.difference(self.contentTypeUids, self.createdContentTypeUids);
-    self.uidToTitleMap = self.mapUidToTitle(self.contentTypes);
-    // remove content types, already created
-    _.remove(this.contentTypes, function (contentType) {
-      return self.contentTypeUids.indexOf(contentType.uid) === -1;
-    });
+      const contentTypeFiles = fileHelper.readdirSync(this.contentTypesFolderPath);
+      for (let contentTypeName of contentTypeFiles) {
+        if (!this.ignoredFilesInContentTypesFolder.has(contentTypeName)) {
+          const contentTypePath = path.join(this.contentTypesFolderPath, contentTypeName);
+          const contentType = await fileHelper.readFile(contentTypePath);
+          if (!this.existingContentTypesUIds?.has(contentType.uid)) {
+            this.contentTypes.push(await fileHelper.readFile(contentTypePath));
+          }
+        }
+      }
+      // this.mapUidToTitle(this.csontentTypes);
+
+      // seed content type
+      addlogs(this.importConfig, 'Started to seed content types', 'info');
+      await executeTask(this.contentTypes, this.seedContentType.bind(this), { concurrency: this.importConcurrency });
+      addlogs(this.importConfig, 'Created content types', 'success');
 
-    return new Promise(function (resolve, reject) {
-      if (self.contentTypes === undefined || _.isEmpty(self.contentTypes)) {
-        addlogs(config, chalk.yellow('No Content types found'), 'success');
-        return resolve({ empty: true })
+      // update content type
+      this.installedExtensions =
+        fileHelper.readFileSync(path.join(this.appMapperFolderPath, 'marketplace-apps.json')) || {};
+      if (isEmpty(this.installedExtensions)) {
+        this.installedExtensions = await getInstalledExtensions(this.importConfig);
       }
-      return Promise.map(
-        self.contentTypeUids,
-        function (contentTypeUid) {
-          return self
-            .seedContentTypes(contentTypeUid, self.uidToTitleMap[contentTypeUid])
-            .catch(reject);
-        },
-        {
-          // seed 3 content types at a time
-          concurrency: reqConcurrency,
-        },
-      )
-        .then(function () {
-          let batches = [];
-          let lenObj = self.contentTypes;
-          for (let i = 0; i < lenObj.length; i += Math.round(requestLimit / 3)) {
-            batches.push(lenObj.slice(i, i + Math.round(requestLimit / 3)));
-          }
 
-          return Promise.map(
-            batches,
-            async function (batch) {
-              return Promise.map(
-                batch,
-                async function (contentType) {
-                  await self.updateContentTypes(contentType)
-                  addlogs(config, contentType.uid + ' was updated successfully!', 'success');
-                },
-                {
-                  concurrency: reqConcurrency,
-                },
-              ).catch((e) => {
-                console.log('Something went wrong while migrating content type batch', e);
-              });
-            },
-            {
-              concurrency: reqConcurrency
-            }
-          ).then(async function () {
-            // eslint-disable-next-line quotes
-            if (field_rules_ct.length > 0) {
-              await fsPromises.writeFile(
-                contentTypesFolderPath + '/field_rules_uid.json',
-                JSON.stringify(field_rules_ct),
-              );
-            }
-
-            if (globalFieldPendingPath && globalFieldPendingPath.length !== 0) {
-              return self
-                .updateGlobalfields()
-                .then(function () {
-                  addlogs(config, chalk.green('Content types have been imported successfully!'), 'success');
-                  return resolve();
-                })
-                .catch((_error) => {
-                  addlogs(config, chalk.green('Error in GlobalFields'), 'success');
-                  return reject();
-                });
-            }
-            addlogs(config, chalk.green('Content types have been imported successfully!'), 'success');
-            return resolve();
-          }).catch((error) => {
-            return reject(error);
+      addlogs(this.importConfig, 'Started to update content types with references', 'info');
+      await executeTask(this.contentTypes, this.updateContentType.bind(this), { concurrency: this.importConcurrency });
+      addlogs(this.importConfig, 'Updated content types with references', 'success');
+
+      // global field update
+      this.pendingGlobalFields = fileHelper.readFileSync(this.globalFieldPendingPath);
+      if (Array.isArray(this.pendingGlobalFields) && this.pendingGlobalFields.length > 0) {
+        this.globalFields = fileHelper.readFileSync(
+          path.resolve(this.globalFieldsFolderPath, this.globalFieldConfig.fileName),
+        );
+        this.existingGlobalFields = fileHelper.readFileSync(this.globalFieldMapperFolderPath);
+        try {
+          addlogs(this.importConfig, 'Started to update pending global field with content type references', 'info');
+          await executeTask(this.pendingGlobalFields, this.updateGlobalFields.bind(this), {
+            concurrency: this.importConcurrency,
           });
-        })
-        .catch((error) => {
-          return reject(error);
-        });
-    });
-  },
-  seedContentTypes: function (uid, title) {
-    let self = this;
-    return new Promise(function (resolve, reject) {
-      let body = _.cloneDeep(self.schemaTemplate);
-      body.content_type.uid = uid;
-      body.content_type.title = title;
-      let requestObject = _.cloneDeep(self.requestOptions);
-      requestObject.json = body;
-
-      return stack
-        .contentType()
-        .create(requestObject.json)
-        .then(resolve)
-        .catch(function (err) {
-          let error = JSON.parse(err.message);
-          if (error.error_code === 115 && (error.errors.uid || error.errors.title)) {
-            // content type uid already exists
-            return resolve();
-          }
-          return reject(error);
-        });
-    });
-  },
-  updateContentTypes: function (contentType) {
-    let self = this;
-    return new Promise(function (resolve, reject) {
-      setTimeout(function () {
-        let requestObject = _.cloneDeep(self.requestOptions);
-        if (contentType.field_rules) {
-          field_rules_ct.push(contentType.uid);
-          delete contentType.field_rules;
+          addlogs(this.importConfig, 'Updated pending global fields with content type with references', 'success');
+        } catch (error) {
+          addlogs(
+            this.importConfig,
+            `Failed to updates global fields with content type reference ${formatError(error)}`,
+            'error',
+          );
         }
-        supress(contentType.schema);
-        requestObject.json.content_type = contentType;
-        let contentTypeResponse = stack.contentType(contentType.uid);
-        Object.assign(contentTypeResponse, _.cloneDeep(contentType));
-        contentTypeResponse
-          .update()
-          .then((_updatedcontentType) => {
-            return resolve();
-          })
-          .catch((err) => {
-            addlogs(config, err, 'error');
-            return reject(err);
-          });
-      }, 1000);
-    });
-  },
-
-  updateGlobalfields: function () {
-    let self = this;
-    return new Promise(function (resolve, reject) {
-      // eslint-disable-next-line no-undef
-      return Promise.map(globalFieldPendingPath, function (globalfield) {
-        let Obj = _.find(self.globalfields, { uid: globalfield });
-        supress(Obj.schema);
-        let globalFieldObj = stack.globalField(globalfield);
-        Object.assign(globalFieldObj, _.cloneDeep(Obj));
-        return globalFieldObj
-          .update()
-          .then((globalFieldResponse) => {
-            let updateObjpos = _.findIndex(globalFieldMapperFolderPath, function (successobj) {
-              let global_field_uid = globalFieldResponse.uid;
-              return global_field_uid === successobj;
-            });
-            globalFieldMapperFolderPath.splice(updateObjpos, 1, Obj);
-            helper.writeFile(globalFieldUpdateFile, globalFieldMapperFolderPath);
-
-            resolve(globalFieldResponse)
-          })
-          .catch(function (err) {
-            let error = JSON.parse(err.message);
-            // eslint-disable-next-line no-console
-            addlogs(config, chalk.red('Global Field failed to update ' + JSON.stringify(error.errors)), 'error');
-          })
-          .catch(function (error) {
-            // failed to update modified schemas back to their original form
-            return reject(error);
-          });
+      }
+
+      // write field rules
+      if (this.fieldRules.length > 0) {
+        try {
+          await fileHelper.writeFile(path.join(this.contentTypesFolderPath, 'field_rules_uid.json', this.fieldRules));
+        } catch (error) {
+          addlogs(this.importConfig, `Failed to write field rules ${formatError(error)}`, 'success');
+        }
+      }
+
+      addlogs(this.importConfig, chalk.green('Content types imported successfully'), 'success');
+    } catch (error) {
+      addlogs(this.importConfig, formatError(error), 'error');
+      throw new Error('Failed to import content types');
+    }
+  }
+
+  async seedContentType(contentType) {
+    const body = cloneDeep(schemaTemplate);
+    body.content_type.uid = contentType.uid;
+    body.content_type.title = contentType.title;
+    const requestObject = cloneDeep(this.requestOptions);
+    requestObject.json = body;
+
+    try {
+      await this.stackAPIClient.contentType().create(requestObject.json);
+    } catch (error) {
+      if (error.error_code === 115 && (error.errors.uid || error.errors.title)) {
+        // content type uid already exists
+        // _.remove(self.contentTypes, { uid: uid });
+        return true;
+      }
+      throw error;
+    }
+  }
+
+  async updateContentType(contentType) {
+    if (typeof contentType !== 'object') return;
+    const requestObject = cloneDeep(this.requestOptions);
+    if (contentType.field_rules) {
+      this.fieldRules.push(contentType.uid);
+      delete contentType.field_rules;
+    }
+
+    supress(contentType.schema, this.importConfig.preserveStackVersion, this.installedExtensions);
+    requestObject.json.content_type = contentType;
+    const contentTypeResponse = this.stackAPIClient.contentType(contentType.uid);
+    Object.assign(contentTypeResponse, cloneDeep(contentType));
+    await contentTypeResponse.update();
+    addlogs(this.importConfig, contentType.uid + ' updated with references', 'success');
+  }
+
+  async updateGlobalFields({ uid }) {
+    const globalField = find(this.globalFields, { uid });
+    supress(globalField.schema, this.importConfig.preserveStackVersion, this.installedExtensions);
+    let globalFieldObj = this.stackAPIClient.globalField(globalField);
+    Object.assign(globalFieldObj, cloneDeep(globalField));
+    try {
+      const globalFieldResponse = await globalFieldObj.update();
+      const existingGlobalField = findIndex(this.existingGlobalFields, (existingGlobalFieldUId) => {
+        return globalFieldResponse.uid === existingGlobalFieldUId;
       });
-    });
-  },
 
-  mapUidToTitle: function (contentTypes) {
-    let result = {};
-    contentTypes.forEach((ct) => {
-      result[ct.uid] = ct.title;
+      // Improve write the updated global fields once all updates are completed
+      this.existingGlobalFields.splice(existingGlobalField, 1, globalField);
+      await fileHelper.writeFile(this.globalFieldMapperFolderPath, this.existingGlobalFields).catch((error) => {
+        addlogs(this.importConfig, `failed to write updated the global field ${uid} ${formatError(error)}`);
+      });
+      addlogs(this.importConfig, `Updated the global field ${uid} with content type references `);
+      return true;
+    } catch (error) {
+      addlogs(this.importConfig, `failed to update the global field ${uid} ${formatError(error)}`);
+    }
+  }
+
+  async mapUidToTitle() {
+    this.contentTypes.forEach((ct) => {
+      this.titleToUIdMap.set(ct.uid, ct.title);
     });
-    return result;
-  },
-};
+  }
+}
 
-module.exports = new importContentTypes();
+module.exports = ContentTypesImport;

package/src/lib/import/custom-roles.js

@@ -0,0 +1,176 @@
+'use strict';
+
+const mkdirp = require('mkdirp');
+const fs = require('fs');
+const path = require('path');
+const chalk = require('chalk');
+const { merge } = require('lodash');
+
+const helper = require('../util/fs');
+const { addlogs } = require('../util/log');
+const { formatError } = require('../util');
+let config = require('../../config/default');
+const stack = require('../util/contentstack-management-sdk');
+
+module.exports = class ImportCustomRoles {
+  fails = [];
+  client = null;
+  labelUids = [];
+  customRolesUidMapper = {};
+  customRolesConfig = config.modules.customRoles;
+
+  constructor(credentialConfig) {
+    this.config = merge(config, credentialConfig);
+    this.client = stack.Client(this.config);
+  }
+
+  async start() {
+    const self = this;
+    addlogs(this.config, chalk.white('Migrating custom-roles'), 'success');
+
+    let customRolesFolderPath = path.resolve(this.config.data, this.customRolesConfig.dirName);
+    let customRolesMapperPath = path.resolve(this.config.data, 'mapper', 'custom-roles');
+    let entriesUidMapperFolderPath = path.resolve(this.config.data, 'mapper', 'entries');
+    let customRolesFailsPath = path.resolve(this.config.data, 'custom-roles', 'fails.json');
+    let environmentsUidMapperFolderPath = path.resolve(this.config.data, 'mapper', 'environments');
+    let customRolesUidMapperPath = path.resolve(this.config.data, 'mapper', 'custom-roles', 'uid-mapping.json');
+    let customRolesLocalesFilePath = path.resolve(
+      customRolesFolderPath,
+      this.customRolesConfig.customRolesLocalesFileName,
+    );
+
+    try {
+      self.customRoles = helper.readFileSync(path.resolve(customRolesFolderPath, this.customRolesConfig.fileName));
+      self.customRolesLocales = helper.readFileSync(customRolesLocalesFilePath);
+      // Mapper file paths.
+
+      if (fs.existsSync(customRolesMapperPath)) {
+        this.customRolesUidMapper = helper.readFileSync(customRolesUidMapperPath) || {};
+      }
+
+      mkdirp.sync(customRolesMapperPath);
+
+      if (!self.customRoles) {
+        addlogs(self.config, chalk.white('No custom-roles found'), 'error');
+        return;
+      }
+      self.customRolesUids = Object.keys(self.customRoles);
+
+      self.localesUidMap = await getLocalesUidMap(self.client, self.config, self.customRolesLocales);
+
+      if (fs.existsSync(environmentsUidMapperFolderPath)) {
+        self.environmentsUidMap = helper.readFileSync(
+          path.resolve(environmentsUidMapperFolderPath, 'uid-mapping.json'),
+        );
+      }
+      if (fs.existsSync(entriesUidMapperFolderPath)) {
+        self.entriesUidMap = helper.readFileSync(path.resolve(entriesUidMapperFolderPath, 'uid-mapping.json'));
+      }
+
+      for (const uid of self.customRolesUids) {
+        const customRole = self.customRoles[uid];
+
+        if (uid in self.customRolesUidMapper) {
+          addlogs(
+            self.config,
+            chalk.white(`The custom-role ${customRole.name} already exists. Skipping it to avoid duplicates!`),
+            'success',
+          );
+          continue;
+        }
+
+        try {
+          customRole.rules.forEach((rule) => {
+            const transformUids = getTransformUidsFactory(rule);
+            rule = transformUids(rule, self.environmentsUidMap, self.localesUidMap, self.entriesUidMap);
+          });
+          // rules.branch is required to create custom roles.
+          const branchRuleExists = customRole.rules.find((rule) => rule.module === 'branch');
+          if (!branchRuleExists) {
+            customRole.rules.push({
+              module: 'branch',
+              branches: ['main'],
+              acl: { read: true },
+            });
+          }
+          const role = await self.client
+            .stack({ api_key: self.config.target_stack, management_token: self.config.management_token })
+            .role()
+            .create({ role: customRole });
+
+          self.customRolesUidMapper[uid] = role;
+          helper.writeFileSync(customRolesUidMapperPath, self.customRolesUidMapper);
+        } catch (error) {
+          self.fails.push(customRole);
+
+          if (error && error.errors && error.errors.name) {
+            addlogs(self.config, chalk.red(`custom-role: ${customRole.name} already exists`), 'error');
+          } else {
+            addlogs(self.config, chalk.red(`custom-role: ${customRole.name} failed`), 'error');
+          }
+
+          addlogs(self.self.config, formatError(error), 'error');
+        }
+      }
+      addlogs(self.config, chalk.green('Custom-roles have been imported successfully!'), 'success');
+    } catch (error) {
+      helper.writeFileSync(customRolesFailsPath, self.fails);
+      addlogs(self.config, chalk.red('Custom-roles import failed'), 'error');
+      addlogs(self.config, formatError(error), 'error');
+
+      throw error;
+    }
+  }
+};
+
+const getTransformUidsFactory = (rule) => {
+  if (rule.module === 'environment') {
+    return environmentUidTransformer;
+  } else if (rule.module === 'locale') {
+    return localeUidTransformer;
+  } else if (rule.module === 'entry') {
+    return entryUidTransformer;
+  } else {
+    return noopTransformer;
+  }
+};
+
+const environmentUidTransformer = (rule, environmentsUidMap) => {
+  rule.environments = rule.environments.map((env) => environmentsUidMap[env]);
+  return rule;
+};
+
+const localeUidTransformer = (rule, environmentsUidMap, localesUidMap) => {
+  rule.locales = rule.locales.map((locale) => localesUidMap[locale]);
+  return rule;
+};
+
+const entryUidTransformer = (rule, environmentsUidMap, localesUidMap, entriesUidMap) => {
+  rule.entries = rule.entries.map((entry) => entriesUidMap[entry]);
+  return rule;
+};
+
+const noopTransformer = (rule) => {
+  return rule;
+};
+
+const getLocalesUidMap = async (client, config, sourceLocales) => {
+  const { items } = await client
+    .stack({ api_key: config.target_stack, management_token: config.management_token })
+    .locale()
+    .query()
+    .find();
+  const [targetLocalesMap, sourceLocalesMap] = [{}, {}];
+
+  items.forEach((locale) => {
+    targetLocalesMap[locale.code] = locale.uid;
+  });
+  for (const key in sourceLocales) {
+    sourceLocalesMap[sourceLocales[key].code] = key;
+  }
+  const localesUidMap = {};
+  for (const key in sourceLocalesMap) {
+    localesUidMap[sourceLocalesMap[key]] = targetLocalesMap[key];
+  }
+  return localesUidMap;
+};