@contentstack/cli-cm-import 1.5.11 → 1.6.0

package/lib/import/modules-js/content-types.js

@@ -0,0 +1,176 @@
+const fs = require('fs');
+const path = require('path');
+const chalk = require('chalk');
+const { cloneDeep, find, findIndex } = require('lodash');
+const { fileHelper, log, executeTask, formatError, schemaTemplate, lookupExtension } = require('../../utils');
+class ContentTypesImport {
+    constructor(importConfig, stackAPIClient) {
+        this.stackAPIClient = stackAPIClient;
+        this.importConfig = importConfig;
+        this.contentTypeConfig = importConfig.modules.content_types;
+        this.globalFieldConfig = importConfig.modules.globalfields;
+        this.importConcurrency = this.contentTypeConfig.importConcurrency || this.importConfig.importConcurrency;
+        this.writeConcurrency = this.contentTypeConfig.writeConcurrency || this.importConfig.writeConcurrency;
+        this.contentTypesFolderPath = path.join(this.importConfig.data, this.contentTypeConfig.dirName);
+        this.mapperFolderPath = path.join(this.importConfig.data, 'mapper', 'content_types');
+        this.existingContentTypesPath = path.join(this.mapperFolderPath, 'success.json');
+        this.globalFieldsFolderPath = path.resolve(this.importConfig.data, this.globalFieldConfig.dirName);
+        this.globalFieldMapperFolderPath = path.join(importConfig.data, 'mapper', 'global_fields', 'success.json');
+        this.globalFieldPendingPath = path.join(importConfig.data, 'mapper', 'global_fields', 'pending_global_fields.js');
+        this.ignoredFilesInContentTypesFolder = new Map([
+            ['__master.json', 'true'],
+            ['__priority.json', 'true'],
+            ['schema.json', 'true'],
+            ['.DS_Store', 'true'],
+        ]);
+        this.contentTypes = [];
+        this.existingContentTypesUIds = [];
+        this.titleToUIdMap = new Map();
+        this.requestOptions = {
+            json: {},
+        };
+        this.fieldRules = [];
+        this.installedExtensions = [];
+        this.globalFields = [];
+        this.existingGlobalFields = [];
+        this.pendingGlobalFields = [];
+    }
+    async start() {
+        try {
+            const appMapperPath = path.join(this.importConfig.data, 'mapper', 'marketplace_apps', 'uid-mapping.json');
+            this.installedExtensions = ((await fileHelper.readFileSync(appMapperPath)) || { extension_uid: {} }).extension_uid;
+            // read content types
+            // remove content types already existing
+            if (fs.existsSync(this.existingContentTypesPath)) {
+                this.existingContentTypesUIds = fileHelper.readFileSync(this.existingContentTypesPath) || [];
+                this.existingContentTypesUIds = new Map(this.existingContentTypesUIds.map((id) => [id, 'true']));
+            }
+            const contentTypeFiles = fileHelper.readdirSync(this.contentTypesFolderPath);
+            for (let contentTypeName of contentTypeFiles) {
+                if (!this.ignoredFilesInContentTypesFolder.has(contentTypeName)) {
+                    const contentTypePath = path.join(this.contentTypesFolderPath, contentTypeName);
+                    const contentType = await fileHelper.readFile(contentTypePath);
+                    if (!this.existingContentTypesUIds.length || !this.existingContentTypesUIds.has(contentType.uid)) {
+                        this.contentTypes.push(await fileHelper.readFile(contentTypePath));
+                    }
+                }
+            }
+            // seed content type
+            log(this.importConfig, 'Started to seed content types', 'info');
+            await executeTask(this.contentTypes, this.seedContentType.bind(this), { concurrency: this.importConcurrency });
+            log(this.importConfig, 'Created content types', 'success');
+            log(this.importConfig, 'Started to update content types with references', 'info');
+            await executeTask(this.contentTypes, this.updateContentType.bind(this), { concurrency: this.importConcurrency });
+            log(this.importConfig, 'Updated content types with references', 'success');
+            // global field update
+            this.pendingGlobalFields = fileHelper.readFileSync(this.globalFieldPendingPath);
+            if (Array.isArray(this.pendingGlobalFields) && this.pendingGlobalFields.length > 0) {
+                this.globalFields = fileHelper.readFileSync(path.resolve(this.globalFieldsFolderPath, this.globalFieldConfig.fileName));
+                this.existingGlobalFields = fileHelper.readFileSync(this.globalFieldMapperFolderPath);
+                try {
+                    log(this.importConfig, 'Started to update pending global field with content type references', 'info');
+                    await executeTask(this.pendingGlobalFields, this.updateGlobalFields.bind(this), {
+                        concurrency: this.importConcurrency,
+                    });
+                    log(this.importConfig, 'Updated pending global fields with content type with references', 'success');
+                }
+                catch (error) {
+                    log(this.importConfig, `Failed to updates global fields with content type reference ${formatError(error)}`, 'error');
+                }
+            }
+            // write field rules
+            if (this.fieldRules.length > 0) {
+                try {
+                    await fileHelper.writeFile(path.join(this.contentTypesFolderPath, 'field_rules_uid.json'), this.fieldRules);
+                }
+                catch (error) {
+                    log(this.importConfig, `Failed to write field rules ${formatError(error)}`, 'success');
+                }
+            }
+            log(this.importConfig, chalk.green('Content types imported successfully'), 'success');
+        }
+        catch (error) {
+            let message_content_type = "";
+            if (error.request !== undefined && JSON.parse(error.request.data).content_type !== undefined) {
+                if (JSON.parse(error.request.data).content_type.uid) {
+                    message_content_type =
+                        ' Update the content type with content_type_uid  - ' + JSON.parse(error.request.data).content_type.uid;
+                }
+                else if (JSON.parse(error.request.data).content_type.title) {
+                    message_content_type =
+                        ' Update the content type with content_type_title  - ' + JSON.parse(error.request.data).content_type.title;
+                }
+                error.errorMessage = error.errorMessage + message_content_type;
+            }
+            log(this.importConfig, formatError(error.errorMessage), 'error');
+            log(this.importConfig, formatError(error), 'error');
+            throw new Error('Failed to import content types');
+        }
+    }
+    async seedContentType(contentType) {
+        const body = cloneDeep(schemaTemplate);
+        body.content_type.uid = contentType.uid;
+        body.content_type.title = contentType.title;
+        const requestObject = cloneDeep(this.requestOptions);
+        requestObject.json = body;
+        try {
+            await this.stackAPIClient.contentType().create(requestObject.json);
+        }
+        catch (error) {
+            if (error.errorCode === 115 && (error.errors.uid || error.errors.title)) {
+                // content type uid already exists
+                // _.remove(self.contentTypes, { uid: uid });
+                return true;
+            }
+            throw error;
+        }
+    }
+    async updateContentType(contentType) {
+        if (typeof contentType !== 'object')
+            return;
+        const requestObject = cloneDeep(this.requestOptions);
+        if (contentType.field_rules) {
+            this.fieldRules.push(contentType.uid);
+            delete contentType.field_rules;
+        }
+        lookupExtension(this.importConfig, contentType.schema, this.importConfig.preserveStackVersion, this.installedExtensions);
+        requestObject.json.content_type = contentType;
+        const contentTypeResponse = this.stackAPIClient.contentType(contentType.uid);
+        Object.assign(contentTypeResponse, cloneDeep(contentType));
+        await contentTypeResponse.update();
+        log(this.importConfig, contentType.uid + ' updated with references', 'success');
+    }
+    async updateGlobalFields(uid) {
+        const globalField = find(this.globalFields, { uid });
+        if (globalField) {
+            lookupExtension(this.importConfig, globalField.schema, this.importConfig.preserveStackVersion, this.installedExtensions);
+            let globalFieldObj = this.stackAPIClient.globalField(globalField.uid);
+            Object.assign(globalFieldObj, cloneDeep(globalField));
+            try {
+                const globalFieldResponse = await globalFieldObj.update();
+                const existingGlobalField = findIndex(this.existingGlobalFields, (existingGlobalFieldUId) => {
+                    return globalFieldResponse.uid === existingGlobalFieldUId;
+                });
+                // Improve write the updated global fields once all updates are completed
+                this.existingGlobalFields.splice(existingGlobalField, 1, globalField);
+                await fileHelper.writeFile(this.globalFieldMapperFolderPath, this.existingGlobalFields).catch((error) => {
+                    log(this.importConfig, `failed to write updated the global field ${uid} ${formatError(error)}`);
+                });
+                log(this.importConfig, `Updated the global field ${uid} with content type references `);
+                return true;
+            }
+            catch (error) {
+                log(this.importConfig, `failed to update the global field ${uid} ${formatError(error)}`);
+            }
+        }
+        else {
+            log(this.importConfig, `Global field ${uid} does not exist, and hence failed to update.`);
+        }
+    }
+    async mapUidToTitle() {
+        this.contentTypes.forEach((ct) => {
+            this.titleToUIdMap.set(ct.uid, ct.title);
+        });
+    }
+}
+module.exports = ContentTypesImport;

package/lib/import/modules-js/custom-roles.d.ts

@@ -0,0 +1,15 @@
+export = ImportCustomRoles;
+declare class ImportCustomRoles {
+    constructor(importConfig: any, stackAPIClient: any);
+    fails: any[];
+    labelUids: any[];
+    customRolesUidMapper: {};
+    customRolesConfig: {
+        dirName: string;
+        fileName: string;
+        customRolesLocalesFileName: string;
+    };
+    config: any;
+    stackAPIClient: any;
+    start(): Promise<void>;
+}

package/lib/import/modules-js/custom-roles.js

@@ -0,0 +1,141 @@
+'use strict';
+const mkdirp = require('mkdirp');
+const fs = require('fs');
+const path = require('path');
+const chalk = require('chalk');
+const { merge } = require('lodash');
+const { fileHelper, log, formatError } = require('../../utils');
+let { default: config } = require('../../config');
+module.exports = class ImportCustomRoles {
+    constructor(importConfig, stackAPIClient) {
+        this.fails = [];
+        this.labelUids = [];
+        this.customRolesUidMapper = {};
+        this.customRolesConfig = config.modules.customRoles;
+        this.config = merge(config, importConfig);
+        this.stackAPIClient = stackAPIClient;
+    }
+    async start() {
+        const self = this;
+        log(this.config, chalk.white('Migrating custom-roles'), 'success');
+        let customRolesFolderPath = path.resolve(this.config.data, this.customRolesConfig.dirName);
+        let customRolesMapperPath = path.resolve(this.config.data, 'mapper', 'custom-roles');
+        let entriesUidMapperFolderPath = path.resolve(this.config.data, 'mapper', 'entries');
+        let customRolesFailsPath = path.resolve(this.config.data, 'custom-roles', 'fails.json');
+        let environmentsUidMapperFolderPath = path.resolve(this.config.data, 'mapper', 'environments');
+        let customRolesUidMapperPath = path.resolve(this.config.data, 'mapper', 'custom-roles', 'uid-mapping.json');
+        let customRolesLocalesFilePath = path.resolve(customRolesFolderPath, this.customRolesConfig.customRolesLocalesFileName);
+        try {
+            self.customRoles = fileHelper.readFileSync(path.resolve(customRolesFolderPath, this.customRolesConfig.fileName));
+            self.customRolesLocales = fileHelper.readFileSync(customRolesLocalesFilePath);
+            // Mapper file paths.
+            if (fs.existsSync(customRolesMapperPath)) {
+                this.customRolesUidMapper = fileHelper.readFileSync(customRolesUidMapperPath) || {};
+            }
+            mkdirp.sync(customRolesMapperPath);
+            if (!self.customRoles) {
+                log(self.config, chalk.white('No custom-roles found'), 'info');
+                return;
+            }
+            self.customRolesUids = Object.keys(self.customRoles);
+            self.localesUidMap = await getLocalesUidMap(self.stackAPIClient, self.config, self.customRolesLocales);
+            if (fs.existsSync(environmentsUidMapperFolderPath)) {
+                self.environmentsUidMap = fileHelper.readFileSync(path.resolve(environmentsUidMapperFolderPath, 'uid-mapping.json'));
+            }
+            if (fs.existsSync(entriesUidMapperFolderPath)) {
+                self.entriesUidMap = fileHelper.readFileSync(path.resolve(entriesUidMapperFolderPath, 'uid-mapping.json'));
+            }
+            for (const uid of self.customRolesUids) {
+                const customRole = self.customRoles[uid];
+                if (uid in self.customRolesUidMapper) {
+                    log(self.config, chalk.white(`The custom-role '${customRole.name}' already exists. Skipping it to avoid duplicates!`), 'success');
+                    continue;
+                }
+                try {
+                    customRole.rules.forEach((rule) => {
+                        const transformUids = getTransformUidsFactory(rule);
+                        rule = transformUids(rule, self.environmentsUidMap, self.localesUidMap, self.entriesUidMap);
+                    });
+                    // rules.branch is required to create custom roles.
+                    const branchRuleExists = customRole.rules.find((rule) => rule.module === 'branch');
+                    if (!branchRuleExists) {
+                        customRole.rules.push({
+                            module: 'branch',
+                            branches: ['main'],
+                            acl: { read: true },
+                        });
+                    }
+                    const role = await self.stackAPIClient.role().create({ role: customRole });
+                    self.customRolesUidMapper[uid] = role;
+                    fileHelper.writeFileSync(customRolesUidMapperPath, self.customRolesUidMapper);
+                }
+                catch (error) {
+                    self.fails.push(customRole);
+                    if (((error && error.errors && error.errors.name) || '').includes('is not a unique.')) {
+                        log(self.config, `custom-role ${customRole.name} already exists`, 'info');
+                    }
+                    else {
+                        if (!(error && error.errors && error.errors.name)) {
+                            log(self.config, `custom-role: ${customRole.name} already exists`, 'error');
+                        }
+                        else {
+                            log(self.config, `custom-role: ${customRole.name} failed`, 'error');
+                        }
+                        log(self.config, formatError(error), 'error');
+                    }
+                }
+            }
+            log(self.config, chalk.green('Custom-roles have been imported successfully!'), 'success');
+        }
+        catch (error) {
+            fileHelper.writeFileSync(customRolesFailsPath, self.fails);
+            log(self.config, 'Custom-roles import failed', 'error');
+            log(self.config, formatError(error), 'error');
+            throw error;
+        }
+    }
+};
+const getTransformUidsFactory = (rule) => {
+    if (rule.module === 'environment') {
+        return environmentUidTransformer;
+    }
+    else if (rule.module === 'locale') {
+        return localeUidTransformer;
+    }
+    else if (rule.module === 'entry') {
+        return entryUidTransformer;
+    }
+    else {
+        return noopTransformer;
+    }
+};
+const environmentUidTransformer = (rule, environmentsUidMap) => {
+    rule.environments = rule.environments.map((env) => environmentsUidMap[env]);
+    return rule;
+};
+const localeUidTransformer = (rule, environmentsUidMap, localesUidMap) => {
+    rule.locales = rule.locales.map((locale) => localesUidMap[locale]);
+    return rule;
+};
+const entryUidTransformer = (rule, environmentsUidMap, localesUidMap, entriesUidMap) => {
+    rule.entries = rule.entries.map((entry) => entriesUidMap[entry]);
+    return rule;
+};
+const noopTransformer = (rule) => {
+    return rule;
+};
+const getLocalesUidMap = async (stackAPIClient, config, sourceLocales) => {
+    const { items } = await stackAPIClient.locale().query().find();
+    const [targetLocalesMap, sourceLocalesMap] = [{}, {}];
+    items.forEach((locale) => {
+        targetLocalesMap[locale.code] = locale.uid;
+    });
+    for (const key in sourceLocales) {
+        sourceLocalesMap[sourceLocales[key].code] = key;
+    }
+    const localesUidMap = {};
+    for (const key in sourceLocalesMap) {
+        localesUidMap[sourceLocalesMap[key]] = targetLocalesMap[key];
+    }
+    return localesUidMap;
+};

package/lib/import/modules-js/entries.d.ts

@@ -0,0 +1,54 @@
+export = ImportEntries;
+declare class ImportEntries {
+    constructor(importConfig: any, stackAPIClient: any);
+    mappedAssetUidPath: string;
+    mappedAssetUrlPath: string;
+    entryMapperPath: string;
+    environmentPath: string;
+    entryUidMapperPath: string;
+    uniqueUidMapperPath: string;
+    modifiedSchemaPath: string;
+    createdEntriesWOUidPath: string;
+    failedWOPath: string;
+    masterLanguage: any;
+    reqConcurrency: any;
+    eConfig: any;
+    ePath: string;
+    ctPath: string;
+    lPath: string;
+    importConcurrency: any;
+    skipFiles: string[];
+    config: any;
+    stackAPIClient: any;
+    ctSchemas: {};
+    refSchemas: any[];
+    ctJsonRte: any[];
+    ctJsonRteWithEntryRefs: any[];
+    jsonRteEntryRefs: {};
+    createdEntriesWOUid: any[];
+    uniqueUids: {};
+    mappedUids: {};
+    success: any[];
+    fails: any[];
+    installedExtensions: any[];
+    environment: any;
+    start(): globalThis.Promise<any>;
+    createEntries(lang: any, mappedAssetUids: any, mappedAssetUrls: any): Promise<any>;
+    getCreatedEntriesWOUid(): Promise<any>;
+    repostEntries(lang: any): Promise<any>;
+    supressFields(): Promise<any>;
+    fetchEntry(query: any): Promise<any>;
+    unSuppressFields(): Promise<any>;
+    removeBuggedEntries(): Promise<any>;
+    field_rules_update(schema: any): Promise<any>;
+    publish(langs: any): Promise<any>;
+    removeEntryRefsFromJSONRTE(entry: any, ctSchema: any): any;
+    doEntryReferencesExist(element: any): any;
+    restoreJsonRteEntryRefs(entry: any, sourceStackEntry: any, ctSchema: any): any;
+    isEntryRef(element: any): boolean;
+    removeUidsFromJsonRteFields(entry: any, ctSchema: any): any;
+    removeUidsFromChildren(children: any): any;
+    setDirtyTrue(jsonRteChild: any): any;
+    resolveAssetRefsInEntryRefsForJsonRte(jsonRteChild: any, mappedAssetUids: any, mappedAssetUrls: any): any;
+}
+import Promise = require("bluebird");