@contentstack/cli-cm-export 1.5.8 → 1.6.1

package/lib/export/modules-js/entries.js

@@ -0,0 +1,141 @@
+const path = require('path');
+const chalk = require('chalk');
+const { executeTask, formatError, fileHelper, log } = require('../../utils');
+class EntriesExport {
+    constructor(exportConfig, stackAPIClient) {
+        this.stackAPIClient = stackAPIClient;
+        this.exportConfig = exportConfig;
+        this.entriesConfig = exportConfig.modules.entries;
+        this.entriesRootPath = path.resolve(exportConfig.data, exportConfig.branchName || '', this.entriesConfig.dirName);
+        this.localesFilePath = path.resolve(exportConfig.data, exportConfig.branchName || '', exportConfig.modules.locales.dirName, exportConfig.modules.locales.fileName);
+        this.schemaFilePath = path.resolve(exportConfig.data, exportConfig.branchName || '', exportConfig.modules.content_types.dirName, 'schema.json');
+        this.fetchConcurrency = this.entriesConfig.fetchConcurrency || exportConfig.fetchConcurrency;
+        this.writeConcurrency = this.entriesConfig.writeConcurrency || exportConfig.writeConcurrency;
+    }
+    async start() {
+        try {
+            log(this.exportConfig, 'Starting entries export', 'info');
+            const locales = await fileHelper.readFile(this.localesFilePath);
+            const contentTypes = await fileHelper.readFile(this.schemaFilePath);
+            if (contentTypes.length === 0) {
+                log(this.exportConfig, 'No content types found to export entries', 'info');
+                return;
+            }
+            const entryRequestOptions = this.createRequestObjects(locales, contentTypes);
+            for (let requestOption of entryRequestOptions) {
+                log(this.exportConfig, `Starting export of entries of content_type - ${requestOption.content_type} locale - ${requestOption.locale}`, 'info');
+                await fileHelper.makeDirectory(path.join(this.entriesRootPath, requestOption.content_type));
+                const entries = await this.getEntries(requestOption);
+                let entriesFilePath = path.join(this.entriesRootPath, requestOption.content_type, requestOption.locale + '.json');
+                await fileHelper.writeLargeFile(entriesFilePath, entries);
+                log(this.exportConfig, `Exported entries of type '${requestOption.content_type}' locale '${requestOption.locale}'`, 'success');
+                if (this.exportConfig.versioning) {
+                    log(this.exportConfig, `Started export versioned entries of type '${requestOption.content_type}' locale '${requestOption.locale}'`, 'info');
+                    for (let entry of entries) {
+                        const versionedEntries = await this.getEntryByVersion(Object.assign(Object.assign({}, requestOption), { uid: entry.uid }), entry._version);
+                        let versionedEntryPath = path.join(this.entriesRootPath, requestOption.locale, requestOption.content_type, entry.uid);
+                        await fileHelper.makeDirectory(versionedEntryPath);
+                        if (versionedEntries.length > 0) {
+                            const write = (versionedEntry) => fileHelper.writeFile(path.join(versionedEntryPath, 'version-' + versionedEntry._version + '.json'), versionedEntry);
+                            await executeTask(versionedEntries, write.bind(this), { concurrency: this.writeConcurrency });
+                            log(this.exportConfig, `Exported versioned entries of type '${requestOption.content_type}' locale '${requestOption.locale}'`, 'success');
+                        }
+                    }
+                }
+            }
+            log(this.exportConfig, chalk.green('Entries exported successfully'), 'success');
+        }
+        catch (error) {
+            log(this.exportConfig, `Failed to export entries ${formatError(error)}`, 'error');
+            throw new Error('Failed to export entries');
+        }
+    }
+    async getEntries(requestOptions, skip = 0, entries = {}) {
+        let requestObject = {
+            locale: requestOptions.locale,
+            skip,
+            limit: this.entriesConfig.limit,
+            include_count: true,
+            include_publish_details: true,
+            query: {
+                locale: requestOptions.locale,
+            },
+        };
+        const entriesSearchResponse = await this.stackAPIClient
+            .contentType(requestOptions.content_type)
+            .entry()
+            .query(requestObject)
+            .find();
+        if (Array.isArray(entriesSearchResponse.items) && entriesSearchResponse.items.length > 0) {
+            // clean up attribs and add to parent entry list
+            this.sanitizeAttribs(entriesSearchResponse.items, entries);
+            skip += this.entriesConfig.limit || 100;
+            if (skip > entriesSearchResponse.count) {
+                return entries;
+            }
+            return await this.getEntries(requestOptions, skip, entries);
+        }
+        return entries;
+    }
+    async getEntryByVersion(requestOptions, version, entries = []) {
+        const queryRequestObject = {
+            locale: requestOptions.locale,
+            except: {
+                BASE: this.entriesConfig.invalidKeys,
+            },
+            version,
+        };
+        const entryResponse = await this.stackAPIClient
+            .contentType(requestOptions.content_type)
+            .entry(requestOptions.uid)
+            .fetch(queryRequestObject);
+        entries.push(entryResponse);
+        if (--version > 0) {
+            return await this.getEntryByVersion(requestOptions, version, entries);
+        }
+        return entries;
+    }
+    async getEntriesCount(requestOptions) {
+        let requestObject = {
+            locale: requestOptions.locale,
+            limit: 1,
+            include_count: true,
+            include_publish_details: true,
+            query: {
+                locale: requestOptions.locale,
+            },
+        };
+        const entriesSearchResponse = await this.stackAPIClient
+            .contentType(requestOptions.content_type)
+            .entry()
+            .query(requestObject)
+            .find();
+        return entriesSearchResponse.count;
+    }
+    createRequestObjects(locales, contentTypes) {
+        let requestObjects = [];
+        contentTypes.forEach((contentType) => {
+            if (Object.keys(locales).length !== 0) {
+                for (let locale in locales) {
+                    requestObjects.push({
+                        content_type: contentType.uid,
+                        locale: locales[locale].code,
+                    });
+                }
+            }
+            requestObjects.push({
+                content_type: contentType.uid,
+                locale: this.exportConfig.master_locale.code,
+            });
+        });
+        return requestObjects;
+    }
+    sanitizeAttribs(entries, entriesList = {}) {
+        entries.forEach((entry) => {
+            this.entriesConfig.invalidKeys.forEach((key) => delete entry[key]);
+            entriesList[entry.uid] = entry;
+        });
+        return entriesList;
+    }
+}
+module.exports = EntriesExport;

package/lib/export/modules-js/environments.d.ts

@@ -0,0 +1,16 @@
+export = ExportEnvironments;
+declare class ExportEnvironments {
+    constructor(exportConfig: any, stackAPIClient: any);
+    config: {};
+    master: {};
+    environments: {};
+    requestOptions: {
+        json: boolean;
+        qs: {
+            asc: string;
+            include_count: boolean;
+        };
+    };
+    stackAPIClient: any;
+    start(): Promise<any>;
+}

package/lib/export/modules-js/environments.js

@@ -0,0 +1,62 @@
+/*!
+ * Contentstack Export
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
+const path = require('path');
+const chalk = require('chalk');
+const mkdirp = require('mkdirp');
+const { merge } = require('lodash');
+const { fileHelper, log, formatError } = require('../../utils');
+module.exports = class ExportEnvironments {
+    constructor(exportConfig, stackAPIClient) {
+        this.config = {};
+        this.master = {};
+        this.environments = {};
+        this.requestOptions = {
+            json: true,
+            qs: {
+                asc: 'updated_at',
+                include_count: true,
+            },
+        };
+        this.config = merge(this.config, exportConfig);
+        this.stackAPIClient = stackAPIClient;
+    }
+    start() {
+        const self = this;
+        const environmentConfig = self.config.modules.environments;
+        const environmentsFolderPath = path.resolve(self.config.data, self.config.branchName || '', environmentConfig.dirName);
+        // Create folder for environments
+        fileHelper.makeDirectory(environmentsFolderPath);
+        log(this.config, 'Starting environment export', 'success');
+        return new Promise(function (resolve, reject) {
+            self.stackAPIClient
+                .environment()
+                .query(self.requestOptions.qs)
+                .find()
+                .then((environmentResponse) => {
+                if (environmentResponse.items.length !== 0) {
+                    for (let i = 0, total = environmentResponse.count; i < total; i++) {
+                        const envUid = environmentResponse.items[i].uid;
+                        self.master[envUid] = '';
+                        self.environments[envUid] = environmentResponse.items[i];
+                        delete self.environments[envUid].uid;
+                        delete self.environments[envUid]['ACL'];
+                    }
+                    fileHelper.writeFileSync(path.join(environmentsFolderPath, environmentConfig.fileName), self.environments);
+                    log(self.config, chalk.green('All the environments have been exported successfully'), 'success');
+                    return resolve();
+                }
+                if (environmentResponse.items.length === 0) {
+                    log(self.config, 'No environments found', 'success');
+                    resolve();
+                }
+            })
+                .catch((error) => {
+                log(self.config, `Environments export failed. ${formatError(error)}`, 'error');
+                reject(error);
+            });
+        });
+    }
+};

package/lib/export/modules-js/extensions.d.ts

@@ -0,0 +1,18 @@
+export = ExportExtensions;
+declare class ExportExtensions {
+    constructor(exportConfig: any, stackAPIClient: any);
+    master: {};
+    extensions: {};
+    extensionConfig: {
+        dirName: string;
+        fileName: string;
+        dependencies?: import("../../types").Modules[];
+    };
+    queryRequestOptions: {
+        asc: string;
+        include_count: boolean;
+    };
+    config: any;
+    stackAPIClient: any;
+    start(): Promise<any>;
+}

package/lib/export/modules-js/extensions.js

@@ -0,0 +1,57 @@
+/*!
+ * Contentstack Export
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
+const mkdirp = require('mkdirp');
+const path = require('path');
+const chalk = require('chalk');
+const { merge } = require('lodash');
+const { formatError, log, fileHelper } = require('../../utils');
+const { default: config } = require('../../config');
+module.exports = class ExportExtensions {
+    constructor(exportConfig, stackAPIClient) {
+        this.master = {};
+        this.extensions = {};
+        this.extensionConfig = config.modules.extensions;
+        this.queryRequestOptions = {
+            asc: 'updated_at',
+            include_count: true,
+        };
+        this.config = merge(config, exportConfig);
+        this.stackAPIClient = stackAPIClient;
+    }
+    start() {
+        log(this.config, 'Starting extension export', 'success');
+        const self = this;
+        const extensionsFolderPath = path.resolve(this.config.data, this.config.branchName || '', this.extensionConfig.dirName);
+        // Create folder for extensions
+        mkdirp.sync(extensionsFolderPath);
+        return new Promise(function (resolve, reject) {
+            self.stackAPIClient
+                .extension()
+                .query(self.queryRequestOptions)
+                .find()
+                .then((extension) => {
+                if (extension.items.length !== 0) {
+                    for (let i = 0, total = extension.count; i < total; i++) {
+                        const extUid = extension.items[i].uid;
+                        self.master[extUid] = '';
+                        self.extensions[extUid] = extension.items[i];
+                        delete self.extensions[extUid].uid;
+                        delete self.extensions[extUid].SYS_ACL;
+                    }
+                    fileHelper.writeFileSync(path.join(extensionsFolderPath, self.extensionConfig.fileName), self.extensions);
+                    log(self.config, chalk.green('All the extensions have been exported successfully'), 'success');
+                    return resolve();
+                }
+                log(self.config, 'No extensions found', 'success');
+                resolve();
+            })
+                .catch((error) => {
+                log(self.config, `Failed to export extensions. ${formatError(error)}`, 'error');
+                reject();
+            });
+        });
+    }
+};

package/lib/export/modules-js/global-fields.d.ts

@@ -0,0 +1,22 @@
+export = ExportGlobalFields;
+declare class ExportGlobalFields {
+    constructor(exportConfig: any, stackAPIClient: any);
+    limit: number;
+    config: {};
+    global_fields: any[];
+    master: {};
+    globalfields: {};
+    requestOptions: {};
+    globalfieldsFolderPath: string;
+    globalfieldsConfig: {
+        dirName: string;
+        fileName: string;
+        validKeys: string[];
+        dependencies?: import("../../types").Modules[];
+    };
+    validKeys: string[];
+    stackAPIClient: any;
+    start(): Promise<any>;
+    getGlobalFields(skip: any, globalFieldConfig: any): Promise<any>;
+    writeGlobalFields(): Promise<any>;
+}

package/lib/export/modules-js/global-fields.js

@@ -0,0 +1,107 @@
+/*!
+ * Contentstack Export
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
+const path = require('path');
+const chalk = require('chalk');
+const mkdirp = require('mkdirp');
+const { merge } = require('lodash');
+const { formatError, log, fileHelper } = require('../../utils');
+const { default: config } = require('../../config');
+module.exports = class ExportGlobalFields {
+    constructor(exportConfig, stackAPIClient) {
+        this.limit = 100;
+        this.config = {};
+        this.global_fields = [];
+        this.master = {};
+        this.globalfields = {};
+        this.requestOptions = {};
+        this.globalfieldsConfig = config.modules.globalfields;
+        this.validKeys = config.modules.globalfields.validKeys;
+        this.requestOptions = {
+            qs: {
+                skip: 0,
+                limit: this.limit,
+                asc: 'updated_at',
+                include_count: true,
+            },
+        };
+        this.config = merge(config, exportConfig);
+        this.stackAPIClient = stackAPIClient;
+        this.globalfieldsFolderPath = path.resolve(this.config.data, this.config.branchName || '', this.globalfieldsConfig.dirName);
+    }
+    start() {
+        const self = this;
+        // Create folder for Global Fields
+        mkdirp.sync(self.globalfieldsFolderPath);
+        log(self.config, 'Starting Global Fields export', 'success');
+        return new Promise(function (resolve, reject) {
+            try {
+                return self
+                    .getGlobalFields(0, self.config)
+                    .then(function (result) {
+                    if (!result) {
+                        return self.writeGlobalFields().then(resolve).catch(reject);
+                    }
+                    return resolve();
+                })
+                    .catch(reject);
+            }
+            catch (error) {
+                log(self.config, error, 'error');
+                return reject(error);
+            }
+        });
+    }
+    getGlobalFields(skip, globalFieldConfig) {
+        const self = this;
+        self.requestOptions.qs.skip = skip;
+        return new Promise(function (resolve, reject) {
+            self.stackAPIClient
+                .globalField()
+                .query(self.requestOptions.qs)
+                .find()
+                .then((globalFieldResponse) => {
+                try {
+                    if (globalFieldResponse.items.length === 0) {
+                        log(globalFieldConfig, 'No global fields found', 'success');
+                        return resolve('No Global Fields');
+                    }
+                    globalFieldResponse.items.forEach(function (globalField) {
+                        for (const key in globalField) {
+                            if (self.validKeys.indexOf(key) === -1) {
+                                delete globalField[key];
+                            }
+                        }
+                        self.global_fields.push(globalField);
+                    });
+                    skip += self.limit;
+                    if (skip >= globalFieldResponse.count) {
+                        return resolve();
+                    }
+                    return self.getGlobalFields(skip, globalFieldConfig).then(resolve).catch(reject);
+                }
+                catch (error) {
+                    log(globalFieldConfig, `Failed to export global-fields. ${formatError(error)}`, 'error');
+                    reject(error);
+                }
+            })
+                .catch(reject);
+        });
+    }
+    writeGlobalFields() {
+        const self = this;
+        return new Promise(function (resolve, reject) {
+            try {
+                fileHelper.writeFileSync(path.join(self.globalfieldsFolderPath, self.globalfieldsConfig.fileName), self.global_fields);
+                log(self.config, chalk.green('Global Fields export completed successfully'), 'success');
+                resolve();
+            }
+            catch (error) {
+                log(self.config, error, 'error');
+                reject(error);
+            }
+        });
+    }
+};

package/lib/export/modules-js/index.d.ts

@@ -0,0 +1 @@
+export default function startModuleExport(modulePayload: any): Promise<any>;

package/lib/export/modules-js/index.js

@@ -0,0 +1,33 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var _a;
+Object.defineProperty(exports, "__esModule", { value: true });
+async function startModuleExport(modulePayload) {
+    const { moduleName, exportConfig, stackAPIClient } = modulePayload;
+    const { default: ModuleRunner } = await (_a = `./${moduleName}`, Promise.resolve().then(() => __importStar(require(_a))));
+    const moduleRunner = new ModuleRunner(exportConfig, stackAPIClient);
+    return moduleRunner.start();
+}
+exports.default = startModuleExport;

package/lib/export/modules-js/labels.d.ts

@@ -0,0 +1,14 @@
+export = ExportLabels;
+declare class ExportLabels {
+    constructor(exportConfig: any, stackAPIClient: any);
+    labels: {};
+    labelConfig: {
+        dirName: string;
+        fileName: string;
+        invalidKeys: string[];
+        dependencies?: import("../../types").Modules[];
+    };
+    config: any;
+    stackAPIClient: any;
+    start(): Promise<any>;
+}

package/lib/export/modules-js/labels.js

@@ -0,0 +1,56 @@
+/*!
+ * Contentstack Export
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
+const path = require('path');
+const chalk = require('chalk');
+const mkdirp = require('mkdirp');
+const { merge } = require('lodash');
+const { formatError, log, fileHelper } = require('../../utils');
+const { default: config } = require('../../config');
+module.exports = class ExportLabels {
+    constructor(exportConfig, stackAPIClient) {
+        this.labels = {};
+        this.labelConfig = config.modules.labels;
+        this.config = merge(config, exportConfig);
+        this.stackAPIClient = stackAPIClient;
+    }
+    start() {
+        log(this.config, 'Starting labels export', 'success');
+        const self = this;
+        const labelsFolderPath = path.resolve(config.data, this.config.branchName || '', self.labelConfig.dirName);
+        // Create locale folder
+        mkdirp.sync(labelsFolderPath);
+        return new Promise(function (resolve, reject) {
+            return self.stackAPIClient
+                .label()
+                .query()
+                .find()
+                .then((response) => {
+                if (response.items.length !== 0) {
+                    response.items.forEach(function (label) {
+                        log(self.config, `'${label.name}' label was exported successfully`, 'success');
+                        self.labels[label.uid] = label;
+                        const deleteItems = self.config.modules.labels.invalidKeys;
+                        deleteItems.forEach((e) => delete label[e]);
+                    });
+                    log(self.config, chalk.green('All the labels have been exported successfully'), 'success');
+                }
+                else {
+                    log(self.config, 'No labels found', 'success');
+                }
+                fileHelper.writeFileSync(path.join(labelsFolderPath, self.labelConfig.fileName), self.labels);
+                resolve();
+            })
+                .catch(function (error) {
+                if (error.statusCode === 401) {
+                    log(self.config, 'You are not allowed to export label, Unless you provide email and password in config', 'error');
+                    return resolve();
+                }
+                log(self.config, `Failed to export labels. ${formatError(error)}`, 'error');
+                reject();
+            });
+        });
+    }
+};

package/lib/export/modules-js/locales.d.ts

@@ -0,0 +1,23 @@
+export = LocaleExport;
+declare class LocaleExport {
+    constructor(exportConfig: any, stackAPIClient: any);
+    stackAPIClient: any;
+    exportConfig: any;
+    localeConfig: any;
+    masterLocaleConfig: any;
+    qs: {
+        include_count: boolean;
+        asc: string;
+        only: {
+            BASE: any;
+        };
+    };
+    localesPath: string;
+    locales: {};
+    masterLocale: {};
+    fetchConcurrency: any;
+    writeConcurrency: any;
+    start(): Promise<void>;
+    getLocales(skip?: number): any;
+    sanitizeAttribs(locales: any): void;
+}

package/lib/export/modules-js/locales.js

@@ -0,0 +1,67 @@
+const path = require('path');
+const chalk = require('chalk');
+const { formatError, log, fileHelper } = require('../../utils');
+class LocaleExport {
+    constructor(exportConfig, stackAPIClient) {
+        this.stackAPIClient = stackAPIClient;
+        this.exportConfig = exportConfig;
+        this.localeConfig = exportConfig.modules.locales;
+        this.masterLocaleConfig = exportConfig.modules.masterLocale;
+        this.qs = {
+            include_count: true,
+            asc: 'updated_at',
+            only: {
+                BASE: this.localeConfig.requiredKeys,
+            },
+        };
+        this.localesPath = path.resolve(exportConfig.data, exportConfig.branchName || '', this.localeConfig.dirName);
+        this.locales = {};
+        this.masterLocale = {};
+        this.fetchConcurrency = this.localeConfig.fetchConcurrency || this.exportConfig.fetchConcurrency;
+        this.writeConcurrency = this.localeConfig.writeConcurrency || this.exportConfig.writeConcurrency;
+    }
+    async start() {
+        try {
+            log(this.exportConfig, 'Starting locale export', 'success');
+            fileHelper.makeDirectory(this.localesPath);
+            await this.getLocales();
+            await fileHelper.writeFile(path.join(this.localesPath, this.localeConfig.fileName), this.locales);
+            await fileHelper.writeFile(path.join(this.localesPath, this.masterLocaleConfig.fileName), this.masterLocale);
+            log(this.exportConfig, 'Completed locale export', 'success');
+        }
+        catch (error) {
+            log(this.exportConfig, `Failed to export locales. ${formatError(error)}`, 'error');
+            throw new Error('Failed to export locales');
+        }
+    }
+    async getLocales(skip = 0) {
+        if (skip) {
+            this.qs.skip = skip;
+        }
+        let localesFetchResponse = await this.stackAPIClient.locale().query(this.qs).find();
+        if (Array.isArray(localesFetchResponse.items) && localesFetchResponse.items.length > 0) {
+            this.sanitizeAttribs(localesFetchResponse.items);
+            skip += this.localeConfig.limit || 100;
+            if (skip > localesFetchResponse.count) {
+                return;
+            }
+            return await this.getLocales(skip);
+        }
+    }
+    sanitizeAttribs(locales) {
+        locales.forEach((locale) => {
+            for (let key in locale) {
+                if (this.localeConfig.requiredKeys.indexOf(key) === -1) {
+                    delete locale[key];
+                }
+            }
+            if (locale.code === this.exportConfig.master_locale.code) {
+                this.masterLocale[locale.uid] = locale;
+            }
+            else {
+                this.locales[locale.uid] = locale;
+            }
+        });
+    }
+}
+module.exports = LocaleExport;

package/lib/export/modules-js/marketplace-apps.d.ts

@@ -0,0 +1,21 @@
+export = ExportMarketplaceApps;
+declare class ExportMarketplaceApps {
+    constructor(credentialConfig: any);
+    client: any;
+    config: any;
+    httpClient: any;
+    nodeCrypto: any;
+    marketplaceAppPath: any;
+    developerHubBaseUrl: any;
+    marketplaceAppConfig: {
+        dirName: string;
+        fileName: string;
+        dependencies?: import("../../types").Modules[];
+    };
+    start(): Promise<void>;
+    getOrgUid(): Promise<void>;
+    createNodeCryptoInstance(): Promise<void>;
+    exportInstalledExtensions(): Promise<void>;
+    getAllStackSpecificApps(listOfApps?: any[], skip?: number): any;
+    getAppConfigurations(sdkClient: any, installedApps: any, [index, appInstallation]: [any, any]): Promise<void>;
+}