@contentstack/cli-cm-export 1.5.9 → 1.7.0

package/lib/export/modules/webhooks.d.ts

@@ -0,0 +1,12 @@
+import BaseClass from './base-class';
+import { ModuleClassParams } from '../../types';
+export default class ExportWebhooks extends BaseClass {
+    private webhooks;
+    private webhookConfig;
+    webhooksFolderPath: string;
+    private qs;
+    constructor({ exportConfig, stackAPIClient }: ModuleClassParams);
+    start(): Promise<void>;
+    getWebhooks(skip?: number): Promise<void>;
+    sanitizeAttribs(webhooks: Record<string, string>[]): void;
+}

package/lib/export/modules/webhooks.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const tslib_1 = require("tslib");
+const omit_1 = tslib_1.__importDefault(require("lodash/omit"));
+const isEmpty_1 = tslib_1.__importDefault(require("lodash/isEmpty"));
+const node_path_1 = require("node:path");
+const config_1 = tslib_1.__importDefault(require("../../config"));
+const base_class_1 = tslib_1.__importDefault(require("./base-class"));
+const utils_1 = require("../../utils");
+class ExportWebhooks extends base_class_1.default {
+    constructor({ exportConfig, stackAPIClient }) {
+        super({ exportConfig, stackAPIClient });
+        this.webhooks = {};
+        this.webhookConfig = config_1.default.modules.webhooks;
+        this.qs = { include_count: true, asc: 'updated_at' };
+    }
+    async start() {
+        (0, utils_1.log)(this.exportConfig, 'Starting webhooks export', 'info');
+        this.webhooksFolderPath = (0, node_path_1.resolve)(this.exportConfig.data, this.exportConfig.branchName || '', this.webhookConfig.dirName);
+        await utils_1.fsUtil.makeDirectory(this.webhooksFolderPath);
+        await this.getWebhooks();
+        if (this.webhooks === undefined || (0, isEmpty_1.default)(this.webhooks)) {
+            (0, utils_1.log)(this.exportConfig, 'No webhooks found', 'info');
+        }
+        else {
+            utils_1.fsUtil.writeFile((0, node_path_1.resolve)(this.webhooksFolderPath, this.webhookConfig.fileName), this.webhooks);
+            (0, utils_1.log)(this.exportConfig, 'All the webhooks have been exported successfully!', 'success');
+        }
+    }
+    async getWebhooks(skip = 0) {
+        if (skip) {
+            this.qs.skip = skip;
+        }
+        await this.stack
+            .webhook()
+            .fetchAll(this.qs)
+            .then(async (data) => {
+            const { items, count } = data;
+            if (items === null || items === void 0 ? void 0 : items.length) {
+                this.sanitizeAttribs(items);
+                skip += this.webhookConfig.limit || 100;
+                if (skip >= count) {
+                    return;
+                }
+                return await this.getWebhooks(skip);
+            }
+        })
+            .catch(({ error }) => {
+            (0, utils_1.log)(this.exportConfig, `Failed to export webhooks.${(0, utils_1.formatError)(error)}`, 'error');
+            (0, utils_1.log)(this.exportConfig, error, 'error');
+        });
+    }
+    sanitizeAttribs(webhooks) {
+        var _a;
+        for (let index = 0; index < (webhooks === null || webhooks === void 0 ? void 0 : webhooks.length); index++) {
+            const webhookUid = webhooks[index].uid;
+            const webhookName = (_a = webhooks[index]) === null || _a === void 0 ? void 0 : _a.name;
+            this.webhooks[webhookUid] = (0, omit_1.default)(webhooks[index], ['SYS_ACL']);
+            (0, utils_1.log)(this.exportConfig, `'${webhookName}' webhook was exported successfully`, 'success');
+        }
+    }
+}
+exports.default = ExportWebhooks;

package/lib/export/modules-js/assets.d.ts

@@ -0,0 +1,43 @@
+export = ExportAssets;
+declare class ExportAssets {
+    constructor(exportConfig: any, stackAPIClient: any);
+    config: any;
+    bLimit: number;
+    vLimit: any;
+    invalidKeys: string[];
+    folderJSONPath: any;
+    folderData: any[];
+    assetsFolderPath: any;
+    assetContents: {};
+    httpClient: HttpClient;
+    assetConfig: {
+        dirName: string;
+        fileName: string;
+        batchLimit: number;
+        host: string;
+        invalidKeys: string[];
+        chunkFileSize: number;
+        downloadLimit: number;
+        fetchConcurrency: number;
+        assetsMetaKeys: string[];
+        securedAssets: boolean;
+        displayExecutionTime: boolean;
+        enableDownloadStatus: boolean;
+        includeVersionedAssets: boolean;
+        dependencies?: import("../../types").Modules[];
+    };
+    stackAPIClient: any;
+    assetDownloadRetry: {};
+    assetDownloadRetryLimit: number;
+    start(): any;
+    assetContentsFile: string;
+    exportFolders(): any;
+    getFolderJSON(skip: any, fCount: any): any;
+    getAssetCount(folder: any): any;
+    getAssetJSON(skip: any): any;
+    getVersionedAssetJSON(uid: any, version: any, bucket: any): any;
+    downloadAsset(asset: any): any;
+    getFolders(): any;
+    getFolderDetails(skip: any, tCount: any): any;
+}
+import { HttpClient } from "@contentstack/cli-utilities/lib/http-client/client";

package/lib/export/modules-js/assets.js

@@ -0,0 +1,391 @@
+/*!
+ * Contentstack Export
+ * Copyright (c) 2019 Contentstack LLC
+ * MIT Licensed
+ */
+const mkdirp = require('mkdirp');
+const path = require('path');
+const fs = require('fs');
+const Promise = require('bluebird');
+const _ = require('lodash');
+const chalk = require('chalk');
+const progress = require('progress-stream');
+const { HttpClient, configHandler } = require('@contentstack/cli-utilities');
+const { fileHelper, log, formatError } = require('../../utils');
+let { default: config } = require('../../config');
+module.exports = class ExportAssets {
+    constructor(exportConfig, stackAPIClient) {
+        this.folderData = [];
+        this.assetContents = {};
+        this.httpClient = HttpClient.create();
+        this.assetConfig = config.modules.assets;
+        this.stackAPIClient = stackAPIClient;
+        this.config = _.merge(config, exportConfig);
+        this.folderData = [];
+        this.assetContents = {};
+        this.assetDownloadRetry = {};
+        this.assetDownloadRetryLimit = 3;
+        this.invalidKeys = this.assetConfig.invalidKeys;
+        this.bLimit = this.assetConfig.batchLimit || 15;
+        this.vLimit = this.assetConfig.downloadLimit || this.config.fetchConcurrency || 3;
+    }
+    start() {
+        const self = this;
+        this.assetsFolderPath = path.resolve(this.config.data, this.config.branchName || '', this.assetConfig.dirName);
+        this.assetContentsFile = path.resolve(this.assetsFolderPath, 'assets.json');
+        this.folderJSONPath = path.resolve(this.assetsFolderPath, 'folders.json');
+        log(this.config, 'Starting assets export', 'success');
+        // Create asset folder
+        mkdirp.sync(this.assetsFolderPath);
+        return new Promise((resolve, reject) => {
+            // TBD: getting all the assets should have optimized
+            return self
+                .getAssetCount()
+                .then((count) => {
+                const assetBatches = [];
+                if (typeof count !== 'number' || count === 0) {
+                    log(self.config, 'No assets found', 'success');
+                    return resolve();
+                }
+                for (let i = 0; i <= count; i += self.bLimit) {
+                    assetBatches.push(i);
+                }
+                return Promise.map(assetBatches, (batch) => {
+                    return self
+                        .getAssetJSON(batch)
+                        .then((assetsJSON) => {
+                        return Promise.map(assetsJSON, (assetJSON) => {
+                            if (self.assetConfig.downloadVersionAssets) {
+                                return self
+                                    .getVersionedAssetJSON(assetJSON.uid, assetJSON._version)
+                                    .then(() => {
+                                    self.assetContents[assetJSON.uid] = assetJSON;
+                                })
+                                    .catch((error) => {
+                                    log(self.config, `Asset '${assetJSON.uid}' failed to download.\n ${formatError(error)}`, 'error');
+                                    log(self.config, error, 'error');
+                                });
+                            }
+                            else {
+                                return self
+                                    .downloadAsset(assetJSON)
+                                    .then(() => {
+                                    self.assetContents[assetJSON.uid] = assetJSON;
+                                })
+                                    .catch((err) => {
+                                    log(self.config, `Asset '${assetJSON.uid}' download failed. ${formatError(err)}`, 'error');
+                                    return err;
+                                });
+                            }
+                        }, { concurrency: self.vLimit })
+                            .then(() => {
+                            log(self.config, 'Batch no ' + (batch + 1) + ' of assets is complete', 'success');
+                            // fileHelper.writeFileSync(this.assetContentsFile, self.assetContents)
+                        })
+                            .catch((error) => {
+                            log(self.config, `Asset batch ${batch + 1} failed to download`, 'error');
+                            log(self.config, formatError(error), 'error');
+                            log(self.config, error, 'error');
+                        });
+                    })
+                        .catch((error) => {
+                        log(self.config, error, 'error');
+                        reject(error);
+                    });
+                }, { concurrency: self.assetConfig.concurrencyLimit || 1 })
+                    .then(() => {
+                    fileHelper.writeFileSync(self.assetContentsFile, self.assetContents);
+                    return self
+                        .exportFolders()
+                        .then(() => {
+                        log(self.config, chalk.green('Asset export completed successfully'), 'success');
+                        return resolve();
+                    })
+                        .catch((error) => {
+                        log(self.config, error, 'error');
+                        reject(error);
+                    });
+                })
+                    .catch((error) => {
+                    fileHelper.writeFileSync(self.assetContentsFile, self.assetContents);
+                    log(self.config, `Asset export failed. ${formatError(error)}`, 'error');
+                    log(self.config, error, 'error');
+                    reject(error);
+                });
+            })
+                .catch((error) => {
+                log(self.config, error, 'error');
+                reject(error);
+            });
+        });
+    }
+    exportFolders() {
+        const self = this;
+        return new Promise((resolve, reject) => {
+            return self
+                .getAssetCount(true)
+                .then((fCount) => {
+                if (fCount === 0) {
+                    log(self.config, 'No folders were found in the stack!', 'success');
+                    return resolve();
+                }
+                return self
+                    .getFolderJSON(0, fCount)
+                    .then(() => {
+                    // asset folders have been successfully exported
+                    log(self.config, 'Asset-folders have been successfully exported!', 'success');
+                    return resolve();
+                })
+                    .catch((error) => {
+                    log(self.config, `Error while exporting asset-folders!\n ${formatError(error)}`, 'error');
+                    return reject(error);
+                });
+            })
+                .catch((error) => {
+                log(self.config, error, 'error');
+                // error while fetching asset folder count
+                return reject(error);
+            });
+        });
+    }
+    getFolderJSON(skip, fCount) {
+        const self = this;
+        return new Promise((resolve, reject) => {
+            if (typeof skip !== 'number') {
+                skip = 0;
+            }
+            if (skip >= fCount) {
+                fileHelper.writeFileSync(self.folderJSONPath, self.folderData);
+                return resolve();
+            }
+            const queryRequestObj = {
+                skip,
+                include_folders: true,
+                query: { is_dir: true },
+            };
+            self.stackAPIClient
+                .asset()
+                .query(queryRequestObj)
+                .find()
+                .then((response) => {
+                skip += 100;
+                self.folderData.push(...response.items);
+                return self.getFolderJSON(skip, fCount).then(resolve).catch(reject);
+            })
+                .catch((error) => reject(error));
+        });
+    }
+    getAssetCount(folder) {
+        const self = this;
+        return new Promise((resolve, reject) => {
+            if (folder && typeof folder === 'boolean') {
+                const queryOptions = {
+                    skip: 99999990,
+                    include_count: true,
+                    include_folders: true,
+                    query: { is_dir: true },
+                };
+                self.stackAPIClient
+                    .asset()
+                    .query(queryOptions)
+                    .find()
+                    .then((asset) => {
+                    return resolve(asset.count);
+                })
+                    .catch((error) => {
+                    log(self.config, error, 'error');
+                });
+            }
+            else {
+                const queryOptions = { skip: 99999990, include_count: true };
+                self.stackAPIClient
+                    .asset()
+                    .query(queryOptions)
+                    .find()
+                    .then(({ count }) => resolve(count))
+                    .catch((error) => {
+                    log(self.config, error, 'error');
+                    reject(error);
+                });
+            }
+        });
+    }
+    getAssetJSON(skip) {
+        const self = this;
+        return new Promise((resolve, reject) => {
+            if (typeof skip !== 'number') {
+                skip = 0;
+            }
+            const queryRequestObj = {
+                skip: skip,
+                limit: self.bLimit,
+                include_publish_details: true,
+                except: {
+                    BASE: self.invalidKeys,
+                },
+            };
+            self.stackAPIClient
+                .asset()
+                .query(queryRequestObj)
+                .find()
+                .then(({ items }) => resolve(items))
+                .catch((error) => {
+                log(self.config, error, 'error');
+                return reject();
+            });
+        });
+    }
+    getVersionedAssetJSON(uid, version, bucket) {
+        const self = this;
+        const assetVersionInfo = bucket || [];
+        return new Promise((resolve, reject) => {
+            if (self.assetDownloadRetry[uid + version] > self.assetDownloadRetryLimit) {
+                console.log('Reached max', self.assetDownloadRetry[uid + version]);
+                return reject(new Error('Asset Max download retry limit exceeded! ' + uid));
+            }
+            if (version <= 0) {
+                const assetVersionInfoFile = path.resolve(self.assetsFolderPath, uid, '_contentstack_' + uid + '.json');
+                fileHelper.writeFileSync(assetVersionInfoFile, assetVersionInfo);
+                return resolve();
+            }
+            const queryrequestOption = {
+                version: version,
+                include_publish_details: true,
+                except: {
+                    BASE: self.invalidKeys,
+                },
+            };
+            self.stackAPIClient
+                .asset(uid)
+                .fetch(queryrequestOption)
+                .then((versionedAssetJSONResponse) => {
+                self
+                    .downloadAsset(versionedAssetJSONResponse)
+                    .then(() => {
+                    assetVersionInfo.splice(0, 0, versionedAssetJSONResponse);
+                    // Remove duplicates
+                    assetVersionInfo = _.uniqWith(assetVersionInfo, _.isEqual);
+                    self.getVersionedAssetJSON(uid, --version, assetVersionInfo).then(resolve).catch(reject);
+                })
+                    .catch(reject);
+            })
+                .catch((error) => {
+                log(self.config, error, 'error');
+                if (error.status === 408) {
+                    console.log('retrying', uid);
+                    // retrying when timeout
+                    self.assetDownloadRetry[uid + version]
+                        ? ++self.assetDownloadRetry[uid + version]
+                        : (self.assetDownloadRetry[uid + version] = 1);
+                    return self.getVersionedAssetJSON(uid, version, assetVersionInfo).then(resolve).catch(reject);
+                }
+                reject(error);
+            });
+        });
+    }
+    downloadAsset(asset) {
+        const self = this;
+        return new Promise(async (resolve, reject) => {
+            const assetFolderPath = path.resolve(self.assetsFolderPath, asset.uid);
+            const assetFilePath = path.resolve(assetFolderPath, asset.filename);
+            if (fs.existsSync(assetFilePath)) {
+                log(self.config, 'Skipping download of { title: ' + asset.filename + ', uid: ' + asset.uid + ' }, as they already exist', 'success');
+                return resolve();
+            }
+            self.assetStream = {
+                url: self.config.securedAssets ? `${asset.url}?authtoken=${configHandler.get('authtoken')}` : asset.url,
+            };
+            fileHelper.makeDirectory(assetFolderPath);
+            const assetFileStream = fs.createWriteStream(assetFilePath);
+            self.assetStream.url = encodeURI(self.assetStream.url);
+            self.httpClient
+                .options({ responseType: 'stream' })
+                .get(self.assetStream.url)
+                .then(({ data: assetStreamRequest }) => {
+                if (self.assetConfig.enableDownloadStatus) {
+                    const str = progress({
+                        time: 5000,
+                        length: assetStreamRequest.headers['content-length'],
+                    });
+                    str.on('progress', (progressData) => {
+                        console.log(`${asset.filename}: ${Math.round(progressData.percentage)}%`);
+                    });
+                    assetStreamRequest.pipe(str).pipe(assetFileStream);
+                }
+                assetStreamRequest.pipe(assetFileStream);
+            })
+                .catch((error) => {
+                log(self.config, error, 'error');
+                reject(error);
+            });
+            assetFileStream
+                .on('close', function () {
+                log(self.config, 'Downloaded ' + asset.filename + ': ' + asset.uid + ' successfully!', 'success');
+                return resolve();
+            })
+                .on('error', (error) => {
+                log(self.config, `Download ${asset.filename}: ${asset.uid} failed!`, 'error');
+                log(self.config, error, 'error');
+                reject(error);
+            });
+        });
+    }
+    getFolders() {
+        const self = this;
+        return new Promise((resolve, reject) => {
+            return self
+                .getAssetCount(true)
+                .then((count) => {
+                if (count === 0) {
+                    log(self.config, 'No folders were found in the stack', 'success');
+                    return resolve();
+                }
+                return self
+                    .getFolderDetails(0, count)
+                    .then(function () {
+                    log(self.config, chalk.green('Exported asset-folders successfully!'), 'success');
+                    return resolve();
+                })
+                    .catch(function (error) {
+                    log(self.config, error, 'error');
+                    reject(error);
+                });
+            })
+                .catch(function (error) {
+                log(self.config, error, 'error');
+                reject(error);
+            });
+        });
+    }
+    getFolderDetails(skip, tCount) {
+        const self = this;
+        return new Promise((resolve, reject) => {
+            if (typeof skip !== 'number') {
+                skip = 0;
+            }
+            if (skip > tCount) {
+                fileHelper.writeFileSync(self.folderJSONPath, self.folderContents);
+                return resolve();
+            }
+            const queryRequestObj = {
+                skip: skip,
+                include_folders: true,
+                query: { is_dir: true },
+            };
+            self.stackAPIClient
+                .asset()
+                .query(queryRequestObj)
+                .find()
+                .then((folderDetailsResponse) => {
+                for (let i in folderDetailsResponse.items) {
+                    self.folderContents.push(folderDetailsResponse.items[i]);
+                }
+                skip += 100;
+                return self.getFolderDetails(skip, tCount).then(resolve).catch(reject);
+            })
+                .catch((error) => {
+                log(self.config, error, 'error');
+            });
+        });
+    }
+};

package/lib/export/modules-js/content-types.d.ts

@@ -0,0 +1,21 @@
+export = ContentTypesExport;
+declare class ContentTypesExport {
+    constructor(exportConfig: any, stackAPIClient: any);
+    stackAPIClient: any;
+    exportConfig: any;
+    contentTypesConfig: any;
+    qs: {
+        include_count: boolean;
+        asc: string;
+        limit: any;
+        include_global_field_schema: boolean;
+    };
+    contentTypesPath: string;
+    contentTypes: any[];
+    fetchConcurrency: any;
+    writeConcurrency: any;
+    start(): Promise<void>;
+    getContentTypes(skip?: number): any;
+    sanitizeAttribs(contentTypes: any): any[];
+    writeContentTypes(contentTypes: any): Promise<any>;
+}

package/lib/export/modules-js/content-types.js

@@ -0,0 +1,75 @@
+const path = require('path');
+const chalk = require('chalk');
+const { fileHelper, executeTask, formatError, log } = require('../../utils');
+class ContentTypesExport {
+    constructor(exportConfig, stackAPIClient) {
+        this.stackAPIClient = stackAPIClient;
+        this.exportConfig = exportConfig;
+        this.contentTypesConfig = exportConfig.modules.content_types;
+        this.qs = {
+            include_count: true,
+            asc: 'updated_at',
+            limit: this.contentTypesConfig.limit,
+            include_global_field_schema: true,
+        };
+        // If content type id is provided then use it as part of query
+        if (Array.isArray(this.exportConfig.contentTypes) && this.exportConfig.length > 0) {
+            this.qs.uid = { $in: this.exportConfig.contentTypes };
+        }
+        this.contentTypesPath = path.resolve(exportConfig.data, exportConfig.branchName || '', this.contentTypesConfig.dirName);
+        this.contentTypes = [];
+        this.fetchConcurrency = this.contentTypesConfig.fetchConcurrency || this.exportConfig.fetchConcurrency;
+        this.writeConcurrency = this.contentTypesConfig.writeConcurrency || this.exportConfig.writeConcurrency;
+    }
+    async start() {
+        try {
+            log(this.exportConfig, 'Starting content type export', 'success');
+            await fileHelper.makeDirectory(this.contentTypesPath);
+            await this.getContentTypes();
+            await this.writeContentTypes(this.contentTypes);
+            log(this.exportConfig, chalk.green('Content type(s) exported successfully'), 'success');
+        }
+        catch (error) {
+            log(this.exportConfig, `Failed to export content types ${formatError(error)}`, 'error');
+            throw new Error('Failed to export content types');
+        }
+    }
+    async getContentTypes(skip = 0) {
+        if (skip) {
+            this.qs.skip = skip;
+        }
+        const contentTypeSearchResponse = await this.stackAPIClient.contentType().query(this.qs).find();
+        if (Array.isArray(contentTypeSearchResponse.items) && contentTypeSearchResponse.items.length > 0) {
+            let updatedContentTypes = this.sanitizeAttribs(contentTypeSearchResponse.items);
+            this.contentTypes.push(...updatedContentTypes);
+            skip += this.contentTypesConfig.limit;
+            if (skip > contentTypeSearchResponse.count) {
+                return;
+            }
+            return await this.getContentTypes(skip);
+        }
+        else {
+            log(this.exportConfig, 'No content types returned for the given query', 'info');
+        }
+    }
+    sanitizeAttribs(contentTypes) {
+        let updatedContentTypes = [];
+        contentTypes.forEach((contentType) => {
+            for (let key in contentType) {
+                if (this.contentTypesConfig.validKeys.indexOf(key) === -1) {
+                    delete contentType[key];
+                }
+            }
+            updatedContentTypes.push(contentType);
+        });
+        return updatedContentTypes;
+    }
+    async writeContentTypes(contentTypes) {
+        function write(contentType) {
+            return fileHelper.writeFile(path.join(this.contentTypesPath, `${contentType.uid === 'schema' ? 'schema|1' : contentType.uid}.json`), contentType);
+        }
+        await executeTask(contentTypes, write.bind(this), { concurrency: this.writeConcurrency });
+        return fileHelper.writeFile(path.join(this.contentTypesPath, 'schema.json'), contentTypes);
+    }
+}
+module.exports = ContentTypesExport;

package/lib/export/modules-js/custom-roles.d.ts

@@ -0,0 +1,21 @@
+export = ExportCustomRoles;
+declare class ExportCustomRoles {
+    constructor(exportConfig: any, stackAPIClient: any);
+    roles: {};
+    customRoles: {};
+    EXISTING_ROLES: {
+        Admin: number;
+        Developer: number;
+        'Content Manager': number;
+    };
+    rolesConfig: {
+        dirName: string;
+        fileName: string;
+        customRolesLocalesFileName: string;
+        dependencies?: import("../../types").Modules[];
+    };
+    config: any;
+    stackAPIClient: any;
+    start(): Promise<void>;
+    getCustomRolesLocales(customRoles: any, customRolesLocalesFilepath: any, stackAPIClient: any, config: any): Promise<void>;
+}