@contentstack/cli-cm-export 1.0.0 → 1.2.0

package/README.md

@@ -38,7 +38,7 @@ $ npm install -g @contentstack/cli-cm-export
 $ csdx COMMAND
 running command...
 $ csdx (-v|--version|version)
-@contentstack/cli-cm-export/1.0.0 linux-x64 node-v16.14.2
+@contentstack/cli-cm-export/1.2.0 darwin-arm64 node-v18.11.0
 $ csdx --help [COMMAND]
 USAGE
   $ csdx COMMAND
@@ -83,5 +83,5 @@ EXAMPLES
   csdx cm:stacks:export --branch [optional] branch name
 ```
 
-_See code: [src/commands/cm/stacks/export.js](https://github.com/contentstack/cli/blob/v1.0.0/packages/contentstack-export/src/commands/cm/stacks/export.js)_
+_See code: [src/commands/cm/stacks/export.js](https://github.com/contentstack/cli/blob/main/packages/contentstack-export/src/commands/cm/stacks/export.js)_
 <!-- commandsstop -->

package/oclif.manifest.json

@@ -1 +1 @@
-{"version":"1.0.0","commands":{"cm:stacks:export":{"id":"cm:stacks:export","description":"Export content from a stack","usage":"cm:stacks:export [-c <value>] [-k <value>] [-d <value>] [-a <value>] [--module <value>] [--content-types <value>] [--branch <value>] [--secured-assets]","pluginName":"@contentstack/cli-cm-export","pluginType":"core","aliases":["cm:export"],"examples":["csdx cm:stacks:export --stack-api-key <stack_api_key> --data-dir <path/of/export/destination/dir>","csdx cm:stacks:export --config <path/to/config/dir>","csdx cm:stacks:export --alias <management_token_alias>","csdx cm:stacks:export --alias <management_token_alias> --data-dir <path/to/export/destination/dir>","csdx cm:stacks:export --alias <management_token_alias> --config <path/to/config/file>","csdx cm:stacks:export --module <single module name>","csdx cm:stacks:export --branch [optional] branch name"],"flags":{"config":{"name":"config","type":"option","char":"c","description":"[optional] path of the config"},"stack-uid":{"name":"stack-uid","type":"option","char":"s","description":"API key of the source stack","hidden":true},"stack-api-key":{"name":"stack-api-key","type":"option","char":"k","description":"API key of the source stack"},"data":{"name":"data","type":"option","description":"path or location to store the data","hidden":true},"data-dir":{"name":"data-dir","type":"option","char":"d","description":"path or location to store the data"},"alias":{"name":"alias","type":"option","char":"a","description":"alias of the management token"},"management-token-alias":{"name":"management-token-alias","type":"option","description":"alias of the management token","hidden":true},"auth-token":{"name":"auth-token","type":"boolean","char":"A","description":"to use auth token","hidden":true,"allowNo":false},"module":{"name":"module","type":"option","char":"m","description":"[optional] specific module name"},"content-types":{"name":"content-types","type":"option","char":"t","description":"[optional] content type"},"branch":{"name":"branch","type":"option","char":"B","description":"[optional] branch name"},"secured-assets":{"name":"secured-assets","type":"boolean","description":"[optional] use when assets are secured","allowNo":false}},"args":[]}}}
+{"version":"1.2.0","commands":{"cm:stacks:export":{"id":"cm:stacks:export","description":"Export content from a stack","usage":"cm:stacks:export [-c <value>] [-k <value>] [-d <value>] [-a <value>] [--module <value>] [--content-types <value>] [--branch <value>] [--secured-assets]","pluginName":"@contentstack/cli-cm-export","pluginType":"core","aliases":["cm:export"],"examples":["csdx cm:stacks:export --stack-api-key <stack_api_key> --data-dir <path/of/export/destination/dir>","csdx cm:stacks:export --config <path/to/config/dir>","csdx cm:stacks:export --alias <management_token_alias>","csdx cm:stacks:export --alias <management_token_alias> --data-dir <path/to/export/destination/dir>","csdx cm:stacks:export --alias <management_token_alias> --config <path/to/config/file>","csdx cm:stacks:export --module <single module name>","csdx cm:stacks:export --branch [optional] branch name"],"flags":{"config":{"name":"config","type":"option","char":"c","description":"[optional] path of the config"},"stack-uid":{"name":"stack-uid","type":"option","char":"s","description":"API key of the source stack","hidden":true},"stack-api-key":{"name":"stack-api-key","type":"option","char":"k","description":"API key of the source stack"},"data":{"name":"data","type":"option","description":"path or location to store the data","hidden":true},"data-dir":{"name":"data-dir","type":"option","char":"d","description":"path or location to store the data"},"alias":{"name":"alias","type":"option","char":"a","description":"alias of the management token"},"management-token-alias":{"name":"management-token-alias","type":"option","description":"alias of the management token","hidden":true},"auth-token":{"name":"auth-token","type":"boolean","char":"A","description":"to use auth token","hidden":true,"allowNo":false},"module":{"name":"module","type":"option","char":"m","description":"[optional] specific module name"},"content-types":{"name":"content-types","type":"option","char":"t","description":"[optional] content type"},"branch":{"name":"branch","type":"option","char":"B","description":"[optional] branch name"},"secured-assets":{"name":"secured-assets","type":"boolean","description":"[optional] use when assets are secured","allowNo":false}},"args":[]}}}

package/package.json

@@ -1,16 +1,17 @@
 {
   "name": "@contentstack/cli-cm-export",
   "description": "Contentstack CLI plugin to export content from stack",
-  "version": "1.0.0",
+  "version": "1.2.0",
   "author": "Contentstack",
   "bugs": "https://github.com/contentstack/cli/issues",
   "dependencies": {
-    "@contentstack/cli-command": "^1.0.0",
-    "@contentstack/cli-utilities": "^1.0.0",
+    "@contentstack/cli-command": "^1.0.2",
+    "@contentstack/cli-utilities": "^1.0.3",
     "@contentstack/management": "^1.3.0",
     "@oclif/command": "^1.8.16",
     "@oclif/config": "^1.18.3",
     "async": "^3.2.4",
+    "big-json": "^3.2.0",
     "bluebird": "^3.7.2",
     "fs": "0.0.1-security",
     "is-valid-path": "^0.1.1",
@@ -18,15 +19,21 @@
     "mkdirp": "^1.0.4",
     "path": "^0.12.7",
     "progress-stream": "^2.0.0",
+    "promise-limit": "^2.7.0",
     "proxyquire": "^2.1.3",
     "winston": "^3.7.2"
   },
   "devDependencies": {
+    "@contentstack/cli-auth": "^1.0.3",
+    "@contentstack/cli-config": "^1.0.2",
+    "@contentstack/cli-dev-dependencies": "^1.0.0",
     "@oclif/dev-cli": "^1.22.2",
     "@oclif/plugin-help": "^5.1.12",
     "@oclif/test": "^1.2.6",
     "assert": "^2.0.0",
     "chai": "^4.2.0",
+    "dotenv": "^16.0.1",
+    "dotenv-expand": "^9.0.0",
     "eslint": "^8.18.0",
     "eslint-config-oclif": "^3.1.0",
     "globby": "^10.0.2",
@@ -54,7 +61,9 @@
     "postpack": "rm -f oclif.manifest.json",
     "prepack": "oclif-dev manifest && oclif-dev readme",
     "test": "nyc mocha --forbid-only \"test/**/*.test.js\"",
-    "version": "oclif-dev readme && git add README.md"
+    "version": "oclif-dev readme && git add README.md",
+    "test:integration": "mocha --forbid-only \"test/run.test.js\" --integration-test",
+    "test:unit": "mocha --forbid-only \"test/unit/*.test.js\" --unit-test"
   },
   "oclif": {
     "commands": "./src/commands",
@@ -62,7 +71,7 @@
     "devPlugins": [
       "@oclif/plugin-help"
     ],
-    "repositoryPrefix": "<%- repo %>/blob/v<%- version %>/packages/contentstack-export/<%- commandPath %>"
+    "repositoryPrefix": "<%- repo %>/blob/main/packages/contentstack-export/<%- commandPath %>"
   },
   "csdxConfig": {
     "expiredCommands": {

package/src/app.js

@@ -1,11 +1,13 @@
 /* eslint-disable no-redeclare */
-var util = require('./lib/util');
-var login = require('./lib/util/login');
-var { addlogs, unlinkFileLogger } = require('./lib/util/log');
-const setupBranches = require('./lib/util/setup-branches');
+const _ = require('lodash');
+const path = require('path');
 const chalk = require('chalk');
-let path = require('path');
-let _ = require('lodash');
+const util = require('./lib/util');
+const { formatError } = require('./lib/util');
+const login = require('./lib/util/login');
+const setupBranches = require('./lib/util/setup-branches');
+const { addlogs, unlinkFileLogger } = require('./lib/util/log');
+const stack = require('./lib/util/contentstack-management-sdk');
 
 exports.initial = async function (config) {
   return new Promise(async function (resolve, reject) {
@@ -17,7 +19,8 @@ exports.initial = async function (config) {
 
     const fetchBranchAndExport = async () => {
       await setupBranches(config, config.branchName);
-      var types = config.modules.types;
+      let types = config.modules.types;
+
       if (Array.isArray(config.branches) && config.branches.length > 0) {
         for (let branch of config.branches) {
           config.branchName = branch.uid;
@@ -28,7 +31,7 @@ exports.initial = async function (config) {
               await allExport(config, types, branch.uid);
             }
           } catch (error) {
-            console.log('failed export branch', branch.uid, error);
+            addlogs(config, `failed export contents ${branch.uid} ${formatError(error)}`, 'error');
           }
         }
       } else {
@@ -39,10 +42,10 @@ exports.initial = async function (config) {
             await allExport(config, types);
           }
         } catch (error) {
-          console.log('failed export contents', error && error.message);
+          addlogs(config, `failed export contents ${formatError(error)}`, 'error');
         }
       }
-    }
+    };
 
     // try {
     if (
@@ -54,8 +57,12 @@ exports.initial = async function (config) {
         .login(config)
         .then(async function () {
           // setup branches
-          await fetchBranchAndExport()
-          unlinkFileLogger();
+          try {
+            await fetchBranchAndExport();
+            unlinkFileLogger();
+          } catch (error) {
+            addlogs(config, `${formatError(error)}`, 'error');
+          }
           resolve();
         })
         .catch((error) => {
@@ -64,19 +71,26 @@ exports.initial = async function (config) {
             addlogs(config, chalk.red('Stack Api key ' + error.errors.api_key[0], 'Please enter valid Key', 'error'));
             addlogs(config, 'The log for this is stored at ' + config.data + '/export/logs', 'success');
           } else {
-            console.log('Stack fail to export');
+            addlogs(config, `${formatError(error)}`, 'error');
           }
-          reject(error);
+          // reject(error);
         });
     } else if (config.management_token) {
-      await fetchBranchAndExport()
+      try {
+        await fetchBranchAndExport();
+      } catch (error) {
+        addlogs(config, `${formatError(error)}`, 'error');
+      }
       resolve();
     }
   });
 };
 
-var singleExport = async (moduleName, types, config, branchName) => {
+const singleExport = async (moduleName, types, config, branchName) => {
   try {
+    const stackClient = stack
+      .Client(config)
+      .stack({ api_key: config.source_stack, management_token: config.management_token });
     if (types.indexOf(moduleName) > -1) {
       let iterateList;
       if (config.modules.dependency && config.modules.dependency[moduleName]) {
@@ -86,10 +100,10 @@ var singleExport = async (moduleName, types, config, branchName) => {
       }
       iterateList.push(moduleName);
 
-      for (let i = 0; i < iterateList.length; i++) {
-        var exportedModule = require('./lib/export/' + iterateList[i]);
-        const result = await exportedModule.start(config, branchName);
-        if (result && iterateList[i] === 'stack') {
+      for (let element of iterateList) {
+        const ExportModule = require('./lib/export/' + element);
+        const result = await new ExportModule(config, stackClient).start(config, branchName);
+        if (result && element === 'stack') {
           let master_locale = {
             master_locale: { code: result.code },
           };
@@ -103,42 +117,41 @@ var singleExport = async (moduleName, types, config, branchName) => {
     }
     return true;
   } catch (error) {
+    addlogs(config, `${formatError(error)}`, 'error');
     addlogs(config, 'Failed to migrate ' + moduleName, 'error');
-    addlogs(config, error, 'error');
     addlogs(config, 'The log for this is stored at ' + path.join(config.data, 'logs', 'export'), 'error');
-    throw error;
   }
 };
 
-var allExport = async (config, types, branchName) => {
-  // eslint-disable-next-line no-async-promise-executor
-  return new Promise(async (resolve, reject) => {
-    try {
-      for (let i = 0; i < types.length; i++) {
-        let type = types[i];
-        var exportedModule = require('./lib/export/' + type);
-        const result = await exportedModule.start(config, branchName);
-        if (result && type === 'stack') {
-          let master_locale = { master_locale: { code: result.code } };
-          config = _.merge(config, master_locale);
-        }
+const allExport = async (config, types, branchName) => {
+  try {
+    const stackClient = stack
+      .Client(config)
+      .stack({ api_key: config.source_stack, management_token: config.management_token });
+
+    for (let type of types) {
+      const ExportModule = require('./lib/export/' + type);
+      const result = await new ExportModule(config, stackClient).start(config, branchName);
+
+      if (result && type === 'stack') {
+        let master_locale = { master_locale: { code: result.code } };
+        config = _.merge(config, master_locale);
       }
-      addlogs(
-        config,
-        chalk.green('The content of the ' + config.source_stack + ' has been exported successfully!'),
-        'success',
-      );
-      addlogs(config, 'The log for this is stored at ' + path.join(config.data, 'logs', 'export'), 'success');
-      return resolve();
-    } catch (error) {
-      addlogs(
-        config,
-        chalk.red('Failed to migrate stack: ' + config.source_stack + '. Please check error logs for more info'),
-        'error',
-      );
-      addlogs(config, chalk.red(error && error.errorMessage), 'error');
-      addlogs(config, 'The log for this is stored at ' + path.join(config.data, 'logs', 'export'), 'error');
-      return reject(error);
     }
-  });
+    addlogs(
+      config,
+      chalk.green('The content of the ' + config.source_stack + ' has been exported successfully!'),
+      'success',
+    );
+    addlogs(config, 'The log for this is stored at ' + path.join(config.data, 'logs', 'export'), 'success');
+    return true;
+  } catch (error) {
+    addlogs(config, `${formatError(error)}`, 'error');
+    addlogs(
+      config,
+      chalk.red('Failed to migrate stack: ' + config.source_stack + '. Please check error logs for more info'),
+      'error',
+    );
+    addlogs(config, 'The log for this is stored at ' + path.join(config.data, 'logs', 'export'), 'error');
+  }
 };

package/src/commands/cm/stacks/export.js

@@ -10,6 +10,7 @@ const {
   parametersWithAuthToken,
   withoutParametersWithAuthToken,
 } = require('../../../lib/util/export-flags');
+const config = require('../../../config/default');
 const { configHandler } = require('@contentstack/cli-utilities');
 
 class ExportCommand extends Command {
@@ -32,6 +33,12 @@ class ExportCommand extends Command {
 
     if (alias) {
       let managementTokens = this.getToken(alias);
+
+      if (alias) {
+        const listOfTokens = configHandler.get('tokens');
+        config.management_token_data = listOfTokens[alias];
+      }
+
       if (managementTokens) {
         if (extConfig) {
           await configWithMToken(
@@ -72,9 +79,9 @@ class ExportCommand extends Command {
       }
     } else if (_authToken) {
       if (extConfig) {
-        configWithAuthToken(extConfig, _authToken, moduleName, host, contentTypes, branchName, securedAssets);
+        await configWithAuthToken(extConfig, _authToken, moduleName, host, contentTypes, branchName, securedAssets);
       } else if (sourceStack && data) {
-        return parametersWithAuthToken(
+        return await parametersWithAuthToken(
           _authToken,
           sourceStack,
           data,
@@ -85,7 +92,7 @@ class ExportCommand extends Command {
           securedAssets,
         );
       } else if (data === undefined && sourceStack === undefined) {
-        withoutParametersWithAuthToken(_authToken, moduleName, host, contentTypes, branchName, securedAssets);
+        await withoutParametersWithAuthToken(_authToken, moduleName, host, contentTypes, branchName, securedAssets);
       } else {
         this.log('Please provide a valid command. Run "csdx cm:export --help" command to view the command usage');
       }

package/src/config/default.js

@@ -1,6 +1,13 @@
 module.exports = {
   versioning: false,
   host: 'https://api.contentstack.io/v3',
+  developerHubUrls: {
+    // NOTE CDA url used as developer-hub url mapper to avoid conflict if user used any custom name
+    'https://api.contentstack.io': 'https://developerhub-api.contentstack.com',
+    'https://eu-api.contentstack.com': 'https://eu-developerhub-api.contentstack.com',
+    'https://azure-na-api.contentstack.com': 'https://azure-na-developerhub-api.contentstack.com',
+    'https://stag-api.csnonprod.com': 'https://stag-developerhub-api.csnonprod.com',
+  },
   // use below hosts for eu region
   // host:'https://eu-api.contentstack.com/v3',
   // use below hosts for azure-na region
@@ -15,15 +22,22 @@ module.exports = {
       'webhooks',
       'global-fields',
       'content-types',
+      'custom-roles',
       'workflows',
       'entries',
       'labels',
+      'marketplace-apps',
     ],
     locales: {
       dirName: 'locales',
       fileName: 'locales.json',
       requiredKeys: ['code', 'uid', 'name', 'fallback_locale'],
     },
+    customRoles: {
+      dirName: 'custom-roles',
+      fileName: 'custom-roles.json',
+      customRolesLocalesFileName: 'custom-roles-locales.json',
+    },
     environments: {
       dirName: 'environments',
       fileName: 'environments.json',
@@ -61,7 +75,7 @@ module.exports = {
       host: 'https://images.contentstack.io',
       invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by', '_metadata', 'published'],
       // no of asset version files (of a single asset) that'll be downloaded parallelly
-      downloadLimit: 5,
+      downloadLimit: 3,
       enableDownloadStatus: false,
     },
     content_types: {
@@ -101,6 +115,10 @@ module.exports = {
     dependency: {
       entries: ['stack', 'locales', 'content-types'],
     },
+    marketplace_apps: {
+      dirName: 'marketplace_apps',
+      fileName: 'marketplace_apps.json',
+    },
   },
   languagesCode: [
     'af-za',
@@ -327,4 +345,6 @@ module.exports = {
     stacks: '/stacks/',
   },
   preserveStackVersion: false,
+  fetchConcurrency: 5,
+  writeConcurrency: 5,
 };