@contentstack/cli-cm-export 1.1.0 → 1.2.0

package/README.md

@@ -38,7 +38,7 @@ $ npm install -g @contentstack/cli-cm-export
 $ csdx COMMAND
 running command...
 $ csdx (-v|--version|version)
-@contentstack/cli-cm-export/1.1.0 darwin-arm64 node-v16.17.0
+@contentstack/cli-cm-export/1.2.0 darwin-arm64 node-v18.11.0
 $ csdx --help [COMMAND]
 USAGE
   $ csdx COMMAND
@@ -83,5 +83,5 @@ EXAMPLES
   csdx cm:stacks:export --branch [optional] branch name
 ```
 
-_See code: [src/commands/cm/stacks/export.js](https://github.com/contentstack/cli/blob/v1.1.0/packages/contentstack-export/src/commands/cm/stacks/export.js)_
+_See code: [src/commands/cm/stacks/export.js](https://github.com/contentstack/cli/blob/main/packages/contentstack-export/src/commands/cm/stacks/export.js)_
 <!-- commandsstop -->

package/oclif.manifest.json

@@ -1 +1 @@
-{"version":"1.1.0","commands":{"cm:stacks:export":{"id":"cm:stacks:export","description":"Export content from a stack","usage":"cm:stacks:export [-c <value>] [-k <value>] [-d <value>] [-a <value>] [--module <value>] [--content-types <value>] [--branch <value>] [--secured-assets]","pluginName":"@contentstack/cli-cm-export","pluginType":"core","aliases":["cm:export"],"examples":["csdx cm:stacks:export --stack-api-key <stack_api_key> --data-dir <path/of/export/destination/dir>","csdx cm:stacks:export --config <path/to/config/dir>","csdx cm:stacks:export --alias <management_token_alias>","csdx cm:stacks:export --alias <management_token_alias> --data-dir <path/to/export/destination/dir>","csdx cm:stacks:export --alias <management_token_alias> --config <path/to/config/file>","csdx cm:stacks:export --module <single module name>","csdx cm:stacks:export --branch [optional] branch name"],"flags":{"config":{"name":"config","type":"option","char":"c","description":"[optional] path of the config"},"stack-uid":{"name":"stack-uid","type":"option","char":"s","description":"API key of the source stack","hidden":true},"stack-api-key":{"name":"stack-api-key","type":"option","char":"k","description":"API key of the source stack"},"data":{"name":"data","type":"option","description":"path or location to store the data","hidden":true},"data-dir":{"name":"data-dir","type":"option","char":"d","description":"path or location to store the data"},"alias":{"name":"alias","type":"option","char":"a","description":"alias of the management token"},"management-token-alias":{"name":"management-token-alias","type":"option","description":"alias of the management token","hidden":true},"auth-token":{"name":"auth-token","type":"boolean","char":"A","description":"to use auth token","hidden":true,"allowNo":false},"module":{"name":"module","type":"option","char":"m","description":"[optional] specific module name"},"content-types":{"name":"content-types","type":"option","char":"t","description":"[optional] content type"},"branch":{"name":"branch","type":"option","char":"B","description":"[optional] branch name"},"secured-assets":{"name":"secured-assets","type":"boolean","description":"[optional] use when assets are secured","allowNo":false}},"args":[]}}}
+{"version":"1.2.0","commands":{"cm:stacks:export":{"id":"cm:stacks:export","description":"Export content from a stack","usage":"cm:stacks:export [-c <value>] [-k <value>] [-d <value>] [-a <value>] [--module <value>] [--content-types <value>] [--branch <value>] [--secured-assets]","pluginName":"@contentstack/cli-cm-export","pluginType":"core","aliases":["cm:export"],"examples":["csdx cm:stacks:export --stack-api-key <stack_api_key> --data-dir <path/of/export/destination/dir>","csdx cm:stacks:export --config <path/to/config/dir>","csdx cm:stacks:export --alias <management_token_alias>","csdx cm:stacks:export --alias <management_token_alias> --data-dir <path/to/export/destination/dir>","csdx cm:stacks:export --alias <management_token_alias> --config <path/to/config/file>","csdx cm:stacks:export --module <single module name>","csdx cm:stacks:export --branch [optional] branch name"],"flags":{"config":{"name":"config","type":"option","char":"c","description":"[optional] path of the config"},"stack-uid":{"name":"stack-uid","type":"option","char":"s","description":"API key of the source stack","hidden":true},"stack-api-key":{"name":"stack-api-key","type":"option","char":"k","description":"API key of the source stack"},"data":{"name":"data","type":"option","description":"path or location to store the data","hidden":true},"data-dir":{"name":"data-dir","type":"option","char":"d","description":"path or location to store the data"},"alias":{"name":"alias","type":"option","char":"a","description":"alias of the management token"},"management-token-alias":{"name":"management-token-alias","type":"option","description":"alias of the management token","hidden":true},"auth-token":{"name":"auth-token","type":"boolean","char":"A","description":"to use auth token","hidden":true,"allowNo":false},"module":{"name":"module","type":"option","char":"m","description":"[optional] specific module name"},"content-types":{"name":"content-types","type":"option","char":"t","description":"[optional] content type"},"branch":{"name":"branch","type":"option","char":"B","description":"[optional] branch name"},"secured-assets":{"name":"secured-assets","type":"boolean","description":"[optional] use when assets are secured","allowNo":false}},"args":[]}}}

package/package.json

@@ -1,16 +1,17 @@
 {
   "name": "@contentstack/cli-cm-export",
   "description": "Contentstack CLI plugin to export content from stack",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "author": "Contentstack",
   "bugs": "https://github.com/contentstack/cli/issues",
   "dependencies": {
-    "@contentstack/cli-command": "^1.0.1",
-    "@contentstack/cli-utilities": "^1.0.2",
+    "@contentstack/cli-command": "^1.0.2",
+    "@contentstack/cli-utilities": "^1.0.3",
     "@contentstack/management": "^1.3.0",
     "@oclif/command": "^1.8.16",
     "@oclif/config": "^1.18.3",
     "async": "^3.2.4",
+    "big-json": "^3.2.0",
     "bluebird": "^3.7.2",
     "fs": "0.0.1-security",
     "is-valid-path": "^0.1.1",
@@ -18,15 +19,21 @@
     "mkdirp": "^1.0.4",
     "path": "^0.12.7",
     "progress-stream": "^2.0.0",
+    "promise-limit": "^2.7.0",
     "proxyquire": "^2.1.3",
     "winston": "^3.7.2"
   },
   "devDependencies": {
+    "@contentstack/cli-auth": "^1.0.3",
+    "@contentstack/cli-config": "^1.0.2",
+    "@contentstack/cli-dev-dependencies": "^1.0.0",
     "@oclif/dev-cli": "^1.22.2",
     "@oclif/plugin-help": "^5.1.12",
     "@oclif/test": "^1.2.6",
     "assert": "^2.0.0",
     "chai": "^4.2.0",
+    "dotenv": "^16.0.1",
+    "dotenv-expand": "^9.0.0",
     "eslint": "^8.18.0",
     "eslint-config-oclif": "^3.1.0",
     "globby": "^10.0.2",
@@ -54,7 +61,9 @@
     "postpack": "rm -f oclif.manifest.json",
     "prepack": "oclif-dev manifest && oclif-dev readme",
     "test": "nyc mocha --forbid-only \"test/**/*.test.js\"",
-    "version": "oclif-dev readme && git add README.md"
+    "version": "oclif-dev readme && git add README.md",
+    "test:integration": "mocha --forbid-only \"test/run.test.js\" --integration-test",
+    "test:unit": "mocha --forbid-only \"test/unit/*.test.js\" --unit-test"
   },
   "oclif": {
     "commands": "./src/commands",
@@ -62,7 +71,7 @@
     "devPlugins": [
       "@oclif/plugin-help"
     ],
-    "repositoryPrefix": "<%- repo %>/blob/v<%- version %>/packages/contentstack-export/<%- commandPath %>"
+    "repositoryPrefix": "<%- repo %>/blob/main/packages/contentstack-export/<%- commandPath %>"
   },
   "csdxConfig": {
     "expiredCommands": {

package/src/app.js

@@ -3,9 +3,11 @@ const _ = require('lodash');
 const path = require('path');
 const chalk = require('chalk');
 const util = require('./lib/util');
+const { formatError } = require('./lib/util');
 const login = require('./lib/util/login');
 const setupBranches = require('./lib/util/setup-branches');
 const { addlogs, unlinkFileLogger } = require('./lib/util/log');
+const stack = require('./lib/util/contentstack-management-sdk');
 
 exports.initial = async function (config) {
   return new Promise(async function (resolve, reject) {
@@ -29,7 +31,7 @@ exports.initial = async function (config) {
               await allExport(config, types, branch.uid);
             }
           } catch (error) {
-            console.log('failed export branch', branch.uid, error);
+            addlogs(config, `failed export contents ${branch.uid} ${formatError(error)}`, 'error');
           }
         }
       } else {
@@ -40,10 +42,10 @@ exports.initial = async function (config) {
             await allExport(config, types);
           }
         } catch (error) {
-          console.log('failed export contents', error && error.message);
+          addlogs(config, `failed export contents ${formatError(error)}`, 'error');
         }
       }
-    }
+    };
 
     // try {
     if (
@@ -55,8 +57,12 @@ exports.initial = async function (config) {
         .login(config)
         .then(async function () {
           // setup branches
-          await fetchBranchAndExport()
-          unlinkFileLogger();
+          try {
+            await fetchBranchAndExport();
+            unlinkFileLogger();
+          } catch (error) {
+            addlogs(config, `${formatError(error)}`, 'error');
+          }
           resolve();
         })
         .catch((error) => {
@@ -65,12 +71,16 @@ exports.initial = async function (config) {
             addlogs(config, chalk.red('Stack Api key ' + error.errors.api_key[0], 'Please enter valid Key', 'error'));
             addlogs(config, 'The log for this is stored at ' + config.data + '/export/logs', 'success');
           } else {
-            console.log('Stack fail to export');
+            addlogs(config, `${formatError(error)}`, 'error');
           }
-          reject(error);
+          // reject(error);
         });
     } else if (config.management_token) {
-      await fetchBranchAndExport()
+      try {
+        await fetchBranchAndExport();
+      } catch (error) {
+        addlogs(config, `${formatError(error)}`, 'error');
+      }
       resolve();
     }
   });
@@ -78,6 +88,9 @@ exports.initial = async function (config) {
 
 const singleExport = async (moduleName, types, config, branchName) => {
   try {
+    const stackClient = stack
+      .Client(config)
+      .stack({ api_key: config.source_stack, management_token: config.management_token });
     if (types.indexOf(moduleName) > -1) {
       let iterateList;
       if (config.modules.dependency && config.modules.dependency[moduleName]) {
@@ -87,12 +100,9 @@ const singleExport = async (moduleName, types, config, branchName) => {
       }
       iterateList.push(moduleName);
 
-      for (const element of iterateList) {
-        let exportedModule = require('./lib/export/' + element);
-        const result = await exportedModule.start(config, branchName)
-          .catch((error) => {
-            console.log(error && error.message)
-          });
+      for (let element of iterateList) {
+        const ExportModule = require('./lib/export/' + element);
+        const result = await new ExportModule(config, stackClient).start(config, branchName);
         if (result && element === 'stack') {
           let master_locale = {
             master_locale: { code: result.code },
@@ -107,44 +117,41 @@ const singleExport = async (moduleName, types, config, branchName) => {
     }
     return true;
   } catch (error) {
+    addlogs(config, `${formatError(error)}`, 'error');
     addlogs(config, 'Failed to migrate ' + moduleName, 'error');
-    addlogs(config, error, 'error');
     addlogs(config, 'The log for this is stored at ' + path.join(config.data, 'logs', 'export'), 'error');
-    throw error;
   }
 };
 
 const allExport = async (config, types, branchName) => {
-  return new Promise(async (resolve, reject) => {
-    try {
-      for (const type of types) {
-        const exportedModule = require('./lib/export/' + type);
-        const result = await exportedModule.start(config, branchName)
-          .catch((error) => {
-            console.log(error && error.message)
-          });
+  try {
+    const stackClient = stack
+      .Client(config)
+      .stack({ api_key: config.source_stack, management_token: config.management_token });
 
-        if (result && type === 'stack') {
-          let master_locale = { master_locale: { code: result.code } };
-          config = _.merge(config, master_locale);
-        }
+    for (let type of types) {
+      const ExportModule = require('./lib/export/' + type);
+      const result = await new ExportModule(config, stackClient).start(config, branchName);
+
+      if (result && type === 'stack') {
+        let master_locale = { master_locale: { code: result.code } };
+        config = _.merge(config, master_locale);
       }
-      addlogs(
-        config,
-        chalk.green('The content of the ' + config.source_stack + ' has been exported successfully!'),
-        'success',
-      );
-      addlogs(config, 'The log for this is stored at ' + path.join(config.data, 'logs', 'export'), 'success');
-      return resolve();
-    } catch (error) {
-      addlogs(
-        config,
-        chalk.red('Failed to migrate stack: ' + config.source_stack + '. Please check error logs for more info'),
-        'error',
-      );
-      addlogs(config, chalk.red(error && error.errorMessage), 'error');
-      addlogs(config, 'The log for this is stored at ' + path.join(config.data, 'logs', 'export'), 'error');
-      return reject(error);
     }
-  });
+    addlogs(
+      config,
+      chalk.green('The content of the ' + config.source_stack + ' has been exported successfully!'),
+      'success',
+    );
+    addlogs(config, 'The log for this is stored at ' + path.join(config.data, 'logs', 'export'), 'success');
+    return true;
+  } catch (error) {
+    addlogs(config, `${formatError(error)}`, 'error');
+    addlogs(
+      config,
+      chalk.red('Failed to migrate stack: ' + config.source_stack + '. Please check error logs for more info'),
+      'error',
+    );
+    addlogs(config, 'The log for this is stored at ' + path.join(config.data, 'logs', 'export'), 'error');
+  }
 };

package/src/commands/cm/stacks/export.js

@@ -79,9 +79,9 @@ class ExportCommand extends Command {
       }
     } else if (_authToken) {
       if (extConfig) {
-        configWithAuthToken(extConfig, _authToken, moduleName, host, contentTypes, branchName, securedAssets);
+        await configWithAuthToken(extConfig, _authToken, moduleName, host, contentTypes, branchName, securedAssets);
       } else if (sourceStack && data) {
-        return parametersWithAuthToken(
+        return await parametersWithAuthToken(
           _authToken,
           sourceStack,
           data,
@@ -92,7 +92,7 @@ class ExportCommand extends Command {
           securedAssets,
         );
       } else if (data === undefined && sourceStack === undefined) {
-        withoutParametersWithAuthToken(_authToken, moduleName, host, contentTypes, branchName, securedAssets);
+        await withoutParametersWithAuthToken(_authToken, moduleName, host, contentTypes, branchName, securedAssets);
       } else {
         this.log('Please provide a valid command. Run "csdx cm:export --help" command to view the command usage');
       }

package/src/config/default.js

@@ -6,7 +6,7 @@ module.exports = {
     'https://api.contentstack.io': 'https://developerhub-api.contentstack.com',
     'https://eu-api.contentstack.com': 'https://eu-developerhub-api.contentstack.com',
     'https://azure-na-api.contentstack.com': 'https://azure-na-developerhub-api.contentstack.com',
-    'https://stag-api.csnonprod.com': 'https://stag-developerhub-api.csnonprod.com'
+    'https://stag-api.csnonprod.com': 'https://stag-developerhub-api.csnonprod.com',
   },
   // use below hosts for eu region
   // host:'https://eu-api.contentstack.com/v3',
@@ -26,7 +26,7 @@ module.exports = {
       'workflows',
       'entries',
       'labels',
-      'marketplace-apps'
+      'marketplace-apps',
     ],
     locales: {
       dirName: 'locales',
@@ -75,7 +75,7 @@ module.exports = {
       host: 'https://images.contentstack.io',
       invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by', '_metadata', 'published'],
       // no of asset version files (of a single asset) that'll be downloaded parallelly
-      downloadLimit: 5,
+      downloadLimit: 3,
       enableDownloadStatus: false,
     },
     content_types: {
@@ -117,8 +117,8 @@ module.exports = {
     },
     marketplace_apps: {
       dirName: 'marketplace_apps',
-      fileName: 'marketplace_apps.json'
-    }
+      fileName: 'marketplace_apps.json',
+    },
   },
   languagesCode: [
     'af-za',
@@ -344,5 +344,7 @@ module.exports = {
     webhooks: '/webhooks/',
     stacks: '/stacks/',
   },
-  preserveStackVersion: false
+  preserveStackVersion: false,
+  fetchConcurrency: 5,
+  writeConcurrency: 5,
 };