@contentstack/cli-cm-export-to-csv 0.1.0-beta → 0.1.0-beta.3

package/oclif.manifest.json

@@ -1 +1 @@
-{"version":"0.1.0-beta","commands":{"cm:export-to-csv":{"id":"cm:export-to-csv","description":"Export entries or organization users to csv using this command\n","pluginName":"@contentstack/cli-cm-export-to-csv","pluginType":"core","aliases":[],"flags":{},"args":[]}}}
+{"version":"0.1.0-beta.3","commands":{"cm:export-to-csv":{"id":"cm:export-to-csv","description":"Export entries or organization users to csv using this command\n","pluginName":"@contentstack/cli-cm-export-to-csv","pluginType":"core","aliases":[],"flags":{},"args":[]}}}

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@contentstack/cli-cm-export-to-csv",
   "description": "Export entities to csv",
-  "version": "0.1.0-beta",
+  "version": "0.1.0-beta.3",
   "author": "Abhinav Gupta @abhinav-from-contentstack",
   "bugs": "https://github.com/contentstack/cli/issues",
   "dependencies": {
-    "@contentstack/cli-command": "^0.1.1-beta.1",
-    "@contentstack/management": "^1.2.1",
+    "@contentstack/cli-command": "^0.1.1-beta.6",
+    "@contentstack/management": "^1.3.0",
     "@oclif/command": "^1.8.0",
     "@oclif/config": "^1.17.0",
     "axios": "^0.21.1",

package/src/commands/cm/export-to-csv.js

@@ -24,8 +24,14 @@ class ExportToCsvCommand extends Command {
 				const environments = await util.getEnvironments(this.managementAPIClient, stack.apiKey) // fetch environments, because in publish details only env uid are available and we need env names
         while(contentTypes.length > 0) {
           let contentType = contentTypes.shift()
-          let entries = await util.getEntries(this.managementAPIClient, stack.apiKey, contentType, language.code) // fetch entries
-          let flatEntries = util.cleanEntries(entries.items, language.code, environments, contentType); // clean entries to be wderitten to file
+
+          const entriesCount = await util.getEntriesCount(this.managementAPIClient, stack.apiKey, contentType, language.code);
+          let flatEntries = [];
+          for (let index = 0; index < entriesCount / 100; index++) {
+            const entriesResult = await util.getEntries(this.managementAPIClient, stack.apiKey, contentType, language.code, index);
+            const flatEntriesResult = util.cleanEntries(entriesResult.items, language.code, environments, contentType);
+            flatEntries = flatEntries.concat(flatEntriesResult);
+          }
           // let dateTime = util.getDateTime()
           // let fileName = `${contentType}_${language.code}_entries_export_${dateTime}.csv`
           let fileName = `${stack.name}_${contentType}_${language.code}_entries_export.csv`

package/src/util/index.js

@@ -173,18 +173,30 @@ function getLanguages(managementAPIClient, stackApiKey) {
 	})
 }
 
-function getEntries(managementAPIClient, stackApiKey, contentType, language) {
+function getEntries(managementAPIClient, stackApiKey, contentType, language, skip) {
 	return new Promise(resolve => {
 		managementAPIClient
 		.stack({api_key: stackApiKey})
 		.contentType(contentType)
 		.entry()
-		.query({include_publish_details: true, locale: language})
+		.query({include_publish_details: true, locale: language, skip: skip * 100})
 		.find()
 		.then(entries => resolve(entries))
 	})
 }
 
+function getEntriesCount(managementAPIClient, stackApiKey, contentType, language) {
+	return new Promise((resolve) => {
+	  managementAPIClient
+		.stack({ api_key: stackApiKey })
+		.contentType(contentType)
+		.entry()
+		.query({ include_publish_details: true, locale: language })
+		.count()
+		.then((entriesData) => resolve(entriesData.entries));
+	});
+  }
+
 function getEnvironments(managementAPIClient, stackApiKey) {
 	let result = {}
 	return managementAPIClient.stack({api_key: stackApiKey}).environment().query().find().then(environments => {
@@ -215,6 +227,7 @@ function cleanEntries(entries, language, environments, contentTypeUid) {
       workflow = entry['_workflow']['name']
       delete entry['_workflow']
     }
+    entry = flatten(entry)
     entry['publish_details'] = envArr
     entry['_workflow'] = workflow
     entry['ACL'] = JSON.stringify({}) // setting ACL to empty obj
@@ -227,6 +240,7 @@ function cleanEntries(entries, language, environments, contentTypeUid) {
     delete entry.publish
     delete entry.unpublish
     delete entry.import
+    delete entry.publishRequest
     return entry
   })
 
@@ -366,6 +380,31 @@ function kebabize(str) {
    return str.split(' ').map((word) => word.toLowerCase()).join('-')
 }
 
+// https://stackoverflow.com/questions/19098797/fastest-way-to-flatten-un-flatten-nested-json-objects
+function flatten(data) {
+    var result = {};
+    function recurse (cur, prop) {
+        if (Object(cur) !== cur) {
+            result[prop] = cur;
+        } else if (Array.isArray(cur)) {
+             for(var i=0, l=cur.length; i<l; i++)
+                 recurse(cur[i], prop + "[" + i + "]");
+            if (l == 0)
+                result[prop] = [];
+        } else {
+            var isEmpty = true;
+            for (var p in cur) {
+                isEmpty = false;
+                recurse(cur[p], prop ? prop+"."+p : p);
+            }
+            if (isEmpty && prop)
+                result[prop] = {};
+        }
+    }
+    recurse(data, "");
+    return result;
+}
+
 module.exports = {
 	chooseOrganization: chooseOrganization,
 	chooseStack: chooseStack,
@@ -385,4 +424,6 @@ module.exports = {
   determineUserOrgRole: determineUserOrgRole,
   getOrganizationsWhereUserIsAdmin: getOrganizationsWhereUserIsAdmin,
   kebabize: kebabize,
+  flatten: flatten,
+  getEntriesCount: getEntriesCount,
 }