@contentstack/cli-cm-bulk-publish 0.1.1-beta.1 → 0.1.1-beta.4

package/src/producer/unpublish.js

@@ -7,11 +7,10 @@ const {getQueue} = require('../util/queue')
 const defaults = require('../config/defaults.json')
 const req = require('../util/request')
 const {
-  bulkUnPublish, UnpublishEntry, UnpublishAsset, initializeLogger,
+  performBulkUnPublish, UnpublishEntry, UnpublishAsset, initializeLogger,
 } = require('../consumer/publish')
 const retryFailedLogs = require('../util/retryfailed')
 const {validateFile} = require('../util/fs')
-const types = 'asset_published,entry_published'
 const queue = getQueue()
 const entryQueue = getQueue()
 const assetQueue = getQueue()
@@ -28,7 +27,7 @@ let filePath
 function setConfig(conf, bup) {
   if (bup) {
     logFileName = 'bulk-unpublish'
-    queue.consumer = bulkUnPublish
+    queue.consumer = performBulkUnPublish
   } else {
     logFileName = 'unpublish'
     entryQueue.consumer = UnpublishEntry
@@ -54,71 +53,70 @@ function getQueryParams(filter) {
   return queryString
 }
 
-async function bulkAction(stack, items, bulkUnpublish, environment, locale) {
-  for (let index = 0; index < items.length; index++) {
-    changedFlag = true
-    if (items[index].data.publish_details) {
-      items[index].data.publish_details.version = items[index].data._version
-    }
-
-    if (bulkUnpublish) {
-      if (bulkUnPublishSet.length < 10 && items[index].type === 'entry_published') {
-        bulkUnPublishSet.push({
-          uid: items[index].data.uid,
-          content_type: items[index].content_type_uid,
-          locale: items[index].data.publish_details.locale || 'en-us',
-          version: items[index].data._version,
-          publish_details: [items[index].data.publish_details] || [],
-        })
-      }
+function bulkAction(stack, items, bulkUnpublish, environment, locale) {
+  return new Promise(async resolve => {
+    for (let index = 0; index < items.length; index++) {
+      changedFlag = true
+
+      if (bulkUnpublish) {
+        if (bulkUnPublishSet.length < 10 && items[index].type === 'entry_published') {
+          bulkUnPublishSet.push({
+            uid: items[index].data.uid,
+            content_type: items[index].content_type_uid,
+            locale: items[index].data.locale || 'en-us',
+            publish_details: [items[index].data.publish_details] || [],
+          })
+        }
 
-      if (bulkUnPulishAssetSet.length < 10 && items[index].type === 'asset_published') {
-        bulkUnPulishAssetSet.push({
-          uid: items[index].data.uid,
-          version: items[index].data._version,
-          publish_details: [items[index].data.publish_details] || [],
-        })
-      }
+        if (bulkUnPulishAssetSet.length < 10 && items[index].type === 'asset_published') {
+          bulkUnPulishAssetSet.push({
+            uid: items[index].data.uid,
+            version: items[index].data._version,
+            publish_details: [items[index].data.publish_details] || [],
+          })
+        }
 
-      if (bulkUnPulishAssetSet.length === 10) {
-        await queue.Enqueue({
-          assets: bulkUnPulishAssetSet, Type: 'asset', locale: locale, environments: [environment], stack: stack
-        })
-        bulkUnPulishAssetSet = []
-      }
+        if (bulkUnPulishAssetSet.length === 10) {
+          await queue.Enqueue({
+            assets: bulkUnPulishAssetSet, Type: 'asset', locale: locale, environments: [environment], stack: stack
+          })
+          bulkUnPulishAssetSet = []
+        }
 
-      if (bulkUnPublishSet.length === 10) {
-        await queue.Enqueue({
-          entries: bulkUnPublishSet, locale: locale, Type: 'entry', environments: [environment], stack: stack
-        })
-        bulkUnPublishSet = []
-      }
-      if (index === items.length - 1 && bulkUnPulishAssetSet.length <= 10 && bulkUnPulishAssetSet.length > 0) {
-        await queue.Enqueue({
-          assets: bulkUnPulishAssetSet, Type: 'asset', locale: locale, environments: [environment], stack: stack
-        })
-        bulkUnPulishAssetSet = []
-      }
+        if (bulkUnPublishSet.length === 10) {
+          await queue.Enqueue({
+            entries: bulkUnPublishSet, locale: locale, Type: 'entry', environments: [environment], stack: stack
+          })
+          bulkUnPublishSet = []
+        }
+        if (index === items.length - 1 && bulkUnPulishAssetSet.length <= 10 && bulkUnPulishAssetSet.length > 0) {
+          await queue.Enqueue({
+            assets: bulkUnPulishAssetSet, Type: 'asset', locale: locale, environments: [environment], stack: stack
+          })
+          bulkUnPulishAssetSet = []
+        }
 
-      if (index === items.length - 1 && bulkUnPublishSet.length <= 10 && bulkUnPublishSet.length > 0) {
-        await queue.Enqueue({
-          entries: bulkUnPublishSet, locale: locale, Type: 'entry', environments: [environment], stack: stack
-        })
-        bulkUnPublishSet = []
-      }
-    } else {
-      if (items[index].type === 'entry_published') {
-        await entryQueue.Enqueue({
-          content_type: items[index].content_type_uid, publish_details: [items[index].data.publish_details], environments: [environment], entryUid: items[index].data.uid, locale: items[index].data.publish_details.locale || 'en-us', Type: 'entry', stack: stack
-        })
-      }
-      if (items[index].type === 'asset_published') {
-        await assetQueue.Enqueue({
-          assetUid: items[index].data.uid, publish_details: [items[index].data.publish_details], environments: [environment], Type: 'entry', stack: stack
-        })
+        if (index === items.length - 1 && bulkUnPublishSet.length <= 10 && bulkUnPublishSet.length > 0) {
+          await queue.Enqueue({
+            entries: bulkUnPublishSet, locale: locale, Type: 'entry', environments: [environment], stack: stack
+          })
+          bulkUnPublishSet = []
+        }
+      } else {
+        if (items[index].type === 'entry_published') {
+          await entryQueue.Enqueue({
+            content_type: items[index].content_type_uid, publish_details: [items[index].data.publish_details], environments: [environment], entryUid: items[index].data.uid, locale: items[index].data.locale || 'en-us', Type: 'entry', stack: stack
+          })
+        }
+        if (items[index].type === 'asset_published') {
+          await assetQueue.Enqueue({
+            assetUid: items[index].data.uid, publish_details: [items[index].data.publish_details], environments: [environment], Type: 'entry', stack: stack
+          })
+        }
       }
     }
-  }
+    return resolve()
+  })
 }
 
 async function getSyncEntries(stack, config, locale, queryParams, bulkUnpublish, environment, deliveryToken, paginationToken = null) {
@@ -134,20 +132,19 @@ async function getSyncEntries(stack, config, locale, queryParams, bulkUnpublish,
       }
       const entriesResponse = await req(conf)
       if (entriesResponse.items.length > 0) {
-        bulkAction(stack, entriesResponse.items, bulkUnpublish, environment, locale)
+        await bulkAction(stack, entriesResponse.items, bulkUnpublish, environment, locale)
       }
       if (entriesResponse.items.length === 0) {
         if (!changedFlag) console.log('No Entries/Assets Found published on specified environment')
         return resolve()
       }
-      setTimeout(() => {
-        getSyncEntries(stack, config, locale, queryParams, bulkUnpublish, environment, deliveryToken, null)
+      setTimeout(async () => {
+        await getSyncEntries(stack, config, locale, queryParams, bulkUnpublish, environment, deliveryToken, null)
       }, 3000)
     } catch (error) {
       reject(error)
     }
   })
-  // return true
 }
 
 async function start({retryFailed, bulkUnpublish, contentType, locale, environment, deliveryToken, onlyAssets, onlyEntries, f_types}, stack, config) {
@@ -159,51 +156,47 @@ async function start({retryFailed, bulkUnpublish, contentType, locale, environme
     } else if (!isSuccessLogEmpty) {
       console.log(`The success log for this session is stored at ${filePath}.success`)
     }
-    process.exit(0)  
+    process.exit(0)
   })
 
-  try {
-    if (retryFailed) {
-      if (typeof retryFailed === 'string' && retryFailed.length > 0) {
-        if (!validateFile(retryFailed, ['unpublish', 'bulk-unpublish'])) {
-          return false
-        }
-
-        bulkUnpublish = retryFailed.match(new RegExp('bulk')) ? true : false
-        setConfig(config, bulkUnpublish)
-
-        if (bulkUnpublish) {
-          await retryFailedLogs(retryFailed, queue, 'bulk')
-        } else {
-          await retryFailedLogs(retryFailed, {entryQueue, assetQueue}, 'publish')
-        }
-      }
-    } else {
-      let filter = {
-        environment,
-        locale,
-      }
-      filter.type = (f_types) ? f_types : types // types mentioned in the config file (f_types) are given preference
-      if (contentType) {
-        filter.content_type_uid = contentType
-      }
-      if (onlyAssets) {
-        filter.type = 'asset_published'
-        delete filter.content_type_uid
-      }
-      if (onlyEntries) {
-        filter.type = 'entry_published'
+  if (retryFailed) {
+    if (typeof retryFailed === 'string' && retryFailed.length > 0) {
+      if (!validateFile(retryFailed, ['unpublish', 'bulk-unpublish'])) {
+        return false
       }
+
+      bulkUnpublish = retryFailed.match(new RegExp('bulk')) ? true : false
       setConfig(config, bulkUnpublish)
-      const queryParams = getQueryParams(filter)
-      try {
-        await getSyncEntries(stack, config, locale, queryParams, bulkUnpublish, environment, deliveryToken)
-      } catch (error) {
-        throw error
+
+      if (bulkUnpublish) {
+        await retryFailedLogs(retryFailed, queue, 'bulk')
+      } else {
+        await retryFailedLogs(retryFailed, {entryQueue, assetQueue}, 'publish')
       }
     }
-  } catch(error) {
-    throw error
+  } else {
+    let filter = {
+      environment,
+      locale,
+    }
+    if (f_types) {
+      filter.type = f_types
+    }
+    // filter.type = (f_types) ? f_types : types // types mentioned in the config file (f_types) are given preference
+    if (contentType) {
+      filter.content_type_uid = contentType
+      filter.type = 'entry_published'
+    }
+    if (onlyAssets) {
+      filter.type = 'asset_published'
+      delete filter.content_type_uid
+    }
+    if (onlyEntries) {
+      filter.type = 'entry_published'
+    }
+    setConfig(config, bulkUnpublish)
+    const queryParams = getQueryParams(filter)
+    await getSyncEntries(stack, config, locale, queryParams, bulkUnpublish, environment, deliveryToken)
   }
 }
 
@@ -214,4 +207,4 @@ module.exports = {
   setConfig,
   getQueryParams,
   start,
-}
+}

package/src/util/client.js

@@ -3,21 +3,26 @@ const {Command} = require('@contentstack/cli-command')
 const command = new Command()
 
 const Configstore = require('configstore')
+// eslint-disable-next-line no-unused-vars
 const config = new Configstore('contentstack_cli')
-const { formatHostname } = require('../util')
+const {formatHostname} = require('../util')
 
 function getStack(data) {
-	const tokenDetails = command.getToken(data.alias)
-	const client = contentstackSdk.client({
-		host: formatHostname(data.host),
-		logHandler: (level, data) => {}
-	})
-	const stack = client.stack({api_key: tokenDetails.apiKey, management_token: tokenDetails.token})
-	stack.alias = data.alias
-	stack.host = data.host
-	return stack
+  const tokenDetails = command.getToken(data.alias)
+  const client = contentstackSdk.client({
+    headers: {
+      branch: data.branch,
+    },
+    host: formatHostname(data.host),
+    // eslint-disable-next-line no-unused-vars
+    logHandler: level => {},
+  })
+  const stack = client.stack({api_key: tokenDetails.apiKey, management_token: tokenDetails.token})
+  stack.alias = data.alias
+  stack.host = data.host
+  return stack
 }
 
 module.exports = {
-  getStack
+  getStack,
 }

package/src/util/queue.js

@@ -7,8 +7,8 @@ class Queue extends EventEmitter {
     this.count = 0;
     this.store = [];
     this.config = {};
-    this.delay = 1.5
-    this.requestBatchSize = 10
+    this.delay = 1
+    this.requestBatchSize = 1
     this.on('dequeue', this.check);
   }
 

package/src/util/request.js

@@ -26,21 +26,23 @@ var makeCall = module.exports = function (req, RETRY) {
         let timeDelay
         if (response.statusCode >= 200 && response.statusCode <= 399) {
           return resolve(JSON.parse(response.body))
-        } if (response.statusCode === 429) {
+        }
+        if (response.statusCode === 429) {
           // eslint-disable-next-line no-mixed-operators
           timeDelay = Math.SQRT2 ** RETRY * 100
           debug(`API rate limit exceeded.\nReceived ${response.statusCode} status\nBody ${JSON.stringify(response)}`)
           debug(`Retrying ${req.uri || req.url} with ${timeDelay} sec delay`)
-          return setTimeout((req, RETRY) => makeCall(req, RETRY)
+          return setTimeout((req1, RETRY1) => makeCall(req1, RETRY1)
           .then(resolve)
           .catch(reject), timeDelay, req, RETRY)
-        } if (response.statusCode >= 500) {
+        }
+        if (response.statusCode >= 500) {
           // retry, with delay
           timeDelay = Math.SQRT2 ** RETRY * 100
           debug(`Recevied ${response.statusCode} status\nBody ${JSON.stringify(response)}`)
           debug(`Retrying ${req.uri || req.url} with ${timeDelay} sec delay`)
           RETRY++
-          return setTimeout((req, RETRY) => makeCall(req, RETRY)
+          return setTimeout((req2, RETRY2) => makeCall(req2, RETRY2)
           .then(resolve)
           .catch(reject), timeDelay, req, RETRY)
         }

package/src/util/store.js

@@ -52,11 +52,7 @@ function get(key, filePath) {
 
 function updateMissing(key, flags) {
   let savedConfig
-  try {
-    savedConfig = get(key, path.resolve(flags.config))
-  } catch(error) {
-    throw error
-  }
+  savedConfig = get(key, path.resolve(flags.config))
   Object.keys(savedConfig).forEach(element => {
     if (flags[element] === undefined) {
       console.log(`Using ${element} from config file`)
@@ -74,13 +70,13 @@ function updateMissing(key, flags) {
 // because both unpublish and cross-publish commands build the filter object
 // internally, and in the original bulk-publish script the filter object was
 // mentioned in the config file itself
-function handleFilterObj(config) {
-  config.environment = config.filter.environment
-  config.contentType = config.filter.content_type_uid
-  config.locale = config.filter.locale
-  config.f_types = config.filter.type // adding f_types to differentiate the types specified in the config.js, and the types defined internally in Unpublish and Cross Publish
-  delete config.filter
-  return config
+function handleFilterObj(config1) {
+  config1.environment = config1.filter.environment
+  config1.contentType = config1.filter.content_type_uid
+  config1.locale = config1.filter.locale
+  config1.f_types = config1.filter.type // adding f_types to differentiate the types specified in the config.js, and the types defined internally in Unpublish and Cross Publish
+  delete config1.filter
+  return config1
 }
 
 module.exports = {