closer-cli 2.16.6 → 2.17.0

package/cmds/importData_cmds/multiple.js

@@ -14,13 +14,14 @@ const {
   importCsvFromFile,
   validateCsvFromFile,
   refreshOrders,
-  truncateTablesByCsvTypes,
   download,
   callBeforeDataImportProcess,
   callAfterDataImportProcess,
   getExecutionTime,
   getArchiveFolder,
-  renameWithTimestamp
+  renameWithTimestamp,
+  sendImportErrorMail,
+  deleteRowsBeforeImportDate
 } = require('../../helpers')
 
 exports.command = 'multiple [options]'
@@ -59,12 +60,13 @@ exports.handler = async argv => {
   argv.out = argv.out || getArchiveFolder(argv.endpoint)
 
   const hrstart = process.hrtime()
-  const report = {
-    start: new Date(),
-    end: null,
+  const importProcess = {
+    importProcessStart: new Date(),
+    importProcessEnd: null,
     time: null,
     endpoint: argv.endpoint,
     mode: null,
+    name: argv.name,
     type: [],
     result: []
   }
@@ -121,6 +123,8 @@ exports.handler = async argv => {
     })
   }
 
+  importProcess.type = files.map(f => f.csvType)
+
   let mode
   if (interactive && !argv.mode) {
     const answers = await prompts(
@@ -151,6 +155,8 @@ exports.handler = async argv => {
     mode = argv.mode
   }
 
+  importProcess.mode = mode
+
   const spinner = new Ora()
 
   const validations = []
@@ -178,7 +184,13 @@ exports.handler = async argv => {
   // Validate CSV headers sequentially
   //
   for (const item of parsedFiles) {
-    const validation = await validateCsvFromFile(item.file, item.csvType, ws, argv.csvDelimiter)
+    const validation = await validateCsvFromFile(
+      item.file,
+      item.csvType,
+      ws,
+      argv.csvDelimiter,
+      importProcess
+    )
     validations.push(validation)
   }
 
@@ -215,67 +227,76 @@ exports.handler = async argv => {
   //
   if (!argv.skipTasksBeforeImport) {
     try {
-      await callBeforeDataImportProcess(spinner, ws)
+      await callBeforeDataImportProcess(spinner, ws, importProcess)
     } catch (e) {
       process.exit()
     }
   }
 
   //
-  // Import FULL?
+  // Import CSV files sequentially
   //
-  if (mode === 'full') {
-    const csvTypes = parsedFiles.map(i => i.csvType)
-    await truncateTablesByCsvTypes(csvTypes, spinner, ws)
+  const partitionsForCsvType = new Map()
+
+  for (const item of parsedFiles) {
+    const csvReport = await importCsvFromFile({
+      file: item.file,
+      csvType: item.csvType,
+      outputDir: argv.out,
+      spinner,
+      ws,
+      delimiter: argv.csvDelimiter,
+      importProcess
+    })
+    csvReports.push(csvReport)
+    if (csvReport.countFailed > 0 || csvReport.countSkipped > 0) {
+      sendImportErrorMail(ws, importProcess, csvReport)
+    }
+    partitionsForCsvType.set(csvReport.csvType, Array.from(csvReport.partitions))
   }
 
   //
-  // Import CSV files sequentially
+  // Import FULL?
   //
-  for (const item of parsedFiles) {
-    const csvReport = await importCsvFromFile(
-      item.file,
-      item.csvType,
-      argv.out,
+  if (mode === 'full') {
+    await deleteRowsBeforeImportDate(
+      importProcess,
+      Object.fromEntries(partitionsForCsvType),
       spinner,
-      ws,
-      argv.csvDelimiter
+      ws
     )
-    csvReports.push(csvReport)
   }
 
   //
   // Refresh Orders of type Draft
   //
   // eslint-disable-next-line no-unused-vars
-  const refreshedOrders = await refreshOrders(spinner, ws)
+  const refreshedOrders = await refreshOrders(spinner, ws, importProcess.name)
 
   //
-  // Report
+  // importProcess
   //
 
-  report.end = new Date()
-  report.time = getExecutionTime(hrstart)
-  report.mode = mode
-  report.type = files.map(f => f.csvType)
+  importProcess.importProcessEnd = new Date()
+  importProcess.time = getExecutionTime(hrstart)
 
-  const reportRows = []
+  const importProcessRows = []
 
   csvReports.forEach(csvReport => {
-    reportRows.push({
+    importProcessRows.push({
       type: 'import.csv',
       payload: csvReport
     })
   })
 
-  reportRows.push({
+  importProcessRows.push({
     type: 'refreshed.orders',
     payload: refreshedOrders
   })
 
-  report.result = reportRows
+  importProcess.result = importProcessRows
 
-  await callAfterDataImportProcess(spinner, ws, report)
+  await callAfterDataImportProcess(spinner, ws, importProcess)
 
   await renameWithTimestamp(argv.out)
 

package/cmds/importData_cmds/single.js

@@ -15,7 +15,8 @@ const {
   download,
   validateCsvFromFile,
   getArchiveFolder,
-  renameWithTimestamp
+  renameWithTimestamp,
+  sendImportErrorMail
 } = require('../../helpers')
 
 exports.command = 'single [options]'
@@ -56,12 +57,13 @@ exports.handler = async argv => {
   const spinner = new Ora()
   const hrstart = process.hrtime()
 
-  const report = {
-    start: new Date(),
-    end: null,
+  const importProcess = {
+    importProcessStart: new Date(),
+    importProcessEnd: null,
     time: null,
     endpoint: argv.endpoint,
     mode: null,
+    name: argv.name,
     type: [],
     result: []
   }
@@ -158,61 +160,66 @@ exports.handler = async argv => {
     spinner.succeed('Downloaded file')
   }
 
-  const validation = await validateCsvFromFile(filepath, type, ws, argv.csvDelimiter)
+  const validation = await validateCsvFromFile(filepath, type, ws, argv.csvDelimiter, importProcess)
 
   if (validation.error) {
     console.log(`ERROR => [${chalk.red(validation.inputFile)}] ${validation.error}`)
     process.exit()
   }
 
+  importProcess.mode = mode
+  importProcess.type = [type]
+
   //
   // Call ImportService.beforeImportProcess (import flag)
   //
   if (!argv.skipTasksBeforeImport) {
     try {
-      await callBeforeDataImportProcess(spinner, ws)
+      await callBeforeDataImportProcess(spinner, ws, importProcess)
     } catch (e) {
       process.exit()
     }
   }
 
   if (argv.fromFile) {
-    csvReport = await importCsvFromFile(
-      argv.fromFile,
-      type,
-      argv.out,
+    csvReport = await importCsvFromFile({
+      file: argv.fromFile,
+      csvType: type,
+      outputDir: argv.out,
       spinner,
       ws,
-      argv.csvDelimiter
-    )
+      delimiter: argv.csvDelimiter,
+      importProcess
+    })
+    if (csvReport.countFailed > 0 || csvReport.countSkipped > 0) {
+      sendImportErrorMail(ws, importProcess, csvReport)
+    }
   }
 
-  const refreshedOrders = await refreshOrders(spinner, ws)
+  const refreshedOrders = await refreshOrders(spinner, ws, importProcess)
 
   //
   // Report
   //
 
-  report.end = new Date()
-  report.time = getExecutionTime(hrstart)
-  report.mode = mode
-  report.type = [type]
+  importProcess.importProcessEnd = new Date()
+  importProcess.time = getExecutionTime(hrstart)
 
-  const reportRows = []
+  const importProcessRows = []
 
-  reportRows.push({
+  importProcessRows.push({
     type: 'import.csv',
     payload: csvReport
   })
 
-  reportRows.push({
+  importProcessRows.push({
     type: 'refreshed.orders',
     payload: refreshedOrders
   })
 
-  report.result = reportRows
+  importProcess.result = importProcessRows
 
-  await callAfterDataImportProcess(spinner, ws, report)
+  await callAfterDataImportProcess(spinner, ws, importProcess)
 
   await renameWithTimestamp(argv.out)
 

package/helpers.js

@@ -47,11 +47,20 @@ exports.getCsvFilesFromDir = getCsvFilesFromDir
 exports.getGoogleSheetCsvUrl = (docId, sheet) =>
   `https://docs.google.com/spreadsheets/d/${docId}/gviz/tq?tqx=out:csv&sheet=${sheet}`
 
-exports.importCsvFromFile = (file, csvType, outputDir, spinner, ws, delimiter = ';') =>
+exports.importCsvFromFile = ({
+  file,
+  csvType,
+  outputDir,
+  spinner,
+  ws,
+  delimiter = ';',
+  importProcess
+}) =>
   new Promise(resolve => {
     const socket = this.createSocketConnection(ws.endpoint, ws.adminSecret)
 
     const hrstart = process.hrtime()
+    const importFileStart = new Date()
 
     let readStream = null
 
@@ -73,16 +82,22 @@ exports.importCsvFromFile = (file, csvType, outputDir, spinner, ws, delimiter =
     let countImported = 0
     let countFailed = 0
     let countSkipped = 0
+    const partitions = new Set()
 
     spinner.start('Loading...')
 
-    const importRowSuccess = () => {
+    const importRowSuccess = csvRow => {
+      partitions.add(csvRow.Partition)
+
       countImported += 1
 
       spinner.text = this.printImportProcessResult({
+        csvType,
         countImported,
         countFailed,
         countSkipped,
+        importFileStart,
+        importFileEnd: new Date(),
         executionTime: this.getExecutionTime(hrstart),
         name: filename
       })
@@ -91,12 +106,17 @@ exports.importCsvFromFile = (file, csvType, outputDir, spinner, ws, delimiter =
     }
 
     const importRowError = csvRow => {
+      partitions.add(csvRow.Partition)
+
       countFailed += 1
 
       spinner.text = this.printImportProcessResult({
+        csvType,
         countImported,
         countFailed,
         countSkipped,
+        importFileStart,
+        importFileEnd: new Date(),
         executionTime: this.getExecutionTime(hrstart),
         name: filename
       })
@@ -112,12 +132,17 @@ exports.importCsvFromFile = (file, csvType, outputDir, spinner, ws, delimiter =
       readStream.resume()
     }
 
-    const importFatal = errorMessage => {
+    const importFatal = (csvRow, errorMessage) => {
+      partitions.add(csvRow.Partition)
+
       spinner.fail(
         this.printImportProcessResult({
+          csvType,
           countImported,
           countFailed,
           countSkipped,
+          importFileStart,
+          importFileEnd: new Date(),
           executionTime: this.getExecutionTime(hrstart),
           name: filename,
           fatalError: errorMessage
@@ -130,8 +155,11 @@ exports.importCsvFromFile = (file, csvType, outputDir, spinner, ws, delimiter =
         countFailed,
         countSkipped,
         executionTime: this.getExecutionTime(hrstart),
+        importFileStart,
+        importFileEnd: new Date(),
         inputFile: file,
-        fatalError: `FATAL: ${errorMessage}`
+        fatalError: `FATAL: ${errorMessage}`,
+        partitions
       })
 
       readStream.destroy()
@@ -154,8 +182,11 @@ exports.importCsvFromFile = (file, csvType, outputDir, spinner, ws, delimiter =
         countImported,
         countFailed,
         countSkipped,
+        importFileStart,
+        importFileEnd: new Date(),
         executionTime: this.getExecutionTime(hrstart),
-        inputFile: file
+        inputFile: file,
+        partitions
       })
     }
 
@@ -181,10 +212,13 @@ exports.importCsvFromFile = (file, csvType, outputDir, spinner, ws, delimiter =
 
                 if (rows.length > 0) {
                   const packet = {
-                    csvRow: rows[0],
-                    csvType,
-                    lineNumber,
-                    type: 'importCsvRow'
+                    type: 'importCsvRow',
+                    payload: {
+                      csvRow: rows[0],
+                      csvType,
+                      lineNumber,
+                      importProcess
+                    }
                   }
                   socket.send(JSON.stringify(packet))
                 } else {
@@ -209,13 +243,13 @@ exports.importCsvFromFile = (file, csvType, outputDir, spinner, ws, delimiter =
 
       switch (type) {
         case 'importRowSuccess':
-          importRowSuccess()
+          importRowSuccess(payload.csvRow)
           break
         case 'importRowError':
           importRowError(payload.csvRow)
           break
         case 'importRowFatal':
-          importFatal(payload.error)
+          importFatal(payload.csvRow, payload.error)
           break
         default:
           break
@@ -227,10 +261,22 @@ exports.importCsvFromFile = (file, csvType, outputDir, spinner, ws, delimiter =
     })
   })
 
-exports.validateCsvFromFile = (file, csvType, ws, delimiter = ';') =>
+exports.sendImportErrorMail = (ws, importProcess, csvReport) => {
+  const socket = this.createSocketConnection(ws.endpoint, ws.adminSecret)
+
+  socket.on('open', () => {
+    socket.send(
+      JSON.stringify({
+        type: 'sendImportErrorMail',
+        payload: { csvReport, importProcess }
+      })
+    )
+  })
+}
+
+exports.validateCsvFromFile = (file, csvType, ws, delimiter = ';', importProcess) =>
   new Promise(resolve => {
     const socket = this.createSocketConnection(ws.endpoint, ws.adminSecret)
-
     const hrstart = process.hrtime()
 
     let readStream = null
@@ -268,9 +314,12 @@ exports.validateCsvFromFile = (file, csvType, ws, delimiter = ';') =>
                 })
 
                 const packet = {
-                  csvColumns: Object.keys(rows[0]),
-                  csvType,
-                  type: 'validateCsvHeaderRequest'
+                  type: 'validateCsvHeaderRequest',
+                  payload: {
+                    csvColumns: Object.keys(rows[0]),
+                    csvType,
+                    importProcess
+                  }
                 }
 
                 socket.send(JSON.stringify(packet))
@@ -324,15 +373,15 @@ exports.validateCsvFromFile = (file, csvType, ws, delimiter = ';') =>
     })
   })
 
-exports.importCsvFromUrl = async (url, csvType, outputDir, spinner, ws) => {
+exports.importCsvFromUrl = async (url, csvType, outputDir, importProcess, spinner, ws) => {
   spinner.start('Downloading file...')
-  const downloadedFile = await this.download(url, outputDir, `${csvType}.csv`)
+  const file = await this.download(url, outputDir, `${csvType}.csv`)
   spinner.succeed('Downloaded file')
-  return this.importCsvFromFile(downloadedFile, csvType, outputDir, spinner, ws)
+  return this.importCsvFromFile({ file, csvType, outputDir, spinner, ws, importProcess })
 }
 
-exports.getExecutionTime = start => {
-  const hrend = process.hrtime(start)
+exports.getExecutionTime = importProcessStart => {
+  const hrend = process.hrtime(importProcessStart)
   return ((hrend[0] * 1e9 + hrend[1]) / 1e9).toFixed(1)
 }
 
@@ -419,7 +468,7 @@ exports.colorGreaterThanZero = (value, color) => {
   return value
 }
 
-exports.refreshOrders = (spinner, ws) =>
+exports.refreshOrders = (spinner, ws, importProcess) =>
   new Promise(resolve => {
     const hrstart = process.hrtime()
 
@@ -428,7 +477,7 @@ exports.refreshOrders = (spinner, ws) =>
     spinner.start('Refreshing draft orders...')
 
     socket.on('open', () => {
-      socket.send(JSON.stringify({ type: 'refreshDraftOrdersRequest' }))
+      socket.send(JSON.stringify({ type: 'refreshDraftOrdersRequest', payload: { importProcess } }))
     })
 
     socket.on('message', packet => {
@@ -444,6 +493,7 @@ exports.refreshOrders = (spinner, ws) =>
             totalCount: payload.totalCount,
             refreshed: payload.refreshed,
             errored: payload.errored,
+            name: importProcess.name,
             executionTime: this.getExecutionTime(hrstart)
           })
           break
@@ -476,20 +526,25 @@ exports.printRefreshedOrders = ({ refreshed, errored, executionTime }) => {
   return message
 }
 
-exports.truncateTablesByCsvTypes = (csvTypes, spinner, ws) =>
+exports.deleteRowsBeforeImportDate = (importProcess, partitionsForCsvType, spinner, ws) =>
   new Promise(resolve => {
     const socket = this.createSocketConnection(ws.endpoint, ws.adminSecret)
 
-    spinner.start('Deleting database records...')
+    spinner.start('Deleting old database records for chosen partitions...')
 
     socket.on('open', () => {
-      socket.send(JSON.stringify({ type: 'truncateTablesByCsvTypesRequest', csvTypes }))
+      socket.send(
+        JSON.stringify({
+          type: 'deleteRowsBeforeImportDateRequest',
+          payload: { importProcess, partitionsForCsvType }
+        })
+      )
     })
 
     socket.on('message', packet => {
       const { type } = JSON.parse(packet)
       switch (type) {
-        case 'truncateTablesByCsvTypesResponse': {
+        case 'deleteRowsBeforeImportDateResponse': {
           spinner.succeed('Database records deleted (soft)')
           resolve()
           break
@@ -530,14 +585,16 @@ exports.download = async (url, destPath, fileName) =>
     downloadStream.pipe(fileWriterStream)
   })
 
-exports.callBeforeDataImportProcess = (spinner, ws) =>
+exports.callBeforeDataImportProcess = (spinner, ws, importProcess) =>
   new Promise((resolve, reject) => {
     const socket = this.createSocketConnection(ws.endpoint, ws.adminSecret)
 
     spinner.start('Executing tasks before start...')
 
     socket.on('open', () => {
-      socket.send(JSON.stringify({ type: 'callBeforeDataImportProcessRequest' }))
+      socket.send(
+        JSON.stringify({ type: 'callBeforeDataImportProcessRequest', payload: { importProcess } })
+      )
     })
 
     socket.on('message', packet => {
@@ -563,14 +620,16 @@ exports.callBeforeDataImportProcess = (spinner, ws) =>
     })
   })
 
-exports.callAfterDataImportProcess = (spinner, ws, report) =>
+exports.callAfterDataImportProcess = (spinner, ws, importProcess) =>
   new Promise(resolve => {
     const socket = this.createSocketConnection(ws.endpoint, ws.adminSecret)
 
     spinner.start('Executing tasks after the end process...')
 
     socket.on('open', () => {
-      socket.send(JSON.stringify({ type: 'callAfterDataImportProcessRequest', payload: report }))
+      socket.send(
+        JSON.stringify({ type: 'callAfterDataImportProcessRequest', payload: { importProcess } })
+      )
     })
 
     socket.on('message', packet => {
@@ -725,7 +784,7 @@ exports.flushMedia = (spinner, ws) =>
 
 exports.getArchiveFolder = instanceEndpoint => {
   const prefix = 'closer'
-  const instance = slugify(instanceEndpoint.split('/')[0].split('.').join('-'), { lower: true })
+  const instance = slugify(instanceEndpoint.split('/')[0], { lower: true })
   const ts = getTimestampForFilename()
   return `${prefix}_${instance}/${ts}`
 }

package/index.js

@@ -22,5 +22,11 @@ require('yargs/yargs')(process.argv.slice(2))
     describe: 'Admin Secret for Closer',
     demandOption: true
   })
+  .option('n', {
+    alias: 'name',
+    describe: 'Attach string name label to import process',
+    type: 'string',
+    demandOption: true
+  })
   .help()
   .version().argv

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "closer-cli",
-  "version": "2.16.6",
+  "version": "2.17.0",
   "description": "",
   "main": "index.js",
   "bin": {

package/cmds/exportData.js

@@ -1,84 +0,0 @@
-/* eslint-disable no-console */
-/* eslint-disable no-await-in-loop */
-/* eslint-disable no-restricted-syntax */
-
-const chalk = require('chalk')
-const Ora = require('ora')
-const prompts = require('prompts')
-const { ALLOWED_CSV_FILES, exportCsv } = require('../helpers')
-
-exports.command = 'export:data'
-exports.desc = 'Export data from Closer'
-
-exports.builder = yargs => {
-  yargs.usage(`Usage: ${chalk.cyan('$0 export:data')} [options]`)
-}
-
-exports.handler = async argv => {
-  const interactive = !argv.nonInteractive
-
-  const csvChoices = Object.keys(ALLOWED_CSV_FILES).map(csvType => ({
-    title: csvType,
-    csvType,
-    value: csvType
-  }))
-
-  let csvTypes
-  if (interactive && !argv.type) {
-    const answers = await prompts(
-      [
-        {
-          instructions: false,
-          name: 'csvTypes',
-          type: 'multiselect',
-          message: 'Select CSV type',
-          choices: csvChoices,
-          min: 1
-        }
-      ],
-      { onCancel: () => process.exit() }
-    )
-
-    csvTypes = answers.csvTypes
-  } else {
-    csvTypes = argv.type.map(type => {
-      const choice = csvChoices.find(csvChoice => csvChoice.csvType === type)
-      return choice.value
-    })
-  }
-
-  const spinner = new Ora()
-  // const hrstart = process.hrtime()
-
-  const ws = {
-    endpoint: argv.endpoint,
-    adminSecret: argv['admin-secret']
-  }
-
-  if (interactive) {
-    console.log('')
-    const { confirmed } = await prompts(
-      [
-        {
-          type: 'confirm',
-          name: 'confirmed',
-          message: 'Continue?',
-          initial: false
-        }
-      ],
-      { onCancel: () => process.exit() }
-    )
-
-    if (!confirmed) {
-      process.exit()
-    }
-  }
-
-  console.log('')
-
-  for (const csvType of csvTypes) {
-    await exportCsv(csvType, argv.out, spinner, ws, argv.csvDelimiter)
-  }
-
-  process.exit()
-}