@mojaloop/ml-testing-toolkit-client-lib 1.6.1 → 1.8.0-snapshot.0

package/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.7.0](https://github.com/mojaloop/ml-testing-toolkit-client-lib/compare/v1.6.1...v1.7.0) (2025-03-13)
+
+
+### Features
+
+* enhance release report structure, implement brief summary option for Slack ([#19](https://github.com/mojaloop/ml-testing-toolkit-client-lib/issues/19)) ([8a6dd8b](https://github.com/mojaloop/ml-testing-toolkit-client-lib/commit/8a6dd8b99182f2931fe3b1cbd14aec7a63b2e316))
+
 ### [1.6.1](https://github.com/mojaloop/ml-testing-toolkit-client-lib/compare/v1.6.0...v1.6.1) (2025-02-25)
 
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mojaloop/ml-testing-toolkit-client-lib",
   "description": "Testing Toolkit Client Library",
-  "version": "1.6.1",
+  "version": "1.8.0-snapshot.0",
   "license": "Apache-2.0",
   "author": "Vijaya Kumar Guthi, ModusBox Inc. ",
   "contributors": [
@@ -63,13 +63,13 @@
     "snapshot": "npx standard-version --no-verify --skip.changelog --prerelease snapshot --releaseCommitMessageFormat 'chore(snapshot): {{currentTag}}'"
   },
   "dependencies": {
-    "@mojaloop/central-services-logger": "11.5.5",
+    "@mojaloop/central-services-logger": "11.6.2",
     "@mojaloop/ml-testing-toolkit-shared-lib": "14.0.4",
-    "@mojaloop/sdk-standard-components": "19.7.0",
-    "@slack/webhook": "7.0.4",
+    "@mojaloop/sdk-standard-components": "19.10.3",
+    "@slack/webhook": "7.0.5",
     "atob": "2.1.2",
     "aws-sdk": "2.1692.0",
-    "axios": "1.7.9",
+    "axios": "1.8.3",
     "cli-table3": "0.6.5",
     "commander": "13.1.0",
     "dotenv": "16.4.7",

package/src/client.js

@@ -47,7 +47,9 @@ program
   .option('--report-auto-filename-enable <reportAutoFilenameEnable>', 'default: false, if true the file name will be generated by the backend')
   .option('--report-target <reportTarget>', 'default: "file://<file_name_genrated_by_backend>"  --- supported targets: "file://path_to_file", "s3://<bucket_name>[/<file_path>]"')
   .option('--slack-webhook-url <slackWebhookUrl>', 'default: "Disabled"  --- supported formats: "https://....."')
+  .option('--slack-webhook-url-for-failed <slackWebhookUrl>', 'default: "Disabled"  --- supported formats: "https://....."')
   .option('--extra-summary-information <Comma separated values in the format key:value>', 'default: none  --- example: "Testcase Name:something,Environment:Dev1"')
+  .option('--brief-summary-prefix <Prefix to use for a brief summary in Slack>', 'default: none  --- example: "environment name, test name"')
   .on('--help', () => { // Extra information on help message
     console.log('')
     console.log(' *** If the option report-target is set to use AWS S3 service, the variables (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION) should be passed in environment')

package/src/extras/release-cd.js

@@ -29,10 +29,21 @@
 const axios = require('axios').default
 const config = require('rc')('release_cd', {})
 
-module.exports = async function (name, result) {
+module.exports = async function (name, { runtimeInformation, test_cases: testCases = [] }) {
   if (!config.reportUrl) return
   const data = {
-    [`tests.${name}`]: result
+    [`tests.${name}`]: {
+      ...runtimeInformation,
+      assertions: Object.fromEntries(testCases.map(
+        testCase => testCase?.requests?.map(
+          request => request?.request?.tests?.assertions
+            .filter(assertion => assertion?.assertionId ?? assertion.id)
+            .map(
+              assertion => [`${testCase.testCaseId ?? testCase.name}.${request.request.requestId ?? request.request.id}.${assertion?.assertionId ?? assertion.id}`, assertion?.resultStatus?.status]
+            )
+        )
+      ).flat(2))
+    }
   }
   console.log(`Sending report to ${config.reportUrl}`, data)
   await axios({

package/src/extras/slack-broadcast.js

@@ -31,24 +31,38 @@ const objectStore = require('../objectStore')
 
 const config = objectStore.get('config')
 
-const generateSlackBlocks = (progress) => {
+const millisecondsToTime = (milliseconds) => {
+  const seconds = Math.floor(milliseconds / 1000)
+  const minutes = Math.floor(seconds / 60)
+  const hours = Math.floor(minutes / 60)
+  return `${String(hours).padStart(2, '0')}:${String(minutes % 60).padStart(2, '0')}:${String(seconds % 60).padStart(2, '0')}`
+}
+
+const generateSlackBlocks = (progress, reportURL) => {
   const slackBlocks = []
   let totalAssertionsCount = 0
   let totalPassedAssertionsCount = 0
   let totalRequestsCount = 0
+  const failedTestCases = []
   progress.test_cases.forEach(testCase => {
     // console.log(fStr.yellow(testCase.name))
     totalRequestsCount += testCase.requests.length
-    // let testCaseAssertionsCount = 0
-    // let testCasePassedAssertionsCount = 0
+    let testCaseAssertionsCount = 0
+    let testCasePassedAssertionsCount = 0
     testCase.requests.forEach(req => {
       const passedAssertionsCount = req.request.tests && req.request.tests.passedAssertionsCount ? req.request.tests.passedAssertionsCount : 0
       const assertionsCount = req.request.tests && req.request.tests.assertions && req.request.tests.assertions.length ? req.request.tests.assertions.length : 0
       totalAssertionsCount += assertionsCount
       totalPassedAssertionsCount += passedAssertionsCount
-      // testCaseAssertionsCount += assertionsCount
-      // testCasePassedAssertionsCount += passedAssertionsCount
+      testCaseAssertionsCount += assertionsCount
+      testCasePassedAssertionsCount += passedAssertionsCount
     })
+    if (testCaseAssertionsCount !== testCasePassedAssertionsCount) {
+      failedTestCases.push({
+        name: testCase.name,
+        failedAssertions: testCaseAssertionsCount - testCasePassedAssertionsCount
+      })
+    }
     // const passed = testCasePassedAssertionsCount === testCaseAssertionsCount
     // // TODO: make sure this list should not be more than 40 because we can add only max 50 blocks in a slack message
     // if(!passed) {
@@ -61,6 +75,40 @@ const generateSlackBlocks = (progress) => {
     //   })
     // }
   })
+
+  // totalAssertionsCount = totalPassedAssertionsCount
+  // failedTestCases.length = 0
+
+  if (config.briefSummaryPrefix) {
+    const top5FailedTestCases = failedTestCases.sort((a, b) => b.failedAssertions - a.failedAssertions).slice(0, 5)
+    return [{
+      type: 'context',
+      elements: [{
+        type: 'mrkdwn',
+        text: [
+          `${totalAssertionsCount === totalPassedAssertionsCount ? '🟢' : '🔴'}`,
+          reportURL ? `<${reportURL}|${config.briefSummaryPrefix}>` : `${config.briefSummaryPrefix}`,
+          `failed: \`${totalAssertionsCount - totalPassedAssertionsCount}/${totalAssertionsCount}`,
+          `(${(100 * ((totalAssertionsCount - totalPassedAssertionsCount) / totalAssertionsCount)).toFixed(2)}%)\`,`,
+          `requests: \`${totalRequestsCount}\`,`,
+          `tests: \`${progress.test_cases.length}\`,`,
+          `duration: \`${millisecondsToTime(progress.runtimeInformation.runDurationMs)}\``,
+          top5FailedTestCases.length > 0 && '\nTop 5 failed test cases:\n',
+          top5FailedTestCases.length > 0 && top5FailedTestCases.map(tc => `• ${tc.name}: \`${tc.failedAssertions}\``).join('\n')
+        ].filter(Boolean).join(' ')
+      }]
+    }]
+  }
+
+  slackBlocks.push({
+    type: 'header',
+    text: {
+      type: 'plain_text',
+      text: 'Testing Toolkit Report',
+      emoji: true
+    }
+  })
+
   let summaryText = ''
 
   summaryText += '>Total assertions: *' + totalAssertionsCount + '*\n'
@@ -107,40 +155,53 @@ const generateSlackBlocks = (progress) => {
     },
     ...additionalParams
   })
+  if (reportURL) {
+    slackBlocks.push({
+      type: 'section',
+      text: {
+        type: 'mrkdwn',
+        text: '<' + reportURL + '|View Report>'
+      }
+    })
+  }
+  slackBlocks.push({
+    type: 'divider'
+  })
   return slackBlocks
 }
 
-const sendSlackNotification = async (progress, reportURL = null) => {
+const sendSlackNotification = async (progress, reportURL = 'http://localhost/') => {
+  console.log('runtimeInformation: ', progress.runtimeInformation)
+
+  let blocks
   if (config.slackWebhookUrl) {
     const url = config.slackWebhookUrl
     const webhook = new IncomingWebhook(url)
-    let slackBlocks = []
-    slackBlocks.push({
-      type: 'header',
-      text: {
-        type: 'plain_text',
-        text: 'Testing Toolkit Report',
-        emoji: true
-      }
-    })
-    slackBlocks = slackBlocks.concat(generateSlackBlocks(progress))
-    if (reportURL) {
-      slackBlocks.push({
-        type: 'section',
-        text: {
-          type: 'mrkdwn',
-          text: '<' + reportURL + '|View Report>'
-        }
+    blocks = generateSlackBlocks(progress, reportURL)
+
+    try {
+      // console.log(JSON.stringify(slackBlocks, null, 2))
+      await webhook.send({
+        text: 'Test Report',
+        blocks
       })
+      console.log('Slack notification sent.')
+    } catch (err) {
+      console.log('ERROR: Sending slack notification failed. ', err.message)
     }
-    slackBlocks.push({
-      type: 'divider'
-    })
+  }
+
+  if (needToNotifyFailed(config, progress)) {
+    const url = config.slackWebhookUrlForFailed
+    const webhook = new IncomingWebhook(url)
+
+    if (!blocks) blocks = generateSlackBlocks(progress, reportURL)
+
     try {
       // console.log(JSON.stringify(slackBlocks, null, 2))
       await webhook.send({
-        text: 'Test Report',
-        blocks: slackBlocks
+        text: 'Failed Tests Report',
+        blocks
       })
       console.log('Slack notification sent.')
     } catch (err) {
@@ -149,6 +210,13 @@ const sendSlackNotification = async (progress, reportURL = null) => {
   }
 }
 
+const needToNotifyFailed = (conf, totalResult) => {
+  return conf.slackWebhookUrlForFailed && (!totalResult?.runtimeInformation?.totalAssertion
+    ? true
+    : totalResult.runtimeInformation.totalPassedAssertions !== totalResult.runtimeInformation.totalAssertion)
+}
+
 module.exports = {
-  sendSlackNotification
+  sendSlackNotification,
+  needToNotifyFailed
 }

package/src/modes/outbound.js

@@ -200,7 +200,7 @@ const handleIncomingProgress = async (progress) => {
         }
       }
       try {
-        await releaseCd(config.reportName, progress.totalResult.runtimeInformation)
+        await releaseCd(config.reportName, progress.totalResult)
       } catch (err) {
         console.error(err)
       }

package/src/router.js

@@ -63,10 +63,12 @@ const cli = (commanderOptions) => {
     reportAutoFilenameEnable: commanderOptions.reportAutoFilenameEnable === 'true' || configFile.reportAutoFilenameEnable === true,
     reportTarget: commanderOptions.reportTarget || configFile.reportTarget,
     slackWebhookUrl: commanderOptions.slackWebhookUrl || configFile.slackWebhookUrl,
+    slackWebhookUrlForFailed: commanderOptions.slackWebhookUrlForFailed || configFile.slackWebhookUrlForFailed,
     slackPassedImage: configFile.slackPassedImage,
     slackFailedImage: configFile.slackFailedImage,
     baseURL: commanderOptions.baseUrl || configFile.baseURL,
     extraSummaryInformation: commanderOptions.extraSummaryInformation || configFile.extraSummaryInformation,
+    briefSummaryPrefix: commanderOptions.briefSummaryPrefix || configFile.briefSummaryPrefix,
     labels: commanderOptions.labels || configFile.labels,
     batchSize: commanderOptions.batchSize || configFile.batchSize || parseInt(process.env.TESTCASES_BATCH_SIZE, 10)
   }

package/test/unit/extras/release-cd.test.js

@@ -42,14 +42,41 @@ describe('Release CD', () => {
   describe('Post test results', () => {
     it('Posts test result to configured URL', async () => {
       const name = 'test'
-      const result = {}
+      const result = {
+        test_cases: [
+          {
+            name: 'test-case',
+            requests: [
+              {
+                request: {
+                  id: 'request-id',
+                  tests: {
+                    assertions: [
+                      {
+                        id: 'assertion-id',
+                        resultStatus: {
+                          status: 'SUCCESS'
+                        }
+                      }
+                    ]
+                  }
+                }
+              }
+            ]
+          }
+        ]
+      }
       config.reportUrl = 'http://example.com'
       await releaseCd(name, result)
       expect(axios.default).toHaveBeenCalledWith({
         method: 'post',
         url: 'http://example.com',
         data: {
-          [`tests.${name}`]: result
+          [`tests.${name}`]: {
+            assertions: {
+              'test-case.request-id.assertion-id': 'SUCCESS'
+            }
+          }
         }
       })
     })

package/test/unit/extras/slack-broadcast.test.js

@@ -102,6 +102,13 @@ describe('Cli client', () => {
         text: expect.any(String),
         blocks: expect.any(Array)
       }))
+      SpySlackSend.mockResolvedValueOnce(null)
+      config.briefSummaryPrefix = 'brief'
+      await expect(slackBroadCast.sendSlackNotification(sampleProgress)).resolves.toBe(undefined)
+      expect(SpySlackSend).toHaveBeenCalledWith(expect.objectContaining({
+        text: expect.any(String),
+        blocks: expect.any(Array)
+      }))
     })
     it('When failed case, it should call slack send function', async () => {
       config.slackWebhookUrl = 'http://some_url'
@@ -127,4 +134,60 @@ describe('Cli client', () => {
       }))
     })
   })
+
+  describe.skip('needToNotify Tests -->', () => {
+    it('should not notify if slackWebhookUrl is not set', () => {
+      expect(slackBroadCast.needToNotify({})).toBeFalsy()
+    })
+
+    it('should notify if slackWebhookUrl is set and slackOnlyFailed not', () => {
+      const conf = { slackWebhookUrl: 'url' }
+      expect(slackBroadCast.needToNotify(conf)).toBe(true)
+    })
+
+    it('should notify if slackWebhookUrl is set and slackOnlyFailed is false', () => {
+      const conf = {
+        slackWebhookUrl: 'url',
+        slackOnlyFailed: false
+      }
+      expect(slackBroadCast.needToNotify(conf)).toBe(true)
+    })
+
+    it('should notify if slackOnlyFailed=true, and tests failed', () => {
+      const conf = {
+        slackWebhookUrl: 'url',
+        slackOnlyFailed: true
+      }
+      const totalResult = {
+        runtimeInformation: {
+          totalPassedAssertions: 0,
+          totalAssertion: 1
+        }
+      }
+      expect(slackBroadCast.needToNotify(conf, totalResult)).toBe(true)
+    })
+
+    it('should notify if slackOnlyFailed=true, and tests passed', () => {
+      const conf = {
+        slackWebhookUrl: 'url',
+        slackOnlyFailed: true
+      }
+      const totalResult = {
+        runtimeInformation: {
+          totalPassedAssertions: 1,
+          totalAssertion: 1
+        }
+      }
+      expect(slackBroadCast.needToNotify(conf, totalResult)).toBe(false)
+    })
+
+    it('should notify if slackOnlyFailed=true, but no totalResult.runtimeInformation', () => {
+      const conf = {
+        slackWebhookUrl: 'url',
+        slackOnlyFailed: true
+      }
+      const totalResult = {}
+      expect(slackBroadCast.needToNotify(conf, totalResult)).toBe(true)
+    })
+  })
 })

package/test/unit/router.test.js

@@ -30,17 +30,19 @@
 
 const spyExit = jest.spyOn(process, 'exit')
 const { cli } = require('../../src/router')
+// const objectStore = require('../../src/objectStore')
 
 jest.mock('../../src/utils/listeners')
 jest.mock('../../src/modes/outbound')
 jest.mock('../../src/modes/testcaseDefinitionReport')
+// jest.mock('../../src/objectStore')
 
 describe('Cli client', () => {
   describe('running router', () => {
     it('when mode is monitoring should not throw an error', async () => {
       const config = {
         "mode": "monitoring"
-      }  
+      }
       spyExit.mockImplementationOnce(jest.fn())
       expect(() => {
         cli(config)
@@ -111,5 +113,14 @@ describe('Cli client', () => {
         cli(config)
       }).not.toThrowError();
     })
+    // it('should have default slackOnlyFailed value', async () => {
+    //   spyExit.mockImplementationOnce(jest.fn())
+    //   cli({
+    //     mode: 'outbound',
+    //     inputFiles: 'test',
+    //     environmentFile: 'test'
+    //   })
+    //   expect(objectStore.set.mock.lastCall[1].slackOnlyFailed).toBe(false)
+    // })
   })
 })