botium-core 1.13.16 → 1.13.18

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botium-core",
-  "version": "1.13.16",
+  "version": "1.13.18",
   "description": "The Selenium for Chatbots",
   "main": "index.js",
   "module": "dist/botium-es.js",
@@ -32,66 +32,66 @@
   },
   "homepage": "https://www.botium.ai",
   "dependencies": {
-    "@babel/runtime": "^7.20.6",
+    "@babel/runtime": "^7.21.5",
     "async": "^3.2.4",
-    "body-parser": "^1.20.1",
+    "body-parser": "^1.20.2",
     "boolean": "^3.2.0",
     "bottleneck": "^2.19.5",
-    "csv-parse": "^5.3.3",
+    "csv-parse": "^5.3.10",
     "debug": "^4.3.4",
     "esprima": "^4.0.1",
     "express": "^4.18.2",
     "globby": "11.0.4",
-    "ioredis": "^5.2.4",
+    "ioredis": "^5.3.2",
     "is-class": "^0.0.9",
     "is-json": "^2.0.1",
     "jsonpath": "^1.1.1",
     "lodash": "^4.17.21",
     "markdown-it": "^13.0.1",
     "mime-types": "^2.1.35",
-    "mkdirp": "^1.0.4",
+    "mkdirp": "^3.0.1",
     "moment": "^2.29.4",
     "mustache": "^4.2.0",
     "promise-retry": "^2.0.1",
     "promise.allsettled": "^1.0.6",
     "randomatic": "^3.1.1",
     "request": "^2.88.2",
-    "rimraf": "^3.0.2",
+    "rimraf": "^5.0.0",
     "sanitize-filename": "^1.6.3",
-    "slugify": "^1.6.5",
-    "socket.io": "^4.5.4",
-    "socket.io-client": "^4.5.4",
+    "slugify": "^1.6.6",
+    "socket.io": "^4.6.1",
+    "socket.io-client": "^4.6.1",
     "socketio-auth": "^0.1.1",
-    "swagger-jsdoc": "^6.2.5",
-    "swagger-ui-express": "^4.6.0",
+    "swagger-jsdoc": "^6.2.8",
+    "swagger-ui-express": "^4.6.3",
     "uuid": "^9.0.0",
-    "vm2": "^3.9.13",
+    "vm2": "^3.9.17",
     "word-error-rate": "0.0.7",
     "write-yaml": "^1.0.0",
     "xlsx": "^0.18.5",
     "xregexp": "^5.1.1",
-    "yaml": "^2.1.3"
+    "yaml": "^2.2.2"
   },
   "devDependencies": {
-    "@babel/core": "^7.20.5",
-    "@babel/node": "^7.20.5",
-    "@babel/plugin-transform-runtime": "^7.19.6",
-    "@babel/preset-env": "^7.20.2",
+    "@babel/core": "^7.21.8",
+    "@babel/node": "^7.20.7",
+    "@babel/plugin-transform-runtime": "^7.21.4",
+    "@babel/preset-env": "^7.21.5",
     "chai": "^4.3.7",
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
-    "eslint": "^8.29.0",
+    "eslint": "^8.40.0",
     "eslint-config-standard": "^17.0.0",
-    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-import": "^2.27.5",
     "eslint-plugin-mocha": "^10.1.0",
-    "eslint-plugin-n": "^15.6.0",
+    "eslint-plugin-n": "^15.7.0",
     "eslint-plugin-promise": "^6.1.1",
     "eslint-plugin-standard": "^4.1.0",
     "license-checker": "^25.0.1",
     "license-compatibility-checker": "^0.3.5",
     "mocha": "^10.2.0",
-    "nock": "^13.2.9",
-    "npm-check-updates": "^16.5.6",
+    "nock": "^13.3.1",
+    "npm-check-updates": "^16.10.12",
     "nyc": "^15.1.0",
     "rollup": "2.79.1",
     "rollup-plugin-babel": "^4.4.0",

package/src/BotDriver.js

@@ -2,7 +2,7 @@ const util = require('util')
 const fs = require('fs')
 const path = require('path')
 const async = require('async')
-const rimraf = require('rimraf')
+const { rimraf } = require('rimraf')
 const mkdirp = require('mkdirp')
 const sanitize = require('sanitize-filename')
 const moment = require('moment')
@@ -168,9 +168,7 @@ module.exports = class BotDriver {
           debug(`BotDriver Build error: ${err}`)
           this.eventEmitter.emit(Events.CONTAINER_BUILD_ERROR, err)
           if (tempDirectory) {
-            rimraf(tempDirectory, (err) => {
-              if (err) debug(`Cleanup temp dir ${tempDirectory} failed: ${util.inspect(err)}`)
-            })
+            rimraf(tempDirectory).catch((err) => debug(`Cleanup temp dir ${tempDirectory} failed: ${util.inspect(err)}`))
           }
           return reject(err)
         }

package/src/Capabilities.js

@@ -68,6 +68,12 @@ module.exports = {
   SIMPLEREST_POLL_INTERVAL: 'SIMPLEREST_POLL_INTERVAL',
   SIMPLEREST_POLL_TIMEOUT: 'SIMPLEREST_PING_TIMEOUT',
   SIMPLEREST_POLL_UPDATE_CONTEXT: 'SIMPLEREST_POLL_UPDATE_CONTEXT',
+  SIMPLEREST_CONTEXT_IGNORE_JSONPATH: 'SIMPLEREST_CONTEXT_IGNORE_JSONPATH',
+  SIMPLEREST_CONTEXT_IGNORE_MATCH: 'SIMPLEREST_CONTEXT_IGNORE_MATCH',
+  SIMPLEREST_CONTEXT_SKIP_JSONPATH: 'SIMPLEREST_CONTEXT_SKIP_JSONPATH',
+  SIMPLEREST_CONTEXT_SKIP_MATCH: 'SIMPLEREST_CONTEXT_SKIP_MATCH',
+  SIMPLEREST_CONTEXT_CONTINUE_JSONPATH: 'SIMPLEREST_CONTEXT_CONTINUE_JSONPATH',
+  SIMPLEREST_CONTEXT_CONTINUE_MATCH: 'SIMPLEREST_CONTEXT_CONTINUE_MATCH',
   SIMPLEREST_BODY_JSONPATH: 'SIMPLEREST_BODY_JSONPATH',
   SIMPLEREST_RESPONSE_JSONPATH: 'SIMPLEREST_RESPONSE_JSONPATH',
   SIMPLEREST_RESPONSE_HOOK: 'SIMPLEREST_RESPONSE_HOOK',
@@ -165,7 +171,5 @@ module.exports = {
   RATELIMIT_USERSAYS_MINTIME: 'RATELIMIT_USERSAYS_MINTIME',
   RATELIMIT_BOTTLENECK_FN: 'RATELIMIT_BOTTLENECK_FN',
   SECURITY_ALLOW_UNSAFE: 'SECURITY_ALLOW_UNSAFE',
-  PRECOMPILERS: 'PRECOMPILERS',
-  // RETRY
-  RETRY_CONVO_ASYNC: 'RETRY_CONVO_ASYNC'
+  PRECOMPILERS: 'PRECOMPILERS'
 }

package/src/Events.js

@@ -13,6 +13,7 @@ module.exports = {
   CONTAINER_CLEANED: 'CONTAINER_CLEANED',
   CONTAINER_CLEAN_ERROR: 'CONTAINER_CLEAN_ERROR',
   BOT_CONNECTED: 'BOT_CONNECTED',
+  CONVO_STEP_NEXT: 'CONVO_STEP_NEXT',
   // Chatbot Events
   MESSAGE_SENTTOBOT: 'MESSAGE_SENTTOBOT',
   MESSAGE_SENDTOBOT_ERROR: 'MESSAGE_SENDTOBOT_ERROR',

package/src/containers/BaseContainer.js

@@ -173,10 +173,12 @@ module.exports = class BaseContainer {
         (rimraffed) => {
           if (this.caps[Capabilities.CLEANUPTEMPDIR]) {
             debug(`Cleanup rimrafing temp dir ${this.tempDirectory}`)
-            rimraf(this.tempDirectory, (err) => {
-              if (err) debug(`Cleanup temp dir ${this.tempDirectory} failed: ${util.inspect(err)}`)
+            try {
+              rimraf.sync(this.tempDirectory)
               rimraffed()
-            })
+            } catch (err) {
+              rimraffed(new Error(`Cleanup temp directory ${this.tempDirectory} failed: ${util.inspect(err)}`))
+            }
           } else {
             rimraffed()
           }

package/src/containers/PluginConnectorContainer.js

@@ -11,16 +11,12 @@ const RetryHelper = require('../helpers/RetryHelper')
 module.exports = class PluginConnectorContainer extends BaseContainer {
   async Validate () {
     await super.Validate()
-    const setAsync = (isAsync) => {
-      this.caps.RETRY_CONVO_ASYNC = isAsync
-    }
     this.pluginInstance = tryLoadPlugin(
       this.caps[Capabilities.CONTAINERMODE],
       this.caps[Capabilities.PLUGINMODULEPATH],
       {
         container: this,
         queueBotSays: (msg) => this._QueueBotSays(msg),
-        setAsync: (isAsync) => setAsync(isAsync),
         bottleneck: this.bottleneck,
         eventEmitter: this.eventEmitter,
         caps: this.caps,

package/src/containers/plugins/SimpleRestContainer.js

@@ -293,6 +293,47 @@ module.exports = class SimpleRestContainer {
       debug(`current session context: ${util.inspect(this.view.context)}`)
     }
 
+    const _isAnyContextJsonPathMatch = (capName, capNameMatch) => {
+      const jsonPaths = getAllCapValues(capName, this.caps)
+      if (jsonPaths.length > 0) {
+        const jsonPathsMatch = getAllCapValues(capNameMatch, this.caps)
+        for (const [index, jsonPath] of jsonPaths.entries()) {
+          const contextNodes = jp.query(this.view.context, jsonPath)
+          if (_.isArray(contextNodes) && contextNodes.length > 0) {
+            if (jsonPathsMatch[index]) {
+              if (contextNodes[0] === jsonPathsMatch[index]) {
+                return {
+                  jsonPath,
+                  match: contextNodes[0]
+                }
+              }
+            } else {
+              return {
+                jsonPath
+              }
+            }
+          }
+        }
+      }
+      return null
+    }
+
+    const ignoreMatch = _isAnyContextJsonPathMatch(Capabilities.SIMPLEREST_CONTEXT_IGNORE_JSONPATH, Capabilities.SIMPLEREST_CONTEXT_IGNORE_MATCH)
+    if (ignoreMatch) {
+      if (ignoreMatch.match) debug(`ignoring response for context match: ${ignoreMatch.jsonPath} = ${ignoreMatch.match}`)
+      else debug(`ignoring response for context: ${ignoreMatch.jsonPath}`)
+      return
+    }
+
+    const skipMatch = _isAnyContextJsonPathMatch(Capabilities.SIMPLEREST_CONTEXT_SKIP_JSONPATH, Capabilities.SIMPLEREST_CONTEXT_SKIP_MATCH)
+    if (skipMatch) {
+      if (skipMatch.match) debug(`skipping response for context match: ${skipMatch.jsonPath} = ${skipMatch.match}`)
+      else debug(`skipping response for context: ${skipMatch.jsonPath}`)
+
+      setTimeout(() => this._doRequest({ messageText: '' }, true, true), 0)
+      return
+    }
+
     const result = []
     if (isFromUser) {
       const _extractFrom = (root, jsonPaths) => {
@@ -442,6 +483,14 @@ module.exports = class SimpleRestContainer {
         }
       }
     }
+
+    const continueMatch = _isAnyContextJsonPathMatch(Capabilities.SIMPLEREST_CONTEXT_CONTINUE_JSONPATH, Capabilities.SIMPLEREST_CONTEXT_CONTINUE_MATCH)
+    if (continueMatch) {
+      if (continueMatch.match) debug(`continue with next response for context match: ${continueMatch.jsonPath} = ${continueMatch.match}`)
+      else debug(`continue with next response for context: ${continueMatch.jsonPath}`)
+
+      setTimeout(() => this._doRequest({ messageText: '' }, true, true), 0)
+    }
     return result
   }
 

package/src/scripting/Convo.js

@@ -1,7 +1,6 @@
 const util = require('util')
 const _ = require('lodash')
 const debug = require('debug')('botium-core-Convo')
-const promiseRetry = require('promise-retry')
 
 const BotiumMockMessage = require('../mocks/BotiumMockMessage')
 const Capabilities = require('../Capabilities')
@@ -9,7 +8,6 @@ const Events = require('../Events')
 const ScriptingMemory = require('./ScriptingMemory')
 const { BotiumError, botiumErrorFromErr, botiumErrorFromList } = require('./BotiumError')
 const { normalizeText, toString, removeBuffers, splitStringInNonEmptyLines } = require('./helper')
-const RetryHelper = require('../helpers/RetryHelper')
 
 const { LOGIC_HOOK_INCLUDE } = require('./logichook/LogicHookConsts')
 
@@ -212,31 +210,6 @@ class Convo {
   }
 
   async Run (container) {
-    if (container.caps.RETRY_CONVO_ASYNC) {
-      return this.RunImpl(container).catch(err => {
-        debug(`Convo failed with error "${err.message || JSON.stringify(err)}".`)
-        throw err
-      })
-    } else {
-      const retryHelper = new RetryHelper(container.caps, 'CONVO')
-      return promiseRetry(async (retry, number) => {
-        const retryRemaining = retryHelper.retrySettings.retries - number + 1
-        return this.RunImpl(container).catch(err => {
-          if (retryHelper.shouldRetry(err)) {
-            debug(`Convo failed with error "${err.message || JSON.stringify(err)}". Retry ${retryRemaining > 0 ? 'enabled' : 'disabled'} (remaining #${retryRemaining}/${retryHelper.retrySettings.retries}, criterion matches)`)
-            retry(err)
-          } else {
-            if (retryHelper.retryErrorPatterns.length > 0) {
-              debug(`Convo failed with error "${err.message || JSON.stringify(err)}". Retry 'disabled' (remaining (#${retryRemaining}/${retryHelper.retrySettings.retries}), criterion does not match)`)
-            }
-            throw err
-          }
-        })
-      }, retryHelper.retrySettings)
-    }
-  }
-
-  async RunImpl (container) {
     const transcript = new Transcript({
       steps: [],
       attachments: [],
@@ -305,6 +278,7 @@ class Convo {
       for (let i = 0; i < this.conversation.length; i++) {
         const convoStep = this.conversation[i]
         const currentStepIndex = i
+        container.eventEmitter.emit(Events.CONVO_STEP_NEXT, container, convoStep, i)
         skipTranscriptStep = false
         const transcriptStep = new TranscriptStep({
           expected: new BotiumMockMessage(convoStep),
@@ -430,6 +404,15 @@ class Convo {
               throw failErr
             }
 
+            if (convoStep.skip === true) {
+              skipTranscriptStep = true
+              const nextConvoStep = this.conversation[i + 1]
+              if (nextConvoStep && nextConvoStep.sender === 'bot') {
+                waitForBotSays = false
+              }
+              continue
+            }
+
             if (!botMsg || (!botMsg.messageText && !botMsg.media && !botMsg.buttons && !botMsg.cards && !botMsg.sourceData && !botMsg.nlp)) {
               const failErr = new BotiumError(`${this.header.name}/${convoStep.stepTag}: bot says nothing`)
               debug(failErr)

package/src/scripting/ScriptingProvider.js

@@ -19,6 +19,7 @@ const { BotiumError, botiumErrorFromList, botiumErrorFromErr } = require('./Boti
 const RetryHelper = require('../helpers/RetryHelper')
 const { getMatchFunction } = require('./MatchFunctions')
 const precompilers = require('./precompilers')
+const { calculateWer, toPercent } = require('./helper')
 
 const globPattern = '**/+(*.convo.txt|*.utterances.txt|*.pconvo.txt|*.scriptingmemory.txt|*.xlsx|*.xlsm|*.convo.csv|*.pconvo.csv|*.utterances.csv|*.yaml|*.yml|*.json|*.md|*.markdown)'
 const skipPattern = /^skip[.\-_]/i
@@ -137,32 +138,57 @@ module.exports = class ScriptingProvider {
         const found = _.find(tomatch, (utt) => this.matchFn(botresponse, utt, this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS]))
         const asserterType = this.caps[Capabilities.SCRIPTING_MATCHING_MODE] === 'wer' ? 'Word Error Rate Asserter' : 'Text Match Asserter'
         if (_.isNil(found)) {
-          let message = `${stepTag}: Bot response `
-          message += meMsg ? `(on ${meMsg}) ` : ''
-          message += botresponse ? ('"' + botresponse + '"') : '<no response>'
-          message += ' expected to match '
-          message += tomatch && tomatch.length > 1 ? 'one of ' : ''
-          message += `${tomatch.map(e => e ? '"' + e + '"' : '<any response>').join(', ')}`
-          throw new BotiumError(
-            message,
-            {
-              type: 'asserter',
-              source: asserterType,
-              params: {
-                matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
-                args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
-              },
-              context: {
-                stepTag
-              },
-              cause: {
-                expected: tomatch,
-                actual: botresponse,
-                matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
-                args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
+          if (this.caps[Capabilities.SCRIPTING_MATCHING_MODE] === 'wer') {
+            const wer = calculateWer(botresponse, tomatch[0])
+            const werArgs = this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS]
+            const threshold = ([',', '.'].find(p => `${werArgs[0]}`.includes(p)) ? parseFloat(werArgs[0]) : parseInt(werArgs[0]) / 100)
+            const message = `${stepTag}: Word Error Rate (${toPercent(wer)}) higher than accepted (${toPercent(threshold)})`
+            throw new BotiumError(
+              message,
+              {
+                type: 'asserter',
+                source: asserterType,
+                params: {
+                  matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
+                  args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
+                },
+                context: {
+                  stepTag
+                },
+                cause: {
+                  expected: `<=${toPercent(threshold)} (${tomatch})`,
+                  actual: `${toPercent(wer)} (${botresponse})`
+                }
               }
-            }
-          )
+            )
+          } else {
+            let message = `${stepTag}: Bot response `
+            message += meMsg ? `(on ${meMsg}) ` : ''
+            message += botresponse ? ('"' + botresponse + '"') : '<no response>'
+            message += ' expected to match '
+            message += tomatch && tomatch.length > 1 ? 'one of ' : ''
+            message += `${tomatch.map(e => e ? '"' + e + '"' : '<any response>').join(', ')}`
+            throw new BotiumError(
+              message,
+              {
+                type: 'asserter',
+                source: asserterType,
+                params: {
+                  matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
+                  args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
+                },
+                context: {
+                  stepTag
+                },
+                cause: {
+                  expected: tomatch,
+                  actual: botresponse,
+                  matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
+                  args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
+                }
+              }
+            )
+          }
         }
       },
       assertBotNotResponse: (botresponse, nottomatch, stepTag, meMsg) => {
@@ -173,33 +199,58 @@ module.exports = class ScriptingProvider {
         const found = _.find(nottomatch, (utt) => this.matchFn(botresponse, utt, this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS]))
         const asserterType = this.caps[Capabilities.SCRIPTING_MATCHING_MODE] === 'wer' ? 'Word Error Rate Asserter' : 'Text Match Asserter'
         if (!_.isNil(found)) {
-          let message = `${stepTag}: Bot response `
-          message += meMsg ? `(on ${meMsg}) ` : ''
-          message += botresponse ? ('"' + botresponse + '"') : '<no response>'
-          message += ' expected NOT to match '
-          message += nottomatch && nottomatch.length > 1 ? 'one of ' : ''
-          message += `${nottomatch.map(e => e ? '"' + e + '"' : '<any response>').join(', ')}`
-          throw new BotiumError(
-            message,
-            {
-              type: 'asserter',
-              source: asserterType,
-              params: {
-                matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
-                args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
-              },
-              context: {
-                stepTag
-              },
-              cause: {
-                not: true,
-                expected: nottomatch,
-                actual: botresponse,
-                matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
-                args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
+          if (this.caps[Capabilities.SCRIPTING_MATCHING_MODE] === 'wer') {
+            const wer = calculateWer(botresponse, nottomatch[0])
+            const werArgs = this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS]
+            const threshold = ([',', '.'].find(p => `${werArgs[0]}`.includes(p)) ? parseFloat(werArgs[0]) : parseInt(werArgs[0]) / 100)
+            const message = `${stepTag}: Word Error Rate (${toPercent(wer)}) lower than accepted (${toPercent(threshold)})`
+            throw new BotiumError(
+              message,
+              {
+                type: 'asserter',
+                source: asserterType,
+                params: {
+                  matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
+                  args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
+                },
+                context: {
+                  stepTag
+                },
+                cause: {
+                  expected: `>=${toPercent(threshold)} (${nottomatch})`,
+                  actual: `${toPercent(wer)} (${botresponse})`
+                }
               }
-            }
-          )
+            )
+          } else {
+            let message = `${stepTag}: Bot response `
+            message += meMsg ? `(on ${meMsg}) ` : ''
+            message += botresponse ? ('"' + botresponse + '"') : '<no response>'
+            message += ' expected NOT to match '
+            message += nottomatch && nottomatch.length > 1 ? 'one of ' : ''
+            message += `${nottomatch.map(e => e ? '"' + e + '"' : '<any response>').join(', ')}`
+            throw new BotiumError(
+              message,
+              {
+                type: 'asserter',
+                source: asserterType,
+                params: {
+                  matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
+                  args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
+                },
+                context: {
+                  stepTag
+                },
+                cause: {
+                  not: true,
+                  expected: nottomatch,
+                  actual: botresponse,
+                  matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
+                  args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
+                }
+              }
+            )
+          }
         }
       },
       fail: null
@@ -889,10 +940,16 @@ module.exports = class ScriptingProvider {
       convoFilter: null
     }, options)
     const expandedConvos = []
+    // The globalContext is going to keep the data even if the Object.assign which happening to create the myContext in _expandConvo function
+    const context = {
+      globalContext: {
+        totalConvoCount: 0
+      }
+    }
     debug(`ExpandConvos - Using utterances expansion mode: ${this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE]}`)
     this.convos.forEach((convo) => {
       convo.expandPartialConvos()
-      for (const expanded of this._expandConvo(convo, options, {})) {
+      for (const expanded of this._expandConvo(convo, options, context)) {
         expanded.header.assertionCount = this.GetAssertionCount(expanded)
         if (options.justHeader) {
           const ConvoWithOnlyHeader = {
@@ -908,6 +965,7 @@ module.exports = class ScriptingProvider {
       }
     })
     this.convos = expandedConvos
+    this.totalConvoCount = context.globalContext.totalConvoCount
     if (!options.justHeader) {
       this._sortConvos()
     } else {
@@ -920,17 +978,24 @@ module.exports = class ScriptingProvider {
       // drop unwanted convos
       convoFilter: null
     }, options)
+    // The globalContext is going to keep the data even if the Object.assign which happening to create the myContext in _expandConvo function
+    const context = {
+      globalContext: {
+        totalConvoCount: 0
+      }
+    }
     debug(`ExpandConvos - Using utterances expansion mode: ${this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE]}`)
     // creating a nested generator, calling the other.
     // We hope this.convos does not changes while this iterator is used
     const _convosIterable = function * (options) {
       for (const convo of this.convos) {
         convo.expandPartialConvos()
-        yield * this._expandConvo(convo, options, {})
+        yield * this._expandConvo(convo, options, context)
       }
     }.bind(this)
 
     this.convosIterable = _convosIterable(options)
+    this.totalConvoCount = context.globalContext.totalConvoCount
   }
 
   /**
@@ -1122,6 +1187,9 @@ module.exports = class ScriptingProvider {
       }
     } else {
       const expanded = Object.assign(_.cloneDeep(currentConvo), { conversation: _.cloneDeep(convoStepsStack) })
+      if (!_.isNil(_.get(context, 'globalContext.totalConvoCount'))) {
+        context.globalContext.totalConvoCount++
+      }
       if (!options.convoFilter || options.convoFilter(expanded)) {
         yield expanded
       }

package/src/scripting/helper.js

@@ -553,6 +553,10 @@ const calculateWer = (str, pattern) => {
   const botMessageWords = botMessage.split(' ').map(bm => bm.trim())
   const utt = _prepareString(utterance)
 
+  // if no wildcards, just calculate WER
+  if (utt.indexOf('*') === -1) return speechScorer.wordErrorRate(botMessage, utt).toFixed(2)
+
+  // if there are wildcards, calculate WER for each wildcard part
   const errors = []
   for (let wildcardPart of utt.split('*')) {
     let wer = 1
@@ -572,7 +576,7 @@ const calculateWer = (str, pattern) => {
       }
     }
     if (_.isNil(subsetPhraseFound)) {
-      throw new Error('Word Error Asserter: Something went wrong here, please try to modify your assertion!')
+      throw new Error('Word Error Asserter: When using wild cards, please make sure that the length of the asserter text is smaller than the bot message!')
     }
     errors.push(_getErrors(_getWords(wildcardPart), _getWords(subsetPhraseFound)))
   }
@@ -586,6 +590,8 @@ const calculateWer = (str, pattern) => {
   return (errCount / allCount).toFixed(2)
 }
 
+const toPercent = (s) => `${(s * 100).toFixed(0)}%`
+
 module.exports = {
   normalizeText,
   splitStringInNonEmptyLines,
@@ -600,5 +606,6 @@ module.exports = {
   validateSender,
   validateConvo,
   linesToScriptingMemories,
-  calculateWer
+  calculateWer,
+  toPercent
 }

package/src/scripting/logichook/asserter/ButtonsAsserter.js

@@ -1,3 +1,4 @@
+const _ = require('lodash')
 const { SCRIPTING_NORMALIZE_TEXT } = require('../../../Capabilities')
 const { BotiumError } = require('../../BotiumError')
 const { buttonsFromMsg } = require('../helpers')
@@ -11,17 +12,29 @@ module.exports = class ButtonsAsserter {
   }
 
   _evalButtons (args, botMsg) {
-    const allButtons = buttonsFromMsg(botMsg, true).map(b => b.text).filter(b => b).map(b => normalizeText(b, !!this.caps[SCRIPTING_NORMALIZE_TEXT]))
+    const allButtons = buttonsFromMsg(botMsg, true).map(b => ({ text: b.text, payload: b.payload })).filter(b => b).map(b => ({ text: normalizeText(b.text, !!this.caps[SCRIPTING_NORMALIZE_TEXT]), payload: b.payload }))
     if (!args || args.length === 0) {
-      return { allButtons, buttonsNotFound: [], buttonsFound: allButtons }
+      return { allButtons, buttonsNotFound: [], buttonsFound: allButtons.map(b => b.text) }
     }
     const buttonsNotFound = []
     const buttonsFound = []
+    const parsePayload = (payload) => {
+      if (_.isNil(payload)) {
+        return undefined
+      }
+      try {
+        return JSON.parse(payload)
+      } catch (e) {
+        return payload
+      }
+    }
     for (let i = 0; i < (args || []).length; i++) {
-      if (allButtons.findIndex(b => this.context.Match(b, normalizeText(args[i], !!this.caps[SCRIPTING_NORMALIZE_TEXT]))) < 0) {
-        buttonsNotFound.push(args[i])
-      } else {
+      const matchByText = allButtons.some(b => this.context.Match(b.text, normalizeText(args[i], !!this.caps[SCRIPTING_NORMALIZE_TEXT])))
+      const matchByPayload = allButtons.some(b => _.isEqual(parsePayload(b.payload), parsePayload(args[i])))
+      if (matchByText || matchByPayload) {
         buttonsFound.push(args[i])
+      } else {
+        buttonsNotFound.push(args[i])
       }
     }
     return { allButtons, buttonsNotFound, buttonsFound }
@@ -42,7 +55,7 @@ module.exports = class ButtonsAsserter {
           cause: {
             not: true,
             expected: args,
-            actual: allButtons,
+            actual: JSON.stringify(allButtons, null, 2),
             diff: buttonsFound
           }
         }
@@ -67,7 +80,7 @@ module.exports = class ButtonsAsserter {
             cause: {
               not: false,
               expected: args,
-              actual: allButtons,
+              actual: JSON.stringify(allButtons, null, 2),
               diff: buttonsNotFound
             }
           }
@@ -85,7 +98,7 @@ module.exports = class ButtonsAsserter {
             cause: {
               not: false,
               expected: args,
-              actual: allButtons,
+              actual: JSON.stringify(allButtons, null, 2),
               diff: buttonsNotFound
             }
           }