botium-core 1.14.9 → 1.15.2

package/src/scripting/Convo.js

@@ -277,6 +277,10 @@ class Convo {
       let skipTranscriptStep = false
       let conditionalGroupId = null
       let conditionMetInGroup = false
+      let skipOptionalStep = false
+      // If there are optional step(s) in the conversation, and the message from the bot fails on each optional bot step(s) and/or mandatory bot step, then we have an unexpected message.
+      // So in this case an unexpected error should be shown instead of the latest assertion error.
+      let optionalStepAssertionError = false
       let globalConvoStepParameters = container.caps[Capabilities.SCRIPTING_CONVO_STEP_PARAMETERS] || {}
       let retryBotMessageTimeoutEnd = null
       let retryBotMessageConvoId = null
@@ -284,6 +288,13 @@ class Convo {
       for (let i = 0; i < this.conversation.length; i = (retryBotMessageDropBotResponse ? i : i + 1)) {
         retryBotMessageDropBotResponse = false
         const convoStep = this.conversation[i]
+        if (!convoStep.optional) {
+          skipOptionalStep = false
+        }
+        if (convoStep.optional && skipOptionalStep) {
+          // If there are multiple optional steps, and the previous optional step was timeout, then the next optional step should be skipped to prevent too long convo run with multiple timeout.
+          continue
+        }
         const rawConvoStepParameters = convoStep.logicHooks.find(lh => lh.name === 'CONVO_STEP_PARAMETERS')?.args
         let convoStepParameters = {}
         if (rawConvoStepParameters && rawConvoStepParameters.length) {
@@ -426,7 +437,8 @@ class Convo {
             } catch (err) {
               transcriptStep.botEnd = new Date()
 
-              if (convoStep.optional) {
+              if (!(err.message.indexOf('Bot did not respond within') < 0) && convoStep.optional) {
+                skipOptionalStep = true
                 continue
               }
 
@@ -512,6 +524,7 @@ class Convo {
                 }
                 waitForBotSays = false
                 skipTranscriptStep = true
+                optionalStepAssertionError = true
                 return true
               } else if (retryOn) {
                 if (!retryBotMessageTimeoutEnd || retryBotMessageConvoId !== convoStep.stepTag) {
@@ -531,9 +544,18 @@ class Convo {
               }
 
               if (container.caps[Capabilities.SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS]) {
-                assertErrors.push(err)
+                if (optionalStepAssertionError) {
+                  optionalStepAssertionError = false
+                  assertErrors.push(new BotiumError(`${this.header.name}: Unexpected message.`))
+                } else {
+                  assertErrors.push(err)
+                }
                 return false
               } else {
+                if (optionalStepAssertionError) {
+                  optionalStepAssertionError = false
+                  throw new BotiumError(`${this.header.name}: Unexpected message.`)
+                }
                 throw err
               }
             }
@@ -547,6 +569,7 @@ class Convo {
               if (convoStep.not) {
                 try {
                   this.scriptingEvents.assertBotNotResponse(response, tomatch, `${this.header.name}/${convoStep.stepTag}`, lastMeConvoStep, convoStepParameters)
+                  optionalStepAssertionError = false
                 } catch (err) {
                   if (isErrorHandledWithOptionConvoStep(err)) {
                     continue
@@ -555,6 +578,7 @@ class Convo {
               } else {
                 try {
                   this.scriptingEvents.assertBotResponse(response, tomatch, `${this.header.name}/${convoStep.stepTag}`, lastMeConvoStep, convoStepParameters)
+                  optionalStepAssertionError = false
                 } catch (err) {
                   if (isErrorHandledWithOptionConvoStep(err)) {
                     continue
@@ -564,6 +588,7 @@ class Convo {
             } else if (convoStep.sourceData) {
               try {
                 this._compareObject(container, scriptingMemory, convoStep, botMsg.sourceData, convoStep.sourceData, botMsg, convoStepParameters)
+                optionalStepAssertionError = false
               } catch (err) {
                 if (isErrorHandledWithOptionConvoStep(err)) {
                   continue
@@ -574,11 +599,13 @@ class Convo {
             try {
               await this.scriptingEvents.assertConvoStep({ convo: this, convoStep, container, scriptingMemory, botMsg, transcript, transcriptStep })
               await this.scriptingEvents.onBotEnd({ convo: this, convoStep, container, scriptingMemory, botMsg, transcript, transcriptStep })
+              optionalStepAssertionError = false
             } catch (err) {
               const nextConvoStep = this.conversation[i + 1]
               if (convoStep.optional && nextConvoStep && nextConvoStep.sender === 'bot') {
                 waitForBotSays = false
                 skipTranscriptStep = true
+                optionalStepAssertionError = true
                 continue
               }
 
@@ -611,8 +638,17 @@ class Convo {
                 } catch (failErr) {
                 }
                 if (container.caps[Capabilities.SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS] && err instanceof BotiumError) {
-                  assertErrors.push(err)
+                  if (optionalStepAssertionError) {
+                    optionalStepAssertionError = false
+                    assertErrors.push(new BotiumError(`${this.header.name}: Unexpected message.`))
+                  } else {
+                    assertErrors.push(err)
+                  }
                 } else {
+                  if (optionalStepAssertionError) {
+                    optionalStepAssertionError = false
+                    throw new BotiumError(`${this.header.name}: Unexpected message.`)
+                  }
                   throw failErr
                 }
               }
@@ -789,7 +825,7 @@ class Convo {
   }
 
   _checkNormalizeText (container, str) {
-    return normalizeText(str, !!container.caps[Capabilities.SCRIPTING_NORMALIZE_TEXT])
+    return normalizeText(str, container.caps)
   }
 
   expandPartialConvos () {

package/src/scripting/ScriptingProvider.js

@@ -83,9 +83,9 @@ module.exports = class ScriptingProvider {
     this.utterances = {}
     this.matchFn = null
     this.asserters = {}
-    this.globalAsserter = {}
+    this.globalAsserters = {}
     this.logicHooks = {}
-    this.globalLogicHook = {}
+    this.globalLogicHooks = {}
     this.userInputs = {}
     this.partialConvos = {}
     this.scriptingMemories = []
@@ -319,8 +319,9 @@ module.exports = class ScriptingProvider {
       }
     }
 
-    const convoAsserter = asserters
-      .filter(a => this.asserters[a.name][asserterType])
+    const localAsserters = (asserters || []).filter(a => this.asserters[a.name][asserterType])
+
+    const convoStepPromises = localAsserters
       .map(a => ({
         asserter: a,
         promise: callAsserter(a, this.asserters[a.name], {
@@ -335,8 +336,12 @@ module.exports = class ScriptingProvider {
       }))
       .map(({ promise, asserter }) => updateExceptionContext(promise, asserter))
 
-    const globalAsserter = Object.values(this.globalAsserter)
+    const globalAsserters = Object.keys(this.globalAsserters)
+      .filter(name => localAsserters.map(a => a.name).indexOf(name) < 0)
+      .reduce((agg, name) => [...agg, this.globalAsserters[name]], [])
       .filter(a => a[asserterType])
+
+    const globalPromises = globalAsserters
       .map(a => ({
         asserter: a,
         promise: p(this.retryHelperAsserter, () => a[asserterType]({
@@ -351,7 +356,7 @@ module.exports = class ScriptingProvider {
       }))
       .map(({ promise, asserter }) => updateExceptionContext(promise, asserter))
 
-    const allPromises = [...convoAsserter, ...globalAsserter]
+    const allPromises = [...convoStepPromises, ...globalPromises]
     if (this.caps[Capabilities.SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS]) {
       return Promise.allSettled(allPromises).then((results) => {
         const rejected = results.filter(result => result.status === 'rejected').map(result => result.reason)
@@ -372,8 +377,7 @@ module.exports = class ScriptingProvider {
       throw Error(`Unknown hookType ${hookType}`)
     }
 
-    const localHooks = (logicHooks || [])
-      .filter(l => this.logicHooks[l.name][hookType])
+    const localHooks = (logicHooks || []).filter(l => this.logicHooks[l.name][hookType])
 
     const convoStepPromises = localHooks
       .map(l => p(this.retryHelperLogicHook, () => this.logicHooks[l.name][hookType]({
@@ -386,7 +390,10 @@ module.exports = class ScriptingProvider {
         ...rest
       })))
 
-    const globalHooks = Object.values(this.globalLogicHook).filter(l => l[hookType])
+    const globalHooks = Object.keys(this.globalLogicHooks)
+      .filter(name => localHooks.map(l => l.name).indexOf(name) < 0)
+      .reduce((agg, name) => [...agg, this.globalLogicHooks[name]], [])
+      .filter(l => l[hookType])
     const globalPromises = globalHooks.map(l => p(this.retryHelperLogicHook, () => l[hookType]({ convo, convoStep, scriptingMemory, container, args: [], isGlobal: true, ...rest })))
 
     const allPromises = [...convoStepPromises, ...globalPromises]
@@ -499,9 +506,9 @@ module.exports = class ScriptingProvider {
 
     const logicHookUtils = new LogicHookUtils({ buildScriptContext: this._buildScriptContext(), caps: this.caps })
     this.asserters = logicHookUtils.asserters
-    this.globalAsserter = logicHookUtils.getGlobalAsserter()
+    this.globalAsserters = logicHookUtils.getGlobalAsserters()
     this.logicHooks = logicHookUtils.logicHooks
-    this.globalLogicHook = logicHookUtils.getGlobalLogicHook()
+    this.globalLogicHooks = logicHookUtils.getGlobalLogicHooks()
     this.userInputs = logicHookUtils.userInputs
   }
 

package/src/scripting/helper.js

@@ -4,9 +4,35 @@ const speechScorer = require('word-error-rate')
 const debug = require('debug')('botium-core-scripting-helper')
 
 const { E_SCRIPTING_MEMORY_COLUMN_MODE } = require('../Enums')
+const Capabilities = require('../Capabilities')
 const WHITE_SPACES_EXCEPT_SPACE_CHAR_AT_THE_END = /[\n\t\r]+$/
 
-const normalizeText = (str, doCleanup) => {
+const normalizeText = (str, doCleanupOrCaps) => {
+  // TODO testlog
+  debug('yxc1', doCleanupOrCaps)
+  let basic
+  let charactersRemove = false
+  let regexpRemove = false
+  if (_.isBoolean(doCleanupOrCaps) || _.isNil(doCleanupOrCaps)) {
+    debug('Normalize text: backward compatibility mode. Use caps instead of boolean flag')
+    basic = !!doCleanupOrCaps
+  } else {
+    const caps = doCleanupOrCaps
+    basic = !!caps[Capabilities.SCRIPTING_NORMALIZE_TEXT]
+    if (caps[Capabilities.SCRIPTING_NORMALIZE_TEXT_REMOVE_CHARACTERES]) {
+      charactersRemove = caps[Capabilities.SCRIPTING_NORMALIZE_TEXT_REMOVE_CHARACTERES]
+      if (_.isString(charactersRemove)) {
+        const splitted = charactersRemove.split(/(?<!\/),/).map(e => e.trim()).map(e => e.split('/,').join(',').split('//').join('/')).filter(c => c.length > 0)
+        charactersRemove = splitted.length ? splitted : [charactersRemove]
+      } else if (!_.isArray(charactersRemove)) {
+        charactersRemove = false
+      }
+    }
+    if (caps[Capabilities.SCRIPTING_NORMALIZE_TEXT_REMOVE_REGEXP]) {
+      regexpRemove = new RegExp(caps[Capabilities.SCRIPTING_NORMALIZE_TEXT_REMOVE_REGEXP], 'ug')
+    }
+  }
+
   if (str && _.isArray(str)) {
     str = str.join(' ')
   } else if (str && !_.isString(str)) {
@@ -16,27 +42,38 @@ const normalizeText = (str, doCleanup) => {
       str = `${str}`
     }
   }
-  if (str && doCleanup) {
-    // remove html tags
-    str = str.replace(/<p[^>]*>/g, ' ')
-    str = str.replace(/<\/p>/g, ' ')
-    str = str.replace(/<br[^>]*>/g, ' ')
-    str = str.replace(/<[^>]*>/g, '')
-    /* eslint-disable no-control-regex */
-    // remove not printable characters
-    str = str.replace(/[\x00-\x1F\x7F]/g, ' ')
-    /* eslint-enable no-control-regex */
-    // replace html entities
-    str = str
-      .replace(/&amp;/g, '&')
-      .replace(/&lt;/g, '<')
-      .replace(/&gt;/g, '>')
-      .replace(/&#39;/g, '\'')
-      .replace(/&quot;/g, '"')
-    // replace two spaces with one
-    str = str.replace(/\s+/g, ' ')
-
-    str = str.split('\n').map(s => s.trim()).join('\n').trim()
+
+  if (str) {
+    if (basic) {
+      // remove html tags
+      str = str.replace(/<p[^>]*>/g, ' ')
+      str = str.replace(/<\/p>/g, ' ')
+      str = str.replace(/<br[^>]*>/g, ' ')
+      str = str.replace(/<[^>]*>/g, '')
+      /* eslint-disable no-control-regex */
+      // remove not printable characters
+      str = str.replace(/[\x00-\x1F\x7F]/g, ' ')
+      /* eslint-enable no-control-regex */
+      // replace html entities
+      str = str
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&#39;/g, '\'')
+        .replace(/&quot;/g, '"')
+      // replace two spaces with one
+      str = str.replace(/\s+/g, ' ')
+
+      str = str.split('\n').map(s => s.trim()).join('\n').trim()
+    }
+    if (charactersRemove) {
+      for (const character of charactersRemove) {
+        str = str.split(character).join('')
+      }
+    }
+    if (regexpRemove) {
+      str = str.replace(regexpRemove, '')
+    }
   }
   return str
 }

package/src/scripting/logichook/LogicHookUtils.js

@@ -24,9 +24,9 @@ const _ = require('lodash')
 module.exports = class LogicHookUtils {
   constructor ({ buildScriptContext, caps }) {
     this.asserters = {}
-    this.globalAsserters = []
+    this.globalAsserterNames = []
     this.logicHooks = {}
-    this.globalLogicHooks = []
+    this.globalLogicHookNames = []
     this.userInputs = {}
     this.buildScriptContext = buildScriptContext
     this.caps = caps
@@ -64,7 +64,7 @@ module.exports = class LogicHookUtils {
         }
         this.asserters[asserter.ref] = this._loadClass(asserter, 'asserter')
         if (asserter.global) {
-          this.globalAsserters.push(asserter.ref)
+          this.globalAsserterNames.push(asserter.ref)
         }
       })
   }
@@ -77,7 +77,7 @@ module.exports = class LogicHookUtils {
         }
         this.logicHooks[logicHook.ref] = this._loadClass(logicHook, 'logichook')
         if (logicHook.global) {
-          this.globalLogicHooks.push(logicHook.ref)
+          this.globalLogicHookNames.push(logicHook.ref)
         }
       })
   }
@@ -92,14 +92,12 @@ module.exports = class LogicHookUtils {
       })
   }
 
-  getGlobalAsserter () {
-    return this.globalAsserters
-      .map(name => this.asserters[name])
+  getGlobalAsserters () {
+    return this.globalAsserterNames.reduce((agg, name) => ({ ...agg, [name]: this.asserters[name] }), {})
   }
 
-  getGlobalLogicHook () {
-    return this.globalLogicHooks
-      .map(name => this.logicHooks[name])
+  getGlobalLogicHooks () {
+    return this.globalLogicHookNames.reduce((agg, name) => ({ ...agg, [name]: this.logicHooks[name] }), {})
   }
 
   _loadClass ({ src, ref, args }, hookType) {

package/src/scripting/logichook/asserter/BaseTextAsserter.js

@@ -1,4 +1,3 @@
-const Capabilities = require('../../../Capabilities')
 const { BotiumError } = require('../../BotiumError')
 const { normalizeText, toString } = require('../../helper')
 const _ = require('lodash')
@@ -16,7 +15,7 @@ module.exports = class BaseTextAsserter {
   }
 
   _checkNormalizeText (str) {
-    return normalizeText(str, !!this.caps[Capabilities.SCRIPTING_NORMALIZE_TEXT])
+    return normalizeText(str, this.caps)
   }
 
   _normalize (botresponse) {

package/src/scripting/logichook/asserter/ButtonsAsserter.js

@@ -1,5 +1,4 @@
 const _ = require('lodash')
-const { SCRIPTING_NORMALIZE_TEXT } = require('../../../Capabilities')
 const { BotiumError } = require('../../BotiumError')
 const { buttonsFromMsg } = require('../helpers')
 const { normalizeText } = require('../../helper')
@@ -12,7 +11,7 @@ module.exports = class ButtonsAsserter {
   }
 
   _evalButtons (args, botMsg) {
-    const allButtons = buttonsFromMsg(botMsg, true).map(b => ({ text: b.text, payload: b.payload })).filter(b => b).map(b => ({ text: normalizeText(b.text, !!this.caps[SCRIPTING_NORMALIZE_TEXT]), payload: b.payload }))
+    const allButtons = buttonsFromMsg(botMsg, true).map(b => ({ text: b.text, payload: b.payload })).filter(b => b).map(b => ({ text: normalizeText(b.text, this.caps), payload: b.payload }))
     if (!args || args.length === 0) {
       return { allButtons, buttonsNotFound: [], buttonsFound: allButtons.map(b => b.text) }
     }
@@ -29,8 +28,8 @@ module.exports = class ButtonsAsserter {
       }
     }
     for (let i = 0; i < (args || []).length; i++) {
-      const matchByText = allButtons.some(b => this.context.Match(b.text, normalizeText(args[i], !!this.caps[SCRIPTING_NORMALIZE_TEXT])))
-      const matchByPayload = allButtons.some(b => this.context.Match(stringifyPayload(b.payload), normalizeText(args[i], !!this.caps[SCRIPTING_NORMALIZE_TEXT])))
+      const matchByText = allButtons.some(b => this.context.Match(b.text, normalizeText(args[i], this.caps)))
+      const matchByPayload = allButtons.some(b => this.context.Match(stringifyPayload(b.payload), normalizeText(args[i], this.caps)))
       if (matchByText || matchByPayload) {
         buttonsFound.push(args[i])
       } else {

package/src/scripting/logichook/userinput/MediaInput.js

@@ -1,7 +1,6 @@
 const fs = require('fs')
 const path = require('path')
-const globby = require('globby')
-const request = require('request')
+const globSync = require('tinyglobby').globSync
 const mime = require('mime-types')
 const url = require('url')
 const _ = require('lodash')
@@ -119,24 +118,22 @@ module.exports = class MediaInput {
           throw new Error(`downloadMedia failed: ${err.message}`)
         }
       } else if (uri.protocol === 'http:' || uri.protocol === 'https:') {
-        return new Promise((resolve, reject) => {
-          request({
-            uri: uri.toString(),
+        try {
+          const response = await fetch(uri.toString(), {
             method: 'GET',
-            followAllRedirects: true,
-            encoding: null,
-            timeout: this.globalArgs.downloadTimeout || 10000
-          }, (err, response, body) => {
-            if (err) {
-              reject(new Error(`downloadMedia failed: ${err.message}`))
-            } else {
-              if (response.statusCode >= 400) {
-                return reject(new Error(`downloadMedia failed: ${response.statusCode}/${response.statusMessage}`))
-              }
-              resolve(body)
-            }
+            redirect: 'follow', // Follows all redirects
+            timeout: this.globalArgs?.downloadTimeout || 10000
           })
-        })
+
+          if (!response.ok) {
+            throw new Error(`downloadMedia failed: ${response.status}/${response.statusText}`)
+          }
+
+          const arrayBuffer = await response.arrayBuffer()
+          return Buffer.from(arrayBuffer)
+        } catch (err) {
+          throw new Error(`downloadMedia failed: ${err.message}`)
+        }
       } else if (uri.protocol === 'data:') {
         return Buffer.from(uri.href.split(',')[1], 'base64')
       }
@@ -161,7 +158,7 @@ module.exports = class MediaInput {
       const baseDir = this._getBaseDir(convo)
       return args.reduce((e, arg) => {
         if (this._isWildcard(arg)) {
-          const mediaFiles = globby.sync(arg, { cwd: baseDir, gitignore: true })
+          const mediaFiles = globSync(arg, { cwd: baseDir })
           mediaFiles.forEach(mf => {
             e.push({
               name: 'MEDIA',

package/test/compiler/compilertxt.spec.js

@@ -252,6 +252,24 @@ describe('compiler.compilertxt', function () {
     assert.equal(convo.conversation[0].logicHooks.length, 0)
     assert.equal(convo.conversation[1].messageText, 'Hi')
   })
+  // connectors might handle json format differently
+  it('should read json as message', async function () {
+    const scriptBuffer = fs.readFileSync(path.resolve(__dirname, CONVOS_DIR, 'convos_jsonmessage.convo.txt'))
+    const context = buildContext()
+
+    const caps = {}
+    const compiler = new Compiler(context, Object.assign({}, DefaultCapabilities, caps))
+
+    compiler.Compile(scriptBuffer, 'SCRIPTING_TYPE_CONVO')
+    assert.deepEqual(context.convos[0].conversation[0].sourceData, {
+      sessionId: '1234567890876543',
+      text: 'Text message',
+      data: {
+        key: 'value'
+      }
+    })
+    assert.equal(!!context.convos[0].conversation[0].messageText, false)
+  })
 
   describe('compiler.compilertxt.logichooks', function () {
     it('should read logicHook if there is just logicHook', async function () {

package/test/compiler/convos/txt/convos_jsonmessage.convo.txt

@@ -0,0 +1,10 @@
+JSON
+
+#me
+{
+  "sessionId": "1234567890876543",
+  "text": "Text message",
+  "data": {
+    "key": "value"
+  }
+}