botium-core 1.14.10 → 1.15.2

package/src/scripting/helper.js

@@ -4,9 +4,35 @@ const speechScorer = require('word-error-rate')
 const debug = require('debug')('botium-core-scripting-helper')
 
 const { E_SCRIPTING_MEMORY_COLUMN_MODE } = require('../Enums')
+const Capabilities = require('../Capabilities')
 const WHITE_SPACES_EXCEPT_SPACE_CHAR_AT_THE_END = /[\n\t\r]+$/
 
-const normalizeText = (str, doCleanup) => {
+const normalizeText = (str, doCleanupOrCaps) => {
+  // TODO testlog
+  debug('yxc1', doCleanupOrCaps)
+  let basic
+  let charactersRemove = false
+  let regexpRemove = false
+  if (_.isBoolean(doCleanupOrCaps) || _.isNil(doCleanupOrCaps)) {
+    debug('Normalize text: backward compatibility mode. Use caps instead of boolean flag')
+    basic = !!doCleanupOrCaps
+  } else {
+    const caps = doCleanupOrCaps
+    basic = !!caps[Capabilities.SCRIPTING_NORMALIZE_TEXT]
+    if (caps[Capabilities.SCRIPTING_NORMALIZE_TEXT_REMOVE_CHARACTERES]) {
+      charactersRemove = caps[Capabilities.SCRIPTING_NORMALIZE_TEXT_REMOVE_CHARACTERES]
+      if (_.isString(charactersRemove)) {
+        const splitted = charactersRemove.split(/(?<!\/),/).map(e => e.trim()).map(e => e.split('/,').join(',').split('//').join('/')).filter(c => c.length > 0)
+        charactersRemove = splitted.length ? splitted : [charactersRemove]
+      } else if (!_.isArray(charactersRemove)) {
+        charactersRemove = false
+      }
+    }
+    if (caps[Capabilities.SCRIPTING_NORMALIZE_TEXT_REMOVE_REGEXP]) {
+      regexpRemove = new RegExp(caps[Capabilities.SCRIPTING_NORMALIZE_TEXT_REMOVE_REGEXP], 'ug')
+    }
+  }
+
   if (str && _.isArray(str)) {
     str = str.join(' ')
   } else if (str && !_.isString(str)) {
@@ -16,27 +42,38 @@ const normalizeText = (str, doCleanup) => {
       str = `${str}`
     }
   }
-  if (str && doCleanup) {
-    // remove html tags
-    str = str.replace(/<p[^>]*>/g, ' ')
-    str = str.replace(/<\/p>/g, ' ')
-    str = str.replace(/<br[^>]*>/g, ' ')
-    str = str.replace(/<[^>]*>/g, '')
-    /* eslint-disable no-control-regex */
-    // remove not printable characters
-    str = str.replace(/[\x00-\x1F\x7F]/g, ' ')
-    /* eslint-enable no-control-regex */
-    // replace html entities
-    str = str
-      .replace(/&amp;/g, '&')
-      .replace(/&lt;/g, '<')
-      .replace(/&gt;/g, '>')
-      .replace(/&#39;/g, '\'')
-      .replace(/&quot;/g, '"')
-    // replace two spaces with one
-    str = str.replace(/\s+/g, ' ')
-
-    str = str.split('\n').map(s => s.trim()).join('\n').trim()
+
+  if (str) {
+    if (basic) {
+      // remove html tags
+      str = str.replace(/<p[^>]*>/g, ' ')
+      str = str.replace(/<\/p>/g, ' ')
+      str = str.replace(/<br[^>]*>/g, ' ')
+      str = str.replace(/<[^>]*>/g, '')
+      /* eslint-disable no-control-regex */
+      // remove not printable characters
+      str = str.replace(/[\x00-\x1F\x7F]/g, ' ')
+      /* eslint-enable no-control-regex */
+      // replace html entities
+      str = str
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&#39;/g, '\'')
+        .replace(/&quot;/g, '"')
+      // replace two spaces with one
+      str = str.replace(/\s+/g, ' ')
+
+      str = str.split('\n').map(s => s.trim()).join('\n').trim()
+    }
+    if (charactersRemove) {
+      for (const character of charactersRemove) {
+        str = str.split(character).join('')
+      }
+    }
+    if (regexpRemove) {
+      str = str.replace(regexpRemove, '')
+    }
   }
   return str
 }

package/src/scripting/logichook/asserter/BaseTextAsserter.js

@@ -1,4 +1,3 @@
-const Capabilities = require('../../../Capabilities')
 const { BotiumError } = require('../../BotiumError')
 const { normalizeText, toString } = require('../../helper')
 const _ = require('lodash')
@@ -16,7 +15,7 @@ module.exports = class BaseTextAsserter {
   }
 
   _checkNormalizeText (str) {
-    return normalizeText(str, !!this.caps[Capabilities.SCRIPTING_NORMALIZE_TEXT])
+    return normalizeText(str, this.caps)
   }
 
   _normalize (botresponse) {

package/src/scripting/logichook/asserter/ButtonsAsserter.js

@@ -1,5 +1,4 @@
 const _ = require('lodash')
-const { SCRIPTING_NORMALIZE_TEXT } = require('../../../Capabilities')
 const { BotiumError } = require('../../BotiumError')
 const { buttonsFromMsg } = require('../helpers')
 const { normalizeText } = require('../../helper')
@@ -12,7 +11,7 @@ module.exports = class ButtonsAsserter {
   }
 
   _evalButtons (args, botMsg) {
-    const allButtons = buttonsFromMsg(botMsg, true).map(b => ({ text: b.text, payload: b.payload })).filter(b => b).map(b => ({ text: normalizeText(b.text, !!this.caps[SCRIPTING_NORMALIZE_TEXT]), payload: b.payload }))
+    const allButtons = buttonsFromMsg(botMsg, true).map(b => ({ text: b.text, payload: b.payload })).filter(b => b).map(b => ({ text: normalizeText(b.text, this.caps), payload: b.payload }))
     if (!args || args.length === 0) {
       return { allButtons, buttonsNotFound: [], buttonsFound: allButtons.map(b => b.text) }
     }
@@ -29,8 +28,8 @@ module.exports = class ButtonsAsserter {
       }
     }
     for (let i = 0; i < (args || []).length; i++) {
-      const matchByText = allButtons.some(b => this.context.Match(b.text, normalizeText(args[i], !!this.caps[SCRIPTING_NORMALIZE_TEXT])))
-      const matchByPayload = allButtons.some(b => this.context.Match(stringifyPayload(b.payload), normalizeText(args[i], !!this.caps[SCRIPTING_NORMALIZE_TEXT])))
+      const matchByText = allButtons.some(b => this.context.Match(b.text, normalizeText(args[i], this.caps)))
+      const matchByPayload = allButtons.some(b => this.context.Match(stringifyPayload(b.payload), normalizeText(args[i], this.caps)))
       if (matchByText || matchByPayload) {
         buttonsFound.push(args[i])
       } else {

package/src/scripting/logichook/userinput/MediaInput.js

@@ -1,7 +1,6 @@
 const fs = require('fs')
 const path = require('path')
 const globSync = require('tinyglobby').globSync
-const request = require('request')
 const mime = require('mime-types')
 const url = require('url')
 const _ = require('lodash')
@@ -119,24 +118,22 @@ module.exports = class MediaInput {
           throw new Error(`downloadMedia failed: ${err.message}`)
         }
       } else if (uri.protocol === 'http:' || uri.protocol === 'https:') {
-        return new Promise((resolve, reject) => {
-          request({
-            uri: uri.toString(),
+        try {
+          const response = await fetch(uri.toString(), {
             method: 'GET',
-            followAllRedirects: true,
-            encoding: null,
-            timeout: this.globalArgs.downloadTimeout || 10000
-          }, (err, response, body) => {
-            if (err) {
-              reject(new Error(`downloadMedia failed: ${err.message}`))
-            } else {
-              if (response.statusCode >= 400) {
-                return reject(new Error(`downloadMedia failed: ${response.statusCode}/${response.statusMessage}`))
-              }
-              resolve(body)
-            }
+            redirect: 'follow', // Follows all redirects
+            timeout: this.globalArgs?.downloadTimeout || 10000
           })
-        })
+
+          if (!response.ok) {
+            throw new Error(`downloadMedia failed: ${response.status}/${response.statusText}`)
+          }
+
+          const arrayBuffer = await response.arrayBuffer()
+          return Buffer.from(arrayBuffer)
+        } catch (err) {
+          throw new Error(`downloadMedia failed: ${err.message}`)
+        }
       } else if (uri.protocol === 'data:') {
         return Buffer.from(uri.href.split(',')[1], 'base64')
       }

package/test/compiler/compilertxt.spec.js

@@ -252,6 +252,24 @@ describe('compiler.compilertxt', function () {
     assert.equal(convo.conversation[0].logicHooks.length, 0)
     assert.equal(convo.conversation[1].messageText, 'Hi')
   })
+  // connectors might handle json format differently
+  it('should read json as message', async function () {
+    const scriptBuffer = fs.readFileSync(path.resolve(__dirname, CONVOS_DIR, 'convos_jsonmessage.convo.txt'))
+    const context = buildContext()
+
+    const caps = {}
+    const compiler = new Compiler(context, Object.assign({}, DefaultCapabilities, caps))
+
+    compiler.Compile(scriptBuffer, 'SCRIPTING_TYPE_CONVO')
+    assert.deepEqual(context.convos[0].conversation[0].sourceData, {
+      sessionId: '1234567890876543',
+      text: 'Text message',
+      data: {
+        key: 'value'
+      }
+    })
+    assert.equal(!!context.convos[0].conversation[0].messageText, false)
+  })
 
   describe('compiler.compilertxt.logichooks', function () {
     it('should read logicHook if there is just logicHook', async function () {

package/test/compiler/convos/txt/convos_jsonmessage.convo.txt

@@ -0,0 +1,10 @@
+JSON
+
+#me
+{
+  "sessionId": "1234567890876543",
+  "text": "Text message",
+  "data": {
+    "key": "value"
+  }
+}