botium-core 1.13.19 → 1.14.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botium-core",
-  "version": "1.13.19",
+  "version": "1.14.1",
   "description": "The Selenium for Chatbots",
   "main": "index.js",
   "module": "dist/botium-es.js",
@@ -18,7 +18,6 @@
     "link": "npm link botium-connector-dialogflow botium-connector-webdriverio botium-connector-directline3 botium-connector-watson botium-connector-alexa-smapi botium-connector-echo",
     "test": "cross-env NODE_PATH=\"./test/plugins/plugindir/fromfolder:./test/plugins/plugindir/fromfile:./test/security/resources\" mocha \"./test/**/*.spec.js\"",
     "coverage:report": "nyc report --reporter=lcov npm test",
-    "license-checker": "license-checker > LICENSES-3RDPARTY.txt",
     "update-dependencies": "npm-check-updates --reject globby,rollup -u --timeout 120000"
   },
   "repository": {
@@ -32,14 +31,13 @@
   },
   "homepage": "https://www.botium.ai",
   "dependencies": {
-    "@babel/runtime": "^7.21.5",
-    "async": "^3.2.4",
+    "@babel/runtime": "^7.23.2",
+    "async": "^3.2.5",
     "body-parser": "^1.20.2",
     "boolean": "^3.2.0",
     "bottleneck": "^2.19.5",
-    "csv-parse": "^5.3.10",
+    "csv-parse": "^5.5.2",
     "debug": "^4.3.4",
-    "esprima": "^4.0.1",
     "express": "^4.18.2",
     "globby": "11.0.4",
     "ioredis": "^5.3.2",
@@ -47,52 +45,49 @@
     "is-json": "^2.0.1",
     "jsonpath": "^1.1.1",
     "lodash": "^4.17.21",
-    "markdown-it": "^13.0.1",
+    "markdown-it": "^13.0.2",
     "mime-types": "^2.1.35",
     "mkdirp": "^3.0.1",
     "moment": "^2.29.4",
     "moment-timezone": "^0.5.43",
     "mustache": "^4.2.0",
     "promise-retry": "^2.0.1",
-    "promise.allsettled": "^1.0.6",
+    "promise.allsettled": "^1.0.7",
     "randomatic": "^3.1.1",
     "request": "^2.88.2",
-    "rimraf": "^5.0.0",
+    "rimraf": "^5.0.5",
     "sanitize-filename": "^1.6.3",
     "slugify": "^1.6.6",
-    "socket.io": "^4.6.1",
-    "socket.io-client": "^4.6.1",
+    "socket.io": "^4.7.2",
+    "socket.io-client": "^4.7.2",
     "socketio-auth": "^0.1.1",
     "swagger-jsdoc": "^6.2.8",
-    "swagger-ui-express": "^4.6.3",
-    "uuid": "^9.0.0",
-    "vm2": "^3.9.17",
+    "swagger-ui-express": "^5.0.0",
+    "uuid": "^9.0.1",
     "word-error-rate": "0.0.7",
     "write-yaml": "^1.0.0",
     "xlsx": "^0.18.5",
     "xregexp": "^5.1.1",
-    "yaml": "^2.2.2"
+    "yaml": "^2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "^7.21.8",
-    "@babel/node": "^7.20.7",
-    "@babel/plugin-transform-runtime": "^7.21.4",
-    "@babel/preset-env": "^7.21.5",
-    "chai": "^4.3.7",
+    "@babel/core": "^7.23.3",
+    "@babel/node": "^7.22.19",
+    "@babel/plugin-transform-runtime": "^7.23.3",
+    "@babel/preset-env": "^7.23.3",
+    "chai": "^4.3.10",
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
-    "eslint": "^8.40.0",
-    "eslint-config-standard": "^17.0.0",
-    "eslint-plugin-import": "^2.27.5",
-    "eslint-plugin-mocha": "^10.1.0",
-    "eslint-plugin-n": "^15.7.0",
+    "eslint": "^8.53.0",
+    "eslint-config-standard": "^17.1.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-mocha": "^10.2.0",
+    "eslint-plugin-n": "^16.3.1",
     "eslint-plugin-promise": "^6.1.1",
     "eslint-plugin-standard": "^4.1.0",
-    "license-checker": "^25.0.1",
-    "license-compatibility-checker": "^0.3.5",
     "mocha": "^10.2.0",
-    "nock": "^13.3.1",
-    "npm-check-updates": "^16.10.12",
+    "nock": "^13.3.8",
+    "npm-check-updates": "^16.14.6",
     "nyc": "^15.1.0",
     "rollup": "2.79.1",
     "rollup-plugin-babel": "^4.4.0",

package/samples/extensions/asserterHooks/botium.json

@@ -7,6 +7,7 @@
       "WATSON_PASSWORD": "ZWDE5xo02sby",
       "WATSON_WORKSPACE_ID": "97513bc0-c581-4bec-ac9f-ea6a8ec308a9",
       "SCRIPTING_ENABLE_MEMORY": true,
+      "SAFEDIR": ".",
       "ASSERTERS": [
         {
           "ref": "DUMMY",

package/samples/extensions/logichooks/botium.json

@@ -3,6 +3,7 @@
     "Capabilities": {
       "PROJECTNAME": "Botium Logichooks Sample",
       "CONTAINERMODE": "echo",
+      "SAFEDIR": ".",
       "ASSERTERS": [
         {
           "ref": "MYASSERTER",

package/samples/extensions/logichooks/custom/MyAsserter.js

@@ -6,17 +6,17 @@ module.exports = class MyAsserter {
     this.caps = caps
   }
 
-  assertConvoBegin ({convo, container, args}) {
+  assertConvoBegin ({ convo, container, args }) {
     console.log(`MyAsserter assertConvoBegin: ${convo.header.name}`)
     return Promise.resolve()
   }
 
-  assertConvoStep ({convo, convoStep, args, botMsg}) {
+  assertConvoStep ({ convo, convoStep, args, botMsg }) {
     console.log(`MyAsserter assertConvoStep, botMessage: ${utils.inspect(botMsg)} ...`)
     return Promise.resolve()
   }
 
-  assertConvoEnd ({convo, container, transcript, args}) {
+  assertConvoEnd ({ convo, container, transcript, args }) {
     console.log(`MyAsserter assertConvoEnd ${convo.header.name}, transcript: ${utils.inspect(transcript)} ...`)
     return Promise.resolve()
   }

package/src/Capabilities.js

@@ -3,6 +3,7 @@ module.exports = {
   TESTSESSIONNAME: 'TESTSESSIONNAME',
   TESTCASENAME: 'TESTCASENAME',
   TEMPDIR: 'TEMPDIR',
+  SAFEDIR: 'SAFEDIR',
   CLEANUPTEMPDIR: 'CLEANUPTEMPDIR',
   WAITFORBOTTIMEOUT: 'WAITFORBOTTIMEOUT',
   CONTAINERMODE: 'CONTAINERMODE',
@@ -12,7 +13,7 @@ module.exports = {
   BOTIUMGRIDURL: 'BOTIUMGRIDURL',
   BOTIUMAPITOKEN: 'BOTIUMAPITOKEN',
   BOTIUMGRIDSLOT: 'BOTIUMGRIDSLOT',
-  // Simple Reset Bot Settings
+  // Simple Rest Bot Settings
   SIMPLEREST_PING_URL: 'SIMPLEREST_PING_URL',
   SIMPLEREST_PING_VERB: 'SIMPLEREST_PING_VERB',
   SIMPLEREST_PING_BODY: 'SIMPLEREST_PING_BODY',

package/src/containers/PluginConnectorContainer.js

@@ -162,4 +162,12 @@ module.exports = class PluginConnectorContainer extends BaseContainer {
       return Promise.reject(new Error(`Clean - Botium plugin failed: ${util.inspect(err)}`))
     }
   }
+
+  GetMetaData () {
+    try {
+      return (this.pluginInstance.GetMetaData ? (this.pluginInstance.GetMetaData() || Promise.resolve()) : Promise.resolve())
+    } catch (err) {
+      return Promise.reject(new Error(`GetMetaData - Botium plugin failed: ${util.inspect(err)}`))
+    }
+  }
 }

package/src/containers/plugins/SimpleRestContainer.js

@@ -336,24 +336,33 @@ module.exports = class SimpleRestContainer {
 
     const result = []
     if (isFromUser) {
-      const _extractFrom = (root, jsonPaths) => {
-        const flattened = []
+      const _extractFrom = (root, jsonPaths, acceptFn = null) => {
+        const result = []
         for (const jsonPath of jsonPaths) {
+          const jsonPathRes = []
           const rb = jp.query(root, jsonPath)
           if (_.isArray(rb)) {
-            _.flattenDeep(rb).forEach(r => flattened.push(r))
-          } else if (rb) {
-            flattened.push(rb)
+            _.flattenDeep(rb).forEach(r => jsonPathRes.push(r))
+          } else {
+            jsonPathRes.push(rb)
+          }
+          if (acceptFn) {
+            result.push(...jsonPathRes.filter(r => acceptFn(root, jsonPath, r)))
+          } else {
+            result.push(...jsonPathRes)
           }
         }
-        return flattened
+        return result
       }
 
       const jsonPathRoots = []
-
       const jsonPathsBody = getAllCapValues(Capabilities.SIMPLEREST_BODY_JSONPATH, this.caps)
       if (jsonPathsBody.length > 0) {
-        jsonPathRoots.push(..._extractFrom(body, jsonPathsBody))
+        jsonPathRoots.push(..._extractFrom(body, jsonPathsBody, (root, jsonPath, r) => {
+          if (r && _.isObject(r)) return true
+          debug(`Ignoring result body from ${jsonPath} - not a querieable object (${util.inspect(r)})`)
+          return false
+        }))
       } else {
         jsonPathRoots.push(body)
       }
@@ -381,7 +390,6 @@ module.exports = class SimpleRestContainer {
           const jsonPathsButtonsPayload = getAllCapValues(`${capPrefix}_PAYLOAD_SUBJSONPATH`, this.caps)
 
           const retrievedButtons = []
-
           const responseButtons = _extractFrom(jsonPathButtonRoot, jsonPathsButtons)
           for (const responseButton of responseButtons) {
             const retrievedButton = {}

package/src/containers/plugins/index.js

@@ -5,7 +5,6 @@ const debug = require('debug')('botium-connector-PluginConnectorContainer-helper
 
 const SimpleRestContainer = require('./SimpleRestContainer')
 const Capabilities = require('../../Capabilities')
-const { BotiumError } = require('../../scripting/BotiumError')
 
 const pluginResolver = (containermode) => {
   if (containermode === 'simplerest') {
@@ -59,23 +58,6 @@ const loadConnectorModule = (PluginClass, args) => {
 
 const tryLoadPlugin = (containermode, modulepath, args) => {
   const pluginLoaderSpec = modulepath || containermode
-  const _checkUnsafe = (caps, mode, cause) => {
-    if (!caps[Capabilities.SECURITY_ALLOW_UNSAFE]) {
-      throw new BotiumError(
-        `Security Error. Using unsafe connector mode "${mode}" is not allowed`,
-        {
-          type: 'security',
-          subtype: 'allow unsafe',
-          source: 'src/containers/plugins/index.js',
-          cause: {
-            SECURITY_ALLOW_UNSAFE: caps[Capabilities.SECURITY_ALLOW_UNSAFE],
-            mode,
-            ...cause
-          }
-        }
-      )
-    }
-  }
 
   if (pluginResolver(pluginLoaderSpec)) {
     const pluginInstance = new (pluginResolver(pluginLoaderSpec))(args)
@@ -88,44 +70,50 @@ const tryLoadPlugin = (containermode, modulepath, args) => {
     return pluginInstance
   }
   const loadErr = []
+  const allowUnsafe = !!args.caps[Capabilities.SECURITY_ALLOW_UNSAFE]
 
   if (_.isString(pluginLoaderSpec)) {
-    const tryLoadFile = path.resolve(process.cwd(), pluginLoaderSpec)
-    if (fs.existsSync(tryLoadFile)) {
-      _checkUnsafe(args.caps, 'Using work dir', { modulepath, containermode })
+    if (args.caps.SAFEDIR) {
+      const tryLoadFile = path.resolve(args.caps.SAFEDIR, pluginLoaderSpec)
+      if (tryLoadFile.startsWith(path.resolve(args.caps.SAFEDIR))) {
+        if (fs.existsSync(tryLoadFile)) {
+          try {
+            let plugin = require(tryLoadFile)
+            if (plugin.default) {
+              plugin = plugin.default
+            }
+            if (!plugin.PluginVersion || !plugin.PluginClass) {
+              loadErr.push(`Invalid Botium plugin loaded from ${tryLoadFile}, expected PluginVersion, PluginClass fields`)
+            } else {
+              const pluginInstance = loadConnectorModule(plugin.PluginClass, args)
+              debug(`Botium plugin loaded from ${tryLoadFile}`)
+              return pluginInstance
+            }
+          } catch (err) {
+            loadErr.push(`Loading Botium plugin from ${tryLoadFile} failed - ${err.message}`)
+          }
+        }
+      }
+    }
+
+    if (allowUnsafe) {
       try {
-        let plugin = require(tryLoadFile)
+        let plugin = require(pluginLoaderSpec)
         if (plugin.default) {
           plugin = plugin.default
         }
         if (!plugin.PluginVersion || !plugin.PluginClass) {
-          loadErr.push(`Invalid Botium plugin loaded from ${tryLoadFile}, expected PluginVersion, PluginClass fields`)
+          loadErr.push(`Invalid Botium plugin loaded from ${pluginLoaderSpec}, expected PluginVersion, PluginClass fields`)
         } else {
           const pluginInstance = loadConnectorModule(plugin.PluginClass, args)
-          debug(`Botium plugin loaded from ${tryLoadFile}`)
+          debug(`Botium plugin loaded from ${pluginLoaderSpec}. Plugin version is ${getModuleVersionSafe(pluginLoaderSpec)}`)
           return pluginInstance
         }
       } catch (err) {
-        loadErr.push(`Loading Botium plugin from ${tryLoadFile} failed - ${err.message}`)
+        loadErr.push(`Loading Botium plugin from ${pluginLoaderSpec} failed - ${err.message}`)
       }
     }
 
-    try {
-      let plugin = require(pluginLoaderSpec)
-      if (plugin.default) {
-        plugin = plugin.default
-      }
-      if (!plugin.PluginVersion || !plugin.PluginClass) {
-        loadErr.push(`Invalid Botium plugin loaded from ${pluginLoaderSpec}, expected PluginVersion, PluginClass fields`)
-      } else {
-        const pluginInstance = loadConnectorModule(plugin.PluginClass, args)
-        debug(`Botium plugin loaded from ${pluginLoaderSpec}. Plugin version is ${getModuleVersionSafe(pluginLoaderSpec)}`)
-        return pluginInstance
-      }
-    } catch (err) {
-      loadErr.push(`Loading Botium plugin from ${pluginLoaderSpec} failed - ${err.message}`)
-    }
-
     const tryLoadPackage = `botium-connector-${pluginLoaderSpec}`
     try {
       let plugin = require(tryLoadPackage)

package/src/helpers/HookUtils.js

@@ -1,48 +1,28 @@
 const util = require('util')
 const path = require('path')
 const fs = require('fs')
-const { NodeVM } = require('vm2')
-const esprima = require('esprima')
 const _ = require('lodash')
 const debug = require('debug')('botium-core-HookUtils')
 
 const Capabilities = require('../Capabilities')
-const { BotiumError } = require('../scripting/BotiumError')
 
-const executeHook = async (caps, hook, args) => {
-  return executeHookSync(caps, hook, args)
+const executeHook = async (caps, hook, ...args) => {
+  return executeHookSync(caps, hook, ...args)
 }
 
-const executeHookSync = (caps, hook, args) => {
+const executeHookSync = (caps, hook, ...args) => {
   if (!hook) {
     return
   }
 
   if (_.isFunction(hook)) {
     try {
-      return hook(args)
+      return hook(...args)
     } catch (err) {
       throw new Error(`Calling Hook function failed: ${err.message}`)
     }
   }
 
-  if (_.isString(hook)) {
-    try {
-      const vm = new NodeVM({
-        eval: false,
-        require: false,
-        sandbox: args
-      })
-      const r = vm.run(hook)
-      if (_.isFunction(r)) {
-        return r(args)
-      } else {
-        return r
-      }
-    } catch (err) {
-      throw new Error(`Calling Hook Javascript code failed: ${err.message}`)
-    }
-  }
   throw new Error(`Unknown hook ${typeof hook}`)
 }
 
@@ -58,56 +38,40 @@ const getHook = (caps, data) => {
   }
 
   if (_.isString(data)) {
-    let resultWithRequire
-    let tryLoadFile = path.resolve(process.cwd(), data)
-    if (fs.existsSync(tryLoadFile)) {
-      try {
-        resultWithRequire = require(tryLoadFile)
-      } catch (err) {
-      }
-    } else {
-      tryLoadFile = data
-      try {
-        resultWithRequire = require(data)
-      } catch (err) {
-      }
-    }
-
-    if (resultWithRequire) {
-      if (!allowUnsafe) {
-        throw new BotiumError(
-          'Security Error. Using unsafe custom hook with require is not allowed',
-          {
-            type: 'security',
-            subtype: 'allow unsafe',
-            source: path.basename(__filename),
-            cause: {
-              SECURITY_ALLOW_UNSAFE: caps[Capabilities.SECURITY_ALLOW_UNSAFE],
-              hookData: data
+    if (caps.SAFEDIR) {
+      const tryLoadFile = path.resolve(caps.SAFEDIR, data)
+      if (tryLoadFile.startsWith(path.resolve(caps.SAFEDIR))) {
+        if (fs.existsSync(tryLoadFile)) {
+          try {
+            const resultWithRequire = require(tryLoadFile)
+            if (_.isFunction(resultWithRequire)) {
+              debug(`found hook, type: safedir, in ${tryLoadFile}`)
+              return resultWithRequire
+            } else {
+              throw new Error(`Expected function from hook specification "${util.inspect(data)}", got: "${util.inspect(resultWithRequire)}"`)
             }
+          } catch (err) {
+            debug(`Failed loading hook, type: safedir, from ${tryLoadFile} failed: ${err.message || err}`)
           }
-        )
-      }
-
-      if (_.isFunction(resultWithRequire)) {
-        debug(`found hook, type: require, in ${tryLoadFile}`)
-        return resultWithRequire
-      } else {
-        throw new Error(`Cant load hook ${tryLoadFile} because it is not a function`)
+        }
       }
     }
-
-    try {
-      esprima.parseScript(data)
-    } catch (err) {
-      throw new Error(`Cant load hook, syntax is not valid - ${util.inspect(err)}`)
+    if (allowUnsafe || data.startsWith('botium-')) {
+      const tryLoadFile = data
+      try {
+        const resultWithRequire = require(tryLoadFile)
+        if (_.isFunction(resultWithRequire)) {
+          debug(`found hook, type: require, in ${tryLoadFile}`)
+          return resultWithRequire
+        } else {
+          throw new Error(`Expected function from hook specification "${util.inspect(data)}", got: "${util.inspect(resultWithRequire)}"`)
+        }
+      } catch (err) {
+        debug(`Failed loading hook, type: require, from ${tryLoadFile} failed: ${err.message || err}`)
+      }
     }
-
-    debug('Found hook, type: JavaScript as String')
-    return data
   }
-
-  throw new Error(`Not valid hook ${util.inspect(data)}`)
+  throw new Error(`Hook specification "${util.inspect(data)}" invalid: no loader available`)
 }
 
 module.exports = {

package/src/scripting/Convo.js

@@ -275,6 +275,8 @@ class Convo {
       let botMsg = null
       let waitForBotSays = true
       let skipTranscriptStep = false
+      let conditionalGroupId = null
+      let conditionMetInGroup = false
       for (let i = 0; i < this.conversation.length; i++) {
         const convoStep = this.conversation[i]
         const currentStepIndex = i
@@ -356,7 +358,6 @@ class Convo {
               throw failErr
             }
           } else if (convoStep.sender === 'bot') {
-            const previousWaitForBotSays = waitForBotSays
             if (waitForBotSays) {
               botMsg = null
             } else {
@@ -406,22 +407,38 @@ class Convo {
             }
 
             if (convoStep.conditional) {
+              waitForBotSays = false
+              let endOfConditionalGroup = false
+              conditionalGroupId = convoStep.logicHooks.find(lh => lh.name.startsWith('CONDITIONAL_STEP')).args[1]
               const nextConvoStep = this.conversation[i + 1]
 
-              if (!previousWaitForBotSays) {
-                skipTranscriptStep = true
-              }
-              waitForBotSays = false
               if (!nextConvoStep || nextConvoStep.sender !== 'bot' || !nextConvoStep.logicHooks || !nextConvoStep.logicHooks.some(lh => lh.name.toUpperCase().startsWith('CONDITIONAL_STEP'))) {
-                waitForBotSays = true
+                endOfConditionalGroup = true
               } else {
-                const conditionalLogicHook = convoStep.logicHooks.find(lh => lh.name.startsWith('CONDITIONAL_STEP'))
                 const nextConditionalLogicHook = nextConvoStep.logicHooks.find(lh => lh.name.startsWith('CONDITIONAL_STEP'))
-                waitForBotSays = conditionalLogicHook.args[1] !== nextConditionalLogicHook.args[1]
+                endOfConditionalGroup = conditionalGroupId !== nextConditionalLogicHook.args[1]
               }
 
-              if (convoStep.conditional.skip) {
+              if (convoStep.conditional.skip || conditionMetInGroup) {
+                skipTranscriptStep = true
+                if (endOfConditionalGroup && !conditionMetInGroup && !convoStep.optional) {
+                  const failErr = new BotiumError(`${this.header.name}/${convoStep.stepTag}: Non of the conditions are met in ${conditionalGroupId ? `'${conditionalGroupId}' ` : ''}condition group`)
+                  debug(failErr)
+                  throw failErr
+                }
+                if (endOfConditionalGroup) {
+                  waitForBotSays = !convoStep.optional
+                  conditionalGroupId = undefined
+                  conditionMetInGroup = false
+                }
                 continue
+              } else {
+                conditionMetInGroup = true
+                if (endOfConditionalGroup) {
+                  waitForBotSays = !convoStep.optional
+                  conditionalGroupId = undefined
+                  conditionMetInGroup = false
+                }
               }
             }
 

package/src/scripting/ScriptingMemory.js

@@ -3,7 +3,6 @@ const debug = require('debug')('botium-core-ScriptingMemory')
 const randomize = require('randomatic')
 const { v1: uuidv1 } = require('uuid')
 const moment = require('moment')
-const { NodeVM } = require('vm2')
 const _ = require('lodash')
 const path = require('path')
 const jp = require('jsonpath')
@@ -181,29 +180,6 @@ const SCRIPTING_FUNCTIONS_RAW = {
       else return ''
     },
     numberOfArguments: 1
-  },
-
-  $func: {
-    handler: (caps, code) => {
-      if (code == null) {
-        throw Error('func function used without args!')
-      }
-      try {
-        const vm = new NodeVM({
-          eval: false,
-          require: false,
-          env: caps[Capabilities.SECURITY_ALLOW_UNSAFE] ? process.env : {},
-          sandbox: {
-            caps,
-            moment
-          }
-        })
-        return vm.run(`module.exports = (${code})`)
-      } catch (err) {
-        throw Error(`func function execution failed - ${err}`)
-      }
-    },
-    numberOfArguments: 1
   }
 }