botium-core 1.12.4 → 1.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botium-core",
-  "version": "1.12.4",
+  "version": "1.13.0",
   "description": "The Selenium for Chatbots",
   "main": "index.js",
   "module": "dist/botium-es.js",
@@ -32,25 +32,25 @@
   },
   "homepage": "https://www.botium.ai",
   "dependencies": {
-    "@babel/runtime": "^7.17.8",
-    "async": "^3.2.3",
-    "body-parser": "^1.19.2",
+    "@babel/runtime": "^7.18.3",
+    "async": "^3.2.4",
+    "body-parser": "^1.20.0",
     "boolean": "^3.2.0",
     "bottleneck": "^2.19.5",
-    "csv-parse": "^5.0.4",
+    "csv-parse": "^5.2.0",
     "debug": "^4.3.4",
     "esprima": "^4.0.1",
-    "express": "^4.17.3",
+    "express": "^4.18.1",
     "globby": "11.0.4",
-    "ioredis": "^4.28.5",
+    "ioredis": "^5.0.6",
     "is-class": "^0.0.9",
     "is-json": "^2.0.1",
     "jsonpath": "^1.1.1",
     "lodash": "^4.17.21",
-    "markdown-it": "^12.3.2",
+    "markdown-it": "^13.0.1",
     "mime-types": "^2.1.35",
     "mkdirp": "^1.0.4",
-    "moment": "^2.29.1",
+    "moment": "^2.29.3",
     "mustache": "^4.2.0",
     "promise-retry": "^2.0.1",
     "promise.allsettled": "^1.0.5",
@@ -59,39 +59,39 @@
     "rimraf": "^3.0.2",
     "sanitize-filename": "^1.6.3",
     "slugify": "^1.6.5",
-    "socket.io": "^4.4.1",
-    "socket.io-client": "^4.4.1",
+    "socket.io": "^4.5.1",
+    "socket.io-client": "^4.5.1",
     "socketio-auth": "^0.1.1",
-    "swagger-jsdoc": "^6.1.0",
-    "swagger-ui-express": "^4.3.0",
+    "swagger-jsdoc": "^6.2.1",
+    "swagger-ui-express": "^4.4.0",
     "uuid": "^8.3.2",
     "vm2": "^3.9.9",
     "write-yaml": "^1.0.0",
-    "xlsx": "^0.18.4",
-    "xregexp": "^5.1.0",
-    "yaml": "^1.10.2"
+    "xlsx": "^0.18.5",
+    "xregexp": "^5.1.1",
+    "yaml": "^2.1.1"
   },
   "devDependencies": {
-    "@babel/core": "^7.17.8",
-    "@babel/node": "^7.16.8",
-    "@babel/plugin-transform-runtime": "^7.17.0",
-    "@babel/preset-env": "^7.16.11",
+    "@babel/core": "^7.18.5",
+    "@babel/node": "^7.18.5",
+    "@babel/plugin-transform-runtime": "^7.18.5",
+    "@babel/preset-env": "^7.18.2",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
-    "eslint": "^8.11.0",
-    "eslint-config-standard": "^16.0.3",
-    "eslint-plugin-import": "^2.25.4",
-    "eslint-plugin-node": "^11.1.0",
+    "eslint": "^8.18.0",
+    "eslint-config-standard": "^17.0.0",
+    "eslint-plugin-import": "^2.26.0",
+    "eslint-plugin-n": "^15.2.3",
     "eslint-plugin-promise": "^6.0.0",
     "eslint-plugin-standard": "^4.1.0",
     "license-checker": "^25.0.1",
     "license-compatibility-checker": "^0.3.5",
-    "mocha": "^9.2.2",
-    "nock": "^13.2.4",
-    "npm-check-updates": "^12.5.4",
+    "mocha": "^10.0.0",
+    "nock": "^13.2.7",
+    "npm-check-updates": "^14.0.1",
     "nyc": "^15.1.0",
-    "rollup": "^2.70.1",
+    "rollup": "^2.75.6",
     "rollup-plugin-babel": "^4.4.0",
     "rollup-plugin-commonjs": "^10.1.0",
     "rollup-plugin-json": "^4.0.0",

package/src/BotDriver.js

@@ -115,6 +115,7 @@ module.exports = class BotDriver {
     this.eventEmitter.emit(Events.CONTAINER_BUILDING)
 
     return new Promise((resolve, reject) => {
+      let tempDirectory = null
       let repo = null
       let container = null
 
@@ -126,9 +127,19 @@ module.exports = class BotDriver {
             .catch(driverValidated)
         },
 
+        (tempDirectoryCreated) => {
+          tempDirectory = path.resolve(process.cwd(), this.caps[Capabilities.TEMPDIR], sanitize(`${this.caps[Capabilities.PROJECTNAME]} ${moment().format('YYYYMMDD HHmmss')} ${randomize('Aa0', 5)}`))
+          try {
+            mkdirp.sync(tempDirectory)
+            tempDirectoryCreated()
+          } catch (err) {
+            tempDirectoryCreated(new Error(`Unable to create temp directory ${tempDirectory}: ${err.message}`))
+          }
+        },
+
         (repoValidated) => {
           try {
-            repo = this._getRepo()
+            repo = this._getRepo(tempDirectory)
           } catch (err) {
             return repoValidated(err)
           }
@@ -141,7 +152,7 @@ module.exports = class BotDriver {
 
         (containerValidated) => {
           try {
-            container = this._getContainer(repo)
+            container = this._getContainer(tempDirectory, repo)
           } catch (err) {
             return containerValidated(err)
           }
@@ -156,9 +167,9 @@ module.exports = class BotDriver {
         if (err) {
           debug(`BotDriver Build error: ${err}`)
           this.eventEmitter.emit(Events.CONTAINER_BUILD_ERROR, err)
-          if (this.tempDirectory) {
-            rimraf(this.tempDirectory, (err) => {
-              if (err) debug(`Cleanup temp dir ${this.tempDirectory} failed: ${util.inspect(err)}`)
+          if (tempDirectory) {
+            rimraf(tempDirectory, (err) => {
+              if (err) debug(`Cleanup temp dir ${tempDirectory} failed: ${util.inspect(err)}`)
             })
           }
           return reject(err)
@@ -311,12 +322,6 @@ module.exports = class BotDriver {
           throw new Error(`Capability '${Capabilities.CONTAINERMODE}' or '${Capabilities.BOTIUMGRIDURL}' missing`)
         }
 
-        this.tempDirectory = path.resolve(process.cwd(), this.caps[Capabilities.TEMPDIR], sanitize(`${this.caps[Capabilities.PROJECTNAME]} ${moment().format('YYYYMMDD HHmmss')} ${randomize('Aa0', 5)}`))
-        try {
-          mkdirp.sync(this.tempDirectory)
-        } catch (err) {
-          throw new Error(`Unable to create temp directory ${this.tempDirectory}: ${err}`)
-        }
         resolve(this)
       } catch (err) {
         reject(err)
@@ -324,35 +329,35 @@ module.exports = class BotDriver {
     })
   }
 
-  _getRepo () {
+  _getRepo (tempDirectory) {
     if (this.caps[Capabilities.BOTIUMGRIDURL]) {
       const NoRepo = require('./repos/NoRepo')
-      return new NoRepo(this.tempDirectory, this.sources)
+      return new NoRepo(tempDirectory, this.sources)
     }
     if (this.sources[Source.GITURL]) {
       const GitRepo = require('./repos/GitRepo')
-      return new GitRepo(this.tempDirectory, this.sources)
+      return new GitRepo(tempDirectory, this.sources)
     }
     if (this.sources[Source.LOCALPATH]) {
       const LocalRepo = require('./repos/LocalRepo')
-      return new LocalRepo(this.tempDirectory, this.sources)
+      return new LocalRepo(tempDirectory, this.sources)
     }
     throw new Error(`No Repo provider found for Sources ${util.inspect(this.sources)}`)
   }
 
-  _getContainer (repo) {
+  _getContainer (tempDirectory, repo) {
     if (this.caps[Capabilities.BOTIUMGRIDURL]) {
       const GridContainer = require('./containers/GridContainer')
-      return new GridContainer(this.eventEmitter, this.tempDirectory, repo, this.caps, this.envs)
+      return new GridContainer(this.eventEmitter, tempDirectory, repo, this.caps, this.envs)
     }
     if (!this.caps[Capabilities.CONTAINERMODE]) {
       throw new Error(`Capability '${Capabilities.CONTAINERMODE}' missing`)
     }
     if (this.caps[Capabilities.CONTAINERMODE] === 'inprocess') {
       const InProcessContainer = require('./containers/InProcessContainer')
-      return new InProcessContainer(this.eventEmitter, this.tempDirectory, repo, this.caps, this.envs)
+      return new InProcessContainer(this.eventEmitter, tempDirectory, repo, this.caps, this.envs)
     }
     const PluginConnectorContainer = require('./containers/PluginConnectorContainer')
-    return new PluginConnectorContainer(this.eventEmitter, this.tempDirectory, repo, this.caps, this.envs)
+    return new PluginConnectorContainer(this.eventEmitter, tempDirectory, repo, this.caps, this.envs)
   }
 }

package/src/Capabilities.js

@@ -91,6 +91,7 @@ module.exports = {
   SIMPLEREST_REDIS_TOPIC: 'SIMPLEREST_REDIS_TOPIC',
   SIMPLEREST_INBOUND_ORDER_UNSETTLED_EVENTS_JSONPATH: 'SIMPLEREST_INBOUND_ORDER_UNSETTLED_EVENTS_JSONPATH',
   SIMPLEREST_INBOUND_DEBOUNCE_TIMEOUT: 'SIMPLEREST_INBOUND_DEBOUNCE_TIMEOUT',
+  SIMPLEREST_COOKIE_REPLICATION: 'SIMPLEREST_COOKIE_REPLICATION',
   // Script Compiler
   SCRIPTING_TXT_EOL: 'SCRIPTING_TXT_EOL',
   // ROW_PER_MESSAGE or QUESTION_ANSWER
@@ -104,6 +105,10 @@ module.exports = {
   SCRIPTING_XLSX_SHEETNAMES_PCONVOS: 'SCRIPTING_XLSX_SHEETNAMES_PCONVOS',
   SCRIPTING_XLSX_SHEETNAMES_UTTERANCES: 'SCRIPTING_XLSX_SHEETNAMES_UTTERANCES',
   SCRIPTING_XLSX_SHEETNAMES_SCRIPTING_MEMORY: 'SCRIPTING_XLSX_SHEETNAMES_SCRIPTING_MEMORY',
+  // hidden capability. All newly in Box created testsets will have this as true. CsvCompiler
+  // - throws less error (Box reads csv files as utterances, and convo. Compiler cant throw exception if a file is correct, but box tries to load it with incorrect script type)
+  // 4 or more colums are compiled just as utterances.
+  SCRIPTING_CSV_LEGACY_MODE_OFF: 'SCRIPTING_CSV_LEGACY_MODE_OFF',
   SCRIPTING_CSV_DELIMITER: 'SCRIPTING_CSV_DELIMITER',
   SCRIPTING_CSV_SKIP_HEADER: 'SCRIPTING_CSV_SKIP_HEADER',
   SCRIPTING_CSV_QUOTE: 'SCRIPTING_CSV_QUOTE',
@@ -113,6 +118,9 @@ module.exports = {
   SCRIPTING_CSV_MULTIROW_COLUMN_TEXT: 'SCRIPTING_CSV_MULTIROW_COLUMN_TEXT',
   SCRIPTING_CSV_QA_COLUMN_QUESTION: 'SCRIPTING_CSV_QA_COLUMN_QUESTION',
   SCRIPTING_CSV_QA_COLUMN_ANSWER: 'SCRIPTING_CSV_QA_COLUMN_ANSWER',
+  SCRIPTING_CSV_UTTERANCE_STARTROW: 'SCRIPTING_CSV_UTTERANCE_STARTROW',
+  SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER: 'SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER',
+  SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY: 'SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY',
   SCRIPTING_NORMALIZE_TEXT: 'SCRIPTING_NORMALIZE_TEXT',
   SCRIPTING_ENABLE_MEMORY: 'SCRIPTING_ENABLE_MEMORY',
   SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS: 'SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS',

package/src/Defaults.js

@@ -34,6 +34,7 @@ module.exports = {
     [Capabilities.SIMPLEREST_STRICT_SSL]: true,
     [Capabilities.SIMPLEREST_INBOUND_UPDATE_CONTEXT]: true,
     [Capabilities.SIMPLEREST_CONTEXT_MERGE_OR_REPLACE]: 'MERGE',
+    [Capabilities.SIMPLEREST_COOKIE_REPLICATION]: true,
     [Capabilities.SCRIPTING_TXT_EOL]: '\n',
     [Capabilities.SCRIPTING_XLSX_EOL_WRITE]: '\r\n',
     [Capabilities.SCRIPTING_XLSX_HASHEADERS]: true,

package/src/containers/plugins/SimpleRestContainer.js

@@ -28,6 +28,7 @@ module.exports = class SimpleRestContainer {
     this.bottleneck = bottleneck || ((fn) => fn())
     this.processInbound = false
     this.redisTopic = this.caps[Capabilities.SIMPLEREST_REDIS_TOPIC] || 'SIMPLEREST_INBOUND_SUBSCRIPTION'
+    this.cookies = {}
 
     if (this.caps[Capabilities.SIMPLEREST_INBOUND_ORDER_UNSETTLED_EVENTS_JSONPATH]) {
       const debounceTimeout = this.caps[Capabilities.SIMPLEREST_INBOUND_DEBOUNCE_TIMEOUT] || 500
@@ -451,7 +452,7 @@ module.exports = class SimpleRestContainer {
 
             if (body) {
               debug(`got response code: ${response.statusCode}, body: ${botiumUtils.shortenJsonString(body)}`)
-
+              this._storeCookiesFromResponse(response)
               try {
                 body = await this._parseResponseBody(body)
               } catch (err) {
@@ -560,8 +561,8 @@ module.exports = class SimpleRestContainer {
       }
     }
     this._addRequestOptions(requestOptions)
-
     await executeHook(this.caps, this.requestHook, Object.assign({ requestOptions }, this.view))
+    this._addRequestCookies(requestOptions)
 
     return requestOptions
   }
@@ -593,6 +594,7 @@ module.exports = class SimpleRestContainer {
         await timeout(pingConfig.timeout)
       } else {
         debug(`_waitForUrlResponse success on url check ${pingConfig.uri}: ${response.statusCode}/${response.statusMessage}`)
+        this._storeCookiesFromResponse(response)
         if (debug.enabled && body) {
           debug(botiumUtils.shortenJsonString(body))
         }
@@ -753,9 +755,9 @@ module.exports = class SimpleRestContainer {
       const timeout = this._getCapValue(Capabilities.SIMPLEREST_POLL_TIMEOUT)
       const pollConfig = {
         method: verb,
-        uri: uri,
+        uri,
         followAllRedirects: true,
-        timeout: timeout
+        timeout
       }
       if (this.caps[Capabilities.SIMPLEREST_POLL_HEADERS]) {
         try {
@@ -783,6 +785,8 @@ module.exports = class SimpleRestContainer {
         debug(`_runPolling: exeucting request hook failed - (${err.message})`)
         return
       }
+      this._addRequestCookies(pollConfig)
+
       request(pollConfig, async (err, response, body) => {
         if (err) {
           debug(`_runPolling: rest request failed: ${err.message}, request: ${JSON.stringify(pollConfig)}`)
@@ -794,7 +798,7 @@ module.exports = class SimpleRestContainer {
             }
           } else if (body) {
             debug(`_runPolling: got response code: ${response.statusCode}, body: ${botiumUtils.shortenJsonString(body)}`)
-
+            this._storeCookiesFromResponse(response)
             try {
               body = await this._parseResponseBody(body)
             } catch (err) {
@@ -835,9 +839,9 @@ module.exports = class SimpleRestContainer {
     const timeout = this._getCapValue(`${capPrefix}_TIMEOUT`) || this._getCapValue(Capabilities.SIMPLEREST_TIMEOUT)
     const httpConfig = {
       method: verb,
-      uri: uri,
+      uri,
       followAllRedirects: true,
-      timeout: timeout
+      timeout
     }
     if (this.caps[`${capPrefix}_HEADERS`]) {
       try {
@@ -856,8 +860,8 @@ module.exports = class SimpleRestContainer {
       }
     }
     this._addRequestOptions(httpConfig)
-
     await executeHook(this.caps, this.requestHooks[capPrefix], Object.assign({ requestOptions: httpConfig }, this.view))
+    this._addRequestCookies(httpConfig)
 
     const retries = this._getCapValue(`${capPrefix}_RETRIES`)
     debug(`_makeCall(${capPrefix}): rest request: ${JSON.stringify(httpConfig)}`)
@@ -874,4 +878,34 @@ module.exports = class SimpleRestContainer {
       _.merge(httpConfig, this.caps[Capabilities.SIMPLEREST_EXTRA_OPTIONS])
     }
   }
+
+  _addRequestCookies (requestOptions) {
+    if (!this.caps[Capabilities.SIMPLEREST_COOKIE_REPLICATION] || !requestOptions) {
+      return
+    }
+    const url = new URL(requestOptions.uri)
+    if (!requestOptions.headers) {
+      requestOptions.headers = {}
+    }
+    requestOptions.headers.Cookie = requestOptions.headers.Cookie ? `${requestOptions.headers.Cookie}; ${this.cookies[url.host]}` : this.cookies[url.host]
+  }
+
+  _storeCookiesFromResponse (response) {
+    if (!this.caps[Capabilities.SIMPLEREST_COOKIE_REPLICATION] || !response) {
+      return
+    }
+    const responseCookies = response.headers['set-cookie']
+    if (!responseCookies) {
+      return
+    }
+    const host = _.get(response, 'request.uri.host')
+    let cookie
+    responseCookies.forEach(cookieString => {
+      cookie = cookie ? `${cookie}; ${cookieString}` : cookieString
+    })
+
+    if (cookie) {
+      this.cookies[host] = cookie
+    }
+  }
 }

package/src/containers/plugins/index.js

@@ -69,7 +69,7 @@ const tryLoadPlugin = (containermode, modulepath, args) => {
           source: 'src/containers/plugins/index.js',
           cause: {
             SECURITY_ALLOW_UNSAFE: caps[Capabilities.SECURITY_ALLOW_UNSAFE],
-            mode: mode,
+            mode,
             ...cause
           }
         }

package/src/helpers/Utils.js

@@ -46,7 +46,7 @@ const toJsonWeak = (stringOrNot) => {
 
 const optionalJson = (json) => {
   const body = isJson(json)
-  return body ? { 'content-type': 'application/json', body: body } : { 'content-type': 'text/plain', body: json }
+  return body ? { 'content-type': 'application/json', body } : { 'content-type': 'text/plain', body: json }
 }
 
 const shortenJsonString = (obj) => {