botium-core 1.12.6 → 1.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botium-core",
-  "version": "1.12.6",
+  "version": "1.13.0",
   "description": "The Selenium for Chatbots",
   "main": "index.js",
   "module": "dist/botium-es.js",
@@ -32,22 +32,22 @@
   },
   "homepage": "https://www.botium.ai",
   "dependencies": {
-    "@babel/runtime": "^7.17.9",
-    "async": "^3.2.3",
+    "@babel/runtime": "^7.18.3",
+    "async": "^3.2.4",
     "body-parser": "^1.20.0",
     "boolean": "^3.2.0",
     "bottleneck": "^2.19.5",
-    "csv-parse": "^5.0.4",
+    "csv-parse": "^5.2.0",
     "debug": "^4.3.4",
     "esprima": "^4.0.1",
-    "express": "^4.17.3",
+    "express": "^4.18.1",
     "globby": "11.0.4",
-    "ioredis": "^5.0.4",
+    "ioredis": "^5.0.6",
     "is-class": "^0.0.9",
     "is-json": "^2.0.1",
     "jsonpath": "^1.1.1",
     "lodash": "^4.17.21",
-    "markdown-it": "^12.3.2",
+    "markdown-it": "^13.0.1",
     "mime-types": "^2.1.35",
     "mkdirp": "^1.0.4",
     "moment": "^2.29.3",
@@ -59,39 +59,39 @@
     "rimraf": "^3.0.2",
     "sanitize-filename": "^1.6.3",
     "slugify": "^1.6.5",
-    "socket.io": "^4.4.1",
-    "socket.io-client": "^4.4.1",
+    "socket.io": "^4.5.1",
+    "socket.io-client": "^4.5.1",
     "socketio-auth": "^0.1.1",
     "swagger-jsdoc": "^6.2.1",
-    "swagger-ui-express": "^4.3.0",
+    "swagger-ui-express": "^4.4.0",
     "uuid": "^8.3.2",
     "vm2": "^3.9.9",
     "write-yaml": "^1.0.0",
     "xlsx": "^0.18.5",
-    "xregexp": "^5.1.0",
-    "yaml": "^2.0.1"
+    "xregexp": "^5.1.1",
+    "yaml": "^2.1.1"
   },
   "devDependencies": {
-    "@babel/core": "^7.17.9",
-    "@babel/node": "^7.16.8",
-    "@babel/plugin-transform-runtime": "^7.17.0",
-    "@babel/preset-env": "^7.16.11",
+    "@babel/core": "^7.18.5",
+    "@babel/node": "^7.18.5",
+    "@babel/plugin-transform-runtime": "^7.18.5",
+    "@babel/preset-env": "^7.18.2",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
-    "eslint": "^8.13.0",
+    "eslint": "^8.18.0",
     "eslint-config-standard": "^17.0.0",
     "eslint-plugin-import": "^2.26.0",
-    "eslint-plugin-n": "^15.1.0",
+    "eslint-plugin-n": "^15.2.3",
     "eslint-plugin-promise": "^6.0.0",
     "eslint-plugin-standard": "^4.1.0",
     "license-checker": "^25.0.1",
     "license-compatibility-checker": "^0.3.5",
-    "mocha": "^9.2.2",
-    "nock": "^13.2.4",
-    "npm-check-updates": "^12.5.9",
+    "mocha": "^10.0.0",
+    "nock": "^13.2.7",
+    "npm-check-updates": "^14.0.1",
     "nyc": "^15.1.0",
-    "rollup": "^2.70.2",
+    "rollup": "^2.75.6",
     "rollup-plugin-babel": "^4.4.0",
     "rollup-plugin-commonjs": "^10.1.0",
     "rollup-plugin-json": "^4.0.0",

package/src/Capabilities.js

@@ -105,6 +105,10 @@ module.exports = {
   SCRIPTING_XLSX_SHEETNAMES_PCONVOS: 'SCRIPTING_XLSX_SHEETNAMES_PCONVOS',
   SCRIPTING_XLSX_SHEETNAMES_UTTERANCES: 'SCRIPTING_XLSX_SHEETNAMES_UTTERANCES',
   SCRIPTING_XLSX_SHEETNAMES_SCRIPTING_MEMORY: 'SCRIPTING_XLSX_SHEETNAMES_SCRIPTING_MEMORY',
+  // hidden capability. All newly in Box created testsets will have this as true. CsvCompiler
+  // - throws less error (Box reads csv files as utterances, and convo. Compiler cant throw exception if a file is correct, but box tries to load it with incorrect script type)
+  // 4 or more colums are compiled just as utterances.
+  SCRIPTING_CSV_LEGACY_MODE_OFF: 'SCRIPTING_CSV_LEGACY_MODE_OFF',
   SCRIPTING_CSV_DELIMITER: 'SCRIPTING_CSV_DELIMITER',
   SCRIPTING_CSV_SKIP_HEADER: 'SCRIPTING_CSV_SKIP_HEADER',
   SCRIPTING_CSV_QUOTE: 'SCRIPTING_CSV_QUOTE',
@@ -114,6 +118,9 @@ module.exports = {
   SCRIPTING_CSV_MULTIROW_COLUMN_TEXT: 'SCRIPTING_CSV_MULTIROW_COLUMN_TEXT',
   SCRIPTING_CSV_QA_COLUMN_QUESTION: 'SCRIPTING_CSV_QA_COLUMN_QUESTION',
   SCRIPTING_CSV_QA_COLUMN_ANSWER: 'SCRIPTING_CSV_QA_COLUMN_ANSWER',
+  SCRIPTING_CSV_UTTERANCE_STARTROW: 'SCRIPTING_CSV_UTTERANCE_STARTROW',
+  SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER: 'SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER',
+  SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY: 'SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY',
   SCRIPTING_NORMALIZE_TEXT: 'SCRIPTING_NORMALIZE_TEXT',
   SCRIPTING_ENABLE_MEMORY: 'SCRIPTING_ENABLE_MEMORY',
   SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS: 'SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS',

package/src/scripting/CompilerCsv.js

@@ -47,6 +47,7 @@ module.exports = class CompilerCsv extends CompilerBase {
     if (scriptData.length === 0) {
       return []
     }
+    const legacyModeOn = !this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_LEGACY_MODE_OFF, false)
 
     let delimiter = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_DELIMITER)
     if (!delimiter) {
@@ -71,7 +72,8 @@ module.exports = class CompilerCsv extends CompilerBase {
         delimiter,
         escape: this.caps[Capabilities.SCRIPTING_CSV_ESCAPE],
         quote: this.caps[Capabilities.SCRIPTING_CSV_QUOTE],
-        columns: false
+        columns: false,
+        relax_column_count: true
       })
     } catch (err) {
       throw new Error(`Invalid CSV: ${err.message || err}`)
@@ -79,123 +81,169 @@ module.exports = class CompilerCsv extends CompilerBase {
     if (rows.length === 0) {
       return []
     }
-    if (rows[0].length === 1) {
-      debug('Found 1-column CSV file, treating it as utterance file')
-      if (scriptType === Constants.SCRIPTING_TYPE_UTTERANCES) {
-        const result = [{ name: rows[0][0], utterances: rows.slice(1).map(r => r[0]) }]
-        this.context.AddUtterances(result)
-        return result
-      } else {
+    const columnCount = rows[0].length
+    debug(`Legacy mode ${legacyModeOn ? 'on' : 'off'} rows ${rows.length} columns ${columnCount}`)
+
+    if ((scriptType === Constants.SCRIPTING_TYPE_CONVO || scriptType === Constants.SCRIPTING_TYPE_PCONVO)) {
+      if (columnCount === 1 || (!legacyModeOn && columnCount > 3)) {
+        debug(`Invalid column count '${columnCount}' in convo mode`)
+        return []
+      }
+      let header = null
+      if (rows.length > 0 && this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER]) {
+        header = rows[0]
+        rows = rows.slice(1)
+      }
+      if (rows.length === 0) {
+        debug('Datarows not found in convo mode')
         return []
       }
-    }
-
-    if (scriptType !== Constants.SCRIPTING_TYPE_CONVO && scriptType !== Constants.SCRIPTING_TYPE_PCONVO) {
-      return []
-    }
-    let header = null
-    if (rows.length > 0 && this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER]) {
-      header = rows[0]
-      rows = rows.slice(1)
-    }
-    if (rows.length === 0) {
-      return []
-    }
-
-    const lineNumberBase = this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER] ? 2 : 1
-    if (rows[0].length === 2) {
-      debug('Found 2-column CSV file, treating it as question/answer file')
 
-      let colQuestion = DEFAULT_QA_COLUMN_QUESTION
-      let colAnswer = DEFAULT_QA_COLUMN_ANSWER
+      const lineNumberBase = this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER] ? 2 : 1
+      if (columnCount === 2) {
+        let colQuestion = DEFAULT_QA_COLUMN_QUESTION
+        let colAnswer = DEFAULT_QA_COLUMN_ANSWER
 
-      if (header) {
-        if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION] !== undefined) {
-          colQuestion = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION])
+        if (header) {
+          if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION] !== undefined) {
+            colQuestion = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION])
+          }
+          if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER] !== undefined) {
+            colAnswer = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER])
+          }
         }
-        if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER] !== undefined) {
-          colAnswer = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER])
+
+        const convos = rows.map((row, i) => new Convo(this.context, {
+          header: {
+            name: `L${i + lineNumberBase}`
+          },
+          conversation: [
+            Object.assign({},
+              linesToConvoStep(
+                [row[colQuestion]],
+                'me',
+                this.context,
+                undefined,
+                true
+              ), { stepTag: `L${i + lineNumberBase}-Question` }),
+            Object.assign({},
+              linesToConvoStep(
+                [row[colAnswer]],
+                'bot',
+                this.context,
+                undefined,
+                true
+              ), { stepTag: `L${i + lineNumberBase}-Answer` })
+          ]
+        }))
+        if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
+          this.context.AddConvos(convos)
+        } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
+          this.context.AddPartialConvos(convos)
         }
+        debug(`Found 2-column CSV file, treating it as question/answer file, extracted ${convos.length} convos`)
+        return convos
       }
 
-      const convos = rows.map((row, i) => new Convo(this.context, {
-        header: {
-          name: `L${i + lineNumberBase}`
-        },
-        conversation: [
-          Object.assign({},
-            linesToConvoStep(
-              [row[colQuestion]],
-              'me',
-              this.context,
-              undefined,
-              true
-            ), { stepTag: `L${i + lineNumberBase}-Question` }),
-          Object.assign({},
-            linesToConvoStep(
-              [row[colAnswer]],
-              'bot',
-              this.context,
-              undefined,
-              true
-            ), { stepTag: `L${i + lineNumberBase}-Answer` })
-        ]
-      }))
-      if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
-        this.context.AddConvos(convos)
-      } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
-        this.context.AddPartialConvos(convos)
-      }
-      return convos
-    }
+      if (columnCount >= 3) {
+        let colConversationId = DEFAULT_MULTIROW_COLUMN_CONVERSATION
+        let colSender = DEFAULT_MULTIROW_COLUMN_SENDER
+        let colText = DEFAULT_MULTIROW_COLUMN_TEXT
 
-    if (rows[0].length >= 3) {
-      debug('Found 3-column CSV file, treating it as multi-row conversation file')
+        if (header) {
+          if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID] !== undefined) {
+            colConversationId = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID])
+          }
+          if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER] !== undefined) {
+            colSender = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER])
+          }
+          if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT] !== undefined) {
+            colText = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT])
+          }
+        }
 
-      let colConversationId = DEFAULT_MULTIROW_COLUMN_CONVERSATION
-      let colSender = DEFAULT_MULTIROW_COLUMN_SENDER
-      let colText = DEFAULT_MULTIROW_COLUMN_TEXT
+        const conversationIds = _.uniq(rows.map(r => r[colConversationId]))
+        const convos = conversationIds.map(conversationId => {
+          const convoRows = rows.map((row, i) => {
+            if (row[colConversationId] === conversationId) {
+              return Object.assign({},
+                linesToConvoStep(
+                  [row[colText]],
+                  row[colSender],
+                  this.context,
+                  undefined,
+                  true
+                ), { stepTag: `L${i + lineNumberBase}` })
+            }
+            return null
+          }).filter(c => c)
+          return new Convo(this.context, {
+            header: {
+              name: conversationId
+            },
+            conversation: convoRows
+          })
+        })
+        if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
+          this.context.AddConvos(convos)
+        } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
+          this.context.AddPartialConvos(convos)
+        }
+        debug(`Found 3-column CSV file, treating it as multi-row conversation file, extracted ${convos.length} convos`)
+        return convos
+      }
+    } else if (scriptType === Constants.SCRIPTING_TYPE_UTTERANCES) {
+      if (columnCount === 2 || columnCount === 3 || (legacyModeOn && columnCount > 4)) {
+        debug(`Invalid column count '${columnCount}' in utterances mode`)
+        return []
+      }
+      const result = []
+      const startRow = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_UTTERANCE_STARTROW, 2) - 1
+      const startRowHeader = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER)
+      const stopOnEmpty = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY)
+
+      for (let col = 0; col < columnCount; col++) {
+        const name = rows[0][col]
+        if (!name || name.trim().length === 0) {
+          debug(`Column ${col + 1} has no header, skipping`)
+          continue
+        }
 
-      if (header) {
-        if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID] !== undefined) {
-          colConversationId = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID])
+        const uttStruct = {
+          name,
+          utterances: []
+        }
+        let skip = !!startRowHeader
+        const getData = (row) => {
+          return rows[row][col] ? rows[row][col].trim() : false
         }
-        if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER] !== undefined) {
-          colSender = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER])
+        //
+        for (let row = startRow; row < rows.length && (skip || !stopOnEmpty || !!getData(row)); row++) { // eslint-disable-line no-unmodified-loop-condition
+          const data = getData(row)
+          if (!data) {
+            continue
+          }
+          if (!skip) {
+            uttStruct.utterances.push(data)
+          } else {
+            if (startRowHeader === rows[row][col]) {
+              skip = false
+            }
+          }
         }
-        if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT] !== undefined) {
-          colText = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT])
+        if (uttStruct.utterances.length === 0) {
+          // liveperson, skipping meta intents
+          debug(`Column ${col + 1} has no utterances, skipping`)
+          continue
         }
+        result.push(uttStruct)
       }
 
-      const conversationIds = _.uniq(rows.map(r => r[colConversationId]))
-      const convos = conversationIds.map(conversationId => {
-        const convoRows = rows.map((row, i) => {
-          if (row[colConversationId] === conversationId) {
-            return Object.assign({},
-              linesToConvoStep(
-                [row[colText]],
-                row[colSender],
-                this.context,
-                undefined,
-                true
-              ), { stepTag: `L${i + lineNumberBase}` })
-          }
-          return null
-        }).filter(c => c)
-        return new Convo(this.context, {
-          header: {
-            name: conversationId
-          },
-          conversation: convoRows
-        })
-      })
-      if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
-        this.context.AddConvos(convos)
-      } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
-        this.context.AddPartialConvos(convos)
-      }
-      return convos
+      debug(`Multi-column utterance file, extracted ${result.length} utterances`)
+      this.context.AddUtterances(result)
+      return result
+    } else {
+      return []
     }
   }
 }

package/src/scripting/ScriptingProvider.js

@@ -20,7 +20,7 @@ const RetryHelper = require('../helpers/RetryHelper')
 const { getMatchFunction } = require('./MatchFunctions')
 const precompilers = require('./precompilers')
 
-const globPattern = '**/+(*.convo.txt|*.utterances.txt|*.pconvo.txt|*.scriptingmemory.txt|*.xlsx|*.xlsm|*.convo.csv|*.pconvo.csv|*.yaml|*.yml|*.json|*.md|*.markdown)'
+const globPattern = '**/+(*.convo.txt|*.utterances.txt|*.pconvo.txt|*.scriptingmemory.txt|*.xlsx|*.xlsm|*.convo.csv|*.pconvo.csv|*.utterances.csv|*.yaml|*.yml|*.json|*.md|*.markdown)'
 const skipPattern = /^skip[.\-_]/i
 
 const p = (retryHelper, fn) => {
@@ -522,6 +522,10 @@ module.exports = class ScriptingProvider {
         result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_CONVO)
       } else if (filename.endsWith('.pconvo.csv')) {
         result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_PCONVO)
+      } else if (filename.endsWith('.pconvo.csv')) {
+        result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_PCONVO)
+      } else if (filename.endsWith('.utterance.csv')) {
+        result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_UTTERANCES)
       } else if (filename.endsWith('.yaml') || filename.endsWith('.yml')) {
         result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_YAML, [Constants.SCRIPTING_TYPE_UTTERANCES, Constants.SCRIPTING_TYPE_PCONVO, Constants.SCRIPTING_TYPE_CONVO, Constants.SCRIPTING_TYPE_SCRIPTING_MEMORY])
       } else if (filename.endsWith('.json')) {