botium-core 1.12.5 → 1.13.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botium-core",
-  "version": "1.12.5",
+  "version": "1.13.1",
   "description": "The Selenium for Chatbots",
   "main": "index.js",
   "module": "dist/botium-es.js",
@@ -32,25 +32,25 @@
   },
   "homepage": "https://www.botium.ai",
   "dependencies": {
-    "@babel/runtime": "^7.17.9",
-    "async": "^3.2.3",
+    "@babel/runtime": "^7.18.6",
+    "async": "^3.2.4",
     "body-parser": "^1.20.0",
     "boolean": "^3.2.0",
     "bottleneck": "^2.19.5",
-    "csv-parse": "^5.0.4",
+    "csv-parse": "^5.3.0",
     "debug": "^4.3.4",
     "esprima": "^4.0.1",
-    "express": "^4.17.3",
+    "express": "^4.18.1",
     "globby": "11.0.4",
-    "ioredis": "^5.0.4",
+    "ioredis": "^5.1.0",
     "is-class": "^0.0.9",
     "is-json": "^2.0.1",
     "jsonpath": "^1.1.1",
     "lodash": "^4.17.21",
-    "markdown-it": "^12.3.2",
+    "markdown-it": "^13.0.1",
     "mime-types": "^2.1.35",
     "mkdirp": "^1.0.4",
-    "moment": "^2.29.3",
+    "moment": "^2.29.4",
     "mustache": "^4.2.0",
     "promise-retry": "^2.0.1",
     "promise.allsettled": "^1.0.5",
@@ -59,39 +59,39 @@
     "rimraf": "^3.0.2",
     "sanitize-filename": "^1.6.3",
     "slugify": "^1.6.5",
-    "socket.io": "^4.4.1",
-    "socket.io-client": "^4.4.1",
+    "socket.io": "^4.5.1",
+    "socket.io-client": "^4.5.1",
     "socketio-auth": "^0.1.1",
     "swagger-jsdoc": "^6.2.1",
-    "swagger-ui-express": "^4.3.0",
+    "swagger-ui-express": "^4.4.0",
     "uuid": "^8.3.2",
-    "vm2": "^3.9.9",
+    "vm2": "^3.9.10",
     "write-yaml": "^1.0.0",
     "xlsx": "^0.18.5",
-    "xregexp": "^5.1.0",
-    "yaml": "^2.0.1"
+    "xregexp": "^5.1.1",
+    "yaml": "^2.1.1"
   },
   "devDependencies": {
-    "@babel/core": "^7.17.9",
-    "@babel/node": "^7.16.8",
-    "@babel/plugin-transform-runtime": "^7.17.0",
-    "@babel/preset-env": "^7.16.11",
+    "@babel/core": "^7.18.6",
+    "@babel/node": "^7.18.6",
+    "@babel/plugin-transform-runtime": "^7.18.6",
+    "@babel/preset-env": "^7.18.6",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
-    "eslint": "^8.13.0",
+    "eslint": "^8.19.0",
     "eslint-config-standard": "^17.0.0",
     "eslint-plugin-import": "^2.26.0",
-    "eslint-plugin-n": "^15.1.0",
+    "eslint-plugin-n": "^15.2.4",
     "eslint-plugin-promise": "^6.0.0",
     "eslint-plugin-standard": "^4.1.0",
     "license-checker": "^25.0.1",
     "license-compatibility-checker": "^0.3.5",
-    "mocha": "^9.2.2",
-    "nock": "^13.2.4",
-    "npm-check-updates": "^12.5.9",
+    "mocha": "^10.0.0",
+    "nock": "^13.2.8",
+    "npm-check-updates": "^15.2.6",
     "nyc": "^15.1.0",
-    "rollup": "^2.70.2",
+    "rollup": "^2.76.0",
     "rollup-plugin-babel": "^4.4.0",
     "rollup-plugin-commonjs": "^10.1.0",
     "rollup-plugin-json": "^4.0.0",

package/src/Capabilities.js

@@ -105,6 +105,10 @@ module.exports = {
   SCRIPTING_XLSX_SHEETNAMES_PCONVOS: 'SCRIPTING_XLSX_SHEETNAMES_PCONVOS',
   SCRIPTING_XLSX_SHEETNAMES_UTTERANCES: 'SCRIPTING_XLSX_SHEETNAMES_UTTERANCES',
   SCRIPTING_XLSX_SHEETNAMES_SCRIPTING_MEMORY: 'SCRIPTING_XLSX_SHEETNAMES_SCRIPTING_MEMORY',
+  // hidden capability. All newly in Box created testsets will have this as true. CsvCompiler
+  // - throws less error (Box reads csv files as utterances, and convo. Compiler cant throw exception if a file is correct, but box tries to load it with incorrect script type)
+  // 4 or more colums are compiled just as utterances.
+  SCRIPTING_CSV_LEGACY_MODE_OFF: 'SCRIPTING_CSV_LEGACY_MODE_OFF',
   SCRIPTING_CSV_DELIMITER: 'SCRIPTING_CSV_DELIMITER',
   SCRIPTING_CSV_SKIP_HEADER: 'SCRIPTING_CSV_SKIP_HEADER',
   SCRIPTING_CSV_QUOTE: 'SCRIPTING_CSV_QUOTE',
@@ -114,6 +118,9 @@ module.exports = {
   SCRIPTING_CSV_MULTIROW_COLUMN_TEXT: 'SCRIPTING_CSV_MULTIROW_COLUMN_TEXT',
   SCRIPTING_CSV_QA_COLUMN_QUESTION: 'SCRIPTING_CSV_QA_COLUMN_QUESTION',
   SCRIPTING_CSV_QA_COLUMN_ANSWER: 'SCRIPTING_CSV_QA_COLUMN_ANSWER',
+  SCRIPTING_CSV_UTTERANCE_STARTROW: 'SCRIPTING_CSV_UTTERANCE_STARTROW',
+  SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER: 'SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER',
+  SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY: 'SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY',
   SCRIPTING_NORMALIZE_TEXT: 'SCRIPTING_NORMALIZE_TEXT',
   SCRIPTING_ENABLE_MEMORY: 'SCRIPTING_ENABLE_MEMORY',
   SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS: 'SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS',

package/src/helpers/RetryHelper.js

@@ -1,13 +1,9 @@
 const util = require('util')
 const _ = require('lodash')
+const debug = require('debug')('botium-core-RetryHelper')
 
 module.exports = class RetryHelper {
-  constructor (caps, section) {
-    this.retrySettings = {
-      retries: caps[`RETRY_${section.toUpperCase()}_NUMRETRIES`] || 1,
-      factor: caps[`RETRY_${section.toUpperCase()}_FACTOR`] || 1,
-      minTimeout: caps[`RETRY_${section.toUpperCase()}_MINTIMEOUT`] || 1000
-    }
+  constructor (caps, section, options = {}) {
     this.retryErrorPatterns = []
     const onErrorRegexp = caps[`RETRY_${section.toUpperCase()}_ONERROR_REGEXP`] || []
     if (onErrorRegexp) {
@@ -22,10 +18,20 @@ module.exports = class RetryHelper {
         this.retryErrorPatterns.push(onErrorRegexp)
       }
     }
+
+    // to turn on retries, NUMRETRIES or ONERROR_REGEXP has to be set
+    this.retrySettings = {
+      retries: caps[`RETRY_${section.toUpperCase()}_NUMRETRIES`] || (!_.isNil(options.numRetries) ? options.numRetries : (this.retryErrorPatterns.length === 0) ? 0 : 1),
+      factor: caps[`RETRY_${section.toUpperCase()}_FACTOR`] || (_.isNil(options.factor) ? 1 : options.factor),
+      minTimeout: caps[`RETRY_${section.toUpperCase()}_MINTIMEOUT`] || (_.isNil(options.minTimeout) ? 1000 : options.minTimeout)
+    }
+
+    debug(`Retry for ${section} is ${this.retrySettings.retries > 0 ? 'enabled' : 'disabled'}. Settings: ${JSON.stringify(this.retrySettings)} Patterns: ${JSON.stringify(this.retryErrorPatterns.map(r => r.toString()))}`)
   }
 
   shouldRetry (err) {
-    if (!err || this.retryErrorPatterns.length === 0) return false
+    if (!err) return false
+    if (this.retryErrorPatterns.length === 0) return true
     const errString = util.inspect(err)
     for (const re of this.retryErrorPatterns) {
       if (errString.match(re)) return true

package/src/scripting/CompilerCsv.js

@@ -47,6 +47,7 @@ module.exports = class CompilerCsv extends CompilerBase {
     if (scriptData.length === 0) {
       return []
     }
+    const legacyModeOn = !this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_LEGACY_MODE_OFF, false)
 
     let delimiter = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_DELIMITER)
     if (!delimiter) {
@@ -71,7 +72,8 @@ module.exports = class CompilerCsv extends CompilerBase {
         delimiter,
         escape: this.caps[Capabilities.SCRIPTING_CSV_ESCAPE],
         quote: this.caps[Capabilities.SCRIPTING_CSV_QUOTE],
-        columns: false
+        columns: false,
+        relax_column_count: true
       })
     } catch (err) {
       throw new Error(`Invalid CSV: ${err.message || err}`)
@@ -79,123 +81,169 @@ module.exports = class CompilerCsv extends CompilerBase {
     if (rows.length === 0) {
       return []
     }
-    if (rows[0].length === 1) {
-      debug('Found 1-column CSV file, treating it as utterance file')
-      if (scriptType === Constants.SCRIPTING_TYPE_UTTERANCES) {
-        const result = [{ name: rows[0][0], utterances: rows.slice(1).map(r => r[0]) }]
-        this.context.AddUtterances(result)
-        return result
-      } else {
+    const columnCount = rows[0].length
+    debug(`Legacy mode ${legacyModeOn ? 'on' : 'off'} rows ${rows.length} columns ${columnCount}`)
+
+    if ((scriptType === Constants.SCRIPTING_TYPE_CONVO || scriptType === Constants.SCRIPTING_TYPE_PCONVO)) {
+      if (columnCount === 1 || (!legacyModeOn && columnCount > 3)) {
+        debug(`Invalid column count '${columnCount}' in convo mode`)
+        return []
+      }
+      let header = null
+      if (rows.length > 0 && this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER]) {
+        header = rows[0]
+        rows = rows.slice(1)
+      }
+      if (rows.length === 0) {
+        debug('Datarows not found in convo mode')
         return []
       }
-    }
-
-    if (scriptType !== Constants.SCRIPTING_TYPE_CONVO && scriptType !== Constants.SCRIPTING_TYPE_PCONVO) {
-      return []
-    }
-    let header = null
-    if (rows.length > 0 && this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER]) {
-      header = rows[0]
-      rows = rows.slice(1)
-    }
-    if (rows.length === 0) {
-      return []
-    }
-
-    const lineNumberBase = this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER] ? 2 : 1
-    if (rows[0].length === 2) {
-      debug('Found 2-column CSV file, treating it as question/answer file')
 
-      let colQuestion = DEFAULT_QA_COLUMN_QUESTION
-      let colAnswer = DEFAULT_QA_COLUMN_ANSWER
+      const lineNumberBase = this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER] ? 2 : 1
+      if (columnCount === 2) {
+        let colQuestion = DEFAULT_QA_COLUMN_QUESTION
+        let colAnswer = DEFAULT_QA_COLUMN_ANSWER
 
-      if (header) {
-        if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION] !== undefined) {
-          colQuestion = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION])
+        if (header) {
+          if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION] !== undefined) {
+            colQuestion = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION])
+          }
+          if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER] !== undefined) {
+            colAnswer = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER])
+          }
         }
-        if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER] !== undefined) {
-          colAnswer = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER])
+
+        const convos = rows.map((row, i) => new Convo(this.context, {
+          header: {
+            name: `L${i + lineNumberBase}`
+          },
+          conversation: [
+            Object.assign({},
+              linesToConvoStep(
+                [row[colQuestion]],
+                'me',
+                this.context,
+                undefined,
+                true
+              ), { stepTag: `L${i + lineNumberBase}-Question` }),
+            Object.assign({},
+              linesToConvoStep(
+                [row[colAnswer]],
+                'bot',
+                this.context,
+                undefined,
+                true
+              ), { stepTag: `L${i + lineNumberBase}-Answer` })
+          ]
+        }))
+        if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
+          this.context.AddConvos(convos)
+        } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
+          this.context.AddPartialConvos(convos)
         }
+        debug(`Found 2-column CSV file, treating it as question/answer file, extracted ${convos.length} convos`)
+        return convos
       }
 
-      const convos = rows.map((row, i) => new Convo(this.context, {
-        header: {
-          name: `L${i + lineNumberBase}`
-        },
-        conversation: [
-          Object.assign({},
-            linesToConvoStep(
-              [row[colQuestion]],
-              'me',
-              this.context,
-              undefined,
-              true
-            ), { stepTag: `L${i + lineNumberBase}-Question` }),
-          Object.assign({},
-            linesToConvoStep(
-              [row[colAnswer]],
-              'bot',
-              this.context,
-              undefined,
-              true
-            ), { stepTag: `L${i + lineNumberBase}-Answer` })
-        ]
-      }))
-      if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
-        this.context.AddConvos(convos)
-      } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
-        this.context.AddPartialConvos(convos)
-      }
-      return convos
-    }
+      if (columnCount >= 3) {
+        let colConversationId = DEFAULT_MULTIROW_COLUMN_CONVERSATION
+        let colSender = DEFAULT_MULTIROW_COLUMN_SENDER
+        let colText = DEFAULT_MULTIROW_COLUMN_TEXT
 
-    if (rows[0].length >= 3) {
-      debug('Found 3-column CSV file, treating it as multi-row conversation file')
+        if (header) {
+          if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID] !== undefined) {
+            colConversationId = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID])
+          }
+          if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER] !== undefined) {
+            colSender = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER])
+          }
+          if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT] !== undefined) {
+            colText = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT])
+          }
+        }
 
-      let colConversationId = DEFAULT_MULTIROW_COLUMN_CONVERSATION
-      let colSender = DEFAULT_MULTIROW_COLUMN_SENDER
-      let colText = DEFAULT_MULTIROW_COLUMN_TEXT
+        const conversationIds = _.uniq(rows.map(r => r[colConversationId]))
+        const convos = conversationIds.map(conversationId => {
+          const convoRows = rows.map((row, i) => {
+            if (row[colConversationId] === conversationId) {
+              return Object.assign({},
+                linesToConvoStep(
+                  [row[colText]],
+                  row[colSender],
+                  this.context,
+                  undefined,
+                  true
+                ), { stepTag: `L${i + lineNumberBase}` })
+            }
+            return null
+          }).filter(c => c)
+          return new Convo(this.context, {
+            header: {
+              name: conversationId
+            },
+            conversation: convoRows
+          })
+        })
+        if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
+          this.context.AddConvos(convos)
+        } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
+          this.context.AddPartialConvos(convos)
+        }
+        debug(`Found 3-column CSV file, treating it as multi-row conversation file, extracted ${convos.length} convos`)
+        return convos
+      }
+    } else if (scriptType === Constants.SCRIPTING_TYPE_UTTERANCES) {
+      if (columnCount === 2 || columnCount === 3 || (legacyModeOn && columnCount > 4)) {
+        debug(`Invalid column count '${columnCount}' in utterances mode`)
+        return []
+      }
+      const result = []
+      const startRow = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_UTTERANCE_STARTROW, 2) - 1
+      const startRowHeader = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER)
+      const stopOnEmpty = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY)
+
+      for (let col = 0; col < columnCount; col++) {
+        const name = rows[0][col]
+        if (!name || name.trim().length === 0) {
+          debug(`Column ${col + 1} has no header, skipping`)
+          continue
+        }
 
-      if (header) {
-        if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID] !== undefined) {
-          colConversationId = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID])
+        const uttStruct = {
+          name,
+          utterances: []
+        }
+        let skip = !!startRowHeader
+        const getData = (row) => {
+          return rows[row][col] ? rows[row][col].trim() : false
         }
-        if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER] !== undefined) {
-          colSender = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER])
+        //
+        for (let row = startRow; row < rows.length && (skip || !stopOnEmpty || !!getData(row)); row++) { // eslint-disable-line no-unmodified-loop-condition
+          const data = getData(row)
+          if (!data) {
+            continue
+          }
+          if (!skip) {
+            uttStruct.utterances.push(data)
+          } else {
+            if (startRowHeader === rows[row][col]) {
+              skip = false
+            }
+          }
         }
-        if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT] !== undefined) {
-          colText = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT])
+        if (uttStruct.utterances.length === 0) {
+          // liveperson, skipping meta intents
+          debug(`Column ${col + 1} has no utterances, skipping`)
+          continue
         }
+        result.push(uttStruct)
       }
 
-      const conversationIds = _.uniq(rows.map(r => r[colConversationId]))
-      const convos = conversationIds.map(conversationId => {
-        const convoRows = rows.map((row, i) => {
-          if (row[colConversationId] === conversationId) {
-            return Object.assign({},
-              linesToConvoStep(
-                [row[colText]],
-                row[colSender],
-                this.context,
-                undefined,
-                true
-              ), { stepTag: `L${i + lineNumberBase}` })
-          }
-          return null
-        }).filter(c => c)
-        return new Convo(this.context, {
-          header: {
-            name: conversationId
-          },
-          conversation: convoRows
-        })
-      })
-      if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
-        this.context.AddConvos(convos)
-      } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
-        this.context.AddPartialConvos(convos)
-      }
-      return convos
+      debug(`Multi-column utterance file, extracted ${result.length} utterances`)
+      this.context.AddUtterances(result)
+      return result
+    } else {
+      return []
     }
   }
 }

package/src/scripting/Convo.js

@@ -1,6 +1,7 @@
 const util = require('util')
 const _ = require('lodash')
 const debug = require('debug')('botium-core-Convo')
+const promiseRetry = require('promise-retry')
 
 const BotiumMockMessage = require('../mocks/BotiumMockMessage')
 const Capabilities = require('../Capabilities')
@@ -8,6 +9,7 @@ const Events = require('../Events')
 const ScriptingMemory = require('./ScriptingMemory')
 const { BotiumError, botiumErrorFromErr, botiumErrorFromList } = require('./BotiumError')
 const { normalizeText, toString, removeBuffers, splitStringInNonEmptyLines } = require('./helper')
+const RetryHelper = require('../helpers/RetryHelper')
 
 const { LOGIC_HOOK_INCLUDE } = require('./logichook/LogicHookConsts')
 
@@ -210,6 +212,24 @@ class Convo {
   }
 
   async Run (container) {
+    const retryHelper = new RetryHelper(container.caps, 'CONVO')
+    return promiseRetry(async (retry, number) => {
+      return this.RunImpl(container).catch(err => {
+        const retryRemaining = retryHelper.retrySettings.retries - number + 1
+        if (retryHelper.shouldRetry(err)) {
+          debug(`Convo failed with error "${err.message || JSON.stringify(err)}". Retry ${retryRemaining > 0 ? 'enabled' : 'disabled'} (remaining #${retryRemaining}/${retryHelper.retrySettings.retries}, criterion matches)`)
+          retry(err)
+        } else {
+          if (retryHelper.retryErrorPatterns.length > 0) {
+            debug(`Convo failed with error "${err.message || JSON.stringify(err)}". Retry 'disabled' (remaining (#${retryRemaining}/${retryHelper.retrySettings.retries}), criterion does not match)`)
+          }
+          throw err
+        }
+      })
+    }, retryHelper.retrySettings)
+  }
+
+  async RunImpl (container) {
     const transcript = new Transcript({
       steps: [],
       attachments: [],
@@ -535,7 +555,7 @@ class Convo {
       if (container.caps[Capabilities.SCRIPTING_ENABLE_SKIP_ASSERT_ERRORS]) {
         const transcriptStepErrs = transcript.steps.filter(s => s.err).map(s => s.err)
         if (transcriptStepErrs && transcriptStepErrs.length > 0) {
-          transcript.err = botiumErrorFromList([transcriptStepErrs, transcript.err].filter(e => e), {})
+          transcript.err = botiumErrorFromList([...transcriptStepErrs.filter(err => err !== transcript.err), transcript.err].filter(e => e), {})
         }
       }
     }

package/src/scripting/ScriptingProvider.js

@@ -20,7 +20,7 @@ const RetryHelper = require('../helpers/RetryHelper')
 const { getMatchFunction } = require('./MatchFunctions')
 const precompilers = require('./precompilers')
 
-const globPattern = '**/+(*.convo.txt|*.utterances.txt|*.pconvo.txt|*.scriptingmemory.txt|*.xlsx|*.xlsm|*.convo.csv|*.pconvo.csv|*.yaml|*.yml|*.json|*.md|*.markdown)'
+const globPattern = '**/+(*.convo.txt|*.utterances.txt|*.pconvo.txt|*.scriptingmemory.txt|*.xlsx|*.xlsm|*.convo.csv|*.pconvo.csv|*.utterances.csv|*.yaml|*.yml|*.json|*.md|*.markdown)'
 const skipPattern = /^skip[.\-_]/i
 
 const p = (retryHelper, fn) => {
@@ -522,6 +522,10 @@ module.exports = class ScriptingProvider {
         result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_CONVO)
       } else if (filename.endsWith('.pconvo.csv')) {
         result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_PCONVO)
+      } else if (filename.endsWith('.pconvo.csv')) {
+        result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_PCONVO)
+      } else if (filename.endsWith('.utterance.csv')) {
+        result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_UTTERANCES)
       } else if (filename.endsWith('.yaml') || filename.endsWith('.yml')) {
         result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_YAML, [Constants.SCRIPTING_TYPE_UTTERANCES, Constants.SCRIPTING_TYPE_PCONVO, Constants.SCRIPTING_TYPE_CONVO, Constants.SCRIPTING_TYPE_SCRIPTING_MEMORY])
       } else if (filename.endsWith('.json')) {

package/src/scripting/helper.js

@@ -77,6 +77,13 @@ const flatString = (str) => {
   return str ? str.split('\n').map(s => s.trim()).join(' ') : ''
 }
 
+const _formatAppendArgs = (args) => {
+  return args ? ` ${args.map(a => _.isString(a) ? a.replace(/\|/g, '\\|') : `${a}`).join('|')}` : ''
+}
+const _parseArgs = (str) => {
+  return (str && str.length > 0 && str.replace(/\\\|/g, '###ESCAPESPLIT###').split('|').map(s => s.replace(/###ESCAPESPLIT###/g, '|').trim())) || []
+}
+
 const linesToConvoStep = (lines, sender, context, eol, singleLineMode = false) => {
   if (!validateSender(sender)) throw new Error(`Failed to parse conversation. Section "${sender}" unknown.`)
 
@@ -106,14 +113,14 @@ const linesToConvoStep = (lines, sender, context, eol, singleLineMode = false) =
       }
       const name = logicLine.split(' ')[0]
       if (sender !== 'me' && context.IsAsserterValid(name)) {
-        const args = (logicLine.length > name.length ? logicLine.substr(name.length + 1).split('|').map(a => a.trim()) : [])
+        const args = (logicLine.length > name.length ? _parseArgs(logicLine.substr(name.length + 1)) : [])
         convoStep.asserters.push({ name, args, not, optional })
       } else if (sender === 'me' && context.IsUserInputValid(name)) {
-        const args = (logicLine.length > name.length ? logicLine.substr(name.length + 1).split('|').map(a => a.trim()) : [])
+        const args = (logicLine.length > name.length ? _parseArgs(logicLine.substr(name.length + 1)) : [])
         convoStep.userInputs.push({ name, args })
         textLinesAccepted = false
       } else if (context.IsLogicHookValid(name)) {
-        const args = (logicLine.length > name.length ? logicLine.substr(name.length + 1).split('|').map(a => a.trim()) : [])
+        const args = (logicLine.length > name.length ? _parseArgs(logicLine.substr(name.length + 1)) : [])
         convoStep.logicHooks.push({ name, args })
         textLinesAccepted = false
       } else {
@@ -422,7 +429,7 @@ const convoStepToLines = (step) => {
   const lines = []
   if (step.sender === 'me') {
     step.forms && step.forms.filter(form => form.value).forEach((form) => {
-      lines.push(`FORM ${form.name}|${form.value}`)
+      lines.push(`FORM${_formatAppendArgs([form.name, form.value])}`)
     })
     if (step.buttons && step.buttons.length > 0) {
       lines.push('BUTTON ' + _decompileButton(step.buttons[0]))
@@ -432,34 +439,34 @@ const convoStepToLines = (step) => {
       lines.push(step.messageText)
     }
     step.userInputs && step.userInputs.forEach((userInput) => {
-      lines.push(userInput.name + (userInput.args ? ' ' + userInput.args.join('|') : ''))
+      lines.push(userInput.name + _formatAppendArgs(userInput.args))
     })
     step.logicHooks && step.logicHooks.forEach((logicHook) => {
-      lines.push(logicHook.name + (logicHook.args ? ' ' + logicHook.args.join('|') : ''))
+      lines.push(logicHook.name + _formatAppendArgs(logicHook.args))
     })
   } else {
     if (step.messageText) {
       lines.push((step.optional ? '?' : '') + (step.not ? '!' : '') + step.messageText)
     }
-    if (step.buttons && step.buttons.length > 0) lines.push('BUTTONS ' + step.buttons.filter(b => b.text).map(b => flatString(b.text)).join('|'))
-    if (step.media && step.media.length > 0) lines.push('MEDIA ' + step.media.filter(m => !m.buffer && m.mediaUri).map(m => m.mediaUri).join('|'))
+    if (step.buttons && step.buttons.length > 0) lines.push('BUTTONS' + _formatAppendArgs(step.buttons.filter(b => b.text).map(b => flatString(b.text))))
+    if (step.media && step.media.length > 0) lines.push('MEDIA' + _formatAppendArgs(step.media.filter(m => !m.buffer && m.mediaUri).map(m => m.mediaUri)))
     if (step.cards && step.cards.length > 0) {
       step.cards.forEach(c => {
         let cardTexts = []
         if (c.text) cardTexts = cardTexts.concat(_.isArray(c.text) ? c.text : [c.text])
         if (c.subtext) cardTexts = cardTexts.concat(_.isArray(c.subtext) ? c.subtext : [c.subtext])
         if (c.content) cardTexts = cardTexts.concat(_.isArray(c.content) ? c.content : [c.content])
-        if (cardTexts.length > 0) lines.push('CARDS ' + cardTexts.map(c => flatString(c)).join('|'))
+        if (cardTexts.length > 0) lines.push('CARDS' + _formatAppendArgs(cardTexts.map(c => flatString(c))))
 
-        if (c.buttons && c.buttons.length > 0) lines.push('BUTTONS ' + c.buttons.filter(b => b.text).map(b => flatString(b.text)).join('|'))
+        if (c.buttons && c.buttons.length > 0) lines.push('BUTTONS' + _formatAppendArgs(c.buttons.filter(b => b.text).map(b => flatString(b.text))))
         if (c.image && !c.image.buffer && c.image.mediaUri) lines.push('MEDIA ' + c.image.mediaUri)
       })
     }
     step.asserters && step.asserters.forEach((asserter) => {
-      lines.push((asserter.optional ? '?' : '') + (asserter.not ? '!' : '') + asserter.name + (asserter.args ? ' ' + asserter.args.join('|') : ''))
+      lines.push((asserter.optional ? '?' : '') + (asserter.not ? '!' : '') + asserter.name + _formatAppendArgs(asserter.args))
     })
     step.logicHooks && step.logicHooks.forEach((logicHook) => {
-      lines.push(logicHook.name + (logicHook.args ? ' ' + logicHook.args.join('|') : ''))
+      lines.push(logicHook.name + _formatAppendArgs(logicHook.args))
     })
   }
   return lines.map(l => l.trim())

package/src/scripting/logichook/LogicHookUtils.js

@@ -192,7 +192,7 @@ module.exports = class LogicHookUtils {
                 })
                 return vm.run(script)
               } catch (err) {
-                throw new Error(`${err.message || err}`)
+                throw new Error(`Script ${key} is not valid - ${err.message || err}`)
               }
             } else {
               throw new Error(`Script "${key}" is not valid - only functions and javascript code accepted`)

package/src/scripting/logichook/asserter/ButtonsAsserter.js

@@ -1,5 +1,7 @@
+const { SCRIPTING_NORMALIZE_TEXT } = require('../../../Capabilities')
 const { BotiumError } = require('../../BotiumError')
 const { buttonsFromMsg } = require('../helpers')
+const { normalizeText } = require('../../helper')
 
 module.exports = class ButtonsAsserter {
   constructor (context, caps = {}) {
@@ -9,14 +11,14 @@ module.exports = class ButtonsAsserter {
   }
 
   _evalButtons (args, botMsg) {
-    const allButtons = buttonsFromMsg(botMsg, true).map(b => b.text)
+    const allButtons = buttonsFromMsg(botMsg, true).map(b => b.text).filter(b => b).map(b => normalizeText(b, !!this.caps[SCRIPTING_NORMALIZE_TEXT]))
     if (!args || args.length === 0) {
       return { allButtons, buttonsNotFound: [], buttonsFound: allButtons }
     }
     const buttonsNotFound = []
     const buttonsFound = []
     for (let i = 0; i < (args || []).length; i++) {
-      if (allButtons.findIndex(b => this.context.Match(b, args[i])) < 0) {
+      if (allButtons.findIndex(b => this.context.Match(b, normalizeText(args[i], !!this.caps[SCRIPTING_NORMALIZE_TEXT]))) < 0) {
         buttonsNotFound.push(args[i])
       } else {
         buttonsFound.push(args[i])