botium-core 1.13.4 → 1.13.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botium-core",
-  "version": "1.13.4",
+  "version": "1.13.5",
   "description": "The Selenium for Chatbots",
   "main": "index.js",
   "module": "dist/botium-es.js",
@@ -32,7 +32,7 @@
   },
   "homepage": "https://www.botium.ai",
   "dependencies": {
-    "@babel/runtime": "^7.18.6",
+    "@babel/runtime": "^7.19.0",
     "async": "^3.2.4",
     "body-parser": "^1.20.0",
     "boolean": "^3.2.0",
@@ -42,7 +42,7 @@
     "esprima": "^4.0.1",
     "express": "^4.18.1",
     "globby": "11.0.4",
-    "ioredis": "^5.1.0",
+    "ioredis": "^5.2.3",
     "is-class": "^0.0.9",
     "is-json": "^2.0.1",
     "jsonpath": "^1.1.1",
@@ -59,13 +59,13 @@
     "rimraf": "^3.0.2",
     "sanitize-filename": "^1.6.3",
     "slugify": "^1.6.5",
-    "socket.io": "^4.5.1",
-    "socket.io-client": "^4.5.1",
+    "socket.io": "^4.5.2",
+    "socket.io-client": "^4.5.2",
     "socketio-auth": "^0.1.1",
-    "swagger-jsdoc": "^6.2.1",
-    "swagger-ui-express": "^4.4.0",
-    "uuid": "^8.3.2",
-    "vm2": "^3.9.10",
+    "swagger-jsdoc": "^6.2.5",
+    "swagger-ui-express": "^4.5.0",
+    "uuid": "^9.0.0",
+    "vm2": "^3.9.11",
     "word-error-rate": "0.0.7",
     "write-yaml": "^1.0.0",
     "xlsx": "^0.18.5",
@@ -73,27 +73,27 @@
     "yaml": "^2.1.1"
   },
   "devDependencies": {
-    "@babel/core": "^7.18.6",
-    "@babel/node": "^7.18.6",
-    "@babel/plugin-transform-runtime": "^7.18.6",
-    "@babel/preset-env": "^7.18.6",
+    "@babel/core": "^7.19.3",
+    "@babel/node": "^7.19.1",
+    "@babel/plugin-transform-runtime": "^7.19.1",
+    "@babel/preset-env": "^7.19.3",
     "chai": "^4.3.6",
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
-    "eslint": "^8.19.0",
+    "eslint": "^8.24.0",
     "eslint-config-standard": "^17.0.0",
     "eslint-plugin-import": "^2.26.0",
     "eslint-plugin-mocha": "^10.1.0",
-    "eslint-plugin-n": "^15.2.4",
-    "eslint-plugin-promise": "^6.0.0",
+    "eslint-plugin-n": "^15.3.0",
+    "eslint-plugin-promise": "^6.0.1",
     "eslint-plugin-standard": "^4.1.0",
     "license-checker": "^25.0.1",
     "license-compatibility-checker": "^0.3.5",
     "mocha": "^10.0.0",
-    "nock": "^13.2.8",
-    "npm-check-updates": "^15.2.6",
+    "nock": "^13.2.9",
+    "npm-check-updates": "^16.3.4",
     "nyc": "^15.1.0",
-    "rollup": "^2.76.0",
+    "rollup": "^2.79.1",
     "rollup-plugin-babel": "^4.4.0",
     "rollup-plugin-commonjs": "^10.1.0",
     "rollup-plugin-json": "^4.0.0",

package/src/scripting/CompilerXlsx.js

@@ -256,6 +256,7 @@ module.exports = class CompilerXlsx extends CompilerBase {
             if (!convo.header.name) {
               convo.header.name = `${convo.header.sheetname}-${this.colnames[convo.header.colindex]}${formatRowIndex(convo.header.rowindex)}`
             }
+            // it is not used anymore?
             convo.header.sort = convo.header.name
             scriptResults.push(convo)
           })

package/src/scripting/ScriptingMemory.js

@@ -73,6 +73,32 @@ const SCRIPTING_FUNCTIONS_RAW = {
     return Date.now()
   },
 
+  $tomorrow: (pattern) => {
+    if (pattern) {
+      return moment().add(1, 'day').format(pattern)
+    }
+    return moment().add(1, 'day').toDate().toLocaleDateString()
+  },
+  $yesterday: (pattern) => {
+    if (pattern) {
+      return moment().subtract(1, 'day').format(pattern)
+    }
+    return moment().subtract(1, 'day').toDate().toLocaleDateString()
+  },
+
+  $date_add: (amount, unit, pattern) => {
+    if (pattern) {
+      return moment().add(amount, unit).format(pattern)
+    }
+    return moment().add(amount, unit).toDate().toLocaleDateString()
+  },
+  $date_subtract: (amount, unit, pattern) => {
+    if (pattern) {
+      return moment().subtract(amount, unit).format(pattern)
+    }
+    return moment().subtract(amount, unit).toDate().toLocaleDateString()
+  },
+
   $year: () => {
     return new Date().getFullYear()
   },
@@ -168,7 +194,8 @@ const SCRIPTING_FUNCTIONS_RAW = {
           require: false,
           env: caps[Capabilities.SECURITY_ALLOW_UNSAFE] ? process.env : {},
           sandbox: {
-            caps
+            caps,
+            moment
           }
         })
         return vm.run(`module.exports = (${code})`)
@@ -246,7 +273,19 @@ const _apply = (scriptingMemory, str, caps, mockMsg) => {
         for (const match of matches) {
           if (match.indexOf('(') > 0) {
             const arg = match.substring(match.indexOf('(') + 1, match.lastIndexOf(')')).replace(/\\\)/g, ')')
-            str = str.replace(match, SCRIPTING_FUNCTIONS[key].handler(caps, arg, mockMsg))
+            let args = [arg]
+            if (SCRIPTING_FUNCTIONS[key].numberOfArguments > 1) {
+              args = arg.split(',')
+            }
+            args = args.map(arg => {
+              arg = arg.trim()
+              if (arg.startsWith('"') && arg.endsWith('"')) {
+                return arg.substring(1, arg.length - 1)
+              } else {
+                return arg
+              }
+            })
+            str = str.replace(match, SCRIPTING_FUNCTIONS[key].handler(caps, ...args, mockMsg))
           } else {
             str = str.replace(match, SCRIPTING_FUNCTIONS[key].handler(caps))
           }

package/src/scripting/ScriptingProvider.js

@@ -868,27 +868,68 @@ module.exports = class ScriptingProvider {
     this._sortConvos()
   }
 
-  ExpandConvos () {
+  ExpandConvos (options = {}) {
+    options = Object.assign({
+      // use skip and keep, or justHeader
+      justHeader: false,
+      // drop unwanted convos
+      convoFilter: null
+    }, options)
+    const context = { count: 0 }
     const expandedConvos = []
     debug(`ExpandConvos - Using utterances expansion mode: ${this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE]}`)
     this.convos.forEach((convo) => {
       convo.expandPartialConvos()
-      this._expandConvo(expandedConvos, convo)
+      for (const expanded of this._expandConvo(convo, options, context)) {
+        expanded.header.assertionCount = this.GetAssertionCount(expanded)
+        if (options.justHeader) {
+          const ConvoWithOnlyHeader = {
+            header: {
+              name: expanded.header.name,
+              assertionCount: expanded.header.assertionCount
+            }
+          }
+          expandedConvos.push(ConvoWithOnlyHeader)
+        } else {
+          expandedConvos.push(expanded)
+        }
+      }
     })
     this.convos = expandedConvos
-    this._sortConvos()
+    if (!options.justHeader) {
+      this._sortConvos()
+    } else {
+      this._updateConvos()
+    }
+  }
+
+  ExpandConvosIterable (options = {}) {
+    options = Object.assign({
+      // drop unwanted convos
+      convoFilter: null
+    }, options)
+    debug(`ExpandConvos - Using utterances expansion mode: ${this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE]}`)
+    // creating a nested generator, calling the other.
+    // We hope this.convos does not changes while this iterator is used
+    const _convosIterable = function * (options) {
+      const context = { count: 0 }
+      for (const convo of this.convos) {
+        convo.expandPartialConvos()
+        yield * this._expandConvo(convo, options, context)
+      }
+    }.bind(this)
+
+    this.convosIterable = _convosIterable(options)
   }
 
   /**
-   *
-   * @param expandedConvos
+   * This is a generator function with yield
    * @param currentConvo
    * @param convoStepIndex
    * @param convoStepsStack list of ConvoSteps
-   * @param context {width: }
    * @private
    */
-  _expandConvo (expandedConvos, currentConvo, convoStepIndex = 0, convoStepsStack = [], context = {}) {
+  * _expandConvo (currentConvo, options, context, convoStepIndex = 0, convoStepsStack = []) {
     const utterancePostfix = (lineTag, uttOrUserInput) => {
       const naming = this.caps[Capabilities.SCRIPTING_UTTEXPANSION_NAMING_MODE] || Defaults.capabilities[Capabilities.SCRIPTING_UTTEXPANSION_NAMING_MODE]
       if (naming === 'justLineTag') {
@@ -908,7 +949,7 @@ module.exports = class ScriptingProvider {
       if (currentStep.sender === 'bot' || currentStep.sender === 'begin' || currentStep.sender === 'end') {
         const currentStepsStack = convoStepsStack.slice()
         currentStepsStack.push(_.cloneDeep(currentStep))
-        this._expandConvo(expandedConvos, currentConvo, convoStepIndex + 1, currentStepsStack, context)
+        yield * this._expandConvo(currentConvo, options, context, convoStepIndex + 1, currentStepsStack)
       } else if (currentStep.sender === 'me') {
         let useUnexpanded = true
         if (currentStep.messageText) {
@@ -925,28 +966,32 @@ module.exports = class ScriptingProvider {
           }
           if (this.utterances[uttName]) {
             const allutterances = this.utterances[uttName].utterances
-            const processSampleUtterances = (sampleutterances, myContext) => {
-              sampleutterances.forEach((utt, index) => {
-                processSampleUtterance(utt, sampleutterances.length, index, Object.assign({ indexExpansionModeIndex: index }, myContext || context))
-              })
+            const processSampleUtterances = function * (sampleutterances, myContext) {
+              for (let index = 0; index < sampleutterances.length; index++) {
+                yield * processSampleUtterance(sampleutterances[index], sampleutterances.length, index, Object.assign({ indexExpansionModeIndex: index }, myContext || context))
+              }
             }
-            const processSampleUtterance = (sampleutterance, length, index, myContext) => {
-              const lineTag = `${index + 1}`.padStart(`${length}`.length, '0')
+            const processSampleUtterance = function * (sampleutterance, length, index, myContext) {
               const currentStepsStack = convoStepsStack.slice()
               if (uttArgs) {
                 sampleutterance = util.format(sampleutterance, ...uttArgs)
               }
               currentStepsStack.push(Object.assign(_.cloneDeep(currentStep), { messageText: sampleutterance }))
               const currentConvoLabeled = _.cloneDeep(currentConvo)
-              Object.assign(currentConvoLabeled.header, { name: `${currentConvo.header.name}/${uttName}-${utterancePostfix(lineTag, sampleutterance)}` })
+              if (length > 1) {
+                const lineTag = `${index + 1}`.padStart(`${length}`.length, '0')
+                Object.assign(currentConvoLabeled.header, { name: `${currentConvo.header.name}/${uttName}-${utterancePostfix(lineTag, sampleutterance)}` })
+              }
               if (!currentConvoLabeled.sourceTag) currentConvoLabeled.sourceTag = {}
               if (!currentConvoLabeled.sourceTag.origConvoName) currentConvoLabeled.sourceTag.origConvoName = currentConvo.header.name
-              this._expandConvo(expandedConvos, currentConvoLabeled, convoStepIndex + 1, currentStepsStack, myContext || context)
-            }
-            if (this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE] === 'index') {
+              yield * this._expandConvo(currentConvoLabeled, options, myContext || context, convoStepIndex + 1, currentStepsStack)
+            }.bind(this)
+            if (allutterances.length === 1) {
+              yield * processSampleUtterances([allutterances[0]], context)
+            } else if (this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE] === 'index') {
               if (_.isNil(context.indexExpansionModeWidth)) {
                 // executed for the first found utterance
-                processSampleUtterances(allutterances, Object.assign({}, context, { indexExpansionModeWidth: allutterances.length }))
+                yield * processSampleUtterances(allutterances, Object.assign({}, context, { indexExpansionModeWidth: allutterances.length }))
               } else {
                 if (_.isNil(context.indexExpansionModeIndex)) {
                   throw new Error('indexExpansionModeIndex must be set!')
@@ -957,58 +1002,67 @@ module.exports = class ScriptingProvider {
                   debug(`While expanding convos by index found in utterance "${uttName}" less examples (${allutterances.length}) as expected (${context.indexExpansionModeWidth})`)
                 }
                 const myContext = Object.assign({}, context, { indexExpansionModeWidth: Math.max(allutterances.length, context.indexExpansionModeWidth) })
-                processSampleUtterance(allutterances[localIndex], allutterances.length, localIndex, myContext)
+                yield * processSampleUtterance(allutterances[localIndex], allutterances.length, localIndex, myContext)
                 if (allutterances.length > context.indexExpansionModeWidth && context.indexExpansionModeIndex + 1 === context.indexExpansionModeWidth) {
                   debug(`While expanding convos by index found in utterance "${uttName}" more examples (${allutterances.length}) as expected (${context.indexExpansionModeWidth})`)
                   for (let i = context.indexExpansionModeWidth; i < allutterances.length; i++) {
                     // if we found a utterance with more examples as any utterances before, we have to start new 'thread'
                     const myContext = Object.assign({}, context, { indexExpansionModeWidth: allutterances.length, indexExpansionModeIndex: i })
-                    processSampleUtterance(allutterances[i], allutterances.length, i, myContext)
+                    yield * processSampleUtterance(allutterances[i], allutterances.length, i, myContext)
                   }
                 }
               }
             } else {
               if (this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE] === 'first') {
-                processSampleUtterances([allutterances[0]])
+                yield * processSampleUtterances([allutterances[0]], context)
               } else if (this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE] === 'random') {
-                processSampleUtterances(allutterances
+                yield * processSampleUtterances(allutterances
                   .map(x => ({ x, r: Math.random() }))
                   .sort((a, b) => a.r - b.r)
                   .map(a => a.x)
-                  .slice(0, this.caps[Capabilities.SCRIPTING_UTTEXPANSION_RANDOM_COUNT]))
+                  .slice(0, this.caps[Capabilities.SCRIPTING_UTTEXPANSION_RANDOM_COUNT]), context)
               } else {
-                processSampleUtterances(allutterances)
+                yield * processSampleUtterances(allutterances, context)
               }
             }
             useUnexpanded = false
           }
         }
         if (currentStep.userInputs && currentStep.userInputs.length > 0) {
-          currentStep.userInputs.forEach((ui, uiIndex) => {
+          for (let uiIndex = 0; uiIndex < currentStep.userInputs.length; uiIndex++) {
+            const ui = currentStep.userInputs[uiIndex]
             const userInput = this.userInputs[ui.name]
             if (userInput && userInput.expandConvo) {
               const expandedUserInputs = userInput.expandConvo({ convo: currentConvo, convoStep: currentStep, args: ui.args })
               if (expandedUserInputs && expandedUserInputs.length > 0) {
                 // let sampleinputs = expandedUserInputs
-                const processSampleInputs = (sampleinputs, myContext, uiIndex) => {
-                  sampleinputs.forEach((input, index) => {
-                    processSampleInput(input, sampleinputs.length, index, Object.assign({ indexExpansionModeIndex: index }, myContext || context), uiIndex)
-                  })
+                const processSampleInputs = function * (sampleinputs, myContext, uiIndex) {
+                  for (let index = 0; index < sampleinputs.length; index++) {
+                    yield * processSampleInput(sampleinputs[index], sampleinputs.length, index, Object.assign({ indexExpansionModeIndex: index }, myContext || context), uiIndex)
+                  }
                 }
-                const processSampleInput = (sampleinput, length, index, myContext, uiIndex) => {
-                  const lineTag = `${index + 1}`.padStart(`${length}`.length, '0')
+                const processSampleInput = function * (sampleinput, length, index, myContext, uiIndex) {
                   const currentStepsStack = convoStepsStack.slice()
                   const currentStepMod = _.cloneDeep(currentStep)
                   currentStepMod.userInputs[uiIndex] = sampleinput
 
                   currentStepsStack.push(currentStepMod)
                   const currentConvoLabeled = _.cloneDeep(currentConvo)
-                  Object.assign(currentConvoLabeled.header, { name: `${currentConvo.header.name}/${ui.name}-${utterancePostfix(lineTag, (sampleinput.args && sampleinput.args.length) ? sampleinput.args.join(', ') : 'no-args')}` })
-                  this._expandConvo(expandedConvos, currentConvoLabeled, convoStepIndex + 1, currentStepsStack, myContext || context)
-                }
-                if (this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE] === 'index') {
+                  if (length > 1) {
+                    if (sampleinput.convoPostfix) {
+                      Object.assign(currentConvoLabeled.header, { name: `${currentConvo.header.name}/${ui.name}-${sampleinput.convoPostfix}` })
+                    } else {
+                      const lineTag = `${index + 1}`.padStart(`${length}`.length, '0')
+                      Object.assign(currentConvoLabeled.header, { name: `${currentConvo.header.name}/${ui.name}-${utterancePostfix(lineTag, (sampleinput.args && sampleinput.args.length) ? sampleinput.args.join(', ') : 'no-args')}` })
+                    }
+                  }
+                  yield * this._expandConvo(currentConvoLabeled, options, myContext || context, convoStepIndex + 1, currentStepsStack)
+                }.bind(this)
+                if (expandedUserInputs.length === 1) {
+                  yield * processSampleInputs([expandedUserInputs[0]], context, uiIndex)
+                } else if (this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE] === 'index') {
                   if (_.isNil(context.indexExpansionModeWidth)) {
-                    processSampleInputs(expandedUserInputs, Object.assign({}, context, { indexExpansionModeWidth: expandedUserInputs.length }), uiIndex)
+                    yield * processSampleInputs(expandedUserInputs, Object.assign({}, context, { indexExpansionModeWidth: expandedUserInputs.length }), uiIndex)
                   } else {
                     if (_.isNil(context.indexExpansionModeIndex)) {
                       throw new Error('indexExpansionModeIndex must be set!')
@@ -1019,20 +1073,20 @@ module.exports = class ScriptingProvider {
                       debug(`While expanding convos by index found user input "${ui.name}, ${ui.args}" less examples (${expandedUserInputs.length}) as expected (${context.indexExpansionModeWidth})`)
                     }
                     const myContext = Object.assign({}, context, { indexExpansionModeWidth: Math.max(expandedUserInputs.length, context.indexExpansionModeWidth) })
-                    processSampleInput(expandedUserInputs[localIndex], expandedUserInputs.length, localIndex, myContext, uiIndex)
+                    yield * processSampleInput(expandedUserInputs[localIndex], expandedUserInputs.length, localIndex, myContext, uiIndex)
                     if (expandedUserInputs.length > context.indexExpansionModeWidth && context.indexExpansionModeIndex + 1 === context.indexExpansionModeWidth) {
                       debug(`While expanding convos by index found user input "${ui.name}, ${ui.args}" more examples (${expandedUserInputs.length}) as expected (${context.indexExpansionModeWidth})`)
                       for (let i = context.indexExpansionModeWidth; i < expandedUserInputs.length; i++) {
                         const myContext = Object.assign({}, context, { indexExpansionModeWidth: expandedUserInputs.length, indexExpansionModeIndex: i })
-                        processSampleInput(expandedUserInputs[i], expandedUserInputs.length, i, myContext, uiIndex)
+                        yield * processSampleInput(expandedUserInputs[i], expandedUserInputs.length, i, myContext, uiIndex)
                       }
                     }
                   }
                 } else {
                   if (this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE] === 'first') {
-                    processSampleInputs([expandedUserInputs[0]], context, uiIndex)
+                    yield * processSampleInputs([expandedUserInputs[0]], context, uiIndex)
                   } else if (this.caps[Capabilities.SCRIPTING_UTTEXPANSION_MODE] === 'random') {
-                    processSampleInputs(expandedUserInputs
+                    yield * processSampleInputs(expandedUserInputs
                       .map(x => ({
                         x,
                         r: Math.random()
@@ -1041,35 +1095,49 @@ module.exports = class ScriptingProvider {
                       .map(a => a.x)
                       .slice(0, this.caps[Capabilities.SCRIPTING_UTTEXPANSION_RANDOM_COUNT]), context, uiIndex)
                   } else {
-                    processSampleInputs(expandedUserInputs, context, uiIndex)
+                    yield * processSampleInputs(expandedUserInputs, context, uiIndex)
                   }
                 }
                 useUnexpanded = false
               }
             }
-          })
+          }
         }
         if (useUnexpanded) {
           const currentStepsStack = convoStepsStack.slice()
           currentStepsStack.push(_.cloneDeep(currentStep))
-          this._expandConvo(expandedConvos, currentConvo, convoStepIndex + 1, currentStepsStack, context)
+          yield * this._expandConvo(currentConvo, options, context, convoStepIndex + 1, currentStepsStack)
         }
       }
     } else {
-      expandedConvos.push(Object.assign(_.cloneDeep(currentConvo), { conversation: _.cloneDeep(convoStepsStack) }))
+      const expanded = Object.assign(_.cloneDeep(currentConvo), { conversation: _.cloneDeep(convoStepsStack) })
+      if (!options.convoFilter || options.convoFilter(expanded)) {
+        context.count++
+        const logPerEntry = context.count < 10 ? 1 : context.count < 100 ? 10 : context.count < 1000 ? 100 : context.count < 10000 ? 1000 : 10000
+        if (context.count % logPerEntry === 0) {
+          debug(`Convo #${context.count} expanded (${expanded.header.name})`)
+        }
+        yield expanded
+      }
     }
   }
 
   _sortConvos () {
     this.convos = _.sortBy(this.convos, [(convo) => convo.header.sort || convo.header.name])
+    this._updateConvos()
+  }
+
+  _updateConvos () {
     let i = 0
     this.convos.forEach((convo) => {
-      convo.header.order = ++i
-      if (!convo.header.projectname) {
-        convo.header.projectname = this.caps[Capabilities.PROJECTNAME]
-      }
-      if (!convo.header.testsessionname) {
-        convo.header.testsessionname = this.caps[Capabilities.TESTSESSIONNAME]
+      if (convo) {
+        convo.header.order = ++i
+        if (!convo.header.projectname) {
+          convo.header.projectname = this.caps[Capabilities.PROJECTNAME]
+        }
+        if (!convo.header.testsessionname) {
+          convo.header.testsessionname = this.caps[Capabilities.TESTSESSIONNAME]
+        }
       }
     })
   }
@@ -1080,7 +1148,11 @@ module.exports = class ScriptingProvider {
     } else if (convos) {
       this.convos.push(convos)
     }
-    this._sortConvos()
+    if (this.convos.filter(c => _.isNil(c))) {
+      this._updateConvos()
+    } else {
+      this._sortConvos()
+    }
   }
 
   AddUtterances (utterances) {
@@ -1334,4 +1406,30 @@ module.exports = class ScriptingProvider {
       ...lines,
       '}'].join('\r\n')
   }
+
+  GetAssertionCount (convo) {
+    if (!convo) {
+      return 0
+    }
+    let counter = 0
+    for (const step of convo.conversation) {
+      if (step.sender === 'bot') {
+        let stepCounter = step.asserters ? step.asserters.length : 0
+        if (step.messageText) {
+          stepCounter++
+        }
+        stepCounter = stepCounter === 0 ? 1 : stepCounter
+        counter += stepCounter
+      }
+    }
+
+    if (convo.convoBegin && convo.convoBegin.asserters) {
+      counter += convo.convoBegin.asserters.length
+    }
+
+    if (convo.convoEnd && convo.convoEnd.asserters) {
+      counter += convo.convoEnd.asserters.length
+    }
+    return counter === 0 ? 1 : counter
+  }
 }

package/src/scripting/helper.js

@@ -437,6 +437,8 @@ const convoStepToLines = (step) => {
       lines.push('MEDIA ' + step.media[0].mediaUri)
     } else if (step.messageText) {
       lines.push(step.messageText)
+    } else if (step.sourceData) {
+      lines.push(JSON.stringify(step.sourceData, null, 2))
     }
     step.userInputs && step.userInputs.forEach((userInput) => {
       lines.push(userInput.name + _formatAppendArgs(userInput.args))

package/src/scripting/logichook/userinput/MediaInput.js

@@ -165,7 +165,8 @@ module.exports = class MediaInput {
           mediaFiles.forEach(mf => {
             e.push({
               name: 'MEDIA',
-              args: [mf]
+              args: [mf],
+              convoPostfix: _.last(mf.split('/'))
             })
           })
         } else {

package/test/compiler/compilercsv.spec.js

@@ -15,8 +15,15 @@ const buildContext = () => {
     AddConvos: (c) => {
       result.convos = result.convos.concat(c)
     },
-    AddUtterances: (u) => {
-      result.utterances = result.utterances.concat(u)
+    AddUtterances: (utteranceStructsToAdd) => {
+      for (const utteranceStructToAdd of utteranceStructsToAdd) {
+        const existing = result.utterances.find(entry => entry.name === utteranceStructToAdd.name)
+        if (existing) {
+          existing.utterances = existing.utterances.concat(utteranceStructToAdd.utterances)
+        } else {
+          result.utterances.push(utteranceStructToAdd)
+        }
+      }
     },
     convos: [],
     utterances: []

package/test/compiler/convos/csv/utterances_variable_row_len.csv

@@ -1,3 +1,3 @@
-variable_row_len,goodbye, sdf, sdf, sdf,
+variable_row_len,goodbye, col1, col2, col3,
 hello,goodbye
 hi,,sd,fs d,f ,sdf ,ds, fsd,f ,sdf, sd,f sd, f,sd f,

package/test/convo/fillAndApplyScriptingMemory.spec.js

@@ -1,4 +1,5 @@
 const path = require('path')
+const moment = require('moment')
 const assert = require('chai').assert
 const BotDriver = require('../../').BotDriver
 const Capabilities = require('../../').Capabilities
@@ -695,6 +696,39 @@ describe('convo.fillAndApplyScriptingMemory', function () {
         assert(result.length === 13, '$timestap is invalid')
       })
 
+      it('tomorrow without format', async function () {
+        const result = ScriptingMemory.apply(
+          { caps: CAPS_ENABLE_SCRIPTING_MEMORY },
+          {},
+          '$tomorrow'
+        )
+        assert.equal(result, moment().add(1, 'day').toDate().toLocaleDateString())
+      })
+      it('tomorrow with format', async function () {
+        const result = ScriptingMemory.apply(
+          { caps: CAPS_ENABLE_SCRIPTING_MEMORY },
+          {},
+          '$tomorrow(YYYY.MM.DD)'
+        )
+        assert.equal(result, moment().add(1, 'day').format('YYYY.MM.DD'))
+      })
+      it('yesterday without format', async function () {
+        const result = ScriptingMemory.apply(
+          { caps: CAPS_ENABLE_SCRIPTING_MEMORY },
+          {},
+          '$yesterday'
+        )
+        assert.equal(result, moment().subtract(1, 'day').toDate().toLocaleDateString())
+      })
+      it('yesterday with format', async function () {
+        const result = ScriptingMemory.apply(
+          { caps: CAPS_ENABLE_SCRIPTING_MEMORY },
+          {},
+          '$yesterday(YYYY.MM.DD)'
+        )
+        assert.equal(result, moment().subtract(1, 'day').format('YYYY.MM.DD'))
+      })
+
       it('year', async function () {
         const result = ScriptingMemory.apply(
           { caps: CAPS_ENABLE_SCRIPTING_MEMORY },
@@ -703,7 +737,7 @@ describe('convo.fillAndApplyScriptingMemory', function () {
         )
 
         const year = parseInt(result)
-        assert(year >= 2019 && year <= 2219, '$year invalid')
+        assert.equal(year, new Date().getFullYear(), '$year invalid')
       })
 
       it('month', async function () {
@@ -712,8 +746,7 @@ describe('convo.fillAndApplyScriptingMemory', function () {
           {},
           '$month'
         )
-
-        assert(result.length >= 2 && result.length <= 10, '$month invalid')
+        assert.equal(result, moment().format('MMMM'), '$month invalid')
       })
       it('month_MM', async function () {
         const result = ScriptingMemory.apply(
@@ -733,7 +766,7 @@ describe('convo.fillAndApplyScriptingMemory', function () {
         )
 
         const dayOfMonth = parseInt(result)
-        assert(dayOfMonth >= 1 && dayOfMonth <= 35, 'day_of_month invalid')
+        assert(dayOfMonth >= 1 && dayOfMonth <= 35, '$day_of_month invalid')
       })
 
       it('day_of_week', async function () {
@@ -746,6 +779,25 @@ describe('convo.fillAndApplyScriptingMemory', function () {
         assert(result.length >= 2 && result.length <= 20, '$day_of_week invalid')
       })
 
+      it('date_add', async function () {
+        const result = ScriptingMemory.apply(
+          { caps: CAPS_ENABLE_SCRIPTING_MEMORY },
+          {},
+          '$date_add(1, "day", YYYY.MM.DD)'
+        )
+
+        assert.equal(result, moment().add(1, 'day').format('YYYY.MM.DD'))
+      })
+      it('date_subtract', async function () {
+        const result = ScriptingMemory.apply(
+          { caps: CAPS_ENABLE_SCRIPTING_MEMORY },
+          {},
+          '$date_subtract(1, "month", YYYY.MM.DD)'
+        )
+
+        assert.equal(result, moment().subtract(1, 'month').format('YYYY.MM.DD'))
+      })
+
       it('random', async function () {
         const result = ScriptingMemory.apply(
           { caps: CAPS_ENABLE_SCRIPTING_MEMORY },
@@ -813,6 +865,14 @@ describe('convo.fillAndApplyScriptingMemory', function () {
         )
         assert.equal(result, 'botium')
       })
+      it('func with moment', async function () {
+        const result = ScriptingMemory.apply(
+          { caps: CAPS_ENABLE_SCRIPTING_MEMORY },
+          {},
+          '$func(moment(\\).subtract(1, "month"\\).startOf("month"\\).format("DD.MM.YYYY"\\))'
+        )
+        assert.equal(result, moment().subtract(1, 'month').startOf('month').format('DD.MM.YYYY'))
+      })
       it('environment variable', async function () {
         process.env.MY_VAR_VALUE = 'botium'
         const result = ScriptingMemory.apply(