botium-core 1.14.4 → 1.14.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botium-core",
-  "version": "1.14.4",
+  "version": "1.14.6",
   "description": "The Selenium for Chatbots",
   "main": "index.js",
   "module": "dist/botium-es.js",
@@ -18,7 +18,7 @@
     "link": "npm link botium-connector-dialogflow botium-connector-webdriverio botium-connector-directline3 botium-connector-watson botium-connector-alexa-smapi botium-connector-echo",
     "test": "cross-env NODE_PATH=\"./test/plugins/plugindir/fromfolder:./test/plugins/plugindir/fromfile:./test/security/resources\" mocha \"./test/**/*.spec.js\"",
     "coverage:report": "nyc report --reporter=lcov npm test",
-    "update-dependencies": "npm-check-updates --reject globby,rollup -u --timeout 120000"
+    "update-dependencies": "npm-check-updates --reject chai,globby,rollup -u --timeout 120000"
   },
   "repository": {
     "type": "git",
@@ -31,7 +31,7 @@
   },
   "homepage": "https://www.botium.ai",
   "dependencies": {
-    "@babel/runtime": "^7.23.5",
+    "@babel/runtime": "^7.23.9",
     "async": "^3.2.5",
     "body-parser": "^1.20.2",
     "boolean": "^3.2.0",
@@ -48,8 +48,8 @@
     "markdown-it": "^14.0.0",
     "mime-types": "^2.1.35",
     "mkdirp": "^3.0.1",
-    "moment": "^2.29.4",
-    "moment-timezone": "^0.5.43",
+    "moment": "^2.30.1",
+    "moment-timezone": "^0.5.45",
     "mustache": "^4.2.0",
     "promise-retry": "^2.0.1",
     "promise.allsettled": "^1.0.7",
@@ -58,8 +58,8 @@
     "rimraf": "^5.0.5",
     "sanitize-filename": "^1.6.3",
     "slugify": "^1.6.6",
-    "socket.io": "^4.7.2",
-    "socket.io-client": "^4.7.2",
+    "socket.io": "^4.7.4",
+    "socket.io-client": "^4.7.4",
     "socketio-auth": "^0.1.1",
     "swagger-jsdoc": "^6.2.8",
     "swagger-ui-express": "^5.0.0",
@@ -71,23 +71,23 @@
     "yaml": "^2.3.4"
   },
   "devDependencies": {
-    "@babel/core": "^7.23.5",
-    "@babel/node": "^7.22.19",
-    "@babel/plugin-transform-runtime": "^7.23.4",
-    "@babel/preset-env": "^7.23.5",
-    "chai": "^4.3.10",
+    "@babel/core": "^7.23.9",
+    "@babel/node": "^7.23.9",
+    "@babel/plugin-transform-runtime": "^7.23.9",
+    "@babel/preset-env": "^7.23.9",
+    "chai": "4.3.10",
     "chai-as-promised": "^7.1.1",
     "cross-env": "^7.0.3",
-    "eslint": "^8.55.0",
+    "eslint": "^8.56.0",
     "eslint-config-standard": "^17.1.0",
-    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-mocha": "^10.2.0",
-    "eslint-plugin-n": "^16.4.0",
+    "eslint-plugin-n": "^16.6.2",
     "eslint-plugin-promise": "^6.1.1",
     "eslint-plugin-standard": "^4.1.0",
-    "mocha": "^10.2.0",
-    "nock": "^13.4.0",
-    "npm-check-updates": "^16.14.11",
+    "mocha": "^10.3.0",
+    "nock": "^13.5.1",
+    "npm-check-updates": "^16.14.15",
     "nyc": "^15.1.0",
     "rollup": "2.79.1",
     "rollup-plugin-babel": "^4.4.0",

package/src/Defaults.js

@@ -34,7 +34,6 @@ module.exports = {
     [Capabilities.SIMPLEREST_INBOUND_UPDATE_CONTEXT]: true,
     [Capabilities.SIMPLEREST_CONTEXT_MERGE_OR_REPLACE]: 'MERGE',
     [Capabilities.SIMPLEREST_COOKIE_REPLICATION]: true,
-    [Capabilities.SCRIPTING_TXT_EOL]: '\n',
     [Capabilities.SCRIPTING_XLSX_EOL_WRITE]: '\r\n',
     [Capabilities.SCRIPTING_XLSX_HASHEADERS]: true,
     [Capabilities.SCRIPTING_CSV_SKIP_HEADER]: true,

package/src/scripting/CompilerMarkdown.js

@@ -3,7 +3,6 @@ const MarkdownIt = require('markdown-it')
 const util = require('util')
 const md = new MarkdownIt()
 
-const Capabilities = require('../Capabilities')
 const CompilerBase = require('./CompilerBase')
 const Constants = require('./Constants')
 const { Convo } = require('./Convo')
@@ -13,13 +12,10 @@ const { linesToConvoStep, validSenders, validateSender } = require('./helper')
 module.exports = class CompilerMarkdown extends CompilerBase {
   constructor (context, caps = {}) {
     super(context, caps)
-
-    this.eol = caps[Capabilities.SCRIPTING_TXT_EOL]
   }
 
   Validate () {
     super.Validate()
-    this._AssertCapabilityExists(Capabilities.SCRIPTING_TXT_EOL)
   }
 
   Compile (scriptBuffer, scriptType = Constants.SCRIPTING_TYPE_CONVO) {
@@ -90,7 +86,7 @@ module.exports = class CompilerMarkdown extends CompilerBase {
                 },
                 linesToConvoStep(step.children.map(child => child.content +
                   (child.children && child.children.length > 0 ? ' ' + child.children.map(child => child.content).join('|') : '')
-                ), sender, this.context, this.eol)
+                ), sender, this.context)
               ))
             } else {
               debug(`Expected sender ${validSenders.map(s => `'${s}'`).join(' or ')} but found ${sender}`)

package/src/scripting/CompilerObjectBase.js

@@ -11,13 +11,10 @@ const { linesToConvoStep, validSenders, linesToScriptingMemories } = require('./
 module.exports = class CompilerObjectBase extends CompilerBase {
   constructor (context, caps = {}) {
     super(context, caps)
-
-    this.eol = caps[Capabilities.SCRIPTING_TXT_EOL]
   }
 
   Validate () {
     super.Validate()
-    this._AssertCapabilityExists(Capabilities.SCRIPTING_TXT_EOL)
   }
 
   GetHeaders (scriptBuffer) {
@@ -69,7 +66,7 @@ module.exports = class CompilerObjectBase extends CompilerBase {
             sender: convoStepSender,
             stepTag: 'Line ' + lineTag
           },
-          linesToConvoStep(convoStepObject, convoStepSender, this.context, this.eol)
+          linesToConvoStep(convoStepObject, convoStepSender, this.context)
         ))
       }
 

package/src/scripting/CompilerTxt.js

@@ -1,27 +1,29 @@
+const _ = require('lodash')
+
 const Capabilities = require('../Capabilities')
 const Constants = require('./Constants')
 const CompilerBase = require('./CompilerBase')
 const Utterance = require('./Utterance')
 const { ConvoHeader, Convo } = require('./Convo')
-const { linesToConvoStep, convoStepToLines, validateConvo, validSenders, linesToScriptingMemories } = require('./helper')
+const { linesToConvoStep, convoStepToLines, validateConvo, validSenders, linesToScriptingMemories, trimExceptSpaceEnd } = require('./helper')
 
 module.exports = class CompilerTxt extends CompilerBase {
   constructor (context, caps = {}) {
     super(context, caps)
 
-    this.eol = caps[Capabilities.SCRIPTING_TXT_EOL]
+    this.eolRead = caps[Capabilities.SCRIPTING_TXT_EOL] || /\r\n|\r|\n/
+    this.eolWrite = (caps[Capabilities.SCRIPTING_TXT_EOL] && !_.isRegExp(caps[Capabilities.SCRIPTING_TXT_EOL])) ? caps[Capabilities.SCRIPTING_TXT_EOL] : '\n'
   }
 
   Validate () {
     super.Validate()
-    this._AssertCapabilityExists(Capabilities.SCRIPTING_TXT_EOL)
   }
 
   GetHeaders (scriptBuffer) {
     let scriptData = scriptBuffer
     if (Buffer.isBuffer(scriptBuffer)) scriptData = scriptData.toString()
 
-    const lines = scriptData.split(this.eol)
+    const lines = scriptData.split(this.eolRead)
 
     const header = { }
 
@@ -35,7 +37,7 @@ module.exports = class CompilerTxt extends CompilerBase {
     let scriptData = scriptBuffer
     if (Buffer.isBuffer(scriptBuffer)) scriptData = scriptData.toString()
 
-    const lines = scriptData.split(this.eol)
+    const lines = scriptData.split(this.eolRead)
 
     if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
       return this._compileConvo(lines, false)
@@ -64,7 +66,7 @@ module.exports = class CompilerTxt extends CompilerBase {
 
     const parseMsg = (lines) => {
       lines = lines || []
-      return linesToConvoStep(lines, convoStepSender, this.context, this.eol)
+      return linesToConvoStep(lines, convoStepSender, this.context)
     }
 
     const pushPrev = () => {
@@ -79,7 +81,7 @@ module.exports = class CompilerTxt extends CompilerBase {
       } else if (!convoStepSender && currentLines) {
         convo.header.name = currentLines[0]
         if (currentLines.length > 1) {
-          convo.header.description = currentLines.slice(1).join(this.eol)
+          convo.header.description = currentLines.slice(1).join(this.eolWrite)
         }
       }
     }
@@ -120,7 +122,7 @@ module.exports = class CompilerTxt extends CompilerBase {
 
   _compileUtterances (lines) {
     if (lines && lines.length > 0) {
-      const result = [new Utterance({ name: lines[0], utterances: lines.length > 1 ? lines.slice(1) : [] })]
+      const result = [new Utterance({ name: lines[0].trim(), utterances: lines.length > 1 ? lines.slice(1).map(line => trimExceptSpaceEnd(line)) : [] })]
       this.context.AddUtterances(result)
       return result
     }
@@ -151,23 +153,23 @@ module.exports = class CompilerTxt extends CompilerBase {
     let script = ''
 
     if (convo.header.name) {
-      script += convo.header.name + this.eol
+      script += convo.header.name + this.eolWrite
     }
     if (convo.header.description) {
-      script += convo.header.description + this.eol
+      script += convo.header.description + this.eolWrite
     }
 
     convo.conversation.forEach((step) => {
-      script += this.eol
+      script += this.eolWrite
 
       script += '#' + step.sender
       if (step.channel && step.channel !== 'default') {
         script += ' ' + step.channel
       }
-      script += this.eol
+      script += this.eolWrite
 
       const stepLines = convoStepToLines(step)
-      if (stepLines && stepLines.length > 0) script += stepLines.join(this.eol) + this.eol
+      if (stepLines && stepLines.length > 0) script += stepLines.join(this.eolWrite) + this.eolWrite
     })
     return script
   }

package/src/scripting/ScriptingProvider.js

@@ -1307,17 +1307,18 @@ module.exports = class ScriptingProvider {
     }
     if (utterances) {
       _.forEach(utterances, (utt) => {
-        const eu = this.utterances[utt.name]
+        const uttName = utt.name?.trim()
+        const eu = this.utterances[uttName]
         if (eu) {
           eu.utterances = _.uniq(_.concat(eu.utterances, utt.utterances))
         } else {
-          this.utterances[utt.name] = utt
+          this.utterances[uttName] = utt
         }
 
-        const { ambiguous, expected } = findAmbiguous(this.utterances[utt.name].utterances)
+        const { ambiguous, expected } = findAmbiguous(this.utterances[uttName].utterances)
 
         if (ambiguous && ambiguous.length > 0) {
-          debug(`Ambigous utterance "${utt.name}", expecting exact ${expected.length ? ('"' + expected.join(', ') + '"') : '<none>'} scripting memory variables in following user examples: ${ambiguous.map(d => `"${d}"`).join(', ')}`)
+          debug(`Ambigous utterance "${uttName}", expecting exact ${expected.length ? ('"' + expected.join(', ') + '"') : '<none>'} scripting memory variables in following user examples: ${ambiguous.map(d => `"${d}"`).join(', ')}`)
         }
       })
     }

package/src/scripting/helper.js

@@ -87,7 +87,7 @@ const _parseArgs = (str) => {
   return (str && str.length > 0 && str.replace(/\\\|/g, '###ESCAPESPLIT###').split('|').map(s => s.replace(/###ESCAPESPLIT###/g, '|').trim())) || []
 }
 
-const linesToConvoStep = (lines, sender, context, eol, singleLineMode = false) => {
+const linesToConvoStep = (lines, sender, context, eol = '\n', singleLineMode = false) => {
   if (!validateSender(sender)) throw new Error(`Failed to parse conversation. Section "${sender}" unknown.`)
 
   const convoStep = { asserters: [], logicHooks: [], userInputs: [], not: false, optional: false, sender }
@@ -248,7 +248,7 @@ const linesToConvoStep = (lines, sender, context, eol, singleLineMode = false) =
         if (eol === null) {
           throw new Error('eol cant be null')
         }
-        convoStep.messageText = textLines.join(eol).replace(WHITE_SPACES_EXCEPT_SPACE_CHAR_AT_THE_END, '')
+        convoStep.messageText = trimExceptSpaceEnd(textLines.join(eol))
       }
     }
   } else {
@@ -271,6 +271,10 @@ const linesToConvoStep = (lines, sender, context, eol, singleLineMode = false) =
   return convoStep
 }
 
+const trimExceptSpaceEnd = (string) => {
+  return string?.replace(WHITE_SPACES_EXCEPT_SPACE_CHAR_AT_THE_END, '')
+}
+
 const convoStepToObject = (step) => {
   const result = []
   if (step.sender === 'me') {
@@ -608,5 +612,6 @@ module.exports = {
   validateConvo,
   linesToScriptingMemories,
   calculateWer,
-  toPercent
+  toPercent,
+  trimExceptSpaceEnd
 }

package/test/compiler/compilertxt.spec.js

@@ -321,5 +321,18 @@ describe('compiler.compilertxt', function () {
       assert.equal(convo.conversation[0].asserters.length, 1)
       assert.deepEqual(convo.conversation[0].asserters[0], { name: 'BUTTONS', args: ['Test|1', 'Test|2'], not: false, optional: false })
     })
+    it('should read file with mixed new line characters', async function () {
+      const scriptBuffer = fs.readFileSync(path.resolve(__dirname, CONVOS_DIR, 'convos_mixed_new_line_characters.convo.txt'))
+      const context = buildContext()
+      const caps = {
+      }
+      const compiler = new Compiler(context, Object.assign({}, DefaultCapabilities, caps))
+      compiler.Compile(scriptBuffer, 'SCRIPTING_TYPE_CONVO')
+      const convo = context.convos[0]
+      assert.equal(convo.header.name, 'mixed new line characters (do not change in text editor!)')
+      assert.equal(convo.conversation[0].messageText, 'n')
+      assert.equal(convo.conversation[1].messageText, 'rn')
+      assert.equal(convo.conversation[2].messageText, 'r')
+    })
   })
 })

package/test/compiler/convos/txt/convos_mixed_new_line_characters.convo.txt

@@ -0,0 +1,10 @@
+mixed new line characters (do not change in text editor!)
+
+#me
+n
+
+#bot
+rn
+
+#bot
+r