botium-core 1.12.6 → 1.13.0

package/dist/botium-es.js

@@ -36,7 +36,7 @@ import express from 'express';
 import bodyParser from 'body-parser';
 
 var name = "botium-core";
-var version$1 = "1.12.6";
+var version$1 = "1.13.0";
 var description = "The Selenium for Chatbots";
 var main = "index.js";
 var module = "dist/botium-es.js";
@@ -68,22 +68,22 @@ var bugs = {
 };
 var homepage = "https://www.botium.ai";
 var dependencies = {
-	"@babel/runtime": "^7.17.9",
-	async: "^3.2.3",
+	"@babel/runtime": "^7.18.3",
+	async: "^3.2.4",
 	"body-parser": "^1.20.0",
 	boolean: "^3.2.0",
 	bottleneck: "^2.19.5",
-	"csv-parse": "^5.0.4",
+	"csv-parse": "^5.2.0",
 	debug: "^4.3.4",
 	esprima: "^4.0.1",
-	express: "^4.17.3",
+	express: "^4.18.1",
 	globby: "11.0.4",
-	ioredis: "^5.0.4",
+	ioredis: "^5.0.6",
 	"is-class": "^0.0.9",
 	"is-json": "^2.0.1",
 	jsonpath: "^1.1.1",
 	lodash: "^4.17.21",
-	"markdown-it": "^12.3.2",
+	"markdown-it": "^13.0.1",
 	"mime-types": "^2.1.35",
 	mkdirp: "^1.0.4",
 	moment: "^2.29.3",
@@ -95,39 +95,39 @@ var dependencies = {
 	rimraf: "^3.0.2",
 	"sanitize-filename": "^1.6.3",
 	slugify: "^1.6.5",
-	"socket.io": "^4.4.1",
-	"socket.io-client": "^4.4.1",
+	"socket.io": "^4.5.1",
+	"socket.io-client": "^4.5.1",
 	"socketio-auth": "^0.1.1",
 	"swagger-jsdoc": "^6.2.1",
-	"swagger-ui-express": "^4.3.0",
+	"swagger-ui-express": "^4.4.0",
 	uuid: "^8.3.2",
 	vm2: "^3.9.9",
 	"write-yaml": "^1.0.0",
 	xlsx: "^0.18.5",
-	xregexp: "^5.1.0",
-	yaml: "^2.0.1"
+	xregexp: "^5.1.1",
+	yaml: "^2.1.1"
 };
 var devDependencies = {
-	"@babel/core": "^7.17.9",
-	"@babel/node": "^7.16.8",
-	"@babel/plugin-transform-runtime": "^7.17.0",
-	"@babel/preset-env": "^7.16.11",
+	"@babel/core": "^7.18.5",
+	"@babel/node": "^7.18.5",
+	"@babel/plugin-transform-runtime": "^7.18.5",
+	"@babel/preset-env": "^7.18.2",
 	chai: "^4.3.6",
 	"chai-as-promised": "^7.1.1",
 	"cross-env": "^7.0.3",
-	eslint: "^8.13.0",
+	eslint: "^8.18.0",
 	"eslint-config-standard": "^17.0.0",
 	"eslint-plugin-import": "^2.26.0",
-	"eslint-plugin-n": "^15.1.0",
+	"eslint-plugin-n": "^15.2.3",
 	"eslint-plugin-promise": "^6.0.0",
 	"eslint-plugin-standard": "^4.1.0",
 	"license-checker": "^25.0.1",
 	"license-compatibility-checker": "^0.3.5",
-	mocha: "^9.2.2",
-	nock: "^13.2.4",
-	"npm-check-updates": "^12.5.9",
+	mocha: "^10.0.0",
+	nock: "^13.2.7",
+	"npm-check-updates": "^14.0.1",
 	nyc: "^15.1.0",
-	rollup: "^2.70.2",
+	rollup: "^2.75.6",
 	"rollup-plugin-babel": "^4.4.0",
 	"rollup-plugin-commonjs": "^10.1.0",
 	"rollup-plugin-json": "^4.0.0",
@@ -276,6 +276,10 @@ var Capabilities = {
   SCRIPTING_XLSX_SHEETNAMES_PCONVOS: 'SCRIPTING_XLSX_SHEETNAMES_PCONVOS',
   SCRIPTING_XLSX_SHEETNAMES_UTTERANCES: 'SCRIPTING_XLSX_SHEETNAMES_UTTERANCES',
   SCRIPTING_XLSX_SHEETNAMES_SCRIPTING_MEMORY: 'SCRIPTING_XLSX_SHEETNAMES_SCRIPTING_MEMORY',
+  // hidden capability. All newly in Box created testsets will have this as true. CsvCompiler
+  // - throws less error (Box reads csv files as utterances, and convo. Compiler cant throw exception if a file is correct, but box tries to load it with incorrect script type)
+  // 4 or more colums are compiled just as utterances.
+  SCRIPTING_CSV_LEGACY_MODE_OFF: 'SCRIPTING_CSV_LEGACY_MODE_OFF',
   SCRIPTING_CSV_DELIMITER: 'SCRIPTING_CSV_DELIMITER',
   SCRIPTING_CSV_SKIP_HEADER: 'SCRIPTING_CSV_SKIP_HEADER',
   SCRIPTING_CSV_QUOTE: 'SCRIPTING_CSV_QUOTE',
@@ -285,6 +289,9 @@ var Capabilities = {
   SCRIPTING_CSV_MULTIROW_COLUMN_TEXT: 'SCRIPTING_CSV_MULTIROW_COLUMN_TEXT',
   SCRIPTING_CSV_QA_COLUMN_QUESTION: 'SCRIPTING_CSV_QA_COLUMN_QUESTION',
   SCRIPTING_CSV_QA_COLUMN_ANSWER: 'SCRIPTING_CSV_QA_COLUMN_ANSWER',
+  SCRIPTING_CSV_UTTERANCE_STARTROW: 'SCRIPTING_CSV_UTTERANCE_STARTROW',
+  SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER: 'SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER',
+  SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY: 'SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY',
   SCRIPTING_NORMALIZE_TEXT: 'SCRIPTING_NORMALIZE_TEXT',
   SCRIPTING_ENABLE_MEMORY: 'SCRIPTING_ENABLE_MEMORY',
   SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS: 'SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS',
@@ -426,6 +433,7 @@ Capabilities.SCRIPTING_XLSX_SHEETNAMES;
 Capabilities.SCRIPTING_XLSX_SHEETNAMES_PCONVOS;
 Capabilities.SCRIPTING_XLSX_SHEETNAMES_UTTERANCES;
 Capabilities.SCRIPTING_XLSX_SHEETNAMES_SCRIPTING_MEMORY;
+Capabilities.SCRIPTING_CSV_LEGACY_MODE_OFF;
 Capabilities.SCRIPTING_CSV_DELIMITER;
 Capabilities.SCRIPTING_CSV_SKIP_HEADER;
 Capabilities.SCRIPTING_CSV_QUOTE;
@@ -435,6 +443,9 @@ Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER;
 Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT;
 Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION;
 Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER;
+Capabilities.SCRIPTING_CSV_UTTERANCE_STARTROW;
+Capabilities.SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER;
+Capabilities.SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY;
 Capabilities.SCRIPTING_NORMALIZE_TEXT;
 Capabilities.SCRIPTING_ENABLE_MEMORY;
 Capabilities.SCRIPTING_ENABLE_MULTIPLE_ASSERT_ERRORS;
@@ -5295,6 +5306,8 @@ var CompilerCsv_1 = class CompilerCsv extends CompilerBase_1 {
       return [];
     }
 
+    const legacyModeOn = !this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_LEGACY_MODE_OFF, false);
+
     let delimiter = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_DELIMITER);
 
     if (!delimiter) {
@@ -5322,7 +5335,8 @@ var CompilerCsv_1 = class CompilerCsv extends CompilerBase_1 {
         delimiter,
         escape: this.caps[Capabilities.SCRIPTING_CSV_ESCAPE],
         quote: this.caps[Capabilities.SCRIPTING_CSV_QUOTE],
-        columns: false
+        columns: false,
+        relax_column_count: true
       });
     } catch (err) {
       throw new Error(`Invalid CSV: ${err.message || err}`);
@@ -5332,120 +5346,175 @@ var CompilerCsv_1 = class CompilerCsv extends CompilerBase_1 {
       return [];
     }
 
-    if (rows[0].length === 1) {
-      debug$c('Found 1-column CSV file, treating it as utterance file');
+    const columnCount = rows[0].length;
+    debug$c(`Legacy mode ${legacyModeOn ? 'on' : 'off'} rows ${rows.length} columns ${columnCount}`);
 
-      if (scriptType === Constants.SCRIPTING_TYPE_UTTERANCES) {
-        const result = [{
-          name: rows[0][0],
-          utterances: rows.slice(1).map(r => r[0])
-        }];
-        this.context.AddUtterances(result);
-        return result;
-      } else {
+    if (scriptType === Constants.SCRIPTING_TYPE_CONVO || scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
+      if (columnCount === 1 || !legacyModeOn && columnCount > 3) {
+        debug$c(`Invalid column count '${columnCount}' in convo mode`);
         return [];
       }
-    }
 
-    if (scriptType !== Constants.SCRIPTING_TYPE_CONVO && scriptType !== Constants.SCRIPTING_TYPE_PCONVO) {
-      return [];
-    }
+      let header = null;
 
-    let header = null;
+      if (rows.length > 0 && this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER]) {
+        header = rows[0];
+        rows = rows.slice(1);
+      }
 
-    if (rows.length > 0 && this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER]) {
-      header = rows[0];
-      rows = rows.slice(1);
-    }
+      if (rows.length === 0) {
+        debug$c('Datarows not found in convo mode');
+        return [];
+      }
 
-    if (rows.length === 0) {
-      return [];
-    }
+      const lineNumberBase = this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER] ? 2 : 1;
 
-    const lineNumberBase = this.caps[Capabilities.SCRIPTING_CSV_SKIP_HEADER] ? 2 : 1;
+      if (columnCount === 2) {
+        let colQuestion = DEFAULT_QA_COLUMN_QUESTION;
+        let colAnswer = DEFAULT_QA_COLUMN_ANSWER;
 
-    if (rows[0].length === 2) {
-      debug$c('Found 2-column CSV file, treating it as question/answer file');
-      let colQuestion = DEFAULT_QA_COLUMN_QUESTION;
-      let colAnswer = DEFAULT_QA_COLUMN_ANSWER;
+        if (header) {
+          if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION] !== undefined) {
+            colQuestion = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION]);
+          }
 
-      if (header) {
-        if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION] !== undefined) {
-          colQuestion = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_QUESTION]);
+          if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER] !== undefined) {
+            colAnswer = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER]);
+          }
         }
 
-        if (this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER] !== undefined) {
-          colAnswer = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_QA_COLUMN_ANSWER]);
-        }
-      }
+        const convos = rows.map((row, i) => new Convo$3(this.context, {
+          header: {
+            name: `L${i + lineNumberBase}`
+          },
+          conversation: [Object.assign({}, linesToConvoStep$2([row[colQuestion]], 'me', this.context, undefined, true), {
+            stepTag: `L${i + lineNumberBase}-Question`
+          }), Object.assign({}, linesToConvoStep$2([row[colAnswer]], 'bot', this.context, undefined, true), {
+            stepTag: `L${i + lineNumberBase}-Answer`
+          })]
+        }));
 
-      const convos = rows.map((row, i) => new Convo$3(this.context, {
-        header: {
-          name: `L${i + lineNumberBase}`
-        },
-        conversation: [Object.assign({}, linesToConvoStep$2([row[colQuestion]], 'me', this.context, undefined, true), {
-          stepTag: `L${i + lineNumberBase}-Question`
-        }), Object.assign({}, linesToConvoStep$2([row[colAnswer]], 'bot', this.context, undefined, true), {
-          stepTag: `L${i + lineNumberBase}-Answer`
-        })]
-      }));
+        if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
+          this.context.AddConvos(convos);
+        } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
+          this.context.AddPartialConvos(convos);
+        }
 
-      if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
-        this.context.AddConvos(convos);
-      } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
-        this.context.AddPartialConvos(convos);
+        debug$c(`Found 2-column CSV file, treating it as question/answer file, extracted ${convos.length} convos`);
+        return convos;
       }
 
-      return convos;
-    }
+      if (columnCount >= 3) {
+        let colConversationId = DEFAULT_MULTIROW_COLUMN_CONVERSATION;
+        let colSender = DEFAULT_MULTIROW_COLUMN_SENDER;
+        let colText = DEFAULT_MULTIROW_COLUMN_TEXT;
 
-    if (rows[0].length >= 3) {
-      debug$c('Found 3-column CSV file, treating it as multi-row conversation file');
-      let colConversationId = DEFAULT_MULTIROW_COLUMN_CONVERSATION;
-      let colSender = DEFAULT_MULTIROW_COLUMN_SENDER;
-      let colText = DEFAULT_MULTIROW_COLUMN_TEXT;
+        if (header) {
+          if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID] !== undefined) {
+            colConversationId = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID]);
+          }
 
-      if (header) {
-        if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID] !== undefined) {
-          colConversationId = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_CONVERSATION_ID]);
-        }
+          if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER] !== undefined) {
+            colSender = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER]);
+          }
 
-        if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER] !== undefined) {
-          colSender = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_SENDER]);
+          if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT] !== undefined) {
+            colText = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT]);
+          }
         }
 
-        if (this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT] !== undefined) {
-          colText = _findColIndex(header, this.caps[Capabilities.SCRIPTING_CSV_MULTIROW_COLUMN_TEXT]);
+        const conversationIds = lodash.uniq(rows.map(r => r[colConversationId]));
+
+        const convos = conversationIds.map(conversationId => {
+          const convoRows = rows.map((row, i) => {
+            if (row[colConversationId] === conversationId) {
+              return Object.assign({}, linesToConvoStep$2([row[colText]], row[colSender], this.context, undefined, true), {
+                stepTag: `L${i + lineNumberBase}`
+              });
+            }
+
+            return null;
+          }).filter(c => c);
+          return new Convo$3(this.context, {
+            header: {
+              name: conversationId
+            },
+            conversation: convoRows
+          });
+        });
+
+        if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
+          this.context.AddConvos(convos);
+        } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
+          this.context.AddPartialConvos(convos);
         }
+
+        debug$c(`Found 3-column CSV file, treating it as multi-row conversation file, extracted ${convos.length} convos`);
+        return convos;
+      }
+    } else if (scriptType === Constants.SCRIPTING_TYPE_UTTERANCES) {
+      if (columnCount === 2 || columnCount === 3 || legacyModeOn && columnCount > 4) {
+        debug$c(`Invalid column count '${columnCount}' in utterances mode`);
+        return [];
       }
 
-      const conversationIds = lodash.uniq(rows.map(r => r[colConversationId]));
+      const result = [];
+      const startRow = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_UTTERANCE_STARTROW, 2) - 1;
 
-      const convos = conversationIds.map(conversationId => {
-        const convoRows = rows.map((row, i) => {
-          if (row[colConversationId] === conversationId) {
-            return Object.assign({}, linesToConvoStep$2([row[colText]], row[colSender], this.context, undefined, true), {
-              stepTag: `L${i + lineNumberBase}`
-            });
+      const startRowHeader = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_UTTERANCE_STARTROW_HEADER);
+
+      const stopOnEmpty = this._GetOptionalCapability(Capabilities.SCRIPTING_CSV_UTTERANCE_STOP_ON_EMPTY);
+
+      for (let col = 0; col < columnCount; col++) {
+        const name = rows[0][col];
+
+        if (!name || name.trim().length === 0) {
+          debug$c(`Column ${col + 1} has no header, skipping`);
+          continue;
+        }
+
+        const uttStruct = {
+          name,
+          utterances: []
+        };
+        let skip = !!startRowHeader;
+
+        const getData = row => {
+          return rows[row][col] ? rows[row][col].trim() : false;
+        }; //
+
+
+        for (let row = startRow; row < rows.length && (skip || !stopOnEmpty || !!getData(row)); row++) {
+          // eslint-disable-line no-unmodified-loop-condition
+          const data = getData(row);
+
+          if (!data) {
+            continue;
           }
 
-          return null;
-        }).filter(c => c);
-        return new Convo$3(this.context, {
-          header: {
-            name: conversationId
-          },
-          conversation: convoRows
-        });
-      });
+          if (!skip) {
+            uttStruct.utterances.push(data);
+          } else {
+            if (startRowHeader === rows[row][col]) {
+              skip = false;
+            }
+          }
+        }
 
-      if (scriptType === Constants.SCRIPTING_TYPE_CONVO) {
-        this.context.AddConvos(convos);
-      } else if (scriptType === Constants.SCRIPTING_TYPE_PCONVO) {
-        this.context.AddPartialConvos(convos);
+        if (uttStruct.utterances.length === 0) {
+          // liveperson, skipping meta intents
+          debug$c(`Column ${col + 1} has no utterances, skipping`);
+          continue;
+        }
+
+        result.push(uttStruct);
       }
 
-      return convos;
+      debug$c(`Multi-column utterance file, extracted ${result.length} utterances`);
+      this.context.AddUtterances(result);
+      return result;
+    } else {
+      return [];
     }
   }
 
@@ -5772,7 +5841,7 @@ const {
 const {
   getMatchFunction
 } = MatchFunctions;
-const globPattern = '**/+(*.convo.txt|*.utterances.txt|*.pconvo.txt|*.scriptingmemory.txt|*.xlsx|*.xlsm|*.convo.csv|*.pconvo.csv|*.yaml|*.yml|*.json|*.md|*.markdown)';
+const globPattern = '**/+(*.convo.txt|*.utterances.txt|*.pconvo.txt|*.scriptingmemory.txt|*.xlsx|*.xlsm|*.convo.csv|*.pconvo.csv|*.utterances.csv|*.yaml|*.yml|*.json|*.md|*.markdown)';
 const skipPattern = /^skip[.\-_]/i;
 
 const p = (retryHelper, fn) => {
@@ -6467,6 +6536,10 @@ var ScriptingProvider_1 = class ScriptingProvider {
         result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_CONVO);
       } else if (filename.endsWith('.pconvo.csv')) {
         result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_PCONVO);
+      } else if (filename.endsWith('.pconvo.csv')) {
+        result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_PCONVO);
+      } else if (filename.endsWith('.utterance.csv')) {
+        result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_CSV, Constants.SCRIPTING_TYPE_UTTERANCES);
       } else if (filename.endsWith('.yaml') || filename.endsWith('.yml')) {
         result = this.ReadScriptFromBuffer(scriptBuffer, Constants.SCRIPTING_FORMAT_YAML, [Constants.SCRIPTING_TYPE_UTTERANCES, Constants.SCRIPTING_TYPE_PCONVO, Constants.SCRIPTING_TYPE_CONVO, Constants.SCRIPTING_TYPE_SCRIPTING_MEMORY]);
       } else if (filename.endsWith('.json')) {