botium-core 1.13.15 → 1.13.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/botium-cjs.js +221 -173
- package/dist/botium-cjs.js.map +1 -1
- package/dist/botium-es.js +213 -166
- package/dist/botium-es.js.map +1 -1
- package/index.js +2 -0
- package/package.json +23 -23
- package/src/BotDriver.js +4 -6
- package/src/Events.js +1 -0
- package/src/containers/BaseContainer.js +4 -5
- package/src/scripting/Convo.js +1 -20
- package/src/scripting/ScriptingProvider.js +102 -51
- package/src/scripting/helper.js +11 -4
- package/src/scripting/logichook/asserter/WerAsserter.js +53 -6
- package/test/scripting/txt/decompile.spec.js +27 -0
- package/test/security/allowUnsafe.spec.js +20 -10
- package/test/security/convos/withscriptingmemoryfunction.convo.txt +1 -0
- package/test/convo/retryconvo.spec.js +0 -134
package/index.js
CHANGED
|
@@ -12,6 +12,8 @@ module.exports = {
|
|
|
12
12
|
HookUtils: require('./src/helpers/HookUtils'),
|
|
13
13
|
TranscriptUtils: require('./src/helpers/TranscriptUtils'),
|
|
14
14
|
|
|
15
|
+
RetryHelper: require('./src/helpers/RetryHelper'),
|
|
16
|
+
|
|
15
17
|
BotiumMockRichMessageTypes: require('./src/mocks/BotiumMockRichMessageTypes'),
|
|
16
18
|
|
|
17
19
|
BotiumError: require('./src/scripting/BotiumError').BotiumError,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "botium-core",
|
|
3
|
-
"version": "1.13.
|
|
3
|
+
"version": "1.13.17",
|
|
4
4
|
"description": "The Selenium for Chatbots",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"module": "dist/botium-es.js",
|
|
@@ -32,66 +32,66 @@
|
|
|
32
32
|
},
|
|
33
33
|
"homepage": "https://www.botium.ai",
|
|
34
34
|
"dependencies": {
|
|
35
|
-
"@babel/runtime": "^7.
|
|
35
|
+
"@babel/runtime": "^7.21.5",
|
|
36
36
|
"async": "^3.2.4",
|
|
37
|
-
"body-parser": "^1.20.
|
|
37
|
+
"body-parser": "^1.20.2",
|
|
38
38
|
"boolean": "^3.2.0",
|
|
39
39
|
"bottleneck": "^2.19.5",
|
|
40
|
-
"csv-parse": "^5.3.
|
|
40
|
+
"csv-parse": "^5.3.10",
|
|
41
41
|
"debug": "^4.3.4",
|
|
42
42
|
"esprima": "^4.0.1",
|
|
43
43
|
"express": "^4.18.2",
|
|
44
44
|
"globby": "11.0.4",
|
|
45
|
-
"ioredis": "^5.2
|
|
45
|
+
"ioredis": "^5.3.2",
|
|
46
46
|
"is-class": "^0.0.9",
|
|
47
47
|
"is-json": "^2.0.1",
|
|
48
48
|
"jsonpath": "^1.1.1",
|
|
49
49
|
"lodash": "^4.17.21",
|
|
50
50
|
"markdown-it": "^13.0.1",
|
|
51
51
|
"mime-types": "^2.1.35",
|
|
52
|
-
"mkdirp": "^
|
|
52
|
+
"mkdirp": "^3.0.1",
|
|
53
53
|
"moment": "^2.29.4",
|
|
54
54
|
"mustache": "^4.2.0",
|
|
55
55
|
"promise-retry": "^2.0.1",
|
|
56
56
|
"promise.allsettled": "^1.0.6",
|
|
57
57
|
"randomatic": "^3.1.1",
|
|
58
58
|
"request": "^2.88.2",
|
|
59
|
-
"rimraf": "^
|
|
59
|
+
"rimraf": "^5.0.0",
|
|
60
60
|
"sanitize-filename": "^1.6.3",
|
|
61
|
-
"slugify": "^1.6.
|
|
62
|
-
"socket.io": "^4.
|
|
63
|
-
"socket.io-client": "^4.
|
|
61
|
+
"slugify": "^1.6.6",
|
|
62
|
+
"socket.io": "^4.6.1",
|
|
63
|
+
"socket.io-client": "^4.6.1",
|
|
64
64
|
"socketio-auth": "^0.1.1",
|
|
65
|
-
"swagger-jsdoc": "^6.2.
|
|
66
|
-
"swagger-ui-express": "^4.6.
|
|
65
|
+
"swagger-jsdoc": "^6.2.8",
|
|
66
|
+
"swagger-ui-express": "^4.6.3",
|
|
67
67
|
"uuid": "^9.0.0",
|
|
68
|
-
"vm2": "^3.9.
|
|
68
|
+
"vm2": "^3.9.17",
|
|
69
69
|
"word-error-rate": "0.0.7",
|
|
70
70
|
"write-yaml": "^1.0.0",
|
|
71
71
|
"xlsx": "^0.18.5",
|
|
72
72
|
"xregexp": "^5.1.1",
|
|
73
|
-
"yaml": "^2.
|
|
73
|
+
"yaml": "^2.2.2"
|
|
74
74
|
},
|
|
75
75
|
"devDependencies": {
|
|
76
|
-
"@babel/core": "^7.
|
|
77
|
-
"@babel/node": "^7.20.
|
|
78
|
-
"@babel/plugin-transform-runtime": "^7.
|
|
79
|
-
"@babel/preset-env": "^7.
|
|
76
|
+
"@babel/core": "^7.21.8",
|
|
77
|
+
"@babel/node": "^7.20.7",
|
|
78
|
+
"@babel/plugin-transform-runtime": "^7.21.4",
|
|
79
|
+
"@babel/preset-env": "^7.21.5",
|
|
80
80
|
"chai": "^4.3.7",
|
|
81
81
|
"chai-as-promised": "^7.1.1",
|
|
82
82
|
"cross-env": "^7.0.3",
|
|
83
|
-
"eslint": "^8.
|
|
83
|
+
"eslint": "^8.40.0",
|
|
84
84
|
"eslint-config-standard": "^17.0.0",
|
|
85
|
-
"eslint-plugin-import": "^2.
|
|
85
|
+
"eslint-plugin-import": "^2.27.5",
|
|
86
86
|
"eslint-plugin-mocha": "^10.1.0",
|
|
87
|
-
"eslint-plugin-n": "^15.
|
|
87
|
+
"eslint-plugin-n": "^15.7.0",
|
|
88
88
|
"eslint-plugin-promise": "^6.1.1",
|
|
89
89
|
"eslint-plugin-standard": "^4.1.0",
|
|
90
90
|
"license-checker": "^25.0.1",
|
|
91
91
|
"license-compatibility-checker": "^0.3.5",
|
|
92
92
|
"mocha": "^10.2.0",
|
|
93
|
-
"nock": "^13.
|
|
94
|
-
"npm-check-updates": "^16.
|
|
93
|
+
"nock": "^13.3.1",
|
|
94
|
+
"npm-check-updates": "^16.10.12",
|
|
95
95
|
"nyc": "^15.1.0",
|
|
96
96
|
"rollup": "2.79.1",
|
|
97
97
|
"rollup-plugin-babel": "^4.4.0",
|
package/src/BotDriver.js
CHANGED
|
@@ -2,8 +2,8 @@ const util = require('util')
|
|
|
2
2
|
const fs = require('fs')
|
|
3
3
|
const path = require('path')
|
|
4
4
|
const async = require('async')
|
|
5
|
-
const rimraf = require('rimraf')
|
|
6
|
-
const
|
|
5
|
+
const { rimraf } = require('rimraf')
|
|
6
|
+
const { mkdirpSync } = require('mkdirp')
|
|
7
7
|
const sanitize = require('sanitize-filename')
|
|
8
8
|
const moment = require('moment')
|
|
9
9
|
const randomize = require('randomatic')
|
|
@@ -130,7 +130,7 @@ module.exports = class BotDriver {
|
|
|
130
130
|
(tempDirectoryCreated) => {
|
|
131
131
|
tempDirectory = path.resolve(process.cwd(), this.caps[Capabilities.TEMPDIR], sanitize(`${this.caps[Capabilities.PROJECTNAME]} ${moment().format('YYYYMMDD HHmmss')} ${randomize('Aa0', 5)}`))
|
|
132
132
|
try {
|
|
133
|
-
|
|
133
|
+
mkdirpSync(tempDirectory)
|
|
134
134
|
tempDirectoryCreated()
|
|
135
135
|
} catch (err) {
|
|
136
136
|
tempDirectoryCreated(new Error(`Unable to create temp directory ${tempDirectory}: ${err.message}`))
|
|
@@ -168,9 +168,7 @@ module.exports = class BotDriver {
|
|
|
168
168
|
debug(`BotDriver Build error: ${err}`)
|
|
169
169
|
this.eventEmitter.emit(Events.CONTAINER_BUILD_ERROR, err)
|
|
170
170
|
if (tempDirectory) {
|
|
171
|
-
rimraf(tempDirectory
|
|
172
|
-
if (err) debug(`Cleanup temp dir ${tempDirectory} failed: ${util.inspect(err)}`)
|
|
173
|
-
})
|
|
171
|
+
rimraf(tempDirectory).catch((err) => debug(`Cleanup temp dir ${tempDirectory} failed: ${util.inspect(err)}`))
|
|
174
172
|
}
|
|
175
173
|
return reject(err)
|
|
176
174
|
}
|
package/src/Events.js
CHANGED
|
@@ -13,6 +13,7 @@ module.exports = {
|
|
|
13
13
|
CONTAINER_CLEANED: 'CONTAINER_CLEANED',
|
|
14
14
|
CONTAINER_CLEAN_ERROR: 'CONTAINER_CLEAN_ERROR',
|
|
15
15
|
BOT_CONNECTED: 'BOT_CONNECTED',
|
|
16
|
+
CONVO_STEP_NEXT: 'CONVO_STEP_NEXT',
|
|
16
17
|
// Chatbot Events
|
|
17
18
|
MESSAGE_SENTTOBOT: 'MESSAGE_SENTTOBOT',
|
|
18
19
|
MESSAGE_SENDTOBOT_ERROR: 'MESSAGE_SENDTOBOT_ERROR',
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const util = require('util')
|
|
2
2
|
const async = require('async')
|
|
3
|
-
const rimraf = require('rimraf')
|
|
3
|
+
const { rimraf } = require('rimraf')
|
|
4
4
|
const Bottleneck = require('bottleneck')
|
|
5
5
|
const _ = require('lodash')
|
|
6
6
|
const request = require('request')
|
|
@@ -173,10 +173,9 @@ module.exports = class BaseContainer {
|
|
|
173
173
|
(rimraffed) => {
|
|
174
174
|
if (this.caps[Capabilities.CLEANUPTEMPDIR]) {
|
|
175
175
|
debug(`Cleanup rimrafing temp dir ${this.tempDirectory}`)
|
|
176
|
-
rimraf(this.tempDirectory
|
|
177
|
-
|
|
178
|
-
rimraffed()
|
|
179
|
-
})
|
|
176
|
+
rimraf(this.tempDirectory)
|
|
177
|
+
.catch((err) => debug(`Cleanup temp dir ${this.tempDirectory} failed: ${util.inspect(err)}`))
|
|
178
|
+
.finally(() => rimraffed())
|
|
180
179
|
} else {
|
|
181
180
|
rimraffed()
|
|
182
181
|
}
|
package/src/scripting/Convo.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
const util = require('util')
|
|
2
2
|
const _ = require('lodash')
|
|
3
3
|
const debug = require('debug')('botium-core-Convo')
|
|
4
|
-
const promiseRetry = require('promise-retry')
|
|
5
4
|
|
|
6
5
|
const BotiumMockMessage = require('../mocks/BotiumMockMessage')
|
|
7
6
|
const Capabilities = require('../Capabilities')
|
|
@@ -9,7 +8,6 @@ const Events = require('../Events')
|
|
|
9
8
|
const ScriptingMemory = require('./ScriptingMemory')
|
|
10
9
|
const { BotiumError, botiumErrorFromErr, botiumErrorFromList } = require('./BotiumError')
|
|
11
10
|
const { normalizeText, toString, removeBuffers, splitStringInNonEmptyLines } = require('./helper')
|
|
12
|
-
const RetryHelper = require('../helpers/RetryHelper')
|
|
13
11
|
|
|
14
12
|
const { LOGIC_HOOK_INCLUDE } = require('./logichook/LogicHookConsts')
|
|
15
13
|
|
|
@@ -212,24 +210,6 @@ class Convo {
|
|
|
212
210
|
}
|
|
213
211
|
|
|
214
212
|
async Run (container) {
|
|
215
|
-
const retryHelper = new RetryHelper(container.caps, 'CONVO')
|
|
216
|
-
return promiseRetry(async (retry, number) => {
|
|
217
|
-
return this.RunImpl(container).catch(err => {
|
|
218
|
-
const retryRemaining = retryHelper.retrySettings.retries - number + 1
|
|
219
|
-
if (retryHelper.shouldRetry(err)) {
|
|
220
|
-
debug(`Convo failed with error "${err.message || JSON.stringify(err)}". Retry ${retryRemaining > 0 ? 'enabled' : 'disabled'} (remaining #${retryRemaining}/${retryHelper.retrySettings.retries}, criterion matches)`)
|
|
221
|
-
retry(err)
|
|
222
|
-
} else {
|
|
223
|
-
if (retryHelper.retryErrorPatterns.length > 0) {
|
|
224
|
-
debug(`Convo failed with error "${err.message || JSON.stringify(err)}". Retry 'disabled' (remaining (#${retryRemaining}/${retryHelper.retrySettings.retries}), criterion does not match)`)
|
|
225
|
-
}
|
|
226
|
-
throw err
|
|
227
|
-
}
|
|
228
|
-
})
|
|
229
|
-
}, retryHelper.retrySettings)
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
async RunImpl (container) {
|
|
233
213
|
const transcript = new Transcript({
|
|
234
214
|
steps: [],
|
|
235
215
|
attachments: [],
|
|
@@ -298,6 +278,7 @@ class Convo {
|
|
|
298
278
|
for (let i = 0; i < this.conversation.length; i++) {
|
|
299
279
|
const convoStep = this.conversation[i]
|
|
300
280
|
const currentStepIndex = i
|
|
281
|
+
container.eventEmitter.emit(Events.CONVO_STEP_NEXT, container, convoStep, i)
|
|
301
282
|
skipTranscriptStep = false
|
|
302
283
|
const transcriptStep = new TranscriptStep({
|
|
303
284
|
expected: new BotiumMockMessage(convoStep),
|
|
@@ -19,6 +19,7 @@ const { BotiumError, botiumErrorFromList, botiumErrorFromErr } = require('./Boti
|
|
|
19
19
|
const RetryHelper = require('../helpers/RetryHelper')
|
|
20
20
|
const { getMatchFunction } = require('./MatchFunctions')
|
|
21
21
|
const precompilers = require('./precompilers')
|
|
22
|
+
const { calculateWer, toPercent } = require('./helper')
|
|
22
23
|
|
|
23
24
|
const globPattern = '**/+(*.convo.txt|*.utterances.txt|*.pconvo.txt|*.scriptingmemory.txt|*.xlsx|*.xlsm|*.convo.csv|*.pconvo.csv|*.utterances.csv|*.yaml|*.yml|*.json|*.md|*.markdown)'
|
|
24
25
|
const skipPattern = /^skip[.\-_]/i
|
|
@@ -137,32 +138,57 @@ module.exports = class ScriptingProvider {
|
|
|
137
138
|
const found = _.find(tomatch, (utt) => this.matchFn(botresponse, utt, this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS]))
|
|
138
139
|
const asserterType = this.caps[Capabilities.SCRIPTING_MATCHING_MODE] === 'wer' ? 'Word Error Rate Asserter' : 'Text Match Asserter'
|
|
139
140
|
if (_.isNil(found)) {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
|
|
162
|
-
args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
|
|
141
|
+
if (this.caps[Capabilities.SCRIPTING_MATCHING_MODE] === 'wer') {
|
|
142
|
+
const wer = calculateWer(botresponse, tomatch[0])
|
|
143
|
+
const werArgs = this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS]
|
|
144
|
+
const threshold = ([',', '.'].find(p => `${werArgs[0]}`.includes(p)) ? parseFloat(werArgs[0]) : parseInt(werArgs[0]) / 100)
|
|
145
|
+
const message = `${stepTag}: Word Error Rate (${toPercent(wer)}) higher than accepted (${toPercent(threshold)})`
|
|
146
|
+
throw new BotiumError(
|
|
147
|
+
message,
|
|
148
|
+
{
|
|
149
|
+
type: 'asserter',
|
|
150
|
+
source: asserterType,
|
|
151
|
+
params: {
|
|
152
|
+
matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
|
|
153
|
+
args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
|
|
154
|
+
},
|
|
155
|
+
context: {
|
|
156
|
+
stepTag
|
|
157
|
+
},
|
|
158
|
+
cause: {
|
|
159
|
+
expected: `<=${toPercent(threshold)} (${tomatch})`,
|
|
160
|
+
actual: `${toPercent(wer)} (${botresponse})`
|
|
161
|
+
}
|
|
163
162
|
}
|
|
164
|
-
|
|
165
|
-
|
|
163
|
+
)
|
|
164
|
+
} else {
|
|
165
|
+
let message = `${stepTag}: Bot response `
|
|
166
|
+
message += meMsg ? `(on ${meMsg}) ` : ''
|
|
167
|
+
message += botresponse ? ('"' + botresponse + '"') : '<no response>'
|
|
168
|
+
message += ' expected to match '
|
|
169
|
+
message += tomatch && tomatch.length > 1 ? 'one of ' : ''
|
|
170
|
+
message += `${tomatch.map(e => e ? '"' + e + '"' : '<any response>').join(', ')}`
|
|
171
|
+
throw new BotiumError(
|
|
172
|
+
message,
|
|
173
|
+
{
|
|
174
|
+
type: 'asserter',
|
|
175
|
+
source: asserterType,
|
|
176
|
+
params: {
|
|
177
|
+
matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
|
|
178
|
+
args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
|
|
179
|
+
},
|
|
180
|
+
context: {
|
|
181
|
+
stepTag
|
|
182
|
+
},
|
|
183
|
+
cause: {
|
|
184
|
+
expected: tomatch,
|
|
185
|
+
actual: botresponse,
|
|
186
|
+
matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
|
|
187
|
+
args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
)
|
|
191
|
+
}
|
|
166
192
|
}
|
|
167
193
|
},
|
|
168
194
|
assertBotNotResponse: (botresponse, nottomatch, stepTag, meMsg) => {
|
|
@@ -173,33 +199,58 @@ module.exports = class ScriptingProvider {
|
|
|
173
199
|
const found = _.find(nottomatch, (utt) => this.matchFn(botresponse, utt, this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS]))
|
|
174
200
|
const asserterType = this.caps[Capabilities.SCRIPTING_MATCHING_MODE] === 'wer' ? 'Word Error Rate Asserter' : 'Text Match Asserter'
|
|
175
201
|
if (!_.isNil(found)) {
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
actual: botresponse,
|
|
198
|
-
matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
|
|
199
|
-
args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
|
|
202
|
+
if (this.caps[Capabilities.SCRIPTING_MATCHING_MODE] === 'wer') {
|
|
203
|
+
const wer = calculateWer(botresponse, nottomatch[0])
|
|
204
|
+
const werArgs = this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS]
|
|
205
|
+
const threshold = ([',', '.'].find(p => `${werArgs[0]}`.includes(p)) ? parseFloat(werArgs[0]) : parseInt(werArgs[0]) / 100)
|
|
206
|
+
const message = `${stepTag}: Word Error Rate (${toPercent(wer)}) lower than accepted (${toPercent(threshold)})`
|
|
207
|
+
throw new BotiumError(
|
|
208
|
+
message,
|
|
209
|
+
{
|
|
210
|
+
type: 'asserter',
|
|
211
|
+
source: asserterType,
|
|
212
|
+
params: {
|
|
213
|
+
matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
|
|
214
|
+
args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
|
|
215
|
+
},
|
|
216
|
+
context: {
|
|
217
|
+
stepTag
|
|
218
|
+
},
|
|
219
|
+
cause: {
|
|
220
|
+
expected: `>=${toPercent(threshold)} (${nottomatch})`,
|
|
221
|
+
actual: `${toPercent(wer)} (${botresponse})`
|
|
222
|
+
}
|
|
200
223
|
}
|
|
201
|
-
|
|
202
|
-
|
|
224
|
+
)
|
|
225
|
+
} else {
|
|
226
|
+
let message = `${stepTag}: Bot response `
|
|
227
|
+
message += meMsg ? `(on ${meMsg}) ` : ''
|
|
228
|
+
message += botresponse ? ('"' + botresponse + '"') : '<no response>'
|
|
229
|
+
message += ' expected NOT to match '
|
|
230
|
+
message += nottomatch && nottomatch.length > 1 ? 'one of ' : ''
|
|
231
|
+
message += `${nottomatch.map(e => e ? '"' + e + '"' : '<any response>').join(', ')}`
|
|
232
|
+
throw new BotiumError(
|
|
233
|
+
message,
|
|
234
|
+
{
|
|
235
|
+
type: 'asserter',
|
|
236
|
+
source: asserterType,
|
|
237
|
+
params: {
|
|
238
|
+
matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
|
|
239
|
+
args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
|
|
240
|
+
},
|
|
241
|
+
context: {
|
|
242
|
+
stepTag
|
|
243
|
+
},
|
|
244
|
+
cause: {
|
|
245
|
+
not: true,
|
|
246
|
+
expected: nottomatch,
|
|
247
|
+
actual: botresponse,
|
|
248
|
+
matchingMode: this.caps[Capabilities.SCRIPTING_MATCHING_MODE],
|
|
249
|
+
args: this.caps[Capabilities.SCRIPTING_MATCHING_MODE_ARGS] || null
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
)
|
|
253
|
+
}
|
|
203
254
|
}
|
|
204
255
|
},
|
|
205
256
|
fail: null
|
package/src/scripting/helper.js
CHANGED
|
@@ -76,11 +76,11 @@ const toString = (value) => {
|
|
|
76
76
|
}
|
|
77
77
|
|
|
78
78
|
const flatString = (str) => {
|
|
79
|
-
return str
|
|
79
|
+
return toString(str).split('\n').map(s => s.trim()).join(' ') || ''
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
const _formatAppendArgs = (args) => {
|
|
83
|
-
return args
|
|
83
|
+
return (args && args.length > 0 && _.isArray(args) && ` ${args.map(a => _.isString(a) ? a.replace(/\|/g, '\\|') : `${a}`).join('|')}`) || ''
|
|
84
84
|
}
|
|
85
85
|
const _parseArgs = (str) => {
|
|
86
86
|
return (str && str.length > 0 && str.replace(/\\\|/g, '###ESCAPESPLIT###').split('|').map(s => s.replace(/###ESCAPESPLIT###/g, '|').trim())) || []
|
|
@@ -553,6 +553,10 @@ const calculateWer = (str, pattern) => {
|
|
|
553
553
|
const botMessageWords = botMessage.split(' ').map(bm => bm.trim())
|
|
554
554
|
const utt = _prepareString(utterance)
|
|
555
555
|
|
|
556
|
+
// if no wildcards, just calculate WER
|
|
557
|
+
if (utt.indexOf('*') === -1) return speechScorer.wordErrorRate(botMessage, utt).toFixed(2)
|
|
558
|
+
|
|
559
|
+
// if there are wildcards, calculate WER for each wildcard part
|
|
556
560
|
const errors = []
|
|
557
561
|
for (let wildcardPart of utt.split('*')) {
|
|
558
562
|
let wer = 1
|
|
@@ -572,7 +576,7 @@ const calculateWer = (str, pattern) => {
|
|
|
572
576
|
}
|
|
573
577
|
}
|
|
574
578
|
if (_.isNil(subsetPhraseFound)) {
|
|
575
|
-
throw new Error('Word Error Asserter:
|
|
579
|
+
throw new Error('Word Error Asserter: When using wild cards, please make sure that the length of the asserter text is smaller than the bot message!')
|
|
576
580
|
}
|
|
577
581
|
errors.push(_getErrors(_getWords(wildcardPart), _getWords(subsetPhraseFound)))
|
|
578
582
|
}
|
|
@@ -586,6 +590,8 @@ const calculateWer = (str, pattern) => {
|
|
|
586
590
|
return (errCount / allCount).toFixed(2)
|
|
587
591
|
}
|
|
588
592
|
|
|
593
|
+
const toPercent = (s) => `${(s * 100).toFixed(0)}%`
|
|
594
|
+
|
|
589
595
|
module.exports = {
|
|
590
596
|
normalizeText,
|
|
591
597
|
splitStringInNonEmptyLines,
|
|
@@ -600,5 +606,6 @@ module.exports = {
|
|
|
600
606
|
validateSender,
|
|
601
607
|
validateConvo,
|
|
602
608
|
linesToScriptingMemories,
|
|
603
|
-
calculateWer
|
|
609
|
+
calculateWer,
|
|
610
|
+
toPercent
|
|
604
611
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// const _ = require('lodash')
|
|
2
2
|
const { BotiumError } = require('../../BotiumError')
|
|
3
|
-
const { calculateWer } = require('../../helper')
|
|
3
|
+
const { calculateWer, toPercent } = require('../../helper')
|
|
4
4
|
|
|
5
5
|
module.exports = class WerAsserter {
|
|
6
6
|
constructor (context, caps = {}) {
|
|
@@ -9,6 +9,55 @@ module.exports = class WerAsserter {
|
|
|
9
9
|
this.name = 'Word Error Rate Asserter'
|
|
10
10
|
}
|
|
11
11
|
|
|
12
|
+
assertNotConvoStep ({ convo, convoStep, args, botMsg }) {
|
|
13
|
+
if (!args || args.length < 1) {
|
|
14
|
+
return Promise.reject(new BotiumError(`${convoStep.stepTag}: ${this.name} - no argument given`,
|
|
15
|
+
{
|
|
16
|
+
type: 'asserter',
|
|
17
|
+
subtype: 'wrong parameters',
|
|
18
|
+
source: this.name,
|
|
19
|
+
cause: { args }
|
|
20
|
+
}
|
|
21
|
+
))
|
|
22
|
+
}
|
|
23
|
+
if (args.length > 2) {
|
|
24
|
+
return Promise.reject(new BotiumError(`${convoStep.stepTag}: ${this.name} - too many arguments "${args}"`,
|
|
25
|
+
{
|
|
26
|
+
type: 'asserter',
|
|
27
|
+
subtype: 'wrong parameters',
|
|
28
|
+
source: this.name,
|
|
29
|
+
cause: { args }
|
|
30
|
+
}
|
|
31
|
+
))
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const utterance = args[0]
|
|
35
|
+
const threshold = ([',', '.'].find(p => `${args[1]}`.includes(p)) ? parseFloat(args[1]) : parseInt(args[1]) / 100).toFixed(2)
|
|
36
|
+
|
|
37
|
+
const wer = calculateWer(botMsg.messageText, utterance)
|
|
38
|
+
|
|
39
|
+
if (wer < threshold) {
|
|
40
|
+
return Promise.reject(new BotiumError(
|
|
41
|
+
`${convoStep.stepTag}: Word Error Rate (${toPercent(wer)}) lower than accepted (${toPercent(threshold)})`,
|
|
42
|
+
{
|
|
43
|
+
type: 'asserter',
|
|
44
|
+
source: this.name,
|
|
45
|
+
context: {
|
|
46
|
+
params: {
|
|
47
|
+
args
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
cause: {
|
|
51
|
+
expected: `>=${toPercent(threshold)} (${utterance})`,
|
|
52
|
+
actual: `${toPercent(wer)} (${botMsg.messageText})`
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
))
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
return Promise.resolve()
|
|
59
|
+
}
|
|
60
|
+
|
|
12
61
|
assertConvoStep ({ convo, convoStep, args, botMsg }) {
|
|
13
62
|
if (!args || args.length < 1) {
|
|
14
63
|
return Promise.reject(new BotiumError(`${convoStep.stepTag}: ${this.name} - no argument given`,
|
|
@@ -37,10 +86,8 @@ module.exports = class WerAsserter {
|
|
|
37
86
|
const wer = calculateWer(botMsg.messageText, utterance)
|
|
38
87
|
|
|
39
88
|
if (wer > threshold) {
|
|
40
|
-
const _toPercent = (s) => `${(s * 100).toFixed(0)}%`
|
|
41
|
-
|
|
42
89
|
return Promise.reject(new BotiumError(
|
|
43
|
-
`${convoStep.stepTag}: Word Error Rate (${
|
|
90
|
+
`${convoStep.stepTag}: Word Error Rate (${toPercent(wer)}) higher than accepted (${toPercent(threshold)})`,
|
|
44
91
|
{
|
|
45
92
|
type: 'asserter',
|
|
46
93
|
source: this.name,
|
|
@@ -50,8 +97,8 @@ module.exports = class WerAsserter {
|
|
|
50
97
|
}
|
|
51
98
|
},
|
|
52
99
|
cause: {
|
|
53
|
-
expected: `<=${
|
|
54
|
-
actual: `${
|
|
100
|
+
expected: `<=${toPercent(threshold)} (${utterance})`,
|
|
101
|
+
actual: `${toPercent(wer)} (${botMsg.messageText})`
|
|
55
102
|
}
|
|
56
103
|
}
|
|
57
104
|
))
|
|
@@ -367,6 +367,33 @@ botText
|
|
|
367
367
|
|
|
368
368
|
#bot
|
|
369
369
|
botText
|
|
370
|
+
`
|
|
371
|
+
)
|
|
372
|
+
})
|
|
373
|
+
it('should decompile button asserter with numeric text', async function () {
|
|
374
|
+
const scriptingProvider = new ScriptingProvider(DefaultCapabilities)
|
|
375
|
+
await scriptingProvider.Build()
|
|
376
|
+
|
|
377
|
+
const convo = {
|
|
378
|
+
header: {
|
|
379
|
+
name: 'test convo'
|
|
380
|
+
},
|
|
381
|
+
conversation: [
|
|
382
|
+
{
|
|
383
|
+
sender: 'bot',
|
|
384
|
+
buttons: [
|
|
385
|
+
{ text: 123, payload: 'buttonpayload1' },
|
|
386
|
+
{ text: 456, payload: 'buttonpayload2' }
|
|
387
|
+
]
|
|
388
|
+
}
|
|
389
|
+
]
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
const script = scriptingProvider.Decompile([convo], 'SCRIPTING_FORMAT_TXT')
|
|
393
|
+
assert.equal(script, `test convo
|
|
394
|
+
|
|
395
|
+
#bot
|
|
396
|
+
BUTTONS 123|456
|
|
370
397
|
`
|
|
371
398
|
)
|
|
372
399
|
})
|
|
@@ -10,13 +10,28 @@ const HookUtils = require('../../src/helpers/HookUtils')
|
|
|
10
10
|
|
|
11
11
|
const myCapsSimpleRest = {
|
|
12
12
|
[Capabilities.CONTAINERMODE]: 'simplerest',
|
|
13
|
-
[Capabilities.SIMPLEREST_URL]: 'http://my-host.com/api/endpoint',
|
|
13
|
+
[Capabilities.SIMPLEREST_URL]: 'http://my-non-existing-botium-host.com/api/endpoint',
|
|
14
14
|
[Capabilities.SIMPLEREST_METHOD]: 'POST',
|
|
15
15
|
[Capabilities.SECURITY_ALLOW_UNSAFE]: false,
|
|
16
16
|
[Capabilities.SCRIPTING_ENABLE_MEMORY]: true,
|
|
17
17
|
[Capabilities.SIMPLEREST_RESPONSE_JSONPATH]: ['$']
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
+
const echoConnector = ({ queueBotSays }) => {
|
|
21
|
+
return {
|
|
22
|
+
UserSays (msg) {
|
|
23
|
+
const botMsg = { sender: 'bot', sourceData: msg.sourceData, messageText: `You said: "${msg.messageText}"` }
|
|
24
|
+
queueBotSays(botMsg)
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const myCapsEcho = {
|
|
30
|
+
[Capabilities.CONTAINERMODE]: echoConnector,
|
|
31
|
+
[Capabilities.SECURITY_ALLOW_UNSAFE]: false,
|
|
32
|
+
[Capabilities.SCRIPTING_ENABLE_MEMORY]: true
|
|
33
|
+
}
|
|
34
|
+
|
|
20
35
|
const _getSimpleRestCaps = (caps) => {
|
|
21
36
|
return Object.assign(
|
|
22
37
|
{},
|
|
@@ -83,20 +98,15 @@ describe('security.allowUnsafe', function () {
|
|
|
83
98
|
|
|
84
99
|
describe('scripting memory', function () {
|
|
85
100
|
it('should not throw security error for using inline function', async function () {
|
|
86
|
-
const driver = new BotDriver(
|
|
101
|
+
const driver = new BotDriver(myCapsEcho)
|
|
87
102
|
const compiler = driver.BuildCompiler()
|
|
88
103
|
const container = await driver.Build()
|
|
89
104
|
await container.Start()
|
|
90
105
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
await compiler.convos[0].Run(container)
|
|
94
|
-
assert.fail('should have failed')
|
|
95
|
-
} catch (err) {
|
|
96
|
-
assert.isFalse(err.message.indexOf('Security Error. Using unsafe scripting memory function $func is not allowed') >= 0)
|
|
97
|
-
}
|
|
106
|
+
compiler.ReadScript(path.resolve(__dirname, 'convos'), 'withscriptingmemoryfunction.convo.txt')
|
|
107
|
+
await compiler.convos[0].Run(container)
|
|
98
108
|
await container.Clean()
|
|
99
|
-
})
|
|
109
|
+
}).timeout(50000)
|
|
100
110
|
})
|
|
101
111
|
|
|
102
112
|
describe('simple rest, scripting memory', function () {
|