momentic 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +1 -0
- package/dist/index.js +2858 -0
- package/package.json +39 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,2858 @@
|
|
|
1
|
+
import * as __WEBPACK_EXTERNAL_MODULE_playwright__ from "playwright";
|
|
2
|
+
import * as __WEBPACK_EXTERNAL_MODULE_zod__ from "zod";
|
|
3
|
+
import * as __WEBPACK_EXTERNAL_MODULE_dedent__ from "dedent";
|
|
4
|
+
import * as __WEBPACK_EXTERNAL_MODULE_diff_lines_24b6f423__ from "diff-lines";
|
|
5
|
+
/******/ var __webpack_modules__ = ({
|
|
6
|
+
|
|
7
|
+
/***/ 909:
|
|
8
|
+
/***/ (function(__unused_webpack_module, exports) {
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
var __assign = (this && this.__assign) || function () {
|
|
12
|
+
__assign = Object.assign || function(t) {
|
|
13
|
+
for (var s, i = 1, n = arguments.length; i < n; i++) {
|
|
14
|
+
s = arguments[i];
|
|
15
|
+
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
|
|
16
|
+
t[p] = s[p];
|
|
17
|
+
}
|
|
18
|
+
return t;
|
|
19
|
+
};
|
|
20
|
+
return __assign.apply(this, arguments);
|
|
21
|
+
};
|
|
22
|
+
Object.defineProperty(exports, "__esModule", ({ value: true }));
|
|
23
|
+
exports.isValidCron = void 0;
|
|
24
|
+
// This comes from the fact that parseInt trims characters coming
|
|
25
|
+
// after digits and consider it a valid int, so `1*` becomes `1`.
|
|
26
|
+
var safeParseInt = function (value) {
|
|
27
|
+
if (/^\d+$/.test(value)) {
|
|
28
|
+
return Number(value);
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
return NaN;
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
var isWildcard = function (value) {
|
|
35
|
+
return value === '*';
|
|
36
|
+
};
|
|
37
|
+
var isQuestionMark = function (value) {
|
|
38
|
+
return value === '?';
|
|
39
|
+
};
|
|
40
|
+
var isInRange = function (value, start, stop) {
|
|
41
|
+
return value >= start && value <= stop;
|
|
42
|
+
};
|
|
43
|
+
var isValidRange = function (value, start, stop) {
|
|
44
|
+
var sides = value.split('-');
|
|
45
|
+
switch (sides.length) {
|
|
46
|
+
case 1:
|
|
47
|
+
return isWildcard(value) || isInRange(safeParseInt(value), start, stop);
|
|
48
|
+
case 2:
|
|
49
|
+
var _a = sides.map(function (side) { return safeParseInt(side); }), small = _a[0], big = _a[1];
|
|
50
|
+
return small <= big && isInRange(small, start, stop) && isInRange(big, start, stop);
|
|
51
|
+
default:
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
var isValidStep = function (value) {
|
|
56
|
+
return value === undefined || (value.search(/[^\d]/) === -1 && safeParseInt(value) > 0);
|
|
57
|
+
};
|
|
58
|
+
var validateForRange = function (value, start, stop) {
|
|
59
|
+
if (value.search(/[^\d-,\/*]/) !== -1) {
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
var list = value.split(',');
|
|
63
|
+
return list.every(function (condition) {
|
|
64
|
+
var splits = condition.split('/');
|
|
65
|
+
// Prevents `*/ * * * *` from being accepted.
|
|
66
|
+
if (condition.trim().endsWith('/')) {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
// Prevents `*/*/* * * * *` from being accepted
|
|
70
|
+
if (splits.length > 2) {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
// If we don't have a `/`, right will be undefined which is considered a valid step if we don't a `/`.
|
|
74
|
+
var left = splits[0], right = splits[1];
|
|
75
|
+
return isValidRange(left, start, stop) && isValidStep(right);
|
|
76
|
+
});
|
|
77
|
+
};
|
|
78
|
+
var hasValidSeconds = function (seconds) {
|
|
79
|
+
return validateForRange(seconds, 0, 59);
|
|
80
|
+
};
|
|
81
|
+
var hasValidMinutes = function (minutes) {
|
|
82
|
+
return validateForRange(minutes, 0, 59);
|
|
83
|
+
};
|
|
84
|
+
var hasValidHours = function (hours) {
|
|
85
|
+
return validateForRange(hours, 0, 23);
|
|
86
|
+
};
|
|
87
|
+
var hasValidDays = function (days, allowBlankDay) {
|
|
88
|
+
return (allowBlankDay && isQuestionMark(days)) || validateForRange(days, 1, 31);
|
|
89
|
+
};
|
|
90
|
+
var monthAlias = {
|
|
91
|
+
jan: '1',
|
|
92
|
+
feb: '2',
|
|
93
|
+
mar: '3',
|
|
94
|
+
apr: '4',
|
|
95
|
+
may: '5',
|
|
96
|
+
jun: '6',
|
|
97
|
+
jul: '7',
|
|
98
|
+
aug: '8',
|
|
99
|
+
sep: '9',
|
|
100
|
+
oct: '10',
|
|
101
|
+
nov: '11',
|
|
102
|
+
dec: '12'
|
|
103
|
+
};
|
|
104
|
+
var hasValidMonths = function (months, alias) {
|
|
105
|
+
// Prevents alias to be used as steps
|
|
106
|
+
if (months.search(/\/[a-zA-Z]/) !== -1) {
|
|
107
|
+
return false;
|
|
108
|
+
}
|
|
109
|
+
if (alias) {
|
|
110
|
+
var remappedMonths = months.toLowerCase().replace(/[a-z]{3}/g, function (match) {
|
|
111
|
+
return monthAlias[match] === undefined ? match : monthAlias[match];
|
|
112
|
+
});
|
|
113
|
+
// If any invalid alias was used, it won't pass the other checks as there will be non-numeric values in the months
|
|
114
|
+
return validateForRange(remappedMonths, 1, 12);
|
|
115
|
+
}
|
|
116
|
+
return validateForRange(months, 1, 12);
|
|
117
|
+
};
|
|
118
|
+
var weekdaysAlias = {
|
|
119
|
+
sun: '0',
|
|
120
|
+
mon: '1',
|
|
121
|
+
tue: '2',
|
|
122
|
+
wed: '3',
|
|
123
|
+
thu: '4',
|
|
124
|
+
fri: '5',
|
|
125
|
+
sat: '6'
|
|
126
|
+
};
|
|
127
|
+
var hasValidWeekdays = function (weekdays, alias, allowBlankDay, allowSevenAsSunday) {
|
|
128
|
+
// If there is a question mark, checks if the allowBlankDay flag is set
|
|
129
|
+
if (allowBlankDay && isQuestionMark(weekdays)) {
|
|
130
|
+
return true;
|
|
131
|
+
}
|
|
132
|
+
else if (!allowBlankDay && isQuestionMark(weekdays)) {
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
// Prevents alias to be used as steps
|
|
136
|
+
if (weekdays.search(/\/[a-zA-Z]/) !== -1) {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
if (alias) {
|
|
140
|
+
var remappedWeekdays = weekdays.toLowerCase().replace(/[a-z]{3}/g, function (match) {
|
|
141
|
+
return weekdaysAlias[match] === undefined ? match : weekdaysAlias[match];
|
|
142
|
+
});
|
|
143
|
+
// If any invalid alias was used, it won't pass the other checks as there will be non-numeric values in the weekdays
|
|
144
|
+
return validateForRange(remappedWeekdays, 0, allowSevenAsSunday ? 7 : 6);
|
|
145
|
+
}
|
|
146
|
+
return validateForRange(weekdays, 0, allowSevenAsSunday ? 7 : 6);
|
|
147
|
+
};
|
|
148
|
+
var hasCompatibleDayFormat = function (days, weekdays, allowBlankDay) {
|
|
149
|
+
return !(allowBlankDay && isQuestionMark(days) && isQuestionMark(weekdays));
|
|
150
|
+
};
|
|
151
|
+
var split = function (cron) {
|
|
152
|
+
return cron.trim().split(/\s+/);
|
|
153
|
+
};
|
|
154
|
+
var defaultOptions = {
|
|
155
|
+
alias: false,
|
|
156
|
+
seconds: false,
|
|
157
|
+
allowBlankDay: false,
|
|
158
|
+
allowSevenAsSunday: false
|
|
159
|
+
};
|
|
160
|
+
exports.isValidCron = function (cron, options) {
|
|
161
|
+
options = __assign(__assign({}, defaultOptions), options);
|
|
162
|
+
var splits = split(cron);
|
|
163
|
+
if (splits.length > (options.seconds ? 6 : 5) || splits.length < 5) {
|
|
164
|
+
return false;
|
|
165
|
+
}
|
|
166
|
+
var checks = [];
|
|
167
|
+
if (splits.length === 6) {
|
|
168
|
+
var seconds = splits.shift();
|
|
169
|
+
if (seconds) {
|
|
170
|
+
checks.push(hasValidSeconds(seconds));
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
// We could only check the steps gradually and return false on the first invalid block,
|
|
174
|
+
// However, this won't have any performance impact so why bother for now.
|
|
175
|
+
var minutes = splits[0], hours = splits[1], days = splits[2], months = splits[3], weekdays = splits[4];
|
|
176
|
+
checks.push(hasValidMinutes(minutes));
|
|
177
|
+
checks.push(hasValidHours(hours));
|
|
178
|
+
checks.push(hasValidDays(days, options.allowBlankDay));
|
|
179
|
+
checks.push(hasValidMonths(months, options.alias));
|
|
180
|
+
checks.push(hasValidWeekdays(weekdays, options.alias, options.allowBlankDay, options.allowSevenAsSunday));
|
|
181
|
+
checks.push(hasCompatibleDayFormat(days, weekdays, options.allowBlankDay));
|
|
182
|
+
return checks.every(Boolean);
|
|
183
|
+
};
|
|
184
|
+
//# sourceMappingURL=index.js.map
|
|
185
|
+
|
|
186
|
+
/***/ }),
|
|
187
|
+
|
|
188
|
+
/***/ 62:
|
|
189
|
+
/***/ ((module) => {
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
module.exports = function (fetch, defaults) {
|
|
194
|
+
defaults = defaults || {};
|
|
195
|
+
if (typeof fetch !== 'function') {
|
|
196
|
+
throw new ArgumentError('fetch must be a function');
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
if (typeof defaults !== 'object') {
|
|
200
|
+
throw new ArgumentError('defaults must be an object');
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
if (defaults.retries !== undefined && !isPositiveInteger(defaults.retries)) {
|
|
204
|
+
throw new ArgumentError('retries must be a positive integer');
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
if (defaults.retryDelay !== undefined && !isPositiveInteger(defaults.retryDelay) && typeof defaults.retryDelay !== 'function') {
|
|
208
|
+
throw new ArgumentError('retryDelay must be a positive integer or a function returning a positive integer');
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if (defaults.retryOn !== undefined && !Array.isArray(defaults.retryOn) && typeof defaults.retryOn !== 'function') {
|
|
212
|
+
throw new ArgumentError('retryOn property expects an array or function');
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
var baseDefaults = {
|
|
216
|
+
retries: 3,
|
|
217
|
+
retryDelay: 1000,
|
|
218
|
+
retryOn: [],
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
defaults = Object.assign(baseDefaults, defaults);
|
|
222
|
+
|
|
223
|
+
return function fetchRetry(input, init) {
|
|
224
|
+
var retries = defaults.retries;
|
|
225
|
+
var retryDelay = defaults.retryDelay;
|
|
226
|
+
var retryOn = defaults.retryOn;
|
|
227
|
+
|
|
228
|
+
if (init && init.retries !== undefined) {
|
|
229
|
+
if (isPositiveInteger(init.retries)) {
|
|
230
|
+
retries = init.retries;
|
|
231
|
+
} else {
|
|
232
|
+
throw new ArgumentError('retries must be a positive integer');
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
if (init && init.retryDelay !== undefined) {
|
|
237
|
+
if (isPositiveInteger(init.retryDelay) || (typeof init.retryDelay === 'function')) {
|
|
238
|
+
retryDelay = init.retryDelay;
|
|
239
|
+
} else {
|
|
240
|
+
throw new ArgumentError('retryDelay must be a positive integer or a function returning a positive integer');
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
if (init && init.retryOn) {
|
|
245
|
+
if (Array.isArray(init.retryOn) || (typeof init.retryOn === 'function')) {
|
|
246
|
+
retryOn = init.retryOn;
|
|
247
|
+
} else {
|
|
248
|
+
throw new ArgumentError('retryOn property expects an array or function');
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// eslint-disable-next-line no-undef
|
|
253
|
+
return new Promise(function (resolve, reject) {
|
|
254
|
+
var wrappedFetch = function (attempt) {
|
|
255
|
+
// As of node 18, this is no longer needed since node comes with native support for fetch:
|
|
256
|
+
/* istanbul ignore next */
|
|
257
|
+
var _input =
|
|
258
|
+
typeof Request !== 'undefined' && input instanceof Request
|
|
259
|
+
? input.clone()
|
|
260
|
+
: input;
|
|
261
|
+
fetch(_input, init)
|
|
262
|
+
.then(function (response) {
|
|
263
|
+
if (Array.isArray(retryOn) && retryOn.indexOf(response.status) === -1) {
|
|
264
|
+
resolve(response);
|
|
265
|
+
} else if (typeof retryOn === 'function') {
|
|
266
|
+
try {
|
|
267
|
+
// eslint-disable-next-line no-undef
|
|
268
|
+
return Promise.resolve(retryOn(attempt, null, response))
|
|
269
|
+
.then(function (retryOnResponse) {
|
|
270
|
+
if(retryOnResponse) {
|
|
271
|
+
retry(attempt, null, response);
|
|
272
|
+
} else {
|
|
273
|
+
resolve(response);
|
|
274
|
+
}
|
|
275
|
+
}).catch(reject);
|
|
276
|
+
} catch (error) {
|
|
277
|
+
reject(error);
|
|
278
|
+
}
|
|
279
|
+
} else {
|
|
280
|
+
if (attempt < retries) {
|
|
281
|
+
retry(attempt, null, response);
|
|
282
|
+
} else {
|
|
283
|
+
resolve(response);
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
})
|
|
287
|
+
.catch(function (error) {
|
|
288
|
+
if (typeof retryOn === 'function') {
|
|
289
|
+
try {
|
|
290
|
+
// eslint-disable-next-line no-undef
|
|
291
|
+
Promise.resolve(retryOn(attempt, error, null))
|
|
292
|
+
.then(function (retryOnResponse) {
|
|
293
|
+
if(retryOnResponse) {
|
|
294
|
+
retry(attempt, error, null);
|
|
295
|
+
} else {
|
|
296
|
+
reject(error);
|
|
297
|
+
}
|
|
298
|
+
})
|
|
299
|
+
.catch(function(error) {
|
|
300
|
+
reject(error);
|
|
301
|
+
});
|
|
302
|
+
} catch(error) {
|
|
303
|
+
reject(error);
|
|
304
|
+
}
|
|
305
|
+
} else if (attempt < retries) {
|
|
306
|
+
retry(attempt, error, null);
|
|
307
|
+
} else {
|
|
308
|
+
reject(error);
|
|
309
|
+
}
|
|
310
|
+
});
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
function retry(attempt, error, response) {
|
|
314
|
+
var delay = (typeof retryDelay === 'function') ?
|
|
315
|
+
retryDelay(attempt, error, response) : retryDelay;
|
|
316
|
+
setTimeout(function () {
|
|
317
|
+
wrappedFetch(++attempt);
|
|
318
|
+
}, delay);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
wrappedFetch(0);
|
|
322
|
+
});
|
|
323
|
+
};
|
|
324
|
+
};
|
|
325
|
+
|
|
326
|
+
function isPositiveInteger(value) {
|
|
327
|
+
return Number.isInteger(value) && value >= 0;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
function ArgumentError(message) {
|
|
331
|
+
this.name = 'ArgumentError';
|
|
332
|
+
this.message = message;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
/***/ })
|
|
337
|
+
|
|
338
|
+
/******/ });
|
|
339
|
+
/************************************************************************/
|
|
340
|
+
/******/ // The module cache
|
|
341
|
+
/******/ var __webpack_module_cache__ = {};
|
|
342
|
+
/******/
|
|
343
|
+
/******/ // The require function
|
|
344
|
+
/******/ function __nccwpck_require__(moduleId) {
|
|
345
|
+
/******/ // Check if module is in cache
|
|
346
|
+
/******/ var cachedModule = __webpack_module_cache__[moduleId];
|
|
347
|
+
/******/ if (cachedModule !== undefined) {
|
|
348
|
+
/******/ return cachedModule.exports;
|
|
349
|
+
/******/ }
|
|
350
|
+
/******/ // Create a new module (and put it into the cache)
|
|
351
|
+
/******/ var module = __webpack_module_cache__[moduleId] = {
|
|
352
|
+
/******/ // no module.id needed
|
|
353
|
+
/******/ // no module.loaded needed
|
|
354
|
+
/******/ exports: {}
|
|
355
|
+
/******/ };
|
|
356
|
+
/******/
|
|
357
|
+
/******/ // Execute the module function
|
|
358
|
+
/******/ var threw = true;
|
|
359
|
+
/******/ try {
|
|
360
|
+
/******/ __webpack_modules__[moduleId].call(module.exports, module, module.exports, __nccwpck_require__);
|
|
361
|
+
/******/ threw = false;
|
|
362
|
+
/******/ } finally {
|
|
363
|
+
/******/ if(threw) delete __webpack_module_cache__[moduleId];
|
|
364
|
+
/******/ }
|
|
365
|
+
/******/
|
|
366
|
+
/******/ // Return the exports of the module
|
|
367
|
+
/******/ return module.exports;
|
|
368
|
+
/******/ }
|
|
369
|
+
/******/
|
|
370
|
+
/************************************************************************/
|
|
371
|
+
/******/ /* webpack/runtime/compat get default export */
|
|
372
|
+
/******/ (() => {
|
|
373
|
+
/******/ // getDefaultExport function for compatibility with non-harmony modules
|
|
374
|
+
/******/ __nccwpck_require__.n = (module) => {
|
|
375
|
+
/******/ var getter = module && module.__esModule ?
|
|
376
|
+
/******/ () => (module['default']) :
|
|
377
|
+
/******/ () => (module);
|
|
378
|
+
/******/ __nccwpck_require__.d(getter, { a: getter });
|
|
379
|
+
/******/ return getter;
|
|
380
|
+
/******/ };
|
|
381
|
+
/******/ })();
|
|
382
|
+
/******/
|
|
383
|
+
/******/ /* webpack/runtime/define property getters */
|
|
384
|
+
/******/ (() => {
|
|
385
|
+
/******/ // define getter functions for harmony exports
|
|
386
|
+
/******/ __nccwpck_require__.d = (exports, definition) => {
|
|
387
|
+
/******/ for(var key in definition) {
|
|
388
|
+
/******/ if(__nccwpck_require__.o(definition, key) && !__nccwpck_require__.o(exports, key)) {
|
|
389
|
+
/******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });
|
|
390
|
+
/******/ }
|
|
391
|
+
/******/ }
|
|
392
|
+
/******/ };
|
|
393
|
+
/******/ })();
|
|
394
|
+
/******/
|
|
395
|
+
/******/ /* webpack/runtime/hasOwnProperty shorthand */
|
|
396
|
+
/******/ (() => {
|
|
397
|
+
/******/ __nccwpck_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))
|
|
398
|
+
/******/ })();
|
|
399
|
+
/******/
|
|
400
|
+
/******/ /* webpack/runtime/compat */
|
|
401
|
+
/******/
|
|
402
|
+
/******/ if (typeof __nccwpck_require__ !== 'undefined') __nccwpck_require__.ab = new URL('.', import.meta.url).pathname.slice(import.meta.url.match(/^file:\/\/\/\w:/) ? 1 : 0, -1) + "/";
|
|
403
|
+
/******/
|
|
404
|
+
/************************************************************************/
|
|
405
|
+
var __webpack_exports__ = {};
|
|
406
|
+
// This entry need to be wrapped in an IIFE because it need to be isolated against other modules in the chunk.
|
|
407
|
+
(() => {
|
|
408
|
+
|
|
409
|
+
// EXPORTS
|
|
410
|
+
__nccwpck_require__.d(__webpack_exports__, {
|
|
411
|
+
"_w": () => (/* reexport */ APIGenerator),
|
|
412
|
+
"Yt": () => (/* reexport */ AgentController),
|
|
413
|
+
"DE": () => (/* reexport */ ChromeBrowser)
|
|
414
|
+
});
|
|
415
|
+
|
|
416
|
+
;// CONCATENATED MODULE: external "playwright"
|
|
417
|
+
var x = y => { var x = {}; __nccwpck_require__.d(x, y); return x; }
|
|
418
|
+
var y = x => () => x
|
|
419
|
+
const external_playwright_namespaceObject = x({ ["chromium"]: () => __WEBPACK_EXTERNAL_MODULE_playwright__.chromium, ["devices"]: () => __WEBPACK_EXTERNAL_MODULE_playwright__.devices });
|
|
420
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/utils/url.ts
|
|
421
|
+
// Returns true if 2 urls are different, ignoring differences in query params.
|
|
422
|
+
const urlChanged = (url1, url2) => {
|
|
423
|
+
const { hostname, pathname } = new URL(url1);
|
|
424
|
+
const { hostname: hostname2, pathname: pathname2 } = new URL(url2);
|
|
425
|
+
return hostname !== hostname2 || pathname !== pathname2;
|
|
426
|
+
};
|
|
427
|
+
|
|
428
|
+
;// CONCATENATED MODULE: ../../packages/types/src/agents.ts
|
|
429
|
+
var AgentType;
|
|
430
|
+
(function (AgentType) {
|
|
431
|
+
AgentType["A11Y"] = "a11y";
|
|
432
|
+
AgentType["HTML"] = "html";
|
|
433
|
+
})(AgentType || (AgentType = {}));
|
|
434
|
+
|
|
435
|
+
;// CONCATENATED MODULE: external "zod"
|
|
436
|
+
var external_zod_x = y => { var x = {}; __nccwpck_require__.d(x, y); return x; }
|
|
437
|
+
var external_zod_y = x => () => x
|
|
438
|
+
const external_zod_namespaceObject = external_zod_x({ ["array"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.array, ["boolean"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.boolean, ["coerce"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.coerce, ["discriminatedUnion"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.discriminatedUnion, ["literal"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.literal, ["nativeEnum"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.nativeEnum, ["null"]: () => __WEBPACK_EXTERNAL_MODULE_zod__["null"], ["number"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.number, ["object"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.object, ["string"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.string, ["union"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.union, ["z"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.z });
|
|
439
|
+
;// CONCATENATED MODULE: ../../packages/types/src/a11y-targets.ts
|
|
440
|
+
|
|
441
|
+
const A11yTargetWithCacheSchema = external_zod_namespaceObject.object({
|
|
442
|
+
// a11y ID
|
|
443
|
+
id: external_zod_namespaceObject.number().int(),
|
|
444
|
+
// additional metadata stored after the action is executed
|
|
445
|
+
// to assist in re-execution
|
|
446
|
+
role: external_zod_namespaceObject.string().optional(),
|
|
447
|
+
name: external_zod_namespaceObject.string().optional(),
|
|
448
|
+
content: external_zod_namespaceObject.string().optional(),
|
|
449
|
+
pathFromRoot: external_zod_namespaceObject.string().optional(),
|
|
450
|
+
serializedForm: external_zod_namespaceObject.string().optional(),
|
|
451
|
+
});
|
|
452
|
+
|
|
453
|
+
;// CONCATENATED MODULE: ../../packages/types/src/assertions.ts
|
|
454
|
+
|
|
455
|
+
// schema that the LLM outputs when evaluating assertions
|
|
456
|
+
const LLMAssertionEvalSchema = external_zod_namespaceObject.z.object({
|
|
457
|
+
thoughts: external_zod_namespaceObject.z.string(),
|
|
458
|
+
result: external_zod_namespaceObject.z.boolean(),
|
|
459
|
+
relevantElements: external_zod_namespaceObject.z.array(external_zod_namespaceObject.z.number()).optional(),
|
|
460
|
+
});
|
|
461
|
+
|
|
462
|
+
;// CONCATENATED MODULE: ../../packages/types/src/errors.ts
|
|
463
|
+
// https://stackoverflow.com/questions/42754270/re-throwing-exception-in-nodejs-and-not-losing-stack-trace
|
|
464
|
+
class BrowserExecutionError extends Error {
|
|
465
|
+
constructor(message, options = {}) {
|
|
466
|
+
super(message, options);
|
|
467
|
+
this.name = "BrowserExecutionError";
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
class errors_CommandParseError extends Error {
|
|
471
|
+
constructor(message, options = {}) {
|
|
472
|
+
super(message, options);
|
|
473
|
+
this.name = "CommandParseError";
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
class EmptyA11yTreeError extends Error {
|
|
477
|
+
constructor(options = {}) {
|
|
478
|
+
super("Got empty a11y tree", options);
|
|
479
|
+
this.name = "EmptyA11yTreeError";
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
;// CONCATENATED MODULE: external "dedent"
|
|
484
|
+
var external_dedent_x = y => { var x = {}; __nccwpck_require__.d(x, y); return x; }
|
|
485
|
+
var external_dedent_y = x => () => x
|
|
486
|
+
const external_dedent_namespaceObject = external_dedent_x({ ["default"]: () => __WEBPACK_EXTERNAL_MODULE_dedent__["default"] });
|
|
487
|
+
;// CONCATENATED MODULE: ../../packages/types/src/preset.ts
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
var preset_PresetCommandType;
|
|
492
|
+
(function (PresetCommandType) {
|
|
493
|
+
PresetCommandType["AI_ASSERTION"] = "AI_ASSERTION";
|
|
494
|
+
PresetCommandType["CLICK"] = "CLICK";
|
|
495
|
+
PresetCommandType["SELECT_OPTION"] = "SELECT_OPTION";
|
|
496
|
+
PresetCommandType["TYPE"] = "TYPE";
|
|
497
|
+
PresetCommandType["PRESS"] = "PRESS";
|
|
498
|
+
PresetCommandType["NAVIGATE"] = "NAVIGATE";
|
|
499
|
+
PresetCommandType["SCROLL_UP"] = "SCROLL_UP";
|
|
500
|
+
PresetCommandType["SCROLL_DOWN"] = "SCROLL_DOWN";
|
|
501
|
+
PresetCommandType["GO_BACK"] = "GO_BACK";
|
|
502
|
+
PresetCommandType["GO_FORWARD"] = "GO_FORWARD";
|
|
503
|
+
PresetCommandType["WAIT"] = "WAIT";
|
|
504
|
+
PresetCommandType["REFRESH"] = "REFRESH";
|
|
505
|
+
})(preset_PresetCommandType || (preset_PresetCommandType = {}));
|
|
506
|
+
const ElementDescriptorSchema = external_zod_namespaceObject.object({
|
|
507
|
+
// natural language passed to LLM
|
|
508
|
+
elementDescriptor: external_zod_namespaceObject.string(),
|
|
509
|
+
// Cached A11y target - when a user creates a preset action, this will not exist
|
|
510
|
+
a11yData: A11yTargetWithCacheSchema.optional(),
|
|
511
|
+
});
|
|
512
|
+
const CommonCommandSchema = external_zod_namespaceObject.object({
|
|
513
|
+
// If the command is suggested by AI, why it did so
|
|
514
|
+
thoughts: external_zod_namespaceObject.string().optional(),
|
|
515
|
+
});
|
|
516
|
+
const NavigateCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
517
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.NAVIGATE),
|
|
518
|
+
url: external_zod_namespaceObject.string(),
|
|
519
|
+
})).describe("NAVIGATE <url> - Go to the specified url");
|
|
520
|
+
const ScrollUpCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
521
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.SCROLL_UP),
|
|
522
|
+
})).describe("SCROLL_UP - Scroll up one page");
|
|
523
|
+
const ScrollDownCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
524
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.SCROLL_DOWN),
|
|
525
|
+
})).describe("SCROLL_DOWN - Scroll down one page");
|
|
526
|
+
const WaitCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
527
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.WAIT),
|
|
528
|
+
delay: external_zod_namespaceObject.number(), // seconds
|
|
529
|
+
}));
|
|
530
|
+
const RefreshCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
531
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.REFRESH),
|
|
532
|
+
}));
|
|
533
|
+
const GoBackCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
534
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.GO_BACK),
|
|
535
|
+
}));
|
|
536
|
+
const GoForwardCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
537
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.GO_FORWARD),
|
|
538
|
+
}));
|
|
539
|
+
const ClickCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
540
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.CLICK),
|
|
541
|
+
target: ElementDescriptorSchema,
|
|
542
|
+
doubleClick: external_zod_namespaceObject.boolean().default(false),
|
|
543
|
+
rightClick: external_zod_namespaceObject.boolean().default(false),
|
|
544
|
+
})).describe(external_dedent_namespaceObject["default"] `CLICK <id> - click on the element that has the specified id.
|
|
545
|
+
You are NOT allowed to click on disabled, hidden or StaticText elements.
|
|
546
|
+
Only click on elements on the Current Page.
|
|
547
|
+
Only click on elements with the following tag names: button, input, link, image, generic.
|
|
548
|
+
`.replace("\n", " "));
|
|
549
|
+
const SelectOptionCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
550
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.SELECT_OPTION),
|
|
551
|
+
target: ElementDescriptorSchema,
|
|
552
|
+
option: external_zod_namespaceObject.string(),
|
|
553
|
+
})).describe(
|
|
554
|
+
// TODO: if we move to a non-mutative way of selecting elements (e.g. by selector), we should update this description
|
|
555
|
+
`SELECT_OPTION <id> "<option>" - select the specified item from the select with the specified id. The item should exist on the page. Use the name of the item instead of the id. Make sure to include quotes around the option.`);
|
|
556
|
+
// Assertions must be user specified today, so no common command schema
|
|
557
|
+
const AIAssertionCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
558
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.AI_ASSERTION),
|
|
559
|
+
assertion: external_zod_namespaceObject.string(),
|
|
560
|
+
useVision: external_zod_namespaceObject.boolean().default(false),
|
|
561
|
+
disableCache: external_zod_namespaceObject.boolean().default(false),
|
|
562
|
+
}));
|
|
563
|
+
const TypeOptionsSchema = external_zod_namespaceObject.object({
|
|
564
|
+
clearContent: external_zod_namespaceObject.boolean().default(true),
|
|
565
|
+
pressKeysSequentially: external_zod_namespaceObject.boolean().default(false),
|
|
566
|
+
});
|
|
567
|
+
const TypeCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
568
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.TYPE),
|
|
569
|
+
target: ElementDescriptorSchema,
|
|
570
|
+
value: external_zod_namespaceObject.string(),
|
|
571
|
+
pressEnter: external_zod_namespaceObject.boolean().default(false),
|
|
572
|
+
}))
|
|
573
|
+
.merge(TypeOptionsSchema)
|
|
574
|
+
.describe(`TYPE <id> "<text>" - type the specified text into the input with the specified id. The text should be specified by the user - do not use text from the EXAMPLES or generate text yourself. Make sure to include quotes around the text.`);
|
|
575
|
+
// https://playwright.dev/docs/api/class-locator#locator-press
|
|
576
|
+
const PressCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
577
|
+
type: external_zod_namespaceObject.literal(preset_PresetCommandType.PRESS),
|
|
578
|
+
value: external_zod_namespaceObject.string(),
|
|
579
|
+
})).describe(`PRESS <key> - press the specified key, such as "ArrowLeft", "Enter", or "a". You must specify at least one key.`);
|
|
580
|
+
const PresetCommandSchema = external_zod_namespaceObject.discriminatedUnion("type", [
|
|
581
|
+
ClickCommandSchema,
|
|
582
|
+
GoBackCommandSchema,
|
|
583
|
+
GoForwardCommandSchema,
|
|
584
|
+
NavigateCommandSchema,
|
|
585
|
+
PressCommandSchema,
|
|
586
|
+
RefreshCommandSchema,
|
|
587
|
+
ScrollDownCommandSchema,
|
|
588
|
+
ScrollUpCommandSchema,
|
|
589
|
+
SelectOptionCommandSchema,
|
|
590
|
+
TypeCommandSchema,
|
|
591
|
+
AIAssertionCommandSchema,
|
|
592
|
+
WaitCommandSchema,
|
|
593
|
+
]);
|
|
594
|
+
// A subset of PresetCommandSchema - these are commands that AI can suggest
|
|
595
|
+
// As we add more training cases and the AI becomes more capable, we can expand this.
|
|
596
|
+
const AISuggestiblePresetCommandSchema = external_zod_namespaceObject.discriminatedUnion("type", [
|
|
597
|
+
ClickCommandSchema,
|
|
598
|
+
TypeCommandSchema,
|
|
599
|
+
PressCommandSchema,
|
|
600
|
+
SelectOptionCommandSchema,
|
|
601
|
+
NavigateCommandSchema,
|
|
602
|
+
ScrollDownCommandSchema,
|
|
603
|
+
ScrollUpCommandSchema,
|
|
604
|
+
]);
|
|
605
|
+
const getDefaultPresetCommand = (type) => {
|
|
606
|
+
switch (type) {
|
|
607
|
+
case preset_PresetCommandType.NAVIGATE:
|
|
608
|
+
return {
|
|
609
|
+
type: preset_PresetCommandType.NAVIGATE,
|
|
610
|
+
url: "",
|
|
611
|
+
};
|
|
612
|
+
case preset_PresetCommandType.GO_BACK:
|
|
613
|
+
case preset_PresetCommandType.GO_FORWARD:
|
|
614
|
+
case preset_PresetCommandType.SCROLL_DOWN:
|
|
615
|
+
case preset_PresetCommandType.SCROLL_UP:
|
|
616
|
+
case preset_PresetCommandType.REFRESH:
|
|
617
|
+
return { type };
|
|
618
|
+
case preset_PresetCommandType.WAIT:
|
|
619
|
+
return {
|
|
620
|
+
type,
|
|
621
|
+
delay: 1,
|
|
622
|
+
};
|
|
623
|
+
case preset_PresetCommandType.CLICK:
|
|
624
|
+
return {
|
|
625
|
+
type,
|
|
626
|
+
target: {
|
|
627
|
+
elementDescriptor: "",
|
|
628
|
+
},
|
|
629
|
+
doubleClick: false,
|
|
630
|
+
rightClick: false,
|
|
631
|
+
};
|
|
632
|
+
case preset_PresetCommandType.TYPE:
|
|
633
|
+
return {
|
|
634
|
+
type,
|
|
635
|
+
target: {
|
|
636
|
+
elementDescriptor: "",
|
|
637
|
+
},
|
|
638
|
+
value: "",
|
|
639
|
+
clearContent: true,
|
|
640
|
+
pressEnter: false,
|
|
641
|
+
pressKeysSequentially: false,
|
|
642
|
+
};
|
|
643
|
+
case preset_PresetCommandType.PRESS:
|
|
644
|
+
return {
|
|
645
|
+
type,
|
|
646
|
+
value: "",
|
|
647
|
+
};
|
|
648
|
+
case preset_PresetCommandType.SELECT_OPTION:
|
|
649
|
+
return {
|
|
650
|
+
type,
|
|
651
|
+
target: {
|
|
652
|
+
elementDescriptor: "",
|
|
653
|
+
},
|
|
654
|
+
option: "",
|
|
655
|
+
};
|
|
656
|
+
case preset_PresetCommandType.AI_ASSERTION:
|
|
657
|
+
return {
|
|
658
|
+
type,
|
|
659
|
+
assertion: "",
|
|
660
|
+
disableCache: true,
|
|
661
|
+
useVision: false,
|
|
662
|
+
};
|
|
663
|
+
default:
|
|
664
|
+
const assertUnreachable = (_x) => {
|
|
665
|
+
throw "If Typescript complains about the line below, you missed a case or break in the switch above";
|
|
666
|
+
};
|
|
667
|
+
return assertUnreachable(type);
|
|
668
|
+
}
|
|
669
|
+
};
|
|
670
|
+
|
|
671
|
+
;// CONCATENATED MODULE: ../../packages/types/src/ai-commands.ts
|
|
672
|
+
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
var ControlFlowCommandType;
|
|
677
|
+
(function (ControlFlowCommandType) {
|
|
678
|
+
ControlFlowCommandType["SUCCESS"] = "SUCCESS";
|
|
679
|
+
ControlFlowCommandType["FAILURE"] = "FAILURE";
|
|
680
|
+
})(ControlFlowCommandType || (ControlFlowCommandType = {}));
|
|
681
|
+
const SuccessCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
682
|
+
type: external_zod_namespaceObject.literal(ControlFlowCommandType.SUCCESS),
|
|
683
|
+
})).describe("SUCCESS - the user goal has been successfully achieved");
|
|
684
|
+
const FailureCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObject.object({
|
|
685
|
+
type: external_zod_namespaceObject.literal(ControlFlowCommandType.FAILURE),
|
|
686
|
+
})).describe("FAILURE - there are no commands to suggest that could make progress that have not already been tried before");
|
|
687
|
+
const ControlFlowCommandSchema = external_zod_namespaceObject.discriminatedUnion("type", [
|
|
688
|
+
SuccessCommandSchema,
|
|
689
|
+
FailureCommandSchema,
|
|
690
|
+
]);
|
|
691
|
+
const ai_commands_AICommandSchema = external_zod_namespaceObject.discriminatedUnion("type", [
|
|
692
|
+
...ControlFlowCommandSchema.options,
|
|
693
|
+
// We allow all preset actions here because users
|
|
694
|
+
// can edit AI commands to be any preset.
|
|
695
|
+
// However, the AI can only suggest items in AISuggestiblePresetCommandSchema
|
|
696
|
+
...AISuggestiblePresetCommandSchema.options,
|
|
697
|
+
]);
|
|
698
|
+
const isControlFlowCommand = (cmd) => {
|
|
699
|
+
return (cmd.type === ControlFlowCommandType.SUCCESS ||
|
|
700
|
+
cmd.type === ControlFlowCommandType.FAILURE);
|
|
701
|
+
};
|
|
702
|
+
// parses the JSON from the LLM into a command object
|
|
703
|
+
function parseCommand(cmd) {
|
|
704
|
+
try {
|
|
705
|
+
return _parseCommand(cmd);
|
|
706
|
+
}
|
|
707
|
+
catch (e) {
|
|
708
|
+
if (e instanceof Error) {
|
|
709
|
+
throw new CommandParseError(`Failed to parse command: ${JSON.stringify(cmd)}`, {
|
|
710
|
+
cause: e,
|
|
711
|
+
});
|
|
712
|
+
}
|
|
713
|
+
throw new Error(`Unexpected throw from parseCommand: ${e}`);
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
const LLMOutputSchema = external_zod_namespaceObject.object({
|
|
717
|
+
command: external_zod_namespaceObject.string(),
|
|
718
|
+
thoughts: external_zod_namespaceObject.string(),
|
|
719
|
+
});
|
|
720
|
+
const NumericStringSchema = external_zod_namespaceObject.string().pipe(external_zod_namespaceObject.coerce.number());
|
|
721
|
+
function _parseCommand(cmd) {
|
|
722
|
+
const { command, thoughts } = LLMOutputSchema.parse(cmd);
|
|
723
|
+
const args = parseArgsStringToArgv(command);
|
|
724
|
+
const type = args[0];
|
|
725
|
+
switch (type) {
|
|
726
|
+
case PresetCommandType.CLICK:
|
|
727
|
+
const clickInput = {
|
|
728
|
+
type,
|
|
729
|
+
target: {
|
|
730
|
+
elementDescriptor: "",
|
|
731
|
+
a11yData: {
|
|
732
|
+
id: NumericStringSchema.parse(args[1]),
|
|
733
|
+
},
|
|
734
|
+
},
|
|
735
|
+
thoughts,
|
|
736
|
+
};
|
|
737
|
+
// We parse again so zod populates the default fields
|
|
738
|
+
return ai_commands_AICommandSchema.parse(clickInput);
|
|
739
|
+
case PresetCommandType.TYPE:
|
|
740
|
+
const typeInput = {
|
|
741
|
+
type,
|
|
742
|
+
target: {
|
|
743
|
+
elementDescriptor: "",
|
|
744
|
+
a11yData: {
|
|
745
|
+
id: NumericStringSchema.parse(args[1]),
|
|
746
|
+
},
|
|
747
|
+
},
|
|
748
|
+
value: z.string().parse(args[2]),
|
|
749
|
+
thoughts,
|
|
750
|
+
};
|
|
751
|
+
return ai_commands_AICommandSchema.parse(typeInput);
|
|
752
|
+
case PresetCommandType.PRESS:
|
|
753
|
+
const pressInput = {
|
|
754
|
+
type,
|
|
755
|
+
value: z.string().parse(args[1]),
|
|
756
|
+
thoughts,
|
|
757
|
+
};
|
|
758
|
+
return ai_commands_AICommandSchema.parse(pressInput);
|
|
759
|
+
case PresetCommandType.SELECT_OPTION:
|
|
760
|
+
return {
|
|
761
|
+
type,
|
|
762
|
+
target: {
|
|
763
|
+
elementDescriptor: "",
|
|
764
|
+
a11yData: {
|
|
765
|
+
id: NumericStringSchema.parse(args[1]),
|
|
766
|
+
},
|
|
767
|
+
},
|
|
768
|
+
option: z.string().parse(args[2]),
|
|
769
|
+
thoughts,
|
|
770
|
+
};
|
|
771
|
+
case PresetCommandType.NAVIGATE:
|
|
772
|
+
return {
|
|
773
|
+
type,
|
|
774
|
+
url: z.string().url().parse(args[1]),
|
|
775
|
+
thoughts,
|
|
776
|
+
};
|
|
777
|
+
case PresetCommandType.SCROLL_DOWN:
|
|
778
|
+
case PresetCommandType.SCROLL_UP:
|
|
779
|
+
return {
|
|
780
|
+
type,
|
|
781
|
+
thoughts,
|
|
782
|
+
};
|
|
783
|
+
case ControlFlowCommandType.SUCCESS:
|
|
784
|
+
case ControlFlowCommandType.FAILURE:
|
|
785
|
+
return {
|
|
786
|
+
type,
|
|
787
|
+
thoughts,
|
|
788
|
+
};
|
|
789
|
+
default:
|
|
790
|
+
throw new Error(`Unknown command type: ${type}`);
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
;// CONCATENATED MODULE: ../../packages/types/src/steps.ts
|
|
795
|
+
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
/**
|
|
799
|
+
* Enum of possible steps inputted by the user.
|
|
800
|
+
*/
|
|
801
|
+
var StepType;
|
|
802
|
+
(function (StepType) {
|
|
803
|
+
// Ask AI to execute an action on the page
|
|
804
|
+
StepType["AI_ACTION"] = "AI_ACTION";
|
|
805
|
+
// Actions that can be executed by the user.
|
|
806
|
+
StepType["PRESET_ACTION"] = "PRESET_ACTION";
|
|
807
|
+
StepType["MODULE"] = "MODULE";
|
|
808
|
+
})(StepType || (StepType = {}));
|
|
809
|
+
const AIActionSchema = external_zod_namespaceObject.object({
|
|
810
|
+
type: external_zod_namespaceObject.literal(StepType.AI_ACTION),
|
|
811
|
+
text: external_zod_namespaceObject.string(),
|
|
812
|
+
// Cached commands for this step
|
|
813
|
+
commands: external_zod_namespaceObject.array(ai_commands_AICommandSchema).optional(),
|
|
814
|
+
});
|
|
815
|
+
const PresetActionSchema = external_zod_namespaceObject.object({
|
|
816
|
+
type: external_zod_namespaceObject.literal(StepType.PRESET_ACTION),
|
|
817
|
+
command: PresetCommandSchema,
|
|
818
|
+
});
|
|
819
|
+
// what is actually saved in the db
|
|
820
|
+
const ModuleStepSchema = external_zod_namespaceObject.object({
|
|
821
|
+
type: external_zod_namespaceObject.literal(StepType.MODULE),
|
|
822
|
+
moduleId: external_zod_namespaceObject.string().uuid(),
|
|
823
|
+
});
|
|
824
|
+
const AllowedModuleStepSchema = external_zod_namespaceObject.union([
|
|
825
|
+
AIActionSchema,
|
|
826
|
+
PresetActionSchema,
|
|
827
|
+
]);
|
|
828
|
+
// used for viewing and editing a module
|
|
829
|
+
const ResolvedModuleStepSchema = external_zod_namespaceObject.object({
|
|
830
|
+
type: external_zod_namespaceObject.literal("RESOLVED_MODULE"),
|
|
831
|
+
moduleId: external_zod_namespaceObject.string().uuid(),
|
|
832
|
+
name: external_zod_namespaceObject.string(),
|
|
833
|
+
steps: AllowedModuleStepSchema.array(),
|
|
834
|
+
});
|
|
835
|
+
const StepSchema = external_zod_namespaceObject.union([
|
|
836
|
+
AIActionSchema,
|
|
837
|
+
PresetActionSchema,
|
|
838
|
+
ModuleStepSchema,
|
|
839
|
+
]);
|
|
840
|
+
// frontend-only, we should probably move this out of 'types' at some point
|
|
841
|
+
const ResolvedStepSchema = external_zod_namespaceObject.union([
|
|
842
|
+
AIActionSchema,
|
|
843
|
+
PresetActionSchema,
|
|
844
|
+
ResolvedModuleStepSchema,
|
|
845
|
+
]);
|
|
846
|
+
const isModule = (step) => {
|
|
847
|
+
return step.type === StepType.MODULE;
|
|
848
|
+
};
|
|
849
|
+
const isResolvedModule = (step) => {
|
|
850
|
+
return step.type === "RESOLVED_MODULE";
|
|
851
|
+
};
|
|
852
|
+
const canBeInModule = (step) => {
|
|
853
|
+
return (step.type === StepType.AI_ACTION || step.type === StepType.PRESET_ACTION);
|
|
854
|
+
};
|
|
855
|
+
const isPresetAction = (step) => {
|
|
856
|
+
return step.type === StepType.PRESET_ACTION;
|
|
857
|
+
};
|
|
858
|
+
const isCommandAllowedInAIAction = (command) => {
|
|
859
|
+
return AICommandSchema.safeParse(command).success;
|
|
860
|
+
};
|
|
861
|
+
|
|
862
|
+
;// CONCATENATED MODULE: ../../packages/types/src/command-results.ts
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
|
|
866
|
+
var ResultStatus;
|
|
867
|
+
(function (ResultStatus) {
|
|
868
|
+
ResultStatus["SUCCESS"] = "SUCCESS";
|
|
869
|
+
ResultStatus["FAILED"] = "FAILED";
|
|
870
|
+
ResultStatus["RUNNING"] = "RUNNING";
|
|
871
|
+
ResultStatus["IDLE"] = "IDLE";
|
|
872
|
+
ResultStatus["CANCELLED"] = "CANCELLED";
|
|
873
|
+
})(ResultStatus || (ResultStatus = {}));
|
|
874
|
+
var CommandStatus;
|
|
875
|
+
(function (CommandStatus) {
|
|
876
|
+
CommandStatus["SUCCESS"] = "SUCCESS";
|
|
877
|
+
CommandStatus["FAILED"] = "FAILED";
|
|
878
|
+
})(CommandStatus || (CommandStatus = {}));
|
|
879
|
+
const CommandMetadataSchema = external_zod_namespaceObject.object({
|
|
880
|
+
beforeUrl: external_zod_namespaceObject.string(),
|
|
881
|
+
beforeScreenshot: external_zod_namespaceObject.string(),
|
|
882
|
+
afterUrl: external_zod_namespaceObject.string().optional(),
|
|
883
|
+
afterScreenshot: external_zod_namespaceObject.string().optional(),
|
|
884
|
+
startedAt: external_zod_namespaceObject.coerce.date(),
|
|
885
|
+
finishedAt: external_zod_namespaceObject.coerce.date(),
|
|
886
|
+
viewport: external_zod_namespaceObject.object({
|
|
887
|
+
height: external_zod_namespaceObject.number(),
|
|
888
|
+
width: external_zod_namespaceObject.number(),
|
|
889
|
+
}),
|
|
890
|
+
status: external_zod_namespaceObject.nativeEnum(CommandStatus),
|
|
891
|
+
error: external_zod_namespaceObject.string().optional(),
|
|
892
|
+
elementInteracted: external_zod_namespaceObject.string().optional(),
|
|
893
|
+
});
|
|
894
|
+
const CommandResultSchema = external_zod_namespaceObject.object({
|
|
895
|
+
command: ai_commands_AICommandSchema,
|
|
896
|
+
})
|
|
897
|
+
.merge(CommandMetadataSchema);
|
|
898
|
+
const StepResultMetadataSchema = external_zod_namespaceObject.object({
|
|
899
|
+
startedAt: external_zod_namespaceObject.coerce.date(),
|
|
900
|
+
finishedAt: external_zod_namespaceObject.coerce.date(),
|
|
901
|
+
status: external_zod_namespaceObject.nativeEnum(ResultStatus),
|
|
902
|
+
error: external_zod_namespaceObject.string().optional(),
|
|
903
|
+
// browser info
|
|
904
|
+
userAgent: external_zod_namespaceObject.string().optional(),
|
|
905
|
+
});
|
|
906
|
+
const PresetActionResultSchema = PresetActionSchema.merge(StepResultMetadataSchema).merge(external_zod_namespaceObject.object({
|
|
907
|
+
// commandresult is saved by actions like AI assertions that have a "return" value
|
|
908
|
+
// commandmetadata is saved by actions like preset click that doesn't have a return value
|
|
909
|
+
// Array just for consistency with other elements, should only ever be one
|
|
910
|
+
results: external_zod_namespaceObject.union([CommandResultSchema, CommandMetadataSchema]).array(),
|
|
911
|
+
}));
|
|
912
|
+
const AIActionResultSchema = AIActionSchema.merge(StepResultMetadataSchema).merge(external_zod_namespaceObject.object({
|
|
913
|
+
// commandresult is saved by actions like AI assertions that have a "return" value
|
|
914
|
+
// commandmetadata is saved by actions like preset click that doesn't have a return value
|
|
915
|
+
results: PresetActionResultSchema.array(),
|
|
916
|
+
}));
|
|
917
|
+
const ModuleResultSchema = ModuleStepSchema.merge(StepResultMetadataSchema).merge(external_zod_namespaceObject.object({
|
|
918
|
+
// nested results
|
|
919
|
+
results: external_zod_namespaceObject.union([AIActionResultSchema, PresetActionResultSchema]).array(),
|
|
920
|
+
}));
|
|
921
|
+
// this maps to a `Step`
|
|
922
|
+
const ResultSchema = external_zod_namespaceObject.discriminatedUnion("type", [
|
|
923
|
+
AIActionResultSchema,
|
|
924
|
+
PresetActionResultSchema,
|
|
925
|
+
ModuleResultSchema,
|
|
926
|
+
]);
|
|
927
|
+
const isCommandResult = (c) => {
|
|
928
|
+
return CommandResultSchema.safeParse(c).success;
|
|
929
|
+
};
|
|
930
|
+
|
|
931
|
+
;// CONCATENATED MODULE: ../../packages/types/src/execute-results.ts
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
var ExecuteResultType;
|
|
936
|
+
(function (ExecuteResultType) {
|
|
937
|
+
ExecuteResultType["COMMAND"] = "command";
|
|
938
|
+
ExecuteResultType["ASSERTION"] = "assertion";
|
|
939
|
+
})(ExecuteResultType || (ExecuteResultType = {}));
|
|
940
|
+
// data persisted about the execution of a command in the controller
|
|
941
|
+
const ExecuteCommandHistoryEntrySchema = external_zod_namespaceObject.object({
|
|
942
|
+
// type of command executed
|
|
943
|
+
type: external_zod_namespaceObject.nativeEnum(StepType),
|
|
944
|
+
// if AI step type, what command was executed
|
|
945
|
+
generatedStep: ai_commands_AICommandSchema.optional(),
|
|
946
|
+
// human readable descriptor for action taken, including element interacted with
|
|
947
|
+
serializedCommand: external_zod_namespaceObject.string().optional(),
|
|
948
|
+
// human readable descriptor for element interacted with
|
|
949
|
+
elementInteracted: external_zod_namespaceObject.string().optional(),
|
|
950
|
+
});
|
|
951
|
+
// result schema for assertion evaluations - this is the type returned to the client.
|
|
952
|
+
// we reuse the success/failure schemas from regular ai commands.
|
|
953
|
+
const execute_results_ExecuteAssertionResultSchema = external_zod_namespaceObject.object({
|
|
954
|
+
type: external_zod_namespaceObject.literal(ExecuteResultType.ASSERTION),
|
|
955
|
+
command: external_zod_namespaceObject.union([SuccessCommandSchema, FailureCommandSchema]),
|
|
956
|
+
relevantElements: external_zod_namespaceObject.array(external_zod_namespaceObject.number()).optional(),
|
|
957
|
+
});
|
|
958
|
+
|
|
959
|
+
;// CONCATENATED MODULE: ../../packages/types/src/goal-splitter.ts
|
|
960
|
+
|
|
961
|
+
const InstructionsSchema = external_zod_namespaceObject.z.string().array();
|
|
962
|
+
|
|
963
|
+
;// CONCATENATED MODULE: ../../packages/types/src/locator.ts
|
|
964
|
+
|
|
965
|
+
const locator_AILocatorSchema = external_zod_namespaceObject.object({
|
|
966
|
+
thoughts: external_zod_namespaceObject.string(),
|
|
967
|
+
// a11y id
|
|
968
|
+
id: external_zod_namespaceObject.number().int(),
|
|
969
|
+
// dropdowns should have options
|
|
970
|
+
options: external_zod_namespaceObject.array(external_zod_namespaceObject.string()).optional(),
|
|
971
|
+
});
|
|
972
|
+
|
|
973
|
+
;// CONCATENATED MODULE: ../../packages/types/src/modules.ts
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
const ModuleMetadataSchema = external_zod_namespaceObject.z.object({
|
|
977
|
+
id: external_zod_namespaceObject.z.string(),
|
|
978
|
+
createdAt: external_zod_namespaceObject.z.coerce.date(),
|
|
979
|
+
createdBy: external_zod_namespaceObject.z.string(),
|
|
980
|
+
organizationId: external_zod_namespaceObject.z.string().or(external_zod_namespaceObject.z["null"]()),
|
|
981
|
+
name: external_zod_namespaceObject.z.string(),
|
|
982
|
+
schemaVersion: external_zod_namespaceObject.z.string(),
|
|
983
|
+
// this is only used in the client and is not stored in the db
|
|
984
|
+
numSteps: external_zod_namespaceObject.z.number(),
|
|
985
|
+
});
|
|
986
|
+
const ModuleSchema = external_zod_namespaceObject.z.object({
|
|
987
|
+
steps: AllowedModuleStepSchema.array(),
|
|
988
|
+
})
|
|
989
|
+
.merge(ModuleMetadataSchema.omit({ numSteps: true }));
|
|
990
|
+
|
|
991
|
+
;// CONCATENATED MODULE: ../../packages/types/src/runs.ts
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
const RunTrigger = {
|
|
995
|
+
WEBHOOK: "WEBHOOK",
|
|
996
|
+
CRON: "CRON",
|
|
997
|
+
MANUAL: "MANUAL",
|
|
998
|
+
};
|
|
999
|
+
const RunStatusEnum = {
|
|
1000
|
+
PENDING: "PENDING",
|
|
1001
|
+
RUNNING: "RUNNING",
|
|
1002
|
+
PASSED: "PASSED",
|
|
1003
|
+
FAILED: "FAILED",
|
|
1004
|
+
CANCELLED: "CANCELLED",
|
|
1005
|
+
};
|
|
1006
|
+
const DateOrStringSchema = external_zod_namespaceObject.z.string().pipe(external_zod_namespaceObject.z.coerce.date()).or(external_zod_namespaceObject.z.date());
|
|
1007
|
+
// No value for test steps or results - just metadata.
|
|
1008
|
+
// This is important bc the endpoints that return this metadata
|
|
1009
|
+
// do not run step migrations.
|
|
1010
|
+
const RunMetadataSchema = external_zod_namespaceObject.z.object({
|
|
1011
|
+
id: external_zod_namespaceObject.z.string(),
|
|
1012
|
+
createdAt: DateOrStringSchema,
|
|
1013
|
+
createdBy: external_zod_namespaceObject.z.string(),
|
|
1014
|
+
organizationId: external_zod_namespaceObject.z.string().or(external_zod_namespaceObject.z["null"]()),
|
|
1015
|
+
scheduledAt: DateOrStringSchema.or(external_zod_namespaceObject.z["null"]()),
|
|
1016
|
+
startedAt: DateOrStringSchema.or(external_zod_namespaceObject.z["null"]()),
|
|
1017
|
+
finishedAt: DateOrStringSchema.or(external_zod_namespaceObject.z["null"]()),
|
|
1018
|
+
testId: external_zod_namespaceObject.z.string().or(external_zod_namespaceObject.z["null"]()),
|
|
1019
|
+
status: external_zod_namespaceObject.z.nativeEnum(RunStatusEnum),
|
|
1020
|
+
trigger: external_zod_namespaceObject.z.nativeEnum(RunTrigger),
|
|
1021
|
+
test: external_zod_namespaceObject.z.object({
|
|
1022
|
+
name: external_zod_namespaceObject.z.string(),
|
|
1023
|
+
id: external_zod_namespaceObject.z.string(),
|
|
1024
|
+
})
|
|
1025
|
+
.or(external_zod_namespaceObject.z["null"]()),
|
|
1026
|
+
});
|
|
1027
|
+
// Metadata + results + test metadata
|
|
1028
|
+
// Use this schema to parse the output of fetchRun
|
|
1029
|
+
const RunWithTestSchema = RunMetadataSchema.merge(external_zod_namespaceObject.z.object({
|
|
1030
|
+
results: ResultSchema.array(),
|
|
1031
|
+
test: external_zod_namespaceObject.z.object({
|
|
1032
|
+
name: external_zod_namespaceObject.z.string(),
|
|
1033
|
+
id: external_zod_namespaceObject.z.string(),
|
|
1034
|
+
baseUrl: external_zod_namespaceObject.z.string(),
|
|
1035
|
+
})
|
|
1036
|
+
.or(external_zod_namespaceObject.z["null"]()),
|
|
1037
|
+
}));
|
|
1038
|
+
|
|
1039
|
+
;// CONCATENATED MODULE: ../../packages/types/src/serialization.ts
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
/**
|
|
1043
|
+
* Clamp text so that it has a maximum char length, cutting off with ... at the end if necessary.
|
|
1044
|
+
* Do not provide a number lower than 3.
|
|
1045
|
+
*/
|
|
1046
|
+
function clampText(text, length) {
|
|
1047
|
+
if (text.length < length) {
|
|
1048
|
+
return text;
|
|
1049
|
+
}
|
|
1050
|
+
return text.slice(0, length - 3) + "[...]";
|
|
1051
|
+
}
|
|
1052
|
+
const AI_COMMAND_DISPLAY_NAMES = {
|
|
1053
|
+
[preset_PresetCommandType.CLICK]: "Click",
|
|
1054
|
+
[preset_PresetCommandType.TYPE]: "Type",
|
|
1055
|
+
[preset_PresetCommandType.SELECT_OPTION]: "Select",
|
|
1056
|
+
[preset_PresetCommandType.PRESS]: "Press",
|
|
1057
|
+
[preset_PresetCommandType.NAVIGATE]: "Navigate",
|
|
1058
|
+
[preset_PresetCommandType.SCROLL_DOWN]: "Scroll down",
|
|
1059
|
+
[preset_PresetCommandType.SCROLL_UP]: "Scroll up",
|
|
1060
|
+
};
|
|
1061
|
+
function serializeAICommand(cmd) {
|
|
1062
|
+
let humanSummary = "";
|
|
1063
|
+
switch (cmd.type) {
|
|
1064
|
+
case ControlFlowCommandType.SUCCESS:
|
|
1065
|
+
humanSummary = `Step complete: ${cmd.thoughts}`;
|
|
1066
|
+
break;
|
|
1067
|
+
case ControlFlowCommandType.FAILURE:
|
|
1068
|
+
humanSummary = `Step failed: ${cmd.thoughts}`;
|
|
1069
|
+
break;
|
|
1070
|
+
default:
|
|
1071
|
+
return serializePresetCommand(cmd);
|
|
1072
|
+
}
|
|
1073
|
+
return humanSummary;
|
|
1074
|
+
}
|
|
1075
|
+
function serializePresetCommand(command) {
|
|
1076
|
+
switch (command.type) {
|
|
1077
|
+
case preset_PresetCommandType.NAVIGATE:
|
|
1078
|
+
return `Go to URL: ${clampText(command.url, 30)}`;
|
|
1079
|
+
case preset_PresetCommandType.GO_BACK:
|
|
1080
|
+
return `Go back to the previous page`;
|
|
1081
|
+
case preset_PresetCommandType.GO_FORWARD:
|
|
1082
|
+
return `Go forward to the next page`;
|
|
1083
|
+
case preset_PresetCommandType.SCROLL_DOWN:
|
|
1084
|
+
return `Scroll down one page`;
|
|
1085
|
+
case preset_PresetCommandType.SCROLL_UP:
|
|
1086
|
+
return `Scroll up one page`;
|
|
1087
|
+
case preset_PresetCommandType.WAIT:
|
|
1088
|
+
return `Wait for ${command.delay} seconds`;
|
|
1089
|
+
case preset_PresetCommandType.REFRESH:
|
|
1090
|
+
return `Refresh the page`;
|
|
1091
|
+
case preset_PresetCommandType.CLICK:
|
|
1092
|
+
return `Click on '${command.target.elementDescriptor}'`;
|
|
1093
|
+
case preset_PresetCommandType.TYPE:
|
|
1094
|
+
let serializedTarget = "";
|
|
1095
|
+
if (command.target.a11yData?.serializedForm) {
|
|
1096
|
+
serializedTarget = ` in element ${command.target.a11yData.serializedForm}`;
|
|
1097
|
+
}
|
|
1098
|
+
else if (command.target.elementDescriptor.length > 0) {
|
|
1099
|
+
serializedTarget = ` in element ${command.target.elementDescriptor}`;
|
|
1100
|
+
}
|
|
1101
|
+
return `Type${serializedTarget}: '${command.value}'`;
|
|
1102
|
+
case preset_PresetCommandType.PRESS:
|
|
1103
|
+
return `Press '${command.value}'`;
|
|
1104
|
+
case preset_PresetCommandType.SELECT_OPTION:
|
|
1105
|
+
return `Select option '${command.option}' in '${command.target.elementDescriptor}'`;
|
|
1106
|
+
case preset_PresetCommandType.AI_ASSERTION:
|
|
1107
|
+
return `${command.useVision ? "Visual assertion" : "Assertion"} successful: '${command.assertion}'`;
|
|
1108
|
+
default:
|
|
1109
|
+
const assertUnreachable = (_x) => {
|
|
1110
|
+
throw "If Typescript complains about the line below, you missed a case or break in the switch above";
|
|
1111
|
+
};
|
|
1112
|
+
return assertUnreachable(command);
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
// EXTERNAL MODULE: ../../node_modules/.pnpm/cron-validator@1.3.1/node_modules/cron-validator/lib/index.js
|
|
1117
|
+
var lib = __nccwpck_require__(909);
|
|
1118
|
+
;// CONCATENATED MODULE: ../../packages/types/src/test-settings.ts
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
const TestAdvancedSettingsSchema = external_zod_namespaceObject.z.object({
|
|
1122
|
+
availableAsModule: external_zod_namespaceObject.z.boolean().default(false),
|
|
1123
|
+
disableAICaching: external_zod_namespaceObject.z.boolean().default(false),
|
|
1124
|
+
});
|
|
1125
|
+
const ScheduleSettingsSchema = external_zod_namespaceObject.z.object({
|
|
1126
|
+
cron: external_zod_namespaceObject.z.string()
|
|
1127
|
+
.refine((v) => {
|
|
1128
|
+
return (0,lib.isValidCron)(v);
|
|
1129
|
+
}, { message: "Invalid cron expression." })
|
|
1130
|
+
// // default crontab is every day
|
|
1131
|
+
.default("0 0 */1 * *"),
|
|
1132
|
+
enabled: external_zod_namespaceObject.z.boolean().default(false),
|
|
1133
|
+
timeZone: external_zod_namespaceObject.z.string().default("America/Los_Angeles"),
|
|
1134
|
+
// this is used for removing repeatable jobs (not set by user)
|
|
1135
|
+
jobKey: external_zod_namespaceObject.z.string().optional(),
|
|
1136
|
+
});
|
|
1137
|
+
const WebhookSchema = external_zod_namespaceObject.z.object({
|
|
1138
|
+
lastStatus: external_zod_namespaceObject.z.number().optional(),
|
|
1139
|
+
url: external_zod_namespaceObject.z.string().url(),
|
|
1140
|
+
});
|
|
1141
|
+
const WebhookSettingsSchema = external_zod_namespaceObject.z.array(WebhookSchema).default([]);
|
|
1142
|
+
const TestSettingsSchema = external_zod_namespaceObject.z.object({
|
|
1143
|
+
name: external_zod_namespaceObject.z.string().min(1),
|
|
1144
|
+
baseUrl: external_zod_namespaceObject.z.string().url(),
|
|
1145
|
+
advanced: TestAdvancedSettingsSchema,
|
|
1146
|
+
});
|
|
1147
|
+
|
|
1148
|
+
;// CONCATENATED MODULE: ../../packages/types/src/test.ts
|
|
1149
|
+
|
|
1150
|
+
|
|
1151
|
+
|
|
1152
|
+
// This is the type shared between FE and BE for a fully resolved, executable test.
|
|
1153
|
+
// This is not the raw storage format, which is the Test type in the orm package.
|
|
1154
|
+
// This is the processed format (e.g. step has been processed into ResolvedSteps)
|
|
1155
|
+
const ResolvedTestSchema = external_zod_namespaceObject.z.object({
|
|
1156
|
+
id: external_zod_namespaceObject.z.string(),
|
|
1157
|
+
name: external_zod_namespaceObject.z.string(),
|
|
1158
|
+
baseUrl: external_zod_namespaceObject.z.string(),
|
|
1159
|
+
steps: external_zod_namespaceObject.z.array(ResolvedStepSchema),
|
|
1160
|
+
createdAt: external_zod_namespaceObject.z.coerce.date(),
|
|
1161
|
+
updatedAt: external_zod_namespaceObject.z.coerce.date(),
|
|
1162
|
+
createdBy: external_zod_namespaceObject.z.string(),
|
|
1163
|
+
organizationId: external_zod_namespaceObject.z.string().or(external_zod_namespaceObject.z["null"]()),
|
|
1164
|
+
schemaVersion: external_zod_namespaceObject.z.string(),
|
|
1165
|
+
advanced: TestAdvancedSettingsSchema,
|
|
1166
|
+
schedule: ScheduleSettingsSchema,
|
|
1167
|
+
webhooks: WebhookSettingsSchema,
|
|
1168
|
+
});
|
|
1169
|
+
|
|
1170
|
+
;// CONCATENATED MODULE: ../../packages/types/src/context.ts
|
|
1171
|
+
|
|
1172
|
+
|
|
1173
|
+
// this is the context that is available to the agent when it is making a decision
|
|
1174
|
+
const DynamicContextSchema = external_zod_namespaceObject.object({
|
|
1175
|
+
// user goal or instruction
|
|
1176
|
+
goal: external_zod_namespaceObject.string(),
|
|
1177
|
+
// current url of the browser
|
|
1178
|
+
url: external_zod_namespaceObject.string(),
|
|
1179
|
+
// serialized page state
|
|
1180
|
+
browserState: external_zod_namespaceObject.string(),
|
|
1181
|
+
// serialized history of previous commands
|
|
1182
|
+
history: external_zod_namespaceObject.string(),
|
|
1183
|
+
// number of previously executed commands
|
|
1184
|
+
numPrevious: external_zod_namespaceObject.number(),
|
|
1185
|
+
// last executed command, if any
|
|
1186
|
+
lastCommand: ExecuteCommandHistoryEntrySchema.or(external_zod_namespaceObject["null"]()),
|
|
1187
|
+
});
|
|
1188
|
+
|
|
1189
|
+
;// CONCATENATED MODULE: ../../packages/types/src/public-api.ts
|
|
1190
|
+
|
|
1191
|
+
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
|
|
1195
|
+
const GeneratorOptionsSchema = external_zod_namespaceObject.object({
|
|
1196
|
+
disableCache: external_zod_namespaceObject.boolean(),
|
|
1197
|
+
});
|
|
1198
|
+
const GetNextCommandBodySchema = DynamicContextSchema.merge(GeneratorOptionsSchema);
|
|
1199
|
+
const GetNextCommandResponseSchema = (/* unused pure expression or super */ null && (AICommandSchema));
|
|
1200
|
+
const GetAssertionResultBodySchema = external_zod_namespaceObject.discriminatedUnion("vision", [
|
|
1201
|
+
DynamicContextSchema.merge(GeneratorOptionsSchema).merge(external_zod_namespaceObject.object({
|
|
1202
|
+
vision: external_zod_namespaceObject.literal(false),
|
|
1203
|
+
})),
|
|
1204
|
+
DynamicContextSchema.pick({
|
|
1205
|
+
goal: true,
|
|
1206
|
+
url: true,
|
|
1207
|
+
})
|
|
1208
|
+
.merge(GeneratorOptionsSchema)
|
|
1209
|
+
.merge(external_zod_namespaceObject.object({
|
|
1210
|
+
// base64 encoded image
|
|
1211
|
+
screenshot: external_zod_namespaceObject.string(),
|
|
1212
|
+
vision: external_zod_namespaceObject.literal(true),
|
|
1213
|
+
})),
|
|
1214
|
+
]);
|
|
1215
|
+
const GetAssertionResponseSchema = (/* unused pure expression or super */ null && (ExecuteAssertionResultSchema));
|
|
1216
|
+
const LocateBodySchema = DynamicContextSchema.pick({
|
|
1217
|
+
browserState: true,
|
|
1218
|
+
goal: true,
|
|
1219
|
+
}).merge(GeneratorOptionsSchema);
|
|
1220
|
+
const LocateResponseSchema = (/* unused pure expression or super */ null && (AILocatorSchema));
|
|
1221
|
+
const SplitGoalBodySchema = DynamicContextSchema.pick({
|
|
1222
|
+
goal: true,
|
|
1223
|
+
url: true,
|
|
1224
|
+
}).merge(GeneratorOptionsSchema);
|
|
1225
|
+
const SplitGoalResponseSchema = external_zod_namespaceObject.string().array();
|
|
1226
|
+
|
|
1227
|
+
;// CONCATENATED MODULE: ../../packages/types/src/index.ts
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
|
|
1231
|
+
|
|
1232
|
+
|
|
1233
|
+
|
|
1234
|
+
|
|
1235
|
+
|
|
1236
|
+
|
|
1237
|
+
|
|
1238
|
+
|
|
1239
|
+
|
|
1240
|
+
|
|
1241
|
+
|
|
1242
|
+
|
|
1243
|
+
|
|
1244
|
+
|
|
1245
|
+
|
|
1246
|
+
|
|
1247
|
+
|
|
1248
|
+
|
|
1249
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/browsers/a11y.ts
|
|
1250
|
+
|
|
1251
|
+
const bannedProperties = new Set(["focusable"]);
|
|
1252
|
+
const alwaysInterestingRoles = new Set([
|
|
1253
|
+
"textbox",
|
|
1254
|
+
"checkbox",
|
|
1255
|
+
"button",
|
|
1256
|
+
"link",
|
|
1257
|
+
]);
|
|
1258
|
+
// do not output IDs in the a11y tree; i.e. prevent llm from clicking these
|
|
1259
|
+
const rolesToOmitID = new Set(["paragraph", "menuitem", "option"]);
|
|
1260
|
+
const defaultA11yNodeSerializeParams = {
|
|
1261
|
+
indentLevel: 0,
|
|
1262
|
+
noID: false,
|
|
1263
|
+
noChildren: false,
|
|
1264
|
+
noProperties: false,
|
|
1265
|
+
};
|
|
1266
|
+
// Serialized format provided to LLM here: packages/web-agent/src/agents/a11y/prompts.ts
|
|
1267
|
+
class ProcessedA11yNode {
|
|
1268
|
+
constructor(params) {
|
|
1269
|
+
this.id = params.id;
|
|
1270
|
+
this.role = params.role;
|
|
1271
|
+
this.name = params.name;
|
|
1272
|
+
this.content = params.content;
|
|
1273
|
+
this.properties = params.properties;
|
|
1274
|
+
this.pathFromRoot = params.pathFromRoot;
|
|
1275
|
+
// this.md5Sum = params.md5Sum;
|
|
1276
|
+
this.children = params.children;
|
|
1277
|
+
this.backendNodeID = params.backendNodeID;
|
|
1278
|
+
}
|
|
1279
|
+
getLogForm() {
|
|
1280
|
+
return JSON.stringify({
|
|
1281
|
+
id: this.id,
|
|
1282
|
+
name: this.name ?? "",
|
|
1283
|
+
role: this.role ?? "",
|
|
1284
|
+
backendNodeId: this.backendNodeID,
|
|
1285
|
+
});
|
|
1286
|
+
}
|
|
1287
|
+
/**
|
|
1288
|
+
* Returns true if the current node contains interesting properties.
|
|
1289
|
+
* Does not go through children.
|
|
1290
|
+
*/
|
|
1291
|
+
isInteresting() {
|
|
1292
|
+
if (alwaysInterestingRoles.has(this.role))
|
|
1293
|
+
return true;
|
|
1294
|
+
// do not prune containers of static text
|
|
1295
|
+
if (this.children.some((child) => child.role === "StaticText"))
|
|
1296
|
+
return true;
|
|
1297
|
+
return !!this.name.trim() || !!this.content;
|
|
1298
|
+
}
|
|
1299
|
+
serialize(opts = defaultA11yNodeSerializeParams) {
|
|
1300
|
+
const { indentLevel, noChildren, noProperties, noID } = Object.assign({}, defaultA11yNodeSerializeParams, opts);
|
|
1301
|
+
const indent = " ".repeat(indentLevel);
|
|
1302
|
+
if (this.role === "StaticText") {
|
|
1303
|
+
return `${indent}${this.name}\n`;
|
|
1304
|
+
}
|
|
1305
|
+
let s = `${indent}<${this.role}`;
|
|
1306
|
+
if (!noID && !rolesToOmitID.has(this.role)) {
|
|
1307
|
+
s += ` id="${this.id}"`;
|
|
1308
|
+
}
|
|
1309
|
+
if (this.name) {
|
|
1310
|
+
s += ` name="${this.name}"`;
|
|
1311
|
+
}
|
|
1312
|
+
if (this.content) {
|
|
1313
|
+
s += ` content="${this.content}"`;
|
|
1314
|
+
}
|
|
1315
|
+
if (Object.keys(this.properties).length > 0 && !noProperties) {
|
|
1316
|
+
Object.entries(this.properties).forEach(([k, v]) => {
|
|
1317
|
+
if (bannedProperties.has(k)) {
|
|
1318
|
+
return;
|
|
1319
|
+
}
|
|
1320
|
+
else if (typeof v === "string") {
|
|
1321
|
+
s += ` ${k}="${v}"`;
|
|
1322
|
+
}
|
|
1323
|
+
else if (typeof v === "boolean") {
|
|
1324
|
+
if (v) {
|
|
1325
|
+
s += ` ${k}`;
|
|
1326
|
+
}
|
|
1327
|
+
else {
|
|
1328
|
+
s += ` ${k}={false}`;
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
else if (typeof v !== "undefined") {
|
|
1332
|
+
s += ` ${k}={${JSON.stringify(v)}}`;
|
|
1333
|
+
}
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
if (this.children.length === 0 || noChildren) {
|
|
1337
|
+
// self-closing tag and return immediately if no children
|
|
1338
|
+
s += " />\n";
|
|
1339
|
+
return s;
|
|
1340
|
+
}
|
|
1341
|
+
else {
|
|
1342
|
+
s += ">\n";
|
|
1343
|
+
}
|
|
1344
|
+
for (const child of this.children) {
|
|
1345
|
+
s += child.serialize({ indentLevel: indentLevel + 2 });
|
|
1346
|
+
}
|
|
1347
|
+
s += `${indent}</${this.role}>\n`;
|
|
1348
|
+
return s;
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
/**
|
|
1352
|
+
* A pruned and enhanced version of the accessibility tree.
|
|
1353
|
+
*/
|
|
1354
|
+
class ProcessedA11yTree {
|
|
1355
|
+
constructor(
|
|
1356
|
+
// root of the tree
|
|
1357
|
+
root,
|
|
1358
|
+
// map of node id to node, for easy access without traversing the entire tree
|
|
1359
|
+
nodeMap) {
|
|
1360
|
+
this.root = root;
|
|
1361
|
+
this.nodeMap = nodeMap;
|
|
1362
|
+
}
|
|
1363
|
+
serialize() {
|
|
1364
|
+
if (!this.root) {
|
|
1365
|
+
return "";
|
|
1366
|
+
}
|
|
1367
|
+
return this.root.serialize();
|
|
1368
|
+
}
|
|
1369
|
+
}
|
|
1370
|
+
/**
|
|
1371
|
+
* Returns a string representation of the node identifier, preferring human readable ones.
|
|
1372
|
+
* May not be unique in a graph.
|
|
1373
|
+
*/
|
|
1374
|
+
function getNodePathIdentifier(node) {
|
|
1375
|
+
if (node.name?.value) {
|
|
1376
|
+
return `"${node.name.value}"`;
|
|
1377
|
+
}
|
|
1378
|
+
if (node.role?.value &&
|
|
1379
|
+
node.role.value !== "none" &&
|
|
1380
|
+
node.role.value !== "generic") {
|
|
1381
|
+
return `"${node.role.value}"`;
|
|
1382
|
+
}
|
|
1383
|
+
return `"${node.nodeId}"`;
|
|
1384
|
+
}
|
|
1385
|
+
function processA11yTreeDFS(node, parent, inputNodeMap, outputNodeMap) {
|
|
1386
|
+
if (!parent && node.parentId) {
|
|
1387
|
+
throw new Error(`Got no parent for accessibility node ${node.nodeId}: ${JSON.stringify(node)}`);
|
|
1388
|
+
}
|
|
1389
|
+
const processedNode = new ProcessedA11yNode({
|
|
1390
|
+
id: node.nodeId,
|
|
1391
|
+
role: node.role?.value || "",
|
|
1392
|
+
name: node.name?.value || "",
|
|
1393
|
+
content: node.value?.value || "",
|
|
1394
|
+
properties: {},
|
|
1395
|
+
children: [],
|
|
1396
|
+
pathFromRoot: (parent ? `${parent.pathFromRoot} ` : "") + getNodePathIdentifier(node),
|
|
1397
|
+
backendNodeID: node.backendDOMNodeId,
|
|
1398
|
+
// md5Sum: "",
|
|
1399
|
+
});
|
|
1400
|
+
if (node.value?.value) {
|
|
1401
|
+
processedNode.content = `${node.value?.value}`;
|
|
1402
|
+
}
|
|
1403
|
+
if (node.properties) {
|
|
1404
|
+
node.properties.forEach((prop) => {
|
|
1405
|
+
processedNode.properties[prop.name] = prop.value.value;
|
|
1406
|
+
});
|
|
1407
|
+
}
|
|
1408
|
+
outputNodeMap.set(processedNode.id, processedNode);
|
|
1409
|
+
const children = node.childIds ?? [];
|
|
1410
|
+
for (const childId of children) {
|
|
1411
|
+
if (!childId) {
|
|
1412
|
+
continue;
|
|
1413
|
+
}
|
|
1414
|
+
const child = inputNodeMap.get(childId);
|
|
1415
|
+
if (!child) {
|
|
1416
|
+
continue;
|
|
1417
|
+
}
|
|
1418
|
+
const processedChildren = processA11yTreeDFS(child, processedNode, inputNodeMap, outputNodeMap);
|
|
1419
|
+
if (!processedChildren.length) {
|
|
1420
|
+
continue;
|
|
1421
|
+
}
|
|
1422
|
+
processedNode.children = processedNode.children.concat(processedChildren);
|
|
1423
|
+
}
|
|
1424
|
+
// StaticText should never have useful children
|
|
1425
|
+
if (processedNode.role === "StaticText") {
|
|
1426
|
+
processedNode.children = [];
|
|
1427
|
+
}
|
|
1428
|
+
// It seems buttons and links often have useless StaticText children that
|
|
1429
|
+
// have the exact same content or are empty. This fn signals to the DFS to ignore those.
|
|
1430
|
+
if (processedNode.children.length === 1 &&
|
|
1431
|
+
processedNode.children[0].role === "StaticText") {
|
|
1432
|
+
const currentName = processedNode.name;
|
|
1433
|
+
const childName = processedNode.children[0]?.name;
|
|
1434
|
+
if (currentName === childName || !childName) {
|
|
1435
|
+
processedNode.children = [];
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
// group adjacent StaticText elements into one
|
|
1439
|
+
const staticTextGroupedChildren = [];
|
|
1440
|
+
for (let i = processedNode.children.length - 1; i >= 0; i--) {
|
|
1441
|
+
const node = processedNode.children[i];
|
|
1442
|
+
if (node.role !== "StaticText") {
|
|
1443
|
+
staticTextGroupedChildren.push(node);
|
|
1444
|
+
continue;
|
|
1445
|
+
}
|
|
1446
|
+
if (i === 0 || processedNode.children[i - 1].role !== "StaticText") {
|
|
1447
|
+
// cannot group with existing
|
|
1448
|
+
staticTextGroupedChildren.push(node);
|
|
1449
|
+
continue;
|
|
1450
|
+
}
|
|
1451
|
+
// group with existing
|
|
1452
|
+
processedNode.children[i - 1].name += ` ${node.name}`;
|
|
1453
|
+
}
|
|
1454
|
+
processedNode.children = staticTextGroupedChildren.reverse();
|
|
1455
|
+
// set parent property on children node - important for multiple attempts at clicking
|
|
1456
|
+
for (const child of processedNode.children) {
|
|
1457
|
+
child.parent = processedNode;
|
|
1458
|
+
}
|
|
1459
|
+
// if current node is not important, try to return children only if possible
|
|
1460
|
+
// (only not possible if we are the root)
|
|
1461
|
+
const interesting = processedNode.isInteresting();
|
|
1462
|
+
if (!interesting) {
|
|
1463
|
+
if (processedNode.children.length === 0) {
|
|
1464
|
+
// delete node entirely
|
|
1465
|
+
return [];
|
|
1466
|
+
}
|
|
1467
|
+
else if (processedNode.children.length === 1) {
|
|
1468
|
+
return [processedNode.children[0]];
|
|
1469
|
+
}
|
|
1470
|
+
else if (node.parentId) {
|
|
1471
|
+
return processedNode.children;
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
// we are going to return the current node; time to compute md5 hash
|
|
1475
|
+
// this is not used right now:
|
|
1476
|
+
/**
|
|
1477
|
+
const md5 = new Md5();
|
|
1478
|
+
["id", "name", "role", "content"].forEach((s: string) => {
|
|
1479
|
+
const attr = s as keyof ProcessedA11yNode;
|
|
1480
|
+
if (typeof processedNode[attr] === "string" && processedNode[attr]) {
|
|
1481
|
+
md5.appendStr(processedNode[attr] as string);
|
|
1482
|
+
}
|
|
1483
|
+
});
|
|
1484
|
+
processedNode.children.forEach((child) => md5.appendStr(child.md5Sum));
|
|
1485
|
+
const md5Sum = md5.end() as string;
|
|
1486
|
+
if (!md5Sum) {
|
|
1487
|
+
throw new Error(`Got empty md5sum for node ${node.nodeId}`);
|
|
1488
|
+
}
|
|
1489
|
+
processedNode.md5Sum = md5Sum;
|
|
1490
|
+
*/
|
|
1491
|
+
return [processedNode];
|
|
1492
|
+
}
|
|
1493
|
+
function processA11yTree(graph) {
|
|
1494
|
+
if (!graph.root) {
|
|
1495
|
+
throw new Error("a11y tree has null root");
|
|
1496
|
+
}
|
|
1497
|
+
// filter out nodes that are no longer rendered on the page
|
|
1498
|
+
graph.allNodes = graph.allNodes.filter((node) => {
|
|
1499
|
+
if (!node.ignored) {
|
|
1500
|
+
return true;
|
|
1501
|
+
}
|
|
1502
|
+
// CDP types are wrong; notRendered is a possible ignored reason
|
|
1503
|
+
return !node.ignoredReasons?.find((reason) => reason.name === "notRendered" &&
|
|
1504
|
+
reason.value?.value);
|
|
1505
|
+
});
|
|
1506
|
+
const nodeMap = new Map();
|
|
1507
|
+
for (const node of graph.allNodes) {
|
|
1508
|
+
nodeMap.set(node.nodeId, node);
|
|
1509
|
+
}
|
|
1510
|
+
const outputNodeMap = new Map();
|
|
1511
|
+
const processedRoot = processA11yTreeDFS(graph.root, null, nodeMap, outputNodeMap);
|
|
1512
|
+
if (processedRoot.length > 1) {
|
|
1513
|
+
throw new Error(`Something went horribly wrong processing the a11y tree, we got: ${JSON.stringify(processedRoot)}`);
|
|
1514
|
+
}
|
|
1515
|
+
else if (processedRoot.length === 0) {
|
|
1516
|
+
throw new EmptyA11yTreeError();
|
|
1517
|
+
}
|
|
1518
|
+
return new ProcessedA11yTree(processedRoot[0], outputNodeMap);
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/browsers/cdp.ts
|
|
1522
|
+
const GREEN = { r: 147, g: 196, b: 125, a: 0.55 };
|
|
1523
|
+
// grabbed from the chrome dev tools "protocol monitor"
|
|
1524
|
+
const NODE_HIGHLIGHT_CONFIG = {
|
|
1525
|
+
showInfo: false,
|
|
1526
|
+
showRulers: false,
|
|
1527
|
+
showStyles: false,
|
|
1528
|
+
showAccessibilityInfo: false,
|
|
1529
|
+
showExtensionLines: false,
|
|
1530
|
+
contrastAlgorithm: "aa",
|
|
1531
|
+
contentColor: GREEN,
|
|
1532
|
+
paddingColor: GREEN,
|
|
1533
|
+
borderColor: GREEN,
|
|
1534
|
+
marginColor: GREEN,
|
|
1535
|
+
eventTargetColor: GREEN,
|
|
1536
|
+
shapeColor: GREEN,
|
|
1537
|
+
shapeMarginColor: GREEN,
|
|
1538
|
+
};
|
|
1539
|
+
const FOCUS_CONFIG_FUNCTION = (/* unused pure expression or super */ null && (`
|
|
1540
|
+
function () {
|
|
1541
|
+
if (this.focus) {
|
|
1542
|
+
this.focus();
|
|
1543
|
+
}
|
|
1544
|
+
return this.click();
|
|
1545
|
+
}
|
|
1546
|
+
`));
|
|
1547
|
+
|
|
1548
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/browsers/constants.ts
|
|
1549
|
+
const RETINA_WINDOW_SCALE_FACTOR = 2;
|
|
1550
|
+
// max time for page to fire load event on navigation
|
|
1551
|
+
const MAX_LOAD_TIMEOUT_MS = 8000;
|
|
1552
|
+
// duration that the network has to remain unchanged before it is considered 'stable'
|
|
1553
|
+
const NETWORK_STABLE_DURATION_MS = 1250;
|
|
1554
|
+
// max time to wait for network idle after page load
|
|
1555
|
+
const NETWORK_IDLE_TIMEOUT_MS = 3000;
|
|
1556
|
+
// how often to check for async conditions to occur (e.g. network idle)
|
|
1557
|
+
const CHECK_INTERVAL_MS = 250;
|
|
1558
|
+
const A11Y_LOAD_TIMEOUT_MS = 1000;
|
|
1559
|
+
// max time to wait for a11y tree to be stable
|
|
1560
|
+
const A11Y_STABLE_TIMEOUT_MS = NETWORK_IDLE_TIMEOUT_MS;
|
|
1561
|
+
// duration that the a11y tree has to remain unchanged before it is considered 'stable'
|
|
1562
|
+
const A11Y_STABLE_DURATION_MS = NETWORK_STABLE_DURATION_MS;
|
|
1563
|
+
// max time an action like a click or typing can take
|
|
1564
|
+
const BROWSER_ACTION_TIMEOUT_MS = 3000;
|
|
1565
|
+
// max time a complicated action like selecting an option can take
|
|
1566
|
+
const COMPLICATED_BROWSER_ACTION_TIMEOUT_MS = MAX_LOAD_TIMEOUT_MS;
|
|
1567
|
+
// duration to highlight an interacted element for
|
|
1568
|
+
const HIGHLIGHT_DURATION_MS = 3000;
|
|
1569
|
+
const CHROME_INTERNAL_URLS = new Set([
|
|
1570
|
+
"about:blank",
|
|
1571
|
+
"chrome-error://chromewebdata/",
|
|
1572
|
+
]);
|
|
1573
|
+
// maximum number of times to attempt to perform a synchronous action
|
|
1574
|
+
// in the browser, like a click
|
|
1575
|
+
const MAX_BROWSER_ACTION_ATTEMPTS = 2;
|
|
1576
|
+
|
|
1577
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/browsers/utils/time.ts
|
|
1578
|
+
const sleep = (ms = 1000) => {
|
|
1579
|
+
return new Promise((resolve) => setTimeout(() => resolve(), ms));
|
|
1580
|
+
};
|
|
1581
|
+
|
|
1582
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/browsers/utils/scripts/cursor.ts
|
|
1583
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
1584
|
+
// @ts-nocheck
|
|
1585
|
+
function addCursorScript() {
|
|
1586
|
+
cursor = document.createElement("img");
|
|
1587
|
+
cursor.setAttribute("src", "data:image/svg+xml;base64,PHN2ZyBoZWlnaHQ9IjMyIiB2aWV3Qm94PSIwIDAgMzIgMzIiIHdpZHRoPSIzMiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48ZyBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiIHRyYW5zZm9ybT0idHJhbnNsYXRlKDEwIDcpIj48cGF0aCBkPSJtNi4xNDggMTguNDczIDEuODYzLTEuMDAzIDEuNjE1LS44MzktMi41NjgtNC44MTZoNC4zMzJsLTExLjM3OS0xMS40MDh2MTYuMDE1bDMuMzE2LTMuMjIxeiIgZmlsbD0iI2ZmZiIvPjxwYXRoIGQ9Im02LjQzMSAxNyAxLjc2NS0uOTQxLTIuNzc1LTUuMjAyaDMuNjA0bC04LjAyNS04LjA0M3YxMS4xODhsMi41My0yLjQ0MnoiIGZpbGw9IiMwMDAiLz48L2c+PC9zdmc+");
|
|
1588
|
+
cursor.setAttribute("id", "selenium_cursor");
|
|
1589
|
+
cursor.setAttribute("style", "position: absolute; z-index: 99999999999; pointer-events: none; left:0; top:0");
|
|
1590
|
+
cursor.style.filter =
|
|
1591
|
+
"invert(0%) sepia(6%) saturate(24%) hue-rotate(315deg) brightness(89%) contrast(110%)";
|
|
1592
|
+
document.body.appendChild(cursor);
|
|
1593
|
+
document.onmousemove = function (e) {
|
|
1594
|
+
e = e || window.event;
|
|
1595
|
+
document.getElementById("selenium_cursor").style.left = e.pageX + "px";
|
|
1596
|
+
document.getElementById("selenium_cursor").style.top = e.pageY + "px";
|
|
1597
|
+
};
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/browsers/utils/scripts/addIDs.ts
|
|
1601
|
+
// It is very important the IDs generated are stable and reproducible
|
|
1602
|
+
// on every page load! This affects caching...a lot!
|
|
1603
|
+
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
|
1604
|
+
// @ts-nocheck
|
|
1605
|
+
function addIDsScript() {
|
|
1606
|
+
// Get all HTML elements on the page
|
|
1607
|
+
const allElements = document.getElementsByTagName("*");
|
|
1608
|
+
let currentID = 1;
|
|
1609
|
+
// Loop through all elements and add the property
|
|
1610
|
+
for (let i = 0; i < allElements.length; i++) {
|
|
1611
|
+
const element = allElements[i];
|
|
1612
|
+
element?.setAttribute("data-momentic-id", currentID);
|
|
1613
|
+
currentID++;
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/browsers/utils/playwright.ts
|
|
1618
|
+
const sometimesRelevantResourceTypes = new Set([
|
|
1619
|
+
"document",
|
|
1620
|
+
"script",
|
|
1621
|
+
"XMLHttpRequest",
|
|
1622
|
+
"fetch",
|
|
1623
|
+
"xhr",
|
|
1624
|
+
]);
|
|
1625
|
+
const alwaysRelevantResourceTypes = new Set(["script", "document"]);
|
|
1626
|
+
const bannedDomains = [
|
|
1627
|
+
"intercom.io",
|
|
1628
|
+
"googletagmanager.com",
|
|
1629
|
+
"google-analytics.com",
|
|
1630
|
+
"www.gstatic.com",
|
|
1631
|
+
"apis.google.com",
|
|
1632
|
+
"sentry.io",
|
|
1633
|
+
"newrelic.com",
|
|
1634
|
+
"p.retool.com",
|
|
1635
|
+
"m.stripe.com",
|
|
1636
|
+
"m.stripe.network",
|
|
1637
|
+
"js.stripe.com",
|
|
1638
|
+
"assets.trybento.co",
|
|
1639
|
+
"udon.trybento.co",
|
|
1640
|
+
"cdn.lr-in-prod.com",
|
|
1641
|
+
"r.lr-in-prod.com",
|
|
1642
|
+
"content.product-usage.assembledhq.com",
|
|
1643
|
+
"data.product-usage.assembledhq.com",
|
|
1644
|
+
"static.zdassets.com",
|
|
1645
|
+
];
|
|
1646
|
+
function serializeRequest(request) {
|
|
1647
|
+
return `${request.resourceType()} ${request.method()} ${request.url()}`;
|
|
1648
|
+
}
|
|
1649
|
+
function stripWWWPrefix(url) {
|
|
1650
|
+
url = url.replace(/^www\./, "");
|
|
1651
|
+
return url;
|
|
1652
|
+
}
|
|
1653
|
+
function isRequestRelevantForPageLoad(request, currentURL) {
|
|
1654
|
+
if (!sometimesRelevantResourceTypes.has(request.resourceType())) {
|
|
1655
|
+
return false;
|
|
1656
|
+
}
|
|
1657
|
+
const parsedCurrentURL = new URL(currentURL);
|
|
1658
|
+
const parsedRequestURL = new URL(request.url());
|
|
1659
|
+
if (bannedDomains.some((domain) => parsedRequestURL.hostname.includes(domain))) {
|
|
1660
|
+
return false;
|
|
1661
|
+
}
|
|
1662
|
+
if (alwaysRelevantResourceTypes.has(request.resourceType())) {
|
|
1663
|
+
return true;
|
|
1664
|
+
}
|
|
1665
|
+
if (request.method() !== "GET") {
|
|
1666
|
+
// XHR requests that modify things are often relevant
|
|
1667
|
+
return true;
|
|
1668
|
+
}
|
|
1669
|
+
// TODO: configure via settings, either for step or globally as to whether we should only wait for the same hostname
|
|
1670
|
+
// allow the request to be to a subdomain (commonly we have api.domain.com or smth like that)
|
|
1671
|
+
return stripWWWPrefix(parsedRequestURL.hostname).includes(stripWWWPrefix(parsedCurrentURL.hostname));
|
|
1672
|
+
}
|
|
1673
|
+
|
|
1674
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/browsers/utils/index.ts
|
|
1675
|
+
|
|
1676
|
+
|
|
1677
|
+
|
|
1678
|
+
|
|
1679
|
+
|
|
1680
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/browsers/chrome.ts
|
|
1681
|
+
|
|
1682
|
+
|
|
1683
|
+
|
|
1684
|
+
|
|
1685
|
+
|
|
1686
|
+
|
|
1687
|
+
/**
|
|
1688
|
+
* Headless chrome browser that has some utility methods to extract DOM information
|
|
1689
|
+
* and perform commands.
|
|
1690
|
+
*
|
|
1691
|
+
* TODO: this should ideally be in its own top-level packages folder because it is not
|
|
1692
|
+
* necessary coupled together with the agent, which covers LLM interaction.
|
|
1693
|
+
*/
|
|
1694
|
+
class ChromeBrowser {
|
|
1695
|
+
constructor({ browser, context, page, baseURL, cdpClient, logger, }) {
|
|
1696
|
+
// key is nodeId, according to the a11y tree
|
|
1697
|
+
this.nodeMap = new Map();
|
|
1698
|
+
this.browser = browser;
|
|
1699
|
+
this.context = context;
|
|
1700
|
+
this.page = page;
|
|
1701
|
+
this.baseURL = baseURL;
|
|
1702
|
+
this.cdpClient = cdpClient;
|
|
1703
|
+
this.logger = logger;
|
|
1704
|
+
}
|
|
1705
|
+
/**
|
|
1706
|
+
* Creates a new browser and waits for navigation to the given test URL.
|
|
1707
|
+
*/
|
|
1708
|
+
static async init(baseURL, logger, onScreenshot, timeout = MAX_LOAD_TIMEOUT_MS) {
|
|
1709
|
+
const browser = await external_playwright_namespaceObject.chromium.launch({ headless: true });
|
|
1710
|
+
const context = await browser.newContext({
|
|
1711
|
+
viewport: {
|
|
1712
|
+
width: 1920,
|
|
1713
|
+
height: 1080,
|
|
1714
|
+
},
|
|
1715
|
+
// comment out the below if you are on Mac OS but you're using a monitor
|
|
1716
|
+
deviceScaleFactor: process.platform === "darwin"
|
|
1717
|
+
? RETINA_WINDOW_SCALE_FACTOR
|
|
1718
|
+
: 1,
|
|
1719
|
+
userAgent: external_playwright_namespaceObject.devices["Desktop Chrome"].userAgent,
|
|
1720
|
+
geolocation: { latitude: 37.7749, longitude: -122.4194 },
|
|
1721
|
+
locale: "en-US",
|
|
1722
|
+
timezoneId: "America/Los_Angeles",
|
|
1723
|
+
});
|
|
1724
|
+
const page = await context.newPage();
|
|
1725
|
+
const cdpClient = await context.newCDPSession(page);
|
|
1726
|
+
const chrome = new ChromeBrowser({
|
|
1727
|
+
browser,
|
|
1728
|
+
context,
|
|
1729
|
+
page,
|
|
1730
|
+
baseURL,
|
|
1731
|
+
cdpClient,
|
|
1732
|
+
logger,
|
|
1733
|
+
});
|
|
1734
|
+
let completed = false;
|
|
1735
|
+
const navigateAndInitCDP = async () => {
|
|
1736
|
+
try {
|
|
1737
|
+
await chrome.navigate(baseURL, false);
|
|
1738
|
+
await cdpClient.send("Accessibility.enable");
|
|
1739
|
+
await cdpClient.send("DOM.enable");
|
|
1740
|
+
await cdpClient.send("Overlay.enable");
|
|
1741
|
+
}
|
|
1742
|
+
catch (err) {
|
|
1743
|
+
logger.error({ err }, "Failed to initialize chrome browser");
|
|
1744
|
+
}
|
|
1745
|
+
finally {
|
|
1746
|
+
completed = true;
|
|
1747
|
+
}
|
|
1748
|
+
};
|
|
1749
|
+
void navigateAndInitCDP();
|
|
1750
|
+
const sendScreenshot = async () => {
|
|
1751
|
+
if (!onScreenshot) {
|
|
1752
|
+
return;
|
|
1753
|
+
}
|
|
1754
|
+
try {
|
|
1755
|
+
onScreenshot({
|
|
1756
|
+
viewport: chrome.viewport,
|
|
1757
|
+
buffer: await chrome.screenshot(),
|
|
1758
|
+
});
|
|
1759
|
+
}
|
|
1760
|
+
catch (err) {
|
|
1761
|
+
logger.error({ err }, "Failed to take screenshot");
|
|
1762
|
+
}
|
|
1763
|
+
};
|
|
1764
|
+
void sendScreenshot();
|
|
1765
|
+
// NOTE: this is a very quick interval because while chome is navigating
|
|
1766
|
+
// we want to show updates to the user ASAP
|
|
1767
|
+
const screenshotInterval = setInterval(() => {
|
|
1768
|
+
void sendScreenshot();
|
|
1769
|
+
}, 250);
|
|
1770
|
+
const startTime = Date.now();
|
|
1771
|
+
while (!completed && Date.now() - startTime < timeout) {
|
|
1772
|
+
await sleep(CHECK_INTERVAL_MS);
|
|
1773
|
+
}
|
|
1774
|
+
clearInterval(screenshotInterval);
|
|
1775
|
+
if (!completed) {
|
|
1776
|
+
logger.warn("Timeout elapsed waiting for browser to initialize - are you sure this page is accessible?");
|
|
1777
|
+
}
|
|
1778
|
+
return chrome;
|
|
1779
|
+
}
|
|
1780
|
+
// Things to do on every page load
|
|
1781
|
+
async pageSetup() {
|
|
1782
|
+
await this.page.evaluate(addCursorScript);
|
|
1783
|
+
await this.page.evaluate(addIDsScript);
|
|
1784
|
+
}
|
|
1785
|
+
async wait(timeoutMs) {
|
|
1786
|
+
await this.page.waitForTimeout(timeoutMs);
|
|
1787
|
+
}
|
|
1788
|
+
async cleanup() {
|
|
1789
|
+
await this.page.close();
|
|
1790
|
+
await this.context.close();
|
|
1791
|
+
await this.browser.close();
|
|
1792
|
+
}
|
|
1793
|
+
get closed() {
|
|
1794
|
+
return this.page.isClosed() || !this.browser.isConnected();
|
|
1795
|
+
}
|
|
1796
|
+
async html() {
|
|
1797
|
+
return await this.page.content();
|
|
1798
|
+
}
|
|
1799
|
+
get url() {
|
|
1800
|
+
return this.page.url();
|
|
1801
|
+
}
|
|
1802
|
+
async screenshot(quality = 100, scale = "device") {
|
|
1803
|
+
return await this.page.screenshot({
|
|
1804
|
+
fullPage: false,
|
|
1805
|
+
quality,
|
|
1806
|
+
scale,
|
|
1807
|
+
type: "jpeg",
|
|
1808
|
+
// allow the blinking text cursor thing to remain there
|
|
1809
|
+
caret: "initial",
|
|
1810
|
+
});
|
|
1811
|
+
}
|
|
1812
|
+
get viewport() {
|
|
1813
|
+
const viewport = this.page.viewportSize();
|
|
1814
|
+
if (!viewport) {
|
|
1815
|
+
throw new Error("failed to get viewport");
|
|
1816
|
+
}
|
|
1817
|
+
return viewport;
|
|
1818
|
+
}
|
|
1819
|
+
async navigate(url,
|
|
1820
|
+
// FIXME: this is an escape hatch to make sure some pages load (assembledhq.com)
|
|
1821
|
+
wrapPossibleNavigation = true) {
|
|
1822
|
+
this.logger.debug(`Navigating to ${url}`);
|
|
1823
|
+
const startTime = Date.now();
|
|
1824
|
+
const doNav = async () => {
|
|
1825
|
+
try {
|
|
1826
|
+
await this.page.goto(url, {
|
|
1827
|
+
timeout: MAX_LOAD_TIMEOUT_MS,
|
|
1828
|
+
});
|
|
1829
|
+
this.logger.debug({ url }, `Got load event in ${Math.floor(Date.now() - startTime)}ms`);
|
|
1830
|
+
}
|
|
1831
|
+
catch (e) {
|
|
1832
|
+
this.logger.warn({ url, type: "navigate", err: e }, "Timeout elapsed waiting for page to load, continuing anyways...");
|
|
1833
|
+
}
|
|
1834
|
+
};
|
|
1835
|
+
if (wrapPossibleNavigation) {
|
|
1836
|
+
await this.wrapPossibleNavigation(doNav);
|
|
1837
|
+
}
|
|
1838
|
+
else {
|
|
1839
|
+
await doNav();
|
|
1840
|
+
}
|
|
1841
|
+
if (CHROME_INTERNAL_URLS.has(this.url) &&
|
|
1842
|
+
process.env.NODE_ENV === "production") {
|
|
1843
|
+
// in dev, this is a little annoying
|
|
1844
|
+
throw new Error(`${url} took too long to load 😞. Please ensure the site and your internet are working.`);
|
|
1845
|
+
}
|
|
1846
|
+
await this.pageSetup();
|
|
1847
|
+
this.logger.debug({ url }, "Navigation complete");
|
|
1848
|
+
}
|
|
1849
|
+
async fill(target, text, options = {}) {
|
|
1850
|
+
const element = await this.click(target, {
|
|
1851
|
+
doubleClick: false,
|
|
1852
|
+
rightClick: false,
|
|
1853
|
+
});
|
|
1854
|
+
await this.type(text, options);
|
|
1855
|
+
return element;
|
|
1856
|
+
}
|
|
1857
|
+
async type(text, options = {}) {
|
|
1858
|
+
const { clearContent = true, pressKeysSequentially = false } = options;
|
|
1859
|
+
if (clearContent) {
|
|
1860
|
+
await this.page.keyboard.press("Meta+A");
|
|
1861
|
+
await this.page.keyboard.press("Backspace");
|
|
1862
|
+
}
|
|
1863
|
+
if (pressKeysSequentially) {
|
|
1864
|
+
await this.page.keyboard.type(text);
|
|
1865
|
+
}
|
|
1866
|
+
else {
|
|
1867
|
+
await this.page.keyboard.insertText(text);
|
|
1868
|
+
}
|
|
1869
|
+
}
|
|
1870
|
+
async clickByA11yID(index, options = {}) {
|
|
1871
|
+
const node = this.nodeMap.get(`${index}`);
|
|
1872
|
+
if (!node) {
|
|
1873
|
+
throw new Error(`Could not find node in DOM with index: ${index}`);
|
|
1874
|
+
}
|
|
1875
|
+
const nodeClicked = await this.clickUsingCDP(node, options);
|
|
1876
|
+
await this.highlightNode(nodeClicked);
|
|
1877
|
+
return node.serialize({ noChildren: true, noProperties: true, noID: true });
|
|
1878
|
+
}
|
|
1879
|
+
async selectOptionByA11yID(index, option) {
|
|
1880
|
+
const node = this.nodeMap.get(`${index}`);
|
|
1881
|
+
if (!node) {
|
|
1882
|
+
throw new Error(`Could not find node in DOM with index: ${index}`);
|
|
1883
|
+
}
|
|
1884
|
+
if (!node.backendNodeID) {
|
|
1885
|
+
throw new Error(`Select target missing backend node id: ${node.getLogForm()}`);
|
|
1886
|
+
}
|
|
1887
|
+
const locator = await this.getLocatorFromBackendID(node.backendNodeID);
|
|
1888
|
+
await locator.selectOption(option, {
|
|
1889
|
+
timeout: COMPLICATED_BROWSER_ACTION_TIMEOUT_MS,
|
|
1890
|
+
});
|
|
1891
|
+
await this.highlightNode(node);
|
|
1892
|
+
return node.serialize({ noChildren: true, noProperties: true, noID: true });
|
|
1893
|
+
}
|
|
1894
|
+
async highlight(target) {
|
|
1895
|
+
try {
|
|
1896
|
+
await this.highlightByA11yID(target.id);
|
|
1897
|
+
}
|
|
1898
|
+
catch (err) {
|
|
1899
|
+
// should never be fatal
|
|
1900
|
+
this.logger.warn({ err, target }, "Failed to highlight target");
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
async highlightByA11yID(index) {
|
|
1904
|
+
const node = this.nodeMap.get(`${index}`);
|
|
1905
|
+
if (!node) {
|
|
1906
|
+
throw new Error(`Could not find node in DOM with index: ${index}`);
|
|
1907
|
+
}
|
|
1908
|
+
if (!node.backendNodeID) {
|
|
1909
|
+
throw new Error(`Select target missing backend node id: ${node.getLogForm()}`);
|
|
1910
|
+
}
|
|
1911
|
+
await this.highlightNode(node);
|
|
1912
|
+
}
|
|
1913
|
+
async highlightNode(node) {
|
|
1914
|
+
try {
|
|
1915
|
+
await this.cdpClient.send("Overlay.highlightNode", {
|
|
1916
|
+
highlightConfig: NODE_HIGHLIGHT_CONFIG,
|
|
1917
|
+
backendNodeId: node.backendNodeID,
|
|
1918
|
+
});
|
|
1919
|
+
}
|
|
1920
|
+
catch (err) {
|
|
1921
|
+
this.logger.warn({ err }, "Failed to add node highlight");
|
|
1922
|
+
}
|
|
1923
|
+
const hideHighlight = async () => {
|
|
1924
|
+
try {
|
|
1925
|
+
await this.cdpClient.send("Overlay.hideHighlight", {
|
|
1926
|
+
backendNodeId: node.backendNodeID,
|
|
1927
|
+
});
|
|
1928
|
+
}
|
|
1929
|
+
catch (err) {
|
|
1930
|
+
// this is okay, purely visual and often occurs due to navigation
|
|
1931
|
+
this.logger.debug({ err }, "Failed to remove node highlight");
|
|
1932
|
+
}
|
|
1933
|
+
};
|
|
1934
|
+
setTimeout(() => {
|
|
1935
|
+
void hideHighlight();
|
|
1936
|
+
}, HIGHLIGHT_DURATION_MS);
|
|
1937
|
+
}
|
|
1938
|
+
async wrapPossibleNavigation(fn, timeoutMS = MAX_LOAD_TIMEOUT_MS) {
|
|
1939
|
+
const startTime = Date.now();
|
|
1940
|
+
const startURL = this.url;
|
|
1941
|
+
let lastRequestReceived = Date.now();
|
|
1942
|
+
const firedRequests = new Map();
|
|
1943
|
+
const finishedRequests = new Map();
|
|
1944
|
+
const requestFinishedListener = (request) => {
|
|
1945
|
+
const key = serializeRequest(request);
|
|
1946
|
+
finishedRequests.set(key, (finishedRequests.get(key) ?? 0) + 1);
|
|
1947
|
+
};
|
|
1948
|
+
const requestFiredListener = (request) => {
|
|
1949
|
+
if (!isRequestRelevantForPageLoad(request, this.url)) {
|
|
1950
|
+
this.logger.debug({
|
|
1951
|
+
uri: serializeRequest(request),
|
|
1952
|
+
}, "Ignoring request for page load network stability");
|
|
1953
|
+
return;
|
|
1954
|
+
}
|
|
1955
|
+
const key = serializeRequest(request);
|
|
1956
|
+
this.logger.debug({
|
|
1957
|
+
uri: key,
|
|
1958
|
+
}, "Request fired on page load, delaying network stability");
|
|
1959
|
+
firedRequests.set(key, (firedRequests.get(key) ?? 0) + 1);
|
|
1960
|
+
lastRequestReceived = Date.now();
|
|
1961
|
+
};
|
|
1962
|
+
this.page.on("requestfinished", requestFinishedListener);
|
|
1963
|
+
this.page.on("request", requestFiredListener);
|
|
1964
|
+
// fire actual function asynchronously
|
|
1965
|
+
// instead of throwing the error, we return it so we can handle it later
|
|
1966
|
+
let rejected = false;
|
|
1967
|
+
const retPromise = fn().catch((e) => {
|
|
1968
|
+
rejected = true;
|
|
1969
|
+
if (e instanceof Error)
|
|
1970
|
+
return e;
|
|
1971
|
+
// we are returning NOT throwing on purpose
|
|
1972
|
+
return new Error(`${e}`);
|
|
1973
|
+
});
|
|
1974
|
+
await sleep(CHECK_INTERVAL_MS);
|
|
1975
|
+
const unwrapAndThrowError = async (p) => {
|
|
1976
|
+
const v = await p;
|
|
1977
|
+
if (v instanceof Error) {
|
|
1978
|
+
throw v;
|
|
1979
|
+
}
|
|
1980
|
+
return v;
|
|
1981
|
+
};
|
|
1982
|
+
// wait for network idle
|
|
1983
|
+
let unfinishedRequests = new Set();
|
|
1984
|
+
const waitForNetworkIdle = async () => {
|
|
1985
|
+
while (!rejected && Date.now() - startTime < timeoutMS) {
|
|
1986
|
+
unfinishedRequests = new Set();
|
|
1987
|
+
await sleep(CHECK_INTERVAL_MS);
|
|
1988
|
+
if (Date.now() - lastRequestReceived <=
|
|
1989
|
+
NETWORK_STABLE_DURATION_MS) {
|
|
1990
|
+
continue;
|
|
1991
|
+
}
|
|
1992
|
+
let anyDifference = false;
|
|
1993
|
+
for (const key of firedRequests.keys()) {
|
|
1994
|
+
if (firedRequests.get(key) !== finishedRequests.get(key)) {
|
|
1995
|
+
this.logger.debug({ uri: key }, "Waiting on request to finish");
|
|
1996
|
+
anyDifference = true;
|
|
1997
|
+
unfinishedRequests.add(key);
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
2000
|
+
if (!anyDifference) {
|
|
2001
|
+
this.logger.debug({
|
|
2002
|
+
url: this.url,
|
|
2003
|
+
requests: JSON.stringify(Array.from(firedRequests.entries())),
|
|
2004
|
+
}, `Network idle in ${Math.floor(Date.now() - startTime)}ms`);
|
|
2005
|
+
return true;
|
|
2006
|
+
}
|
|
2007
|
+
}
|
|
2008
|
+
if (!rejected) {
|
|
2009
|
+
this.logger.warn({
|
|
2010
|
+
url: this.url,
|
|
2011
|
+
requests: JSON.stringify(Array.from(unfinishedRequests.entries())),
|
|
2012
|
+
}, "Timeout elapsed waiting for network idle, continuing anyways...");
|
|
2013
|
+
}
|
|
2014
|
+
return false;
|
|
2015
|
+
};
|
|
2016
|
+
const waitResult = await waitForNetworkIdle();
|
|
2017
|
+
this.page.off("requestfinished", requestFinishedListener);
|
|
2018
|
+
this.page.off("request", requestFiredListener);
|
|
2019
|
+
if (!waitResult) {
|
|
2020
|
+
return unwrapAndThrowError(retPromise);
|
|
2021
|
+
}
|
|
2022
|
+
if (!rejected && urlChanged(this.url, startURL)) {
|
|
2023
|
+
this.logger.debug(`Detected url change in wrapPossibleNavigation, waiting for load state`);
|
|
2024
|
+
try {
|
|
2025
|
+
await this.page.waitForLoadState("load", {
|
|
2026
|
+
timeout: timeoutMS - (Date.now() - startTime),
|
|
2027
|
+
});
|
|
2028
|
+
}
|
|
2029
|
+
catch (e) {
|
|
2030
|
+
this.logger.warn({ url: this.url }, "Timeout elapsed waiting for load state to fire, continuing anyways...");
|
|
2031
|
+
}
|
|
2032
|
+
}
|
|
2033
|
+
return unwrapAndThrowError(retPromise);
|
|
2034
|
+
}
|
|
2035
|
+
async click(target, options = {}) {
|
|
2036
|
+
const elementInteracted = await this.wrapPossibleNavigation(() => this.clickByA11yID(target.id, options));
|
|
2037
|
+
return elementInteracted;
|
|
2038
|
+
}
|
|
2039
|
+
async selectOption(target, option) {
|
|
2040
|
+
return this.selectOptionByA11yID(target.id, option);
|
|
2041
|
+
}
|
|
2042
|
+
async press(key) {
|
|
2043
|
+
await this.wrapPossibleNavigation(() => this.page.keyboard.press(key));
|
|
2044
|
+
}
|
|
2045
|
+
async refresh() {
|
|
2046
|
+
await this.page.reload();
|
|
2047
|
+
await this.pageSetup();
|
|
2048
|
+
}
|
|
2049
|
+
async getA11yTree() {
|
|
2050
|
+
let processedTree = null;
|
|
2051
|
+
let attempt = 0;
|
|
2052
|
+
const url = this.url;
|
|
2053
|
+
while (!processedTree) {
|
|
2054
|
+
try {
|
|
2055
|
+
this.logger.debug(`Getting a11y tree at ${url}`);
|
|
2056
|
+
const graph = await this.getRawA11yTree();
|
|
2057
|
+
if (!graph.root || graph.allNodes.length === 0) {
|
|
2058
|
+
// throw specific error class
|
|
2059
|
+
throw new Error("No a11y tree found on page");
|
|
2060
|
+
}
|
|
2061
|
+
processedTree = processA11yTree(graph);
|
|
2062
|
+
}
|
|
2063
|
+
catch (e) {
|
|
2064
|
+
this.logger.error({ err: e, url }, "Error fetching a11y tree");
|
|
2065
|
+
if (attempt === 0) {
|
|
2066
|
+
await sleep(1000);
|
|
2067
|
+
attempt++;
|
|
2068
|
+
}
|
|
2069
|
+
else {
|
|
2070
|
+
throw new Error(`Max retries exceeded fetching a11y tree: ${e}`);
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
}
|
|
2074
|
+
if (!processedTree.root) {
|
|
2075
|
+
// could be valid!
|
|
2076
|
+
this.logger.warn("A11y tree was pruned entirely");
|
|
2077
|
+
}
|
|
2078
|
+
this.nodeMap = processedTree.nodeMap;
|
|
2079
|
+
return processedTree;
|
|
2080
|
+
}
|
|
2081
|
+
async getRawA11yTree() {
|
|
2082
|
+
const url = this.page.url();
|
|
2083
|
+
let lastTreeUpdateTimestamp = Date.now();
|
|
2084
|
+
const treeUpdateListener = () => {
|
|
2085
|
+
lastTreeUpdateTimestamp = Date.now();
|
|
2086
|
+
};
|
|
2087
|
+
this.cdpClient.addListener("Accessibility.nodesUpdated", treeUpdateListener);
|
|
2088
|
+
let accessibilityTreeLoadFired = false;
|
|
2089
|
+
const accessibilityLoadListener = () => {
|
|
2090
|
+
this.logger.info({ url }, `A11y tree load event fired`);
|
|
2091
|
+
accessibilityTreeLoadFired = true;
|
|
2092
|
+
};
|
|
2093
|
+
this.cdpClient.addListener("Accessibility.loadComplete", accessibilityLoadListener);
|
|
2094
|
+
// make sure the a11y tree hasn't updated in the last 1 second
|
|
2095
|
+
// and the a11y event has fired
|
|
2096
|
+
const a11yLoadStart = Date.now();
|
|
2097
|
+
let timeoutTriggered = true;
|
|
2098
|
+
while (Date.now() - a11yLoadStart < A11Y_STABLE_TIMEOUT_MS) {
|
|
2099
|
+
await sleep(CHECK_INTERVAL_MS);
|
|
2100
|
+
if (!accessibilityTreeLoadFired &&
|
|
2101
|
+
Date.now() - a11yLoadStart < A11Y_LOAD_TIMEOUT_MS) {
|
|
2102
|
+
// some websites never fire the a11y load event
|
|
2103
|
+
// thus, we only allocate 1 second for catching this event
|
|
2104
|
+
this.logger.debug({ url }, `A11y tree not loaded yet, waiting...`);
|
|
2105
|
+
continue;
|
|
2106
|
+
}
|
|
2107
|
+
if (Date.now() - lastTreeUpdateTimestamp >=
|
|
2108
|
+
A11Y_STABLE_DURATION_MS) {
|
|
2109
|
+
this.logger.debug({ url }, `A11y tree not stable yet, waiting...`);
|
|
2110
|
+
continue;
|
|
2111
|
+
}
|
|
2112
|
+
timeoutTriggered = false;
|
|
2113
|
+
break;
|
|
2114
|
+
}
|
|
2115
|
+
this.logger.debug({
|
|
2116
|
+
duration: Date.now() - a11yLoadStart,
|
|
2117
|
+
eventReceived: accessibilityTreeLoadFired,
|
|
2118
|
+
timeoutTriggered,
|
|
2119
|
+
}, "A11y wait phase completed");
|
|
2120
|
+
const { node: root } = await this.cdpClient.send("Accessibility.getRootAXNode");
|
|
2121
|
+
const { nodes } = await this.cdpClient.send("Accessibility.queryAXTree", {
|
|
2122
|
+
backendNodeId: root.backendDOMNodeId,
|
|
2123
|
+
});
|
|
2124
|
+
this.cdpClient.removeListener("Accessibility.loadComplete", accessibilityLoadListener);
|
|
2125
|
+
this.cdpClient.removeListener("Accessibility.nodesUpdated", treeUpdateListener);
|
|
2126
|
+
return {
|
|
2127
|
+
root,
|
|
2128
|
+
allNodes: nodes,
|
|
2129
|
+
};
|
|
2130
|
+
}
|
|
2131
|
+
async clickUsingVisualCoordinates(backendNodeId) {
|
|
2132
|
+
const location = await this.getElementLocation(backendNodeId);
|
|
2133
|
+
if (!location) {
|
|
2134
|
+
throw new Error(`Could not find element location with backend node id: ${backendNodeId}`);
|
|
2135
|
+
}
|
|
2136
|
+
this.logger.debug({ location }, "Executing mouse click");
|
|
2137
|
+
await this.page.mouse.click(location.centerX, location.centerY);
|
|
2138
|
+
}
|
|
2139
|
+
// Get the "id" attribute value from an HTML element.
|
|
2140
|
+
async getIDAttributeUsingCDP(objectId) {
|
|
2141
|
+
// https://bugs.chromium.org/p/chromium/issues/detail?id=1374241
|
|
2142
|
+
await this.cdpClient.send("DOM.getDocument", { depth: 0 });
|
|
2143
|
+
const cdpNodeResult = await this.cdpClient.send("DOM.requestNode", {
|
|
2144
|
+
objectId,
|
|
2145
|
+
});
|
|
2146
|
+
const attrResult = await this.cdpClient.send("DOM.getAttributes", {
|
|
2147
|
+
nodeId: cdpNodeResult.nodeId,
|
|
2148
|
+
});
|
|
2149
|
+
const attributes = attrResult.attributes;
|
|
2150
|
+
const indexAttr = attributes.findIndex((s) => s === "data-momentic-id");
|
|
2151
|
+
if (indexAttr === -1) {
|
|
2152
|
+
return "";
|
|
2153
|
+
}
|
|
2154
|
+
return attributes[indexAttr + 1] || "";
|
|
2155
|
+
}
|
|
2156
|
+
async getLocatorFromBackendID(backendNodeId) {
|
|
2157
|
+
await this.page.evaluate(addIDsScript);
|
|
2158
|
+
// get a remote javascript object from the a11y backend node ID
|
|
2159
|
+
const cdpResolveResult = await this.cdpClient.send("DOM.resolveNode", {
|
|
2160
|
+
backendNodeId,
|
|
2161
|
+
});
|
|
2162
|
+
if (!cdpResolveResult || !cdpResolveResult.object.objectId) {
|
|
2163
|
+
throw new Error(`Could not resolve backend node ${backendNodeId}`);
|
|
2164
|
+
}
|
|
2165
|
+
try {
|
|
2166
|
+
const id = await this.getIDAttributeUsingCDP(cdpResolveResult.object.objectId);
|
|
2167
|
+
if (!id) {
|
|
2168
|
+
throw new Error("Failed getting data-momentic-id attribute using CDP");
|
|
2169
|
+
}
|
|
2170
|
+
return this.page.locator(`[data-momentic-id="${id}"]`);
|
|
2171
|
+
}
|
|
2172
|
+
catch (err) {
|
|
2173
|
+
this.logger.error({
|
|
2174
|
+
err,
|
|
2175
|
+
}, "Failed to get ID attribute");
|
|
2176
|
+
throw err;
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
async clickUsingCDP(originalNode, options = {}) {
|
|
2180
|
+
let clickAttempts = 0;
|
|
2181
|
+
let candidateNode = originalNode;
|
|
2182
|
+
while (clickAttempts < MAX_BROWSER_ACTION_ATTEMPTS) {
|
|
2183
|
+
// Did we reach the root?
|
|
2184
|
+
if (!candidateNode || candidateNode.role === "RootWebArea") {
|
|
2185
|
+
throw new Error(`Attempted to click node with no clickable surrounding elements: ${originalNode.getLogForm()}`);
|
|
2186
|
+
}
|
|
2187
|
+
// Check disqualifying conditions for clicks - these don't count as "attempts"
|
|
2188
|
+
if (candidateNode.role === "StaticText") {
|
|
2189
|
+
// static text corresponds to html text nodes that are not clickable
|
|
2190
|
+
candidateNode = candidateNode.parent;
|
|
2191
|
+
continue;
|
|
2192
|
+
}
|
|
2193
|
+
const candidateNodeID = candidateNode.backendNodeID;
|
|
2194
|
+
if (!candidateNodeID) {
|
|
2195
|
+
this.logger.warn({ node: candidateNode.getLogForm() }, "Click candidate had no backend node ID");
|
|
2196
|
+
candidateNode = candidateNode.parent;
|
|
2197
|
+
continue;
|
|
2198
|
+
}
|
|
2199
|
+
// Attempt to click
|
|
2200
|
+
try {
|
|
2201
|
+
const locator = await this.getLocatorFromBackendID(candidateNodeID);
|
|
2202
|
+
// this timeout is important because playwright checks for clickability/visibility
|
|
2203
|
+
// before clicking, and a timeout indicates obstruction, or disabled state
|
|
2204
|
+
// see: https://playwright.dev/docs/actionability#introduction
|
|
2205
|
+
if (options.doubleClick) {
|
|
2206
|
+
await locator.dblclick({
|
|
2207
|
+
timeout: BROWSER_ACTION_TIMEOUT_MS,
|
|
2208
|
+
});
|
|
2209
|
+
}
|
|
2210
|
+
else {
|
|
2211
|
+
await locator.click({
|
|
2212
|
+
timeout: BROWSER_ACTION_TIMEOUT_MS,
|
|
2213
|
+
button: options.rightClick ? "right" : "left",
|
|
2214
|
+
});
|
|
2215
|
+
}
|
|
2216
|
+
if (candidateNode.id !== originalNode.id) {
|
|
2217
|
+
this.logger.info({
|
|
2218
|
+
oldNode: originalNode.getLogForm(),
|
|
2219
|
+
newNode: candidateNode.getLogForm(),
|
|
2220
|
+
}, `Redirected click successfully to new element`);
|
|
2221
|
+
}
|
|
2222
|
+
return candidateNode;
|
|
2223
|
+
}
|
|
2224
|
+
catch (err) {
|
|
2225
|
+
this.logger.error({ err, node: candidateNode.getLogForm() }, "Failed click or click timed out");
|
|
2226
|
+
clickAttempts++;
|
|
2227
|
+
// try to click the parent instead
|
|
2228
|
+
// we could re-prompt the LLM in the future
|
|
2229
|
+
candidateNode = candidateNode.parent;
|
|
2230
|
+
}
|
|
2231
|
+
}
|
|
2232
|
+
throw new Error(`Max click redirection attempts exhausted on original element: ${originalNode.getLogForm()}`);
|
|
2233
|
+
}
|
|
2234
|
+
/**
|
|
2235
|
+
* Currently unused, but could be useful for vision model integration.
|
|
2236
|
+
* Gets x/y position of an a11y node.
|
|
2237
|
+
*/
|
|
2238
|
+
async getElementLocation(backendNodeId) {
|
|
2239
|
+
const tree = await this.cdpClient.send("DOMSnapshot.captureSnapshot", {
|
|
2240
|
+
computedStyles: [],
|
|
2241
|
+
includeDOMRects: true,
|
|
2242
|
+
includePaintOrder: true,
|
|
2243
|
+
});
|
|
2244
|
+
let devicePixelRatio = await this.page.evaluate(() => window.devicePixelRatio);
|
|
2245
|
+
// it lies on macos lolol
|
|
2246
|
+
// this apparently isn't working when the browser is dragged onto a monitor either
|
|
2247
|
+
if (process.platform === "darwin" && devicePixelRatio === 1) {
|
|
2248
|
+
// UNCOMMENT THE BELOW IF YOU ARE ON MAC OS AND NOT USING A MONITOR
|
|
2249
|
+
// COMMENT THE BELOW OUT IF YOU ARE USING A MONITOR OR NOT ON MAC OS
|
|
2250
|
+
devicePixelRatio = RETINA_WINDOW_SCALE_FACTOR;
|
|
2251
|
+
}
|
|
2252
|
+
const document = tree["documents"][0];
|
|
2253
|
+
const layout = document["layout"];
|
|
2254
|
+
const nodes = document["nodes"];
|
|
2255
|
+
const nodeNames = nodes["nodeName"] || [];
|
|
2256
|
+
const backendNodeIds = nodes["backendNodeId"] || [];
|
|
2257
|
+
const layoutNodeIndex = layout["nodeIndex"];
|
|
2258
|
+
const bounds = layout["bounds"];
|
|
2259
|
+
let cursor = -1;
|
|
2260
|
+
for (let i = 0; i < nodeNames.length; i++) {
|
|
2261
|
+
if (backendNodeIds[i] === backendNodeId) {
|
|
2262
|
+
cursor = layoutNodeIndex.indexOf(i);
|
|
2263
|
+
break;
|
|
2264
|
+
}
|
|
2265
|
+
}
|
|
2266
|
+
if (cursor === -1) {
|
|
2267
|
+
throw new Error(`Could not find any backend node with ID ${backendNodeId}`);
|
|
2268
|
+
}
|
|
2269
|
+
let [x = 0, y = 0, width = 0, height = 0] = bounds[cursor];
|
|
2270
|
+
x /= devicePixelRatio;
|
|
2271
|
+
y /= devicePixelRatio;
|
|
2272
|
+
width /= devicePixelRatio;
|
|
2273
|
+
height /= devicePixelRatio;
|
|
2274
|
+
const centerX = x + width / 2;
|
|
2275
|
+
const centerY = y + height / 2;
|
|
2276
|
+
return { centerX, centerY };
|
|
2277
|
+
}
|
|
2278
|
+
async scrollUp() {
|
|
2279
|
+
// TODO: this works pretty well for full page scroll, in the future we'd need to figure out how to scroll nested containers
|
|
2280
|
+
await this.page.evaluate(() => {
|
|
2281
|
+
(document.scrollingElement || document.body).scrollTop =
|
|
2282
|
+
(document.scrollingElement || document.body).scrollTop -
|
|
2283
|
+
window.innerHeight;
|
|
2284
|
+
});
|
|
2285
|
+
await this.page.evaluate(() => {
|
|
2286
|
+
(document.scrollingElement || document.body).scrollTop =
|
|
2287
|
+
(document.scrollingElement || document.body).scrollTop +
|
|
2288
|
+
window.innerHeight;
|
|
2289
|
+
});
|
|
2290
|
+
}
|
|
2291
|
+
async scrollDown() {
|
|
2292
|
+
await this.page.evaluate(() => {
|
|
2293
|
+
(document.scrollingElement || document.body).scrollTop =
|
|
2294
|
+
(document.scrollingElement || document.body).scrollTop +
|
|
2295
|
+
window.innerHeight;
|
|
2296
|
+
});
|
|
2297
|
+
}
|
|
2298
|
+
async goForward() {
|
|
2299
|
+
await this.wrapPossibleNavigation(() => this.page.goForward({ timeout: MAX_LOAD_TIMEOUT_MS }));
|
|
2300
|
+
await this.pageSetup();
|
|
2301
|
+
}
|
|
2302
|
+
async goBack() {
|
|
2303
|
+
await this.wrapPossibleNavigation(() => this.page.goBack({ timeout: MAX_LOAD_TIMEOUT_MS }));
|
|
2304
|
+
await this.pageSetup();
|
|
2305
|
+
}
|
|
2306
|
+
}
|
|
2307
|
+
ChromeBrowser.USER_AGENT = external_playwright_namespaceObject.devices["Desktop Chrome"].userAgent;
|
|
2308
|
+
|
|
2309
|
+
|
|
2310
|
+
;// CONCATENATED MODULE: external "diff-lines"
|
|
2311
|
+
var external_diff_lines_x = y => { var x = {}; __nccwpck_require__.d(x, y); return x; }
|
|
2312
|
+
var external_diff_lines_y = x => () => x
|
|
2313
|
+
const external_diff_lines_namespaceObject = external_diff_lines_x({ ["default"]: () => __WEBPACK_EXTERNAL_MODULE_diff_lines_24b6f423__["default"] });
|
|
2314
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/controller.ts
|
|
2315
|
+
|
|
2316
|
+
// @ts-expect-error: no types from this library
|
|
2317
|
+
|
|
2318
|
+
|
|
2319
|
+
|
|
2320
|
+
const MAX_HISTORY_CHAR_LENGTH = 10000;
|
|
2321
|
+
class AgentController {
|
|
2322
|
+
constructor({ browser, config, generator, logger }) {
|
|
2323
|
+
this.browser = browser;
|
|
2324
|
+
this.generator = generator;
|
|
2325
|
+
this.config = config;
|
|
2326
|
+
this.logger = logger;
|
|
2327
|
+
this.pendingInstructions = [];
|
|
2328
|
+
this.commandHistory = [];
|
|
2329
|
+
}
|
|
2330
|
+
/**
|
|
2331
|
+
* Get copy of executed commands in human readable form. Most recent is last.
|
|
2332
|
+
* Only commands that have completed execution are returned.
|
|
2333
|
+
*/
|
|
2334
|
+
get history() {
|
|
2335
|
+
return this.commandHistory.filter((cmd) => cmd.state === "DONE");
|
|
2336
|
+
}
|
|
2337
|
+
get lastExecutedCommand() {
|
|
2338
|
+
const history = this.history;
|
|
2339
|
+
if (history.length === 0)
|
|
2340
|
+
return null;
|
|
2341
|
+
const lastEntry = history[history.length - 1];
|
|
2342
|
+
return lastEntry;
|
|
2343
|
+
}
|
|
2344
|
+
/**
|
|
2345
|
+
* Reset the command history provided to agents.
|
|
2346
|
+
* Should be called due to a logical break between commands
|
|
2347
|
+
* such as a SUCCESS being issued.
|
|
2348
|
+
*/
|
|
2349
|
+
resetHistory() {
|
|
2350
|
+
this.commandHistory = [];
|
|
2351
|
+
this.pendingInstructions = [];
|
|
2352
|
+
}
|
|
2353
|
+
/**
|
|
2354
|
+
* Reset controller and browser state.
|
|
2355
|
+
*/
|
|
2356
|
+
async resetState() {
|
|
2357
|
+
this.resetHistory();
|
|
2358
|
+
await this.browser.navigate(this.browser.baseURL);
|
|
2359
|
+
}
|
|
2360
|
+
/**
|
|
2361
|
+
* Get the browser state as a string
|
|
2362
|
+
*/
|
|
2363
|
+
async getBrowserState() {
|
|
2364
|
+
const a11yTree = await this.browser.getA11yTree();
|
|
2365
|
+
return a11yTree.serialize();
|
|
2366
|
+
}
|
|
2367
|
+
getSerializedHistory(url, currentBrowserState) {
|
|
2368
|
+
let history;
|
|
2369
|
+
if (this.config.useHistory === "diff") {
|
|
2370
|
+
history = this.getDiffHistory(url, currentBrowserState);
|
|
2371
|
+
}
|
|
2372
|
+
else {
|
|
2373
|
+
history = this.getListHistory();
|
|
2374
|
+
}
|
|
2375
|
+
return history;
|
|
2376
|
+
}
|
|
2377
|
+
async splitUserGoal(type, goal, disableCache) {
|
|
2378
|
+
if (type === StepType.AI_ACTION &&
|
|
2379
|
+
goal.match(/[,!;.]|(?:and)|(?:then)/) &&
|
|
2380
|
+
this.config.useGoalSplitter) {
|
|
2381
|
+
const granularInstructions = await this.generator.getGranularGoals({ goal, url: this.browser.url }, disableCache);
|
|
2382
|
+
// convert into a stack (last element is first to be executed)
|
|
2383
|
+
this.pendingInstructions = granularInstructions.reverse();
|
|
2384
|
+
}
|
|
2385
|
+
else {
|
|
2386
|
+
this.pendingInstructions = [goal];
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
/**
|
|
2390
|
+
* Given previously executed commands, generate command for the current prompt.
|
|
2391
|
+
* Should only be used for AI action.
|
|
2392
|
+
*/
|
|
2393
|
+
async promptToCommand(type, goal, disableCache) {
|
|
2394
|
+
// are we out of granular instructions to execute?
|
|
2395
|
+
if (this.pendingInstructions.length === 0) {
|
|
2396
|
+
// stores granular instructions in this.pendingInstructions, which functions like a stack
|
|
2397
|
+
await this.splitUserGoal(type, goal, disableCache);
|
|
2398
|
+
}
|
|
2399
|
+
const currInstruction = this.pendingInstructions[this.pendingInstructions.length - 1];
|
|
2400
|
+
this.logger.info({ goal: currInstruction }, "Starting prompt translation");
|
|
2401
|
+
const getBrowserStateStart = Date.now();
|
|
2402
|
+
const url = this.browser.url;
|
|
2403
|
+
const browserState = await this.getBrowserState();
|
|
2404
|
+
this.logger.info({
|
|
2405
|
+
duration: Date.now() - getBrowserStateStart,
|
|
2406
|
+
url,
|
|
2407
|
+
}, "Got browser state");
|
|
2408
|
+
const numPrevious = this.commandHistory.length;
|
|
2409
|
+
this.commandHistory.push({
|
|
2410
|
+
state: "PENDING",
|
|
2411
|
+
browserStateBeforeCommand: browserState,
|
|
2412
|
+
urlBeforeCommand: url,
|
|
2413
|
+
type,
|
|
2414
|
+
});
|
|
2415
|
+
const history = this.getSerializedHistory(url, browserState);
|
|
2416
|
+
const getCommandProposalStart = Date.now();
|
|
2417
|
+
const proposedCommand = await this.generator.getProposedCommand({
|
|
2418
|
+
url,
|
|
2419
|
+
numPrevious,
|
|
2420
|
+
browserState,
|
|
2421
|
+
history,
|
|
2422
|
+
goal: currInstruction,
|
|
2423
|
+
lastCommand: this.lastExecutedCommand,
|
|
2424
|
+
}, disableCache);
|
|
2425
|
+
this.logger.info({ duration: Date.now() - getCommandProposalStart }, "Got proposed command");
|
|
2426
|
+
if (proposedCommand.type === ControlFlowCommandType.SUCCESS) {
|
|
2427
|
+
const finishedInstruction = this.pendingInstructions.pop();
|
|
2428
|
+
this.logger.info({
|
|
2429
|
+
finishedInstruction,
|
|
2430
|
+
remainingInstructions: this.pendingInstructions,
|
|
2431
|
+
}, "Removing pending instruction due to SUCCESS");
|
|
2432
|
+
// promptToCommand will pick the next instruction to execute off the stack
|
|
2433
|
+
if (this.pendingInstructions.length !== 0) {
|
|
2434
|
+
// remove the last command from the history since it was an intermediate command from goalSplitter
|
|
2435
|
+
this.commandHistory.pop();
|
|
2436
|
+
return this.promptToCommand(type, "", disableCache);
|
|
2437
|
+
}
|
|
2438
|
+
}
|
|
2439
|
+
else if (
|
|
2440
|
+
// on failure, we don't continue to execute
|
|
2441
|
+
proposedCommand.type === ControlFlowCommandType.FAILURE) {
|
|
2442
|
+
this.logger.info({
|
|
2443
|
+
remainingInstructions: this.pendingInstructions,
|
|
2444
|
+
}, "Removing pending instructions due to FAILURE");
|
|
2445
|
+
this.pendingInstructions = [];
|
|
2446
|
+
}
|
|
2447
|
+
return proposedCommand;
|
|
2448
|
+
}
|
|
2449
|
+
async locateElement(description, disableCache) {
|
|
2450
|
+
const locator = await this.generator.getElementLocation({ browserState: await this.getBrowserState(), goal: description }, disableCache);
|
|
2451
|
+
if (locator.id < 0) {
|
|
2452
|
+
throw new Error(`Unable to locate element with description: ${description}`);
|
|
2453
|
+
}
|
|
2454
|
+
return locator;
|
|
2455
|
+
}
|
|
2456
|
+
/**
|
|
2457
|
+
* Construct a detailed history that can be passed to the LLM.
|
|
2458
|
+
* History includes commands executed as well as browser state changes that occurred
|
|
2459
|
+
* at each step.
|
|
2460
|
+
*/
|
|
2461
|
+
getDiffHistory(currentURL, currentPageState) {
|
|
2462
|
+
// only include ai actions in the history
|
|
2463
|
+
const doneCommands = this.history.filter((h) => h.type === StepType.AI_ACTION);
|
|
2464
|
+
if (doneCommands.length === 0)
|
|
2465
|
+
return "<NONE/>";
|
|
2466
|
+
const historyLines = [
|
|
2467
|
+
"\nYou have already executed the following commands successfully (most recent listed first)",
|
|
2468
|
+
"-".repeat(10),
|
|
2469
|
+
];
|
|
2470
|
+
doneCommands.reverse().forEach((log, i) => {
|
|
2471
|
+
historyLines.push(`COMMAND ${doneCommands.length - i}${i === 0 ? " (command just executed)" : ""}: ${log.serializedCommand}`);
|
|
2472
|
+
if (i === 0) {
|
|
2473
|
+
// generate page diff, if there was one
|
|
2474
|
+
if (urlChanged(log.urlBeforeCommand, currentURL)) {
|
|
2475
|
+
historyLines.push(` URL CHANGE: '${log.urlBeforeCommand}' -> '${currentURL}'`);
|
|
2476
|
+
}
|
|
2477
|
+
else {
|
|
2478
|
+
const browserStateDiff = (0,external_diff_lines_namespaceObject["default"])(log.browserStateBeforeCommand, currentPageState, {
|
|
2479
|
+
n_surrounding: 1,
|
|
2480
|
+
});
|
|
2481
|
+
if (!browserStateDiff) {
|
|
2482
|
+
historyLines.push("PAGE CONTENT CHANGE: <NONE/>");
|
|
2483
|
+
}
|
|
2484
|
+
else if (browserStateDiff.length < MAX_HISTORY_CHAR_LENGTH) {
|
|
2485
|
+
historyLines.push("PAGE CONTENT CHANGE:");
|
|
2486
|
+
browserStateDiff
|
|
2487
|
+
.split("\n")
|
|
2488
|
+
.forEach((l) => historyLines.push(` ${l}`));
|
|
2489
|
+
}
|
|
2490
|
+
else {
|
|
2491
|
+
historyLines.push("PAGE CONTENT CHANGE: <TOO_LONG_TO_DISPLAY/>");
|
|
2492
|
+
}
|
|
2493
|
+
}
|
|
2494
|
+
}
|
|
2495
|
+
historyLines.push("-".repeat(10));
|
|
2496
|
+
});
|
|
2497
|
+
historyLines.push(`STARTING URL: ${this.browser.baseURL}`);
|
|
2498
|
+
return historyLines.join("\n");
|
|
2499
|
+
}
|
|
2500
|
+
getListHistory() {
|
|
2501
|
+
return external_dedent_namespaceObject["default"] `Here are the commands that you have successfully executed:
|
|
2502
|
+
${this.commandHistory
|
|
2503
|
+
.filter((cmd) => cmd.type === StepType.AI_ACTION)
|
|
2504
|
+
.map((cmd) => `- ${cmd.serializedCommand}`)
|
|
2505
|
+
.join("\n")}`;
|
|
2506
|
+
}
|
|
2507
|
+
/**
|
|
2508
|
+
* Given a command, interact with the chromium browser to actually execute the actions
|
|
2509
|
+
* @param [stateless=false] Execute this command in a stateless fashion, without modifying any controller state such as
|
|
2510
|
+
* pending instructions. Useful when executing cached instructions.
|
|
2511
|
+
*/
|
|
2512
|
+
async executeCommand(command, disableCache, stateless = false) {
|
|
2513
|
+
const pendingHistory = this.commandHistory[this.commandHistory.length - 1];
|
|
2514
|
+
if (!stateless) {
|
|
2515
|
+
// if we're not using cached commands, we must be executing a pending command
|
|
2516
|
+
// generated by promptToCommand
|
|
2517
|
+
if (!pendingHistory || pendingHistory.state !== "PENDING") {
|
|
2518
|
+
throw new Error("Executing command but there is no pending entry in the history");
|
|
2519
|
+
}
|
|
2520
|
+
}
|
|
2521
|
+
else {
|
|
2522
|
+
// cached commands can rely on things like a11y IDs - we need to populate this state in the chrome browser.
|
|
2523
|
+
// currently, all necessary side effects are accomplished by getting the a11y tree
|
|
2524
|
+
await this.browser.getA11yTree();
|
|
2525
|
+
}
|
|
2526
|
+
let result;
|
|
2527
|
+
try {
|
|
2528
|
+
const executionStart = Date.now();
|
|
2529
|
+
result = await this.sendCommandToBrowser(command, disableCache);
|
|
2530
|
+
this.logger.info({ result, duration: Date.now() - executionStart }, "Got execution result");
|
|
2531
|
+
}
|
|
2532
|
+
catch (e) {
|
|
2533
|
+
if (e instanceof Error) {
|
|
2534
|
+
throw new BrowserExecutionError(`Failed to execute command: ${e}`, {
|
|
2535
|
+
cause: e,
|
|
2536
|
+
});
|
|
2537
|
+
}
|
|
2538
|
+
throw new BrowserExecutionError(`Unexpected throw from executing command`, {
|
|
2539
|
+
cause: new Error(`${e}`),
|
|
2540
|
+
});
|
|
2541
|
+
}
|
|
2542
|
+
if (result.succeedImmediately && !stateless) {
|
|
2543
|
+
// pop the last command off the stack since we won't get a real SUCCESS command within promptToCommand
|
|
2544
|
+
this.pendingInstructions.pop();
|
|
2545
|
+
if (this.pendingInstructions.length > 0) {
|
|
2546
|
+
// we still have pending instructions queued up
|
|
2547
|
+
// override the immediate success
|
|
2548
|
+
result.succeedImmediately = false;
|
|
2549
|
+
}
|
|
2550
|
+
}
|
|
2551
|
+
// TODO(ENG-139): Save other a11y stuff as well.
|
|
2552
|
+
// Update the command with the targeted element
|
|
2553
|
+
// if this is the first time the command was generated
|
|
2554
|
+
if (result.elementInteracted &&
|
|
2555
|
+
"target" in command &&
|
|
2556
|
+
!command.target.elementDescriptor) {
|
|
2557
|
+
// Save the serialized element interacted as the "descriptor" for now
|
|
2558
|
+
// In the future, we can ask the LLM for a more human-readable descriptor
|
|
2559
|
+
command.target.elementDescriptor = result.elementInteracted;
|
|
2560
|
+
}
|
|
2561
|
+
if (!stateless) {
|
|
2562
|
+
// the conditional at the beginning of this function validates that pendingHistory isn't undefined
|
|
2563
|
+
// if stateless is false
|
|
2564
|
+
pendingHistory.generatedStep = command;
|
|
2565
|
+
pendingHistory.serializedCommand = serializeAICommand(command);
|
|
2566
|
+
pendingHistory.state = "DONE";
|
|
2567
|
+
}
|
|
2568
|
+
return result;
|
|
2569
|
+
}
|
|
2570
|
+
/**
|
|
2571
|
+
* Executes a preset command.
|
|
2572
|
+
* For most cases, the execution result contains metadata about the command executed.
|
|
2573
|
+
* For assertions, an AssertionResult with thoughts is returned.
|
|
2574
|
+
*/
|
|
2575
|
+
async executePresetStep(command, disableCache) {
|
|
2576
|
+
const urlBeforeCommand = this.browser.url;
|
|
2577
|
+
switch (command.type) {
|
|
2578
|
+
case preset_PresetCommandType.AI_ASSERTION: {
|
|
2579
|
+
let params;
|
|
2580
|
+
if (command.useVision) {
|
|
2581
|
+
params = {
|
|
2582
|
+
goal: command.assertion,
|
|
2583
|
+
url: urlBeforeCommand,
|
|
2584
|
+
// used for vision only
|
|
2585
|
+
screenshot: await this.browser.screenshot(),
|
|
2586
|
+
// unused for visual assertion
|
|
2587
|
+
browserState: "",
|
|
2588
|
+
history: "",
|
|
2589
|
+
numPrevious: -1,
|
|
2590
|
+
lastCommand: null,
|
|
2591
|
+
};
|
|
2592
|
+
}
|
|
2593
|
+
else {
|
|
2594
|
+
const browserState = await this.getBrowserState();
|
|
2595
|
+
const history = this.getSerializedHistory(urlBeforeCommand, browserState);
|
|
2596
|
+
params = {
|
|
2597
|
+
goal: command.assertion,
|
|
2598
|
+
url: urlBeforeCommand,
|
|
2599
|
+
// used for text only
|
|
2600
|
+
browserState,
|
|
2601
|
+
history,
|
|
2602
|
+
lastCommand: this.lastExecutedCommand,
|
|
2603
|
+
numPrevious: this.commandHistory.length,
|
|
2604
|
+
};
|
|
2605
|
+
}
|
|
2606
|
+
const result = await this.generator.getAssertionResult(params, command.useVision, command.disableCache);
|
|
2607
|
+
if (result.relevantElements) {
|
|
2608
|
+
// highlight relevant elements
|
|
2609
|
+
void Promise.all(result.relevantElements.map((id) => this.browser.highlight({ id })));
|
|
2610
|
+
}
|
|
2611
|
+
return result;
|
|
2612
|
+
}
|
|
2613
|
+
case preset_PresetCommandType.NAVIGATE:
|
|
2614
|
+
await this.browser.navigate(command.url);
|
|
2615
|
+
break;
|
|
2616
|
+
case preset_PresetCommandType.GO_BACK:
|
|
2617
|
+
await this.browser.goBack();
|
|
2618
|
+
break;
|
|
2619
|
+
case preset_PresetCommandType.GO_FORWARD:
|
|
2620
|
+
await this.browser.goForward();
|
|
2621
|
+
break;
|
|
2622
|
+
case preset_PresetCommandType.SCROLL_DOWN:
|
|
2623
|
+
await this.browser.scrollDown();
|
|
2624
|
+
break;
|
|
2625
|
+
case preset_PresetCommandType.SCROLL_UP:
|
|
2626
|
+
await this.browser.scrollUp();
|
|
2627
|
+
break;
|
|
2628
|
+
case preset_PresetCommandType.WAIT:
|
|
2629
|
+
await this.browser.wait(command.delay * 1000);
|
|
2630
|
+
break;
|
|
2631
|
+
case preset_PresetCommandType.REFRESH:
|
|
2632
|
+
await this.browser.refresh();
|
|
2633
|
+
break;
|
|
2634
|
+
case preset_PresetCommandType.CLICK: {
|
|
2635
|
+
let id;
|
|
2636
|
+
if (command.target.a11yData) {
|
|
2637
|
+
id = command.target.a11yData?.id;
|
|
2638
|
+
}
|
|
2639
|
+
else {
|
|
2640
|
+
const locator = await this.locateElement(command.target.elementDescriptor, disableCache);
|
|
2641
|
+
id = locator.id;
|
|
2642
|
+
}
|
|
2643
|
+
const elementInteracted = await this.browser.click({
|
|
2644
|
+
id,
|
|
2645
|
+
}, {
|
|
2646
|
+
doubleClick: command.doubleClick,
|
|
2647
|
+
rightClick: command.rightClick,
|
|
2648
|
+
});
|
|
2649
|
+
const result = {
|
|
2650
|
+
type: ExecuteResultType.COMMAND,
|
|
2651
|
+
urlAfterCommand: this.browser.url,
|
|
2652
|
+
succeedImmediately: false,
|
|
2653
|
+
elementInteracted,
|
|
2654
|
+
};
|
|
2655
|
+
if (urlChanged(urlBeforeCommand, result.urlAfterCommand)) {
|
|
2656
|
+
result.succeedImmediately = true;
|
|
2657
|
+
result.succeedImmediatelyReason = "URL changed";
|
|
2658
|
+
}
|
|
2659
|
+
return result;
|
|
2660
|
+
}
|
|
2661
|
+
case preset_PresetCommandType.SELECT_OPTION: {
|
|
2662
|
+
let id;
|
|
2663
|
+
if (command.target.a11yData) {
|
|
2664
|
+
id = command.target.a11yData?.id;
|
|
2665
|
+
}
|
|
2666
|
+
else {
|
|
2667
|
+
const locator = await this.locateElement(command.target.elementDescriptor, disableCache);
|
|
2668
|
+
id = locator.id;
|
|
2669
|
+
}
|
|
2670
|
+
const elementInteracted = await this.browser.selectOption({
|
|
2671
|
+
id,
|
|
2672
|
+
}, command.option);
|
|
2673
|
+
return {
|
|
2674
|
+
type: ExecuteResultType.COMMAND,
|
|
2675
|
+
succeedImmediately: false,
|
|
2676
|
+
urlAfterCommand: this.browser.url,
|
|
2677
|
+
elementInteracted,
|
|
2678
|
+
};
|
|
2679
|
+
}
|
|
2680
|
+
case preset_PresetCommandType.TYPE: {
|
|
2681
|
+
let elementInteracted;
|
|
2682
|
+
const target = command.target;
|
|
2683
|
+
if (target.a11yData) {
|
|
2684
|
+
elementInteracted = await this.browser.click({
|
|
2685
|
+
id: target.a11yData.id,
|
|
2686
|
+
});
|
|
2687
|
+
}
|
|
2688
|
+
else if (target.elementDescriptor.length > 0) {
|
|
2689
|
+
const locator = await this.locateElement(command.target.elementDescriptor, disableCache);
|
|
2690
|
+
elementInteracted = await this.browser.click({
|
|
2691
|
+
id: locator.id,
|
|
2692
|
+
});
|
|
2693
|
+
}
|
|
2694
|
+
await this.browser.type(command.value, {
|
|
2695
|
+
clearContent: command.clearContent,
|
|
2696
|
+
pressKeysSequentially: command.pressKeysSequentially,
|
|
2697
|
+
});
|
|
2698
|
+
if (command.pressEnter) {
|
|
2699
|
+
await this.browser.press("Enter");
|
|
2700
|
+
}
|
|
2701
|
+
const result = {
|
|
2702
|
+
type: ExecuteResultType.COMMAND,
|
|
2703
|
+
urlAfterCommand: this.browser.url,
|
|
2704
|
+
succeedImmediately: false,
|
|
2705
|
+
elementInteracted,
|
|
2706
|
+
};
|
|
2707
|
+
if (urlChanged(urlBeforeCommand, result.urlAfterCommand)) {
|
|
2708
|
+
result.succeedImmediately = true;
|
|
2709
|
+
result.succeedImmediatelyReason = "URL changed";
|
|
2710
|
+
}
|
|
2711
|
+
return result;
|
|
2712
|
+
}
|
|
2713
|
+
case preset_PresetCommandType.PRESS:
|
|
2714
|
+
await this.browser.press(command.value);
|
|
2715
|
+
const result = {
|
|
2716
|
+
type: ExecuteResultType.COMMAND,
|
|
2717
|
+
urlAfterCommand: this.browser.url,
|
|
2718
|
+
succeedImmediately: false,
|
|
2719
|
+
};
|
|
2720
|
+
if (urlChanged(urlBeforeCommand, result.urlAfterCommand)) {
|
|
2721
|
+
result.succeedImmediately = true;
|
|
2722
|
+
result.succeedImmediatelyReason = "URL changed";
|
|
2723
|
+
}
|
|
2724
|
+
return result;
|
|
2725
|
+
default:
|
|
2726
|
+
const assertUnreachable = (_x) => {
|
|
2727
|
+
throw "If Typescript complains about the line below, you missed a case or break in the switch above";
|
|
2728
|
+
};
|
|
2729
|
+
return assertUnreachable(command);
|
|
2730
|
+
}
|
|
2731
|
+
return {
|
|
2732
|
+
type: ExecuteResultType.COMMAND,
|
|
2733
|
+
succeedImmediately: false,
|
|
2734
|
+
urlAfterCommand: this.browser.url,
|
|
2735
|
+
};
|
|
2736
|
+
}
|
|
2737
|
+
async sendCommandToBrowser(command, disableCache) {
|
|
2738
|
+
switch (command.type) {
|
|
2739
|
+
/**
|
|
2740
|
+
* Control flow
|
|
2741
|
+
*/
|
|
2742
|
+
case ControlFlowCommandType.SUCCESS:
|
|
2743
|
+
case ControlFlowCommandType.FAILURE:
|
|
2744
|
+
return {
|
|
2745
|
+
type: ExecuteResultType.COMMAND,
|
|
2746
|
+
succeedImmediately: false,
|
|
2747
|
+
urlAfterCommand: this.browser.url,
|
|
2748
|
+
};
|
|
2749
|
+
/**
|
|
2750
|
+
* Preset action
|
|
2751
|
+
*/
|
|
2752
|
+
default:
|
|
2753
|
+
const result = await this.executePresetStep(command, disableCache);
|
|
2754
|
+
if (result.type !== "command") {
|
|
2755
|
+
// AI should never generate 'assertion' results
|
|
2756
|
+
throw new Error(`Unexpected preset result type ${result.type} from executing AI action`);
|
|
2757
|
+
}
|
|
2758
|
+
return result;
|
|
2759
|
+
}
|
|
2760
|
+
}
|
|
2761
|
+
}
|
|
2762
|
+
|
|
2763
|
+
// EXTERNAL MODULE: ../../node_modules/.pnpm/fetch-retry@5.0.6/node_modules/fetch-retry/index.js
|
|
2764
|
+
var fetch_retry = __nccwpck_require__(62);
|
|
2765
|
+
var fetch_retry_default = /*#__PURE__*/__nccwpck_require__.n(fetch_retry);
|
|
2766
|
+
;// CONCATENATED MODULE: ../../packages/web-agent/src/generators/api-generator.ts
|
|
2767
|
+
|
|
2768
|
+
|
|
2769
|
+
|
|
2770
|
+
const fetch = fetch_retry_default()(global.fetch);
|
|
2771
|
+
const API_VERSION = "v1";
|
|
2772
|
+
class APIGenerator {
|
|
2773
|
+
constructor(params) {
|
|
2774
|
+
this.baseURL = params.baseURL;
|
|
2775
|
+
this.apiKey = params.apiKey;
|
|
2776
|
+
}
|
|
2777
|
+
async getElementLocation(context, disableCache) {
|
|
2778
|
+
const result = await this.sendRequest(`/${API_VERSION}/web-agent/locate-element`, {
|
|
2779
|
+
browserState: context.browserState,
|
|
2780
|
+
goal: context.goal,
|
|
2781
|
+
disableCache,
|
|
2782
|
+
});
|
|
2783
|
+
return locator_AILocatorSchema.parse(result);
|
|
2784
|
+
}
|
|
2785
|
+
async getAssertionResult(context, useVision, disableCache) {
|
|
2786
|
+
if (useVision) {
|
|
2787
|
+
const result = await this.sendRequest(`/${API_VERSION}/web-agent/assertion`, {
|
|
2788
|
+
url: context.url,
|
|
2789
|
+
goal: context.goal,
|
|
2790
|
+
screenshot: context.screenshot?.toString("base64"),
|
|
2791
|
+
disableCache,
|
|
2792
|
+
vision: true,
|
|
2793
|
+
});
|
|
2794
|
+
return execute_results_ExecuteAssertionResultSchema.parse(result);
|
|
2795
|
+
}
|
|
2796
|
+
const result = await this.sendRequest(`/${API_VERSION}/web-agent/assertion`, {
|
|
2797
|
+
url: context.url,
|
|
2798
|
+
browserState: context.browserState,
|
|
2799
|
+
goal: context.goal,
|
|
2800
|
+
history: context.history,
|
|
2801
|
+
numPrevious: context.numPrevious,
|
|
2802
|
+
lastCommand: context.lastCommand,
|
|
2803
|
+
disableCache,
|
|
2804
|
+
vision: false,
|
|
2805
|
+
});
|
|
2806
|
+
return execute_results_ExecuteAssertionResultSchema.parse(result);
|
|
2807
|
+
}
|
|
2808
|
+
async getProposedCommand(context, disableCache) {
|
|
2809
|
+
const result = await this.sendRequest(`/${API_VERSION}/web-agent/next-command`, {
|
|
2810
|
+
url: context.url,
|
|
2811
|
+
browserState: context.browserState,
|
|
2812
|
+
goal: context.goal,
|
|
2813
|
+
history: context.history,
|
|
2814
|
+
numPrevious: context.numPrevious,
|
|
2815
|
+
lastCommand: context.lastCommand,
|
|
2816
|
+
disableCache,
|
|
2817
|
+
});
|
|
2818
|
+
return ai_commands_AICommandSchema.parse(result);
|
|
2819
|
+
}
|
|
2820
|
+
async getGranularGoals(context, disableCache) {
|
|
2821
|
+
const result = await this.sendRequest(`/${API_VERSION}/web-agent/split-goal`, {
|
|
2822
|
+
url: context.url,
|
|
2823
|
+
goal: context.goal,
|
|
2824
|
+
disableCache,
|
|
2825
|
+
});
|
|
2826
|
+
return external_zod_namespaceObject.string().array().parse(result);
|
|
2827
|
+
}
|
|
2828
|
+
async sendRequest(path, body) {
|
|
2829
|
+
const response = await fetch(`${this.baseURL}${path}`, {
|
|
2830
|
+
retries: 3,
|
|
2831
|
+
retryDelay: 1000,
|
|
2832
|
+
method: "POST",
|
|
2833
|
+
body: JSON.stringify(body),
|
|
2834
|
+
headers: {
|
|
2835
|
+
"Content-Type": "application/json",
|
|
2836
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
2837
|
+
},
|
|
2838
|
+
});
|
|
2839
|
+
if (!response.ok) {
|
|
2840
|
+
throw new Error(`Request to ${path} failed with status ${response.status}: ${await response.text()}`);
|
|
2841
|
+
}
|
|
2842
|
+
return response.json();
|
|
2843
|
+
}
|
|
2844
|
+
}
|
|
2845
|
+
|
|
2846
|
+
;// CONCATENATED MODULE: ./src/index.ts
|
|
2847
|
+
// NOTE: these paths are using the direct paths to these files to support treeshaking so we don't bundle unnecessary code
|
|
2848
|
+
|
|
2849
|
+
|
|
2850
|
+
|
|
2851
|
+
|
|
2852
|
+
|
|
2853
|
+
})();
|
|
2854
|
+
|
|
2855
|
+
var __webpack_exports__APIGenerator = __webpack_exports__._w;
|
|
2856
|
+
var __webpack_exports__AgentController = __webpack_exports__.Yt;
|
|
2857
|
+
var __webpack_exports__ChromeBrowser = __webpack_exports__.DE;
|
|
2858
|
+
export { __webpack_exports__APIGenerator as APIGenerator, __webpack_exports__AgentController as AgentController, __webpack_exports__ChromeBrowser as ChromeBrowser };
|