momentic 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/dist/index.js +1107 -978
  2. package/package.json +2 -3
package/dist/index.js CHANGED
@@ -1,8 +1,6 @@
1
- import * as __WEBPACK_EXTERNAL_MODULE_playwright__ from "playwright";
2
- import * as __WEBPACK_EXTERNAL_MODULE_zod__ from "zod";
3
- import * as __WEBPACK_EXTERNAL_MODULE_dedent__ from "dedent";
4
- import * as __WEBPACK_EXTERNAL_MODULE_diff_lines_24b6f423__ from "diff-lines";
5
- /******/ var __webpack_modules__ = ({
1
+ /******/ (() => { // webpackBootstrap
2
+ /******/ "use strict";
3
+ /******/ var __webpack_modules__ = ({
6
4
 
7
5
  /***/ 909:
8
6
  /***/ (function(__unused_webpack_module, exports) {
@@ -335,88 +333,99 @@ function ArgumentError(message) {
335
333
 
336
334
  /***/ })
337
335
 
338
- /******/ });
336
+ /******/ });
339
337
  /************************************************************************/
340
- /******/ // The module cache
341
- /******/ var __webpack_module_cache__ = {};
342
- /******/
343
- /******/ // The require function
344
- /******/ function __nccwpck_require__(moduleId) {
345
- /******/ // Check if module is in cache
346
- /******/ var cachedModule = __webpack_module_cache__[moduleId];
347
- /******/ if (cachedModule !== undefined) {
348
- /******/ return cachedModule.exports;
349
- /******/ }
350
- /******/ // Create a new module (and put it into the cache)
351
- /******/ var module = __webpack_module_cache__[moduleId] = {
352
- /******/ // no module.id needed
353
- /******/ // no module.loaded needed
354
- /******/ exports: {}
355
- /******/ };
356
- /******/
357
- /******/ // Execute the module function
358
- /******/ var threw = true;
359
- /******/ try {
360
- /******/ __webpack_modules__[moduleId].call(module.exports, module, module.exports, __nccwpck_require__);
361
- /******/ threw = false;
362
- /******/ } finally {
363
- /******/ if(threw) delete __webpack_module_cache__[moduleId];
338
+ /******/ // The module cache
339
+ /******/ var __webpack_module_cache__ = {};
340
+ /******/
341
+ /******/ // The require function
342
+ /******/ function __nccwpck_require__(moduleId) {
343
+ /******/ // Check if module is in cache
344
+ /******/ var cachedModule = __webpack_module_cache__[moduleId];
345
+ /******/ if (cachedModule !== undefined) {
346
+ /******/ return cachedModule.exports;
347
+ /******/ }
348
+ /******/ // Create a new module (and put it into the cache)
349
+ /******/ var module = __webpack_module_cache__[moduleId] = {
350
+ /******/ // no module.id needed
351
+ /******/ // no module.loaded needed
352
+ /******/ exports: {}
353
+ /******/ };
354
+ /******/
355
+ /******/ // Execute the module function
356
+ /******/ var threw = true;
357
+ /******/ try {
358
+ /******/ __webpack_modules__[moduleId].call(module.exports, module, module.exports, __nccwpck_require__);
359
+ /******/ threw = false;
360
+ /******/ } finally {
361
+ /******/ if(threw) delete __webpack_module_cache__[moduleId];
362
+ /******/ }
363
+ /******/
364
+ /******/ // Return the exports of the module
365
+ /******/ return module.exports;
364
366
  /******/ }
365
- /******/
366
- /******/ // Return the exports of the module
367
- /******/ return module.exports;
368
- /******/ }
369
- /******/
367
+ /******/
370
368
  /************************************************************************/
371
- /******/ /* webpack/runtime/compat get default export */
372
- /******/ (() => {
373
- /******/ // getDefaultExport function for compatibility with non-harmony modules
374
- /******/ __nccwpck_require__.n = (module) => {
375
- /******/ var getter = module && module.__esModule ?
376
- /******/ () => (module['default']) :
377
- /******/ () => (module);
378
- /******/ __nccwpck_require__.d(getter, { a: getter });
379
- /******/ return getter;
380
- /******/ };
381
- /******/ })();
382
- /******/
383
- /******/ /* webpack/runtime/define property getters */
384
- /******/ (() => {
385
- /******/ // define getter functions for harmony exports
386
- /******/ __nccwpck_require__.d = (exports, definition) => {
387
- /******/ for(var key in definition) {
388
- /******/ if(__nccwpck_require__.o(definition, key) && !__nccwpck_require__.o(exports, key)) {
389
- /******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });
369
+ /******/ /* webpack/runtime/compat get default export */
370
+ /******/ (() => {
371
+ /******/ // getDefaultExport function for compatibility with non-harmony modules
372
+ /******/ __nccwpck_require__.n = (module) => {
373
+ /******/ var getter = module && module.__esModule ?
374
+ /******/ () => (module['default']) :
375
+ /******/ () => (module);
376
+ /******/ __nccwpck_require__.d(getter, { a: getter });
377
+ /******/ return getter;
378
+ /******/ };
379
+ /******/ })();
380
+ /******/
381
+ /******/ /* webpack/runtime/define property getters */
382
+ /******/ (() => {
383
+ /******/ // define getter functions for harmony exports
384
+ /******/ __nccwpck_require__.d = (exports, definition) => {
385
+ /******/ for(var key in definition) {
386
+ /******/ if(__nccwpck_require__.o(definition, key) && !__nccwpck_require__.o(exports, key)) {
387
+ /******/ Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });
388
+ /******/ }
390
389
  /******/ }
391
- /******/ }
392
- /******/ };
393
- /******/ })();
394
- /******/
395
- /******/ /* webpack/runtime/hasOwnProperty shorthand */
396
- /******/ (() => {
397
- /******/ __nccwpck_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))
398
- /******/ })();
399
- /******/
400
- /******/ /* webpack/runtime/compat */
401
- /******/
402
- /******/ if (typeof __nccwpck_require__ !== 'undefined') __nccwpck_require__.ab = new URL('.', import.meta.url).pathname.slice(import.meta.url.match(/^file:\/\/\/\w:/) ? 1 : 0, -1) + "/";
403
- /******/
390
+ /******/ };
391
+ /******/ })();
392
+ /******/
393
+ /******/ /* webpack/runtime/hasOwnProperty shorthand */
394
+ /******/ (() => {
395
+ /******/ __nccwpck_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))
396
+ /******/ })();
397
+ /******/
398
+ /******/ /* webpack/runtime/make namespace object */
399
+ /******/ (() => {
400
+ /******/ // define __esModule on exports
401
+ /******/ __nccwpck_require__.r = (exports) => {
402
+ /******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) {
403
+ /******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
404
+ /******/ }
405
+ /******/ Object.defineProperty(exports, '__esModule', { value: true });
406
+ /******/ };
407
+ /******/ })();
408
+ /******/
409
+ /******/ /* webpack/runtime/compat */
410
+ /******/
411
+ /******/ if (typeof __nccwpck_require__ !== 'undefined') __nccwpck_require__.ab = __dirname + "/";
412
+ /******/
404
413
  /************************************************************************/
405
414
  var __webpack_exports__ = {};
406
415
  // This entry need to be wrapped in an IIFE because it need to be isolated against other modules in the chunk.
407
416
  (() => {
417
+ // ESM COMPAT FLAG
418
+ __nccwpck_require__.r(__webpack_exports__);
408
419
 
409
420
  // EXPORTS
410
421
  __nccwpck_require__.d(__webpack_exports__, {
411
- "_w": () => (/* reexport */ APIGenerator),
412
- "Yt": () => (/* reexport */ AgentController),
413
- "DE": () => (/* reexport */ ChromeBrowser)
422
+ "APIGenerator": () => (/* reexport */ APIGenerator),
423
+ "AgentController": () => (/* reexport */ AgentController),
424
+ "ChromeBrowser": () => (/* reexport */ ChromeBrowser)
414
425
  });
415
426
 
416
427
  ;// CONCATENATED MODULE: external "playwright"
417
- var x = y => { var x = {}; __nccwpck_require__.d(x, y); return x; }
418
- var y = x => () => x
419
- const external_playwright_namespaceObject = x({ ["chromium"]: () => __WEBPACK_EXTERNAL_MODULE_playwright__.chromium, ["devices"]: () => __WEBPACK_EXTERNAL_MODULE_playwright__.devices });
428
+ const external_playwright_namespaceObject = require("playwright");
420
429
  ;// CONCATENATED MODULE: ../../packages/web-agent/src/utils/url.ts
421
430
  // Returns true if 2 urls are different, ignoring differences in query params.
422
431
  const urlChanged = (url1, url2) => {
@@ -433,9 +442,7 @@ var AgentType;
433
442
  })(AgentType || (AgentType = {}));
434
443
 
435
444
  ;// CONCATENATED MODULE: external "zod"
436
- var external_zod_x = y => { var x = {}; __nccwpck_require__.d(x, y); return x; }
437
- var external_zod_y = x => () => x
438
- const external_zod_namespaceObject = external_zod_x({ ["array"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.array, ["boolean"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.boolean, ["coerce"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.coerce, ["discriminatedUnion"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.discriminatedUnion, ["literal"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.literal, ["nativeEnum"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.nativeEnum, ["null"]: () => __WEBPACK_EXTERNAL_MODULE_zod__["null"], ["number"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.number, ["object"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.object, ["string"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.string, ["union"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.union, ["z"]: () => __WEBPACK_EXTERNAL_MODULE_zod__.z });
445
+ const external_zod_namespaceObject = require("zod");
439
446
  ;// CONCATENATED MODULE: ../../packages/types/src/a11y-targets.ts
440
447
 
441
448
  const A11yTargetWithCacheSchema = external_zod_namespaceObject.object({
@@ -481,9 +488,8 @@ class EmptyA11yTreeError extends Error {
481
488
  }
482
489
 
483
490
  ;// CONCATENATED MODULE: external "dedent"
484
- var external_dedent_x = y => { var x = {}; __nccwpck_require__.d(x, y); return x; }
485
- var external_dedent_y = x => () => x
486
- const external_dedent_namespaceObject = external_dedent_x({ ["default"]: () => __WEBPACK_EXTERNAL_MODULE_dedent__["default"] });
491
+ const external_dedent_namespaceObject = require("dedent");
492
+ var external_dedent_default = /*#__PURE__*/__nccwpck_require__.n(external_dedent_namespaceObject);
487
493
  ;// CONCATENATED MODULE: ../../packages/types/src/preset.ts
488
494
 
489
495
 
@@ -541,7 +547,7 @@ const ClickCommandSchema = CommonCommandSchema.merge(external_zod_namespaceObjec
541
547
  target: ElementDescriptorSchema,
542
548
  doubleClick: external_zod_namespaceObject.boolean().default(false),
543
549
  rightClick: external_zod_namespaceObject.boolean().default(false),
544
- })).describe(external_dedent_namespaceObject["default"] `CLICK <id> - click on the element that has the specified id.
550
+ })).describe((external_dedent_default()) `CLICK <id> - click on the element that has the specified id.
545
551
  You are NOT allowed to click on disabled, hidden or StaticText elements.
546
552
  Only click on elements on the Current Page.
547
553
  Only click on elements with the following tag names: button, input, link, image, generic.
@@ -1073,6 +1079,7 @@ function serializeAICommand(cmd) {
1073
1079
  return humanSummary;
1074
1080
  }
1075
1081
  function serializePresetCommand(command) {
1082
+ var _a;
1076
1083
  switch (command.type) {
1077
1084
  case preset_PresetCommandType.NAVIGATE:
1078
1085
  return `Go to URL: ${clampText(command.url, 30)}`;
@@ -1092,7 +1099,7 @@ function serializePresetCommand(command) {
1092
1099
  return `Click on '${command.target.elementDescriptor}'`;
1093
1100
  case preset_PresetCommandType.TYPE:
1094
1101
  let serializedTarget = "";
1095
- if (command.target.a11yData?.serializedForm) {
1102
+ if ((_a = command.target.a11yData) === null || _a === void 0 ? void 0 : _a.serializedForm) {
1096
1103
  serializedTarget = ` in element ${command.target.a11yData.serializedForm}`;
1097
1104
  }
1098
1105
  else if (command.target.elementDescriptor.length > 0) {
@@ -1277,10 +1284,11 @@ class ProcessedA11yNode {
1277
1284
  this.backendNodeID = params.backendNodeID;
1278
1285
  }
1279
1286
  getLogForm() {
1287
+ var _a, _b;
1280
1288
  return JSON.stringify({
1281
1289
  id: this.id,
1282
- name: this.name ?? "",
1283
- role: this.role ?? "",
1290
+ name: (_a = this.name) !== null && _a !== void 0 ? _a : "",
1291
+ role: (_b = this.role) !== null && _b !== void 0 ? _b : "",
1284
1292
  backendNodeId: this.backendNodeID,
1285
1293
  });
1286
1294
  }
@@ -1372,10 +1380,11 @@ class ProcessedA11yTree {
1372
1380
  * May not be unique in a graph.
1373
1381
  */
1374
1382
  function getNodePathIdentifier(node) {
1375
- if (node.name?.value) {
1383
+ var _a, _b;
1384
+ if ((_a = node.name) === null || _a === void 0 ? void 0 : _a.value) {
1376
1385
  return `"${node.name.value}"`;
1377
1386
  }
1378
- if (node.role?.value &&
1387
+ if (((_b = node.role) === null || _b === void 0 ? void 0 : _b.value) &&
1379
1388
  node.role.value !== "none" &&
1380
1389
  node.role.value !== "generic") {
1381
1390
  return `"${node.role.value}"`;
@@ -1383,22 +1392,23 @@ function getNodePathIdentifier(node) {
1383
1392
  return `"${node.nodeId}"`;
1384
1393
  }
1385
1394
  function processA11yTreeDFS(node, parent, inputNodeMap, outputNodeMap) {
1395
+ var _a, _b, _c, _d, _e, _f, _g;
1386
1396
  if (!parent && node.parentId) {
1387
1397
  throw new Error(`Got no parent for accessibility node ${node.nodeId}: ${JSON.stringify(node)}`);
1388
1398
  }
1389
1399
  const processedNode = new ProcessedA11yNode({
1390
1400
  id: node.nodeId,
1391
- role: node.role?.value || "",
1392
- name: node.name?.value || "",
1393
- content: node.value?.value || "",
1401
+ role: ((_a = node.role) === null || _a === void 0 ? void 0 : _a.value) || "",
1402
+ name: ((_b = node.name) === null || _b === void 0 ? void 0 : _b.value) || "",
1403
+ content: ((_c = node.value) === null || _c === void 0 ? void 0 : _c.value) || "",
1394
1404
  properties: {},
1395
1405
  children: [],
1396
1406
  pathFromRoot: (parent ? `${parent.pathFromRoot} ` : "") + getNodePathIdentifier(node),
1397
1407
  backendNodeID: node.backendDOMNodeId,
1398
1408
  // md5Sum: "",
1399
1409
  });
1400
- if (node.value?.value) {
1401
- processedNode.content = `${node.value?.value}`;
1410
+ if ((_d = node.value) === null || _d === void 0 ? void 0 : _d.value) {
1411
+ processedNode.content = `${(_e = node.value) === null || _e === void 0 ? void 0 : _e.value}`;
1402
1412
  }
1403
1413
  if (node.properties) {
1404
1414
  node.properties.forEach((prop) => {
@@ -1406,7 +1416,7 @@ function processA11yTreeDFS(node, parent, inputNodeMap, outputNodeMap) {
1406
1416
  });
1407
1417
  }
1408
1418
  outputNodeMap.set(processedNode.id, processedNode);
1409
- const children = node.childIds ?? [];
1419
+ const children = (_f = node.childIds) !== null && _f !== void 0 ? _f : [];
1410
1420
  for (const childId of children) {
1411
1421
  if (!childId) {
1412
1422
  continue;
@@ -1430,7 +1440,7 @@ function processA11yTreeDFS(node, parent, inputNodeMap, outputNodeMap) {
1430
1440
  if (processedNode.children.length === 1 &&
1431
1441
  processedNode.children[0].role === "StaticText") {
1432
1442
  const currentName = processedNode.name;
1433
- const childName = processedNode.children[0]?.name;
1443
+ const childName = (_g = processedNode.children[0]) === null || _g === void 0 ? void 0 : _g.name;
1434
1444
  if (currentName === childName || !childName) {
1435
1445
  processedNode.children = [];
1436
1446
  }
@@ -1496,12 +1506,16 @@ function processA11yTree(graph) {
1496
1506
  }
1497
1507
  // filter out nodes that are no longer rendered on the page
1498
1508
  graph.allNodes = graph.allNodes.filter((node) => {
1509
+ var _a;
1499
1510
  if (!node.ignored) {
1500
1511
  return true;
1501
1512
  }
1502
1513
  // CDP types are wrong; notRendered is a possible ignored reason
1503
- return !node.ignoredReasons?.find((reason) => reason.name === "notRendered" &&
1504
- reason.value?.value);
1514
+ return !((_a = node.ignoredReasons) === null || _a === void 0 ? void 0 : _a.find((reason) => {
1515
+ var _a;
1516
+ return reason.name === "notRendered" &&
1517
+ ((_a = reason.value) === null || _a === void 0 ? void 0 : _a.value);
1518
+ }));
1505
1519
  });
1506
1520
  const nodeMap = new Map();
1507
1521
  for (const node of graph.allNodes) {
@@ -1609,7 +1623,7 @@ function addIDsScript() {
1609
1623
  // Loop through all elements and add the property
1610
1624
  for (let i = 0; i < allElements.length; i++) {
1611
1625
  const element = allElements[i];
1612
- element?.setAttribute("data-momentic-id", currentID);
1626
+ element === null || element === void 0 ? void 0 : element.setAttribute("data-momentic-id", currentID);
1613
1627
  currentID++;
1614
1628
  }
1615
1629
  }
@@ -1678,6 +1692,15 @@ function isRequestRelevantForPageLoad(request, currentURL) {
1678
1692
 
1679
1693
 
1680
1694
  ;// CONCATENATED MODULE: ../../packages/web-agent/src/browsers/chrome.ts
1695
+ var __awaiter = (undefined && undefined.__awaiter) || function (thisArg, _arguments, P, generator) {
1696
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
1697
+ return new (P || (P = Promise))(function (resolve, reject) {
1698
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
1699
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
1700
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
1701
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
1702
+ });
1703
+ };
1681
1704
 
1682
1705
 
1683
1706
 
@@ -1705,108 +1728,120 @@ class ChromeBrowser {
1705
1728
  /**
1706
1729
  * Creates a new browser and waits for navigation to the given test URL.
1707
1730
  */
1708
- static async init(baseURL, logger, onScreenshot, timeout = MAX_LOAD_TIMEOUT_MS) {
1709
- const browser = await external_playwright_namespaceObject.chromium.launch({ headless: true });
1710
- const context = await browser.newContext({
1711
- viewport: {
1712
- width: 1920,
1713
- height: 1080,
1714
- },
1715
- // comment out the below if you are on Mac OS but you're using a monitor
1716
- deviceScaleFactor: process.platform === "darwin"
1717
- ? RETINA_WINDOW_SCALE_FACTOR
1718
- : 1,
1719
- userAgent: external_playwright_namespaceObject.devices["Desktop Chrome"].userAgent,
1720
- geolocation: { latitude: 37.7749, longitude: -122.4194 },
1721
- locale: "en-US",
1722
- timezoneId: "America/Los_Angeles",
1723
- });
1724
- const page = await context.newPage();
1725
- const cdpClient = await context.newCDPSession(page);
1726
- const chrome = new ChromeBrowser({
1727
- browser,
1728
- context,
1729
- page,
1730
- baseURL,
1731
- cdpClient,
1732
- logger,
1733
- });
1734
- let completed = false;
1735
- const navigateAndInitCDP = async () => {
1736
- try {
1737
- await chrome.navigate(baseURL, false);
1738
- await cdpClient.send("Accessibility.enable");
1739
- await cdpClient.send("DOM.enable");
1740
- await cdpClient.send("Overlay.enable");
1741
- }
1742
- catch (err) {
1743
- logger.error({ err }, "Failed to initialize chrome browser");
1744
- }
1745
- finally {
1746
- completed = true;
1747
- }
1748
- };
1749
- void navigateAndInitCDP();
1750
- const sendScreenshot = async () => {
1751
- if (!onScreenshot) {
1752
- return;
1753
- }
1754
- try {
1755
- onScreenshot({
1756
- viewport: chrome.viewport,
1757
- buffer: await chrome.screenshot(),
1758
- });
1731
+ static init(baseURL, logger, onScreenshot, timeout = MAX_LOAD_TIMEOUT_MS) {
1732
+ return __awaiter(this, void 0, void 0, function* () {
1733
+ const browser = yield external_playwright_namespaceObject.chromium.launch({ headless: true });
1734
+ const context = yield browser.newContext({
1735
+ viewport: {
1736
+ width: 1920,
1737
+ height: 1080,
1738
+ },
1739
+ // comment out the below if you are on Mac OS but you're using a monitor
1740
+ deviceScaleFactor: process.platform === "darwin"
1741
+ ? RETINA_WINDOW_SCALE_FACTOR
1742
+ : 1,
1743
+ userAgent: external_playwright_namespaceObject.devices["Desktop Chrome"].userAgent,
1744
+ geolocation: { latitude: 37.7749, longitude: -122.4194 },
1745
+ locale: "en-US",
1746
+ timezoneId: "America/Los_Angeles",
1747
+ });
1748
+ const page = yield context.newPage();
1749
+ const cdpClient = yield context.newCDPSession(page);
1750
+ const chrome = new ChromeBrowser({
1751
+ browser,
1752
+ context,
1753
+ page,
1754
+ baseURL,
1755
+ cdpClient,
1756
+ logger,
1757
+ });
1758
+ let completed = false;
1759
+ const navigateAndInitCDP = () => __awaiter(this, void 0, void 0, function* () {
1760
+ try {
1761
+ yield chrome.navigate(baseURL, false);
1762
+ yield cdpClient.send("Accessibility.enable");
1763
+ yield cdpClient.send("DOM.enable");
1764
+ yield cdpClient.send("Overlay.enable");
1765
+ }
1766
+ catch (err) {
1767
+ logger.error({ err }, "Failed to initialize chrome browser");
1768
+ }
1769
+ finally {
1770
+ completed = true;
1771
+ }
1772
+ });
1773
+ void navigateAndInitCDP();
1774
+ const sendScreenshot = () => __awaiter(this, void 0, void 0, function* () {
1775
+ if (!onScreenshot) {
1776
+ return;
1777
+ }
1778
+ try {
1779
+ onScreenshot({
1780
+ viewport: chrome.viewport,
1781
+ buffer: yield chrome.screenshot(),
1782
+ });
1783
+ }
1784
+ catch (err) {
1785
+ logger.error({ err }, "Failed to take screenshot");
1786
+ }
1787
+ });
1788
+ void sendScreenshot();
1789
+ // NOTE: this is a very quick interval because while chome is navigating
1790
+ // we want to show updates to the user ASAP
1791
+ const screenshotInterval = setInterval(() => {
1792
+ void sendScreenshot();
1793
+ }, 250);
1794
+ const startTime = Date.now();
1795
+ while (!completed && Date.now() - startTime < timeout) {
1796
+ yield sleep(CHECK_INTERVAL_MS);
1759
1797
  }
1760
- catch (err) {
1761
- logger.error({ err }, "Failed to take screenshot");
1798
+ clearInterval(screenshotInterval);
1799
+ if (!completed) {
1800
+ logger.warn("Timeout elapsed waiting for browser to initialize - are you sure this page is accessible?");
1762
1801
  }
1763
- };
1764
- void sendScreenshot();
1765
- // NOTE: this is a very quick interval because while chome is navigating
1766
- // we want to show updates to the user ASAP
1767
- const screenshotInterval = setInterval(() => {
1768
- void sendScreenshot();
1769
- }, 250);
1770
- const startTime = Date.now();
1771
- while (!completed && Date.now() - startTime < timeout) {
1772
- await sleep(CHECK_INTERVAL_MS);
1773
- }
1774
- clearInterval(screenshotInterval);
1775
- if (!completed) {
1776
- logger.warn("Timeout elapsed waiting for browser to initialize - are you sure this page is accessible?");
1777
- }
1778
- return chrome;
1802
+ return chrome;
1803
+ });
1779
1804
  }
1780
1805
  // Things to do on every page load
1781
- async pageSetup() {
1782
- await this.page.evaluate(addCursorScript);
1783
- await this.page.evaluate(addIDsScript);
1806
+ pageSetup() {
1807
+ return __awaiter(this, void 0, void 0, function* () {
1808
+ yield this.page.evaluate(addCursorScript);
1809
+ yield this.page.evaluate(addIDsScript);
1810
+ });
1784
1811
  }
1785
- async wait(timeoutMs) {
1786
- await this.page.waitForTimeout(timeoutMs);
1812
+ wait(timeoutMs) {
1813
+ return __awaiter(this, void 0, void 0, function* () {
1814
+ yield this.page.waitForTimeout(timeoutMs);
1815
+ });
1787
1816
  }
1788
- async cleanup() {
1789
- await this.page.close();
1790
- await this.context.close();
1791
- await this.browser.close();
1817
+ cleanup() {
1818
+ return __awaiter(this, void 0, void 0, function* () {
1819
+ yield this.page.close();
1820
+ yield this.context.close();
1821
+ yield this.browser.close();
1822
+ });
1792
1823
  }
1793
1824
  get closed() {
1794
1825
  return this.page.isClosed() || !this.browser.isConnected();
1795
1826
  }
1796
- async html() {
1797
- return await this.page.content();
1827
+ html() {
1828
+ return __awaiter(this, void 0, void 0, function* () {
1829
+ return yield this.page.content();
1830
+ });
1798
1831
  }
1799
1832
  get url() {
1800
1833
  return this.page.url();
1801
1834
  }
1802
- async screenshot(quality = 100, scale = "device") {
1803
- return await this.page.screenshot({
1804
- fullPage: false,
1805
- quality,
1806
- scale,
1807
- type: "jpeg",
1808
- // allow the blinking text cursor thing to remain there
1809
- caret: "initial",
1835
+ screenshot(quality = 100, scale = "device") {
1836
+ return __awaiter(this, void 0, void 0, function* () {
1837
+ return yield this.page.screenshot({
1838
+ fullPage: false,
1839
+ quality,
1840
+ scale,
1841
+ type: "jpeg",
1842
+ // allow the blinking text cursor thing to remain there
1843
+ caret: "initial",
1844
+ });
1810
1845
  });
1811
1846
  }
1812
1847
  get viewport() {
@@ -1816,502 +1851,560 @@ class ChromeBrowser {
1816
1851
  }
1817
1852
  return viewport;
1818
1853
  }
1819
- async navigate(url,
1854
+ navigate(url,
1820
1855
  // FIXME: this is an escape hatch to make sure some pages load (assembledhq.com)
1821
1856
  wrapPossibleNavigation = true) {
1822
- this.logger.debug(`Navigating to ${url}`);
1823
- const startTime = Date.now();
1824
- const doNav = async () => {
1825
- try {
1826
- await this.page.goto(url, {
1827
- timeout: MAX_LOAD_TIMEOUT_MS,
1828
- });
1829
- this.logger.debug({ url }, `Got load event in ${Math.floor(Date.now() - startTime)}ms`);
1857
+ return __awaiter(this, void 0, void 0, function* () {
1858
+ this.logger.debug(`Navigating to ${url}`);
1859
+ const startTime = Date.now();
1860
+ const doNav = () => __awaiter(this, void 0, void 0, function* () {
1861
+ try {
1862
+ yield this.page.goto(url, {
1863
+ timeout: MAX_LOAD_TIMEOUT_MS,
1864
+ });
1865
+ this.logger.debug({ url }, `Got load event in ${Math.floor(Date.now() - startTime)}ms`);
1866
+ }
1867
+ catch (e) {
1868
+ this.logger.warn({ url, type: "navigate", err: e }, "Timeout elapsed waiting for page to load, continuing anyways...");
1869
+ }
1870
+ });
1871
+ if (wrapPossibleNavigation) {
1872
+ yield this.wrapPossibleNavigation(doNav);
1830
1873
  }
1831
- catch (e) {
1832
- this.logger.warn({ url, type: "navigate", err: e }, "Timeout elapsed waiting for page to load, continuing anyways...");
1874
+ else {
1875
+ yield doNav();
1833
1876
  }
1834
- };
1835
- if (wrapPossibleNavigation) {
1836
- await this.wrapPossibleNavigation(doNav);
1837
- }
1838
- else {
1839
- await doNav();
1840
- }
1841
- if (CHROME_INTERNAL_URLS.has(this.url) &&
1842
- process.env.NODE_ENV === "production") {
1843
- // in dev, this is a little annoying
1844
- throw new Error(`${url} took too long to load 😞. Please ensure the site and your internet are working.`);
1845
- }
1846
- await this.pageSetup();
1847
- this.logger.debug({ url }, "Navigation complete");
1848
- }
1849
- async fill(target, text, options = {}) {
1850
- const element = await this.click(target, {
1851
- doubleClick: false,
1852
- rightClick: false,
1877
+ if (CHROME_INTERNAL_URLS.has(this.url) &&
1878
+ process.env.NODE_ENV === "production") {
1879
+ // in dev, this is a little annoying
1880
+ throw new Error(`${url} took too long to load 😞. Please ensure the site and your internet are working.`);
1881
+ }
1882
+ yield this.pageSetup();
1883
+ this.logger.debug({ url }, "Navigation complete");
1853
1884
  });
1854
- await this.type(text, options);
1855
- return element;
1856
- }
1857
- async type(text, options = {}) {
1858
- const { clearContent = true, pressKeysSequentially = false } = options;
1859
- if (clearContent) {
1860
- await this.page.keyboard.press("Meta+A");
1861
- await this.page.keyboard.press("Backspace");
1862
- }
1863
- if (pressKeysSequentially) {
1864
- await this.page.keyboard.type(text);
1865
- }
1866
- else {
1867
- await this.page.keyboard.insertText(text);
1868
- }
1869
1885
  }
1870
- async clickByA11yID(index, options = {}) {
1871
- const node = this.nodeMap.get(`${index}`);
1872
- if (!node) {
1873
- throw new Error(`Could not find node in DOM with index: ${index}`);
1874
- }
1875
- const nodeClicked = await this.clickUsingCDP(node, options);
1876
- await this.highlightNode(nodeClicked);
1877
- return node.serialize({ noChildren: true, noProperties: true, noID: true });
1878
- }
1879
- async selectOptionByA11yID(index, option) {
1880
- const node = this.nodeMap.get(`${index}`);
1881
- if (!node) {
1882
- throw new Error(`Could not find node in DOM with index: ${index}`);
1883
- }
1884
- if (!node.backendNodeID) {
1885
- throw new Error(`Select target missing backend node id: ${node.getLogForm()}`);
1886
- }
1887
- const locator = await this.getLocatorFromBackendID(node.backendNodeID);
1888
- await locator.selectOption(option, {
1889
- timeout: COMPLICATED_BROWSER_ACTION_TIMEOUT_MS,
1886
+ fill(target, text, options = {}) {
1887
+ return __awaiter(this, void 0, void 0, function* () {
1888
+ const element = yield this.click(target, {
1889
+ doubleClick: false,
1890
+ rightClick: false,
1891
+ });
1892
+ yield this.type(text, options);
1893
+ return element;
1890
1894
  });
1891
- await this.highlightNode(node);
1892
- return node.serialize({ noChildren: true, noProperties: true, noID: true });
1893
1895
  }
1894
- async highlight(target) {
1895
- try {
1896
- await this.highlightByA11yID(target.id);
1897
- }
1898
- catch (err) {
1899
- // should never be fatal
1900
- this.logger.warn({ err, target }, "Failed to highlight target");
1901
- }
1896
+ type(text, options = {}) {
1897
+ return __awaiter(this, void 0, void 0, function* () {
1898
+ const { clearContent = true, pressKeysSequentially = false } = options;
1899
+ if (clearContent) {
1900
+ yield this.page.keyboard.press("Meta+A");
1901
+ yield this.page.keyboard.press("Backspace");
1902
+ }
1903
+ if (pressKeysSequentially) {
1904
+ yield this.page.keyboard.type(text);
1905
+ }
1906
+ else {
1907
+ yield this.page.keyboard.insertText(text);
1908
+ }
1909
+ });
1902
1910
  }
1903
- async highlightByA11yID(index) {
1904
- const node = this.nodeMap.get(`${index}`);
1905
- if (!node) {
1906
- throw new Error(`Could not find node in DOM with index: ${index}`);
1907
- }
1908
- if (!node.backendNodeID) {
1909
- throw new Error(`Select target missing backend node id: ${node.getLogForm()}`);
1910
- }
1911
- await this.highlightNode(node);
1911
+ clickByA11yID(index, options = {}) {
1912
+ return __awaiter(this, void 0, void 0, function* () {
1913
+ const node = this.nodeMap.get(`${index}`);
1914
+ if (!node) {
1915
+ throw new Error(`Could not find node in DOM with index: ${index}`);
1916
+ }
1917
+ const nodeClicked = yield this.clickUsingCDP(node, options);
1918
+ yield this.highlightNode(nodeClicked);
1919
+ return node.serialize({ noChildren: true, noProperties: true, noID: true });
1920
+ });
1912
1921
  }
1913
- async highlightNode(node) {
1914
- try {
1915
- await this.cdpClient.send("Overlay.highlightNode", {
1916
- highlightConfig: NODE_HIGHLIGHT_CONFIG,
1917
- backendNodeId: node.backendNodeID,
1922
+ selectOptionByA11yID(index, option) {
1923
+ return __awaiter(this, void 0, void 0, function* () {
1924
+ const node = this.nodeMap.get(`${index}`);
1925
+ if (!node) {
1926
+ throw new Error(`Could not find node in DOM with index: ${index}`);
1927
+ }
1928
+ if (!node.backendNodeID) {
1929
+ throw new Error(`Select target missing backend node id: ${node.getLogForm()}`);
1930
+ }
1931
+ const locator = yield this.getLocatorFromBackendID(node.backendNodeID);
1932
+ yield locator.selectOption(option, {
1933
+ timeout: COMPLICATED_BROWSER_ACTION_TIMEOUT_MS,
1918
1934
  });
1919
- }
1920
- catch (err) {
1921
- this.logger.warn({ err }, "Failed to add node highlight");
1922
- }
1923
- const hideHighlight = async () => {
1935
+ yield this.highlightNode(node);
1936
+ return node.serialize({ noChildren: true, noProperties: true, noID: true });
1937
+ });
1938
+ }
1939
+ highlight(target) {
1940
+ return __awaiter(this, void 0, void 0, function* () {
1924
1941
  try {
1925
- await this.cdpClient.send("Overlay.hideHighlight", {
1926
- backendNodeId: node.backendNodeID,
1927
- });
1942
+ yield this.highlightByA11yID(target.id);
1928
1943
  }
1929
1944
  catch (err) {
1930
- // this is okay, purely visual and often occurs due to navigation
1931
- this.logger.debug({ err }, "Failed to remove node highlight");
1945
+ // should never be fatal
1946
+ this.logger.warn({ err, target }, "Failed to highlight target");
1932
1947
  }
1933
- };
1934
- setTimeout(() => {
1935
- void hideHighlight();
1936
- }, HIGHLIGHT_DURATION_MS);
1937
- }
1938
- async wrapPossibleNavigation(fn, timeoutMS = MAX_LOAD_TIMEOUT_MS) {
1939
- const startTime = Date.now();
1940
- const startURL = this.url;
1941
- let lastRequestReceived = Date.now();
1942
- const firedRequests = new Map();
1943
- const finishedRequests = new Map();
1944
- const requestFinishedListener = (request) => {
1945
- const key = serializeRequest(request);
1946
- finishedRequests.set(key, (finishedRequests.get(key) ?? 0) + 1);
1947
- };
1948
- const requestFiredListener = (request) => {
1949
- if (!isRequestRelevantForPageLoad(request, this.url)) {
1950
- this.logger.debug({
1951
- uri: serializeRequest(request),
1952
- }, "Ignoring request for page load network stability");
1953
- return;
1948
+ });
1949
+ }
1950
+ highlightByA11yID(index) {
1951
+ return __awaiter(this, void 0, void 0, function* () {
1952
+ const node = this.nodeMap.get(`${index}`);
1953
+ if (!node) {
1954
+ throw new Error(`Could not find node in DOM with index: ${index}`);
1954
1955
  }
1955
- const key = serializeRequest(request);
1956
- this.logger.debug({
1957
- uri: key,
1958
- }, "Request fired on page load, delaying network stability");
1959
- firedRequests.set(key, (firedRequests.get(key) ?? 0) + 1);
1960
- lastRequestReceived = Date.now();
1961
- };
1962
- this.page.on("requestfinished", requestFinishedListener);
1963
- this.page.on("request", requestFiredListener);
1964
- // fire actual function asynchronously
1965
- // instead of throwing the error, we return it so we can handle it later
1966
- let rejected = false;
1967
- const retPromise = fn().catch((e) => {
1968
- rejected = true;
1969
- if (e instanceof Error)
1970
- return e;
1971
- // we are returning NOT throwing on purpose
1972
- return new Error(`${e}`);
1956
+ if (!node.backendNodeID) {
1957
+ throw new Error(`Select target missing backend node id: ${node.getLogForm()}`);
1958
+ }
1959
+ yield this.highlightNode(node);
1973
1960
  });
1974
- await sleep(CHECK_INTERVAL_MS);
1975
- const unwrapAndThrowError = async (p) => {
1976
- const v = await p;
1977
- if (v instanceof Error) {
1978
- throw v;
1961
+ }
1962
+ highlightNode(node) {
1963
+ return __awaiter(this, void 0, void 0, function* () {
1964
+ try {
1965
+ yield this.cdpClient.send("Overlay.highlightNode", {
1966
+ highlightConfig: NODE_HIGHLIGHT_CONFIG,
1967
+ backendNodeId: node.backendNodeID,
1968
+ });
1979
1969
  }
1980
- return v;
1981
- };
1982
- // wait for network idle
1983
- let unfinishedRequests = new Set();
1984
- const waitForNetworkIdle = async () => {
1985
- while (!rejected && Date.now() - startTime < timeoutMS) {
1986
- unfinishedRequests = new Set();
1987
- await sleep(CHECK_INTERVAL_MS);
1988
- if (Date.now() - lastRequestReceived <=
1989
- NETWORK_STABLE_DURATION_MS) {
1990
- continue;
1970
+ catch (err) {
1971
+ this.logger.warn({ err }, "Failed to add node highlight");
1972
+ }
1973
+ const hideHighlight = () => __awaiter(this, void 0, void 0, function* () {
1974
+ try {
1975
+ yield this.cdpClient.send("Overlay.hideHighlight", {
1976
+ backendNodeId: node.backendNodeID,
1977
+ });
1991
1978
  }
1992
- let anyDifference = false;
1993
- for (const key of firedRequests.keys()) {
1994
- if (firedRequests.get(key) !== finishedRequests.get(key)) {
1995
- this.logger.debug({ uri: key }, "Waiting on request to finish");
1996
- anyDifference = true;
1997
- unfinishedRequests.add(key);
1998
- }
1979
+ catch (err) {
1980
+ // this is okay, purely visual and often occurs due to navigation
1981
+ this.logger.debug({ err }, "Failed to remove node highlight");
1999
1982
  }
2000
- if (!anyDifference) {
1983
+ });
1984
+ setTimeout(() => {
1985
+ void hideHighlight();
1986
+ }, HIGHLIGHT_DURATION_MS);
1987
+ });
1988
+ }
1989
+ wrapPossibleNavigation(fn, timeoutMS = MAX_LOAD_TIMEOUT_MS) {
1990
+ return __awaiter(this, void 0, void 0, function* () {
1991
+ const startTime = Date.now();
1992
+ const startURL = this.url;
1993
+ let lastRequestReceived = Date.now();
1994
+ const firedRequests = new Map();
1995
+ const finishedRequests = new Map();
1996
+ const requestFinishedListener = (request) => {
1997
+ var _a;
1998
+ const key = serializeRequest(request);
1999
+ finishedRequests.set(key, ((_a = finishedRequests.get(key)) !== null && _a !== void 0 ? _a : 0) + 1);
2000
+ };
2001
+ const requestFiredListener = (request) => {
2002
+ var _a;
2003
+ if (!isRequestRelevantForPageLoad(request, this.url)) {
2001
2004
  this.logger.debug({
2005
+ uri: serializeRequest(request),
2006
+ }, "Ignoring request for page load network stability");
2007
+ return;
2008
+ }
2009
+ const key = serializeRequest(request);
2010
+ this.logger.debug({
2011
+ uri: key,
2012
+ }, "Request fired on page load, delaying network stability");
2013
+ firedRequests.set(key, ((_a = firedRequests.get(key)) !== null && _a !== void 0 ? _a : 0) + 1);
2014
+ lastRequestReceived = Date.now();
2015
+ };
2016
+ this.page.on("requestfinished", requestFinishedListener);
2017
+ this.page.on("request", requestFiredListener);
2018
+ // fire actual function asynchronously
2019
+ // instead of throwing the error, we return it so we can handle it later
2020
+ let rejected = false;
2021
+ const retPromise = fn().catch((e) => {
2022
+ rejected = true;
2023
+ if (e instanceof Error)
2024
+ return e;
2025
+ // we are returning NOT throwing on purpose
2026
+ return new Error(`${e}`);
2027
+ });
2028
+ yield sleep(CHECK_INTERVAL_MS);
2029
+ const unwrapAndThrowError = (p) => __awaiter(this, void 0, void 0, function* () {
2030
+ const v = yield p;
2031
+ if (v instanceof Error) {
2032
+ throw v;
2033
+ }
2034
+ return v;
2035
+ });
2036
+ // wait for network idle
2037
+ let unfinishedRequests = new Set();
2038
+ const waitForNetworkIdle = () => __awaiter(this, void 0, void 0, function* () {
2039
+ while (!rejected && Date.now() - startTime < timeoutMS) {
2040
+ unfinishedRequests = new Set();
2041
+ yield sleep(CHECK_INTERVAL_MS);
2042
+ if (Date.now() - lastRequestReceived <=
2043
+ NETWORK_STABLE_DURATION_MS) {
2044
+ continue;
2045
+ }
2046
+ let anyDifference = false;
2047
+ for (const key of firedRequests.keys()) {
2048
+ if (firedRequests.get(key) !== finishedRequests.get(key)) {
2049
+ this.logger.debug({ uri: key }, "Waiting on request to finish");
2050
+ anyDifference = true;
2051
+ unfinishedRequests.add(key);
2052
+ }
2053
+ }
2054
+ if (!anyDifference) {
2055
+ this.logger.debug({
2056
+ url: this.url,
2057
+ requests: JSON.stringify(Array.from(firedRequests.entries())),
2058
+ }, `Network idle in ${Math.floor(Date.now() - startTime)}ms`);
2059
+ return true;
2060
+ }
2061
+ }
2062
+ if (!rejected) {
2063
+ this.logger.warn({
2002
2064
  url: this.url,
2003
- requests: JSON.stringify(Array.from(firedRequests.entries())),
2004
- }, `Network idle in ${Math.floor(Date.now() - startTime)}ms`);
2005
- return true;
2065
+ requests: JSON.stringify(Array.from(unfinishedRequests.entries())),
2066
+ }, "Timeout elapsed waiting for network idle, continuing anyways...");
2006
2067
  }
2068
+ return false;
2069
+ });
2070
+ const waitResult = yield waitForNetworkIdle();
2071
+ this.page.off("requestfinished", requestFinishedListener);
2072
+ this.page.off("request", requestFiredListener);
2073
+ if (!waitResult) {
2074
+ return unwrapAndThrowError(retPromise);
2007
2075
  }
2008
- if (!rejected) {
2009
- this.logger.warn({
2010
- url: this.url,
2011
- requests: JSON.stringify(Array.from(unfinishedRequests.entries())),
2012
- }, "Timeout elapsed waiting for network idle, continuing anyways...");
2076
+ if (!rejected && urlChanged(this.url, startURL)) {
2077
+ this.logger.debug(`Detected url change in wrapPossibleNavigation, waiting for load state`);
2078
+ try {
2079
+ yield this.page.waitForLoadState("load", {
2080
+ timeout: timeoutMS - (Date.now() - startTime),
2081
+ });
2082
+ }
2083
+ catch (e) {
2084
+ this.logger.warn({ url: this.url }, "Timeout elapsed waiting for load state to fire, continuing anyways...");
2085
+ }
2013
2086
  }
2014
- return false;
2015
- };
2016
- const waitResult = await waitForNetworkIdle();
2017
- this.page.off("requestfinished", requestFinishedListener);
2018
- this.page.off("request", requestFiredListener);
2019
- if (!waitResult) {
2020
2087
  return unwrapAndThrowError(retPromise);
2021
- }
2022
- if (!rejected && urlChanged(this.url, startURL)) {
2023
- this.logger.debug(`Detected url change in wrapPossibleNavigation, waiting for load state`);
2024
- try {
2025
- await this.page.waitForLoadState("load", {
2026
- timeout: timeoutMS - (Date.now() - startTime),
2027
- });
2028
- }
2029
- catch (e) {
2030
- this.logger.warn({ url: this.url }, "Timeout elapsed waiting for load state to fire, continuing anyways...");
2031
- }
2032
- }
2033
- return unwrapAndThrowError(retPromise);
2088
+ });
2034
2089
  }
2035
- async click(target, options = {}) {
2036
- const elementInteracted = await this.wrapPossibleNavigation(() => this.clickByA11yID(target.id, options));
2037
- return elementInteracted;
2090
+ click(target, options = {}) {
2091
+ return __awaiter(this, void 0, void 0, function* () {
2092
+ const elementInteracted = yield this.wrapPossibleNavigation(() => this.clickByA11yID(target.id, options));
2093
+ return elementInteracted;
2094
+ });
2038
2095
  }
2039
- async selectOption(target, option) {
2040
- return this.selectOptionByA11yID(target.id, option);
2096
+ selectOption(target, option) {
2097
+ return __awaiter(this, void 0, void 0, function* () {
2098
+ return this.selectOptionByA11yID(target.id, option);
2099
+ });
2041
2100
  }
2042
- async press(key) {
2043
- await this.wrapPossibleNavigation(() => this.page.keyboard.press(key));
2101
+ press(key) {
2102
+ return __awaiter(this, void 0, void 0, function* () {
2103
+ yield this.wrapPossibleNavigation(() => this.page.keyboard.press(key));
2104
+ });
2044
2105
  }
2045
- async refresh() {
2046
- await this.page.reload();
2047
- await this.pageSetup();
2106
+ refresh() {
2107
+ return __awaiter(this, void 0, void 0, function* () {
2108
+ yield this.page.reload();
2109
+ yield this.pageSetup();
2110
+ });
2048
2111
  }
2049
- async getA11yTree() {
2050
- let processedTree = null;
2051
- let attempt = 0;
2052
- const url = this.url;
2053
- while (!processedTree) {
2054
- try {
2055
- this.logger.debug(`Getting a11y tree at ${url}`);
2056
- const graph = await this.getRawA11yTree();
2057
- if (!graph.root || graph.allNodes.length === 0) {
2058
- // throw specific error class
2059
- throw new Error("No a11y tree found on page");
2060
- }
2061
- processedTree = processA11yTree(graph);
2062
- }
2063
- catch (e) {
2064
- this.logger.error({ err: e, url }, "Error fetching a11y tree");
2065
- if (attempt === 0) {
2066
- await sleep(1000);
2067
- attempt++;
2112
+ getA11yTree() {
2113
+ return __awaiter(this, void 0, void 0, function* () {
2114
+ let processedTree = null;
2115
+ let attempt = 0;
2116
+ const url = this.url;
2117
+ while (!processedTree) {
2118
+ try {
2119
+ this.logger.debug(`Getting a11y tree at ${url}`);
2120
+ const graph = yield this.getRawA11yTree();
2121
+ if (!graph.root || graph.allNodes.length === 0) {
2122
+ // throw specific error class
2123
+ throw new Error("No a11y tree found on page");
2124
+ }
2125
+ processedTree = processA11yTree(graph);
2068
2126
  }
2069
- else {
2070
- throw new Error(`Max retries exceeded fetching a11y tree: ${e}`);
2127
+ catch (e) {
2128
+ this.logger.error({ err: e, url }, "Error fetching a11y tree");
2129
+ if (attempt === 0) {
2130
+ yield sleep(1000);
2131
+ attempt++;
2132
+ }
2133
+ else {
2134
+ throw new Error(`Max retries exceeded fetching a11y tree: ${e}`);
2135
+ }
2071
2136
  }
2072
2137
  }
2073
- }
2074
- if (!processedTree.root) {
2075
- // could be valid!
2076
- this.logger.warn("A11y tree was pruned entirely");
2077
- }
2078
- this.nodeMap = processedTree.nodeMap;
2079
- return processedTree;
2080
- }
2081
- async getRawA11yTree() {
2082
- const url = this.page.url();
2083
- let lastTreeUpdateTimestamp = Date.now();
2084
- const treeUpdateListener = () => {
2085
- lastTreeUpdateTimestamp = Date.now();
2086
- };
2087
- this.cdpClient.addListener("Accessibility.nodesUpdated", treeUpdateListener);
2088
- let accessibilityTreeLoadFired = false;
2089
- const accessibilityLoadListener = () => {
2090
- this.logger.info({ url }, `A11y tree load event fired`);
2091
- accessibilityTreeLoadFired = true;
2092
- };
2093
- this.cdpClient.addListener("Accessibility.loadComplete", accessibilityLoadListener);
2094
- // make sure the a11y tree hasn't updated in the last 1 second
2095
- // and the a11y event has fired
2096
- const a11yLoadStart = Date.now();
2097
- let timeoutTriggered = true;
2098
- while (Date.now() - a11yLoadStart < A11Y_STABLE_TIMEOUT_MS) {
2099
- await sleep(CHECK_INTERVAL_MS);
2100
- if (!accessibilityTreeLoadFired &&
2101
- Date.now() - a11yLoadStart < A11Y_LOAD_TIMEOUT_MS) {
2102
- // some websites never fire the a11y load event
2103
- // thus, we only allocate 1 second for catching this event
2104
- this.logger.debug({ url }, `A11y tree not loaded yet, waiting...`);
2105
- continue;
2106
- }
2107
- if (Date.now() - lastTreeUpdateTimestamp >=
2108
- A11Y_STABLE_DURATION_MS) {
2109
- this.logger.debug({ url }, `A11y tree not stable yet, waiting...`);
2110
- continue;
2138
+ if (!processedTree.root) {
2139
+ // could be valid!
2140
+ this.logger.warn("A11y tree was pruned entirely");
2111
2141
  }
2112
- timeoutTriggered = false;
2113
- break;
2114
- }
2115
- this.logger.debug({
2116
- duration: Date.now() - a11yLoadStart,
2117
- eventReceived: accessibilityTreeLoadFired,
2118
- timeoutTriggered,
2119
- }, "A11y wait phase completed");
2120
- const { node: root } = await this.cdpClient.send("Accessibility.getRootAXNode");
2121
- const { nodes } = await this.cdpClient.send("Accessibility.queryAXTree", {
2122
- backendNodeId: root.backendDOMNodeId,
2142
+ this.nodeMap = processedTree.nodeMap;
2143
+ return processedTree;
2123
2144
  });
2124
- this.cdpClient.removeListener("Accessibility.loadComplete", accessibilityLoadListener);
2125
- this.cdpClient.removeListener("Accessibility.nodesUpdated", treeUpdateListener);
2126
- return {
2127
- root,
2128
- allNodes: nodes,
2129
- };
2130
- }
2131
- async clickUsingVisualCoordinates(backendNodeId) {
2132
- const location = await this.getElementLocation(backendNodeId);
2133
- if (!location) {
2134
- throw new Error(`Could not find element location with backend node id: ${backendNodeId}`);
2135
- }
2136
- this.logger.debug({ location }, "Executing mouse click");
2137
- await this.page.mouse.click(location.centerX, location.centerY);
2138
2145
  }
2139
- // Get the "id" attribute value from an HTML element.
2140
- async getIDAttributeUsingCDP(objectId) {
2141
- // https://bugs.chromium.org/p/chromium/issues/detail?id=1374241
2142
- await this.cdpClient.send("DOM.getDocument", { depth: 0 });
2143
- const cdpNodeResult = await this.cdpClient.send("DOM.requestNode", {
2144
- objectId,
2145
- });
2146
- const attrResult = await this.cdpClient.send("DOM.getAttributes", {
2147
- nodeId: cdpNodeResult.nodeId,
2146
+ getRawA11yTree() {
2147
+ return __awaiter(this, void 0, void 0, function* () {
2148
+ const url = this.page.url();
2149
+ let lastTreeUpdateTimestamp = Date.now();
2150
+ const treeUpdateListener = () => {
2151
+ lastTreeUpdateTimestamp = Date.now();
2152
+ };
2153
+ this.cdpClient.addListener("Accessibility.nodesUpdated", treeUpdateListener);
2154
+ let accessibilityTreeLoadFired = false;
2155
+ const accessibilityLoadListener = () => {
2156
+ this.logger.info({ url }, `A11y tree load event fired`);
2157
+ accessibilityTreeLoadFired = true;
2158
+ };
2159
+ this.cdpClient.addListener("Accessibility.loadComplete", accessibilityLoadListener);
2160
+ // make sure the a11y tree hasn't updated in the last 1 second
2161
+ // and the a11y event has fired
2162
+ const a11yLoadStart = Date.now();
2163
+ let timeoutTriggered = true;
2164
+ while (Date.now() - a11yLoadStart < A11Y_STABLE_TIMEOUT_MS) {
2165
+ yield sleep(CHECK_INTERVAL_MS);
2166
+ if (!accessibilityTreeLoadFired &&
2167
+ Date.now() - a11yLoadStart < A11Y_LOAD_TIMEOUT_MS) {
2168
+ // some websites never fire the a11y load event
2169
+ // thus, we only allocate 1 second for catching this event
2170
+ this.logger.debug({ url }, `A11y tree not loaded yet, waiting...`);
2171
+ continue;
2172
+ }
2173
+ if (Date.now() - lastTreeUpdateTimestamp >=
2174
+ A11Y_STABLE_DURATION_MS) {
2175
+ this.logger.debug({ url }, `A11y tree not stable yet, waiting...`);
2176
+ continue;
2177
+ }
2178
+ timeoutTriggered = false;
2179
+ break;
2180
+ }
2181
+ this.logger.debug({
2182
+ duration: Date.now() - a11yLoadStart,
2183
+ eventReceived: accessibilityTreeLoadFired,
2184
+ timeoutTriggered,
2185
+ }, "A11y wait phase completed");
2186
+ const { node: root } = yield this.cdpClient.send("Accessibility.getRootAXNode");
2187
+ const { nodes } = yield this.cdpClient.send("Accessibility.queryAXTree", {
2188
+ backendNodeId: root.backendDOMNodeId,
2189
+ });
2190
+ this.cdpClient.removeListener("Accessibility.loadComplete", accessibilityLoadListener);
2191
+ this.cdpClient.removeListener("Accessibility.nodesUpdated", treeUpdateListener);
2192
+ return {
2193
+ root,
2194
+ allNodes: nodes,
2195
+ };
2148
2196
  });
2149
- const attributes = attrResult.attributes;
2150
- const indexAttr = attributes.findIndex((s) => s === "data-momentic-id");
2151
- if (indexAttr === -1) {
2152
- return "";
2153
- }
2154
- return attributes[indexAttr + 1] || "";
2155
2197
  }
2156
- async getLocatorFromBackendID(backendNodeId) {
2157
- await this.page.evaluate(addIDsScript);
2158
- // get a remote javascript object from the a11y backend node ID
2159
- const cdpResolveResult = await this.cdpClient.send("DOM.resolveNode", {
2160
- backendNodeId,
2198
+ clickUsingVisualCoordinates(backendNodeId) {
2199
+ return __awaiter(this, void 0, void 0, function* () {
2200
+ const location = yield this.getElementLocation(backendNodeId);
2201
+ if (!location) {
2202
+ throw new Error(`Could not find element location with backend node id: ${backendNodeId}`);
2203
+ }
2204
+ this.logger.debug({ location }, "Executing mouse click");
2205
+ yield this.page.mouse.click(location.centerX, location.centerY);
2161
2206
  });
2162
- if (!cdpResolveResult || !cdpResolveResult.object.objectId) {
2163
- throw new Error(`Could not resolve backend node ${backendNodeId}`);
2164
- }
2165
- try {
2166
- const id = await this.getIDAttributeUsingCDP(cdpResolveResult.object.objectId);
2167
- if (!id) {
2168
- throw new Error("Failed getting data-momentic-id attribute using CDP");
2207
+ }
2208
+ // Get the "id" attribute value from an HTML element.
2209
+ getIDAttributeUsingCDP(objectId) {
2210
+ return __awaiter(this, void 0, void 0, function* () {
2211
+ // https://bugs.chromium.org/p/chromium/issues/detail?id=1374241
2212
+ yield this.cdpClient.send("DOM.getDocument", { depth: 0 });
2213
+ const cdpNodeResult = yield this.cdpClient.send("DOM.requestNode", {
2214
+ objectId,
2215
+ });
2216
+ const attrResult = yield this.cdpClient.send("DOM.getAttributes", {
2217
+ nodeId: cdpNodeResult.nodeId,
2218
+ });
2219
+ const attributes = attrResult.attributes;
2220
+ const indexAttr = attributes.findIndex((s) => s === "data-momentic-id");
2221
+ if (indexAttr === -1) {
2222
+ return "";
2169
2223
  }
2170
- return this.page.locator(`[data-momentic-id="${id}"]`);
2171
- }
2172
- catch (err) {
2173
- this.logger.error({
2174
- err,
2175
- }, "Failed to get ID attribute");
2176
- throw err;
2177
- }
2224
+ return attributes[indexAttr + 1] || "";
2225
+ });
2178
2226
  }
2179
- async clickUsingCDP(originalNode, options = {}) {
2180
- let clickAttempts = 0;
2181
- let candidateNode = originalNode;
2182
- while (clickAttempts < MAX_BROWSER_ACTION_ATTEMPTS) {
2183
- // Did we reach the root?
2184
- if (!candidateNode || candidateNode.role === "RootWebArea") {
2185
- throw new Error(`Attempted to click node with no clickable surrounding elements: ${originalNode.getLogForm()}`);
2227
+ getLocatorFromBackendID(backendNodeId) {
2228
+ return __awaiter(this, void 0, void 0, function* () {
2229
+ yield this.page.evaluate(addIDsScript);
2230
+ // get a remote javascript object from the a11y backend node ID
2231
+ const cdpResolveResult = yield this.cdpClient.send("DOM.resolveNode", {
2232
+ backendNodeId,
2233
+ });
2234
+ if (!cdpResolveResult || !cdpResolveResult.object.objectId) {
2235
+ throw new Error(`Could not resolve backend node ${backendNodeId}`);
2186
2236
  }
2187
- // Check disqualifying conditions for clicks - these don't count as "attempts"
2188
- if (candidateNode.role === "StaticText") {
2189
- // static text corresponds to html text nodes that are not clickable
2190
- candidateNode = candidateNode.parent;
2191
- continue;
2237
+ try {
2238
+ const id = yield this.getIDAttributeUsingCDP(cdpResolveResult.object.objectId);
2239
+ if (!id) {
2240
+ throw new Error("Failed getting data-momentic-id attribute using CDP");
2241
+ }
2242
+ return this.page.locator(`[data-momentic-id="${id}"]`);
2192
2243
  }
2193
- const candidateNodeID = candidateNode.backendNodeID;
2194
- if (!candidateNodeID) {
2195
- this.logger.warn({ node: candidateNode.getLogForm() }, "Click candidate had no backend node ID");
2196
- candidateNode = candidateNode.parent;
2197
- continue;
2244
+ catch (err) {
2245
+ this.logger.error({
2246
+ err,
2247
+ }, "Failed to get ID attribute");
2248
+ throw err;
2198
2249
  }
2199
- // Attempt to click
2200
- try {
2201
- const locator = await this.getLocatorFromBackendID(candidateNodeID);
2202
- // this timeout is important because playwright checks for clickability/visibility
2203
- // before clicking, and a timeout indicates obstruction, or disabled state
2204
- // see: https://playwright.dev/docs/actionability#introduction
2205
- if (options.doubleClick) {
2206
- await locator.dblclick({
2207
- timeout: BROWSER_ACTION_TIMEOUT_MS,
2208
- });
2250
+ });
2251
+ }
2252
+ clickUsingCDP(originalNode, options = {}) {
2253
+ return __awaiter(this, void 0, void 0, function* () {
2254
+ let clickAttempts = 0;
2255
+ let candidateNode = originalNode;
2256
+ while (clickAttempts < MAX_BROWSER_ACTION_ATTEMPTS) {
2257
+ // Did we reach the root?
2258
+ if (!candidateNode || candidateNode.role === "RootWebArea") {
2259
+ throw new Error(`Attempted to click node with no clickable surrounding elements: ${originalNode.getLogForm()}`);
2209
2260
  }
2210
- else {
2211
- await locator.click({
2212
- timeout: BROWSER_ACTION_TIMEOUT_MS,
2213
- button: options.rightClick ? "right" : "left",
2214
- });
2261
+ // Check disqualifying conditions for clicks - these don't count as "attempts"
2262
+ if (candidateNode.role === "StaticText") {
2263
+ // static text corresponds to html text nodes that are not clickable
2264
+ candidateNode = candidateNode.parent;
2265
+ continue;
2215
2266
  }
2216
- if (candidateNode.id !== originalNode.id) {
2217
- this.logger.info({
2218
- oldNode: originalNode.getLogForm(),
2219
- newNode: candidateNode.getLogForm(),
2220
- }, `Redirected click successfully to new element`);
2267
+ const candidateNodeID = candidateNode.backendNodeID;
2268
+ if (!candidateNodeID) {
2269
+ this.logger.warn({ node: candidateNode.getLogForm() }, "Click candidate had no backend node ID");
2270
+ candidateNode = candidateNode.parent;
2271
+ continue;
2272
+ }
2273
+ // Attempt to click
2274
+ try {
2275
+ const locator = yield this.getLocatorFromBackendID(candidateNodeID);
2276
+ // this timeout is important because playwright checks for clickability/visibility
2277
+ // before clicking, and a timeout indicates obstruction, or disabled state
2278
+ // see: https://playwright.dev/docs/actionability#introduction
2279
+ if (options.doubleClick) {
2280
+ yield locator.dblclick({
2281
+ timeout: BROWSER_ACTION_TIMEOUT_MS,
2282
+ });
2283
+ }
2284
+ else {
2285
+ yield locator.click({
2286
+ timeout: BROWSER_ACTION_TIMEOUT_MS,
2287
+ button: options.rightClick ? "right" : "left",
2288
+ });
2289
+ }
2290
+ if (candidateNode.id !== originalNode.id) {
2291
+ this.logger.info({
2292
+ oldNode: originalNode.getLogForm(),
2293
+ newNode: candidateNode.getLogForm(),
2294
+ }, `Redirected click successfully to new element`);
2295
+ }
2296
+ return candidateNode;
2297
+ }
2298
+ catch (err) {
2299
+ this.logger.error({ err, node: candidateNode.getLogForm() }, "Failed click or click timed out");
2300
+ clickAttempts++;
2301
+ // try to click the parent instead
2302
+ // we could re-prompt the LLM in the future
2303
+ candidateNode = candidateNode.parent;
2221
2304
  }
2222
- return candidateNode;
2223
- }
2224
- catch (err) {
2225
- this.logger.error({ err, node: candidateNode.getLogForm() }, "Failed click or click timed out");
2226
- clickAttempts++;
2227
- // try to click the parent instead
2228
- // we could re-prompt the LLM in the future
2229
- candidateNode = candidateNode.parent;
2230
2305
  }
2231
- }
2232
- throw new Error(`Max click redirection attempts exhausted on original element: ${originalNode.getLogForm()}`);
2306
+ throw new Error(`Max click redirection attempts exhausted on original element: ${originalNode.getLogForm()}`);
2307
+ });
2233
2308
  }
2234
2309
  /**
2235
2310
  * Currently unused, but could be useful for vision model integration.
2236
2311
  * Gets x/y position of an a11y node.
2237
2312
  */
2238
- async getElementLocation(backendNodeId) {
2239
- const tree = await this.cdpClient.send("DOMSnapshot.captureSnapshot", {
2240
- computedStyles: [],
2241
- includeDOMRects: true,
2242
- includePaintOrder: true,
2243
- });
2244
- let devicePixelRatio = await this.page.evaluate(() => window.devicePixelRatio);
2245
- // it lies on macos lolol
2246
- // this apparently isn't working when the browser is dragged onto a monitor either
2247
- if (process.platform === "darwin" && devicePixelRatio === 1) {
2248
- // UNCOMMENT THE BELOW IF YOU ARE ON MAC OS AND NOT USING A MONITOR
2249
- // COMMENT THE BELOW OUT IF YOU ARE USING A MONITOR OR NOT ON MAC OS
2250
- devicePixelRatio = RETINA_WINDOW_SCALE_FACTOR;
2251
- }
2252
- const document = tree["documents"][0];
2253
- const layout = document["layout"];
2254
- const nodes = document["nodes"];
2255
- const nodeNames = nodes["nodeName"] || [];
2256
- const backendNodeIds = nodes["backendNodeId"] || [];
2257
- const layoutNodeIndex = layout["nodeIndex"];
2258
- const bounds = layout["bounds"];
2259
- let cursor = -1;
2260
- for (let i = 0; i < nodeNames.length; i++) {
2261
- if (backendNodeIds[i] === backendNodeId) {
2262
- cursor = layoutNodeIndex.indexOf(i);
2263
- break;
2313
+ getElementLocation(backendNodeId) {
2314
+ return __awaiter(this, void 0, void 0, function* () {
2315
+ const tree = yield this.cdpClient.send("DOMSnapshot.captureSnapshot", {
2316
+ computedStyles: [],
2317
+ includeDOMRects: true,
2318
+ includePaintOrder: true,
2319
+ });
2320
+ let devicePixelRatio = yield this.page.evaluate(() => window.devicePixelRatio);
2321
+ // it lies on macos lolol
2322
+ // this apparently isn't working when the browser is dragged onto a monitor either
2323
+ if (process.platform === "darwin" && devicePixelRatio === 1) {
2324
+ // UNCOMMENT THE BELOW IF YOU ARE ON MAC OS AND NOT USING A MONITOR
2325
+ // COMMENT THE BELOW OUT IF YOU ARE USING A MONITOR OR NOT ON MAC OS
2326
+ devicePixelRatio = RETINA_WINDOW_SCALE_FACTOR;
2264
2327
  }
2265
- }
2266
- if (cursor === -1) {
2267
- throw new Error(`Could not find any backend node with ID ${backendNodeId}`);
2268
- }
2269
- let [x = 0, y = 0, width = 0, height = 0] = bounds[cursor];
2270
- x /= devicePixelRatio;
2271
- y /= devicePixelRatio;
2272
- width /= devicePixelRatio;
2273
- height /= devicePixelRatio;
2274
- const centerX = x + width / 2;
2275
- const centerY = y + height / 2;
2276
- return { centerX, centerY };
2277
- }
2278
- async scrollUp() {
2279
- // TODO: this works pretty well for full page scroll, in the future we'd need to figure out how to scroll nested containers
2280
- await this.page.evaluate(() => {
2281
- (document.scrollingElement || document.body).scrollTop =
2282
- (document.scrollingElement || document.body).scrollTop -
2283
- window.innerHeight;
2328
+ const document = tree["documents"][0];
2329
+ const layout = document["layout"];
2330
+ const nodes = document["nodes"];
2331
+ const nodeNames = nodes["nodeName"] || [];
2332
+ const backendNodeIds = nodes["backendNodeId"] || [];
2333
+ const layoutNodeIndex = layout["nodeIndex"];
2334
+ const bounds = layout["bounds"];
2335
+ let cursor = -1;
2336
+ for (let i = 0; i < nodeNames.length; i++) {
2337
+ if (backendNodeIds[i] === backendNodeId) {
2338
+ cursor = layoutNodeIndex.indexOf(i);
2339
+ break;
2340
+ }
2341
+ }
2342
+ if (cursor === -1) {
2343
+ throw new Error(`Could not find any backend node with ID ${backendNodeId}`);
2344
+ }
2345
+ let [x = 0, y = 0, width = 0, height = 0] = bounds[cursor];
2346
+ x /= devicePixelRatio;
2347
+ y /= devicePixelRatio;
2348
+ width /= devicePixelRatio;
2349
+ height /= devicePixelRatio;
2350
+ const centerX = x + width / 2;
2351
+ const centerY = y + height / 2;
2352
+ return { centerX, centerY };
2284
2353
  });
2285
- await this.page.evaluate(() => {
2286
- (document.scrollingElement || document.body).scrollTop =
2287
- (document.scrollingElement || document.body).scrollTop +
2288
- window.innerHeight;
2354
+ }
2355
+ scrollUp() {
2356
+ return __awaiter(this, void 0, void 0, function* () {
2357
+ // TODO: this works pretty well for full page scroll, in the future we'd need to figure out how to scroll nested containers
2358
+ yield this.page.evaluate(() => {
2359
+ (document.scrollingElement || document.body).scrollTop =
2360
+ (document.scrollingElement || document.body).scrollTop -
2361
+ window.innerHeight;
2362
+ });
2363
+ yield this.page.evaluate(() => {
2364
+ (document.scrollingElement || document.body).scrollTop =
2365
+ (document.scrollingElement || document.body).scrollTop +
2366
+ window.innerHeight;
2367
+ });
2289
2368
  });
2290
2369
  }
2291
- async scrollDown() {
2292
- await this.page.evaluate(() => {
2293
- (document.scrollingElement || document.body).scrollTop =
2294
- (document.scrollingElement || document.body).scrollTop +
2295
- window.innerHeight;
2370
+ scrollDown() {
2371
+ return __awaiter(this, void 0, void 0, function* () {
2372
+ yield this.page.evaluate(() => {
2373
+ (document.scrollingElement || document.body).scrollTop =
2374
+ (document.scrollingElement || document.body).scrollTop +
2375
+ window.innerHeight;
2376
+ });
2296
2377
  });
2297
2378
  }
2298
- async goForward() {
2299
- await this.wrapPossibleNavigation(() => this.page.goForward({ timeout: MAX_LOAD_TIMEOUT_MS }));
2300
- await this.pageSetup();
2379
+ goForward() {
2380
+ return __awaiter(this, void 0, void 0, function* () {
2381
+ yield this.wrapPossibleNavigation(() => this.page.goForward({ timeout: MAX_LOAD_TIMEOUT_MS }));
2382
+ yield this.pageSetup();
2383
+ });
2301
2384
  }
2302
- async goBack() {
2303
- await this.wrapPossibleNavigation(() => this.page.goBack({ timeout: MAX_LOAD_TIMEOUT_MS }));
2304
- await this.pageSetup();
2385
+ goBack() {
2386
+ return __awaiter(this, void 0, void 0, function* () {
2387
+ yield this.wrapPossibleNavigation(() => this.page.goBack({ timeout: MAX_LOAD_TIMEOUT_MS }));
2388
+ yield this.pageSetup();
2389
+ });
2305
2390
  }
2306
2391
  }
2307
2392
  ChromeBrowser.USER_AGENT = external_playwright_namespaceObject.devices["Desktop Chrome"].userAgent;
2308
2393
 
2309
2394
 
2310
2395
  ;// CONCATENATED MODULE: external "diff-lines"
2311
- var external_diff_lines_x = y => { var x = {}; __nccwpck_require__.d(x, y); return x; }
2312
- var external_diff_lines_y = x => () => x
2313
- const external_diff_lines_namespaceObject = external_diff_lines_x({ ["default"]: () => __WEBPACK_EXTERNAL_MODULE_diff_lines_24b6f423__["default"] });
2396
+ const external_diff_lines_namespaceObject = require("diff-lines");
2397
+ var external_diff_lines_default = /*#__PURE__*/__nccwpck_require__.n(external_diff_lines_namespaceObject);
2314
2398
  ;// CONCATENATED MODULE: ../../packages/web-agent/src/controller.ts
2399
+ var controller_awaiter = (undefined && undefined.__awaiter) || function (thisArg, _arguments, P, generator) {
2400
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
2401
+ return new (P || (P = Promise))(function (resolve, reject) {
2402
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
2403
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
2404
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
2405
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
2406
+ });
2407
+ };
2315
2408
 
2316
2409
  // @ts-expect-error: no types from this library
2317
2410
 
@@ -2353,16 +2446,20 @@ class AgentController {
2353
2446
  /**
2354
2447
  * Reset controller and browser state.
2355
2448
  */
2356
- async resetState() {
2357
- this.resetHistory();
2358
- await this.browser.navigate(this.browser.baseURL);
2449
+ resetState() {
2450
+ return controller_awaiter(this, void 0, void 0, function* () {
2451
+ this.resetHistory();
2452
+ yield this.browser.navigate(this.browser.baseURL);
2453
+ });
2359
2454
  }
2360
2455
  /**
2361
2456
  * Get the browser state as a string
2362
2457
  */
2363
- async getBrowserState() {
2364
- const a11yTree = await this.browser.getA11yTree();
2365
- return a11yTree.serialize();
2458
+ getBrowserState() {
2459
+ return controller_awaiter(this, void 0, void 0, function* () {
2460
+ const a11yTree = yield this.browser.getA11yTree();
2461
+ return a11yTree.serialize();
2462
+ });
2366
2463
  }
2367
2464
  getSerializedHistory(url, currentBrowserState) {
2368
2465
  let history;
@@ -2374,84 +2471,90 @@ class AgentController {
2374
2471
  }
2375
2472
  return history;
2376
2473
  }
2377
- async splitUserGoal(type, goal, disableCache) {
2378
- if (type === StepType.AI_ACTION &&
2379
- goal.match(/[,!;.]|(?:and)|(?:then)/) &&
2380
- this.config.useGoalSplitter) {
2381
- const granularInstructions = await this.generator.getGranularGoals({ goal, url: this.browser.url }, disableCache);
2382
- // convert into a stack (last element is first to be executed)
2383
- this.pendingInstructions = granularInstructions.reverse();
2384
- }
2385
- else {
2386
- this.pendingInstructions = [goal];
2387
- }
2474
+ splitUserGoal(type, goal, disableCache) {
2475
+ return controller_awaiter(this, void 0, void 0, function* () {
2476
+ if (type === StepType.AI_ACTION &&
2477
+ goal.match(/[,!;.]|(?:and)|(?:then)/) &&
2478
+ this.config.useGoalSplitter) {
2479
+ const granularInstructions = yield this.generator.getGranularGoals({ goal, url: this.browser.url }, disableCache);
2480
+ // convert into a stack (last element is first to be executed)
2481
+ this.pendingInstructions = granularInstructions.reverse();
2482
+ }
2483
+ else {
2484
+ this.pendingInstructions = [goal];
2485
+ }
2486
+ });
2388
2487
  }
2389
2488
  /**
2390
2489
  * Given previously executed commands, generate command for the current prompt.
2391
2490
  * Should only be used for AI action.
2392
2491
  */
2393
- async promptToCommand(type, goal, disableCache) {
2394
- // are we out of granular instructions to execute?
2395
- if (this.pendingInstructions.length === 0) {
2396
- // stores granular instructions in this.pendingInstructions, which functions like a stack
2397
- await this.splitUserGoal(type, goal, disableCache);
2398
- }
2399
- const currInstruction = this.pendingInstructions[this.pendingInstructions.length - 1];
2400
- this.logger.info({ goal: currInstruction }, "Starting prompt translation");
2401
- const getBrowserStateStart = Date.now();
2402
- const url = this.browser.url;
2403
- const browserState = await this.getBrowserState();
2404
- this.logger.info({
2405
- duration: Date.now() - getBrowserStateStart,
2406
- url,
2407
- }, "Got browser state");
2408
- const numPrevious = this.commandHistory.length;
2409
- this.commandHistory.push({
2410
- state: "PENDING",
2411
- browserStateBeforeCommand: browserState,
2412
- urlBeforeCommand: url,
2413
- type,
2414
- });
2415
- const history = this.getSerializedHistory(url, browserState);
2416
- const getCommandProposalStart = Date.now();
2417
- const proposedCommand = await this.generator.getProposedCommand({
2418
- url,
2419
- numPrevious,
2420
- browserState,
2421
- history,
2422
- goal: currInstruction,
2423
- lastCommand: this.lastExecutedCommand,
2424
- }, disableCache);
2425
- this.logger.info({ duration: Date.now() - getCommandProposalStart }, "Got proposed command");
2426
- if (proposedCommand.type === ControlFlowCommandType.SUCCESS) {
2427
- const finishedInstruction = this.pendingInstructions.pop();
2428
- this.logger.info({
2429
- finishedInstruction,
2430
- remainingInstructions: this.pendingInstructions,
2431
- }, "Removing pending instruction due to SUCCESS");
2432
- // promptToCommand will pick the next instruction to execute off the stack
2433
- if (this.pendingInstructions.length !== 0) {
2434
- // remove the last command from the history since it was an intermediate command from goalSplitter
2435
- this.commandHistory.pop();
2436
- return this.promptToCommand(type, "", disableCache);
2492
+ promptToCommand(type, goal, disableCache) {
2493
+ return controller_awaiter(this, void 0, void 0, function* () {
2494
+ // are we out of granular instructions to execute?
2495
+ if (this.pendingInstructions.length === 0) {
2496
+ // stores granular instructions in this.pendingInstructions, which functions like a stack
2497
+ yield this.splitUserGoal(type, goal, disableCache);
2437
2498
  }
2438
- }
2439
- else if (
2440
- // on failure, we don't continue to execute
2441
- proposedCommand.type === ControlFlowCommandType.FAILURE) {
2499
+ const currInstruction = this.pendingInstructions[this.pendingInstructions.length - 1];
2500
+ this.logger.info({ goal: currInstruction }, "Starting prompt translation");
2501
+ const getBrowserStateStart = Date.now();
2502
+ const url = this.browser.url;
2503
+ const browserState = yield this.getBrowserState();
2442
2504
  this.logger.info({
2443
- remainingInstructions: this.pendingInstructions,
2444
- }, "Removing pending instructions due to FAILURE");
2445
- this.pendingInstructions = [];
2446
- }
2447
- return proposedCommand;
2505
+ duration: Date.now() - getBrowserStateStart,
2506
+ url,
2507
+ }, "Got browser state");
2508
+ const numPrevious = this.commandHistory.length;
2509
+ this.commandHistory.push({
2510
+ state: "PENDING",
2511
+ browserStateBeforeCommand: browserState,
2512
+ urlBeforeCommand: url,
2513
+ type,
2514
+ });
2515
+ const history = this.getSerializedHistory(url, browserState);
2516
+ const getCommandProposalStart = Date.now();
2517
+ const proposedCommand = yield this.generator.getProposedCommand({
2518
+ url,
2519
+ numPrevious,
2520
+ browserState,
2521
+ history,
2522
+ goal: currInstruction,
2523
+ lastCommand: this.lastExecutedCommand,
2524
+ }, disableCache);
2525
+ this.logger.info({ duration: Date.now() - getCommandProposalStart }, "Got proposed command");
2526
+ if (proposedCommand.type === ControlFlowCommandType.SUCCESS) {
2527
+ const finishedInstruction = this.pendingInstructions.pop();
2528
+ this.logger.info({
2529
+ finishedInstruction,
2530
+ remainingInstructions: this.pendingInstructions,
2531
+ }, "Removing pending instruction due to SUCCESS");
2532
+ // promptToCommand will pick the next instruction to execute off the stack
2533
+ if (this.pendingInstructions.length !== 0) {
2534
+ // remove the last command from the history since it was an intermediate command from goalSplitter
2535
+ this.commandHistory.pop();
2536
+ return this.promptToCommand(type, "", disableCache);
2537
+ }
2538
+ }
2539
+ else if (
2540
+ // on failure, we don't continue to execute
2541
+ proposedCommand.type === ControlFlowCommandType.FAILURE) {
2542
+ this.logger.info({
2543
+ remainingInstructions: this.pendingInstructions,
2544
+ }, "Removing pending instructions due to FAILURE");
2545
+ this.pendingInstructions = [];
2546
+ }
2547
+ return proposedCommand;
2548
+ });
2448
2549
  }
2449
- async locateElement(description, disableCache) {
2450
- const locator = await this.generator.getElementLocation({ browserState: await this.getBrowserState(), goal: description }, disableCache);
2451
- if (locator.id < 0) {
2452
- throw new Error(`Unable to locate element with description: ${description}`);
2453
- }
2454
- return locator;
2550
+ locateElement(description, disableCache) {
2551
+ return controller_awaiter(this, void 0, void 0, function* () {
2552
+ const locator = yield this.generator.getElementLocation({ browserState: yield this.getBrowserState(), goal: description }, disableCache);
2553
+ if (locator.id < 0) {
2554
+ throw new Error(`Unable to locate element with description: ${description}`);
2555
+ }
2556
+ return locator;
2557
+ });
2455
2558
  }
2456
2559
  /**
2457
2560
  * Construct a detailed history that can be passed to the LLM.
@@ -2475,7 +2578,7 @@ class AgentController {
2475
2578
  historyLines.push(` URL CHANGE: '${log.urlBeforeCommand}' -> '${currentURL}'`);
2476
2579
  }
2477
2580
  else {
2478
- const browserStateDiff = (0,external_diff_lines_namespaceObject["default"])(log.browserStateBeforeCommand, currentPageState, {
2581
+ const browserStateDiff = external_diff_lines_default()(log.browserStateBeforeCommand, currentPageState, {
2479
2582
  n_surrounding: 1,
2480
2583
  });
2481
2584
  if (!browserStateDiff) {
@@ -2498,7 +2601,7 @@ class AgentController {
2498
2601
  return historyLines.join("\n");
2499
2602
  }
2500
2603
  getListHistory() {
2501
- return external_dedent_namespaceObject["default"] `Here are the commands that you have successfully executed:
2604
+ return (external_dedent_default()) `Here are the commands that you have successfully executed:
2502
2605
  ${this.commandHistory
2503
2606
  .filter((cmd) => cmd.type === StepType.AI_ACTION)
2504
2607
  .map((cmd) => `- ${cmd.serializedCommand}`)
@@ -2509,254 +2612,261 @@ class AgentController {
2509
2612
  * @param [stateless=false] Execute this command in a stateless fashion, without modifying any controller state such as
2510
2613
  * pending instructions. Useful when executing cached instructions.
2511
2614
  */
2512
- async executeCommand(command, disableCache, stateless = false) {
2513
- const pendingHistory = this.commandHistory[this.commandHistory.length - 1];
2514
- if (!stateless) {
2515
- // if we're not using cached commands, we must be executing a pending command
2516
- // generated by promptToCommand
2517
- if (!pendingHistory || pendingHistory.state !== "PENDING") {
2518
- throw new Error("Executing command but there is no pending entry in the history");
2615
+ executeCommand(command, disableCache, stateless = false) {
2616
+ return controller_awaiter(this, void 0, void 0, function* () {
2617
+ const pendingHistory = this.commandHistory[this.commandHistory.length - 1];
2618
+ if (!stateless) {
2619
+ // if we're not using cached commands, we must be executing a pending command
2620
+ // generated by promptToCommand
2621
+ if (!pendingHistory || pendingHistory.state !== "PENDING") {
2622
+ throw new Error("Executing command but there is no pending entry in the history");
2623
+ }
2519
2624
  }
2520
- }
2521
- else {
2522
- // cached commands can rely on things like a11y IDs - we need to populate this state in the chrome browser.
2523
- // currently, all necessary side effects are accomplished by getting the a11y tree
2524
- await this.browser.getA11yTree();
2525
- }
2526
- let result;
2527
- try {
2528
- const executionStart = Date.now();
2529
- result = await this.sendCommandToBrowser(command, disableCache);
2530
- this.logger.info({ result, duration: Date.now() - executionStart }, "Got execution result");
2531
- }
2532
- catch (e) {
2533
- if (e instanceof Error) {
2534
- throw new BrowserExecutionError(`Failed to execute command: ${e}`, {
2535
- cause: e,
2625
+ else {
2626
+ // cached commands can rely on things like a11y IDs - we need to populate this state in the chrome browser.
2627
+ // currently, all necessary side effects are accomplished by getting the a11y tree
2628
+ yield this.browser.getA11yTree();
2629
+ }
2630
+ let result;
2631
+ try {
2632
+ const executionStart = Date.now();
2633
+ result = yield this.sendCommandToBrowser(command, disableCache);
2634
+ this.logger.info({ result, duration: Date.now() - executionStart }, "Got execution result");
2635
+ }
2636
+ catch (e) {
2637
+ if (e instanceof Error) {
2638
+ throw new BrowserExecutionError(`Failed to execute command: ${e}`, {
2639
+ cause: e,
2640
+ });
2641
+ }
2642
+ throw new BrowserExecutionError(`Unexpected throw from executing command`, {
2643
+ cause: new Error(`${e}`),
2536
2644
  });
2537
2645
  }
2538
- throw new BrowserExecutionError(`Unexpected throw from executing command`, {
2539
- cause: new Error(`${e}`),
2540
- });
2541
- }
2542
- if (result.succeedImmediately && !stateless) {
2543
- // pop the last command off the stack since we won't get a real SUCCESS command within promptToCommand
2544
- this.pendingInstructions.pop();
2545
- if (this.pendingInstructions.length > 0) {
2546
- // we still have pending instructions queued up
2547
- // override the immediate success
2548
- result.succeedImmediately = false;
2646
+ if (result.succeedImmediately && !stateless) {
2647
+ // pop the last command off the stack since we won't get a real SUCCESS command within promptToCommand
2648
+ this.pendingInstructions.pop();
2649
+ if (this.pendingInstructions.length > 0) {
2650
+ // we still have pending instructions queued up
2651
+ // override the immediate success
2652
+ result.succeedImmediately = false;
2653
+ }
2549
2654
  }
2550
- }
2551
- // TODO(ENG-139): Save other a11y stuff as well.
2552
- // Update the command with the targeted element
2553
- // if this is the first time the command was generated
2554
- if (result.elementInteracted &&
2555
- "target" in command &&
2556
- !command.target.elementDescriptor) {
2557
- // Save the serialized element interacted as the "descriptor" for now
2558
- // In the future, we can ask the LLM for a more human-readable descriptor
2559
- command.target.elementDescriptor = result.elementInteracted;
2560
- }
2561
- if (!stateless) {
2562
- // the conditional at the beginning of this function validates that pendingHistory isn't undefined
2563
- // if stateless is false
2564
- pendingHistory.generatedStep = command;
2565
- pendingHistory.serializedCommand = serializeAICommand(command);
2566
- pendingHistory.state = "DONE";
2567
- }
2568
- return result;
2655
+ // TODO(ENG-139): Save other a11y stuff as well.
2656
+ // Update the command with the targeted element
2657
+ // if this is the first time the command was generated
2658
+ if (result.elementInteracted &&
2659
+ "target" in command &&
2660
+ !command.target.elementDescriptor) {
2661
+ // Save the serialized element interacted as the "descriptor" for now
2662
+ // In the future, we can ask the LLM for a more human-readable descriptor
2663
+ command.target.elementDescriptor = result.elementInteracted;
2664
+ }
2665
+ if (!stateless) {
2666
+ // the conditional at the beginning of this function validates that pendingHistory isn't undefined
2667
+ // if stateless is false
2668
+ pendingHistory.generatedStep = command;
2669
+ pendingHistory.serializedCommand = serializeAICommand(command);
2670
+ pendingHistory.state = "DONE";
2671
+ }
2672
+ return result;
2673
+ });
2569
2674
  }
2570
2675
  /**
2571
2676
  * Executes a preset command.
2572
2677
  * For most cases, the execution result contains metadata about the command executed.
2573
2678
  * For assertions, an AssertionResult with thoughts is returned.
2574
2679
  */
2575
- async executePresetStep(command, disableCache) {
2576
- const urlBeforeCommand = this.browser.url;
2577
- switch (command.type) {
2578
- case preset_PresetCommandType.AI_ASSERTION: {
2579
- let params;
2580
- if (command.useVision) {
2581
- params = {
2582
- goal: command.assertion,
2583
- url: urlBeforeCommand,
2584
- // used for vision only
2585
- screenshot: await this.browser.screenshot(),
2586
- // unused for visual assertion
2587
- browserState: "",
2588
- history: "",
2589
- numPrevious: -1,
2590
- lastCommand: null,
2591
- };
2680
+ executePresetStep(command, disableCache) {
2681
+ var _a, _b;
2682
+ return controller_awaiter(this, void 0, void 0, function* () {
2683
+ const urlBeforeCommand = this.browser.url;
2684
+ switch (command.type) {
2685
+ case preset_PresetCommandType.AI_ASSERTION: {
2686
+ let params;
2687
+ if (command.useVision) {
2688
+ params = {
2689
+ goal: command.assertion,
2690
+ url: urlBeforeCommand,
2691
+ // used for vision only
2692
+ screenshot: yield this.browser.screenshot(),
2693
+ // unused for visual assertion
2694
+ browserState: "",
2695
+ history: "",
2696
+ numPrevious: -1,
2697
+ lastCommand: null,
2698
+ };
2699
+ }
2700
+ else {
2701
+ const browserState = yield this.getBrowserState();
2702
+ const history = this.getSerializedHistory(urlBeforeCommand, browserState);
2703
+ params = {
2704
+ goal: command.assertion,
2705
+ url: urlBeforeCommand,
2706
+ // used for text only
2707
+ browserState,
2708
+ history,
2709
+ lastCommand: this.lastExecutedCommand,
2710
+ numPrevious: this.commandHistory.length,
2711
+ };
2712
+ }
2713
+ const result = yield this.generator.getAssertionResult(params, command.useVision, command.disableCache);
2714
+ if (result.relevantElements) {
2715
+ // highlight relevant elements
2716
+ void Promise.all(result.relevantElements.map((id) => this.browser.highlight({ id })));
2717
+ }
2718
+ return result;
2592
2719
  }
2593
- else {
2594
- const browserState = await this.getBrowserState();
2595
- const history = this.getSerializedHistory(urlBeforeCommand, browserState);
2596
- params = {
2597
- goal: command.assertion,
2598
- url: urlBeforeCommand,
2599
- // used for text only
2600
- browserState,
2601
- history,
2602
- lastCommand: this.lastExecutedCommand,
2603
- numPrevious: this.commandHistory.length,
2720
+ case preset_PresetCommandType.NAVIGATE:
2721
+ yield this.browser.navigate(command.url);
2722
+ break;
2723
+ case preset_PresetCommandType.GO_BACK:
2724
+ yield this.browser.goBack();
2725
+ break;
2726
+ case preset_PresetCommandType.GO_FORWARD:
2727
+ yield this.browser.goForward();
2728
+ break;
2729
+ case preset_PresetCommandType.SCROLL_DOWN:
2730
+ yield this.browser.scrollDown();
2731
+ break;
2732
+ case preset_PresetCommandType.SCROLL_UP:
2733
+ yield this.browser.scrollUp();
2734
+ break;
2735
+ case preset_PresetCommandType.WAIT:
2736
+ yield this.browser.wait(command.delay * 1000);
2737
+ break;
2738
+ case preset_PresetCommandType.REFRESH:
2739
+ yield this.browser.refresh();
2740
+ break;
2741
+ case preset_PresetCommandType.CLICK: {
2742
+ let id;
2743
+ if (command.target.a11yData) {
2744
+ id = (_a = command.target.a11yData) === null || _a === void 0 ? void 0 : _a.id;
2745
+ }
2746
+ else {
2747
+ const locator = yield this.locateElement(command.target.elementDescriptor, disableCache);
2748
+ id = locator.id;
2749
+ }
2750
+ const elementInteracted = yield this.browser.click({
2751
+ id,
2752
+ }, {
2753
+ doubleClick: command.doubleClick,
2754
+ rightClick: command.rightClick,
2755
+ });
2756
+ const result = {
2757
+ type: ExecuteResultType.COMMAND,
2758
+ urlAfterCommand: this.browser.url,
2759
+ succeedImmediately: false,
2760
+ elementInteracted,
2604
2761
  };
2762
+ if (urlChanged(urlBeforeCommand, result.urlAfterCommand)) {
2763
+ result.succeedImmediately = true;
2764
+ result.succeedImmediatelyReason = "URL changed";
2765
+ }
2766
+ return result;
2605
2767
  }
2606
- const result = await this.generator.getAssertionResult(params, command.useVision, command.disableCache);
2607
- if (result.relevantElements) {
2608
- // highlight relevant elements
2609
- void Promise.all(result.relevantElements.map((id) => this.browser.highlight({ id })));
2610
- }
2611
- return result;
2612
- }
2613
- case preset_PresetCommandType.NAVIGATE:
2614
- await this.browser.navigate(command.url);
2615
- break;
2616
- case preset_PresetCommandType.GO_BACK:
2617
- await this.browser.goBack();
2618
- break;
2619
- case preset_PresetCommandType.GO_FORWARD:
2620
- await this.browser.goForward();
2621
- break;
2622
- case preset_PresetCommandType.SCROLL_DOWN:
2623
- await this.browser.scrollDown();
2624
- break;
2625
- case preset_PresetCommandType.SCROLL_UP:
2626
- await this.browser.scrollUp();
2627
- break;
2628
- case preset_PresetCommandType.WAIT:
2629
- await this.browser.wait(command.delay * 1000);
2630
- break;
2631
- case preset_PresetCommandType.REFRESH:
2632
- await this.browser.refresh();
2633
- break;
2634
- case preset_PresetCommandType.CLICK: {
2635
- let id;
2636
- if (command.target.a11yData) {
2637
- id = command.target.a11yData?.id;
2638
- }
2639
- else {
2640
- const locator = await this.locateElement(command.target.elementDescriptor, disableCache);
2641
- id = locator.id;
2642
- }
2643
- const elementInteracted = await this.browser.click({
2644
- id,
2645
- }, {
2646
- doubleClick: command.doubleClick,
2647
- rightClick: command.rightClick,
2648
- });
2649
- const result = {
2650
- type: ExecuteResultType.COMMAND,
2651
- urlAfterCommand: this.browser.url,
2652
- succeedImmediately: false,
2653
- elementInteracted,
2654
- };
2655
- if (urlChanged(urlBeforeCommand, result.urlAfterCommand)) {
2656
- result.succeedImmediately = true;
2657
- result.succeedImmediatelyReason = "URL changed";
2658
- }
2659
- return result;
2660
- }
2661
- case preset_PresetCommandType.SELECT_OPTION: {
2662
- let id;
2663
- if (command.target.a11yData) {
2664
- id = command.target.a11yData?.id;
2665
- }
2666
- else {
2667
- const locator = await this.locateElement(command.target.elementDescriptor, disableCache);
2668
- id = locator.id;
2669
- }
2670
- const elementInteracted = await this.browser.selectOption({
2671
- id,
2672
- }, command.option);
2673
- return {
2674
- type: ExecuteResultType.COMMAND,
2675
- succeedImmediately: false,
2676
- urlAfterCommand: this.browser.url,
2677
- elementInteracted,
2678
- };
2679
- }
2680
- case preset_PresetCommandType.TYPE: {
2681
- let elementInteracted;
2682
- const target = command.target;
2683
- if (target.a11yData) {
2684
- elementInteracted = await this.browser.click({
2685
- id: target.a11yData.id,
2686
- });
2768
+ case preset_PresetCommandType.SELECT_OPTION: {
2769
+ let id;
2770
+ if (command.target.a11yData) {
2771
+ id = (_b = command.target.a11yData) === null || _b === void 0 ? void 0 : _b.id;
2772
+ }
2773
+ else {
2774
+ const locator = yield this.locateElement(command.target.elementDescriptor, disableCache);
2775
+ id = locator.id;
2776
+ }
2777
+ const elementInteracted = yield this.browser.selectOption({
2778
+ id,
2779
+ }, command.option);
2780
+ return {
2781
+ type: ExecuteResultType.COMMAND,
2782
+ succeedImmediately: false,
2783
+ urlAfterCommand: this.browser.url,
2784
+ elementInteracted,
2785
+ };
2687
2786
  }
2688
- else if (target.elementDescriptor.length > 0) {
2689
- const locator = await this.locateElement(command.target.elementDescriptor, disableCache);
2690
- elementInteracted = await this.browser.click({
2691
- id: locator.id,
2787
+ case preset_PresetCommandType.TYPE: {
2788
+ let elementInteracted;
2789
+ const target = command.target;
2790
+ if (target.a11yData) {
2791
+ elementInteracted = yield this.browser.click({
2792
+ id: target.a11yData.id,
2793
+ });
2794
+ }
2795
+ else if (target.elementDescriptor.length > 0) {
2796
+ const locator = yield this.locateElement(command.target.elementDescriptor, disableCache);
2797
+ elementInteracted = yield this.browser.click({
2798
+ id: locator.id,
2799
+ });
2800
+ }
2801
+ yield this.browser.type(command.value, {
2802
+ clearContent: command.clearContent,
2803
+ pressKeysSequentially: command.pressKeysSequentially,
2692
2804
  });
2805
+ if (command.pressEnter) {
2806
+ yield this.browser.press("Enter");
2807
+ }
2808
+ const result = {
2809
+ type: ExecuteResultType.COMMAND,
2810
+ urlAfterCommand: this.browser.url,
2811
+ succeedImmediately: false,
2812
+ elementInteracted,
2813
+ };
2814
+ if (urlChanged(urlBeforeCommand, result.urlAfterCommand)) {
2815
+ result.succeedImmediately = true;
2816
+ result.succeedImmediatelyReason = "URL changed";
2817
+ }
2818
+ return result;
2693
2819
  }
2694
- await this.browser.type(command.value, {
2695
- clearContent: command.clearContent,
2696
- pressKeysSequentially: command.pressKeysSequentially,
2697
- });
2698
- if (command.pressEnter) {
2699
- await this.browser.press("Enter");
2700
- }
2701
- const result = {
2702
- type: ExecuteResultType.COMMAND,
2703
- urlAfterCommand: this.browser.url,
2704
- succeedImmediately: false,
2705
- elementInteracted,
2706
- };
2707
- if (urlChanged(urlBeforeCommand, result.urlAfterCommand)) {
2708
- result.succeedImmediately = true;
2709
- result.succeedImmediatelyReason = "URL changed";
2710
- }
2711
- return result;
2820
+ case preset_PresetCommandType.PRESS:
2821
+ yield this.browser.press(command.value);
2822
+ const result = {
2823
+ type: ExecuteResultType.COMMAND,
2824
+ urlAfterCommand: this.browser.url,
2825
+ succeedImmediately: false,
2826
+ };
2827
+ if (urlChanged(urlBeforeCommand, result.urlAfterCommand)) {
2828
+ result.succeedImmediately = true;
2829
+ result.succeedImmediatelyReason = "URL changed";
2830
+ }
2831
+ return result;
2832
+ default:
2833
+ const assertUnreachable = (_x) => {
2834
+ throw "If Typescript complains about the line below, you missed a case or break in the switch above";
2835
+ };
2836
+ return assertUnreachable(command);
2712
2837
  }
2713
- case preset_PresetCommandType.PRESS:
2714
- await this.browser.press(command.value);
2715
- const result = {
2716
- type: ExecuteResultType.COMMAND,
2717
- urlAfterCommand: this.browser.url,
2718
- succeedImmediately: false,
2719
- };
2720
- if (urlChanged(urlBeforeCommand, result.urlAfterCommand)) {
2721
- result.succeedImmediately = true;
2722
- result.succeedImmediatelyReason = "URL changed";
2723
- }
2724
- return result;
2725
- default:
2726
- const assertUnreachable = (_x) => {
2727
- throw "If Typescript complains about the line below, you missed a case or break in the switch above";
2728
- };
2729
- return assertUnreachable(command);
2730
- }
2731
- return {
2732
- type: ExecuteResultType.COMMAND,
2733
- succeedImmediately: false,
2734
- urlAfterCommand: this.browser.url,
2735
- };
2736
- }
2737
- async sendCommandToBrowser(command, disableCache) {
2738
- switch (command.type) {
2739
- /**
2740
- * Control flow
2741
- */
2742
- case ControlFlowCommandType.SUCCESS:
2743
- case ControlFlowCommandType.FAILURE:
2744
- return {
2745
- type: ExecuteResultType.COMMAND,
2746
- succeedImmediately: false,
2747
- urlAfterCommand: this.browser.url,
2748
- };
2749
- /**
2750
- * Preset action
2751
- */
2752
- default:
2753
- const result = await this.executePresetStep(command, disableCache);
2754
- if (result.type !== "command") {
2755
- // AI should never generate 'assertion' results
2756
- throw new Error(`Unexpected preset result type ${result.type} from executing AI action`);
2757
- }
2758
- return result;
2759
- }
2838
+ return {
2839
+ type: ExecuteResultType.COMMAND,
2840
+ succeedImmediately: false,
2841
+ urlAfterCommand: this.browser.url,
2842
+ };
2843
+ });
2844
+ }
2845
+ sendCommandToBrowser(command, disableCache) {
2846
+ return controller_awaiter(this, void 0, void 0, function* () {
2847
+ switch (command.type) {
2848
+ /**
2849
+ * Control flow
2850
+ */
2851
+ case ControlFlowCommandType.SUCCESS:
2852
+ case ControlFlowCommandType.FAILURE:
2853
+ return {
2854
+ type: ExecuteResultType.COMMAND,
2855
+ succeedImmediately: false,
2856
+ urlAfterCommand: this.browser.url,
2857
+ };
2858
+ /**
2859
+ * Preset action
2860
+ */
2861
+ default:
2862
+ const result = yield this.executePresetStep(command, disableCache);
2863
+ if (result.type !== "command") {
2864
+ // AI should never generate 'assertion' results
2865
+ throw new Error(`Unexpected preset result type ${result.type} from executing AI action`);
2866
+ }
2867
+ return result;
2868
+ }
2869
+ });
2760
2870
  }
2761
2871
  }
2762
2872
 
@@ -2764,6 +2874,15 @@ class AgentController {
2764
2874
  var fetch_retry = __nccwpck_require__(62);
2765
2875
  var fetch_retry_default = /*#__PURE__*/__nccwpck_require__.n(fetch_retry);
2766
2876
  ;// CONCATENATED MODULE: ../../packages/web-agent/src/generators/api-generator.ts
2877
+ var api_generator_awaiter = (undefined && undefined.__awaiter) || function (thisArg, _arguments, P, generator) {
2878
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
2879
+ return new (P || (P = Promise))(function (resolve, reject) {
2880
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
2881
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
2882
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
2883
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
2884
+ });
2885
+ };
2767
2886
 
2768
2887
 
2769
2888
 
@@ -2774,72 +2893,83 @@ class APIGenerator {
2774
2893
  this.baseURL = params.baseURL;
2775
2894
  this.apiKey = params.apiKey;
2776
2895
  }
2777
- async getElementLocation(context, disableCache) {
2778
- const result = await this.sendRequest(`/${API_VERSION}/web-agent/locate-element`, {
2779
- browserState: context.browserState,
2780
- goal: context.goal,
2781
- disableCache,
2896
+ getElementLocation(context, disableCache) {
2897
+ return api_generator_awaiter(this, void 0, void 0, function* () {
2898
+ const result = yield this.sendRequest(`/${API_VERSION}/web-agent/locate-element`, {
2899
+ browserState: context.browserState,
2900
+ goal: context.goal,
2901
+ disableCache,
2902
+ });
2903
+ return locator_AILocatorSchema.parse(result);
2782
2904
  });
2783
- return locator_AILocatorSchema.parse(result);
2784
2905
  }
2785
- async getAssertionResult(context, useVision, disableCache) {
2786
- if (useVision) {
2787
- const result = await this.sendRequest(`/${API_VERSION}/web-agent/assertion`, {
2906
+ getAssertionResult(context, useVision, disableCache) {
2907
+ var _a;
2908
+ return api_generator_awaiter(this, void 0, void 0, function* () {
2909
+ if (useVision) {
2910
+ const result = yield this.sendRequest(`/${API_VERSION}/web-agent/assertion`, {
2911
+ url: context.url,
2912
+ goal: context.goal,
2913
+ screenshot: (_a = context.screenshot) === null || _a === void 0 ? void 0 : _a.toString("base64"),
2914
+ disableCache,
2915
+ vision: true,
2916
+ });
2917
+ return execute_results_ExecuteAssertionResultSchema.parse(result);
2918
+ }
2919
+ const result = yield this.sendRequest(`/${API_VERSION}/web-agent/assertion`, {
2788
2920
  url: context.url,
2921
+ browserState: context.browserState,
2789
2922
  goal: context.goal,
2790
- screenshot: context.screenshot?.toString("base64"),
2923
+ history: context.history,
2924
+ numPrevious: context.numPrevious,
2925
+ lastCommand: context.lastCommand,
2791
2926
  disableCache,
2792
- vision: true,
2927
+ vision: false,
2793
2928
  });
2794
2929
  return execute_results_ExecuteAssertionResultSchema.parse(result);
2795
- }
2796
- const result = await this.sendRequest(`/${API_VERSION}/web-agent/assertion`, {
2797
- url: context.url,
2798
- browserState: context.browserState,
2799
- goal: context.goal,
2800
- history: context.history,
2801
- numPrevious: context.numPrevious,
2802
- lastCommand: context.lastCommand,
2803
- disableCache,
2804
- vision: false,
2805
2930
  });
2806
- return execute_results_ExecuteAssertionResultSchema.parse(result);
2807
- }
2808
- async getProposedCommand(context, disableCache) {
2809
- const result = await this.sendRequest(`/${API_VERSION}/web-agent/next-command`, {
2810
- url: context.url,
2811
- browserState: context.browserState,
2812
- goal: context.goal,
2813
- history: context.history,
2814
- numPrevious: context.numPrevious,
2815
- lastCommand: context.lastCommand,
2816
- disableCache,
2931
+ }
2932
+ getProposedCommand(context, disableCache) {
2933
+ return api_generator_awaiter(this, void 0, void 0, function* () {
2934
+ const result = yield this.sendRequest(`/${API_VERSION}/web-agent/next-command`, {
2935
+ url: context.url,
2936
+ browserState: context.browserState,
2937
+ goal: context.goal,
2938
+ history: context.history,
2939
+ numPrevious: context.numPrevious,
2940
+ lastCommand: context.lastCommand,
2941
+ disableCache,
2942
+ });
2943
+ return ai_commands_AICommandSchema.parse(result);
2817
2944
  });
2818
- return ai_commands_AICommandSchema.parse(result);
2819
2945
  }
2820
- async getGranularGoals(context, disableCache) {
2821
- const result = await this.sendRequest(`/${API_VERSION}/web-agent/split-goal`, {
2822
- url: context.url,
2823
- goal: context.goal,
2824
- disableCache,
2946
+ getGranularGoals(context, disableCache) {
2947
+ return api_generator_awaiter(this, void 0, void 0, function* () {
2948
+ const result = yield this.sendRequest(`/${API_VERSION}/web-agent/split-goal`, {
2949
+ url: context.url,
2950
+ goal: context.goal,
2951
+ disableCache,
2952
+ });
2953
+ return external_zod_namespaceObject.string().array().parse(result);
2825
2954
  });
2826
- return external_zod_namespaceObject.string().array().parse(result);
2827
- }
2828
- async sendRequest(path, body) {
2829
- const response = await fetch(`${this.baseURL}${path}`, {
2830
- retries: 3,
2831
- retryDelay: 1000,
2832
- method: "POST",
2833
- body: JSON.stringify(body),
2834
- headers: {
2835
- "Content-Type": "application/json",
2836
- Authorization: `Bearer ${this.apiKey}`,
2837
- },
2955
+ }
2956
+ sendRequest(path, body) {
2957
+ return api_generator_awaiter(this, void 0, void 0, function* () {
2958
+ const response = yield fetch(`${this.baseURL}${path}`, {
2959
+ retries: 3,
2960
+ retryDelay: 1000,
2961
+ method: "POST",
2962
+ body: JSON.stringify(body),
2963
+ headers: {
2964
+ "Content-Type": "application/json",
2965
+ Authorization: `Bearer ${this.apiKey}`,
2966
+ },
2967
+ });
2968
+ if (!response.ok) {
2969
+ throw new Error(`Request to ${path} failed with status ${response.status}: ${yield response.text()}`);
2970
+ }
2971
+ return response.json();
2838
2972
  });
2839
- if (!response.ok) {
2840
- throw new Error(`Request to ${path} failed with status ${response.status}: ${await response.text()}`);
2841
- }
2842
- return response.json();
2843
2973
  }
2844
2974
  }
2845
2975
 
@@ -2852,7 +2982,6 @@ class APIGenerator {
2852
2982
 
2853
2983
  })();
2854
2984
 
2855
- var __webpack_exports__APIGenerator = __webpack_exports__._w;
2856
- var __webpack_exports__AgentController = __webpack_exports__.Yt;
2857
- var __webpack_exports__ChromeBrowser = __webpack_exports__.DE;
2858
- export { __webpack_exports__APIGenerator as APIGenerator, __webpack_exports__AgentController as AgentController, __webpack_exports__ChromeBrowser as ChromeBrowser };
2985
+ module.exports = __webpack_exports__;
2986
+ /******/ })()
2987
+ ;