automation_model 1.0.410-dev → 1.0.410-stage

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,9 +2,9 @@
2
2
  import { expect } from "@playwright/test";
3
3
  import dayjs from "dayjs";
4
4
  import fs from "fs";
5
+ import { Jimp } from "jimp";
5
6
  import path from "path";
6
7
  import reg_parser from "regex-parser";
7
- import sharp from "sharp";
8
8
  import { findDateAlternatives, findNumberAlternatives } from "./analyze_helper.js";
9
9
  import { getDateTimeValue } from "./date_time.js";
10
10
  import drawRectangle from "./drawRect.js";
@@ -14,6 +14,8 @@ import objectPath from "object-path";
14
14
  import { decrypt } from "./utils.js";
15
15
  import csv from "csv-parser";
16
16
  import { Readable } from "node:stream";
17
+ import readline from "readline";
18
+ import { getContext } from "./init_browser.js";
17
19
  const Types = {
18
20
  CLICK: "click_element",
19
21
  NAVIGATE: "navigate",
@@ -41,15 +43,19 @@ const Types = {
41
43
  LOAD_DATA: "load_data",
42
44
  SET_INPUT: "set_input",
43
45
  };
46
+ export const apps = {};
44
47
  class StableBrowser {
45
- constructor(browser, page, logger = null, context = null) {
48
+ constructor(browser, page, logger = null, context = null, world = null) {
46
49
  this.browser = browser;
47
50
  this.page = page;
48
51
  this.logger = logger;
49
52
  this.context = context;
53
+ this.world = world;
50
54
  this.project_path = null;
51
55
  this.webLogFile = null;
56
+ this.networkLogger = null;
52
57
  this.configuration = null;
58
+ this.appName = "main";
53
59
  if (!this.logger) {
54
60
  this.logger = console;
55
61
  }
@@ -75,23 +81,34 @@ class StableBrowser {
75
81
  this.logger.error("unable to read ai_config.json");
76
82
  }
77
83
  const logFolder = path.join(this.project_path, "logs", "web");
78
- this.webLogFile = this.getWebLogFile(logFolder);
79
- this.registerConsoleLogListener(page, context, this.webLogFile);
80
- this.registerRequestListener();
84
+ this.world = world;
81
85
  context.pages = [this.page];
82
86
  context.pageLoading = { status: false };
87
+ this.registerEventListeners(this.context);
88
+ }
89
+ registerEventListeners(context) {
90
+ this.registerConsoleLogListener(this.page, context);
91
+ this.registerRequestListener(this.page, context, this.webLogFile);
92
+ if (!context.pageLoading) {
93
+ context.pageLoading = { status: false };
94
+ }
83
95
  context.playContext.on("page", async function (page) {
84
96
  context.pageLoading.status = true;
85
97
  this.page = page;
86
98
  context.page = page;
87
99
  context.pages.push(page);
88
100
  page.on("close", async () => {
89
- if (this.context && this.context.pages && this.context.pages.length > 0) {
101
+ if (this.context && this.context.pages && this.context.pages.length > 1) {
90
102
  this.context.pages.pop();
91
103
  this.page = this.context.pages[this.context.pages.length - 1];
92
104
  this.context.page = this.page;
93
- let title = await this.page.title();
94
- console.log("Switched to page " + title);
105
+ try {
106
+ let title = await this.page.title();
107
+ console.log("Switched to page " + title);
108
+ }
109
+ catch (error) {
110
+ console.error("Error on page close", error);
111
+ }
95
112
  }
96
113
  });
97
114
  try {
@@ -104,6 +121,36 @@ class StableBrowser {
104
121
  context.pageLoading.status = false;
105
122
  }.bind(this));
106
123
  }
124
+ async switchApp(appName) {
125
+ // check if the current app (this.appName) is the same as the new app
126
+ if (this.appName === appName) {
127
+ return;
128
+ }
129
+ let navigate = false;
130
+ if (!apps[appName]) {
131
+ let newContext = await getContext(null, false, this.logger, appName, false, this);
132
+ navigate = true;
133
+ apps[appName] = {
134
+ context: newContext,
135
+ browser: newContext.browser,
136
+ page: newContext.page,
137
+ };
138
+ }
139
+ const tempContext = {};
140
+ this._copyContext(this, tempContext);
141
+ this._copyContext(apps[appName], this);
142
+ apps[this.appName] = tempContext;
143
+ this.appName = appName;
144
+ if (navigate) {
145
+ await this.goto(this.context.environment.baseUrl);
146
+ await this.waitForPageLoad();
147
+ }
148
+ }
149
+ _copyContext(from, to) {
150
+ to.browser = from.browser;
151
+ to.page = from.page;
152
+ to.context = from.context;
153
+ }
107
154
  getWebLogFile(logFolder) {
108
155
  if (!fs.existsSync(logFolder)) {
109
156
  fs.mkdirSync(logFolder, { recursive: true });
@@ -115,37 +162,63 @@ class StableBrowser {
115
162
  const fileName = nextIndex + ".json";
116
163
  return path.join(logFolder, fileName);
117
164
  }
118
- registerConsoleLogListener(page, context, logFile) {
165
+ registerConsoleLogListener(page, context) {
119
166
  if (!this.context.webLogger) {
120
167
  this.context.webLogger = [];
121
168
  }
122
169
  page.on("console", async (msg) => {
123
- this.context.webLogger.push({
170
+ var _a;
171
+ const obj = {
124
172
  type: msg.type(),
125
173
  text: msg.text(),
126
174
  location: msg.location(),
127
175
  time: new Date().toISOString(),
128
- });
129
- await fs.promises.writeFile(logFile, JSON.stringify(this.context.webLogger, null, 2));
176
+ };
177
+ this.context.webLogger.push(obj);
178
+ (_a = this.world) === null || _a === void 0 ? void 0 : _a.attach(JSON.stringify(obj), { mediaType: "application/json+log" });
130
179
  });
131
180
  }
132
- registerRequestListener() {
133
- this.page.on("request", async (data) => {
181
+ registerRequestListener(page, context, logFile) {
182
+ if (!this.context.networkLogger) {
183
+ this.context.networkLogger = [];
184
+ }
185
+ page.on("request", async (data) => {
186
+ var _a;
187
+ const startTime = new Date().getTime();
134
188
  try {
135
- const pageUrl = new URL(this.page.url());
189
+ const pageUrl = new URL(page.url());
136
190
  const requestUrl = new URL(data.url());
137
191
  if (pageUrl.hostname === requestUrl.hostname) {
138
192
  const method = data.method();
139
- if (method === "POST" || method === "GET" || method === "PUT" || method === "DELETE" || method === "PATCH") {
193
+ if (["POST", "GET", "PUT", "DELETE", "PATCH"].includes(method)) {
140
194
  const token = await data.headerValue("Authorization");
141
195
  if (token) {
142
- this.context.authtoken = token;
196
+ context.authtoken = token;
143
197
  }
144
198
  }
145
199
  }
200
+ const response = await data.response();
201
+ const endTime = new Date().getTime();
202
+ const obj = {
203
+ url: data.url(),
204
+ method: data.method(),
205
+ postData: data.postData(),
206
+ error: data.failure() ? data.failure().errorText : null,
207
+ duration: endTime - startTime,
208
+ startTime,
209
+ };
210
+ context.networkLogger.push(obj);
211
+ (_a = this.world) === null || _a === void 0 ? void 0 : _a.attach(JSON.stringify(obj), { mediaType: "application/json+network" });
146
212
  }
147
213
  catch (error) {
148
214
  console.error("Error in request listener", error);
215
+ context.networkLogger.push({
216
+ error: "not able to listen",
217
+ message: error.message,
218
+ stack: error.stack,
219
+ time: new Date().toISOString(),
220
+ });
221
+ // await fs.promises.writeFile(logFile, JSON.stringify(context.networkLogger, null, 2));
149
222
  }
150
223
  });
151
224
  }
@@ -215,6 +288,7 @@ class StableBrowser {
215
288
  }
216
289
  _getLocator(locator, scope, _params) {
217
290
  locator = this._fixLocatorUsingParams(locator, _params);
291
+ let locatorReturn;
218
292
  if (locator.role) {
219
293
  if (locator.role[1].nameReg) {
220
294
  locator.role[1].name = reg_parser(locator.role[1].nameReg);
@@ -223,10 +297,10 @@ class StableBrowser {
223
297
  // if (locator.role[1].name) {
224
298
  // locator.role[1].name = this._fixUsingParams(locator.role[1].name, _params);
225
299
  // }
226
- return scope.getByRole(locator.role[0], locator.role[1]);
300
+ locatorReturn = scope.getByRole(locator.role[0], locator.role[1]);
227
301
  }
228
302
  if (locator.css) {
229
- return scope.locator(locator.css);
303
+ locatorReturn = scope.locator(locator.css);
230
304
  }
231
305
  // handle role/name locators
232
306
  // locator.selector will be something like: textbox[name="Username"i]
@@ -237,23 +311,33 @@ class StableBrowser {
237
311
  const role = match[1];
238
312
  const name = match[3];
239
313
  const flags = match[4];
240
- return scope.getByRole(role, { name }, { exact: flags === "i" });
314
+ locatorReturn = scope.getByRole(role, { name }, { exact: flags === "i" });
241
315
  }
242
316
  }
243
- // handle css locators
244
- if (locator.engine === "css") {
245
- return scope.locator(locator.selector);
246
- }
247
- if ((locator === null || locator === void 0 ? void 0 : locator.engine) && (locator === null || locator === void 0 ? void 0 : locator.score) <= 520) {
248
- if (locator.engine === "internal:attr") {
249
- selector = `[${selector}]`;
317
+ if (locator === null || locator === void 0 ? void 0 : locator.engine) {
318
+ if (locator.engine === "css") {
319
+ locatorReturn = scope.locator(locator.selector);
320
+ }
321
+ else {
322
+ let selector = locator.selector;
323
+ if (locator.engine === "internal:attr") {
324
+ if (!selector.startsWith("[")) {
325
+ selector = `[${selector}]`;
326
+ }
327
+ }
328
+ locatorReturn = scope.locator(`${locator.engine}=${selector}`);
250
329
  }
251
- const newLocator = scope.locator(`${locator.engine}=${selector}`);
252
- return newLocator;
253
330
  }
254
- throw new Error("unknown locator type");
331
+ if (!locatorReturn) {
332
+ console.error(locator);
333
+ throw new Error("Locator undefined");
334
+ }
335
+ return locatorReturn;
255
336
  }
256
337
  async _locateElmentByTextClimbCss(scope, text, climb, css, _params) {
338
+ if (css && css.locator) {
339
+ css = css.locator;
340
+ }
257
341
  let result = await this._locateElementByText(scope, this._fixUsingParams(text, _params), "*", false, true, _params);
258
342
  if (result.elementCount === 0) {
259
343
  return;
@@ -462,6 +546,8 @@ class StableBrowser {
462
546
  if (result.foundElements.length > 0) {
463
547
  let dialogCloseLocator = result.foundElements[0].locator;
464
548
  await dialogCloseLocator.click();
549
+ // wait for the dialog to close
550
+ await dialogCloseLocator.waitFor({ state: "hidden" });
465
551
  return { rerun: true };
466
552
  }
467
553
  }
@@ -470,7 +556,7 @@ class StableBrowser {
470
556
  }
471
557
  async _locate(selectors, info, _params, timeout = 30000) {
472
558
  for (let i = 0; i < 3; i++) {
473
- info.log += "attempt " + i + ": totoal locators " + selectors.locators.length + "\n";
559
+ info.log += "attempt " + i + ": total locators " + selectors.locators.length + "\n";
474
560
  for (let j = 0; j < selectors.locators.length; j++) {
475
561
  let selector = selectors.locators[j];
476
562
  info.log += "searching for locator " + j + ":" + JSON.stringify(selector) + "\n";
@@ -490,9 +576,30 @@ class StableBrowser {
490
576
  //let arrayMode = Array.isArray(selectors);
491
577
  let scope = this.page;
492
578
  if (selectors.iframe_src || selectors.frameLocators) {
579
+ const findFrame = (frame, framescope) => {
580
+ for (let i = 0; i < frame.selectors.length; i++) {
581
+ let frameLocator = frame.selectors[i];
582
+ if (frameLocator.css) {
583
+ framescope = framescope.frameLocator(frameLocator.css);
584
+ if (frameLocator.index) {
585
+ framescope = framescope.nth(frameLocator.index);
586
+ }
587
+ break;
588
+ }
589
+ }
590
+ if (frame.children) {
591
+ return findFrame(frame.children, framescope);
592
+ }
593
+ return framescope;
594
+ };
493
595
  info.log += "searching for iframe " + selectors.iframe_src + "/" + selectors.frameLocators + "\n";
494
596
  while (true) {
495
597
  let frameFound = false;
598
+ if (selectors.nestFrmLoc) {
599
+ scope = findFrame(selectors.nestFrmLoc, scope);
600
+ frameFound = true;
601
+ break;
602
+ }
496
603
  if (selectors.frameLocators) {
497
604
  for (let i = 0; i < selectors.frameLocators.length; i++) {
498
605
  let frameLocator = selectors.frameLocators[i];
@@ -656,6 +763,9 @@ class StableBrowser {
656
763
  async click(selectors, _params, options = {}, world = null) {
657
764
  this._validateSelectors(selectors);
658
765
  const startTime = Date.now();
766
+ if (options && options.context) {
767
+ selectors.locators[0].text = options.context;
768
+ }
659
769
  const info = {};
660
770
  info.log = "***** click on " + selectors.element_name + " *****\n";
661
771
  info.operation = "click";
@@ -669,14 +779,14 @@ class StableBrowser {
669
779
  ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
670
780
  try {
671
781
  await this._highlightElements(element);
672
- await element.click({ timeout: 5000 });
782
+ await element.click();
673
783
  await new Promise((resolve) => setTimeout(resolve, 1000));
674
784
  }
675
785
  catch (e) {
676
786
  // await this.closeUnexpectedPopups();
677
787
  info.log += "click failed, will try again" + "\n";
678
788
  element = await this._locate(selectors, info, _params);
679
- await element.click({ timeout: 10000, force: true });
789
+ await element.dispatchEvent("click");
680
790
  await new Promise((resolve) => setTimeout(resolve, 1000));
681
791
  }
682
792
  await this.waitForPageLoad();
@@ -729,7 +839,7 @@ class StableBrowser {
729
839
  ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
730
840
  try {
731
841
  await this._highlightElements(element);
732
- await element.setChecked(checked, { timeout: 5000 });
842
+ await element.setChecked(checked);
733
843
  await new Promise((resolve) => setTimeout(resolve, 1000));
734
844
  }
735
845
  catch (e) {
@@ -793,7 +903,7 @@ class StableBrowser {
793
903
  ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
794
904
  try {
795
905
  await this._highlightElements(element);
796
- await element.hover({ timeout: 10000 });
906
+ await element.hover();
797
907
  await new Promise((resolve) => setTimeout(resolve, 1000));
798
908
  }
799
909
  catch (e) {
@@ -855,7 +965,7 @@ class StableBrowser {
855
965
  ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
856
966
  try {
857
967
  await this._highlightElements(element);
858
- await element.selectOption(values, { timeout: 5000 });
968
+ await element.selectOption(values);
859
969
  }
860
970
  catch (e) {
861
971
  //await this.closeUnexpectedPopups();
@@ -1258,7 +1368,7 @@ class StableBrowser {
1258
1368
  let element = await this._locate(selectors, info, _params);
1259
1369
  ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
1260
1370
  await this._highlightElements(element);
1261
- await element.fill(value, { timeout: 10000 });
1371
+ await element.fill(value);
1262
1372
  await element.dispatchEvent("change");
1263
1373
  if (enter) {
1264
1374
  await new Promise((resolve) => setTimeout(resolve, 2000));
@@ -1477,7 +1587,7 @@ class StableBrowser {
1477
1587
  return info;
1478
1588
  }
1479
1589
  catch (e) {
1480
- //await this.closeUnexpectedPopups();
1590
+ await this.closeUnexpectedPopups();
1481
1591
  this.logger.error("verify element contains text failed " + info.log);
1482
1592
  ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
1483
1593
  info.screenshotPath = screenshotPath;
@@ -1525,6 +1635,29 @@ class StableBrowser {
1525
1635
  }
1526
1636
  return dataFile;
1527
1637
  }
1638
+ async waitForUserInput(message, world = null) {
1639
+ if (!message) {
1640
+ message = "# Wait for user input. Press any key to continue";
1641
+ }
1642
+ else {
1643
+ message = "# Wait for user input. " + message;
1644
+ }
1645
+ message += "\n";
1646
+ const value = await new Promise((resolve) => {
1647
+ const rl = readline.createInterface({
1648
+ input: process.stdin,
1649
+ output: process.stdout,
1650
+ });
1651
+ rl.question(message, (answer) => {
1652
+ rl.close();
1653
+ resolve(answer);
1654
+ });
1655
+ });
1656
+ if (value) {
1657
+ this.logger.info(`{{userInput}} was set to: ${value}`);
1658
+ }
1659
+ this.setTestData({ userInput: value }, world);
1660
+ }
1528
1661
  setTestData(testData, world = null) {
1529
1662
  if (!testData) {
1530
1663
  return;
@@ -1712,7 +1845,6 @@ class StableBrowser {
1712
1845
  }
1713
1846
  async takeScreenshot(screenshotPath) {
1714
1847
  const playContext = this.context.playContext;
1715
- const client = await playContext.newCDPSession(this.page);
1716
1848
  // Using CDP to capture the screenshot
1717
1849
  const viewportWidth = Math.max(...(await this.page.evaluate(() => [
1718
1850
  document.body.scrollWidth,
@@ -1722,41 +1854,40 @@ class StableBrowser {
1722
1854
  document.body.clientWidth,
1723
1855
  document.documentElement.clientWidth,
1724
1856
  ])));
1725
- const viewportHeight = Math.max(...(await this.page.evaluate(() => [
1726
- document.body.scrollHeight,
1727
- document.documentElement.scrollHeight,
1728
- document.body.offsetHeight,
1729
- document.documentElement.offsetHeight,
1730
- document.body.clientHeight,
1731
- document.documentElement.clientHeight,
1732
- ])));
1733
- const { data } = await client.send("Page.captureScreenshot", {
1734
- format: "png",
1735
- clip: {
1736
- x: 0,
1737
- y: 0,
1738
- width: viewportWidth,
1739
- height: viewportHeight,
1740
- scale: 1,
1741
- },
1742
- });
1743
- if (!screenshotPath) {
1744
- return data;
1745
- }
1746
- let screenshotBuffer = Buffer.from(data, "base64");
1747
- const sharpBuffer = sharp(screenshotBuffer);
1748
- const metadata = await sharpBuffer.metadata();
1749
- //check if you are on retina display and reduce the quality of the image
1750
- if (metadata.width > viewportWidth || metadata.height > viewportHeight) {
1751
- screenshotBuffer = await sharpBuffer
1752
- .resize(viewportWidth, viewportHeight, {
1753
- fit: sharp.fit.inside,
1754
- withoutEnlargement: true,
1755
- })
1756
- .toBuffer();
1757
- }
1758
- fs.writeFileSync(screenshotPath, screenshotBuffer);
1759
- await client.detach();
1857
+ let screenshotBuffer = null;
1858
+ if (this.context.browserName === "chromium") {
1859
+ const client = await playContext.newCDPSession(this.page);
1860
+ const { data } = await client.send("Page.captureScreenshot", {
1861
+ format: "png",
1862
+ // clip: {
1863
+ // x: 0,
1864
+ // y: 0,
1865
+ // width: viewportWidth,
1866
+ // height: viewportHeight,
1867
+ // scale: 1,
1868
+ // },
1869
+ });
1870
+ await client.detach();
1871
+ if (!screenshotPath) {
1872
+ return data;
1873
+ }
1874
+ screenshotBuffer = Buffer.from(data, "base64");
1875
+ }
1876
+ else {
1877
+ screenshotBuffer = await this.page.screenshot();
1878
+ }
1879
+ let image = await Jimp.read(screenshotBuffer);
1880
+ // Get the image dimensions
1881
+ const { width, height } = image.bitmap;
1882
+ const resizeRatio = viewportWidth / width;
1883
+ // Resize the image to fit within the viewport dimensions without enlarging
1884
+ if (width > viewportWidth) {
1885
+ image = image.resize({ w: viewportWidth, h: height * resizeRatio }); // Resize the image while maintaining aspect ratio
1886
+ await image.write(screenshotPath);
1887
+ }
1888
+ else {
1889
+ fs.writeFileSync(screenshotPath, screenshotBuffer);
1890
+ }
1760
1891
  }
1761
1892
  async verifyElementExistInPage(selectors, _params = null, options = {}, world = null) {
1762
1893
  this._validateSelectors(selectors);
@@ -2564,13 +2695,13 @@ class StableBrowser {
2564
2695
  }
2565
2696
  catch (e) {
2566
2697
  if (e.label === "networkidle") {
2567
- console.log("waitted for the network to be idle timeout");
2698
+ console.log("waited for the network to be idle timeout");
2568
2699
  }
2569
2700
  else if (e.label === "load") {
2570
- console.log("waitted for the load timeout");
2701
+ console.log("waited for the load timeout");
2571
2702
  }
2572
2703
  else if (e.label === "domcontentloaded") {
2573
- console.log("waitted for the domcontent loaded timeout");
2704
+ console.log("waited for the domcontent loaded timeout");
2574
2705
  }
2575
2706
  console.log(".");
2576
2707
  }
@@ -2713,33 +2844,18 @@ class StableBrowser {
2713
2844
  }
2714
2845
  async scrollIfNeeded(element, info) {
2715
2846
  try {
2716
- let didScroll = await element.evaluate((node) => {
2717
- const rect = node.getBoundingClientRect();
2718
- if (rect &&
2719
- rect.top >= 0 &&
2720
- rect.left >= 0 &&
2721
- rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
2722
- rect.right <= (window.innerWidth || document.documentElement.clientWidth)) {
2723
- return false;
2724
- }
2725
- else {
2726
- node.scrollIntoView({
2727
- behavior: "smooth",
2728
- block: "center",
2729
- inline: "center",
2730
- });
2731
- return true;
2732
- }
2847
+ await element.scrollIntoViewIfNeeded({
2848
+ timeout: 2000,
2733
2849
  });
2734
- if (didScroll) {
2735
- await new Promise((resolve) => setTimeout(resolve, 500));
2736
- if (info) {
2737
- info.box = await element.boundingBox();
2738
- }
2850
+ await new Promise((resolve) => setTimeout(resolve, 500));
2851
+ if (info) {
2852
+ info.box = await element.boundingBox({
2853
+ timeout: 1000,
2854
+ });
2739
2855
  }
2740
2856
  }
2741
2857
  catch (e) {
2742
- console.log("scroll failed");
2858
+ console.log("#-#");
2743
2859
  }
2744
2860
  }
2745
2861
  _reportToWorld(world, properties) {