automation_model 1.0.412-dev → 1.0.412-stage

package/lib/stable_browser.js

@@ -2,9 +2,9 @@
 import { expect } from "@playwright/test";
 import dayjs from "dayjs";
 import fs from "fs";
+import { Jimp } from "jimp";
 import path from "path";
 import reg_parser from "regex-parser";
-import sharp from "sharp";
 import { findDateAlternatives, findNumberAlternatives } from "./analyze_helper.js";
 import { getDateTimeValue } from "./date_time.js";
 import drawRectangle from "./drawRect.js";
@@ -14,6 +14,8 @@ import objectPath from "object-path";
 import { decrypt } from "./utils.js";
 import csv from "csv-parser";
 import { Readable } from "node:stream";
+import readline from "readline";
+import { getContext } from "./init_browser.js";
 const Types = {
     CLICK: "click_element",
     NAVIGATE: "navigate",
@@ -41,15 +43,19 @@ const Types = {
     LOAD_DATA: "load_data",
     SET_INPUT: "set_input",
 };
+export const apps = {};
 class StableBrowser {
-    constructor(browser, page, logger = null, context = null) {
+    constructor(browser, page, logger = null, context = null, world = null) {
         this.browser = browser;
         this.page = page;
         this.logger = logger;
         this.context = context;
+        this.world = world;
         this.project_path = null;
         this.webLogFile = null;
+        this.networkLogger = null;
         this.configuration = null;
+        this.appName = "main";
         if (!this.logger) {
             this.logger = console;
         }
@@ -75,23 +81,34 @@ class StableBrowser {
             this.logger.error("unable to read ai_config.json");
         }
         const logFolder = path.join(this.project_path, "logs", "web");
-        this.webLogFile = this.getWebLogFile(logFolder);
-        this.registerConsoleLogListener(page, context, this.webLogFile);
-        this.registerRequestListener();
+        this.world = world;
         context.pages = [this.page];
         context.pageLoading = { status: false };
+        this.registerEventListeners(this.context);
+    }
+    registerEventListeners(context) {
+        this.registerConsoleLogListener(this.page, context);
+        this.registerRequestListener(this.page, context, this.webLogFile);
+        if (!context.pageLoading) {
+            context.pageLoading = { status: false };
+        }
         context.playContext.on("page", async function (page) {
             context.pageLoading.status = true;
             this.page = page;
             context.page = page;
             context.pages.push(page);
             page.on("close", async () => {
-                if (this.context && this.context.pages && this.context.pages.length > 0) {
+                if (this.context && this.context.pages && this.context.pages.length > 1) {
                     this.context.pages.pop();
                     this.page = this.context.pages[this.context.pages.length - 1];
                     this.context.page = this.page;
-                    let title = await this.page.title();
-                    console.log("Switched to page " + title);
+                    try {
+                        let title = await this.page.title();
+                        console.log("Switched to page " + title);
+                    }
+                    catch (error) {
+                        console.error("Error on page close", error);
+                    }
                 }
             });
             try {
@@ -104,6 +121,36 @@ class StableBrowser {
             context.pageLoading.status = false;
         }.bind(this));
     }
+    async switchApp(appName) {
+        // check if the current app (this.appName) is the same as the new app
+        if (this.appName === appName) {
+            return;
+        }
+        let navigate = false;
+        if (!apps[appName]) {
+            let newContext = await getContext(null, false, this.logger, appName, false, this);
+            navigate = true;
+            apps[appName] = {
+                context: newContext,
+                browser: newContext.browser,
+                page: newContext.page,
+            };
+        }
+        const tempContext = {};
+        this._copyContext(this, tempContext);
+        this._copyContext(apps[appName], this);
+        apps[this.appName] = tempContext;
+        this.appName = appName;
+        if (navigate) {
+            await this.goto(this.context.environment.baseUrl);
+            await this.waitForPageLoad();
+        }
+    }
+    _copyContext(from, to) {
+        to.browser = from.browser;
+        to.page = from.page;
+        to.context = from.context;
+    }
     getWebLogFile(logFolder) {
         if (!fs.existsSync(logFolder)) {
             fs.mkdirSync(logFolder, { recursive: true });
@@ -115,37 +162,63 @@ class StableBrowser {
         const fileName = nextIndex + ".json";
         return path.join(logFolder, fileName);
     }
-    registerConsoleLogListener(page, context, logFile) {
+    registerConsoleLogListener(page, context) {
         if (!this.context.webLogger) {
             this.context.webLogger = [];
         }
         page.on("console", async (msg) => {
-            this.context.webLogger.push({
+            var _a;
+            const obj = {
                 type: msg.type(),
                 text: msg.text(),
                 location: msg.location(),
                 time: new Date().toISOString(),
-            });
-            await fs.promises.writeFile(logFile, JSON.stringify(this.context.webLogger, null, 2));
+            };
+            this.context.webLogger.push(obj);
+            (_a = this.world) === null || _a === void 0 ? void 0 : _a.attach(JSON.stringify(obj), { mediaType: "application/json+log" });
         });
     }
-    registerRequestListener() {
-        this.page.on("request", async (data) => {
+    registerRequestListener(page, context, logFile) {
+        if (!this.context.networkLogger) {
+            this.context.networkLogger = [];
+        }
+        page.on("request", async (data) => {
+            var _a;
+            const startTime = new Date().getTime();
             try {
-                const pageUrl = new URL(this.page.url());
+                const pageUrl = new URL(page.url());
                 const requestUrl = new URL(data.url());
                 if (pageUrl.hostname === requestUrl.hostname) {
                     const method = data.method();
-                    if (method === "POST" || method === "GET" || method === "PUT" || method === "DELETE" || method === "PATCH") {
+                    if (["POST", "GET", "PUT", "DELETE", "PATCH"].includes(method)) {
                         const token = await data.headerValue("Authorization");
                         if (token) {
-                            this.context.authtoken = token;
+                            context.authtoken = token;
                         }
                     }
                 }
+                const response = await data.response();
+                const endTime = new Date().getTime();
+                const obj = {
+                    url: data.url(),
+                    method: data.method(),
+                    postData: data.postData(),
+                    error: data.failure() ? data.failure().errorText : null,
+                    duration: endTime - startTime,
+                    startTime,
+                };
+                context.networkLogger.push(obj);
+                (_a = this.world) === null || _a === void 0 ? void 0 : _a.attach(JSON.stringify(obj), { mediaType: "application/json+network" });
             }
             catch (error) {
                 console.error("Error in request listener", error);
+                context.networkLogger.push({
+                    error: "not able to listen",
+                    message: error.message,
+                    stack: error.stack,
+                    time: new Date().toISOString(),
+                });
+                // await fs.promises.writeFile(logFile, JSON.stringify(context.networkLogger, null, 2));
             }
         });
     }
@@ -226,7 +299,7 @@ class StableBrowser {
             // }
             locatorReturn = scope.getByRole(locator.role[0], locator.role[1]);
         }
-        if (locator.css || locator.engine === "css") {
+        if (locator.css) {
             locatorReturn = scope.locator(locator.css);
         }
         // handle role/name locators
@@ -242,25 +315,29 @@ class StableBrowser {
             }
         }
         if (locator === null || locator === void 0 ? void 0 : locator.engine) {
-            if (locator.engine === "internal:attr") {
-                selector = `[${selector}]`;
+            if (locator.engine === "css") {
+                locatorReturn = scope.locator(locator.selector);
+            }
+            else {
+                let selector = locator.selector;
+                if (locator.engine === "internal:attr") {
+                    if (!selector.startsWith("[")) {
+                        selector = `[${selector}]`;
+                    }
+                }
+                locatorReturn = scope.locator(`${locator.engine}=${selector}`);
             }
-            locatorReturn = scope.locator(`${locator.engine}=${selector}`);
         }
         if (!locatorReturn) {
             console.error(locator);
             throw new Error("Locator undefined");
-            // } else {
-            //   const count = locatorReturn.count();
-            //   if (count === 0) {
-            //     throw new Error("Elements not found");
-            //   } else if (count > 1) {
-            //     throw new Error("Multiple elements found");
-            //   }
         }
         return locatorReturn;
     }
     async _locateElmentByTextClimbCss(scope, text, climb, css, _params) {
+        if (css && css.locator) {
+            css = css.locator;
+        }
         let result = await this._locateElementByText(scope, this._fixUsingParams(text, _params), "*", false, true, _params);
         if (result.elementCount === 0) {
             return;
@@ -288,6 +365,15 @@ class StableBrowser {
                 return false;
             }
             document.isParent = isParent;
+            function getRegex(str) {
+                const match = str.match(/^\/(.*?)\/([gimuy]*)$/);
+                if (!match) {
+                    return null;
+                }
+                let [_, pattern, flags] = match;
+                return new RegExp(pattern, flags);
+            }
+            document.getRegex = getRegex;
             function collectAllShadowDomElements(element, result = []) {
                 // Check and add the element if it has a shadow root
                 if (element.shadowRoot) {
@@ -306,6 +392,10 @@ class StableBrowser {
             if (!tag) {
                 tag = "*";
             }
+            let regexpSearch = document.getRegex(text);
+            if (regexpSearch) {
+                regex = true;
+            }
             let elements = Array.from(document.querySelectorAll(tag));
             let shadowHosts = [];
             document.collectAllShadowDomElements(document, shadowHosts);
@@ -321,7 +411,9 @@ class StableBrowser {
             let randomToken = null;
             const foundElements = [];
             if (regex) {
-                let regexpSearch = new RegExp(text, "im");
+                if (!regexpSearch) {
+                    regexpSearch = new RegExp(text, "im");
+                }
                 for (let i = 0; i < elements.length; i++) {
                     const element = elements[i];
                     if ((element.innerText && regexpSearch.test(element.innerText)) ||
@@ -335,8 +427,8 @@ class StableBrowser {
                 for (let i = 0; i < elements.length; i++) {
                     const element = elements[i];
                     if (partial) {
-                        if ((element.innerText && element.innerText.trim().includes(text)) ||
-                            (element.value && element.value.includes(text))) {
+                        if ((element.innerText && element.innerText.toLowerCase().trim().includes(text.toLowerCase())) ||
+                            (element.value && element.value.toLowerCase().includes(text.toLowerCase()))) {
                             foundElements.push(element);
                         }
                     }
@@ -469,6 +561,8 @@ class StableBrowser {
                 if (result.foundElements.length > 0) {
                     let dialogCloseLocator = result.foundElements[0].locator;
                     await dialogCloseLocator.click();
+                    // wait for the dialog to close
+                    await dialogCloseLocator.waitFor({ state: "hidden" });
                     return { rerun: true };
                 }
             }
@@ -477,7 +571,7 @@ class StableBrowser {
     }
     async _locate(selectors, info, _params, timeout = 30000) {
         for (let i = 0; i < 3; i++) {
-            info.log += "attempt " + i + ": totoal locators " + selectors.locators.length + "\n";
+            info.log += "attempt " + i + ": total locators " + selectors.locators.length + "\n";
             for (let j = 0; j < selectors.locators.length; j++) {
                 let selector = selectors.locators[j];
                 info.log += "searching for locator " + j + ":" + JSON.stringify(selector) + "\n";
@@ -497,9 +591,30 @@ class StableBrowser {
         //let arrayMode = Array.isArray(selectors);
         let scope = this.page;
         if (selectors.iframe_src || selectors.frameLocators) {
+            const findFrame = (frame, framescope) => {
+                for (let i = 0; i < frame.selectors.length; i++) {
+                    let frameLocator = frame.selectors[i];
+                    if (frameLocator.css) {
+                        framescope = framescope.frameLocator(frameLocator.css);
+                        if (frameLocator.index) {
+                            framescope = framescope.nth(frameLocator.index);
+                        }
+                        break;
+                    }
+                }
+                if (frame.children) {
+                    return findFrame(frame.children, framescope);
+                }
+                return framescope;
+            };
             info.log += "searching for iframe " + selectors.iframe_src + "/" + selectors.frameLocators + "\n";
             while (true) {
                 let frameFound = false;
+                if (selectors.nestFrmLoc) {
+                    scope = findFrame(selectors.nestFrmLoc, scope);
+                    frameFound = true;
+                    break;
+                }
                 if (selectors.frameLocators) {
                     for (let i = 0; i < selectors.frameLocators.length; i++) {
                         let frameLocator = selectors.frameLocators[i];
@@ -660,14 +775,12 @@ class StableBrowser {
         }
         return result;
     }
-    async contextClick(selectors, text, _params, options = {}, world = null) {
-        this._validateSelectors(selectors);
-        selectors.locators[0].text = text;
-        await this.click(selectors, _params, options, world);
-    }
     async click(selectors, _params, options = {}, world = null) {
         this._validateSelectors(selectors);
         const startTime = Date.now();
+        if (options && options.context) {
+            selectors.locators[0].text = options.context;
+        }
         const info = {};
         info.log = "***** click on " + selectors.element_name + " *****\n";
         info.operation = "click";
@@ -681,14 +794,14 @@ class StableBrowser {
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             try {
                 await this._highlightElements(element);
-                await element.click({ timeout: 5000 });
+                await element.click();
                 await new Promise((resolve) => setTimeout(resolve, 1000));
             }
             catch (e) {
                 // await this.closeUnexpectedPopups();
                 info.log += "click failed, will try again" + "\n";
                 element = await this._locate(selectors, info, _params);
-                await element.click({ timeout: 10000, force: true });
+                await element.dispatchEvent("click");
                 await new Promise((resolve) => setTimeout(resolve, 1000));
             }
             await this.waitForPageLoad();
@@ -741,7 +854,7 @@ class StableBrowser {
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             try {
                 await this._highlightElements(element);
-                await element.setChecked(checked, { timeout: 5000 });
+                await element.setChecked(checked);
                 await new Promise((resolve) => setTimeout(resolve, 1000));
             }
             catch (e) {
@@ -805,7 +918,7 @@ class StableBrowser {
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             try {
                 await this._highlightElements(element);
-                await element.hover({ timeout: 10000 });
+                await element.hover();
                 await new Promise((resolve) => setTimeout(resolve, 1000));
             }
             catch (e) {
@@ -867,7 +980,7 @@ class StableBrowser {
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             try {
                 await this._highlightElements(element);
-                await element.selectOption(values, { timeout: 5000 });
+                await element.selectOption(values);
             }
             catch (e) {
                 //await this.closeUnexpectedPopups();
@@ -1270,7 +1383,7 @@ class StableBrowser {
             let element = await this._locate(selectors, info, _params);
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             await this._highlightElements(element);
-            await element.fill(value, { timeout: 10000 });
+            await element.fill(value);
             await element.dispatchEvent("change");
             if (enter) {
                 await new Promise((resolve) => setTimeout(resolve, 2000));
@@ -1489,7 +1602,7 @@ class StableBrowser {
             return info;
         }
         catch (e) {
-            //await this.closeUnexpectedPopups();
+            await this.closeUnexpectedPopups();
             this.logger.error("verify element contains text failed " + info.log);
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             info.screenshotPath = screenshotPath;
@@ -1537,6 +1650,29 @@ class StableBrowser {
         }
         return dataFile;
     }
+    async waitForUserInput(message, world = null) {
+        if (!message) {
+            message = "# Wait for user input. Press any key to continue";
+        }
+        else {
+            message = "# Wait for user input. " + message;
+        }
+        message += "\n";
+        const value = await new Promise((resolve) => {
+            const rl = readline.createInterface({
+                input: process.stdin,
+                output: process.stdout,
+            });
+            rl.question(message, (answer) => {
+                rl.close();
+                resolve(answer);
+            });
+        });
+        if (value) {
+            this.logger.info(`{{userInput}} was set to: ${value}`);
+        }
+        this.setTestData({ userInput: value }, world);
+    }
     setTestData(testData, world = null) {
         if (!testData) {
             return;
@@ -1724,7 +1860,6 @@ class StableBrowser {
     }
     async takeScreenshot(screenshotPath) {
         const playContext = this.context.playContext;
-        const client = await playContext.newCDPSession(this.page);
         // Using CDP to capture the screenshot
         const viewportWidth = Math.max(...(await this.page.evaluate(() => [
             document.body.scrollWidth,
@@ -1734,41 +1869,40 @@ class StableBrowser {
             document.body.clientWidth,
             document.documentElement.clientWidth,
         ])));
-        const viewportHeight = Math.max(...(await this.page.evaluate(() => [
-            document.body.scrollHeight,
-            document.documentElement.scrollHeight,
-            document.body.offsetHeight,
-            document.documentElement.offsetHeight,
-            document.body.clientHeight,
-            document.documentElement.clientHeight,
-        ])));
-        const { data } = await client.send("Page.captureScreenshot", {
-            format: "png",
-            clip: {
-                x: 0,
-                y: 0,
-                width: viewportWidth,
-                height: viewportHeight,
-                scale: 1,
-            },
-        });
-        if (!screenshotPath) {
-            return data;
-        }
-        let screenshotBuffer = Buffer.from(data, "base64");
-        const sharpBuffer = sharp(screenshotBuffer);
-        const metadata = await sharpBuffer.metadata();
-        //check if you are on retina display and reduce the quality of the image
-        if (metadata.width > viewportWidth || metadata.height > viewportHeight) {
-            screenshotBuffer = await sharpBuffer
-                .resize(viewportWidth, viewportHeight, {
-                fit: sharp.fit.inside,
-                withoutEnlargement: true,
-            })
-                .toBuffer();
-        }
-        fs.writeFileSync(screenshotPath, screenshotBuffer);
-        await client.detach();
+        let screenshotBuffer = null;
+        if (this.context.browserName === "chromium") {
+            const client = await playContext.newCDPSession(this.page);
+            const { data } = await client.send("Page.captureScreenshot", {
+                format: "png",
+                // clip: {
+                //   x: 0,
+                //   y: 0,
+                //   width: viewportWidth,
+                //   height: viewportHeight,
+                //   scale: 1,
+                // },
+            });
+            await client.detach();
+            if (!screenshotPath) {
+                return data;
+            }
+            screenshotBuffer = Buffer.from(data, "base64");
+        }
+        else {
+            screenshotBuffer = await this.page.screenshot();
+        }
+        let image = await Jimp.read(screenshotBuffer);
+        // Get the image dimensions
+        const { width, height } = image.bitmap;
+        const resizeRatio = viewportWidth / width;
+        // Resize the image to fit within the viewport dimensions without enlarging
+        if (width > viewportWidth) {
+            image = image.resize({ w: viewportWidth, h: height * resizeRatio }); // Resize the image while maintaining aspect ratio
+            await image.write(screenshotPath);
+        }
+        else {
+            fs.writeFileSync(screenshotPath, screenshotBuffer);
+        }
     }
     async verifyElementExistInPage(selectors, _params = null, options = {}, world = null) {
         this._validateSelectors(selectors);
@@ -2576,13 +2710,13 @@ class StableBrowser {
         }
         catch (e) {
             if (e.label === "networkidle") {
-                console.log("waitted for the network to be idle timeout");
+                console.log("waited for the network to be idle timeout");
             }
             else if (e.label === "load") {
-                console.log("waitted for the load timeout");
+                console.log("waited for the load timeout");
             }
             else if (e.label === "domcontentloaded") {
-                console.log("waitted for the domcontent loaded timeout");
+                console.log("waited for the domcontent loaded timeout");
             }
             console.log(".");
         }
@@ -2725,33 +2859,18 @@ class StableBrowser {
     }
     async scrollIfNeeded(element, info) {
         try {
-            let didScroll = await element.evaluate((node) => {
-                const rect = node.getBoundingClientRect();
-                if (rect &&
-                    rect.top >= 0 &&
-                    rect.left >= 0 &&
-                    rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
-                    rect.right <= (window.innerWidth || document.documentElement.clientWidth)) {
-                    return false;
-                }
-                else {
-                    node.scrollIntoView({
-                        behavior: "smooth",
-                        block: "center",
-                        inline: "center",
-                    });
-                    return true;
-                }
+            await element.scrollIntoViewIfNeeded({
+                timeout: 2000,
             });
-            if (didScroll) {
-                await new Promise((resolve) => setTimeout(resolve, 500));
-                if (info) {
-                    info.box = await element.boundingBox();
-                }
+            await new Promise((resolve) => setTimeout(resolve, 500));
+            if (info) {
+                info.box = await element.boundingBox({
+                    timeout: 1000,
+                });
             }
         }
         catch (e) {
-            console.log("scroll failed");
+            console.log("#-#");
         }
     }
     _reportToWorld(world, properties) {