automation_model 1.0.417-dev → 1.0.417-stage

package/lib/stable_browser.js

@@ -2,9 +2,9 @@
 import { expect } from "@playwright/test";
 import dayjs from "dayjs";
 import fs from "fs";
+import { Jimp } from "jimp";
 import path from "path";
 import reg_parser from "regex-parser";
-import sharp from "sharp";
 import { findDateAlternatives, findNumberAlternatives } from "./analyze_helper.js";
 import { getDateTimeValue } from "./date_time.js";
 import drawRectangle from "./drawRect.js";
@@ -14,6 +14,9 @@ import objectPath from "object-path";
 import { decrypt } from "./utils.js";
 import csv from "csv-parser";
 import { Readable } from "node:stream";
+import readline from "readline";
+import { getContext } from "./init_browser.js";
+import { locate_element } from "./locate_element.js";
 const Types = {
     CLICK: "click_element",
     NAVIGATE: "navigate",
@@ -41,15 +44,19 @@ const Types = {
     LOAD_DATA: "load_data",
     SET_INPUT: "set_input",
 };
+export const apps = {};
 class StableBrowser {
-    constructor(browser, page, logger = null, context = null) {
+    constructor(browser, page, logger = null, context = null, world = null) {
         this.browser = browser;
         this.page = page;
         this.logger = logger;
         this.context = context;
+        this.world = world;
         this.project_path = null;
         this.webLogFile = null;
+        this.networkLogger = null;
         this.configuration = null;
+        this.appName = "main";
         if (!this.logger) {
             this.logger = console;
         }
@@ -75,23 +82,34 @@ class StableBrowser {
             this.logger.error("unable to read ai_config.json");
         }
         const logFolder = path.join(this.project_path, "logs", "web");
-        this.webLogFile = this.getWebLogFile(logFolder);
-        this.registerConsoleLogListener(page, context, this.webLogFile);
-        this.registerRequestListener();
+        this.world = world;
         context.pages = [this.page];
         context.pageLoading = { status: false };
+        this.registerEventListeners(this.context);
+    }
+    registerEventListeners(context) {
+        this.registerConsoleLogListener(this.page, context);
+        this.registerRequestListener(this.page, context, this.webLogFile);
+        if (!context.pageLoading) {
+            context.pageLoading = { status: false };
+        }
         context.playContext.on("page", async function (page) {
             context.pageLoading.status = true;
             this.page = page;
             context.page = page;
             context.pages.push(page);
             page.on("close", async () => {
-                if (this.context && this.context.pages && this.context.pages.length > 0) {
+                if (this.context && this.context.pages && this.context.pages.length > 1) {
                     this.context.pages.pop();
                     this.page = this.context.pages[this.context.pages.length - 1];
                     this.context.page = this.page;
-                    let title = await this.page.title();
-                    console.log("Switched to page " + title);
+                    try {
+                        let title = await this.page.title();
+                        console.log("Switched to page " + title);
+                    }
+                    catch (error) {
+                        console.error("Error on page close", error);
+                    }
                 }
             });
             try {
@@ -104,6 +122,36 @@ class StableBrowser {
             context.pageLoading.status = false;
         }.bind(this));
     }
+    async switchApp(appName) {
+        // check if the current app (this.appName) is the same as the new app
+        if (this.appName === appName) {
+            return;
+        }
+        let navigate = false;
+        if (!apps[appName]) {
+            let newContext = await getContext(null, false, this.logger, appName, false, this);
+            navigate = true;
+            apps[appName] = {
+                context: newContext,
+                browser: newContext.browser,
+                page: newContext.page,
+            };
+        }
+        const tempContext = {};
+        this._copyContext(this, tempContext);
+        this._copyContext(apps[appName], this);
+        apps[this.appName] = tempContext;
+        this.appName = appName;
+        if (navigate) {
+            await this.goto(this.context.environment.baseUrl);
+            await this.waitForPageLoad();
+        }
+    }
+    _copyContext(from, to) {
+        to.browser = from.browser;
+        to.page = from.page;
+        to.context = from.context;
+    }
     getWebLogFile(logFolder) {
         if (!fs.existsSync(logFolder)) {
             fs.mkdirSync(logFolder, { recursive: true });
@@ -115,37 +163,65 @@ class StableBrowser {
         const fileName = nextIndex + ".json";
         return path.join(logFolder, fileName);
     }
-    registerConsoleLogListener(page, context, logFile) {
+    registerConsoleLogListener(page, context) {
         if (!this.context.webLogger) {
             this.context.webLogger = [];
         }
         page.on("console", async (msg) => {
-            this.context.webLogger.push({
+            var _a;
+            const obj = {
                 type: msg.type(),
                 text: msg.text(),
                 location: msg.location(),
                 time: new Date().toISOString(),
-            });
-            await fs.promises.writeFile(logFile, JSON.stringify(this.context.webLogger, null, 2));
+            };
+            this.context.webLogger.push(obj);
+            if (msg.type() === "error") {
+                (_a = this.world) === null || _a === void 0 ? void 0 : _a.attach(JSON.stringify(obj), { mediaType: "application/json+log" });
+            }
         });
     }
-    registerRequestListener() {
-        this.page.on("request", async (data) => {
+    registerRequestListener(page, context, logFile) {
+        if (!this.context.networkLogger) {
+            this.context.networkLogger = [];
+        }
+        page.on("request", async (data) => {
+            var _a;
+            const startTime = new Date().getTime();
             try {
-                const pageUrl = new URL(this.page.url());
+                const pageUrl = new URL(page.url());
                 const requestUrl = new URL(data.url());
                 if (pageUrl.hostname === requestUrl.hostname) {
                     const method = data.method();
-                    if (method === "POST" || method === "GET" || method === "PUT" || method === "DELETE" || method === "PATCH") {
+                    if (["POST", "GET", "PUT", "DELETE", "PATCH"].includes(method)) {
                         const token = await data.headerValue("Authorization");
                         if (token) {
-                            this.context.authtoken = token;
+                            context.authtoken = token;
                         }
                     }
                 }
+                const response = await data.response();
+                const endTime = new Date().getTime();
+                const obj = {
+                    url: data.url(),
+                    method: data.method(),
+                    postData: data.postData(),
+                    error: data.failure() ? data.failure().errorText : null,
+                    duration: endTime - startTime,
+                    startTime,
+                };
+                context.networkLogger.push(obj);
+                (_a = this.world) === null || _a === void 0 ? void 0 : _a.attach(JSON.stringify(obj), { mediaType: "application/json+network" });
             }
             catch (error) {
                 console.error("Error in request listener", error);
+                context.networkLogger.push({
+                    error: "not able to listen",
+                    message: error.message,
+                    stack: error.stack,
+                    time: new Date().toISOString(),
+                });
+                // await fs.promises.writeFile(logFile, JSON.stringify(context.networkLogger, null, 2));
             }
         });
     }
@@ -248,7 +324,9 @@ class StableBrowser {
             else {
                 let selector = locator.selector;
                 if (locator.engine === "internal:attr") {
-                    selector = `[${selector}]`;
+                    if (!selector.startsWith("[")) {
+                        selector = `[${selector}]`;
+                    }
                 }
                 locatorReturn = scope.locator(`${locator.engine}=${selector}`);
             }
@@ -260,6 +338,9 @@ class StableBrowser {
         return locatorReturn;
     }
     async _locateElmentByTextClimbCss(scope, text, climb, css, _params) {
+        if (css && css.locator) {
+            css = css.locator;
+        }
         let result = await this._locateElementByText(scope, this._fixUsingParams(text, _params), "*", false, true, _params);
         if (result.elementCount === 0) {
             return;
@@ -287,6 +368,15 @@ class StableBrowser {
                 return false;
             }
             document.isParent = isParent;
+            function getRegex(str) {
+                const match = str.match(/^\/(.*?)\/([gimuy]*)$/);
+                if (!match) {
+                    return null;
+                }
+                let [_, pattern, flags] = match;
+                return new RegExp(pattern, flags);
+            }
+            document.getRegex = getRegex;
             function collectAllShadowDomElements(element, result = []) {
                 // Check and add the element if it has a shadow root
                 if (element.shadowRoot) {
@@ -305,6 +395,10 @@ class StableBrowser {
             if (!tag) {
                 tag = "*";
             }
+            let regexpSearch = document.getRegex(text);
+            if (regexpSearch) {
+                regex = true;
+            }
             let elements = Array.from(document.querySelectorAll(tag));
             let shadowHosts = [];
             document.collectAllShadowDomElements(document, shadowHosts);
@@ -320,7 +414,9 @@ class StableBrowser {
             let randomToken = null;
             const foundElements = [];
             if (regex) {
-                let regexpSearch = new RegExp(text, "im");
+                if (!regexpSearch) {
+                    regexpSearch = new RegExp(text, "im");
+                }
                 for (let i = 0; i < elements.length; i++) {
                     const element = elements[i];
                     if ((element.innerText && regexpSearch.test(element.innerText)) ||
@@ -334,8 +430,8 @@ class StableBrowser {
                 for (let i = 0; i < elements.length; i++) {
                     const element = elements[i];
                     if (partial) {
-                        if ((element.innerText && element.innerText.trim().includes(text)) ||
-                            (element.value && element.value.includes(text))) {
+                        if ((element.innerText && element.innerText.toLowerCase().trim().includes(text.toLowerCase())) ||
+                            (element.value && element.value.toLowerCase().includes(text.toLowerCase()))) {
                             foundElements.push(element);
                         }
                     }
@@ -380,6 +476,12 @@ class StableBrowser {
     }
     async _collectLocatorInformation(selectorHierarchy, index = 0, scope, foundLocators, _params, info, visibleOnly = true) {
         let locatorSearch = selectorHierarchy[index];
+        try {
+            locatorSearch = JSON.parse(this._fixUsingParams(JSON.stringify(locatorSearch), _params));
+        }
+        catch (e) {
+            console.error(e);
+        }
         //info.log += "searching for locator " + JSON.stringify(locatorSearch) + "\n";
         let locator = null;
         if (locatorSearch.climb && locatorSearch.climb >= 0) {
@@ -468,6 +570,8 @@ class StableBrowser {
                 if (result.foundElements.length > 0) {
                     let dialogCloseLocator = result.foundElements[0].locator;
                     await dialogCloseLocator.click();
+                    // wait for the dialog to close
+                    await dialogCloseLocator.waitFor({ state: "hidden" });
                     return { rerun: true };
                 }
             }
@@ -476,7 +580,7 @@ class StableBrowser {
     }
     async _locate(selectors, info, _params, timeout = 30000) {
         for (let i = 0; i < 3; i++) {
-            info.log += "attempt " + i + ": totoal locators " + selectors.locators.length + "\n";
+            info.log += "attempt " + i + ": total locators " + selectors.locators.length + "\n";
             for (let j = 0; j < selectors.locators.length; j++) {
                 let selector = selectors.locators[j];
                 info.log += "searching for locator " + j + ":" + JSON.stringify(selector) + "\n";
@@ -495,10 +599,44 @@ class StableBrowser {
         let locatorsCount = 0;
         //let arrayMode = Array.isArray(selectors);
         let scope = this.page;
+        // for the simple click usecase
+        if (selectors.frame) {
+            scope = selectors.frame;
+        }
         if (selectors.iframe_src || selectors.frameLocators) {
+            const findFrame = async (frame, framescope) => {
+                for (let i = 0; i < frame.selectors.length; i++) {
+                    let frameLocator = frame.selectors[i];
+                    if (frameLocator.css) {
+                        let testframescope = framescope.frameLocator(frameLocator.css);
+                        if (frameLocator.index) {
+                            testframescope = framescope.nth(frameLocator.index);
+                        }
+                        try {
+                            await testframescope.owner().evaluateHandle(() => true, null, {
+                                timeout: 5000,
+                            });
+                            framescope = testframescope;
+                            break;
+                        }
+                        catch (error) {
+                            console.error("frame not found " + frameLocator.css);
+                        }
+                    }
+                }
+                if (frame.children) {
+                    return await findFrame(frame.children, framescope);
+                }
+                return framescope;
+            };
             info.log += "searching for iframe " + selectors.iframe_src + "/" + selectors.frameLocators + "\n";
             while (true) {
                 let frameFound = false;
+                if (selectors.nestFrmLoc) {
+                    scope = await findFrame(selectors.nestFrmLoc, scope);
+                    frameFound = true;
+                    break;
+                }
                 if (selectors.frameLocators) {
                     for (let i = 0; i < selectors.frameLocators.length; i++) {
                         let frameLocator = selectors.frameLocators[i];
@@ -659,6 +797,66 @@ class StableBrowser {
         }
         return result;
     }
+    async simpleClick(elementDescription, _params, options = {}, world = null) {
+        const startTime = Date.now();
+        let timeout = 30000;
+        if (options && options.timeout) {
+            timeout = options.timeout;
+        }
+        while (true) {
+            try {
+                const result = await locate_element(this.context, elementDescription, "click");
+                if ((result === null || result === void 0 ? void 0 : result.elementNumber) >= 0) {
+                    const selectors = {
+                        frame: result === null || result === void 0 ? void 0 : result.frame,
+                        locators: [
+                            {
+                                css: result === null || result === void 0 ? void 0 : result.css,
+                            },
+                        ],
+                    };
+                    await this.click(selectors, _params, options, world);
+                    return;
+                }
+            }
+            catch (e) {
+                if (performance.now() - startTime > timeout) {
+                    throw e;
+                }
+            }
+            await new Promise((resolve) => setTimeout(resolve, 3000));
+        }
+    }
+    async simpleClickType(elementDescription, value, _params, options = {}, world = null) {
+        const startTime = Date.now();
+        let timeout = 30000;
+        if (options && options.timeout) {
+            timeout = options.timeout;
+        }
+        while (true) {
+            try {
+                const result = await locate_element(this.context, elementDescription, "fill", value);
+                if ((result === null || result === void 0 ? void 0 : result.elementNumber) >= 0) {
+                    const selectors = {
+                        frame: result === null || result === void 0 ? void 0 : result.frame,
+                        locators: [
+                            {
+                                css: result === null || result === void 0 ? void 0 : result.css,
+                            },
+                        ],
+                    };
+                    await this.clickType(selectors, value, false, _params, options, world);
+                    return;
+                }
+            }
+            catch (e) {
+                if (performance.now() - startTime > timeout) {
+                    throw e;
+                }
+            }
+            await new Promise((resolve) => setTimeout(resolve, 3000));
+        }
+    }
     async click(selectors, _params, options = {}, world = null) {
         this._validateSelectors(selectors);
         const startTime = Date.now();
@@ -678,14 +876,14 @@ class StableBrowser {
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             try {
                 await this._highlightElements(element);
-                await element.click({ timeout: 5000 });
+                await element.click();
                 await new Promise((resolve) => setTimeout(resolve, 1000));
             }
             catch (e) {
                 // await this.closeUnexpectedPopups();
                 info.log += "click failed, will try again" + "\n";
                 element = await this._locate(selectors, info, _params);
-                await element.click({ timeout: 10000, force: true });
+                await element.dispatchEvent("click");
                 await new Promise((resolve) => setTimeout(resolve, 1000));
             }
             await this.waitForPageLoad();
@@ -738,7 +936,7 @@ class StableBrowser {
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             try {
                 await this._highlightElements(element);
-                await element.setChecked(checked, { timeout: 5000 });
+                await element.setChecked(checked);
                 await new Promise((resolve) => setTimeout(resolve, 1000));
             }
             catch (e) {
@@ -802,7 +1000,7 @@ class StableBrowser {
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             try {
                 await this._highlightElements(element);
-                await element.hover({ timeout: 10000 });
+                await element.hover();
                 await new Promise((resolve) => setTimeout(resolve, 1000));
             }
             catch (e) {
@@ -864,7 +1062,7 @@ class StableBrowser {
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             try {
                 await this._highlightElements(element);
-                await element.selectOption(values, { timeout: 5000 });
+                await element.selectOption(values);
             }
             catch (e) {
                 //await this.closeUnexpectedPopups();
@@ -1152,7 +1350,6 @@ class StableBrowser {
             let element = await this._locate(selectors, info, _params);
             //insert red border around the element
             await this.scrollIfNeeded(element, info);
-            ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             await this._highlightElements(element);
             if (options === null || options === undefined || !options.press) {
                 try {
@@ -1202,6 +1399,7 @@ class StableBrowser {
                     await new Promise((resolve) => setTimeout(resolve, 500));
                 }
             }
+            ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             if (enter === true) {
                 await new Promise((resolve) => setTimeout(resolve, 2000));
                 await this.page.keyboard.press("Enter");
@@ -1267,7 +1465,7 @@ class StableBrowser {
             let element = await this._locate(selectors, info, _params);
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             await this._highlightElements(element);
-            await element.fill(value, { timeout: 10000 });
+            await element.fill(value);
             await element.dispatchEvent("change");
             if (enter) {
                 await new Promise((resolve) => setTimeout(resolve, 2000));
@@ -1486,7 +1684,7 @@ class StableBrowser {
             return info;
         }
         catch (e) {
-            //await this.closeUnexpectedPopups();
+            await this.closeUnexpectedPopups();
             this.logger.error("verify element contains text failed " + info.log);
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             info.screenshotPath = screenshotPath;
@@ -1534,6 +1732,29 @@ class StableBrowser {
         }
         return dataFile;
     }
+    async waitForUserInput(message, world = null) {
+        if (!message) {
+            message = "# Wait for user input. Press any key to continue";
+        }
+        else {
+            message = "# Wait for user input. " + message;
+        }
+        message += "\n";
+        const value = await new Promise((resolve) => {
+            const rl = readline.createInterface({
+                input: process.stdin,
+                output: process.stdout,
+            });
+            rl.question(message, (answer) => {
+                rl.close();
+                resolve(answer);
+            });
+        });
+        if (value) {
+            this.logger.info(`{{userInput}} was set to: ${value}`);
+        }
+        this.setTestData({ userInput: value }, world);
+    }
     setTestData(testData, world = null) {
         if (!testData) {
             return;
@@ -1721,7 +1942,6 @@ class StableBrowser {
     }
     async takeScreenshot(screenshotPath) {
         const playContext = this.context.playContext;
-        const client = await playContext.newCDPSession(this.page);
         // Using CDP to capture the screenshot
         const viewportWidth = Math.max(...(await this.page.evaluate(() => [
             document.body.scrollWidth,
@@ -1731,41 +1951,40 @@ class StableBrowser {
             document.body.clientWidth,
             document.documentElement.clientWidth,
         ])));
-        const viewportHeight = Math.max(...(await this.page.evaluate(() => [
-            document.body.scrollHeight,
-            document.documentElement.scrollHeight,
-            document.body.offsetHeight,
-            document.documentElement.offsetHeight,
-            document.body.clientHeight,
-            document.documentElement.clientHeight,
-        ])));
-        const { data } = await client.send("Page.captureScreenshot", {
-            format: "png",
-            // clip: {
-            //   x: 0,
-            //   y: 0,
-            //   width: viewportWidth,
-            //   height: viewportHeight,
-            //   scale: 1,
-            // },
-        });
-        if (!screenshotPath) {
-            return data;
-        }
-        let screenshotBuffer = Buffer.from(data, "base64");
-        const sharpBuffer = sharp(screenshotBuffer);
-        const metadata = await sharpBuffer.metadata();
-        //check if you are on retina display and reduce the quality of the image
-        if (metadata.width > viewportWidth || metadata.height > viewportHeight) {
-            screenshotBuffer = await sharpBuffer
-                .resize(viewportWidth, viewportHeight, {
-                fit: sharp.fit.inside,
-                withoutEnlargement: true,
-            })
-                .toBuffer();
-        }
-        fs.writeFileSync(screenshotPath, screenshotBuffer);
-        await client.detach();
+        let screenshotBuffer = null;
+        if (this.context.browserName === "chromium") {
+            const client = await playContext.newCDPSession(this.page);
+            const { data } = await client.send("Page.captureScreenshot", {
+                format: "png",
+                // clip: {
+                //   x: 0,
+                //   y: 0,
+                //   width: viewportWidth,
+                //   height: viewportHeight,
+                //   scale: 1,
+                // },
+            });
+            await client.detach();
+            if (!screenshotPath) {
+                return data;
+            }
+            screenshotBuffer = Buffer.from(data, "base64");
+        }
+        else {
+            screenshotBuffer = await this.page.screenshot();
+        }
+        let image = await Jimp.read(screenshotBuffer);
+        // Get the image dimensions
+        const { width, height } = image.bitmap;
+        const resizeRatio = viewportWidth / width;
+        // Resize the image to fit within the viewport dimensions without enlarging
+        if (width > viewportWidth) {
+            image = image.resize({ w: viewportWidth, h: height * resizeRatio }); // Resize the image while maintaining aspect ratio
+            await image.write(screenshotPath);
+        }
+        else {
+            fs.writeFileSync(screenshotPath, screenshotBuffer);
+        }
     }
     async verifyElementExistInPage(selectors, _params = null, options = {}, world = null) {
         this._validateSelectors(selectors);
@@ -2573,13 +2792,13 @@ class StableBrowser {
         }
         catch (e) {
             if (e.label === "networkidle") {
-                console.log("waitted for the network to be idle timeout");
+                console.log("waited for the network to be idle timeout");
             }
             else if (e.label === "load") {
-                console.log("waitted for the load timeout");
+                console.log("waited for the load timeout");
             }
             else if (e.label === "domcontentloaded") {
-                console.log("waitted for the domcontent loaded timeout");
+                console.log("waited for the domcontent loaded timeout");
             }
             console.log(".");
         }
@@ -2722,33 +2941,18 @@ class StableBrowser {
     }
     async scrollIfNeeded(element, info) {
         try {
-            let didScroll = await element.evaluate((node) => {
-                const rect = node.getBoundingClientRect();
-                if (rect &&
-                    rect.top >= 0 &&
-                    rect.left >= 0 &&
-                    rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
-                    rect.right <= (window.innerWidth || document.documentElement.clientWidth)) {
-                    return false;
-                }
-                else {
-                    node.scrollIntoView({
-                        behavior: "smooth",
-                        block: "center",
-                        inline: "center",
-                    });
-                    return true;
-                }
+            await element.scrollIntoViewIfNeeded({
+                timeout: 2000,
             });
-            if (didScroll) {
-                await new Promise((resolve) => setTimeout(resolve, 500));
-                if (info) {
-                    info.box = await element.boundingBox();
-                }
+            await new Promise((resolve) => setTimeout(resolve, 500));
+            if (info) {
+                info.box = await element.boundingBox({
+                    timeout: 1000,
+                });
             }
         }
         catch (e) {
-            console.log("scroll failed");
+            console.log("#-#");
         }
     }
     _reportToWorld(world, properties) {