automation_model 1.0.373-dev.0 → 1.0.373-stage

package/lib/stable_browser.js

@@ -12,6 +12,8 @@ import drawRectangle from "./drawRect.js";
 import { getTableCells, getTableData } from "./table_analyze.js";
 import objectPath from "object-path";
 import { decrypt } from "./utils.js";
+import csv from "csv-parser";
+import { Readable } from "node:stream";
 const Types = {
     CLICK: "click_element",
     NAVIGATE: "navigate",
@@ -27,6 +29,7 @@ const Types = {
     SELECT: "select_combobox",
     VERIFY_PAGE_PATH: "verify_page_path",
     TYPE_PRESS: "type_press",
+    PRESS: "press_key",
     HOVER: "hover_element",
     CHECK: "check_element",
     UNCHECK: "uncheck_element",
@@ -35,6 +38,8 @@ const Types = {
     SET_DATE_TIME: "set_date_time",
     SET_VIEWPORT: "set_viewport",
     VERIFY_VISUAL: "verify_visual",
+    LOAD_DATA: "load_data",
+    SET_INPUT: "set_input",
 };
 class StableBrowser {
     constructor(browser, page, logger = null, context = null) {
@@ -80,9 +85,15 @@ class StableBrowser {
             this.page = page;
             context.page = page;
             context.pages.push(page);
-            this.webLogFile = this.getWebLogFile(logFolder);
-            this.registerConsoleLogListener(page, context, this.webLogFile);
-            this.registerRequestListener();
+            page.on("close", async () => {
+                if (this.context && this.context.pages && this.context.pages.length > 0) {
+                    this.context.pages.pop();
+                    this.page = this.context.pages[this.context.pages.length - 1];
+                    this.context.page = this.page;
+                    let title = await this.page.title();
+                    console.log("Switched to page " + title);
+                }
+            });
             try {
                 await this.waitForPageLoad();
                 console.log("Switch page: " + (await page.title()));
@@ -91,7 +102,7 @@ class StableBrowser {
                 this.logger.error("error on page load " + e);
             }
             context.pageLoading.status = false;
-        });
+        }.bind(this));
     }
     getWebLogFile(logFolder) {
         if (!fs.existsSync(logFolder)) {
@@ -177,21 +188,78 @@ class StableBrowser {
         }
         return text;
     }
+    _fixLocatorUsingParams(locator, _params) {
+        // check if not null
+        if (!locator) {
+            return locator;
+        }
+        // clone the locator
+        locator = JSON.parse(JSON.stringify(locator));
+        this.scanAndManipulate(locator, _params);
+        return locator;
+    }
+    _isObject(value) {
+        return value && typeof value === "object" && value.constructor === Object;
+    }
+    scanAndManipulate(currentObj, _params) {
+        for (const key in currentObj) {
+            if (typeof currentObj[key] === "string") {
+                // Perform string manipulation
+                currentObj[key] = this._fixUsingParams(currentObj[key], _params);
+            }
+            else if (this._isObject(currentObj[key])) {
+                // Recursively scan nested objects
+                this.scanAndManipulate(currentObj[key], _params);
+            }
+        }
+    }
     _getLocator(locator, scope, _params) {
+        locator = this._fixLocatorUsingParams(locator, _params);
+        let locatorReturn;
         if (locator.role) {
             if (locator.role[1].nameReg) {
                 locator.role[1].name = reg_parser(locator.role[1].nameReg);
                 delete locator.role[1].nameReg;
             }
-            if (locator.role[1].name) {
-                locator.role[1].name = this._fixUsingParams(locator.role[1].name, _params);
-            }
-            return scope.getByRole(locator.role[0], locator.role[1]);
+            // if (locator.role[1].name) {
+            //   locator.role[1].name = this._fixUsingParams(locator.role[1].name, _params);
+            // }
+            locatorReturn = scope.getByRole(locator.role[0], locator.role[1]);
         }
         if (locator.css) {
-            return scope.locator(this._fixUsingParams(locator.css, _params));
+            locatorReturn = scope.locator(locator.css);
+        }
+        // handle role/name locators
+        // locator.selector will be something like: textbox[name="Username"i]
+        if (locator.engine === "internal:role") {
+            // extract the role, name and the i/s flags using regex
+            const match = locator.selector.match(/(.*)\[(.*)="(.*)"(.*)\]/);
+            if (match) {
+                const role = match[1];
+                const name = match[3];
+                const flags = match[4];
+                locatorReturn = scope.getByRole(role, { name }, { exact: flags === "i" });
+            }
+        }
+        if (locator === null || locator === void 0 ? void 0 : locator.engine) {
+            if (locator.engine === "css") {
+                locatorReturn = scope.locator(locator.selector);
+            }
+            else {
+                let selector = locator.selector;
+                if (locator.engine === "internal:attr") {
+                    if (!selector.startsWith("[")) {
+                        selector = `[${selector}]`;
+                    }
+                }
+                locatorReturn = scope.locator(`${locator.engine}=${selector}`);
+            }
         }
-        throw new Error("unknown locator type");
+        if (!locatorReturn) {
+            console.error(locator);
+            throw new Error("Locator undefined");
+        }
+        return locatorReturn;
     }
     async _locateElmentByTextClimbCss(scope, text, climb, css, _params) {
         let result = await this._locateElementByText(scope, this._fixUsingParams(text, _params), "*", false, true, _params);
@@ -314,10 +382,13 @@ class StableBrowser {
     }
     async _collectLocatorInformation(selectorHierarchy, index = 0, scope, foundLocators, _params, info, visibleOnly = true) {
         let locatorSearch = selectorHierarchy[index];
-        info.log += "searching for locator " + JSON.stringify(locatorSearch) + "\n";
+        //info.log += "searching for locator " + JSON.stringify(locatorSearch) + "\n";
         let locator = null;
         if (locatorSearch.climb && locatorSearch.climb >= 0) {
             let locatorString = await this._locateElmentByTextClimbCss(scope, locatorSearch.text, locatorSearch.climb, locatorSearch.css, _params);
+            if (!locatorString) {
+                return;
+            }
             locator = this._getLocator({ css: locatorString }, scope, _params);
         }
         else if (locatorSearch.text) {
@@ -339,7 +410,7 @@ class StableBrowser {
         //   cssHref = true;
         // }
         let count = await locator.count();
-        info.log += "total elements found " + count + "\n";
+        //info.log += "total elements found " + count + "\n";
         //let visibleCount = 0;
         let visibleLocator = null;
         if (locatorSearch.index && locatorSearch.index < count) {
@@ -349,16 +420,20 @@ class StableBrowser {
         for (let j = 0; j < count; j++) {
             let visible = await locator.nth(j).isVisible();
             const enabled = await locator.nth(j).isEnabled();
-            info.log += "element " + j + " visible " + visible + " enabled " + enabled + "\n";
             if (!visibleOnly) {
                 visible = true;
             }
             if (visible && enabled) {
                 foundLocators.push(locator.nth(j));
-                // if (cssHref) {
-                //   info.log += "css href locator found, will ignore all others" + "\n";
-                //   break;
-                // }
+            }
+            else {
+                if (!info.printMessages) {
+                    info.printMessages = {};
+                }
+                if (!info.printMessages[j.toString()]) {
+                    info.log += "element " + locator + " visible " + visible + " enabled " + enabled + "\n";
+                    info.printMessages[j.toString()] = true;
+                }
             }
         }
     }
@@ -404,6 +479,10 @@ class StableBrowser {
     async _locate(selectors, info, _params, timeout = 30000) {
         for (let i = 0; i < 3; i++) {
             info.log += "attempt " + i + ": totoal locators " + selectors.locators.length + "\n";
+            for (let j = 0; j < selectors.locators.length; j++) {
+                let selector = selectors.locators[j];
+                info.log += "searching for locator " + j + ":" + JSON.stringify(selector) + "\n";
+            }
             let element = await this._locate_internal(selectors, info, _params, timeout);
             if (!element.rerun) {
                 return element;
@@ -498,6 +577,7 @@ class StableBrowser {
             let foundElements = result.foundElements;
             if (foundElements.length === 1 && foundElements[0].unique) {
                 info.box = foundElements[0].box;
+                info.log += "unique element was found, locator: " + foundElements[0].locator + "\n";
                 return foundElements[0].locator;
             }
             //info.log += "total elements found " + foundElements.length);
@@ -527,6 +607,7 @@ class StableBrowser {
                     }
                 }
                 if (maxCountElement) {
+                    info.log += "unique element was found, locator: " + maxCountElement.locator + "\n";
                     info.box = await maxCountElement.locator.boundingBox();
                     return maxCountElement.locator;
                 }
@@ -535,9 +616,11 @@ class StableBrowser {
                 break;
             }
             if (performance.now() - startTime > highPriorityTimeout) {
+                info.log += "high priority timeout, will try all elements" + "\n";
                 highPriorityOnly = false;
             }
             if (performance.now() - startTime > visibleOnlyTimeout) {
+                info.log += "visible only timeout, will try all elements" + "\n";
                 visibleOnly = false;
             }
             await new Promise((resolve) => setTimeout(resolve, 1000));
@@ -581,6 +664,9 @@ class StableBrowser {
     async click(selectors, _params, options = {}, world = null) {
         this._validateSelectors(selectors);
         const startTime = Date.now();
+        if (options && options.context) {
+            selectors.locators[0].text = options.context;
+        }
         const info = {};
         info.log = "***** click on " + selectors.element_name + " *****\n";
         info.operation = "click";
@@ -889,71 +975,45 @@ class StableBrowser {
             });
         }
     }
-    async setDateTime(selectors, value, format = null, enter = false, _params = null, options = {}, world = null) {
+    async setInputValue(selectors, value, _params = null, options = {}, world = null) {
+        // set input value for non fillable inputs like date, time, range, color, etc.
         this._validateSelectors(selectors);
         const startTime = Date.now();
-        let error = null;
-        let screenshotId = null;
-        let screenshotPath = null;
         const info = {};
-        info.log = "";
-        info.operation = Types.SET_DATE_TIME;
+        info.log = "***** set input value " + selectors.element_name + " *****\n";
+        info.operation = "setInputValue";
         info.selectors = selectors;
+        value = this._fixUsingParams(value, _params);
         info.value = value;
+        let error = null;
+        let screenshotId = null;
+        let screenshotPath = null;
         try {
             value = await this._replaceWithLocalData(value, this);
             let element = await this._locate(selectors, info, _params);
-            //insert red border around the element
             await this.scrollIfNeeded(element, info);
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             await this._highlightElements(element);
             try {
-                await element.click();
-                await new Promise((resolve) => setTimeout(resolve, 500));
-                if (format) {
-                    value = dayjs(value).format(format);
-                    await element.fill(value);
-                }
-                else {
-                    const dateTimeValue = await getDateTimeValue({ value, element });
-                    await element.evaluateHandle((el, dateTimeValue) => {
-                        el.value = ""; // clear input
-                        el.value = dateTimeValue;
-                    }, dateTimeValue);
-                }
-                if (enter) {
-                    await new Promise((resolve) => setTimeout(resolve, 2000));
-                    await this.page.keyboard.press("Enter");
-                    await this.waitForPageLoad();
-                }
+                await element.evaluateHandle((el, value) => {
+                    el.value = value;
+                }, value);
             }
             catch (error) {
-                //await this.closeUnexpectedPopups();
-                this.logger.error("setting date time input failed " + JSON.stringify(info));
-                this.logger.info("Trying again")(({ screenshotId, screenshotPath } = await this._screenShot(options, world, info)));
+                this.logger.error("setInputValue failed, will try again");
+                ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
                 info.screenshotPath = screenshotPath;
                 Object.assign(error, { info: info });
-                await element.click();
-                await new Promise((resolve) => setTimeout(resolve, 500));
-                if (format) {
-                    value = dayjs(value).format(format);
-                    await element.fill(value);
-                }
-                else {
-                    const dateTimeValue = await getDateTimeValue({ value, element });
-                    await element.evaluateHandle((el, dateTimeValue) => {
-                        el.value = ""; // clear input
-                        el.value = dateTimeValue;
-                    }, dateTimeValue);
-                }
-                if (enter) {
-                    await new Promise((resolve) => setTimeout(resolve, 2000));
-                    await this.page.keyboard.press("Enter");
-                    await this.waitForPageLoad();
-                }
+                await element.evaluateHandle((el, value) => {
+                    el.value = value;
+                });
             }
         }
-        catch (error) {
+        catch (e) {
+            this.logger.error("setInputValue failed " + info.log);
+            ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
+            info.screenshotPath = screenshotPath;
+            Object.assign(e, { info: info });
             error = e;
             throw e;
         }
@@ -961,10 +1021,10 @@ class StableBrowser {
             const endTime = Date.now();
             this._reportToWorld(world, {
                 element_name: selectors.element_name,
-                type: Types.SET_DATE_TIME,
-                screenshotId,
+                type: Types.SET_INPUT,
+                text: `Set input value`,
                 value: value,
-                text: `setDateTime input with value: ${value}`,
+                screenshotId,
                 result: error
                     ? {
                         status: "FAILED",
@@ -981,7 +1041,7 @@ class StableBrowser {
             });
         }
     }
-    async setDateTime(selectors, value, enter = false, _params = null, options = {}, world = null) {
+    async setDateTime(selectors, value, format = null, enter = false, _params = null, options = {}, world = null) {
         this._validateSelectors(selectors);
         const startTime = Date.now();
         let error = null;
@@ -993,6 +1053,7 @@ class StableBrowser {
         info.selectors = selectors;
         info.value = value;
         try {
+            value = await this._replaceWithLocalData(value, this);
             let element = await this._locate(selectors, info, _params);
             //insert red border around the element
             await this.scrollIfNeeded(element, info);
@@ -1001,28 +1062,51 @@ class StableBrowser {
             try {
                 await element.click();
                 await new Promise((resolve) => setTimeout(resolve, 500));
-                const dateTimeValue = await getDateTimeValue({ value, element });
-                await element.evaluateHandle((el, dateTimeValue) => {
-                    el.value = ""; // clear input
-                    el.value = dateTimeValue;
-                }, dateTimeValue);
+                if (format) {
+                    value = dayjs(value).format(format);
+                    await element.fill(value);
+                }
+                else {
+                    const dateTimeValue = await getDateTimeValue({ value, element });
+                    await element.evaluateHandle((el, dateTimeValue) => {
+                        el.value = ""; // clear input
+                        el.value = dateTimeValue;
+                    }, dateTimeValue);
+                }
+                if (enter) {
+                    await new Promise((resolve) => setTimeout(resolve, 2000));
+                    await this.page.keyboard.press("Enter");
+                    await this.waitForPageLoad();
+                }
             }
-            catch (error) {
+            catch (err) {
                 //await this.closeUnexpectedPopups();
                 this.logger.error("setting date time input failed " + JSON.stringify(info));
-                this.logger.info("Trying again")(({ screenshotId, screenshotPath } = await this._screenShot(options, world, info)));
+                this.logger.info("Trying again");
+                ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
                 info.screenshotPath = screenshotPath;
-                Object.assign(error, { info: info });
+                Object.assign(err, { info: info });
                 await element.click();
                 await new Promise((resolve) => setTimeout(resolve, 500));
-                const dateTimeValue = await getDateTimeValue({ value, element });
-                await element.evaluateHandle((el, dateTimeValue) => {
-                    el.value = ""; // clear input
-                    el.value = dateTimeValue;
-                }, dateTimeValue);
+                if (format) {
+                    value = dayjs(value).format(format);
+                    await element.fill(value);
+                }
+                else {
+                    const dateTimeValue = await getDateTimeValue({ value, element });
+                    await element.evaluateHandle((el, dateTimeValue) => {
+                        el.value = ""; // clear input
+                        el.value = dateTimeValue;
+                    }, dateTimeValue);
+                }
+                if (enter) {
+                    await new Promise((resolve) => setTimeout(resolve, 2000));
+                    await this.page.keyboard.press("Enter");
+                    await this.waitForPageLoad();
+                }
             }
         }
-        catch (error) {
+        catch (e) {
             error = e;
             throw e;
         }
@@ -1072,20 +1156,32 @@ class StableBrowser {
             await this.scrollIfNeeded(element, info);
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             await this._highlightElements(element);
-            try {
-                let currentValue = await element.inputValue();
-                if (currentValue) {
-                    await element.fill("");
+            if (options === null || options === undefined || !options.press) {
+                try {
+                    let currentValue = await element.inputValue();
+                    if (currentValue) {
+                        await element.fill("");
+                    }
+                }
+                catch (e) {
+                    this.logger.info("unable to clear input value");
                 }
             }
-            catch (e) {
-                this.logger.info("unable to clear input value");
-            }
-            try {
-                await element.click({ timeout: 5000 });
+            if (options === null || options === undefined || options.press) {
+                try {
+                    await element.click({ timeout: 5000 });
+                }
+                catch (e) {
+                    await element.dispatchEvent("click");
+                }
             }
-            catch (e) {
-                await element.dispatchEvent("click");
+            else {
+                try {
+                    await element.focus();
+                }
+                catch (e) {
+                    await element.dispatchEvent("focus");
+                }
             }
             await new Promise((resolve) => setTimeout(resolve, 500));
             const valueSegment = _value.split("&&");
@@ -1281,7 +1377,8 @@ class StableBrowser {
         let screenshotId = null;
         let screenshotPath = null;
         const info = {};
-        info.log = "***** verify element " + selectors.element_name + " contains pattern  " + pattern + "/" + text + " *****\n";
+        info.log =
+            "***** verify element " + selectors.element_name + " contains pattern  " + pattern + "/" + text + " *****\n";
         info.operation = "containsPattern";
         info.selectors = selectors;
         info.value = text;
@@ -1337,7 +1434,6 @@ class StableBrowser {
         }
     }
     async containsText(selectors, text, climb, _params = null, options = {}, world = null) {
-        var _a, _b, _c;
         this._validateSelectors(selectors);
         if (!text) {
             throw new Error("text is null");
@@ -1358,46 +1454,48 @@ class StableBrowser {
         info.value = text;
         let foundObj = null;
         try {
-            foundObj = await this._getText(selectors, climb, _params, options, info, world);
-            if (foundObj && foundObj.element) {
-                await this.scrollIfNeeded(foundObj.element, info);
-            }
-            ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
-            const dateAlternatives = findDateAlternatives(text);
-            const numberAlternatives = findNumberAlternatives(text);
-            if (dateAlternatives.date) {
-                for (let i = 0; i < dateAlternatives.dates.length; i++) {
-                    if ((foundObj === null || foundObj === void 0 ? void 0 : foundObj.text.includes(dateAlternatives.dates[i])) ||
-                        ((_a = foundObj === null || foundObj === void 0 ? void 0 : foundObj.value) === null || _a === void 0 ? void 0 : _a.includes(dateAlternatives.dates[i]))) {
-                        return info;
-                    }
-                }
-                throw new Error("element doesn't contain text " + text);
-            }
-            else if (numberAlternatives.number) {
-                for (let i = 0; i < numberAlternatives.numbers.length; i++) {
-                    if ((foundObj === null || foundObj === void 0 ? void 0 : foundObj.text.includes(numberAlternatives.numbers[i])) ||
-                        ((_b = foundObj === null || foundObj === void 0 ? void 0 : foundObj.value) === null || _b === void 0 ? void 0 : _b.includes(numberAlternatives.numbers[i]))) {
-                        return info;
-                    }
-                }
-                throw new Error("element doesn't contain text " + text);
-            }
-            else if (!(foundObj === null || foundObj === void 0 ? void 0 : foundObj.text.includes(text)) && !((_c = foundObj === null || foundObj === void 0 ? void 0 : foundObj.value) === null || _c === void 0 ? void 0 : _c.includes(text))) {
-                info.foundText = foundObj === null || foundObj === void 0 ? void 0 : foundObj.text;
-                info.value = foundObj === null || foundObj === void 0 ? void 0 : foundObj.value;
-                throw new Error("element doesn't contain text " + text);
-            }
-            return info;
+            // foundObj = await this._getText(selectors, climb, _params, options, info, world);
+            // if (foundObj && foundObj.element) {
+            //   await this.scrollIfNeeded(foundObj.element, info);
+            // }
+            // ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
+            // const dateAlternatives = findDateAlternatives(text);
+            // const numberAlternatives = findNumberAlternatives(text);
+            // if (dateAlternatives.date) {
+            //   for (let i = 0; i < dateAlternatives.dates.length; i++) {
+            //     if (
+            //       foundObj?.text.includes(dateAlternatives.dates[i]) ||
+            //       foundObj?.value?.includes(dateAlternatives.dates[i])
+            //     ) {
+            //       return info;
+            //     }
+            //   }
+            //   throw new Error("element doesn't contain text " + text);
+            // } else if (numberAlternatives.number) {
+            //   for (let i = 0; i < numberAlternatives.numbers.length; i++) {
+            //     if (
+            //       foundObj?.text.includes(numberAlternatives.numbers[i]) ||
+            //       foundObj?.value?.includes(numberAlternatives.numbers[i])
+            //     ) {
+            //       return info;
+            //     }
+            //   }
+            //   throw new Error("element doesn't contain text " + text);
+            // } else if (!foundObj?.text.includes(text) && !foundObj?.value?.includes(text)) {
+            //   info.foundText = foundObj?.text;
+            //   info.value = foundObj?.value;
+            //   throw new Error("element doesn't contain text " + text);
+            // }
+            // return info;
         }
         catch (e) {
             //await this.closeUnexpectedPopups();
-            this.logger.error("verify element contains text failed " + info.log);
-            ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
-            info.screenshotPath = screenshotPath;
-            Object.assign(e, { info: info });
-            error = e;
-            throw e;
+            // this.logger.error("verify element contains text failed " + info.log);
+            // ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
+            // info.screenshotPath = screenshotPath;
+            // Object.assign(e, { info: info });
+            // error = e;
+            // throw e;
         }
         finally {
             const endTime = Date.now();
@@ -1451,15 +1549,62 @@ class StableBrowser {
         // save the data to the file
         fs.writeFileSync(dataFile, JSON.stringify(data, null, 2));
     }
+    _getDataFilePath(fileName) {
+        let dataFile = path.join(this.project_path, "data", fileName);
+        if (fs.existsSync(dataFile)) {
+            return dataFile;
+        }
+        dataFile = path.join(this.project_path, fileName);
+        if (fs.existsSync(dataFile)) {
+            return dataFile;
+        }
+        throw new Error("data file not found " + fileName);
+    }
+    _parseCSVSync(filePath) {
+        const data = fs.readFileSync(filePath, "utf8");
+        const results = [];
+        return new Promise((resolve, reject) => {
+            const readableStream = new Readable();
+            readableStream._read = () => { }; // _read is required but you can noop it
+            readableStream.push(data);
+            readableStream.push(null);
+            readableStream
+                .pipe(csv())
+                .on("data", (data) => results.push(data))
+                .on("end", () => resolve(results))
+                .on("error", (error) => reject(error));
+        });
+    }
     loadTestData(type, dataSelector, world = null) {
         switch (type) {
             case "users":
-                // check if file users.json exists
-                if (!fs.existsSync(path.join(this.project_path, "users.json"))) {
-                    throw new Error("users.json file not found");
+                // get the users.json file path
+                let dataFile = this._getDataFilePath("users.json");
+                // read the file and return the data
+                const users = JSON.parse(fs.readFileSync(dataFile, "utf8"));
+                for (let i = 0; i < users.length; i++) {
+                    if (users[i].username === dataSelector) {
+                        const userObj = {
+                            username: users[i].username,
+                            password: "secret:" + users[i].password,
+                            totp: users[i].secretKey ? "totp:" + users[i].secretKey : null,
+                        };
+                        this.setTestData(userObj, world);
+                        return userObj;
+                    }
                 }
+                throw new Error("user not found " + dataSelector);
+            default:
+                throw new Error("unknown type " + type);
+        }
+    }
+    async loadTestDataAsync(type, dataSelector, world = null) {
+        switch (type) {
+            case "users": {
+                // get the users.json file path
+                let dataFile = this._getDataFilePath("users.json");
                 // read the file and return the data
-                const users = JSON.parse(fs.readFileSync(path.join(this.project_path, "users.json"), "utf8"));
+                const users = JSON.parse(fs.readFileSync(dataFile, "utf8"));
                 for (let i = 0; i < users.length; i++) {
                     if (users[i].username === dataSelector) {
                         const userObj = {
@@ -1472,6 +1617,29 @@ class StableBrowser {
                     }
                 }
                 throw new Error("user not found " + dataSelector);
+            }
+            case "csv": {
+                // the dataSelector should start with the file name followed by the row number: data.csv:1, if no row number is provided, it will default to 1
+                const parts = dataSelector.split(":");
+                let rowNumber = 0;
+                if (parts.length > 1) {
+                    rowNumber = parseInt(parts[1]);
+                }
+                let dataFile = this._getDataFilePath(parts[0]);
+                const results = await this._parseCSVSync(dataFile);
+                // result stracture:
+                // [
+                //   { NAME: 'Daffy Duck', AGE: '24' },
+                //   { NAME: 'Bugs Bunny', AGE: '22' }
+                // ]
+                // verify the row number is within the range
+                if (rowNumber >= results.length) {
+                    throw new Error("row number is out of range " + rowNumber);
+                }
+                const data = results[rowNumber];
+                this.setTestData(data, world);
+                return data;
+            }
             default:
                 throw new Error("unknown type " + type);
         }
@@ -1576,13 +1744,13 @@ class StableBrowser {
         ])));
         const { data } = await client.send("Page.captureScreenshot", {
             format: "png",
-            clip: {
-                x: 0,
-                y: 0,
-                width: viewportWidth,
-                height: viewportHeight,
-                scale: 1,
-            },
+            // clip: {
+            //   x: 0,
+            //   y: 0,
+            //   width: viewportWidth,
+            //   height: viewportHeight,
+            //   scale: 1,
+            // },
         });
         if (!screenshotPath) {
             return data;
@@ -1688,7 +1856,8 @@ class StableBrowser {
             if (world) {
                 world[variable] = info.value;
             }
-            this.logger.info("world." + variable + "=" + info.value);
+            this.setTestData({ [variable]: info.value }, world);
+            this.logger.info("set test data: " + variable + "=" + info.value);
             return info;
         }
         catch (e) {
@@ -1725,6 +1894,91 @@ class StableBrowser {
             });
         }
     }
+    async extractEmailData(emailAddress, options, world) {
+        if (!emailAddress) {
+            throw new Error("email address is null");
+        }
+        // check if address contain @
+        if (emailAddress.indexOf("@") === -1) {
+            emailAddress = emailAddress + "@blinq-mail.io";
+        }
+        else {
+            if (!emailAddress.toLowerCase().endsWith("@blinq-mail.io")) {
+                throw new Error("email address should end with @blinq-mail.io");
+            }
+        }
+        const startTime = Date.now();
+        let timeout = 60000;
+        if (options && options.timeout) {
+            timeout = options.timeout;
+        }
+        const serviceUrl = this._getServerUrl() + "/api/mail/createLinkOrCodeFromEmail";
+        const request = {
+            method: "POST",
+            url: serviceUrl,
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${process.env.TOKEN}`,
+            },
+            data: JSON.stringify({
+                email: emailAddress,
+            }),
+        };
+        let errorCount = 0;
+        while (true) {
+            try {
+                let result = await this.context.api.request(request);
+                // the response body expected to be the following:
+                // {
+                //  "status": true,
+                //  "content": {
+                //    "url": "",
+                //    "code": "112112",
+                //    "name": "generate_link_or_code"
+                //  }
+                //}
+                if ((result && result.data, result.data.status === true)) {
+                    let codeOrUrlFound = false;
+                    let emailCode = null;
+                    let emailUrl = null;
+                    // check if a code is returned
+                    if (result.data.content && result.data.content.code) {
+                        let code = result.data.content.code;
+                        this.setTestData({ emailCode: code }, world);
+                        this.logger.info("set test data: emailCode = " + code);
+                        emailCode = code;
+                        codeOrUrlFound = true;
+                    }
+                    // check if a url is returned
+                    if (result.data.content && result.data.content.url) {
+                        let url = result.data.content.url;
+                        this.setTestData({ emailUrl: url }, world);
+                        this.logger.info("set test data: emailUrl = " + url);
+                        emailUrl = url;
+                        codeOrUrlFound = true;
+                    }
+                    if (codeOrUrlFound) {
+                        return { emailUrl, emailCode };
+                    }
+                    else {
+                        this.logger.info("an email received but no code or url found");
+                    }
+                }
+            }
+            catch (e) {
+                errorCount++;
+                if (errorCount > 3) {
+                    throw e;
+                }
+                // ignore
+            }
+            // check if the timeout is reached
+            if (Date.now() - startTime > timeout) {
+                throw new Error("timeout reached");
+            }
+            await new Promise((resolve) => setTimeout(resolve, 5000));
+        }
+    }
     async _highlightElements(scope, css) {
         try {
             if (!scope) {
@@ -1906,8 +2160,10 @@ class StableBrowser {
                     const dataAttribute = `[data-blinq-id="blinq-id-${resultWithElementsFound[0].randomToken}"]`;
                     await this._highlightElements(frame, dataAttribute);
                     const element = await frame.$(dataAttribute);
-                    await this.scrollIfNeeded(element, info);
-                    await element.dispatchEvent("bvt_verify_page_contains_text");
+                    if (element) {
+                        await this.scrollIfNeeded(element, info);
+                        await element.dispatchEvent("bvt_verify_page_contains_text");
+                    }
                 }
                 ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
                 return info;
@@ -1945,6 +2201,16 @@ class StableBrowser {
             });
         }
     }
+    _getServerUrl() {
+        let serviceUrl = "https://api.blinq.io";
+        if (process.env.NODE_ENV_BLINQ === "dev") {
+            serviceUrl = "https://dev.api.blinq.io";
+        }
+        else if (process.env.NODE_ENV_BLINQ === "stage") {
+            serviceUrl = "https://stage.api.blinq.io";
+        }
+        return serviceUrl;
+    }
     async visualVerification(text, options = {}, world = null) {
         const startTime = Date.now();
         let error = null;
@@ -1959,13 +2225,7 @@ class StableBrowser {
             throw new Error("TOKEN is not set");
         }
         try {
-            let serviceUrl = "https://api.blinq.io";
-            if (process.env.NODE_ENV_BLINQ === "dev") {
-                serviceUrl = "https://dev.api.blinq.io";
-            }
-            else if (process.env.NODE_ENV_BLINQ === "stage") {
-                serviceUrl = "https://stage.api.blinq.io";
-            }
+            let serviceUrl = this._getServerUrl();
             ({ screenshotId, screenshotPath } = await this._screenShot(options, world, info));
             info.screenshotPath = screenshotPath;
             const screenshot = await this.takeScreenshot();
@@ -2114,7 +2374,16 @@ class StableBrowser {
         let screenshotId = null;
         let screenshotPath = null;
         const info = {};
-        info.log = "***** analyze table " + selectors.element_name + " query " + query + " operator " + operator + " value " + value + " *****\n";
+        info.log =
+            "***** analyze table " +
+                selectors.element_name +
+                " query " +
+                query +
+                " operator " +
+                operator +
+                " value " +
+                value +
+                " *****\n";
         info.operation = "analyzeTable";
         info.selectors = selectors;
         info.query = query;
@@ -2348,13 +2617,6 @@ class StableBrowser {
         const info = {};
         try {
             await this.page.close();
-            if (this.context && this.context.pages && this.context.pages.length > 0) {
-                this.context.pages.pop();
-                this.page = this.context.pages[this.context.pages.length - 1];
-                this.context.page = this.page;
-                let title = await this.page.title();
-                console.log("Switched to page " + title);
-            }
         }
         catch (e) {
             console.log(".");