automation_model 1.0.418-dev → 1.0.418

package/lib/init_browser.js

@@ -7,10 +7,16 @@ import { StableBrowser } from "./stable_browser.js";
 import { Api } from "./api.js";
 // let environment = null;
 // init browser create context and page, if context and page are not null
-const getContext = async function (environment, headless = false, logger) {
+const getContext = async function (environment, headless = false, world, logger, appName, createStable = true, stable = null, moveToRight = -1, reportFolder = null, initScripts = null, storageState = null) {
     if (environment === null) {
         environment = initEnvironment();
     }
+    if (appName && !environment.apps && !environment.apps[appName]) {
+        throw new Error(`App ${appName} not found in environment`);
+    }
+    if (appName) {
+        environment = environment.apps[appName];
+    }
     const { cookies, origins } = environment;
     if (cookies) {
         for (let i = 0; i < cookies.length; i++) {
@@ -22,31 +28,133 @@ const getContext = async function (environment, headless = false, logger) {
     }
     let extensionPath = undefined;
     let userDataDirPath = undefined;
+    let userAgent = undefined;
     let aiConfigFile = "ai_config.json";
+    let channel = undefined;
     if (process.env.PROJECT_PATH) {
         aiConfigFile = path.join(process.env.PROJECT_PATH, "ai_config.json");
     }
+    let configuration = {};
     if (fs.existsSync(aiConfigFile)) {
-        const configuration = JSON.parse(fs.readFileSync(aiConfigFile, "utf8"));
+        configuration = JSON.parse(fs.readFileSync(aiConfigFile, "utf8"));
         if (configuration.userDataDirPath) {
             userDataDirPath = configuration.userDataDirPath;
         }
         if (configuration.extensionPath) {
             extensionPath = configuration.extensionPath;
         }
+        if (configuration.useGoogleChrome === true) {
+            channel = "chrome";
+        }
+        else if (configuration.useMicrosoftEdge === true) {
+            channel = "msedge";
+        }
+        if (configuration.overrideUserAgent) {
+            userAgent = configuration.overrideUserAgent;
+        }
+    }
+    let usedStorageState = null;
+    usedStorageState = { cookies, origins };
+    let downloadsPath = "downloads";
+    if (reportFolder) {
+        downloadsPath = path.join(reportFolder, "downloads");
+    }
+    else if (stable && stable.context && stable.context.reportFolder) {
+        reportFolder = stable.context.reportFolder;
+        downloadsPath = path.join(stable.context.reportFolder, "downloads");
+    }
+    if (world) {
+        world.downloadsPath = downloadsPath;
+    }
+    if (stable && stable.context) {
+        stable.context.downloadsPath = downloadsPath;
+    }
+    // check if data.json exists in the report folder
+    // and if it contain storageState field, if so, use it
+    if (reportFolder) {
+        const dataFile = path.join(reportFolder, "data.json");
+        if (fs.existsSync(dataFile)) {
+            const data = fs.readFileSync(dataFile, "utf8");
+            const dataObject = JSON.parse(data);
+            if (dataObject.storageState) {
+                console.log("Init browser with storage state");
+                usedStorageState = dataObject.storageState;
+            }
+        }
+    }
+    if (storageState) {
+        usedStorageState = storageState;
     }
-    const storageState = { cookies, origins };
-    let browser = await browserManager.getBrowser(headless, storageState, extensionPath, userDataDirPath);
+    let browser = await browserManager.createBrowser(headless, usedStorageState, extensionPath, userDataDirPath, reportFolder ? reportFolder : ".", userAgent, channel, configuration, initScripts);
     let context = new TestContext();
     context.browser = browser.browser;
+    context.browserObject = browser;
     context.playContext = browser.context;
     context.page = browser.page;
+    context.headless = headless;
     context.environment = environment;
-    context.stable = new StableBrowser(context.browser, context.page, logger, context);
+    context.browserName = browser.browser ? browser.browser.browserType().name() : "unknown";
+    context.reportFolder = reportFolder;
+    context.initScripts = initScripts;
+    if (createStable) {
+        context.stable = new StableBrowser(context.browser, context.page, logger, context, world);
+    }
+    else {
+        context.stable = stable;
+    }
     context.api = new Api(logger);
+    if (moveToRight > 0 && context.browserName === "chromium") {
+        // move the borwser to the top right corner of the screen
+        // create a cdp session
+        // Get CDP session
+        const playContext = context.playContext;
+        const client = await playContext.newCDPSession(context.page);
+        // Get window ID for the current target
+        const { windowId } = await client.send("Browser.getWindowForTarget");
+        //console.log(windowId);
+        // get the window for the current target
+        const window = await client.send("Browser.getWindowBounds", {
+            windowId,
+        });
+        //console.log(window);
+        await client.send("Browser.setWindowBounds", {
+            windowId,
+            bounds: {
+                left: window.bounds.left + moveToRight,
+            },
+        });
+        // close cdp
+        await client.detach();
+    }
     // await _initCookies(context);
     return context;
 };
+const refreshBrowser = async function (stable, sessionPath, world) {
+    await stable.context.browserObject.close();
+    stable.context.pages = [];
+    let storageState = null;
+    if (sessionPath) {
+        if (!fs.existsSync(sessionPath)) {
+            throw new Error("Session path not found: " + sessionPath);
+        }
+        const data = fs.readFileSync(sessionPath, "utf8");
+        storageState = JSON.parse(data).storageState;
+    }
+    const newContext = await getContext(stable.context.environment, stable.context.headless, world, null, stable.context.appName, false, stable, -1, stable.context.reportFolder, stable.context.initScripts, storageState);
+    // clone all the new context properties to the old context
+    stable.context.browser = newContext.browser;
+    stable.context.browserObject = newContext.browserObject;
+    stable.context.playContext = newContext.playContext;
+    stable.context.page = newContext.page;
+    stable.page = newContext.page;
+    stable.context.pages.push(newContext.page);
+    stable.context.headless = newContext.headless;
+    stable.context.environment = newContext.environment;
+    stable.context.browserName = newContext.browserName;
+    stable.context.reportFolder = newContext.reportFolder;
+    stable.context.initScripts = newContext.initScripts;
+    await stable.goto(stable.context.environment.baseUrl);
+};
 const closeBrowser = async function (browser) {
     await browserManager.closeBrowser(browser);
 };
@@ -75,7 +183,10 @@ const initEnvironment = function () {
                 }
             }
         }
-        // console.log("envFile: ", envFile);
+        // check if the envFile is not empty and exists
+        if (!envFile || !fs.existsSync(envFile)) {
+            throw new Error(envFile + " not found");
+        }
         const data = fs.readFileSync(envFile, "utf8");
         //console.log("data", data);
         const envObject = JSON.parse(data);
@@ -101,5 +212,5 @@ const checkForEnvArg = function () {
     }
     return null;
 };
-export { getContext, closeBrowser };
+export { getContext, closeBrowser, refreshBrowser };
 //# sourceMappingURL=init_browser.js.map

package/lib/init_browser.js.map

@@ -1 +1 @@
-{"version":3,"file":"init_browser.js","sourceRoot":"","sources":["../../src/init_browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,0BAA0B;AAE1B,yEAAyE;AACzE,MAAM,UAAU,GAAG,KAAK,WAAW,WAA+B,EAAE,QAAQ,GAAG,KAAK,EAAE,MAAa;IACjG,IAAI,WAAW,KAAK,IAAI,EAAE;QACxB,WAAW,GAAG,eAAe,EAAE,CAAC;KACjC;IACD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,WAAW,CAAC;IACzC,IAAI,OAAO,EAAE;QACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,MAAM,CAAC,OAAO,KAAK,WAAW,EAAE;gBAClC,OAAO,MAAM,CAAC,OAAO,CAAC;aACvB;SACF;KACF;IACD,IAAI,aAAa,GAAG,SAAS,CAAC;IAC9B,IAAI,eAAe,GAAG,SAAS,CAAC;IAChC,IAAI,YAAY,GAAG,gBAAgB,CAAC;IACpC,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE;QAC5B,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;KACtE;IACD,IAAI,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QAC/B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;QACxE,IAAI,aAAa,CAAC,eAAe,EAAE;YACjC,eAAe,GAAG,aAAa,CAAC,eAAe,CAAC;SACjD;QACD,IAAI,aAAa,CAAC,aAAa,EAAE;YAC/B,aAAa,GAAG,aAAa,CAAC,aAAa,CAAC;SAC7C;KACF;IACD,MAAM,YAAY,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC1C,IAAI,OAAO,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,eAAe,CAAC,CAAC;IACtG,IAAI,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAChC,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAClC,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;IACtC,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5B,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;IAElC,OAAO,CAAC,MAAM,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,OAAQ,EAAE,OAAO,CAAC,IAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,+BAA+B;IAC/B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,KAAK,WAAW,OAAqC;IACxE,MAAM,cAAc,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG;IACtB,8BAA8B;IAC9B,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IACtC,IAAI;QACF,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,cAAc,EAAE,CAAC;QACnC,IAAI,SAAS,EAAE;YACb,OAAO,GAAG,SAAS,CAAC;SACrB;aAAM,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE;YAChC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;SACjC;aAAM,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC,EAAE;YAC9D,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;SAChD;aAAM;YACL,IAAI,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;YACrE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACrC,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACpB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;oBAC1B,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;oBACzD,MAAM;iBACP;aACF;SACF;QACD,qCAAqC;QACrC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9C,4BAA4B;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,sCAAsC;QACtC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACtC,kCAAkC;QAClC,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;KACjD;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;KAC9C;IACD,IAAI;IACJ,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC;AACF,MAAM,cAAc,GAAG;IACrB,KAAK,IAAI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QACrC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;YACxB,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE;gBAC1B,OAAO,KAAK,CAAC;aACd;SACF;KACF;IACD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AACF,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC"}
+{"version":3,"file":"init_browser.js","sourceRoot":"","sources":["../../src/init_browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAG/B,0BAA0B;AAE1B,yEAAyE;AACzE,MAAM,UAAU,GAAG,KAAK,WACtB,WAA+B,EAC/B,QAAQ,GAAG,KAAK,EAChB,KAAU,EACV,MAAa,EACb,OAAuB,EACvB,YAAY,GAAG,IAAI,EACnB,SAA+B,IAAI,EACnC,WAAW,GAAG,CAAC,CAAC,EAChB,eAA8B,IAAI,EAClC,cAAkC,IAAI,EACtC,eAA2B,IAAI;IAE/B,IAAI,WAAW,KAAK,IAAI,EAAE;QACxB,WAAW,GAAG,eAAe,EAAE,CAAC;KACjC;IACD,IAAI,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;QAC9D,MAAM,IAAI,KAAK,CAAC,OAAO,OAAO,2BAA2B,CAAC,CAAC;KAC5D;IACD,IAAI,OAAO,EAAE;QACX,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;KACzC;IACD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,WAAW,CAAC;IACzC,IAAI,OAAO,EAAE;QACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,MAAM,CAAC,OAAO,KAAK,WAAW,EAAE;gBAClC,OAAO,MAAM,CAAC,OAAO,CAAC;aACvB;SACF;KACF;IACD,IAAI,aAAa,GAAG,SAAS,CAAC;IAC9B,IAAI,eAAe,GAAG,SAAS,CAAC;IAChC,IAAI,SAAS,GAAG,SAAS,CAAC;IAC1B,IAAI,YAAY,GAAG,gBAAgB,CAAC;IACpC,IAAI,OAAO,GAAG,SAAS,CAAC;IACxB,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE;QAC5B,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;KACtE;IACD,IAAI,aAAa,GAAQ,EAAE,CAAC;IAC5B,IAAI,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QAC/B,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;QAClE,IAAI,aAAa,CAAC,eAAe,EAAE;YACjC,eAAe,GAAG,aAAa,CAAC,eAAe,CAAC;SACjD;QACD,IAAI,aAAa,CAAC,aAAa,EAAE;YAC/B,aAAa,GAAG,aAAa,CAAC,aAAa,CAAC;SAC7C;QAED,IAAI,aAAa,CAAC,eAAe,KAAK,IAAI,EAAE;YAC1C,OAAO,GAAG,QAAQ,CAAC;SACpB;aAAM,IAAI,aAAa,CAAC,gBAAgB,KAAK,IAAI,EAAE;YAClD,OAAO,GAAG,QAAQ,CAAC;SACpB;QAED,IAAI,aAAa,CAAC,iBAAiB,EAAE;YACnC,SAAS,GAAG,aAAa,CAAC,iBAAiB,CAAC;SAC7C;KACF;IACD,IAAI,gBAAgB,GAAG,IAAI,CAAC;IAC5B,gBAAgB,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IACxC,IAAI,aAAa,GAAG,WAAW,CAAC;IAChC,IAAI,YAAY,EAAE;QAChB,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;KACtD;SAAM,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE;QAClE,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;QAC3C,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;KACrE;IACD,IAAI,KAAK,EAAE;QACT,KAAK,CAAC,aAAa,GAAG,aAAa,CAAC;KACrC;IACD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE;QAC5B,MAAM,CAAC,OAAO,CAAC,aAAa,GAAG,aAAa,CAAC;KAC9C;IACD,iDAAiD;IACjD,sDAAsD;IACtD,IAAI,YAAY,EAAE;QAChB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QACtD,IAAI,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;YAC3B,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,UAAU,CAAC,YAAY,EAAE;gBAC3B,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;gBAC/C,gBAAgB,GAAG,UAAU,CAAC,YAAY,CAAC;aAC5C;SACF;KACF;IACD,IAAI,YAAY,EAAE;QAChB,gBAAgB,GAAG,YAAY,CAAC;KACjC;IACD,IAAI,OAAO,GAAG,MAAM,cAAc,CAAC,aAAa,CAC9C,QAAQ,EACR,gBAAgB,EAChB,aAAa,EACb,eAAe,EACf,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,EACjC,SAAS,EACT,OAAO,EACP,aAAa,EACb,WAAW,CACZ,CAAC;IACF,IAAI,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAChC,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAClC,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC;IAChC,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;IACtC,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5B,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC5B,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;IAClC,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACzF,OAAO,CAAC,YAAY,GAAG,YAAY,CAAC;IACpC,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;IAElC,IAAI,YAAY,EAAE;QAChB,OAAO,CAAC,MAAM,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,OAAQ,EAAE,OAAO,CAAC,IAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;KAC7F;SAAM;QACL,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;KACzB;IACD,OAAO,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,WAAW,GAAG,CAAC,IAAI,OAAO,CAAC,WAAW,KAAK,UAAU,EAAE;QACzD,yDAAyD;QACzD,uBAAuB;QACvB,kBAAkB;QAClB,MAAM,WAAW,GAAQ,OAAO,CAAC,WAAW,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAE7D,uCAAuC;QACvC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QACrE,wBAAwB;QAExB,wCAAwC;QACxC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;YAC1D,QAAQ;SACT,CAAC,CAAC;QACH,sBAAsB;QACtB,MAAM,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;YAC3C,QAAQ;YACR,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,WAAW;aACvC;SACF,CAAC,CAAC;QACH,YAAY;QACZ,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;KACvB;IAED,+BAA+B;IAC/B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AACF,MAAM,cAAc,GAAG,KAAK,WAAW,MAAW,EAAE,WAAmB,EAAE,KAAU;IACjF,MAAM,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;IAC3C,MAAM,CAAC,OAAO,CAAC,KAAK,GAAG,EAAE,CAAC;IAE1B,IAAI,YAAY,GAAG,IAAI,CAAC;IACxB,IAAI,WAAW,EAAE;QACf,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,WAAW,CAAC,CAAC;SAC3D;QACD,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QAClD,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC;KAC9C;IACD,MAAM,UAAU,GAAG,MAAM,UAAU,CACjC,MAAM,CAAC,OAAO,CAAC,WAAW,EAC1B,MAAM,CAAC,OAAO,CAAC,QAAQ,EACvB,KAAK,EACL,IAAI,EACJ,MAAM,CAAC,OAAO,CAAC,OAAO,EACtB,KAAK,EACL,MAAM,EACN,CAAC,CAAC,EACF,MAAM,CAAC,OAAO,CAAC,YAAY,EAC3B,MAAM,CAAC,OAAO,CAAC,WAAW,EAC1B,YAAY,CACb,CAAC;IACF,0DAA0D;IAC1D,MAAM,CAAC,OAAO,CAAC,OAAO,GAAG,UAAU,CAAC,OAAO,CAAC;IAC5C,MAAM,CAAC,OAAO,CAAC,aAAa,GAAG,UAAU,CAAC,aAAa,CAAC;IACxD,MAAM,CAAC,OAAO,CAAC,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC;IACpD,MAAM,CAAC,OAAO,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;IACtC,MAAM,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC;IAC9B,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,CAAC,OAAO,CAAC,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;IAC9C,MAAM,CAAC,OAAO,CAAC,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC;IACpD,MAAM,CAAC,OAAO,CAAC,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC;IACpD,MAAM,CAAC,OAAO,CAAC,YAAY,GAAG,UAAU,CAAC,YAAY,CAAC;IACtD,MAAM,CAAC,OAAO,CAAC,WAAW,GAAG,UAAU,CAAC,WAAW,CAAC;IACpD,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;AACxD,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,KAAK,WAAW,OAAqC;IACxE,MAAM,cAAc,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG;IACtB,8BAA8B;IAC9B,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IACtC,IAAI;QACF,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,cAAc,EAAE,CAAC;QACnC,IAAI,SAAS,EAAE;YACb,OAAO,GAAG,SAAS,CAAC;SACrB;aAAM,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE;YAChC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;SACjC;aAAM,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC,EAAE;YAC9D,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;SAChD;aAAM;YACL,IAAI,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;YACrE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACrC,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACpB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;oBAC1B,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;oBACzD,MAAM;iBACP;aACF;SACF;QACD,+CAA+C;QAC/C,IAAI,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;YACvC,MAAM,IAAI,KAAK,CAAC,OAAO,GAAG,YAAY,CAAC,CAAC;SACzC;QACD,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9C,4BAA4B;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,sCAAsC;QACtC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACtC,kCAAkC;QAClC,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;KACjD;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;KAC9C;IACD,IAAI;IACJ,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC;AACF,MAAM,cAAc,GAAG;IACrB,KAAK,IAAI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QACrC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;YACxB,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE;gBAC1B,OAAO,KAAK,CAAC;aACd;SACF;KACF;IACD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AACF,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC"}

package/lib/locate_element.d.ts

@@ -0,0 +1,7 @@
+export declare function locate_element(context: any, elementDescription: string, operation?: string, value?: string | null): Promise<{
+    reason: any;
+    elementNumber: any;
+    frameIndex: number;
+    frame: null;
+    css: string;
+} | null>;

package/lib/locate_element.js

@@ -0,0 +1,215 @@
+import axios from "axios";
+import * as path from "path";
+import { fileURLToPath } from "url";
+import { _getServerUrl } from "./utils.js";
+// Get __filename and __dirname in ESM
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+export async function locate_element(context, elementDescription, operation = "click", value = null) {
+    // load the axe-core library to all of the frames in the page
+    // read the axe-core library from the file system placed in the same folder as the script file, name axe.mini.js
+    // Construct the path to axe.min.js relative to the current file
+    // let axeMinJsPath = path.join(__dirname, "..", "scripts", "axe.mini.js");
+    // // Check if the file exists
+    // if (!fs.existsSync(axeMinJsPath)) {
+    //   axeMinJsPath = path.join(__dirname, "scripts", "axe.mini.js");
+    // }
+    // // Read the content of axe.min.js synchronously
+    // const axeMinJsContent = fs.readFileSync(axeMinJsPath, "utf-8");
+    // await Promise.all(context.stable.page.frames().map((frame: any) => frame.evaluate(axeMinJsContent)));
+    const frames = await context.stable.page.frames();
+    // for each frame create a tree of the accessibility nodes
+    const frameDump = [];
+    let iframeIndex = 0;
+    let nextFrameIndex = 1;
+    let actionableElementIndex = 0;
+    const randomToken = Math.random().toString(36).substring(7);
+    const actionableElements = [];
+    for (let frame of frames) {
+        // @ts-ignore
+        let result = null;
+        try {
+            result = await frame.evaluate(
+            // @ts-ignore
+            ([iframeIndex, nextFrameIndex, actionableElementIndex, randomToken]) => {
+                let iframeCount = 1;
+                const actionableElements = [];
+                const actionableRoles = [
+                    "link",
+                    "button",
+                    "tab",
+                    "menuitem",
+                    "menuitemcheckbox",
+                    "menuitemradio",
+                    "checkbox",
+                    "textbox",
+                    "combobox",
+                    "listitem",
+                    "radio",
+                    "searchbox",
+                    "option",
+                    "Date",
+                    "row",
+                    "DateTime",
+                    "InputTime",
+                    "treeitem",
+                ];
+                function isAccessibilityElement(element) {
+                    if (!element) {
+                        return false;
+                    }
+                    // @ts-ignore
+                    let hidden = axe.commons.dom.isHiddenForEveryone(element);
+                    if (hidden) {
+                        return false;
+                    }
+                    if (element.tagName === "IFRAME") {
+                        return true;
+                    }
+                    // @ts-ignore
+                    let roles = axe.commons.aria.getRole(element);
+                    if (!roles) {
+                        return false;
+                    }
+                    return true;
+                }
+                function getAttributes(element) {
+                    const attrs = element.attributes;
+                    const result = {};
+                    for (let i = 0; i < attrs.length; i++) {
+                        result[attrs[i].name] = attrs[i].value;
+                        if (attrs[i].name === "class") {
+                            // clean the spaces
+                            result[attrs[i].name] = attrs[i].value.replace(/\s+/g, " ").trim();
+                        }
+                    }
+                    return result;
+                }
+                function createAccessibilityNode(element, actionableElements) {
+                    // check if the element is an iframe
+                    const result = {
+                        tag: element.tagName,
+                        // @ts-ignore
+                        role: axe.commons.aria.getRole(element),
+                        // @ts-ignore
+                        visible: axe.commons.dom.isVisible(element),
+                        // @ts-ignore
+                        name: axe.commons.text.accessibleText(element),
+                        attributes: getAttributes(element),
+                        // @ts-ignore
+                        frameIndex: iframeIndex,
+                    };
+                    if (actionableRoles.includes(result.role)) {
+                        // @ts-ignore
+                        result.elementNumber = actionableElementIndex;
+                        element.setAttribute("data-blinq-id", "blinq-id-" + randomToken + "-" + actionableElementIndex);
+                        //console.log("attribute set to " + "blinq-id-" + randomToken + "-" + actionableElementIndex);
+                        actionableElementIndex++;
+                        actionableElements.push(result);
+                    }
+                    if (element.tagName === "IFRAME") {
+                        // @ts-ignore
+                        result.targetFrameIndex = nextFrameIndex;
+                        nextFrameIndex++;
+                    }
+                    return result;
+                }
+                // a function that traverses the DOM tree and builds a tree of accessibility nodes
+                function buildAccessibilityTree(node, treeRoot) {
+                    //const foundAccessibilityNodes = [];
+                    if (!treeRoot.children) {
+                        treeRoot.children = [];
+                    }
+                    //console.log(node.tagName + " " + node.children.length);
+                    for (let child of node.children) {
+                        if (isAccessibilityElement(child)) {
+                            const accessibilityNode = createAccessibilityNode(child, actionableElements);
+                            // @ts-ignore
+                            // accessibilityNode.children = buildAccessibilityTree(child, accessibilityNode),
+                            // foundAccessibilityNodes.push(accessibilityNode);
+                            treeRoot.children.push(accessibilityNode);
+                            buildAccessibilityTree(child, accessibilityNode);
+                        }
+                        else {
+                            buildAccessibilityTree(child, treeRoot);
+                        }
+                    }
+                }
+                // @ts-ignore
+                axe.utils.getFlattenedTree(document);
+                const root = createAccessibilityNode(document.body, actionableElements);
+                buildAccessibilityTree(document.body, root);
+                return JSON.stringify({ root, nextFrameIndex, actionableElementIndex, actionableElements }, null, 2);
+            }, [iframeIndex, nextFrameIndex, actionableElementIndex, randomToken]);
+            const resultData = JSON.parse(result);
+            nextFrameIndex = resultData.nextFrameIndex;
+            actionableElementIndex = resultData.actionableElementIndex;
+            frameDump.push(resultData.root);
+            actionableElements.push(...resultData.actionableElements);
+            iframeIndex++;
+        }
+        catch (e) {
+            // ignore
+        }
+    }
+    function traverseDFS(node) {
+        // Recursively traverse each child node
+        if (node.children && node.children.length > 0) {
+            node.children.forEach((child) => {
+                if (child.tag === "IFRAME") {
+                    if (child.targetFrameIndex && frameDump.length >= child.targetFrameIndex) {
+                        child.children = [frameDump[child.targetFrameIndex]];
+                    }
+                }
+                if (child.shadowRoot) {
+                    traverseDFS(child.shadowRoot);
+                }
+                else {
+                    traverseDFS(child);
+                }
+            });
+        }
+    }
+    traverseDFS(frameDump[0]);
+    let serviceUrl = _getServerUrl();
+    const config = {
+        method: "post",
+        url: `${serviceUrl}/api/runs/locate-element/locate`,
+        headers: {
+            "x-source": "true",
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${process.env.TOKEN}`,
+        },
+        data: JSON.stringify({
+            elementDescription: elementDescription,
+            operation: operation,
+            dump: frameDump[0],
+            value: value,
+        }),
+    };
+    let result = await axios.request(config);
+    //console.log(JSON.stringify(frameDump[0]));
+    if (result.status !== 200 || !result.data || result.data.status !== true || !result.data.result) {
+        console.error("Failed to locate element");
+        return null;
+    }
+    const returnData = {
+        reason: result.data.result.reason,
+        elementNumber: result.data.result.elementNumber,
+        frameIndex: -1,
+        frame: null,
+        css: "",
+    };
+    if (result.data.result.elementNumber !== -1) {
+        // find the iframe index, the element is in, by identifying the actionable element
+        const actionableElement = actionableElements.find((element) => element.elementNumber === result.data.result.elementNumber);
+        if (actionableElement) {
+            returnData.frameIndex = actionableElement.frameIndex;
+            returnData.frame = frames[returnData.frameIndex];
+            returnData.css = `[data-blinq-id="blinq-id-${randomToken}-${returnData.elementNumber}"]`;
+        }
+    }
+    //console.log(dumpToHtml(frameDump[0]));
+    return returnData;
+}
+//# sourceMappingURL=locate_element.js.map

package/lib/locate_element.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"locate_element.js","sourceRoot":"","sources":["../../src/locate_element.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,sCAAsC;AACtC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAY,EACZ,kBAA0B,EAC1B,SAAS,GAAG,OAAO,EACnB,QAAuB,IAAI;IAE3B,6DAA6D;IAC7D,gHAAgH;IAChH,gEAAgE;IAChE,2EAA2E;IAC3E,8BAA8B;IAC9B,sCAAsC;IACtC,mEAAmE;IACnE,IAAI;IAEJ,kDAAkD;IAClD,kEAAkE;IAClE,wGAAwG;IAExG,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IAClD,0DAA0D;IAC1D,MAAM,SAAS,GAAU,EAAE,CAAC;IAC5B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,sBAAsB,GAAG,CAAC,CAAC;IAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,kBAAkB,GAAU,EAAE,CAAC;IACrC,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE;QACxB,aAAa;QACb,IAAI,MAAM,GAAQ,IAAI,CAAC;QACvB,IAAI;YACF,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ;YAC3B,aAAa;YACb,CAAC,CAAC,WAAW,EAAE,cAAc,EAAE,sBAAsB,EAAE,WAAW,CAAC,EAAE,EAAE;gBACrE,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,MAAM,kBAAkB,GAAU,EAAE,CAAC;gBACrC,MAAM,eAAe,GAAG;oBACtB,MAAM;oBACN,QAAQ;oBACR,KAAK;oBACL,UAAU;oBACV,kBAAkB;oBAClB,eAAe;oBACf,UAAU;oBACV,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,QAAQ;oBACR,MAAM;oBACN,KAAK;oBACL,UAAU;oBACV,WAAW;oBACX,UAAU;iBACX,CAAC;gBAEF,SAAS,sBAAsB,CAAC,OAAY;oBAC1C,IAAI,CAAC,OAAO,EAAE;wBACZ,OAAO,KAAK,CAAC;qBACd;oBACD,aAAa;oBACb,IAAI,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;oBAC1D,IAAI,MAAM,EAAE;wBACV,OAAO,KAAK,CAAC;qBACd;oBACD,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE;wBAChC,OAAO,IAAI,CAAC;qBACb;oBACD,aAAa;oBACb,IAAI,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;oBAC9C,IAAI,CAAC,KAAK,EAAE;wBACV,OAAO,KAAK,CAAC;qBACd;oBACD,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,SAAS,aAAa,CAAC,OAAY;oBACjC,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC;oBACjC,MAAM,MAAM,GAAQ,EAAE,CAAC;oBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;wBACrC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;wBACvC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE;4BAC7B,mBAAmB;4BACnB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;yBACpE;qBACF;oBACD,OAAO,MAAM,CAAC;gBAChB,CAAC;gBACD,SAAS,uBAAuB,CAAC,OAAY,EAAE,kBAAuB;oBACpE,oCAAoC;oBAEpC,MAAM,MAAM,GAAG;wBACb,GAAG,EAAE,OAAO,CAAC,OAAO;wBACpB,aAAa;wBACb,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;wBACvC,aAAa;wBACb,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC;wBAC3C,aAAa;wBACb,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;wBAC9C,UAAU,EAAE,aAAa,CAAC,OAAO,CAAC;wBAClC,aAAa;wBACb,UAAU,EAAE,WAAW;qBACxB,CAAC;oBACF,IAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;wBACzC,aAAa;wBACb,MAAM,CAAC,aAAa,GAAG,sBAAsB,CAAC;wBAC9C,OAAO,CAAC,YAAY,CAAC,eAAe,EAAE,WAAW,GAAG,WAAW,GAAG,GAAG,GAAG,sBAAsB,CAAC,CAAC;wBAChG,8FAA8F;wBAC9F,sBAAsB,EAAE,CAAC;wBACzB,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;qBACjC;oBACD,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE;wBAChC,aAAa;wBACb,MAAM,CAAC,gBAAgB,GAAG,cAAc,CAAC;wBACzC,cAAc,EAAE,CAAC;qBAClB;oBAED,OAAO,MAAM,CAAC;gBAChB,CAAC;gBACD,kFAAkF;gBAClF,SAAS,sBAAsB,CAAC,IAAS,EAAE,QAAa;oBACtD,qCAAqC;oBACrC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE;wBACtB,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC;qBACxB;oBACD,yDAAyD;oBACzD,KAAK,IAAI,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE;wBAC/B,IAAI,sBAAsB,CAAC,KAAK,CAAC,EAAE;4BACjC,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAC;4BAC7E,aAAa;4BACb,iFAAiF;4BACjF,mDAAmD;4BACnD,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;4BAC1C,sBAAsB,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;yBAClD;6BAAM;4BACL,sBAAsB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;yBACzC;qBACF;gBACH,CAAC;gBACD,aAAa;gBACb,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;gBACrC,MAAM,IAAI,GAAG,uBAAuB,CAAC,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;gBACxE,sBAAsB,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACvG,CAAC,EACD,CAAC,WAAW,EAAE,cAAc,EAAE,sBAAsB,EAAE,WAAW,CAAC,CACnE,CAAC;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACtC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;YAC3C,sBAAsB,GAAG,UAAU,CAAC,sBAAsB,CAAC;YAC3D,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YAChC,kBAAkB,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAAC;YAC1D,WAAW,EAAE,CAAC;SACf;QAAC,OAAO,CAAC,EAAE;YACV,SAAS;SACV;KACF;IACD,SAAS,WAAW,CAAC,IAAS;QAC5B,uCAAuC;QACvC,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YAC7C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAU,EAAE,EAAE;gBACnC,IAAI,KAAK,CAAC,GAAG,KAAK,QAAQ,EAAE;oBAC1B,IAAI,KAAK,CAAC,gBAAgB,IAAI,SAAS,CAAC,MAAM,IAAI,KAAK,CAAC,gBAAgB,EAAE;wBACxE,KAAK,CAAC,QAAQ,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;qBACtD;iBACF;gBACD,IAAI,KAAK,CAAC,UAAU,EAAE;oBACpB,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;iBAC/B;qBAAM;oBACL,WAAW,CAAC,KAAK,CAAC,CAAC;iBACpB;YACH,CAAC,CAAC,CAAC;SACJ;IACH,CAAC;IACD,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,UAAU,GAAG,aAAa,EAAE,CAAC;IACjC,MAAM,MAAM,GAAG;QACb,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG,UAAU,iCAAiC;QACnD,OAAO,EAAE;YACP,UAAU,EAAE,MAAM;YAClB,cAAc,EAAE,kBAAkB;YAClC,aAAa,EAAE,UAAU,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE;SAC7C;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,kBAAkB,EAAE,kBAAkB;YACtC,SAAS,EAAE,SAAS;YACpB,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;YAClB,KAAK,EAAE,KAAK;SACb,CAAC;KACH,CAAC;IAEF,IAAI,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,4CAA4C;IAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE;QAC/F,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC1C,OAAO,IAAI,CAAC;KACb;IACD,MAAM,UAAU,GAAG;QACjB,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM;QACjC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa;QAC/C,UAAU,EAAE,CAAC,CAAC;QACd,KAAK,EAAE,IAAI;QACX,GAAG,EAAE,EAAE;KACR,CAAC;IACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,KAAK,CAAC,CAAC,EAAE;QAC3C,kFAAkF;QAClF,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,IAAI,CAC/C,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,aAAa,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CACxE,CAAC;QACF,IAAI,iBAAiB,EAAE;YACrB,UAAU,CAAC,UAAU,GAAG,iBAAiB,CAAC,UAAU,CAAC;YACrD,UAAU,CAAC,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YACjD,UAAU,CAAC,GAAG,GAAG,4BAA4B,WAAW,IAAI,UAAU,CAAC,aAAa,IAAI,CAAC;SAC1F;KACF;IACD,wCAAwC;IACxC,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/lib/locator.d.ts

@@ -17,6 +17,42 @@ declare function getTableData(table: Element): {
     columnNames: never[];
     rows: never[];
 };
+declare function getTableData2(table: any): {
+    rowsCount: number;
+    columnsCount: number;
+    nodes: never[];
+    rows: never[];
+    columnHeaders: never[];
+};
+declare function getVisibleText(thElement: any): any;
+declare const Roles: {
+    LINK: string;
+    BUTTON: string;
+    CHECKBOX: string;
+    RADIO: string;
+    SLIDER: string;
+    TEXTBOX: string;
+    PRESENTATION: string;
+    IMG: string;
+    LIST: string;
+    LISTITEM: string;
+    TABLE: string;
+    COLUMNHEADER: string;
+    ROWHEADER: string;
+    CELL: string;
+    ROW: string;
+    NAVIGATION: string;
+    BANNER: string;
+    CONTENTINFO: string;
+    MAIN: string;
+    ARTICLE: string;
+    REGION: string;
+    COMPLEMENTARY: string;
+    FORM: string;
+    LISTBOX: string;
+    COMBOBOX: string;
+};
+declare function getRole(element: any): any;
 declare function processTableQuery(table: Element, query: string): {
     cells: any[];
     error?: undefined;

package/lib/locator.js

@@ -45,6 +45,171 @@ function getTableData(table) {
 }
 // @ts-ignore
 document.getTableData = getTableData;
+function getTableData2(table) {
+    // Scan the table and build a tree
+    function scanTable(root, nodeArray, tableData) {
+        const role = document.getRole(root);
+        if (role &&
+            (role === Roles.CELL || role === Roles.ROW || role === Roles.COLUMNHEADER || role === Roles.ROWHEADER)) {
+            const rec = root.getBoundingClientRect();
+            const node = {
+                tag: root.tagName,
+                role,
+                rec: rec ? [rec.x, rec.y, rec.width, rec.height] : null,
+                children: [],
+            };
+            if (role === Roles.CELL || role === Roles.COLUMNHEADER || role === Roles.ROWHEADER) {
+                node.text = document.getVisibleText(root);
+            }
+            if (role === Roles.COLUMNHEADER) {
+                tableData.columnsCount++;
+                tableData.columnHeaders.push(node);
+            }
+            if (role === Roles.ROW) {
+                tableData.rowsCount++;
+                tableData.rows.push(node);
+            }
+            nodeArray.push(node);
+            nodeArray = node.children;
+        }
+        const children = Array.from(root.children);
+        for (let i = 0; i < children.length; i++) {
+            scanTable(children[i], nodeArray, tableData);
+        }
+    }
+    const tableData = {
+        rowsCount: 0,
+        columnsCount: 0,
+        nodes: [],
+        rows: [],
+        columnHeaders: [],
+    };
+    // Pass the root table element and initialize the tree
+    scanTable(table, tableData.nodes, tableData);
+    return tableData;
+}
+function getVisibleText(thElement) {
+    // if (!(thElement instanceof HTMLTableCellElement && thElement.tagName === 'TH')) {
+    //     throw new Error('Provided element is not a valid TH element.');
+    // }
+    // Helper function to recursively collect visible text
+    function getText(node) {
+        if (node.nodeType === Node.TEXT_NODE) {
+            return node.textContent.trim(); // Return text content of text nodes
+        }
+        if (node.nodeType === Node.ELEMENT_NODE) {
+            const tagName = node.tagName.toUpperCase();
+            if (tagName === "BR") {
+                return "\n"; // Treat <br> as a newline
+            }
+            if (getComputedStyle(node).display === "none") {
+                return ""; // Ignore invisible elements
+            }
+            // Recursively process child nodes
+            return Array.from(node.childNodes).map(getText).join("");
+        }
+        return ""; // Ignore other node types
+    }
+    return getText(thElement)
+        .replace(/\n\s*\n/g, "\n")
+        .trim(); // Clean up multiple newlines
+}
+document.getVisibleText = getVisibleText;
+// @ts-ignore
+document.getTableData2 = getTableData2;
+const Roles = {
+    LINK: "link",
+    BUTTON: "button",
+    CHECKBOX: "checkbox",
+    RADIO: "radio",
+    SLIDER: "slider",
+    TEXTBOX: "textbox",
+    PRESENTATION: "presentation",
+    IMG: "img",
+    LIST: "list",
+    LISTITEM: "listitem",
+    TABLE: "table",
+    COLUMNHEADER: "columnheader",
+    ROWHEADER: "rowheader",
+    CELL: "cell",
+    ROW: "row",
+    NAVIGATION: "navigation",
+    BANNER: "banner",
+    CONTENTINFO: "contentinfo",
+    MAIN: "main",
+    ARTICLE: "article",
+    REGION: "region",
+    COMPLEMENTARY: "complementary",
+    FORM: "form",
+    LISTBOX: "listbox",
+    COMBOBOX: "combobox",
+};
+function getRole(element) {
+    if (!element || typeof element.getAttribute !== "function") {
+        return null;
+    }
+    const tagName = element.tagName.toLowerCase();
+    function getInputRole(el) {
+        const type = el.type;
+        if (["button", "submit", "reset", "image"].includes(type)) {
+            return Roles.BUTTON;
+        }
+        if (type === "checkbox")
+            return Roles.CHECKBOX;
+        if (type === "radio")
+            return Roles.RADIO;
+        if (type === "range")
+            return Roles.SLIDER;
+        if (["email", "tel", "text", "search", "url", "password"].includes(type)) {
+            return Roles.TEXTBOX;
+        }
+        return null;
+    }
+    function getImageRole(el) {
+        return el.getAttribute("alt") === "" ? Roles.PRESENTATION : Roles.IMG;
+    }
+    const implicitRoles = {
+        a: (el) => (el.hasAttribute("href") ? Roles.LINK : null),
+        button: Roles.BUTTON,
+        input: getInputRole,
+        select: (el) => (el.hasAttribute("multiple") || el.size > 1 ? Roles.LISTBOX : Roles.COMBOBOX),
+        textarea: Roles.TEXTBOX,
+        img: getImageRole,
+        ul: Roles.LIST,
+        ol: Roles.LIST,
+        li: Roles.LISTITEM,
+        table: Roles.TABLE,
+        th: (el) => {
+            const scope = el.getAttribute("scope");
+            if (scope === "col" || el.closest("thead")) {
+                return Roles.COLUMNHEADER;
+            }
+            if (scope === "row") {
+                return Roles.ROWHEADER;
+            }
+            // Fallback: infer based on context
+            const parentRow = el.closest("tr");
+            return parentRow && parentRow.parentNode.tagName.toLowerCase() === "thead" ? Roles.COLUMNHEADER : Roles.ROWHEADER;
+        },
+        td: Roles.CELL,
+        tr: Roles.ROW,
+        nav: Roles.NAVIGATION,
+        header: Roles.BANNER,
+        footer: Roles.CONTENTINFO,
+        form: (el) => (el.hasAttribute("name") ? Roles.FORM : null),
+        main: Roles.MAIN,
+        article: Roles.ARTICLE,
+        section: Roles.REGION,
+        aside: Roles.COMPLEMENTARY,
+    };
+    const role = element.getAttribute("role");
+    if (role) {
+        return role;
+    }
+    const implicitRole = implicitRoles[tagName];
+    return typeof implicitRole === "function" ? implicitRole(element) : implicitRole || null;
+}
+document.getRole = getRole;
 // locator                        | operator | value
 // table.rows                     | >=       | 1
 // table[1]["Availability"].text  | equals   | 100%