automation_model 1.0.411-dev → 1.0.411

package/lib/error-messages.js

@@ -0,0 +1,200 @@
+function classifyPlaywrightError(error) {
+    const errorMessage = error.message.toLowerCase();
+    // Timeout Errors
+    if (error.name === "TimeoutError" || errorMessage.includes("timeout")) {
+        return {
+            errorType: "TimeoutError",
+            errorMessage: error.message,
+        };
+    }
+    // Network Errors
+    if (errorMessage.includes("connect econnrefused") ||
+        errorMessage.includes("net::") ||
+        errorMessage.includes("network") ||
+        errorMessage.includes("connection refused") ||
+        errorMessage.includes("failed to fetch")) {
+        return {
+            errorType: "NetworkError",
+            errorMessage: error.message,
+        };
+    }
+    // Selector Errors
+    if (errorMessage.includes("no element matches selector") ||
+        errorMessage.includes("no node found for selector") ||
+        errorMessage.includes("resolved to") ||
+        errorMessage.includes("element not found")) {
+        return {
+            errorType: "SelectorError",
+            errorMessage: error.message,
+        };
+    }
+    // Frame Errors
+    if (errorMessage.includes("frame was detached") ||
+        errorMessage.includes("frame not found") ||
+        errorMessage.includes("execution context was destroyed")) {
+        return {
+            errorType: "FrameError",
+            errorMessage: error.message,
+        };
+    }
+    // Element State Errors
+    if (errorMessage.includes("element is not clickable") ||
+        errorMessage.includes("element is outside of viewport") ||
+        errorMessage.includes("element is not visible") ||
+        errorMessage.includes("element is disabled")) {
+        return {
+            errorType: "ElementStateError",
+            errorMessage: error.message,
+        };
+    }
+    // Browser Context Errors
+    if (errorMessage.includes("target closed") ||
+        errorMessage.includes("browser has been closed") ||
+        errorMessage.includes("connection closed")) {
+        return {
+            errorType: "BrowserContextError",
+            errorMessage: error.message,
+        };
+    }
+    // Screenshot Errors
+    if (errorMessage.includes("failed to save screenshot") ||
+        errorMessage.includes("screenshot") ||
+        (errorMessage.includes("enoent") && errorMessage.includes("screenshots"))) {
+        return {
+            errorType: "ScreenshotError",
+            errorMessage: error.message,
+        };
+    }
+    // Type Errors
+    if (errorMessage.includes("cannot type") ||
+        errorMessage.includes("element is not an <input>") ||
+        errorMessage.includes("element is not focusable")) {
+        return {
+            errorType: "TypeError",
+            errorMessage: error.message,
+        };
+    }
+    // Evaluation Errors
+    if (errorMessage.includes("evaluation failed") ||
+        errorMessage.includes("execution context was destroyed") ||
+        errorMessage.includes("cannot execute in detached frame")) {
+        return {
+            errorType: "EvaluationError",
+            errorMessage: error.message,
+        };
+    }
+    // Assertion Errors
+    if (error.name === "AssertionError" || errorMessage.includes("expect(") || errorMessage.includes("assertion")) {
+        return {
+            errorType: "AssertionError",
+            errorMessage: error.message,
+        };
+    }
+    // Default case for unrecognized errors
+    return {
+        errorType: "UnknownError",
+        errorMessage: error.message,
+    };
+}
+function classifyJSError(error) {
+    const errorMessage = error.message.toLowerCase();
+    // Syntax Errors
+    if (error.name === "SyntaxError" || errorMessage.includes("syntax error")) {
+        return {
+            errorType: "SyntaxError",
+            errorMessage: error.message,
+        };
+    }
+    // Reference Errors
+    if (error.name === "ReferenceError" || errorMessage.includes("reference error")) {
+        return {
+            errorType: "ReferenceError",
+            errorMessage: error.message,
+        };
+    }
+    // Type Errors
+    if (error.name === "TypeError" || errorMessage.includes("type error")) {
+        return {
+            errorType: "TypeError",
+            errorMessage: error.message,
+        };
+    }
+    // Range Errors
+    if (error.name === "RangeError" || errorMessage.includes("range error")) {
+        return {
+            errorType: "RangeError",
+            errorMessage: error.message,
+        };
+    }
+    // Default case for unrecognized errors
+    return {
+        errorType: "UnknownError",
+        errorMessage: error.message,
+    };
+}
+const classifyErrorFromInfo = (error, info) => {
+    const failCause = info?.failCause;
+    if (!failCause) {
+        return {
+            errorType: "UnknownError",
+            errorMessage: error.message,
+        };
+    }
+    if (failCause.enabled === false) {
+        return {
+            errorType: "ElementDisabled",
+            errorMessage: failCause.lastError,
+        };
+    }
+    if (failCause.visible === false) {
+        return {
+            errorType: "ElementNotVisible",
+            errorMessage: failCause.lastError,
+        };
+    }
+    if (failCause.textNotFound) {
+        return {
+            errorType: "TextNotFoundError",
+            errorMessage: failCause.lastError,
+        };
+    }
+    if (failCause.iframeNotFound) {
+        return {
+            errorType: "IframeNotFoundError",
+            errorMessage: failCause.lastError,
+        };
+    }
+    if (failCause.locatorNotFound) {
+        return {
+            errorType: "ElementNotFoundError",
+            errorMessage: failCause.lastError,
+        };
+    }
+    if (failCause.foundMultiple) {
+        return {
+            errorType: "MultipleElementsFoundError",
+            errorMessage: failCause.lastError ?? `Found ${failCause.count} elements`,
+        };
+    }
+    return {
+        errorType: "UnknownError",
+        errorMessage: error.message,
+    };
+};
+const getHumanReadableErrorMessage = (error, info) => {
+    // @ts-ignore
+    if (error.errors && error.errors.length > 0) {
+        // @ts-ignore
+        return getHumanReadableErrorMessage(error.errors[0], info);
+    }
+    let errorClassification = classifyErrorFromInfo(error, info);
+    if (errorClassification.errorType === "UnknownError") {
+        errorClassification = classifyPlaywrightError(error);
+    }
+    if (errorClassification.errorType === "UnknownError") {
+        errorClassification = classifyJSError(error);
+    }
+    return errorClassification;
+};
+export { getHumanReadableErrorMessage };
+//# sourceMappingURL=error-messages.js.map

package/lib/error-messages.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"error-messages.js","sourceRoot":"","sources":["../../src/error-messages.ts"],"names":[],"mappings":"AAKA,SAAS,uBAAuB,CAAC,KAAY;IAC3C,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAEjD,iBAAiB;IACjB,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;QACrE,OAAO;YACL,SAAS,EAAE,cAAc;YACzB,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,iBAAiB;IACjB,IACE,YAAY,CAAC,QAAQ,CAAC,sBAAsB,CAAC;QAC7C,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC9B,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC;QAChC,YAAY,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QAC3C,YAAY,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EACxC;QACA,OAAO;YACL,SAAS,EAAE,cAAc;YACzB,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,kBAAkB;IAClB,IACE,YAAY,CAAC,QAAQ,CAAC,6BAA6B,CAAC;QACpD,YAAY,CAAC,QAAQ,CAAC,4BAA4B,CAAC;QACnD,YAAY,CAAC,QAAQ,CAAC,aAAa,CAAC;QACpC,YAAY,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAC1C;QACA,OAAO;YACL,SAAS,EAAE,eAAe;YAC1B,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,eAAe;IACf,IACE,YAAY,CAAC,QAAQ,CAAC,oBAAoB,CAAC;QAC3C,YAAY,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QACxC,YAAY,CAAC,QAAQ,CAAC,iCAAiC,CAAC,EACxD;QACA,OAAO;YACL,SAAS,EAAE,YAAY;YACvB,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,uBAAuB;IACvB,IACE,YAAY,CAAC,QAAQ,CAAC,0BAA0B,CAAC;QACjD,YAAY,CAAC,QAAQ,CAAC,gCAAgC,CAAC;QACvD,YAAY,CAAC,QAAQ,CAAC,wBAAwB,CAAC;QAC/C,YAAY,CAAC,QAAQ,CAAC,qBAAqB,CAAC,EAC5C;QACA,OAAO;YACL,SAAS,EAAE,mBAAmB;YAC9B,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,yBAAyB;IACzB,IACE,YAAY,CAAC,QAAQ,CAAC,eAAe,CAAC;QACtC,YAAY,CAAC,QAAQ,CAAC,yBAAyB,CAAC;QAChD,YAAY,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAC1C;QACA,OAAO;YACL,SAAS,EAAE,qBAAqB;YAChC,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,oBAAoB;IACpB,IACE,YAAY,CAAC,QAAQ,CAAC,2BAA2B,CAAC;QAClD,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC;QACnC,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,EACzE;QACA,OAAO;YACL,SAAS,EAAE,iBAAiB;YAC5B,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,cAAc;IACd,IACE,YAAY,CAAC,QAAQ,CAAC,aAAa,CAAC;QACpC,YAAY,CAAC,QAAQ,CAAC,2BAA2B,CAAC;QAClD,YAAY,CAAC,QAAQ,CAAC,0BAA0B,CAAC,EACjD;QACA,OAAO;YACL,SAAS,EAAE,WAAW;YACtB,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,oBAAoB;IACpB,IACE,YAAY,CAAC,QAAQ,CAAC,mBAAmB,CAAC;QAC1C,YAAY,CAAC,QAAQ,CAAC,iCAAiC,CAAC;QACxD,YAAY,CAAC,QAAQ,CAAC,kCAAkC,CAAC,EACzD;QACA,OAAO;YACL,SAAS,EAAE,iBAAiB;YAC5B,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,mBAAmB;IACnB,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,IAAI,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE;QAC7G,OAAO;YACL,SAAS,EAAE,gBAAgB;YAC3B,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,uCAAuC;IACvC,OAAO;QACL,SAAS,EAAE,cAAc;QACzB,YAAY,EAAE,KAAK,CAAC,OAAO;KAC5B,CAAC;AACJ,CAAC;AACD,SAAS,eAAe,CAAC,KAAY;IACnC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAEjD,gBAAgB;IAChB,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,IAAI,YAAY,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE;QACzE,OAAO;YACL,SAAS,EAAE,aAAa;YACxB,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,mBAAmB;IACnB,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,IAAI,YAAY,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE;QAC/E,OAAO;YACL,SAAS,EAAE,gBAAgB;YAC3B,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,cAAc;IACd,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE;QACrE,OAAO;YACL,SAAS,EAAE,WAAW;YACtB,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,eAAe;IACf,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,YAAY,CAAC,QAAQ,CAAC,aAAa,CAAC,EAAE;QACvE,OAAO;YACL,SAAS,EAAE,YAAY;YACvB,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IAED,uCAAuC;IACvC,OAAO;QACL,SAAS,EAAE,cAAc;QACzB,YAAY,EAAE,KAAK,CAAC,OAAO;KAC5B,CAAC;AACJ,CAAC;AACD,MAAM,qBAAqB,GAAG,CAAC,KAAY,EAAE,IAAS,EAAuB,EAAE;IAC7E,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,CAAC;IAClC,IAAI,CAAC,SAAS,EAAE;QACd,OAAO;YACL,SAAS,EAAE,cAAc;YACzB,YAAY,EAAE,KAAK,CAAC,OAAO;SAC5B,CAAC;KACH;IACD,IAAI,SAAS,CAAC,OAAO,KAAK,KAAK,EAAE;QAC/B,OAAO;YACL,SAAS,EAAE,iBAAiB;YAC5B,YAAY,EAAE,SAAS,CAAC,SAAS;SAClC,CAAC;KACH;IACD,IAAI,SAAS,CAAC,OAAO,KAAK,KAAK,EAAE;QAC/B,OAAO;YACL,SAAS,EAAE,mBAAmB;YAC9B,YAAY,EAAE,SAAS,CAAC,SAAS;SAClC,CAAC;KACH;IACD,IAAI,SAAS,CAAC,YAAY,EAAE;QAC1B,OAAO;YACL,SAAS,EAAE,mBAAmB;YAC9B,YAAY,EAAE,SAAS,CAAC,SAAS;SAClC,CAAC;KACH;IACD,IAAI,SAAS,CAAC,cAAc,EAAE;QAC5B,OAAO;YACL,SAAS,EAAE,qBAAqB;YAChC,YAAY,EAAE,SAAS,CAAC,SAAS;SAClC,CAAC;KACH;IACD,IAAI,SAAS,CAAC,eAAe,EAAE;QAC7B,OAAO;YACL,SAAS,EAAE,sBAAsB;YACjC,YAAY,EAAE,SAAS,CAAC,SAAS;SAClC,CAAC;KACH;IACD,IAAI,SAAS,CAAC,aAAa,EAAE;QAC3B,OAAO;YACL,SAAS,EAAE,4BAA4B;YACvC,YAAY,EAAE,SAAS,CAAC,SAAS,IAAI,SAAS,SAAS,CAAC,KAAK,WAAW;SACzE,CAAC;KACH;IACD,OAAO;QACL,SAAS,EAAE,cAAc;QACzB,YAAY,EAAE,KAAK,CAAC,OAAO;KAC5B,CAAC;AACJ,CAAC,CAAC;AACF,MAAM,4BAA4B,GAAG,CAAC,KAAY,EAAE,IAAS,EAAuB,EAAE;IACpF,aAAa;IACb,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;QAC3C,aAAa;QACb,OAAO,4BAA4B,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;KAC5D;IACD,IAAI,mBAAmB,GAAG,qBAAqB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC7D,IAAI,mBAAmB,CAAC,SAAS,KAAK,cAAc,EAAE;QACpD,mBAAmB,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;KACtD;IACD,IAAI,mBAAmB,CAAC,SAAS,KAAK,cAAc,EAAE;QACpD,mBAAmB,GAAG,eAAe,CAAC,KAAK,CAAC,CAAC;KAC9C;IACD,OAAO,mBAAmB,CAAC;AAC7B,CAAC,CAAC;AACF,OAAO,EAAE,4BAA4B,EAAE,CAAC"}

package/lib/generation_scripts.d.ts

@@ -0,0 +1,4 @@
+export interface InitScripts {
+    recorderCjs: string | null;
+    scripts: string[] | null;
+}

package/lib/generation_scripts.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=generation_scripts.js.map

package/lib/generation_scripts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"generation_scripts.js","sourceRoot":"","sources":["../../src/generation_scripts.ts"],"names":[],"mappings":""}

package/lib/index.d.ts

@@ -7,3 +7,4 @@ export * from "./browser_manager.js";
 export * from "./analyze_helper.js";
 export * from "./find_function.js";
 export * from "./utils.js";
+export * from "./table.js";

package/lib/index.js

@@ -7,4 +7,5 @@ export * from "./browser_manager.js";
 export * from "./analyze_helper.js";
 export * from "./find_function.js";
 export * from "./utils.js";
+export * from "./table.js";
 //# sourceMappingURL=index.js.map

package/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC"}

package/lib/init_browser.d.ts

@@ -1,7 +1,9 @@
 import { Environment } from "./environment.js";
 import { TestContext } from "./test_context.js";
+import { StableBrowser } from "./stable_browser.js";
 import { Browser as PlaywrightBrowser } from "playwright";
 import { Browser } from "./browser_manager.js";
-declare const getContext: (environment: Environment | null, headless?: boolean, logger?: null) => Promise<TestContext>;
+import { InitScripts } from "./generation_scripts.js";
+declare const getContext: (environment: Environment | null, headless: boolean | undefined, world: any, logger?: null, appName?: string | null, createStable?: boolean, stable?: StableBrowser | null, moveToRight?: number, reportFolder?: string | null, initScripts?: InitScripts | null) => Promise<TestContext>;
 declare const closeBrowser: (browser?: Browser | PlaywrightBrowser) => Promise<void>;
 export { getContext, closeBrowser };

package/lib/init_browser.js

@@ -7,10 +7,16 @@ import { StableBrowser } from "./stable_browser.js";
 import { Api } from "./api.js";
 // let environment = null;
 // init browser create context and page, if context and page are not null
-const getContext = async function (environment, headless = false, logger) {
+const getContext = async function (environment, headless = false, world, logger, appName, createStable = true, stable = null, moveToRight = -1, reportFolder = null, initScripts = null) {
     if (environment === null) {
         environment = initEnvironment();
     }
+    if (appName && !environment.apps && !environment.apps[appName]) {
+        throw new Error(`App ${appName} not found in environment`);
+    }
+    if (appName) {
+        environment = environment.apps[appName];
+    }
     const { cookies, origins } = environment;
     if (cookies) {
         for (let i = 0; i < cookies.length; i++) {
@@ -22,28 +28,85 @@ const getContext = async function (environment, headless = false, logger) {
     }
     let extensionPath = undefined;
     let userDataDirPath = undefined;
+    let userAgent = undefined;
     let aiConfigFile = "ai_config.json";
+    let channel = undefined;
     if (process.env.PROJECT_PATH) {
         aiConfigFile = path.join(process.env.PROJECT_PATH, "ai_config.json");
     }
+    let configuration = {};
     if (fs.existsSync(aiConfigFile)) {
-        const configuration = JSON.parse(fs.readFileSync(aiConfigFile, "utf8"));
+        configuration = JSON.parse(fs.readFileSync(aiConfigFile, "utf8"));
         if (configuration.userDataDirPath) {
             userDataDirPath = configuration.userDataDirPath;
         }
         if (configuration.extensionPath) {
             extensionPath = configuration.extensionPath;
         }
+        if (configuration.useGoogleChrome === true) {
+            channel = "chrome";
+        }
+        else if (configuration.useMicrosoftEdge === true) {
+            channel = "msedge";
+        }
+        if (configuration.overrideUserAgent) {
+            userAgent = configuration.overrideUserAgent;
+        }
     }
     const storageState = { cookies, origins };
-    let browser = await browserManager.getBrowser(headless, storageState, extensionPath, userDataDirPath);
+    let downloadsPath = "downloads";
+    if (reportFolder) {
+        downloadsPath = path.join(reportFolder, "downloads");
+    }
+    else if (stable && stable.context && stable.context.reportFolder) {
+        reportFolder = stable.context.reportFolder;
+        downloadsPath = path.join(stable.context.reportFolder, "downloads");
+    }
+    if (world) {
+        world.downloadsPath = downloadsPath;
+    }
+    if (stable && stable.context) {
+        stable.context.downloadsPath = downloadsPath;
+    }
+    let browser = await browserManager.createBrowser(headless, storageState, extensionPath, userDataDirPath, reportFolder ? reportFolder : ".", userAgent, channel, configuration, initScripts);
     let context = new TestContext();
     context.browser = browser.browser;
+    context.browserObject = browser;
     context.playContext = browser.context;
     context.page = browser.page;
     context.environment = environment;
-    context.stable = new StableBrowser(context.browser, context.page, logger, context);
+    context.browserName = browser.browser ? browser.browser.browserType().name() : "unknown";
+    context.reportFolder = reportFolder;
+    if (createStable) {
+        context.stable = new StableBrowser(context.browser, context.page, logger, context, world);
+    }
+    else {
+        context.stable = stable;
+    }
     context.api = new Api(logger);
+    if (moveToRight > 0 && context.browserName === "chromium") {
+        // move the borwser to the top right corner of the screen
+        // create a cdp session
+        // Get CDP session
+        const playContext = context.playContext;
+        const client = await playContext.newCDPSession(context.page);
+        // Get window ID for the current target
+        const { windowId } = await client.send("Browser.getWindowForTarget");
+        //console.log(windowId);
+        // get the window for the current target
+        const window = await client.send("Browser.getWindowBounds", {
+            windowId,
+        });
+        //console.log(window);
+        await client.send("Browser.setWindowBounds", {
+            windowId,
+            bounds: {
+                left: window.bounds.left + moveToRight,
+            },
+        });
+        // close cdp
+        await client.detach();
+    }
     // await _initCookies(context);
     return context;
 };

package/lib/init_browser.js.map

@@ -1 +1 @@
-{"version":3,"file":"init_browser.js","sourceRoot":"","sources":["../../src/init_browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAE/B,0BAA0B;AAE1B,yEAAyE;AACzE,MAAM,UAAU,GAAG,KAAK,WAAW,WAA+B,EAAE,QAAQ,GAAG,KAAK,EAAE,MAAa;IACjG,IAAI,WAAW,KAAK,IAAI,EAAE;QACxB,WAAW,GAAG,eAAe,EAAE,CAAC;KACjC;IACD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,WAAW,CAAC;IACzC,IAAI,OAAO,EAAE;QACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,MAAM,CAAC,OAAO,KAAK,WAAW,EAAE;gBAClC,OAAO,MAAM,CAAC,OAAO,CAAC;aACvB;SACF;KACF;IACD,IAAI,aAAa,GAAG,SAAS,CAAC;IAC9B,IAAI,eAAe,GAAG,SAAS,CAAC;IAChC,IAAI,YAAY,GAAG,gBAAgB,CAAC;IACpC,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE;QAC5B,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;KACtE;IACD,IAAI,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QAC/B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;QACxE,IAAI,aAAa,CAAC,eAAe,EAAE;YACjC,eAAe,GAAG,aAAa,CAAC,eAAe,CAAC;SACjD;QACD,IAAI,aAAa,CAAC,aAAa,EAAE;YAC/B,aAAa,GAAG,aAAa,CAAC,aAAa,CAAC;SAC7C;KACF;IACD,MAAM,YAAY,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC1C,IAAI,OAAO,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,eAAe,CAAC,CAAC;IACtG,IAAI,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAChC,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAClC,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;IACtC,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5B,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;IAElC,OAAO,CAAC,MAAM,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,OAAQ,EAAE,OAAO,CAAC,IAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;IACrF,OAAO,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,+BAA+B;IAC/B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,KAAK,WAAW,OAAqC;IACxE,MAAM,cAAc,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG;IACtB,8BAA8B;IAC9B,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IACtC,IAAI;QACF,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,cAAc,EAAE,CAAC;QACnC,IAAI,SAAS,EAAE;YACb,OAAO,GAAG,SAAS,CAAC;SACrB;aAAM,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE;YAChC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;SACjC;aAAM,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC,EAAE;YAC9D,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;SAChD;aAAM;YACL,IAAI,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;YACrE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACrC,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACpB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;oBAC1B,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;oBACzD,MAAM;iBACP;aACF;SACF;QACD,qCAAqC;QACrC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9C,4BAA4B;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,sCAAsC;QACtC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACtC,kCAAkC;QAClC,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;KACjD;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;KAC9C;IACD,IAAI;IACJ,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC;AACF,MAAM,cAAc,GAAG;IACrB,KAAK,IAAI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QACrC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;YACxB,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE;gBAC1B,OAAO,KAAK,CAAC;aACd;SACF;KACF;IACD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AACF,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC"}
+{"version":3,"file":"init_browser.js","sourceRoot":"","sources":["../../src/init_browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAG/B,0BAA0B;AAE1B,yEAAyE;AACzE,MAAM,UAAU,GAAG,KAAK,WACtB,WAA+B,EAC/B,QAAQ,GAAG,KAAK,EAChB,KAAU,EACV,MAAa,EACb,OAAuB,EACvB,YAAY,GAAG,IAAI,EACnB,SAA+B,IAAI,EACnC,WAAW,GAAG,CAAC,CAAC,EAChB,eAA8B,IAAI,EAClC,cAAkC,IAAI;IAEtC,IAAI,WAAW,KAAK,IAAI,EAAE;QACxB,WAAW,GAAG,eAAe,EAAE,CAAC;KACjC;IACD,IAAI,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;QAC9D,MAAM,IAAI,KAAK,CAAC,OAAO,OAAO,2BAA2B,CAAC,CAAC;KAC5D;IACD,IAAI,OAAO,EAAE;QACX,WAAW,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;KACzC;IACD,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,WAAW,CAAC;IACzC,IAAI,OAAO,EAAE;QACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACvC,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,MAAM,CAAC,OAAO,KAAK,WAAW,EAAE;gBAClC,OAAO,MAAM,CAAC,OAAO,CAAC;aACvB;SACF;KACF;IACD,IAAI,aAAa,GAAG,SAAS,CAAC;IAC9B,IAAI,eAAe,GAAG,SAAS,CAAC;IAChC,IAAI,SAAS,GAAG,SAAS,CAAC;IAC1B,IAAI,YAAY,GAAG,gBAAgB,CAAC;IACpC,IAAI,OAAO,GAAG,SAAS,CAAC;IACxB,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE;QAC5B,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;KACtE;IACD,IAAI,aAAa,GAAQ,EAAE,CAAC;IAC5B,IAAI,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE;QAC/B,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC;QAClE,IAAI,aAAa,CAAC,eAAe,EAAE;YACjC,eAAe,GAAG,aAAa,CAAC,eAAe,CAAC;SACjD;QACD,IAAI,aAAa,CAAC,aAAa,EAAE;YAC/B,aAAa,GAAG,aAAa,CAAC,aAAa,CAAC;SAC7C;QAED,IAAI,aAAa,CAAC,eAAe,KAAK,IAAI,EAAE;YAC1C,OAAO,GAAG,QAAQ,CAAC;SACpB;aAAM,IAAI,aAAa,CAAC,gBAAgB,KAAK,IAAI,EAAE;YAClD,OAAO,GAAG,QAAQ,CAAC;SACpB;QAED,IAAI,aAAa,CAAC,iBAAiB,EAAE;YACnC,SAAS,GAAG,aAAa,CAAC,iBAAiB,CAAC;SAC7C;KACF;IACD,MAAM,YAAY,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC1C,IAAI,aAAa,GAAG,WAAW,CAAC;IAChC,IAAI,YAAY,EAAE;QAChB,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;KACtD;SAAM,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE;QAClE,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;QAC3C,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;KACrE;IACD,IAAI,KAAK,EAAE;QACT,KAAK,CAAC,aAAa,GAAG,aAAa,CAAC;KACrC;IACD,IAAI,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE;QAC5B,MAAM,CAAC,OAAO,CAAC,aAAa,GAAG,aAAa,CAAC;KAC9C;IACD,IAAI,OAAO,GAAG,MAAM,cAAc,CAAC,aAAa,CAC9C,QAAQ,EACR,YAAY,EACZ,aAAa,EACb,eAAe,EACf,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,EACjC,SAAS,EACT,OAAO,EACP,aAAa,EACb,WAAW,CACZ,CAAC;IACF,IAAI,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAChC,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAClC,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC;IAChC,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;IACtC,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5B,OAAO,CAAC,WAAW,GAAG,WAAW,CAAC;IAClC,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACzF,OAAO,CAAC,YAAY,GAAG,YAAY,CAAC;IAEpC,IAAI,YAAY,EAAE;QAChB,OAAO,CAAC,MAAM,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,OAAQ,EAAE,OAAO,CAAC,IAAK,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;KAC7F;SAAM;QACL,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;KACzB;IACD,OAAO,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,WAAW,GAAG,CAAC,IAAI,OAAO,CAAC,WAAW,KAAK,UAAU,EAAE;QACzD,yDAAyD;QACzD,uBAAuB;QACvB,kBAAkB;QAClB,MAAM,WAAW,GAAQ,OAAO,CAAC,WAAW,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAE7D,uCAAuC;QACvC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QACrE,wBAAwB;QAExB,wCAAwC;QACxC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;YAC1D,QAAQ;SACT,CAAC,CAAC;QACH,sBAAsB;QACtB,MAAM,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;YAC3C,QAAQ;YACR,MAAM,EAAE;gBACN,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,WAAW;aACvC;SACF,CAAC,CAAC;QACH,YAAY;QACZ,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;KACvB;IAED,+BAA+B;IAC/B,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,KAAK,WAAW,OAAqC;IACxE,MAAM,cAAc,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF,MAAM,eAAe,GAAG;IACtB,8BAA8B;IAC9B,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;IACtC,IAAI;QACF,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,cAAc,EAAE,CAAC;QACnC,IAAI,SAAS,EAAE;YACb,OAAO,GAAG,SAAS,CAAC;SACrB;aAAM,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE;YAChC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;SACjC;aAAM,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC,EAAE;YAC9D,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;SAChD;aAAM;YACL,IAAI,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC,CAAC;YACrE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACrC,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACpB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE;oBAC1B,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,IAAI,CAAC,CAAC;oBACzD,MAAM;iBACP;aACF;SACF;QACD,qCAAqC;QACrC,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9C,4BAA4B;QAC5B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,sCAAsC;QACtC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QACtC,kCAAkC;QAClC,OAAO,CAAC,GAAG,CAAC,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;KACjD;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;KAC9C;IACD,IAAI;IACJ,OAAO,WAAW,CAAC;AACrB,CAAC,CAAC;AACF,MAAM,cAAc,GAAG;IACrB,KAAK,IAAI,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;QACrC,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;YACxB,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACpC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE;gBAC1B,OAAO,KAAK,CAAC;aACd;SACF;KACF;IACD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AACF,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC"}

package/lib/locate_element.d.ts

@@ -0,0 +1,7 @@
+export declare function locate_element(context: any, elementDescription: string, operation?: string, value?: string | null): Promise<{
+    reason: any;
+    elementNumber: any;
+    frameIndex: number;
+    frame: null;
+    css: string;
+} | null>;

package/lib/locate_element.js

@@ -0,0 +1,215 @@
+import axios from "axios";
+import * as path from "path";
+import { fileURLToPath } from "url";
+import { _getServerUrl } from "./utils.js";
+// Get __filename and __dirname in ESM
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+export async function locate_element(context, elementDescription, operation = "click", value = null) {
+    // load the axe-core library to all of the frames in the page
+    // read the axe-core library from the file system placed in the same folder as the script file, name axe.mini.js
+    // Construct the path to axe.min.js relative to the current file
+    // let axeMinJsPath = path.join(__dirname, "..", "scripts", "axe.mini.js");
+    // // Check if the file exists
+    // if (!fs.existsSync(axeMinJsPath)) {
+    //   axeMinJsPath = path.join(__dirname, "scripts", "axe.mini.js");
+    // }
+    // // Read the content of axe.min.js synchronously
+    // const axeMinJsContent = fs.readFileSync(axeMinJsPath, "utf-8");
+    // await Promise.all(context.stable.page.frames().map((frame: any) => frame.evaluate(axeMinJsContent)));
+    const frames = await context.stable.page.frames();
+    // for each frame create a tree of the accessibility nodes
+    const frameDump = [];
+    let iframeIndex = 0;
+    let nextFrameIndex = 1;
+    let actionableElementIndex = 0;
+    const randomToken = Math.random().toString(36).substring(7);
+    const actionableElements = [];
+    for (let frame of frames) {
+        // @ts-ignore
+        let result = null;
+        try {
+            result = await frame.evaluate(
+            // @ts-ignore
+            ([iframeIndex, nextFrameIndex, actionableElementIndex, randomToken]) => {
+                let iframeCount = 1;
+                const actionableElements = [];
+                const actionableRoles = [
+                    "link",
+                    "button",
+                    "tab",
+                    "menuitem",
+                    "menuitemcheckbox",
+                    "menuitemradio",
+                    "checkbox",
+                    "textbox",
+                    "combobox",
+                    "listitem",
+                    "radio",
+                    "searchbox",
+                    "option",
+                    "Date",
+                    "row",
+                    "DateTime",
+                    "InputTime",
+                    "treeitem",
+                ];
+                function isAccessibilityElement(element) {
+                    if (!element) {
+                        return false;
+                    }
+                    // @ts-ignore
+                    let hidden = axe.commons.dom.isHiddenForEveryone(element);
+                    if (hidden) {
+                        return false;
+                    }
+                    if (element.tagName === "IFRAME") {
+                        return true;
+                    }
+                    // @ts-ignore
+                    let roles = axe.commons.aria.getRole(element);
+                    if (!roles) {
+                        return false;
+                    }
+                    return true;
+                }
+                function getAttributes(element) {
+                    const attrs = element.attributes;
+                    const result = {};
+                    for (let i = 0; i < attrs.length; i++) {
+                        result[attrs[i].name] = attrs[i].value;
+                        if (attrs[i].name === "class") {
+                            // clean the spaces
+                            result[attrs[i].name] = attrs[i].value.replace(/\s+/g, " ").trim();
+                        }
+                    }
+                    return result;
+                }
+                function createAccessibilityNode(element, actionableElements) {
+                    // check if the element is an iframe
+                    const result = {
+                        tag: element.tagName,
+                        // @ts-ignore
+                        role: axe.commons.aria.getRole(element),
+                        // @ts-ignore
+                        visible: axe.commons.dom.isVisible(element),
+                        // @ts-ignore
+                        name: axe.commons.text.accessibleText(element),
+                        attributes: getAttributes(element),
+                        // @ts-ignore
+                        frameIndex: iframeIndex,
+                    };
+                    if (actionableRoles.includes(result.role)) {
+                        // @ts-ignore
+                        result.elementNumber = actionableElementIndex;
+                        element.setAttribute("data-blinq-id", "blinq-id-" + randomToken + "-" + actionableElementIndex);
+                        //console.log("attribute set to " + "blinq-id-" + randomToken + "-" + actionableElementIndex);
+                        actionableElementIndex++;
+                        actionableElements.push(result);
+                    }
+                    if (element.tagName === "IFRAME") {
+                        // @ts-ignore
+                        result.targetFrameIndex = nextFrameIndex;
+                        nextFrameIndex++;
+                    }
+                    return result;
+                }
+                // a function that traverses the DOM tree and builds a tree of accessibility nodes
+                function buildAccessibilityTree(node, treeRoot) {
+                    //const foundAccessibilityNodes = [];
+                    if (!treeRoot.children) {
+                        treeRoot.children = [];
+                    }
+                    //console.log(node.tagName + " " + node.children.length);
+                    for (let child of node.children) {
+                        if (isAccessibilityElement(child)) {
+                            const accessibilityNode = createAccessibilityNode(child, actionableElements);
+                            // @ts-ignore
+                            // accessibilityNode.children = buildAccessibilityTree(child, accessibilityNode),
+                            // foundAccessibilityNodes.push(accessibilityNode);
+                            treeRoot.children.push(accessibilityNode);
+                            buildAccessibilityTree(child, accessibilityNode);
+                        }
+                        else {
+                            buildAccessibilityTree(child, treeRoot);
+                        }
+                    }
+                }
+                // @ts-ignore
+                axe.utils.getFlattenedTree(document);
+                const root = createAccessibilityNode(document.body, actionableElements);
+                buildAccessibilityTree(document.body, root);
+                return JSON.stringify({ root, nextFrameIndex, actionableElementIndex, actionableElements }, null, 2);
+            }, [iframeIndex, nextFrameIndex, actionableElementIndex, randomToken]);
+            const resultData = JSON.parse(result);
+            nextFrameIndex = resultData.nextFrameIndex;
+            actionableElementIndex = resultData.actionableElementIndex;
+            frameDump.push(resultData.root);
+            actionableElements.push(...resultData.actionableElements);
+            iframeIndex++;
+        }
+        catch (e) {
+            // ignore
+        }
+    }
+    function traverseDFS(node) {
+        // Recursively traverse each child node
+        if (node.children && node.children.length > 0) {
+            node.children.forEach((child) => {
+                if (child.tag === "IFRAME") {
+                    if (child.targetFrameIndex && frameDump.length >= child.targetFrameIndex) {
+                        child.children = [frameDump[child.targetFrameIndex]];
+                    }
+                }
+                if (child.shadowRoot) {
+                    traverseDFS(child.shadowRoot);
+                }
+                else {
+                    traverseDFS(child);
+                }
+            });
+        }
+    }
+    traverseDFS(frameDump[0]);
+    let serviceUrl = _getServerUrl();
+    const config = {
+        method: "post",
+        url: `${serviceUrl}/api/runs/locate-element/locate`,
+        headers: {
+            "x-source": "true",
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${process.env.TOKEN}`,
+        },
+        data: JSON.stringify({
+            elementDescription: elementDescription,
+            operation: operation,
+            dump: frameDump[0],
+            value: value,
+        }),
+    };
+    let result = await axios.request(config);
+    //console.log(JSON.stringify(frameDump[0]));
+    if (result.status !== 200 || !result.data || result.data.status !== true || !result.data.result) {
+        console.error("Failed to locate element");
+        return null;
+    }
+    const returnData = {
+        reason: result.data.result.reason,
+        elementNumber: result.data.result.elementNumber,
+        frameIndex: -1,
+        frame: null,
+        css: "",
+    };
+    if (result.data.result.elementNumber !== -1) {
+        // find the iframe index, the element is in, by identifying the actionable element
+        const actionableElement = actionableElements.find((element) => element.elementNumber === result.data.result.elementNumber);
+        if (actionableElement) {
+            returnData.frameIndex = actionableElement.frameIndex;
+            returnData.frame = frames[returnData.frameIndex];
+            returnData.css = `[data-blinq-id="blinq-id-${randomToken}-${returnData.elementNumber}"]`;
+        }
+    }
+    //console.log(dumpToHtml(frameDump[0]));
+    return returnData;
+}
+//# sourceMappingURL=locate_element.js.map

package/lib/locate_element.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"locate_element.js","sourceRoot":"","sources":["../../src/locate_element.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,sCAAsC;AACtC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,OAAY,EACZ,kBAA0B,EAC1B,SAAS,GAAG,OAAO,EACnB,QAAuB,IAAI;IAE3B,6DAA6D;IAC7D,gHAAgH;IAChH,gEAAgE;IAChE,2EAA2E;IAC3E,8BAA8B;IAC9B,sCAAsC;IACtC,mEAAmE;IACnE,IAAI;IAEJ,kDAAkD;IAClD,kEAAkE;IAClE,wGAAwG;IAExG,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;IAClD,0DAA0D;IAC1D,MAAM,SAAS,GAAU,EAAE,CAAC;IAC5B,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,sBAAsB,GAAG,CAAC,CAAC;IAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,kBAAkB,GAAU,EAAE,CAAC;IACrC,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE;QACxB,aAAa;QACb,IAAI,MAAM,GAAQ,IAAI,CAAC;QACvB,IAAI;YACF,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ;YAC3B,aAAa;YACb,CAAC,CAAC,WAAW,EAAE,cAAc,EAAE,sBAAsB,EAAE,WAAW,CAAC,EAAE,EAAE;gBACrE,IAAI,WAAW,GAAG,CAAC,CAAC;gBACpB,MAAM,kBAAkB,GAAU,EAAE,CAAC;gBACrC,MAAM,eAAe,GAAG;oBACtB,MAAM;oBACN,QAAQ;oBACR,KAAK;oBACL,UAAU;oBACV,kBAAkB;oBAClB,eAAe;oBACf,UAAU;oBACV,SAAS;oBACT,UAAU;oBACV,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,QAAQ;oBACR,MAAM;oBACN,KAAK;oBACL,UAAU;oBACV,WAAW;oBACX,UAAU;iBACX,CAAC;gBAEF,SAAS,sBAAsB,CAAC,OAAY;oBAC1C,IAAI,CAAC,OAAO,EAAE;wBACZ,OAAO,KAAK,CAAC;qBACd;oBACD,aAAa;oBACb,IAAI,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;oBAC1D,IAAI,MAAM,EAAE;wBACV,OAAO,KAAK,CAAC;qBACd;oBACD,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE;wBAChC,OAAO,IAAI,CAAC;qBACb;oBACD,aAAa;oBACb,IAAI,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;oBAC9C,IAAI,CAAC,KAAK,EAAE;wBACV,OAAO,KAAK,CAAC;qBACd;oBACD,OAAO,IAAI,CAAC;gBACd,CAAC;gBACD,SAAS,aAAa,CAAC,OAAY;oBACjC,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC;oBACjC,MAAM,MAAM,GAAQ,EAAE,CAAC;oBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;wBACrC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;wBACvC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,EAAE;4BAC7B,mBAAmB;4BACnB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;yBACpE;qBACF;oBACD,OAAO,MAAM,CAAC;gBAChB,CAAC;gBACD,SAAS,uBAAuB,CAAC,OAAY,EAAE,kBAAuB;oBACpE,oCAAoC;oBAEpC,MAAM,MAAM,GAAG;wBACb,GAAG,EAAE,OAAO,CAAC,OAAO;wBACpB,aAAa;wBACb,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;wBACvC,aAAa;wBACb,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC;wBAC3C,aAAa;wBACb,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC;wBAC9C,UAAU,EAAE,aAAa,CAAC,OAAO,CAAC;wBAClC,aAAa;wBACb,UAAU,EAAE,WAAW;qBACxB,CAAC;oBACF,IAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;wBACzC,aAAa;wBACb,MAAM,CAAC,aAAa,GAAG,sBAAsB,CAAC;wBAC9C,OAAO,CAAC,YAAY,CAAC,eAAe,EAAE,WAAW,GAAG,WAAW,GAAG,GAAG,GAAG,sBAAsB,CAAC,CAAC;wBAChG,8FAA8F;wBAC9F,sBAAsB,EAAE,CAAC;wBACzB,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;qBACjC;oBACD,IAAI,OAAO,CAAC,OAAO,KAAK,QAAQ,EAAE;wBAChC,aAAa;wBACb,MAAM,CAAC,gBAAgB,GAAG,cAAc,CAAC;wBACzC,cAAc,EAAE,CAAC;qBAClB;oBAED,OAAO,MAAM,CAAC;gBAChB,CAAC;gBACD,kFAAkF;gBAClF,SAAS,sBAAsB,CAAC,IAAS,EAAE,QAAa;oBACtD,qCAAqC;oBACrC,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE;wBACtB,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC;qBACxB;oBACD,yDAAyD;oBACzD,KAAK,IAAI,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE;wBAC/B,IAAI,sBAAsB,CAAC,KAAK,CAAC,EAAE;4BACjC,MAAM,iBAAiB,GAAG,uBAAuB,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAC;4BAC7E,aAAa;4BACb,iFAAiF;4BACjF,mDAAmD;4BACnD,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;4BAC1C,sBAAsB,CAAC,KAAK,EAAE,iBAAiB,CAAC,CAAC;yBAClD;6BAAM;4BACL,sBAAsB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;yBACzC;qBACF;gBACH,CAAC;gBACD,aAAa;gBACb,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;gBACrC,MAAM,IAAI,GAAG,uBAAuB,CAAC,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;gBACxE,sBAAsB,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACvG,CAAC,EACD,CAAC,WAAW,EAAE,cAAc,EAAE,sBAAsB,EAAE,WAAW,CAAC,CACnE,CAAC;YACF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACtC,cAAc,GAAG,UAAU,CAAC,cAAc,CAAC;YAC3C,sBAAsB,GAAG,UAAU,CAAC,sBAAsB,CAAC;YAC3D,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YAChC,kBAAkB,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,kBAAkB,CAAC,CAAC;YAC1D,WAAW,EAAE,CAAC;SACf;QAAC,OAAO,CAAC,EAAE;YACV,SAAS;SACV;KACF;IACD,SAAS,WAAW,CAAC,IAAS;QAC5B,uCAAuC;QACvC,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YAC7C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAU,EAAE,EAAE;gBACnC,IAAI,KAAK,CAAC,GAAG,KAAK,QAAQ,EAAE;oBAC1B,IAAI,KAAK,CAAC,gBAAgB,IAAI,SAAS,CAAC,MAAM,IAAI,KAAK,CAAC,gBAAgB,EAAE;wBACxE,KAAK,CAAC,QAAQ,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;qBACtD;iBACF;gBACD,IAAI,KAAK,CAAC,UAAU,EAAE;oBACpB,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;iBAC/B;qBAAM;oBACL,WAAW,CAAC,KAAK,CAAC,CAAC;iBACpB;YACH,CAAC,CAAC,CAAC;SACJ;IACH,CAAC;IACD,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,UAAU,GAAG,aAAa,EAAE,CAAC;IACjC,MAAM,MAAM,GAAG;QACb,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG,UAAU,iCAAiC;QACnD,OAAO,EAAE;YACP,UAAU,EAAE,MAAM;YAClB,cAAc,EAAE,kBAAkB;YAClC,aAAa,EAAE,UAAU,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE;SAC7C;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,kBAAkB,EAAE,kBAAkB;YACtC,SAAS,EAAE,SAAS;YACpB,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;YAClB,KAAK,EAAE,KAAK;SACb,CAAC;KACH,CAAC;IAEF,IAAI,MAAM,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACzC,4CAA4C;IAC5C,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE;QAC/F,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC1C,OAAO,IAAI,CAAC;KACb;IACD,MAAM,UAAU,GAAG;QACjB,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM;QACjC,aAAa,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa;QAC/C,UAAU,EAAE,CAAC,CAAC;QACd,KAAK,EAAE,IAAI;QACX,GAAG,EAAE,EAAE;KACR,CAAC;IACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,KAAK,CAAC,CAAC,EAAE;QAC3C,kFAAkF;QAClF,MAAM,iBAAiB,GAAG,kBAAkB,CAAC,IAAI,CAC/C,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,aAAa,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CACxE,CAAC;QACF,IAAI,iBAAiB,EAAE;YACrB,UAAU,CAAC,UAAU,GAAG,iBAAiB,CAAC,UAAU,CAAC;YACrD,UAAU,CAAC,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;YACjD,UAAU,CAAC,GAAG,GAAG,4BAA4B,WAAW,IAAI,UAAU,CAAC,aAAa,IAAI,CAAC;SAC1F;KACF;IACD,wCAAwC;IACxC,OAAO,UAAU,CAAC;AACpB,CAAC"}