ff-automationv2 2.2.33 → 2.2.34-beta.0

package/dist/automation/actions/interaction/clear/clear.js

@@ -9,7 +9,10 @@ async function clear(args) {
         case "android":
         case "ios":
             try {
-                const { ffNumber, xpathPath } = (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
+                const isWebView = args.pageDOM?.includes("ff-inspect");
+                const { ffNumber, xpathPath } = isWebView
+                    ? (0, enterActionHelper_js_1.enterActionHelper)(args.pageDOM, args.fireflinkIndex)
+                    : (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
                 await args.driver.$(xpathPath).clearValue();
                 const ffElementNumber = `Fire-Flink-${ffNumber}`;
                 (0, appendScriptData_js_1.appendScriptData)({

package/dist/automation/actions/interaction/clear/clearAndEnter.js

@@ -10,7 +10,10 @@ async function clearAndEnter(args) {
         case "android":
         case "ios":
             try {
-                const { ffNumber, xpathPath } = (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
+                const isWebView = args.pageDOM?.includes("ff-inspect");
+                const { ffNumber, xpathPath } = isWebView
+                    ? (0, enterActionHelper_js_1.enterActionHelper)(args.pageDOM, args.fireflinkIndex)
+                    : (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
                 const element = args.driver.$(xpathPath);
                 await element.clearValue();
                 await element.setValue(args.value);

package/dist/automation/actions/interaction/elementlessActions/getOtp.js

@@ -46,7 +46,7 @@ async function GetOtpAction(args) {
             (0, appendScriptData_js_1.appendScriptData)({
                 scriptDataAppender: args.scriptDataAppender,
                 nlpName: "MOB_FetchOtpFromMessages",
-                stepInputs: [existingSms.address, timeFrom + "s", waitTime],
+                stepInputs: [existingSms.address, timeFrom + "s", waitTime, _regex],
                 platform: "android",
                 returnTypeVariableName: args.returnTypeVariableName
             });
@@ -84,7 +84,7 @@ async function GetOtpAction(args) {
                 (0, appendScriptData_js_1.appendScriptData)({
                     scriptDataAppender: args.scriptDataAppender,
                     nlpName: "MOB_FetchOtpFromMessages",
-                    stepInputs: [latestSms.address, timeFrom + "s", waitTime],
+                    stepInputs: [latestSms.address, timeFrom + "s", waitTime, regex],
                     platform: "android",
                     returnTypeVariableName: args.returnTypeVariableName
                 });

package/dist/automation/actions/interaction/enterActions/EnterInputIntoElementFromClipBoardInterface_mob.js

@@ -7,7 +7,10 @@ const logData_js_1 = require("../../../../utils/logger/logData.js");
 const saveStepInputData_js_1 = require("../../../../utils/helpers/saveStepInputData.js");
 async function MOB_EnterInputIntoElementFromClipBoard(args) {
     try {
-        const { ffNumber, xpathPath } = (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
+        const isWebView = args.pageDOM?.includes("ff-inspect");
+        const { ffNumber, xpathPath } = isWebView
+            ? (0, enterActionHelper_js_1.enterActionHelper)(args.pageDOM, args.fireflinkIndex)
+            : (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
         const element = await args.driver.$(xpathPath);
         const base64Value = await args.driver.getClipboard('plaintext');
         const value = Buffer.from(base64Value, 'base64').toString('utf-8');

package/dist/automation/actions/interaction/enterActions/enterInput.js

@@ -44,7 +44,10 @@ async function enterInput(args) {
         case "android":
         case "ios":
             try {
-                const { ffNumber, xpathPath } = (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
+                const isWebView = args.pageDOM?.includes("ff-inspect");
+                const { ffNumber, xpathPath } = isWebView
+                    ? (0, enterActionHelper_js_1.enterActionHelper)(args.pageDOM, args.fireflinkIndex)
+                    : (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
                 const element = await args.driver.$(xpathPath);
                 if (args.stepIndex + 1 < args.filteredSteps.length &&
                     args.filteredSteps[args.stepIndex].toLowerCase().includes("enter")) {
@@ -54,7 +57,7 @@ async function enterInput(args) {
                 else {
                     await element.setValue(args.value);
                 }
-                if (args.platform === "android") {
+                if (args.platform === "android" && !isWebView) {
                     try {
                         const text = await element.getAttribute("text");
                         if (text !== args.value) {

package/dist/automation/actions/interaction/enterActions/enterInputAndPress.js

@@ -8,7 +8,10 @@ async function enterInputAndPress(args) {
     switch (args.platform) {
         case "android":
             try {
-                const { ffNumber, xpathPath } = (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
+                const isWebView = args.pageDOM?.includes("ff-inspect");
+                const { ffNumber, xpathPath } = isWebView
+                    ? (0, enterActionHelper_js_1.enterActionHelper)(args.pageDOM, args.fireflinkIndex)
+                    : (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
                 const element = await args.driver.$(xpathPath);
                 const parts = String(args.value).split(",");
                 const inputValue = parts[0]?.trim() || "";

package/dist/automation/actions/interaction/enterActions/waitAndEnter.js

@@ -19,7 +19,10 @@ async function waitAndEnter(args) {
         // ---------------- ANDROID ----------------
         case "android":
             try {
-                const { ffNumber, xpathPath } = (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
+                const isWebView = args.pageDOM?.includes("ff-inspect");
+                const { ffNumber, xpathPath } = isWebView
+                    ? (0, enterActionHelper_js_1.enterActionHelper)(args.pageDOM, args.fireflinkIndex)
+                    : (0, enterActionHelper_js_1.enterActionHelperMob)(args.pageDOM, args.fireflinkIndex);
                 const { inputValue, waitTimeSeconds } = parseValue(args.value);
                 const element = await args.driver.$(xpathPath);
                 // Wait before typing

package/dist/automation/actions/interaction/get/getText.js

@@ -11,8 +11,15 @@ const getText = async (args) => {
         case "ios":
             try {
                 const element = await args.driver.$(args.selector);
-                const attrName = args.platform === "ios" ? "value" : "text";
-                const _text = await element.getAttribute(attrName);
+                const isWebView = args.pageDOM?.includes("ff-inspect");
+                let _text = null;
+                if (isWebView) {
+                    _text = await element.getText();
+                }
+                else {
+                    const attrName = args.platform === "ios" ? "value" : "text";
+                    _text = await element.getAttribute(attrName);
+                }
                 logData_js_1.logger.info("Text", _text);
                 (0, saveStepInputData_js_1.saveStepInputData)(args, _text);
                 (0, appendScriptData_js_1.appendScriptData)({

package/dist/automation/actions/interaction/iframe/switchToDefaultContent.js

@@ -4,7 +4,7 @@ exports.switchToDefaultContent = switchToDefaultContent;
 const appendScriptData_js_1 = require("../../../../fireflinkData/fireflinkScript/appendScriptData.js");
 async function switchToDefaultContent(args) {
     try {
-        await args.driver.switchToFrame(null);
+        await args.driver.switchFrame(null);
         (0, appendScriptData_js_1.appendScriptData)({
             scriptDataAppender: args.scriptDataAppender,
             nlpName: "SwitchToDefaultContent"

package/dist/automation/actions/interaction/verify/VerifyElementNotContainsText.js

@@ -7,7 +7,10 @@ async function verifyElementNotContainsText(args) {
         case "android":
             try {
                 const element = await args.driver.$(args.selector);
-                const expected = await element.getAttribute("text");
+                const isWebView = args.pageDOM?.includes("ff-inspect");
+                const expected = isWebView
+                    ? (await element.getText()) || ""
+                    : (await element.getAttribute("text")) || "";
                 const caseSensitive = false;
                 if (expected == args.value) {
                     throw new Error(`Element do contain the text. Expected: not containing "${args.value}", Actual: contains "${expected}"`);

package/dist/automation/actions/interaction/verify/verifypartialtext.js

@@ -10,8 +10,15 @@ async function verifyPartialText(args) {
         case "ios":
             try {
                 const element = await args.driver.$(args.selector);
-                const attributeName = args.platform === "ios" ? "value" : "text";
-                const actual = (await element.getAttribute(attributeName)) || "";
+                const isWebView = args.pageDOM?.includes("ff-inspect");
+                let actual;
+                if (isWebView) {
+                    actual = (await element.getText()) || "";
+                }
+                else {
+                    const attributeName = args.platform === "ios" ? "value" : "text";
+                    actual = (await element.getAttribute(attributeName)) || "";
+                }
                 const expected = args.value;
                 if (!actual?.includes(expected)) {
                     throw new Error(`Element does not contain the expected text. Expected to contain: "${args.value}", Actual: "${actual}"`);

package/dist/automation/actions/interaction/verify/verifytext.js

@@ -10,8 +10,15 @@ async function verifyText(args) {
         case "ios":
             try {
                 const element = await args.driver.$(args.selector);
-                const attributeName = args.platform === "ios" ? "value" : "text";
-                const text = (await element.getAttribute(attributeName)) || "";
+                const isWebView = args.pageDOM?.includes("ff-inspect");
+                let text;
+                if (isWebView) {
+                    text = (await element.getText()) || "";
+                }
+                else {
+                    const attributeName = args.platform === "ios" ? "value" : "text";
+                    text = (await element.getAttribute(attributeName)) || "";
+                }
                 const expected = (args.value || "").toLowerCase();
                 const isMatched = text.toLowerCase().trim() === expected.trim();
                 logData_js_1.logger.info("text", { text });

package/dist/core/helpers/elementSelectorResolver.d.ts

@@ -3,7 +3,7 @@ import { ShadowIframeResult } from "../interfaces/automationRunnerInterface.js";
 export { ShadowIframeResult };
 export declare function generateUniqueName(currentName: string | undefined, usedNames: string[]): string | undefined;
 export declare function normalizeSvgSelector(selector: string): string;
-export declare function resolveElementSelector(doc: any, domInfo: any, fireflinkIndex: string, platform: PlatformType, driver: any): Promise<{
+export declare function resolveElementSelector(doc: any, domInfo: any, fireflinkIndex: string, platform: PlatformType, driver: any, fallbackAttempt: number, driverContext?: any): Promise<{
     selector: string;
     doc: any;
     domInfo: any;

package/dist/core/helpers/elementSelectorResolver.js

@@ -31,11 +31,11 @@ function normalizeSvgSelector(selector) {
     }
     return selector;
 }
-async function resolveElementSelector(doc, domInfo, fireflinkIndex, platform, driver) {
+async function resolveElementSelector(doc, domInfo, fireflinkIndex, platform, driver, fallbackAttempt, driverContext) {
     let selector;
     let updatedDoc = doc;
     let updatedDomInfo = domInfo;
-    if (platform === "web") {
+    if (platform === "web" || (driverContext && fallbackAttempt > 1)) {
         if (!updatedDoc) {
             logData_js_1.logger.info("doc is undefined. Rebuilding document from domInfo.");
             if (!updatedDomInfo || !updatedDomInfo.dom) {
@@ -57,6 +57,7 @@ async function resolveElementSelector(doc, domInfo, fireflinkIndex, platform, dr
         selector = result.xpathPath;
     }
     selector = normalizeSvgSelector(selector);
+    selector = selector.replace(/\/div\[0\]\/div\[0\]/g, "/div[1]").replace(/\/div\[0\]/g, "/div[1]");
     return { selector, doc: updatedDoc, domInfo: updatedDomInfo };
 }
 async function handleShadowAndIframe(driver, stepAction, selector, fireflinkIndex, domInfo, actionHandlers, elementName, elementType) {

package/dist/core/helpers/runnerDomPipeline.d.ts

@@ -4,6 +4,6 @@ import { DomCaptureResult, DomProcessResult } from "../interfaces/automationRunn
 export { DomCaptureResult, DomProcessResult };
 export declare class RunnerDomPipeline {
     static parseDomDocument(domHtml: string, platform: PlatformType): any;
-    static captureAndParseDom(driver: any, platform: PlatformType, stepCount: number, action: string, isRetry?: boolean): Promise<DomCaptureResult>;
-    static processRelevantDom(domProcessor: DomProcessingEngine, domInfo: any, keywords: any, stepCount: number, action: string, currentStep: string, platform: PlatformType, driver: any, isRetry?: boolean): Promise<DomProcessResult>;
+    static captureAndParseDom(driver: any, platform: PlatformType, stepCount: number, action: string, isRetry?: boolean, driverContext?: any): Promise<DomCaptureResult>;
+    static processRelevantDom(domProcessor: DomProcessingEngine, domInfo: any, keywords: any, stepCount: number, action: string, currentStep: string, platform: PlatformType, driver: any, isRetry?: boolean, driverContext?: any): Promise<DomProcessResult>;
 }

package/dist/core/helpers/runnerDomPipeline.js

@@ -19,7 +19,7 @@ class RunnerDomPipeline {
             return parser.window.document;
         }
     }
-    static async captureAndParseDom(driver, platform, stepCount, action, isRetry = false) {
+    static async captureAndParseDom(driver, platform, stepCount, action, isRetry = false, driverContext) {
         try {
             let domInfo;
             let doc;
@@ -32,12 +32,23 @@ class RunnerDomPipeline {
                 doc = RunnerDomPipeline.parseDomDocument(domInfo.dom, "web");
             }
             else {
-                domInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOMForMobile)(driver);
-                const fileName = isRetry
-                    ? `annotated-dom-Mob-retry-${stepCount}`
-                    : `annotated-dom-Mob-${stepCount}`;
-                logData_js_1.logger.saveDOM(domInfo.dom, fileName);
-                doc = RunnerDomPipeline.parseDomDocument(domInfo.dom, platform);
+                if (driverContext) {
+                    await driver.switchContext(driverContext);
+                    domInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOM)(driver);
+                    const fileName = isRetry
+                        ? `annotated-dom-retry-webview-${stepCount}`
+                        : `annotated-dom-${stepCount}`;
+                    logData_js_1.logger.saveDOM(domInfo.dom, fileName);
+                    doc = RunnerDomPipeline.parseDomDocument(domInfo.dom, "web");
+                }
+                else {
+                    domInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOMForMobile)(driver);
+                    const fileName = isRetry
+                        ? `annotated-dom-Mob-retry-${stepCount}`
+                        : `annotated-dom-Mob-${stepCount}`;
+                    logData_js_1.logger.saveDOM(domInfo.dom, fileName);
+                    doc = RunnerDomPipeline.parseDomDocument(domInfo.dom, platform);
+                }
             }
             return { domInfo, doc };
         }
@@ -48,11 +59,11 @@ class RunnerDomPipeline {
             throw error;
         }
     }
-    static async processRelevantDom(domProcessor, domInfo, keywords, stepCount, action, currentStep, platform, driver, isRetry = false) {
+    static async processRelevantDom(domProcessor, domInfo, keywords, stepCount, action, currentStep, platform, driver, isRetry = false, driverContext) {
         try {
             let extractedRelevantDom;
             let toonDom;
-            if (platform === "web") {
+            if (platform === "web" || driverContext) {
                 extractedRelevantDom = await domProcessor.webProcess({
                     keywords: keywords,
                     rawDom: domInfo?.dom,

package/dist/core/helpers/visionFallbackHandler.js

@@ -10,7 +10,7 @@ const runnerPostActionHelper_js_1 = require("./runnerPostActionHelper.js");
 const runnerDomPipeline_js_1 = require("./runnerDomPipeline.js");
 const logData_js_1 = require("../../utils/logger/logData.js");
 async function executeVisionFallback(params) {
-    const { driver, platform, action, step, stepAction, inputValue, stepCount, fallbackAttempt, priorAndNextSteps, listOfSteps, currentElementName, extractorElementName, stepElementType, iframeDomHtml, beforeHandles, context, stepProcessor, actionHandlers, reporter, causeError, } = params;
+    const { driver, platform, action, step, stepAction, inputValue, stepCount, fallbackAttempt, priorAndNextSteps, listOfSteps, currentElementName, extractorElementName, stepElementType, iframeDomHtml, beforeHandles, context, stepProcessor, actionHandlers, reporter, causeError, driverContext } = params;
     let domInfo = params.domInfo;
     let doc = params.doc;
     logData_js_1.logger.error(`execution failed for step "${step}": ${causeError} and going for vision`);
@@ -20,7 +20,7 @@ async function executeVisionFallback(params) {
         if (action === "scroll") {
             throw new Error("Scroll action failed element not found", { cause: causeError });
         }
-        if (platform === "web") {
+        if (platform === "web" || (driverContext && fallbackAttempt > 1)) {
             annotatedScreenshot = await (0, generateAnnotatedScreenshots_js_1.generateAnnotatedScreenshots)(driver);
         }
         else {
@@ -36,7 +36,7 @@ async function executeVisionFallback(params) {
             bounds = visionResult.boxes;
         }
         let stepResultVision;
-        if (platform === "web") {
+        if (platform === "web" || (driverContext && fallbackAttempt > 1)) {
             stepResultVision = await stepProcessor.getLLMResponse({
                 platform,
                 type: promptType_js_1.PromptType.VISION_PROMPT,
@@ -73,7 +73,7 @@ async function executeVisionFallback(params) {
             throw new Error(`vision unable to find element for step "${step}"`, { cause: causeError });
         }
         let xpath;
-        if (platform === "android" || platform === "ios") {
+        if ((platform === "android" || platform === "ios") && (driverContext && fallbackAttempt < 2)) {
             const index = Number(fireflinkIndex.split("-").pop());
             const box = bounds[index - 1];
             if (!box) {

package/dist/core/interfaces/automationRunnerInterface.d.ts

@@ -52,6 +52,7 @@ export interface VisionFallbackParams {
     actionHandlers: Record<string, any>;
     reporter: RunnerKafkaReporter;
     causeError: any;
+    driverContext?: any;
 }
 export interface VisionFallbackResult {
     preCapturedDomInfo: any | null;

package/dist/core/main/runAutomationScript.js

@@ -205,6 +205,7 @@ class AutomationRunner {
             currentPlatform = context.activePlatform;
             (0, platformContext_js_1.setPlatform)(currentPlatform);
             let stepResult;
+            let driverContext = null;
             let stepAction;
             let fireflinkIndex;
             let selector;
@@ -383,6 +384,8 @@ class AutomationRunner {
                         }
                         else {
                             driver = await context.mobileSession.getCurrentDriver();
+                            const contexts = await driver.getContexts();
+                            driverContext = contexts.find((context) => String(context).startsWith("WEBVIEW")) || null;
                         }
                     }
                     catch (error) {
@@ -518,10 +521,10 @@ class AutomationRunner {
                                 selector = "";
                                 iframeDomHtml = null;
                                 beforeHandles = null;
-                                const retryDomResult = await runnerDomPipeline_js_1.RunnerDomPipeline.captureAndParseDom(driver, currentPlatform, stepCount, action, true);
+                                const retryDomResult = await runnerDomPipeline_js_1.RunnerDomPipeline.captureAndParseDom(driver, currentPlatform, stepCount, action, true, driverContext);
                                 domInfo = retryDomResult.domInfo;
                                 doc = retryDomResult.doc;
-                                const retryRelevantResult = await runnerDomPipeline_js_1.RunnerDomPipeline.processRelevantDom(domProcessor, domInfo, keywords, stepCount, action, currentStep, currentPlatform, driver, true);
+                                const retryRelevantResult = await runnerDomPipeline_js_1.RunnerDomPipeline.processRelevantDom(domProcessor, domInfo, keywords, stepCount, action, currentStep, currentPlatform, driver, true, driverContext);
                                 extractedRelevantDom = retryRelevantResult.extractedRelevantDom;
                                 const retryToonDom = retryRelevantResult.toonDom;
                                 const retryActionResolution = await (0, stepActionInspector_js_1.resolveStepActionFromDom)(stepProcessor, currentPlatform, action, retryToonDom, priorAndNextSteps, currentStep, null);
@@ -560,7 +563,7 @@ class AutomationRunner {
                                 logData_js_1.logger.error(`element not found for step "${step}"`);
                                 throw new Error(`element not found for step "${step}"`);
                             }
-                            const selectorResult = await (0, elementSelectorResolver_js_1.resolveElementSelector)(doc, domInfo, fireflinkIndex, currentPlatform, driver);
+                            const selectorResult = await (0, elementSelectorResolver_js_1.resolveElementSelector)(doc, domInfo, fireflinkIndex, currentPlatform, driver, fallbackAttempt, driverContext);
                             selector = selectorResult.selector;
                             doc = selectorResult.doc;
                             domInfo = selectorResult.domInfo;
@@ -598,6 +601,9 @@ class AutomationRunner {
                             });
                             preCapturedDomInfo = await (0, runnerPostActionHelper_js_1.handlePostActionWeb)(currentPlatform, driver, action, beforeHandles, context.scriptAppender, stepCount);
                             if (fallbackAttempt === 2) {
+                                if (driverContext) {
+                                    await driver.switchContext("NATIVE_APP");
+                                }
                                 await reporter.sendRetry("Retry successful. Moving on to the next step.");
                             }
                             break;
@@ -632,11 +638,15 @@ class AutomationRunner {
                                 actionHandlers,
                                 reporter,
                                 causeError: error,
+                                driverContext,
                             });
                             if (visionResult.retryNeeded) {
                                 continue;
                             }
                             preCapturedDomInfo = visionResult.preCapturedDomInfo;
+                            if (driverContext) {
+                                await driver.switchContext("NATIVE_APP");
+                            }
                             break;
                         }
                     }

package/dist/fireflinkData/fireflinkLocators/getListOfLocators.d.ts

@@ -1,6 +1,7 @@
+import { platform } from "../../core/types/scriptTypesType.js";
 import { LocatorWorkerPool } from "../../utils/workers/LocatorWorkerPool.js";
 export declare class ElementGetter {
     private readonly workerPool;
     constructor(workerPool: LocatorWorkerPool);
-    getFireFlinkElement(html: string, attribute: string): Promise<any>;
+    getFireFlinkElement(html: string, attribute: string, targetPlatform?: platform): Promise<any>;
 }

package/dist/fireflinkData/fireflinkLocators/getListOfLocators.js

@@ -8,8 +8,8 @@ class ElementGetter {
     constructor(workerPool) {
         this.workerPool = workerPool;
     }
-    async getFireFlinkElement(html, attribute) {
-        const platform = (0, platformContext_js_1.getPlatform)();
+    async getFireFlinkElement(html, attribute, targetPlatform) {
+        const platform = targetPlatform || (html?.includes("ff-inspect") ? "web" : (0, platformContext_js_1.getPlatform)());
         // const formattedAttribute = attribute.replace('[', '//*[@');
         try {
             const result = await this.workerPool.getFireFlinkElement(html, attribute, platform);

package/dist/fireflinkData/fireflinkScript/appendScriptData.js

@@ -3,10 +3,37 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.appendScriptData = appendScriptData;
 const logData_js_1 = require("../../utils/logger/logData.js");
 async function appendScriptData(args) {
+    const isMobile = args.platform === "android" || args.platform === "ios";
+    const isWebView = args.pageDOM?.includes("ff-inspect");
+    if (isMobile) {
+        if (args.scriptDataAppender.currentMobileView === undefined) {
+            args.scriptDataAppender.currentMobileView = "NATIVE";
+        }
+        if (isWebView && args.scriptDataAppender.currentMobileView !== "WEBVIEW") {
+            args.scriptDataAppender.add(async () => ({
+                nlpName: "MOB_SwitchToView",
+                stepInputs: ["WEBVIEW"],
+                elementsData: []
+            }));
+            args.scriptDataAppender.currentMobileView = "WEBVIEW";
+        }
+        else if (args.pageDOM && !isWebView && args.scriptDataAppender.currentMobileView === "WEBVIEW") {
+            args.scriptDataAppender.add(async () => ({
+                nlpName: "MOB_SwitchToNativeView",
+                stepInputs: [],
+                elementsData: []
+            }));
+            args.scriptDataAppender.currentMobileView = "NATIVE";
+        }
+    }
     args.scriptDataAppender.add(async () => {
-        const searchAttribute = args.platform === "web"
-            ? `[ff-inspect="${args.ffNumber}"]`
-            : `[FF-inspecter="${args.ffNumber}"]`;
+        const isWebView = args.pageDOM?.includes("ff-inspect");
+        const effectivePlatform = isWebView ? "web" : args.platform;
+        const searchAttribute = args.ffNumber
+            ? (effectivePlatform === "web"
+                ? `[ff-inspect="${args.ffNumber}"]`
+                : `[FF-inspecter="${args.ffNumber}"]`)
+            : args.selector;
         const data = {
             nlpName: args.nlpName
         };
@@ -19,9 +46,12 @@ async function appendScriptData(args) {
         if (args.elementName) {
             let locators;
             try {
-                const ffElement = await args.elementGetter.getFireFlinkElement(args.pageDOM, searchAttribute);
+                if (!searchAttribute) {
+                    throw new Error("searchAttribute is not found");
+                }
+                const ffElement = await args.elementGetter.getFireFlinkElement(args.pageDOM, searchAttribute, effectivePlatform);
                 // locators = [{ name: "xpath", value: args.selector }];
-                if (ffElement?.locators?.length === 0) {
+                if (!ffElement?.locators || ffElement.locators.length === 0) {
                     throw new Error("locators is not found");
                 }
                 else {
@@ -30,7 +60,13 @@ async function appendScriptData(args) {
             }
             catch (error) {
                 logData_js_1.logger.info(`Element is not found ${error.message}`);
-                locators = [{ name: "xpath", value: args.selector }];
+                locators = [{
+                        name: "xpath",
+                        type: "static",
+                        value: args.selector,
+                        isRecorded: "N",
+                        isAiRecorded: "Y"
+                    }];
             }
             data.elementsData = [{
                     name: args.elementName,

package/dist/fireflinkData/fireflinkScript/scriptGenrationData.d.ts

@@ -1,5 +1,6 @@
 import { IScriptGenerationData } from "../../core/interfaces/scriptGenrationDataInterface.js";
 export declare class ScriptDataAppender {
+    currentMobileView: "NATIVE" | "WEBVIEW";
     private data;
     private pending;
     private errorNLP;

package/dist/fireflinkData/fireflinkScript/scriptGenrationData.js

@@ -4,6 +4,7 @@ exports.ScriptDataAppender = void 0;
 const platformSyntax_js_1 = require("../../utils/helpers/platformSyntax.js");
 class ScriptDataAppender {
     constructor() {
+        this.currentMobileView = "NATIVE";
         this.data = [];
         this.pending = [];
         this.errorNLP = "";

package/dist/imageAnalysisWeb/annotateScreenshot.d.ts

@@ -1,2 +1,2 @@
 import { NodeItem } from "./screenshotInterface.js";
-export declare function annotateScreenshot(screenshotBase64: string, nodes: NodeItem[]): Promise<Buffer>;
+export declare function annotateScreenshot(screenshotBase64: string, deviceWidth: number, deviceHeight: number, nodes: NodeItem[]): Promise<Buffer>;

package/dist/imageAnalysisWeb/annotateScreenshot.js

@@ -59,10 +59,16 @@ function findFreeTagPosition(box, tagW, tagH, elementBoxes, usedTagRects, imgWid
     usedTagRects.push([fallbackX, fallbackY, fallbackX + tagW, fallbackY + tagH]);
     return { x: fallbackX, y: fallbackY };
 }
-async function annotateScreenshot(screenshotBase64, nodes) {
+async function annotateScreenshot(screenshotBase64, deviceWidth, deviceHeight, nodes) {
     const buffer = Buffer.from(screenshotBase64, "base64");
     const image = await (0, canvas_1.loadImage)(buffer);
-    const scale = image.width / 1920;
+    // Actual screenshot resolution
+    const screenshotWidth = image.width;
+    const screenshotHeight = image.height;
+    // Device / WebView resolution from which bbox coordinates are coming
+    const scaleX = screenshotWidth / deviceWidth;
+    const scaleY = screenshotHeight / deviceHeight;
+    const scale = Math.min(scaleX, scaleY);
     const fontSize = Math.max(16, Math.floor(20 * scale));
     const canvas = (0, canvas_1.createCanvas)(image.width, image.height);
     const ctx = canvas.getContext("2d");
@@ -71,20 +77,28 @@ async function annotateScreenshot(screenshotBase64, nodes) {
     ctx.textBaseline = "middle";
     ctx.textAlign = "center";
     const usedTagRects = [];
+    /*
+    * Convert all bounding boxes from device/WebView
+    * coordinates to screenshot coordinates.
+    */
     const elementBoxes = nodes.map((n) => [
-        Math.floor(n.bbox.x),
-        Math.floor(n.bbox.y),
-        Math.floor(n.bbox.x + n.bbox.w),
-        Math.floor(n.bbox.y + n.bbox.h),
+        Math.floor(n.bbox.x * scaleX),
+        Math.floor(n.bbox.y * scaleY),
+        Math.floor((n.bbox.x + n.bbox.w) * scaleX),
+        Math.floor((n.bbox.y + n.bbox.h) * scaleY),
     ]);
     nodes.forEach((item, i) => {
         const color = COLORS[i % COLORS.length];
         const colorStr = `rgb(${color[0]},${color[1]},${color[2]})`;
         const label = item.ff.split("-").pop() || "";
-        const x1 = Math.floor(item.bbox.x);
-        const y1 = Math.floor(item.bbox.y);
-        const x2 = Math.floor(item.bbox.x + item.bbox.w);
-        const y2 = Math.floor(item.bbox.y + item.bbox.h);
+        /*
+        * Convert current node bbox to actual
+        * screenshot coordinates.
+        */
+        const x1 = Math.floor(item.bbox.x * scaleX);
+        const y1 = Math.floor(item.bbox.y * scaleY);
+        const x2 = Math.floor((item.bbox.x + item.bbox.w) * scaleX);
+        const y2 = Math.floor((item.bbox.y + item.bbox.h) * scaleY);
         ctx.strokeStyle = colorStr;
         ctx.lineWidth = Math.max(2, Math.floor(2 * scale));
         ctx.strokeRect(x1, y1, x2 - x1, y2 - y1);

package/dist/imageAnalysisWeb/generateAnnotatedScreenshots.js

@@ -14,13 +14,16 @@ async function generateAnnotatedScreenshots(driver) {
     //   driver.execute(JS_PRIMARY),
     //   // driver.execute(JS_SECONDARY),
     // ]);
+    const windowSize = await driver.getWindowSize();
+    const deviceWidth = windowSize.width;
+    const deviceHeight = windowSize.height;
     const primary = primaryRaw ?? [];
     let secondary = secondaryRaw ?? [];
     const primaryIds = new Set(primary.map((item) => item.ff));
     secondary = secondary.filter((item) => !primaryIds.has(item.ff));
     const [primaryAnnotated, secondaryAnnotated] = await Promise.all([
-        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, primary),
-        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, secondary)
+        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, deviceWidth, deviceHeight, primary),
+        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, deviceWidth, deviceHeight, secondary)
     ]);
     // Uncomment only when debugging
     logData_js_1.logger.saveImage(primaryAnnotated, "annotated_primary.png");