ff-automationv2 2.2.33 → 2.2.34-beta.1

package/dist/core/helpers/visionFallbackHandler.js

@@ -10,7 +10,7 @@ const runnerPostActionHelper_js_1 = require("./runnerPostActionHelper.js");
 const runnerDomPipeline_js_1 = require("./runnerDomPipeline.js");
 const logData_js_1 = require("../../utils/logger/logData.js");
 async function executeVisionFallback(params) {
-    const { driver, platform, action, step, stepAction, inputValue, stepCount, fallbackAttempt, priorAndNextSteps, listOfSteps, currentElementName, extractorElementName, stepElementType, iframeDomHtml, beforeHandles, context, stepProcessor, actionHandlers, reporter, causeError, } = params;
+    const { driver, platform, action, step, stepAction, inputValue, stepCount, fallbackAttempt, priorAndNextSteps, listOfSteps, currentElementName, extractorElementName, stepElementType, iframeDomHtml, beforeHandles, context, stepProcessor, actionHandlers, reporter, causeError, driverContext } = params;
     let domInfo = params.domInfo;
     let doc = params.doc;
     logData_js_1.logger.error(`execution failed for step "${step}": ${causeError} and going for vision`);
@@ -20,7 +20,7 @@ async function executeVisionFallback(params) {
         if (action === "scroll") {
             throw new Error("Scroll action failed element not found", { cause: causeError });
         }
-        if (platform === "web") {
+        if (platform === "web" || (driverContext && fallbackAttempt > 1)) {
             annotatedScreenshot = await (0, generateAnnotatedScreenshots_js_1.generateAnnotatedScreenshots)(driver);
         }
         else {
@@ -36,7 +36,7 @@ async function executeVisionFallback(params) {
             bounds = visionResult.boxes;
         }
         let stepResultVision;
-        if (platform === "web") {
+        if (platform === "web" || (driverContext && fallbackAttempt > 1)) {
             stepResultVision = await stepProcessor.getLLMResponse({
                 platform,
                 type: promptType_js_1.PromptType.VISION_PROMPT,
@@ -73,7 +73,7 @@ async function executeVisionFallback(params) {
             throw new Error(`vision unable to find element for step "${step}"`, { cause: causeError });
         }
         let xpath;
-        if (platform === "android" || platform === "ios") {
+        if ((platform === "android" || platform === "ios") && (driverContext && fallbackAttempt < 2)) {
             const index = Number(fireflinkIndex.split("-").pop());
             const box = bounds[index - 1];
             if (!box) {

package/dist/core/interfaces/automationRunnerInterface.d.ts

@@ -52,6 +52,7 @@ export interface VisionFallbackParams {
     actionHandlers: Record<string, any>;
     reporter: RunnerKafkaReporter;
     causeError: any;
+    driverContext?: any;
 }
 export interface VisionFallbackResult {
     preCapturedDomInfo: any | null;

package/dist/core/interfaces/executionDetails.d.ts

@@ -23,7 +23,7 @@ export interface AutomationRequest {
     userName: string;
     serverHost: string;
     browserName: BrowserName;
-    platform: platform;
+    platform?: platform;
     token: string;
     appiumPort: number;
     realDeviceId: string;
@@ -38,4 +38,5 @@ export interface AutomationRequest {
     adbPath?: string;
     topic: string;
     aiExecutionAction: string;
+    hasMoreSteps: boolean;
 }

package/dist/core/interfaces/fireflinkScriptPayloadInterface.d.ts

@@ -9,8 +9,7 @@ export interface IStep {
 }
 export interface IPayload {
     scriptName: string;
-    scriptType: string;
-    type?: string;
+    type: string;
     projectId?: string;
     testCaseId?: string;
     pageDetails?: any;
@@ -20,7 +19,6 @@ export interface IPayload {
     projectType?: string;
     tokensConsumed?: number;
     error?: IErrorInfo;
-    platform?: string;
     serviceProvider?: string;
     model?: string;
 }

package/dist/core/interfaces/promptInterface.d.ts

@@ -4,6 +4,11 @@ export interface PromptArgsMap {
     [PromptType.KEYWORD_EXTRACTOR]: {
         priorAndNextSteps: string[];
     };
+    [PromptType.MULTICHANNEL_KEYWORD_EXTRACTOR]: {
+        priorAndNextSteps: string[];
+        allSteps: string[];
+        previousPlatform?: string | null;
+    };
     [PromptType.ERROR_DESCRIPTION]: Record<string, any>;
     [PromptType.FF_INSPECTOR]: {
         stepAction: string;

package/dist/core/main/actionHandlerFactory.js

@@ -265,7 +265,7 @@ function createActionHandlers(context) {
         },
         openApp: async (result) => {
             try {
-                const isIOS = context.request.platform?.toLowerCase() === "ios";
+                const isIOS = context.activePlatform === "ios";
                 const appPackage = result.keywords
                     .find((item) => item.startsWith("appPackage:"))
                     ?.substring("appPackage:".length);
@@ -340,7 +340,7 @@ function createActionHandlers(context) {
         closeApp: async () => {
             try {
                 await context.mobileSession.close();
-                const nlpName = context.request.platform.toLowerCase() === "ios" ? "IOS_CloseApp" : "MOB_CloseApp";
+                const nlpName = context.activePlatform === "ios" ? "IOS_CloseApp" : "MOB_CloseApp";
                 context.scriptAppender.add(async () => ({
                     nlpName: nlpName,
                     elementsData: [],
@@ -357,7 +357,7 @@ function createActionHandlers(context) {
                 if (!context.browserOpened) {
                     logData_js_1.logger.info("Browser not initialized, auto-opening before navigate...");
                     await context.session.open();
-                    context.executor = new executor_js_1.ActionExecutor(await context.session.getCurrentDriver(), context.scriptAppender, context.locator, context.request.platform, context.adbPath);
+                    context.executor = new executor_js_1.ActionExecutor(await context.session.getCurrentDriver(), context.scriptAppender, context.locator, context.activePlatform, context.adbPath);
                     context.browserOpened = true;
                     context.scriptAppender.add(async () => ({
                         nlpName: "Open Browser",

package/dist/core/main/executionContext.d.ts

@@ -18,6 +18,8 @@ export declare class ExecutionContext {
     activePlatform: platform;
     browserOpened: boolean;
     stepsInputData: any[];
+    allManualSteps: string[];
     constructor(request: AutomationRequest, workerPool: LocatorWorkerPool);
     initializeMobileDriver(caps: any): Promise<void>;
+    syncExecutorToActivePlatform(): Promise<void>;
 }

package/dist/core/main/executionContext.js

@@ -15,15 +15,16 @@ class ExecutionContext {
         this.executor = null;
         this.browserOpened = false;
         this.stepsInputData = [];
+        this.allManualSteps = [];
         this.session = new initiateBrowserSession_js_1.BrowserSession(browserType_js_1.browserName[request.browserName], request.isCloud, request.capabilities, defaultValues_js_1.DEFAULT_PAGE_LOAD, request.cloudConfig);
         this.locator = new getListOfLocators_js_1.ElementGetter(workerPool);
         this.scriptAppender = new scriptGenrationData_js_1.ScriptDataAppender();
         this.mobileSession = new initiateMobileSession_js_1.MobileSession(request.appiumPort, request.isCloud, request.capabilities, request.cloudConfig, request.adbPath);
         this.adbPath = request.adbPath || "";
-        this.activePlatform = request.platform.toLowerCase();
+        this.activePlatform = (request.platform?.toLowerCase() || "web");
     }
     async initializeMobileDriver(caps) {
-        const isIOS = this.request.platform.toLowerCase() === "ios";
+        const isIOS = this.activePlatform === "ios";
         const appiumCaps = {};
         const allowedCapabilities = [
             "app",
@@ -107,7 +108,21 @@ class ExecutionContext {
                 ...appiumCaps
             };
         await this.mobileSession.createDriver(capabilities);
-        this.executor = new executor_js_1.ActionExecutor(await this.mobileSession.getCurrentDriver(), this.scriptAppender, this.locator, this.request.platform, this.adbPath, this.stepsInputData);
+        this.executor = new executor_js_1.ActionExecutor(await this.mobileSession.getCurrentDriver(), this.scriptAppender, this.locator, this.activePlatform, this.adbPath, this.stepsInputData);
+    }
+    async syncExecutorToActivePlatform() {
+        try {
+            if (this.activePlatform === "web") {
+                const webDriver = await this.session.getCurrentDriver();
+                this.executor = new executor_js_1.ActionExecutor(webDriver, this.scriptAppender, this.locator, "web", this.adbPath, this.stepsInputData);
+                return;
+            }
+            const mobileDriver = await this.mobileSession.getCurrentDriver();
+            this.executor = new executor_js_1.ActionExecutor(mobileDriver, this.scriptAppender, this.locator, this.activePlatform, this.adbPath, this.stepsInputData);
+        }
+        catch {
+            // Driver for the detected platform may not exist yet (open/openApp comes later).
+        }
     }
 }
 exports.ExecutionContext = ExecutionContext;

package/dist/core/main/runAutomationScript.d.ts

@@ -1,6 +1,7 @@
 import "../../utils/sslConfig/apiCallPatch.js";
 import { IAutomationRunner } from "../interfaces/automationRunnerInterface.js";
 import { AutomationRequest } from "../../core/interfaces/executionDetails.js";
+import { platform } from "../types/scriptTypesType.js";
 export declare class AutomationRunner implements IAutomationRunner {
     static sessionTerminationDetails: Record<string, boolean>;
     static sessionNames: Record<string, string>;
@@ -9,6 +10,7 @@ export declare class AutomationRunner implements IAutomationRunner {
     static updateSessionTerminationInfo(testCaseId: string): void;
     static sessionName(testCaseId: string, sessionName: string): void;
     static getSessionName(testCaseId: string): string;
+    static resolveDetectedPlatform(detectedPlatform?: string, previousPlatform?: platform): platform;
     static normalizeScriptType(scriptType?: string): string;
     private cleanup;
     private saveSessionStepData;

package/dist/core/main/runAutomationScript.js

@@ -24,7 +24,6 @@ const isAlertPresent_js_1 = require("../../utils/alertPopup/isAlertPresent.js");
 const logData_js_1 = require("../../utils/logger/logData.js");
 const sameActionsHelper_js_1 = require("../../utils/helpers/sameActionsHelper.js");
 const messageType_js_1 = require("../types/messageType.js");
-const platformSyntax_js_1 = require("../../utils/helpers/platformSyntax.js");
 const driverState_js_1 = require("../../automation/driverState.js");
 const SessionManager_js_1 = require("./SessionManager.js");
 const runnerKafkaReporter_js_1 = require("../helpers/runnerKafkaReporter.js");
@@ -49,6 +48,18 @@ class AutomationRunner {
     static getSessionName(testCaseId) {
         return AutomationRunner.sessionNames[testCaseId];
     }
+    static resolveDetectedPlatform(detectedPlatform, previousPlatform) {
+        const value = (detectedPlatform ?? "").trim().toLowerCase();
+        if (value === "ios")
+            return "ios";
+        if (value === "android")
+            return "android";
+        if (value === "web")
+            return "web";
+        if (previousPlatform)
+            return previousPlatform;
+        return "web";
+    }
     static normalizeScriptType(scriptType) {
         const rawValue = (scriptType ?? "Web").trim();
         if (!rawValue) {
@@ -150,15 +161,16 @@ class AutomationRunner {
         const usedElementNames = [];
         const usedVariableNames = [];
         (0, logData_js_1.initLogger)(request.sessionPath);
-        request.type = request.platform;
-        request.platform = AutomationRunner.normalizeScriptType(request.platform);
-        const isWebMobile = (request.type?.toLowerCase().includes("web") &&
-            request.type?.toLowerCase().includes("android")) ||
+        const isMultichannel = request.type?.toLowerCase().includes("Multi-Channel");
+        if (!isMultichannel) {
+            request.platform = AutomationRunner.normalizeScriptType(request.platform);
+        }
+        const isWebMobile = isMultichannel ||
+            (request.type?.toLowerCase().includes("web") &&
+                request.type?.toLowerCase().includes("android")) ||
             (request.type?.toLowerCase() === "android");
         logData_js_1.logger.info("User Given Step", request.userStory);
-        const data = request.scriptName ? JSON.parse(request.scriptName) : {};
-        const hasMoreSteps = data?.hasMoreSteps ?? false;
-        request.scriptName = data?.automationScriptName ?? "automationScriptName";
+        const hasMoreSteps = request.hasMoreSteps;
         if (!request.isTerminate)
             AutomationRunner.sessionName(request.testCaseId, request.scriptName);
         if (request.isTerminate) {
@@ -170,6 +182,7 @@ class AutomationRunner {
         const instanceDetails = await fireflinkServiceInstance.runGetInstancesDetails(request.projectId, request.token, request.serverHost, request.licenseId);
         const aiInfo = (0, getInstancesInfo_js_1.getAiInstanceInfo)(instanceDetails.responseObject);
         let context = SessionManager_js_1.SessionManager.get(request);
+        const hadExistingSession = Boolean(context);
         if (!context) {
             context = SessionManager_js_1.SessionManager.create(request);
         }
@@ -183,11 +196,13 @@ class AutomationRunner {
         const stepProcessor = new stepProcessor_js_1.StepProcessor(llm);
         let currentPlatform = context.activePlatform;
         (0, platformContext_js_1.setPlatform)(currentPlatform);
+        let previousDetectedPlatform = isMultichannel && hadExistingSession ? context.activePlatform : undefined;
         const actionHandlers = (0, actionHandlerFactory_js_1.createActionHandlers)(context);
         const domProcessor = new getRelaventElements_js_1.DomProcessingEngine();
         let stepCount = 1;
         const stepsInputData = context.stepsInputData || (context.stepsInputData = []);
         const listOfSteps = request.userStory.map((step) => String(step));
+        context.allManualSteps.push(...listOfSteps);
         let driver;
         if (listOfSteps.length === 0 && request?.aiExecutionAction !== "stopExecution") {
             logData_js_1.logger.info(listOfSteps);
@@ -205,6 +220,7 @@ class AutomationRunner {
             currentPlatform = context.activePlatform;
             (0, platformContext_js_1.setPlatform)(currentPlatform);
             let stepResult;
+            let driverContext = null;
             let stepAction;
             let fireflinkIndex;
             let selector;
@@ -226,14 +242,40 @@ class AutomationRunner {
                 const start = Math.max(0, index - 3);
                 const end = Math.min(listOfSteps.length, index + 4);
                 const priorAndNextSteps = listOfSteps.slice(start, end);
-                const result = await stepProcessor.getLLMResponse({
-                    platform: currentPlatform,
-                    type: promptType_js_1.PromptType.KEYWORD_EXTRACTOR,
-                    args: { priorAndNextSteps },
-                    input: { currentStep: step },
-                });
+                const result = isMultichannel
+                    ? await stepProcessor.getLLMResponse({
+                        platform: currentPlatform || "web",
+                        type: promptType_js_1.PromptType.MULTICHANNEL_KEYWORD_EXTRACTOR,
+                        args: {
+                            priorAndNextSteps,
+                            allSteps: context.allManualSteps,
+                            previousPlatform: previousDetectedPlatform ?? null,
+                        },
+                        input: { currentStep: step },
+                    })
+                    : await stepProcessor.getLLMResponse({
+                        platform: currentPlatform,
+                        type: promptType_js_1.PromptType.KEYWORD_EXTRACTOR,
+                        args: { priorAndNextSteps },
+                        input: { currentStep: step },
+                    });
+                if (isMultichannel) {
+                    const firstStepPlatform = AutomationRunner.resolveDetectedPlatform(result.response?.steps?.[0]?.platform ?? result.response?.platform, previousDetectedPlatform);
+                    currentPlatform = firstStepPlatform;
+                    previousDetectedPlatform = firstStepPlatform;
+                    context.activePlatform = firstStepPlatform;
+                    context.request.platform = firstStepPlatform;
+                    (0, platformContext_js_1.setPlatform)(firstStepPlatform);
+                }
                 if (currentPlatform === "android" || currentPlatform === "ios") {
-                    if (!driver) {
+                    let mobileDriver = null;
+                    try {
+                        mobileDriver = await context.mobileSession.getCurrentDriver();
+                    }
+                    catch {
+                        mobileDriver = null;
+                    }
+                    if (!mobileDriver) {
                         const initialKeywords = result.response?.keywords ||
                             (Array.isArray(result.response?.steps) ? result.response.steps[0]?.keywords : undefined);
                         await context.initializeMobileDriver(initialKeywords);
@@ -252,6 +294,15 @@ class AutomationRunner {
                     const stepObj = stepsArray[i];
                     const action = stepObj.action;
                     const keywords = stepObj.keywords;
+                    if (isMultichannel) {
+                        const stepPlatform = AutomationRunner.resolveDetectedPlatform(stepObj.platform, previousDetectedPlatform);
+                        currentPlatform = stepPlatform;
+                        previousDetectedPlatform = stepPlatform;
+                        context.activePlatform = stepPlatform;
+                        context.request.platform = stepPlatform;
+                        (0, platformContext_js_1.setPlatform)(stepPlatform);
+                        await context.syncExecutorToActivePlatform();
+                    }
                     const currentStep = action?.toLowerCase() === "getotp" ? step : stepObj.step || step;
                     let currentElementName = stepObj.elementName;
                     let currentVariableName;
@@ -383,6 +434,8 @@ class AutomationRunner {
                         }
                         else {
                             driver = await context.mobileSession.getCurrentDriver();
+                            const contexts = await driver.getContexts();
+                            driverContext = contexts.find((context) => String(context).startsWith("WEBVIEW")) || null;
                         }
                     }
                     catch (error) {
@@ -518,10 +571,10 @@ class AutomationRunner {
                                 selector = "";
                                 iframeDomHtml = null;
                                 beforeHandles = null;
-                                const retryDomResult = await runnerDomPipeline_js_1.RunnerDomPipeline.captureAndParseDom(driver, currentPlatform, stepCount, action, true);
+                                const retryDomResult = await runnerDomPipeline_js_1.RunnerDomPipeline.captureAndParseDom(driver, currentPlatform, stepCount, action, true, driverContext);
                                 domInfo = retryDomResult.domInfo;
                                 doc = retryDomResult.doc;
-                                const retryRelevantResult = await runnerDomPipeline_js_1.RunnerDomPipeline.processRelevantDom(domProcessor, domInfo, keywords, stepCount, action, currentStep, currentPlatform, driver, true);
+                                const retryRelevantResult = await runnerDomPipeline_js_1.RunnerDomPipeline.processRelevantDom(domProcessor, domInfo, keywords, stepCount, action, currentStep, currentPlatform, driver, true, driverContext);
                                 extractedRelevantDom = retryRelevantResult.extractedRelevantDom;
                                 const retryToonDom = retryRelevantResult.toonDom;
                                 const retryActionResolution = await (0, stepActionInspector_js_1.resolveStepActionFromDom)(stepProcessor, currentPlatform, action, retryToonDom, priorAndNextSteps, currentStep, null);
@@ -560,7 +613,7 @@ class AutomationRunner {
                                 logData_js_1.logger.error(`element not found for step "${step}"`);
                                 throw new Error(`element not found for step "${step}"`);
                             }
-                            const selectorResult = await (0, elementSelectorResolver_js_1.resolveElementSelector)(doc, domInfo, fireflinkIndex, currentPlatform, driver);
+                            const selectorResult = await (0, elementSelectorResolver_js_1.resolveElementSelector)(doc, domInfo, fireflinkIndex, currentPlatform, driver, fallbackAttempt, driverContext);
                             selector = selectorResult.selector;
                             doc = selectorResult.doc;
                             domInfo = selectorResult.domInfo;
@@ -598,6 +651,9 @@ class AutomationRunner {
                             });
                             preCapturedDomInfo = await (0, runnerPostActionHelper_js_1.handlePostActionWeb)(currentPlatform, driver, action, beforeHandles, context.scriptAppender, stepCount);
                             if (fallbackAttempt === 2) {
+                                if (driverContext) {
+                                    await driver.switchContext("NATIVE_APP");
+                                }
                                 await reporter.sendRetry("Retry successful. Moving on to the next step.");
                             }
                             break;
@@ -632,11 +688,15 @@ class AutomationRunner {
                                 actionHandlers,
                                 reporter,
                                 causeError: error,
+                                driverContext,
                             });
                             if (visionResult.retryNeeded) {
                                 continue;
                             }
                             preCapturedDomInfo = visionResult.preCapturedDomInfo;
+                            if (driverContext) {
+                                await driver.switchContext("NATIVE_APP");
+                            }
                             break;
                         }
                     }
@@ -676,7 +736,6 @@ class AutomationRunner {
         const payload = {
             scriptName: AutomationRunner.getSessionName(request.testCaseId) ?? "SCRIPT_NAME",
             type: request.type,
-            scriptType: request.type,
             projectId: request.projectId,
             testCaseId: request.testCaseId,
             pageDetails: request.pageDetails,
@@ -684,7 +743,6 @@ class AutomationRunner {
             licenseId: request.licenseId,
             userId: request.userId,
             projectType: request.projectType,
-            platform: (0, platformSyntax_js_1.getPlatformSyntax)(currentPlatform),
             tokensConsumed: (await stepProcessor.getResultTokenUsage()).totalTokens,
         };
         this.KafkaPayload.topic = request.topic;

package/dist/core/types/promptType.d.ts

@@ -1,6 +1,7 @@
 export declare enum PromptType {
     USER_STORY_TO_LIST = "userStoryToList",
     KEYWORD_EXTRACTOR = "keywordExtractorPrompt",
+    MULTICHANNEL_KEYWORD_EXTRACTOR = "multichannelKeywordExtractorPrompt",
     ERROR_DESCRIPTION = "errorDescriptionPrompt",
     FF_INSPECTOR = "ffInspectorNumExtractor",
     VISION_PROMPT = "visionPrompt",

package/dist/core/types/promptType.js

@@ -5,6 +5,7 @@ var PromptType;
 (function (PromptType) {
     PromptType["USER_STORY_TO_LIST"] = "userStoryToList";
     PromptType["KEYWORD_EXTRACTOR"] = "keywordExtractorPrompt";
+    PromptType["MULTICHANNEL_KEYWORD_EXTRACTOR"] = "multichannelKeywordExtractorPrompt";
     PromptType["ERROR_DESCRIPTION"] = "errorDescriptionPrompt";
     PromptType["FF_INSPECTOR"] = "ffInspectorNumExtractor";
     PromptType["VISION_PROMPT"] = "visionPrompt";

package/dist/fireflinkData/fireflinkLocators/getListOfLocators.d.ts

@@ -1,6 +1,7 @@
+import { platform } from "../../core/types/scriptTypesType.js";
 import { LocatorWorkerPool } from "../../utils/workers/LocatorWorkerPool.js";
 export declare class ElementGetter {
     private readonly workerPool;
     constructor(workerPool: LocatorWorkerPool);
-    getFireFlinkElement(html: string, attribute: string): Promise<any>;
+    getFireFlinkElement(html: string, attribute: string, targetPlatform?: platform): Promise<any>;
 }

package/dist/fireflinkData/fireflinkLocators/getListOfLocators.js

@@ -8,8 +8,8 @@ class ElementGetter {
     constructor(workerPool) {
         this.workerPool = workerPool;
     }
-    async getFireFlinkElement(html, attribute) {
-        const platform = (0, platformContext_js_1.getPlatform)();
+    async getFireFlinkElement(html, attribute, targetPlatform) {
+        const platform = targetPlatform || (html?.includes("ff-inspect") ? "web" : (0, platformContext_js_1.getPlatform)());
         // const formattedAttribute = attribute.replace('[', '//*[@');
         try {
             const result = await this.workerPool.getFireFlinkElement(html, attribute, platform);

package/dist/fireflinkData/fireflinkScript/appendScriptData.js

@@ -3,10 +3,37 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.appendScriptData = appendScriptData;
 const logData_js_1 = require("../../utils/logger/logData.js");
 async function appendScriptData(args) {
+    const isMobile = args.platform === "android" || args.platform === "ios";
+    const isWebView = args.pageDOM?.includes("ff-inspect");
+    if (isMobile) {
+        if (args.scriptDataAppender.currentMobileView === undefined) {
+            args.scriptDataAppender.currentMobileView = "NATIVE";
+        }
+        if (isWebView && args.scriptDataAppender.currentMobileView !== "WEBVIEW") {
+            args.scriptDataAppender.add(async () => ({
+                nlpName: "MOB_SwitchToView",
+                stepInputs: ["WEBVIEW"],
+                elementsData: []
+            }));
+            args.scriptDataAppender.currentMobileView = "WEBVIEW";
+        }
+        else if (args.pageDOM && !isWebView && args.scriptDataAppender.currentMobileView === "WEBVIEW") {
+            args.scriptDataAppender.add(async () => ({
+                nlpName: "MOB_SwitchToNativeView",
+                stepInputs: [],
+                elementsData: []
+            }));
+            args.scriptDataAppender.currentMobileView = "NATIVE";
+        }
+    }
     args.scriptDataAppender.add(async () => {
-        const searchAttribute = args.platform === "web"
-            ? `[ff-inspect="${args.ffNumber}"]`
-            : `[FF-inspecter="${args.ffNumber}"]`;
+        const isWebView = args.pageDOM?.includes("ff-inspect");
+        const effectivePlatform = isWebView ? "web" : args.platform;
+        const searchAttribute = args.ffNumber
+            ? (effectivePlatform === "web"
+                ? `[ff-inspect="${args.ffNumber}"]`
+                : `[FF-inspecter="${args.ffNumber}"]`)
+            : args.selector;
         const data = {
             nlpName: args.nlpName
         };
@@ -18,20 +45,40 @@ async function appendScriptData(args) {
         }
         if (args.elementName) {
             let locators;
-            try {
-                const ffElement = await args.elementGetter.getFireFlinkElement(args.pageDOM, searchAttribute);
-                // locators = [{ name: "xpath", value: args.selector }];
-                if (ffElement?.locators?.length === 0) {
-                    throw new Error("locators is not found");
+            const isJsPath = args.selector?.includes(".shadowRoot") || args.selector?.startsWith("document.");
+            if (isJsPath) {
+                locators = [{
+                        name: "JsPath",
+                        type: "Static",
+                        value: args.selector,
+                        isRecorded: "N",
+                        isAiRecorded: "Y"
+                    }];
+            }
+            else {
+                try {
+                    if (!searchAttribute) {
+                        throw new Error("searchAttribute is not found");
+                    }
+                    const ffElement = await args.elementGetter.getFireFlinkElement(args.pageDOM, searchAttribute, effectivePlatform);
+                    if (!ffElement?.locators || ffElement.locators.length === 0) {
+                        throw new Error("locators is not found");
+                    }
+                    else {
+                        locators = ffElement.locators;
+                    }
                 }
-                else {
-                    locators = ffElement.locators;
+                catch (error) {
+                    logData_js_1.logger.info(`Element is not found ${error.message}`);
+                    locators = [{
+                            name: "xpath",
+                            type: "static",
+                            value: args.selector,
+                            isRecorded: "N",
+                            isAiRecorded: "Y"
+                        }];
                 }
             }
-            catch (error) {
-                logData_js_1.logger.info(`Element is not found ${error.message}`);
-                locators = [{ name: "xpath", value: args.selector }];
-            }
             data.elementsData = [{
                     name: args.elementName,
                     type: args.elementType,

package/dist/fireflinkData/fireflinkScript/scriptGenrationData.d.ts

@@ -1,5 +1,6 @@
 import { IScriptGenerationData } from "../../core/interfaces/scriptGenrationDataInterface.js";
 export declare class ScriptDataAppender {
+    currentMobileView: "NATIVE" | "WEBVIEW";
     private data;
     private pending;
     private errorNLP;

package/dist/fireflinkData/fireflinkScript/scriptGenrationData.js

@@ -4,6 +4,7 @@ exports.ScriptDataAppender = void 0;
 const platformSyntax_js_1 = require("../../utils/helpers/platformSyntax.js");
 class ScriptDataAppender {
     constructor() {
+        this.currentMobileView = "NATIVE";
         this.data = [];
         this.pending = [];
         this.errorNLP = "";

package/dist/imageAnalysisWeb/annotateScreenshot.d.ts

@@ -1,2 +1,2 @@
 import { NodeItem } from "./screenshotInterface.js";
-export declare function annotateScreenshot(screenshotBase64: string, nodes: NodeItem[]): Promise<Buffer>;
+export declare function annotateScreenshot(screenshotBase64: string, deviceWidth: number, deviceHeight: number, nodes: NodeItem[]): Promise<Buffer>;

package/dist/imageAnalysisWeb/annotateScreenshot.js

@@ -59,10 +59,16 @@ function findFreeTagPosition(box, tagW, tagH, elementBoxes, usedTagRects, imgWid
     usedTagRects.push([fallbackX, fallbackY, fallbackX + tagW, fallbackY + tagH]);
     return { x: fallbackX, y: fallbackY };
 }
-async function annotateScreenshot(screenshotBase64, nodes) {
+async function annotateScreenshot(screenshotBase64, deviceWidth, deviceHeight, nodes) {
     const buffer = Buffer.from(screenshotBase64, "base64");
     const image = await (0, canvas_1.loadImage)(buffer);
-    const scale = image.width / 1920;
+    // Actual screenshot resolution
+    const screenshotWidth = image.width;
+    const screenshotHeight = image.height;
+    // Device / WebView resolution from which bbox coordinates are coming
+    const scaleX = screenshotWidth / deviceWidth;
+    const scaleY = screenshotHeight / deviceHeight;
+    const scale = Math.min(scaleX, scaleY);
     const fontSize = Math.max(16, Math.floor(20 * scale));
     const canvas = (0, canvas_1.createCanvas)(image.width, image.height);
     const ctx = canvas.getContext("2d");
@@ -71,20 +77,28 @@ async function annotateScreenshot(screenshotBase64, nodes) {
     ctx.textBaseline = "middle";
     ctx.textAlign = "center";
     const usedTagRects = [];
+    /*
+    * Convert all bounding boxes from device/WebView
+    * coordinates to screenshot coordinates.
+    */
     const elementBoxes = nodes.map((n) => [
-        Math.floor(n.bbox.x),
-        Math.floor(n.bbox.y),
-        Math.floor(n.bbox.x + n.bbox.w),
-        Math.floor(n.bbox.y + n.bbox.h),
+        Math.floor(n.bbox.x * scaleX),
+        Math.floor(n.bbox.y * scaleY),
+        Math.floor((n.bbox.x + n.bbox.w) * scaleX),
+        Math.floor((n.bbox.y + n.bbox.h) * scaleY),
     ]);
     nodes.forEach((item, i) => {
         const color = COLORS[i % COLORS.length];
         const colorStr = `rgb(${color[0]},${color[1]},${color[2]})`;
         const label = item.ff.split("-").pop() || "";
-        const x1 = Math.floor(item.bbox.x);
-        const y1 = Math.floor(item.bbox.y);
-        const x2 = Math.floor(item.bbox.x + item.bbox.w);
-        const y2 = Math.floor(item.bbox.y + item.bbox.h);
+        /*
+        * Convert current node bbox to actual
+        * screenshot coordinates.
+        */
+        const x1 = Math.floor(item.bbox.x * scaleX);
+        const y1 = Math.floor(item.bbox.y * scaleY);
+        const x2 = Math.floor((item.bbox.x + item.bbox.w) * scaleX);
+        const y2 = Math.floor((item.bbox.y + item.bbox.h) * scaleY);
         ctx.strokeStyle = colorStr;
         ctx.lineWidth = Math.max(2, Math.floor(2 * scale));
         ctx.strokeRect(x1, y1, x2 - x1, y2 - y1);

package/dist/imageAnalysisWeb/generateAnnotatedScreenshots.js

@@ -14,13 +14,16 @@ async function generateAnnotatedScreenshots(driver) {
     //   driver.execute(JS_PRIMARY),
     //   // driver.execute(JS_SECONDARY),
     // ]);
+    const windowSize = await driver.getWindowSize();
+    const deviceWidth = windowSize.width;
+    const deviceHeight = windowSize.height;
     const primary = primaryRaw ?? [];
     let secondary = secondaryRaw ?? [];
     const primaryIds = new Set(primary.map((item) => item.ff));
     secondary = secondary.filter((item) => !primaryIds.has(item.ff));
     const [primaryAnnotated, secondaryAnnotated] = await Promise.all([
-        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, primary),
-        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, secondary)
+        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, deviceWidth, deviceHeight, primary),
+        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, deviceWidth, deviceHeight, secondary)
     ]);
     // Uncomment only when debugging
     logData_js_1.logger.saveImage(primaryAnnotated, "annotated_primary.png");