ff-automationv2 2.2.30 → 2.2.32-beta.0

package/dist/core/constants/supportedActions.js

@@ -28,6 +28,7 @@ exports.ELEMENTLESS_ACTION = [
     "SwitchToParentWindow",
     "SwitchToWindowIfTitleContainsString",
     "SwitchToNewWindowIfUrlContainsString",
+    "ScrollPageToSpecificLocation",
     "GetPageTitle",
     "GetBrowserVersion",
     "GetAllWindowHandles",
@@ -68,6 +69,9 @@ exports.ELEMENTLESS_ACTION = [
     "GetNetworkRouteTime",
     "GetPerformanceMetrics",
     "GetUserHomeDirectory",
+    "ConcatString",
+    "GetLastModifiedFile",
+    "VerifyPDF",
     "MOB_GetCurrentWindowHandle",
     "MOB_GetDriverInstance",
     "MOB_GetADBLogsForGivenAppPackage",

package/dist/core/helpers/elementSelectorResolver.d.ts

@@ -0,0 +1,11 @@
+import { platform as PlatformType } from "../types/scriptTypesType.js";
+import { ShadowIframeResult } from "../interfaces/automationRunnerInterface.js";
+export { ShadowIframeResult };
+export declare function generateUniqueName(currentName: string | undefined, usedNames: string[]): string | undefined;
+export declare function normalizeSvgSelector(selector: string): string;
+export declare function resolveElementSelector(doc: any, domInfo: any, fireflinkIndex: string, platform: PlatformType, driver: any): Promise<{
+    selector: string;
+    doc: any;
+    domInfo: any;
+}>;
+export declare function handleShadowAndIframe(driver: any, stepAction: string, selector: string, fireflinkIndex: string, domInfo: any, actionHandlers: Record<string, any>, elementName: string, elementType: string): Promise<ShadowIframeResult>;

package/dist/core/helpers/elementSelectorResolver.js

@@ -0,0 +1,102 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateUniqueName = generateUniqueName;
+exports.normalizeSvgSelector = normalizeSvgSelector;
+exports.resolveElementSelector = resolveElementSelector;
+exports.handleShadowAndIframe = handleShadowAndIframe;
+const xpathcreation_js_1 = require("../../utils/helpers/xpathcreation.js");
+const jsForAttributeInjection_js_1 = require("../../utils/DomExtraction/jsForAttributeInjection.js");
+const jsFindElement_js_1 = require("../../utils/javascript/jsFindElement.js");
+const runnerDomPipeline_js_1 = require("./runnerDomPipeline.js");
+const logData_js_1 = require("../../utils/logger/logData.js");
+function generateUniqueName(currentName, usedNames) {
+    if (!currentName) {
+        return currentName;
+    }
+    const baseName = currentName;
+    let newName = baseName;
+    let count = 0;
+    while (usedNames.includes(newName)) {
+        count++;
+        newName = `${baseName}_${count}`;
+    }
+    usedNames.push(newName);
+    return newName;
+}
+function normalizeSvgSelector(selector) {
+    if (selector.includes("svg")) {
+        const match = /svg\[(\d+)\]/.exec(selector);
+        const svgIndex = match ? match[1] : "1";
+        return selector.replace(/svg\[\d+\]/, `*[name()='svg'][${svgIndex}]`);
+    }
+    return selector;
+}
+async function resolveElementSelector(doc, domInfo, fireflinkIndex, platform, driver) {
+    let selector;
+    let updatedDoc = doc;
+    let updatedDomInfo = domInfo;
+    if (platform === "web") {
+        if (!updatedDoc) {
+            logData_js_1.logger.info("doc is undefined. Rebuilding document from domInfo.");
+            if (!updatedDomInfo || !updatedDomInfo.dom) {
+                updatedDomInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOM)(driver);
+            }
+            updatedDoc = runnerDomPipeline_js_1.RunnerDomPipeline.parseDomDocument(updatedDomInfo.dom, "web");
+        }
+        const ele = updatedDoc.querySelector(`[ff-inspect="${fireflinkIndex}"]`);
+        if (ele) {
+            selector = ele.getAttribute("ff-xpath") || "";
+        }
+        else {
+            const result = (0, xpathcreation_js_1.getWebXPathByFFIndex)(updatedDomInfo.dom, fireflinkIndex);
+            selector = result.xpathPath;
+        }
+    }
+    else {
+        const result = await (0, xpathcreation_js_1.getFFIndexAndXPath)(updatedDomInfo.dom, fireflinkIndex);
+        selector = result.xpathPath;
+    }
+    selector = normalizeSvgSelector(selector);
+    return { selector, doc: updatedDoc, domInfo: updatedDomInfo };
+}
+async function handleShadowAndIframe(driver, stepAction, selector, fireflinkIndex, domInfo, actionHandlers, elementName, elementType) {
+    let updatedStepAction = stepAction;
+    let updatedSelector = selector;
+    let iframeDomHtml = null;
+    if (updatedSelector.includes("shadow")) {
+        updatedStepAction = updatedStepAction + "_shadow";
+        logData_js_1.logger.info(updatedStepAction);
+    }
+    else if (updatedSelector.includes("iframe")) {
+        const defaultHandler = actionHandlers["switchToDefaultContent"];
+        await defaultHandler({ driver });
+        const result_if = await driver.executeAsyncScript(jsFindElement_js_1.Find_element_by_ff_js_script, [fireflinkIndex]);
+        if (!result_if?.success) {
+            logData_js_1.logger.info("Not found");
+            return {
+                stepAction: updatedStepAction,
+                selector: updatedSelector,
+                iframeDomHtml: null,
+                handled: false,
+            };
+        }
+        const { xpath_if, frames, iframe_dom_html } = result_if;
+        updatedSelector = xpath_if;
+        iframeDomHtml = iframe_dom_html;
+        const frameHandler = actionHandlers["switchToFrame"];
+        await frameHandler({
+            selector: updatedSelector,
+            frames: frames,
+            elementName,
+            elementType,
+            pageDOM: domInfo.dom,
+            fireflinkIndex,
+        });
+    }
+    return {
+        stepAction: updatedStepAction,
+        selector: updatedSelector,
+        iframeDomHtml,
+        handled: true,
+    };
+}

package/dist/core/helpers/runnerDomPipeline.d.ts

@@ -0,0 +1,9 @@
+import { DomProcessingEngine } from "../../domAnalysis/getRelaventElements.js";
+import { platform as PlatformType } from "../types/scriptTypesType.js";
+import { DomCaptureResult, DomProcessResult } from "../interfaces/automationRunnerInterface.js";
+export { DomCaptureResult, DomProcessResult };
+export declare class RunnerDomPipeline {
+    static parseDomDocument(domHtml: string, platform: PlatformType): any;
+    static captureAndParseDom(driver: any, platform: PlatformType, stepCount: number, action: string, isRetry?: boolean): Promise<DomCaptureResult>;
+    static processRelevantDom(domProcessor: DomProcessingEngine, domInfo: any, keywords: any, stepCount: number, action: string, currentStep: string, platform: PlatformType, driver: any, isRetry?: boolean): Promise<DomProcessResult>;
+}

package/dist/core/helpers/runnerDomPipeline.js

@@ -0,0 +1,97 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RunnerDomPipeline = void 0;
+const jsdom_1 = require("jsdom");
+const toon_1 = require("@toon-format/toon");
+const jsForAttributeInjection_js_1 = require("../../utils/DomExtraction/jsForAttributeInjection.js");
+const domSanitizer_js_1 = require("../../utils/helpers/domSanitizer.js");
+const logData_js_1 = require("../../utils/logger/logData.js");
+class RunnerDomPipeline {
+    static parseDomDocument(domHtml, platform) {
+        if (platform === "web") {
+            const parser = new jsdom_1.JSDOM((0, domSanitizer_js_1.sanitizeWebDom)(domHtml));
+            return parser.window.document;
+        }
+        else {
+            const parser = new jsdom_1.JSDOM(domHtml, {
+                contentType: "text/xml",
+            });
+            return parser.window.document;
+        }
+    }
+    static async captureAndParseDom(driver, platform, stepCount, action, isRetry = false) {
+        try {
+            let domInfo;
+            let doc;
+            if (platform === "web") {
+                domInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOM)(driver);
+                const fileName = isRetry
+                    ? `annotated-dom-retry-${stepCount}`
+                    : `annotated-dom-${stepCount}`;
+                logData_js_1.logger.saveDOM(domInfo.dom, fileName);
+                doc = RunnerDomPipeline.parseDomDocument(domInfo.dom, "web");
+            }
+            else {
+                domInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOMForMobile)(driver);
+                const fileName = isRetry
+                    ? `annotated-dom-Mob-retry-${stepCount}`
+                    : `annotated-dom-Mob-${stepCount}`;
+                logData_js_1.logger.saveDOM(domInfo.dom, fileName);
+                doc = RunnerDomPipeline.parseDomDocument(domInfo.dom, platform);
+            }
+            return { domInfo, doc };
+        }
+        catch (error) {
+            if (action === "verify") {
+                return { domInfo: null, doc: null };
+            }
+            throw error;
+        }
+    }
+    static async processRelevantDom(domProcessor, domInfo, keywords, stepCount, action, currentStep, platform, driver, isRetry = false) {
+        try {
+            let extractedRelevantDom;
+            let toonDom;
+            if (platform === "web") {
+                extractedRelevantDom = await domProcessor.webProcess({
+                    keywords: keywords,
+                    rawDom: domInfo?.dom,
+                    stepCount: stepCount,
+                    stepAction: action,
+                    step: currentStep,
+                    platform: platform,
+                    driver: driver,
+                });
+                toonDom = (0, toon_1.encode)(extractedRelevantDom || []);
+                const fileName = isRetry
+                    ? `relevant-dom-toon-retry-${stepCount}`
+                    : `relevant-dom-toon-${stepCount}`;
+                logData_js_1.logger.saveText(toonDom, fileName);
+            }
+            else {
+                extractedRelevantDom = await domProcessor.mobileProcess({
+                    keywords: keywords,
+                    rawDom: domInfo?.dom,
+                    stepCount: stepCount,
+                    stepAction: action,
+                    step: currentStep,
+                    platform: platform,
+                    driver: driver,
+                });
+                toonDom = (0, toon_1.encode)(extractedRelevantDom || []);
+                const fileName = isRetry
+                    ? `relevant-dom-toon-Mob-retry-${stepCount}`
+                    : `relevant-dom-toon-Mob-${stepCount}`;
+                logData_js_1.logger.saveText(toonDom, fileName);
+            }
+            return { extractedRelevantDom, toonDom };
+        }
+        catch (error) {
+            if (action === "verify") {
+                return { extractedRelevantDom: null, toonDom: undefined };
+            }
+            throw error;
+        }
+    }
+}
+exports.RunnerDomPipeline = RunnerDomPipeline;

package/dist/core/helpers/runnerKafkaReporter.d.ts

@@ -0,0 +1,17 @@
+import { fireflinkService } from "../../service/fireflink.service.js";
+import { AutomationRequest } from "../interfaces/executionDetails.js";
+import { IKafkaPayload } from "../interfaces/fireflinkScriptPayloadInterface.js";
+export declare class RunnerKafkaReporter {
+    private fireflinkServiceInstance;
+    private request;
+    constructor(fireflinkServiceInstance: fireflinkService, request: AutomationRequest);
+    sendLiveLog(message: any, messageType: string): Promise<void>;
+    sendOngoing(message: string): Promise<void>;
+    sendRetry(message: string): Promise<void>;
+    sendTermination(): Promise<void>;
+    sendError(errorDescription?: string): Promise<void>;
+    sendExecutionStopped(): Promise<void>;
+    sendNextBatch(): Promise<void>;
+    sendCompleted(): Promise<void>;
+    sendMetadata(kafkaPayload: IKafkaPayload): Promise<void>;
+}

package/dist/core/helpers/runnerKafkaReporter.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RunnerKafkaReporter = void 0;
+const messageType_js_1 = require("../types/messageType.js");
+const logData_js_1 = require("../../utils/logger/logData.js");
+class RunnerKafkaReporter {
+    constructor(fireflinkServiceInstance, request) {
+        this.fireflinkServiceInstance = fireflinkServiceInstance;
+        this.request = request;
+    }
+    async sendLiveLog(message, messageType) {
+        await this.fireflinkServiceInstance.produceMessageToKafka(this.request.kafkaMessageKey, {
+            topic: this.request.topic,
+            message: {
+                topic: "",
+                service: "",
+                chatId: "",
+                aiPromptPayload: {
+                    display: false,
+                    message: "",
+                    messageId: "",
+                    branchId: "",
+                    userId: this.request.userId,
+                    scriptId: "",
+                    scriptType: "",
+                    type: "logs",
+                    sessionId: "",
+                    licenseId: this.request.licenseId,
+                    projectId: this.request.projectId,
+                    liveLogs: {
+                        executionId: this.request.kafkaMessageKey,
+                        message,
+                        messageType,
+                    },
+                    attachment: [],
+                },
+            },
+        });
+    }
+    async sendOngoing(message) {
+        await this.sendLiveLog(message, messageType_js_1.MessageType.LiveLog.Ongoing);
+        logData_js_1.logger.info(`Message produced to Kafka topic -  ${JSON.stringify({ messageType: messageType_js_1.MessageType.LiveLog.Ongoing })}`);
+    }
+    async sendRetry(message) {
+        await this.sendLiveLog(message, messageType_js_1.MessageType.LiveLog.retry);
+    }
+    async sendTermination() {
+        logData_js_1.logger.info(`Execution terminated successfully. Script: ${this.request.scriptName}, Test Case ID: ${this.request.testCaseId}`);
+        await this.sendLiveLog("Execution terminated by the user", messageType_js_1.MessageType.LiveLog.Termination);
+    }
+    async sendError(errorDescription) {
+        await this.sendLiveLog(errorDescription ?? "errorDescription unknown", messageType_js_1.MessageType.LiveLog.Error);
+        logData_js_1.logger.info(`Message produced to Kafka topic -  ${JSON.stringify({ messageType: messageType_js_1.MessageType.LiveLog.Error })}`);
+    }
+    async sendExecutionStopped() {
+        await this.sendLiveLog("Execution stopped", messageType_js_1.MessageType.LiveLog.ExecutionStopped);
+        logData_js_1.logger.info(`Message produced to Kafka topic -  ${JSON.stringify({ messageType: messageType_js_1.MessageType.LiveLog.ExecutionStopped })}`);
+    }
+    async sendNextBatch() {
+        await this.sendLiveLog("Invoke the generate_script_from_scenario tool with the remaining test steps to continue the automation execution from where it left off.", messageType_js_1.MessageType.nextBatch);
+        logData_js_1.logger.info(`Message produced to Kafka topic -  ${JSON.stringify({ messageType: messageType_js_1.MessageType.nextBatch })}`);
+    }
+    async sendCompleted() {
+        await this.sendLiveLog("Execution completed", messageType_js_1.MessageType.LiveLog.Completed);
+        logData_js_1.logger.info(`Message produced to Kafka topic -  ${JSON.stringify({ messageType: messageType_js_1.MessageType.LiveLog.Completed })}`);
+    }
+    async sendMetadata(kafkaPayload) {
+        await this.sendLiveLog(kafkaPayload, messageType_js_1.MessageType.AutomationScriptMetaData);
+    }
+}
+exports.RunnerKafkaReporter = RunnerKafkaReporter;

package/dist/core/helpers/runnerPostActionHelper.d.ts

@@ -0,0 +1,2 @@
+import { platform as PlatformType } from "../types/scriptTypesType.js";
+export declare function handlePostActionWeb(platform: PlatformType, driver: any, action: string, beforeHandles: any, scriptAppender: any, stepCount: number): Promise<any | null>;

package/dist/core/helpers/runnerPostActionHelper.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handlePostActionWeb = handlePostActionWeb;
+const isAlertPresent_js_1 = require("../../utils/alertPopup/isAlertPresent.js");
+const jsForToaster_js_1 = require("../../utils/javascript/jsForToaster.js");
+const jsForAttributeInjection_js_1 = require("../../utils/DomExtraction/jsForAttributeInjection.js");
+const newTabTracker_js_1 = require("../../utils/newTab/newTabTracker.js");
+const logData_js_1 = require("../../utils/logger/logData.js");
+async function handlePostActionWeb(platform, driver, action, beforeHandles, scriptAppender, stepCount) {
+    let preCapturedDomInfo = null;
+    if (platform === "web") {
+        const alertPresent = await (0, isAlertPresent_js_1.isAlertPresent)(driver);
+        if (!alertPresent) {
+            if (action === "click") {
+                await driver.executeScript(jsForToaster_js_1.TOAST_WATCHER_JS, []);
+                await new Promise((resolve) => setTimeout(resolve, 100));
+                const postClickToastDetected = await driver.executeScript("return window.__toastDetected === true;", []);
+                if (postClickToastDetected) {
+                    logData_js_1.logger.info("Toast detected after click");
+                    preCapturedDomInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOM)(driver);
+                    logData_js_1.logger.saveDOM(preCapturedDomInfo.dom, `toaster-dom-${stepCount}`);
+                }
+            }
+        }
+        if (beforeHandles?.length > 0) {
+            await (0, newTabTracker_js_1.newTabTracker)(driver, action, beforeHandles, scriptAppender);
+        }
+    }
+    return preCapturedDomInfo;
+}

package/dist/core/helpers/stepActionInspector.d.ts

@@ -0,0 +1,5 @@
+import { StepProcessor } from "../main/stepProcessor.js";
+import { platform as PlatformType } from "../types/scriptTypesType.js";
+import { ResolvedStepAction } from "../interfaces/automationRunnerInterface.js";
+export { ResolvedStepAction };
+export declare function resolveStepActionFromDom(stepProcessor: StepProcessor, platform: PlatformType, action: string, toonDom: any, priorAndNextSteps: string[], currentStep: string, fileType?: string | null): Promise<ResolvedStepAction>;

package/dist/core/helpers/stepActionInspector.js

@@ -0,0 +1,113 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveStepActionFromDom = resolveStepActionFromDom;
+const promptType_js_1 = require("../types/promptType.js");
+const sameActionsHelper_js_1 = require("../../utils/helpers/sameActionsHelper.js");
+const InvalidFileTypeError_js_1 = require("../Errors/InvalidFileTypeError.js");
+async function resolveStepActionFromDom(stepProcessor, platform, action, toonDom, priorAndNextSteps, currentStep, fileType) {
+    let stepResult;
+    let stepAction;
+    if (action === "verify") {
+        if (fileType) {
+            const normalizedFileType = fileType.toLowerCase();
+            if (normalizedFileType !== "pdf") {
+                throw new InvalidFileTypeError_js_1.InvalidFileTypeError(normalizedFileType);
+            }
+            stepResult = await stepProcessor.getLLMResponse({
+                platform,
+                type: promptType_js_1.PromptType.FILE_VERIFY_PROMPT,
+                args: {
+                    extractedDomJson: toonDom,
+                    priorAndNextSteps,
+                },
+                input: { currentStep },
+            });
+        }
+        else {
+            stepResult = await stepProcessor.getLLMResponse({
+                platform,
+                type: promptType_js_1.PromptType.VERIFY_PROMPT,
+                args: {
+                    extractedDomJson: toonDom,
+                    priorAndNextSteps,
+                },
+                input: { currentStep },
+            });
+        }
+        stepAction = stepResult.response.nlpName;
+    }
+    else if (action === "get") {
+        stepResult = await stepProcessor.getLLMResponse({
+            platform,
+            type: promptType_js_1.PromptType.GET_PROMPT,
+            args: {
+                extractedDomJson: toonDom,
+                priorAndNextSteps,
+            },
+            input: { currentStep },
+        });
+        stepAction = stepResult.response.nlpName;
+    }
+    else if (action === "wait") {
+        stepResult = await stepProcessor.getLLMResponse({
+            platform,
+            type: promptType_js_1.PromptType.WAIT_ACTION_EXTRACTOR_PROMPT,
+            args: {
+                extractedDomJson: toonDom,
+                priorAndNextSteps,
+            },
+            input: { currentStep },
+        });
+        stepAction = stepResult.response.nlpName;
+    }
+    else if (action === "combined") {
+        stepResult = await stepProcessor.getLLMResponse({
+            platform,
+            type: promptType_js_1.PromptType.COMBINED_ACTION_EXTRACTOR_PROMPT,
+            args: {
+                extractedDomJson: toonDom,
+                priorAndNextSteps,
+            },
+            input: { currentStep },
+        });
+        stepAction = stepResult.response.nlpName;
+    }
+    else if (action === "drag_and_drop") {
+        stepResult = await stepProcessor.getLLMResponse({
+            platform,
+            type: promptType_js_1.PromptType.FF_INSPECTOR,
+            args: {
+                extractedDomJson: toonDom,
+                priorAndNextSteps,
+                isDrag: true,
+                isAlert: false,
+            },
+            input: { currentStep },
+        });
+        stepAction = stepResult.response.nlpName;
+    }
+    else {
+        stepResult = await stepProcessor.getLLMResponse({
+            platform,
+            type: promptType_js_1.PromptType.FF_INSPECTOR,
+            args: {
+                stepAction: action,
+                extractedDomJson: toonDom,
+                priorAndNextSteps,
+                isAlert: false,
+                isDrag: false,
+            },
+            input: { currentStep },
+        });
+        if (platform === "web") {
+            stepAction = stepResult.response.action;
+        }
+        else {
+            stepAction = stepResult.response.nlpName;
+        }
+    }
+    if (platform === "android" || platform === "ios") {
+        stepAction = sameActionsHelper_js_1.sameActionHelper[stepAction] || stepAction;
+    }
+    return { stepResult, stepAction };
+}

package/dist/core/helpers/visionFallbackHandler.d.ts

@@ -0,0 +1,3 @@
+import { VisionFallbackParams, VisionFallbackResult } from "../interfaces/automationRunnerInterface.js";
+export { VisionFallbackParams, VisionFallbackResult };
+export declare function executeVisionFallback(params: VisionFallbackParams): Promise<VisionFallbackResult>;

package/dist/core/helpers/visionFallbackHandler.js

@@ -0,0 +1,138 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeVisionFallback = executeVisionFallback;
+const generateAnnotatedScreenshots_js_1 = require("../../imageAnalysisWeb/generateAnnotatedScreenshots.js");
+const jsForAttributeInjection_js_1 = require("../../utils/DomExtraction/jsForAttributeInjection.js");
+const annotatedScreenshotMobile_js_1 = require("../../imageAnalysisMobile/annotatedScreenshotMobile.js");
+const xpathcreation_js_1 = require("../../utils/helpers/xpathcreation.js");
+const promptType_js_1 = require("../types/promptType.js");
+const runnerPostActionHelper_js_1 = require("./runnerPostActionHelper.js");
+const runnerDomPipeline_js_1 = require("./runnerDomPipeline.js");
+const logData_js_1 = require("../../utils/logger/logData.js");
+async function executeVisionFallback(params) {
+    const { driver, platform, action, step, stepAction, inputValue, stepCount, fallbackAttempt, priorAndNextSteps, listOfSteps, currentElementName, extractorElementName, stepElementType, iframeDomHtml, beforeHandles, context, stepProcessor, actionHandlers, reporter, causeError, } = params;
+    let domInfo = params.domInfo;
+    let doc = params.doc;
+    logData_js_1.logger.error(`execution failed for step "${step}": ${causeError} and going for vision`);
+    try {
+        let annotatedScreenshot;
+        let bounds;
+        if (action === "scroll") {
+            throw new Error("Scroll action failed element not found", { cause: causeError });
+        }
+        if (platform === "web") {
+            annotatedScreenshot = await (0, generateAnnotatedScreenshots_js_1.generateAnnotatedScreenshots)(driver);
+        }
+        else {
+            domInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOMForMobile)(driver);
+            let visionResult;
+            if (platform === "android") {
+                visionResult = await (0, annotatedScreenshotMobile_js_1.annotateScreenshot)(driver, domInfo.dom);
+            }
+            else {
+                visionResult = await (0, annotatedScreenshotMobile_js_1.annotateIosScreenshot)(driver, domInfo.dom);
+            }
+            annotatedScreenshot = visionResult.boxedImageBuffer;
+            bounds = visionResult.boxes;
+        }
+        let stepResultVision;
+        if (platform === "web") {
+            stepResultVision = await stepProcessor.getLLMResponse({
+                platform,
+                type: promptType_js_1.PromptType.VISION_PROMPT,
+                args: { priorAndNextSteps },
+                input: { currentStep: step, annotatedScreenshot: annotatedScreenshot.primary },
+                isVision: true,
+            });
+            if (stepResultVision.response.fireflinkIndex === "Fire-Flink-0") {
+                stepResultVision = await stepProcessor.getLLMResponse({
+                    platform,
+                    type: promptType_js_1.PromptType.VISION_PROMPT,
+                    args: { priorAndNextSteps },
+                    input: { currentStep: step, annotatedScreenshot: annotatedScreenshot.secondary },
+                    isVision: true,
+                });
+            }
+        }
+        else {
+            stepResultVision = await stepProcessor.getLLMResponse({
+                platform,
+                type: promptType_js_1.PromptType.VISION_PROMPT,
+                args: { priorAndNextSteps },
+                input: { currentStep: step, annotatedScreenshot },
+                isVision: true,
+            });
+        }
+        let fireflinkIndex = stepResultVision.response.fireflinkIndex;
+        logData_js_1.logger.info(`fireflinkIndex: ${fireflinkIndex}`);
+        if (fireflinkIndex === "Fire-Flink-0") {
+            logData_js_1.logger.error(`vision unable to find element for step "${step}"`);
+            if (platform === "web") {
+                context.scriptAppender.setErrorNLP(stepAction);
+            }
+            throw new Error(`vision unable to find element for step "${step}"`, { cause: causeError });
+        }
+        let xpath;
+        if (platform === "android" || platform === "ios") {
+            const index = Number(fireflinkIndex.split("-").pop());
+            const box = bounds[index - 1];
+            if (!box) {
+                throw new Error(`No bounds found for index ${index}`, { cause: causeError });
+            }
+            const { x1, y1, x2, y2 } = box;
+            const boundsStr = `[${x1},${y1}][${x2},${y2}]`;
+            const result = (0, xpathcreation_js_1.getFFIndexAndXPath)(domInfo.dom, boundsStr, platform);
+            xpath = result.xpathPath;
+            fireflinkIndex = result.ffInspector ?? fireflinkIndex;
+        }
+        else {
+            if (!domInfo || !domInfo.dom || !doc) {
+                logData_js_1.logger.info("Re-fetching annotated DOM for vision fallback");
+                domInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOM)(driver);
+                doc = runnerDomPipeline_js_1.RunnerDomPipeline.parseDomDocument(domInfo.dom, "web");
+            }
+            const ele = doc.querySelector(`[ff-inspect="${fireflinkIndex}"]`);
+            if (ele) {
+                xpath = ele.getAttribute("ff-xpath") || "";
+            }
+            else {
+                const result = (0, xpathcreation_js_1.getWebXPathByFFIndex)(domInfo.dom, fireflinkIndex);
+                xpath = result.xpathPath;
+            }
+            if (!xpath) {
+                context.scriptAppender.setErrorNLP(stepAction);
+                throw new Error(`Unable to resolve xpath for ${fireflinkIndex}`, { cause: causeError });
+            }
+        }
+        const pageDOM = iframeDomHtml ?? domInfo.dom;
+        const handler = actionHandlers[stepAction];
+        await handler({
+            selector: xpath,
+            value: inputValue,
+            fireflinkIndex,
+            pageDOM,
+            elementName: currentElementName || extractorElementName,
+            elementType: stepElementType,
+            stepIndex: params.stepIndex ?? stepCount,
+            currentStep: params.currentStep ?? step,
+            manualStep: params.manualStep ?? step,
+            stepsInputData: params.stepsInputData,
+            filteredSteps: listOfSteps,
+            returnTypeVariableName: params.returnTypeVariableName,
+            ...(params.variableName ? { variableName: params.variableName } : {}),
+        });
+        const preCapturedDomInfo = await (0, runnerPostActionHelper_js_1.handlePostActionWeb)(platform, driver, action, beforeHandles, context.scriptAppender, stepCount);
+        if (fallbackAttempt === 2) {
+            await reporter.sendRetry("Retry successful. Moving on to the next step.");
+        }
+        return { preCapturedDomInfo, retryNeeded: false };
+    }
+    catch (error) {
+        logData_js_1.logger.error(`vision execution failed for step "${step}": ${error}`);
+        if (fallbackAttempt < 2) {
+            await reporter.sendRetry("Step failed to execute. Waiting a few seconds before reattempting the same step.");
+            return { preCapturedDomInfo: null, retryNeeded: true };
+        }
+        throw error;
+    }
+}