ff-automationv2 2.2.35 → 2.2.36-beta.2

package/dist/ai/llmprompts/systemPrompts/sliderSwipeVisionPrompt.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.swipeVisionPrompt = swipeVisionPrompt;
+async function swipeVisionPrompt({ priorAndNextSteps }) {
+    const prompt = `
+You are a mobile automation assistant analyzing a swipe action from a screenshot.
+
+Given:
+- A screenshot with UI elements labeled using Fire-Flink indexes.
+- The user's swipe step.
+- Context: ${JSON.stringify(priorAndNextSteps)}.
+
+Your first task is to identify the TARGET ELEMENT mentioned or implied by the user's step.
+
+Then determine whether that exact target element is visually a SLIDER/DRAG CONTROL in the screenshot.
+
+IMPORTANT:
+- Return NormalSwipe=false ONLY if ALL conditions are true:
+  1. The target from the user's step is present in the screenshot.
+  2. The identified target corresponds to a specific UI control.
+  3. That control is visually designed to be dragged/slid from one position to another.
+  4. A parent/container defining the slider movement area can be identified.
+- If the target is a normal text, button, link, image, icon, label, or any other non-slider element, return NormalSwipe=true.
+- If the target exists but is NOT a slider/drag control, return NormalSwipe=true.
+- If the target cannot be found, return NormalSwipe=true.
+- Never classify an element as a slider only because the step contains words such as "swipe", "slide", "to", or because the element has a suitable size.
+- Do not assume that any horizontally or vertically shaped element is a slider.
+- The slider must have a clear visual indication that a specific control/thumb can be dragged.
+- Do not use surrounding elements as the target unless they are actually the draggable control.
+- For a valid slider, identify both:
+  - targetIndex = draggable thumb/control
+  - parentIndex = slider/container defining the movement boundary
+- Determine the direction from the user's step and UI.
+
+For a valid slider:
+{
+  "NormalSwipe": false,
+  "parentIndex": "Fire-Flink-x",
+  "targetIndex": "Fire-Flink-x",
+  "direction": "left|right|up|down"
+}
+
+For every other case:
+{
+  "NormalSwipe": true,
+  "parentIndex": "Fire-Flink-0",
+  "targetIndex": "Fire-Flink-0",
+  "direction": "left|right|up|down|none"
+}
+
+Return ONLY valid JSON.
+`.trim();
+    return prompt;
+}

package/dist/ai/llmprompts/systemPrompts/verifyActionExtractorPromptMob.js

@@ -2,51 +2,51 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.verifyActionExtractorPromptMob = verifyActionExtractorPromptMob;
 async function verifyActionExtractorPromptMob({ extractedDomJson, priorAndNextSteps, }) {
-    const nlpList = `  "MOB_VerifyElementIsDisplayed,MOB_VerifyElementIsDisabled,MOB_VerifyElementIsEnabled,MOB_VerifyElementIsSelected,MOB_VerifyText,MOB_VerifyAttributeValue,MOB_VerifyPartialAttributeValue,"
-    "MOB_VerifyElementIsDisplayedInSpecifiedTime,MOB_VerifyElementIsEnabledInSpecifiedTime,"
-    "MOB_VerifyBatteryLevel,MOB_VerifyPartialText,MOB_VerifyElementIsClickable,"
-    "MOB_VerifyIfKeyboardIsHidden,MOB_VerifyIfKeyboardIsShown,MOB_VerifyIfXAndYCoordinateOfElement,"
-    "MOB_VerifyHeightOfElement,MOB_VerifyXLocationOfAndroidElement,MOB_VerifyIfDeviceIsUnlocked,"
-    "MOB_VerifyIfWifiOff,MOB_VerifyElementIsSelectedInSpecifiedTime,MOB_VerifyIfWifiOn,MOB_VerifyIfDeviceIsLocked,"
-    "MOB_VerifyIfAirplaneModeOff,MOB_VerifyIfGpsOff,MOB_VerifyHexCodeForGivenXYCoordinates,"
-    "MOB_VerifyYLocationOfAndroidElement,MOB_VerifyIfGpsOn,MOB_VerifyIfAirplaneModeOn,"
-    "MOB_VerifyWidthOfElement,MOB_VerifyElementIsNotDisplayed,MOB_VerifyElementNotContainsText,"
+    const nlpList = `  "MOB_VerifyElementIsDisplayed,MOB_VerifyElementIsDisabled,MOB_VerifyElementIsEnabled,MOB_VerifyElementIsSelected,MOB_VerifyText,MOB_VerifyAttributeValue,MOB_VerifyPartialAttributeValue,"
+    "MOB_VerifyElementIsDisplayedInSpecifiedTime,MOB_VerifyElementIsEnabledInSpecifiedTime,"
+    "MOB_VerifyBatteryLevel,MOB_VerifyPartialText,MOB_VerifyElementIsClickable,"
+    "MOB_VerifyIfKeyboardIsHidden,MOB_VerifyIfKeyboardIsShown,MOB_VerifyIfXAndYCoordinateOfElement,"
+    "MOB_VerifyHeightOfElement,MOB_VerifyXLocationOfAndroidElement,MOB_VerifyIfDeviceIsUnlocked,"
+    "MOB_VerifyIfWifiOff,MOB_VerifyElementIsSelectedInSpecifiedTime,MOB_VerifyIfWifiOn,MOB_VerifyIfDeviceIsLocked,"
+    "MOB_VerifyIfAirplaneModeOff,MOB_VerifyIfGpsOff,MOB_VerifyHexCodeForGivenXYCoordinates,"
+    "MOB_VerifyYLocationOfAndroidElement,MOB_VerifyIfGpsOn,MOB_VerifyIfAirplaneModeOn,"
+    "MOB_VerifyWidthOfElement,MOB_VerifyElementIsNotDisplayed,MOB_VerifyElementNotContainsText,"
     "MOB_VerifyIfAppIsClosed, MOB_CheckIfElementIsDisplayed"`;
     const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"];
-    const prompt = `You are an intelligent assistant that extracts structured UI action data.
-Given the step: step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
-
-Select the perfect matching NLP name from the list that best fits the step's intent.
-
-Return **only valid JSON** in the following format:
-{
-  "attributeValue": "Fire-Flink-x",
-  "nlpName": "x",
-  "inputText":"x"
-  "elementType": "x",
-  "locatorType":"x"
-}
-Rules:
-- nlpName must be exactly one from the provided list.
-- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
-- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
-- For some steps you can see two inputs but you need to uderstand step and send the requred inputText.
-        ex: Verify the flights matches the flight,Verify the home has 30 and 50 coordinates
-        here keyword : "flights" , input : flight, here input:30,50
-- If step mention time and element name then inputText should be time.
-        example: Verify the home is displayed for 30 seconds 
-        here keyword : "home" , input : 30 only they mention seconds as input in step
-- If step mention verify text then only pick MOB_VerifyText nlp otherwise pick MOB_VerifyElementIsDisplayed.
-- If step mention verify x and y coordinate of element then only pick MOB_VerifyIfXAndYCoordinateOfElement.
-- Extract inputText from the step that is which is being verified in the step if they mention verifying two values in inputs take as it is,if they mention one value take only that as input text, if you can't find any input text in the step, return keyword as inputText.
-- if the step is about checking the element is displayed then pick MOB_CheckIfElementIsDisplayed nlp.
-- For MOB_VerifyHeightOfElement just provide height mentioned in the step as the inputText don't add anything else 
-- Extract keyword from the step, if the step contains any keyword.
-- Use the closest semantic match for the step; return attributeValue as Fire-Flink-0, only if nothing is found.
-- Based on step give most relevant type of element. use this list to choose elementType: ${elementType} and Never change syntax of elementType, follow the syntax of elementType in list.if elementType is not there in list return 'link'.
-- Extract locator_type if mentioned in the step, like(text,resource-id,content-desc) . if not found return 'empty'.
-- Return ONLY a valid JSON object; the response must start with '{' and end with '}', with no markdown, code fences, explanations, or extra text.
-- Respond with valid JSON only.
+    const prompt = `You are an intelligent assistant that extracts structured UI action data.
+Given the step: step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
+
+Select the perfect matching NLP name from the list that best fits the step's intent.
+
+Return **only valid JSON** in the following format:
+{
+  "attributeValue": "Fire-Flink-x",
+  "nlpName": "x",
+  "inputText":"x"
+  "elementType": "x",
+  "locatorType":"x"
+}
+Rules:
+- nlpName must be exactly one from the provided list.
+- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
+- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
+- For some steps you can see two inputs but you need to uderstand step and send the requred inputText.
+        ex: Verify the flights matches the flight,Verify the home has 30 and 50 coordinates
+        here keyword : "flights" , input : flight, here input:30,50
+- If step mention time and element name then inputText should be time.
+        example: Verify the home is displayed for 30 seconds 
+        here keyword : "home" , input : 30 only they mention seconds as input in step
+- If step mention verify text then only pick MOB_VerifyText nlp otherwise pick MOB_VerifyElementIsDisplayed.
+- If step mention verify x and y coordinate of element then only pick MOB_VerifyIfXAndYCoordinateOfElement.
+- Extract inputText from the step that is which is being verified in the step if they mention verifying two values in inputs take as it is,if they mention one value take only that as input text, if you can't find any input text in the step, return keyword as inputText.
+- if the step is about checking the element is displayed then pick MOB_CheckIfElementIsDisplayed nlp.
+- For MOB_VerifyHeightOfElement just provide height mentioned in the step as the inputText don't add anything else 
+- Extract keyword from the step, if the step contains any keyword.
+- Use the closest semantic match for the step; return attributeValue as Fire-Flink-0, only if nothing is found.
+- Based on step give most relevant type of element. use this list to choose elementType: ${elementType} and Never change syntax of elementType, follow the syntax of elementType in list.if elementType is not there in list return 'link'.
+- Extract locator_type if mentioned in the step, like(text,resource-id,content-desc) . if not found return 'empty'.
+- Return ONLY a valid JSON object; the response must start with '{' and end with '}', with no markdown, code fences, explanations, or extra text.
+- Respond with valid JSON only.
 `;
     return prompt;
 }

package/dist/ai/llmprompts/userPrompts/userPrompt.js

@@ -38,4 +38,25 @@ exports.userInputFormatters = {
         ];
         return content;
     },
+    [promptType_js_1.PromptType.SWIPE_VISION_PROMPT]: (userInput) => {
+        let imageBase64 = null;
+        if (userInput.annotatedScreenshot) {
+            imageBase64 = (0, imageUtils_js_1.formatImageBase64)(userInput.annotatedScreenshot);
+        }
+        if (!imageBase64) {
+            throw new Error("Vision image is missing. No annotatedScreenshot provided.");
+        }
+        const content = [
+            {
+                type: "input_text",
+                text: `Analyze the following step for swipe action:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
+            },
+            {
+                type: "input_image",
+                image_url: imageBase64,
+                detail: "auto",
+            },
+        ];
+        return content;
+    },
 };

package/dist/automation/actions/executor.d.ts

@@ -359,6 +359,7 @@ export declare class ActionExecutor implements IActionExecutor {
     swipeRightToElement(value: string, elementName: string, elementType: string, numOfScrolls: number): Promise<void>;
     swipeNTimes(numOfScrolls: string, direction?: string): Promise<void>;
     swipeDirectionNTimes(numOfScrolls: string, direction: string): Promise<void>;
+    swipeSlider(parentXpath: string, targetXpath: string, direction?: string, elementName?: string, elementType?: string, parentIndex?: string, targetIndex?: string, pageDOM?: string): Promise<void>;
     swipeToElementForMWeb(value: string, elementName: string, elementType: string, pageDOM: string, fireflinkIndex: string, numOfScrolls: number): Promise<void>;
     swipeUsingReferenceElement(value: string, elementName: string, elementType: string, numOfScrolls: string, direction: string, pageDOM: string, fireflinkIndex: string): Promise<void>;
     MOB_EnterInputIntoElementFromClipBoard(selector: string, elementName: string, elementType: string, pageDOM: string, fireflinkIndex: string): Promise<void>;

package/dist/automation/actions/executor.js

@@ -268,6 +268,7 @@ const swipeDownToElement_js_1 = require("./interaction/swipe/swipeDownToElement.
 const swipeLeftToElement_js_1 = require("./interaction/swipe/swipeLeftToElement.js");
 const swipeRightToElement_js_1 = require("./interaction/swipe/swipeRightToElement.js");
 const swipeDirectionNTimes_js_1 = require("./interaction/swipe/swipeDirectionNTimes.js");
+const slider_Swipe_js_1 = require("./interaction/swipe/slider_Swipe.js");
 const swipeUsingReferenceElement_js_1 = require("./interaction/swipe/swipeUsingReferenceElement.js");
 const swipeUpToElementForMWeb_js_1 = require("./interaction/swipe/swipeUpToElementForMWeb.js");
 const EnterInputIntoElementFromClipBoardInterface_mob_js_1 = require("./interaction/enterActions/EnterInputIntoElementFromClipBoardInterface_mob.js");
@@ -6428,6 +6429,28 @@ class ActionExecutor {
             throw error;
         }
     }
+    async swipeSlider(parentXpath, targetXpath, direction, elementName, elementType, parentIndex, targetIndex, pageDOM) {
+        try {
+            await (0, slider_Swipe_js_1.swipeSlider)({
+                driver: this.driver,
+                parentXpath,
+                targetXpath,
+                direction,
+                platform: this.platform,
+                scriptDataAppender: this.scriptDataAppender,
+                elementName,
+                elementType,
+                parentIndex,
+                targetIndex,
+                pageDOM,
+                elementGetter: this.elementGetter
+            });
+        }
+        catch (error) {
+            logData_js_1.logger.error("Error in [ActionExecutor]", error);
+            throw error;
+        }
+    }
     async swipeToElementForMWeb(value, elementName, elementType, pageDOM, fireflinkIndex, numOfScrolls) {
         try {
             await (0, swipeUpToElementForMWeb_js_1.swipeUpToElementForMWeb)({

package/dist/automation/actions/interaction/swipe/slider_Swipe.d.ts

@@ -0,0 +1,2 @@
+import type { swipeSliderI } from "../../interface/swipeActionInterface.js";
+export declare function swipeSlider(args: swipeSliderI): Promise<void>;

package/dist/automation/actions/interaction/swipe/slider_Swipe.js

@@ -0,0 +1,129 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.swipeSlider = swipeSlider;
+const logData_js_1 = require("../../../../utils/logger/logData.js");
+const appendScriptData_js_1 = require("../../../../fireflinkData/fireflinkScript/appendScriptData.js");
+async function swipeSlider(args) {
+    const nlpName = args.platform === "ios"
+        ? "IOS_Swipe"
+        : "MOB_SwipeNTimes";
+    try {
+        const parent = await args.driver.$(args.parentXpath);
+        const target = await args.driver.$(args.targetXpath);
+        if (!(await parent.isExisting())) {
+            throw new Error(`Slider parent not found: ${args.parentXpath}`);
+        }
+        if (!(await target.isExisting())) {
+            throw new Error(`Slider target not found: ${args.targetXpath}`);
+        }
+        const parentLocation = await parent.getLocation();
+        const parentSize = await parent.getSize();
+        const targetLocation = await target.getLocation();
+        const targetSize = await target.getSize();
+        const parentRect = {
+            x: parentLocation.x,
+            y: parentLocation.y,
+            width: parentSize.width,
+            height: parentSize.height
+        };
+        const targetRect = {
+            x: targetLocation.x,
+            y: targetLocation.y,
+            width: targetSize.width,
+            height: targetSize.height
+        };
+        const direction = (args.direction || "right")
+            .toLowerCase()
+            .trim();
+        let startX;
+        let startY;
+        let endX;
+        let endY;
+        switch (direction) {
+            case "right":
+                startX = targetRect.x + targetRect.width / 2;
+                startY = targetRect.y + targetRect.height / 2;
+                endX = parentRect.x + parentRect.width - 10;
+                endY = startY;
+                break;
+            case "left":
+                startX = targetRect.x + targetRect.width / 2;
+                startY = targetRect.y + targetRect.height / 2;
+                endX = parentRect.x + 10;
+                endY = startY;
+                break;
+            case "down":
+                startX = targetRect.x + targetRect.width / 2;
+                startY = targetRect.y + targetRect.height / 2;
+                endX = startX;
+                endY = parentRect.y + parentRect.height - 10;
+                break;
+            case "up":
+                startX = targetRect.x + targetRect.width / 2;
+                startY = targetRect.y + targetRect.height / 2;
+                endX = startX;
+                endY = parentRect.y + 10;
+                break;
+            default:
+                throw new Error(`Unsupported swipe direction: ${direction}`);
+        }
+        startX = Math.round(startX);
+        startY = Math.round(startY);
+        endX = Math.round(endX);
+        endY = Math.round(endY);
+        logData_js_1.logger.info(`Slider swipe: (${startX}, ${startY}) -> (${endX}, ${endY})`);
+        logData_js_1.logger.info(`direction = ${direction}`);
+        logData_js_1.logger.info(`startX = ${startX}`);
+        logData_js_1.logger.info(`startY = ${startY}`);
+        logData_js_1.logger.info(`endX = ${endX}`);
+        logData_js_1.logger.info(`endY = ${endY}`);
+        await args.driver.performActions([
+            {
+                type: "pointer",
+                id: "finger1",
+                parameters: {
+                    pointerType: "touch"
+                },
+                actions: [
+                    {
+                        type: "pointerMove",
+                        duration: 0,
+                        x: startX,
+                        y: startY
+                    },
+                    {
+                        type: "pointerDown",
+                        button: 0
+                    },
+                    {
+                        type: "pause",
+                        duration: 100
+                    },
+                    {
+                        type: "pointerMove",
+                        duration: 1000,
+                        x: endX,
+                        y: endY
+                    },
+                    {
+                        type: "pointerUp",
+                        button: 0
+                    }
+                ]
+            }
+        ]);
+        await args.driver.releaseActions();
+        await args.driver.pause(1000);
+        const stepInputs = args.platform === "ios" ? [startX, startY, endX, endY] : [startX, startY, endX, endY, 1];
+        (0, appendScriptData_js_1.appendScriptData)({
+            scriptDataAppender: args.scriptDataAppender,
+            nlpName,
+            platform: args.platform,
+            stepInputs: stepInputs,
+        });
+    }
+    catch (error) {
+        args.scriptDataAppender.setErrorNLP(nlpName);
+        throw new Error("Slider swipe action failed", { cause: error });
+    }
+}

package/dist/automation/actions/interface/FireFlinkResponseSchema.d.ts

@@ -210,6 +210,12 @@ export declare const DefaultInteractionSchema: z.ZodObject<{
 export declare const VisionSchema: z.ZodObject<{
     fireflinkIndex: z.ZodString;
 }, z.core.$strip>;
+export declare const SwipeVisionSchema: z.ZodObject<{
+    NormalSwipe: z.ZodBoolean;
+    parentIndex: z.ZodString;
+    targetIndex: z.ZodString;
+    direction: z.ZodString;
+}, z.core.$strip>;
 export declare const UserStorySchema: z.ZodObject<{
     manualSteps: z.ZodArray<z.ZodString>;
 }, z.core.$strip>;

package/dist/automation/actions/interface/FireFlinkResponseSchema.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ArithmeticSchema = exports.InputContextSchema = exports.OtpExtractorSchema = exports.ErrorDescriptionSchema = exports.UserStorySchema = exports.VisionSchema = exports.DefaultInteractionSchema = exports.MouseActionSchema = exports.UploadActionSchema = exports.GetOtpActionSchema = exports.ClearTextActionSchema = exports.EnterActionSchema = exports.TapActionSchema = exports.ClickActionSchema = exports.SwitchActionSchema = exports.MobileScrollSchema = exports.WebScrollSchema = exports.DragDropActionSchema = exports.MobileCombinedSchema = exports.CombinedActionSchema = exports.WaitActionSchema = exports.MobileGetSchema = exports.MobileFileVerifySchema = exports.MobileVerifySchema = exports.WebGetSchema = exports.WebFileVerifySchema = exports.WebVerifySchema = exports.AlertShema = exports.MultichannelKeywordSchema = exports.MultichannelStepItemSchema = exports.KeywordSchema = exports.StepItemSchema = void 0;
+exports.ArithmeticSchema = exports.InputContextSchema = exports.OtpExtractorSchema = exports.ErrorDescriptionSchema = exports.UserStorySchema = exports.SwipeVisionSchema = exports.VisionSchema = exports.DefaultInteractionSchema = exports.MouseActionSchema = exports.UploadActionSchema = exports.GetOtpActionSchema = exports.ClearTextActionSchema = exports.EnterActionSchema = exports.TapActionSchema = exports.ClickActionSchema = exports.SwitchActionSchema = exports.MobileScrollSchema = exports.WebScrollSchema = exports.DragDropActionSchema = exports.MobileCombinedSchema = exports.CombinedActionSchema = exports.WaitActionSchema = exports.MobileGetSchema = exports.MobileFileVerifySchema = exports.MobileVerifySchema = exports.WebGetSchema = exports.WebFileVerifySchema = exports.WebVerifySchema = exports.AlertShema = exports.MultichannelKeywordSchema = exports.MultichannelStepItemSchema = exports.KeywordSchema = exports.StepItemSchema = void 0;
 exports.getSchemaForPrompt = getSchemaForPrompt;
 const zod_1 = require("zod");
 const promptType_js_1 = require("../../../core/types/promptType.js");
@@ -183,6 +183,12 @@ exports.DefaultInteractionSchema = zod_1.z.object({
 exports.VisionSchema = zod_1.z.object({
     fireflinkIndex: zod_1.z.string(),
 });
+exports.SwipeVisionSchema = zod_1.z.object({
+    NormalSwipe: zod_1.z.boolean(),
+    parentIndex: zod_1.z.string(),
+    targetIndex: zod_1.z.string(),
+    direction: zod_1.z.string(),
+});
 exports.UserStorySchema = zod_1.z.object({
     manualSteps: zod_1.z.array(zod_1.z.string()),
 });
@@ -210,6 +216,7 @@ const schemaMap = {
     [promptType_js_1.PromptType.KEYWORD_EXTRACTOR]: exports.KeywordSchema,
     [promptType_js_1.PromptType.MULTICHANNEL_KEYWORD_EXTRACTOR]: exports.MultichannelKeywordSchema,
     [promptType_js_1.PromptType.VISION_PROMPT]: exports.VisionSchema,
+    [promptType_js_1.PromptType.SWIPE_VISION_PROMPT]: exports.SwipeVisionSchema,
     [promptType_js_1.PromptType.USER_STORY_TO_LIST]: exports.UserStorySchema,
     [promptType_js_1.PromptType.ERROR_DESCRIPTION]: exports.ErrorDescriptionSchema,
     [promptType_js_1.PromptType.WAIT_ACTION_EXTRACTOR_PROMPT]: exports.WaitActionSchema,

package/dist/automation/actions/interface/swipeActionInterface.d.ts

@@ -102,3 +102,18 @@ export interface swipe {
     scriptDataAppender: ScriptDataAppender;
     platform: string;
 }
+export interface swipeSliderI {
+    driver: WebdriverIO.Browser;
+    parentXpath: string;
+    targetXpath: string;
+    xpath?: string;
+    platform: string;
+    direction?: string;
+    scriptDataAppender: ScriptDataAppender;
+    elementName?: string;
+    elementType?: string;
+    parentIndex?: string;
+    targetIndex?: string;
+    elementGetter: ElementGetter;
+    pageDOM?: any;
+}

package/dist/core/helpers/mobileSwipeHelper.d.ts

@@ -0,0 +1,3 @@
+import { MobileSwipeSliderParams, MobileSwipeSliderResult } from "../interfaces/mobileSwipeSliderInterface.js";
+export { MobileSwipeSliderParams, MobileSwipeSliderResult };
+export declare function checkMobileSwipeSlider(params: MobileSwipeSliderParams): Promise<MobileSwipeSliderResult>;

package/dist/core/helpers/mobileSwipeHelper.js

@@ -0,0 +1,78 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.checkMobileSwipeSlider = checkMobileSwipeSlider;
+const annotatedScreenshotMobile_js_1 = require("../../imageAnalysisMobile/annotatedScreenshotMobile.js");
+const jsForAttributeInjection_js_1 = require("../../utils/DomExtraction/jsForAttributeInjection.js");
+const promptType_js_1 = require("../types/promptType.js");
+const xpathcreation_js_1 = require("../../utils/helpers/xpathcreation.js");
+async function checkMobileSwipeSlider(params) {
+    const { driver, currentPlatform, stepAction, step, stepProcessor, priorAndNextSteps, } = params;
+    let domInfo = params.domInfo;
+    const doc = params.doc;
+    if (stepAction !== "MOB_SwipeToElement" || (currentPlatform !== "android" && currentPlatform !== "ios")) {
+        return { isSlider: false, domInfo, doc };
+    }
+    if (!domInfo?.dom) {
+        domInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOMForMobile)(driver);
+    }
+    let visionAnnotationResult;
+    if (currentPlatform === "android") {
+        visionAnnotationResult = await (0, annotatedScreenshotMobile_js_1.annotateScreenshot)(driver, domInfo.dom);
+    }
+    else if (currentPlatform === "ios") {
+        visionAnnotationResult = await (0, annotatedScreenshotMobile_js_1.annotateIosScreenshot)(driver, domInfo.dom);
+    }
+    const stepResultVision = await stepProcessor.getLLMResponse({
+        platform: currentPlatform,
+        type: promptType_js_1.PromptType.SWIPE_VISION_PROMPT,
+        args: { priorAndNextSteps },
+        input: {
+            currentStep: step,
+            annotatedScreenshot: visionAnnotationResult?.boxedImageBuffer,
+        },
+        isVision: true,
+    });
+    const NormalSwipe = stepResultVision.response.NormalSwipe;
+    const parentIndex = stepResultVision.response.parentIndex || stepResultVision.response.fireflinkIndex;
+    const targetIndex = stepResultVision.response.targetIndex || stepResultVision.response.fireflinkIndex;
+    const direction = stepResultVision.response.direction?.toLowerCase()?.trim() || "right";
+    if (!NormalSwipe &&
+        parentIndex &&
+        targetIndex &&
+        parentIndex !== "Fire-Flink-0" &&
+        targetIndex !== "Fire-Flink-0") {
+        const bounds = visionAnnotationResult?.boxes;
+        const resolveBoundsAndXpath = (idxStr) => {
+            const idxNum = Number(idxStr.split("-").pop());
+            const box = bounds?.[idxNum - 1];
+            if (!box) {
+                throw new Error(`No bounds found for index ${idxStr} (${idxNum})`);
+            }
+            const { x1, y1, x2, y2 } = box;
+            const boundsStr = `[${x1},${y1}][${x2},${y2}]`;
+            const result = (0, xpathcreation_js_1.getFFIndexAndXPath)(domInfo.dom, boundsStr, currentPlatform);
+            if (!result.xpathPath) {
+                throw new Error(`Unable to resolve xpath for element ${idxStr}`);
+            }
+            return {
+                xpath: result.xpathPath,
+                resolvedIndex: result.ffInspector ?? idxStr,
+            };
+        };
+        const parentResult = resolveBoundsAndXpath(parentIndex);
+        const targetResult = resolveBoundsAndXpath(targetIndex);
+        return {
+            isSlider: true,
+            parentXpath: parentResult.xpath,
+            targetXpath: targetResult.xpath,
+            sliderXpath: targetResult.xpath,
+            parentIndex: parentResult.resolvedIndex,
+            targetIndex: targetResult.resolvedIndex,
+            fireflinkIndex: targetResult.resolvedIndex,
+            direction,
+            domInfo,
+            doc,
+        };
+    }
+    return { isSlider: false, domInfo, doc };
+}

package/dist/core/helpers/visionFallbackHandler.js

@@ -73,7 +73,7 @@ async function executeVisionFallback(params) {
             throw new Error(`vision unable to find element for step "${step}"`, { cause: causeError });
         }
         let xpath;
-        if ((platform === "android" || platform === "ios") && (driverContext && fallbackAttempt < 2)) {
+        if ((platform === "android" || platform === "ios") && (!driverContext || fallbackAttempt < 2)) {
             const index = Number(fireflinkIndex.split("-").pop());
             const box = bounds[index - 1];
             if (!box) {
@@ -99,10 +99,12 @@ async function executeVisionFallback(params) {
                 const result = (0, xpathcreation_js_1.getWebXPathByFFIndex)(domInfo.dom, fireflinkIndex);
                 xpath = result.xpathPath;
             }
-            if (!xpath) {
+        }
+        if (!xpath) {
+            if (platform === "web") {
                 context.scriptAppender.setErrorNLP(stepAction);
-                throw new Error(`Unable to resolve xpath for ${fireflinkIndex}`, { cause: causeError });
             }
+            throw new Error(`Unable to resolve xpath for ${fireflinkIndex}`, { cause: causeError });
         }
         const pageDOM = iframeDomHtml ?? domInfo.dom;
         const handler = actionHandlers[stepAction];

package/dist/core/interfaces/actionInterface.d.ts

@@ -233,6 +233,7 @@ export interface IActionExecutor {
     swipeRightToElement(value: string, elementName: string, elementType: string, nomOfScrolls?: number): Promise<void>;
     swipeNTimes(nomOfScrolls: string, direction: string): Promise<void>;
     swipeDirectionNTimes(nomOfScrolls: string, direction: string): Promise<void>;
+    swipeSlider(parentXpath: string, targetXpath: string, direction?: string, elementName?: string, elementType?: string, parentIndex?: string, targetIndex?: string, pageDOM?: string): Promise<void>;
     swipeToElementForMWeb(value: string, elementName: string, elementType: string, pageDOM: string, fireflinkIndex: string, nomOfScrolls?: number): Promise<void>;
     swipeUsingReferenceElement(value: string, elementName: string, elementType: string, nomOfScrolls: string, direction: string, pageDOM: string, fireflinkIndex: string): Promise<void>;
     MOB_EnterInputIntoElementFromClipBoard(selector: string, elementName: string, elementType: string, pageDOM: string, fireflinkIndex: string): Promise<void>;

package/dist/core/interfaces/mobileSwipeSliderInterface.d.ts

@@ -0,0 +1,25 @@
+import { platform as PlatformType } from "../types/scriptTypesType.js";
+export interface MobileSwipeSliderParams {
+    driver: any;
+    currentPlatform: PlatformType;
+    stepAction: string;
+    step: string;
+    domInfo: any;
+    doc: any;
+    stepProcessor: any;
+    priorAndNextSteps: string[];
+    fallbackAttempt: number;
+    driverContext?: any;
+}
+export interface MobileSwipeSliderResult {
+    isSlider: boolean;
+    parentXpath?: string;
+    targetXpath?: string;
+    sliderXpath?: string;
+    parentIndex?: string;
+    targetIndex?: string;
+    fireflinkIndex?: string;
+    direction?: string;
+    domInfo: any;
+    doc: any;
+}

package/dist/core/interfaces/mobileSwipeSliderInterface.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/interfaces/promptInterface.d.ts

@@ -21,6 +21,10 @@ export interface PromptArgsMap {
         currentStep: string;
         priorAndNextSteps: string[];
     };
+    [PromptType.SWIPE_VISION_PROMPT]: {
+        currentStep?: string;
+        priorAndNextSteps: string[];
+    };
     [PromptType.VERIFY_PROMPT]: {
         stepAction: string;
         extractedDomJson: any;

package/dist/core/main/actionHandlerFactory.js

@@ -1425,8 +1425,8 @@ function createActionHandlers(context) {
         MOB_SwipeNTimes: withExecutor(async (executor, result) => {
             await executor.swipeNTimes(result.numOfScrolls, result.direction);
         }),
-        MOB_Builtin_Swipe: withExecutor(async (executor, result) => {
-            await executor.swipeDirectionNTimes(result.numOfScrolls, result.direction);
+        MOB_SwipeSlider: withExecutor(async (executor, result) => {
+            await executor.swipeSlider(result.parentXpath || result.xpath || result.selector, result.targetXpath || result.xpath || result.selector, result.direction, result.elementName, result.elementType, result.parentIndex, result.targetIndex, result.pageDOM);
         }),
         MOB_SwipeDirectionNTimes: withExecutor(async (executor, result) => {
             await executor.swipeDirectionNTimes(result.numOfScrolls, result.direction);