ff-automationv2 2.2.35 → 2.2.36-beta.2

package/dist/ai/llmcalls/llmAction.js

@@ -31,8 +31,9 @@ class llmAction {
         const promptBuilder = promptRegistry_js_1.prompts[platform][type];
         const systemPrompt = await promptBuilder(args);
         const userPrompt = userPrompt_js_1.userInputFormatters[type](userInput);
-        if (type === promptType_js_1.PromptType.VISION_PROMPT && Array.isArray(userPrompt)) {
-            return this.getLLMResponseWithVision(systemPrompt, userPrompt, isVision);
+        if ((type === promptType_js_1.PromptType.VISION_PROMPT || type === promptType_js_1.PromptType.SWIPE_VISION_PROMPT) && Array.isArray(userPrompt)) {
+            const schemaToUse = (0, FireFlinkResponseSchema_js_1.getSchemaForPrompt)(type, platform, args);
+            return this.getLLMResponseWithVision(systemPrompt, userPrompt, isVision, schemaToUse);
         }
         if (typeof userPrompt !== "string") {
             throw new Error("Invalid non-vision user prompt format");
@@ -116,7 +117,7 @@ class llmAction {
         }
         return this.provider;
     }
-    async getLLMResponseWithVision(system, userPrompt, isVision = false) {
+    async getLLMResponseWithVision(system, userPrompt, isVision = false, schemaToUse = FireFlinkResponseSchema_js_1.VisionSchema) {
         let attempt = 0;
         const MAX_RETRIES = 5;
         let content;
@@ -149,7 +150,7 @@ class llmAction {
                         { role: "user", content: parts },
                     ],
                     output: ai_1.Output.object({
-                        schema: FireFlinkResponseSchema_js_1.VisionSchema,
+                        schema: schemaToUse,
                     }),
                     providerOptions: this.getProviderOptions()
                 });
@@ -167,7 +168,7 @@ class llmAction {
                 logData_js_1.logger.error(`Vision LLM Error Attempt ${attempt}:`, error);
                 if (ai_1.NoObjectGeneratedError.isInstance(error) && error.text) {
                     try {
-                        const recovered = this.extractValidatedJson(error.text, FireFlinkResponseSchema_js_1.VisionSchema);
+                        const recovered = this.extractValidatedJson(error.text, schemaToUse);
                         logData_js_1.logger.info(`Vision LLM Attempt ${attempt} - Recovered via regex fallback:`, recovered);
                         await logData_js_1.logger.info("Time taken to get LLM response (vision, recovered):", Date.now() - startTime, "ms");
                         return {

package/dist/ai/llmprompts/llmPromptTypes/systemPromptTypes.d.ts

@@ -29,6 +29,9 @@ export type visionPrompt = {
 export type visionPromptMobile = {
     priorAndNextSteps: string[];
 };
+export type swipeVisionPrompt = {
+    priorAndNextSteps: string[];
+};
 export type waitActionArgs = {
     extractedDomJson: string;
     priorAndNextSteps: string[];

package/dist/ai/llmprompts/promptRegistry.js

@@ -22,6 +22,7 @@ const inputContextPrompt_js_1 = require("./systemPrompts/inputContextPrompt.js")
 const otpExtractorPrompt_js_1 = require("./systemPrompts/otpExtractorPrompt.js");
 const fileVerifyActionExtractorPrompt_js_1 = require("./systemPrompts/fileVerifyActionExtractorPrompt.js");
 const arithmeticPrompt_js_1 = require("./systemPrompts/arithmeticPrompt.js");
+const sliderSwipeVisionPrompt_js_1 = require("./systemPrompts/sliderSwipeVisionPrompt.js");
 exports.prompts = {
     web: {
         userStoryToList: userStoryToListPrompt_js_1.buildStepExtractionPrompt,
@@ -31,6 +32,7 @@ exports.prompts = {
         ffInspectorNumExtractor: fireflinkElementIndexExtactors_js_1.ffInspectorNumExtractor,
         verifyActionExtractorPrompt: verifyActionExtractorPrompt_js_1.verifyActionExtractorPrompt,
         visionPrompt: visionPrompt_js_1.visionPrompt,
+        swipe_vision_prompt: sliderSwipeVisionPrompt_js_1.swipeVisionPrompt,
         getActionExtractorPrompt: getActionExtractorPrompt_js_1.getActionExtractorPrompt,
         combinedActionExtractorPrompt: combinedActionExtractorPromptMob_js_1.combinedActionExtractorPromptMob,
         waitActionExtractorPrompt: waitActionExtractorPrompt_js_1.waitActionExtractorPrompt,
@@ -47,6 +49,7 @@ exports.prompts = {
         ffInspectorNumExtractor: fireflinkElementIndexExtractor_Mob_js_1.ffInspectorNumExtractorMob,
         verifyActionExtractorPrompt: verifyActionExtractorPromptMob_js_1.verifyActionExtractorPromptMob,
         visionPrompt: visionPromptMobile_js_1.visionPromptMobile,
+        swipe_vision_prompt: sliderSwipeVisionPrompt_js_1.swipeVisionPrompt,
         getActionExtractorPrompt: getActionExtractorPromptMob_js_1.getActionExtractorPromptMob,
         combinedActionExtractorPrompt: combinedActionExtractorPromptMob_js_1.combinedActionExtractorPromptMob,
         waitActionExtractorPrompt: waitActionExtractorPromptMob_js_1.waitActionExtractorPromptMob,
@@ -63,6 +66,7 @@ exports.prompts = {
         ffInspectorNumExtractor: fireflinkElementIndexExtractor_Mob_js_1.ffInspectorNumExtractorMob,
         verifyActionExtractorPrompt: verifyActionExtractorPromptMob_js_1.verifyActionExtractorPromptMob,
         visionPrompt: visionPromptMobile_js_1.visionPromptMobile,
+        swipe_vision_prompt: sliderSwipeVisionPrompt_js_1.swipeVisionPrompt,
         getActionExtractorPrompt: getActionExtractorPromptMob_js_1.getActionExtractorPromptMob,
         combinedActionExtractorPrompt: combinedActionExtractorPromptMob_js_1.combinedActionExtractorPromptMob,
         waitActionExtractorPrompt: waitActionExtractorPromptMob_js_1.waitActionExtractorPromptMob,

package/dist/ai/llmprompts/systemPrompts/actionExtractorPrompt.js

@@ -3,139 +3,139 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.keywordExtractor = keywordExtractor;
 async function keywordExtractor({ priorAndNextSteps }) {
     const allowedActions = ["enter", "refresh", "wait", "verify", "scroll", "navigate", "navigateBack", "click", "maximize", "minimize", "get", "upload", "close", "open", "drag_and_drop", "switch", "cleartext", "notwebopenaction", "mouseAction", "switchToAndroid", "getOtp", "arithmetic"];
-    const prompt = `
-You are an expert in Web application testing.
-    From the step, extract ONLY the meaningful keywords so that i can search for the element in the dom.
-    Rules:
-    - Only give response for the current step.
-    - understand the step and context from the ${JSON.stringify(priorAndNextSteps)}.and keywords should be from step. it should not be related to other steps.
-    - Keywords must represent ONLY the element label or visible text, never add action or instruction words in keywords.
-    - Element name + type keywords: strip action words, leaving the core element name and any
-    type word(s) attached to it (icon, button, dropdown, filter, tab, field, etc., in the order
-    they appear in the step). Build keywords as a growing chain starting from the core name and
-    adding one more type word each time: [name], [name+type1], [name+type1+type2], ...
-    The core name alone is ALWAYS keyword 1 — never drop it, no matter how many type words follow.
-      Treat "input field", "text field", "search box", "check box", "dropdown" as ONE type word,
-    not two.
-  No type word in the step → only [name] as the keyword.
-    Example: "click X filter dropdown" → ["X", "X filter", "X filter dropdown"]
-  If more than 3 type words are attached to the element (chain would exceed 5 keywords), keep the core name plus only the 2-3 type words CLOSEST to the element name and drop the rest, so total keywords stay within the 3-5 max.
-    - When a step contains two elements connected by a positional or relational phrase, keywords MUST contain at least one keyword from each element — TARGET element first, then the reference element. Never omit the reference element. This includes positional connectors (above, below, next to, under, near) AND possessive/associative connectors (of, for) that link an attribute/property element to a named reference element. Do NOT merge the two elements into a single combined keyword string.
-    Example: "Click on x above y" → x is the target, y is the reference → ["x", "y"].
-    Example: "get Price of Learn Selenium" → Price is the target attribute, "Learn Selenium" is the reference element → keywords ["Price", "Learn Selenium"], elementName "Price".
-    Example: "verify Rating for Learn Selenium is 300" → ["Rating", "Learn Selenium"], elementName "Rating".
-    - 3 to 5 keywords maximum. dont give more then 5 keywords. and no need for case sensitive.
-    - General ordering rule: whenever a step involves both a target element and a reference/context element, the TARGET element's keyword(s) must always come FIRST, followed by the reference/context element's keyword(s). This applies to positional/relational steps, selection steps, and any other multi-element step.
-    - For the open action if they are not provided anything like profile, capablities in the step in that case don't add them check whether the profile and the capablities are present in the step if not please don't add them.
-    - For the Open action, if a browser path and profile are provided, return keywords as ["path:<path_value>", "profile:<profile_value>"]. Should not remove backslash from path.
-    example: step open browser,x:\\x\\x\\x,default -> ["path:c:\\x\\x\\x","profile:default"] 
-    - For the Open action, if the user requests specific browser capabilities/arguments, extract them into the keywords array using the "caps:" prefix followed by one of these exact allowed values: 
-      - "headless" (for running in headless / no-gui mode)
-      - "incognito" (for private / incognito mode)
-      - "ignoreCertificates" (for ignoring SSL/certificate errors)
-      - "startMaximized" (for starting maximized)
-      - "disableGpu" (for disabling hardware/GPU acceleration)
-      - "disableNotifications" (for blocking push notifications)
-      - "disableExtensions" (for disabling browser extensions)
-      - "disablePopups" (for disabling popup blocking)
-      - "muteAudio" (for muting browser audio)
-      - "noSandbox" (for disabling sandbox)
-      - "disableWebSecurity" (for disabling web security, CORS, or same-origin policy checks)
-      - "disableLocalNetwork" (for disabling WebRTC local IP discovery or local network access constraints)
-      - "geolocation" (for blocking or allowing geolocation access)
-      - "camera" (for blocking or allowing camera access)
-      - "microphone" (for blocking or allowing microphone access)
-      - "passwordManager" (for disabling credentials and password manager prompts)
-      Do NOT extract or generate any capability strings outside of this list.
-    - Do NOT split the keyword into individual words or generate variations such as ["sign", "in"] or ["add", "to", "cart"] for ["Sign In","Add to Cart"]. Only include the original phrase should not add type for first keyword.
-    - If the step is about selecting/choosing/picking a SINGLE element with no separate reference element present (e.g. "select the Agree checkbox", "select India"), treat it as ONE step: keywords = [target element] only, action = "click".
-    - If the step is about entering text or Uploading file, Should NOT include input value from the step into keywords.
-    - If the step has words like tag name audio, video, image,svg, checkbox etc, include them in the keywords.
-    - **If icon is mentioned in step then 'svg' should add in keywords ex: click on x icon -> [x, x icon,svg] and for Upload action first keyword should be 'file'. ex: upload file in x  -> [file,x].**
-    - Do NOT split single-word keywords and do NOT include relation terms (above, below, next to, of, for, etc.) in keywords.
-    - Treat each keyword independently never merge different keywords.
-    - Keywords can be string or number if the step contains a number as part of the ELEMENT's name/label (e.g. "click on Step 2 tab" -> "Step 2" is a keyword). This does NOT apply to numbers that are the value being entered/typed/uploaded — those stay excluded per the "entering text or Uploading file" rule below, even if numeric (e.g. "Enter 5 in Quantity field" -> "5" is NOT a keyword).
-    - Do Not include any other unrelated keywords for step. 
-    - Do NOT include generic UI words (button, field, etc) and action words (tap, click, press, etc).
-    - Do NOT include status/technical words (displayed, enabled, authenticate, visible).
-    - If the step is for the switching to android then provide the action as the switchToAndroid
-    - element_name: extract name of the element that mentioned in the the step.(eg:tap on x -> element_name:x) keep element_name as short as possible and make the first letter of first word of the element_name as capital. because element_name is also used to find element in the dom. and if element_name is not mentioned in step then return action of the step as element_name.
-    - Set action to notwebopenaction only when the instruction is related to a mobile application, app, application, installed package, or bundle .
-    - action: click for taping, clicking or selecting, enter for entering input, wait for waiting or sleeping,scroll for scrolling and swiping, navigate for navigating to page using url, navigateBack for navigateing back to previous page, get for getting,fetching,capturing,reading element or text,maximize for maximizing browser window, close for closing browser window,open for opening browser window, upload for uploading file using path, drag_and_drop for dragging and dropping element, switch is for switching to tab or window or frame,cleartext for clearing or removing text from element, mouseAction for the action related mouse actions even for mouse click , refresh is for refreshing or reloading pages, getOtp for fetching/reading an OTP value, arithmetic for performing arithmetic operations (multiply, divide, add, subtract, calculate).
-    - If mouse action related step is there select mouseAction from the allowed actions list check for the step and provide it.
-    - action must be one of from this list  ${JSON.stringify(allowedActions)}.if not one of them, return '0'. if step about set or find action return '0'
-    - For navigate action, keywords should contain only one keyword which is full url from the step and should not include any other text. and if step has another actions, including navigate action, don't return navigate action return another action which is in the step. and element_name should be "URL".
-    - Navigate action rules:
-      - Use action "navigate" ONLY when the step gives an explicit instruction to go to a URL (e.g., "Navigate to https://x.com"). keywords must contain only the full URL, nothing else. elementName should be "URL".
-      - If the step ONLY checks/confirms the current or already-navigated URL, with NO "go to/navigate to" instruction present in that same step (e.g., "Verify navigated URL", "Get navigated URL"), return whichever action word the step itself uses (verify, get, check, confirm, etc. — never "navigate"), and do NOT split it — it is already a single atomic step. keywords should be ["URL"], elementName "URL".
-    CRITICAL RULES:
-    1.If the step contains multiple actions, you MUST split it into multiple objects in the "steps" array in the same sequence and generate the response object in the same flow.
-      - Each object must represent EXACTLY ONE action.
-      - Preserve the ORIGINAL ORDER of actions strictly.
-      Example 1: "Enter  username , Password as Username and Password and click on button", should be split into 3 steps: 1.Enter Username in Username, 2.Enter Password in Password, 3.Click on button. and elementName for first step should be Username, for second step should be Password and for third step should be click on button.
-    
-    2.If the step contains Select, choose, pick and also contains a connecting word to a reference element ("from", "in", "on", "under") then you must always split into exactly two steps, click on the reference element first and then click on the target value. elementName for the first step should be the reference element (mentioned after the connecting word), and for the second step it should be the target value (mentioned after select/choose/pick). if the step has more than one connecting word then consider only the first one for elementName and keyword extraction.
-      Example: "Select 'India' from Country" → 1.Click on Country, 2.Click on India. elementName for first step should be Country and for second step should be India.
-      Example: "select 'x' in 'y'" → 1.Click on y, 2.Click on x. elementName for first step should be y and for second step should be x.
-    3.If the step contains a compound action (e.g., "Enter text in field and click button"), you MUST split it into separate steps for each action, ensuring that each step is represented as a distinct object in the "steps" array with its own keywords and elementName.
-      Example: "Enter 'John' in Name field and click Submit" → 1.Enter 'John' in Name field, 2.Click on Submit. and elementName for first step should be Name field and for second step should be Submit.
-    4. ONLY IF the step text itself contains BOTH an explicit "go to/navigate to URL" instruction AND a separate URL-checking instruction (e.g., "Navigate to url and verify navigated url"), split into atomic steps with navigate FIRST, then the checking action SECOND, using whichever action word the step itself uses for the second part. Do not split a step that only checks the URL with no navigate instruction present — that stays a single step per the Navigate action rules above.
-    5.Keywords in the multiple steps should be extracted based on single step and it should not contain any keywords from other steps.
-    6. If the step performs the SAME action on TWO OR MORE DIFFERENT target elements joined by "and"/"," (with or without a value), split into separate steps — one per target element — each step keeping the same action (and same value, if any) with its own elementName and keywords. The "step" field for EACH split step must repeat the full action phrase from the original step, not just the element name — this is the one case where "step" is reconstructed rather than a literal substring, since the original sentence never repeated the action for each element.
-      Example (with value): "enter x in both a and b" -> 1. Enter x in a (elementName "a", keywords ["a"], step: "enter x in a"), 2. Enter x in b (elementName "b", keywords ["b"], step: "enter x in b").
-      Example (no value): "get text from a and b" -> 1. Get text from a (elementName "a", keywords ["a"], step: "get text from a"), 2. Get text from b (elementName "b", keywords ["b"], step: "get text from b").
-    
-    - fileType: If the verify action step mentions verifying or checking any file extension or document (e.g. "pdf" or any file extension), extract that file extension in lowercase (without dot, e.g. "pdf") and set it in fileType. For all non-file/UI verification steps, set fileType to null.
-
-    Respond ONLY with valid JSON in this EXACT format:
-
-    {
-      "steps": [
-        {
-          "keywords": ["key1", "key2"],
-          "elementName": "element name",
-          "action": ${allowedActions.join("|")},
-          "isStepValueDependable": boolean,
-          "liveLog": "Should Generate a short, human-readable description of what the AI is about to do.",
-          "fileType": "pdf" | other lowercase file extension | null,
-          "step": Return the original step exactly as-is if it contains a single action; if multiple actions exist, split into atomic steps without changing any wording, spelling, casing, spacing, or punctuation.(Rules: No rephrasing, no additions/removals—each output step must be an exact substring of the input.)
-        }
-      ]
-    }
-
-
-    STEP RULES (VERY IMPORTANT):
-
-    - Each "step" MUST represent EXACTLY ONE action.
-** isStepValueDependable (boolean):
-  • Set true when action is 'verify' or 'enter' and the step contains a value, comparison value, or referenced subject that represents data from another step, calculated result, or a referenced element/variable. Any explicit or implicit reference to existing, extracted, fetched, retrieved, copied, entered, calculated, or relationally referenced data must be treated as dependable (e.g., "enter calculated final result", "verify calculated value", "enter atpValue in field"). Resolve the reference using priorAndNextSteps when applicable; never consider future steps.
-  • Set true when action is 'arithmetic' and the step depends on values or variables from earlier steps (e.g., "Multiply atpValue by 3", "Subtract atpdivValue from atpValue", "Store captured value in variable atpValue").
-  • Set false only when the action has no reference to existing or earlier step data.
-  • Example: "verify extracted x text is x" → true when "extracted x text" refers to previously obtained data.
-  • Example: "enter that value in x" → true when "that value" refers to previously obtained data.
-  • Example: "Multiply atpValue by 3" → true when "atpValue" refers to a value captured in an earlier step.
-
-    - Step MUST be rewritten as a CLEAN and COMPLETE instruction.
-    - Step MUST contain:
-      • action
-      • target element
-      • value (if applicable)
-    - DO NOT copy original sentence if it contains multiple actions.
-    - EXCEPTION: for steps split under Critical Rule 6 (same action repeated across multiple elements), the "step" field is reconstructed to repeat the action phrase per element (see Rule 6) — it will not be a literal substring of the original in this one case.
-    - DO NOT keep unnecessary words like:
-      "select", "choose", "pick", "credentials", "details", etc.
-
-    - ALWAYS normalize action wording:
-
-      Examples:
-      "Select X from Y"
-      "Click on Y"
-      "Click on X"
-
-    - Each step must be SELF-CONTAINED and EXECUTABLE.
-
-
-    No other text.
+    const prompt = `
+You are an expert in Web application testing.
+    From the step, extract ONLY the meaningful keywords so that i can search for the element in the dom.
+    Rules:
+    - Only give response for the current step.
+    - understand the step and context from the ${JSON.stringify(priorAndNextSteps)}.and keywords should be from step. it should not be related to other steps.
+    - Keywords must represent ONLY the element label or visible text, never add action or instruction words in keywords.
+    - Element name + type keywords: strip action words, leaving the core element name and any
+    type word(s) attached to it (icon, button, dropdown, filter, tab, field, etc., in the order
+    they appear in the step). Build keywords as a growing chain starting from the core name and
+    adding one more type word each time: [name], [name+type1], [name+type1+type2], ...
+    The core name alone is ALWAYS keyword 1 — never drop it, no matter how many type words follow.
+      Treat "input field", "text field", "search box", "check box", "dropdown" as ONE type word,
+    not two.
+  No type word in the step → only [name] as the keyword.
+    Example: "click X filter dropdown" → ["X", "X filter", "X filter dropdown"]
+  If more than 3 type words are attached to the element (chain would exceed 5 keywords), keep the core name plus only the 2-3 type words CLOSEST to the element name and drop the rest, so total keywords stay within the 3-5 max.
+    - When a step contains two elements connected by a positional or relational phrase, keywords MUST contain at least one keyword from each element — TARGET element first, then the reference element. Never omit the reference element. This includes positional connectors (above, below, next to, under, near) AND possessive/associative connectors (of, for) that link an attribute/property element to a named reference element. Do NOT merge the two elements into a single combined keyword string.
+    Example: "Click on x above y" → x is the target, y is the reference → ["x", "y"].
+    Example: "get Price of Learn Selenium" → Price is the target attribute, "Learn Selenium" is the reference element → keywords ["Price", "Learn Selenium"], elementName "Price".
+    Example: "verify Rating for Learn Selenium is 300" → ["Rating", "Learn Selenium"], elementName "Rating".
+    - 3 to 5 keywords maximum. dont give more then 5 keywords. and no need for case sensitive.
+    - General ordering rule: whenever a step involves both a target element and a reference/context element, the TARGET element's keyword(s) must always come FIRST, followed by the reference/context element's keyword(s). This applies to positional/relational steps, selection steps, and any other multi-element step.
+    - For the open action if they are not provided anything like profile, capablities in the step in that case don't add them check whether the profile and the capablities are present in the step if not please don't add them.
+    - For the Open action, if a browser path and profile are provided, return keywords as ["path:<path_value>", "profile:<profile_value>"]. Should not remove backslash from path.
+    example: step open browser,x:\\x\\x\\x,default -> ["path:c:\\x\\x\\x","profile:default"] 
+    - For the Open action, if the user requests specific browser capabilities/arguments, extract them into the keywords array using the "caps:" prefix followed by one of these exact allowed values: 
+      - "headless" (for running in headless / no-gui mode)
+      - "incognito" (for private / incognito mode)
+      - "ignoreCertificates" (for ignoring SSL/certificate errors)
+      - "startMaximized" (for starting maximized)
+      - "disableGpu" (for disabling hardware/GPU acceleration)
+      - "disableNotifications" (for blocking push notifications)
+      - "disableExtensions" (for disabling browser extensions)
+      - "disablePopups" (for disabling popup blocking)
+      - "muteAudio" (for muting browser audio)
+      - "noSandbox" (for disabling sandbox)
+      - "disableWebSecurity" (for disabling web security, CORS, or same-origin policy checks)
+      - "disableLocalNetwork" (for disabling WebRTC local IP discovery or local network access constraints)
+      - "geolocation" (for blocking or allowing geolocation access)
+      - "camera" (for blocking or allowing camera access)
+      - "microphone" (for blocking or allowing microphone access)
+      - "passwordManager" (for disabling credentials and password manager prompts)
+      Do NOT extract or generate any capability strings outside of this list.
+    - Do NOT split the keyword into individual words or generate variations such as ["sign", "in"] or ["add", "to", "cart"] for ["Sign In","Add to Cart"]. Only include the original phrase should not add type for first keyword.
+    - If the step is about selecting/choosing/picking a SINGLE element with no separate reference element present (e.g. "select the Agree checkbox", "select India"), treat it as ONE step: keywords = [target element] only, action = "click".
+    - If the step is about entering text or Uploading file, Should NOT include input value from the step into keywords.
+    - If the step has words like tag name audio, video, image,svg, checkbox etc, include them in the keywords.
+    - **If icon is mentioned in step then 'svg' should add in keywords ex: click on x icon -> [x, x icon,svg] and for Upload action first keyword should be 'file'. ex: upload file in x  -> [file,x].**
+    - Do NOT split single-word keywords and do NOT include relation terms (above, below, next to, of, for, etc.) in keywords.
+    - Treat each keyword independently never merge different keywords.
+    - Keywords can be string or number if the step contains a number as part of the ELEMENT's name/label (e.g. "click on Step 2 tab" -> "Step 2" is a keyword). This does NOT apply to numbers that are the value being entered/typed/uploaded — those stay excluded per the "entering text or Uploading file" rule below, even if numeric (e.g. "Enter 5 in Quantity field" -> "5" is NOT a keyword).
+    - Do Not include any other unrelated keywords for step. 
+    - Do NOT include generic UI words (button, field, etc) and action words (tap, click, press, etc).
+    - Do NOT include status/technical words (displayed, enabled, authenticate, visible).
+    - If the step is for the switching to android then provide the action as the switchToAndroid
+    - element_name: extract name of the element that mentioned in the the step.(eg:tap on x -> element_name:x) keep element_name as short as possible and make the first letter of first word of the element_name as capital. because element_name is also used to find element in the dom. and if element_name is not mentioned in step then return action of the step as element_name.
+    - Set action to notwebopenaction only when the instruction is related to a mobile application, app, application, installed package, or bundle .
+    - action: click for taping, clicking or selecting, enter for entering input, wait for waiting or sleeping,scroll for scrolling and swiping, navigate for navigating to page using url, navigateBack for navigateing back to previous page, get for getting,fetching,capturing,reading element or text,maximize for maximizing browser window, close for closing browser window,open for opening browser window, upload for uploading file using path, drag_and_drop for dragging and dropping element, switch is for switching to tab or window or frame,cleartext for clearing or removing text from element, mouseAction for the action related mouse actions even for mouse click , refresh is for refreshing or reloading pages, getOtp for fetching/reading an OTP value, arithmetic for performing arithmetic operations (multiply, divide, add, subtract, calculate).
+    - If mouse action related step is there select mouseAction from the allowed actions list check for the step and provide it.
+    - action must be one of from this list  ${JSON.stringify(allowedActions)}.if not one of them, return '0'. if step about set or find action return '0'
+    - For navigate action, keywords should contain only one keyword which is full url from the step and should not include any other text. and if step has another actions, including navigate action, don't return navigate action return another action which is in the step. and element_name should be "URL".
+    - Navigate action rules:
+      - Use action "navigate" ONLY when the step gives an explicit instruction to go to a URL (e.g., "Navigate to https://x.com"). keywords must contain only the full URL, nothing else. elementName should be "URL".
+      - If the step ONLY checks/confirms the current or already-navigated URL, with NO "go to/navigate to" instruction present in that same step (e.g., "Verify navigated URL", "Get navigated URL"), return whichever action word the step itself uses (verify, get, check, confirm, etc. — never "navigate"), and do NOT split it — it is already a single atomic step. keywords should be ["URL"], elementName "URL".
+    CRITICAL RULES:
+    1.If the step contains multiple actions, you MUST split it into multiple objects in the "steps" array in the same sequence and generate the response object in the same flow.
+      - Each object must represent EXACTLY ONE action.
+      - Preserve the ORIGINAL ORDER of actions strictly.
+      Example 1: "Enter  username , Password as Username and Password and click on button", should be split into 3 steps: 1.Enter Username in Username, 2.Enter Password in Password, 3.Click on button. and elementName for first step should be Username, for second step should be Password and for third step should be click on button.
+    
+    2.If the step contains Select, choose, pick and also contains a connecting word to a reference element ("from", "in", "on", "under") then you must always split into exactly two steps, click on the reference element first and then click on the target value. elementName for the first step should be the reference element (mentioned after the connecting word), and for the second step it should be the target value (mentioned after select/choose/pick). if the step has more than one connecting word then consider only the first one for elementName and keyword extraction.
+      Example: "Select 'India' from Country" → 1.Click on Country, 2.Click on India. elementName for first step should be Country and for second step should be India.
+      Example: "select 'x' in 'y'" → 1.Click on y, 2.Click on x. elementName for first step should be y and for second step should be x.
+    3.If the step contains a compound action (e.g., "Enter text in field and click button"), you MUST split it into separate steps for each action, ensuring that each step is represented as a distinct object in the "steps" array with its own keywords and elementName.
+      Example: "Enter 'John' in Name field and click Submit" → 1.Enter 'John' in Name field, 2.Click on Submit. and elementName for first step should be Name field and for second step should be Submit.
+    4. ONLY IF the step text itself contains BOTH an explicit "go to/navigate to URL" instruction AND a separate URL-checking instruction (e.g., "Navigate to url and verify navigated url"), split into atomic steps with navigate FIRST, then the checking action SECOND, using whichever action word the step itself uses for the second part. Do not split a step that only checks the URL with no navigate instruction present — that stays a single step per the Navigate action rules above.
+    5.Keywords in the multiple steps should be extracted based on single step and it should not contain any keywords from other steps.
+    6. If the step performs the SAME action on TWO OR MORE DIFFERENT target elements joined by "and"/"," (with or without a value), split into separate steps — one per target element — each step keeping the same action (and same value, if any) with its own elementName and keywords. The "step" field for EACH split step must repeat the full action phrase from the original step, not just the element name — this is the one case where "step" is reconstructed rather than a literal substring, since the original sentence never repeated the action for each element.
+      Example (with value): "enter x in both a and b" -> 1. Enter x in a (elementName "a", keywords ["a"], step: "enter x in a"), 2. Enter x in b (elementName "b", keywords ["b"], step: "enter x in b").
+      Example (no value): "get text from a and b" -> 1. Get text from a (elementName "a", keywords ["a"], step: "get text from a"), 2. Get text from b (elementName "b", keywords ["b"], step: "get text from b").
+    
+    - fileType: If the verify action step mentions verifying or checking any file extension or document (e.g. "pdf" or any file extension), extract that file extension in lowercase (without dot, e.g. "pdf") and set it in fileType. For all non-file/UI verification steps, set fileType to null.
+
+    Respond ONLY with valid JSON in this EXACT format:
+
+    {
+      "steps": [
+        {
+          "keywords": ["key1", "key2"],
+          "elementName": "element name",
+          "action": ${allowedActions.join("|")},
+          "isStepValueDependable": boolean,
+          "liveLog": "Should Generate a short, human-readable description of what the AI is about to do.",
+          "fileType": "pdf" | other lowercase file extension | null,
+          "step": Return the original step exactly as-is if it contains a single action; if multiple actions exist, split into atomic steps without changing any wording, spelling, casing, spacing, or punctuation.(Rules: No rephrasing, no additions/removals—each output step must be an exact substring of the input.)
+        }
+      ]
+    }
+
+
+    STEP RULES (VERY IMPORTANT):
+
+    - Each "step" MUST represent EXACTLY ONE action.
+** isStepValueDependable (boolean):
+  • Set true when action is 'verify' or 'enter' and the step contains a value, comparison value, or referenced subject that represents data from another step, calculated result, or a referenced element/variable. Any explicit or implicit reference to existing, extracted, fetched, retrieved, copied, entered, calculated, or relationally referenced data must be treated as dependable (e.g., "enter calculated final result", "verify calculated value", "enter atpValue in field"). Resolve the reference using priorAndNextSteps when applicable; never consider future steps.
+  • Set true when action is 'arithmetic' and the step depends on values or variables from earlier steps (e.g., "Multiply atpValue by 3", "Subtract atpdivValue from atpValue", "Store captured value in variable atpValue").
+  • Set false only when the action has no reference to existing or earlier step data.
+  • Example: "verify extracted x text is x" → true when "extracted x text" refers to previously obtained data.
+  • Example: "enter that value in x" → true when "that value" refers to previously obtained data.
+  • Example: "Multiply atpValue by 3" → true when "atpValue" refers to a value captured in an earlier step.
+
+    - Step MUST be rewritten as a CLEAN and COMPLETE instruction.
+    - Step MUST contain:
+      • action
+      • target element
+      • value (if applicable)
+    - DO NOT copy original sentence if it contains multiple actions.
+    - EXCEPTION: for steps split under Critical Rule 6 (same action repeated across multiple elements), the "step" field is reconstructed to repeat the action phrase per element (see Rule 6) — it will not be a literal substring of the original in this one case.
+    - DO NOT keep unnecessary words like:
+      "select", "choose", "pick", "credentials", "details", etc.
+
+    - ALWAYS normalize action wording:
+
+      Examples:
+      "Select X from Y"
+      "Click on Y"
+      "Click on X"
+
+    - Each step must be SELF-CONTAINED and EXECUTABLE.
+
+
+    No other text.
 `;
     return prompt;
 }

package/dist/ai/llmprompts/systemPrompts/arithmeticPrompt.js

@@ -3,46 +3,46 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.arithmeticPrompt = arithmeticPrompt;
 async function arithmeticPrompt({ inputStepsContext, }) {
     const arithmetic = ["multiply", "divide", "add", "subtract"];
-    const prompt = `
-You are an intelligent assistant that extracts arithmetic operations and operands from test steps.
-
-Previous steps data:
-${JSON.stringify(inputStepsContext, null, 2)}
-
-Instructions:
-1. Analyze the current step and use Previous steps data to resolve any operand that refers to data from an earlier step.
-2. Extract exactly two operands, their corresponding variable names when available, the arithmetic operation, and the target variable name.
-
-Rules for "operation":
-  - Return ONLY one of: "multiply", "divide", "add", "subtract".
-- Determine the operation directly from the current step.
-
-Rules for "operandOne" and "operandTwo":
-  - If an operand is a literal number, return only the numeric value as a string.Remove currency symbols, special characters, commas, or other non - numeric symbols.Example: "$2" -> "2".
-- If an operand refers to an earlier step or its value using contextual or relational wording, find the matching step in Previous steps data and use that step's inputText/value as the operand and its variableName as the corresponding operandVariableName.
-  - If the referenced earlier step contains both inputText and variableName, always use the inputText for the operand and the exact variableName for its operandVariableName.
-- If an operand is a variable already present in Previous steps data, use its exact variableName.
-- If no variableName exists for a literal operand, set its operandVariableName to null.
-- Do not include symbols, descriptions, element names, or explanations in operandOne or operandTwo.
-
-Rules for "subtract":
-  - "Subtract X from Y" -> operandOne = Y, operandTwo = X.
-- For other subtraction wording, follow the operand order expressed by the step.
-
-Rules for "variableName":
-  - If the current step explicitly provides a target variable name, use that exact name without modifying it.
-- Otherwise, generate a valid descriptive variable name based on the operation and result.
-- The generated name must contain only letters, digits, or underscores and must not start with a digit.
-
-Respond ONLY with valid JSON in this exact format:
-{
-  "operandOne": "x",
-  "operandTwo": "y",
-  "operandOneVariableName": "variableName1 | null",
-  "operandTwoVariableName": "variableName2 | null",
-  "operation": ${arithmetic.join("|")},
-  "variableName": "targetVariableName"
-}
+    const prompt = `
+You are an intelligent assistant that extracts arithmetic operations and operands from test steps.
+
+Previous steps data:
+${JSON.stringify(inputStepsContext, null, 2)}
+
+Instructions:
+1. Analyze the current step and use Previous steps data to resolve any operand that refers to data from an earlier step.
+2. Extract exactly two operands, their corresponding variable names when available, the arithmetic operation, and the target variable name.
+
+Rules for "operation":
+  - Return ONLY one of: "multiply", "divide", "add", "subtract".
+- Determine the operation directly from the current step.
+
+Rules for "operandOne" and "operandTwo":
+  - If an operand is a literal number, return only the numeric value as a string.Remove currency symbols, special characters, commas, or other non - numeric symbols.Example: "$2" -> "2".
+- If an operand refers to an earlier step or its value using contextual or relational wording, find the matching step in Previous steps data and use that step's inputText/value as the operand and its variableName as the corresponding operandVariableName.
+  - If the referenced earlier step contains both inputText and variableName, always use the inputText for the operand and the exact variableName for its operandVariableName.
+- If an operand is a variable already present in Previous steps data, use its exact variableName.
+- If no variableName exists for a literal operand, set its operandVariableName to null.
+- Do not include symbols, descriptions, element names, or explanations in operandOne or operandTwo.
+
+Rules for "subtract":
+  - "Subtract X from Y" -> operandOne = Y, operandTwo = X.
+- For other subtraction wording, follow the operand order expressed by the step.
+
+Rules for "variableName":
+  - If the current step explicitly provides a target variable name, use that exact name without modifying it.
+- Otherwise, generate a valid descriptive variable name based on the operation and result.
+- The generated name must contain only letters, digits, or underscores and must not start with a digit.
+
+Respond ONLY with valid JSON in this exact format:
+{
+  "operandOne": "x",
+  "operandTwo": "y",
+  "operandOneVariableName": "variableName1 | null",
+  "operandTwoVariableName": "variableName2 | null",
+  "operation": ${arithmetic.join("|")},
+  "variableName": "targetVariableName"
+}
 `;
     return prompt;
 }

package/dist/ai/llmprompts/systemPrompts/fileVerifyActionExtractorPrompt.js

@@ -4,42 +4,42 @@ exports.fileVerifyActionExtractorPrompt = fileVerifyActionExtractorPrompt;
 async function fileVerifyActionExtractorPrompt({ extractedDomJson, priorAndNextSteps, }) {
     const nlpList = `VerifyPDF,CompareTwoPdf,VerifyPDFExist,CheckAccessibilityForPdfUsingPython,CheckIfPdfIsXfaFrom,VerifyElementIsDisplayed,VerifyElementIsNotDisplayed,VerifyElementIsEnabled,VerifyElementIsDisabled,VerifyElementIsClickable,VerifyNavigateURL,VerifyIfLinkIsWorking`;
     const elementType = ['link', 'textfield', 'icon', 'button', 'radioButton', 'text', 'textarea', 'image', 'dropdown', 'checkbox', 'tab', 'action overflow button', 'hamburger icon', 'toggle button', 'suggestion', 'time picker', 'date picker', 'toaster message', 'card', 'tooltip', 'option', 'calender', 'sliders', 'visual testing'];
-    const prompt = `You are an intelligent assistant that extracts structured UI action data for file and element verification.
-    Given the step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList} don't provide the other nlp names which are not present in the list in any manner.
-
-    Select the perfect matching NLP name from the list that best fits the step's intent.
-    **The NLP name must match exactly one value from the approved NLP list. Do not generate or suggest any new NLP names under any condition.**
-    step may give any way, you need to get most matching and most related nlp name from the list that relates to step.
-
-    - Return ONLY a valid JSON object; the response must start with '{' and end with '}', with no markdown, code fences, explanations, or extra text.
-    Return **only valid JSON** in the following format:
-    {
-    "attributeValue": "Fire-Flink-x",
-    "nlpName": "x",
-    "inputText": "x",
-    "elementType": "x",
-    "isDownload": true,
-    "path": null
-    }
-
-    Rules:
-    - PDF steps: if step specifies checking/verifying text, content, or phrases inside a PDF → choose VerifyPDF and extract the expected text into inputText | if step only verifies the presence, creation, or download of the file without checking inner text → choose VerifyPDFExist with inputText: "".
-    - nlpName must be exactly one from the provided list.
-    - While matching nlpName with step (ignore spaces, case differences). If multiple NLPs are similar, choose the MOST SPECIFIC and EXACT match.
-    - Use context from the steps: ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
-    - **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or any other attributes best match the step in the simplified DOM.**
-    - If no matching element found for the step or if step is file/pdf verification, return attributeValue as 'Fire-Flink-0'.
-    - **inputText rules**: Extract ONLY the expected text phrase to verify inside the PDF or element. Strip any wrapping quotes (' or "). **NEVER include the file path or file name in inputText**. If no text is being verified, return "".
-    - Based on step give most relevant type of element. Use this list to choose elementType: ${elementType}. If elementType is not there in list return 'link'.
-    - **isDownload & path rules**:
-      - If the step contains both a file path and expected text (e.g. "Verify existing PDF contains text 'path/to/doc.pdf',Expected Text" or "Verify PDF 'path/to/doc.pdf' contains 'Expected Text'"):
-        • Extract the file path into "path" (stripped of surrounding quotes).
-        • Extract ONLY the search phrase into "inputText" (do NOT include the file path in inputText).
-        • Set isDownload: false.
-      - If verifying an existing file at a specified path or location (e.g. contains drive letter C:, /, or file extension .pdf) → set isDownload: false and path to the extracted file path string.
-      - If the step is about verifying a file downloaded via the browser (e.g. "verify downloaded pdf", "verify pdf", or no explicit file path is given) → set isDownload: true and path: null.
-      - **path formatting**: Extract the file path from the step and add extra //// for path like "C:////Users/////User/////Downloads/////file.pdf" (strip surrounding quotes).
-    - **Respond with valid JSON only. don't return any other text or don't return response in list format**
+    const prompt = `You are an intelligent assistant that extracts structured UI action data for file and element verification.
+    Given the step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList} don't provide the other nlp names which are not present in the list in any manner.
+
+    Select the perfect matching NLP name from the list that best fits the step's intent.
+    **The NLP name must match exactly one value from the approved NLP list. Do not generate or suggest any new NLP names under any condition.**
+    step may give any way, you need to get most matching and most related nlp name from the list that relates to step.
+
+    - Return ONLY a valid JSON object; the response must start with '{' and end with '}', with no markdown, code fences, explanations, or extra text.
+    Return **only valid JSON** in the following format:
+    {
+    "attributeValue": "Fire-Flink-x",
+    "nlpName": "x",
+    "inputText": "x",
+    "elementType": "x",
+    "isDownload": true,
+    "path": null
+    }
+
+    Rules:
+    - PDF steps: if step specifies checking/verifying text, content, or phrases inside a PDF → choose VerifyPDF and extract the expected text into inputText | if step only verifies the presence, creation, or download of the file without checking inner text → choose VerifyPDFExist with inputText: "".
+    - nlpName must be exactly one from the provided list.
+    - While matching nlpName with step (ignore spaces, case differences). If multiple NLPs are similar, choose the MOST SPECIFIC and EXACT match.
+    - Use context from the steps: ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
+    - **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or any other attributes best match the step in the simplified DOM.**
+    - If no matching element found for the step or if step is file/pdf verification, return attributeValue as 'Fire-Flink-0'.
+    - **inputText rules**: Extract ONLY the expected text phrase to verify inside the PDF or element. Strip any wrapping quotes (' or "). **NEVER include the file path or file name in inputText**. If no text is being verified, return "".
+    - Based on step give most relevant type of element. Use this list to choose elementType: ${elementType}. If elementType is not there in list return 'link'.
+    - **isDownload & path rules**:
+      - If the step contains both a file path and expected text (e.g. "Verify existing PDF contains text 'path/to/doc.pdf',Expected Text" or "Verify PDF 'path/to/doc.pdf' contains 'Expected Text'"):
+        • Extract the file path into "path" (stripped of surrounding quotes).
+        • Extract ONLY the search phrase into "inputText" (do NOT include the file path in inputText).
+        • Set isDownload: false.
+      - If verifying an existing file at a specified path or location (e.g. contains drive letter C:, /, or file extension .pdf) → set isDownload: false and path to the extracted file path string.
+      - If the step is about verifying a file downloaded via the browser (e.g. "verify downloaded pdf", "verify pdf", or no explicit file path is given) → set isDownload: true and path: null.
+      - **path formatting**: Extract the file path from the step and add extra //// for path like "C:////Users/////User/////Downloads/////file.pdf" (strip surrounding quotes).
+    - **Respond with valid JSON only. don't return any other text or don't return response in list format**
     `;
     return prompt;
 }

package/dist/ai/llmprompts/systemPrompts/fireflinkElementIndexExtractor_Mob.js

@@ -88,7 +88,7 @@ Rules:
     else if (stepAction === "swipe" || stepAction === "scroll") {
         prompt = `
   You are an intelligent assistant that extracts structured mobile scroll/swipe action data.
-Given the step: ${JSON.stringify(priorAndNextSteps)}, the simplified DOM (for scroll context): ${extractedDomJson}, and the list of NLP names: ${SwipeActions}.
+Given the step: ${JSON.stringify(priorAndNextSteps)}, the simplified DOM (for scroll context): ${extractedDomJson}, and the list of NLP names: ${SwipeActions.join("|")}.
 
 Select the perfect matching NLP name from the list that best fits the step's intent.
 
@@ -98,7 +98,7 @@ Return **only valid JSON** in the following format:
   "nlpName": "x",
   "inputText": "x",
   "keyword": "x",
-  "type": "x",
+  "elementType": "x",
   "numOfScrolls": "x",
   "direction": "x"
 }}
@@ -120,10 +120,10 @@ example: swipe up to "Contact Us" using "Settings" as reference.
 - **Extraction**:
     - inputText: The target element in the step.e.g., "swipe to submit text",then input text must be "submit",Dont include its type in inputText like text,button,icon etc...Leave empty if just swiping N times, if coordinates are specified then give coordinates in inputText like (x1,y1,x2,y2) where x1,y1 is starting coordinate and x2,y2 is ending coordinate.**
     - keyword: Crucial keyword from step (e.g., "Settings").
-    - nomOfScrolls: Extract count if specified (e.g., "scroll 3 times"). Default to 0 if not specified but action implies generic scroll.
+    - nomOfScrolls: Extract  if specified (e.g., "scroll 3 times"). Default to 0 if not specified but action implies generic scroll.
     - direction: "up", "down", "left", "right". Infer from step (e.g., "Scroll down" -> "down"). If direction is not specified, default direction to "none".
     - attributeValue: If an element is visible and matches the target, provide its FF-inspecter. Otherwise "Fire-Flink-2".
-    - type: Based on step give most relevant type of element. use this list to choose elementType: ${elementType} and Never change syntax of elementType, follow the syntax of elementType in list.if elementType is not there in list return 'link'.
+    - elementType: Based on step give most relevant type of element. use this list to choose elementType: ${elementType.join("|")} and Never change syntax of elementType, follow the syntax of elementType in list.if elementType is not there in list return 'link'.
     - Return ONLY a valid JSON object; the response must start with '{' and end with '}', with no markdown, code fences, explanations, or extra text.
 
 - response:valid json only.`;