ff-automationv2 2.2.29 → 2.2.30

package/dist/ai/llmcalls/llmAction.js

@@ -44,11 +44,18 @@ class llmAction {
         const MAX_RETRIES = 5;
         const startTime = new Date().getTime();
         const schemaToUse = (0, FireFlinkResponseSchema_js_1.getSchemaForPrompt)(type, platform, args);
+        let modelInstance;
+        if (this.serviceProvider === "Groq" || this.serviceProvider === "DefaultFireFlink") {
+            modelInstance = this.provider.responses(this.model);
+        }
+        else {
+            modelInstance = this.provider(this.model);
+        }
         while (attempt < MAX_RETRIES) {
             attempt++;
             try {
                 const response = await (0, ai_1.generateText)({
-                    model: this.provider(this.model),
+                    model: modelInstance,
                     messages: [
                         { role: "system", content: system },
                         { role: "user", content: user },
@@ -58,8 +65,7 @@ class llmAction {
                     }),
                     providerOptions: this.getProviderOptions()
                 });
-                const rawText = response.output;
-                const parsedContent = rawText;
+                const parsedContent = response.output;
                 logData_js_1.logger.info(`LLM Attempt ${attempt} - Raw Response:`, parsedContent);
                 if (this.isInvalidLLMOutput(parsedContent)) {
                     logData_js_1.logger.error(`Invalid LLM output. Retrying (${attempt})`);
@@ -73,6 +79,9 @@ class llmAction {
             }
             catch (error) {
                 logData_js_1.logger.error(`LLM Error Attempt ${attempt}:`, error);
+                if (attempt < MAX_RETRIES) {
+                    continue;
+                }
                 logData_js_1.logger.error(`Cause is ${error.cause}`, error.cause);
                 if (ai_1.NoObjectGeneratedError.isInstance(error) && error.text) {
                     try {
@@ -201,27 +210,42 @@ class llmAction {
         return false;
     }
     handleKnownErrors(error) {
-        if (error?.status === 429) {
+        if (error?.lastError?.statusCode === 429) {
             return {
                 error: "Rate Limit Exceeded (429)",
-                errorDescription: "FireFlink has exceeded the API quota or rate limit for the selected AI model. \n This is common with free or low-tier API plans \n Please wait a moment before retrying or check your API usage limits.",
+                errorDescription: "Service has exceeded the API quota or rate limit for the selected AI model. \n This is common with free or low-tier API plans \n Please wait a moment before retrying or check your API usage limits.",
                 rawError: "Rate limit reached for model. HTTP 429 returned by model"
             };
         }
-        if (error?.status === 400 &&
+        else if (error?.statusCode === 400 &&
             error?.error?.message?.toLowerCase().includes("context")) {
             return {
                 error: "Context Length Exceeded (400)",
-                errorDescription: "FireFlink could not process the request because the input exceeds the maximum context length supported by the selected AI model.\nThis typically happens when the input is very long",
+                errorDescription: "Service could not process the request because the input exceeds the maximum context length supported by the selected AI model.\nThis typically happens when the input is very long",
                 rawError: "This model's maximum context length has been exceeded. HTTP 400 returned by model"
             };
         }
+        else if (error?.statusCode === 401 &&
+            error?.error?.message?.toLowerCase().includes("Incorrect api key")) {
+            return {
+                error: "Invalid API key",
+                errorDescription: "Service could not process the request because the API key is invalid. Please check your API key and try again.",
+                rawError: "Invalid API key"
+            };
+        }
+        else if (error?.statusCode !== 200) {
+            return {
+                error: error.message,
+                errorDescription: "Service has can't process the request. Because of service Provider internal error",
+                rawError: error,
+            };
+        }
         return null;
     }
     buildRetryFailure() {
         return {
             error: "LLM Empty Response After Retry",
-            errorDescription: "FireFlink attempted multiple times but received empty or invalid responses.",
+            errorDescription: "Service attempted multiple times but received empty or invalid responses.",
             rawError: "Max retry limit reached",
         };
     }
@@ -243,11 +267,23 @@ class llmAction {
                     },
                 };
             case "Anthropic":
-                return {
-                    anthropic: {
-                        effort: "low",
-                    },
-                };
+                if (this.model.startsWith("claude-haiku")) {
+                    return {
+                        anthropic: {
+                            thinking: {
+                                type: "enabled",
+                                budgetTokens: 1024
+                            }
+                        },
+                    };
+                }
+                else {
+                    return {
+                        anthropic: {
+                            effort: "low",
+                        },
+                    };
+                }
             case "Gemini":
                 if (this.model.startsWith("gemini-2")) {
                     return {

package/dist/ai/llmcalls/parseLlmOputput.js

@@ -15,7 +15,7 @@ class LLMResultParser {
         return JSON.parse(match[0]);
     }
     async fetchResult(llmOutput) {
-        if (typeof llmOutput?.error === "string" && llmOutput.error.includes("429")) {
+        if (llmOutput?.error) {
             throw llmOutput;
         }
         try {
@@ -45,10 +45,10 @@ class LLMResultParser {
                 }
                 parsedContent = content;
             }
-            const usage = llmOutput?.usage ?? {};
-            this.inputTokens += Number(usage.input_tokens ?? 0);
-            this.outputTokens += Number(usage.output_tokens ?? 0);
-            this.totalTokens += Number(usage.total_tokens ?? 0);
+            const usage = llmOutput?.totalUsage ?? {};
+            this.inputTokens += Number(usage.inputTokens ?? 0);
+            this.outputTokens += Number(usage.outputTokens ?? 0);
+            this.totalTokens += Number(usage.totalTokens ?? 0);
             // logger.info(`tokens : ${usage.output_tokens} + ${usage.input_tokens} = ${usage.total_tokens}`);
             // logger.info(`Total tokens for this call is : ${usage.total_tokens}`);
             return { response: parsedContent };

package/dist/ai/llmprompts/promptRegistry.js

@@ -17,6 +17,7 @@ const combinedActionExtractorPromptMob_js_1 = require("./systemPrompts/combinedA
 const waitActionExtractorPromptMob_js_1 = require("./systemPrompts/waitActionExtractorPromptMob.js");
 const waitActionExtractorPrompt_js_1 = require("./systemPrompts/waitActionExtractorPrompt.js");
 const combinedActionExtractorPrompt_js_1 = require("./systemPrompts/combinedActionExtractorPrompt.js");
+const otpExtractorPrompt_js_1 = require("./systemPrompts/otpExtractorPrompt.js");
 exports.prompts = {
     web: {
         userStoryToList: userStoryToListPrompt_js_1.buildStepExtractionPrompt,
@@ -28,6 +29,7 @@ exports.prompts = {
         getActionExtractorPrompt: getActionExtractorPrompt_js_1.getActionExtractorPrompt,
         combinedActionExtractorPrompt: combinedActionExtractorPrompt_js_1.combinedActionExtractorPrompt,
         waitActionExtractorPrompt: waitActionExtractorPrompt_js_1.waitActionExtractorPrompt,
+        otpExtractorPrompt: otpExtractorPrompt_js_1.otpExtractorPrompt,
     },
     android: {
         userStoryToList: userStoryToListPrompt_js_1.androidbuildStepExtractionPrompt,
@@ -39,6 +41,7 @@ exports.prompts = {
         getActionExtractorPrompt: getActionExtractorPromptMob_js_1.getActionExtractorPromptMob,
         combinedActionExtractorPrompt: combinedActionExtractorPromptMob_js_1.combinedActionExtractorPromptMob,
         waitActionExtractorPrompt: waitActionExtractorPromptMob_js_1.waitActionExtractorPromptMob,
+        otpExtractorPrompt: otpExtractorPrompt_js_1.otpExtractorPrompt,
     },
     ios: {
         userStoryToList: userStoryToListPrompt_js_1.androidbuildStepExtractionPrompt,
@@ -50,5 +53,6 @@ exports.prompts = {
         getActionExtractorPrompt: getActionExtractorPromptMob_js_1.getActionExtractorPromptMob,
         combinedActionExtractorPrompt: combinedActionExtractorPromptMob_js_1.combinedActionExtractorPromptMob,
         waitActionExtractorPrompt: waitActionExtractorPromptMob_js_1.waitActionExtractorPromptMob,
+        otpExtractorPrompt: otpExtractorPrompt_js_1.otpExtractorPrompt,
     }
 };

package/dist/ai/llmprompts/systemPrompts/actionExtractorPrompt.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.keywordExtractor = keywordExtractor;
 async function keywordExtractor({ priorAndNextSteps }) {
-    const allowedActions = ["enter", "wait", "verify", "scroll", "navigate", "navigateBack", "click", "maximize", "minimize", "get", "upload", "close", "open", "drag_and_drop", "switch", "cleartext", "notwebopenaction", "mouseAction"];
+    const allowedActions = ["enter", "refresh", "wait", "verify", "scroll", "navigate", "navigateBack", "click", "maximize", "minimize", "get", "upload", "close", "open", "drag_and_drop", "switch", "cleartext", "notwebopenaction", "mouseAction", "switchToAndroid", "getOtp", "enterOtp"];
     const prompt = `
 You are an expert in Web application testing.
     From the step, extract ONLY the meaningful keywords so that i can search for the element in the dom.
@@ -10,18 +10,44 @@ You are an expert in Web application testing.
     - Only give response for the current step.
     - understand the step and context from the ${JSON.stringify(priorAndNextSteps)}.and keywords should be from step. it should not be related to other steps.
     - Keywords must represent ONLY the element label or visible text, never add action or instruction words in keywords.
-    - If an element type is mentioned (e.g., icon, button, dropdown, text) in step.You should not add type is not mentioned in step , generate two keywords:
-       First keyword must be the pure element name
-        Second keyword should combine element name + type
-        Example: "click on x icon" → "x", "x icon" 
-    -  When a step contains two elements connected by a positional or relational phrase, keywords MUST contain at least one keyword from each element (reference and targe). Never omit the reference element. Example: "Click on x above y" → ["x", "y"].
+    - Element name + type keywords: strip action words, leaving the core element name and any
+  type word(s) attached to it (icon, button, dropdown, filter, tab, field, etc., in the order
+  they appear in the step). Build keywords as a growing chain starting from the core name and
+  adding one more type word each time: [name], [name+type1], [name+type1+type2], ...
+  The core name alone is ALWAYS keyword 1 — never drop it, no matter how many type words follow.
+  Treat "input field", "text field", "search box", "check box", "drop down" as ONE type word,
+  not two.
+  No type word in the step → only [name] as the keyword.
+  Example: "click X filter dropdown" → ["X", "X filter", "X filter dropdown"]
+    - When a step contains two elements connected by a positional or relational phrase, keywords MUST contain at least one keyword from each element — TARGET element first, then the reference element. Never omit the reference element. Example: "Click on x above y" → x is the target, y is the reference → ["x", "y"].
     - 3 to 5 keywords maximum. dont give more than 5 keywords. and no need for casesensitive.
+    - General ordering rule: whenever a step involves both a target element and a reference/context element, the TARGET element's keyword(s) must always come FIRST, followed by the reference/context element's keyword(s). This applies to positional/relational steps, selection steps, and any other multi-element step.
+    - For the open action if they are not provided anything like profile, capablities in the step in that case don't add them check whether the profile and the capablities are present in the step if not please don't add them.
     - For the Open action, if a browser path and profile are provided, return keywords as ["path:<path_value>", "profile:<profile_value>"]. Should not remove backslach from path.
     example: step open browser,x:\\x\\x\\x,default -> ["path:c:\\x\\x\\x","profile:default"] 
-    - Do NOT split the keyword into individual words or generate variations such as ["sign", "in"] or ["add", "to", "cart"] for ["Sign In","Add to Cart"]. Only include the original phrase.
+    - For the Open action, if the user requests specific browser capabilities/arguments, extract them into the keywords array using the "caps:" prefix followed by one of these exact allowed values: 
+      - "headless" (for running in headless / no-gui mode)
+      - "incognito" (for private / incognito mode)
+      - "ignoreCertificates" (for ignoring SSL/certificate errors)
+      - "startMaximized" (for starting maximized)
+      - "disableGpu" (for disabling hardware/GPU acceleration)
+      - "disableNotifications" (for blocking push notifications)
+      - "disableExtensions" (for disabling browser extensions)
+      - "disablePopups" (for disabling popup blocking)
+      - "muteAudio" (for muting browser audio)
+      - "noSandbox" (for disabling sandbox)
+      - "disableWebSecurity" (for disabling web security, CORS, or same-origin policy checks)
+      - "disableLocalNetwork" (for disabling WebRTC local IP discovery or local network access constraints)
+      - "geolocation" (for blocking or allowing geolocation access)
+      - "camera" (for blocking or allowing camera access)
+      - "microphone" (for blocking or allowing microphone access)
+      - "passwordManager" (for disabling credentials and password manager prompts)
+      Do NOT extract or generate any capability strings outside of this list.
+    - Do NOT split the keyword into individual words or generate variations such as ["sign", "in"] or ["add", "to", "cart"] for ["Sign In","Add to Cart"]. Only include the original phrase should not add type for first keyword.
     - If icon is metioned in step than 'svg' should add in keywords and for Upload action first keyword should be 'file'.
-    - If step is about selecting, include keywords for both the source and the selected value (example: select 'x' in 'y' → "y", "x"), ensuring both are meaningful.
+    - If the step is about selecting, include keywords for both the target value and the source/reference, with the TARGET value FIRST and the source/reference SECOND (example: select 'x' in 'y' → "x", "y"), ensuring both are meaningful and action should set to 'click'.
     - If the step is about entering text or Uploading file, Should NOT include input value from the step into keywords.
+    - If the step is about entering the OTP then give the action as the enterOtp, just only for the entering OTP only.
     - If the step has words like tag name audio, video, image,svg, checkbox etc, include them in the keywords.
     - **If icon is metioned in step than 'svg' should add in keywords ex: click on x icon -> [x, x icon,svg] and for Upload action first keyword should be 'file'. ex: upload file in x  -> [file,x].**
     - Do NOT split single-word keywords and do NOT include relation terms (above, below, next to, etc.) in keywords.
@@ -30,9 +56,10 @@ You are an expert in Web application testing.
     - Do Not include any other unrelated keywords for step. 
     - Do NOT include generic UI words (button, field, etc) and action words (tap, click, press, etc).
     - Do NOT include status/technical words (displayed, enabled, authenticate, visible).
+    - If the step is for the switching to android then provide the action as the switchToAndroid
     - element_name: extract name of the element that mentioned in the the step.(eg:tap on x -> element_name:x) keep element_name as short as possible and make the first letter of first word of the element_name as capital. beacuse element_name is also used to find element in the dom. and if element_name is not mentioned in step than return action of the step as element_name.
-    - Set action to notwebopenaction only when the instruction is related to a desktop application, mobile application, app, application, installed package, or bundle .
-    - action: click for taping, clicking or selecting, enter for entering input, wait for waiting or sleeping,scroll for scrolling and swiping, navigate for navigating to page using url, navigateBack for navigateing back to previous page, get for getting,fetching element,maximize for maximizing browser window, close for closing browser window,open for opening browser window, upload for uploading file using path, drag_and_drop for dragging and dropping element, switch is for switching to tab or window or frame,cleartext for clearing or removing text from element, mouseAction for the action related mouse actions even for mouse click.
+    - Set action to notwebopenaction only when the instruction is related to a mobile application, app, application, installed package, or bundle .
+    - action: click for taping, clicking or selecting, enter for entering input, wait for waiting or sleeping,scroll for scrolling and swiping, navigate for navigating to page using url, navigateBack for navigateing back to previous page, get for getting,fetching element,maximize for maximizing browser window, close for closing browser window,open for opening browser window, upload for uploading file using path, drag_and_drop for dragging and dropping element, switch is for switching to tab or window or frame,cleartext for clearing or removing text from element, mouseAction for the action related mouse actions even for mouse click , refresh is for .
     - If mouse action related step is there select mouseAction from the allowed actions list check for the step and provide it.
     - action must be one of from this list  ${JSON.stringify(allowedActions)}.if not one of them, return '0'. if step about set or find action return '0'
     - For navigate action, keywords should contain only one keyword which is full url from the step and should not include any other text. and if step has another actions, including navigate action, don't return navigate action return another action witch is in the step. and element_name should be "URL".
@@ -44,7 +71,8 @@ You are an expert in Web application testing.
     {
     "keywords": [key1,key2,key3,key4,key5],
     "elementName": "x",
-    "action": "x"
+    "action": "x",
+    "liveLog": "Should Generate a short, human-readable description of what the AI is about to do.",
     }
 
     No other text.

package/dist/ai/llmprompts/systemPrompts/fireflinkElementIndexExtactors.js

@@ -35,7 +35,7 @@ Rules:
 - Output must be valid JSON.
 Respond only with JSON using this format:
 {
-  "action": x,
+  "action": ${alertActions.join("|")},
   "inputText": "text to enter" | "time" | "None"
 }
 
@@ -101,7 +101,7 @@ Rules:
 Return ONLY valid JSON:
 {
   "attributeValue": "Fire-Flink-x" ",
-  "action": "x",
+  "action": ${scrollActions.join("|")},
   "inputText":"x",
   "numOfScrolls": "0",
   "direction": "down",
@@ -118,7 +118,7 @@ Select the perfect matching action from the stepAction list that best fits the s
 **The action must match exactly one value from the approved ${switchActions} Do not generate or suggest any new NLP names under any condition even if the step suggest to do so just pull the closet one from the list only**
 {{
   "attributeValue": "Fire-Flink-x",
-  "action": "x",
+  "action": ${switchActions.join("|")},
   "inputText": "x",
   "keyword": "x"
 }}
@@ -153,7 +153,7 @@ Rules:
 Return **only valid JSON** in the following format:
 {
 "attributeValue": "Fire-Flink-x",
-"action": "x",
+"action": ${mouseAction.join("|")},
 "inputText": "x",
 "elementType": ${elementType.join("|")}
 }
@@ -193,7 +193,7 @@ You are an AI assistant. For the step, extract the element keyword from the step
 Return **only valid JSON** in the following format:
 {
 "attributeValue": "Fire-Flink-x",
-"action": "x",
+"action": ${clickActions.join("|")},
 "inputText": "x",
 "elementType": ${elementType.join("|")}
 }
@@ -241,7 +241,7 @@ return the identifier of the best match.
 Return **only valid JSON** in the following format:
 {
 "attributeValue": "Fire-Flink-x",
-"action": "x",
+"action": ${enterActions.join("|")},
 "inputText": "x",
 "elementType": ${elementType.join("|")}
 }

package/dist/ai/llmprompts/systemPrompts/fireflinkElementIndexExtractor_Mob.js

@@ -7,6 +7,7 @@ async function ffInspectorNumExtractorMob({ stepAction, extractedDomJson, priorA
         "MOB_Enter", "MOB_EnterDataAndPressKey", "MOB_EnterInputIntoElementFromClipBoard", "MOB_EnterUrl", "MOB_Clear", "MOB_ClearThenEnterInput",
         "MOB_LongPress", "MOB_PressEnterKey", "MOB_PressSpaceKey", "MOB_PressBackSpaceKey", "MOB_PressAnyKey", "MOB_PressAnyKeyNTimes", "MOB_EnterDataAndPressKey", "MOB_PressHomeKey", "MOB_PressBackKey"];
     const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"];
+    const OtpAction = ["getOtp"];
     let prompt = "";
     if (stepAction === "tap" || stepAction === "enter") {
         prompt = `
@@ -49,6 +50,37 @@ You are an AI assistant. For the step, extract the element keyword from the step
 
 filtered dom JSON: ${extractedDomJson}
 `;
+    }
+    else if (stepAction === "getOtp") {
+        prompt = `
+  You are an intelligent assistant that extracts structured mobile getOTP action data.
+  Given the step: ${JSON.stringify(priorAndNextSteps)},and the list of NLP names: ${OtpAction}.
+
+Select the perfect matching NLP name from the list that best fits the step's intent.
+
+Return **only valid JSON** in the following format:
+{{ 
+  "nlpName": "x",
+  "inputText": "x",
+  "timeFrom": "x",
+  "waitTime": "x",
+  "activity": "x"
+}}
+
+Rules:
+- nlpName must be exactly one from the provided list.
+- If the steps are provided with the message header and the messaging app provide the message header in the inputText and messaging app in the bundleID in the activity field.
+- If the step does not provides the messaging app don't provide example provide "activity":"" empty.
+- If the step does not provides the time range don't provide example provide "timeFrom":"" and "waitTime":"" empty.
+
+- **Extraction**:
+    - inputText: The header of the message mentioned in the step.
+    - activity: The messaging app from which the OTP is to be extracted.
+    - timeFrom: The time range for which the OTP is to be extracted just give number like "60" for 60 seconds like that don't add any extra words with that.
+    - waitTime: The time to wait for the OTP to be extracted just give number like "60" for 60 seconds like that don't add any extra words with that.
+    - Return ONLY a valid JSON object; the response must start with '{' and end with '}', with no markdown, code fences, explanations, or extra text.
+
+- response:valid json only.`;
     }
     else if (stepAction === "swipe" || stepAction === "scroll") {
         prompt = `

package/dist/ai/llmprompts/systemPrompts/mobileKeywordExtractor.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.keywordExtractorMobile = keywordExtractorMobile;
 async function keywordExtractorMobile({ priorAndNextSteps }) {
-    const allowedActions = ["openApp", "tap", "enter", "wait", "verify", "scroll", "get", "closeApp", "combined"];
+    const allowedActions = ["openApp", "tap", "enter", "wait", "verify", "scroll", "get", "closeApp", "combined", "switchToWeb", "getOtp", "enterOtp"];
     const prompt = `
 You are an expert in mobile app testing.
 From the step, extract ONLY the meaningful keywords so that i can search for the element in the dom.
@@ -10,28 +10,37 @@ Rules:
 - understand the step and context from the ${priorAndNextSteps}.and only extract keywords from current step and dont extract from prior and next steps.
 - 3 to 5 keywords maximum.
 - If step is contains more than one keyword  like tap on x button of y ,then include both in keywords like this [x,y].and include full keywords rather than insering part of it.
-- If the step is about luanching an app or opening app, like (action, param1, param2) or(action,'param1,param2') then wrap them into array and give them as keywords like ['com.app.android', 'com.app.activity'],if it is (action,param1) like (launch app, 'com.app.android') then give it as one keyword ['com.app.android'].
-and also give them in the format (launch app, 'com.app.android', 'com.app.activity') like first package name and second activity name. if any separator exists (- / | :), split on it.
-If they mention package:com.app.android and activity:com.app.activity remove variable name and then convert them into ['com.app.android, 'com.app.activity']
 - If the step is about entering text, Should NOT include entered input values from the step in keywords.
 - First keywords should be from step next Keywords must be distinct and based on the element's label meaning only.
 - **Keywords can be string or number if the step contains a number it can be any number,must include that number in keywords and give it in string format.**
+- If the step is about launching an app or opening app and if it has capbilities add them in keywords like ["key1:value1","key2:value2"...] and if step has app package and app activity without capability names like this 'com.app.android', 'com.app.activity' in keywords add with capability names
+- For the Open App / Launch App action, if the user mentions any Android app capability, extract it into the keywords array as a key-value pair using the exact format "<capability>:<value>".
+- The only allowed Android capabilities are:
+      - "app" - APK/app path
+      - "appPackage" - Android application package name
+      - "appActivity" - Android activity name
+      - "noReset" - whether the app state should be reset
+      - "fullReset" - whether a complete reset should be performed
+      - "autoGrantPermissions" - whether applicable Android runtime permissions should be automatically granted
+- Every capability mentioned by the user MUST be returned as a key-value pair and capability name must be in above capability names, Never return a capability without its value.
 - if step has keyword with type give two keywords with type and without type. ex: tap on leaving from text field. keywords = ["leaving from",leaving from text field].
 - Should NOT include any other unrelated keywords for step.
 - Should NOT include generic UI words (button, field, etc) and action words (tap, click, press, etc).example:tap on x button -> ['x']
 - Should NOT include status/technical words (displayed, enabled, authenticate, visible).
+- If the step is for the switching to web then provide the action as the switchToWeb
 - If an element label contains multiple words (e.g., "Sign In", "Add to Cart"), keep them together as ONE keyword and do not split them and also for keywords you generated, do not split them.
 -** element_name: extract name of the element that mentioned in the step  not from keywords or other steps.(eg:tap on x -> element_name:x). always try to retuen short and meaning full element name from step**
-- action: openApp for opening or launching of app and not for open app with apk and activate app is there in step give action as combined, tap for taping or selecting or clicking or pressing, enter for entering input, wait for waiting or sleeping, verify for verifying or checking,scroll for scrolling and swiping, get for getting and fetching element and gtting logs and getting driver or instance details, closeApp for closing the app but not for terminating app step.
+- action: openApp for opening or launching of app  and activate app is there in step give action as combined, tap for taping or selecting or clicking or pressing, enter for entering input, wait for waiting or sleeping, verify for verifying or checking,scroll for scrolling and swiping, get for getting and fetching element and gtting logs and getting driver or instance details, closeApp for closing the app but not for terminating app step.
 - If step is press any key give action as tap.if step is clear or clearandenter give action as enter and if step is open chrome browser app por application and it doesnot have app package or bundle id give action as combined but if step has app package or bundle id give action as openApp.
 - action must be one of from this list ${allowedActions}.if not one of them, return action as 'combined'. if step about set or setting or find, install apk or ipa, uninstall apk or ipa,activate app with app package or bundle id, pinch in or pinch out or related turn wifi or airplane mode return action as 'combined'
-- if the step action is about finding or check app is installed or open chrome browser or open app with apk path or open notification bar or terminate app using app package or bumdle id or running app in the background for some seconds then provide action as combined.
+- if the step action is about finding or check app is installed or open chrome browser  or open notification bar or terminate app using app package or bumdle id or running app in the background for some seconds then provide action as combined.
 - Return ONLY a valid JSON object; the response must start with '{' and end with '}', with no markdown, code fences, explanations, or extra text.
 Respond only with JSON using this format:
     {
-    "keywords": [key1,key2,key3,key4,key5] if step is opening app then keywords should be like [app package,app activity],
+    "keywords": [key1,key2,key3,key4,key5] if step is opening app then keywords should be like ["key1:value1","key2:value2"...],
     "elementName": "x",
-    "action": "x"
+    "action": "x",
+    "liveLog": "Should Generate a short, human-readable description of what the AI is about to do.for open app dont include whole step jsut give Opening app",
     }
 
     No other text.

package/dist/ai/llmprompts/systemPrompts/otpExtractorPrompt.d.ts

@@ -0,0 +1 @@
+export declare function otpExtractorPrompt(): Promise<string>;

package/dist/ai/llmprompts/systemPrompts/otpExtractorPrompt.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.otpExtractorPrompt = otpExtractorPrompt;
+async function otpExtractorPrompt() {
+    return `
+You extract one-time passwords (OTPs) from SMS messages and also the provider of the regex to extract that OTP from the message.
+
+
+Return only valid JSON in exactly this format:
+{
+  "otp": ""
+  "regex":""
+}
+
+Rules:
+- Extract only the OTP/code stated in the supplied SMS body.
+- Do not infer, generate, or guess an OTP.
+- If no single unambiguous OTP is present, return an empty string.
+- Return the regex that can be used to extract that OTP number itself from that message body also don't include any text only give the expression.
+- Return no text, Markdown, or keys other than "otp".
+`;
+}

package/dist/ai/llmprompts/systemPrompts/verifyActionExtractorPrompt.js

@@ -54,6 +54,8 @@ Return **only valid JSON** in the following format:
 
 Rules:
 - nlpName must be exactly one from the provided list.
+- URL steps (check in order, stop at first match): "title" mentioned → VerifyLinkNavigatesToTitle | contains/includes → VerifyUrlContainsExpectedUrl (if step says "browser"/"window"/"tab" url → VerifyBrowserWindowUrlContainsExpectedUrl instead) | literal word "exactly" or "equals" present → VerifyUrlIsString (if step says "browser"/"window"/"tab" url → VerifyBrowserWindowUrlIsString instead) | everything else, including plain "is"/"navigated to" phrasing → VerifyNavigateURL.
+  example: "verify navigated url is x" → VerifyNavigateURL ("is" here is just a linking verb, not the keyword "exactly"/"equals")
 -while matching nlpName with step (ignore spaces, case differences).If multiple NLPs are similar, choose the MOST SPECIFIC and EXACT match.
 - Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
 - **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or any other attributes best match the step in the simplified DOM.**
@@ -64,7 +66,7 @@ Rules:
 and also If a step requires comparing two values, provide the input in the format inputText: "value1,value2".
 - if step is about tag name then inputText: "tag name". and if step is about verifying tag name and with count then inputText: "tag name,count". and if any step is about verifying number of elements with xpath then inputText should be "xpath,count"
 - if step is about verifying location then inputText: "coordinates".
-- If a step involves a URL, map navigation/action keywords (navigate) to VerifyNavigateURL, partial match keywords (contains, includes) to VerifyUrlContainsExpectedUrl or VerifyBrowserWindowUrlContainsExpectedUrl exact match keywords (equals, is, exactly) to VerifyUrlIsString or VerifyBrowserWindowUrlIsString based on step  if they mention navigate to title match with (VerifyLinkNavigatesToTitle) default to exact match if unclear, and prioritize navigation if both intents exist.
+- If the step requires verifying text from an element, only select elements capable of containing text. Do not select image or image-related elements (e.g., img, svg, picture, source).
 - If no explicit option is mentioned in the step, map it to the most relevant non-option verification NLP based on the intent of the step.
 - **Extract inputText from the step that is which is being verified in the step, if you can't find any input text in the step, return keyword as inputText.**
 if step is verifying value of element then extract value which is being verified only not any other words in the step. 

package/dist/ai/llmprompts/systemPrompts/visionPrompt.js

@@ -5,19 +5,27 @@ async function visionPrompt({ priorAndNextSteps }) {
     const customPromptText = `
 You are a precise automation assistant.
 Context: ${priorAndNextSteps}
+
+- **Do not provide any additional text or explanation. The output must be strictly in this formate only Fire-Flink-x.**
 - Use the context to understand the step and select the closest semantic match to Step and return only its index, else Fire-Flink-0.
-- **If u can't find the element and also If there is no element with number, return Fire-Flink-0. dont return any other elements index.**
-- Using the provided annotated screenshot and Step, identify the correct element index from the image.
-- Match the index to the text or action described in Step as accurately as possible.
+- Using the provided annotated screenshot and Step, identify the correct target element from the image.
+- First identify the target element that matches the Step using its text, action, role, or visual characteristics.
+- Match the text or action described in Step to the correct target element as accurately as possible.
+- Check the color and the arrow/connector in the screenshot to map the target element to its Fire-Flink index.
+- **The annotation arrow/connector is the primary source of truth for mapping the target element to its Fire-Flink index. First identify the target element, then trace its annotation arrow/connector to the corresponding Fire-Flink number.**
+- **Do not select an index based only on the physical position or proximity of the number to the target element.**
+- **In crowded screenshots, Fire-Flink index numbers may overlap, appear inside other elements, or be close to unrelated elements. Ignore the number's physical position and follow the annotation arrow/connector to determine which index belongs to the target element.**
+- Use color as a secondary confirmation when tracing the annotation arrow/connector.
 - Allow partial and case-insensitive matching. If a clear match exists, return its index.
 - If the Chosen element has more than one index has been provided for it, return the index that smaller in number.
+- **If u can't find the element OR the target element's annotation cannot be reliably mapped to a numbered Fire-Flink index, return Fire-Flink-0. Do not return any other element's index.**
 - Return only the Fire-Flink-x for the target element.
-- **Never return Fire-Flink-1 only return Fire-Flink-0.**
-- Do not provide any additional text or explanation. The output must be strictly Fire-Flink-x.
+- if the element that is related to step is there in image but it does not have any bounding box around it, dont give random fire flink index number by asssuming the position, return only "Fire-Flink-0".
+-**return only "Fire-Flink-0" If given image does not have color bounding boxes around elements or if image is not even loaded fully.**
 - Return ONLY a valid JSON object; the response must start with '{' and end with '}', with no markdown, code fences, explanations, or extra text.
+
 Respond only with JSON using this format:
 {"fireflinkIndex": "Fire-Flink-x" | "Fire-Flink-0"}
-
 `.trim();
     return customPromptText;
 }

package/dist/ai/llmprompts/systemPrompts/visionPromptMobile.js

@@ -4,6 +4,7 @@ exports.visionPromptMobile = visionPromptMobile;
 async function visionPromptMobile({ priorAndNextSteps }) {
     const customPromptText = `
 You are a precise mobile automation assistant. 
+- **Do not provide any additional text or explanation. The output must be strictly in this formate only Fire-Flink-x.**
 - You are given an image containing multiple elements outlined with bounding boxes.Each bounding box has a small black index number label drawn above it (like 1, 2, 3...).
 - Do not treat numbers appearing in the step as index numbers.if any number appears in the step it is not an index number.
 - Use the context from the all steps ${priorAndNextSteps}
@@ -14,6 +15,8 @@ You are a precise mobile automation assistant.
 - Return the **index** number as shown in the image (1-based).
 - if there are two elements that match step and one have box and other doesn't have box then prefer the one with box and return that index number.
 - if no correct element found or if none of correct elements has no box,then return 0.
+- **If u can't find the element and also If there is no element with number, return Fire-Flink-0. dont return any other elements index.**
+- if the element that is related to step is there in image but it does not have any bounding box around it, dont give random fire flink index number by asssuming the position, return only "Fire-Flink-0".
 - Return ONLY a valid JSON object; the response must start with '{' and end with '}', with no markdown, code fences, explanations, or extra text.
 Respond only with JSON using this format:
 {fireflinkIndex: Fire-Flink-x | Fire-Flink-0}

package/dist/ai/llmprompts/userPrompts/userPrompt.js

@@ -12,6 +12,7 @@ exports.userInputFormatters = {
     [promptType_js_1.PromptType.GET_PROMPT]: (userInput) => `Inspect the DOM for the following get context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
     [promptType_js_1.PromptType.COMBINED_ACTION_EXTRACTOR_PROMPT]: (userInput) => `Inspect the DOM for the following wait context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
     [promptType_js_1.PromptType.WAIT_ACTION_EXTRACTOR_PROMPT]: (userInput) => `Inspect the DOM for the following wait context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
+    [promptType_js_1.PromptType.OTP_EXTRACTOR]: (userInput) => `Extract the OTP from this SMS body:\n\n${JSON.stringify(userInput.smsBody, null, 2)}`,
     [promptType_js_1.PromptType.VISION_PROMPT]: (userInput) => {
         let imageBase64 = null;
         if (userInput.annotatedScreenshot) {

package/dist/automation/actions/executor.d.ts

@@ -366,4 +366,8 @@ export declare class ActionExecutor implements IActionExecutor {
     MouseDoubleClickAtCursorPoint(): Promise<void>;
     ReleaseLeftMouseButton(pageDOM: string, selector: string, fireflinkIndex: string, elementName: string, elementType: string): Promise<void>;
     MouseClickOnCurrentCursorPointNTimes(value: string): Promise<void>;
+    getOtp(selector: string, timeFrom: string, waitTime: string, activity: string, otpExtractor?: (smsBody: string) => Promise<{
+        otp: string;
+        regex: string;
+    } | null>): Promise<void>;
 }

package/dist/automation/actions/executor.js

@@ -357,6 +357,7 @@ const mouseClickOnCurrentCursorPoint_js_1 = require("./interaction/mouse/mouseCl
 const mouseDoubleClickAtCursorPoint_js_1 = require("./interaction/mouse/mouseDoubleClickAtCursorPoint.js");
 const mouseReleaseLeftMouseButton_js_1 = require("./interaction/mouse/mouseReleaseLeftMouseButton.js");
 const mouseClickOnCurrentCursorNTImes_js_1 = require("./interaction/mouse/mouseClickOnCurrentCursorNTImes.js");
+const getOtp_js_1 = require("./interaction/elementlessActions/getOtp.js");
 class ActionExecutor {
     constructor(driver, scriptDataAppender, elementGetter, platform, adbPath) {
         this.driver = driver;
@@ -5969,5 +5970,24 @@ class ActionExecutor {
             throw error;
         }
     }
+    async getOtp(selector, timeFrom, waitTime, activity, otpExtractor) {
+        try {
+            await (0, getOtp_js_1.GetOtpAction)({
+                driver: this.driver,
+                adbPath: this.adbPath,
+                selector,
+                timeFrom,
+                waitTime,
+                activity,
+                scriptDataAppender: this.scriptDataAppender,
+                platform: this.platform,
+                otpExtractor
+            });
+        }
+        catch (error) {
+            logData_js_1.logger.error("Error in [ActionExecutor]", error);
+            throw error;
+        }
+    }
 }
 exports.ActionExecutor = ActionExecutor;

package/dist/automation/actions/interaction/clear/clear.js

@@ -45,7 +45,7 @@ async function clear(args) {
                     ffNumber: ffElementNumber,
                     elementName: args.elementName,
                     elementType: args.elementType,
-                    selector: args.selector,
+                    selector: xpathPath,
                     platform: args.platform
                 });
             }

package/dist/automation/actions/interaction/clear/clearAndEnter.js

@@ -49,7 +49,7 @@ async function clearAndEnter(args) {
                     ffNumber: ffElementNumber,
                     elementName: args.elementName,
                     elementType: args.elementType,
-                    selector: args.selector,
+                    selector: xpathPath,
                     platform: args.platform
                 });
             }