ff-automationv2 2.2.28 → 2.2.30

package/dist/ai/llmcalls/llmAction.d.ts

@@ -2,15 +2,20 @@ import { FireFlinkLLMResponse } from "../../core/types/llmResponseType.js";
 import { PromptArgsMap } from "../../core/interfaces/promptInterface.js";
 import { platform } from "../../core/types/scriptTypesType.js";
 export declare class llmAction {
-    private client;
-    private visionClient;
     private model;
     private visionApiKey;
-    constructor(apiKey: string, baseURL: string, model: string, visionApiKey: string);
-    getLLMResponse<K extends keyof PromptArgsMap>(platform: platform, type: K, args: PromptArgsMap[K], userInput: Record<string, any>): Promise<FireFlinkLLMResponse>;
+    private apiKey;
+    private provider;
+    private serviceProvider;
+    private visionProvider;
+    constructor(apiKey: string, provider: any, model: string, visionApiKey: string, accountType: string, enterpriseConfigFile: any, serviceProvider: string);
+    getLLMResponse<K extends keyof PromptArgsMap>(platform: platform, type: K, args: PromptArgsMap[K], userInput: Record<string, any>, isVision: boolean): Promise<FireFlinkLLMResponse>;
     private getLLMResponseText;
+    private getProviderToUse;
     private getLLMResponseWithVision;
     private isInvalidLLMOutput;
     private handleKnownErrors;
     private buildRetryFailure;
+    private extractValidatedJson;
+    private getProviderOptions;
 }

package/dist/ai/llmcalls/llmAction.js

@@ -1,65 +1,101 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.llmAction = void 0;
 const promptRegistry_js_1 = require("../llmprompts/promptRegistry.js");
-const openai_1 = __importDefault(require("openai"));
-const decodeApiKey_js_1 = require("./decodeApiKey.js");
 const userPrompt_js_1 = require("../../ai/llmprompts/userPrompts/userPrompt.js");
 const promptType_js_1 = require("../../core/types/promptType.js");
 const logData_js_1 = require("../../utils/logger/logData.js");
+const ai_1 = require("ai");
+const FireFlinkResponseSchema_js_1 = require("../../automation/actions/interface/FireFlinkResponseSchema.js");
+const llmConfiguration_js_1 = require("../../llmConfig/llmConfiguration.js");
 class llmAction {
-    constructor(apiKey, baseURL, model, visionApiKey) {
-        this.model = model;
+    constructor(apiKey, provider, model, visionApiKey, accountType, enterpriseConfigFile, serviceProvider) {
+        this.apiKey = apiKey;
+        this.provider = provider;
+        this.serviceProvider = serviceProvider;
+        if (model.startsWith("publishers/google/models/")) {
+            this.model = `${model.split("/").pop()}`;
+        }
+        else if (model.startsWith("publishers/meta/models/")) {
+            this.model = `${model.split("/").pop()}`;
+        }
+        else if (model.startsWith("models/")) {
+            this.model = model.split("/").pop();
+        }
+        else {
+            this.model = model;
+        }
         this.visionApiKey = visionApiKey;
-        this.client = new openai_1.default({
-            apiKey: (0, decodeApiKey_js_1.decodeApiKey)(apiKey),
-            baseURL: baseURL,
-        });
-        this.visionClient = new openai_1.default({
-            apiKey: (0, decodeApiKey_js_1.decodeApiKey)(this.visionApiKey ?? "c2stcHJvai1HVTVlZGt4UDJEQXVBa29IbzB0MXdsTWFrWjljNnlQQjEtdUdGLXloZmhIclhEYnZubHo0M0l4QUoyME5Pc0hGYUw2OUFkdG5kOVQzQmxia0ZKQzUxTFJZX09aQzNzRHBOYjN5XzM5SUY4UXpBZElSU2ZBQ21wZjRtZG1OUzBlWXRhUjhudGhWdUktb2djeVBSMW1PdVpEdU83UUE="),
-        });
     }
-    async getLLMResponse(platform, type, args, userInput) {
+    async getLLMResponse(platform, type, args, userInput, isVision) {
         const promptBuilder = promptRegistry_js_1.prompts[platform][type];
         const systemPrompt = await promptBuilder(args);
         const userPrompt = userPrompt_js_1.userInputFormatters[type](userInput);
         if (type === promptType_js_1.PromptType.VISION_PROMPT && Array.isArray(userPrompt)) {
-            return this.getLLMResponseWithVision(systemPrompt, userPrompt);
+            return this.getLLMResponseWithVision(systemPrompt, userPrompt, isVision);
         }
         if (typeof userPrompt !== "string") {
             throw new Error("Invalid non-vision user prompt format");
         }
-        return this.getLLMResponseText(systemPrompt, userPrompt);
+        return this.getLLMResponseText(platform, systemPrompt, userPrompt, type, args);
     }
-    async getLLMResponseText(system, user) {
+    async getLLMResponseText(platform, system, user, type, args) {
         let attempt = 0;
         const MAX_RETRIES = 5;
-        // const startTime = new Date().getTime()
+        const startTime = new Date().getTime();
+        const schemaToUse = (0, FireFlinkResponseSchema_js_1.getSchemaForPrompt)(type, platform, args);
+        let modelInstance;
+        if (this.serviceProvider === "Groq" || this.serviceProvider === "DefaultFireFlink") {
+            modelInstance = this.provider.responses(this.model);
+        }
+        else {
+            modelInstance = this.provider(this.model);
+        }
         while (attempt < MAX_RETRIES) {
             attempt++;
             try {
-                const response = await this.client.chat.completions.create({
-                    model: this.model,
+                const response = await (0, ai_1.generateText)({
+                    model: modelInstance,
                     messages: [
                         { role: "system", content: system },
                         { role: "user", content: user },
                     ],
-                    response_format: { type: "json_object" },
+                    output: ai_1.Output.object({
+                        schema: schemaToUse,
+                    }),
+                    providerOptions: this.getProviderOptions()
                 });
-                const content = response.choices?.[0]?.message?.content;
-                logData_js_1.logger.info(`LLM Attempt ${attempt} - Raw Response:`, content);
-                if (this.isInvalidLLMOutput(content)) {
+                const parsedContent = response.output;
+                logData_js_1.logger.info(`LLM Attempt ${attempt} - Raw Response:`, parsedContent);
+                if (this.isInvalidLLMOutput(parsedContent)) {
                     logData_js_1.logger.error(`Invalid LLM output. Retrying (${attempt})`);
                     continue;
                 }
-                // logger.info("Time taken to get LLM response (Text):", Date.now() - startTime, "ms");
-                return response;
+                await logData_js_1.logger.info("Time taken to get LLM response (Text):", Date.now() - startTime, "ms");
+                return {
+                    ...response,
+                    output: parsedContent
+                };
             }
             catch (error) {
                 logData_js_1.logger.error(`LLM Error Attempt ${attempt}:`, error);
+                if (attempt < MAX_RETRIES) {
+                    continue;
+                }
+                logData_js_1.logger.error(`Cause is ${error.cause}`, error.cause);
+                if (ai_1.NoObjectGeneratedError.isInstance(error) && error.text) {
+                    try {
+                        const recovered = this.extractValidatedJson(error.text, schemaToUse);
+                        await logData_js_1.logger.info("Time taken to get LLM response (Text, recovered):", Date.now() - startTime, "ms");
+                        return {
+                            text: error.text,
+                            output: recovered
+                        };
+                    }
+                    catch (recoveryError) {
+                        logData_js_1.logger.error(`Regex recovery also failed on attempt ${attempt}:`, recoveryError);
+                    }
+                }
                 const structuredError = this.handleKnownErrors(error);
                 if (structuredError)
                     return structuredError;
@@ -67,32 +103,82 @@ class llmAction {
         }
         return this.buildRetryFailure();
     }
-    async getLLMResponseWithVision(system, userPrompt) {
+    async getProviderToUse(isVision) {
+        if (this.serviceProvider === "DefaultFireFlink" && isVision) {
+            if (!this.visionProvider) {
+                const configProvider = new llmConfiguration_js_1.ServiceProviderProviderConfiguration();
+                this.visionProvider = await configProvider.getProvider({
+                    serviceProvider: "OpenAi",
+                    apiKey: this.visionApiKey,
+                });
+            }
+            return this.visionProvider;
+        }
+        return this.provider;
+    }
+    async getLLMResponseWithVision(system, userPrompt, isVision = false) {
         let attempt = 0;
         const MAX_RETRIES = 5;
-        // const startTime = new Date().getTime()
+        let content;
+        const startTime = new Date().getTime();
+        const providerToUse = await this.getProviderToUse(isVision);
+        const modelToUse = (this.serviceProvider === "DefaultFireFlink" && isVision) ? "gpt-4.1" : this.model;
         while (attempt < MAX_RETRIES) {
             attempt++;
             try {
-                const response = await this.visionClient.chat.completions.create({
-                    model: "gpt-4.1",
+                const parts = userPrompt.map((item) => {
+                    if (item.type === "input_text") {
+                        return {
+                            type: "text",
+                            text: item.text,
+                        };
+                    }
+                    let imageValue = item.image_url;
+                    if (item.image_url.startsWith("http://") || item.image_url.startsWith("https://")) {
+                        imageValue = new URL(item.image_url);
+                    }
+                    return {
+                        type: "image",
+                        image: imageValue,
+                    };
+                });
+                const response = await (0, ai_1.generateText)({
+                    model: providerToUse(modelToUse),
                     messages: [
                         { role: "system", content: system },
-                        { role: "user", content: userPrompt },
+                        { role: "user", content: parts },
                     ],
-                    response_format: { type: "json_object" },
+                    output: ai_1.Output.object({
+                        schema: FireFlinkResponseSchema_js_1.VisionSchema,
+                    }),
+                    providerOptions: this.getProviderOptions()
                 });
-                const content = response.choices?.[0]?.message?.content;
-                logData_js_1.logger.info(`Vision LLM Attempt ${attempt} - Raw Response:`, content);
+                content = response.output;
+                const parsedContent = content;
+                logData_js_1.logger.info(`Vision LLM Attempt ${attempt} - Raw Response:`, parsedContent);
                 if (this.isInvalidLLMOutput(content)) {
                     logData_js_1.logger.error(`Invalid Vision output. Retrying (${attempt})`);
                     continue;
                 }
-                // logger.info("Time taken to get LLM response (vision):", Date.now() - startTime, "ms");
+                await logData_js_1.logger.info("Time taken to get LLM response (vision):", Date.now() - startTime, "ms");
                 return response;
             }
             catch (error) {
-                logData_js_1.logger.error(`Vision LLM Error Attempt ${attempt}:`);
+                logData_js_1.logger.error(`Vision LLM Error Attempt ${attempt}:`, error);
+                if (ai_1.NoObjectGeneratedError.isInstance(error) && error.text) {
+                    try {
+                        const recovered = this.extractValidatedJson(error.text, FireFlinkResponseSchema_js_1.VisionSchema);
+                        logData_js_1.logger.info(`Vision LLM Attempt ${attempt} - Recovered via regex fallback:`, recovered);
+                        await logData_js_1.logger.info("Time taken to get LLM response (vision, recovered):", Date.now() - startTime, "ms");
+                        return {
+                            text: error.text,
+                            output: recovered
+                        };
+                    }
+                    catch (recoveryError) {
+                        logData_js_1.logger.error(`Regex recovery also failed on attempt ${attempt}:`, recoveryError);
+                    }
+                }
                 const structuredError = this.handleKnownErrors(error);
                 if (structuredError)
                     return structuredError;
@@ -124,29 +210,115 @@ class llmAction {
         return false;
     }
     handleKnownErrors(error) {
-        if (error?.status === 429) {
+        if (error?.lastError?.statusCode === 429) {
             return {
                 error: "Rate Limit Exceeded (429)",
-                errorDescription: "FireFlink has exceeded the API quota or rate limit for the selected AI model. \n This is common with free or low-tier API plans \n Please wait a moment before retrying or check your API usage limits.",
+                errorDescription: "Service has exceeded the API quota or rate limit for the selected AI model. \n This is common with free or low-tier API plans \n Please wait a moment before retrying or check your API usage limits.",
                 rawError: "Rate limit reached for model. HTTP 429 returned by model"
             };
         }
-        if (error?.status === 400 &&
+        else if (error?.statusCode === 400 &&
             error?.error?.message?.toLowerCase().includes("context")) {
             return {
                 error: "Context Length Exceeded (400)",
-                errorDescription: "FireFlink could not process the request because the input exceeds the maximum context length supported by the selected AI model.\nThis typically happens when the input is very long",
+                errorDescription: "Service could not process the request because the input exceeds the maximum context length supported by the selected AI model.\nThis typically happens when the input is very long",
                 rawError: "This model's maximum context length has been exceeded. HTTP 400 returned by model"
             };
         }
+        else if (error?.statusCode === 401 &&
+            error?.error?.message?.toLowerCase().includes("Incorrect api key")) {
+            return {
+                error: "Invalid API key",
+                errorDescription: "Service could not process the request because the API key is invalid. Please check your API key and try again.",
+                rawError: "Invalid API key"
+            };
+        }
+        else if (error?.statusCode !== 200) {
+            return {
+                error: error.message,
+                errorDescription: "Service has can't process the request. Because of service Provider internal error",
+                rawError: error,
+            };
+        }
         return null;
     }
     buildRetryFailure() {
         return {
             error: "LLM Empty Response After Retry",
-            errorDescription: "FireFlink attempted multiple times but received empty or invalid responses.",
+            errorDescription: "Service attempted multiple times but received empty or invalid responses.",
             rawError: "Max retry limit reached",
         };
     }
+    extractValidatedJson(rawText, schemaToUse) {
+        const trimmed = rawText.trim();
+        const jsonMatch = trimmed.match(/\{[\s\S]*\}/);
+        if (!jsonMatch) {
+            throw new Error("No JSON object found in LLM response");
+        }
+        const parsedJson = JSON.parse(jsonMatch[0]);
+        return schemaToUse.parse(parsedJson);
+    }
+    getProviderOptions() {
+        switch (this.serviceProvider) {
+            case "OpenAI":
+                return {
+                    openai: {
+                        reasoningEffort: "low",
+                    },
+                };
+            case "Anthropic":
+                if (this.model.startsWith("claude-haiku")) {
+                    return {
+                        anthropic: {
+                            thinking: {
+                                type: "enabled",
+                                budgetTokens: 1024
+                            }
+                        },
+                    };
+                }
+                else {
+                    return {
+                        anthropic: {
+                            effort: "low",
+                        },
+                    };
+                }
+            case "Gemini":
+                if (this.model.startsWith("gemini-2")) {
+                    return {
+                        google: {
+                            thinkingConfig: {
+                                thinkingBudget: 128,
+                            },
+                        },
+                    };
+                }
+                if (this.model.startsWith("gemini-3")) {
+                    return {
+                        google: {
+                            thinkingConfig: {
+                                thinkingLevel: "low",
+                            },
+                        },
+                    };
+                }
+                return {};
+            case "Groq":
+                return {
+                    groq: {
+                        reasoningEffort: "low",
+                    },
+                };
+            case "Azure_AI":
+                return {
+                    openai: {
+                        reasoningEffort: "low",
+                    },
+                };
+            default:
+                return {};
+        }
+    }
 }
 exports.llmAction = llmAction;

package/dist/ai/llmcalls/parseLlmOputput.js

@@ -15,29 +15,41 @@ class LLMResultParser {
         return JSON.parse(match[0]);
     }
     async fetchResult(llmOutput) {
-        if (typeof llmOutput?.error === "string" && llmOutput.error.includes("429")) {
+        if (llmOutput?.error) {
             throw llmOutput;
         }
         try {
-            const content = llmOutput?.choices?.[0]?.message?.content;
+            let content;
+            if (llmOutput?.output) {
+                content = llmOutput?.output;
+            }
+            else if (llmOutput?.output_text) {
+                content = llmOutput?.output_text;
+            }
             if (!content) {
                 throw new Error("No content in LLM output");
             }
+            // if (typeof content === "string") {
+            //     content = content
+            //         .replace(/^[\s\S]*?(?=\{)|(?<=\})[\s\S]*$/g, "")
+            //         .trim();
+            // }
             let parsedContent;
             try {
                 parsedContent = JSON.parse(content);
             }
             catch {
-                const extracted = this.extractJsonObject(content);
-                if (!extracted) {
+                // const extracted = this.extractJsonObject(content);
+                if (!content) {
                     throw new Error("Could not extract valid JSON from LLM response");
                 }
-                parsedContent = extracted;
+                parsedContent = content;
             }
-            const usage = llmOutput?.usage ?? {};
-            this.inputTokens += Number(usage.prompt_tokens ?? 0);
-            this.outputTokens += Number(usage.completion_tokens ?? 0);
-            this.totalTokens += Number(usage.total_tokens ?? 0);
+            const usage = llmOutput?.totalUsage ?? {};
+            this.inputTokens += Number(usage.inputTokens ?? 0);
+            this.outputTokens += Number(usage.outputTokens ?? 0);
+            this.totalTokens += Number(usage.totalTokens ?? 0);
+            // logger.info(`tokens : ${usage.output_tokens} + ${usage.input_tokens} = ${usage.total_tokens}`);
             // logger.info(`Total tokens for this call is : ${usage.total_tokens}`);
             return { response: parsedContent };
         }

package/dist/ai/llmprompts/promptRegistry.js

@@ -17,6 +17,7 @@ const combinedActionExtractorPromptMob_js_1 = require("./systemPrompts/combinedA
 const waitActionExtractorPromptMob_js_1 = require("./systemPrompts/waitActionExtractorPromptMob.js");
 const waitActionExtractorPrompt_js_1 = require("./systemPrompts/waitActionExtractorPrompt.js");
 const combinedActionExtractorPrompt_js_1 = require("./systemPrompts/combinedActionExtractorPrompt.js");
+const otpExtractorPrompt_js_1 = require("./systemPrompts/otpExtractorPrompt.js");
 exports.prompts = {
     web: {
         userStoryToList: userStoryToListPrompt_js_1.buildStepExtractionPrompt,
@@ -28,6 +29,7 @@ exports.prompts = {
         getActionExtractorPrompt: getActionExtractorPrompt_js_1.getActionExtractorPrompt,
         combinedActionExtractorPrompt: combinedActionExtractorPrompt_js_1.combinedActionExtractorPrompt,
         waitActionExtractorPrompt: waitActionExtractorPrompt_js_1.waitActionExtractorPrompt,
+        otpExtractorPrompt: otpExtractorPrompt_js_1.otpExtractorPrompt,
     },
     android: {
         userStoryToList: userStoryToListPrompt_js_1.androidbuildStepExtractionPrompt,
@@ -39,5 +41,18 @@ exports.prompts = {
         getActionExtractorPrompt: getActionExtractorPromptMob_js_1.getActionExtractorPromptMob,
         combinedActionExtractorPrompt: combinedActionExtractorPromptMob_js_1.combinedActionExtractorPromptMob,
         waitActionExtractorPrompt: waitActionExtractorPromptMob_js_1.waitActionExtractorPromptMob,
+        otpExtractorPrompt: otpExtractorPrompt_js_1.otpExtractorPrompt,
+    },
+    ios: {
+        userStoryToList: userStoryToListPrompt_js_1.androidbuildStepExtractionPrompt,
+        keywordExtractorPrompt: mobileKeywordExtractor_js_1.keywordExtractorMobile,
+        errorDescriptionPrompt: errorDescriptionPrompt_js_1.buildErrorDescriptionPrompt,
+        ffInspectorNumExtractor: fireflinkElementIndexExtractor_Mob_js_1.ffInspectorNumExtractorMob,
+        verifyActionExtractorPrompt: verifyActionExtractorPromptMob_js_1.verifyActionExtractorPromptMob,
+        visionPrompt: visionPromptMobile_js_1.visionPromptMobile,
+        getActionExtractorPrompt: getActionExtractorPromptMob_js_1.getActionExtractorPromptMob,
+        combinedActionExtractorPrompt: combinedActionExtractorPromptMob_js_1.combinedActionExtractorPromptMob,
+        waitActionExtractorPrompt: waitActionExtractorPromptMob_js_1.waitActionExtractorPromptMob,
+        otpExtractorPrompt: otpExtractorPrompt_js_1.otpExtractorPrompt,
     }
 };

package/dist/ai/llmprompts/systemPrompts/actionExtractorPrompt.js

@@ -2,40 +2,77 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.keywordExtractor = keywordExtractor;
 async function keywordExtractor({ priorAndNextSteps }) {
-    const allowedActions = ["enter", "wait", "verify", "scroll", "navigate", "navigateBack", "click", "maximize", "minimize", "get", "upload", "close", "open", "drag_and_drop", "switch", "cleartext"];
+    const allowedActions = ["enter", "refresh", "wait", "verify", "scroll", "navigate", "navigateBack", "click", "maximize", "minimize", "get", "upload", "close", "open", "drag_and_drop", "switch", "cleartext", "notwebopenaction", "mouseAction", "switchToAndroid", "getOtp", "enterOtp"];
     const prompt = `
 You are an expert in Web application testing.
     From the step, extract ONLY the meaningful keywords so that i can search for the element in the dom.
     Rules:
     - Only give response for the current step.
     - understand the step and context from the ${JSON.stringify(priorAndNextSteps)}.and keywords should be from step. it should not be related to other steps.
-    - 3 to 5 keywords maximum. dont give more than 5 keywords. and no need for casesensitive.And first keyword should be full keyword from step same as mentioned.like (click on search bar.in -> "search bar.in")
+    - Keywords must represent ONLY the element label or visible text, never add action or instruction words in keywords.
+    - Element name + type keywords: strip action words, leaving the core element name and any
+  type word(s) attached to it (icon, button, dropdown, filter, tab, field, etc., in the order
+  they appear in the step). Build keywords as a growing chain starting from the core name and
+  adding one more type word each time: [name], [name+type1], [name+type1+type2], ...
+  The core name alone is ALWAYS keyword 1 — never drop it, no matter how many type words follow.
+  Treat "input field", "text field", "search box", "check box", "drop down" as ONE type word,
+  not two.
+  No type word in the step → only [name] as the keyword.
+  Example: "click X filter dropdown" → ["X", "X filter", "X filter dropdown"]
+    - When a step contains two elements connected by a positional or relational phrase, keywords MUST contain at least one keyword from each element — TARGET element first, then the reference element. Never omit the reference element. Example: "Click on x above y" → x is the target, y is the reference → ["x", "y"].
+    - 3 to 5 keywords maximum. dont give more than 5 keywords. and no need for casesensitive.
+    - General ordering rule: whenever a step involves both a target element and a reference/context element, the TARGET element's keyword(s) must always come FIRST, followed by the reference/context element's keyword(s). This applies to positional/relational steps, selection steps, and any other multi-element step.
+    - For the open action if they are not provided anything like profile, capablities in the step in that case don't add them check whether the profile and the capablities are present in the step if not please don't add them.
+    - For the Open action, if a browser path and profile are provided, return keywords as ["path:<path_value>", "profile:<profile_value>"]. Should not remove backslach from path.
+    example: step open browser,x:\\x\\x\\x,default -> ["path:c:\\x\\x\\x","profile:default"] 
+    - For the Open action, if the user requests specific browser capabilities/arguments, extract them into the keywords array using the "caps:" prefix followed by one of these exact allowed values: 
+      - "headless" (for running in headless / no-gui mode)
+      - "incognito" (for private / incognito mode)
+      - "ignoreCertificates" (for ignoring SSL/certificate errors)
+      - "startMaximized" (for starting maximized)
+      - "disableGpu" (for disabling hardware/GPU acceleration)
+      - "disableNotifications" (for blocking push notifications)
+      - "disableExtensions" (for disabling browser extensions)
+      - "disablePopups" (for disabling popup blocking)
+      - "muteAudio" (for muting browser audio)
+      - "noSandbox" (for disabling sandbox)
+      - "disableWebSecurity" (for disabling web security, CORS, or same-origin policy checks)
+      - "disableLocalNetwork" (for disabling WebRTC local IP discovery or local network access constraints)
+      - "geolocation" (for blocking or allowing geolocation access)
+      - "camera" (for blocking or allowing camera access)
+      - "microphone" (for blocking or allowing microphone access)
+      - "passwordManager" (for disabling credentials and password manager prompts)
+      Do NOT extract or generate any capability strings outside of this list.
+    - Do NOT split the keyword into individual words or generate variations such as ["sign", "in"] or ["add", "to", "cart"] for ["Sign In","Add to Cart"]. Only include the original phrase should not add type for first keyword.
+    - If icon is metioned in step than 'svg' should add in keywords and for Upload action first keyword should be 'file'.
+    - If the step is about selecting, include keywords for both the target value and the source/reference, with the TARGET value FIRST and the source/reference SECOND (example: select 'x' in 'y' → "x", "y"), ensuring both are meaningful and action should set to 'click'.
     - If the step is about entering text or Uploading file, Should NOT include input value from the step into keywords.
+    - If the step is about entering the OTP then give the action as the enterOtp, just only for the entering OTP only.
     - If the step has words like tag name audio, video, image,svg, checkbox etc, include them in the keywords.
-    - If icon is metioned in step than 'svg' should add in keywords and for Upload action first keyword should be 'file'.
-    - First keywords should be from step next Keywords must be distinct and based on the element's label meaning only.
-    - If a keyword contains exactly two words, you MUST always include both the spaced and concatenated (no-space) versions; omission of either is invalid.
+    - **If icon is metioned in step than 'svg' should add in keywords ex: click on x icon -> [x, x icon,svg] and for Upload action first keyword should be 'file'. ex: upload file in x  -> [file,x].**
     - Do NOT split single-word keywords and do NOT include relation terms (above, below, next to, etc.) in keywords.
-    - Treat each keyword independently—never merge different keywords or combine them with relation words.
-    - Example: Click on Sign In above Create Account button → ["Sign In", "SignIn", "Create Account", "CreateAccount"] never break kewword like ["create", "account"]
+    - Treat each keyword independently never merge different keywords.
     - Keywords can be string or number if the step contains a number ,add it also as keyword.
     - Do Not include any other unrelated keywords for step. 
     - Do NOT include generic UI words (button, field, etc) and action words (tap, click, press, etc).
     - Do NOT include status/technical words (displayed, enabled, authenticate, visible).
-    - If an element label contains multiple words (e.g., "Sign In", "Add to Cart"), keep them together as ONE keyword like ["Sign In", "SignIn", "Add to Cart", "AddToCart"] and Never split them and also for keywords you generated, should not split them like ["sign", "in","add","to","cart"].
+    - If the step is for the switching to android then provide the action as the switchToAndroid
     - element_name: extract name of the element that mentioned in the the step.(eg:tap on x -> element_name:x) keep element_name as short as possible and make the first letter of first word of the element_name as capital. beacuse element_name is also used to find element in the dom. and if element_name is not mentioned in step than return action of the step as element_name.
-    - action: click for taping, clicking or selecting, enter for entering input, wait for waiting or sleeping,scroll for scrolling and swiping, navigate for navigating to page using url, navigateBack for navigateing back to previous page, get for getting,fetching element,maximize for maximizing browser window, close for closing browser window,open for opening browser window, upload for uploading file using path, drag_and_drop for dragging and dropping element, switch is for switching to tab or window or frame,cleartext for clearing or removing text from element.
+    - Set action to notwebopenaction only when the instruction is related to a mobile application, app, application, installed package, or bundle .
+    - action: click for taping, clicking or selecting, enter for entering input, wait for waiting or sleeping,scroll for scrolling and swiping, navigate for navigating to page using url, navigateBack for navigateing back to previous page, get for getting,fetching element,maximize for maximizing browser window, close for closing browser window,open for opening browser window, upload for uploading file using path, drag_and_drop for dragging and dropping element, switch is for switching to tab or window or frame,cleartext for clearing or removing text from element, mouseAction for the action related mouse actions even for mouse click , refresh is for .
+    - If mouse action related step is there select mouseAction from the allowed actions list check for the step and provide it.
     - action must be one of from this list  ${JSON.stringify(allowedActions)}.if not one of them, return '0'. if step about set or find action return '0'
     - For navigate action, keywords should contain only one keyword which is full url from the step and should not include any other text. and if step has another actions, including navigate action, don't return navigate action return another action witch is in the step. and element_name should be "URL".
     Navigate action is allowed only when the step intent is purely navigation (e.g., "Navigate to URL").
     If the step includes verification intent (verify/check/confirm/etc), you MUST return "verify" and MUST NOT return "navigate".
     only give navigate action if step has only navigate action.
-         
+    - Return ONLY a valid JSON object; the response must start with '{' and end with '}', with no markdown, code fences, explanations, or extra text.     
     Respond only with JSON using this format:
     {
     "keywords": [key1,key2,key3,key4,key5],
     "elementName": "x",
-    "action": "x"
+    "action": "x",
+    "liveLog": "Should Generate a short, human-readable description of what the AI is about to do.",
     }
 
     No other text.

package/dist/ai/llmprompts/systemPrompts/combinedActionExtractorPrompt.js

@@ -12,23 +12,24 @@ Select the perfect matching NLP name from the list that best fits the step's int
 
 Return **only valid JSON** in the following format:
 {
-  "attribute_value": "Fire-Flink-x",
+  "attributeValue": "Fire-Flink-x",
   "nlpName": "x",
-  "input_text":"x",
+  "inputText":"x",
   "keyword": "x",
-  "element_type": "x",
+  "elementType": "x",
   "keycode": "x"
 }
 Rules:
 - nlpName must be exactly one from the provided list.
 - Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
 - **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
-- If step has mutipul inputs add all in input_text and all matched attribute_value also.
-- Extract input_text from the step, if you can't find any input text in the step, return keyword as input_text.input can be number, app package, apk file path.dont include extra words.
+- If step has mutipul inputs add all in inputText and all matched attributeValue also.
+- Extract inputText from the step, if you can't find any input text in the step, return keyword as inputText.input can be number, app package, apk file path.dont include extra words.
 - Extract keyword from the step, if the step contains any keyword.
 - Extract keycode of keys if the step contains any key name like ENTER,SPACE,BACKSPACE etc and give keycode of the given key.
-- Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
+- Use the closest semantic match for the step; return attributeValue as Fire-Flink-0, only if nothing is found.
 - Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
+-  Return ONLY a valid JSON object; the response must start with '{' and end with '}', with no markdown, code fences, explanations, or extra text.
 - Respond with valid JSON only.
 `;
     return prompt;