ff-automationv2 2.1.3 → 2.2.2

package/README.md

@@ -1,28 +1,28 @@
-# 🔥 FireFlink AI Automation Platform
-
-Enterprise-grade AI-powered automation engine that transforms natural
-language user stories into executable browser automation scripts using
-LLM-driven reasoning and intelligent DOM abstraction.
-
-## Executive Summary
-
-FireFlink enables organizations to: - Convert user stories into
-automated test scripts - Reduce manual scripting effort - Improve
-element resilience using AI-based DOM interpretation - Track LLM token
-usage and automation generation cost - Integrate automation results
-directly into backend infrastructure
-
-## Core Capabilities
-
--   AI Step Generation
--   Intelligent Element Resolution
--   DOM Annotation Engine (FF-inspecter system)
--   Modular Action Handler Framework
--   Failure Isolation & Safe Termination
--   Token Usage Monitoring
--   API Integration Layer
-
-## High-Level Execution Lifecycle
-
-User Story → LLM Steps → DOM Annotation → Element Resolution → Action
-Execution → Script Generation → API Submission
+# 🔥 FireFlink AI Automation Platform
+
+Enterprise-grade AI-powered automation engine that transforms natural
+language user stories into executable browser automation scripts using
+LLM-driven reasoning and intelligent DOM abstraction.
+
+## Executive Summary
+
+FireFlink enables organizations to: - Convert user stories into
+automated test scripts - Reduce manual scripting effort - Improve
+element resilience using AI-based DOM interpretation - Track LLM token
+usage and automation generation cost - Integrate automation results
+directly into backend infrastructure
+
+## Core Capabilities
+
+-   AI Step Generation
+-   Intelligent Element Resolution
+-   DOM Annotation Engine (FF-inspecter system)
+-   Modular Action Handler Framework
+-   Failure Isolation & Safe Termination
+-   Token Usage Monitoring
+-   API Integration Layer
+
+## High-Level Execution Lifecycle
+
+User Story → LLM Steps → DOM Annotation → Element Resolution → Action
+Execution → Script Generation → API Submission

package/dist/ai/llmcalls/decodeApiKey.d.ts

@@ -0,0 +1 @@
+export declare function decodeApiKey(apiKey: string): string;

package/dist/ai/llmcalls/decodeApiKey.js

@@ -0,0 +1,11 @@
+export function decodeApiKey(apiKey) {
+    if (!apiKey) {
+        throw new Error("API key is required");
+    }
+    try {
+        return Buffer.from(apiKey, "base64").toString("utf-8");
+    }
+    catch (error) {
+        throw new Error("Failed to decode API key", { cause: error });
+    }
+}

package/dist/ai/llmcalls/llmAction.d.ts

@@ -0,0 +1,16 @@
+import { FireFlinkLLMResponse } from "../../core/types/llmResponseType.js";
+import { PromptArgsMap } from "../../core/interfaces/promptInterface.js";
+import { platform } from "../../core/types/scriptTypesType.js";
+export declare class llmAction {
+    private client;
+    private visionClient;
+    private model;
+    private visionApiKey;
+    constructor(apiKey: string, baseURL: string, model: string, visionApiKey: string);
+    getLLMResponse<K extends keyof PromptArgsMap>(platform: platform, type: K, args: PromptArgsMap[K], userInput: Record<string, any>): Promise<FireFlinkLLMResponse>;
+    private getLLMResponseText;
+    private getLLMResponseWithVision;
+    private isInvalidLLMOutput;
+    private handleKnownErrors;
+    private buildRetryFailure;
+}

package/dist/ai/llmcalls/llmAction.js

@@ -0,0 +1,145 @@
+import { prompts } from "../llmprompts/promptRegistry.js";
+import OpenAI from "openai";
+import { decodeApiKey } from "./decodeApiKey.js";
+import { userInputFormatters } from "../../ai/llmprompts/userPrompts/userPrompt.js";
+import { PromptType } from "../../core/types/promptType.js";
+import { logger } from "../../utils/logger/logData.js";
+export class llmAction {
+    constructor(apiKey, baseURL, model, visionApiKey) {
+        this.model = model;
+        this.visionApiKey = visionApiKey;
+        this.client = new OpenAI({
+            apiKey: decodeApiKey(apiKey),
+            baseURL: baseURL,
+        });
+        this.visionClient = new OpenAI({
+            apiKey: decodeApiKey(this.visionApiKey),
+        });
+    }
+    async getLLMResponse(platform, type, args, userInput) {
+        const promptBuilder = prompts[platform][type];
+        const systemPrompt = await promptBuilder(args);
+        const userPrompt = userInputFormatters[type](userInput);
+        if (type === PromptType.VISION_PROMPT && Array.isArray(userPrompt)) {
+            return this.getLLMResponseWithVision(systemPrompt, userPrompt);
+        }
+        if (typeof userPrompt !== "string") {
+            throw new Error("Invalid non-vision user prompt format");
+        }
+        return this.getLLMResponseText(systemPrompt, userPrompt);
+    }
+    async getLLMResponseText(system, user) {
+        let attempt = 0;
+        const MAX_RETRIES = 5;
+        const startTime = new Date().getTime();
+        while (attempt < MAX_RETRIES) {
+            attempt++;
+            try {
+                const response = await this.client.chat.completions.create({
+                    model: this.model,
+                    messages: [
+                        { role: "system", content: system },
+                        { role: "user", content: user },
+                    ],
+                    response_format: { type: "json_object" },
+                });
+                const content = response.choices?.[0]?.message?.content;
+                logger.info(`LLM Attempt ${attempt} - Raw Response:`, content);
+                if (this.isInvalidLLMOutput(content)) {
+                    logger.error(`Invalid LLM output. Retrying (${attempt})`);
+                    continue;
+                }
+                logger.info("Time taken to get LLM response (Text):", Date.now() - startTime, "ms");
+                return response;
+            }
+            catch (error) {
+                logger.error(`LLM Error Attempt ${attempt}:`, error);
+                const structuredError = this.handleKnownErrors(error);
+                if (structuredError)
+                    return structuredError;
+            }
+        }
+        return this.buildRetryFailure();
+    }
+    async getLLMResponseWithVision(system, userPrompt) {
+        let attempt = 0;
+        const MAX_RETRIES = 5;
+        const startTime = new Date().getTime();
+        while (attempt < MAX_RETRIES) {
+            attempt++;
+            try {
+                const response = await this.visionClient.chat.completions.create({
+                    model: "gpt-4.1",
+                    messages: [
+                        { role: "system", content: system },
+                        { role: "user", content: userPrompt },
+                    ],
+                    response_format: { type: "json_object" },
+                });
+                const content = response.choices?.[0]?.message?.content;
+                logger.info(`Vision LLM Attempt ${attempt} - Raw Response:`, content);
+                if (this.isInvalidLLMOutput(content)) {
+                    logger.error(`Invalid Vision output. Retrying (${attempt})`);
+                    continue;
+                }
+                logger.info("Time taken to get LLM response (vision):", Date.now() - startTime, "ms");
+                return response;
+            }
+            catch (error) {
+                logger.error(`Vision LLM Error Attempt ${attempt}:`, error);
+                const structuredError = this.handleKnownErrors(error);
+                if (structuredError)
+                    return structuredError;
+            }
+        }
+        return this.buildRetryFailure();
+    }
+    isInvalidLLMOutput(content) {
+        if (!content)
+            return true;
+        if (typeof content === "string") {
+            const trimmed = content.trim();
+            if (!trimmed)
+                return true;
+            if (trimmed === "{}")
+                return true;
+            try {
+                const parsed = JSON.parse(trimmed);
+                if (typeof parsed === "object" &&
+                    parsed !== null &&
+                    Object.keys(parsed).length === 0) {
+                    return true;
+                }
+            }
+            catch {
+                return true;
+            }
+        }
+        return false;
+    }
+    handleKnownErrors(error) {
+        if (error?.status === 429) {
+            return {
+                error: "Rate Limit Exceeded (429)",
+                errorDescription: "FireFlink has exceeded the API quota or rate limit for the selected AI model. \n This is common with free or low-tier API plans \n Please wait a moment before retrying or check your API usage limits.",
+                rawError: "Rate limit reached for model. HTTP 429 returned by model"
+            };
+        }
+        if (error?.status === 400 &&
+            error?.error?.message?.toLowerCase().includes("context")) {
+            return {
+                error: "Context Length Exceeded (400)",
+                errorDescription: "FireFlink could not process the request because the input exceeds the maximum context length supported by the selected AI model.\nThis typically happens when the input is very long",
+                rawError: "This model's maximum context length has been exceeded. HTTP 400 returned by model"
+            };
+        }
+        return null;
+    }
+    buildRetryFailure() {
+        return {
+            error: "LLM Empty Response After Retry",
+            errorDescription: "FireFlink attempted multiple times but received empty or invalid responses.",
+            rawError: "Max retry limit reached",
+        };
+    }
+}

package/dist/ai/llmcalls/parseLlmOputput.d.ts

@@ -0,0 +1,15 @@
+import { FireFlinkLLMResponse } from "../../core/types/llmResponseType.js";
+export declare class LLMResultParser {
+    private inputTokens;
+    private outputTokens;
+    private totalTokens;
+    private extractJsonObject;
+    fetchResult(llmOutput: any): Promise<{
+        response: FireFlinkLLMResponse;
+    }>;
+    getTokenUsage(): {
+        inputTokens: number;
+        outputTokens: number;
+        totalTokens: number;
+    };
+}

package/{src/ai/llmcalls/parseLlmOputput.ts → dist/ai/llmcalls/parseLlmOputput.js}

@@ -1,68 +1,50 @@
-import { FireFlinkLLMResponse } from "../../core/types/llmResponseType.js";
 import { logger } from "../../utils/logger/logData.js";
-
-
 export class LLMResultParser {
-    private inputTokens: number = 0;
-    private outputTokens: number = 0;
-    private totalTokens: number = 0;
-
-    private async extractJsonObject(response: string) {
+    constructor() {
+        this.inputTokens = 0;
+        this.outputTokens = 0;
+        this.totalTokens = 0;
+    }
+    async extractJsonObject(response) {
         const match = response.match(/\{[\s\S]*\}/);
-        if (!match) return null;
+        if (!match)
+            return null;
         return JSON.parse(match[0]);
     }
-    public async fetchResult(llmOutput: any): Promise<{
-        response: FireFlinkLLMResponse
-    }> {
+    async fetchResult(llmOutput) {
         try {
             const content = llmOutput?.choices?.[0]?.message?.content;
-
             if (!content) {
                 throw new Error("No content in LLM output");
             }
-
-            let parsedContent: FireFlinkLLMResponse;
-
+            let parsedContent;
             try {
                 // Try direct parse first
                 parsedContent = JSON.parse(content);
-            } catch {
+            }
+            catch {
                 // If LLM added explanation text, extract JSON block
                 const extracted = this.extractJsonObject(content);
-
                 if (!extracted) {
                     throw new Error("Could not extract valid JSON from LLM response");
                 }
-
-                parsedContent = extracted as FireFlinkLLMResponse;
+                parsedContent = extracted;
             }
-
             const usage = llmOutput?.usage ?? {};
-
             const inputTokens = Number(usage.prompt_tokens ?? 0);
             const outputTokens = Number(usage.completion_tokens ?? 0);
             const totalTokens = Number(usage.total_tokens ?? 0);
-
             this.inputTokens += inputTokens;
             this.outputTokens += outputTokens;
             this.totalTokens += totalTokens;
-
             logger.info(`Total tokens for this call is : ${usage.total_tokens}`);
-
             return { response: parsedContent };
-
-        } catch (error) {
-            throw new Error(
-                "Failed to parse LLM output",
-                { cause: error }
-            );
         }
-
+        catch (error) {
+            throw new Error("Failed to parse LLM output", { cause: error });
+        }
     }
-
-
-    public getTokenUsage() {
+    getTokenUsage() {
         return {
             inputTokens: this.inputTokens,
             outputTokens: this.outputTokens,

package/dist/ai/llmprompts/llmPromptTypes/systemPromptTypes.d.ts

@@ -0,0 +1,37 @@
+export type keywordExtractor = {
+    priorAndNextSteps: string[];
+};
+export type FFInspectorArgs = {
+    stepAction: string;
+    extractedDomJson: string;
+    priorAndNextSteps: string[];
+    isAlert?: boolean;
+    isDrag?: boolean;
+};
+export type FFInspectorArgsMob = {
+    stepAction: string;
+    extractedDomJson: string;
+    priorAndNextSteps: string[];
+};
+export type getActionArgs = {
+    extractedDomJson: string;
+    priorAndNextSteps: string[];
+};
+export type verifyActionArgs = {
+    extractedDomJson: string;
+    priorAndNextSteps: string[];
+};
+export type visionPrompt = {
+    priorAndNextSteps: string[];
+};
+export type visionPromptMobile = {
+    priorAndNextSteps: string[];
+};
+export type waitActionArgs = {
+    extractedDomJson: string;
+    priorAndNextSteps: string[];
+};
+export type combinedActionArgs = {
+    extractedDomJson: string;
+    priorAndNextSteps: string[];
+};

package/dist/ai/llmprompts/llmPromptTypes/systemPromptTypes.js

@@ -0,0 +1 @@
+export {};

package/dist/ai/llmprompts/promptRegistry.d.ts

@@ -0,0 +1,2 @@
+import { PromptMap } from "../../core/types/promptMap.js";
+export declare const prompts: PromptMap;

package/dist/ai/llmprompts/promptRegistry.js

@@ -0,0 +1,40 @@
+import { keywordExtractor } from "./systemPrompts/actionExtractorPrompt.js";
+import { buildStepExtractionPrompt, androidbuildStepExtractionPrompt } from "./systemPrompts/userStoryToListPrompt.js";
+import { buildErrorDescriptionPrompt } from "./systemPrompts/errorDescriptionPrompt.js";
+import { ffInspectorNumExtractor } from "./systemPrompts/fireflinkElementIndexExtactors.js";
+import { ffInspectorNumExtractorMob } from "./systemPrompts/fireflinkElementIndexExtractor_Mob.js";
+import { visionPrompt } from "./systemPrompts/visionPrompt.js";
+import { visionPromptMobile } from "./systemPrompts/visionPromptMobile.js";
+import { verifyActionExtractorPrompt } from "./systemPrompts/verifyActionExtractorPrompt.js";
+import { getActionExtractorPrompt } from "./systemPrompts/getActionExtractorPrompt.js";
+import { verifyActionExtractorPromptMob } from "./systemPrompts/verifyActionExtractorPromptMob.js";
+import { getActionExtractorPromptMob } from "./systemPrompts/getActionExtractorPromptMob.js";
+import { keywordExtractorMobile } from "./systemPrompts/mobileKeywordExtractor.js";
+import { combinedActionExtractorPromptMob } from "./systemPrompts/combinedActionExtractorPromptMob.js";
+import { waitActionExtractorPromptMob } from "./systemPrompts/waitActionExtractorPromptMob.js";
+import { waitActionExtractorPrompt } from "./systemPrompts/waitActionExtractorPrompt.js";
+import { combinedActionExtractorPrompt } from "./systemPrompts/combinedActionExtractorPrompt.js";
+export const prompts = {
+    web: {
+        userStoryToList: buildStepExtractionPrompt,
+        keywordExtractorPrompt: keywordExtractor,
+        errorDescriptionPrompt: buildErrorDescriptionPrompt,
+        ffInspectorNumExtractor: ffInspectorNumExtractor,
+        verifyActionExtractorPrompt: verifyActionExtractorPrompt,
+        visionPrompt: visionPrompt,
+        getActionExtractorPrompt: getActionExtractorPrompt,
+        combinedActionExtractorPrompt: combinedActionExtractorPrompt,
+        waitActionExtractorPrompt: waitActionExtractorPrompt,
+    },
+    android: {
+        userStoryToList: androidbuildStepExtractionPrompt,
+        keywordExtractorPrompt: keywordExtractorMobile,
+        errorDescriptionPrompt: buildErrorDescriptionPrompt,
+        ffInspectorNumExtractor: ffInspectorNumExtractorMob,
+        verifyActionExtractorPrompt: verifyActionExtractorPromptMob,
+        visionPrompt: visionPromptMobile,
+        getActionExtractorPrompt: getActionExtractorPromptMob,
+        combinedActionExtractorPrompt: combinedActionExtractorPromptMob,
+        waitActionExtractorPrompt: waitActionExtractorPromptMob,
+    }
+};

package/dist/ai/llmprompts/systemPrompts/actionExtractorPrompt.d.ts

@@ -0,0 +1,2 @@
+import { keywordExtractor } from "../llmPromptTypes/systemPromptTypes";
+export declare function keywordExtractor({ priorAndNextSteps }: keywordExtractor): Promise<string>;

package/{src/ai/llmprompts/systemPrompts/actionExtractorPrompt.ts → dist/ai/llmprompts/systemPrompts/actionExtractorPrompt.js}

@@ -1,51 +1,41 @@
-
-type keywordExtractor = {
-  priorAndNextSteps: string[];
-
-};
-export async function keywordExtractor(
-  { priorAndNextSteps }: keywordExtractor
-): Promise<string> {
-
-  const allowedActions: string[] = ["enter", "wait", "verify", "scroll", "navigate", "click", "maximize", "get", "upload", "close", "open", "drag_and_drop", "switch", "cleartext"];
-
-  const prompt = `
-You are an expert in Web application testing.
-    From the step, extract ONLY the meaningful keywords so that i can search for the element in the dom.
-    Rules:
-    - Only give response for the current step.
-    - understand the step and context from the ${JSON.stringify(priorAndNextSteps)}.and keywords should be from step. it should not be related to other steps.
-    - 3 to 5 keywords maximum. dont give more than 5 keywords. and no need for casesensitive. 
-    - If the step is about entering text or Uploading file, Should NOT include input value from the step into keywords.
-    - If the step has words like tag name audio, video, image,svg, checkbox etc, include them in the keywords.
-    - If icon is metioned in step than 'svg' should add in keywords and for Upload action first keyword should be 'file'.
-    - First keywords should be from step next Keywords must be distinct and based on the element's label meaning only.
-    - If a keyword contains exactly two words, you MUST always include both the spaced and concatenated (no-space) versions; omission of either is invalid.
-    - Do NOT split single-word keywords and do NOT include relation terms (above, below, next to, etc.) in keywords.
-    - Treat each keyword independently—never merge different keywords or combine them with relation words.
-    - Example: Click on Sign In above Create Account button → ["Sign In", "SignIn", "Create Account", "CreateAccount"] never break kewword like ["create", "account"]
-    - Keywords can be string or number if the step contains a number ,add it also as keyword.
-    - Do Not include any other unrelated keywords for step. 
-    - Do NOT include generic UI words (button, field, etc) and action words (tap, click, press, etc).
-    - Do NOT include status/technical words (displayed, enabled, authenticate, visible).
-    - If an element label contains multiple words (e.g., "Sign In", "Add to Cart"), keep them together as ONE keyword and do not split them and also for keywords you generated, do not split them.
-    - element_name: extract name of the element that mentioned in the the step.(eg:tap on x -> element_name:x) keep element_name as short as possible and make the first letter of first word of the element_name as capital. beacuse element_name is also used to find element in the dom. and if element_name is not mentioned in step than return action of the step as element_name.
-    - action: click for taping, clicking or selecting, enter for entering input, wait for waiting or sleeping,scroll for scrolling and swiping, navigate for navigating to page using url or navigateing back to previous page, get for getting,fetching element,maximize for maximizing browser window, close for closing browser window,open for opening browser window, upload for uploading file using path, drag_and_drop for dragging and dropping element, switch is for switching to tab or window or frame,cleartext for clearing or removing text from element.
-    - action must be one of from this list  ${JSON.stringify(allowedActions)}.if not one of them, return '0'. if step about set or find action return '0'
-    - For navigate action, keywords should contain only one keyword which is full url from the step and should not include any other text. and if step has another actions, including navigate action, don't return navigate action return another action witch is in the step. and element_name should be "URL".
-    Navigate action is allowed only when the step intent is purely navigation (e.g., "Navigate to URL").
-    If the step includes verification intent (verify/check/confirm/etc), you MUST return "verify" and MUST NOT return "navigate".
-    only give navigate action if step has only navigate action.
-         
-    Respond only with JSON using this format:
-    {
-    "keywords": [key1,key2,key3,key4,key5],
-    "elementName": "x",
-    "action": "x"
-    }
-
-    No other text.
+export async function keywordExtractor({ priorAndNextSteps }) {
+    const allowedActions = ["enter", "wait", "verify", "scroll", "navigate", "click", "maximize", "minimize", "get", "upload", "close", "open", "drag_and_drop", "switch", "cleartext"];
+    const prompt = `
+You are an expert in Web application testing.
+    From the step, extract ONLY the meaningful keywords so that i can search for the element in the dom.
+    Rules:
+    - Only give response for the current step.
+    - understand the step and context from the ${JSON.stringify(priorAndNextSteps)}.and keywords should be from step. it should not be related to other steps.
+    - 3 to 5 keywords maximum. dont give more than 5 keywords. and no need for casesensitive.And first keyword should be full keyword from step same as mentioned.like (click on search bar.in -> "search bar.in")
+    - If the step is about entering text or Uploading file, Should NOT include input value from the step into keywords.
+    - If the step has words like tag name audio, video, image,svg, checkbox etc, include them in the keywords.
+    - If icon is metioned in step than 'svg' should add in keywords and for Upload action first keyword should be 'file'.
+    - First keywords should be from step next Keywords must be distinct and based on the element's label meaning only.
+    - If a keyword contains exactly two words, you MUST always include both the spaced and concatenated (no-space) versions; omission of either is invalid.
+    - Do NOT split single-word keywords and do NOT include relation terms (above, below, next to, etc.) in keywords.
+    - Treat each keyword independently—never merge different keywords or combine them with relation words.
+    - Example: Click on Sign In above Create Account button → ["Sign In", "SignIn", "Create Account", "CreateAccount"] never break kewword like ["create", "account"]
+    - Keywords can be string or number if the step contains a number ,add it also as keyword.
+    - Do Not include any other unrelated keywords for step. 
+    - Do NOT include generic UI words (button, field, etc) and action words (tap, click, press, etc).
+    - Do NOT include status/technical words (displayed, enabled, authenticate, visible).
+    - If an element label contains multiple words (e.g., "Sign In", "Add to Cart"), keep them together as ONE keyword like ["Sign In", "SignIn", "Add to Cart", "AddToCart"] and Never split them and also for keywords you generated, should not split them like ["sign", "in","add","to","cart"].
+    - element_name: extract name of the element that mentioned in the the step.(eg:tap on x -> element_name:x) keep element_name as short as possible and make the first letter of first word of the element_name as capital. beacuse element_name is also used to find element in the dom. and if element_name is not mentioned in step than return action of the step as element_name.
+    - action: click for taping, clicking or selecting, enter for entering input, wait for waiting or sleeping,scroll for scrolling and swiping, navigate for navigating to page using url or navigateing back to previous page, get for getting,fetching element,maximize for maximizing browser window, close for closing browser window,open for opening browser window, upload for uploading file using path, drag_and_drop for dragging and dropping element, switch is for switching to tab or window or frame,cleartext for clearing or removing text from element.
+    - action must be one of from this list  ${JSON.stringify(allowedActions)}.if not one of them, return '0'. if step about set or find action return '0'
+    - For navigate action, keywords should contain only one keyword which is full url from the step and should not include any other text. and if step has another actions, including navigate action, don't return navigate action return another action witch is in the step. and element_name should be "URL".
+    Navigate action is allowed only when the step intent is purely navigation (e.g., "Navigate to URL").
+    If the step includes verification intent (verify/check/confirm/etc), you MUST return "verify" and MUST NOT return "navigate".
+    only give navigate action if step has only navigate action.
+         
+    Respond only with JSON using this format:
+    {
+    "keywords": [key1,key2,key3,key4,key5],
+    "elementName": "x",
+    "action": "x"
+    }
+
+    No other text.
 `;
-
-  return prompt;
+    return prompt;
 }

package/dist/ai/llmprompts/systemPrompts/combinedActionExtractorPrompt.d.ts

@@ -0,0 +1,2 @@
+import { combinedActionArgs } from "../llmPromptTypes/systemPromptTypes";
+export declare function combinedActionExtractorPrompt({ extractedDomJson, priorAndNextSteps, }: combinedActionArgs): Promise<string>;

package/dist/ai/llmprompts/systemPrompts/combinedActionExtractorPrompt.js

@@ -0,0 +1,32 @@
+export async function combinedActionExtractorPrompt({ extractedDomJson, priorAndNextSteps, }) {
+    const nlpList = `"dummy"`;
+    const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"];
+    const prompt = `
+    You are an intelligent assistant that extracts structured UI action data.
+Given the step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
+
+Select the perfect matching NLP name from the list that best fits the step's intent.
+
+Return **only valid JSON** in the following format:
+{
+  "attribute_value": "Fire-Flink-x",
+  "nlpName": "x",
+  "input_text":"x",
+  "keyword": "x",
+  "element_type": "x",
+  "keycode": "x"
+}
+Rules:
+- nlpName must be exactly one from the provided list.
+- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
+- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
+- If step has mutipul inputs add all in input_text and all matched attribute_value also.
+- Extract input_text from the step, if you can't find any input text in the step, return keyword as input_text.input can be number, app package, apk file path.dont include extra words.
+- Extract keyword from the step, if the step contains any keyword.
+- Extract keycode of keys if the step contains any key name like ENTER,SPACE,BACKSPACE etc and give keycode of the given key.
+- Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
+- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
+- Respond with valid JSON only.
+`;
+    return prompt;
+}

package/dist/ai/llmprompts/systemPrompts/combinedActionExtractorPromptMob.d.ts

@@ -0,0 +1,2 @@
+import { combinedActionArgs } from "../llmPromptTypes/systemPromptTypes";
+export declare function combinedActionExtractorPromptMob({ extractedDomJson, priorAndNextSteps, }: combinedActionArgs): Promise<string>;

package/dist/ai/llmprompts/systemPrompts/combinedActionExtractorPromptMob.js

@@ -0,0 +1,44 @@
+export async function combinedActionExtractorPromptMob({ extractedDomJson, priorAndNextSteps, }) {
+    const nlpList = `"MOB_SetCapability","MOB_EnterUrl","MOB_OpenMobileApplication",
+"MOB_IsAppInstalled","MOB_SetEmulatorPowerPercentage","MOB_ReceiveMessageOnEmulator","MOB_UninstallAPK",
+"MOB_InstallAPK","MOB_SetClipBoardText","MOB_PressAnyKeyNTimes",
+"MOB_SetContext","MOB_TerminateAppUsingAppPackage","MOB_OpenAppWithApkFilePath","MOB_ActivateAppUsingAppPackage",
+"MOB_ExecuteJavaScript","MOB_LongPress","MOB_Clear","MOB_EnterInputIntoElementFromClipBoard",
+"MOB_FindElement","MOB_FindElements","MOB_OpenNotification","MOB_PressEnterKey","MOB_ResetApp","MOB_HideKeyboard",
+"MOB_SetEmulatorPowerStateOff","MOB_OpenChromeBrowser","MOB_SetDeviceOrientationAsPortrait","MOB_SetDeviceOrientationAsLandscape",
+"MOB_SetEmulatorPowerStateOn","MOB_AirplaneModeSwitchOn","MOB_LockDevice","MOB_WifiSwitchOff",
+"Continue","MOB_GpsSwitchOn","MOB_GpsSwitchOff","MOB_RestartCurrentDevice","MOB_AirplaneModeSwitchOff","Break","MOB_WifiSwitchOn",
+"MOB_PinchInByPercent","MOB_PinchOutByPercent","MOB_Click",
+"MOB_CreateCapabilitiesInstance","MOB_PressAnyKey","MOB_PressBackSpaceKey","MOB_PressSpaceKey","MOB_RunAppInBackground",
+"MOB_SetAppiumDriverSettings","MOB_SetDriverInstance","MOB_SetImplicitTimeOut",
+"MOB_TakeScreenShotAndCompareWithImage","MOB_TakeScreenshotAndCompareSectionWithImage","MOB_TakeScreenshotAndCompareWithImageIgnoringSpecifiedSection"`;
+    const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"];
+    const prompt = `
+    You are an intelligent assistant that extracts structured UI action data.
+Given the step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
+
+Select the perfect matching NLP name from the list that best fits the step's intent.
+
+Return **only valid JSON** in the following format:
+{
+  "attribute_value": "Fire-Flink-x",
+  "nlpName": "x",
+  "input_text":"x",
+  "keyword": "x",
+  "element_type": "x",
+  "keycode": "x"
+}
+Rules:
+- nlpName must be exactly one from the provided list.
+- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
+- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
+- If step has mutipul inputs add all in input_text and all matched attribute_value also.
+- Extract input_text from the step, if you can't find any input text in the step, return keyword as input_text.input can be number, app package, apk file path.dont include extra words.
+- Extract keyword from the step, if the step contains any keyword.
+- Extract keycode of keys if the step contains any key name like ENTER,SPACE,BACKSPACE etc and give keycode of the given key.
+- Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
+- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
+- Respond with valid JSON only.
+`;
+    return prompt;
+}

package/dist/ai/llmprompts/systemPrompts/errorDescriptionPrompt.d.ts

@@ -0,0 +1 @@
+export declare function buildErrorDescriptionPrompt(): Promise<string>;