ff-automationv2 2.1.3 → 2.2.0

package/.env

@@ -0,0 +1 @@
+KAFKAJS_NO_PARTITIONER_WARNING=1

package/.husky/pre-commit

@@ -0,0 +1,5 @@
+echo "Linting code..."
+npm run lint
+
+echo "Running TypeScript check..."
+npm run type-check

package/ARCHITECTURE.md

@@ -71,6 +71,8 @@ AutomationRunner
                 ↓
             FOR EACH STEP
                 ↓
+            Is termination check
+                ↓
             LLM → Action + Keywords
                 ↓
             Inject Annotated DOM
@@ -93,16 +95,17 @@ AutomationRunner
         │             │
        NO            YES
         │             │
-        │      Append CloseBrowser
+        │      Append CloseBrowser 
+        | (add manual step, nlp name if available)
         │             ↓
-        │      Stop Execution
+        │      Add error info via LLM 
         │             |
         ↓             |
 Continue Next Step    |
         ↓             |
 (all Steps Complete)  |
         |             |
-        |             |
+        |       Stop Execution
         └──────┬──────┘
         Collect Token Usage
                ↓

package/Note.md

@@ -0,0 +1,7 @@
+API calls are often faster in TypeScript (Node.js) because:
+
+Non-blocking networking
+Single event loop for async tasks
+Efficient socket and connection reuse
+Optimized JSON parsing in V8
+Lower runtime overhead per request

package/RELEASE_GUIDE.md

@@ -3,7 +3,7 @@
 git checkout feature/your-feature-name
 git add .
 git commit -m "feat: new changes"
-git pull --rebase
+git pull 
 npm version prerelease --preid=beta
 git push --follow-tags
 

package/bitbucket-pipelines.yml

@@ -2,35 +2,14 @@ image: node:20
 
 pipelines:
   branches:
-    "feature/automation_V2":
-      - step:
-          name: Publish Beta
-          caches:
-            - node
-          script:
-            - echo "Installing dependencies"
-            - npm ci
-
-            - echo "Building project"
-            - npm run build
-
-            - echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc
-
-            - echo "Publishing as beta"
-            - npm publish --tag beta --access public
-    "release/automation_V2":
+    release/automation_V2:
       - step:
           name: Publish Production
           caches:
             - node
           script:
-            - echo "Installing dependencies"
             - npm ci
-
-            - echo "Building project"
+            - npm run lint
             - npm run build
-
-            - echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc
-
-            - echo "Publishing as latest"
-            - npm publish --access public
+            - echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc
+            - npm publish --access public

package/eslint.config.ts

@@ -4,6 +4,9 @@ import tseslint from "typescript-eslint";
 import { defineConfig } from "eslint/config";
 
 export default defineConfig([
+  {
+    ignores: ["src/tests/**", "dist/**", "node_modules/**"]
+  },
   {
     files: ["**/*.{js,mjs,cjs,ts,mts,cts}"],
     plugins: { js },
@@ -15,6 +18,7 @@ export default defineConfig([
 
   {
     rules: {
+      "no-useless-escape": "off",
       "no-console": ["error", { allow: ["warn", "error"] }],
       "@typescript-eslint/no-explicit-any": "off",
       "@typescript-eslint/no-unused-vars": [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ff-automationv2",
-  "version": "2.1.3",
+  "version": "2.2.0",
   "private": false,
   "type": "module",
   "description": "This lib is used to automate the manual testcase",
@@ -9,45 +9,57 @@
   "exports": {
     ".": {
       "import": "./dist/index.js",
+      "require": "./dist/index.js",
       "types": "./dist/index.d.ts"
     }
   },
   "scripts": {
+    "type-check": "tsc --noEmit",
     "builds": "npm run lint && tsc",
-    "build": "tsc",
-    "prepare": "npm run build",
+    "build": "eslint . && tsc && echo Build Successful",
+    "prepare": "husky",
     "lint": "eslint src --max-warnings=0",
     "test": "npm run build && node dist/tests/test12.js",
     "llm": "npm run build && node dist/tests/testllm.js",
     "dev": "npm run build && node dist/index.js",
     "start": "node dist/index.js",
-    "file": "node dist/domAnalysis/getXpath.js"
+    "file": "node dist/domAnalysis/getXpath.js",
+    "error": "node dist/tests/test12.js",
+    "bro": "node dist/tests/test3.js"
   },
   "keywords": [],
   "author": "Diwahar R",
   "license": "ISC",
   "devDependencies": {
     "@eslint/js": "10.0.1",
+    "@types/fs-extra": "^11.0.4",
     "@types/js-beautify": "^1.14.3",
     "@types/jsdom": "27.0.0",
-    "@types/node": "25.2.2",
+    "@types/node": "^25.4.0",
     "eslint": "10.0.0",
     "globals": "17.3.0",
+    "husky": "^9.1.7",
     "jiti": "2.6.1",
     "nodemon": "3.1.11",
     "ts-node": "10.9.2",
-    "typescript": "5.9.3",
+    "typescript": "^5.9.3",
     "typescript-eslint": "8.56.0"
   },
   "dependencies": {
-    "axios": "1.13.5",
+    "@xmldom/xmldom": "^0.8.11",
+    "axios": "^1.13.5",
+    "canvas": "3.2.1",
     "cheerio": "1.2.0",
+    "devtools": "^8.42.0",
+    "dotenv": "^17.3.1",
+    "fast-fuzzy": "^1.12.0",
     "fs-extra": "11.3.3",
-    "fuzzball": "2.2.3",
     "js-beautify": "1.15.1",
     "jsdom": "27.4.0",
+    "kafkajs": "^2.2.4",
     "openai": "6.18.0",
     "uuid": "13.0.0",
-    "webdriverio": "9.23.3"
+    "webdriverio": "8.46.0",
+    "xpath": "^0.0.34"
   }
 }

package/src/ai/llmcalls/decodeApiKey.ts

@@ -7,7 +7,7 @@ export function decodeApiKey(apiKey: string): string {
   try {
     return Buffer.from(apiKey, "base64").toString("utf-8");
   } catch (error) {
-   throw new Error("Failed to decode API key", { cause: error });
+    throw new Error("Failed to decode API key", { cause: error });
   }
 }
 

package/src/ai/llmcalls/llmAction.ts

@@ -1,89 +1,196 @@
 import { prompts } from "../llmprompts/promptRegistry.js";
 import OpenAI from "openai";
 import { decodeApiKey } from "./decodeApiKey.js";
-import { FireFlinkLLMResponse } from "../../core/types/llmResponseType.js"
-import { PromptArgsMap } from "../../core/interfaces/promptInterface.js"
-import { userInputFormatters } from "../../ai/llmprompts/userPrompts/userPrompt.js"
-import { visionPromptMessage } from "../../core/types/visionllmInputType.js"
+import { FireFlinkLLMResponse } from "../../core/types/llmResponseType.js";
+import { PromptArgsMap } from "../../core/interfaces/promptInterface.js";
+import { userInputFormatters } from "../../ai/llmprompts/userPrompts/userPrompt.js";
+import { visionPromptMessage } from "../../core/types/visionllmInputType.js";
 import { PromptType } from "../../core/types/promptType.js";
+import { platform } from "../../core/types/scriptTypesType.js"
+import { logger } from "../../utils/logger/logData.js";
+
 export class llmAction {
     private client: OpenAI;
+    private visionClient: OpenAI;
     private model: string;
-    private visionClient: OpenAI
+    private visionApiKey: string
+
     constructor(
         apiKey: string,
         baseURL: string,
         model: string,
-        visionApiKey: string
+        visionApiKey: string,
     ) {
         this.model = model;
+        this.visionApiKey = visionApiKey;
         this.client = new OpenAI({
             apiKey: decodeApiKey(apiKey),
             baseURL: baseURL,
         });
         this.visionClient = new OpenAI({
-            apiKey: decodeApiKey(visionApiKey),
+            apiKey: decodeApiKey(this.visionApiKey),
         });
 
     }
 
-    private async getLLMResponseText(system: string, user: string): Promise<FireFlinkLLMResponse> {
-        const response = await this.client.chat.completions.create({
-            model: this.model,
-            messages: [
-                {
-                    role: "system",
-                    content: system
-                },
-                {
-                    role: "user",
-                    content: user
-                }
-            ],
-            response_format: { type: "json_object" }
-        });
-        try {
 
-            return response;
-        } catch {
-            throw new Error(`Invalid JSON returned from LLM `);
-        }
-    }
     async getLLMResponse<K extends keyof PromptArgsMap>(
+        platform: platform,
         type: K,
         args: PromptArgsMap[K],
         userInput: Record<string, any>
     ): Promise<FireFlinkLLMResponse> {
-        const promptBuilder: (args: PromptArgsMap[K]) => Promise<string> =
-            prompts[type];
+        const promptBuilder = prompts[platform][type];
         const systemPrompt = await promptBuilder(args);
-        const userPrompt: string | visionPromptMessage = userInputFormatters[type](userInput);
+        const userPrompt = userInputFormatters[type](userInput);
+
         if (type === PromptType.VISION_PROMPT && Array.isArray(userPrompt)) {
             return this.getLLMResponseWithVision(systemPrompt, userPrompt);
         }
+
         if (typeof userPrompt !== "string") {
             throw new Error("Invalid non-vision user prompt format");
         }
+
         return this.getLLMResponseText(systemPrompt, userPrompt);
     }
 
-    private async getLLMResponseWithVision(system: any, userPrompt: visionPromptMessage): Promise<string> {
-        const response = await this.visionClient.chat.completions.create({
-            model: "gpt-4.1",
-            messages: [
-                {
-                    role: "system",
-                    content: system
-                },
-                {
-                    role: "user",
-                    content: userPrompt
+    private async getLLMResponseText(
+        system: string,
+        user: string
+    ): Promise<FireFlinkLLMResponse> {
+        let attempt = 0;
+        const MAX_RETRIES = 5
+        const startTime = new Date().getTime()
+        while (attempt < MAX_RETRIES) {
+            attempt++;
+            try {
+                const response = await this.client.chat.completions.create({
+                    model: this.model,
+                    messages: [
+                        { role: "system", content: system },
+                        { role: "user", content: user },
+                    ],
+                    response_format: { type: "json_object" },
+                });
+
+                const content = response.choices?.[0]?.message?.content;
+
+                logger.info(`LLM Attempt ${attempt} - Raw Response:`, content);
+
+                if (this.isInvalidLLMOutput(content)) {
+                    logger.error(`Invalid LLM output. Retrying (${attempt})`);
+                    continue;
                 }
-            ],
-            response_format: { type: "json_object" }
-        });
-        return response.choices[0].message.content || "";
+                logger.info("Time taken to get LLM response (Text):", Date.now() - startTime, "ms");
+                return response as FireFlinkLLMResponse;
+            } catch (error: any) {
+                logger.error(`LLM Error Attempt ${attempt}:`, error);
+
+                const structuredError = this.handleKnownErrors(error);
+                if (structuredError) return structuredError;
+
+            }
+        }
+
+        return this.buildRetryFailure();
+    }
+    private async getLLMResponseWithVision(
+        system: string,
+        userPrompt: visionPromptMessage
+    ): Promise<FireFlinkLLMResponse> {
+        let attempt = 0;
+        const MAX_RETRIES = 5
+        const startTime = new Date().getTime()
+        while (attempt < MAX_RETRIES) {
+            attempt++;
+
+            try {
+                const response = await this.visionClient.chat.completions.create({
+                    model: "gpt-4.1",
+                    messages: [
+                        { role: "system", content: system },
+                        { role: "user", content: userPrompt },
+                    ],
+                    response_format: { type: "json_object" },
+                });
+
+                const content = response.choices?.[0]?.message?.content;
+
+                logger.info(`Vision LLM Attempt ${attempt} - Raw Response:`, content);
+
+                if (this.isInvalidLLMOutput(content)) {
+                    logger.error(`Invalid Vision output. Retrying (${attempt})`);
+                    continue;
+                }
+                logger.info("Time taken to get LLM response (vision):", Date.now() - startTime, "ms");
+                return response as FireFlinkLLMResponse;
+            } catch (error: any) {
+                logger.error(`Vision LLM Error Attempt ${attempt}:`, error);
+
+                const structuredError = this.handleKnownErrors(error);
+                if (structuredError) return structuredError;
+            }
+        }
+
+        return this.buildRetryFailure();
     }
 
+    private isInvalidLLMOutput(content: any): boolean {
+        if (!content) return true;
+
+        if (typeof content === "string") {
+            const trimmed = content.trim();
+            if (!trimmed) return true;
+            if (trimmed === "{}") return true;
+
+            try {
+                const parsed = JSON.parse(trimmed);
+                if (
+                    typeof parsed === "object" &&
+                    parsed !== null &&
+                    Object.keys(parsed).length === 0
+                ) {
+                    return true;
+                }
+            } catch {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private handleKnownErrors(error: any): FireFlinkLLMResponse | null {
+        if (error?.status === 429) {
+            return {
+                error: "Rate Limit Exceeded (429)",
+                errorDescription: "FireFlink has exceeded the API quota or rate limit for the selected AI model. \n This is common with free or low-tier API plans \n Please wait a moment before retrying or check your API usage limits.",
+                rawError: "Rate limit reached for model. HTTP 429 returned by model"
+            };
+        }
+
+        if (
+            error?.status === 400 &&
+            error?.error?.message?.toLowerCase().includes("context")
+        ) {
+            return {
+                error: "Context Length Exceeded (400)",
+                errorDescription: "FireFlink could not process the request because the input exceeds the maximum context length supported by the selected AI model.\nThis typically happens when the input is very long",
+                rawError: "This model's maximum context length has been exceeded. HTTP 400 returned by model"
+            };
+        }
+
+        return null;
+    }
+
+    private buildRetryFailure(): FireFlinkLLMResponse {
+        return {
+            error: "LLM Empty Response After Retry",
+            errorDescription:
+                "FireFlink attempted multiple times but received empty or invalid responses.",
+            rawError: "Max retry limit reached",
+        };
+    }
 
-}
+}

package/src/ai/llmprompts/llmPromptTypes/systemPromptTypes.ts

@@ -0,0 +1,45 @@
+export type keywordExtractor = {
+  priorAndNextSteps: string[];
+
+};
+
+export type FFInspectorArgs = {
+  stepAction: string;
+  extractedDomJson: string;
+  priorAndNextSteps: string[];
+  isAlert?: boolean;
+  isDrag?: boolean;
+};
+export type FFInspectorArgsMob = {
+  stepAction: string;
+  extractedDomJson: string;
+  priorAndNextSteps: string[];
+};
+
+export type getActionArgs = {
+  extractedDomJson: string;
+  priorAndNextSteps: string[];
+}
+
+export type verifyActionArgs = {
+  extractedDomJson: string;
+  priorAndNextSteps: string[];
+}
+
+export type visionPrompt = {
+  priorAndNextSteps: string[]
+}
+
+export type visionPromptMobile = {
+  priorAndNextSteps: string[]
+}
+
+export type waitActionArgs = {
+  extractedDomJson: string;
+  priorAndNextSteps: string[]
+}
+
+export type combinedActionArgs = {
+  extractedDomJson: string;
+  priorAndNextSteps: string[]
+}

package/src/ai/llmprompts/promptRegistry.ts

@@ -1,16 +1,42 @@
 import { keywordExtractor } from "./systemPrompts/actionExtractorPrompt.js";
-import { buildStepExtractionPrompt } from "./systemPrompts/userStoryToListPrompt.js";
+import { buildStepExtractionPrompt, androidbuildStepExtractionPrompt } from "./systemPrompts/userStoryToListPrompt.js";
 import { buildErrorDescriptionPrompt } from "./systemPrompts/errorDescriptionPrompt.js";
 import { ffInspectorNumExtractor } from "./systemPrompts/fireflinkElementIndexExtactors.js";
+import { ffInspectorNumExtractorMob } from "./systemPrompts/fireflinkElementIndexExtractor_Mob.js";
 import { visionPrompt } from "./systemPrompts/visionPrompt.js";
+import { visionPromptMobile } from "./systemPrompts/visionPromptMobile.js";
 import { PromptMap } from "../../core/types/promptMap.js";
-
-
+import { verifyActionExtractorPrompt } from "./systemPrompts/verifyActionExtractorPrompt.js";
+import { getActionExtractorPrompt } from "./systemPrompts/getActionExtractorPrompt.js";
+import { verifyActionExtractorPromptMob } from "./systemPrompts/verifyActionExtractorPromptMob.js";
+import { getActionExtractorPromptMob } from "./systemPrompts/getActionExtractorPromptMob.js";
+import { keywordExtractorMobile } from "./systemPrompts/mobileKeywordExtractor.js";
+import { combinedActionExtractorPromptMob } from "./systemPrompts/combinedActionExtractorPromptMob.js";
+import { waitActionExtractorPromptMob } from "./systemPrompts/waitActionExtractorPromptMob.js";
+import{waitActionExtractorPrompt} from "./systemPrompts/waitActionExtractorPrompt.js";
+import { combinedActionExtractorPrompt } from "./systemPrompts/combinedActionExtractorPrompt.js";
 export const prompts: PromptMap = {
-    userStoryToList: buildStepExtractionPrompt,
-    keywordExtractorPrompt: keywordExtractor,
-    errorDescriptionPrompt: buildErrorDescriptionPrompt,
-    ffInspectorNumExtractor: ffInspectorNumExtractor,
-    visionPrompt: visionPrompt,
+    web: {
+        userStoryToList: buildStepExtractionPrompt,
+        keywordExtractorPrompt: keywordExtractor,
+        errorDescriptionPrompt: buildErrorDescriptionPrompt,
+        ffInspectorNumExtractor: ffInspectorNumExtractor,
+        verifyActionExtractorPrompt: verifyActionExtractorPrompt,
+        visionPrompt: visionPrompt,
+        getActionExtractorPrompt: getActionExtractorPrompt,
+        combinedActionExtractorPrompt: combinedActionExtractorPrompt,
+        waitActionExtractorPrompt: waitActionExtractorPrompt,
+    },
+    android: {
+        userStoryToList: androidbuildStepExtractionPrompt,
+        keywordExtractorPrompt: keywordExtractorMobile,
+        errorDescriptionPrompt: buildErrorDescriptionPrompt,
+        ffInspectorNumExtractor: ffInspectorNumExtractorMob,
+        verifyActionExtractorPrompt: verifyActionExtractorPromptMob,
+        visionPrompt: visionPromptMobile,
+        getActionExtractorPrompt: getActionExtractorPromptMob,
+        combinedActionExtractorPrompt: combinedActionExtractorPromptMob,
+        waitActionExtractorPrompt: waitActionExtractorPromptMob,
+    }
 };
 

package/src/ai/llmprompts/systemPrompts/actionExtractorPrompt.ts

@@ -1,13 +1,10 @@
 
-type keywordExtractor = {
-  priorAndNextSteps: string[];
-
-};
+import { keywordExtractor } from "../llmPromptTypes/systemPromptTypes"
 export async function keywordExtractor(
   { priorAndNextSteps }: keywordExtractor
 ): Promise<string> {
 
-  const allowedActions: string[] = ["enter", "wait", "verify", "scroll", "navigate", "click", "maximize", "get", "upload", "close", "open", "drag_and_drop", "switch", "cleartext"];
+  const allowedActions: string[] = ["enter", "wait", "verify", "scroll", "navigate", "click", "maximize", "minimize", "get", "upload", "close", "open", "drag_and_drop", "switch", "cleartext"];
 
   const prompt = `
 You are an expert in Web application testing.
@@ -15,7 +12,7 @@ You are an expert in Web application testing.
     Rules:
     - Only give response for the current step.
     - understand the step and context from the ${JSON.stringify(priorAndNextSteps)}.and keywords should be from step. it should not be related to other steps.
-    - 3 to 5 keywords maximum. dont give more than 5 keywords. and no need for casesensitive. 
+    - 3 to 5 keywords maximum. dont give more than 5 keywords. and no need for casesensitive.And first keyword should be full keyword from step same as mentioned.like (click on search bar.in -> "search bar.in")
     - If the step is about entering text or Uploading file, Should NOT include input value from the step into keywords.
     - If the step has words like tag name audio, video, image,svg, checkbox etc, include them in the keywords.
     - If icon is metioned in step than 'svg' should add in keywords and for Upload action first keyword should be 'file'.
@@ -28,7 +25,7 @@ You are an expert in Web application testing.
     - Do Not include any other unrelated keywords for step. 
     - Do NOT include generic UI words (button, field, etc) and action words (tap, click, press, etc).
     - Do NOT include status/technical words (displayed, enabled, authenticate, visible).
-    - If an element label contains multiple words (e.g., "Sign In", "Add to Cart"), keep them together as ONE keyword and do not split them and also for keywords you generated, do not split them.
+    - If an element label contains multiple words (e.g., "Sign In", "Add to Cart"), keep them together as ONE keyword like ["Sign In", "SignIn", "Add to Cart", "AddToCart"] and Never split them and also for keywords you generated, should not split them like ["sign", "in","add","to","cart"].
     - element_name: extract name of the element that mentioned in the the step.(eg:tap on x -> element_name:x) keep element_name as short as possible and make the first letter of first word of the element_name as capital. beacuse element_name is also used to find element in the dom. and if element_name is not mentioned in step than return action of the step as element_name.
     - action: click for taping, clicking or selecting, enter for entering input, wait for waiting or sleeping,scroll for scrolling and swiping, navigate for navigating to page using url or navigateing back to previous page, get for getting,fetching element,maximize for maximizing browser window, close for closing browser window,open for opening browser window, upload for uploading file using path, drag_and_drop for dragging and dropping element, switch is for switching to tab or window or frame,cleartext for clearing or removing text from element.
     - action must be one of from this list  ${JSON.stringify(allowedActions)}.if not one of them, return '0'. if step about set or find action return '0'

package/src/ai/llmprompts/systemPrompts/combinedActionExtractorPrompt.ts

@@ -0,0 +1,41 @@
+import { combinedActionArgs } from "../llmPromptTypes/systemPromptTypes"
+
+export async function combinedActionExtractorPrompt({
+    extractedDomJson,
+    priorAndNextSteps,
+
+}: combinedActionArgs
+): Promise<string> {
+    const nlpList = `"dummy"`;
+
+
+    const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"]
+    const prompt = `
+    You are an intelligent assistant that extracts structured UI action data.
+Given the step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
+
+Select the perfect matching NLP name from the list that best fits the step's intent.
+
+Return **only valid JSON** in the following format:
+{
+  "attribute_value": "Fire-Flink-x",
+  "nlpName": "x",
+  "input_text":"x",
+  "keyword": "x",
+  "element_type": "x",
+  "keycode": "x"
+}
+Rules:
+- nlpName must be exactly one from the provided list.
+- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
+- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
+- If step has mutipul inputs add all in input_text and all matched attribute_value also.
+- Extract input_text from the step, if you can't find any input text in the step, return keyword as input_text.input can be number, app package, apk file path.dont include extra words.
+- Extract keyword from the step, if the step contains any keyword.
+- Extract keycode of keys if the step contains any key name like ENTER,SPACE,BACKSPACE etc and give keycode of the given key.
+- Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
+- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
+- Respond with valid JSON only.
+`;
+    return prompt;
+}