ff-automationv2 2.1.1-beta.3 → 2.1.1

package/bitbucket-pipelines.yml

@@ -2,8 +2,7 @@ image: node:20
 
 pipelines:
   branches:
-
-    "feature/*":
+    "feature/automation_V2":
       - step:
           name: Publish Beta
           caches:
@@ -19,8 +18,7 @@ pipelines:
 
             - echo "Publishing as beta"
             - npm publish --tag beta --access public
-
-    release:
+    "release/automation_V2":
       - step:
           name: Publish Production
           caches:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ff-automationv2",
-  "version": "2.1.1-beta.3",
+  "version": "2.1.1",
   "private": false,
   "type": "module",
   "description": "This lib is used to automate the manual testcase",
@@ -28,6 +28,7 @@
   "license": "ISC",
   "devDependencies": {
     "@eslint/js": "10.0.1",
+    "@types/js-beautify": "^1.14.3",
     "@types/jsdom": "27.0.0",
     "@types/node": "25.2.2",
     "eslint": "10.0.0",
@@ -43,6 +44,7 @@
     "cheerio": "1.2.0",
     "fs-extra": "11.3.3",
     "fuzzball": "2.2.3",
+    "js-beautify": "1.15.1",
     "jsdom": "27.4.0",
     "openai": "6.18.0",
     "uuid": "13.0.0",

package/src/ai/llmprompts/systemPrompts/actionExtractorPrompt.ts

@@ -7,63 +7,44 @@ export async function keywordExtractor(
   { priorAndNextSteps }: keywordExtractor
 ): Promise<string> {
 
-  const allowedActions: string[] = [
-    "enter",
-    "wait",
-    "verify",
-    "scroll",
-    "navigate",
-    "click",
-    "maximize",
-    "get",
-    "upload",
-    "close",
-    "open",
-    "drag_and_drop",
-    "switch"
-  ];
+  const allowedActions: string[] = ["enter", "wait", "verify", "scroll", "navigate", "click", "maximize", "get", "upload", "close", "open", "drag_and_drop", "switch", "cleartext"];
 
   const prompt = `
 You are an expert in Web application testing.
-From the step: step, extract ONLY the meaningful keywords so that i can search for the element in the dom.
-
-Rules:
-- Only give response for the current step .
-- understand the step and context from the ${JSON.stringify(priorAndNextSteps)} and keywords should be from step.
-- 3 to 5 keywords maximum.
-- If the step is about entering text or Uploading file, Should NOT include input value from the step into keywords.
-- If the step has words like tag name audio, video, image, svg, checkbox etc, include them in the keywords.
-- If icon is mentioned in step then 'svg' should add in keywords and for Upload action first keyword should be 'file'.
-- First keyword should be from step. Next keywords must be distinct and based on element label meaning only.
-- If keyword is two words, then create an additional keyword by combining them.
-  Example: "Sign In" → "Sign In", "SignIn"
-- Keywords can be string or number.
-- Do NOT include generic UI words or action words.
-- Do NOT include status or technical words.
-- If element label contains multiple words, keep them together as ONE keyword.
-- element_name: extract name of the element from step.
-  Capitalize first letter.
-  If not mentioned, return action as element_name.
-- action rules:
-  click, enter, wait, scroll, navigate, get, maximize, close, open,
-  upload, drag_and_drop, switch
-- action must be one of ${JSON.stringify(allowedActions)}.
-  If not, return "0".
-- For navigate:
-  - keywords must contain ONLY full URL
-  - element_name must be "URL"
-  - navigate allowed ONLY if step intent is purely navigation
-- If verification intent exists, MUST return "verify"
-
-Respond only with JSON using this format:
-{
-  "keywords": [],
-  "elementName": "",
-  "elementType":""
-  "action": ""
-}
-
-No other text.
+    From the step, extract ONLY the meaningful keywords so that i can search for the element in the dom.
+    Rules:
+    - Only give response for the current step.
+    - understand the step and context from the ${JSON.stringify(priorAndNextSteps)}.and keywords should be from step. it should not be related to other steps.
+    - 3 to 5 keywords maximum.
+    - If the step is about entering text or Uploading file, Should NOT include input value from the step into keywords.
+    - If the step has words like tag name audio, video, image,svg, checkbox etc, include them in the keywords.
+    - If icon is metioned in step than 'svg' should add in keywords and for Upload action first keyword should be 'file'.
+    - First keywords should be from step next Keywords must be distinct and based on the element's label meaning only.
+    - If a keyword contains exactly two words, you MUST always include both the spaced and concatenated (no-space) versions; omission of either is invalid.
+    - Do NOT split single-word keywords and do NOT include relation terms (above, below, next to, etc.) in keywords.
+    - Treat each keyword independently—never merge different keywords or combine them with relation words.
+    - Example: Click on Sign In above Create Account button → ["Sign In", "SignIn", "Create Account", "CreateAccount"]
+    - Keywords can be string or number if the step contains a number ,add it also as keyword.
+    - Do Not include any other unrelated keywords for step. 
+    - Do NOT include generic UI words (button, field, etc) and action words (tap, click, press, etc).
+    - Do NOT include status/technical words (displayed, enabled, authenticate, visible).
+    - If an element label contains multiple words (e.g., "Sign In", "Add to Cart"), keep them together as ONE keyword and do not split them and also for keywords you generated, do not split them.
+    - element_name: extract name of the element that mentioned in the the step.(eg:tap on x -> element_name:x) keep element_name as short as possible and make the first letter of first word of the element_name as capital. beacuse element_name is also used to find element in the dom. and if element_name is not mentioned in step than return action of the step as element_name.
+    - action: click for taping, clicking or selecting, enter for entering input, wait for waiting or sleeping,scroll for scrolling and swiping, navigate for navigating to page using url or navigateing back to previous page, get for getting,fetching element,maximize for maximizing browser window, close for closing browser window,open for opening browser window, upload for uploading file using path, drag_and_drop for dragging and dropping element, switch is for switching to tab or window or frame,cleartext for clearing or removing text from element.
+    - action must be one of from this list  ${JSON.stringify(allowedActions)}.if not one of them, return '0'. if step about set or find action return '0'
+    - For navigate action, keywords should contain only one keyword which is full url from the step and should not include any other text. and if step has another actions, including navigate action, don't return navigate action return another action witch is in the step. and element_name should be "URL".
+    Navigate action is allowed only when the step intent is purely navigation (e.g., "Navigate to URL").
+    If the step includes verification intent (verify/check/confirm/etc), you MUST return "verify" and MUST NOT return "navigate".
+    only give navigate action if step has only navigate action.
+         
+    Respond only with JSON using this format:
+    {
+    "keywords": [key1,key2,key3,key4,key5],
+    "element_name": "x",
+    "action": "x"
+    }
+
+    No other text.
 `;
 
   return prompt;

package/src/ai/llmprompts/systemPrompts/fireflinkElementIndexExtactors.ts

@@ -26,8 +26,8 @@ export async function ffInspectorNumExtractor({
     "WaitTillAlertIsPresent",
     "VerifyTextPresentOnAlertPopup"
   ];
-  const elementType = ["button", "link"]
-
+  const elementType = ['link', 'textfield', 'icon', 'button', 'radioButton', 'text', 'textarea', 'image', 'dropdown', 'checkbox', 'tab', 'action overflow button', 'hamburger icon', 'toggle button', 'suggestion', 'time picker', 'date picker', 'toaster message', 'card', 'tooltip', 'option', 'calender', 'sliders', 'visual testing'];
+  const enterActions = ["enter", "clearandenter"];
   let prompt;
 
   // ---------------- ALERT ----------------
@@ -123,75 +123,50 @@ Return ONLY valid JSON:
   // ---------------- DEFAULT (CLICK / ENTER / UPLOAD etc.) ----------------
   else {
     prompt = `
-You are a deterministic UI action extraction engine.
-
-OBJECTIVE:
-Identify the SINGLE best matching element from the Simplified JSON
-for the given step and return structured automation data.
-
------------------------------------------
-
-CONTEXT STEPS:
-${JSON.stringify(priorAndNextSteps)}
-
-CLICK ACTION WORDS:
-${JSON.stringify(clickActions)}
-
-ALLOWED ELEMENT TYPES:
-${elementType}
------------------------------------------
-
-CRITICAL MATCHING LOGIC:
-
-1. First understand intent:
-   - click words → click
-   - enter/type/write → enter_text
-   - upload → upload
-   - drag_and_drop → drag_and_drop
-
-2. For ENTER actions:
-   DO NOT pick first input blindly.
-   Score candidates using priority:
-
-   Priority Order:
-   (1) Exact label text match
-   (2) Placeholder match
-   (3) Name attribute semantic match
-   (4) ID semantic match
-   (5) type="email" if step contains "email"
-   (6) DOM sibling label relationship
-
-   Choose highest scoring match.
-
-   If NO strong semantic match → return:
-   attribute_value = "Fire-Flink-0"
-
-3. For CLICK actions:
-   - Prefer exact text match
-   - If multiple, choose closest contextual match
-   - If icon + svg exists, prefer svg element
-
-4. Never hallucinate.
-5. Only use FF values from provided JSON.
-6. No null values. Use empty string "".
-7. Output must be valid JSON only.
-
------------------------------------------
-SIMPLIFIED JSON:
-${extractedDomJson}
------------------------------------------
-
-Respond ONLY with JSON:
 
+-You are an intelligent assistant that extracts structured UI action data.
+    -Given a structured UI JSON representation with uniquely identified elements (ff-inspect values like Fire-Flink-1, Fire-Flink-2, Fire-Flink-3... in DOM order), 
+    -locate the most appropriate element for an automation step by performing keyword-based matching using exact, partial, and fuzzy strategies, and 
+    return the identifier of the best match.
+    
+Return **only valid JSON** in the following format:
 {
-  "attribute_value": "Fire-Flink-x",
-  "action": "click | enter | upload | drag_and_drop",
-  "input_text": "",
-  "keyword": "",
-  "num_of_scrolls": "0",
-  "direction": "down",
-  "elementType": ""
+"attribute_value": "Fire-Flink-x",
+"action": "x",
+"input_text": "x",
+"keyword": "x",
+"num_of_scrolls": "0",
+"direction": "down",
+"element_type": "x"
 }
+
+Rules:
+You are an AI assistant. For the step, extract the element keyword from the step (the field name like 'Leaving from', 'Going To').
+- Use context from the ${JSON.stringify(priorAndNextSteps)}, keyword and filtered dom to search for FF-inspecter. 
+- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the atep.**
+- If multiple matches exist, use the reference element and choose the closest match by ff-inspect distance.
+  **important example: step: "click on login below user login" 
+            simplified json: 
+                        login - ff-inspect-300
+                        user login - ff-inspect-790
+                        login - ff-inspect-803
+                        then should return login - ff-inspect-803 witch is near to user reference element user login - ff-inspect-790**
+- Respect direction (up/down), default is down.
+- Do not return elements far away from the reference.
+- Select the closest semantic match to step and return only its attribute_value, else Fire-Flink-0.
+- Action: ${clickActions} for clicking or selecting, ${enterActions} for entering text, 'upload' for uploading file, 'drag_and_drop' for dragging and dropping, 'cleartext' for clearing text. 
+- For click action, if only step involving 'clicking on icon' then try to return related svg tags attribute_value, if they are available in simplified json.
+- For Action ${enterActions} extract input text from step.
+- **For ${enterActions} actions, never directly select a input tag if should be realted to step; if u can't find input tag related to step then prefer the closest label/span/div/b/i etc, it can be any tag related to the step.**
+- **For ${enterActions} step if you can't find perfect matching element return attribute_value as Fire-Flink-0 for the step. it should be exact match for the step.**
+- For ${enterActions} step, If the step implies autogenerated data or random data(email, phone, credentials, identifiers, etc.), generate a valid dummy input_text suitable for the field. 
+example: step: "Enter sss.@gmail.com in email field" and step : 
+        you should check only for email in Simplified JSON and input_text should be "sss.@gmail.com" dont add any extra text.
+- For Action "upload" extract file path from step.
+- if u cant find any input_text or any other dont give null just return them "" empty.
+- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
+Simplified JSON: ${extractedDomJson}
+
 `;
   }
   return prompt;

package/src/core/main/runAutomationScript.ts

@@ -40,7 +40,7 @@ export class AutomationRunner implements IAutomationRunner {
       this.implicit
     );
     const domProcessor = new DomProcessingEngine();
-    let stepCount = 0;
+    let stepCount = 1;
     const listOfSteps = stepResult.response.manualSteps
     if (listOfSteps.length === 0) {
       throw new Error("No executable manual steps were returned by the LLM.");
@@ -57,12 +57,14 @@ export class AutomationRunner implements IAutomationRunner {
           input: { currentStep: step }
         });
 
+        logger.info(JSON.stringify(result, null, 2))
+
         const action = result.response.action?.toLowerCase();
         const handler = actionHandlers[action];
 
         logger.info
           (
-            `Processing step: "${step}" with action: "${action}" and keywords: ${JSON.stringify(result.response.keywords)}`
+            `Processing step ${stepCount}: "${step}" with action: "${action}" and keywords: ${JSON.stringify(result.response.keywords)}`
           );
 
         if (!handler) {
@@ -93,15 +95,19 @@ export class AutomationRunner implements IAutomationRunner {
           continue;
         }
 
+
         const browser = await context.session.getCurrentBrowser();
         domInfo = await getAnnotatedDOM(browser);
+        await logger.saveDOM(domInfo.dom, `annotated-dom-${stepCount}`);
 
         extractedRelevantDom = domProcessor.process({
           keywords: result.response.keywords,
           rawDom: domInfo.dom,
-          stepIndex: stepCount
+          stepCount: stepCount
         });
 
+        await logger.saveJSON(extractedRelevantDom, `relevant-dom-${stepCount}`);
+
         const stepResult = await stepProcessor.getLLMResponse({
           type: PromptType.FF_INSPECTOR,
           args: {
@@ -114,12 +120,17 @@ export class AutomationRunner implements IAutomationRunner {
           input: { currentStep: step }
         });
 
+        logger.info(JSON.stringify(stepResult, null, 2))
+
         const fireflinkIndex = stepResult.response.attribute_value;
         const xpath = domInfo.selectors[fireflinkIndex];
 
         if (!xpath) {
           throw new Error(`Unable to resolve xpath for ${fireflinkIndex}`);
         }
+        else if (fireflinkIndex == 0) {
+          throw new Error(`Unable to find element for ${step}`);
+        }
 
         await handler({
           selector: xpath,

package/src/domAnalysis/getRelaventElements.ts

@@ -1,12 +1,15 @@
 import { DomSimplifier } from "./simplifyAndFlatten.js";
 import { DomSearcher } from "./searchBest.js";
 import { DomRelationshipBuilder } from "./relativeElementsFromDom.js";
+import { logger } from "../utils/logger/logData.js";
+
 export interface DomProcessingRequest {
     keywords: string[] | string;
     rawDom: string;
-    stepIndex: number;
+    stepCount: number;
 }
 
+
 export class DomProcessingEngine {
 
     private simplifier = new DomSimplifier();
@@ -16,6 +19,7 @@ export class DomProcessingEngine {
     public process(request: DomProcessingRequest) {
 
         const flat = this.simplifier.simplify(request.rawDom);
+        logger.saveJSON(flat, `flat-dom-${request.stepCount}`);
         const searched = this.searcher.search(flat, request.keywords);
         const structured = this.relationshipBuilder.build(searched);
 

package/src/domAnalysis/searchBest.ts

@@ -123,13 +123,19 @@ export class DomSearcher {
             }
 
             if (collected.length < topN * 4) {
-                fuzzyList.sort((a, b) => b[0] - a[0])
-                    .forEach(([_, idx]) => addWithContext(idx, collected));
+                fuzzyList
+                    .sort((a, b) => b[0] - a[0])
+                    .forEach(([_, idx]) => {
+                        if (collected.length < topN * 4) {
+                            addWithContext(idx, collected);
+                        }
+                    });
             }
 
             if (collected.length < topN * 4) {
                 for (const idx of tagMatches) {
                     addWithContext(idx, collected);
+                    if (collected.length >= topN * 4) break;
                 }
             }
 

package/src/utils/logger/logData.ts

@@ -1,12 +1,18 @@
 import fs from "fs";
 import path from "path";
+import beautify from "js-beautify";
 
 const logsDir = path.resolve(process.cwd(), "logs");
+const domDir = path.resolve(process.cwd(), "dom");
 
 if (!fs.existsSync(logsDir)) {
     fs.mkdirSync(logsDir, { recursive: true });
 }
 
+if (!fs.existsSync(domDir)) {
+    fs.mkdirSync(domDir, { recursive: true });
+}
+
 const logFilePath = path.join(logsDir, "ai-execution-logs.txt");
 
 export const logger = {
@@ -32,5 +38,44 @@ export const logger = {
         const message = `[ERROR] ${new Date().toISOString()} - ${formattedArgs.join(" ")}\n`;
 
         fs.appendFileSync(logFilePath, message, "utf-8");
+    },
+
+    saveDOM: (domContent: string, fileName?: string) => {
+
+        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+        const finalFileName = fileName
+            ? `${fileName}.html`
+            : `dom-${timestamp}.html`;
+
+        const filePath = path.join(domDir, finalFileName);
+
+        const formattedHTML = beautify.html(domContent, {
+            indent_size: 2,
+            preserve_newlines: false,
+            wrap_line_length: 120
+        });
+
+        fs.writeFileSync(filePath, formattedHTML, "utf-8");
+    },
+
+    saveJSON: (data: unknown, fileName?: string) => {
+        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+
+        const finalFileName = fileName
+            ? `${fileName}.json`
+            : `data-${timestamp}.json`;
+
+        const filePath = path.join(domDir, finalFileName);
+
+        fs.writeFileSync(
+            filePath,
+            JSON.stringify(data, null, 2),
+            "utf-8"
+        );
+
+
     }
+
+
 };
+