ff-automationv2 2.1.2 → 2.1.3-beta.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ff-automationv2",
-  "version": "2.1.2",
+  "version": "2.1.3-beta.1",
   "private": false,
   "type": "module",
   "description": "This lib is used to automate the manual testcase",

package/src/ai/llmcalls/parseLlmOputput.ts

@@ -1,4 +1,5 @@
 import { FireFlinkLLMResponse } from "../../core/types/llmResponseType.js";
+import { logger } from "../../utils/logger/logData.js";
 
 
 export class LLMResultParser {
@@ -47,6 +48,8 @@ export class LLMResultParser {
             this.outputTokens += outputTokens;
             this.totalTokens += totalTokens;
 
+            logger.info(`Total tokens for this call is : ${usage.total_tokens}`);
+
             return { response: parsedContent };
 
         } catch (error) {

package/src/ai/llmprompts/systemPrompts/actionExtractorPrompt.ts

@@ -15,7 +15,7 @@ You are an expert in Web application testing.
     Rules:
     - Only give response for the current step.
     - understand the step and context from the ${JSON.stringify(priorAndNextSteps)}.and keywords should be from step. it should not be related to other steps.
-    - 3 to 5 keywords maximum.
+    - 3 to 5 keywords maximum. dont give more than 5 keywords. and no need for casesensitive. 
     - If the step is about entering text or Uploading file, Should NOT include input value from the step into keywords.
     - If the step has words like tag name audio, video, image,svg, checkbox etc, include them in the keywords.
     - If icon is metioned in step than 'svg' should add in keywords and for Upload action first keyword should be 'file'.
@@ -23,7 +23,7 @@ You are an expert in Web application testing.
     - If a keyword contains exactly two words, you MUST always include both the spaced and concatenated (no-space) versions; omission of either is invalid.
     - Do NOT split single-word keywords and do NOT include relation terms (above, below, next to, etc.) in keywords.
     - Treat each keyword independently—never merge different keywords or combine them with relation words.
-    - Example: Click on Sign In above Create Account button → ["Sign In", "SignIn", "Create Account", "CreateAccount"]
+    - Example: Click on Sign In above Create Account button → ["Sign In", "SignIn", "Create Account", "CreateAccount"] never break kewword like ["create", "account"]
     - Keywords can be string or number if the step contains a number ,add it also as keyword.
     - Do Not include any other unrelated keywords for step. 
     - Do NOT include generic UI words (button, field, etc) and action words (tap, click, press, etc).
@@ -40,7 +40,7 @@ You are an expert in Web application testing.
     Respond only with JSON using this format:
     {
     "keywords": [key1,key2,key3,key4,key5],
-    "element_name": "x",
+    "elementName": "x",
     "action": "x"
     }
 

package/src/ai/llmprompts/systemPrompts/fireflinkElementIndexExtactors.ts

@@ -137,7 +137,7 @@ Return **only valid JSON** in the following format:
 "keyword": "x",
 "num_of_scrolls": "0",
 "direction": "down",
-"element_type": "x"
+"elementType": "x"
 }
 
 Rules:
@@ -165,7 +165,7 @@ example: step: "Enter sss.@gmail.com in email field" and step :
 - For Action "upload" extract file path from step.
 - if u cant find any input_text or any other dont give null just return them "" empty.
 - Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
-Simplified JSON: ${extractedDomJson}
+Simplified JSON: ${extractedDomJson} if this simplified json is empty return Fire-Flink-0.
 
 `;
   }

package/src/automation/actions/interaction/enterInput.ts

@@ -1,12 +1,20 @@
 import { EnterInputInterface } from "../interface/interactionActionInterface.js";
+import { enterActionHelper } from "../../../utils/helpers/enterActionHelper.js";
 export async function enterInput(args: EnterInputInterface): Promise<void> {
+
     try {
-        const element = args.driver.$(args.selector);
+
+        const { ffNumber, xpathPath } = enterActionHelper(
+            args.pageDOM,
+            args.fireflinkIndex
+        );
+
+        const element = await args.driver.$(xpathPath);
         await element.scrollIntoView({ block: 'center', inline: 'center' });
         await element.setValue(args.value);
         const ffElement: any = await args.elementGetter.getFireFlinkElement(
             args.pageDOM,
-            `[ff-inspect="${args.fireflinkIndex}"]`
+            `[ff-inspect="Fire-Flink-${ffNumber}"]`
         );
         args.scriptDataAppender.add({
             nlpName: 'SendKeys',
@@ -21,6 +29,7 @@ export async function enterInput(args: EnterInputInterface): Promise<void> {
 
     }
     catch (error: any) {
+        console.log(error);
         throw new Error("enter action failed", { cause: error });
     }
 

package/src/core/main/runAutomationScript.ts

@@ -14,12 +14,64 @@ import { DomProcessingEngine } from "../../domAnalysis/getRelaventElements.js"
 import { getAnnotatedDOM } from "../../utils/DomExtraction/jsForAttributeInjection.js"
 import { logger } from "../../utils/logger/logData.js"
 export class AutomationRunner implements IAutomationRunner {
+  static sessionTerminationDetails: Record<string, boolean> = {};
   constructor(
     private readonly request: AutomationRequest,
     private pageLoad: number = 20000,
     private implicit: number = 15000,
     private tokensConsumed: number = 0
-  ) { }
+
+  ) { AutomationRunner.sessionTerminationDetails[request.testCaseId] = false }
+
+  static getSessionTerminationInfo(testCaseId: string): boolean {
+    return AutomationRunner.sessionTerminationDetails[testCaseId]
+  }
+  static updateSessionTerminationInfo(testCaseId: string): void {
+    AutomationRunner.sessionTerminationDetails[testCaseId] = true
+  }
+
+  private async cleanup(
+    context: ExecutionContext,
+    domInfo: any,
+    extractedRelevantDom: any,
+    stepProcessor: StepProcessor,
+    scriptRunner: ScriptRunner
+  ) {
+    try {
+      logger.info("Starting cleanup process...");
+      if (context?.session) {
+        try {
+          const browser = await context.session.getCurrentBrowser();
+          // if (browser) {
+          //   await browser.deleteSession?.();
+          // }
+        } catch (e) {
+          logger.error("Browser cleanup failed:", e);
+        }
+      }
+
+      domInfo = null;
+      extractedRelevantDom = null;
+
+      if (stepProcessor) {
+        (stepProcessor as any).llm = null;
+      }
+
+      scriptRunner = null as any;
+
+      delete AutomationRunner.sessionTerminationDetails[this.request.testCaseId];
+
+      if (global.gc) {
+        global.gc();
+      }
+
+      logger.info("Cleanup completed successfully.");
+    }
+    catch (error) {
+      logger.error("Cleanup error:", error);
+    }
+  }
+
 
   async run(): Promise<void> {
     const apiService = new FireFlinkApiService();
@@ -42,11 +94,16 @@ export class AutomationRunner implements IAutomationRunner {
     const domProcessor = new DomProcessingEngine();
     let stepCount = 1;
     const listOfSteps = stepResult.response.manualSteps
-    if (listOfSteps.length === 0) {
+    if (listOfSteps.length == 0) {
       throw new Error("No executable manual steps were returned by the LLM.");
     }
+
+    logger.info(listOfSteps)
+
     for (const step of listOfSteps) {
       try {
+        if (AutomationRunner.getSessionTerminationInfo(this.request.testCaseId)) { break; }
+
         const start = Math.max(0, stepCount - 3);
         const end = Math.min(listOfSteps.length, stepCount + 3);
         const priorAndNextSteps = listOfSteps.slice(start, end);
@@ -128,7 +185,7 @@ export class AutomationRunner implements IAutomationRunner {
         if (!xpath) {
           throw new Error(`Unable to resolve xpath for ${fireflinkIndex}`);
         }
-        else if (fireflinkIndex == 0) {
+        else if (fireflinkIndex == "Fire-Flink-0") {
           throw new Error(`Unable to find element for ${step}`);
         }
 
@@ -184,5 +241,14 @@ export class AutomationRunner implements IAutomationRunner {
         "Failed to send payload to FireFlink API:", { cause: error }
       );
     }
+    finally {
+      await this.cleanup(
+        context,
+        domInfo,
+        extractedRelevantDom,
+        stepProcessor,
+        scriptRunner
+      );
+    }
   }
 }

package/src/domAnalysis/searchBest.ts

@@ -88,7 +88,7 @@ export class DomSearcher {
 
                 let matched = false;
 
-                if (searchText === kw) {
+                if (searchText == kw) {
                     exact.push(i);
                     matched = true;
                 }
@@ -105,7 +105,7 @@ export class DomSearcher {
                     }
                 }
 
-                if (kw === tagName || tagName.includes(kw)) {
+                if (kw == tagName || tagName.includes(kw)) {
                     tagMatches.push(i);
                 }
             }

package/src/utils/DomExtraction/jsForAttributeInjection.ts

@@ -19,19 +19,25 @@ export async function getAnnotatedDOM(
         // ==========================================================
         function getCoverageStatus(el: Element): string {
 
-            try {
-                if (!el || !el.getBoundingClientRect) return "any";
+            if (!el || !el.getBoundingClientRect) return "unknown";
+
+            const rect = el.getBoundingClientRect();
+            if (rect.width === 0 || rect.height === 0) return "false";
 
-                const rect = el.getBoundingClientRect();
-                if (rect.width === 0 || rect.height === 0) return "false";
+            const testPoints = [
+                [rect.left + 5, rect.top + 5],
+                [rect.right - 5, rect.top + 5],
+                [rect.left + 5, rect.bottom - 5],
+                [rect.right - 5, rect.bottom - 5],
+                [rect.left + rect.width / 2, rect.top + rect.height / 2]
+            ];
 
-                const x = rect.left + rect.width / 2;
-                const y = rect.top + rect.height / 2;
+            for (let [x, y] of testPoints) {
 
-                const topEl = el.ownerDocument.elementFromPoint(x, y);
-                if (!topEl) return "false";
+                let topEl = document.elementFromPoint(x, y);
+                if (!topEl) continue;
 
-                if (topEl === el || el.contains(topEl)) return "false";
+                if (topEl === el || el.contains(topEl)) continue;
 
                 const style = getComputedStyle(topEl);
 
@@ -39,213 +45,275 @@ export async function getAnnotatedDOM(
                     style.pointerEvents === "none" ||
                     style.opacity === "0" ||
                     style.visibility === "hidden"
-                ) return "false";
+                ) continue;
 
-                return "true";
-            } catch {
-                return "any";
+                if (el.closest("label")?.contains(topEl)) continue;
+                if (topEl.closest("label")?.contains(el)) continue;
+                if (el.parentElement?.contains(topEl)) continue;
+
+                const isBlocking =
+                    ["BUTTON", "A"].includes(topEl.tagName) ||
+                    (topEl as HTMLElement).onclick ||
+                    topEl.getAttribute("role") === "button";
+
+                if (isBlocking) return "true";
             }
+
+            return "false";
         }
 
         // ==========================================================
-        // INTERACTIVITY METADATA
+        // VISIBILITY CHECK
         // ==========================================================
-        function annotateInteractivity(el: Element): void {
+        function isUsable(node: Element): boolean {
+
+            const s = getComputedStyle(node);
 
-            const s = getComputedStyle(el);
+            if (
+                s.display === "none" ||
+                s.visibility === "hidden" ||
+                s.opacity === "0" ||
+                s.pointerEvents === "none"
+            ) return false;
 
-            el.setAttribute("element-visibility", s.visibility || "");
-            el.setAttribute("element-pointer-events", s.pointerEvents || "");
-            el.setAttribute("element-zindex", s.zIndex || "");
-            el.setAttribute("element-covered", getCoverageStatus(el));
+            const rect = node.getBoundingClientRect();
+            if (rect.width === 0 || rect.height === 0) return false;
+
+            return true;
         }
 
         // ==========================================================
-        // GLOBAL XPATH (Shadow + Iframe Safe)
+        // ANNOTATION
+        // ==========================================================
+        function annotate(node: Element): void {
+
+            const s = getComputedStyle(node);
+
+            node.setAttribute("element-visibility", s.visibility || "");
+            node.setAttribute("element-pointer-events", s.pointerEvents || "");
+            node.setAttribute("element-zindex", s.zIndex || "");
+
+            if (!isUsable(node)) {
+                node.setAttribute("element-covered", "true");
+                return;
+            }
+
+            try {
+                node.setAttribute(
+                    "element-covered",
+                    getCoverageStatus(node)
+                );
+            } catch {
+                node.setAttribute("element-covered", "unknown");
+            }
+        }
+
+        // ==========================================================
+        // XPATH (Shadow + iframe safe)
         // ==========================================================
         function getXPath(node: Element): string {
 
             if (!node || node.nodeType !== 1) return "";
 
-            const segments: string[] = [];
-            let current: Node | null = node;
+            let segments: string[] = [];
+            let current: Element | null = node;
 
             while (current && current.nodeType === 1) {
 
-                const el = current as Element;
-                const parent = el.parentNode;
+                const parent = current.parentNode;
+
+                if (parent instanceof ShadowRoot) {
+
+                    const host = parent.host;
+
+                    const siblings = Array.from(parent.children)
+                        .filter(n => n.tagName === current!.tagName);
+
+                    const index = siblings.indexOf(current) + 1;
+
+                    segments.unshift(
+                        "shadow/" +
+                        current.tagName.toLowerCase() +
+                        "[" + index + "]"
+                    );
+
+                    current = host as Element;
+                    continue;
+                }
 
                 let index = 1;
 
                 if (parent && (parent as Element).children) {
+
                     const siblings = Array.from(
                         (parent as Element).children
-                    ).filter(n => n.tagName === el.tagName);
+                    ).filter(n => n.tagName === current!.tagName);
 
-                    index = siblings.indexOf(el) + 1;
+                    index = siblings.indexOf(current) + 1;
                 }
 
                 segments.unshift(
-                    el.tagName.toLowerCase() + "[" + index + "]"
+                    current.tagName.toLowerCase() +
+                    "[" + index + "]"
                 );
 
-                // ---------------------------
-                // Shadow DOM handling
-                // ---------------------------
-                if (parent instanceof ShadowRoot) {
-                    segments.unshift("shadow");
-                    current = parent.host;
-                    continue;
-                }
+                if (!parent || parent.nodeType !== 1) break;
 
-                // ---------------------------
-                // Iframe root handling
-                // ---------------------------
-                if (
-                    el.ownerDocument !== document &&
-                    el === el.ownerDocument.documentElement
-                ) {
-                    const frameEl =
-                        el.ownerDocument.defaultView?.frameElement as Element | null;
-
-                    if (frameEl && frameEl.getAttribute("ff-xpath")) {
-                        return (
-                            frameEl.getAttribute("ff-xpath") +
-                            "/" +
-                            segments.join("/")
-                        );
-                    }
-                }
+                current = parent as Element;
+            }
 
-                if (!parent || parent.nodeType !== 1) break;
+            let path = "/" + segments.join("/");
+
+            if (node.ownerDocument !== document) {
+
+                const frameEl =
+                    node.ownerDocument.defaultView?.frameElement;
+
+                if (frameEl) {
 
-                current = parent;
+                    let framePath =
+                        frameEl.getAttribute("ff-xpath");
+
+                    if (!framePath) {
+                        framePath = getXPath(frameEl);
+                    }
+
+                    path = framePath + path;
+                }
             }
 
-            return "/" + segments.join("/");
+            return path;
         }
 
         // ==========================================================
-        // PROCESS NODE
+        // TRUE NODE WALK (TreeWalker)
         // ==========================================================
-        function processNode(node: Element): void {
+        function walk(root: Node): void {
 
-            if (!node || node.nodeType !== 1) return;
+            const walker = document.createTreeWalker(
+                root,
+                NodeFilter.SHOW_ELEMENT,
+                null
+            );
 
-            const fireId = `Fire-Flink-${nodeIndex++}`;
-            const xpath = getXPath(node);
+            let node = walker.currentNode as Element | null;
 
-            node.setAttribute("ff-inspect", fireId);
-            node.setAttribute("ff-xpath", xpath);
+            while (node) {
 
-            xpaths[fireId] = xpath;
+                if (
+                    node.nodeType === 1 &&
+                    !["META", "STYLE", "SCRIPT"]
+                        .includes(node.tagName)
+                ) {
 
-            annotateInteractivity(node);
+                    const fireId =
+                        "Fire-Flink-" + nodeIndex++;
 
-            // ---------------------------
-            // Shadow DOM
-            // ---------------------------
-            const shadowRoot = (node as HTMLElement).shadowRoot;
-            if (shadowRoot) {
-                shadowRoot.querySelectorAll("*")
-                    .forEach(el => processNode(el as Element));
-            }
+                    const xpath = getXPath(node);
 
-            // ---------------------------
-            // Same-origin iframe
-            // ---------------------------
-            if (node.tagName === "IFRAME") {
-                try {
-                    const doc =
-                        (node as HTMLIFrameElement).contentDocument;
+                    node.setAttribute("ff-inspect", fireId);
+                    node.setAttribute("ff-xpath", xpath);
 
-                    if (doc) {
-                        processNode(doc.documentElement);
-                        doc.querySelectorAll("*")
-                            .forEach(el => processNode(el as Element));
+                    xpaths[fireId] = xpath;
+
+                    annotate(node);
+                }
+
+                if ((node as HTMLElement).shadowRoot) {
+                    walk((node as HTMLElement).shadowRoot!);
+                }
+
+                if (node.tagName === "IFRAME") {
+                    try {
+                        const doc =
+                            (node as HTMLIFrameElement)
+                                .contentDocument;
+
+                        if (doc?.documentElement) {
+                            walk(doc.documentElement);
+                        }
+                    } catch {
+                        node.setAttribute(
+                            "ff-cross-origin",
+                            "true"
+                        );
                     }
-                } catch {
-                    node.setAttribute("ff-cross-origin", "true");
                 }
-            }
 
-            node.querySelectorAll(":scope > *")
-                .forEach(el => processNode(el as Element));
+                node = walker.nextNode() as Element | null;
+            }
         }
 
-        // ==========================================================
-        // INITIAL INJECTION
-        // ==========================================================
-        document.querySelectorAll("*")
-            .forEach(el => processNode(el as Element));
+        // Start traversal
+        walk(document.documentElement);
 
         // ==========================================================
         // SERIALIZER
         // ==========================================================
         function serializeNode(node: Node): string {
 
-            if (node.nodeType === Node.TEXT_NODE) {
+            if (node.nodeType === Node.TEXT_NODE)
                 return node.textContent || "";
-            }
 
-            if (node.nodeType !== Node.ELEMENT_NODE) return "";
+            if (node.nodeType !== Node.ELEMENT_NODE)
+                return "";
 
             const el = node as Element;
             const tag = el.tagName.toLowerCase();
-            let html = `<${tag}`;
 
-            for (const attr of Array.from(el.attributes)) {
+            let html = "<" + tag;
 
-                // Avoid dumping large srcdoc inline
+            for (const attr of Array.from(el.attributes)) {
                 if (attr.name === "srcdoc") continue;
 
-                html += ` ${attr.name}="${attr.value.replace(/"/g, "&quot;")}"`;
+                html +=
+                    " " + attr.name +
+                    "=\"" +
+                    attr.value.replace(/"/g, "&quot;") +
+                    "\"";
             }
 
             html += ">";
 
-            // Shadow DOM serialization
-            const shadowRoot = (el as HTMLElement).shadowRoot;
-            if (shadowRoot) {
-                html += "<!--#shadow-root-open-->";
-                shadowRoot.querySelectorAll("*")
+            if ((el as HTMLElement).shadowRoot) {
+                html += "<!--#shadow-root-->";
+                (el as HTMLElement)
+                    .shadowRoot!
+                    .childNodes
                     .forEach(child => {
                         html += serializeNode(child);
                     });
                 html += "<!--#/shadow-root-->";
             }
 
-            // Iframe serialization
             if (
                 tag === "iframe" &&
                 el.getAttribute("ff-cross-origin") !== "true"
             ) {
                 try {
                     const doc =
-                        (el as HTMLIFrameElement).contentDocument;
+                        (el as HTMLIFrameElement)
+                            .contentDocument;
 
                     if (doc) {
-                        html += "<!--#iframe-content-->";
-                        html += serializeNode(doc.documentElement);
-                        html += "<!--#/iframe-content-->";
+                        html += serializeNode(
+                            doc.documentElement
+                        );
                     }
-                } catch {
-                    // cross-origin iframe, cannot access content 
-                }
+                } catch { }
             }
 
-            for (const child of Array.from(el.childNodes)) {
+            el.childNodes.forEach(child => {
                 html += serializeNode(child);
-            }
-
-            html += `</${tag}>`;
+            });
 
+            html += "</" + tag + ">";
             return html;
         }
 
         return {
             dom:
-                "<!DOCTYPE html>\\n" +
+                "<!DOCTYPE html>\n" +
                 serializeNode(document.documentElement),
             selectors: xpaths
         };

package/src/utils/helpers/enterActionHelper.ts

@@ -0,0 +1,94 @@
+import { JSDOM } from "jsdom";
+
+export interface EnterActionResult {
+    ffNumber: number;
+    xpathPath: string;
+}
+
+export const enterActionHelper = (
+    pageDOM: string,
+    ffNumber: string
+): EnterActionResult => {
+
+    //  Extract numeric part
+    const match = ffNumber.match(/\d+$/);
+    let numericFF = match ? parseInt(match[0], 10) : 0;
+
+    const parser = new JSDOM(pageDOM);
+    const doc = parser.window.document;
+
+    const isEditable = (el: Element | null): boolean => {
+        if (!el) return false;
+
+        const tag = el.tagName?.toLowerCase();
+
+        return (
+            ["input", "textarea", "select"].includes(tag) ||
+            el.getAttribute("contenteditable") === "true"
+        );
+    };
+
+    const getXPath = (el: Element | null): string => {
+        return el?.getAttribute("ff-xpath") || "";
+    };
+
+    let element = doc.querySelector(
+        `[ff-inspect="Fire-Flink-${numericFF}"]`
+    );
+
+    if (!element) {
+        return { ffNumber: numericFF, xpathPath: "" };
+    }
+
+    let xpathPath = "";
+
+    //  Current element editable
+    if (isEditable(element)) {
+        xpathPath = getXPath(element);
+
+    } else {
+
+        // Search editable child
+        let editableChild: Element | null = null;
+
+        for (const child of Array.from(element.querySelectorAll("*"))) {
+            if (isEditable(child)) {
+                editableChild = child;
+                break;
+            }
+        }
+
+        if (editableChild) {
+            xpathPath = getXPath(editableChild);
+
+        } else {
+
+            // Try next 10 FF elements
+            for (let offset = 1; offset <= 10; offset++) {
+                const nextNum = numericFF + offset;
+
+                const nextTarget = doc.querySelector(
+                    `[ff-inspect="Fire-Flink-${nextNum}"]`
+                );
+
+                if (isEditable(nextTarget)) {
+                    numericFF = nextNum;
+                    xpathPath = getXPath(nextTarget);
+                    break;
+                }
+            }
+
+            // Final fallback
+            if (!xpathPath) {
+                xpathPath = getXPath(element);
+            }
+        }
+    }
+
+    // console.log("xpathPath", xpathPath);
+
+    return {
+        ffNumber: numericFF,
+        xpathPath
+    };
+};