ff-automationv2 2.1.3 → 2.2.2

package/dist/utils/javascript/jsForVision.js

@@ -0,0 +1,148 @@
+export const JS_PRIMARY = `return (function() {
+
+    function isReallyVisible(n) {
+    const r = n.getBoundingClientRect();
+    if (r.width === 0 || r.height === 0) return false;
+
+    const cx = r.x + r.width / 2;
+    const cy = r.y + r.height / 2;
+
+    let topEl = document.elementFromPoint(cx, cy);
+    if (!topEl) return false;
+
+    return n === topEl || n.contains(topEl);
+}
+
+   function isPrimary(n) {
+    const tag = n.tagName.toLowerCase();
+    const text = (n.innerText || "").trim();
+    const style = window.getComputedStyle(n);
+
+    if (["input", "button", "select", "textarea"].includes(tag))
+        return true;
+
+    if (tag === "label" && text.length > 0)
+        return true;
+
+    if (tag === "a" && text.length > 0)
+        return true;
+
+    if (tag === "li") {
+        if (text.length > 0 && style.cursor === "pointer")
+            return true;
+    }
+
+    return false;
+}
+    function getBBox(n) {
+        const r = n.getBoundingClientRect();
+        return {x:r.x, y:r.y, w:r.width, h:r.height};
+    }
+
+    function collectShadowAware(root, out) {
+        if (!root) return;
+
+        if (root.nodeType === Node.ELEMENT_NODE && root.hasAttribute("ff-inspect"))
+            out.push(root);
+
+        for (const child of root.children) {
+            collectShadowAware(child, out);
+            if (child.shadowRoot) collectShadowAware(child.shadowRoot, out);
+        }
+    }
+
+    const all = [];
+    collectShadowAware(document.documentElement, all);
+
+    const out = [];
+
+    for (let n of all) {
+        if (isReallyVisible(n) && isPrimary(n)) {
+            out.push({
+                ff: n.getAttribute("ff-inspect"),
+                xpath: n.getAttribute("ff-xpath"),
+                bbox: getBBox(n),
+                tag: n.tagName.toLowerCase(),
+                text: (n.innerText || "").trim()
+            });
+        }
+    }
+
+    return out;
+})();
+`;
+export const JS_SECONDARY = `return (function() {
+
+    function isReallyVisible(n) {
+    const r = n.getBoundingClientRect();
+    if (r.width === 0 || r.height === 0) return false;
+
+    const cx = r.x + r.width / 2;
+    const cy = r.y + r.height / 2;
+
+    let topEl = document.elementFromPoint(cx, cy);
+    if (!topEl) return false;
+
+    return n === topEl || n.contains(topEl);
+}
+
+    function isSecondary(n) {
+        const r = n.getBoundingClientRect();
+        const tag = n.tagName.toLowerCase();
+        const text = (n.innerText || "").trim();
+        const style = window.getComputedStyle(n);
+
+        if (r.y < 120 && text.length > 0 && text.length <= 12)
+            return true;
+
+        if (["div", "span"].includes(tag)) {
+            if (text.length > 0 && text.length <= 20 && r.height > 10)
+                return true;
+            if (style.cursor === "pointer")
+                return true;
+        }
+
+        if (tag === "li") {
+            if (text.length > 0 || style.cursor === "pointer")
+                return true;
+        }
+
+        return false;
+    }
+
+    function getBBox(n) {
+        const r = n.getBoundingClientRect();
+        return {x:r.x, y:r.y, w:r.width, h:r.height};
+    }
+
+    function collectShadowAware(root, out) {
+        if (!root) return;
+
+        if (root.nodeType === Node.ELEMENT_NODE && root.hasAttribute("ff-inspect"))
+            out.push(root);
+
+        for (const child of root.children) {
+            collectShadowAware(child, out);
+            if (child.shadowRoot) collectShadowAware(child.shadowRoot, out);
+        }
+    }
+
+    const all = [];
+    collectShadowAware(document.documentElement, all);
+
+    const out = [];
+
+    for (let n of all) {
+        if (isReallyVisible(n) && isSecondary(n)) {
+            out.push({
+                ff: n.getAttribute("ff-inspect"),
+                xpath: n.getAttribute("ff-xpath"),
+                bbox: getBBox(n),
+                tag: n.tagName.toLowerCase(),
+                text: (n.innerText || "").trim()
+            });
+        }
+    }
+
+    return out;
+})();`;

package/dist/utils/logger/logData.d.ts

@@ -0,0 +1,8 @@
+export declare const logger: {
+    info: (...args: any[]) => Promise<void>;
+    error: (...args: any[]) => Promise<void>;
+    getError: (...args: any[]) => string;
+    saveDOM: (domContent: string, fileName?: string) => Promise<void>;
+    saveJSON: (data: unknown, fileName?: string) => Promise<void>;
+    saveImage: (imageBuffer: Buffer, fileName: string) => Promise<void>;
+};

package/dist/utils/logger/logData.js

@@ -0,0 +1,107 @@
+import fs from "fs/promises";
+import path from "path";
+import beautify from "js-beautify";
+const logsDir = path.resolve(process.cwd(), "logs");
+const domDir = path.resolve(process.cwd(), "dom");
+const screenshotsDir = path.resolve(process.cwd(), "screenshots");
+const now = new Date();
+const istDate = new Intl.DateTimeFormat("sv-SE", {
+    timeZone: "Asia/Kolkata",
+    year: "numeric",
+    month: "2-digit",
+    day: "2-digit",
+    hour: "2-digit",
+    minute: "2-digit",
+    second: "2-digit",
+    fractionalSecondDigits: 3,
+    hour12: false,
+})
+    .format(now)
+    .replace(" ", "T");
+const logFilePath = path.join(logsDir, "ai-execution-logs.txt");
+async function ensureDirs() {
+    await fs.mkdir(logsDir, { recursive: true });
+    await fs.mkdir(domDir, { recursive: true });
+    await fs.mkdir(screenshotsDir, { recursive: true });
+}
+ensureDirs();
+export const logger = {
+    info: async (...args) => {
+        const formattedArgs = args.map((arg) => typeof arg === "object" ? JSON.stringify(arg, null, 2) : arg);
+        const message = `[INFO] ${istDate}+05:30 - ${formattedArgs.join(" ")}\n`;
+        await fs.appendFile(logFilePath, message, "utf-8");
+    },
+    error: async (...args) => {
+        const formatted = args.map((arg) => {
+            if (arg instanceof Error) {
+                let output = arg.stack || `${arg.name}: ${arg.message}`;
+                output = output.replace(/invalid session id:\s*invalid session id:/gi, "invalid session id:");
+                if (arg.cause instanceof Error) {
+                    let causeStack = arg.cause.stack || "";
+                    causeStack = causeStack.replace(/invalid session id:\s*invalid session id:/gi, "invalid session id:");
+                    output += `\nCaused by: ${causeStack}`;
+                }
+                return output;
+            }
+            if (typeof arg === "object") {
+                return JSON.stringify(arg, null, 2);
+            }
+            return String(arg);
+        });
+        const message = `[ERROR] ${new Date().toISOString()} - ${formatted.join(" ")}\n`;
+        await fs.appendFile(logFilePath, message, "utf-8");
+    },
+    getError: (...args) => {
+        const formatted = args.map((arg) => {
+            if (arg instanceof Error) {
+                let output = arg.stack || `${arg.name}: ${arg.message}`;
+                output = output.replace(/invalid session id:\s*invalid session id:/gi, "invalid session id:");
+                if (arg.cause instanceof Error) {
+                    let causeStack = arg.cause.stack || "";
+                    causeStack = causeStack.replace(/invalid session id:\s*invalid session id:/gi, "invalid session id:");
+                    output += `\nCaused by: ${causeStack}`;
+                }
+                return output;
+            }
+            if (typeof arg === "object") {
+                return JSON.stringify(arg, null, 2);
+            }
+            return String(arg);
+        });
+        return formatted.join(" ");
+    },
+    saveDOM: async (domContent, fileName) => {
+        const timestamp = new Date().toLocaleString().replace(/[:.]/g, "-");
+        let finalFileName;
+        let formattedContent;
+        if (fileName?.includes("Mob")) {
+            finalFileName = fileName ? `${fileName}.xml` : `dom-${timestamp}.xml`;
+            formattedContent = domContent;
+        }
+        else {
+            finalFileName = fileName ? `${fileName}.html` : `dom-${timestamp}.html`;
+            formattedContent = beautify.html(domContent, {
+                indent_size: 2,
+                preserve_newlines: false,
+                wrap_line_length: 120,
+            });
+        }
+        const filePath = path.join(domDir, finalFileName);
+        await fs.writeFile(filePath, formattedContent, "utf-8");
+    },
+    saveJSON: async (data, fileName) => {
+        const timestamp = new Date().toLocaleString().replace(/[:.]/g, "-");
+        const finalFileName = fileName ? `${fileName}.json` : `data-${timestamp}.json`;
+        const filePath = path.join(domDir, finalFileName);
+        const safeData = JSON.stringify(data, (key, value) => {
+            if (key === "element")
+                return undefined; // remove DOM node
+            return value;
+        }, 2);
+        await fs.writeFile(filePath, safeData, "utf-8");
+    },
+    saveImage: async (imageBuffer, fileName) => {
+        const filePath = path.join(screenshotsDir, fileName);
+        await fs.writeFile(filePath, imageBuffer);
+    },
+};

package/dist/utils/mobileXpath/findXpathFromIndex.d.ts

@@ -0,0 +1,2 @@
+import { IMobileXpathFinder } from "./findXpathFromIndexInterface.js";
+export declare function findXpathFromIndex(args: IMobileXpathFinder): string;

package/dist/utils/mobileXpath/findXpathFromIndex.js

@@ -0,0 +1,61 @@
+import * as cheerio from "cheerio";
+export function findXpathFromIndex(args) {
+    const $ = cheerio.load(args.dom, { xmlMode: true });
+    const centers = [];
+    const clickableBoxes = [];
+    const traverse = (node) => {
+        if (node?.type === "tag") {
+            const attr = node.attribs;
+            if (attr?.bounds) {
+                const clickable = attr.clickable === "true";
+                const enabled = attr.enabled === "true";
+                const clazz = attr.class || "";
+                if ((clickable && enabled) ||
+                    clazz.toLowerCase().includes("edittext") ||
+                    clazz.toLowerCase().includes("autocompletetextview")) {
+                    const match = attr.bounds.match(/\d+/g);
+                    if (match && match.length === 4) {
+                        const x1 = parseInt(match[0]);
+                        const y1 = parseInt(match[1]);
+                        const x2 = parseInt(match[2]);
+                        const y2 = parseInt(match[3]);
+                        clickableBoxes.push([x1, y1, x2, y2]);
+                        centers.push([Math.floor((x1 + x2) / 2), Math.floor((y1 + y2) / 2)]);
+                    }
+                }
+            }
+        }
+        if (node.children) {
+            node.children.forEach(traverse);
+        }
+    };
+    traverse($.root()[0]);
+    const index = Number(args.fireflinkIndex.split("-").pop());
+    if (isNaN(index) || index < 1 || index > centers.length) {
+        throw new Error(`Index ${index} out of range (Total clickable items: ${centers.length})`);
+    }
+    const [x1, y1, x2, y2] = clickableBoxes[index - 1];
+    const boundsStr = `[${x1},${y1}][${x2},${y2}]`;
+    const element = $(`[bounds="${boundsStr}"]`).first();
+    if (!element.length) {
+        throw new Error(`Element with bounds ${boundsStr} not found`);
+    }
+    return getXPath(element);
+}
+function getXPath(element) {
+    let xpath = "";
+    let current = element;
+    while (current.length && current[0].tagName !== "hierarchy") {
+        const tag = current[0].tagName;
+        const siblings = current.parent().children(tag);
+        if (siblings.length > 1) {
+            const index = siblings.index(current) + 1;
+            xpath = `/${tag}[${index}]` + xpath;
+        }
+        else {
+            xpath = `/${tag}` + xpath;
+        }
+        current = current.parent();
+    }
+    return `/hierarchy${xpath}`;
+}

package/dist/utils/mobileXpath/findXpathFromIndexInterface.d.ts

@@ -0,0 +1,4 @@
+export interface IMobileXpathFinder {
+    fireflinkIndex: string;
+    dom: string;
+}

package/dist/utils/mobileXpath/findXpathFromIndexInterface.js

@@ -0,0 +1 @@
+export {};

package/dist/utils/newTab/newTabTracker.d.ts

@@ -0,0 +1,2 @@
+import { ScriptDataAppender } from "../../fireflinkData/fireflinkScript/scriptGenrationData.js";
+export declare function newTabTracker(driver: WebdriverIO.Browser, stepAction: string, previousHandles: string[], scriptAppender: ScriptDataAppender): Promise<void>;

package/dist/utils/newTab/newTabTracker.js

@@ -0,0 +1,38 @@
+export async function newTabTracker(driver, stepAction, previousHandles, scriptAppender) {
+    if (stepAction === "switch")
+        return;
+    const currentHandle = await driver.getWindowHandle();
+    const currentHandles = await driver.getWindowHandles();
+    if (currentHandles.length <= previousHandles.length) {
+        return;
+    }
+    const newHandles = currentHandles.filter(handle => !previousHandles.includes(handle));
+    for (const handle of newHandles) {
+        try {
+            await driver.switchToWindow(handle);
+            const url = await driver.getUrl();
+            // Skip unwanted system URLs
+            if (!url ||
+                url.startsWith("devtools://") ||
+                url.startsWith("chrome://") ||
+                url === "about:blank") {
+                await driver.switchToWindow(currentHandle);
+                continue;
+            }
+            // Valid URL → accept it
+            scriptAppender.add(async () => ({
+                nlpName: "SwitchToWindowBasedOnURL",
+                stepInputs: [url],
+                elementsData: []
+            }));
+            return;
+        }
+        catch (error) {
+            // Restore original window before failing
+            await driver.switchToWindow(currentHandle);
+            throw new Error(`Error switching to new window: ${error}`, { cause: error });
+        }
+    }
+    // If no valid window found, ensure we are back on original
+    await driver.switchToWindow(currentHandle);
+}

package/package.json

@@ -1,53 +1,68 @@
-{
-  "name": "ff-automationv2",
-  "version": "2.1.3",
-  "private": false,
-  "type": "module",
-  "description": "This lib is used to automate the manual testcase",
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
-  "exports": {
-    ".": {
-      "import": "./dist/index.js",
-      "types": "./dist/index.d.ts"
-    }
-  },
-  "scripts": {
-    "builds": "npm run lint && tsc",
-    "build": "tsc",
-    "prepare": "npm run build",
-    "lint": "eslint src --max-warnings=0",
-    "test": "npm run build && node dist/tests/test12.js",
-    "llm": "npm run build && node dist/tests/testllm.js",
-    "dev": "npm run build && node dist/index.js",
-    "start": "node dist/index.js",
-    "file": "node dist/domAnalysis/getXpath.js"
-  },
-  "keywords": [],
-  "author": "Diwahar R",
-  "license": "ISC",
-  "devDependencies": {
-    "@eslint/js": "10.0.1",
-    "@types/js-beautify": "^1.14.3",
-    "@types/jsdom": "27.0.0",
-    "@types/node": "25.2.2",
-    "eslint": "10.0.0",
-    "globals": "17.3.0",
-    "jiti": "2.6.1",
-    "nodemon": "3.1.11",
-    "ts-node": "10.9.2",
-    "typescript": "5.9.3",
-    "typescript-eslint": "8.56.0"
-  },
-  "dependencies": {
-    "axios": "1.13.5",
-    "cheerio": "1.2.0",
-    "fs-extra": "11.3.3",
-    "fuzzball": "2.2.3",
-    "js-beautify": "1.15.1",
-    "jsdom": "27.4.0",
-    "openai": "6.18.0",
-    "uuid": "13.0.0",
-    "webdriverio": "9.23.3"
-  }
-}
+{
+  "name": "ff-automationv2",
+  "version": "2.2.2",
+  "private": false,
+  "type": "module",
+  "description": "This lib is used to automate the manual testcase",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "type-check": "tsc --noEmit",
+    "builds": "npm run lint && tsc",
+    "build": "eslint . && tsc && echo Build Successful",
+    "prepare": "husky",
+    "lint": "eslint src --max-warnings=0",
+    "test": "npm run build && node dist/tests/test12.js",
+    "llm": "npm run build && node dist/tests/testllm.js",
+    "dev": "npm run build && node dist/index.js",
+    "start": "node dist/index.js",
+    "file": "node dist/domAnalysis/getXpath.js",
+    "error": "node dist/tests/test12.js",
+    "bro": "node dist/tests/test3.js"
+  },
+  "keywords": [],
+  "author": "Diwahar R",
+  "license": "ISC",
+  "devDependencies": {
+    "@eslint/js": "10.0.1",
+    "@types/fs-extra": "^11.0.4",
+    "@types/js-beautify": "^1.14.3",
+    "@types/jsdom": "27.0.0",
+    "@types/node": "^25.4.0",
+    "eslint": "10.0.0",
+    "globals": "17.3.0",
+    "husky": "^9.1.7",
+    "jiti": "2.6.1",
+    "nodemon": "3.1.11",
+    "ts-node": "10.9.2",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "8.56.0"
+  },
+  "dependencies": {
+    "@xmldom/xmldom": "^0.8.11",
+    "axios": "^1.13.5",
+    "canvas": "3.2.1",
+    "cheerio": "1.2.0",
+    "devtools": "^8.42.0",
+    "dotenv": "^17.3.1",
+    "fast-fuzzy": "^1.12.0",
+    "fs-extra": "11.3.3",
+    "js-beautify": "1.15.1",
+    "jsdom": "27.4.0",
+    "kafkajs": "^2.2.4",
+    "openai": "6.18.0",
+    "uuid": "13.0.0",
+    "webdriverio": "8.46.0",
+    "xpath": "^0.0.34"
+  }
+}

package/ARCHITECTURE.md

@@ -1,112 +0,0 @@
-# 📊 FireFlink System Architecture
-
-## Layered Architecture
-
-```
-┌────────────────────────────────────────────┐
-│ User Story Layer                           │
-└────────────────────────────────────────────┘
-                    ↓
-┌────────────────────────────────────────────┐
-│ LLM Reasoning Layer                        │
-│ - USER_STORY_TO_LIST                       │
-│ - KEYWORD_EXTRACTOR                        │
-│ - FF_INSPECTOR                             │
-└────────────────────────────────────────────┘
-                    ↓
-┌────────────────────────────────────────────┐
-│ Orchestration Layer                        │
-│ (AutomationRunner)                         │
-└────────────────────────────────────────────┘
-                    ↓
-┌────────────────────────────────────────────┐
-│ DOM Abstraction Layer                      │
-│ - Inject FF-inspecter attributes           │
-│ - Serialize DOM                            │
-│ - Build FF → XPath mapping                 │
-└────────────────────────────────────────────┘
-                    ↓
-┌────────────────────────────────────────────┐
-│ DOM Processing Engine                      │
-│ - Keyword-based DOM filtering              │
-└────────────────────────────────────────────┘
-                    ↓
-┌────────────────────────────────────────────┐
-│ Action Execution Layer                     │
-│ - Click Handler                            │
-│ - Enter Handler                            │
-│ - Navigate Handler                         │
-└────────────────────────────────────────────┘
-                    ↓
-┌────────────────────────────────────────────┐
-│ Script Generation Layer                    │
-│ - ScriptAppender                           │
-│ - Payload Builder                          │
-└────────────────────────────────────────────┘
-                    ↓
-┌────────────────────────────────────────────┐
-│ FireFlink Backend API                      │
-└────────────────────────────────────────────┘
-
-```
-
-## Execution Sequence
-
-AutomationRunner
-→ StepProcessor (LLM)
-→ Browser (Inject DOM)
-→ DomProcessingEngine
-→ StepProcessor (FF selection)
-→ ActionHandler
-→ ScriptAppender
-→ ScriptRunner
-→ FireFlink API
-
-## high Level flow
-
-```
-            User Story
-                ↓
-            LLM → Manual Steps
-                ↓
-            FOR EACH STEP
-                ↓
-            LLM → Action + Keywords
-                ↓
-            Inject Annotated DOM
-                ↓
-            Filter Relevant DOM (DomProcessingEngine)
-                ↓
-            LLM → Choose FF-inspecter Index
-                ↓
-            Map FF Index → XPath
-                ↓
-            Execute Action
-                ↓
-            Append Script Data
-                ↓
-┌─────────────────────────────┐
-│        Error Occurred?      │
-└──────────────┬──────────────┘
-               │
-        ┌──────┴──────┐
-        │             │
-       NO            YES
-        │             │
-        │      Append CloseBrowser
-        │             ↓
-        │      Stop Execution
-        │             |
-        ↓             |
-Continue Next Step    |
-        ↓             |
-(all Steps Complete)  |
-        |             |
-        |             |
-        └──────┬──────┘
-        Collect Token Usage
-               ↓
-        Build Payload
-               ↓
-        Send to FireFlink API
-```