ff-automationv2 2.1.3 → 2.2.0

package/src/utils/javascript/jsForVision.ts

@@ -0,0 +1,151 @@
+export const JS_PRIMARY = `return (function() {
+
+    function isReallyVisible(n) {
+    const r = n.getBoundingClientRect();
+    if (r.width === 0 || r.height === 0) return false;
+
+    const cx = r.x + r.width / 2;
+    const cy = r.y + r.height / 2;
+
+    let topEl = document.elementFromPoint(cx, cy);
+    if (!topEl) return false;
+
+    return n === topEl || n.contains(topEl);
+}
+
+   function isPrimary(n) {
+    const tag = n.tagName.toLowerCase();
+    const text = (n.innerText || "").trim();
+    const style = window.getComputedStyle(n);
+
+    if (["input", "button", "select", "textarea"].includes(tag))
+        return true;
+
+    if (tag === "label" && text.length > 0)
+        return true;
+
+    if (tag === "a" && text.length > 0)
+        return true;
+
+    if (tag === "li") {
+        if (text.length > 0 && style.cursor === "pointer")
+            return true;
+    }
+
+    return false;
+}
+    function getBBox(n) {
+        const r = n.getBoundingClientRect();
+        return {x:r.x, y:r.y, w:r.width, h:r.height};
+    }
+
+    function collectShadowAware(root, out) {
+        if (!root) return;
+
+        if (root.nodeType === Node.ELEMENT_NODE && root.hasAttribute("ff-inspect"))
+            out.push(root);
+
+        for (const child of root.children) {
+            collectShadowAware(child, out);
+            if (child.shadowRoot) collectShadowAware(child.shadowRoot, out);
+        }
+    }
+
+    const all = [];
+    collectShadowAware(document.documentElement, all);
+
+    const out = [];
+
+    for (let n of all) {
+        if (isReallyVisible(n) && isPrimary(n)) {
+            out.push({
+                ff: n.getAttribute("ff-inspect"),
+                xpath: n.getAttribute("ff-xpath"),
+                bbox: getBBox(n),
+                tag: n.tagName.toLowerCase(),
+                text: (n.innerText || "").trim()
+            });
+        }
+    }
+
+    return out;
+})();
+`
+
+
+
+export const JS_SECONDARY = `return (function() {
+
+    function isReallyVisible(n) {
+    const r = n.getBoundingClientRect();
+    if (r.width === 0 || r.height === 0) return false;
+
+    const cx = r.x + r.width / 2;
+    const cy = r.y + r.height / 2;
+
+    let topEl = document.elementFromPoint(cx, cy);
+    if (!topEl) return false;
+
+    return n === topEl || n.contains(topEl);
+}
+
+    function isSecondary(n) {
+        const r = n.getBoundingClientRect();
+        const tag = n.tagName.toLowerCase();
+        const text = (n.innerText || "").trim();
+        const style = window.getComputedStyle(n);
+
+        if (r.y < 120 && text.length > 0 && text.length <= 12)
+            return true;
+
+        if (["div", "span"].includes(tag)) {
+            if (text.length > 0 && text.length <= 20 && r.height > 10)
+                return true;
+            if (style.cursor === "pointer")
+                return true;
+        }
+
+        if (tag === "li") {
+            if (text.length > 0 || style.cursor === "pointer")
+                return true;
+        }
+
+        return false;
+    }
+
+    function getBBox(n) {
+        const r = n.getBoundingClientRect();
+        return {x:r.x, y:r.y, w:r.width, h:r.height};
+    }
+
+    function collectShadowAware(root, out) {
+        if (!root) return;
+
+        if (root.nodeType === Node.ELEMENT_NODE && root.hasAttribute("ff-inspect"))
+            out.push(root);
+
+        for (const child of root.children) {
+            collectShadowAware(child, out);
+            if (child.shadowRoot) collectShadowAware(child.shadowRoot, out);
+        }
+    }
+
+    const all = [];
+    collectShadowAware(document.documentElement, all);
+
+    const out = [];
+
+    for (let n of all) {
+        if (isReallyVisible(n) && isSecondary(n)) {
+            out.push({
+                ff: n.getAttribute("ff-inspect"),
+                xpath: n.getAttribute("ff-xpath"),
+                bbox: getBBox(n),
+                tag: n.tagName.toLowerCase(),
+                text: (n.innerText || "").trim()
+            });
+        }
+    }
+
+    return out;
+})();` 

package/src/utils/logger/logData.ts

@@ -1,81 +1,160 @@
-import fs from "fs";
+import fs from "fs/promises";
 import path from "path";
 import beautify from "js-beautify";
 
 const logsDir = path.resolve(process.cwd(), "logs");
 const domDir = path.resolve(process.cwd(), "dom");
+const screenshotsDir = path.resolve(process.cwd(), "screenshots");
+
+const now = new Date();
+
+const istDate = new Intl.DateTimeFormat("sv-SE", {
+    timeZone: "Asia/Kolkata",
+    year: "numeric",
+    month: "2-digit",
+    day: "2-digit",
+    hour: "2-digit",
+    minute: "2-digit",
+    second: "2-digit",
+    fractionalSecondDigits: 3,
+    hour12: false,
+})
+    .format(now)
+    .replace(" ", "T");
 
-if (!fs.existsSync(logsDir)) {
-    fs.mkdirSync(logsDir, { recursive: true });
-}
+const logFilePath = path.join(logsDir, "ai-execution-logs.txt");
 
-if (!fs.existsSync(domDir)) {
-    fs.mkdirSync(domDir, { recursive: true });
+async function ensureDirs() {
+    await fs.mkdir(logsDir, { recursive: true });
+    await fs.mkdir(domDir, { recursive: true });
+    await fs.mkdir(screenshotsDir, { recursive: true });
 }
 
-const logFilePath = path.join(logsDir, "ai-execution-logs.txt");
+ensureDirs();
 
 export const logger = {
-    info: (...args: any[]) => {
-        const formattedArgs = args.map(arg =>
-            typeof arg === "object"
-                ? JSON.stringify(arg, null, 2)
-                : arg
+    info: async (...args: any[]) => {
+        const formattedArgs = args.map((arg) =>
+            typeof arg === "object" ? JSON.stringify(arg, null, 2) : arg
         );
 
-        const message = `[INFO] ${new Date().toISOString()} - ${formattedArgs.join(" ")}\n`;
+        const message = `[INFO] ${istDate}+05:30 - ${formattedArgs.join(" ")}\n`;
 
-        fs.appendFileSync(logFilePath, message, "utf-8");
+        await fs.appendFile(logFilePath, message, "utf-8");
     },
 
-    error: (...args: any[]) => {
-        const formattedArgs = args.map(arg =>
-            typeof arg === "object"
-                ? JSON.stringify(arg, null, 2)
-                : arg
-        );
+    error: async (...args: any[]) => {
+        const formatted = args.map((arg) => {
+            if (arg instanceof Error) {
+                let output = arg.stack || `${arg.name}: ${arg.message}`;
 
-        const message = `[ERROR] ${new Date().toISOString()} - ${formattedArgs.join(" ")}\n`;
+                output = output.replace(
+                    /invalid session id:\s*invalid session id:/gi,
+                    "invalid session id:"
+                );
 
-        fs.appendFileSync(logFilePath, message, "utf-8");
-    },
+                if (arg.cause instanceof Error) {
+                    let causeStack = arg.cause.stack || "";
 
-    saveDOM: (domContent: string, fileName?: string) => {
+                    causeStack = causeStack.replace(
+                        /invalid session id:\s*invalid session id:/gi,
+                        "invalid session id:"
+                    );
 
-        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
-        const finalFileName = fileName
-            ? `${fileName}.html`
-            : `dom-${timestamp}.html`;
+                    output += `\nCaused by: ${causeStack}`;
+                }
 
-        const filePath = path.join(domDir, finalFileName);
+                return output;
+            }
 
-        const formattedHTML = beautify.html(domContent, {
-            indent_size: 2,
-            preserve_newlines: false,
-            wrap_line_length: 120
+            if (typeof arg === "object") {
+                return JSON.stringify(arg, null, 2);
+            }
+
+            return String(arg);
         });
 
-        fs.writeFileSync(filePath, formattedHTML, "utf-8");
+        const message = `[ERROR] ${new Date().toISOString()} - ${formatted.join(
+            " "
+        )}\n`;
+
+        await fs.appendFile(logFilePath, message, "utf-8");
     },
 
-    saveJSON: (data: unknown, fileName?: string) => {
-        const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    getError: (...args: any[]) => {
+        const formatted = args.map((arg) => {
+            if (arg instanceof Error) {
+                let output = arg.stack || `${arg.name}: ${arg.message}`;
 
-        const finalFileName = fileName
-            ? `${fileName}.json`
-            : `data-${timestamp}.json`;
+                output = output.replace(
+                    /invalid session id:\s*invalid session id:/gi,
+                    "invalid session id:"
+                );
 
-        const filePath = path.join(domDir, finalFileName);
+                if (arg.cause instanceof Error) {
+                    let causeStack = arg.cause.stack || "";
 
-        fs.writeFileSync(
-            filePath,
-            JSON.stringify(data, null, 2),
-            "utf-8"
-        );
+                    causeStack = causeStack.replace(
+                        /invalid session id:\s*invalid session id:/gi,
+                        "invalid session id:"
+                    );
+
+                    output += `\nCaused by: ${causeStack}`;
+                }
+
+                return output;
+            }
 
+            if (typeof arg === "object") {
+                return JSON.stringify(arg, null, 2);
+            }
 
-    }
+            return String(arg);
+        });
+
+        return formatted.join(" ");
+    },
+
+    saveDOM: async (domContent: string, fileName?: string) => {
+        const timestamp = new Date().toLocaleString().replace(/[:.]/g, "-");
+        let finalFileName: string;
+        let formattedContent: string;
+
+        if (fileName?.includes("Mob")) {
+            finalFileName = fileName ? `${fileName}.xml` : `dom-${timestamp}.xml`;
+            formattedContent = domContent;
+        } else {
+            finalFileName = fileName ? `${fileName}.html` : `dom-${timestamp}.html`;
+            formattedContent = beautify.html(domContent, {
+                indent_size: 2,
+                preserve_newlines: false,
+                wrap_line_length: 120,
+            });
+        }
+
+        const filePath = path.join(domDir, finalFileName);
+        await fs.writeFile(filePath, formattedContent, "utf-8");
+    },
+
+    saveJSON: async (data: unknown, fileName?: string) => {
+        const timestamp = new Date().toLocaleString().replace(/[:.]/g, "-");
+        const finalFileName = fileName ? `${fileName}.json` : `data-${timestamp}.json`;
+        const filePath = path.join(domDir, finalFileName);
 
+        const safeData = JSON.stringify(
+            data,
+            (key, value) => {
+                if (key === "element") return undefined; // remove DOM node
+                return value;
+            },
+            2
+        );
 
-};
+        await fs.writeFile(filePath, safeData, "utf-8");
+    },
 
+    saveImage: async (imageBuffer: Buffer, fileName: string) => {
+        const filePath = path.join(screenshotsDir, fileName);
+        await fs.writeFile(filePath, imageBuffer);
+    },
+};

package/src/utils/mobileXpath/findXpathFromIndex.ts

@@ -0,0 +1,87 @@
+import * as cheerio from "cheerio"
+import { IMobileXpathFinder } from "./findXpathFromIndexInterface.js";
+export function findXpathFromIndex(
+    args: IMobileXpathFinder
+): string {
+    const $ = cheerio.load(args.dom, { xmlMode: true });
+    const centers: [number, number][] = [];
+    const clickableBoxes: [number, number, number, number][] = [];
+
+    const traverse = (node: any) => {
+        if (node?.type === "tag") {
+            const attr = node.attribs;
+            if (attr?.bounds) {
+                const clickable = attr.clickable === "true";
+                const enabled = attr.enabled === "true";
+                const clazz = attr.class || "";
+
+                if (
+                    (clickable && enabled) ||
+                    clazz.toLowerCase().includes("edittext") ||
+                    clazz.toLowerCase().includes("autocompletetextview")
+                ) {
+                    const match = attr.bounds.match(/\d+/g);
+                    if (match && match.length === 4) {
+                        const x1 = parseInt(match[0]);
+                        const y1 = parseInt(match[1]);
+                        const x2 = parseInt(match[2]);
+                        const y2 = parseInt(match[3]);
+
+                        clickableBoxes.push([x1, y1, x2, y2]);
+                        centers.push([Math.floor((x1 + x2) / 2), Math.floor((y1 + y2) / 2)]);
+                    }
+                }
+            }
+        }
+
+        if (node.children) {
+            node.children.forEach(traverse);
+        }
+    };
+
+    traverse($.root()[0]);
+
+    const index = Number(args.fireflinkIndex.split("-").pop());
+
+    if (isNaN(index) || index < 1 || index > centers.length) {
+        throw new Error(`Index ${index} out of range (Total clickable items: ${centers.length})`);
+    }
+
+    const [x1, y1, x2, y2] = clickableBoxes[index - 1];
+    const boundsStr = `[${x1},${y1}][${x2},${y2}]`;
+
+    const element = $(`[bounds="${boundsStr}"]`).first();
+
+    if (!element.length) {
+        throw new Error(`Element with bounds ${boundsStr} not found`);
+    }
+
+    return getXPath(element);
+}
+
+function getXPath(element: cheerio.Cheerio<any>): string {
+
+    let xpath = ""
+    let current: any = element
+
+    while (current.length && current[0].tagName !== "hierarchy") {
+
+        const tag = current[0].tagName
+
+        const siblings = current.parent().children(tag)
+
+        if (siblings.length > 1) {
+
+            const index = siblings.index(current) + 1
+            xpath = `/${tag}[${index}]` + xpath
+
+        } else {
+
+            xpath = `/${tag}` + xpath
+        }
+
+        current = current.parent()
+    }
+
+    return `/hierarchy${xpath}`
+}

package/src/utils/mobileXpath/findXpathFromIndexInterface.ts

@@ -0,0 +1,4 @@
+export interface IMobileXpathFinder {
+    fireflinkIndex: string,
+    dom: string
+}

package/src/utils/newTab/newTabTracker.ts

@@ -0,0 +1,62 @@
+import { ScriptDataAppender } from "../../fireflinkData/fireflinkScript/scriptGenrationData.js";
+
+export async function newTabTracker(
+    driver: WebdriverIO.Browser,
+    stepAction: string,
+    previousHandles: string[],
+    scriptAppender: ScriptDataAppender
+): Promise<void> {
+
+    if (stepAction === "switch") return;
+
+    const currentHandle = await driver.getWindowHandle();
+    const currentHandles = await driver.getWindowHandles();
+
+    if (currentHandles.length <= previousHandles.length) {
+        return;
+    }
+
+    const newHandles = currentHandles.filter(
+        handle => !previousHandles.includes(handle)
+    );
+
+    for (const handle of newHandles) {
+        try {
+            await driver.switchToWindow(handle);
+
+            const url = await driver.getUrl();
+
+            // Skip unwanted system URLs
+            if (
+                !url ||
+                url.startsWith("devtools://") ||
+                url.startsWith("chrome://") ||
+                url === "about:blank"
+            ) {
+                await driver.switchToWindow(currentHandle);
+                continue;
+            }
+
+            // Valid URL → accept it
+            scriptAppender.add(async () => ({
+                nlpName: "SwitchToWindowBasedOnURL",
+                stepInputs: [url],
+                elementsData: []
+            }));
+
+            return;
+
+        } catch (error) {
+            // Restore original window before failing
+            await driver.switchToWindow(currentHandle);
+
+            throw new Error(
+                `Error switching to new window: ${error}`,
+                { cause: error }
+            );
+        }
+    }
+
+    // If no valid window found, ensure we are back on original
+    await driver.switchToWindow(currentHandle);
+}

package/src/automation/actions/interaction/click.ts

@@ -1,25 +0,0 @@
-import { ClickInterface } from "../interface/interactionActionInterface.js";
-export async function click(args: ClickInterface): Promise<void> {
-    try {
-        const element = await args.driver.$(args.selector);
-        await element.scrollIntoView({ block: 'center', inline: 'center' });
-        await element.click();
-        const ffElement: any = await args.elementGetter.getFireFlinkElement(
-            args.pageDOM,
-            `[ff-inspect="${args.fireflinkIndex}"]`
-        );
-        args.scriptDataAppender.add(
-            {
-                nlpName: 'Click',
-                stepInputs: [],
-                elementsData: [{
-                    elementName: args.elementName,
-                    type: args.elementType,
-                    locators: ffElement.locators,
-                    platform: args.platform
-                }]
-            });
-    } catch (error: any) {
-        throw new Error("Click action failed", { cause: error });
-    }
-}

package/src/automation/actions/interaction/enterInput.ts

@@ -1,36 +0,0 @@
-import { EnterInputInterface } from "../interface/interactionActionInterface.js";
-import { enterActionHelper } from "../../../utils/helpers/enterActionHelper.js";
-export async function enterInput(args: EnterInputInterface): Promise<void> {
-
-    try {
-
-        const { ffNumber, xpathPath } = enterActionHelper(
-            args.pageDOM,
-            args.fireflinkIndex
-        );
-
-        const element = await args.driver.$(xpathPath);
-        await element.scrollIntoView({ block: 'center', inline: 'center' });
-        await element.setValue(args.value);
-        const ffElement: any = await args.elementGetter.getFireFlinkElement(
-            args.pageDOM,
-            `[ff-inspect="Fire-Flink-${ffNumber}"]`
-        );
-        args.scriptDataAppender.add({
-            nlpName: 'SendKeys',
-            stepInputs: [args.value],
-            elementsData: [{
-                elementName: args.elementName,
-                type: args.elementType,
-                locators: ffElement.locators,
-                platform: args.platform
-            }]
-        });
-
-    }
-    catch (error: any) {
-        console.log(error);
-        throw new Error("enter action failed", { cause: error });
-    }
-
-}

package/src/automation/actions/navigation/getTitle.ts

@@ -1,9 +0,0 @@
-import { getTitleInterface } from "../interface/navigationActionInterface.js";
-export async function getTitle(args: getTitleInterface): Promise<void> {
-    try {
-        const title = await args.driver.getTitle();
-        args.scriptDataAppender.add({ nlpName: 'getTitle', elementsData: [], stepInputs: [title] });
-    } catch (error: any) {
-        throw new Error("Get title action failed", { cause: error });
-    }
-}

package/src/automation/actions/navigation/goBack.ts

@@ -1,9 +0,0 @@
-import { goBackInterface } from "../interface/navigationActionInterface.js";
-export async function goBack(args: goBackInterface): Promise<void> {
-    try {
-        await args.driver.back();
-        args.scriptDataAppender.add({ nlpName: 'goBack', elementsData: [], stepInputs: [] });
-    } catch (error: any) {
-        throw new Error("Navigate action failed", { cause: error });
-    }
-}

package/src/automation/actions/navigation/navigate.ts

@@ -1,10 +0,0 @@
-import { navigateToUrlInterface } from "../interface/navigationActionInterface.js";
-export async function navigate(args: navigateToUrlInterface): Promise<void> {
-    try {
-        await args.driver.url(args.url);
-        args.scriptDataAppender.add({ nlpName: 'NavigateToURL', elementsData: [], stepInputs: [args.url] });
-
-    } catch (error: any) {
-        throw new Error("Navigate action failed", { cause: error });
-    }
-}

package/src/automation/actions/navigation/refresh.ts

@@ -1,9 +0,0 @@
-import { refreshInterface } from "../interface/navigationActionInterface.js";
-export async function pageRefresh(args: refreshInterface): Promise<void> {
-    try {
-        await args.driver.refresh();
-        args.scriptDataAppender.add({ nlpName: 'refresh', elementsData: [], stepInputs: [] });
-    } catch (error: any) {
-        throw new Error("Refresh action failed", { cause: error });
-    }
-}

package/src/automation/actions/wait/wait.ts

@@ -1,10 +0,0 @@
-import { waitInterface } from "../interface/waitActionInterface.js";
-export async function wait(args: waitInterface): Promise<void> {
-    try {
-        await args.driver.pause(Number(args.time) * 1000);
-        args.scriptDataAppender.add({ nlpName: 'Sleep', elementsData: [], stepInputs: [args.time] });
-
-    } catch (error: any) {
-        throw new Error("Navigate action failed", { cause: error });
-    }
-}