ff-automationv2 2.1.3 → 2.2.0

package/src/core/main/stepProcessor.ts

@@ -2,8 +2,10 @@ import { llmAction } from "../../ai/llmcalls/llmAction.js";
 import { LLMResultParser } from "../../ai/llmcalls/parseLlmOputput.js";
 import { FireFlinkLLMResponse } from "../types/llmResponseType.js"
 import { PromptType } from "../types/promptType.js";
+import { platform } from "../types/scriptTypesType.js"
 
 export interface LLMRequest {
+    platform: platform;
     type: PromptType;
     args: Record<string, any>;
     input: Record<string, any>;
@@ -21,8 +23,8 @@ export class StepProcessor {
     }
 
 
-    async getLLMResponse({ type, args, input }: LLMRequest): Promise<{ response: FireFlinkLLMResponse }> {
-        const response = await this.llm.getLLMResponse(type, args, input);
+    async getLLMResponse({ platform, type, args, input }: LLMRequest): Promise<{ response: FireFlinkLLMResponse }> {
+        const response = await this.llm.getLLMResponse(platform, type, args, input);
         return this.resultParser.fetchResult(response);
     }
 }

package/src/core/types/llmResponseType.ts

@@ -1,11 +1,15 @@
-import { IKeywordAction, IDragDropAction, IInputAction, IScrollOrAttributeAction, IManualSteps, IFireFlinkIndex } from "../interfaces/llmResponseInterface.js";
+import { IKeywordAction, IDragDropAction, IAlertAction, IScrollOrAttributeAction, IManualSteps, IVisionAction , IWaitAction, IClickAction, IVerifyAction, IGetAction} from "../interfaces/llmResponseInterface.js";
 
 export type FireFlinkLLMResponse =
 
     | IKeywordAction
-    | IInputAction
+    | IAlertAction
     | IDragDropAction
     | IScrollOrAttributeAction
     | IManualSteps
-    | IFireFlinkIndex
+    | IVisionAction
+    | IWaitAction
+    | IClickAction
+    | IVerifyAction
+    | IGetAction
     | any

package/src/core/types/promptMap.ts

@@ -1,7 +1,10 @@
 import { PromptArgsMap } from "../interfaces/promptInterface.js";
+import { platform } from "../types/scriptTypesType.js"
 
 export type PromptMap = {
-    [K in keyof PromptArgsMap]: (
-        args: PromptArgsMap[K]
-    ) => Promise<string>;
-};
+    [P in platform]: {
+        [K in keyof PromptArgsMap]: (
+            args: PromptArgsMap[K]
+        ) => Promise<string>;
+    };
+};

package/src/core/types/promptType.ts

@@ -4,4 +4,10 @@ export enum PromptType {
     ERROR_DESCRIPTION = "errorDescriptionPrompt",
     FF_INSPECTOR = "ffInspectorNumExtractor",
     VISION_PROMPT = "visionPrompt",
+    VERIFY_PROMPT = "verifyActionExtractorPrompt",
+    GET_PROMPT = "getActionExtractorPrompt",
+    WAIT_ACTION_EXTRACTOR_PROMPT = "waitActionExtractorPrompt",
+    COMBINED_ACTION_EXTRACTOR_PROMPT = "combinedActionExtractorPrompt"
 }
+
+

package/src/core/types/scriptTypesType.ts

@@ -0,0 +1 @@
+export type platform = "web" | "android";

package/src/domAnalysis/getRelaventElements.ts

@@ -16,7 +16,7 @@ export class DomProcessingEngine {
     private searcher = new DomSearcher();
     private relationshipBuilder = new DomRelationshipBuilder();
 
-    public process(request: DomProcessingRequest) {
+    public webProcess(request: DomProcessingRequest) {
 
         const flat = this.simplifier.simplify(request.rawDom);
         logger.saveJSON(flat, `flat-dom-${request.stepCount}`);
@@ -25,4 +25,13 @@ export class DomProcessingEngine {
 
         return structured;
     }
+    public mobileProcess(request: DomProcessingRequest) {
+
+        const  { flatNodes, parentMap }  = this.simplifier.simplifyMobile(request.rawDom);
+        logger.saveJSON(flatNodes, `flat-dom-Mob-${request.stepCount}`);
+        const { results, indexMap } = this.searcher.searchBestMobile(flatNodes, request.keywords);
+        const grouped = this.relationshipBuilder.buildGroupedResultsMobile(results,flatNodes,indexMap,parentMap);
+
+        return grouped;
+    }
 }

package/src/domAnalysis/relativeElementsFromDom.ts

@@ -91,4 +91,209 @@ export class DomRelationshipBuilder {
 
     return structured;
   }
+
+  public buildGroupedResultsMobile(
+    results: Record<string, any[]>,
+    nodes: any[],
+    indexMap: Record<string, number>,
+    parentMap: Map<Element, Element | null>
+) {
+
+    const groupedResults: any[][] = [];
+    const usedNodes = new Set<string>();
+
+    const slim = (n: Record<string, any>) => {
+        const keys = [
+            "tag",
+            "FF-inspecter",
+            "text",
+            "content-desc",
+            "resource-id",
+            "class",
+            "pane-title",
+            "hint",
+            "clickable",
+            "editable",
+            "enabled"
+        ];
+
+        const out: Record<string, any> = {};
+        for (const k of keys) {
+            if (k in n) out[k] = n[k];
+        }
+        return out;
+    };
+
+    const addNodeSafe = (
+        nodeDict: Record<string, any>,
+        group: any[],
+        localUsed: Set<string>
+    ) => {
+
+        const ff = nodeDict["FF-inspecter"];
+
+        if (!ff || localUsed.has(ff)) return;
+
+        group.push(slim(nodeDict));
+        localUsed.add(ff);
+    };
+
+    for (const matchedNodes of Object.values(results)) {
+
+        for (const matchedNode of matchedNodes) {
+
+            const selectedFF = matchedNode["FF-inspecter"];
+
+            if (!(selectedFF in indexMap)) continue;
+            if (usedNodes.has(selectedFF)) continue;
+
+            const item = nodes[indexMap[selectedFF]];
+            const element = item.element;
+            const parent = parentMap.get(element) || null;
+
+            const group: any[] = [];
+            const localUsed = new Set<string>();
+
+            addNodeSafe(matchedNode, group, localUsed);
+
+            if (matchedNode["clickable"] === "true") {
+                groupedResults.push(group);
+                localUsed.forEach(ff => usedNodes.add(ff));
+                continue;
+            }
+
+            if (!parent) {
+                groupedResults.push(group);
+                localUsed.forEach(ff => usedNodes.add(ff));
+                continue;
+            }
+
+            // parent check
+            if (
+                parent.getAttribute("clickable") === "true" &&
+                parent.getAttribute("enabled") !== "false"
+            ) {
+                addNodeSafe(
+                    Object.fromEntries(Array.from(parent?.attributes ?? []).map(a => [a.name, a.value])),
+                    group,
+                    localUsed
+                );
+            }
+
+            const siblings = Array.from(parent?.children ?? []);
+            const idx = siblings.indexOf(element);
+
+            // upper siblings
+            if (idx > 0) {
+                for (let j = idx - 1; j >= 0; j--) {
+
+                    const upper = siblings[j];
+
+                    if (
+                        upper.getAttribute("clickable") === "true" &&
+                        upper.getAttribute("enabled") !== "false"
+                    ) {
+                        addNodeSafe(
+                            Object.fromEntries(Array.from(upper?.attributes ?? []).map(a => [a.name, a.value])),
+                            group,
+                            localUsed
+                        );
+                        break;
+                    }
+
+                    const upperParent = parentMap.get(upper);
+
+                    if (
+                        upperParent &&
+                        upperParent.getAttribute("clickable") === "true" &&
+                        upperParent.getAttribute("enabled") !== "false"
+                    ) {
+                        addNodeSafe(
+                            Object.fromEntries(Array.from(upperParent?.attributes ?? []).map(a => [a.name, a.value])),
+                            group,
+                            localUsed
+                        );
+                        break;
+                    }
+                }
+            }
+
+            // lower siblings
+            if (idx !== -1) {
+                for (let j = idx + 1; j < siblings.length; j++) {
+
+                    const lower = siblings[j];
+
+                    if (
+                        lower.getAttribute("clickable") === "true" &&
+                        lower.getAttribute("enabled") !== "false"
+                    ) {
+                        addNodeSafe(
+                            Object.fromEntries(Array.from(lower?.attributes ?? []).map(a => [a.name, a.value])),
+                            group,
+                            localUsed
+                        );
+                        break;
+                    }
+
+                    for (const child of Array.from(lower?.children ?? [])) {
+                        if (
+                            child.getAttribute("clickable") === "true" &&
+                            child.getAttribute("enabled") !== "false"
+                        ) {
+                            addNodeSafe(
+                                Object.fromEntries(Array.from(child?.attributes ?? []).map(a => [a.name, a.value])),
+                                group,
+                                localUsed
+                            );
+                            break;
+                        }
+                    }
+                }
+            }
+
+            groupedResults.push(group);
+            localUsed.forEach(ff => usedNodes.add(ff));
+        }
+    }
+
+    const firstMultiIndex: Record<string, number> = {};
+
+    groupedResults.forEach((group, i) => {
+        if (group.length > 1) {
+            group.forEach(node => {
+                const ff = node["FF-inspecter"];
+                if (!(ff in firstMultiIndex)) {
+                    firstMultiIndex[ff] = i;
+                }
+            });
+        }
+    });
+
+    const finalResults: any[][] = [];
+    const usedFF = new Set<string>();
+
+    groupedResults.forEach((group, i) => {
+
+        const newGroup: any[] = [];
+
+        group.forEach(node => {
+
+            const ff = node["FF-inspecter"];
+
+            if (usedFF.has(ff)) return;
+
+            if (group.length === 1 && ff in firstMultiIndex) return;
+
+            if (ff in firstMultiIndex && firstMultiIndex[ff] !== i) return;
+
+            newGroup.push(node);
+            usedFF.add(ff);
+        });
+
+        if (newGroup.length) finalResults.push(newGroup);
+    });
+
+    return finalResults;
+}  
 }

package/src/domAnalysis/searchBest.ts

@@ -1,6 +1,5 @@
-import * as fuzz from "fuzzball";
-
-import { FlatNode } from "./simplifyAndFlatten.js";
+import { fuzzy } from "fast-fuzzy";
+import { FlatNode } from "../core/interfaces/simplifyAndFlattenInterface";
 
 export class DomSearcher {
 
@@ -8,7 +7,7 @@ export class DomSearcher {
         nodes: FlatNode[],
         keywords: string[] | string,
         topN = 20,
-        threshold = 60
+        threshold = 40
     ) {
 
         if (typeof keywords === "string") {
@@ -99,9 +98,10 @@ export class DomSearcher {
                 }
 
                 if (!matched) {
-                    const score = fuzz.token_set_ratio(searchText, kw);
-                    if (score >= threshold) {
-                        fuzzyList.push([score, i]);
+                    const score = fuzzy(kw, searchText);
+
+                    if (score >= threshold / 100) {
+                        fuzzyList.push([score * 100, i]);
                     }
                 }
 
@@ -144,17 +144,15 @@ export class DomSearcher {
         }
 
         const seen = new Set<string>();
-        const unique: any[] = [];
-
-        Object.values(results).forEach(list => {
-            list.forEach(n => {
-                const key = n["FF-inspecter"];
-                if (key && !seen.has(key)) {
-                    seen.add(key);
-                    unique.push(n);
-                }
+
+        const unique = Object.values(results)
+            .flat()
+            .filter(node => {
+                const ff = node["FF-inspecter"];
+                if (!ff || seen.has(ff)) return false;
+                seen.add(ff);
+                return true;
             });
-        });
 
         unique.sort((a, b) =>
             ffNumber(a["FF-inspecter"]) - ffNumber(b["FF-inspecter"])
@@ -162,4 +160,119 @@ export class DomSearcher {
 
         return unique;
     }
+
+    public searchBestMobile(
+        nodes: FlatNode[],
+        keywords: string[] | string,
+        topN = 5,
+        threshold = 60
+    ) {
+
+        if (typeof keywords === "string") {
+            keywords = keywords.replace(/[\[\]]/g, "").split(",").map(k => k.trim());
+        }
+
+        keywords = keywords.map(k => k.toLowerCase());
+
+        const slim = (n: Record<string, any>) => {
+            const allowed = [
+                "tag",
+                "FF-inspecter",
+                "text",
+                "content-desc",
+                "resource-id",
+                "class",
+                "pane-title",
+                "hint",
+                "clickable",
+                "editable",
+                "enabled"
+            ];
+
+            const result: Record<string, any> = {};
+
+            for (const key of allowed) {
+                if (key in n) result[key] = n[key];
+            }
+
+            return result;
+        };
+
+        const indexMap: Record<string, number> = {};
+
+        nodes.forEach((item, i) => {
+            const ff = item.node["FF-inspecter"];
+            if (ff) indexMap[ff] = i;
+        });
+
+        const results: Record<string, any[]> = {};
+
+        for (const kw of keywords) {
+
+            const exact: Record<string, any>[] = [];
+            const strong: Record<string, any>[] = [];
+            const fuzzyMatches: [number, Record<string, any>][] = [];
+
+            for (const item of nodes) {
+
+                const node = item.node;
+
+                const text = (item.search || "").toLowerCase();
+                const t = (node["text"] || "").toLowerCase();
+                const cd = (node["content-desc"] || "").toLowerCase();
+                const rid = (node["resource-id"] || "").toLowerCase();
+                const hint = (node["hint"] || "").toLowerCase();
+
+                // EXACT MATCH
+                if (
+                    text === kw ||
+                    t === kw ||
+                    cd === kw ||
+                    rid === kw ||
+                    hint === kw
+                ) {
+                    exact.push(node);
+                    continue;
+                }
+
+                // STRONG PARTIAL MATCH
+                if (
+                    text.includes(kw) ||
+                    t.includes(kw) ||
+                    cd.includes(kw) ||
+                    rid.includes(kw) ||
+                    hint.includes(kw)
+                ) {
+                    strong.push(node);
+                    continue;
+                }
+
+                // FUZZY MATCH
+                const score = fuzzy(kw, text) * 100;
+
+                if (score >= threshold) {
+                    fuzzyMatches.push([score, node]);
+                }
+            }
+
+            let selected: Record<string, any>[];
+
+            if (exact.length) {
+                selected = exact.slice(0, topN);
+            } else if (strong.length) {
+                selected = strong.slice(0, topN);
+            } else {
+                fuzzyMatches.sort((a, b) => b[0] - a[0]);
+                selected = fuzzyMatches.slice(0, topN).map(f => f[1]);
+            }
+
+            results[kw] = selected.map(slim);
+        }
+
+        return {
+            results,
+            indexMap
+        };
+    }
+
 }

package/src/domAnalysis/simplifyAndFlatten.ts

@@ -1,16 +1,12 @@
 import * as cheerio from "cheerio";
 import { Element } from "domhandler";
-
-export interface FlatNode {
-    node: Record<string, any>;
-    search: string;
-}
-
+import { DOMParser } from "@xmldom/xmldom";
+import { FlatNode, FlatNodeMob, SimplifiedDomResult } from "../core/interfaces/simplifyAndFlattenInterface.js"
 export class DomSimplifier {
 
     public simplify(rawDom: string): FlatNode[] {
 
-        const $ = cheerio.load(rawDom);
+        const $ = cheerio.load(rawDom, { xml: true });
         const flatNodes: FlatNode[] = [];
 
         $("*").each((_, node) => {
@@ -21,13 +17,12 @@ export class DomSimplifier {
             const tag = el.tagName.toLowerCase();
             if (!tag || tag === "meta") return;
 
-            const attribs = el.attribs ?? {};   // ✅ declared only once
+            const attribs = el.attribs ?? {};
 
             if (tag !== "select" && tag !== "option") {
                 if (
                     (attribs["element-visibility"] === "hidden" ||
-                        attribs["element-covered"] === "true") &&
-                    attribs["element-covered-by-tag"] !== "iframe"
+                        attribs["element-covered"] === "true")
                 ) {
                     return;
                 }
@@ -115,4 +110,138 @@ export class DomSimplifier {
 
         return flatNodes;
     }
+
+
+
+    public simplifyMobile(rawDom: string): SimplifiedDomResult {
+        const parser = new DOMParser();
+        const xmlDoc = parser.parseFromString(rawDom, "text/xml");
+
+        const root = xmlDoc.documentElement;
+
+        const flatNodes: FlatNodeMob[] = [];
+
+        // parentMap: child → parent
+        const parentMap = new Map<globalThis.Element, globalThis.Element>();
+
+        function xmlToTree(elem: globalThis.Element): any {
+
+            const children: any[] = [];
+
+            for (let i = 0; i < elem.childNodes.length; i++) {
+
+                const child = elem.childNodes[i];
+
+                if (child.nodeType === 1) {
+
+                    const childElement = child as globalThis.Element;
+
+                    // store parent relationship
+                    parentMap.set(childElement, elem);
+
+                    children.push(xmlToTree(childElement));
+                }
+            }
+
+            const attributes: Record<string, string> = {};
+
+            for (let i = 0; i < elem.attributes.length; i++) {
+                const attr = elem.attributes[i];
+                attributes[attr.name] = attr.value;
+            }
+
+            return {
+                tag: elem.tagName,
+                attributes,
+                text: (elem.textContent || "").trim(),
+                _element: elem,
+                children
+            };
+        }
+
+        const xmlTree = xmlToTree(root);
+
+        function simplifyAndFlatten(node: any): any {
+
+            const simplified: Record<string, any> = {};
+            const attrs = node.attributes || {};
+
+            simplified["tag"] = node.tag;
+
+            const skipNode = attrs["enabled"] === "false";
+
+            if (attrs["FF-inspecter"]) {
+                simplified["FF-inspecter"] = attrs["FF-inspecter"];
+            }
+
+            simplified["text"] = node.text || attrs["text"] || "";
+
+            const importantAttrs = [
+                "resource-id",
+                "content-desc",
+                "pane-title",
+                "clickable",
+                "editable",
+                "enabled",
+                "class",
+                "hint"
+            ];
+
+            for (const attr of importantAttrs) {
+                if (attrs[attr]) {
+                    simplified[attr] = attrs[attr];
+                }
+            }
+
+            const searchParts: string[] = [];
+
+            if (simplified.text && simplified.text.trim()) {
+                searchParts.push(simplified.text);
+            }
+
+            for (const key of [
+                "content-desc",
+                "pane-title",
+                "resource-id",
+                "hint"
+            ]) {
+                const val = simplified[key];
+                if (val && typeof val === "string" && val.trim()) {
+                    searchParts.push(val);
+                }
+            }
+
+            simplified["search"] = searchParts.join(" ").trim().toLowerCase();
+
+            if (!skipNode) {
+                flatNodes.push({
+                    node: simplified,
+                    search: simplified["search"],
+                    element: node._element
+                });
+            }
+
+            const children: any[] = [];
+
+            for (const child of node.children || []) {
+                const simplifiedChild = simplifyAndFlatten(child);
+
+                if (simplifiedChild && simplifiedChild.enabled !== "false") {
+                    children.push(simplifiedChild);
+                }
+            }
+
+            if (children.length) {
+                simplified["children"] = children;
+            }
+
+            return simplified;
+        }
+
+        simplifyAndFlatten(xmlTree);
+
+        return {
+            flatNodes, parentMap
+        };
+    }
 }

package/src/fireflinkData/fireflinkLocators/getListOfLocators.ts

@@ -4,7 +4,7 @@ export class ElementGetter {
     async getFireFlinkElement(html: string, attribute: string): Promise<any> {
         const formattedAttribute = attribute.replace('[', '//*[@');
         try {
-            const result = await getElementsFromHTML(
+            const result = getElementsFromHTML(
                 [
                     {
                         name: "xpath",
@@ -17,7 +17,6 @@ export class ElementGetter {
                 ],
                 html
             );
-
             return Array.isArray(result) && result.length > 0 ? JSON.parse(JSON.stringify(result[0], null, 4)) : null;
         }
         catch (error) {