ff-automationv2 2.1.3 → 2.2.2

package/dist/core/types/promptType.d.ts

@@ -0,0 +1,11 @@
+export declare enum PromptType {
+    USER_STORY_TO_LIST = "userStoryToList",
+    KEYWORD_EXTRACTOR = "keywordExtractorPrompt",
+    ERROR_DESCRIPTION = "errorDescriptionPrompt",
+    FF_INSPECTOR = "ffInspectorNumExtractor",
+    VISION_PROMPT = "visionPrompt",
+    VERIFY_PROMPT = "verifyActionExtractorPrompt",
+    GET_PROMPT = "getActionExtractorPrompt",
+    WAIT_ACTION_EXTRACTOR_PROMPT = "waitActionExtractorPrompt",
+    COMBINED_ACTION_EXTRACTOR_PROMPT = "combinedActionExtractorPrompt"
+}

package/dist/core/types/promptType.js

@@ -0,0 +1,12 @@
+export var PromptType;
+(function (PromptType) {
+    PromptType["USER_STORY_TO_LIST"] = "userStoryToList";
+    PromptType["KEYWORD_EXTRACTOR"] = "keywordExtractorPrompt";
+    PromptType["ERROR_DESCRIPTION"] = "errorDescriptionPrompt";
+    PromptType["FF_INSPECTOR"] = "ffInspectorNumExtractor";
+    PromptType["VISION_PROMPT"] = "visionPrompt";
+    PromptType["VERIFY_PROMPT"] = "verifyActionExtractorPrompt";
+    PromptType["GET_PROMPT"] = "getActionExtractorPrompt";
+    PromptType["WAIT_ACTION_EXTRACTOR_PROMPT"] = "waitActionExtractorPrompt";
+    PromptType["COMBINED_ACTION_EXTRACTOR_PROMPT"] = "combinedActionExtractorPrompt";
+})(PromptType || (PromptType = {}));

package/dist/core/types/scriptTypesType.d.ts

@@ -0,0 +1 @@
+export type platform = "web" | "android";

package/dist/core/types/scriptTypesType.js

@@ -0,0 +1 @@
+export {};

package/dist/core/types/visionllmInputType.d.ts

@@ -0,0 +1,9 @@
+export type visionPromptMessage = ({
+    type: "text";
+    text: string;
+} | {
+    type: "image_url";
+    image_url: {
+        url: string;
+    };
+})[];

package/dist/core/types/visionllmInputType.js

@@ -0,0 +1 @@
+export {};

package/dist/domAnalysis/getRelaventElements.d.ts

@@ -0,0 +1,12 @@
+export interface DomProcessingRequest {
+    keywords: string[] | string;
+    rawDom: string;
+    stepCount: number;
+}
+export declare class DomProcessingEngine {
+    private simplifier;
+    private searcher;
+    private relationshipBuilder;
+    webProcess(request: DomProcessingRequest): any[];
+    mobileProcess(request: DomProcessingRequest): any[][];
+}

package/dist/domAnalysis/getRelaventElements.js

@@ -0,0 +1,25 @@
+import { DomSimplifier } from "./simplifyAndFlatten.js";
+import { DomSearcher } from "./searchBest.js";
+import { DomRelationshipBuilder } from "./relativeElementsFromDom.js";
+import { logger } from "../utils/logger/logData.js";
+export class DomProcessingEngine {
+    constructor() {
+        this.simplifier = new DomSimplifier();
+        this.searcher = new DomSearcher();
+        this.relationshipBuilder = new DomRelationshipBuilder();
+    }
+    webProcess(request) {
+        const flat = this.simplifier.simplify(request.rawDom);
+        logger.saveJSON(flat, `flat-dom-${request.stepCount}`);
+        const searched = this.searcher.search(flat, request.keywords);
+        const structured = this.relationshipBuilder.build(searched);
+        return structured;
+    }
+    mobileProcess(request) {
+        const { flatNodes, parentMap } = this.simplifier.simplifyMobile(request.rawDom);
+        logger.saveJSON(flatNodes, `flat-dom-Mob-${request.stepCount}`);
+        const { results, indexMap } = this.searcher.searchBestMobile(flatNodes, request.keywords);
+        const grouped = this.relationshipBuilder.buildGroupedResultsMobile(results, flatNodes, indexMap, parentMap);
+        return grouped;
+    }
+}

package/dist/domAnalysis/relativeElementsFromDom.d.ts

@@ -0,0 +1,4 @@
+export declare class DomRelationshipBuilder {
+    build(nodes: any[]): any[];
+    buildGroupedResultsMobile(results: Record<string, any[]>, nodes: any[], indexMap: Record<string, number>, parentMap: Map<Element, Element | null>): any[][];
+}

package/dist/domAnalysis/relativeElementsFromDom.js

@@ -0,0 +1,193 @@
+export class DomRelationshipBuilder {
+    build(nodes) {
+        const ffToXPath = new Map();
+        nodes.forEach(n => {
+            if (n["FF-inspecter"] && n["ff-xpath"]) {
+                ffToXPath.set(n["FF-inspecter"], n["ff-xpath"]);
+            }
+        });
+        const parentXPath = (x) => x.includes("/") ? x.substring(0, x.lastIndexOf("/")) : undefined;
+        const ffToNode = new Map();
+        nodes.forEach(n => {
+            ffToNode.set(n["FF-inspecter"], n);
+        });
+        const structured = [];
+        for (const node of nodes) {
+            const ff = node["FF-inspecter"];
+            const xp = node["ff-xpath"];
+            let parent;
+            const children = [];
+            const siblings = [];
+            if (ff && xp) {
+                const px = parentXPath(xp);
+                for (const [otherFF, otherXP] of ffToXPath.entries()) {
+                    if (otherFF === ff)
+                        continue;
+                    if (otherXP.startsWith(xp + "/")) {
+                        children.push(otherFF);
+                    }
+                    else if (px && parentXPath(otherXP) === px) {
+                        siblings.push(otherFF);
+                    }
+                }
+                for (const [otherFF, otherXP] of ffToXPath.entries()) {
+                    if (otherXP === px) {
+                        parent = otherFF;
+                        break;
+                    }
+                }
+            }
+            const slim = Object.fromEntries(Object.entries(node).filter(([k, v]) => v && !["search", "ff-xpath", "href", "src"].includes(k)));
+            const rel = {};
+            const mini = (n) => Object.fromEntries(Object.entries(n).filter(([k, v]) => v && !["search", "ff-xpath", "href", "src"].includes(k)));
+            if (parent && ffToNode.has(parent)) {
+                rel.parent = mini(ffToNode.get(parent));
+            }
+            if (children.length) {
+                rel.children = children.slice(0, 2)
+                    .map(c => mini(ffToNode.get(c)));
+            }
+            if (siblings.length) {
+                rel.siblings = siblings.slice(0, 2)
+                    .map(s => mini(ffToNode.get(s)));
+            }
+            if (Object.keys(rel).length) {
+                slim["relationship"] = rel;
+            }
+            structured.push(slim);
+        }
+        return structured;
+    }
+    buildGroupedResultsMobile(results, nodes, indexMap, parentMap) {
+        const groupedResults = [];
+        const usedNodes = new Set();
+        const slim = (n) => {
+            const keys = [
+                "tag",
+                "FF-inspecter",
+                "text",
+                "content-desc",
+                "resource-id",
+                "class",
+                "pane-title",
+                "hint",
+                "clickable",
+                "editable",
+                "enabled"
+            ];
+            const out = {};
+            for (const k of keys) {
+                if (k in n)
+                    out[k] = n[k];
+            }
+            return out;
+        };
+        const addNodeSafe = (nodeDict, group, localUsed) => {
+            const ff = nodeDict["FF-inspecter"];
+            if (!ff || localUsed.has(ff))
+                return;
+            group.push(slim(nodeDict));
+            localUsed.add(ff);
+        };
+        for (const matchedNodes of Object.values(results)) {
+            for (const matchedNode of matchedNodes) {
+                const selectedFF = matchedNode["FF-inspecter"];
+                if (!(selectedFF in indexMap))
+                    continue;
+                if (usedNodes.has(selectedFF))
+                    continue;
+                const item = nodes[indexMap[selectedFF]];
+                const element = item.element;
+                const parent = parentMap.get(element) || null;
+                const group = [];
+                const localUsed = new Set();
+                addNodeSafe(matchedNode, group, localUsed);
+                if (matchedNode["clickable"] === "true") {
+                    groupedResults.push(group);
+                    localUsed.forEach(ff => usedNodes.add(ff));
+                    continue;
+                }
+                if (!parent) {
+                    groupedResults.push(group);
+                    localUsed.forEach(ff => usedNodes.add(ff));
+                    continue;
+                }
+                // parent check
+                if (parent.getAttribute("clickable") === "true" &&
+                    parent.getAttribute("enabled") !== "false") {
+                    addNodeSafe(Object.fromEntries(Array.from(parent?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
+                }
+                const siblings = Array.from(parent?.children ?? []);
+                const idx = siblings.indexOf(element);
+                // upper siblings
+                if (idx > 0) {
+                    for (let j = idx - 1; j >= 0; j--) {
+                        const upper = siblings[j];
+                        if (upper.getAttribute("clickable") === "true" &&
+                            upper.getAttribute("enabled") !== "false") {
+                            addNodeSafe(Object.fromEntries(Array.from(upper?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
+                            break;
+                        }
+                        const upperParent = parentMap.get(upper);
+                        if (upperParent &&
+                            upperParent.getAttribute("clickable") === "true" &&
+                            upperParent.getAttribute("enabled") !== "false") {
+                            addNodeSafe(Object.fromEntries(Array.from(upperParent?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
+                            break;
+                        }
+                    }
+                }
+                // lower siblings
+                if (idx !== -1) {
+                    for (let j = idx + 1; j < siblings.length; j++) {
+                        const lower = siblings[j];
+                        if (lower.getAttribute("clickable") === "true" &&
+                            lower.getAttribute("enabled") !== "false") {
+                            addNodeSafe(Object.fromEntries(Array.from(lower?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
+                            break;
+                        }
+                        for (const child of Array.from(lower?.children ?? [])) {
+                            if (child.getAttribute("clickable") === "true" &&
+                                child.getAttribute("enabled") !== "false") {
+                                addNodeSafe(Object.fromEntries(Array.from(child?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
+                                break;
+                            }
+                        }
+                    }
+                }
+                groupedResults.push(group);
+                localUsed.forEach(ff => usedNodes.add(ff));
+            }
+        }
+        const firstMultiIndex = {};
+        groupedResults.forEach((group, i) => {
+            if (group.length > 1) {
+                group.forEach(node => {
+                    const ff = node["FF-inspecter"];
+                    if (!(ff in firstMultiIndex)) {
+                        firstMultiIndex[ff] = i;
+                    }
+                });
+            }
+        });
+        const finalResults = [];
+        const usedFF = new Set();
+        groupedResults.forEach((group, i) => {
+            const newGroup = [];
+            group.forEach(node => {
+                const ff = node["FF-inspecter"];
+                if (usedFF.has(ff))
+                    return;
+                if (group.length === 1 && ff in firstMultiIndex)
+                    return;
+                if (ff in firstMultiIndex && firstMultiIndex[ff] !== i)
+                    return;
+                newGroup.push(node);
+                usedFF.add(ff);
+            });
+            if (newGroup.length)
+                finalResults.push(newGroup);
+        });
+        return finalResults;
+    }
+}

package/dist/domAnalysis/searchBest.d.ts

@@ -0,0 +1,8 @@
+import { FlatNode } from "../core/interfaces/simplifyAndFlattenInterface";
+export declare class DomSearcher {
+    search(nodes: FlatNode[], keywords: string[] | string, topN?: number, threshold?: number): any[];
+    searchBestMobile(nodes: FlatNode[], keywords: string[] | string, topN?: number, threshold?: number): {
+        results: Record<string, any[]>;
+        indexMap: Record<string, number>;
+    };
+}

package/dist/domAnalysis/searchBest.js

@@ -0,0 +1,208 @@
+import { fuzzy } from "fast-fuzzy";
+export class DomSearcher {
+    search(nodes, keywords, topN = 20, threshold = 40) {
+        if (typeof keywords === "string") {
+            keywords = keywords.replace(/[\[\]]/g, "");
+            keywords = keywords.split(",").map(k => k.trim());
+        }
+        keywords = keywords.map(k => k.replace(/[^a-zA-Z0-9\s]/g, " ").trim().toLowerCase());
+        const ffNumber = (val) => {
+            if (!val)
+                return 0;
+            const m = val.match(/(\d+)$/);
+            return m ? parseInt(m[1]) : 0;
+        };
+        const isChild = (p, c) => c.startsWith(p + "/");
+        const isSibling = (x1, x2) => x1.substring(0, x1.lastIndexOf("/")) ===
+            x2.substring(0, x2.lastIndexOf("/"));
+        const addWithContext = (idx, collected) => {
+            const total = nodes.length;
+            for (const i of [idx - 1, idx, idx + 1]) {
+                if (i >= 0 && i < total) {
+                    if (nodes[i].node["FF-inspecter"]) {
+                        collected.push(i);
+                    }
+                }
+            }
+            const current = nodes[idx].node;
+            const cx = current["ff-xpath"];
+            if (!cx)
+                return;
+            for (let j = 0; j < total; j++) {
+                const nx = nodes[j].node["ff-xpath"];
+                if (nx && isChild(cx, nx)) {
+                    if (nodes[j].node["FF-inspecter"]) {
+                        collected.push(j);
+                    }
+                    break;
+                }
+            }
+            for (let j = 0; j < total; j++) {
+                const nx = nodes[j].node["ff-xpath"];
+                if (nx && isSibling(cx, nx) && j !== idx) {
+                    if (nodes[j].node["FF-inspecter"]) {
+                        collected.push(j);
+                    }
+                    break;
+                }
+            }
+        };
+        const results = {};
+        for (const kw of keywords) {
+            const exact = [];
+            const strong = [];
+            const fuzzyList = [];
+            const tagMatches = [];
+            for (let i = 0; i < nodes.length; i++) {
+                const node = nodes[i].node;
+                const searchText = (node.search || "").toLowerCase();
+                const tagName = (node.tag || "").toLowerCase();
+                let matched = false;
+                if (searchText == kw) {
+                    exact.push(i);
+                    matched = true;
+                }
+                if (searchText.includes(kw)) {
+                    strong.push(i);
+                    matched = true;
+                }
+                if (!matched) {
+                    const score = fuzzy(kw, searchText);
+                    if (score >= threshold / 100) {
+                        fuzzyList.push([score * 100, i]);
+                    }
+                }
+                if (kw == tagName || tagName.includes(kw)) {
+                    tagMatches.push(i);
+                }
+            }
+            const collected = [];
+            for (const idx of exact) {
+                addWithContext(idx, collected);
+                if (collected.length >= topN * 4)
+                    break;
+            }
+            for (const idx of strong) {
+                addWithContext(idx, collected);
+                if (collected.length >= topN * 4)
+                    break;
+            }
+            if (collected.length < topN * 4) {
+                fuzzyList
+                    .sort((a, b) => b[0] - a[0])
+                    .forEach(([_, idx]) => {
+                    if (collected.length < topN * 4) {
+                        addWithContext(idx, collected);
+                    }
+                });
+            }
+            if (collected.length < topN * 4) {
+                for (const idx of tagMatches) {
+                    addWithContext(idx, collected);
+                    if (collected.length >= topN * 4)
+                        break;
+                }
+            }
+            const uniqueIdx = Array.from(new Set(collected));
+            results[kw] = uniqueIdx.map(i => nodes[i].node);
+        }
+        const seen = new Set();
+        const unique = Object.values(results)
+            .flat()
+            .filter(node => {
+            const ff = node["FF-inspecter"];
+            if (!ff || seen.has(ff))
+                return false;
+            seen.add(ff);
+            return true;
+        });
+        unique.sort((a, b) => ffNumber(a["FF-inspecter"]) - ffNumber(b["FF-inspecter"]));
+        return unique;
+    }
+    searchBestMobile(nodes, keywords, topN = 5, threshold = 60) {
+        if (typeof keywords === "string") {
+            keywords = keywords.replace(/[\[\]]/g, "").split(",").map(k => k.trim());
+        }
+        keywords = keywords.map(k => k.toLowerCase());
+        const slim = (n) => {
+            const allowed = [
+                "tag",
+                "FF-inspecter",
+                "text",
+                "content-desc",
+                "resource-id",
+                "class",
+                "pane-title",
+                "hint",
+                "clickable",
+                "editable",
+                "enabled"
+            ];
+            const result = {};
+            for (const key of allowed) {
+                if (key in n)
+                    result[key] = n[key];
+            }
+            return result;
+        };
+        const indexMap = {};
+        nodes.forEach((item, i) => {
+            const ff = item.node["FF-inspecter"];
+            if (ff)
+                indexMap[ff] = i;
+        });
+        const results = {};
+        for (const kw of keywords) {
+            const exact = [];
+            const strong = [];
+            const fuzzyMatches = [];
+            for (const item of nodes) {
+                const node = item.node;
+                const text = (item.search || "").toLowerCase();
+                const t = (node["text"] || "").toLowerCase();
+                const cd = (node["content-desc"] || "").toLowerCase();
+                const rid = (node["resource-id"] || "").toLowerCase();
+                const hint = (node["hint"] || "").toLowerCase();
+                // EXACT MATCH
+                if (text === kw ||
+                    t === kw ||
+                    cd === kw ||
+                    rid === kw ||
+                    hint === kw) {
+                    exact.push(node);
+                    continue;
+                }
+                // STRONG PARTIAL MATCH
+                if (text.includes(kw) ||
+                    t.includes(kw) ||
+                    cd.includes(kw) ||
+                    rid.includes(kw) ||
+                    hint.includes(kw)) {
+                    strong.push(node);
+                    continue;
+                }
+                // FUZZY MATCH
+                const score = fuzzy(kw, text) * 100;
+                if (score >= threshold) {
+                    fuzzyMatches.push([score, node]);
+                }
+            }
+            let selected;
+            if (exact.length) {
+                selected = exact.slice(0, topN);
+            }
+            else if (strong.length) {
+                selected = strong.slice(0, topN);
+            }
+            else {
+                fuzzyMatches.sort((a, b) => b[0] - a[0]);
+                selected = fuzzyMatches.slice(0, topN).map(f => f[1]);
+            }
+            results[kw] = selected.map(slim);
+        }
+        return {
+            results,
+            indexMap
+        };
+    }
+}

package/dist/domAnalysis/simplifyAndFlatten.d.ts

@@ -0,0 +1,5 @@
+import { FlatNode, SimplifiedDomResult } from "../core/interfaces/simplifyAndFlattenInterface.js";
+export declare class DomSimplifier {
+    simplify(rawDom: string): FlatNode[];
+    simplifyMobile(rawDom: string): SimplifiedDomResult;
+}