ff-automationv2 2.2.28 → 2.2.30

package/dist/core/main/stepProcessor.d.ts

@@ -7,6 +7,7 @@ export interface LLMRequest {
     type: PromptType;
     args: Record<string, any>;
     input: Record<string, any>;
+    isVision?: boolean;
 }
 export declare class StepProcessor {
     private llm;
@@ -17,7 +18,7 @@ export declare class StepProcessor {
         outputTokens: number;
         totalTokens: number;
     }>;
-    getLLMResponse({ platform, type, args, input }: LLMRequest): Promise<{
+    getLLMResponse({ platform, type, args, input, isVision }: LLMRequest): Promise<{
         response: FireFlinkLLMResponse;
     }>;
 }

package/dist/core/main/stepProcessor.js

@@ -10,9 +10,14 @@ class StepProcessor {
     async getResultTokenUsage() {
         return this.resultParser.getTokenUsage();
     }
-    async getLLMResponse({ platform, type, args, input }) {
-        const response = await this.llm.getLLMResponse(platform, type, args, input);
-        return this.resultParser.fetchResult(response);
+    async getLLMResponse({ platform, type, args, input, isVision = false }) {
+        try {
+            const response = await this.llm.getLLMResponse(platform, type, args, input, isVision);
+            return this.resultParser.fetchResult(response);
+        }
+        catch (error) {
+            throw new Error("failed to get response from llm", { cause: error });
+        }
     }
 }
 exports.StepProcessor = StepProcessor;

package/dist/core/types/browserType.d.ts

@@ -1,8 +1,8 @@
-export declare const browsers: {
+export declare const browserName: {
     readonly "Microsoft Edge": "edge";
     readonly "Google Chrome": "chrome";
     readonly "Mozilla Firefox": "firefox";
     readonly Safari: "safari";
 };
-export type BrowserName = keyof typeof browsers;
-export type BrowserId = typeof browsers[BrowserName];
+export type BrowserName = keyof typeof browserName;
+export type BrowserId = typeof browserName[BrowserName];

package/dist/core/types/browserType.js

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.browsers = void 0;
-exports.browsers = {
+exports.browserName = void 0;
+exports.browserName = {
     "Microsoft Edge": "edge",
     "Google Chrome": "chrome",
     "Mozilla Firefox": "firefox",

package/dist/core/types/messageType.d.ts

@@ -0,0 +1,13 @@
+export declare const MessageType: {
+    readonly LiveLog: {
+        readonly Ongoing: "liveLogOngoing";
+        readonly Completed: "liveLogCompleted";
+        readonly Error: "liveLogError";
+        readonly Termination: "termination";
+        readonly ExecutionStopped: "executionStopped";
+        readonly retry: "retry";
+    };
+    readonly nextBatch: "nextBatch";
+    readonly AutomationScriptMetaData: "AutomationScriptMetaData";
+};
+export type MessageType = typeof MessageType[keyof typeof MessageType] | typeof MessageType.LiveLog[keyof typeof MessageType.LiveLog];

package/dist/core/types/messageType.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MessageType = void 0;
+exports.MessageType = {
+    LiveLog: {
+        Ongoing: "liveLogOngoing",
+        Completed: "liveLogCompleted",
+        Error: "liveLogError",
+        Termination: "termination",
+        ExecutionStopped: "executionStopped",
+        retry: "retry",
+    },
+    nextBatch: "nextBatch",
+    AutomationScriptMetaData: "AutomationScriptMetaData",
+};

package/dist/core/types/promptType.d.ts

@@ -7,5 +7,6 @@ export declare enum PromptType {
     VERIFY_PROMPT = "verifyActionExtractorPrompt",
     GET_PROMPT = "getActionExtractorPrompt",
     WAIT_ACTION_EXTRACTOR_PROMPT = "waitActionExtractorPrompt",
-    COMBINED_ACTION_EXTRACTOR_PROMPT = "combinedActionExtractorPrompt"
+    COMBINED_ACTION_EXTRACTOR_PROMPT = "combinedActionExtractorPrompt",
+    OTP_EXTRACTOR = "otpExtractorPrompt"
 }

package/dist/core/types/promptType.js

@@ -12,4 +12,5 @@ var PromptType;
     PromptType["GET_PROMPT"] = "getActionExtractorPrompt";
     PromptType["WAIT_ACTION_EXTRACTOR_PROMPT"] = "waitActionExtractorPrompt";
     PromptType["COMBINED_ACTION_EXTRACTOR_PROMPT"] = "combinedActionExtractorPrompt";
+    PromptType["OTP_EXTRACTOR"] = "otpExtractorPrompt";
 })(PromptType || (exports.PromptType = PromptType = {}));

package/dist/core/types/scriptTypesType.d.ts

@@ -1 +1 @@
-export type platform = "web" | "android";
+export type platform = "web" | "android" | "ios";

package/dist/core/types/visionllmInputType.d.ts

@@ -1,9 +1,8 @@
 export type visionPromptMessage = ({
-    type: "text";
+    type: "input_text";
     text: string;
 } | {
-    type: "image_url";
-    image_url: {
-        url: string;
-    };
+    type: "input_image";
+    image_url: string;
+    detail: "auto" | "low" | "high";
 })[];

package/dist/domAnalysis/getRelaventElements.d.ts

@@ -2,11 +2,15 @@ export interface DomProcessingRequest {
     keywords: string[] | string;
     rawDom: string;
     stepCount: number;
+    stepAction: string;
+    step: string;
+    platform: string;
+    driver: any;
 }
 export declare class DomProcessingEngine {
     private simplifier;
     private searcher;
     private relationshipBuilder;
     webProcess(request: DomProcessingRequest): any[];
-    mobileProcess(request: DomProcessingRequest): any[][];
+    mobileProcess(request: DomProcessingRequest): Promise<any[]>;
 }

package/dist/domAnalysis/getRelaventElements.js

@@ -18,11 +18,20 @@ class DomProcessingEngine {
         const structured = this.relationshipBuilder.build(searched);
         return structured;
     }
-    mobileProcess(request) {
-        const { flatNodes, parentMap } = this.simplifier.simplifyMobile(request.rawDom);
-        logData_js_1.logger.saveJSON(flatNodes, `flat-dom-Mob-${request.stepCount}`);
-        const { results, indexMap } = this.searcher.searchBestMobile(flatNodes, request.keywords);
-        const grouped = this.relationshipBuilder.buildGroupedResultsMobile(results, flatNodes, indexMap, parentMap);
+    async mobileProcess(request) {
+        let grouped;
+        if (request.platform.toLowerCase() === "ios") {
+            const { flatNodes, parentMap, runtimeElements } = await this.simplifier.simplifyIosDom(request.rawDom, request.driver);
+            logData_js_1.logger.saveJSON(flatNodes, `flat-dom-ios-${request.stepCount}`);
+            const { results, indexMap } = await this.searcher.searchBestIos(flatNodes, request.keywords);
+            grouped = await this.relationshipBuilder.buildGroupedResultsIos(results, flatNodes, indexMap, parentMap, runtimeElements);
+        }
+        else {
+            const { flatNodes, parentMap } = await this.simplifier.simplifyAndroidDom(request.rawDom, request.stepAction, request.step);
+            logData_js_1.logger.saveJSON(flatNodes, `flat-dom-android-${request.stepCount}`);
+            const { results, indexMap } = await this.searcher.searchBestAndroid(flatNodes, request.keywords);
+            grouped = await this.relationshipBuilder.buildGroupedResultsAndroid(results, flatNodes, indexMap, parentMap);
+        }
         return grouped;
     }
 }

package/dist/domAnalysis/relativeElementsFromDom.d.ts

@@ -1,4 +1,5 @@
 export declare class DomRelationshipBuilder {
     build(nodes: any[]): any[];
-    buildGroupedResultsMobile(results: Record<string, any[]>, nodes: any[], indexMap: Record<string, number>, parentMap: Map<Element, Element | null>): any[][];
+    buildGroupedResultsAndroid(results: Record<string, any[]>, nodes: any[], indexMap: Record<string, number>, parentMap: Map<Element, Element | null>): any[][];
+    buildGroupedResultsIos(results: Record<string, any[]>, nodes: any[], indexMap: Record<string, number>, parentMap: Map<any, any>, runtimeElements: Record<number, any>): any[][];
 }

package/dist/domAnalysis/relativeElementsFromDom.js

@@ -61,7 +61,7 @@ class DomRelationshipBuilder {
         }
         return structured;
     }
-    buildGroupedResultsMobile(results, nodes, indexMap, parentMap) {
+    buildGroupedResultsAndroid(results, nodes, indexMap, parentMap) {
         const groupedResults = [];
         const usedNodes = new Set();
         const slim = (n) => {
@@ -193,5 +193,70 @@ class DomRelationshipBuilder {
         });
         return finalResults;
     }
+    buildGroupedResultsIos(results, nodes, indexMap, parentMap, runtimeElements) {
+        const grouped = new Map();
+        const slimIOS = (n) => {
+            const keys = [
+                "tag",
+                "FF-inspecter",
+                "value",
+                "name",
+                "label",
+                "class",
+                "pane-title",
+                "accessible",
+                "editable",
+                "enabled",
+                "traits"
+            ];
+            const out = {};
+            for (const k of keys) {
+                if (k in n)
+                    out[k] = n[k];
+            }
+            return out;
+        };
+        for (const matchedNodes of Object.values(results)) {
+            for (const node of matchedNodes) {
+                const ff = node["FF-inspecter"];
+                if (!ff || !(ff in indexMap))
+                    continue;
+                const item = nodes[indexMap[ff]];
+                const element = runtimeElements[item.id];
+                const parent = parentMap.get(element);
+                const parentKey = parent ? parent : ff;
+                if (!grouped.has(parentKey)) {
+                    grouped.set(parentKey, []);
+                }
+                grouped.get(parentKey).push(slimIOS(node));
+            }
+        }
+        const groupedResults = [];
+        for (const group of grouped.values()) {
+            const seenFF = new Set();
+            const uniqueGroup = [];
+            for (const node of group) {
+                const ff = node["FF-inspecter"];
+                if (!ff || seenFF.has(ff))
+                    continue;
+                seenFF.add(ff);
+                uniqueGroup.push(node);
+            }
+            if (uniqueGroup.length) {
+                groupedResults.push(uniqueGroup);
+            }
+        }
+        const ffNumber = (node) => {
+            const ff = node["FF-inspecter"] || "";
+            const parts = ff.split("-");
+            return parseInt(parts[parts.length - 1]) || 0;
+        };
+        groupedResults.sort((a, b) => {
+            const minA = Math.min(...a.map(ffNumber));
+            const minB = Math.min(...b.map(ffNumber));
+            return minA - minB;
+        });
+        return groupedResults;
+    }
 }
 exports.DomRelationshipBuilder = DomRelationshipBuilder;

package/dist/domAnalysis/searchBest.d.ts

@@ -1,7 +1,11 @@
 import { FlatNode } from "../core/interfaces/simplifyAndFlattenInterface";
 export declare class DomSearcher {
     search(nodes: FlatNode[], keywords: string[] | string, topN?: number, threshold?: number): any[];
-    searchBestMobile(nodes: FlatNode[], keywords: string[] | string, topN?: number, threshold?: number): {
+    searchBestAndroid(nodes: FlatNode[], keywords: string[] | string, topN?: number, threshold?: number): {
+        results: Record<string, any[]>;
+        indexMap: Record<string, number>;
+    };
+    searchBestIos(nodes: any[], keywords: string[] | string, topN?: number, threshold?: number): {
         results: Record<string, any[]>;
         indexMap: Record<string, number>;
     };

package/dist/domAnalysis/searchBest.js

@@ -1,14 +1,20 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DomSearcher = void 0;
+const fastest_levenshtein_1 = require("fastest-levenshtein");
 const fast_fuzzy_1 = require("fast-fuzzy");
+// import { logger } from "../utils/logger/logData";
 class DomSearcher {
     search(nodes, keywords, topN = 20, threshold = 40) {
         if (typeof keywords === "string") {
             keywords = keywords.replace(/[\[\]]/g, "");
             keywords = keywords.split(",").map(k => k.trim());
         }
-        keywords = keywords.map(k => k.replace(/[^a-zA-Z0-9\s]/g, " ").trim().toLowerCase());
+        keywords = keywords.map(k => (typeof k === "string" ? k : String(k))
+            .replace(/[^a-zA-Z0-9\s]/g, " ")
+            .trim()
+            .toLowerCase()
+            .replace(/\s+/g, ""));
         const ffNumber = (val) => {
             if (!val)
                 return 0;
@@ -23,7 +29,7 @@ class DomSearcher {
             for (const i of [idx - 1, idx, idx + 1]) {
                 if (i >= 0 && i < total) {
                     if (nodes[i].node["FF-inspecter"]) {
-                        collected.push(i);
+                        collected.add(i);
                     }
                 }
             }
@@ -35,7 +41,7 @@ class DomSearcher {
                 const nx = nodes[j].node["ff-xpath"];
                 if (nx && isChild(cx, nx)) {
                     if (nodes[j].node["FF-inspecter"]) {
-                        collected.push(j);
+                        collected.add(j);
                     }
                     break;
                 }
@@ -44,70 +50,100 @@ class DomSearcher {
                 const nx = nodes[j].node["ff-xpath"];
                 if (nx && isSibling(cx, nx) && j !== idx) {
                     if (nodes[j].node["FF-inspecter"]) {
-                        collected.push(j);
+                        collected.add(j);
                     }
                     break;
                 }
             }
         };
         const results = {};
+        const tempResults = {};
+        let hasAnyMatch = false;
         for (const kw of keywords) {
             const exact = [];
             const strong = [];
-            const fuzzyList = [];
             const tagMatches = [];
             for (let i = 0; i < nodes.length; i++) {
                 const node = nodes[i].node;
                 const searchText = (node.search || "").toLowerCase();
                 const tagName = (node.tag || "").toLowerCase();
-                let matched = false;
-                if (searchText == kw) {
+                if (searchText === kw) {
                     exact.push(i);
-                    matched = true;
                 }
-                if (searchText.includes(kw)) {
+                else if (searchText.includes(kw)) {
                     strong.push(i);
-                    matched = true;
                 }
-                if (!matched) {
-                    const score = (0, fast_fuzzy_1.fuzzy)(kw, searchText);
-                    if (score >= threshold / 100) {
-                        fuzzyList.push([score * 100, i]);
-                    }
-                }
-                if (kw == tagName || tagName.includes(kw)) {
+                else if (kw === tagName || tagName.includes(kw)) {
                     tagMatches.push(i);
                 }
             }
-            const collected = [];
+            const collected = new Set();
             for (const idx of exact) {
                 addWithContext(idx, collected);
-                if (collected.length >= topN * 4)
+                if (collected.size >= topN * 4)
                     break;
             }
             for (const idx of strong) {
                 addWithContext(idx, collected);
-                if (collected.length >= topN * 4)
+                if (collected.size >= topN * 4)
+                    break;
+            }
+            for (const idx of tagMatches) {
+                addWithContext(idx, collected);
+                if (collected.size >= topN * 4)
                     break;
             }
-            if (collected.length < topN * 4) {
+            if (collected.size > 0) {
+                hasAnyMatch = true;
+            }
+            tempResults[kw] = Array.from(collected);
+        }
+        if (!hasAnyMatch) {
+            for (const kw of keywords) {
+                const fuzzyList = [];
+                const tokens1 = Array.from(new Set(kw.toLowerCase().split(/\s+/)));
+                for (let i = 0; i < nodes.length; i++) {
+                    const node = nodes[i].node;
+                    const searchText = (node.search || "").toLowerCase();
+                    const tokens2 = searchText.split(/\s+/);
+                    // For each kw token, find best matching token in searchText
+                    const tokenScores = tokens1.map(t1 => {
+                        let best = 0;
+                        for (const t2 of tokens2) {
+                            if (t1 === t2) {
+                                best = 100;
+                                break;
+                            }
+                            const sim = (1 - (0, fastest_levenshtein_1.distance)(t1, t2) / Math.max(t1.length, t2.length))
+                                * 100;
+                            if (sim > best)
+                                best = sim;
+                        }
+                        return best;
+                    });
+                    const allMatched = tokenScores.every(s => s >= 50);
+                    if (!allMatched)
+                        continue;
+                    const score = tokenScores.reduce((a, b) => a + b, 0) / tokenScores.length;
+                    if (score >= threshold) {
+                        fuzzyList.push([score, i]);
+                    }
+                }
+                const collected = new Set();
                 fuzzyList
                     .sort((a, b) => b[0] - a[0])
                     .forEach(([_, idx]) => {
-                    if (collected.length < topN * 4) {
+                    if (collected.size < topN * 4) {
                         addWithContext(idx, collected);
                     }
                 });
+                results[kw] = Array.from(collected).map(i => nodes[i].node);
             }
-            if (collected.length < topN * 4) {
-                for (const idx of tagMatches) {
-                    addWithContext(idx, collected);
-                    if (collected.length >= topN * 4)
-                        break;
-                }
+        }
+        else {
+            for (const kw of keywords) {
+                results[kw] = tempResults[kw].map(i => nodes[i].node);
             }
-            const uniqueIdx = Array.from(new Set(collected));
-            results[kw] = uniqueIdx.map(i => nodes[i].node);
         }
         const seen = new Set();
         const unique = Object.values(results)
@@ -122,13 +158,13 @@ class DomSearcher {
         unique.sort((a, b) => ffNumber(a["FF-inspecter"]) - ffNumber(b["FF-inspecter"]));
         return unique;
     }
-    searchBestMobile(nodes, keywords, topN = 5, threshold = 60) {
+    searchBestAndroid(nodes, keywords, topN = 5, threshold = 40) {
         if (typeof keywords === "string") {
             keywords = keywords.replace(/[\[\]]/g, "")
                 .split(",")
                 .map(k => k.trim());
         }
-        keywords = keywords.map(k => String(k).toLowerCase());
+        keywords = keywords.map(k => String(k).toLowerCase().replace(/\s+/g, ""));
         const slim = (n) => {
             const allowed = [
                 "tag",
@@ -157,58 +193,124 @@ class DomSearcher {
                 indexMap[ff] = i;
         });
         const results = {};
+        const tempResults = {};
+        let hasAnyMatch = false;
         for (const kw of keywords) {
             const exact = [];
             const strong = [];
-            const fuzzyMatches = [];
             for (const item of nodes) {
                 const node = item.node;
                 const text = (item.search || "").toLowerCase();
-                const t = (node["text"] || "").toLowerCase();
-                const cd = (node["content-desc"] || "").toLowerCase();
-                const rid = (node["resource-id"] || "").toLowerCase();
                 const hint = (node["hint"] || "").toLowerCase();
                 // EXACT MATCH
-                if (text === kw ||
-                    t === kw ||
-                    cd === kw ||
-                    rid === kw ||
-                    hint === kw) {
+                if (text === kw || hint === kw) {
                     exact.push(node);
-                    continue;
                 }
-                // STRONG PARTIAL MATCH
-                if (text.includes(kw) ||
-                    t.includes(kw) ||
-                    cd.includes(kw) ||
-                    rid.includes(kw) ||
-                    hint.includes(kw)) {
+                // STRONG MATCH
+                else if (text.includes(kw) || hint.includes(kw)) {
                     strong.push(node);
-                    continue;
-                }
-                // FUZZY MATCH
-                const score = (0, fast_fuzzy_1.fuzzy)(kw, text) * 100;
-                if (score >= threshold) {
-                    fuzzyMatches.push([score, node]);
                 }
             }
-            let selected;
+            let selected = [];
             if (exact.length) {
                 selected = exact.slice(0, topN);
             }
             else if (strong.length) {
                 selected = strong.slice(0, topN);
             }
-            else {
+            if (selected.length > 0) {
+                hasAnyMatch = true;
+            }
+            tempResults[kw] = selected;
+        }
+        if (!hasAnyMatch) {
+            for (const kw of keywords) {
+                const fuzzyMatches = [];
+                for (const item of nodes) {
+                    const node = item.node;
+                    const text = (item.search || "").toLowerCase();
+                    const score = (0, fast_fuzzy_1.fuzzy)(kw, text) * 100;
+                    if (score >= threshold) {
+                        fuzzyMatches.push([score, node]);
+                    }
+                }
                 fuzzyMatches.sort((a, b) => b[0] - a[0]);
-                selected = fuzzyMatches.slice(0, topN).map(f => f[1]);
+                const selected = fuzzyMatches
+                    .slice(0, topN)
+                    .map(f => f[1]);
+                results[kw] = selected.map(slim);
+            }
+        }
+        else {
+            for (const kw of keywords) {
+                results[kw] = tempResults[kw].map(slim);
             }
-            results[kw] = selected.map(slim);
         }
         return {
             results,
             indexMap
         };
     }
+    searchBestIos(nodes, keywords, topN = 5, threshold = 40) {
+        if (typeof keywords === "string") {
+            keywords = keywords.replace(/[\[\]]/g, "")
+                .split(",")
+                .map(k => k.trim());
+        }
+        keywords = keywords.map(k => k.toLowerCase().replace(/\s+/g, ""));
+        const slim = (n) => {
+            const allowed = [
+                "tag", "FF-inspecter", "value", "name",
+                "label", "class", "pane-title",
+                "accessible", "editable", "enabled", "traits"
+            ];
+            const res = {};
+            for (const k of allowed)
+                if (k in n)
+                    res[k] = n[k];
+            return res;
+        };
+        const indexMap = {};
+        nodes.forEach((item, i) => {
+            const ff = item.node?.["FF-inspecter"];
+            if (ff)
+                indexMap[ff] = i;
+        });
+        const results = {};
+        for (const kw of keywords) {
+            const exact = [];
+            const strong = [];
+            const fuzzyMatches = [];
+            for (const item of nodes) {
+                const node = item.node || {};
+                const text = (item.search || "").toLowerCase();
+                const value = (node["value"] || "").toLowerCase();
+                const name = (node["name"] || "").toLowerCase();
+                const label = (node["label"] || "").toLowerCase();
+                if (text === kw || value === kw || name === kw || label === kw) {
+                    exact.push(node);
+                    continue;
+                }
+                if (text.includes(kw) || value.includes(kw) || name.includes(kw) || label.includes(kw)) {
+                    strong.push(node);
+                    continue;
+                }
+                const score = (0, fast_fuzzy_1.fuzzy)(kw, text) * 100;
+                if (score >= threshold)
+                    fuzzyMatches.push([score, node]);
+            }
+            let selected;
+            if (exact.length)
+                selected = exact.slice(0, topN);
+            else if (strong.length)
+                selected = strong.slice(0, topN);
+            else {
+                fuzzyMatches.sort((a, b) => b[0] - a[0]);
+                selected = fuzzyMatches.slice(0, topN).map(f => f[1]);
+            }
+            results[kw] = selected.map(slim);
+        }
+        return { results, indexMap };
+    }
 }
 exports.DomSearcher = DomSearcher;

package/dist/domAnalysis/simplifyAndFlatten.d.ts

@@ -1,5 +1,6 @@
-import { FlatNode, SimplifiedDomResult } from "../core/interfaces/simplifyAndFlattenInterface.js";
+import { FlatNode, SimplifiedDomResult, SimplifiedDomResultIos } from "../core/interfaces/simplifyAndFlattenInterface.js";
 export declare class DomSimplifier {
     simplify(rawDom: string): FlatNode[];
-    simplifyMobile(rawDom: string): SimplifiedDomResult;
+    simplifyAndroidDom(rawDom: string, stepAction: string, step: string): SimplifiedDomResult;
+    simplifyIosDom(rawDom: string, driver: WebdriverIO.Browser): Promise<SimplifiedDomResultIos>;
 }