ff-automationv2 2.2.36-beta.0 → 2.2.36-beta.2

package/dist/core/main/stepProcessor.d.ts

@@ -0,0 +1,24 @@
+import { llmAction } from "../../ai/llmcalls/llmAction.js";
+import { FireFlinkLLMResponse } from "../types/llmResponseType.js";
+import { PromptType } from "../types/promptType.js";
+import { platform } from "../types/scriptTypesType.js";
+export interface LLMRequest {
+    platform: platform;
+    type: PromptType;
+    args: Record<string, any>;
+    input: Record<string, any>;
+    isVision?: boolean;
+}
+export declare class StepProcessor {
+    private llm;
+    private resultParser;
+    constructor(llm: llmAction);
+    getResultTokenUsage(): Promise<{
+        inputTokens: number;
+        outputTokens: number;
+        totalTokens: number;
+    }>;
+    getLLMResponse({ platform, type, args, input, isVision }: LLMRequest): Promise<{
+        response: FireFlinkLLMResponse;
+    }>;
+}

package/dist/core/main/stepProcessor.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StepProcessor = void 0;
+const parseLlmOputput_js_1 = require("../../ai/llmcalls/parseLlmOputput.js");
+class StepProcessor {
+    constructor(llm) {
+        this.llm = llm;
+        this.resultParser = new parseLlmOputput_js_1.LLMResultParser();
+    }
+    async getResultTokenUsage() {
+        return this.resultParser.getTokenUsage();
+    }
+    async getLLMResponse({ platform, type, args, input, isVision = false }) {
+        try {
+            const response = await this.llm.getLLMResponse(platform, type, args, input, isVision);
+            return this.resultParser.fetchResult(response);
+        }
+        catch (error) {
+            throw new Error("failed to get response from llm", { cause: error });
+        }
+    }
+}
+exports.StepProcessor = StepProcessor;

package/dist/core/types/browserType.d.ts

@@ -0,0 +1,8 @@
+export declare const browserName: {
+    readonly "Microsoft Edge": "edge";
+    readonly "Google Chrome": "chrome";
+    readonly "Mozilla Firefox": "firefox";
+    readonly Safari: "safari";
+};
+export type BrowserName = keyof typeof browserName;
+export type BrowserId = typeof browserName[BrowserName];

package/dist/core/types/browserType.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.browserName = void 0;
+exports.browserName = {
+    "Microsoft Edge": "edge",
+    "Google Chrome": "chrome",
+    "Mozilla Firefox": "firefox",
+    "Safari": "safari"
+};

package/dist/core/types/cloudServerPortType.d.ts

@@ -0,0 +1,4 @@
+export declare const CloudPort: {
+    readonly https: 443;
+    readonly http: 80;
+};

package/dist/core/types/cloudServerPortType.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CloudPort = void 0;
+exports.CloudPort = {
+    https: 443,
+    http: 80
+};

package/dist/core/types/cloudprotocolType.d.ts

@@ -0,0 +1 @@
+export type Protocol = "http" | "https";

package/dist/core/types/cloudprotocolType.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/types/llmResponseType.d.ts

@@ -0,0 +1,2 @@
+import { IKeywordAction, IDragDropAction, IAlertAction, IScrollOrAttributeAction, IManualSteps, IVisionAction, IWaitAction, IClickAction, IVerifyAction, IGetAction, IInputTextExtractor } from "../interfaces/llmResponseInterface.js";
+export type FireFlinkLLMResponse = IKeywordAction | IAlertAction | IDragDropAction | IScrollOrAttributeAction | IManualSteps | IVisionAction | IWaitAction | IClickAction | IVerifyAction | IGetAction | IInputTextExtractor | any;

package/dist/core/types/llmResponseType.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/types/messageType.d.ts

@@ -0,0 +1,13 @@
+export declare const MessageType: {
+    readonly LiveLog: {
+        readonly Ongoing: "liveLogOngoing";
+        readonly Completed: "liveLogCompleted";
+        readonly Error: "liveLogError";
+        readonly Termination: "termination";
+        readonly ExecutionStopped: "executionStopped";
+        readonly retry: "retry";
+    };
+    readonly nextBatch: "nextBatch";
+    readonly AutomationScriptMetaData: "AutomationScriptMetaData";
+};
+export type MessageType = typeof MessageType[keyof typeof MessageType] | typeof MessageType.LiveLog[keyof typeof MessageType.LiveLog];

package/dist/core/types/messageType.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MessageType = void 0;
+exports.MessageType = {
+    LiveLog: {
+        Ongoing: "liveLogOngoing",
+        Completed: "liveLogCompleted",
+        Error: "liveLogError",
+        Termination: "termination",
+        ExecutionStopped: "executionStopped",
+        retry: "retry",
+    },
+    nextBatch: "nextBatch",
+    AutomationScriptMetaData: "AutomationScriptMetaData",
+};

package/dist/core/types/promptMap.d.ts

@@ -0,0 +1,7 @@
+import { PromptArgsMap } from "../interfaces/promptInterface.js";
+import { platform } from "../types/scriptTypesType.js";
+export type PromptMap = {
+    [P in platform]: {
+        [K in keyof PromptArgsMap]: (args: PromptArgsMap[K]) => Promise<string>;
+    };
+};

package/dist/core/types/promptMap.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/types/promptType.d.ts

@@ -0,0 +1,17 @@
+export declare enum PromptType {
+    USER_STORY_TO_LIST = "userStoryToList",
+    KEYWORD_EXTRACTOR = "keywordExtractorPrompt",
+    MULTICHANNEL_KEYWORD_EXTRACTOR = "multichannelKeywordExtractorPrompt",
+    ERROR_DESCRIPTION = "errorDescriptionPrompt",
+    FF_INSPECTOR = "ffInspectorNumExtractor",
+    VISION_PROMPT = "visionPrompt",
+    SWIPE_VISION_PROMPT = "swipe_vision_prompt",
+    VERIFY_PROMPT = "verifyActionExtractorPrompt",
+    GET_PROMPT = "getActionExtractorPrompt",
+    WAIT_ACTION_EXTRACTOR_PROMPT = "waitActionExtractorPrompt",
+    COMBINED_ACTION_EXTRACTOR_PROMPT = "combinedActionExtractorPrompt",
+    OTP_EXTRACTOR = "otpExtractorPrompt",
+    FILE_VERIFY_PROMPT = "fileVerifyActionExtractorPrompt",
+    INPUT_CONTEXT_PROMPT = "inputContextPrompt",
+    ARITHMETIC_PROMPT = "arithmeticPrompt"
+}

package/dist/core/types/promptType.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PromptType = void 0;
+var PromptType;
+(function (PromptType) {
+    PromptType["USER_STORY_TO_LIST"] = "userStoryToList";
+    PromptType["KEYWORD_EXTRACTOR"] = "keywordExtractorPrompt";
+    PromptType["MULTICHANNEL_KEYWORD_EXTRACTOR"] = "multichannelKeywordExtractorPrompt";
+    PromptType["ERROR_DESCRIPTION"] = "errorDescriptionPrompt";
+    PromptType["FF_INSPECTOR"] = "ffInspectorNumExtractor";
+    PromptType["VISION_PROMPT"] = "visionPrompt";
+    PromptType["SWIPE_VISION_PROMPT"] = "swipe_vision_prompt";
+    PromptType["VERIFY_PROMPT"] = "verifyActionExtractorPrompt";
+    PromptType["GET_PROMPT"] = "getActionExtractorPrompt";
+    PromptType["WAIT_ACTION_EXTRACTOR_PROMPT"] = "waitActionExtractorPrompt";
+    PromptType["COMBINED_ACTION_EXTRACTOR_PROMPT"] = "combinedActionExtractorPrompt";
+    PromptType["OTP_EXTRACTOR"] = "otpExtractorPrompt";
+    PromptType["FILE_VERIFY_PROMPT"] = "fileVerifyActionExtractorPrompt";
+    PromptType["INPUT_CONTEXT_PROMPT"] = "inputContextPrompt";
+    PromptType["ARITHMETIC_PROMPT"] = "arithmeticPrompt";
+})(PromptType || (exports.PromptType = PromptType = {}));

package/dist/core/types/scriptTypesType.d.ts

@@ -0,0 +1 @@
+export type platform = "web" | "android" | "ios";

package/dist/core/types/scriptTypesType.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/types/visionllmInputType.d.ts

@@ -0,0 +1,8 @@
+export type visionPromptMessage = ({
+    type: "input_text";
+    text: string;
+} | {
+    type: "input_image";
+    image_url: string;
+    detail: "auto" | "low" | "high";
+})[];

package/dist/core/types/visionllmInputType.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/domAnalysis/getRelaventElements.d.ts

@@ -0,0 +1,16 @@
+export interface DomProcessingRequest {
+    keywords: string[] | string;
+    rawDom: string;
+    stepCount: number;
+    stepAction: string;
+    step: string;
+    platform: string;
+    driver: any;
+}
+export declare class DomProcessingEngine {
+    private simplifier;
+    private searcher;
+    private relationshipBuilder;
+    webProcess(request: DomProcessingRequest): any[];
+    mobileProcess(request: DomProcessingRequest): Promise<any[]>;
+}

package/dist/domAnalysis/getRelaventElements.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DomProcessingEngine = void 0;
+const simplifyAndFlatten_js_1 = require("./simplifyAndFlatten.js");
+const searchBest_js_1 = require("./searchBest.js");
+const relativeElementsFromDom_js_1 = require("./relativeElementsFromDom.js");
+const logData_js_1 = require("../utils/logger/logData.js");
+class DomProcessingEngine {
+    constructor() {
+        this.simplifier = new simplifyAndFlatten_js_1.DomSimplifier();
+        this.searcher = new searchBest_js_1.DomSearcher();
+        this.relationshipBuilder = new relativeElementsFromDom_js_1.DomRelationshipBuilder();
+    }
+    webProcess(request) {
+        const flat = this.simplifier.simplify(request.rawDom);
+        logData_js_1.logger.saveJSON(flat, `flat-dom-${request.stepCount}`);
+        const searched = this.searcher.search(flat, request.keywords);
+        const structured = this.relationshipBuilder.build(searched);
+        return structured;
+    }
+    async mobileProcess(request) {
+        let grouped;
+        if (request.platform.toLowerCase() === "ios") {
+            const { flatNodes, parentMap, runtimeElements } = await this.simplifier.simplifyIosDom(request.rawDom, request.driver);
+            logData_js_1.logger.saveJSON(flatNodes, `flat-dom-ios-${request.stepCount}`);
+            const { results, indexMap } = await this.searcher.searchBestIos(flatNodes, request.keywords);
+            grouped = await this.relationshipBuilder.buildGroupedResultsIos(results, flatNodes, indexMap, parentMap, runtimeElements);
+        }
+        else {
+            const { flatNodes, parentMap } = await this.simplifier.simplifyAndroidDom(request.rawDom, request.stepAction, request.step);
+            logData_js_1.logger.saveJSON(flatNodes, `flat-dom-android-${request.stepCount}`);
+            const { results, indexMap } = await this.searcher.searchBestAndroid(flatNodes, request.keywords);
+            grouped = await this.relationshipBuilder.buildGroupedResultsAndroid(results, flatNodes, indexMap, parentMap);
+        }
+        return grouped;
+    }
+}
+exports.DomProcessingEngine = DomProcessingEngine;

package/dist/domAnalysis/relativeElementsFromDom.d.ts

@@ -0,0 +1,5 @@
+export declare class DomRelationshipBuilder {
+    build(nodes: any[]): any[];
+    buildGroupedResultsAndroid(results: Record<string, any[]>, nodes: any[], indexMap: Record<string, number>, parentMap: Map<Element, Element | null>): any[][];
+    buildGroupedResultsIos(results: Record<string, any[]>, nodes: any[], indexMap: Record<string, number>, parentMap: Map<any, any>, runtimeElements: Record<number, any>): any[][];
+}

package/dist/domAnalysis/relativeElementsFromDom.js

@@ -0,0 +1,262 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DomRelationshipBuilder = void 0;
+class DomRelationshipBuilder {
+    build(nodes) {
+        const ffToXPath = new Map();
+        nodes.forEach(n => {
+            if (n["FF-inspecter"] && n["ff-xpath"]) {
+                ffToXPath.set(n["FF-inspecter"], n["ff-xpath"]);
+            }
+        });
+        const parentXPath = (x) => x.includes("/") ? x.substring(0, x.lastIndexOf("/")) : undefined;
+        const ffToNode = new Map();
+        nodes.forEach(n => {
+            ffToNode.set(n["FF-inspecter"], n);
+        });
+        const structured = [];
+        for (const node of nodes) {
+            const ff = node["FF-inspecter"];
+            const xp = node["ff-xpath"];
+            let parent;
+            const children = [];
+            const siblings = [];
+            if (ff && xp) {
+                const px = parentXPath(xp);
+                for (const [otherFF, otherXP] of ffToXPath.entries()) {
+                    if (otherFF === ff)
+                        continue;
+                    if (otherXP.startsWith(xp + "/")) {
+                        children.push(otherFF);
+                    }
+                    else if (px && parentXPath(otherXP) === px) {
+                        siblings.push(otherFF);
+                    }
+                }
+                for (const [otherFF, otherXP] of ffToXPath.entries()) {
+                    if (otherXP === px) {
+                        parent = otherFF;
+                        break;
+                    }
+                }
+            }
+            const slim = Object.fromEntries(Object.entries(node).filter(([k, v]) => v && !["search", "ff-xpath", "href", "src"].includes(k)));
+            const rel = {};
+            const mini = (n) => Object.fromEntries(Object.entries(n).filter(([k, v]) => v && !["search", "ff-xpath", "href", "src"].includes(k)));
+            if (parent && ffToNode.has(parent)) {
+                rel.parent = mini(ffToNode.get(parent));
+            }
+            if (children.length) {
+                rel.children = children.slice(0, 2)
+                    .map(c => mini(ffToNode.get(c)));
+            }
+            if (siblings.length) {
+                rel.siblings = siblings.slice(0, 2)
+                    .map(s => mini(ffToNode.get(s)));
+            }
+            if (Object.keys(rel).length) {
+                slim["relationship"] = rel;
+            }
+            structured.push(slim);
+        }
+        return structured;
+    }
+    buildGroupedResultsAndroid(results, nodes, indexMap, parentMap) {
+        const groupedResults = [];
+        const usedNodes = new Set();
+        const slim = (n) => {
+            const keys = [
+                "tag",
+                "FF-inspecter",
+                "text",
+                "content-desc",
+                "resource-id",
+                "class",
+                "pane-title",
+                "hint",
+                "clickable",
+                "editable",
+                "enabled"
+            ];
+            const out = {};
+            for (const k of keys) {
+                if (k in n)
+                    out[k] = n[k];
+            }
+            return out;
+        };
+        const addNodeSafe = (nodeDict, group, localUsed) => {
+            const ff = nodeDict["FF-inspecter"];
+            if (!ff || localUsed.has(ff))
+                return;
+            group.push(slim(nodeDict));
+            localUsed.add(ff);
+        };
+        for (const matchedNodes of Object.values(results)) {
+            for (const matchedNode of matchedNodes) {
+                const selectedFF = matchedNode["FF-inspecter"];
+                if (!(selectedFF in indexMap))
+                    continue;
+                if (usedNodes.has(selectedFF))
+                    continue;
+                const item = nodes[indexMap[selectedFF]];
+                const element = item.element;
+                const parent = parentMap.get(element) || null;
+                const group = [];
+                const localUsed = new Set();
+                addNodeSafe(matchedNode, group, localUsed);
+                if (matchedNode["clickable"] === "true") {
+                    groupedResults.push(group);
+                    localUsed.forEach(ff => usedNodes.add(ff));
+                    continue;
+                }
+                if (!parent) {
+                    groupedResults.push(group);
+                    localUsed.forEach(ff => usedNodes.add(ff));
+                    continue;
+                }
+                // parent check
+                if (parent.getAttribute("clickable") === "true" &&
+                    parent.getAttribute("enabled") !== "false") {
+                    addNodeSafe(Object.fromEntries(Array.from(parent?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
+                }
+                const siblings = Array.from(parent?.children ?? []);
+                const idx = siblings.indexOf(element);
+                // upper siblings
+                if (idx > 0) {
+                    for (let j = idx - 1; j >= 0; j--) {
+                        const upper = siblings[j];
+                        if (upper.getAttribute("clickable") === "true" &&
+                            upper.getAttribute("enabled") !== "false") {
+                            addNodeSafe(Object.fromEntries(Array.from(upper?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
+                            break;
+                        }
+                        const upperParent = parentMap.get(upper);
+                        if (upperParent &&
+                            upperParent.getAttribute("clickable") === "true" &&
+                            upperParent.getAttribute("enabled") !== "false") {
+                            addNodeSafe(Object.fromEntries(Array.from(upperParent?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
+                            break;
+                        }
+                    }
+                }
+                // lower siblings
+                if (idx !== -1) {
+                    for (let j = idx + 1; j < siblings.length; j++) {
+                        const lower = siblings[j];
+                        if (lower.getAttribute("clickable") === "true" &&
+                            lower.getAttribute("enabled") !== "false") {
+                            addNodeSafe(Object.fromEntries(Array.from(lower?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
+                            break;
+                        }
+                        for (const child of Array.from(lower?.children ?? [])) {
+                            if (child.getAttribute("clickable") === "true" &&
+                                child.getAttribute("enabled") !== "false") {
+                                addNodeSafe(Object.fromEntries(Array.from(child?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
+                                break;
+                            }
+                        }
+                    }
+                }
+                groupedResults.push(group);
+                localUsed.forEach(ff => usedNodes.add(ff));
+            }
+        }
+        const firstMultiIndex = {};
+        groupedResults.forEach((group, i) => {
+            if (group.length > 1) {
+                group.forEach(node => {
+                    const ff = node["FF-inspecter"];
+                    if (!(ff in firstMultiIndex)) {
+                        firstMultiIndex[ff] = i;
+                    }
+                });
+            }
+        });
+        const finalResults = [];
+        const usedFF = new Set();
+        groupedResults.forEach((group, i) => {
+            const newGroup = [];
+            group.forEach(node => {
+                const ff = node["FF-inspecter"];
+                if (usedFF.has(ff))
+                    return;
+                if (group.length === 1 && ff in firstMultiIndex)
+                    return;
+                if (ff in firstMultiIndex && firstMultiIndex[ff] !== i)
+                    return;
+                newGroup.push(node);
+                usedFF.add(ff);
+            });
+            if (newGroup.length)
+                finalResults.push(newGroup);
+        });
+        return finalResults;
+    }
+    buildGroupedResultsIos(results, nodes, indexMap, parentMap, runtimeElements) {
+        const grouped = new Map();
+        const slimIOS = (n) => {
+            const keys = [
+                "tag",
+                "FF-inspecter",
+                "value",
+                "name",
+                "label",
+                "class",
+                "pane-title",
+                "accessible",
+                "editable",
+                "enabled",
+                "traits"
+            ];
+            const out = {};
+            for (const k of keys) {
+                if (k in n)
+                    out[k] = n[k];
+            }
+            return out;
+        };
+        for (const matchedNodes of Object.values(results)) {
+            for (const node of matchedNodes) {
+                const ff = node["FF-inspecter"];
+                if (!ff || !(ff in indexMap))
+                    continue;
+                const item = nodes[indexMap[ff]];
+                const element = runtimeElements[item.id];
+                const parent = parentMap.get(element);
+                const parentKey = parent ? parent : ff;
+                if (!grouped.has(parentKey)) {
+                    grouped.set(parentKey, []);
+                }
+                grouped.get(parentKey).push(slimIOS(node));
+            }
+        }
+        const groupedResults = [];
+        for (const group of grouped.values()) {
+            const seenFF = new Set();
+            const uniqueGroup = [];
+            for (const node of group) {
+                const ff = node["FF-inspecter"];
+                if (!ff || seenFF.has(ff))
+                    continue;
+                seenFF.add(ff);
+                uniqueGroup.push(node);
+            }
+            if (uniqueGroup.length) {
+                groupedResults.push(uniqueGroup);
+            }
+        }
+        const ffNumber = (node) => {
+            const ff = node["FF-inspecter"] || "";
+            const parts = ff.split("-");
+            return parseInt(parts[parts.length - 1]) || 0;
+        };
+        groupedResults.sort((a, b) => {
+            const minA = Math.min(...a.map(ffNumber));
+            const minB = Math.min(...b.map(ffNumber));
+            return minA - minB;
+        });
+        return groupedResults;
+    }
+}
+exports.DomRelationshipBuilder = DomRelationshipBuilder;

package/dist/domAnalysis/searchBest.d.ts

@@ -0,0 +1,12 @@
+import { FlatNode } from "../core/interfaces/simplifyAndFlattenInterface";
+export declare class DomSearcher {
+    search(nodes: FlatNode[], keywords: string[] | string, topN?: number, threshold?: number): any[];
+    searchBestAndroid(nodes: FlatNode[], keywords: string[] | string, topN?: number, threshold?: number): {
+        results: Record<string, any[]>;
+        indexMap: Record<string, number>;
+    };
+    searchBestIos(nodes: any[], keywords: string[] | string, topN?: number, threshold?: number): {
+        results: Record<string, any[]>;
+        indexMap: Record<string, number>;
+    };
+}