ff-automationv2 2.2.35 → 2.2.36-beta.0

package/dist/core/main/stepProcessor.d.ts

@@ -1,24 +0,0 @@
-import { llmAction } from "../../ai/llmcalls/llmAction.js";
-import { FireFlinkLLMResponse } from "../types/llmResponseType.js";
-import { PromptType } from "../types/promptType.js";
-import { platform } from "../types/scriptTypesType.js";
-export interface LLMRequest {
-    platform: platform;
-    type: PromptType;
-    args: Record<string, any>;
-    input: Record<string, any>;
-    isVision?: boolean;
-}
-export declare class StepProcessor {
-    private llm;
-    private resultParser;
-    constructor(llm: llmAction);
-    getResultTokenUsage(): Promise<{
-        inputTokens: number;
-        outputTokens: number;
-        totalTokens: number;
-    }>;
-    getLLMResponse({ platform, type, args, input, isVision }: LLMRequest): Promise<{
-        response: FireFlinkLLMResponse;
-    }>;
-}

package/dist/core/main/stepProcessor.js

@@ -1,23 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.StepProcessor = void 0;
-const parseLlmOputput_js_1 = require("../../ai/llmcalls/parseLlmOputput.js");
-class StepProcessor {
-    constructor(llm) {
-        this.llm = llm;
-        this.resultParser = new parseLlmOputput_js_1.LLMResultParser();
-    }
-    async getResultTokenUsage() {
-        return this.resultParser.getTokenUsage();
-    }
-    async getLLMResponse({ platform, type, args, input, isVision = false }) {
-        try {
-            const response = await this.llm.getLLMResponse(platform, type, args, input, isVision);
-            return this.resultParser.fetchResult(response);
-        }
-        catch (error) {
-            throw new Error("failed to get response from llm", { cause: error });
-        }
-    }
-}
-exports.StepProcessor = StepProcessor;

package/dist/core/types/browserType.d.ts

@@ -1,8 +0,0 @@
-export declare const browserName: {
-    readonly "Microsoft Edge": "edge";
-    readonly "Google Chrome": "chrome";
-    readonly "Mozilla Firefox": "firefox";
-    readonly Safari: "safari";
-};
-export type BrowserName = keyof typeof browserName;
-export type BrowserId = typeof browserName[BrowserName];

package/dist/core/types/browserType.js

@@ -1,9 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.browserName = void 0;
-exports.browserName = {
-    "Microsoft Edge": "edge",
-    "Google Chrome": "chrome",
-    "Mozilla Firefox": "firefox",
-    "Safari": "safari"
-};

package/dist/core/types/cloudServerPortType.d.ts

@@ -1,4 +0,0 @@
-export declare const CloudPort: {
-    readonly https: 443;
-    readonly http: 80;
-};

package/dist/core/types/cloudServerPortType.js

@@ -1,7 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.CloudPort = void 0;
-exports.CloudPort = {
-    https: 443,
-    http: 80
-};

package/dist/core/types/cloudprotocolType.d.ts

@@ -1 +0,0 @@
-export type Protocol = "http" | "https";

package/dist/core/types/cloudprotocolType.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/types/llmResponseType.d.ts

@@ -1,2 +0,0 @@
-import { IKeywordAction, IDragDropAction, IAlertAction, IScrollOrAttributeAction, IManualSteps, IVisionAction, IWaitAction, IClickAction, IVerifyAction, IGetAction, IInputTextExtractor } from "../interfaces/llmResponseInterface.js";
-export type FireFlinkLLMResponse = IKeywordAction | IAlertAction | IDragDropAction | IScrollOrAttributeAction | IManualSteps | IVisionAction | IWaitAction | IClickAction | IVerifyAction | IGetAction | IInputTextExtractor | any;

package/dist/core/types/llmResponseType.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/types/messageType.d.ts

@@ -1,13 +0,0 @@
-export declare const MessageType: {
-    readonly LiveLog: {
-        readonly Ongoing: "liveLogOngoing";
-        readonly Completed: "liveLogCompleted";
-        readonly Error: "liveLogError";
-        readonly Termination: "termination";
-        readonly ExecutionStopped: "executionStopped";
-        readonly retry: "retry";
-    };
-    readonly nextBatch: "nextBatch";
-    readonly AutomationScriptMetaData: "AutomationScriptMetaData";
-};
-export type MessageType = typeof MessageType[keyof typeof MessageType] | typeof MessageType.LiveLog[keyof typeof MessageType.LiveLog];

package/dist/core/types/messageType.js

@@ -1,15 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.MessageType = void 0;
-exports.MessageType = {
-    LiveLog: {
-        Ongoing: "liveLogOngoing",
-        Completed: "liveLogCompleted",
-        Error: "liveLogError",
-        Termination: "termination",
-        ExecutionStopped: "executionStopped",
-        retry: "retry",
-    },
-    nextBatch: "nextBatch",
-    AutomationScriptMetaData: "AutomationScriptMetaData",
-};

package/dist/core/types/promptMap.d.ts

@@ -1,7 +0,0 @@
-import { PromptArgsMap } from "../interfaces/promptInterface.js";
-import { platform } from "../types/scriptTypesType.js";
-export type PromptMap = {
-    [P in platform]: {
-        [K in keyof PromptArgsMap]: (args: PromptArgsMap[K]) => Promise<string>;
-    };
-};

package/dist/core/types/promptMap.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/types/promptType.d.ts

@@ -1,16 +0,0 @@
-export declare enum PromptType {
-    USER_STORY_TO_LIST = "userStoryToList",
-    KEYWORD_EXTRACTOR = "keywordExtractorPrompt",
-    MULTICHANNEL_KEYWORD_EXTRACTOR = "multichannelKeywordExtractorPrompt",
-    ERROR_DESCRIPTION = "errorDescriptionPrompt",
-    FF_INSPECTOR = "ffInspectorNumExtractor",
-    VISION_PROMPT = "visionPrompt",
-    VERIFY_PROMPT = "verifyActionExtractorPrompt",
-    GET_PROMPT = "getActionExtractorPrompt",
-    WAIT_ACTION_EXTRACTOR_PROMPT = "waitActionExtractorPrompt",
-    COMBINED_ACTION_EXTRACTOR_PROMPT = "combinedActionExtractorPrompt",
-    OTP_EXTRACTOR = "otpExtractorPrompt",
-    FILE_VERIFY_PROMPT = "fileVerifyActionExtractorPrompt",
-    INPUT_CONTEXT_PROMPT = "inputContextPrompt",
-    ARITHMETIC_PROMPT = "arithmeticPrompt"
-}

package/dist/core/types/promptType.js

@@ -1,20 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PromptType = void 0;
-var PromptType;
-(function (PromptType) {
-    PromptType["USER_STORY_TO_LIST"] = "userStoryToList";
-    PromptType["KEYWORD_EXTRACTOR"] = "keywordExtractorPrompt";
-    PromptType["MULTICHANNEL_KEYWORD_EXTRACTOR"] = "multichannelKeywordExtractorPrompt";
-    PromptType["ERROR_DESCRIPTION"] = "errorDescriptionPrompt";
-    PromptType["FF_INSPECTOR"] = "ffInspectorNumExtractor";
-    PromptType["VISION_PROMPT"] = "visionPrompt";
-    PromptType["VERIFY_PROMPT"] = "verifyActionExtractorPrompt";
-    PromptType["GET_PROMPT"] = "getActionExtractorPrompt";
-    PromptType["WAIT_ACTION_EXTRACTOR_PROMPT"] = "waitActionExtractorPrompt";
-    PromptType["COMBINED_ACTION_EXTRACTOR_PROMPT"] = "combinedActionExtractorPrompt";
-    PromptType["OTP_EXTRACTOR"] = "otpExtractorPrompt";
-    PromptType["FILE_VERIFY_PROMPT"] = "fileVerifyActionExtractorPrompt";
-    PromptType["INPUT_CONTEXT_PROMPT"] = "inputContextPrompt";
-    PromptType["ARITHMETIC_PROMPT"] = "arithmeticPrompt";
-})(PromptType || (exports.PromptType = PromptType = {}));

package/dist/core/types/scriptTypesType.d.ts

@@ -1 +0,0 @@
-export type platform = "web" | "android" | "ios";

package/dist/core/types/scriptTypesType.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/types/visionllmInputType.d.ts

@@ -1,8 +0,0 @@
-export type visionPromptMessage = ({
-    type: "input_text";
-    text: string;
-} | {
-    type: "input_image";
-    image_url: string;
-    detail: "auto" | "low" | "high";
-})[];

package/dist/core/types/visionllmInputType.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/domAnalysis/getRelaventElements.d.ts

@@ -1,16 +0,0 @@
-export interface DomProcessingRequest {
-    keywords: string[] | string;
-    rawDom: string;
-    stepCount: number;
-    stepAction: string;
-    step: string;
-    platform: string;
-    driver: any;
-}
-export declare class DomProcessingEngine {
-    private simplifier;
-    private searcher;
-    private relationshipBuilder;
-    webProcess(request: DomProcessingRequest): any[];
-    mobileProcess(request: DomProcessingRequest): Promise<any[]>;
-}

package/dist/domAnalysis/getRelaventElements.js

@@ -1,38 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DomProcessingEngine = void 0;
-const simplifyAndFlatten_js_1 = require("./simplifyAndFlatten.js");
-const searchBest_js_1 = require("./searchBest.js");
-const relativeElementsFromDom_js_1 = require("./relativeElementsFromDom.js");
-const logData_js_1 = require("../utils/logger/logData.js");
-class DomProcessingEngine {
-    constructor() {
-        this.simplifier = new simplifyAndFlatten_js_1.DomSimplifier();
-        this.searcher = new searchBest_js_1.DomSearcher();
-        this.relationshipBuilder = new relativeElementsFromDom_js_1.DomRelationshipBuilder();
-    }
-    webProcess(request) {
-        const flat = this.simplifier.simplify(request.rawDom);
-        logData_js_1.logger.saveJSON(flat, `flat-dom-${request.stepCount}`);
-        const searched = this.searcher.search(flat, request.keywords);
-        const structured = this.relationshipBuilder.build(searched);
-        return structured;
-    }
-    async mobileProcess(request) {
-        let grouped;
-        if (request.platform.toLowerCase() === "ios") {
-            const { flatNodes, parentMap, runtimeElements } = await this.simplifier.simplifyIosDom(request.rawDom, request.driver);
-            logData_js_1.logger.saveJSON(flatNodes, `flat-dom-ios-${request.stepCount}`);
-            const { results, indexMap } = await this.searcher.searchBestIos(flatNodes, request.keywords);
-            grouped = await this.relationshipBuilder.buildGroupedResultsIos(results, flatNodes, indexMap, parentMap, runtimeElements);
-        }
-        else {
-            const { flatNodes, parentMap } = await this.simplifier.simplifyAndroidDom(request.rawDom, request.stepAction, request.step);
-            logData_js_1.logger.saveJSON(flatNodes, `flat-dom-android-${request.stepCount}`);
-            const { results, indexMap } = await this.searcher.searchBestAndroid(flatNodes, request.keywords);
-            grouped = await this.relationshipBuilder.buildGroupedResultsAndroid(results, flatNodes, indexMap, parentMap);
-        }
-        return grouped;
-    }
-}
-exports.DomProcessingEngine = DomProcessingEngine;

package/dist/domAnalysis/relativeElementsFromDom.d.ts

@@ -1,5 +0,0 @@
-export declare class DomRelationshipBuilder {
-    build(nodes: any[]): any[];
-    buildGroupedResultsAndroid(results: Record<string, any[]>, nodes: any[], indexMap: Record<string, number>, parentMap: Map<Element, Element | null>): any[][];
-    buildGroupedResultsIos(results: Record<string, any[]>, nodes: any[], indexMap: Record<string, number>, parentMap: Map<any, any>, runtimeElements: Record<number, any>): any[][];
-}

package/dist/domAnalysis/relativeElementsFromDom.js

@@ -1,262 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.DomRelationshipBuilder = void 0;
-class DomRelationshipBuilder {
-    build(nodes) {
-        const ffToXPath = new Map();
-        nodes.forEach(n => {
-            if (n["FF-inspecter"] && n["ff-xpath"]) {
-                ffToXPath.set(n["FF-inspecter"], n["ff-xpath"]);
-            }
-        });
-        const parentXPath = (x) => x.includes("/") ? x.substring(0, x.lastIndexOf("/")) : undefined;
-        const ffToNode = new Map();
-        nodes.forEach(n => {
-            ffToNode.set(n["FF-inspecter"], n);
-        });
-        const structured = [];
-        for (const node of nodes) {
-            const ff = node["FF-inspecter"];
-            const xp = node["ff-xpath"];
-            let parent;
-            const children = [];
-            const siblings = [];
-            if (ff && xp) {
-                const px = parentXPath(xp);
-                for (const [otherFF, otherXP] of ffToXPath.entries()) {
-                    if (otherFF === ff)
-                        continue;
-                    if (otherXP.startsWith(xp + "/")) {
-                        children.push(otherFF);
-                    }
-                    else if (px && parentXPath(otherXP) === px) {
-                        siblings.push(otherFF);
-                    }
-                }
-                for (const [otherFF, otherXP] of ffToXPath.entries()) {
-                    if (otherXP === px) {
-                        parent = otherFF;
-                        break;
-                    }
-                }
-            }
-            const slim = Object.fromEntries(Object.entries(node).filter(([k, v]) => v && !["search", "ff-xpath", "href", "src"].includes(k)));
-            const rel = {};
-            const mini = (n) => Object.fromEntries(Object.entries(n).filter(([k, v]) => v && !["search", "ff-xpath", "href", "src"].includes(k)));
-            if (parent && ffToNode.has(parent)) {
-                rel.parent = mini(ffToNode.get(parent));
-            }
-            if (children.length) {
-                rel.children = children.slice(0, 2)
-                    .map(c => mini(ffToNode.get(c)));
-            }
-            if (siblings.length) {
-                rel.siblings = siblings.slice(0, 2)
-                    .map(s => mini(ffToNode.get(s)));
-            }
-            if (Object.keys(rel).length) {
-                slim["relationship"] = rel;
-            }
-            structured.push(slim);
-        }
-        return structured;
-    }
-    buildGroupedResultsAndroid(results, nodes, indexMap, parentMap) {
-        const groupedResults = [];
-        const usedNodes = new Set();
-        const slim = (n) => {
-            const keys = [
-                "tag",
-                "FF-inspecter",
-                "text",
-                "content-desc",
-                "resource-id",
-                "class",
-                "pane-title",
-                "hint",
-                "clickable",
-                "editable",
-                "enabled"
-            ];
-            const out = {};
-            for (const k of keys) {
-                if (k in n)
-                    out[k] = n[k];
-            }
-            return out;
-        };
-        const addNodeSafe = (nodeDict, group, localUsed) => {
-            const ff = nodeDict["FF-inspecter"];
-            if (!ff || localUsed.has(ff))
-                return;
-            group.push(slim(nodeDict));
-            localUsed.add(ff);
-        };
-        for (const matchedNodes of Object.values(results)) {
-            for (const matchedNode of matchedNodes) {
-                const selectedFF = matchedNode["FF-inspecter"];
-                if (!(selectedFF in indexMap))
-                    continue;
-                if (usedNodes.has(selectedFF))
-                    continue;
-                const item = nodes[indexMap[selectedFF]];
-                const element = item.element;
-                const parent = parentMap.get(element) || null;
-                const group = [];
-                const localUsed = new Set();
-                addNodeSafe(matchedNode, group, localUsed);
-                if (matchedNode["clickable"] === "true") {
-                    groupedResults.push(group);
-                    localUsed.forEach(ff => usedNodes.add(ff));
-                    continue;
-                }
-                if (!parent) {
-                    groupedResults.push(group);
-                    localUsed.forEach(ff => usedNodes.add(ff));
-                    continue;
-                }
-                // parent check
-                if (parent.getAttribute("clickable") === "true" &&
-                    parent.getAttribute("enabled") !== "false") {
-                    addNodeSafe(Object.fromEntries(Array.from(parent?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
-                }
-                const siblings = Array.from(parent?.children ?? []);
-                const idx = siblings.indexOf(element);
-                // upper siblings
-                if (idx > 0) {
-                    for (let j = idx - 1; j >= 0; j--) {
-                        const upper = siblings[j];
-                        if (upper.getAttribute("clickable") === "true" &&
-                            upper.getAttribute("enabled") !== "false") {
-                            addNodeSafe(Object.fromEntries(Array.from(upper?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
-                            break;
-                        }
-                        const upperParent = parentMap.get(upper);
-                        if (upperParent &&
-                            upperParent.getAttribute("clickable") === "true" &&
-                            upperParent.getAttribute("enabled") !== "false") {
-                            addNodeSafe(Object.fromEntries(Array.from(upperParent?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
-                            break;
-                        }
-                    }
-                }
-                // lower siblings
-                if (idx !== -1) {
-                    for (let j = idx + 1; j < siblings.length; j++) {
-                        const lower = siblings[j];
-                        if (lower.getAttribute("clickable") === "true" &&
-                            lower.getAttribute("enabled") !== "false") {
-                            addNodeSafe(Object.fromEntries(Array.from(lower?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
-                            break;
-                        }
-                        for (const child of Array.from(lower?.children ?? [])) {
-                            if (child.getAttribute("clickable") === "true" &&
-                                child.getAttribute("enabled") !== "false") {
-                                addNodeSafe(Object.fromEntries(Array.from(child?.attributes ?? []).map(a => [a.name, a.value])), group, localUsed);
-                                break;
-                            }
-                        }
-                    }
-                }
-                groupedResults.push(group);
-                localUsed.forEach(ff => usedNodes.add(ff));
-            }
-        }
-        const firstMultiIndex = {};
-        groupedResults.forEach((group, i) => {
-            if (group.length > 1) {
-                group.forEach(node => {
-                    const ff = node["FF-inspecter"];
-                    if (!(ff in firstMultiIndex)) {
-                        firstMultiIndex[ff] = i;
-                    }
-                });
-            }
-        });
-        const finalResults = [];
-        const usedFF = new Set();
-        groupedResults.forEach((group, i) => {
-            const newGroup = [];
-            group.forEach(node => {
-                const ff = node["FF-inspecter"];
-                if (usedFF.has(ff))
-                    return;
-                if (group.length === 1 && ff in firstMultiIndex)
-                    return;
-                if (ff in firstMultiIndex && firstMultiIndex[ff] !== i)
-                    return;
-                newGroup.push(node);
-                usedFF.add(ff);
-            });
-            if (newGroup.length)
-                finalResults.push(newGroup);
-        });
-        return finalResults;
-    }
-    buildGroupedResultsIos(results, nodes, indexMap, parentMap, runtimeElements) {
-        const grouped = new Map();
-        const slimIOS = (n) => {
-            const keys = [
-                "tag",
-                "FF-inspecter",
-                "value",
-                "name",
-                "label",
-                "class",
-                "pane-title",
-                "accessible",
-                "editable",
-                "enabled",
-                "traits"
-            ];
-            const out = {};
-            for (const k of keys) {
-                if (k in n)
-                    out[k] = n[k];
-            }
-            return out;
-        };
-        for (const matchedNodes of Object.values(results)) {
-            for (const node of matchedNodes) {
-                const ff = node["FF-inspecter"];
-                if (!ff || !(ff in indexMap))
-                    continue;
-                const item = nodes[indexMap[ff]];
-                const element = runtimeElements[item.id];
-                const parent = parentMap.get(element);
-                const parentKey = parent ? parent : ff;
-                if (!grouped.has(parentKey)) {
-                    grouped.set(parentKey, []);
-                }
-                grouped.get(parentKey).push(slimIOS(node));
-            }
-        }
-        const groupedResults = [];
-        for (const group of grouped.values()) {
-            const seenFF = new Set();
-            const uniqueGroup = [];
-            for (const node of group) {
-                const ff = node["FF-inspecter"];
-                if (!ff || seenFF.has(ff))
-                    continue;
-                seenFF.add(ff);
-                uniqueGroup.push(node);
-            }
-            if (uniqueGroup.length) {
-                groupedResults.push(uniqueGroup);
-            }
-        }
-        const ffNumber = (node) => {
-            const ff = node["FF-inspecter"] || "";
-            const parts = ff.split("-");
-            return parseInt(parts[parts.length - 1]) || 0;
-        };
-        groupedResults.sort((a, b) => {
-            const minA = Math.min(...a.map(ffNumber));
-            const minB = Math.min(...b.map(ffNumber));
-            return minA - minB;
-        });
-        return groupedResults;
-    }
-}
-exports.DomRelationshipBuilder = DomRelationshipBuilder;

package/dist/domAnalysis/searchBest.d.ts

@@ -1,12 +0,0 @@
-import { FlatNode } from "../core/interfaces/simplifyAndFlattenInterface";
-export declare class DomSearcher {
-    search(nodes: FlatNode[], keywords: string[] | string, topN?: number, threshold?: number): any[];
-    searchBestAndroid(nodes: FlatNode[], keywords: string[] | string, topN?: number, threshold?: number): {
-        results: Record<string, any[]>;
-        indexMap: Record<string, number>;
-    };
-    searchBestIos(nodes: any[], keywords: string[] | string, topN?: number, threshold?: number): {
-        results: Record<string, any[]>;
-        indexMap: Record<string, number>;
-    };
-}