ff-automationv2 2.2.32 → 2.2.34-beta.0

package/dist/core/main/runAutomationScript.js

@@ -205,6 +205,7 @@ class AutomationRunner {
             currentPlatform = context.activePlatform;
             (0, platformContext_js_1.setPlatform)(currentPlatform);
             let stepResult;
+            let driverContext = null;
             let stepAction;
             let fireflinkIndex;
             let selector;
@@ -266,7 +267,7 @@ class AutomationRunner {
                     stepsInputData.currentStepIndex = currentStepIndex;
                     context.executor?.setStepContext(currentStepIndex, currentStep, stepsInputData, step);
                     logData_js_1.logger.info("stepsInputData", stepsInputData);
-                    const inputContextPromise = isStepValueDependable && stepsInputData.length > 0
+                    const inputContextPromise = action !== "arithmetic" && isStepValueDependable && stepsInputData.length > 0
                         ? stepProcessor.getLLMResponse({
                             platform: currentPlatform,
                             type: promptType_js_1.PromptType.INPUT_CONTEXT_PROMPT,
@@ -297,6 +298,36 @@ class AutomationRunner {
                         stepCount++;
                         continue;
                     }
+                    if (action === "arithmetic") {
+                        const arithmeticResult = await stepProcessor.getLLMResponse({
+                            platform: currentPlatform,
+                            type: promptType_js_1.PromptType.ARITHMETIC_PROMPT,
+                            args: {
+                                inputStepsContext: isStepValueDependable ? stepsInputData : [],
+                            },
+                            input: { currentStep },
+                        });
+                        let targetVarName = arithmeticResult?.response?.variableName?.trim() || "";
+                        if (targetVarName) {
+                            targetVarName = (0, elementSelectorResolver_js_1.generateUniqueName)(targetVarName, usedVariableNames);
+                        }
+                        handler = actionHandlers["arithmetic"];
+                        await handler({
+                            operandOne: arithmeticResult?.response?.operandOne,
+                            operandTwo: arithmeticResult?.response?.operandTwo,
+                            operandOneVariableName: arithmeticResult?.response?.operandOneVariableName,
+                            operandTwoVariableName: arithmeticResult?.response?.operandTwoVariableName,
+                            operation: arithmeticResult?.response?.operation,
+                            variableName: targetVarName,
+                            returnTypeVariableName: targetVarName,
+                            stepIndex: currentStepIndex,
+                            currentStep,
+                            manualStep: step,
+                            stepsInputData,
+                        });
+                        stepCount++;
+                        continue;
+                    }
                     if (action === "getOtp") {
                         if (!isWebMobile) {
                             throw new Error(`Selected platform is invalid for the step: ${step}`);
@@ -353,6 +384,8 @@ class AutomationRunner {
                         }
                         else {
                             driver = await context.mobileSession.getCurrentDriver();
+                            const contexts = await driver.getContexts();
+                            driverContext = contexts.find((context) => String(context).startsWith("WEBVIEW")) || null;
                         }
                     }
                     catch (error) {
@@ -410,10 +443,15 @@ class AutomationRunner {
                     if (inputContextResult?.response?.variableName && inputContextResult.response.variableName.trim() !== "") {
                         currentVariableName = inputContextResult.response.variableName.trim();
                     }
-                    inputValue =
-                        inputText && inputText.trim().length > 0
-                            ? inputText
-                            : stepResult?.response?.inputText;
+                    if (action === "verify") {
+                        inputValue = stepResult?.response?.inputText;
+                    }
+                    else {
+                        inputValue =
+                            inputText && inputText.trim().length > 0
+                                ? inputText
+                                : stepResult?.response?.inputText;
+                    }
                     if (supportedActions_js_1.SKIPPED_ACTIONS.includes(stepAction)) {
                         handler = actionHandlers[stepAction];
                         await handler();
@@ -483,10 +521,10 @@ class AutomationRunner {
                                 selector = "";
                                 iframeDomHtml = null;
                                 beforeHandles = null;
-                                const retryDomResult = await runnerDomPipeline_js_1.RunnerDomPipeline.captureAndParseDom(driver, currentPlatform, stepCount, action, true);
+                                const retryDomResult = await runnerDomPipeline_js_1.RunnerDomPipeline.captureAndParseDom(driver, currentPlatform, stepCount, action, true, driverContext);
                                 domInfo = retryDomResult.domInfo;
                                 doc = retryDomResult.doc;
-                                const retryRelevantResult = await runnerDomPipeline_js_1.RunnerDomPipeline.processRelevantDom(domProcessor, domInfo, keywords, stepCount, action, currentStep, currentPlatform, driver, true);
+                                const retryRelevantResult = await runnerDomPipeline_js_1.RunnerDomPipeline.processRelevantDom(domProcessor, domInfo, keywords, stepCount, action, currentStep, currentPlatform, driver, true, driverContext);
                                 extractedRelevantDom = retryRelevantResult.extractedRelevantDom;
                                 const retryToonDom = retryRelevantResult.toonDom;
                                 const retryActionResolution = await (0, stepActionInspector_js_1.resolveStepActionFromDom)(stepProcessor, currentPlatform, action, retryToonDom, priorAndNextSteps, currentStep, null);
@@ -525,7 +563,7 @@ class AutomationRunner {
                                 logData_js_1.logger.error(`element not found for step "${step}"`);
                                 throw new Error(`element not found for step "${step}"`);
                             }
-                            const selectorResult = await (0, elementSelectorResolver_js_1.resolveElementSelector)(doc, domInfo, fireflinkIndex, currentPlatform, driver);
+                            const selectorResult = await (0, elementSelectorResolver_js_1.resolveElementSelector)(doc, domInfo, fireflinkIndex, currentPlatform, driver, fallbackAttempt, driverContext);
                             selector = selectorResult.selector;
                             doc = selectorResult.doc;
                             domInfo = selectorResult.domInfo;
@@ -563,6 +601,9 @@ class AutomationRunner {
                             });
                             preCapturedDomInfo = await (0, runnerPostActionHelper_js_1.handlePostActionWeb)(currentPlatform, driver, action, beforeHandles, context.scriptAppender, stepCount);
                             if (fallbackAttempt === 2) {
+                                if (driverContext) {
+                                    await driver.switchContext("NATIVE_APP");
+                                }
                                 await reporter.sendRetry("Retry successful. Moving on to the next step.");
                             }
                             break;
@@ -597,11 +638,15 @@ class AutomationRunner {
                                 actionHandlers,
                                 reporter,
                                 causeError: error,
+                                driverContext,
                             });
                             if (visionResult.retryNeeded) {
                                 continue;
                             }
                             preCapturedDomInfo = visionResult.preCapturedDomInfo;
+                            if (driverContext) {
+                                await driver.switchContext("NATIVE_APP");
+                            }
                             break;
                         }
                     }

package/dist/core/types/promptType.d.ts

@@ -10,5 +10,6 @@ export declare enum PromptType {
     COMBINED_ACTION_EXTRACTOR_PROMPT = "combinedActionExtractorPrompt",
     OTP_EXTRACTOR = "otpExtractorPrompt",
     FILE_VERIFY_PROMPT = "fileVerifyActionExtractorPrompt",
-    INPUT_CONTEXT_PROMPT = "inputContextPrompt"
+    INPUT_CONTEXT_PROMPT = "inputContextPrompt",
+    ARITHMETIC_PROMPT = "arithmeticPrompt"
 }

package/dist/core/types/promptType.js

@@ -15,4 +15,5 @@ var PromptType;
     PromptType["OTP_EXTRACTOR"] = "otpExtractorPrompt";
     PromptType["FILE_VERIFY_PROMPT"] = "fileVerifyActionExtractorPrompt";
     PromptType["INPUT_CONTEXT_PROMPT"] = "inputContextPrompt";
+    PromptType["ARITHMETIC_PROMPT"] = "arithmeticPrompt";
 })(PromptType || (exports.PromptType = PromptType = {}));

package/dist/fireflinkData/fireflinkLocators/getListOfLocators.d.ts

@@ -1,6 +1,7 @@
+import { platform } from "../../core/types/scriptTypesType.js";
 import { LocatorWorkerPool } from "../../utils/workers/LocatorWorkerPool.js";
 export declare class ElementGetter {
     private readonly workerPool;
     constructor(workerPool: LocatorWorkerPool);
-    getFireFlinkElement(html: string, attribute: string): Promise<any>;
+    getFireFlinkElement(html: string, attribute: string, targetPlatform?: platform): Promise<any>;
 }

package/dist/fireflinkData/fireflinkLocators/getListOfLocators.js

@@ -8,8 +8,8 @@ class ElementGetter {
     constructor(workerPool) {
         this.workerPool = workerPool;
     }
-    async getFireFlinkElement(html, attribute) {
-        const platform = (0, platformContext_js_1.getPlatform)();
+    async getFireFlinkElement(html, attribute, targetPlatform) {
+        const platform = targetPlatform || (html?.includes("ff-inspect") ? "web" : (0, platformContext_js_1.getPlatform)());
         // const formattedAttribute = attribute.replace('[', '//*[@');
         try {
             const result = await this.workerPool.getFireFlinkElement(html, attribute, platform);

package/dist/fireflinkData/fireflinkScript/appendScriptData.js

@@ -3,10 +3,37 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.appendScriptData = appendScriptData;
 const logData_js_1 = require("../../utils/logger/logData.js");
 async function appendScriptData(args) {
+    const isMobile = args.platform === "android" || args.platform === "ios";
+    const isWebView = args.pageDOM?.includes("ff-inspect");
+    if (isMobile) {
+        if (args.scriptDataAppender.currentMobileView === undefined) {
+            args.scriptDataAppender.currentMobileView = "NATIVE";
+        }
+        if (isWebView && args.scriptDataAppender.currentMobileView !== "WEBVIEW") {
+            args.scriptDataAppender.add(async () => ({
+                nlpName: "MOB_SwitchToView",
+                stepInputs: ["WEBVIEW"],
+                elementsData: []
+            }));
+            args.scriptDataAppender.currentMobileView = "WEBVIEW";
+        }
+        else if (args.pageDOM && !isWebView && args.scriptDataAppender.currentMobileView === "WEBVIEW") {
+            args.scriptDataAppender.add(async () => ({
+                nlpName: "MOB_SwitchToNativeView",
+                stepInputs: [],
+                elementsData: []
+            }));
+            args.scriptDataAppender.currentMobileView = "NATIVE";
+        }
+    }
     args.scriptDataAppender.add(async () => {
-        const searchAttribute = args.platform === "web"
-            ? `[ff-inspect="${args.ffNumber}"]`
-            : `[FF-inspecter="${args.ffNumber}"]`;
+        const isWebView = args.pageDOM?.includes("ff-inspect");
+        const effectivePlatform = isWebView ? "web" : args.platform;
+        const searchAttribute = args.ffNumber
+            ? (effectivePlatform === "web"
+                ? `[ff-inspect="${args.ffNumber}"]`
+                : `[FF-inspecter="${args.ffNumber}"]`)
+            : args.selector;
         const data = {
             nlpName: args.nlpName
         };
@@ -19,9 +46,12 @@ async function appendScriptData(args) {
         if (args.elementName) {
             let locators;
             try {
-                const ffElement = await args.elementGetter.getFireFlinkElement(args.pageDOM, searchAttribute);
+                if (!searchAttribute) {
+                    throw new Error("searchAttribute is not found");
+                }
+                const ffElement = await args.elementGetter.getFireFlinkElement(args.pageDOM, searchAttribute, effectivePlatform);
                 // locators = [{ name: "xpath", value: args.selector }];
-                if (ffElement?.locators?.length === 0) {
+                if (!ffElement?.locators || ffElement.locators.length === 0) {
                     throw new Error("locators is not found");
                 }
                 else {
@@ -30,7 +60,13 @@ async function appendScriptData(args) {
             }
             catch (error) {
                 logData_js_1.logger.info(`Element is not found ${error.message}`);
-                locators = [{ name: "xpath", value: args.selector }];
+                locators = [{
+                        name: "xpath",
+                        type: "static",
+                        value: args.selector,
+                        isRecorded: "N",
+                        isAiRecorded: "Y"
+                    }];
             }
             data.elementsData = [{
                     name: args.elementName,

package/dist/fireflinkData/fireflinkScript/scriptGenrationData.d.ts

@@ -1,5 +1,6 @@
 import { IScriptGenerationData } from "../../core/interfaces/scriptGenrationDataInterface.js";
 export declare class ScriptDataAppender {
+    currentMobileView: "NATIVE" | "WEBVIEW";
     private data;
     private pending;
     private errorNLP;

package/dist/fireflinkData/fireflinkScript/scriptGenrationData.js

@@ -4,6 +4,7 @@ exports.ScriptDataAppender = void 0;
 const platformSyntax_js_1 = require("../../utils/helpers/platformSyntax.js");
 class ScriptDataAppender {
     constructor() {
+        this.currentMobileView = "NATIVE";
         this.data = [];
         this.pending = [];
         this.errorNLP = "";

package/dist/imageAnalysisWeb/annotateScreenshot.d.ts

@@ -1,2 +1,2 @@
 import { NodeItem } from "./screenshotInterface.js";
-export declare function annotateScreenshot(screenshotBase64: string, nodes: NodeItem[]): Promise<Buffer>;
+export declare function annotateScreenshot(screenshotBase64: string, deviceWidth: number, deviceHeight: number, nodes: NodeItem[]): Promise<Buffer>;

package/dist/imageAnalysisWeb/annotateScreenshot.js

@@ -59,10 +59,16 @@ function findFreeTagPosition(box, tagW, tagH, elementBoxes, usedTagRects, imgWid
     usedTagRects.push([fallbackX, fallbackY, fallbackX + tagW, fallbackY + tagH]);
     return { x: fallbackX, y: fallbackY };
 }
-async function annotateScreenshot(screenshotBase64, nodes) {
+async function annotateScreenshot(screenshotBase64, deviceWidth, deviceHeight, nodes) {
     const buffer = Buffer.from(screenshotBase64, "base64");
     const image = await (0, canvas_1.loadImage)(buffer);
-    const scale = image.width / 1920;
+    // Actual screenshot resolution
+    const screenshotWidth = image.width;
+    const screenshotHeight = image.height;
+    // Device / WebView resolution from which bbox coordinates are coming
+    const scaleX = screenshotWidth / deviceWidth;
+    const scaleY = screenshotHeight / deviceHeight;
+    const scale = Math.min(scaleX, scaleY);
     const fontSize = Math.max(16, Math.floor(20 * scale));
     const canvas = (0, canvas_1.createCanvas)(image.width, image.height);
     const ctx = canvas.getContext("2d");
@@ -71,20 +77,28 @@ async function annotateScreenshot(screenshotBase64, nodes) {
     ctx.textBaseline = "middle";
     ctx.textAlign = "center";
     const usedTagRects = [];
+    /*
+    * Convert all bounding boxes from device/WebView
+    * coordinates to screenshot coordinates.
+    */
     const elementBoxes = nodes.map((n) => [
-        Math.floor(n.bbox.x),
-        Math.floor(n.bbox.y),
-        Math.floor(n.bbox.x + n.bbox.w),
-        Math.floor(n.bbox.y + n.bbox.h),
+        Math.floor(n.bbox.x * scaleX),
+        Math.floor(n.bbox.y * scaleY),
+        Math.floor((n.bbox.x + n.bbox.w) * scaleX),
+        Math.floor((n.bbox.y + n.bbox.h) * scaleY),
     ]);
     nodes.forEach((item, i) => {
         const color = COLORS[i % COLORS.length];
         const colorStr = `rgb(${color[0]},${color[1]},${color[2]})`;
         const label = item.ff.split("-").pop() || "";
-        const x1 = Math.floor(item.bbox.x);
-        const y1 = Math.floor(item.bbox.y);
-        const x2 = Math.floor(item.bbox.x + item.bbox.w);
-        const y2 = Math.floor(item.bbox.y + item.bbox.h);
+        /*
+        * Convert current node bbox to actual
+        * screenshot coordinates.
+        */
+        const x1 = Math.floor(item.bbox.x * scaleX);
+        const y1 = Math.floor(item.bbox.y * scaleY);
+        const x2 = Math.floor((item.bbox.x + item.bbox.w) * scaleX);
+        const y2 = Math.floor((item.bbox.y + item.bbox.h) * scaleY);
         ctx.strokeStyle = colorStr;
         ctx.lineWidth = Math.max(2, Math.floor(2 * scale));
         ctx.strokeRect(x1, y1, x2 - x1, y2 - y1);

package/dist/imageAnalysisWeb/generateAnnotatedScreenshots.js

@@ -14,13 +14,16 @@ async function generateAnnotatedScreenshots(driver) {
     //   driver.execute(JS_PRIMARY),
     //   // driver.execute(JS_SECONDARY),
     // ]);
+    const windowSize = await driver.getWindowSize();
+    const deviceWidth = windowSize.width;
+    const deviceHeight = windowSize.height;
     const primary = primaryRaw ?? [];
     let secondary = secondaryRaw ?? [];
     const primaryIds = new Set(primary.map((item) => item.ff));
     secondary = secondary.filter((item) => !primaryIds.has(item.ff));
     const [primaryAnnotated, secondaryAnnotated] = await Promise.all([
-        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, primary),
-        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, secondary)
+        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, deviceWidth, deviceHeight, primary),
+        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, deviceWidth, deviceHeight, secondary)
     ]);
     // Uncomment only when debugging
     logData_js_1.logger.saveImage(primaryAnnotated, "annotated_primary.png");