ff-automationv2 2.2.28 → 2.2.29

package/dist/core/main/runAutomationScript.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.AutomationRunner = void 0;
+require("../../utils/sslConfig/apiCallPatch.js");
+const platformContext_js_1 = require("../../utils/helpers/platformContext.js");
 const llmConfiguration_js_1 = require("../../llmConfig/llmConfiguration.js");
 const llmAction_js_1 = require("../../ai/llmcalls/llmAction.js");
 const executionContext_js_1 = require("./executionContext.js");
@@ -26,6 +28,9 @@ const logData_js_1 = require("../../utils/logger/logData.js");
 const sameActionsHelper_js_1 = require("../../utils/helpers/sameActionsHelper.js");
 const node_console_1 = require("node:console");
 const annotatedScreenshotMobile_js_1 = require("../../imageAnalysisMobile/annotatedScreenshotMobile.js");
+const toon_1 = require("@toon-format/toon");
+const platformSyntax_js_1 = require("../../utils/helpers/platformSyntax.js");
+// import { encode } from '@toon-format/toon';
 class AutomationRunner {
     constructor() {
         this.KafkaPayload = {};
@@ -41,14 +46,19 @@ class AutomationRunner {
             logData_js_1.logger.info("Starting cleanup process...");
             domInfo = null;
             extractedRelevantDom = null;
-            if ((context?.session) || (context?.androidSession)) {
+            if ((context?.session) || (context?.mobileSession)) {
                 try {
                     const platformHandlers = {
                         android: async () => {
                             if (driver?.sessionId) {
                                 await driver.deleteSession();
+                                logData_js_1.logger.info("Android session deleted successfully");
+                            }
+                        },
+                        ios: async () => {
+                            if (driver?.sessionId) {
+                                logData_js_1.logger.info("iOS session is not deleted");
                             }
-                            logData_js_1.logger.info("Android session deleted successfully");
                         },
                         web: async () => {
                             if (request.isCloud) {
@@ -91,18 +101,19 @@ class AutomationRunner {
         AutomationRunner.sessionTerminationDetails[request.testCaseId] = request.isTerminate;
         const apiService = new fireflinkApi_service_js_1.FireFlinkApiService();
         const fireflinkServiceInstance = new fireflink_service_js_1.fireflinkService(apiService, request.broker, request.sslConfig);
-        const instanceDetails = await fireflinkServiceInstance.runGetInstancesDetails(request.projectId, request.token, request.serverHost);
+        const instanceDetails = await fireflinkServiceInstance.runGetInstancesDetails(request.projectId, request.token, request.serverHost, request.licenseId);
         const aiInfo = (0, getInstancesInfo_js_1.getAiInstanceInfo)(instanceDetails.responseObject);
         const context = new executionContext_js_1.ExecutionContext(request);
-        const configProvider = new llmConfiguration_js_1.ServiceProviderBaseUrlProvider();
-        const baseUrl = configProvider.getBaseUrl(aiInfo);
+        const configProvider = new llmConfiguration_js_1.ServiceProviderProviderConfiguration();
+        const provider = await configProvider.getProvider(aiInfo);
         let domInfo = null;
         let extractedRelevantDom = null;
         let errorInfo = {};
-        const llm = new llmAction_js_1.llmAction(aiInfo.apiKey, baseUrl, aiInfo.model, aiInfo.visionApiKey);
+        const llm = new llmAction_js_1.llmAction(aiInfo.apiKey, provider, aiInfo.model, aiInfo.visionApiKey, aiInfo.accountType, aiInfo.enterpriseConfigFile, aiInfo.serviceProvider);
         let preCapturedDomInfo = null;
         const stepProcessor = new stepProcessor_js_1.StepProcessor(llm);
         const platform = request.platform.toLocaleLowerCase();
+        (0, platformContext_js_1.setPlatform)(platform);
         const actionHandlers = (0, actionHandlerFactory_js_1.createActionHandlers)(context);
         const domProcessor = new getRelaventElements_js_1.DomProcessingEngine();
         let stepCount = 1;
@@ -112,6 +123,7 @@ class AutomationRunner {
             logData_js_1.logger.info(listOfSteps);
             throw new Error("No executable manual steps were returned by the LLM.");
         }
+        logData_js_1.logger.info(`Service Provider --> ${aiInfo.serviceProvider} & Model --> ${aiInfo.model} `);
         const start_time = new Date().getTime();
         for (const step of listOfSteps) {
             let stepResult;
@@ -140,6 +152,19 @@ class AutomationRunner {
                     input: { currentStep: step }
                 });
                 const action = result.response.action;
+                if (action === 'notwebopenaction') {
+                    throw new Error(`Selected platform is invalid for the step: ${step}`);
+                }
+                if (!allAction_js_1.allKeywordAction.includes(action)) {
+                    throw new Error(`Unsupported action at action or keyword exracter: ${action}`);
+                }
+                let handler = actionHandlers[action];
+                if (supportedActions_js_1.INPUTLESS_ACTIONS.includes(action)) {
+                    logData_js_1.logger.info(result.response.keywords);
+                    await handler({ keywords: result.response.keywords, step: step });
+                    stepCount++;
+                    continue;
+                }
                 let elementName = result.response.elementName;
                 if (elementName) {
                     const baseName = elementName;
@@ -153,30 +178,30 @@ class AutomationRunner {
                     result.response.elementName = elementName;
                     usedElementNames.push(elementName);
                 }
-                if (!allAction_js_1.allKeywordAction.includes(action)) {
-                    throw new Error(`Unsupported action at action or keyword exracter: ${action}`);
-                }
-                let handler = actionHandlers[action];
-                if (supportedActions_js_1.INPUTLESS_ACTIONS.includes(action)) {
-                    logData_js_1.logger.info(result.response.keywords);
-                    await handler({ keywords: result.response.keywords, step: step });
-                    stepCount++;
-                    continue;
-                }
-                if (platform == "web") {
-                    driver = await context.session.getCurrentDriver();
+                try {
+                    if (platform == "web") {
+                        driver = await context.session.getCurrentDriver();
+                    }
+                    else {
+                        driver = await context.mobileSession.getCurrentDriver();
+                    }
                 }
-                else {
-                    driver = await context.androidSession.getCurrentDriver();
+                catch (error) {
+                    if (action == "verify") {
+                        driver = null;
+                    }
+                    else {
+                        throw error;
+                    }
                 }
-                if (await (0, isAlertPresent_js_1.isAlertPresent)(driver) && action != "wait") {
+                if (await (0, isAlertPresent_js_1.isAlertPresent)(driver) && !["wait", "verify"].includes(action)) {
                     logData_js_1.logger.info("Alert is present");
                     stepResult = await stepProcessor.getLLMResponse({
                         platform: platform,
                         type: promptType_js_1.PromptType.FF_INSPECTOR,
                         args: {
                             stepAction: action,
-                            extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
+                            extractedDomJson: null,
                             priorAndNextSteps,
                             isAlert: true,
                             isDrag: false
@@ -186,50 +211,78 @@ class AutomationRunner {
                     stepAction = stepResult.response.action;
                     handler = actionHandlers[stepAction];
                     await handler({
-                        value: stepResult.response.input_text
+                        value: stepResult.response.inputText
                     });
+                    stepCount++;
                     continue;
                 }
                 else {
-                    if (preCapturedDomInfo) {
+                    let toonDom;
+                    if (preCapturedDomInfo && platform === "web") {
                         logData_js_1.logger.info("Using pre-captured DOM from previous toaster detection");
                         domInfo = preCapturedDomInfo;
                         preCapturedDomInfo = null;
                     }
                     else {
-                        if (platform === "web") {
-                            domInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOM)(driver);
-                            logData_js_1.logger.saveDOM(domInfo.dom, `annotated-dom-${stepCount}`);
+                        try {
+                            if (platform === "web") {
+                                domInfo = await (0, jsForAttributeInjection_js_1.getAnnotatedDOM)(driver);
+                                logData_js_1.logger.saveDOM(domInfo.dom, `annotated-dom-${stepCount}`);
+                                const parser = new jsdom_1.JSDOM(domInfo.dom);
+                                doc = parser.window.document;
+                            }
+                            else {
+                                domInfo = await (0, jsForAttributeInjection_js_2.getAnnotatedDOMForMobile)(driver);
+                                logData_js_1.logger.saveDOM(domInfo.dom, `annotated-dom-Mob-${stepCount}`);
+                                const parser = new jsdom_1.JSDOM(domInfo.dom, {
+                                    contentType: "text/xml"
+                                });
+                                doc = parser.window.document;
+                            }
+                            if (platform === "web") {
+                                extractedRelevantDom = await domProcessor.webProcess({
+                                    keywords: result.response.keywords,
+                                    rawDom: domInfo.dom,
+                                    stepCount: stepCount,
+                                    stepAction: action,
+                                    step: step,
+                                    platform: platform,
+                                    driver: driver
+                                });
+                                toonDom = (0, toon_1.encode)(extractedRelevantDom || []);
+                                // logger.saveJSON(extractedRelevantDom, `relevant-dom-${stepCount}`);
+                                logData_js_1.logger.saveText(toonDom, `relevant-dom-toon-${stepCount}`);
+                            }
+                            else {
+                                extractedRelevantDom = await domProcessor.mobileProcess({
+                                    keywords: result.response.keywords,
+                                    rawDom: domInfo.dom,
+                                    stepCount: stepCount,
+                                    stepAction: action,
+                                    step: step,
+                                    platform: platform,
+                                    driver: driver
+                                });
+                                toonDom = (0, toon_1.encode)(extractedRelevantDom || []);
+                                // logger.saveJSON(extractedRelevantDom, `relevant-dom-Mob-${stepCount}`);
+                                logData_js_1.logger.saveText(toonDom, `relevant-dom-toon-Mob-${stepCount}`);
+                            }
                         }
-                        else {
-                            domInfo = await (0, jsForAttributeInjection_js_2.getAnnotatedDOMForMobile)(driver);
-                            logData_js_1.logger.saveDOM(domInfo.dom, `annotated-dom-Mob-${stepCount}`);
+                        catch (error) {
+                            if (action == "verify") {
+                                extractedRelevantDom = null;
+                            }
+                            else {
+                                throw error;
+                            }
                         }
                     }
-                    const parser = new jsdom_1.JSDOM(domInfo.dom);
-                    doc = parser.window.document;
-                    if (platform === "web") {
-                        extractedRelevantDom = domProcessor.webProcess({
-                            keywords: result.response.keywords,
-                            rawDom: domInfo.dom,
-                            stepCount: stepCount
-                        });
-                        logData_js_1.logger.saveJSON(extractedRelevantDom, `relevant-dom-${stepCount}`);
-                    }
-                    else {
-                        extractedRelevantDom = domProcessor.mobileProcess({
-                            keywords: result.response.keywords,
-                            rawDom: domInfo.dom,
-                            stepCount: stepCount
-                        });
-                        logData_js_1.logger.saveJSON(extractedRelevantDom, `relevant-dom-Mob-${stepCount}`);
-                    }
                     if (action == "verify") {
                         stepResult = await stepProcessor.getLLMResponse({
                             platform: platform,
                             type: promptType_js_1.PromptType.VERIFY_PROMPT,
                             args: {
-                                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
+                                extractedDomJson: toonDom,
                                 priorAndNextSteps,
                             },
                             input: { currentStep: step }
@@ -241,7 +294,7 @@ class AutomationRunner {
                             platform: platform,
                             type: promptType_js_1.PromptType.GET_PROMPT,
                             args: {
-                                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
+                                extractedDomJson: toonDom,
                                 priorAndNextSteps,
                             },
                             input: { currentStep: step }
@@ -253,7 +306,7 @@ class AutomationRunner {
                             platform: platform,
                             type: promptType_js_1.PromptType.WAIT_ACTION_EXTRACTOR_PROMPT,
                             args: {
-                                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
+                                extractedDomJson: toonDom,
                                 priorAndNextSteps,
                             },
                             input: { currentStep: step }
@@ -265,7 +318,7 @@ class AutomationRunner {
                             platform: platform,
                             type: promptType_js_1.PromptType.COMBINED_ACTION_EXTRACTOR_PROMPT,
                             args: {
-                                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
+                                extractedDomJson: toonDom,
                                 priorAndNextSteps,
                             },
                             input: { currentStep: step }
@@ -277,7 +330,7 @@ class AutomationRunner {
                             platform: platform,
                             type: promptType_js_1.PromptType.FF_INSPECTOR,
                             args: {
-                                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
+                                extractedDomJson: toonDom,
                                 priorAndNextSteps,
                                 isDrag: true,
                                 isAlert: false
@@ -292,7 +345,7 @@ class AutomationRunner {
                             type: promptType_js_1.PromptType.FF_INSPECTOR,
                             args: {
                                 stepAction: action,
-                                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
+                                extractedDomJson: toonDom,
                                 priorAndNextSteps,
                                 isAlert: false,
                                 isDrag: false
@@ -307,9 +360,15 @@ class AutomationRunner {
                         }
                     }
                 }
-                if (platform === "android") {
+                if (platform === "android" || platform === "ios") {
                     stepAction = sameActionsHelper_js_1.sameActionHelper[stepAction] || stepAction;
-                    logData_js_1.logger.info(`step action: ${stepAction}`);
+                    // logger.info(`step action: ${stepAction}`);
+                }
+                if (supportedActions_js_1.SKIPPED_ACTIONS.includes(stepAction)) {
+                    handler = actionHandlers[stepAction];
+                    await handler();
+                    stepCount++;
+                    continue;
                 }
                 if (supportedActions_js_1.ELEMENTLESS_ACTION.includes(stepAction)) {
                     let currentDriver = null;
@@ -319,10 +378,15 @@ class AutomationRunner {
                         beforeHandles = await currentDriver.getWindowHandles();
                     }
                     else {
-                        currentDriver = await context.androidSession.getCurrentDriver();
+                        currentDriver = await context.mobileSession.getCurrentDriver();
                     }
                     handler = actionHandlers[stepAction];
-                    await handler({ value: stepResult.response.input_text });
+                    await handler({
+                        value: stepResult.response.inputText,
+                        selector: stepResult.response.attributeValue,
+                        elementName: result.response.elementName,
+                        elementType: stepResult.response.elementType
+                    });
                     if ((currentDriver) && (platform == "web")) {
                         await (0, newTabTracker_js_1.newTabTracker)(currentDriver, action, beforeHandles, context.scriptAppender);
                     }
@@ -357,14 +421,20 @@ class AutomationRunner {
                         });
                         continue;
                     }
-                    fireflinkIndex = stepResult.response.attribute_value;
+                    fireflinkIndex = stepResult.response.attributeValue;
                     if (fireflinkIndex == "Fire-Flink-0") {
                         logData_js_1.logger.error(`element not found for step "${step}"`);
                         throw new Error(`element not found for step "${step}"`, { cause: node_console_1.error });
                     }
                     if (platform === "web") {
                         const ele = doc.querySelector(`[ff-inspect="${fireflinkIndex}"]`);
-                        selector = ele.getAttribute("ff-xpath");
+                        if (ele) {
+                            selector = ele.getAttribute("ff-xpath") || "";
+                        }
+                        else {
+                            const result = (0, xpathcreation_js_1.getWebXPathByFFIndex)(domInfo.dom, fireflinkIndex);
+                            selector = result.xpathPath;
+                        }
                     }
                     else {
                         const result = await (0, xpathcreation_js_1.getFFIndexAndXPath)(domInfo.dom, fireflinkIndex);
@@ -399,12 +469,18 @@ class AutomationRunner {
                         handler = actionHandlers[frameAction];
                         await handler({
                             selector: selector,
-                            fireflinkIndex,
                             frames: frames,
                             elementName: result.response.elementName,
-                            elementType: stepResult.response.elementType
+                            elementType: stepResult.response.elementType,
+                            pageDOM: domInfo.dom,
+                            fireflinkIndex
                         });
                     }
+                    if (selector.includes('svg')) {
+                        const match = /svg\[(\d+)\]/.exec(selector);
+                        const svgIndex = match ? match[1] : "1";
+                        selector = selector.replace(/svg\[\d+\]/, `*[name()='svg'][${svgIndex}]`);
+                    }
                     handler = actionHandlers[stepAction];
                     if (iframeDomHtml != null) {
                         domInfo = iframeDomHtml;
@@ -412,15 +488,20 @@ class AutomationRunner {
                     else {
                         domInfo = domInfo.dom;
                     }
+                    // logger.info(`xpath ------>: ${selector}`)
                     await handler({
+                        nlpName: stepResult.response.nlpName,
                         selector: selector,
-                        value: stepResult.response.input_text,
+                        value: stepResult.response.inputText,
                         fireflinkIndex,
                         pageDOM: domInfo,
                         elementName: result.response.elementName,
                         elementType: stepResult.response.elementType,
-                        num_of_scrolls: stepResult.response.num_of_scrolls,
+                        numOfScrolls: stepResult.response.numOfScrolls,
                         direction: stepResult.response.direction,
+                        stepIndex: stepCount,
+                        filteredSteps: listOfSteps,
+                        attributeType: stepResult.response.attributeType,
                     });
                     if (platform === "web") {
                         if (await (0, isAlertPresent_js_1.isAlertPresent)(driver) != true) {
@@ -439,34 +520,71 @@ class AutomationRunner {
                     }
                 }
                 catch (error) {
-                    logData_js_1.logger.error(`execution failed for step "${step}": ${error.message} and going for vision`);
+                    logData_js_1.logger.error(`execution failed for step "${step}": ${error} and going for vision`);
                     try {
                         let annotatedScreenshot;
                         let bounds;
+                        if (action === "scroll") {
+                            throw new Error(`Scroll action failed element not found`, { cause: error });
+                        }
                         if (platform === "web") {
                             annotatedScreenshot = await (0, generateAnnotatedScreenshots_js_1.generateAnnotatedScreenshots)(driver);
                         }
                         else {
-                            const VissionResult = await (0, annotatedScreenshotMobile_js_1.annotateScreenshot)(driver, domInfo.dom);
+                            domInfo = await (0, jsForAttributeInjection_js_2.getAnnotatedDOMForMobile)(driver);
+                            let VissionResult;
+                            if (platform === "android") {
+                                VissionResult = await (0, annotatedScreenshotMobile_js_1.annotateScreenshot)(driver, domInfo.dom);
+                            }
+                            else {
+                                VissionResult = await (0, annotatedScreenshotMobile_js_1.annotateIosScreenshot)(driver, domInfo.dom);
+                            }
                             annotatedScreenshot = VissionResult.boxedImageBuffer;
                             bounds = VissionResult.boxes;
                         }
-                        stepResultVision = await stepProcessor.getLLMResponse({
-                            platform: platform,
-                            type: promptType_js_1.PromptType.VISION_PROMPT,
-                            args: {
-                                priorAndNextSteps
-                            },
-                            input: { currentStep: step, annotatedScreenshot: annotatedScreenshot }
-                        });
+                        if (platform == "web") {
+                            stepResultVision = await stepProcessor.getLLMResponse({
+                                platform: platform,
+                                type: promptType_js_1.PromptType.VISION_PROMPT,
+                                args: {
+                                    priorAndNextSteps
+                                },
+                                input: { currentStep: step, annotatedScreenshot: annotatedScreenshot.primary },
+                                isVision: true
+                            });
+                            if (stepResultVision.response.fireflinkIndex == "Fire-Flink-0") {
+                                stepResultVision = await stepProcessor.getLLMResponse({
+                                    platform: platform,
+                                    type: promptType_js_1.PromptType.VISION_PROMPT,
+                                    args: {
+                                        priorAndNextSteps
+                                    },
+                                    input: { currentStep: step, annotatedScreenshot: annotatedScreenshot.secondary },
+                                    isVision: true
+                                });
+                            }
+                        }
+                        else {
+                            stepResultVision = await stepProcessor.getLLMResponse({
+                                platform: platform,
+                                type: promptType_js_1.PromptType.VISION_PROMPT,
+                                args: {
+                                    priorAndNextSteps
+                                },
+                                input: { currentStep: step, annotatedScreenshot: annotatedScreenshot },
+                                isVision: true
+                            });
+                        }
                         fireflinkIndex = stepResultVision.response.fireflinkIndex;
                         logData_js_1.logger.info(`fireflinkIndex: ${fireflinkIndex}`);
                         if (fireflinkIndex == "Fire-Flink-0") {
                             logData_js_1.logger.error(`vision unable to find element for step "${step}"`);
-                            context.scriptAppender.setErrorNLP(stepAction);
+                            if (platform === "web") {
+                                context.scriptAppender.setErrorNLP(stepAction);
+                            }
                             throw new Error(`vision unable to find element for step "${step}"`, { cause: error });
                         }
-                        if (platform == "android") {
+                        if (platform == "android" || platform == "ios") {
                             const index = Number(fireflinkIndex.split("-").pop());
                             const box = bounds[index - 1];
                             if (!box) {
@@ -474,17 +592,14 @@ class AutomationRunner {
                             }
                             const { x1, y1, x2, y2 } = box;
                             const boundsStr = `[${x1},${y1}][${x2},${y2}]`;
-                            const result = (0, xpathcreation_js_1.getFFIndexAndXPath)(domInfo.dom, boundsStr);
+                            const result = (0, xpathcreation_js_1.getFFIndexAndXPath)(domInfo.dom, boundsStr, platform);
                             xpath = result.xpathPath;
                             fireflinkIndex = result.ffInspector ?? fireflinkIndex;
-                            logData_js_1.logger.info(`fireflinkIndex: ${fireflinkIndex}`);
-                            logData_js_1.logger.info(`xpath: ${xpath}`);
                         }
                         else {
                             const ele = doc.querySelector(`[ff-inspect="${fireflinkIndex}"]`);
                             xpath = ele.getAttribute("ff-xpath");
                             if (!xpath) {
-                                logData_js_1.logger.info(`Unable to resolve xpath for ${fireflinkIndex}`);
                                 context.scriptAppender.setErrorNLP(stepAction);
                                 throw new Error(`Unable to resolve xpath for ${fireflinkIndex}`, { cause: error });
                             }
@@ -495,15 +610,16 @@ class AutomationRunner {
                         else {
                             domInfo = domInfo.dom;
                         }
-                        logData_js_1.logger.info(`elementType: ${stepResult.response.elementType}`);
                         handler = actionHandlers[stepAction];
                         await handler({
                             selector: xpath,
-                            value: stepResult.response.input_text,
+                            value: stepResult.response.inputText,
                             fireflinkIndex,
                             pageDOM: domInfo,
                             elementName: result.response.elementName,
-                            elementType: stepResult.response.elementType
+                            elementType: stepResult.response.elementType,
+                            stepIndex: stepCount,
+                            filteredSteps: listOfSteps
                         });
                         if (await (0, isAlertPresent_js_1.isAlertPresent)(driver) != true) {
                             if (action === "click") {
@@ -565,23 +681,26 @@ class AutomationRunner {
             licenseId: request.licenseId,
             userId: request.userId,
             projectType: request.projectType,
+            // serviceProvider: aiInfo.serviceProvider,
+            // model: aiInfo.model,
+            platform: (0, platformSyntax_js_1.getPlatformSyntax)(platform),
             tokensConsumed: (await stepProcessor.getResultTokenUsage()).totalTokens,
             ...(errorInfo && Object.keys(errorInfo).length > 0
                 ? { error: errorInfo }
                 : {})
         };
         try {
+            // logger.info('payload', payload)
             await context.scriptAppender.waitForAllSteps();
-            logData_js_1.logger.info(context.scriptAppender.getData());
             this.KafkaPayload.topic = `Topic-FIREFLINK-api-requests-${request.env}`;
             this.KafkaPayload.message = {
                 ...payload,
                 stepData: context.scriptAppender.getData(),
                 serverHost: request.serverHost
             };
-            delete payload.tokensConsumed;
-            logData_js_1.logger.info({ ...payload, topic: this.KafkaPayload.topic });
-            await fireflinkServiceInstance.produceMessageToKafka(this.KafkaPayload);
+            // delete payload.tokensConsumed;
+            logData_js_1.logger.info(await fireflinkServiceInstance.produceMessageToKafka(this.KafkaPayload));
+            logData_js_1.logger.info(`Message produced to Kafka topic ${JSON.stringify(this.KafkaPayload, null, 2)} successfully.`);
         }
         catch (error) {
             logData_js_1.logger.error("Failed to produce message :", logData_js_1.logger.getError(error));
@@ -589,7 +708,7 @@ class AutomationRunner {
         }
         finally {
             await this.cleanup(context, domInfo, extractedRelevantDom, stepProcessor, fireflinkServiceInstance, request, platform, (await context.session.getCurrentDriver().catch(() => null)) ??
-                (await context.androidSession.getCurrentDriver().catch(() => null)));
+                (await context.mobileSession.getCurrentDriver().catch(() => null)));
         }
     }
 }

package/dist/core/main/stepProcessor.d.ts

@@ -7,6 +7,7 @@ export interface LLMRequest {
     type: PromptType;
     args: Record<string, any>;
     input: Record<string, any>;
+    isVision?: boolean;
 }
 export declare class StepProcessor {
     private llm;
@@ -17,7 +18,7 @@ export declare class StepProcessor {
         outputTokens: number;
         totalTokens: number;
     }>;
-    getLLMResponse({ platform, type, args, input }: LLMRequest): Promise<{
+    getLLMResponse({ platform, type, args, input, isVision }: LLMRequest): Promise<{
         response: FireFlinkLLMResponse;
     }>;
 }

package/dist/core/main/stepProcessor.js

@@ -10,8 +10,8 @@ class StepProcessor {
     async getResultTokenUsage() {
         return this.resultParser.getTokenUsage();
     }
-    async getLLMResponse({ platform, type, args, input }) {
-        const response = await this.llm.getLLMResponse(platform, type, args, input);
+    async getLLMResponse({ platform, type, args, input, isVision = false }) {
+        const response = await this.llm.getLLMResponse(platform, type, args, input, isVision);
         return this.resultParser.fetchResult(response);
     }
 }

package/dist/core/types/scriptTypesType.d.ts

@@ -1 +1 @@
-export type platform = "web" | "android";
+export type platform = "web" | "android" | "ios";

package/dist/core/types/visionllmInputType.d.ts

@@ -1,9 +1,8 @@
 export type visionPromptMessage = ({
-    type: "text";
+    type: "input_text";
     text: string;
 } | {
-    type: "image_url";
-    image_url: {
-        url: string;
-    };
+    type: "input_image";
+    image_url: string;
+    detail: "auto" | "low" | "high";
 })[];

package/dist/domAnalysis/getRelaventElements.d.ts

@@ -2,11 +2,15 @@ export interface DomProcessingRequest {
     keywords: string[] | string;
     rawDom: string;
     stepCount: number;
+    stepAction: string;
+    step: string;
+    platform: string;
+    driver: any;
 }
 export declare class DomProcessingEngine {
     private simplifier;
     private searcher;
     private relationshipBuilder;
     webProcess(request: DomProcessingRequest): any[];
-    mobileProcess(request: DomProcessingRequest): any[][];
+    mobileProcess(request: DomProcessingRequest): Promise<any[]>;
 }

package/dist/domAnalysis/getRelaventElements.js

@@ -18,11 +18,20 @@ class DomProcessingEngine {
         const structured = this.relationshipBuilder.build(searched);
         return structured;
     }
-    mobileProcess(request) {
-        const { flatNodes, parentMap } = this.simplifier.simplifyMobile(request.rawDom);
-        logData_js_1.logger.saveJSON(flatNodes, `flat-dom-Mob-${request.stepCount}`);
-        const { results, indexMap } = this.searcher.searchBestMobile(flatNodes, request.keywords);
-        const grouped = this.relationshipBuilder.buildGroupedResultsMobile(results, flatNodes, indexMap, parentMap);
+    async mobileProcess(request) {
+        let grouped;
+        if (request.platform.toLowerCase() === "ios") {
+            const { flatNodes, parentMap, runtimeElements } = await this.simplifier.simplifyIosDom(request.rawDom, request.driver);
+            logData_js_1.logger.saveJSON(flatNodes, `flat-dom-ios-${request.stepCount}`);
+            const { results, indexMap } = await this.searcher.searchBestIos(flatNodes, request.keywords);
+            grouped = await this.relationshipBuilder.buildGroupedResultsIos(results, flatNodes, indexMap, parentMap, runtimeElements);
+        }
+        else {
+            const { flatNodes, parentMap } = await this.simplifier.simplifyAndroidDom(request.rawDom, request.stepAction, request.step);
+            logData_js_1.logger.saveJSON(flatNodes, `flat-dom-android-${request.stepCount}`);
+            const { results, indexMap } = await this.searcher.searchBestAndroid(flatNodes, request.keywords);
+            grouped = await this.relationshipBuilder.buildGroupedResultsAndroid(results, flatNodes, indexMap, parentMap);
+        }
         return grouped;
     }
 }

package/dist/domAnalysis/relativeElementsFromDom.d.ts

@@ -1,4 +1,5 @@
 export declare class DomRelationshipBuilder {
     build(nodes: any[]): any[];
-    buildGroupedResultsMobile(results: Record<string, any[]>, nodes: any[], indexMap: Record<string, number>, parentMap: Map<Element, Element | null>): any[][];
+    buildGroupedResultsAndroid(results: Record<string, any[]>, nodes: any[], indexMap: Record<string, number>, parentMap: Map<Element, Element | null>): any[][];
+    buildGroupedResultsIos(results: Record<string, any[]>, nodes: any[], indexMap: Record<string, number>, parentMap: Map<any, any>, runtimeElements: Record<number, any>): any[][];
 }