ff-automationv2 2.2.0 → 2.2.5

package/src/core/main/executionContext.ts

@@ -1,22 +0,0 @@
-import { BrowserSession } from "../../automation/browserSession/initiateBrowserSession.js";
-import { MobileSession } from "../../automation/mobileSession/initiateMobileSession.js";
-import { ActionExecutor } from "../../automation/actions/executor.js";
-import { ElementGetter } from "../../fireflinkData/fireflinkLocators/getListOfLocators.js";
-import { ScriptDataAppender } from "../../fireflinkData/fireflinkScript/scriptGenrationData.js";
-import { AutomationRequest } from "../interfaces/executionDetails.js";
-import { DEFAULT_PAGE_LOAD } from "../constants/defaultValues.js"
-
-export class ExecutionContext {
-    public session: BrowserSession;
-    public androidSession: MobileSession;
-    public executor: ActionExecutor | null = null;
-    public locator: ElementGetter;
-    public scriptAppender: ScriptDataAppender;
-
-    constructor(public request: AutomationRequest) {
-        this.session = new BrowserSession(request.browser.browserName, DEFAULT_PAGE_LOAD);
-        this.locator = new ElementGetter();
-        this.scriptAppender = new ScriptDataAppender();
-        this.androidSession = new MobileSession(request.appiumPort);
-    }
-}

package/src/core/main/runAutomationScript.ts

@@ -1,659 +0,0 @@
-import { IAutomationRunner } from "../interfaces/automationRunnerInterface.js";
-import { AutomationRequest } from "../../core/interfaces/executionDetails.js";
-import { ServiceProviderBaseUrlProvider } from "../../llmConfig/llmConfiguration.js";
-import { llmAction } from "../../ai/llmcalls/llmAction.js";
-import { ExecutionContext } from "./executionContext.js";
-import { createActionHandlers } from "../../core/main/actionHandlerFactory.js";
-import { StepProcessor } from "./stepProcessor.js";
-import { FireFlinkApiService } from "../../service/api/fireflinkApi.service.js";
-import { ScriptRunner } from "../../service/scriptRunner.service.js";
-import { IPayload } from "../interfaces/fireflinkScriptPayloadInterface.js";
-import { PromptType } from "../types/promptType.js";
-import { INPUTLESS_ACTIONS, ELEMENTLESS_ACTION } from "../constants/supportedActions.js";
-import { DomProcessingEngine } from "../../domAnalysis/getRelaventElements.js"
-import { getAnnotatedDOM } from "../../utils/DomExtraction/jsForAttributeInjection.js"
-import { JSDOM } from "jsdom";
-import { getAnnotatedDOMForMobile } from "../../utils/DomExtraction/jsForAttributeInjection.js"
-import { allKeywordAction } from "../constants/allAction.js";
-import { IErrorInfo } from "../interfaces/errorInfoInterface.js";
-import { newTabTracker } from "../../utils/newTab/newTabTracker.js";
-import { generateAnnotatedScreenshots } from "../../imageAnalysisWeb/generateAnnotatedScreenshots.js";
-import { getAiInstanceInfo } from "../../utils/InstancesDetails/getInstancesInfo.js"
-import { isAlertPresent } from "../../utils/alertPopup/isAlertPresent.js";
-import { TOAST_WATCHER_JS } from "../../utils/javascript/jsForToaster.js";
-import { Find_element_by_ff_js_script } from "../../utils/javascript/jsFindElement.js"
-import { getFFIndexAndXPath } from "../../utils/helpers/xpathcreation.js"
-import { platform } from "../types/scriptTypesType.js"
-import { logger } from "../../utils/logger/logData.js"
-import { findXpathFromIndex } from "../../utils/mobileXpath/findXpathFromIndex.js"
-import { sameActionHelper } from "../../utils/helpers/sameActionsHelper.js"
-import { error } from "node:console";
-import { annotateScreenshot } from "../../imageAnalysisMobile/annotatedScreenshotMobile.js";
-import { IKafkaPayload } from "../interfaces/fireflinkScriptPayloadInterface.js"
-export class AutomationRunner implements IAutomationRunner {
-  static sessionTerminationDetails: Record<string, boolean> = {};
-  private usedElementNames: string[] = [];
-  private KafkaPayload: IKafkaPayload = {} as IKafkaPayload;
-  static getSessionTerminationInfo(testCaseId: string): boolean {
-    return AutomationRunner.sessionTerminationDetails[testCaseId]
-  }
-  static updateSessionTerminationInfo(testCaseId: string): void {
-    AutomationRunner.sessionTerminationDetails[testCaseId] = true
-  }
-
-  private async cleanup(
-    context: ExecutionContext,
-    domInfo: any,
-    extractedRelevantDom: any,
-    stepProcessor: StepProcessor,
-    scriptRunner: ScriptRunner,
-    request: AutomationRequest,
-    platform: platform,
-    driver: any
-  ) {
-    try {
-
-      logger.info("Starting cleanup process...");
-      if ((context?.session) || (context?.androidSession)) {
-        try {
-
-          const platformHandlers: Record<platform, () => Promise<void>> = {
-
-            android: async () => {
-              if (driver?.sessionId) {
-                await driver.deleteSession();
-              }
-              logger.info("Android session deleted successfully");
-            },
-
-            web: async () => {
-              logger.info(
-                "Still browser session is active, session id :",
-                driver.sessionId
-              );
-            }
-
-          };
-
-          await (platformHandlers[platform] ?? platformHandlers.web)();
-
-        } catch {
-          logger.error("Browser session is deactivated");
-        }
-      }
-
-
-
-      domInfo = null;
-      extractedRelevantDom = null;
-
-      if (stepProcessor) {
-        (stepProcessor as any).llm = null;
-      }
-
-      (scriptRunner as any) = null;
-
-      delete AutomationRunner.sessionTerminationDetails[request.testCaseId];
-
-      if (global.gc) {
-        global.gc();
-      }
-
-      logger.info("Cleanup completed successfully.");
-
-    } catch (error) {
-      logger.error("Cleanup error:", error);
-    }
-  }
-
-  async run(request: AutomationRequest): Promise<void> {
-    this.usedElementNames = [];
-    AutomationRunner.sessionTerminationDetails[request.testCaseId] = false;
-    const apiService = new FireFlinkApiService();
-    const scriptRunner = new ScriptRunner(apiService, request.kafkaProducerInstance);
-    const instanceDetails = await scriptRunner.runGetInstancesDetails(
-      request.projectId,
-      request.token,
-      request.serverHost
-    );
-
-    const aiInfo = getAiInstanceInfo(instanceDetails.responseObject)
-
-    const context = new ExecutionContext(request);
-
-    const configProvider = new ServiceProviderBaseUrlProvider();
-    const baseUrl = configProvider.getBaseUrl(aiInfo.serviceProvider);
-    let domInfo: any = null;
-    let extractedRelevantDom: any = null;
-    let errorInfo: IErrorInfo = {};
-    const llm = new llmAction(aiInfo.apiKey, baseUrl, aiInfo.model, aiInfo.visionApiKey);
-    let preCapturedDomInfo: any = null;
-    const stepProcessor = new StepProcessor(llm);
-    const platform = request.platform.toLocaleLowerCase() as platform
-    const actionHandlers = createActionHandlers(
-      context,
-    );
-    const domProcessor = new DomProcessingEngine();
-    let stepCount = 1;
-    const listOfSteps = request.userStory.map(step => String(step));
-    let driver: any;
-
-    if (listOfSteps.length == 0) {
-      logger.info(listOfSteps)
-      throw new Error("No executable manual steps were returned by the LLM.");
-    }
-    const start_time = new Date().getTime()
-
-    for (const step of listOfSteps) {
-      try {
-        if (AutomationRunner.getSessionTerminationInfo(request.testCaseId)) { break; }
-        logger.info(`Processing step ${stepCount}: ${step}`);
-        const start = Math.max(0, stepCount - 3);
-        const end = Math.min(listOfSteps.length, stepCount + 3);
-        const priorAndNextSteps = listOfSteps.slice(start, end);
-        const result = await stepProcessor.getLLMResponse({
-          platform: platform,
-          type: PromptType.KEYWORD_EXTRACTOR,
-          args: { priorAndNextSteps },
-          input: { currentStep: step }
-        });
-        const action = result.response.action;
-
-        let elementName = result.response.elementName;
-        if (elementName) {
-          const baseName = elementName;
-          let newName = baseName;
-          let count = 0;
-          while (this.usedElementNames.includes(newName)) {
-            count++;
-            newName = `${baseName}_${count}`;
-          }
-          elementName = newName;
-          result.response.elementName = elementName;
-          this.usedElementNames.push(elementName);
-        }
-
-        if (!allKeywordAction.includes(action)) {
-          throw new Error(`Unsupported action at action or keyword exracter: ${action}`);
-        }
-        let handler = actionHandlers[action];
-        if (INPUTLESS_ACTIONS.includes(action)) {
-          logger.info(result.response.keywords)
-          await handler({ keywords: result.response.keywords, step: step });
-          stepCount++;
-          continue;
-        }
-
-        let stepResult: any;
-        let stepAction: string;
-        let fireflinkIndex: string;
-        let selector: string;
-        let xpath: string;
-        let frameAction: string | null = null;
-        let iframeDomHtml: string | null = null;
-        let doc: any = null;
-        let beforeHandles: any = null;
-        if (platform == "web") {
-          driver = await context.session.getCurrentDriver();
-        } else {
-          driver = await context.androidSession.getCurrentDriver();
-        }
-
-        if (await isAlertPresent(driver) && action != "wait") {
-          logger.info("Alert is present");
-          stepResult = await stepProcessor.getLLMResponse({
-            platform: platform,
-            type: PromptType.FF_INSPECTOR,
-            args: {
-              stepAction: action,
-              extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
-              priorAndNextSteps,
-              isAlert: true,
-              isDrag: false
-            },
-            input: { currentStep: step }
-          });
-          stepAction = stepResult.response.action;
-
-          handler = actionHandlers[stepAction];
-          await handler({
-            value: stepResult.response.input_text
-          });
-          continue;
-        }
-        else {
-          if (preCapturedDomInfo) {
-            logger.info("Using pre-captured DOM from previous toaster detection");
-            domInfo = preCapturedDomInfo;
-            preCapturedDomInfo = null;
-          }
-          else {
-            if (platform === "web") {
-              domInfo = await getAnnotatedDOM(driver);
-              logger.saveDOM(domInfo.dom, `annotated-dom-${stepCount}`);
-            }
-            else {
-              domInfo = await getAnnotatedDOMForMobile(driver);
-              logger.saveDOM(domInfo.dom, `annotated-dom-Mob-${stepCount}`);
-            }
-          }
-
-          const parser = new JSDOM(domInfo.dom);
-          doc = parser.window.document;
-
-          if (platform === "web") {
-            extractedRelevantDom = domProcessor.webProcess({
-              keywords: result.response.keywords,
-              rawDom: domInfo.dom,
-              stepCount: stepCount
-            });
-            logger.saveJSON(extractedRelevantDom, `relevant-dom-${stepCount}`);
-          }
-          else {
-            extractedRelevantDom = domProcessor.mobileProcess({
-              keywords: result.response.keywords,
-              rawDom: domInfo.dom,
-              stepCount: stepCount
-            });
-            logger.saveJSON(extractedRelevantDom, `relevant-dom-Mob-${stepCount}`);
-          }
-
-
-
-          if (action == "verify") {
-            stepResult = await stepProcessor.getLLMResponse({
-              platform: platform,
-              type: PromptType.VERIFY_PROMPT,
-              args: {
-                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
-                priorAndNextSteps,
-              },
-              input: { currentStep: step }
-            });
-            stepAction = stepResult.response.nlpName;
-          }
-          else if (action == "get") {
-            stepResult = await stepProcessor.getLLMResponse({
-              platform: platform,
-              type: PromptType.GET_PROMPT,
-              args: {
-                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
-                priorAndNextSteps,
-              },
-              input: { currentStep: step }
-            });
-            stepAction = stepResult.response.nlpName;
-          }
-          else if (action == "wait") {
-            stepResult = await stepProcessor.getLLMResponse({
-              platform: platform,
-              type: PromptType.WAIT_ACTION_EXTRACTOR_PROMPT,
-              args: {
-                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
-                priorAndNextSteps,
-              },
-              input: { currentStep: step }
-            });
-            stepAction = stepResult.response.nlpName;
-          }
-          else if (action == "combined") {
-            stepResult = await stepProcessor.getLLMResponse({
-              platform: platform,
-              type: PromptType.COMBINED_ACTION_EXTRACTOR_PROMPT,
-              args: {
-                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
-                priorAndNextSteps,
-              },
-              input: { currentStep: step }
-            });
-            stepAction = stepResult.response.nlpName;
-          }
-          else if (action == "drag_and_drop") {
-            stepResult = await stepProcessor.getLLMResponse({
-              platform: platform,
-              type: PromptType.FF_INSPECTOR,
-              args: {
-                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
-                priorAndNextSteps,
-                isDrag: true,
-                isAlert: false
-              },
-              input: { currentStep: step }
-            });
-            stepAction = stepResult.response.nlpName;
-          }
-
-          else {
-            stepResult = await stepProcessor.getLLMResponse({
-              platform: platform,
-              type: PromptType.FF_INSPECTOR,
-              args: {
-                stepAction: action,
-                extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
-                priorAndNextSteps,
-                isAlert: false,
-                isDrag: false
-              },
-              input: { currentStep: step }
-            });
-            if (platform == "web") {
-              stepAction = stepResult.response.action;
-            }
-            else {
-              stepAction = stepResult.response.nlpName;
-            }
-          }
-        }
-        if (platform === "android") {
-          stepAction = sameActionHelper[stepAction] || stepAction;
-          logger.info(`step action: ${stepAction}`);
-        }
-
-        if (ELEMENTLESS_ACTION.includes(stepAction)) {
-          let currentDriver: any = null;
-          let beforeHandles: any = null;
-          if (platform == "web") {
-            currentDriver = await context.session.getCurrentDriver();
-            beforeHandles = await currentDriver.getWindowHandles();
-          } else {
-            currentDriver = await context.androidSession.getCurrentDriver();
-          }
-
-          handler = actionHandlers[stepAction];
-
-          await handler({ value: stepResult.response.input_text });
-          if ((currentDriver) && (platform == "web")) {
-            await newTabTracker(currentDriver, action, beforeHandles, context.scriptAppender);
-          }
-          stepCount++;
-          continue;
-        }
-
-
-        try {
-
-          fireflinkIndex = stepResult.response.attribute_value;
-          if (fireflinkIndex == "Fire-Flink-0") {
-            logger.error(`element not found for step "${step}"`);
-            throw new Error(`element not found for step "${step}"`, { cause: error });
-          }
-
-          if (platform === "web") {
-            const ele = doc.querySelector(
-              `[ff-inspect="${fireflinkIndex}"]`
-            );
-            selector = ele.getAttribute("ff-xpath");
-          }
-          else {
-            selector = await getFFIndexAndXPath(domInfo.dom, fireflinkIndex)
-          }
-
-          if (!selector) {
-            logger.info(`Unable to resolve xpath for ${fireflinkIndex}`);
-            throw new Error(`Unable to resolve xpath for ${fireflinkIndex}`, { cause: error });
-          }
-
-          if (platform === "web") {
-            beforeHandles = await driver.getWindowHandles();
-          }
-
-          if (stepAction == "Builtin_DragAndDropToElement") {
-            handler = actionHandlers[stepAction];
-            const fireflinkIndex1 = stepResult.response.drag;
-            const ele1 = doc.querySelector(`[ff-inspect="${fireflinkIndex1}"]`);
-            const selector1 = ele1?.getAttribute("ff-xpath");
-            const fireflinkIndex2 = stepResult.response.drop;
-            const ele2 = doc.querySelector(`[ff-inspect="${fireflinkIndex2}"]`);
-            const selector2 = ele2?.getAttribute("ff-xpath");
-            if (!selector2 && !selector1) {
-              throw new Error(`Unable to resolve xpath for ${fireflinkIndex2}`);
-            }
-            else if (fireflinkIndex2 == "Fire-Flink-0" && fireflinkIndex1 == "Fire-Flink-0") {
-              throw new Error(`Unable to find element for ${step}`);
-            }
-            await handler({
-              selector1: selector1,
-              selector2: selector2,
-              fireflinkIndex1: fireflinkIndex1,
-              fireflinkIndex2: fireflinkIndex2,
-              pageDOM: domInfo.dom,
-              elementName1: stepResult.response.drag_element_name,
-              elementName2: stepResult.response.drop_element_name,
-              elementType1: stepResult.response.drag_element_type,
-              elementType2: stepResult.response.drop_element_type
-
-            });
-            continue;
-          }
-
-
-          if (selector.includes("shadow")) {
-            stepAction = stepAction + "_shadow";
-            logger.info(stepAction);
-          }
-          else if (selector.includes("iframe")) {
-            frameAction = "switchToDefaultContent";
-            handler = actionHandlers[frameAction];
-            await handler({
-              driver: driver,
-            });
-
-            const result_if = await driver.executeAsyncScript(
-              Find_element_by_ff_js_script,
-              [fireflinkIndex]
-            );
-            if (!result_if?.success) {
-              logger.info("Not found");
-              return;
-            }
-            const { xpath_if, frames, iframe_dom_html } = result_if;
-            selector = xpath_if
-            iframeDomHtml = iframe_dom_html
-            frameAction = "switchToFrame";
-            handler = actionHandlers[frameAction];
-            await handler({
-              selector: selector,
-              fireflinkIndex,
-              frames: frames,
-              elementName: result.response.elementName,
-              elementType: stepResult.response.elementType
-            });
-
-          }
-
-          handler = actionHandlers[stepAction];
-          if (iframeDomHtml != null) {
-            domInfo = iframeDomHtml
-          }
-          else {
-            domInfo = domInfo.dom
-          }
-          await handler({
-            selector: selector,
-            value: stepResult.response.input_text,
-            fireflinkIndex,
-            pageDOM: domInfo,
-            elementName: result.response.elementName,
-            elementType: stepResult.response.elementType
-          });
-          if (platform === "web") {
-            if (await isAlertPresent(driver) != true) {
-              if (action === "click") {
-                await driver.executeScript(TOAST_WATCHER_JS, []);
-
-                await new Promise(resolve => setTimeout(resolve, 100));
-                const postClickToastDetected = await driver.executeScript(
-                  "return window.__toastDetected === true;",
-                  []
-                );
-                if (postClickToastDetected) {
-                  logger.info(`Toast detected after click`);
-                  preCapturedDomInfo = await getAnnotatedDOM(driver);
-                  logger.saveDOM(preCapturedDomInfo.dom, `toaster-dom-${stepCount}`);
-                }
-              }
-            }
-            await newTabTracker(driver, action, beforeHandles, context.scriptAppender);
-
-          }
-
-
-        }
-        catch (error: any) {
-          logger.error(`execution failed for step "${step}": ${error.message} and going for vision`);
-          try {
-            let annotatedScreenshot: any;
-            if (platform === "web") {
-              annotatedScreenshot = await generateAnnotatedScreenshots(driver);
-            }
-            else {
-              annotatedScreenshot = await annotateScreenshot(driver, domInfo.dom);
-            }
-            stepResult = await stepProcessor.getLLMResponse({
-              platform: platform,
-              type: PromptType.VISION_PROMPT,
-              args: {
-                priorAndNextSteps
-              },
-              input: { currentStep: step, annotatedScreenshot: annotatedScreenshot }
-            });
-            fireflinkIndex = stepResult.response.fireflinkIndex;
-            logger.info(`fireflinkIndex: ${fireflinkIndex}`);
-            if (fireflinkIndex == "Fire-Flink-0") {
-              logger.error(`vision unable to find element for step "${step}"`);
-              context.scriptAppender.setErrorNLP(stepAction)
-              throw new Error(`vision unable to find element for step "${step}"`, { cause: error });
-            }
-            if (platform == "android") {
-              xpath = findXpathFromIndex({
-                fireflinkIndex,
-                dom: domInfo.dom
-              })
-            }
-            else {
-              const ele = doc.querySelector(
-                `[ff-inspect="${fireflinkIndex}"]`
-              );
-              xpath = ele.getAttribute("ff-xpath");
-              if (!xpath) {
-                logger.info(`Unable to resolve xpath for ${fireflinkIndex}`);
-                context.scriptAppender.setErrorNLP(stepAction)
-                throw new Error(`Unable to resolve xpath for ${fireflinkIndex}`, { cause: error });
-              }
-
-            }
-
-
-            if (iframeDomHtml != null) {
-              domInfo = iframeDomHtml
-            }
-            else {
-              domInfo = domInfo.dom
-            }
-
-            handler = actionHandlers[stepAction];
-            await handler({
-              selector: xpath,
-              value: stepResult.response.input_text,
-              fireflinkIndex,
-              pageDOM: domInfo,
-              elementName: result.response.elementName,
-              elementType: stepResult.response.elementType
-            });
-            if (await isAlertPresent(driver) != true) {
-
-              if (action === "click") {
-                await driver.executeScript(TOAST_WATCHER_JS, []);
-
-                await new Promise(resolve => setTimeout(resolve, 100));
-                const postClickToastDetected = await driver.executeScript(
-                  "return window.__toastDetected === true;",
-                  []
-                );
-                if (postClickToastDetected) {
-                  logger.info(`Toast detected after click`);
-                  preCapturedDomInfo = await getAnnotatedDOM(driver);
-                  logger.saveDOM(preCapturedDomInfo.dom, `toaster-dom-${stepCount}`);
-                }
-              }
-            }
-
-          }
-          catch (error: any) {
-            logger.error(`vision execution failed for step "${step}": ${error}`);
-            throw error;
-          }
-
-        }
-        domInfo = null;
-        extractedRelevantDom = null;
-        stepCount++;
-      }
-
-      catch (error: any) {
-        logger.error(`Error executing step "${step}":`, error);
-        errorInfo = (await stepProcessor.getLLMResponse({ platform: platform, type: PromptType.ERROR_DESCRIPTION, args: {}, input: { error: logger.getError(error) } })).response;
-        if (
-          errorInfo.error === "Context Length Exceeded (400)" ||
-          errorInfo.error === "Rate Limit Exceeded (429)"
-        ) {
-          break;
-        }
-        if (context.scriptAppender.getErrorNLP()) { errorInfo.nlp = context.scriptAppender.getErrorNLP() }
-        errorInfo.erroredManualStep = step
-        errorInfo.rawError = logger.getError(error);
-        break;
-      }
-    }
-    logger.info('execution  time->', new Date().getTime() - start_time)
-
-
-
-    const payload: IPayload = {
-      scriptName: request.scriptName,
-      version: request.version,
-      type: request.type,
-      scriptType: request.scriptType,
-      projectId: request.projectId,
-      testCaseId: request.testCaseId,
-      promptId: request.promptId,
-      pageDetails: request.pageDetails,
-      generatedBy: request.generatedBy,
-      webSocketId: request.webSocketId,
-      licenseType: request.licenseType,
-      licenseId: request.licenseId,
-      userId: request.userId,
-      topic: request.topic,
-      projectType: request.projectType,
-      tokensConsumed: (await stepProcessor.getResultTokenUsage()).totalTokens,
-      errorInfo: errorInfo
-    }
-    try {
-      await context.scriptAppender.waitForAllSteps()
-      this.KafkaPayload.topic = request.topic;
-      this.KafkaPayload.message = {
-        ...payload,
-        scriptGenerationData: context.scriptAppender.getData(),
-        serverHost: request.serverHost
-      };
-      await scriptRunner.produceMessageToKafka(this.KafkaPayload)
-    } catch (error: any) {
-
-      throw new Error(
-        "Failed to send payload to FireFlink API:", { cause: error }
-      );
-    }
-    finally {
-      await this.cleanup(
-        context,
-        domInfo,
-        extractedRelevantDom,
-        stepProcessor,
-        scriptRunner,
-        request,
-        platform,
-        driver
-      );
-    }
-  }
-}