ff-automationv2 2.1.3 → 2.2.2

package/src/core/main/runAutomationScript.ts

@@ -1,254 +0,0 @@
-import { IAutomationRunner } from "../interfaces/automationRunnerInterface.js";
-import { AutomationRequest } from "../../core/interfaces/executionDetails.js";
-import { ServiceProviderBaseUrlProvider } from "../../llmConfig/llmConfiguration.js";
-import { llmAction } from "../../ai/llmcalls/llmAction.js";
-import { ExecutionContext } from "./executionContext.js";
-import { createActionHandlers } from "../../core/main/actionHandlerFactory.js";
-import { StepProcessor } from "./stepProcessor.js";
-import { FireFlinkApiService } from "../../service/fireflinkApi.service.js";
-import { ScriptRunner } from "../../service/scriptRunner.service.js";
-import { IPayload } from "../interfaces/fireflinkScriptPayloadInterface.js";
-import { PromptType } from "../types/promptType.js";
-import { INPUTLESS_ACTIONS, ELEMENTLESS_ACTION } from "../constants/supportedActions.js";
-import { DomProcessingEngine } from "../../domAnalysis/getRelaventElements.js"
-import { getAnnotatedDOM } from "../../utils/DomExtraction/jsForAttributeInjection.js"
-import { logger } from "../../utils/logger/logData.js"
-export class AutomationRunner implements IAutomationRunner {
-  static sessionTerminationDetails: Record<string, boolean> = {};
-  constructor(
-    private readonly request: AutomationRequest,
-    private pageLoad: number = 20000,
-    private implicit: number = 15000,
-    private tokensConsumed: number = 0
-
-  ) { AutomationRunner.sessionTerminationDetails[request.testCaseId] = false }
-
-  static getSessionTerminationInfo(testCaseId: string): boolean {
-    return AutomationRunner.sessionTerminationDetails[testCaseId]
-  }
-  static updateSessionTerminationInfo(testCaseId: string): void {
-    AutomationRunner.sessionTerminationDetails[testCaseId] = true
-  }
-
-  private async cleanup(
-    context: ExecutionContext,
-    domInfo: any,
-    extractedRelevantDom: any,
-    stepProcessor: StepProcessor,
-    scriptRunner: ScriptRunner
-  ) {
-    try {
-      logger.info("Starting cleanup process...");
-      if (context?.session) {
-        try {
-          const browser = await context.session.getCurrentBrowser();
-          // if (browser) {
-          //   await browser.deleteSession?.();
-          // }
-        } catch (e) {
-          logger.error("Browser cleanup failed:", e);
-        }
-      }
-
-      domInfo = null;
-      extractedRelevantDom = null;
-
-      if (stepProcessor) {
-        (stepProcessor as any).llm = null;
-      }
-
-      scriptRunner = null as any;
-
-      delete AutomationRunner.sessionTerminationDetails[this.request.testCaseId];
-
-      if (global.gc) {
-        global.gc();
-      }
-
-      logger.info("Cleanup completed successfully.");
-    }
-    catch (error) {
-      logger.error("Cleanup error:", error);
-    }
-  }
-
-
-  async run(): Promise<void> {
-    const apiService = new FireFlinkApiService();
-    const scriptRunner = new ScriptRunner(apiService);
-    const context = new ExecutionContext(this.request);
-    const configProvider = new ServiceProviderBaseUrlProvider();
-    const baseUrl = configProvider.getBaseUrl(this.request.serviceProvider);
-    let domInfo: any = null;
-    let extractedRelevantDom: any = null;
-    const llm = new llmAction(this.request.apiKey, baseUrl, this.request.model, this.request.visionApikey);
-    const stepProcessor = new StepProcessor(llm);
-
-    const stepResult = await stepProcessor.getLLMResponse({ type: PromptType.USER_STORY_TO_LIST, args: {}, input: { userStory: this.request.userStory } });
-
-    const actionHandlers = createActionHandlers(
-      context,
-      this.pageLoad,
-      this.implicit
-    );
-    const domProcessor = new DomProcessingEngine();
-    let stepCount = 1;
-    const listOfSteps = stepResult.response.manualSteps
-    if (listOfSteps.length == 0) {
-      throw new Error("No executable manual steps were returned by the LLM.");
-    }
-
-    logger.info(listOfSteps)
-
-    for (const step of listOfSteps) {
-      try {
-        if (AutomationRunner.getSessionTerminationInfo(this.request.testCaseId)) { break; }
-
-        const start = Math.max(0, stepCount - 3);
-        const end = Math.min(listOfSteps.length, stepCount + 3);
-        const priorAndNextSteps = listOfSteps.slice(start, end);
-
-        const result = await stepProcessor.getLLMResponse({
-          type: PromptType.KEYWORD_EXTRACTOR,
-          args: { priorAndNextSteps },
-          input: { currentStep: step }
-        });
-
-        logger.info(JSON.stringify(result, null, 2))
-
-        const action = result.response.action?.toLowerCase();
-        const handler = actionHandlers[action];
-
-        logger.info
-          (
-            `Processing step ${stepCount}: "${step}" with action: "${action}" and keywords: ${JSON.stringify(result.response.keywords)}`
-          );
-
-        if (!handler) {
-          throw new Error(`Unsupported action: ${action}`);
-        }
-
-        if (INPUTLESS_ACTIONS.includes(action)) {
-          await handler(result);
-          stepCount++;
-          continue;
-        }
-
-        if (ELEMENTLESS_ACTION.includes(action)) {
-          const stepResult = await stepProcessor.getLLMResponse({
-            type: PromptType.FF_INSPECTOR,
-            args: {
-              stepAction: action,
-              extractedDomJson: "",
-              priorAndNextSteps,
-              isAlert: false,
-              isDrag: false
-            },
-            input: { currentStep: step }
-          });
-
-          await handler({ value: stepResult.response.input_text });
-          stepCount++;
-          continue;
-        }
-
-
-        const driver = await context.session.getCurrentBrowser();
-        domInfo = await getAnnotatedDOM(driver);
-        await logger.saveDOM(domInfo.dom, `annotated-dom-${stepCount}`);
-
-        extractedRelevantDom = domProcessor.process({
-          keywords: result.response.keywords,
-          rawDom: domInfo.dom,
-          stepCount: stepCount
-        });
-
-        await logger.saveJSON(extractedRelevantDom, `relevant-dom-${stepCount}`);
-
-        const stepResult = await stepProcessor.getLLMResponse({
-          type: PromptType.FF_INSPECTOR,
-          args: {
-            stepAction: action,
-            extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
-            priorAndNextSteps,
-            isAlert: false,
-            isDrag: false
-          },
-          input: { currentStep: step }
-        });
-
-        logger.info(JSON.stringify(stepResult, null, 2))
-
-        const fireflinkIndex = stepResult.response.attribute_value;
-        const xpath = domInfo.selectors[fireflinkIndex];
-
-        if (!xpath) {
-          throw new Error(`Unable to resolve xpath for ${fireflinkIndex}`);
-        }
-        else if (fireflinkIndex == "Fire-Flink-0") {
-          throw new Error(`Unable to find element for ${step}`);
-        }
-
-        await handler({
-          selector: xpath,
-          value: stepResult.response.input_text,
-          fireflinkIndex,
-          pageDOM: domInfo.dom,
-          elementName: result.response.elementName,
-          elementType: stepResult.response.elementType
-        });
-        domInfo = null;
-        extractedRelevantDom = null;
-        stepCount++;
-        await new Promise(resolve => setImmediate(resolve));
-      }
-      catch (error: any) {
-        context.scriptAppender.add({
-          nlpName: "CloseBrowser",
-          elementsData: [],
-          stepInputs: []
-        });
-
-        logger.error(`Error executing step "${step}":`, error.message);
-        break;
-      }
-    }
-    const payload: IPayload = {
-      scriptName: this.request.scriptName,
-      scriptType: this.request.scriptType,
-      projectId: this.request.projectId,
-      testCaseId: this.request.testCaseId,
-      promptId: this.request.promptId,
-      pageDetails: this.request.pageDetails,
-      generatedBy: this.request.generatedBy,
-      webSocketId: this.request.webSocketId,
-      licenseType: this.request.licenseType,
-      licenseId: this.request.licenseId,
-      userId: this.request.userId,
-      topic: this.request.topic,
-      projectType: this.request.projectType,
-      tokensConsumed: (await stepProcessor.getResultTokenUsage()).totalTokens,
-    }
-    try {
-      await scriptRunner.runScriptFromPayload(
-        context.scriptAppender.getData(),
-        payload,
-        this.request.token,
-        this.request.serverHost
-      );
-    } catch (error: any) {
-      throw new Error(
-        "Failed to send payload to FireFlink API:", { cause: error }
-      );
-    }
-    finally {
-      await this.cleanup(
-        context,
-        domInfo,
-        extractedRelevantDom,
-        stepProcessor,
-        scriptRunner
-      );
-    }
-  }
-}

package/src/core/main/stepProcessor.ts

@@ -1,28 +0,0 @@
-import { llmAction } from "../../ai/llmcalls/llmAction.js";
-import { LLMResultParser } from "../../ai/llmcalls/parseLlmOputput.js";
-import { FireFlinkLLMResponse } from "../types/llmResponseType.js"
-import { PromptType } from "../types/promptType.js";
-
-export interface LLMRequest {
-    type: PromptType;
-    args: Record<string, any>;
-    input: Record<string, any>;
-}
-
-export class StepProcessor {
-    private resultParser: LLMResultParser;
-
-    constructor(private llm: llmAction) {
-        this.resultParser = new LLMResultParser();
-    }
-
-    async getResultTokenUsage() {
-        return this.resultParser.getTokenUsage();
-    }
-
-
-    async getLLMResponse({ type, args, input }: LLMRequest): Promise<{ response: FireFlinkLLMResponse }> {
-        const response = await this.llm.getLLMResponse(type, args, input);
-        return this.resultParser.fetchResult(response);
-    }
-}

package/src/core/types/llmResponseType.ts

@@ -1,11 +0,0 @@
-import { IKeywordAction, IDragDropAction, IInputAction, IScrollOrAttributeAction, IManualSteps, IFireFlinkIndex } from "../interfaces/llmResponseInterface.js";
-
-export type FireFlinkLLMResponse =
-
-    | IKeywordAction
-    | IInputAction
-    | IDragDropAction
-    | IScrollOrAttributeAction
-    | IManualSteps
-    | IFireFlinkIndex
-    | any

package/src/core/types/promptMap.ts

@@ -1,7 +0,0 @@
-import { PromptArgsMap } from "../interfaces/promptInterface.js";
-
-export type PromptMap = {
-    [K in keyof PromptArgsMap]: (
-        args: PromptArgsMap[K]
-    ) => Promise<string>;
-};

package/src/core/types/promptType.ts

@@ -1,7 +0,0 @@
-export enum PromptType {
-    USER_STORY_TO_LIST = "userStoryToList",
-    KEYWORD_EXTRACTOR = "keywordExtractorPrompt",
-    ERROR_DESCRIPTION = "errorDescriptionPrompt",
-    FF_INSPECTOR = "ffInspectorNumExtractor",
-    VISION_PROMPT = "visionPrompt",
-}

package/src/core/types/visionllmInputType.ts

@@ -1,4 +0,0 @@
-export type visionPromptMessage = (
-    | { type: "text"; text: string }
-    | { type: "image_url"; image_url: { url: string } }
-)[];

package/src/domAnalysis/getRelaventElements.ts

@@ -1,28 +0,0 @@
-import { DomSimplifier } from "./simplifyAndFlatten.js";
-import { DomSearcher } from "./searchBest.js";
-import { DomRelationshipBuilder } from "./relativeElementsFromDom.js";
-import { logger } from "../utils/logger/logData.js";
-
-export interface DomProcessingRequest {
-    keywords: string[] | string;
-    rawDom: string;
-    stepCount: number;
-}
-
-
-export class DomProcessingEngine {
-
-    private simplifier = new DomSimplifier();
-    private searcher = new DomSearcher();
-    private relationshipBuilder = new DomRelationshipBuilder();
-
-    public process(request: DomProcessingRequest) {
-
-        const flat = this.simplifier.simplify(request.rawDom);
-        logger.saveJSON(flat, `flat-dom-${request.stepCount}`);
-        const searched = this.searcher.search(flat, request.keywords);
-        const structured = this.relationshipBuilder.build(searched);
-
-        return structured;
-    }
-}

package/src/domAnalysis/relativeElementsFromDom.ts

@@ -1,94 +0,0 @@
-export class DomRelationshipBuilder {
-
-  public build(nodes: any[]) {
-
-    const ffToXPath = new Map<string, string>();
-
-    nodes.forEach(n => {
-      if (n["FF-inspecter"] && n["ff-xpath"]) {
-        ffToXPath.set(n["FF-inspecter"], n["ff-xpath"]);
-      }
-    });
-
-    const parentXPath = (x: string) =>
-      x.includes("/") ? x.substring(0, x.lastIndexOf("/")) : undefined;
-
-    const ffToNode = new Map<string, any>();
-    nodes.forEach(n => {
-      ffToNode.set(n["FF-inspecter"], n);
-    });
-
-    const structured: any[] = [];
-
-    for (const node of nodes) {
-
-      const ff = node["FF-inspecter"];
-      const xp = node["ff-xpath"];
-
-      let parent: string | undefined;
-      const children: string[] = [];
-      const siblings: string[] = [];
-
-      if (ff && xp) {
-
-        const px = parentXPath(xp);
-
-        for (const [otherFF, otherXP] of ffToXPath.entries()) {
-
-          if (otherFF === ff) continue;
-
-          if (otherXP.startsWith(xp + "/")) {
-            children.push(otherFF);
-          }
-          else if (px && parentXPath(otherXP) === px) {
-            siblings.push(otherFF);
-          }
-        }
-
-        for (const [otherFF, otherXP] of ffToXPath.entries()) {
-          if (otherXP === px) {
-            parent = otherFF;
-            break;
-          }
-        }
-      }
-
-      const slim = Object.fromEntries(
-        Object.entries(node).filter(([k, v]) =>
-          v && !["search", "ff-xpath", "href", "src"].includes(k)
-        )
-      );
-
-      const rel: any = {};
-
-      const mini = (n: any) =>
-        Object.fromEntries(
-          Object.entries(n).filter(([k, v]) =>
-            v && !["search", "ff-xpath", "href", "src"].includes(k)
-          )
-        );
-
-      if (parent && ffToNode.has(parent)) {
-        rel.parent = mini(ffToNode.get(parent));
-      }
-
-      if (children.length) {
-        rel.children = children.slice(0, 2)
-          .map(c => mini(ffToNode.get(c)));
-      }
-
-      if (siblings.length) {
-        rel.siblings = siblings.slice(0, 2)
-          .map(s => mini(ffToNode.get(s)));
-      }
-
-      if (Object.keys(rel).length) {
-        slim["relationship"] = rel;
-      }
-
-      structured.push(slim);
-    }
-
-    return structured;
-  }
-}

package/src/domAnalysis/searchBest.ts

@@ -1,165 +0,0 @@
-import * as fuzz from "fuzzball";
-
-import { FlatNode } from "./simplifyAndFlatten.js";
-
-export class DomSearcher {
-
-    public search(
-        nodes: FlatNode[],
-        keywords: string[] | string,
-        topN = 20,
-        threshold = 60
-    ) {
-
-        if (typeof keywords === "string") {
-            keywords = keywords.replace(/[\[\]]/g, "");
-            keywords = keywords.split(",").map(k => k.trim());
-        }
-
-        keywords = keywords.map(k =>
-            k.replace(/[^a-zA-Z0-9\s]/g, " ").trim().toLowerCase()
-        );
-
-        const ffNumber = (val?: string) => {
-            if (!val) return 0;
-            const m = val.match(/(\d+)$/);
-            return m ? parseInt(m[1]) : 0;
-        };
-
-        const isChild = (p: string, c: string) => c.startsWith(p + "/");
-        const isSibling = (x1: string, x2: string) =>
-            x1.substring(0, x1.lastIndexOf("/")) ===
-            x2.substring(0, x2.lastIndexOf("/"));
-
-        const addWithContext = (
-            idx: number,
-            collected: number[]
-        ) => {
-
-            const total = nodes.length;
-
-            for (const i of [idx - 1, idx, idx + 1]) {
-                if (i >= 0 && i < total) {
-                    if (nodes[i].node["FF-inspecter"]) {
-                        collected.push(i);
-                    }
-                }
-            }
-
-            const current = nodes[idx].node;
-            const cx = current["ff-xpath"];
-            if (!cx) return;
-
-            for (let j = 0; j < total; j++) {
-                const nx = nodes[j].node["ff-xpath"];
-                if (nx && isChild(cx, nx)) {
-                    if (nodes[j].node["FF-inspecter"]) {
-                        collected.push(j);
-                    }
-                    break;
-                }
-            }
-
-            for (let j = 0; j < total; j++) {
-                const nx = nodes[j].node["ff-xpath"];
-                if (nx && isSibling(cx, nx) && j !== idx) {
-                    if (nodes[j].node["FF-inspecter"]) {
-                        collected.push(j);
-                    }
-                    break;
-                }
-            }
-        };
-
-        const results: Record<string, any[]> = {};
-
-        for (const kw of keywords) {
-
-            const exact: number[] = [];
-            const strong: number[] = [];
-            const fuzzyList: [number, number][] = [];
-            const tagMatches: number[] = [];
-
-            for (let i = 0; i < nodes.length; i++) {
-
-                const node = nodes[i].node;
-                const searchText = (node.search || "").toLowerCase();
-                const tagName = (node.tag || "").toLowerCase();
-
-                let matched = false;
-
-                if (searchText == kw) {
-                    exact.push(i);
-                    matched = true;
-                }
-
-                if (searchText.includes(kw)) {
-                    strong.push(i);
-                    matched = true;
-                }
-
-                if (!matched) {
-                    const score = fuzz.token_set_ratio(searchText, kw);
-                    if (score >= threshold) {
-                        fuzzyList.push([score, i]);
-                    }
-                }
-
-                if (kw == tagName || tagName.includes(kw)) {
-                    tagMatches.push(i);
-                }
-            }
-
-            const collected: number[] = [];
-
-            for (const idx of exact) {
-                addWithContext(idx, collected);
-                if (collected.length >= topN * 4) break;
-            }
-
-            for (const idx of strong) {
-                addWithContext(idx, collected);
-                if (collected.length >= topN * 4) break;
-            }
-
-            if (collected.length < topN * 4) {
-                fuzzyList
-                    .sort((a, b) => b[0] - a[0])
-                    .forEach(([_, idx]) => {
-                        if (collected.length < topN * 4) {
-                            addWithContext(idx, collected);
-                        }
-                    });
-            }
-
-            if (collected.length < topN * 4) {
-                for (const idx of tagMatches) {
-                    addWithContext(idx, collected);
-                    if (collected.length >= topN * 4) break;
-                }
-            }
-
-            const uniqueIdx = Array.from(new Set(collected));
-            results[kw] = uniqueIdx.map(i => nodes[i].node);
-        }
-
-        const seen = new Set<string>();
-        const unique: any[] = [];
-
-        Object.values(results).forEach(list => {
-            list.forEach(n => {
-                const key = n["FF-inspecter"];
-                if (key && !seen.has(key)) {
-                    seen.add(key);
-                    unique.push(n);
-                }
-            });
-        });
-
-        unique.sort((a, b) =>
-            ffNumber(a["FF-inspecter"]) - ffNumber(b["FF-inspecter"])
-        );
-
-        return unique;
-    }
-}