ff-automationv2 1.0.0 → 2.0.1-beta.0

package/src/core/main/runAutomationScript.ts

@@ -1,177 +1,177 @@
-import { IAutomationRunner } from "../interfaces/automationRunnerInterface.js";
-import { AutomationRequest } from "../../core/interfaces/executionDetails.js";
-import { ServiceProviderBaseUrlProvider } from "../../llmConfig/llmConfiguration.js";
-import { llmAction } from "../../ai/llmcalls/llmAction.js";
-import { ExecutionContext } from "./executionContext.js";
-import { createActionHandlers } from "../../core/main/actionHandlerFactory.js";
-import { StepProcessor } from "./stepProcessor.js";
-import { FireFlinkApiService } from "../../service/fireflinkApi.service.js";
-import { ScriptRunner } from "../../service/scriptRunner.service.js";
-import { IPayload } from "../interfaces/fireflinkScriptPayloadInterface.js";
-import { PromptType } from "../types/promptType.js";
-import { INPUTLESS_ACTIONS, ELEMENTLESS_ACTION } from "../constants/supportedActions.js";
-import { DomProcessingEngine } from "../../domAnalysis/getRelaventElements.js"
-import { getAnnotatedDOM } from "../../utils/DomExtraction/jsForAttributeInjection.js"
-import { logger } from "../../utils/logger/logData.js"
-export class AutomationRunner implements IAutomationRunner {
-  constructor(
-    private readonly request: AutomationRequest,
-    private pageLoad: number = 20000,
-    private implicit: number = 15000,
-    private tokensConsumed: number = 0
-  ) { }
-
-  async run(): Promise<void> {
-    const apiService = new FireFlinkApiService();
-    const scriptRunner = new ScriptRunner(apiService);
-    const context = new ExecutionContext(this.request);
-    const configProvider = new ServiceProviderBaseUrlProvider();
-    const baseUrl = configProvider.getBaseUrl(this.request.serviceProvider);
-    let domInfo: any = null;
-    let extractedRelevantDom: any = null;
-    const llm = new llmAction(this.request.apiKey, baseUrl, this.request.model, this.request.visionApikey);
-    const stepProcessor = new StepProcessor(llm);
-
-    const stepResult = await stepProcessor.getLLMResponse({ type: PromptType.USER_STORY_TO_LIST, args: {}, input: { userStory: this.request.userStory } });
-
-    const actionHandlers = createActionHandlers(
-      context,
-      this.pageLoad,
-      this.implicit
-    );
-    const domProcessor = new DomProcessingEngine();
-    let stepCount = 0;
-    const listOfSteps = stepResult.response.manualSteps
-    if (listOfSteps.length === 0) {
-      throw new Error("No executable manual steps were returned by the LLM.");
-    }
-    for (const step of listOfSteps) {
-      try {
-        const start = Math.max(0, stepCount - 3);
-        const end = Math.min(listOfSteps.length, stepCount + 3);
-        const priorAndNextSteps = listOfSteps.slice(start, end);
-
-        const result = await stepProcessor.getLLMResponse({
-          type: PromptType.KEYWORD_EXTRACTOR,
-          args: { priorAndNextSteps },
-          input: { currentStep: step }
-        });
-
-        const action = result.response.action?.toLowerCase();
-        const handler = actionHandlers[action];
-
-        logger.info
-          (
-            `Processing step: "${step}" with action: "${action}" and keywords: ${JSON.stringify(result.response.keywords)}`
-          );
-
-        if (!handler) {
-          throw new Error(`Unsupported action: ${action}`);
-        }
-
-        if (INPUTLESS_ACTIONS.includes(action)) {
-          await handler(result);
-          stepCount++;
-          continue;
-        }
-
-        if (ELEMENTLESS_ACTION.includes(action)) {
-          const stepResult = await stepProcessor.getLLMResponse({
-            type: PromptType.FF_INSPECTOR,
-            args: {
-              stepAction: action,
-              extractedDomJson: "",
-              priorAndNextSteps,
-              isAlert: false,
-              isDrag: false
-            },
-            input: { currentStep: step }
-          });
-
-          await handler({ value: stepResult.response.input_text });
-          stepCount++;
-          continue;
-        }
-
-        const browser = await context.session.getCurrentBrowser();
-        domInfo = await getAnnotatedDOM(browser);
-
-        extractedRelevantDom = domProcessor.process({
-          keywords: result.response.keywords,
-          rawDom: domInfo.dom,
-          stepIndex: stepCount
-        });
-
-        const stepResult = await stepProcessor.getLLMResponse({
-          type: PromptType.FF_INSPECTOR,
-          args: {
-            stepAction: action,
-            extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
-            priorAndNextSteps,
-            isAlert: false,
-            isDrag: false
-          },
-          input: { currentStep: step }
-        });
-
-        const fireflinkIndex = stepResult.response.attribute_value;
-        const xpath = domInfo.selectors[fireflinkIndex];
-
-        if (!xpath) {
-          throw new Error(`Unable to resolve xpath for ${fireflinkIndex}`);
-        }
-
-        await handler({
-          selector: xpath,
-          value: stepResult.response.input_text,
-          fireflinkIndex,
-          pageDOM: domInfo.dom,
-          elementName: result.response.elementName,
-          elementType: stepResult.response.elementType
-        });
-        domInfo = null;
-        extractedRelevantDom = null;
-        stepCount++;
-        await new Promise(resolve => setImmediate(resolve));
-      }
-      catch (error: any) {
-        context.scriptAppender.add({
-          nlpName: "CloseBrowser",
-          elementsData: [],
-          stepInputs: []
-        });
-
-        logger.error(`Error executing step "${step}":`, error.message);
-        break;
-      }
-    }
-    const payload: IPayload = {
-      scriptName: this.request.scriptName,
-      scriptType: this.request.scriptType,
-      projectId: this.request.projectId,
-      testCaseId: this.request.testCaseId,
-      promptId: this.request.promptId,
-      pageDetails: this.request.pageDetails,
-      generatedBy: this.request.generatedBy,
-      webSocketId: this.request.webSocketId,
-      licenseType: this.request.licenseType,
-      licenseId: this.request.licenseId,
-      userId: this.request.userId,
-      topic: this.request.topic,
-      projectType: this.request.projectType,
-      tokensConsumed: (await stepProcessor.getResultTokenUsage()).totalTokens,
-    }
-    try {
-      await scriptRunner.runScriptFromPayload(
-        context.scriptAppender.getData(),
-        payload,
-        this.request.token,
-        this.request.serverHost
-      );
-    } catch (error: any) {
-      throw new Error(
-        "Failed to send payload to FireFlink API:", { cause: error }
-      );
-    }
-  }
-}
+import { IAutomationRunner } from "../interfaces/automationRunnerInterface.js";
+import { AutomationRequest } from "../../core/interfaces/executionDetails.js";
+import { ServiceProviderBaseUrlProvider } from "../../llmConfig/llmConfiguration.js";
+import { llmAction } from "../../ai/llmcalls/llmAction.js";
+import { ExecutionContext } from "./executionContext.js";
+import { createActionHandlers } from "../../core/main/actionHandlerFactory.js";
+import { StepProcessor } from "./stepProcessor.js";
+import { FireFlinkApiService } from "../../service/fireflinkApi.service.js";
+import { ScriptRunner } from "../../service/scriptRunner.service.js";
+import { IPayload } from "../interfaces/fireflinkScriptPayloadInterface.js";
+import { PromptType } from "../types/promptType.js";
+import { INPUTLESS_ACTIONS, ELEMENTLESS_ACTION } from "../constants/supportedActions.js";
+import { DomProcessingEngine } from "../../domAnalysis/getRelaventElements.js"
+import { getAnnotatedDOM } from "../../utils/DomExtraction/jsForAttributeInjection.js"
+import { logger } from "../../utils/logger/logData.js"
+export class AutomationRunner implements IAutomationRunner {
+  constructor(
+    private readonly request: AutomationRequest,
+    private pageLoad: number = 20000,
+    private implicit: number = 15000,
+    private tokensConsumed: number = 0
+  ) { }
+
+  async run(): Promise<void> {
+    const apiService = new FireFlinkApiService();
+    const scriptRunner = new ScriptRunner(apiService);
+    const context = new ExecutionContext(this.request);
+    const configProvider = new ServiceProviderBaseUrlProvider();
+    const baseUrl = configProvider.getBaseUrl(this.request.serviceProvider);
+    let domInfo: any = null;
+    let extractedRelevantDom: any = null;
+    const llm = new llmAction(this.request.apiKey, baseUrl, this.request.model, this.request.visionApikey);
+    const stepProcessor = new StepProcessor(llm);
+
+    const stepResult = await stepProcessor.getLLMResponse({ type: PromptType.USER_STORY_TO_LIST, args: {}, input: { userStory: this.request.userStory } });
+
+    const actionHandlers = createActionHandlers(
+      context,
+      this.pageLoad,
+      this.implicit
+    );
+    const domProcessor = new DomProcessingEngine();
+    let stepCount = 0;
+    const listOfSteps = stepResult.response.manualSteps
+    if (listOfSteps.length === 0) {
+      throw new Error("No executable manual steps were returned by the LLM.");
+    }
+    for (const step of listOfSteps) {
+      try {
+        const start = Math.max(0, stepCount - 3);
+        const end = Math.min(listOfSteps.length, stepCount + 3);
+        const priorAndNextSteps = listOfSteps.slice(start, end);
+
+        const result = await stepProcessor.getLLMResponse({
+          type: PromptType.KEYWORD_EXTRACTOR,
+          args: { priorAndNextSteps },
+          input: { currentStep: step }
+        });
+
+        const action = result.response.action?.toLowerCase();
+        const handler = actionHandlers[action];
+
+        logger.info
+          (
+            `Processing step: "${step}" with action: "${action}" and keywords: ${JSON.stringify(result.response.keywords)}`
+          );
+
+        if (!handler) {
+          throw new Error(`Unsupported action: ${action}`);
+        }
+
+        if (INPUTLESS_ACTIONS.includes(action)) {
+          await handler(result);
+          stepCount++;
+          continue;
+        }
+
+        if (ELEMENTLESS_ACTION.includes(action)) {
+          const stepResult = await stepProcessor.getLLMResponse({
+            type: PromptType.FF_INSPECTOR,
+            args: {
+              stepAction: action,
+              extractedDomJson: "",
+              priorAndNextSteps,
+              isAlert: false,
+              isDrag: false
+            },
+            input: { currentStep: step }
+          });
+
+          await handler({ value: stepResult.response.input_text });
+          stepCount++;
+          continue;
+        }
+
+        const browser = await context.session.getCurrentBrowser();
+        domInfo = await getAnnotatedDOM(browser);
+
+        extractedRelevantDom = domProcessor.process({
+          keywords: result.response.keywords,
+          rawDom: domInfo.dom,
+          stepIndex: stepCount
+        });
+
+        const stepResult = await stepProcessor.getLLMResponse({
+          type: PromptType.FF_INSPECTOR,
+          args: {
+            stepAction: action,
+            extractedDomJson: JSON.stringify(extractedRelevantDom, null, 2),
+            priorAndNextSteps,
+            isAlert: false,
+            isDrag: false
+          },
+          input: { currentStep: step }
+        });
+
+        const fireflinkIndex = stepResult.response.attribute_value;
+        const xpath = domInfo.selectors[fireflinkIndex];
+
+        if (!xpath) {
+          throw new Error(`Unable to resolve xpath for ${fireflinkIndex}`);
+        }
+
+        await handler({
+          selector: xpath,
+          value: stepResult.response.input_text,
+          fireflinkIndex,
+          pageDOM: domInfo.dom,
+          elementName: result.response.elementName,
+          elementType: stepResult.response.elementType
+        });
+        domInfo = null;
+        extractedRelevantDom = null;
+        stepCount++;
+        await new Promise(resolve => setImmediate(resolve));
+      }
+      catch (error: any) {
+        context.scriptAppender.add({
+          nlpName: "CloseBrowser",
+          elementsData: [],
+          stepInputs: []
+        });
+
+        logger.error(`Error executing step "${step}":`, error.message);
+        break;
+      }
+    }
+    const payload: IPayload = {
+      scriptName: this.request.scriptName,
+      scriptType: this.request.scriptType,
+      projectId: this.request.projectId,
+      testCaseId: this.request.testCaseId,
+      promptId: this.request.promptId,
+      pageDetails: this.request.pageDetails,
+      generatedBy: this.request.generatedBy,
+      webSocketId: this.request.webSocketId,
+      licenseType: this.request.licenseType,
+      licenseId: this.request.licenseId,
+      userId: this.request.userId,
+      topic: this.request.topic,
+      projectType: this.request.projectType,
+      tokensConsumed: (await stepProcessor.getResultTokenUsage()).totalTokens,
+    }
+    try {
+      await scriptRunner.runScriptFromPayload(
+        context.scriptAppender.getData(),
+        payload,
+        this.request.token,
+        this.request.serverHost
+      );
+    } catch (error: any) {
+      throw new Error(
+        "Failed to send payload to FireFlink API:", { cause: error }
+      );
+    }
+  }
+}

package/src/core/main/stepProcessor.ts

@@ -1,28 +1,28 @@
-import { llmAction } from "../../ai/llmcalls/llmAction.js";
-import { LLMResultParser } from "../../ai/llmcalls/parseLlmOputput.js";
-import { FireFlinkLLMResponse } from "../types/llmResponseType.js"
-import { PromptType } from "../types/promptType.js";
-
-export interface LLMRequest {
-    type: PromptType;
-    args: Record<string, any>;
-    input: Record<string, any>;
-}
-
-export class StepProcessor {
-    private resultParser: LLMResultParser;
-
-    constructor(private llm: llmAction) {
-        this.resultParser = new LLMResultParser();
-    }
-
-    async getResultTokenUsage() {
-        return this.resultParser.getTokenUsage();
-    }
-
-
-    async getLLMResponse({ type, args, input }: LLMRequest): Promise<{ response: FireFlinkLLMResponse }> {
-        const response = await this.llm.getLLMResponse(type, args, input);
-        return this.resultParser.fetchResult(response);
-    }
-}
+import { llmAction } from "../../ai/llmcalls/llmAction.js";
+import { LLMResultParser } from "../../ai/llmcalls/parseLlmOputput.js";
+import { FireFlinkLLMResponse } from "../types/llmResponseType.js"
+import { PromptType } from "../types/promptType.js";
+
+export interface LLMRequest {
+    type: PromptType;
+    args: Record<string, any>;
+    input: Record<string, any>;
+}
+
+export class StepProcessor {
+    private resultParser: LLMResultParser;
+
+    constructor(private llm: llmAction) {
+        this.resultParser = new LLMResultParser();
+    }
+
+    async getResultTokenUsage() {
+        return this.resultParser.getTokenUsage();
+    }
+
+
+    async getLLMResponse({ type, args, input }: LLMRequest): Promise<{ response: FireFlinkLLMResponse }> {
+        const response = await this.llm.getLLMResponse(type, args, input);
+        return this.resultParser.fetchResult(response);
+    }
+}

package/src/core/types/llmResponseType.ts

@@ -1,11 +1,11 @@
-import { IKeywordAction, IDragDropAction, IInputAction, IScrollOrAttributeAction, IManualSteps, IFireFlinkIndex } from "../interfaces/llmResponseInterface.js";
-
-export type FireFlinkLLMResponse =
-
-    | IKeywordAction
-    | IInputAction
-    | IDragDropAction
-    | IScrollOrAttributeAction
-    | IManualSteps
-    | IFireFlinkIndex
+import { IKeywordAction, IDragDropAction, IInputAction, IScrollOrAttributeAction, IManualSteps, IFireFlinkIndex } from "../interfaces/llmResponseInterface.js";
+
+export type FireFlinkLLMResponse =
+
+    | IKeywordAction
+    | IInputAction
+    | IDragDropAction
+    | IScrollOrAttributeAction
+    | IManualSteps
+    | IFireFlinkIndex
     | any

package/src/core/types/visionllmInputType.ts

@@ -1,4 +1,4 @@
-export type visionPromptMessage = (
-    | { type: "text"; text: string }
-    | { type: "image_url"; image_url: { url: string } }
-)[];
+export type visionPromptMessage = (
+    | { type: "text"; text: string }
+    | { type: "image_url"; image_url: { url: string } }
+)[];

package/src/domAnalysis/getRelaventElements.ts

@@ -1,24 +1,24 @@
-import { DomSimplifier } from "./simplifyAndFlatten.js";
-import { DomSearcher } from "./searchBest.js";
-import { DomRelationshipBuilder } from "./relativeElementsFromDom.js";
-export interface DomProcessingRequest {
-    keywords: string[] | string;
-    rawDom: string;
-    stepIndex: number;
-}
-
-export class DomProcessingEngine {
-
-    private simplifier = new DomSimplifier();
-    private searcher = new DomSearcher();
-    private relationshipBuilder = new DomRelationshipBuilder();
-
-    public process(request: DomProcessingRequest) {
-
-        const flat = this.simplifier.simplify(request.rawDom);
-        const searched = this.searcher.search(flat, request.keywords);
-        const structured = this.relationshipBuilder.build(searched);
-
-        return structured;
-    }
-}
+import { DomSimplifier } from "./simplifyAndFlatten.js";
+import { DomSearcher } from "./searchBest.js";
+import { DomRelationshipBuilder } from "./relativeElementsFromDom.js";
+export interface DomProcessingRequest {
+    keywords: string[] | string;
+    rawDom: string;
+    stepIndex: number;
+}
+
+export class DomProcessingEngine {
+
+    private simplifier = new DomSimplifier();
+    private searcher = new DomSearcher();
+    private relationshipBuilder = new DomRelationshipBuilder();
+
+    public process(request: DomProcessingRequest) {
+
+        const flat = this.simplifier.simplify(request.rawDom);
+        const searched = this.searcher.search(flat, request.keywords);
+        const structured = this.relationshipBuilder.build(searched);
+
+        return structured;
+    }
+}