ff-automationv2 2.1.3 → 2.2.2

package/FireFlink_Architecture.drawio

@@ -1,68 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<mxfile host="app.diagrams.net">
-  <diagram name="FireFlink Architecture">
-    <mxGraphModel dx="1000" dy="1000" grid="1" gridSize="10" guides="1" tooltips="1"
-      connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100"
-      math="0" shadow="0">
-      <root>
-        <mxCell id="0"/>
-        <mxCell id="1" parent="0"/>
-        
-        <mxCell id="2" value="User Story" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
-          <mxGeometry x="300" y="20" width="200" height="60" as="geometry"/>
-        </mxCell>
-
-        <mxCell id="3" value="LLM Reasoning Layer" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
-          <mxGeometry x="300" y="120" width="200" height="60" as="geometry"/>
-        </mxCell>
-
-        <mxCell id="4" value="AutomationRunner (Orchestration)" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
-          <mxGeometry x="300" y="220" width="200" height="60" as="geometry"/>
-        </mxCell>
-
-        <mxCell id="5" value="DOM Abstraction Layer (FF-inspecter Injection)" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
-          <mxGeometry x="300" y="320" width="200" height="60" as="geometry"/>
-        </mxCell>
-
-        <mxCell id="6" value="DOM Processing Engine" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
-          <mxGeometry x="300" y="420" width="200" height="60" as="geometry"/>
-        </mxCell>
-
-        <mxCell id="7" value="Action Execution Layer" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
-          <mxGeometry x="300" y="520" width="200" height="60" as="geometry"/>
-        </mxCell>
-
-        <mxCell id="8" value="Script Generation Layer" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
-          <mxGeometry x="300" y="620" width="200" height="60" as="geometry"/>
-        </mxCell>
-
-        <mxCell id="9" value="FireFlink Backend API" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
-          <mxGeometry x="300" y="720" width="200" height="60" as="geometry"/>
-        </mxCell>
-
-        <mxCell id="10" style="endArrow=block;html=1;" edge="1" parent="1" source="2" target="3">
-          <mxGeometry relative="1" as="geometry"/>
-        </mxCell>
-        <mxCell id="11" style="endArrow=block;html=1;" edge="1" parent="1" source="3" target="4">
-          <mxGeometry relative="1" as="geometry"/>
-        </mxCell>
-        <mxCell id="12" style="endArrow=block;html=1;" edge="1" parent="1" source="4" target="5">
-          <mxGeometry relative="1" as="geometry"/>
-        </mxCell>
-        <mxCell id="13" style="endArrow=block;html=1;" edge="1" parent="1" source="5" target="6">
-          <mxGeometry relative="1" as="geometry"/>
-        </mxCell>
-        <mxCell id="14" style="endArrow=block;html=1;" edge="1" parent="1" source="6" target="7">
-          <mxGeometry relative="1" as="geometry"/>
-        </mxCell>
-        <mxCell id="15" style="endArrow=block;html=1;" edge="1" parent="1" source="7" target="8">
-          <mxGeometry relative="1" as="geometry"/>
-        </mxCell>
-        <mxCell id="16" style="endArrow=block;html=1;" edge="1" parent="1" source="8" target="9">
-          <mxGeometry relative="1" as="geometry"/>
-        </mxCell>
-
-      </root>
-    </mxGraphModel>
-  </diagram>
-</mxfile>

package/ONBOARDING.md

@@ -1,29 +0,0 @@
-# 🧪 Developer Onboarding Guide
-
-## Getting Started
-
-Entry Point: AutomationRunner.ts
-
-Run: new AutomationRunner(request).run();
-
-## Execution Flow
-
-1.  Convert user story to steps
-2.  Extract action & keywords
-3.  Inject annotated DOM
-4.  Filter relevant DOM
-5.  Resolve FF-index
-6.  Execute action
-7.  Send script to backend
-
-## Adding a New Action
-
--   Create handler
--   Register in actionHandlerFactory
--   Add to INPUTLESS_ACTIONS if applicable
-
-## Debugging
-
--   Log DOM using saveDomToFile()
--   Validate FF-index mapping
--   Check token usage tracking

package/RELEASE_GUIDE.md

@@ -1,15 +0,0 @@
-#### for testing env
-
-git checkout feature/your-feature-name
-git add .
-git commit -m "feat: new changes"
-git pull --rebase
-npm version prerelease --preid=beta
-git push --follow-tags
-
-#### for Prodcut
-
-#### merge to release branch
-
-npm version patch
-git push --follow-tags

package/TECHNICAL_DEEP_DIVE.md

@@ -1,26 +0,0 @@
-# 🧠 Technical Deep Dive
-
-## Core Concept
-
-Each DOM element is assigned: FF-inspecter="Fire-Flink-X"
-
-This allows deterministic mapping from AI selection to XPath.
-
-## Token Optimization Strategy
-
-Instead of sending full DOM to LLM, FireFlink filters relevant nodes
-using DomProcessingEngine to reduce token usage.
-
-## Failure Handling Model
-
-Execution stops on: - Unsupported action - Invalid FF-index - XPath
-resolution failure - Handler execution failure
-
-CloseBrowser step is appended before termination.
-
-## Extensibility
-
--   Add new PromptType
--   Extend StepProcessor
--   Add ActionHandler
--   Modify payload schema

package/bitbucket-pipelines.yml

@@ -1,36 +0,0 @@
-image: node:20
-
-pipelines:
-  branches:
-    "feature/automation_V2":
-      - step:
-          name: Publish Beta
-          caches:
-            - node
-          script:
-            - echo "Installing dependencies"
-            - npm ci
-
-            - echo "Building project"
-            - npm run build
-
-            - echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc
-
-            - echo "Publishing as beta"
-            - npm publish --tag beta --access public
-    "release/automation_V2":
-      - step:
-          name: Publish Production
-          caches:
-            - node
-          script:
-            - echo "Installing dependencies"
-            - npm ci
-
-            - echo "Building project"
-            - npm run build
-
-            - echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > .npmrc
-
-            - echo "Publishing as latest"
-            - npm publish --access public

package/eslint.config.ts

@@ -1,29 +0,0 @@
-import js from "@eslint/js";
-import globals from "globals";
-import tseslint from "typescript-eslint";
-import { defineConfig } from "eslint/config";
-
-export default defineConfig([
-  {
-    files: ["**/*.{js,mjs,cjs,ts,mts,cts}"],
-    plugins: { js },
-    extends: ["js/recommended"],
-    languageOptions: { globals: globals.browser },
-  },
-
-  ...tseslint.configs.recommended,
-
-  {
-    rules: {
-      "no-console": ["error", { allow: ["warn", "error"] }],
-      "@typescript-eslint/no-explicit-any": "off",
-      "@typescript-eslint/no-unused-vars": [
-        "error",
-        {
-          argsIgnorePattern: "^_",
-          varsIgnorePattern: "^_",
-        },
-      ],
-    },
-  },
-]);

package/src/ai/llmcalls/decodeApiKey.ts

@@ -1,14 +0,0 @@
-
-export function decodeApiKey(apiKey: string): string {
-  if (!apiKey) {
-    throw new Error("API key is required");
-  }
-
-  try {
-    return Buffer.from(apiKey, "base64").toString("utf-8");
-  } catch (error) {
-   throw new Error("Failed to decode API key", { cause: error });
-  }
-}
-
-

package/src/ai/llmcalls/llmAction.ts

@@ -1,89 +0,0 @@
-import { prompts } from "../llmprompts/promptRegistry.js";
-import OpenAI from "openai";
-import { decodeApiKey } from "./decodeApiKey.js";
-import { FireFlinkLLMResponse } from "../../core/types/llmResponseType.js"
-import { PromptArgsMap } from "../../core/interfaces/promptInterface.js"
-import { userInputFormatters } from "../../ai/llmprompts/userPrompts/userPrompt.js"
-import { visionPromptMessage } from "../../core/types/visionllmInputType.js"
-import { PromptType } from "../../core/types/promptType.js";
-export class llmAction {
-    private client: OpenAI;
-    private model: string;
-    private visionClient: OpenAI
-    constructor(
-        apiKey: string,
-        baseURL: string,
-        model: string,
-        visionApiKey: string
-    ) {
-        this.model = model;
-        this.client = new OpenAI({
-            apiKey: decodeApiKey(apiKey),
-            baseURL: baseURL,
-        });
-        this.visionClient = new OpenAI({
-            apiKey: decodeApiKey(visionApiKey),
-        });
-
-    }
-
-    private async getLLMResponseText(system: string, user: string): Promise<FireFlinkLLMResponse> {
-        const response = await this.client.chat.completions.create({
-            model: this.model,
-            messages: [
-                {
-                    role: "system",
-                    content: system
-                },
-                {
-                    role: "user",
-                    content: user
-                }
-            ],
-            response_format: { type: "json_object" }
-        });
-        try {
-
-            return response;
-        } catch {
-            throw new Error(`Invalid JSON returned from LLM `);
-        }
-    }
-    async getLLMResponse<K extends keyof PromptArgsMap>(
-        type: K,
-        args: PromptArgsMap[K],
-        userInput: Record<string, any>
-    ): Promise<FireFlinkLLMResponse> {
-        const promptBuilder: (args: PromptArgsMap[K]) => Promise<string> =
-            prompts[type];
-        const systemPrompt = await promptBuilder(args);
-        const userPrompt: string | visionPromptMessage = userInputFormatters[type](userInput);
-        if (type === PromptType.VISION_PROMPT && Array.isArray(userPrompt)) {
-            return this.getLLMResponseWithVision(systemPrompt, userPrompt);
-        }
-        if (typeof userPrompt !== "string") {
-            throw new Error("Invalid non-vision user prompt format");
-        }
-        return this.getLLMResponseText(systemPrompt, userPrompt);
-    }
-
-    private async getLLMResponseWithVision(system: any, userPrompt: visionPromptMessage): Promise<string> {
-        const response = await this.visionClient.chat.completions.create({
-            model: "gpt-4.1",
-            messages: [
-                {
-                    role: "system",
-                    content: system
-                },
-                {
-                    role: "user",
-                    content: userPrompt
-                }
-            ],
-            response_format: { type: "json_object" }
-        });
-        return response.choices[0].message.content || "";
-    }
-
-
-}

package/src/ai/llmprompts/promptRegistry.ts

@@ -1,16 +0,0 @@
-import { keywordExtractor } from "./systemPrompts/actionExtractorPrompt.js";
-import { buildStepExtractionPrompt } from "./systemPrompts/userStoryToListPrompt.js";
-import { buildErrorDescriptionPrompt } from "./systemPrompts/errorDescriptionPrompt.js";
-import { ffInspectorNumExtractor } from "./systemPrompts/fireflinkElementIndexExtactors.js";
-import { visionPrompt } from "./systemPrompts/visionPrompt.js";
-import { PromptMap } from "../../core/types/promptMap.js";
-
-
-export const prompts: PromptMap = {
-    userStoryToList: buildStepExtractionPrompt,
-    keywordExtractorPrompt: keywordExtractor,
-    errorDescriptionPrompt: buildErrorDescriptionPrompt,
-    ffInspectorNumExtractor: ffInspectorNumExtractor,
-    visionPrompt: visionPrompt,
-};
-

package/src/ai/llmprompts/systemPrompts/errorDescriptionPrompt.ts

@@ -1,23 +0,0 @@
-export async function buildErrorDescriptionPrompt(): Promise<string> {
-  const description_prompt = `
-
-Analyze the error and execution history to provide diagnostics in JSON format.
-
-Return a JSON object with the following keys:
-1. "error": A concise, single-line, selenium type(error).__name__ (e.g., "InvalidSessionIdException", "InvalidElementStateException", "ElementNotInteractableException", "NoSuchElementException" etc).
-2. "errorDescription": A very simple, non-technical, and user-friendly explanation of why the automation might have failed.
-   - Use plain English.
-   - All the lines should contain FireFlink. example: "FireFlink was unable to find the element..."
-   - Do not mention code, XPaths, or internal technical terms.
-   - The output MUST be a single paragraph of exactly 10 to 15 lines of text.
-   - Focus on potential real-world causes.
-
-Respond only with JSON using this format:
-{
-  "error": ",
-  "errorDescription": "",
-}
-`;
-
-  return description_prompt;
-}

package/src/ai/llmprompts/systemPrompts/fireflinkElementIndexExtactors.ts

@@ -1,173 +0,0 @@
-type FFInspectorArgs = {
-  stepAction: string;
-  extractedDomJson: string;
-  priorAndNextSteps: string[];
-  isAlert?: boolean;
-  isDrag?: boolean;
-};
-export async function ffInspectorNumExtractor({
-  stepAction,
-  extractedDomJson,
-  priorAndNextSteps,
-  isAlert = false,
-  isDrag = false,
-}: FFInspectorArgs
-): Promise<
-  string
-> {
-  const clickActions = ["click", "doubleclick", "rightclick"];
-  const alertActions = [
-    "ClickOkOnAlertPopup",
-    "ClickCancelOnAlertPopup",
-    "EnterDataToAlert",
-    "GetTextPresentOnAlertPopup",
-    "VerifyAlertPopupIsDisplayed",
-    "VerifyAlertPopUpMessageContainsString",
-    "WaitTillAlertIsPresent",
-    "VerifyTextPresentOnAlertPopup"
-  ];
-  const elementType = ['link', 'textfield', 'icon', 'button', 'radioButton', 'text', 'textarea', 'image', 'dropdown', 'checkbox', 'tab', 'action overflow button', 'hamburger icon', 'toggle button', 'suggestion', 'time picker', 'date picker', 'toaster message', 'card', 'tooltip', 'option', 'calender', 'sliders', 'visual testing'];
-  const enterActions = ["enter", "clearandenter"];
-  let prompt;
-
-  // ---------------- ALERT ----------------
-  if (isAlert) {
-    prompt = `
-You are given a Selenium test step that requires interaction with a browser alert popup.
-
-Rules:
-- if the alert requires entering text or verification comparison, extract that text.
-- If no input is needed, set "input_text" to "None".
-- If step verifies text presence → VerifyTextPresentOnAlertPopup.
-- If step verifies text contains → VerifyAlertPopUpMessageContainsString.
-- Do not add explanations or extra text.
-- Return ONE action from this list only: ${JSON.stringify(alertActions)}.
-- Output must be valid JSON.
-Respond only with JSON using this format:
-
-{
-  "action": x,
-  "input_text": "text to enter" | "None"
-}
-
-`;
-  }
-
-  // ---------------- DRAG & DROP ----------------
-  else if (isDrag) {
-    prompt = `
-You are an intelligent assistant that extracts structured UI action data.
-
-Given a structured UI JSON representation with uniquely identified elements
-(ff-inspect values like Fire-Flink-1, Fire-Flink-2... in DOM order),
-locate the most appropriate elements for a drag-and-drop action.
-
-
-
-Rules:
-- Step: for the given step
-- Use the simplified JSON to find elements.
-- Identify source (drag) and destination (drop).
-- If not found, return Fire-Flink-0 for both.
-- Extract element names from step if possible.
-- Respond with JSON only.
-Simplified JSON:
-${extractedDomJson}
-
-Respond only with JSON using this format:
-{
-  "drag": "Fire-Flink-x",
-  "drop": "Fire-Flink-x",
-  "action": "Builtin_DragAndDropToElement",
-  "drag_element_name": "x",
-  "drop_element_name": "x"
-}
-`;
-  }
-
-  // ---------------- WAIT / SCROLL ----------------
-  else if (stepAction === "wait" || stepAction === "scroll") {
-    prompt = `
-You are an intelligent assistant that extracts structured UI action data.
-
-
-
-Rules:
-- Step: for the given step
-- Use context from test steps: ${JSON.stringify(priorAndNextSteps)}
-- Use simplified JSON to find best FF-inspect match.
-- Choose closest semantic + DOM-distance match.
-- Default direction is down.
-- Action:
-  - scroll → extract num_of_scrolls & direction
-  - wait → extract wait time (number only)
-- No nulls; use empty strings if needed.
-- The element type should be from this list ${elementType}
-
-Simplified JSON:
-${extractedDomJson}
-
-Return ONLY valid JSON:
-{
-  "attribute_value": "Fire-Flink-x",
-  "action": "x",
-  "input_text": "seconds in Interger | input text",
-  "keyword": "x",
-  "num_of_scrolls": "0",
-  "direction": "down",
-  "elementType":"x"
-}
-`;
-  }
-
-  // ---------------- DEFAULT (CLICK / ENTER / UPLOAD etc.) ----------------
-  else {
-    prompt = `
-
--You are an intelligent assistant that extracts structured UI action data.
-    -Given a structured UI JSON representation with uniquely identified elements (ff-inspect values like Fire-Flink-1, Fire-Flink-2, Fire-Flink-3... in DOM order), 
-    -locate the most appropriate element for an automation step by performing keyword-based matching using exact, partial, and fuzzy strategies, and 
-    return the identifier of the best match.
-    
-Return **only valid JSON** in the following format:
-{
-"attribute_value": "Fire-Flink-x",
-"action": "x",
-"input_text": "x",
-"keyword": "x",
-"num_of_scrolls": "0",
-"direction": "down",
-"elementType": "x"
-}
-
-Rules:
-You are an AI assistant. For the step, extract the element keyword from the step (the field name like 'Leaving from', 'Going To').
-- Use context from the ${JSON.stringify(priorAndNextSteps)}, keyword and filtered dom to search for FF-inspecter. 
-- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the atep.**
-- If multiple matches exist, use the reference element and choose the closest match by ff-inspect distance.
-  **important example: step: "click on login below user login" 
-            simplified json: 
-                        login - ff-inspect-300
-                        user login - ff-inspect-790
-                        login - ff-inspect-803
-                        then should return login - ff-inspect-803 witch is near to user reference element user login - ff-inspect-790**
-- Respect direction (up/down), default is down.
-- Do not return elements far away from the reference.
-- Select the closest semantic match to step and return only its attribute_value, else Fire-Flink-0.
-- Action: ${clickActions} for clicking or selecting, ${enterActions} for entering text, 'upload' for uploading file, 'drag_and_drop' for dragging and dropping, 'cleartext' for clearing text. 
-- For click action, if only step involving 'clicking on icon' then try to return related svg tags attribute_value, if they are available in simplified json.
-- For Action ${enterActions} extract input text from step.
-- **For ${enterActions} actions, never directly select a input tag if should be realted to step; if u can't find input tag related to step then prefer the closest label/span/div/b/i etc, it can be any tag related to the step.**
-- **For ${enterActions} step if you can't find perfect matching element return attribute_value as Fire-Flink-0 for the step. it should be exact match for the step.**
-- For ${enterActions} step, If the step implies autogenerated data or random data(email, phone, credentials, identifiers, etc.), generate a valid dummy input_text suitable for the field. 
-example: step: "Enter sss.@gmail.com in email field" and step : 
-        you should check only for email in Simplified JSON and input_text should be "sss.@gmail.com" dont add any extra text.
-- For Action "upload" extract file path from step.
-- if u cant find any input_text or any other dont give null just return them "" empty.
-- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
-Simplified JSON: ${extractedDomJson} if this simplified json is empty return Fire-Flink-0.
-
-`;
-  }
-  return prompt;
-}

package/src/ai/llmprompts/systemPrompts/userStoryToListPrompt.ts

@@ -1,24 +0,0 @@
-
-export async function buildStepExtractionPrompt(
-): Promise<string> {
-  const prompt = `
-You are a structured automation assistant. Your task is to extract step data from a provided user story for web automation.
-
-Requirements
-Each step must be a single string (not a tuple or any other data type).
-Preserve the original wording of each step.
-You may rewrite for clarity or formatting consistency only.
-Do not add new steps.
-Do not remove any existing steps.
-Do not introduce additional actions.
-Do not include the example list in the response.
-Do not include any extra text, commentary, markdown, or code fences.
-Output Format
-
-Respond strictly in valid JSON using the following structure:
-{
-"manaulSteps": ["step 1", "step 2", "step 3"]
-}
-`;
-  return prompt;
-}

package/src/ai/llmprompts/systemPrompts/visionPrompt.ts

@@ -1,28 +0,0 @@
-
-type visionPrompt = {
-    currentStep: string,
-    priorAndNextSteps: string[]
-}
-export async function visionPrompt(
-    { currentStep, priorAndNextSteps }: visionPrompt
-): Promise<string> {
-
-    const customPromptText = `
-You are a precise automation assistant.
-Context: ${priorAndNextSteps}
-- Use the context to understand the step and select the closest semantic match to ${currentStep} and return only its index, else Fire-Flink-0.
-- **If u can't find the element, return Fire-Flink-0. dont return any other elements index.**
-- Using the provided annotated screenshot and ${currentStep}, identify the correct element index from the image.
-- Match the index to the text or action described in ${currentStep} as accurately as possible.
-- Return only the Fire-Flink-xx for the target element.
-- Do not provide any additional text or explanation. The output must be strictly Fire-Flink-xx.
-Now, Analyze the step:
-Step: "${currentStep}"
-
-Respond only with JSON using this format:
-{fireflinkIndex: Fire-Flink-xx.}
-
-`.trim();
-
-    return customPromptText;
-}

package/src/ai/llmprompts/userPrompts/userPrompt.ts

@@ -1,41 +0,0 @@
-import { PromptType } from "../../../core/types/promptType.js";
-import { visionPromptMessage } from "../../../core/types/visionllmInputType.js";
-
-export const userInputFormatters: {
-    [K in PromptType]: (
-        userInput: Record<string, any>
-    ) => string | visionPromptMessage;
-} = {
-    [PromptType.USER_STORY_TO_LIST]: (userInput) =>
-        `Convert the following user story into steps:\n\n${JSON.stringify(userInput.userStory, null, 2)}`,
-
-    [PromptType.KEYWORD_EXTRACTOR]: (userInput) =>
-        `Extract keywords from the following input:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
-
-    [PromptType.ERROR_DESCRIPTION]: (userInput) =>
-        `Analyze the following error context:\n\n${JSON.stringify(userInput.error, null, 2)}`,
-
-    [PromptType.FF_INSPECTOR]: (userInput) =>
-        `Inspect the DOM for the following action context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
-
-    [PromptType.VISION_PROMPT]: (userInput) => {
-        const content: visionPromptMessage = [
-            {
-                type: "text",
-                text: `Analyze the following step:\n\n${JSON.stringify(
-                    userInput.currentStep,
-                    null,
-                    2
-                )}`
-            },
-            {
-                type: "image_url",
-                image_url: {
-                    url: String(userInput.imageBase64 ?? "")
-                }
-            }
-        ];
-        return content;
-    }
-
-};