ff-automationv2 1.0.0

package/src/ai/llmprompts/systemPrompts/fireflinkElementIndexExtactors.ts

@@ -0,0 +1,198 @@
+type FFInspectorArgs = {
+  stepAction: string;
+  extractedDomJson: string;
+  priorAndNextSteps: string[];
+  isAlert?: boolean;
+  isDrag?: boolean;
+};
+export async function ffInspectorNumExtractor({
+  stepAction,
+  extractedDomJson,
+  priorAndNextSteps,
+  isAlert = false,
+  isDrag = false,
+}: FFInspectorArgs
+): Promise<
+  string
+> {
+  const clickActions = ["click", "doubleclick", "rightclick"];
+  const alertActions = [
+    "ClickOkOnAlertPopup",
+    "ClickCancelOnAlertPopup",
+    "EnterDataToAlert",
+    "GetTextPresentOnAlertPopup",
+    "VerifyAlertPopupIsDisplayed",
+    "VerifyAlertPopUpMessageContainsString",
+    "WaitTillAlertIsPresent",
+    "VerifyTextPresentOnAlertPopup"
+  ];
+  const elementType = ["button", "link"]
+
+  let prompt;
+
+  // ---------------- ALERT ----------------
+  if (isAlert) {
+    prompt = `
+You are given a Selenium test step that requires interaction with a browser alert popup.
+
+Rules:
+- if the alert requires entering text or verification comparison, extract that text.
+- If no input is needed, set "input_text" to "None".
+- If step verifies text presence → VerifyTextPresentOnAlertPopup.
+- If step verifies text contains → VerifyAlertPopUpMessageContainsString.
+- Do not add explanations or extra text.
+- Return ONE action from this list only: ${JSON.stringify(alertActions)}.
+- Output must be valid JSON.
+Respond only with JSON using this format:
+
+{
+  "action": x,
+  "input_text": "text to enter" | "None"
+}
+
+`;
+  }
+
+  // ---------------- DRAG & DROP ----------------
+  else if (isDrag) {
+    prompt = `
+You are an intelligent assistant that extracts structured UI action data.
+
+Given a structured UI JSON representation with uniquely identified elements
+(ff-inspect values like Fire-Flink-1, Fire-Flink-2... in DOM order),
+locate the most appropriate elements for a drag-and-drop action.
+
+
+
+Rules:
+- Step: for the given step
+- Use the simplified JSON to find elements.
+- Identify source (drag) and destination (drop).
+- If not found, return Fire-Flink-0 for both.
+- Extract element names from step if possible.
+- Respond with JSON only.
+Simplified JSON:
+${extractedDomJson}
+
+Respond only with JSON using this format:
+{
+  "drag": "Fire-Flink-x",
+  "drop": "Fire-Flink-x",
+  "action": "Builtin_DragAndDropToElement",
+  "drag_element_name": "x",
+  "drop_element_name": "x"
+}
+`;
+  }
+
+  // ---------------- WAIT / SCROLL ----------------
+  else if (stepAction === "wait" || stepAction === "scroll") {
+    prompt = `
+You are an intelligent assistant that extracts structured UI action data.
+
+
+
+Rules:
+- Step: for the given step
+- Use context from test steps: ${JSON.stringify(priorAndNextSteps)}
+- Use simplified JSON to find best FF-inspect match.
+- Choose closest semantic + DOM-distance match.
+- Default direction is down.
+- Action:
+  - scroll → extract num_of_scrolls & direction
+  - wait → extract wait time (number only)
+- No nulls; use empty strings if needed.
+- The element type should be from this list ${elementType}
+
+Simplified JSON:
+${extractedDomJson}
+
+Return ONLY valid JSON:
+{
+  "attribute_value": "Fire-Flink-x",
+  "action": "x",
+  "input_text": "seconds in Interger | input text",
+  "keyword": "x",
+  "num_of_scrolls": "0",
+  "direction": "down",
+  "elementType":"x"
+}
+`;
+  }
+
+  // ---------------- DEFAULT (CLICK / ENTER / UPLOAD etc.) ----------------
+  else {
+    prompt = `
+You are a deterministic UI action extraction engine.
+
+OBJECTIVE:
+Identify the SINGLE best matching element from the Simplified JSON
+for the given step and return structured automation data.
+
+-----------------------------------------
+
+CONTEXT STEPS:
+${JSON.stringify(priorAndNextSteps)}
+
+CLICK ACTION WORDS:
+${JSON.stringify(clickActions)}
+
+ALLOWED ELEMENT TYPES:
+${elementType}
+-----------------------------------------
+
+CRITICAL MATCHING LOGIC:
+
+1. First understand intent:
+   - click words → click
+   - enter/type/write → enter_text
+   - upload → upload
+   - drag_and_drop → drag_and_drop
+
+2. For ENTER actions:
+   DO NOT pick first input blindly.
+   Score candidates using priority:
+
+   Priority Order:
+   (1) Exact label text match
+   (2) Placeholder match
+   (3) Name attribute semantic match
+   (4) ID semantic match
+   (5) type="email" if step contains "email"
+   (6) DOM sibling label relationship
+
+   Choose highest scoring match.
+
+   If NO strong semantic match → return:
+   attribute_value = "Fire-Flink-0"
+
+3. For CLICK actions:
+   - Prefer exact text match
+   - If multiple, choose closest contextual match
+   - If icon + svg exists, prefer svg element
+
+4. Never hallucinate.
+5. Only use FF values from provided JSON.
+6. No null values. Use empty string "".
+7. Output must be valid JSON only.
+
+-----------------------------------------
+SIMPLIFIED JSON:
+${extractedDomJson}
+-----------------------------------------
+
+Respond ONLY with JSON:
+
+{
+  "attribute_value": "Fire-Flink-x",
+  "action": "click | enter | upload | drag_and_drop",
+  "input_text": "",
+  "keyword": "",
+  "num_of_scrolls": "0",
+  "direction": "down",
+  "elementType": ""
+}
+`;
+  }
+  return prompt;
+}

package/src/ai/llmprompts/systemPrompts/userStoryToListPrompt.ts

@@ -0,0 +1,24 @@
+
+export async function buildStepExtractionPrompt(
+): Promise<string> {
+  const prompt = `
+You are a structured automation assistant. Your task is to extract step data from a provided user story for web automation.
+
+Requirements
+Each step must be a single string (not a tuple or any other data type).
+Preserve the original wording of each step.
+You may rewrite for clarity or formatting consistency only.
+Do not add new steps.
+Do not remove any existing steps.
+Do not introduce additional actions.
+Do not include the example list in the response.
+Do not include any extra text, commentary, markdown, or code fences.
+Output Format
+
+Respond strictly in valid JSON using the following structure:
+{
+"manaulSteps": ["step 1", "step 2", "step 3"]
+}
+`;
+  return prompt;
+}

package/src/ai/llmprompts/systemPrompts/visionPrompt.ts

@@ -0,0 +1,28 @@
+
+type visionPrompt = {
+    currentStep: string,
+    priorAndNextSteps: string[]
+}
+export async function visionPrompt(
+    { currentStep, priorAndNextSteps }: visionPrompt
+): Promise<string> {
+
+    const customPromptText = `
+You are a precise automation assistant.
+Context: ${priorAndNextSteps}
+- Use the context to understand the step and select the closest semantic match to ${currentStep} and return only its index, else Fire-Flink-0.
+- **If u can't find the element, return Fire-Flink-0. dont return any other elements index.**
+- Using the provided annotated screenshot and ${currentStep}, identify the correct element index from the image.
+- Match the index to the text or action described in ${currentStep} as accurately as possible.
+- Return only the Fire-Flink-xx for the target element.
+- Do not provide any additional text or explanation. The output must be strictly Fire-Flink-xx.
+Now, Analyze the step:
+Step: "${currentStep}"
+
+Respond only with JSON using this format:
+{fireflinkIndex: Fire-Flink-xx.}
+
+`.trim();
+
+    return customPromptText;
+}

package/src/ai/llmprompts/userPrompts/userPrompt.ts

@@ -0,0 +1,41 @@
+import { PromptType } from "../../../core/types/promptType.js";
+import { visionPromptMessage } from "../../../core/types/visionllmInputType.js";
+
+export const userInputFormatters: {
+    [K in PromptType]: (
+        userInput: Record<string, any>
+    ) => string | visionPromptMessage;
+} = {
+    [PromptType.USER_STORY_TO_LIST]: (userInput) =>
+        `Convert the following user story into steps:\n\n${JSON.stringify(userInput.userStory, null, 2)}`,
+
+    [PromptType.KEYWORD_EXTRACTOR]: (userInput) =>
+        `Extract keywords from the following input:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
+
+    [PromptType.ERROR_DESCRIPTION]: (userInput) =>
+        `Analyze the following error context:\n\n${JSON.stringify(userInput.error, null, 2)}`,
+
+    [PromptType.FF_INSPECTOR]: (userInput) =>
+        `Inspect the DOM for the following action context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
+
+    [PromptType.VISION_PROMPT]: (userInput) => {
+        const content: visionPromptMessage = [
+            {
+                type: "text",
+                text: `Analyze the following step:\n\n${JSON.stringify(
+                    userInput.currentStep,
+                    null,
+                    2
+                )}`
+            },
+            {
+                type: "image_url",
+                image_url: {
+                    url: String(userInput.imageBase64 ?? "")
+                }
+            }
+        ];
+        return content;
+    }
+
+};

package/src/automation/actions/executor.ts

@@ -0,0 +1,75 @@
+import type { IActionExecutor } from "../../core/interfaces/actionInterface.js";
+import { navigate } from './navigation/navigate.js';
+import { goBack } from './navigation/goBack.js';
+import { pageRefresh } from './navigation/refresh.js';
+import { click } from './interaction/click.js';
+import { wait } from "./wait/wait.js"
+import { enterInput } from "./interaction/enterInput.js";
+import { ScriptDataAppender } from "../../fireflinkData/fireflinkScript/scriptGenrationData.js";
+import { ElementGetter } from "../../fireflinkData/fireflinkLocators/getListOfLocators.js";
+export class ActionExecutor implements IActionExecutor {
+    constructor(private readonly browser: WebdriverIO.Browser,
+        private scriptDataAppender: ScriptDataAppender,
+        private elementGetter: ElementGetter,
+        private platform: string) { }
+
+
+
+    async navigate(url: string): Promise<void> {
+        await navigate({
+            browser: this.browser,
+            url,
+            scriptDataAppender: this.scriptDataAppender
+        });
+    }
+
+    async goBack(): Promise<void> {
+        await goBack({
+            browser: this.browser,
+            scriptDataAppender: this.scriptDataAppender
+        });
+    }
+
+    async refresh(): Promise<void> {
+        await pageRefresh({
+            browser: this.browser,
+            scriptDataAppender: this.scriptDataAppender
+        });
+    }
+
+
+    async wait(time: string): Promise<void> {
+        await wait({
+            browser: this.browser,
+            time,
+            scriptDataAppender: this.scriptDataAppender
+        });
+    }
+    async click(pageDOM: string, selector: string, fireflinkIndex: string, elementName: string, elementType: string): Promise<void> {
+        await click({
+            browser: this.browser,
+            selector,
+            scriptDataAppender: this.scriptDataAppender,
+            elementGetter: this.elementGetter,
+            fireflinkIndex,
+            pageDOM,
+            elementName,
+            elementType,
+            platform: this.platform
+        });
+    }
+    async enterInput(selector: string, value: string, fireflinkIndex: string, pageDOM: string, elementName: string, elementType: string): Promise<void> {
+        await enterInput({
+            browser: this.browser,
+            selector: selector,
+            value,
+            scriptDataAppender: this.scriptDataAppender,
+            elementGetter: this.elementGetter,
+            fireflinkIndex,
+            pageDOM,
+            elementName,
+            elementType,
+            platform: this.platform
+        });
+    }
+}

package/src/automation/actions/interaction/click.ts

@@ -0,0 +1,25 @@
+import { ClickInterface } from "../interface/interactionActionInterface.js";
+export async function click(args: ClickInterface): Promise<void> {
+    try {
+        const element = await args.browser.$(args.selector);
+        await element.scrollIntoView({ block: 'center', inline: 'center' });
+        await element.click();
+        const ffElement: any = await args.elementGetter.getFireFlinkElement(
+            args.pageDOM,
+            `[ff-inspect="${args.fireflinkIndex}"]`
+        );
+        args.scriptDataAppender.add(
+            {
+                nlpName: 'Click',
+                stepInputs: [],
+                elementsData: [{
+                    elementName: args.elementName,
+                    type: args.elementType,
+                    locators: ffElement.locators,
+                    platform: args.platform
+                }]
+            });
+    } catch (error: any) {
+        throw new Error("Click action failed", { cause: error });
+    }
+}

package/src/automation/actions/interaction/enterInput.ts

@@ -0,0 +1,27 @@
+import { EnterInputInterface } from "../interface/interactionActionInterface.js";
+export async function enterInput(args: EnterInputInterface): Promise<void> {
+    try {
+        const element = args.browser.$(args.selector);
+        await element.scrollIntoView({ block: 'center', inline: 'center' });
+        await element.setValue(args.value);
+        const ffElement: any = await args.elementGetter.getFireFlinkElement(
+            args.pageDOM,
+            `[ff-inspect="${args.fireflinkIndex}"]`
+        );
+        args.scriptDataAppender.add({
+            nlpName: 'SendKeys',
+            stepInputs: [args.value],
+            elementsData: [{
+                elementName: args.elementName,
+                type: args.elementType,
+                locators: ffElement.locators,
+                platform: args.platform
+            }]
+        });
+
+    }
+    catch (error: any) {
+        throw new Error("enter action failed", { cause: error });
+    }
+
+}

package/src/automation/actions/interface/interactionActionInterface.ts

@@ -0,0 +1,27 @@
+import { ScriptDataAppender } from "../../../fireflinkData/fireflinkScript/scriptGenrationData.js";
+import { ElementGetter } from "../../../fireflinkData/fireflinkLocators/getListOfLocators.js";
+export interface ClickInterface {
+  browser: WebdriverIO.Browser;
+  selector: string;
+  scriptDataAppender: ScriptDataAppender;
+  elementGetter: ElementGetter;
+  fireflinkIndex: string;
+  pageDOM: string;
+  elementName: string;
+  elementType: any;
+  platform: string;
+}
+
+
+export interface EnterInputInterface {
+  browser: WebdriverIO.Browser;
+  selector: string;
+  value: string;
+  scriptDataAppender: ScriptDataAppender;
+  elementGetter: ElementGetter;
+  fireflinkIndex: string;
+  pageDOM: string;
+  elementName: string;
+  elementType: any;
+  platform: string;
+}

package/src/automation/actions/interface/navigationActionInterface.ts

@@ -0,0 +1,22 @@
+import { ScriptDataAppender } from "../../../fireflinkData/fireflinkScript/scriptGenrationData.js";
+
+export interface getTitleInterface {
+  browser: WebdriverIO.Browser;
+  scriptDataAppender: ScriptDataAppender;
+}
+
+export interface goBackInterface {
+  browser: WebdriverIO.Browser;
+  scriptDataAppender: ScriptDataAppender;
+}
+
+export interface navigateToUrlInterface {
+  browser: WebdriverIO.Browser;
+  scriptDataAppender: ScriptDataAppender;
+    url: string;
+}
+
+export interface refreshInterface {
+  browser: WebdriverIO.Browser;
+  scriptDataAppender: ScriptDataAppender;
+}

package/src/automation/actions/interface/waitActionInterface.ts

@@ -0,0 +1,6 @@
+import { ScriptDataAppender } from "../../../fireflinkData/fireflinkScript/scriptGenrationData.js";
+export interface waitInterface {
+    browser: WebdriverIO.Browser;
+    scriptDataAppender: ScriptDataAppender;
+    time: string;
+}

package/src/automation/actions/navigation/getTitle.ts

@@ -0,0 +1,9 @@
+import { getTitleInterface } from "../interface/navigationActionInterface.js";
+export async function getTitle(args: getTitleInterface): Promise<void> {
+    try {
+        const title = await args.browser.getTitle();
+        args.scriptDataAppender.add({ nlpName: 'getTitle', elementsData: [], stepInputs: [title] });
+    } catch (error: any) {
+        throw new Error("Get title action failed", { cause: error });
+    }
+}

package/src/automation/actions/navigation/goBack.ts

@@ -0,0 +1,9 @@
+import { goBackInterface } from "../interface/navigationActionInterface.js";
+export async function goBack(args: goBackInterface): Promise<void> {
+    try {
+        await args.browser.back();
+        args.scriptDataAppender.add({ nlpName: 'goBack', elementsData: [], stepInputs: [] });
+    } catch (error: any) {
+        throw new Error("Navigate action failed", { cause: error });
+    }
+}

package/src/automation/actions/navigation/navigate.ts

@@ -0,0 +1,10 @@
+import { navigateToUrlInterface } from "../interface/navigationActionInterface.js";
+export async function navigate(args: navigateToUrlInterface): Promise<void> {
+    try {
+        await args.browser.url(args.url);
+        args.scriptDataAppender.add({ nlpName: 'NavigateToURL', elementsData: [], stepInputs: [args.url] });
+
+    } catch (error: any) {
+        throw new Error("Navigate action failed", { cause: error });
+    }
+}

package/src/automation/actions/navigation/refresh.ts

@@ -0,0 +1,9 @@
+import { refreshInterface } from "../interface/navigationActionInterface.js";
+export async function pageRefresh(args: refreshInterface): Promise<void> {
+    try {
+        await args.browser.refresh();
+        args.scriptDataAppender.add({ nlpName: 'refresh', elementsData: [], stepInputs: [] });
+    } catch (error: any) {
+        throw new Error("Refresh action failed", { cause: error });
+    }
+}

package/src/automation/actions/wait/wait.ts

@@ -0,0 +1,10 @@
+import { waitInterface } from "../interface/waitActionInterface.js";
+export async function wait(args: waitInterface): Promise<void> {
+    try {
+        await args.browser.pause(Number(args.time) * 1000);
+        args.scriptDataAppender.add({ nlpName: 'Sleep', elementsData: [], stepInputs: [args.time] });
+
+    } catch (error: any) {
+        throw new Error("Navigate action failed", { cause: error });
+    }
+}

package/src/automation/browserSession/initiateBrowserSession.ts

@@ -0,0 +1,81 @@
+import { remote } from "webdriverio";
+import { browserCapabilities } from "../../core/interfaces/browserCapabilitiesInterface.js";
+
+export class BrowserSession {
+    private browser: WebdriverIO.Browser | null = null;
+
+    private async openBrowser(capabilities: browserCapabilities): Promise<WebdriverIO.Browser> {
+        try {
+            const browser = await remote(
+                {
+                    capabilities: {
+                        browserName: capabilities.browserName,
+                    },
+                    logLevel: "silent"
+                }
+            );
+            browser.setTimeout({ 'pageLoad': capabilities.pageLoad, 'implicit': capabilities.implicit });
+            await browser.waitUntil(async () => {
+                const state = await browser.execute(() => document.readyState);
+                return state === "complete";
+            });
+
+            return browser;
+        } catch (error: any) {
+            throw new Error("Browser failed to open", { cause: error });
+        }
+    }
+
+
+    async open(capabilities: browserCapabilities): Promise<void> {
+        this.browser = await this.openBrowser(capabilities);
+
+    }
+
+    async close(): Promise<void> {
+        if (this.browser) {
+            await this.browser.deleteSession();
+            this.browser = null;
+        }
+    }
+
+    async getCurrentBrowser(): Promise<WebdriverIO.Browser> {
+        if (!this.browser) {
+            throw new Error("Browser is not initialized");
+        }
+        return this.browser;
+    }
+
+
+    async waitForStableDOM(timeout = 10000): Promise<void> {
+        if (!this.browser) throw new Error("Browser session not found");
+
+        let lastHTML = "";
+        let stableCount = 0;
+
+        try {
+            await this.browser.waitUntil(async () => {
+                if (!this.browser) return;
+                const currentHTML = await this.browser.execute(() =>
+                    document.documentElement.outerHTML
+                );
+                if (currentHTML === lastHTML) {
+                    stableCount++;
+                } else {
+                    stableCount = 0;
+                }
+
+                lastHTML = currentHTML;
+
+                return stableCount >= 3;
+            }, {
+                timeout,
+                interval: 500
+            });
+        } catch {
+            return;
+        }
+    }
+
+
+}

package/src/core/constants/supportedActions.ts

@@ -0,0 +1,8 @@
+export const INPUTLESS_ACTIONS = [
+    "open",
+    "goBack",
+    "refresh",
+    "navigate"
+];
+
+export const ELEMENTLESS_ACTION = ["wait"]

package/src/core/interfaces/StableDomInterface.ts

@@ -0,0 +1,6 @@
+export interface StableDOM {
+    browser: WebdriverIO.Browser | null,
+    timeout: number,
+    idleMs: number
+
+}

package/src/core/interfaces/actionInterface.ts

@@ -0,0 +1,13 @@
+
+
+// Main action executor interface
+export interface IActionExecutor {
+  navigate(url: string): Promise<void>;
+  goBack(): Promise<void>;
+  refresh(): Promise<void>;
+  wait(time: string): Promise<void>;
+  click(pageDOM: string, selector: string, fireflinkIndex: string, elementName: string, elementType: string): Promise<void>;
+  enterInput(selector: string, value: string, fireflinkIndex: string, pageDOM: string, elementName: string, elementType: string, platform: string): Promise<void>;
+}
+
+

package/src/core/interfaces/automationRunnerInterface.ts

@@ -0,0 +1,3 @@
+export interface IAutomationRunner {
+  run(): Promise<void>;
+}

package/src/core/interfaces/browserCapabilitiesInterface.ts

@@ -0,0 +1,5 @@
+export interface browserCapabilities {
+    browserName: string;
+    pageLoad: number;
+    implicit: number;
+}

package/src/core/interfaces/browserConfigurationInterface.ts

@@ -0,0 +1,3 @@
+export interface BrowserConfig {
+    browserName: string;
+}