ff-automationv2 1.0.0

package/ARCHITECTURE.md

@@ -0,0 +1,112 @@
+# 📊 FireFlink System Architecture
+
+## Layered Architecture
+
+```
+┌────────────────────────────────────────────┐
+│ User Story Layer                           │
+└────────────────────────────────────────────┘
+                    ↓
+┌────────────────────────────────────────────┐
+│ LLM Reasoning Layer                        │
+│ - USER_STORY_TO_LIST                       │
+│ - KEYWORD_EXTRACTOR                        │
+│ - FF_INSPECTOR                             │
+└────────────────────────────────────────────┘
+                    ↓
+┌────────────────────────────────────────────┐
+│ Orchestration Layer                        │
+│ (AutomationRunner)                         │
+└────────────────────────────────────────────┘
+                    ↓
+┌────────────────────────────────────────────┐
+│ DOM Abstraction Layer                      │
+│ - Inject FF-inspecter attributes           │
+│ - Serialize DOM                            │
+│ - Build FF → XPath mapping                 │
+└────────────────────────────────────────────┘
+                    ↓
+┌────────────────────────────────────────────┐
+│ DOM Processing Engine                      │
+│ - Keyword-based DOM filtering              │
+└────────────────────────────────────────────┘
+                    ↓
+┌────────────────────────────────────────────┐
+│ Action Execution Layer                     │
+│ - Click Handler                            │
+│ - Enter Handler                            │
+│ - Navigate Handler                         │
+└────────────────────────────────────────────┘
+                    ↓
+┌────────────────────────────────────────────┐
+│ Script Generation Layer                    │
+│ - ScriptAppender                           │
+│ - Payload Builder                          │
+└────────────────────────────────────────────┘
+                    ↓
+┌────────────────────────────────────────────┐
+│ FireFlink Backend API                      │
+└────────────────────────────────────────────┘
+
+```
+
+## Execution Sequence
+
+AutomationRunner
+→ StepProcessor (LLM)
+→ Browser (Inject DOM)
+→ DomProcessingEngine
+→ StepProcessor (FF selection)
+→ ActionHandler
+→ ScriptAppender
+→ ScriptRunner
+→ FireFlink API
+
+## high Level flow
+
+```
+            User Story
+                ↓
+            LLM → Manual Steps
+                ↓
+            FOR EACH STEP
+                ↓
+            LLM → Action + Keywords
+                ↓
+            Inject Annotated DOM
+                ↓
+            Filter Relevant DOM (DomProcessingEngine)
+                ↓
+            LLM → Choose FF-inspecter Index
+                ↓
+            Map FF Index → XPath
+                ↓
+            Execute Action
+                ↓
+            Append Script Data
+                ↓
+┌─────────────────────────────┐
+│        Error Occurred?      │
+└──────────────┬──────────────┘
+               │
+        ┌──────┴──────┐
+        │             │
+       NO            YES
+        │             │
+        │      Append CloseBrowser
+        │             ↓
+        │      Stop Execution
+        │             |
+        ↓             |
+Continue Next Step    |
+        ↓             |
+(all Steps Complete)  |
+        |             |
+        |             |
+        └──────┬──────┘
+        Collect Token Usage
+               ↓
+        Build Payload
+               ↓
+        Send to FireFlink API
+```

package/FireFlink_Architecture.drawio

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<mxfile host="app.diagrams.net">
+  <diagram name="FireFlink Architecture">
+    <mxGraphModel dx="1000" dy="1000" grid="1" gridSize="10" guides="1" tooltips="1"
+      connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="850" pageHeight="1100"
+      math="0" shadow="0">
+      <root>
+        <mxCell id="0"/>
+        <mxCell id="1" parent="0"/>
+        
+        <mxCell id="2" value="User Story" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="300" y="20" width="200" height="60" as="geometry"/>
+        </mxCell>
+
+        <mxCell id="3" value="LLM Reasoning Layer" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="300" y="120" width="200" height="60" as="geometry"/>
+        </mxCell>
+
+        <mxCell id="4" value="AutomationRunner (Orchestration)" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="300" y="220" width="200" height="60" as="geometry"/>
+        </mxCell>
+
+        <mxCell id="5" value="DOM Abstraction Layer (FF-inspecter Injection)" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="300" y="320" width="200" height="60" as="geometry"/>
+        </mxCell>
+
+        <mxCell id="6" value="DOM Processing Engine" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="300" y="420" width="200" height="60" as="geometry"/>
+        </mxCell>
+
+        <mxCell id="7" value="Action Execution Layer" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="300" y="520" width="200" height="60" as="geometry"/>
+        </mxCell>
+
+        <mxCell id="8" value="Script Generation Layer" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="300" y="620" width="200" height="60" as="geometry"/>
+        </mxCell>
+
+        <mxCell id="9" value="FireFlink Backend API" style="rounded=1;whiteSpace=wrap;html=1;" vertex="1" parent="1">
+          <mxGeometry x="300" y="720" width="200" height="60" as="geometry"/>
+        </mxCell>
+
+        <mxCell id="10" style="endArrow=block;html=1;" edge="1" parent="1" source="2" target="3">
+          <mxGeometry relative="1" as="geometry"/>
+        </mxCell>
+        <mxCell id="11" style="endArrow=block;html=1;" edge="1" parent="1" source="3" target="4">
+          <mxGeometry relative="1" as="geometry"/>
+        </mxCell>
+        <mxCell id="12" style="endArrow=block;html=1;" edge="1" parent="1" source="4" target="5">
+          <mxGeometry relative="1" as="geometry"/>
+        </mxCell>
+        <mxCell id="13" style="endArrow=block;html=1;" edge="1" parent="1" source="5" target="6">
+          <mxGeometry relative="1" as="geometry"/>
+        </mxCell>
+        <mxCell id="14" style="endArrow=block;html=1;" edge="1" parent="1" source="6" target="7">
+          <mxGeometry relative="1" as="geometry"/>
+        </mxCell>
+        <mxCell id="15" style="endArrow=block;html=1;" edge="1" parent="1" source="7" target="8">
+          <mxGeometry relative="1" as="geometry"/>
+        </mxCell>
+        <mxCell id="16" style="endArrow=block;html=1;" edge="1" parent="1" source="8" target="9">
+          <mxGeometry relative="1" as="geometry"/>
+        </mxCell>
+
+      </root>
+    </mxGraphModel>
+  </diagram>
+</mxfile>

package/ONBOARDING.md

@@ -0,0 +1,29 @@
+# 🧪 Developer Onboarding Guide
+
+## Getting Started
+
+Entry Point: AutomationRunner.ts
+
+Run: new AutomationRunner(request).run();
+
+## Execution Flow
+
+1.  Convert user story to steps
+2.  Extract action & keywords
+3.  Inject annotated DOM
+4.  Filter relevant DOM
+5.  Resolve FF-index
+6.  Execute action
+7.  Send script to backend
+
+## Adding a New Action
+
+-   Create handler
+-   Register in actionHandlerFactory
+-   Add to INPUTLESS_ACTIONS if applicable
+
+## Debugging
+
+-   Log DOM using saveDomToFile()
+-   Validate FF-index mapping
+-   Check token usage tracking

package/README.md

@@ -0,0 +1,28 @@
+# 🔥 FireFlink AI Automation Platform
+
+Enterprise-grade AI-powered automation engine that transforms natural
+language user stories into executable browser automation scripts using
+LLM-driven reasoning and intelligent DOM abstraction.
+
+## Executive Summary
+
+FireFlink enables organizations to: - Convert user stories into
+automated test scripts - Reduce manual scripting effort - Improve
+element resilience using AI-based DOM interpretation - Track LLM token
+usage and automation generation cost - Integrate automation results
+directly into backend infrastructure
+
+## Core Capabilities
+
+-   AI Step Generation
+-   Intelligent Element Resolution
+-   DOM Annotation Engine (FF-inspecter system)
+-   Modular Action Handler Framework
+-   Failure Isolation & Safe Termination
+-   Token Usage Monitoring
+-   API Integration Layer
+
+## High-Level Execution Lifecycle
+
+User Story → LLM Steps → DOM Annotation → Element Resolution → Action
+Execution → Script Generation → API Submission

package/TECHNICAL_DEEP_DIVE.md

@@ -0,0 +1,26 @@
+# 🧠 Technical Deep Dive
+
+## Core Concept
+
+Each DOM element is assigned: FF-inspecter="Fire-Flink-X"
+
+This allows deterministic mapping from AI selection to XPath.
+
+## Token Optimization Strategy
+
+Instead of sending full DOM to LLM, FireFlink filters relevant nodes
+using DomProcessingEngine to reduce token usage.
+
+## Failure Handling Model
+
+Execution stops on: - Unsupported action - Invalid FF-index - XPath
+resolution failure - Handler execution failure
+
+CloseBrowser step is appended before termination.
+
+## Extensibility
+
+-   Add new PromptType
+-   Extend StepProcessor
+-   Add ActionHandler
+-   Modify payload schema

package/eslint.config.ts

@@ -0,0 +1,29 @@
+import js from "@eslint/js";
+import globals from "globals";
+import tseslint from "typescript-eslint";
+import { defineConfig } from "eslint/config";
+
+export default defineConfig([
+  {
+    files: ["**/*.{js,mjs,cjs,ts,mts,cts}"],
+    plugins: { js },
+    extends: ["js/recommended"],
+    languageOptions: { globals: globals.browser },
+  },
+
+  ...tseslint.configs.recommended,
+
+  {
+    rules: {
+      "no-console": ["error", { allow: ["warn", "error"] }],
+      "@typescript-eslint/no-explicit-any": "off",
+      "@typescript-eslint/no-unused-vars": [
+        "error",
+        {
+          argsIgnorePattern: "^_",
+          varsIgnorePattern: "^_",
+        },
+      ],
+    },
+  },
+]);

package/package.json

@@ -0,0 +1,51 @@
+{
+  "name": "ff-automationv2",
+  "version": "1.0.0",
+  "private": false,
+  "type": "module",
+  "description": "This lib is used to automate the manual testcase",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "builds": "npm run lint && tsc",
+    "build": "tsc",
+    "prepare": "npm run build",
+    "lint": "eslint src --max-warnings=0",
+    "test": "npm run build && node dist/tests/test12.js",
+    "llm": "npm run build && node dist/tests/testllm.js",
+    "dev": "npm run build && node dist/index.js",
+    "start": "node dist/index.js",
+    "file": "node dist/domAnalysis/getXpath.js"
+  },
+  "keywords": [],
+  "author": "Diwahar R",
+  "license": "ISC",
+  "devDependencies": {
+    "@eslint/js": "10.0.1",
+    "@types/jsdom": "27.0.0",
+    "@types/node": "25.2.2",
+    "eslint": "10.0.0",
+    "globals": "17.3.0",
+    "jiti": "2.6.1",
+    "nodemon": "3.1.11",
+    "ts-node": "10.9.2",
+    "typescript": "5.9.3",
+    "typescript-eslint": "8.56.0"
+  },
+  "dependencies": {
+    "axios": "1.13.5",
+    "cheerio": "1.2.0",
+    "fs-extra": "11.3.3",
+    "fuzzball": "2.2.3",
+    "jsdom": "27.4.0",
+    "openai": "6.18.0",
+    "uuid": "13.0.0",
+    "webdriverio": "9.23.3"
+  }
+}

package/src/ai/llmcalls/decodeApiKey.ts

@@ -0,0 +1,14 @@
+
+export function decodeApiKey(apiKey: string): string {
+  if (!apiKey) {
+    throw new Error("API key is required");
+  }
+
+  try {
+    return Buffer.from(apiKey, "base64").toString("utf-8");
+  } catch (error) {
+   throw new Error("Failed to decode API key", { cause: error });
+  }
+}
+
+

package/src/ai/llmcalls/llmAction.ts

@@ -0,0 +1,89 @@
+import { prompts } from "../llmprompts/promptRegistry.js";
+import OpenAI from "openai";
+import { decodeApiKey } from "./decodeApiKey.js";
+import { FireFlinkLLMResponse } from "../../core/types/llmResponseType.js"
+import { PromptArgsMap } from "../../core/interfaces/promptInterface.js"
+import { userInputFormatters } from "../../ai/llmprompts/userPrompts/userPrompt.js"
+import { visionPromptMessage } from "../../core/types/visionllmInputType.js"
+import { PromptType } from "../../core/types/promptType.js";
+export class llmAction {
+    private client: OpenAI;
+    private model: string;
+    private visionClient: OpenAI
+    constructor(
+        apiKey: string,
+        baseURL: string,
+        model: string,
+        visionApiKey: string
+    ) {
+        this.model = model;
+        this.client = new OpenAI({
+            apiKey: decodeApiKey(apiKey),
+            baseURL: baseURL,
+        });
+        this.visionClient = new OpenAI({
+            apiKey: decodeApiKey(visionApiKey),
+        });
+
+    }
+
+    private async getLLMResponseText(system: string, user: string): Promise<FireFlinkLLMResponse> {
+        const response = await this.client.chat.completions.create({
+            model: this.model,
+            messages: [
+                {
+                    role: "system",
+                    content: system
+                },
+                {
+                    role: "user",
+                    content: user
+                }
+            ],
+            response_format: { type: "json_object" }
+        });
+        try {
+
+            return response;
+        } catch {
+            throw new Error(`Invalid JSON returned from LLM `);
+        }
+    }
+    async getLLMResponse<K extends keyof PromptArgsMap>(
+        type: K,
+        args: PromptArgsMap[K],
+        userInput: Record<string, any>
+    ): Promise<FireFlinkLLMResponse> {
+        const promptBuilder: (args: PromptArgsMap[K]) => Promise<string> =
+            prompts[type];
+        const systemPrompt = await promptBuilder(args);
+        const userPrompt: string | visionPromptMessage = userInputFormatters[type](userInput);
+        if (type === PromptType.VISION_PROMPT && Array.isArray(userPrompt)) {
+            return this.getLLMResponseWithVision(systemPrompt, userPrompt);
+        }
+        if (typeof userPrompt !== "string") {
+            throw new Error("Invalid non-vision user prompt format");
+        }
+        return this.getLLMResponseText(systemPrompt, userPrompt);
+    }
+
+    private async getLLMResponseWithVision(system: any, userPrompt: visionPromptMessage): Promise<string> {
+        const response = await this.visionClient.chat.completions.create({
+            model: "gpt-4.1",
+            messages: [
+                {
+                    role: "system",
+                    content: system
+                },
+                {
+                    role: "user",
+                    content: userPrompt
+                }
+            ],
+            response_format: { type: "json_object" }
+        });
+        return response.choices[0].message.content || "";
+    }
+
+
+}

package/src/ai/llmcalls/parseLlmOputput.ts

@@ -0,0 +1,69 @@
+import { FireFlinkLLMResponse } from "../../core/types/llmResponseType.js";
+
+
+export class LLMResultParser {
+    private inputTokens: number = 0;
+    private outputTokens: number = 0;
+    private totalTokens: number = 0;
+
+    private async extractJsonObject(response: string) {
+        const match = response.match(/\{[\s\S]*\}/);
+        if (!match) return null;
+        return JSON.parse(match[0]);
+    }
+    public async fetchResult(llmOutput: any): Promise<{
+        response: FireFlinkLLMResponse
+    }> {
+        try {
+            const content = llmOutput?.choices?.[0]?.message?.content;
+
+            if (!content) {
+                throw new Error("No content in LLM output");
+            }
+
+            let parsedContent: FireFlinkLLMResponse;
+
+            try {
+                // Try direct parse first
+                parsedContent = JSON.parse(content);
+            } catch {
+                // If LLM added explanation text, extract JSON block
+                const extracted = this.extractJsonObject(content);
+
+                if (!extracted) {
+                    throw new Error("Could not extract valid JSON from LLM response");
+                }
+
+                parsedContent = extracted as FireFlinkLLMResponse;
+            }
+
+            const usage = llmOutput?.usage ?? {};
+
+            const inputTokens = Number(usage.prompt_tokens ?? 0);
+            const outputTokens = Number(usage.completion_tokens ?? 0);
+            const totalTokens = Number(usage.total_tokens ?? 0);
+
+            this.inputTokens += inputTokens;
+            this.outputTokens += outputTokens;
+            this.totalTokens += totalTokens;
+
+            return { response: parsedContent };
+
+        } catch (error) {
+            throw new Error(
+                "Failed to parse LLM output",
+                { cause: error }
+            );
+        }
+
+    }
+
+
+    public getTokenUsage() {
+        return {
+            inputTokens: this.inputTokens,
+            outputTokens: this.outputTokens,
+            totalTokens: this.totalTokens,
+        };
+    }
+}

package/src/ai/llmprompts/promptRegistry.ts

@@ -0,0 +1,16 @@
+import { keywordExtractor } from "./systemPrompts/actionExtractorPrompt.js";
+import { buildStepExtractionPrompt } from "./systemPrompts/userStoryToListPrompt.js";
+import { buildErrorDescriptionPrompt } from "./systemPrompts/errorDescriptionPrompt.js";
+import { ffInspectorNumExtractor } from "./systemPrompts/fireflinkElementIndexExtactors.js";
+import { visionPrompt } from "./systemPrompts/visionPrompt.js";
+import { PromptMap } from "../../core/types/promptMap.js";
+
+
+export const prompts: PromptMap = {
+    userStoryToList: buildStepExtractionPrompt,
+    keywordExtractorPrompt: keywordExtractor,
+    errorDescriptionPrompt: buildErrorDescriptionPrompt,
+    ffInspectorNumExtractor: ffInspectorNumExtractor,
+    visionPrompt: visionPrompt,
+};
+

package/src/ai/llmprompts/systemPrompts/actionExtractorPrompt.ts

@@ -0,0 +1,70 @@
+
+type keywordExtractor = {
+  priorAndNextSteps: string[];
+
+};
+export async function keywordExtractor(
+  { priorAndNextSteps }: keywordExtractor
+): Promise<string> {
+
+  const allowedActions: string[] = [
+    "enter",
+    "wait",
+    "verify",
+    "scroll",
+    "navigate",
+    "click",
+    "maximize",
+    "get",
+    "upload",
+    "close",
+    "open",
+    "drag_and_drop",
+    "switch"
+  ];
+
+  const prompt = `
+You are an expert in Web application testing.
+From the step: step, extract ONLY the meaningful keywords so that i can search for the element in the dom.
+
+Rules:
+- Only give response for the current step .
+- understand the step and context from the ${JSON.stringify(priorAndNextSteps)} and keywords should be from step.
+- 3 to 5 keywords maximum.
+- If the step is about entering text or Uploading file, Should NOT include input value from the step into keywords.
+- If the step has words like tag name audio, video, image, svg, checkbox etc, include them in the keywords.
+- If icon is mentioned in step then 'svg' should add in keywords and for Upload action first keyword should be 'file'.
+- First keyword should be from step. Next keywords must be distinct and based on element label meaning only.
+- If keyword is two words, then create an additional keyword by combining them.
+  Example: "Sign In" → "Sign In", "SignIn"
+- Keywords can be string or number.
+- Do NOT include generic UI words or action words.
+- Do NOT include status or technical words.
+- If element label contains multiple words, keep them together as ONE keyword.
+- element_name: extract name of the element from step.
+  Capitalize first letter.
+  If not mentioned, return action as element_name.
+- action rules:
+  click, enter, wait, scroll, navigate, get, maximize, close, open,
+  upload, drag_and_drop, switch
+- action must be one of ${JSON.stringify(allowedActions)}.
+  If not, return "0".
+- For navigate:
+  - keywords must contain ONLY full URL
+  - element_name must be "URL"
+  - navigate allowed ONLY if step intent is purely navigation
+- If verification intent exists, MUST return "verify"
+
+Respond only with JSON using this format:
+{
+  "keywords": [],
+  "elementName": "",
+  "elementType":""
+  "action": ""
+}
+
+No other text.
+`;
+
+  return prompt;
+}

package/src/ai/llmprompts/systemPrompts/errorDescriptionPrompt.ts

@@ -0,0 +1,23 @@
+export async function buildErrorDescriptionPrompt(): Promise<string> {
+  const description_prompt = `
+
+Analyze the error and execution history to provide diagnostics in JSON format.
+
+Return a JSON object with the following keys:
+1. "error": A concise, single-line, selenium type(error).__name__ (e.g., "InvalidSessionIdException", "InvalidElementStateException", "ElementNotInteractableException", "NoSuchElementException" etc).
+2. "errorDescription": A very simple, non-technical, and user-friendly explanation of why the automation might have failed.
+   - Use plain English.
+   - All the lines should contain FireFlink. example: "FireFlink was unable to find the element..."
+   - Do not mention code, XPaths, or internal technical terms.
+   - The output MUST be a single paragraph of exactly 10 to 15 lines of text.
+   - Focus on potential real-world causes.
+
+Respond only with JSON using this format:
+{
+  "error": ",
+  "errorDescription": "",
+}
+`;
+
+  return description_prompt;
+}