ff-automationv2 2.2.0 → 2.2.2

package/src/fireflinkData/fireflinkLocators/typeList.ts

@@ -1,36 +0,0 @@
-const elementType: { [key: string]: string } = {
-    'Link': 'link',
-    'Text Field': 'textfield',
-    'Icon': 'icon',
-    'Button': 'button',
-    'Radio Button': 'radioButton',
-    'Text': 'text',
-    'Textarea': 'textarea',
-    'Image': 'image',
-    'Dropdown': 'dropdown',
-    'Checkbox': 'checkbox',
-    'Tab': 'tab',
-    'Action Overflow button': 'action overflow button',
-    'Hamburger menu': 'hamburger icon',
-    'Toggle Button': 'toggle button',
-    'Suggestion': 'suggestion',
-    'Time Picker': 'time picker',
-    'Date Picker': 'date picker',
-    'Toaster Message': 'toaster message',
-    'Card': 'card',
-    'Tooltip': 'tooltip',
-    'Option': 'option',
-    'Calender': 'calender',
-    'Sliders': 'sliders',
-    'Visual Testing': 'visual testing'
-} as const;
-
-export const normalizedElementType: Record<string, string> =
-    Object.fromEntries(
-        Object.entries(elementType).map(([key, value]) => [
-            key.toLowerCase().replace(/\s+/g, ""),
-            value,
-        ])
-    );
-
-

package/src/fireflinkData/fireflinkScript/scriptGenrationData.ts

@@ -1,79 +0,0 @@
-import { IScriptGenerationData } from "../../core/interfaces/scriptGenrationDataInterface.js";
-
-export class ScriptDataAppender {
-
-    private data: IScriptGenerationData[] = [];
-    private queue: (() => Promise<void>)[] = [];
-    private errorNLP: string = "";
-
-    private concurrencyLimit = 5;
-
-    add(task: () => Promise<IScriptGenerationData>) {
-
-        const job = async () => {
-            await this.retry(async () => {
-                const item = await task();
-                this.process(item);
-            }, 3);
-        };
-
-        this.queue.push(job);
-    }
-
-    private process(item: IScriptGenerationData) {
-        this.data.push(item);
-    }
-
-    private async retry(fn: () => Promise<void>, retries: number) {
-
-        for (let i = 0; i <= retries; i++) {
-            try {
-                return await fn();
-            } catch (err: any) {
-
-                if (i === retries) {
-                    console.error("Step failed:", err);
-                    this.errorNLP = err?.message ?? "Unknown error";
-                    throw err;
-                }
-
-            }
-        }
-    }
-
-    async waitForAllSteps() {
-
-        const workers: Promise<void>[] = [];
-
-        while (this.queue.length > 0 || workers.length > 0) {
-
-            while (workers.length < this.concurrencyLimit && this.queue.length > 0) {
-
-                const job = this.queue.shift()!;
-                const worker = job().finally(() => {
-                    const index = workers.indexOf(worker);
-                    if (index > -1) workers.splice(index, 1);
-                });
-
-                workers.push(worker);
-            }
-
-            if (workers.length > 0) {
-                await Promise.race(workers);
-            }
-
-        }
-    }
-
-    getData() {
-        return this.data;
-    }
-
-    setErrorNLP(nlp: string) {
-        this.errorNLP = nlp;
-    }
-
-    getErrorNLP() {
-        return this.errorNLP;
-    }
-}

package/src/imageAnalysisMobile/annotatedScreenshotMobile.ts

@@ -1,137 +0,0 @@
-import { createCanvas, loadImage } from "canvas"
-import { XMLParser } from "fast-xml-parser"
-import { Bounds } from "./annotatedScreenshotMobileInterface.js"
-import { logger } from "../utils/logger/logData.js"
-
-function parseBounds(bounds: string): Bounds | null {
-
-    const nums = bounds.match(/\d+/g)
-
-    if (!nums || nums.length !== 4) return null
-
-    return {
-        x1: parseInt(nums[0]),
-        y1: parseInt(nums[1]),
-        x2: parseInt(nums[2]),
-        y2: parseInt(nums[3])
-    }
-}
-
-
-function extractBoxes(xmlSource: string): Bounds[] {
-
-    const parser = new XMLParser({
-        ignoreAttributes: false,
-        attributeNamePrefix: ""
-    })
-
-    const tree = parser.parse(xmlSource)
-
-    const boxes: Bounds[] = []
-
-    const traverse = (node: any) => {
-
-        if (node?.bounds) {
-
-            const clickable = node.clickable === "true"
-            const enabled = node.enabled === "true"
-            const clazz = node.class || ""
-
-            if (
-                (clickable && enabled) ||
-                clazz.toLowerCase().includes("edittext") ||
-                clazz.toLowerCase().includes("autocompletetextview")
-            ) {
-
-                const bounds = parseBounds(node.bounds)
-
-                if (bounds) boxes.push(bounds)
-            }
-        }
-
-        for (const key in node) {
-            if (typeof node[key] === "object") {
-                traverse(node[key])
-            }
-        }
-    }
-
-    traverse(tree)
-
-    return boxes
-}
-
-
-const COLORS: [number, number, number][] = [
-    [255, 0, 0],
-    [0, 128, 255],
-    [0, 180, 0],
-    [255, 140, 0],
-    [148, 0, 211],
-    [0, 180, 180],
-    [180, 0, 0],
-    [255, 105, 180],
-];
-
-async function drawBoxes(
-    screenshotBuffer: Buffer,
-    boxes: Bounds[]
-): Promise<Buffer> {
-
-    const img = await loadImage(screenshotBuffer)
-
-    const canvas = createCanvas(img.width, img.height)
-    const ctx = canvas.getContext("2d")
-
-    ctx.drawImage(img, 0, 0)
-
-    const scale = img.width / 1080
-    const fontSize = Math.max(20, Math.floor(28 * scale))
-
-    ctx.font = `${fontSize}px Arial`
-    ctx.textBaseline = "top"
-
-    boxes.forEach((box, index) => {
-
-        const color = COLORS[index % COLORS.length]
-        const colorStr = `rgb(${color[0]},${color[1]},${color[2]})`
-
-        ctx.strokeStyle = colorStr
-        ctx.lineWidth = Math.max(2, Math.floor(3 * scale))
-
-        ctx.strokeRect(
-            box.x1,
-            box.y1,
-            box.x2 - box.x1,
-            box.y2 - box.y1
-        )
-
-        ctx.fillStyle = colorStr
-        ctx.fillText(
-            String(index + 1),
-            box.x1 + 6,
-            box.y1 + 6
-        )
-    })
-
-    return canvas.toBuffer("image/png")
-}
-
-
-export async function annotateScreenshot(
-    driver: any,
-    xmlSource: string
-): Promise<Buffer> {
-
-    const screenshotBase64 = await driver.takeScreenshot()
-    const screenshotBuffer = Buffer.from(screenshotBase64, "base64")
-
-    const boxes = extractBoxes(xmlSource)
-
-    const boxedImageBuffer = await drawBoxes(screenshotBuffer, boxes)
-
-    // Uncomment only when debugging
-    await logger.saveImage(boxedImageBuffer, "boxed.png");
-
-    return boxedImageBuffer
-}

package/src/imageAnalysisMobile/annotatedScreenshotMobileInterface.ts

@@ -1,6 +0,0 @@
-export interface Bounds {
-  x1: number
-  y1: number
-  x2: number
-  y2: number
-}

package/src/imageAnalysisWeb/annotateScreenshot.ts

@@ -1,196 +0,0 @@
-import { createCanvas, loadImage, CanvasRenderingContext2D } from "canvas";
-import { Rect, NodeItem } from "./screenshotInterface.js";
-
-
-const COLORS: [number, number, number][] = [
-  [255, 0, 0],
-  [0, 128, 255],
-  [0, 180, 0],
-  [255, 140, 0],
-  [148, 0, 211],
-  [0, 180, 180],
-  [180, 0, 0],
-  [255, 105, 180],
-];
-
-const TAG_PADDING = 4;
-
-function measureText(
-  ctx: CanvasRenderingContext2D,
-  text: string
-) {
-  const metrics = ctx.measureText(text);
-  const height =
-    metrics.actualBoundingBoxAscent + metrics.actualBoundingBoxDescent;
-  return { width: metrics.width, height };
-}
-
-function rectsIntersect(a: Rect, b: Rect): boolean {
-  const [ax1, ay1, ax2, ay2] = a;
-  const [bx1, by1, bx2, by2] = b;
-  return !(ax2 < bx1 || ax1 > bx2 || ay2 < by1 || ay1 > by2);
-}
-
-function insideScreen(rect: Rect, width: number, height: number): boolean {
-  const [x1, y1, x2, y2] = rect;
-  return x1 >= 0 && y1 >= 0 && x2 <= width && y2 <= height;
-}
-
-function findFreeTagPosition(
-  box: Rect,
-  tagW: number,
-  tagH: number,
-  elementBoxes: Rect[],
-  usedTagRects: Rect[],
-  imgWidth: number,
-  imgHeight: number
-) {
-  const [bx1, by1, bx2, by2] = box;
-  const cx = Math.floor((bx1 + bx2) / 2);
-  const cy = Math.floor((by1 + by2) / 2);
-
-  const maxRadius = 120;
-  const step = 6;
-
-  for (let radius = 6; radius < maxRadius; radius += step) {
-    for (let angle = 0; angle < 360; angle += 30) {
-      const rad = (angle * Math.PI) / 180;
-
-      const lx = Math.floor(cx + radius * Math.cos(rad) - tagW / 2);
-      const ly = Math.floor(cy + radius * Math.sin(rad) - tagH / 2);
-
-      const rect: Rect = [
-        lx - TAG_PADDING,
-        ly - TAG_PADDING,
-        lx + tagW + TAG_PADDING,
-        ly + tagH + TAG_PADDING,
-      ];
-
-      if (!insideScreen(rect, imgWidth, imgHeight)) continue;
-      if (elementBoxes.some((b) => rectsIntersect(rect, b))) continue;
-      if (usedTagRects.some((t) => rectsIntersect(rect, t))) continue;
-
-      usedTagRects.push(rect);
-      return { x: lx, y: ly };
-    }
-  }
-
-  const fallbackX = Math.max(0, bx1 - tagW - 10);
-  const fallbackY = Math.max(0, by1 - tagH - 10);
-  usedTagRects.push([fallbackX, fallbackY, fallbackX + tagW, fallbackY + tagH]);
-
-  return { x: fallbackX, y: fallbackY };
-}
-
-export async function annotateScreenshot(
-  screenshotBase64: string,
-  nodes: NodeItem[],
-): Promise<Buffer> {
-
-  const buffer = Buffer.from(screenshotBase64, "base64");
-  const image = await loadImage(buffer);
-  const scale = image.width / 1920;
-  const fontSize = Math.max(10, Math.floor(14 * scale));
-
-  const canvas = createCanvas(image.width, image.height);
-  const ctx = canvas.getContext("2d");
-
-  ctx.drawImage(image, 0, 0);
-  ctx.font = `${fontSize}px Tahoma`;
-  ctx.textBaseline = "middle";
-  ctx.textAlign = "center";
-
-  const usedTagRects: Rect[] = [];
-
-  const elementBoxes: Rect[] = nodes.map((n) => [
-    Math.floor(n.bbox.x),
-    Math.floor(n.bbox.y),
-    Math.floor(n.bbox.x + n.bbox.w),
-    Math.floor(n.bbox.y + n.bbox.h),
-  ]);
-
-  nodes.forEach((item, i) => {
-
-    const color = COLORS[i % COLORS.length];
-    const colorStr = `rgb(${color[0]},${color[1]},${color[2]})`;
-
-    const label = item.ff.split("-").pop() || "";
-
-    const x1 = Math.floor(item.bbox.x);
-    const y1 = Math.floor(item.bbox.y);
-    const x2 = Math.floor(item.bbox.x + item.bbox.w);
-    const y2 = Math.floor(item.bbox.y + item.bbox.h);
-
-    ctx.strokeStyle = colorStr;
-    ctx.lineWidth = Math.max(2, Math.floor(2 * scale));
-    ctx.strokeRect(x1, y1, x2 - x1, y2 - y1);
-
-    const { width: tw, height: th } = measureText(ctx, label);
-
-    const tagW = Math.max(22, tw + 10);
-    const tagH = Math.max(18, th + 8);
-
-    const { x: lx, y: ly } = findFreeTagPosition(
-      [x1, y1, x2, y2],
-      tagW,
-      tagH,
-      elementBoxes,
-      usedTagRects,
-      canvas.width,
-      canvas.height
-    );
-
-    ctx.fillStyle = "white";
-    ctx.strokeStyle = colorStr;
-    ctx.lineWidth = 2;
-    ctx.fillRect(lx, ly, tagW, tagH);
-    ctx.strokeRect(lx, ly, tagW, tagH);
-
-    ctx.fillStyle = colorStr;
-    ctx.fillText(label, lx + tagW / 2, ly + tagH / 2);
-
-    const tagCx = lx + tagW / 2;
-    const tagCy = ly + tagH / 2;
-    const boxCx = (x1 + x2) / 2;
-    const boxCy = (y1 + y2) / 2;
-
-    const offset = 2;
-
-    let lineStartX: number;
-    let lineStartY: number;
-    let lineEndX: number;
-    let lineEndY: number;
-
-    if (tagCx < x1) {
-      lineStartX = lx + tagW;
-      lineStartY = tagCy;
-      lineEndX = x1 - offset;
-      lineEndY = boxCy;
-    } else if (tagCx > x2) {
-      lineStartX = lx;
-      lineStartY = tagCy;
-      lineEndX = x2 + offset;
-      lineEndY = boxCy;
-    } else if (tagCy < y1) {
-      lineStartX = tagCx;
-      lineStartY = ly + tagH;
-      lineEndX = boxCx;
-      lineEndY = y1 - offset;
-    } else {
-      lineStartX = tagCx;
-      lineStartY = ly;
-      lineEndX = boxCx;
-      lineEndY = y2 + offset;
-    }
-
-    ctx.beginPath();
-    ctx.moveTo(lineStartX, lineStartY);
-    ctx.lineTo(lineEndX, lineEndY);
-    ctx.stroke();
-  });
-
-
-  const boxedImageBuffer = canvas.toBuffer("image/png");
-
-  return boxedImageBuffer;
-}

package/src/imageAnalysisWeb/generateAnnotatedScreenshots.ts

@@ -1,43 +0,0 @@
-import { annotateScreenshot } from "./annotateScreenshot.js";
-import { JS_PRIMARY } from "../utils/javascript/jsForVision.js";
-import { NodeItem } from "./screenshotInterface.js";
-// import { logger } from "../utils/logger/logData.js";
-
-export async function generateAnnotatedScreenshots(
-  driver: WebdriverIO.Browser
-): Promise<Buffer> {
-
-  const screenshot = await driver.takeScreenshot();
-
-  // const [primaryRaw, secondaryRaw] = await Promise.all([
-  //   driver.execute(JS_PRIMARY),
-  //   driver.execute(JS_SECONDARY),
-  // ]);
-  const [primaryRaw] = await Promise.all([
-    driver.execute(JS_PRIMARY),
-    // driver.execute(JS_SECONDARY),
-  ]);
-
-  const primary = (primaryRaw as NodeItem[]) ?? [];
-  // let secondary = (secondaryRaw as NodeItem[]) ?? [];
-
-  // const primaryIds = new Set(primary.map((item) => item.ff));
-  // secondary = secondary.filter((item) => !primaryIds.has(item.ff));
-
-  const [primaryAnnotated] = await Promise.all([
-    annotateScreenshot(
-      screenshot,
-      primary,
-    ),
-    // annotateScreenshot(
-    //   screenshot,
-    //   secondary,
-    // )
-  ]);
-
-  // Uncomment only when debugging
-  // await logger.saveImage(primaryAnnotated, "annotated_primary.png");
-
-
-  return primaryAnnotated;
-}

package/src/imageAnalysisWeb/screenshotInterface.ts

@@ -1,18 +0,0 @@
-
-export interface BBox {
-    x: number;
-    y: number;
-    w: number;
-    h: number;
-}
-
-export interface NodeItem {
-    ff: string;
-    xpath: string;
-    bbox: BBox;
-    tag: string;
-    text: string;
-}
-
-export type Rect = [number, number, number, number];
-

package/src/index.ts

@@ -1,5 +0,0 @@
-import { AutomationRunner } from "./core/main/runAutomationScript.js";
-import dotenv from "dotenv";
-dotenv.config();
-export { AutomationRunner };
-

package/src/llmConfig/llmConfiguration.ts

@@ -1,26 +0,0 @@
-
-import { IServiceProviderBaseUrlConfiguration } from "../core/interfaces/llmConfigurationInterface.js";
-export class ServiceProviderBaseUrlProvider
-    implements IServiceProviderBaseUrlConfiguration {
-
-    private readonly baseUrls: Record<string, string> = {
-        Groq: "https://api.groq.com/openai/v1",
-        DefaultFireFlink: "https://api.groq.com/openai/v1",
-        OpenAi: "https://api.openai.com/v1",
-        Anthropic: "https://api.anthropic.com/v1/",
-        Gemini: "https://generativelanguage.googleapis.com/v1beta/openai/"
-    };
-
-    public getBaseUrl(provider: string): string {
-        const url = this.baseUrls[provider];
-
-        if (!url) {
-            throw new Error(`Unsupported service provider: ${provider}`);
-        }
-
-        return url;
-    }
-}
-
-
-

package/src/service/api/fireflinkApi.service.ts

@@ -1,44 +0,0 @@
-import axios, { AxiosError } from "axios";
-import { IFireFlinkApiService } from "../../core/interfaces/fireflinkScriptPayloadInterface.js";
-
-export class FireFlinkApiService implements IFireFlinkApiService {
-    async getInstancesDetailsApi(
-        headers: Record<string, string>,
-        url: string
-    ): Promise<any> {
-
-        try {
-            const response = await axios.get(url, {
-                headers,
-                timeout: 30000,
-            });
-
-            return response.data;
-
-        } catch (error: any) {
-
-            const err = error as AxiosError;
-
-            if (err.code === "ECONNABORTED") {
-                throw new Error(
-                    "Request to FireFlink API timed out for getting the instances details",
-                    { cause: error }
-                );
-            }
-
-            if (err.response) {
-                throw new Error(
-                    `HTTP Error ${err.response.status}: ${JSON.stringify(
-                        err.response.data
-                    )}`,
-                    { cause: error }
-                );
-            }
-
-            throw new Error(
-                "Unexpected error while calling FireFlink API for getting the instances details",
-                { cause: error }
-            );
-        }
-    }
-}

package/src/service/kafka/fireflinkKafka.service.ts

@@ -1,25 +0,0 @@
-import { Producer } from "kafkajs";
-import { IKafkaPayload } from "../../core/interfaces/fireflinkScriptPayloadInterface.js";
-import { logger } from "../../utils/logger/logData.js";
-
-export class KafkaProducerService {
-
-    constructor(private kafkaProducer: Producer) { }
-
-    async produceMessage(kafkaArgs: IKafkaPayload) {
-        try {
-            await this.kafkaProducer.connect();
-            logger.info("Kafka connected successfully");
-
-            await this.kafkaProducer.send({
-                topic: kafkaArgs.topic,
-                messages: [{ value: JSON.stringify(kafkaArgs.message) }],
-            });
-            await this.kafkaProducer.disconnect()
-            logger.info("Kafka produced message successfully and Kafka produced disconnected successfully");
-        } catch (error) {
-            await this.kafkaProducer.disconnect();
-            logger.error("An error occurred and the Kafka producer disconnected without sending the message", error);
-        }
-    }
-}

package/src/service/scriptRunner.service.ts

@@ -1,64 +0,0 @@
-import {
-    IScriptRunner,
-    IFireFlinkApiService,
-    IKafkaPayload
-} from "../core/interfaces/fireflinkScriptPayloadInterface.js";
-
-import { KafkaProducerService } from "../service/kafka/fireflinkKafka.service.js"
-import { Producer } from "kafkajs";
-
-import { logger } from "../utils/logger/logData.js"
-
-export class ScriptRunner implements IScriptRunner {
-    private kafkaProducer: KafkaProducerService;
-    constructor(private apiService: IFireFlinkApiService,
-        private kafkaProducerInstance: Producer,
-    ) {
-        this.kafkaProducer = new KafkaProducerService(this.kafkaProducerInstance)
-
-    }
-    async runGetInstancesDetails(
-        projectId: string,
-        token: string,
-        serverHost: string
-    ): Promise<any> {
-        const headers = {
-            Accept: "*/*",
-            "Accept-Language": "en-US,en;q=0.9",
-            "Content-Type": "application/json",
-            projectId: projectId || "",
-            Authorization: `Bearer ${token}`,
-        };
-
-        const url = `${serverHost}/project/optimize/v3/ai-service-provider/selected-instance?projectId=${projectId}`;
-        try {
-            const finalResult = await this.apiService.getInstancesDetailsApi(
-                headers,
-                url
-            );
-            if (finalResult) {
-                logger.error(
-                    "Successfully submitted automation script to FireFlink API."
-                );
-            } else {
-                throw new Error(
-                    "FireFlink API responded with an error: " + JSON.stringify(finalResult));
-            }
-            return finalResult;
-        } catch (error: any) {
-            logger.error(
-                `Failed to submit automation script, ${error}`
-            );
-            throw new Error(
-                "Failed to submit automation script", { cause: error }
-            );
-        }
-
-    }
-    async produceMessageToKafka(kafkaArgs: IKafkaPayload) {
-        this.kafkaProducer.produceMessage(kafkaArgs)
-
-    }
-}
-
-