ff-automationv2 2.1.3 → 2.2.0

package/src/fireflinkData/fireflinkScript/scriptGenrationData.ts

@@ -1,30 +1,79 @@
 import { IScriptGenerationData } from "../../core/interfaces/scriptGenrationDataInterface.js";
-import { IPayload } from "../../core/interfaces/fireflinkScriptPayloadInterface.js";
-import { FireFlinkApiService } from "../../service/fireflinkApi.service.js";
-import { ScriptRunner } from "../../service/scriptRunner.service.js";
-
-const apiService = new FireFlinkApiService();
-const scriptRunner = new ScriptRunner(apiService);
 
 export class ScriptDataAppender {
+
     private data: IScriptGenerationData[] = [];
-    add(item: IScriptGenerationData): void {
+    private queue: (() => Promise<void>)[] = [];
+    private errorNLP: string = "";
+
+    private concurrencyLimit = 5;
+
+    add(task: () => Promise<IScriptGenerationData>) {
+
+        const job = async () => {
+            await this.retry(async () => {
+                const item = await task();
+                this.process(item);
+            }, 3);
+        };
+
+        this.queue.push(job);
+    }
+
+    private process(item: IScriptGenerationData) {
         this.data.push(item);
     }
-    getData(): IScriptGenerationData[] {
+
+    private async retry(fn: () => Promise<void>, retries: number) {
+
+        for (let i = 0; i <= retries; i++) {
+            try {
+                return await fn();
+            } catch (err: any) {
+
+                if (i === retries) {
+                    console.error("Step failed:", err);
+                    this.errorNLP = err?.message ?? "Unknown error";
+                    throw err;
+                }
+
+            }
+        }
+    }
+
+    async waitForAllSteps() {
+
+        const workers: Promise<void>[] = [];
+
+        while (this.queue.length > 0 || workers.length > 0) {
+
+            while (workers.length < this.concurrencyLimit && this.queue.length > 0) {
+
+                const job = this.queue.shift()!;
+                const worker = job().finally(() => {
+                    const index = workers.indexOf(worker);
+                    if (index > -1) workers.splice(index, 1);
+                });
+
+                workers.push(worker);
+            }
+
+            if (workers.length > 0) {
+                await Promise.race(workers);
+            }
+
+        }
+    }
+
+    getData() {
         return this.data;
     }
-    produceScriptGenerationData(
-        result: any,
-        payload: IPayload,
-        token: string,
-        serverHost: string
-    ): Promise<any> {
-        return scriptRunner.runScriptFromPayload(
-            result,
-            payload,
-            token,
-            serverHost
-        );
+
+    setErrorNLP(nlp: string) {
+        this.errorNLP = nlp;
+    }
+
+    getErrorNLP() {
+        return this.errorNLP;
     }
-}
+}

package/src/imageAnalysisMobile/annotatedScreenshotMobile.ts

@@ -0,0 +1,137 @@
+import { createCanvas, loadImage } from "canvas"
+import { XMLParser } from "fast-xml-parser"
+import { Bounds } from "./annotatedScreenshotMobileInterface.js"
+import { logger } from "../utils/logger/logData.js"
+
+function parseBounds(bounds: string): Bounds | null {
+
+    const nums = bounds.match(/\d+/g)
+
+    if (!nums || nums.length !== 4) return null
+
+    return {
+        x1: parseInt(nums[0]),
+        y1: parseInt(nums[1]),
+        x2: parseInt(nums[2]),
+        y2: parseInt(nums[3])
+    }
+}
+
+
+function extractBoxes(xmlSource: string): Bounds[] {
+
+    const parser = new XMLParser({
+        ignoreAttributes: false,
+        attributeNamePrefix: ""
+    })
+
+    const tree = parser.parse(xmlSource)
+
+    const boxes: Bounds[] = []
+
+    const traverse = (node: any) => {
+
+        if (node?.bounds) {
+
+            const clickable = node.clickable === "true"
+            const enabled = node.enabled === "true"
+            const clazz = node.class || ""
+
+            if (
+                (clickable && enabled) ||
+                clazz.toLowerCase().includes("edittext") ||
+                clazz.toLowerCase().includes("autocompletetextview")
+            ) {
+
+                const bounds = parseBounds(node.bounds)
+
+                if (bounds) boxes.push(bounds)
+            }
+        }
+
+        for (const key in node) {
+            if (typeof node[key] === "object") {
+                traverse(node[key])
+            }
+        }
+    }
+
+    traverse(tree)
+
+    return boxes
+}
+
+
+const COLORS: [number, number, number][] = [
+    [255, 0, 0],
+    [0, 128, 255],
+    [0, 180, 0],
+    [255, 140, 0],
+    [148, 0, 211],
+    [0, 180, 180],
+    [180, 0, 0],
+    [255, 105, 180],
+];
+
+async function drawBoxes(
+    screenshotBuffer: Buffer,
+    boxes: Bounds[]
+): Promise<Buffer> {
+
+    const img = await loadImage(screenshotBuffer)
+
+    const canvas = createCanvas(img.width, img.height)
+    const ctx = canvas.getContext("2d")
+
+    ctx.drawImage(img, 0, 0)
+
+    const scale = img.width / 1080
+    const fontSize = Math.max(20, Math.floor(28 * scale))
+
+    ctx.font = `${fontSize}px Arial`
+    ctx.textBaseline = "top"
+
+    boxes.forEach((box, index) => {
+
+        const color = COLORS[index % COLORS.length]
+        const colorStr = `rgb(${color[0]},${color[1]},${color[2]})`
+
+        ctx.strokeStyle = colorStr
+        ctx.lineWidth = Math.max(2, Math.floor(3 * scale))
+
+        ctx.strokeRect(
+            box.x1,
+            box.y1,
+            box.x2 - box.x1,
+            box.y2 - box.y1
+        )
+
+        ctx.fillStyle = colorStr
+        ctx.fillText(
+            String(index + 1),
+            box.x1 + 6,
+            box.y1 + 6
+        )
+    })
+
+    return canvas.toBuffer("image/png")
+}
+
+
+export async function annotateScreenshot(
+    driver: any,
+    xmlSource: string
+): Promise<Buffer> {
+
+    const screenshotBase64 = await driver.takeScreenshot()
+    const screenshotBuffer = Buffer.from(screenshotBase64, "base64")
+
+    const boxes = extractBoxes(xmlSource)
+
+    const boxedImageBuffer = await drawBoxes(screenshotBuffer, boxes)
+
+    // Uncomment only when debugging
+    await logger.saveImage(boxedImageBuffer, "boxed.png");
+
+    return boxedImageBuffer
+}

package/src/imageAnalysisMobile/annotatedScreenshotMobileInterface.ts

@@ -0,0 +1,6 @@
+export interface Bounds {
+  x1: number
+  y1: number
+  x2: number
+  y2: number
+}

package/src/imageAnalysisWeb/annotateScreenshot.ts

@@ -0,0 +1,196 @@
+import { createCanvas, loadImage, CanvasRenderingContext2D } from "canvas";
+import { Rect, NodeItem } from "./screenshotInterface.js";
+
+
+const COLORS: [number, number, number][] = [
+  [255, 0, 0],
+  [0, 128, 255],
+  [0, 180, 0],
+  [255, 140, 0],
+  [148, 0, 211],
+  [0, 180, 180],
+  [180, 0, 0],
+  [255, 105, 180],
+];
+
+const TAG_PADDING = 4;
+
+function measureText(
+  ctx: CanvasRenderingContext2D,
+  text: string
+) {
+  const metrics = ctx.measureText(text);
+  const height =
+    metrics.actualBoundingBoxAscent + metrics.actualBoundingBoxDescent;
+  return { width: metrics.width, height };
+}
+
+function rectsIntersect(a: Rect, b: Rect): boolean {
+  const [ax1, ay1, ax2, ay2] = a;
+  const [bx1, by1, bx2, by2] = b;
+  return !(ax2 < bx1 || ax1 > bx2 || ay2 < by1 || ay1 > by2);
+}
+
+function insideScreen(rect: Rect, width: number, height: number): boolean {
+  const [x1, y1, x2, y2] = rect;
+  return x1 >= 0 && y1 >= 0 && x2 <= width && y2 <= height;
+}
+
+function findFreeTagPosition(
+  box: Rect,
+  tagW: number,
+  tagH: number,
+  elementBoxes: Rect[],
+  usedTagRects: Rect[],
+  imgWidth: number,
+  imgHeight: number
+) {
+  const [bx1, by1, bx2, by2] = box;
+  const cx = Math.floor((bx1 + bx2) / 2);
+  const cy = Math.floor((by1 + by2) / 2);
+
+  const maxRadius = 120;
+  const step = 6;
+
+  for (let radius = 6; radius < maxRadius; radius += step) {
+    for (let angle = 0; angle < 360; angle += 30) {
+      const rad = (angle * Math.PI) / 180;
+
+      const lx = Math.floor(cx + radius * Math.cos(rad) - tagW / 2);
+      const ly = Math.floor(cy + radius * Math.sin(rad) - tagH / 2);
+
+      const rect: Rect = [
+        lx - TAG_PADDING,
+        ly - TAG_PADDING,
+        lx + tagW + TAG_PADDING,
+        ly + tagH + TAG_PADDING,
+      ];
+
+      if (!insideScreen(rect, imgWidth, imgHeight)) continue;
+      if (elementBoxes.some((b) => rectsIntersect(rect, b))) continue;
+      if (usedTagRects.some((t) => rectsIntersect(rect, t))) continue;
+
+      usedTagRects.push(rect);
+      return { x: lx, y: ly };
+    }
+  }
+
+  const fallbackX = Math.max(0, bx1 - tagW - 10);
+  const fallbackY = Math.max(0, by1 - tagH - 10);
+  usedTagRects.push([fallbackX, fallbackY, fallbackX + tagW, fallbackY + tagH]);
+
+  return { x: fallbackX, y: fallbackY };
+}
+
+export async function annotateScreenshot(
+  screenshotBase64: string,
+  nodes: NodeItem[],
+): Promise<Buffer> {
+
+  const buffer = Buffer.from(screenshotBase64, "base64");
+  const image = await loadImage(buffer);
+  const scale = image.width / 1920;
+  const fontSize = Math.max(10, Math.floor(14 * scale));
+
+  const canvas = createCanvas(image.width, image.height);
+  const ctx = canvas.getContext("2d");
+
+  ctx.drawImage(image, 0, 0);
+  ctx.font = `${fontSize}px Tahoma`;
+  ctx.textBaseline = "middle";
+  ctx.textAlign = "center";
+
+  const usedTagRects: Rect[] = [];
+
+  const elementBoxes: Rect[] = nodes.map((n) => [
+    Math.floor(n.bbox.x),
+    Math.floor(n.bbox.y),
+    Math.floor(n.bbox.x + n.bbox.w),
+    Math.floor(n.bbox.y + n.bbox.h),
+  ]);
+
+  nodes.forEach((item, i) => {
+
+    const color = COLORS[i % COLORS.length];
+    const colorStr = `rgb(${color[0]},${color[1]},${color[2]})`;
+
+    const label = item.ff.split("-").pop() || "";
+
+    const x1 = Math.floor(item.bbox.x);
+    const y1 = Math.floor(item.bbox.y);
+    const x2 = Math.floor(item.bbox.x + item.bbox.w);
+    const y2 = Math.floor(item.bbox.y + item.bbox.h);
+
+    ctx.strokeStyle = colorStr;
+    ctx.lineWidth = Math.max(2, Math.floor(2 * scale));
+    ctx.strokeRect(x1, y1, x2 - x1, y2 - y1);
+
+    const { width: tw, height: th } = measureText(ctx, label);
+
+    const tagW = Math.max(22, tw + 10);
+    const tagH = Math.max(18, th + 8);
+
+    const { x: lx, y: ly } = findFreeTagPosition(
+      [x1, y1, x2, y2],
+      tagW,
+      tagH,
+      elementBoxes,
+      usedTagRects,
+      canvas.width,
+      canvas.height
+    );
+
+    ctx.fillStyle = "white";
+    ctx.strokeStyle = colorStr;
+    ctx.lineWidth = 2;
+    ctx.fillRect(lx, ly, tagW, tagH);
+    ctx.strokeRect(lx, ly, tagW, tagH);
+
+    ctx.fillStyle = colorStr;
+    ctx.fillText(label, lx + tagW / 2, ly + tagH / 2);
+
+    const tagCx = lx + tagW / 2;
+    const tagCy = ly + tagH / 2;
+    const boxCx = (x1 + x2) / 2;
+    const boxCy = (y1 + y2) / 2;
+
+    const offset = 2;
+
+    let lineStartX: number;
+    let lineStartY: number;
+    let lineEndX: number;
+    let lineEndY: number;
+
+    if (tagCx < x1) {
+      lineStartX = lx + tagW;
+      lineStartY = tagCy;
+      lineEndX = x1 - offset;
+      lineEndY = boxCy;
+    } else if (tagCx > x2) {
+      lineStartX = lx;
+      lineStartY = tagCy;
+      lineEndX = x2 + offset;
+      lineEndY = boxCy;
+    } else if (tagCy < y1) {
+      lineStartX = tagCx;
+      lineStartY = ly + tagH;
+      lineEndX = boxCx;
+      lineEndY = y1 - offset;
+    } else {
+      lineStartX = tagCx;
+      lineStartY = ly;
+      lineEndX = boxCx;
+      lineEndY = y2 + offset;
+    }
+
+    ctx.beginPath();
+    ctx.moveTo(lineStartX, lineStartY);
+    ctx.lineTo(lineEndX, lineEndY);
+    ctx.stroke();
+  });
+
+
+  const boxedImageBuffer = canvas.toBuffer("image/png");
+
+  return boxedImageBuffer;
+}

package/src/imageAnalysisWeb/generateAnnotatedScreenshots.ts

@@ -0,0 +1,43 @@
+import { annotateScreenshot } from "./annotateScreenshot.js";
+import { JS_PRIMARY } from "../utils/javascript/jsForVision.js";
+import { NodeItem } from "./screenshotInterface.js";
+// import { logger } from "../utils/logger/logData.js";
+
+export async function generateAnnotatedScreenshots(
+  driver: WebdriverIO.Browser
+): Promise<Buffer> {
+
+  const screenshot = await driver.takeScreenshot();
+
+  // const [primaryRaw, secondaryRaw] = await Promise.all([
+  //   driver.execute(JS_PRIMARY),
+  //   driver.execute(JS_SECONDARY),
+  // ]);
+  const [primaryRaw] = await Promise.all([
+    driver.execute(JS_PRIMARY),
+    // driver.execute(JS_SECONDARY),
+  ]);
+
+  const primary = (primaryRaw as NodeItem[]) ?? [];
+  // let secondary = (secondaryRaw as NodeItem[]) ?? [];
+
+  // const primaryIds = new Set(primary.map((item) => item.ff));
+  // secondary = secondary.filter((item) => !primaryIds.has(item.ff));
+
+  const [primaryAnnotated] = await Promise.all([
+    annotateScreenshot(
+      screenshot,
+      primary,
+    ),
+    // annotateScreenshot(
+    //   screenshot,
+    //   secondary,
+    // )
+  ]);
+
+  // Uncomment only when debugging
+  // await logger.saveImage(primaryAnnotated, "annotated_primary.png");
+
+
+  return primaryAnnotated;
+}

package/src/imageAnalysisWeb/screenshotInterface.ts

@@ -0,0 +1,18 @@
+
+export interface BBox {
+    x: number;
+    y: number;
+    w: number;
+    h: number;
+}
+
+export interface NodeItem {
+    ff: string;
+    xpath: string;
+    bbox: BBox;
+    tag: string;
+    text: string;
+}
+
+export type Rect = [number, number, number, number];
+

package/src/index.ts

@@ -1,5 +1,5 @@
-
 import { AutomationRunner } from "./core/main/runAutomationScript.js";
-
+import dotenv from "dotenv";
+dotenv.config();
 export { AutomationRunner };
 

package/src/service/{fireflinkApi.service.ts → api/fireflinkApi.service.ts}

@@ -1,16 +1,14 @@
 import axios, { AxiosError } from "axios";
-import { IFireFlinkApiService } from "../core/interfaces/fireflinkScriptPayloadInterface.js";
+import { IFireFlinkApiService } from "../../core/interfaces/fireflinkScriptPayloadInterface.js";
 
 export class FireFlinkApiService implements IFireFlinkApiService {
-
-    async callFireFlinkApi(
+    async getInstancesDetailsApi(
         headers: Record<string, string>,
-        body: any,
         url: string
     ): Promise<any> {
 
         try {
-            const response = await axios.post(url, body, {
+            const response = await axios.get(url, {
                 headers,
                 timeout: 30000,
             });
@@ -23,7 +21,7 @@ export class FireFlinkApiService implements IFireFlinkApiService {
 
             if (err.code === "ECONNABORTED") {
                 throw new Error(
-                    "Request to FireFlink API timed out",
+                    "Request to FireFlink API timed out for getting the instances details",
                     { cause: error }
                 );
             }
@@ -38,7 +36,7 @@ export class FireFlinkApiService implements IFireFlinkApiService {
             }
 
             throw new Error(
-                "Unexpected error while calling FireFlink API",
+                "Unexpected error while calling FireFlink API for getting the instances details",
                 { cause: error }
             );
         }

package/src/service/kafka/fireflinkKafka.service.ts

@@ -0,0 +1,25 @@
+import { Producer } from "kafkajs";
+import { IKafkaPayload } from "../../core/interfaces/fireflinkScriptPayloadInterface.js";
+import { logger } from "../../utils/logger/logData.js";
+
+export class KafkaProducerService {
+
+    constructor(private kafkaProducer: Producer) { }
+
+    async produceMessage(kafkaArgs: IKafkaPayload) {
+        try {
+            await this.kafkaProducer.connect();
+            logger.info("Kafka connected successfully");
+
+            await this.kafkaProducer.send({
+                topic: kafkaArgs.topic,
+                messages: [{ value: JSON.stringify(kafkaArgs.message) }],
+            });
+            await this.kafkaProducer.disconnect()
+            logger.info("Kafka produced message successfully and Kafka produced disconnected successfully");
+        } catch (error) {
+            await this.kafkaProducer.disconnect();
+            logger.error("An error occurred and the Kafka producer disconnected without sending the message", error);
+        }
+    }
+}

package/src/service/scriptRunner.service.ts

@@ -1,65 +1,39 @@
 import {
-    IPayload,
     IScriptRunner,
-    IStep,
     IFireFlinkApiService,
-} from "../core/interfaces/fireflinkScriptPayloadInterface";
+    IKafkaPayload
+} from "../core/interfaces/fireflinkScriptPayloadInterface.js";
+
+import { KafkaProducerService } from "../service/kafka/fireflinkKafka.service.js"
+import { Producer } from "kafkajs";
 
 import { logger } from "../utils/logger/logData.js"
 
 export class ScriptRunner implements IScriptRunner {
-    constructor(private apiService: IFireFlinkApiService) { }
+    private kafkaProducer: KafkaProducerService;
+    constructor(private apiService: IFireFlinkApiService,
+        private kafkaProducerInstance: Producer,
+    ) {
+        this.kafkaProducer = new KafkaProducerService(this.kafkaProducerInstance)
 
-    async runScriptFromPayload(
-        result: any[] | [],
-        payload: IPayload,
+    }
+    async runGetInstancesDetails(
+        projectId: string,
         token: string,
         serverHost: string
     ): Promise<any> {
-        let stepData: IStep[];
-
-        try {
-            stepData =
-                typeof result === "string" ? JSON.parse(result) : result;
-
-        } catch (error) {
-            logger.error(`Failed to parse final llm data: ${error}`);
-            return;
-        }
-        logger.info("Sending script payload to FireFlink");
-
-        const body = {
-            stepData: stepData,
-            scriptName: payload.scriptName,
-            scriptType: payload.scriptType,
-            projectId: payload.projectId,
-            testCaseId: payload.testCaseId,
-            promptId: payload.promptId,
-            pageDetails: payload.pageDetails,
-            generatedBy: payload.generatedBy,
-            webSocketId: payload.webSocketId,
-            licenseType: payload.licenseType,
-            licenseId: payload.licenseId,
-            userId: payload.userId,
-            topic: payload.topic,
-            projectType: payload.projectType,
-            tokensConsumed: payload.tokensConsumed,
-        };
-        logger.info("PayLoad for Fireflink : ", body)
         const headers = {
             Accept: "*/*",
             "Accept-Language": "en-US,en;q=0.9",
             "Content-Type": "application/json",
-            projectId: payload.projectId || "",
+            projectId: projectId || "",
             Authorization: `Bearer ${token}`,
         };
 
-        const url = `${serverHost}/project/optimize/v3/bulk/scripts/script-payload`;
-
+        const url = `${serverHost}/project/optimize/v3/ai-service-provider/selected-instance?projectId=${projectId}`;
         try {
-            const finalResult = await this.apiService.callFireFlinkApi(
+            const finalResult = await this.apiService.getInstancesDetailsApi(
                 headers,
-                JSON.stringify(body),
                 url
             );
             if (finalResult) {
@@ -79,5 +53,12 @@ export class ScriptRunner implements IScriptRunner {
                 "Failed to submit automation script", { cause: error }
             );
         }
+
+    }
+    async produceMessageToKafka(kafkaArgs: IKafkaPayload) {
+        this.kafkaProducer.produceMessage(kafkaArgs)
+
     }
 }
+
+