ff-automationv2 2.2.27 → 2.2.29

package/dist/fireflinkData/fireflinkScript/appendScriptData.d.ts

@@ -0,0 +1,2 @@
+import { AppendScriptDataArgs } from "../../core/interfaces/scriptGenrationDataInterface.js";
+export declare function appendScriptData(args: AppendScriptDataArgs): void;

package/dist/fireflinkData/fireflinkScript/appendScriptData.js

@@ -0,0 +1,51 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.appendScriptData = appendScriptData;
+const logData_js_1 = require("../../utils/logger/logData.js");
+function appendScriptData(args) {
+    args.scriptDataAppender.add(async () => {
+        const searchAttribute = args.platform === "web"
+            ? `[ff-inspect="${args.ffNumber}"]`
+            : `[FF-inspecter="${args.ffNumber}"]`;
+        const data = {
+            nlpName: args.nlpName
+        };
+        if (args.stepInputs) {
+            data.stepInputs = args.stepInputs;
+        }
+        else {
+            data.stepInputs = [];
+        }
+        if (args.elementName) {
+            let locators;
+            const locator = [{ name: "xpath", value: args.selector }];
+            try {
+                const ffElement = await args.elementGetter.getFireFlinkElement(args.pageDOM, searchAttribute);
+                if (ffElement?.locators?.length >= 1) {
+                    locators = ffElement.locators;
+                }
+                else {
+                    locators = [
+                        ...locator,
+                        ...ffElement.locators.flat(Infinity)
+                    ];
+                }
+            }
+            catch (error) {
+                logData_js_1.logger.info(`Element is not found ${error.message}`);
+                locators = [{ name: "xpath", value: args.selector }];
+            }
+            data.elementsData = [{
+                    name: args.elementName,
+                    type: args.elementType,
+                    locators: locators,
+                    platform: args.platform
+                }];
+        }
+        else {
+            data.elementsData = [];
+        }
+        return data;
+    });
+}
+;

package/dist/fireflinkData/fireflinkScript/scriptGenrationData.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ScriptDataAppender = void 0;
+const platformSyntax_js_1 = require("../../utils/helpers/platformSyntax.js");
 class ScriptDataAppender {
     constructor() {
         this.data = [];
@@ -14,6 +15,9 @@ class ScriptDataAppender {
         const job = async () => {
             await this.retry(async () => {
                 const item = await task();
+                item.elementsData?.forEach(element => {
+                    element.platform = (0, platformSyntax_js_1.getPlatformSyntax)(element.platform);
+                });
                 this.process(item, index);
             }, 3);
         };

package/dist/imageAnalysisMobile/annotatedScreenshotMobile.d.ts

@@ -2,3 +2,7 @@ export declare function annotateScreenshot(driver: any, xmlSource: string): Prom
     boxedImageBuffer: Buffer;
     boxes: any[];
 }>;
+export declare function annotateIosScreenshot(driver: any, xmlSource: string): Promise<{
+    boxedImageBuffer: Buffer;
+    boxes: any[];
+}>;

package/dist/imageAnalysisMobile/annotatedScreenshotMobile.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.annotateScreenshot = annotateScreenshot;
+exports.annotateIosScreenshot = annotateIosScreenshot;
 const canvas_1 = require("@napi-rs/canvas");
 const fast_xml_parser_1 = require("fast-xml-parser");
 const logData_js_1 = require("../utils/logger/logData.js");
@@ -31,8 +32,67 @@ function extractBoxes(xmlSource) {
                 clazz.toLowerCase().includes("edittext") ||
                 clazz.toLowerCase().includes("autocompletetextview")) {
                 const bounds = parseBounds(node.bounds);
-                if (bounds)
+                if (bounds) {
                     boxes.push(bounds);
+                }
+            }
+        }
+        for (const key in node) {
+            if (typeof node[key] === "object") {
+                traverse(node[key]);
+            }
+        }
+    };
+    traverse(tree);
+    return boxes;
+}
+function extractIosBoxes(xmlSource, screenWidth, screenHeight) {
+    const parser = new fast_xml_parser_1.XMLParser({
+        ignoreAttributes: false,
+        attributeNamePrefix: ""
+    });
+    const tree = parser.parse(xmlSource);
+    const boxes = [];
+    const seen = new Set();
+    const traverse = (node) => {
+        if (!node)
+            return;
+        if (node.x !== undefined && node.y !== undefined && node.width !== undefined && node.height !== undefined) {
+            const x = parseInt(node.x, 10);
+            const y = parseInt(node.y, 10);
+            const w = parseInt(node.width, 10);
+            const h = parseInt(node.height, 10);
+            const clazz = node.type || node.class || "";
+            const skipNode = (String(node.enabled) === "false" ||
+                String(node.visible) === "false" ||
+                String(node.traits).includes("PlaysSound") ||
+                String(node.traits).includes("KeyboardKey") ||
+                w <= 0 || h <= 0);
+            const isIOSClass = [
+                "XCUIElementTypeButton",
+                "XCUIElementTypeCell",
+                "XCUIElementTypeImage",
+                "XCUIElementTypeOther",
+                "XCUIElementTypeTextField",
+                // "XCUIElementTypeStaticText"
+            ].includes(clazz);
+            if (!skipNode && isIOSClass) {
+                // isOnScreen check if screen dimensions are known
+                const isOnScreen = (screenWidth && screenHeight)
+                    ? ((x + w) > 0 && x < screenWidth && (y + h) > 0 && y < screenHeight)
+                    : true;
+                if (isOnScreen) {
+                    const boxStr = `${x},${y},${x + w},${y + h}`;
+                    if (!seen.has(boxStr)) {
+                        seen.add(boxStr);
+                        boxes.push({
+                            x1: x,
+                            y1: y,
+                            x2: x + w,
+                            y2: y + h
+                        });
+                    }
+                }
             }
         }
         for (const key in node) {
@@ -54,11 +114,41 @@ const COLORS = [
     [180, 0, 0],
     [255, 105, 180],
 ];
-async function drawBoxes(screenshotBuffer, boxes) {
+async function drawBoxes(screenshotBuffer, boxes, deviceWidth, deviceHeight) {
     const img = await (0, canvas_1.loadImage)(screenshotBuffer);
     const canvas = (0, canvas_1.createCanvas)(img.width, img.height);
     const ctx = canvas.getContext("2d");
     ctx.drawImage(img, 0, 0);
+    const scaleX = img.width / deviceWidth;
+    const scaleY = img.height / deviceHeight;
+    const fontScale = Math.min(scaleX, scaleY);
+    const fontSize = Math.max(20, Math.floor(40 * fontScale));
+    ctx.font = `bold ${fontSize}px Arial`;
+    ctx.textBaseline = "top";
+    boxes.forEach((box, index) => {
+        const color = COLORS[index % COLORS.length];
+        const colorStr = `rgb(${color[0]},${color[1]},${color[2]})`;
+        const scaledX1 = box.x1 * scaleX;
+        const scaledY1 = box.y1 * scaleY;
+        const scaledWidth = (box.x2 - box.x1) * scaleX;
+        const scaledHeight = (box.y2 - box.y1) * scaleY;
+        ctx.strokeStyle = colorStr;
+        ctx.lineWidth = Math.max(2, Math.floor(3 * fontScale));
+        ctx.strokeRect(scaledX1, scaledY1, scaledWidth, scaledHeight);
+        ctx.fillStyle = colorStr;
+        ctx.textBaseline = "middle";
+        ctx.fillText(String(index + 1), box.x1 + 8, box.y1 // stays aligned across all boxes
+        );
+    });
+    return canvas.toBuffer("image/png");
+}
+async function drawBoxesIos(screenshotBuffer, boxes, screenWidth, screenHeight) {
+    const img = await (0, canvas_1.loadImage)(screenshotBuffer);
+    const canvas = (0, canvas_1.createCanvas)(img.width, img.height);
+    const ctx = canvas.getContext("2d");
+    ctx.drawImage(img, 0, 0);
+    const scaleX = screenWidth ? img.width / screenWidth : 1;
+    const scaleY = screenHeight ? img.height / screenHeight : 1;
     const scale = img.width / 1080;
     const fontSize = Math.max(28, Math.floor(40 * scale));
     ctx.font = `bold ${fontSize}px Arial`;
@@ -68,19 +158,37 @@ async function drawBoxes(screenshotBuffer, boxes) {
         const colorStr = `rgb(${color[0]},${color[1]},${color[2]})`;
         ctx.strokeStyle = colorStr;
         ctx.lineWidth = Math.max(2, Math.floor(3 * scale));
-        ctx.strokeRect(box.x1, box.y1, box.x2 - box.x1, box.y2 - box.y1);
+        const x = box.x1 * scaleX;
+        const y = box.y1 * scaleY;
+        const w = (box.x2 - box.x1) * scaleX;
+        const h = (box.y2 - box.y1) * scaleY;
+        ctx.strokeRect(x, y, w, h);
         ctx.fillStyle = colorStr;
-        ctx.fillText(String(index + 1), box.x1 - fontSize * 0.6, box.y1 - fontSize * 0.6);
+        ctx.textBaseline = "middle";
+        ctx.fillText(String(index + 1), x + 8, y // stays aligned across all boxes
+        );
     });
     return canvas.toBuffer("image/png");
 }
 async function annotateScreenshot(driver, xmlSource) {
+    const rect = await driver.getWindowRect();
     const screenshotBase64 = await driver.takeScreenshot();
     const screenshotBuffer = Buffer.from(screenshotBase64, "base64");
     const boxes = extractBoxes(xmlSource);
-    const boxedImageBuffer = await drawBoxes(screenshotBuffer, boxes);
-    // Uncomment only when debugging
-    await logData_js_1.logger.saveImage(boxedImageBuffer, "boxed.png");
+    const boxedImageBuffer = await drawBoxes(screenshotBuffer, boxes, rect.width, rect.height);
+    logData_js_1.logger.saveImage(boxedImageBuffer, "boxed.png");
+    return {
+        boxedImageBuffer: boxedImageBuffer,
+        boxes: boxes
+    };
+}
+async function annotateIosScreenshot(driver, xmlSource) {
+    const screenshotBase64 = await driver.takeScreenshot();
+    const screenshotBuffer = Buffer.from(screenshotBase64, "base64");
+    const windowSize = await driver.getWindowRect();
+    const boxes = extractIosBoxes(xmlSource, windowSize.width, windowSize.height);
+    const boxedImageBuffer = await drawBoxesIos(screenshotBuffer, boxes, windowSize.width, windowSize.height);
+    logData_js_1.logger.saveImage(boxedImageBuffer, "ios_boxed.png");
     return {
         boxedImageBuffer: boxedImageBuffer,
         boxes: boxes

package/dist/imageAnalysisWeb/annotateScreenshot.js

@@ -12,7 +12,7 @@ const COLORS = [
     [180, 0, 0],
     [255, 105, 180],
 ];
-const TAG_PADDING = 4;
+const TAG_PADDING = 8;
 function measureText(ctx, text) {
     const metrics = ctx.measureText(text);
     const height = metrics.actualBoundingBoxAscent + metrics.actualBoundingBoxDescent;
@@ -63,7 +63,7 @@ async function annotateScreenshot(screenshotBase64, nodes) {
     const buffer = Buffer.from(screenshotBase64, "base64");
     const image = await (0, canvas_1.loadImage)(buffer);
     const scale = image.width / 1920;
-    const fontSize = Math.max(10, Math.floor(14 * scale));
+    const fontSize = Math.max(16, Math.floor(20 * scale));
     const canvas = (0, canvas_1.createCanvas)(image.width, image.height);
     const ctx = canvas.getContext("2d");
     ctx.drawImage(image, 0, 0);
@@ -89,8 +89,8 @@ async function annotateScreenshot(screenshotBase64, nodes) {
         ctx.lineWidth = Math.max(2, Math.floor(2 * scale));
         ctx.strokeRect(x1, y1, x2 - x1, y2 - y1);
         const { width: tw, height: th } = measureText(ctx, label);
-        const tagW = Math.max(22, tw + 10);
-        const tagH = Math.max(18, th + 8);
+        const tagW = Math.max(28, tw + 12);
+        const tagH = Math.max(25, th + 12);
         const { x: lx, y: ly } = findFreeTagPosition([x1, y1, x2, y2], tagW, tagH, elementBoxes, usedTagRects, canvas.width, canvas.height);
         ctx.fillStyle = "white";
         ctx.strokeStyle = colorStr;

package/dist/imageAnalysisWeb/generateAnnotatedScreenshots.d.ts

@@ -1 +1,4 @@
-export declare function generateAnnotatedScreenshots(driver: WebdriverIO.Browser): Promise<Buffer>;
+export declare function generateAnnotatedScreenshots(driver: WebdriverIO.Browser): Promise<{
+    primary: Buffer;
+    secondary: Buffer;
+}>;

package/dist/imageAnalysisWeb/generateAnnotatedScreenshots.js

@@ -3,29 +3,27 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.generateAnnotatedScreenshots = generateAnnotatedScreenshots;
 const annotateScreenshot_js_1 = require("./annotateScreenshot.js");
 const jsForVision_js_1 = require("../utils/javascript/jsForVision.js");
-// import { logger } from "../utils/logger/logData.js";
+const logData_js_1 = require("../utils/logger/logData.js");
 async function generateAnnotatedScreenshots(driver) {
     const screenshot = await driver.takeScreenshot();
-    // const [primaryRaw, secondaryRaw] = await Promise.all([
-    //   driver.execute(JS_PRIMARY),
-    //   driver.execute(JS_SECONDARY),
-    // ]);
-    const [primaryRaw] = await Promise.all([
+    const [primaryRaw, secondaryRaw] = await Promise.all([
         driver.execute(jsForVision_js_1.JS_PRIMARY),
-        // driver.execute(JS_SECONDARY),
+        driver.execute(jsForVision_js_1.JS_SECONDARY),
     ]);
+    // const [primaryRaw] = await Promise.all([
+    //   driver.execute(JS_PRIMARY),
+    //   // driver.execute(JS_SECONDARY),
+    // ]);
     const primary = primaryRaw ?? [];
-    // let secondary = (secondaryRaw as NodeItem[]) ?? [];
-    // const primaryIds = new Set(primary.map((item) => item.ff));
-    // secondary = secondary.filter((item) => !primaryIds.has(item.ff));
-    const [primaryAnnotated] = await Promise.all([
+    let secondary = secondaryRaw ?? [];
+    const primaryIds = new Set(primary.map((item) => item.ff));
+    secondary = secondary.filter((item) => !primaryIds.has(item.ff));
+    const [primaryAnnotated, secondaryAnnotated] = await Promise.all([
         (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, primary),
-        // annotateScreenshot(
-        //   screenshot,
-        //   secondary,
-        // )
+        (0, annotateScreenshot_js_1.annotateScreenshot)(screenshot, secondary)
     ]);
     // Uncomment only when debugging
-    // await logger.saveImage(primaryAnnotated, "annotated_primary.png");
-    return primaryAnnotated;
+    logData_js_1.logger.saveImage(primaryAnnotated, "annotated_primary.png");
+    logData_js_1.logger.saveImage(secondaryAnnotated, "annotated_secondary.png");
+    return { primary: primaryAnnotated, secondary: secondaryAnnotated };
 }

package/dist/llmConfig/llmConfiguration.d.ts

@@ -1,4 +1,5 @@
-import { IServiceProviderBaseUrlConfiguration } from "../core/interfaces/llmConfigurationInterface.js";
-export declare class ServiceProviderBaseUrlProvider implements IServiceProviderBaseUrlConfiguration {
-    getBaseUrl(aiInfo: Record<string, string>): string;
+import { IServiceProviderProviderConfiguration } from "../core/interfaces/llmConfigurationInterface.js";
+import { AIInstanceInfo } from "../utils/InstancesDetails/getInstancesInfo.js";
+export declare class ServiceProviderProviderConfiguration implements IServiceProviderProviderConfiguration {
+    getProvider(aiInfo: AIInstanceInfo): Promise<any>;
 }

package/dist/llmConfig/llmConfiguration.js

@@ -1,24 +1,47 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ServiceProviderBaseUrlProvider = void 0;
-const BASE_URLS = {
-    Groq: "https://api.groq.com/openai/v1",
-    DefaultFireFlink: "https://api.groq.com/openai/v1",
-    OpenAi: "https://api.openai.com/v1",
-    Anthropic: "https://api.anthropic.com/v1/",
-    Gemini: "https://generativelanguage.googleapis.com/v1beta/openai/",
+exports.ServiceProviderProviderConfiguration = void 0;
+const openai_1 = require("@ai-sdk/openai");
+const anthropic_1 = require("@ai-sdk/anthropic");
+const groq_1 = require("@ai-sdk/groq");
+const google_1 = require("@ai-sdk/google");
+const google_vertex_1 = require("@ai-sdk/google-vertex");
+const azure_1 = require("@ai-sdk/azure");
+const PROVIDERS = {
+    Groq: groq_1.createGroq,
+    DefaultFireFlink: groq_1.createGroq,
+    OpenAi: openai_1.createOpenAI,
+    Anthropic: anthropic_1.createAnthropic,
 };
-class ServiceProviderBaseUrlProvider {
-    getBaseUrl(aiInfo) {
-        if (aiInfo.serviceProvider === "Azure") {
-            if (!aiInfo.resource)
-                throw new Error("Resource not found");
-            return `https://${aiInfo.resource}.openai.azure.com/openai/v1/`;
+class ServiceProviderProviderConfiguration {
+    async getProvider(aiInfo) {
+        if (aiInfo.serviceProvider === "Azure_AI") {
+            return (0, azure_1.createAzure)({
+                apiKey: aiInfo.apiKey,
+                resourceName: aiInfo.resource,
+            });
         }
-        const url = BASE_URLS[aiInfo.serviceProvider];
-        if (!url)
-            throw new Error(`Unsupported service provider: ${aiInfo.serviceProvider}`);
-        return url;
+        if (aiInfo.serviceProvider === "Gemini") {
+            if (aiInfo.accountType === "Enterprise Level") {
+                return (0, google_vertex_1.createVertex)({
+                    project: aiInfo.projectId,
+                    location: aiInfo.region,
+                    googleAuthOptions: {
+                        credentials: aiInfo.enterpriseConfigFile,
+                    },
+                });
+            }
+            return (0, google_1.createGoogleGenerativeAI)({
+                apiKey: aiInfo.apiKey,
+            });
+        }
+        const providerFactory = PROVIDERS[aiInfo.serviceProvider];
+        if (!providerFactory) {
+            throw new Error(`Unsupported provider: ${aiInfo.serviceProvider}`);
+        }
+        return providerFactory({
+            apiKey: aiInfo.apiKey,
+        });
     }
 }
-exports.ServiceProviderBaseUrlProvider = ServiceProviderBaseUrlProvider;
+exports.ServiceProviderProviderConfiguration = ServiceProviderProviderConfiguration;

package/dist/service/fireflink.service.d.ts

@@ -3,6 +3,6 @@ export declare class fireflinkService implements IFireflinkService {
     private apiService;
     private kafkaProducer;
     constructor(apiService: IFireFlinkApiService, broker: string[], sslConfig: any);
-    runGetInstancesDetails(projectId: string, token: string, serverHost: string): Promise<any>;
+    runGetInstancesDetails(projectId: string, token: string, serverHost: string, licenseId: string): Promise<any>;
     produceMessageToKafka(kafkaArgs: IKafkaPayload): Promise<void>;
 }

package/dist/service/fireflink.service.js

@@ -8,19 +8,19 @@ class fireflinkService {
         this.apiService = apiService;
         this.kafkaProducer = new fireflinkKafka_service_js_1.KafkaProducerService(broker, sslConfig);
     }
-    async runGetInstancesDetails(projectId, token, serverHost) {
+    async runGetInstancesDetails(projectId, token, serverHost, licenseId) {
         const headers = {
             Accept: "*/*",
             "Accept-Language": "en-US,en;q=0.9",
             "Content-Type": "application/json",
             projectId: projectId || "",
-            Authorization: `Bearer ${token}`,
+            // Authorization: `Bearer ${token}`,
         };
-        const url = `${serverHost}/project/optimize/v3/ai-service-provider/selected-instance?projectId=${projectId}`;
+        const url = `${serverHost}/project/optimize/v3/ai-service-provider/internal/selected-instance?projectId=${projectId}&licenseId=${licenseId}`;
         try {
             const finalResult = await this.apiService.getInstancesDetailsApi(headers, url);
             if (finalResult) {
-                logData_js_1.logger.error("Successfully submitted automation script to FireFlink API.");
+                logData_js_1.logger.info("Successfully submitted automation script to FireFlink API.");
             }
             else {
                 throw new Error("FireFlink API responded with an error: " + JSON.stringify(finalResult));

package/dist/tests/poc.d.ts

@@ -1 +0,0 @@
-export {};