ff-automationv2 2.2.0 → 2.2.2

package/src/ai/llmprompts/systemPrompts/verifyActionExtractorPromptMob.ts

@@ -1,58 +0,0 @@
-
-import { verifyActionArgs } from "../llmPromptTypes/systemPromptTypes"
-
-
-export async function verifyActionExtractorPromptMob({
-    extractedDomJson,
-    priorAndNextSteps,
-
-}: verifyActionArgs
-): Promise<string> {
-    const nlpList = `  "MOB_VerifyElementIsDisplayed,MOB_VerifyElementIsEnabled,MOB_VerifyElementIsSelected,MOB_VerifyText,MOB_VerifyAttributeValue,MOB_VerifyPartialAttributeValue,"
-    "MOB_VerifyElementIsDisplayedInSpecifiedTime,MOB_VerifyElementIsEnabledInSpecifiedTime,MOB_VerifyElementIsDisabledInSpecifiedTime,"
-    "MOB_VerifyBatteryLevel,MOB_VerifyPartialText,MOB_VerifyElementIsClickable,"
-    "MOB_VerifyIfKeyboardIsHidden,MOB_VerifyIfKeyboardIsShown,MOB_VerifyIfXAndYCoordinateOfElement,"
-    "MOB_VerifyHeightOfElement,MOB_VerifyXLocationOfAndroidElement,MOB_VerifyIfDeviceIsUnlocked,"
-    "MOB_VerifyIfWifiOff,MOB_VerifyElementIsSelectedInSpecifiedTime,MOB_VerifyIfWifiOn,MOB_VerifyIfDeviceIsLocked,"
-    "MOB_VerifyIfAirplaneModeOff,MOB_VerifyIfGpsOff,MOB_VerifyHexCodeForGivenXYCoordinates,"
-    "MOB_VerifyYLocationOfAndroidElement,MOB_VerifyIfGpsOn,MOB_VerifyIfAirplaneModeOn,"
-    "MOB_VerifyWidthOfElement,MOB_VerifyElementIsNotDisplayed,MOB_VerifyElementNotContainsText,"
-    "MOB_VerifyIfAppIsClosed"`;
-
-
-    const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"]
-
-    const prompt = `You are an intelligent assistant that extracts structured UI action data.
-Given the step: step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
-
-Select the perfect matching NLP name from the list that best fits the step's intent.
-
-Return **only valid JSON** in the following format:
-{
-  "FF_inspector_value": "Fire-Flink-x",
-  "nlpName": "x",
-  "input_text":"x"
-  "keyword": "x",
-  "element_type": "x",
-  "locator_type":"x"
-}
-Rules:
-- nlpName must be exactly one from the provided list.
-- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
-- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
-- For some steps you can see two inputs but you need to uderstand step and send the requred input_text.
-        ex: Verify the flights matches the flight,Verify the home has 30 and 50 coordinates
-        here keyword : "flights" , input : flight, here input:30,50
-- If step mention time and element name then Input_text should be elementname,time.
-        example: Verify the home is displayed for 30 seconds 
-        here keyword : "home" , input : home,30
-- If step mention verify text then only pick MOB_VerifyText nlp otherwise pick MOB_VerifyElementIsDisplayed.
-- Extract input_text from the step that is which is being verified in the {step}, if you can't find any input text in the step, return keyword as input_text.
-- Extract keyword from the step, if the step contains any keyword.
-- Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
-- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
-- Extract locator_type if mentioned in the step, like(text,resource-id,content-desc) . if not found return 'empty'.
-- Respond with valid JSON only.
-`;
-    return prompt;
-}

package/src/ai/llmprompts/systemPrompts/visionPrompt.ts

@@ -1,22 +0,0 @@
-import { visionPrompt } from "../llmPromptTypes/systemPromptTypes"
-
-export async function visionPrompt(
-    { priorAndNextSteps }: visionPrompt
-): Promise<string> {
-
-    const customPromptText = `
-You are a precise automation assistant.
-Context: ${priorAndNextSteps}
-- Use the context to understand the step and select the closest semantic match to Step and return only its index, else Fire-Flink-0.
-- **If u can't find the element, return Fire-Flink-0. dont return any other elements index.**
-- Using the provided annotated screenshot and Step, identify the correct element index from the image.
-- Match the index to the text or action described in Step as accurately as possible.
-- Return only the Fire-Flink-x for the target element.
-- Do not provide any additional text or explanation. The output must be strictly Fire-Flink-x.
-Respond only with JSON using this format:
-{fireflinkIndex: Fire-Flink-x | Fire-Flink-0}
-
-`.trim();
-
-    return customPromptText;
-}

package/src/ai/llmprompts/systemPrompts/visionPromptMobile.ts

@@ -1,25 +0,0 @@
-import { visionPromptMobile } from "../llmPromptTypes/systemPromptTypes"
-
-export async function visionPromptMobile(
-    { priorAndNextSteps }: visionPromptMobile
-): Promise<string> {
-
-    const customPromptText = `
-You are a precise mobile automation assistant. 
-- You are given an image containing multiple elements outlined with Red bounding boxes.Each bounding box has a small black index number label drawn above it (like 1, 2, 3...).
-- Do not treat numbers appearing in the step as index numbers.if any number appears in the step it is not an index number.
-- Use the context from the all steps ${priorAndNextSteps}
-- **Analyze the user's step and action done by that step action determine which UI element in the screenshot corresponds semantically to that action.
-- for action enter check for edit text field and choose index number of that box.
-- **pick the index number that is on the box where element present that best matches semantically or visually to that action of the given step.**
-- Identify which element number (from the image) best matches that step.
-- Return the **index** number as shown in the image (1-based).
-- if there are two elements that match step and one have box and other doesn't have box then prefer the one with box and return that index number.
-- if no correct element found or if none of correct elements has no box,then return 0.
-Respond only with JSON using this format:
-{fireflinkIndex: Fire-Flink-x | Fire-Flink-0}
-
-`.trim();
-
-    return customPromptText;
-}

package/src/ai/llmprompts/systemPrompts/waitActionExtractorPrompt.ts

@@ -1,65 +0,0 @@
-import { waitActionArgs } from "../llmPromptTypes/systemPromptTypes"
-
-export async function waitActionExtractorPrompt({
-    extractedDomJson,
-    priorAndNextSteps,
-
-}: waitActionArgs
-): Promise<string> {
-    const nlpList = `
-    "WaitTillUlrOfPageContainsString",
-    "WaitTillTitleOfPageIsString",
-    "WaitTillUrlOfPageIsString",
-    "WaitTitleOfPageContainsString",
-    "WaitTillElementVisible",
-    "WaitTillAlertIsPresent",
-    "WaitUntilElementIsInvisible",
-    "WaitUntilElementIsClickable",
-    "WaitAttributeOfElementContainsString",
-    "WaitTillAttributeOfElementIsString",
-    "WaitUntilElementIsSelected",
-    "WaitUntilAllElementIsInvisible",
-    "WaitTillPresenceOfAllElements",
-    "WaitTillElementContainsText",
-    "WaitTillPresenceOfElement",
-    "WaitTillElementHasText",
-    "WaitTillPageLoads",
-    "WaitTillAttributeOfElementIsNotEmpty",
-    "WaitTillValueAttributeOfElementHasText",
-    "WaitAndSendKeys",
-    "WaitTillStalenessOfElement",
-    "WaitAndClick",
-    "Sleep"`;
-
-
-    const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"]
-    const prompt = `
-    You are an intelligent assistant that extracts structured UI action data.
-Given the step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
-
-Select the perfect matching NLP name from the list that best fits the step's intent.
-
-Return **only valid JSON** in the following format:
-{
-  "attribute_value": "Fire-Flink-x",
-  "nlpName": "x",
-  "input_text":"x"
-  "keyword": "x",
-  "element_type": "x"
-}
-Rules:
-- nlpName must be exactly one from the provided list.
-- For steps like wait for some time or wait for 5 secondes. give input in number only and nlpName should be "Sleep".
-- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
-- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
-- For some steps you can see two inputs but you need to uderstand step and send the requred input_text.
-        ex: Wait until the flights has the flight text
-        here keyword : "flights" , input : flight
-- Extract input_text from the step, if you can't find any input text in the step, return keyword as input_text.
-- Extract keyword from the step, if the step contains any keyword.
-- Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
-- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
-- Respond with valid JSON only.
-`;
-    return prompt;
-}

package/src/ai/llmprompts/systemPrompts/waitActionExtractorPromptMob.ts

@@ -1,45 +0,0 @@
-import { waitActionArgs } from "../llmPromptTypes/systemPromptTypes"
-
-export async function waitActionExtractorPromptMob({
-    extractedDomJson,
-    priorAndNextSteps,
-
-}: waitActionArgs
-): Promise<string> {
-    const nlpList = `"MOB_WaitTillPresenceOfElement","MOB_WaitTillElementIsSelected","MOB_WaitTillElementHasText","MOB_WaitTillElementIsEnabled",
-"MOB_WaitTillElementIsVisible","MOB_WaitTillElementContainsText","MOB_WaitTillAttributeOfElementContainsString","MOB_WaitTillAllElementsAreInvisible",
-"MOB_WaitTillElementIsInvisible","MOB_WaitTillElementIsClickable","MOB_WaitTillAttributeOfElementIsString","MOB_WaitTillPresenceOfAllElements",
-"MOB_WaitTillStalenessOfElement","MOB_WaitAndSendKeys","MOB_WaitAndClick","Sleep"`;
-
-
-    const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"]
-    const prompt = `
-    You are an intelligent assistant that extracts structured UI action data.
-Given the step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
-
-Select the perfect matching NLP name from the list that best fits the step's intent.
-
-Return **only valid JSON** in the following format:
-{
-  "attribute_value": "Fire-Flink-x",
-  "nlpName": "x",
-  "input_text":"x"
-  "keyword": "x",
-  "element_type": "x"
-}
-Rules:
-- nlpName must be exactly one from the provided list.
-- For steps like wait for some time or wait for 5 secondes. give input in number only and nlpName should be "Sleep".
-- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
-- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
-- For some steps you can see two inputs but you need to uderstand step and send the requred input_text.
-        ex: Wait until the flights has the flight text
-        here keyword : "flights" , input : flight
-- Extract input_text from the step, if you can't find any input text in the step, return keyword as input_text.
-- Extract keyword from the step, if the step contains any keyword.
-- Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
-- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
-- Respond with valid JSON only.
-`;
-    return prompt;
-}

package/src/ai/llmprompts/userPrompts/userPrompt.ts

@@ -1,65 +0,0 @@
-import { PromptType } from "../../../core/types/promptType.js";
-import { visionPromptMessage } from "../../../core/types/visionllmInputType.js";
-import { formatImageBase64 } from "../../../utils/helpers/imageUtils.js";
-
-export const userInputFormatters: {
-    [K in PromptType]: (
-        userInput: Record<string, any>,
-    ) => string | visionPromptMessage;
-} = {
-    [PromptType.USER_STORY_TO_LIST]: (userInput) =>
-        `Convert the following user story into steps:\n\n${JSON.stringify(userInput.userStory, null, 2)}`,
-
-    [PromptType.KEYWORD_EXTRACTOR]: (userInput) =>
-        `Extract keywords from the following input:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
-
-    [PromptType.ERROR_DESCRIPTION]: (error) =>
-        `Analyze the following error context:\n\n${JSON.stringify(error.error, null, 2)}`,
-
-    [PromptType.FF_INSPECTOR]: (userInput) =>
-        `Inspect the DOM for the following action context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
-
-    [PromptType.VERIFY_PROMPT]: (userInput) =>
-        `Inspect the DOM for the following verify context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
-
-    [PromptType.GET_PROMPT]: (userInput) =>
-        `Inspect the DOM for the following get context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
-
-    [PromptType.COMBINED_ACTION_EXTRACTOR_PROMPT]: (userInput) =>
-        `Inspect the DOM for the following wait context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
-
-    [PromptType.WAIT_ACTION_EXTRACTOR_PROMPT]: (userInput) =>
-        `Inspect the DOM for the following wait context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
-
-    [PromptType.VISION_PROMPT]: (userInput) => {
-        let imageBase64: string | null = null;
-
-        if (userInput.annotatedScreenshot) {
-            imageBase64 = formatImageBase64(userInput.annotatedScreenshot);
-        }
-
-        if (!imageBase64) {
-            throw new Error("Vision image is missing. No annotatedScreenshot provided.");
-        }
-
-        const content: visionPromptMessage = [
-            {
-                type: "text",
-                text: `Analyze the following step:\n\n${JSON.stringify(
-                    userInput.currentStep,
-                    null,
-                    2
-                )}`
-            },
-            {
-                type: "image_url",
-                image_url: {
-                    url: imageBase64
-                }
-            }
-        ];
-
-        return content;
-    }
-
-};