ff-automationv2 2.1.3 → 2.2.0

package/src/ai/llmprompts/systemPrompts/userStoryToListPrompt.ts

@@ -2,23 +2,60 @@
 export async function buildStepExtractionPrompt(
 ): Promise<string> {
   const prompt = `
-You are a structured automation assistant. Your task is to extract step data from a provided user story for web automation.
+You are a precise assistant.
+Extract Steps from the following user story for web automation.
 
-Requirements
-Each step must be a single string (not a tuple or any other data type).
-Preserve the original wording of each step.
-You may rewrite for clarity or formatting consistency only.
-Do not add new steps.
-Do not remove any existing steps.
-Do not introduce additional actions.
-Do not include the example list in the response.
-Do not include any extra text, commentary, markdown, or code fences.
+Rules:
+1. Each step must be a string — not a tuple.
+2. Keep the wording the same except:
+   - Do not add new steps, only rewrite existing ones.
+   - Do not remove any steps.
+   - Do not remove quotes from the step. if these quotes are present in the step, keep them as it is.
+3. Output only a **valid TypeScript array of strings**. No explanations, markdown, or code fences..
+4. Please paste the “below example array” here, so I can make it dynamic and generic strictly in the same format.
+**Dont include any other text in the array. and also dont include example array in the response.**
 Output Format
 
 Respond strictly in valid JSON using the following structure:
 {
-"manaulSteps": ["step 1", "step 2", "step 3"]
+"manualSteps": ["step 1", "step 2", "step 3"]
 }
 `;
   return prompt;
 }
+
+
+export async function androidbuildStepExtractionPrompt(
+): Promise<string> {
+  const prompt = `
+You are a precise assistant.  
+Extract **step data** from the following mobile {user_story_raw}.
+
+Rules:
+Dont take text from one step and use it in another step.
+1. If the step is about luanching an app, like (action, param1, param2) or(action,'param1,param2') then wrap them into a tuple.
+and also give them in the format (launch app, 'com.app.android', 'com.app.activity') like first package name and second activity name. if any separator exists (- / | :), split on it.
+If they mention package:com.app.android and activity:com.app.activity remove variable name and then convert them into (launch app, 'com.app.android, 'com.app.activity')
+2.Dont remove parameters from the step. and never combine or merge steps and dont create any extra steps.
+step may be any formate for enter if input is give seperatly with comma use this
+ example:'enter username in text field','testuser' → ('enter username in text field','testuser').
+if not given seperatly with comma use and mention in step it self than this
+ example:'enter testuser in text field' → ('enter testuser in text field','testuser').
+3. If it's just a single string value, keep it as a string.
+4. Always output a **Always output a valid list of strings.** containing strings or tuples only. 
+ - Do not add explanations, reasoning, or extra text outside the list.
+ - Do not make any assumption if step inputs are missing in user_story_raw
+ - **Dont change the order of steps from user story**
+5. If the step is tuple keep it as tuple like this (step). if not just keep it as a string like this 'step'.
+**strictly follow the output format**
+[
+    ('Launch  Mobile App','com.app.android','com.app.activity'),
+    'Tap on cancel button',
+    'Tap on signin with email',
+    ('enter username in text field','testuser'),
+    ('enter testuser in text field','testuser'),
+    'verify element is displayed'
+]
+`;
+  return prompt;
+}

package/src/ai/llmprompts/systemPrompts/verifyActionExtractorPrompt.ts

@@ -0,0 +1,82 @@
+
+import { verifyActionArgs } from "../llmPromptTypes/systemPromptTypes"
+
+
+export async function verifyActionExtractorPrompt({
+    extractedDomJson,
+    priorAndNextSteps,
+
+}: verifyActionArgs
+): Promise<string> {
+    const nlpList = `VerifyElementIsDisabled,VerifyElementIsEnabled,VerifyElementIsSelected,VerifyIfOptionWithIndexIsSelectedInListBox,VerifyWidthOfElement,VerifyBrowserWindowXLocation,
+    VerifyIfCookieNameIsDeleted,VerifyBrowserWindowWidth,VerifyValueOfElementIsCleared,
+    VerifyTagName,VerifyIfCookieNameIsAdded,VerifyLinkNavigatesToTitle,VerifyXLocationOfElement,
+    VerifyAllOptionsAreDeselected,VerifyBrowserHtmlCodeContainsString,VerifyBrowserWindowYLocation,
+    VerifyNumberOfElementsByTagName,VerifyListBoxOptionsAreSorted,VerifyNavigateURL,
+    VerifyHeightOfWebElement,VerifyValueOfElementContainsString,VerifyBrowserWindowHeight,VerifyIfLinksAreBroken,
+    VerifyIfLinkIsBroken,VerifyValueOfElementIsString,VerifyYLocationOfElement,
+    VerifyNumberOfElementsByXpath,VerifyIfOptionWithIndexIsDeselectedInElement,VerifyAttributeValue,
+    VerifyElementIsDisplayed,VerifyAllOptionsAreSelected,VerifyIfOptionWithValueIsDeselected,
+    VerifyBrowserWindowTitleContainsString,VerifyBrowserWindowTitleIsString,
+    VerifyTitleOfCurrentPage,VerifyPartialTitleOfCurrentPage,VerifyElementIsNotDisplayed,
+    VerifyCssAttributeValue,VerifyPartialAttributeValue,VerifyIfSpecifiedOptionIsSelected,
+    VerifyTitleInAllPage,VerifyIfOptionWithValueIsSelected,VerifyIfVideoIsPlaying,
+    VerifyIfGivenOptionIsDuplicate,VerifyRadioButtonIsSelected,VerifyRadioButtonIsNotSelected,
+    VerifyThatSpecifiedOptionIsNotSelected,
+    VerifyText,VerifyPartialText,VerifyElementNotContainsText,VerifyElementIsClickable,
+    VerifyCheckBoxIsSelected,VerifyCheckBoxIsNotSelected,
+    VerifyUrlContainsExpectedUrl,VerifyUrlIsString,
+    VerifyIfBrowserWindowIsClosed,VerifyAllBrowserWindowsClosed,
+    VerifyIfAllCookiesAreDeleted,VerifyIfCookieObjectIsPresent,
+    VerifyIfXAndYCoordinateOfElement,VerifyIfVideoIsMuted,VerifyIfVideoIsPaused,
+    VerifyIfVideoEnded,VerifyIfVideoPlayingInMute,VerifyIfVideoPlayingInUnmute,
+    VerifyIfAudioIsPlaying,VerifyIfAudioIsPaused,VerifyIfAudioIsMuted,VerifyIfAudioEnded,
+    VerifyIfLinkIsWorking,VerifyIfLinksAreWorking,
+    VerifyIfImageIsBroken,VerifyAllBrokenImages,VerifyIfStatusCodeOfALinkIsStatusCode,
+    VerifyIfStatusCodesOfLinks,VerifyIfOptionWithVisibleTextIsSelected,VerifyBrowserWindowUrlIsString,
+    VerifyIfOptionWithTextIsDeselected,VerifyPerformanceScore,VerifyAccessibilityScore,VerifyBrowserWindowUrlContainsExpectedUrl,
+    VerifyBestPracticesScore,VerifySeoScore,VerifyPwaScore,VerifyHexCodeForGivenXYCoordinates`;
+
+
+    const elementType = ['link', 'textfield', 'icon', 'button', 'radioButton', 'text', 'textarea', 'image', 'dropdown', 'checkbox', 'tab', 'action overflow button', 'hamburger icon', 'toggle button', 'suggestion', 'time picker', 'date picker', 'toaster message', 'card', 'tooltip', 'option', 'calender', 'sliders', 'visual testing'];
+    const prompt = `You are an intelligent assistant that extracts structured UI action data.
+Given the step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList} don't provide the other nlp names which are not present in the list in any manner.
+
+
+Select the perfect matching NLP name from the list that best fits the step's intent.
+**The NLP name must match exactly one value from the approved NLP list. Do not generate or suggest any new NLP names under any condition.**
+step may give any way,you need to get most matching and most related nlp name from the list that relates to step.
+for these nlp's VerifyIfSpecifiedOptionIsSelected,VerifyThatSpecifiedOptionIsNotSelected,VerifyIfOptionWithValueIsSelected,VerifyIfOptionWithValueIsDeselected,in step if they mention word value then only match with these VerifyIfOptionWithValueIsSelected,VerifyIfOptionWithValueIsDeselected.Otherwise go with specified option nlp's.
+example: verify if email element is present
+nlp name should be VerifyElementIsDisplayed
+
+
+{
+  "attribute_value": "Fire-Flink-x",
+  "nlpName": "x",
+  "input_text":"x"
+  "elementType": "x"
+}
+
+Rules:
+- nlpName must be exactly one from the provided list.
+- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
+- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or any other attributes best match the step in the simplified DOM.**
+- For some steps you can see two inputs but you need to understand step and send the required input_text. input_text can be string, number, url or xpath.
+        ex: Verify the flights matches the flight, Verify the home has 30 and 50 coordinates if they give x-30,y-50 just send 30,50 and verify hex code '#FFFFFF' for given (34,11) coordinates or they will give x-34,y-11 hexcode-#FFFFFF format than fill input_text in this order "#FFFFFF,34,11"
+        here keyword : "flights" , input : "flight", here input:"30,50"  , here input:"#FFFFFF,34,11" 
+- If step is about checking attribute value then map to attribute nlps and input_text: "attribute_name,element_name" here attribute_name can be id,text,href etc not fire-flink-x. input_text should be in this format.if they mention css attribute value then input_text should be "css_property,expected_value".
+and also If a step requires comparing two values, provide the input in the format input_text: "value1,value2".
+- if step is about tag name then input_text: "tag name". and if step is about verifying tag name and with count then input_text: "tag name,count". and if any step is about verifying number of elements with xpath then input_text should be "xpath,count"
+- if step is about verifying location then input_text: "coordinates".
+- If a step involves a URL, map navigation/action keywords (navigate) to VerifyNavigateURL, partial match keywords (contains, includes) to VerifyUrlContainsExpectedUrl or VerifyBrowserWindowUrlContainsExpectedUrl exact match keywords (equals, is, exactly) to VerifyUrlIsString or VerifyBrowserWindowUrlIsString based on step  if they mention navigate to title match with (VerifyLinkNavigatesToTitle) default to exact match if unclear, and prioritize navigation if both intents exist.
+- If no explicit option is mentioned in the step, map it to the most relevant non-option verification NLP based on the intent of the step.
+- **Extract input_text from the step that is which is being verified in the step, if you can't find any input text in the step, return keyword as input_text.**
+if step is verifying value of element then extract value which is being verified only not any other words in the step. 
+- Extract keyword from the step, if the step contains any keyword.
+- if no matching element found for the step return attribute_value as only 'Fire-Flink-0' dont give Fire-Flink-x or any string .
+- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
+- **Respond with valid JSON only. don't return any other text or don't return response in list format**
+`;
+    return prompt;
+}

package/src/ai/llmprompts/systemPrompts/verifyActionExtractorPromptMob.ts

@@ -0,0 +1,58 @@
+
+import { verifyActionArgs } from "../llmPromptTypes/systemPromptTypes"
+
+
+export async function verifyActionExtractorPromptMob({
+    extractedDomJson,
+    priorAndNextSteps,
+
+}: verifyActionArgs
+): Promise<string> {
+    const nlpList = `  "MOB_VerifyElementIsDisplayed,MOB_VerifyElementIsEnabled,MOB_VerifyElementIsSelected,MOB_VerifyText,MOB_VerifyAttributeValue,MOB_VerifyPartialAttributeValue,"
+    "MOB_VerifyElementIsDisplayedInSpecifiedTime,MOB_VerifyElementIsEnabledInSpecifiedTime,MOB_VerifyElementIsDisabledInSpecifiedTime,"
+    "MOB_VerifyBatteryLevel,MOB_VerifyPartialText,MOB_VerifyElementIsClickable,"
+    "MOB_VerifyIfKeyboardIsHidden,MOB_VerifyIfKeyboardIsShown,MOB_VerifyIfXAndYCoordinateOfElement,"
+    "MOB_VerifyHeightOfElement,MOB_VerifyXLocationOfAndroidElement,MOB_VerifyIfDeviceIsUnlocked,"
+    "MOB_VerifyIfWifiOff,MOB_VerifyElementIsSelectedInSpecifiedTime,MOB_VerifyIfWifiOn,MOB_VerifyIfDeviceIsLocked,"
+    "MOB_VerifyIfAirplaneModeOff,MOB_VerifyIfGpsOff,MOB_VerifyHexCodeForGivenXYCoordinates,"
+    "MOB_VerifyYLocationOfAndroidElement,MOB_VerifyIfGpsOn,MOB_VerifyIfAirplaneModeOn,"
+    "MOB_VerifyWidthOfElement,MOB_VerifyElementIsNotDisplayed,MOB_VerifyElementNotContainsText,"
+    "MOB_VerifyIfAppIsClosed"`;
+
+
+    const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"]
+
+    const prompt = `You are an intelligent assistant that extracts structured UI action data.
+Given the step: step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
+
+Select the perfect matching NLP name from the list that best fits the step's intent.
+
+Return **only valid JSON** in the following format:
+{
+  "FF_inspector_value": "Fire-Flink-x",
+  "nlpName": "x",
+  "input_text":"x"
+  "keyword": "x",
+  "element_type": "x",
+  "locator_type":"x"
+}
+Rules:
+- nlpName must be exactly one from the provided list.
+- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
+- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
+- For some steps you can see two inputs but you need to uderstand step and send the requred input_text.
+        ex: Verify the flights matches the flight,Verify the home has 30 and 50 coordinates
+        here keyword : "flights" , input : flight, here input:30,50
+- If step mention time and element name then Input_text should be elementname,time.
+        example: Verify the home is displayed for 30 seconds 
+        here keyword : "home" , input : home,30
+- If step mention verify text then only pick MOB_VerifyText nlp otherwise pick MOB_VerifyElementIsDisplayed.
+- Extract input_text from the step that is which is being verified in the {step}, if you can't find any input text in the step, return keyword as input_text.
+- Extract keyword from the step, if the step contains any keyword.
+- Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
+- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
+- Extract locator_type if mentioned in the step, like(text,resource-id,content-desc) . if not found return 'empty'.
+- Respond with valid JSON only.
+`;
+    return prompt;
+}

package/src/ai/llmprompts/systemPrompts/visionPrompt.ts

@@ -1,26 +1,20 @@
+import { visionPrompt } from "../llmPromptTypes/systemPromptTypes"
 
-type visionPrompt = {
-    currentStep: string,
-    priorAndNextSteps: string[]
-}
 export async function visionPrompt(
-    { currentStep, priorAndNextSteps }: visionPrompt
+    { priorAndNextSteps }: visionPrompt
 ): Promise<string> {
 
     const customPromptText = `
 You are a precise automation assistant.
 Context: ${priorAndNextSteps}
-- Use the context to understand the step and select the closest semantic match to ${currentStep} and return only its index, else Fire-Flink-0.
+- Use the context to understand the step and select the closest semantic match to Step and return only its index, else Fire-Flink-0.
 - **If u can't find the element, return Fire-Flink-0. dont return any other elements index.**
-- Using the provided annotated screenshot and ${currentStep}, identify the correct element index from the image.
-- Match the index to the text or action described in ${currentStep} as accurately as possible.
-- Return only the Fire-Flink-xx for the target element.
-- Do not provide any additional text or explanation. The output must be strictly Fire-Flink-xx.
-Now, Analyze the step:
-Step: "${currentStep}"
-
+- Using the provided annotated screenshot and Step, identify the correct element index from the image.
+- Match the index to the text or action described in Step as accurately as possible.
+- Return only the Fire-Flink-x for the target element.
+- Do not provide any additional text or explanation. The output must be strictly Fire-Flink-x.
 Respond only with JSON using this format:
-{fireflinkIndex: Fire-Flink-xx.}
+{fireflinkIndex: Fire-Flink-x | Fire-Flink-0}
 
 `.trim();
 

package/src/ai/llmprompts/systemPrompts/visionPromptMobile.ts

@@ -0,0 +1,25 @@
+import { visionPromptMobile } from "../llmPromptTypes/systemPromptTypes"
+
+export async function visionPromptMobile(
+    { priorAndNextSteps }: visionPromptMobile
+): Promise<string> {
+
+    const customPromptText = `
+You are a precise mobile automation assistant. 
+- You are given an image containing multiple elements outlined with Red bounding boxes.Each bounding box has a small black index number label drawn above it (like 1, 2, 3...).
+- Do not treat numbers appearing in the step as index numbers.if any number appears in the step it is not an index number.
+- Use the context from the all steps ${priorAndNextSteps}
+- **Analyze the user's step and action done by that step action determine which UI element in the screenshot corresponds semantically to that action.
+- for action enter check for edit text field and choose index number of that box.
+- **pick the index number that is on the box where element present that best matches semantically or visually to that action of the given step.**
+- Identify which element number (from the image) best matches that step.
+- Return the **index** number as shown in the image (1-based).
+- if there are two elements that match step and one have box and other doesn't have box then prefer the one with box and return that index number.
+- if no correct element found or if none of correct elements has no box,then return 0.
+Respond only with JSON using this format:
+{fireflinkIndex: Fire-Flink-x | Fire-Flink-0}
+
+`.trim();
+
+    return customPromptText;
+}

package/src/ai/llmprompts/systemPrompts/waitActionExtractorPrompt.ts

@@ -0,0 +1,65 @@
+import { waitActionArgs } from "../llmPromptTypes/systemPromptTypes"
+
+export async function waitActionExtractorPrompt({
+    extractedDomJson,
+    priorAndNextSteps,
+
+}: waitActionArgs
+): Promise<string> {
+    const nlpList = `
+    "WaitTillUlrOfPageContainsString",
+    "WaitTillTitleOfPageIsString",
+    "WaitTillUrlOfPageIsString",
+    "WaitTitleOfPageContainsString",
+    "WaitTillElementVisible",
+    "WaitTillAlertIsPresent",
+    "WaitUntilElementIsInvisible",
+    "WaitUntilElementIsClickable",
+    "WaitAttributeOfElementContainsString",
+    "WaitTillAttributeOfElementIsString",
+    "WaitUntilElementIsSelected",
+    "WaitUntilAllElementIsInvisible",
+    "WaitTillPresenceOfAllElements",
+    "WaitTillElementContainsText",
+    "WaitTillPresenceOfElement",
+    "WaitTillElementHasText",
+    "WaitTillPageLoads",
+    "WaitTillAttributeOfElementIsNotEmpty",
+    "WaitTillValueAttributeOfElementHasText",
+    "WaitAndSendKeys",
+    "WaitTillStalenessOfElement",
+    "WaitAndClick",
+    "Sleep"`;
+
+
+    const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"]
+    const prompt = `
+    You are an intelligent assistant that extracts structured UI action data.
+Given the step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
+
+Select the perfect matching NLP name from the list that best fits the step's intent.
+
+Return **only valid JSON** in the following format:
+{
+  "attribute_value": "Fire-Flink-x",
+  "nlpName": "x",
+  "input_text":"x"
+  "keyword": "x",
+  "element_type": "x"
+}
+Rules:
+- nlpName must be exactly one from the provided list.
+- For steps like wait for some time or wait for 5 secondes. give input in number only and nlpName should be "Sleep".
+- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
+- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
+- For some steps you can see two inputs but you need to uderstand step and send the requred input_text.
+        ex: Wait until the flights has the flight text
+        here keyword : "flights" , input : flight
+- Extract input_text from the step, if you can't find any input text in the step, return keyword as input_text.
+- Extract keyword from the step, if the step contains any keyword.
+- Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
+- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
+- Respond with valid JSON only.
+`;
+    return prompt;
+}

package/src/ai/llmprompts/systemPrompts/waitActionExtractorPromptMob.ts

@@ -0,0 +1,45 @@
+import { waitActionArgs } from "../llmPromptTypes/systemPromptTypes"
+
+export async function waitActionExtractorPromptMob({
+    extractedDomJson,
+    priorAndNextSteps,
+
+}: waitActionArgs
+): Promise<string> {
+    const nlpList = `"MOB_WaitTillPresenceOfElement","MOB_WaitTillElementIsSelected","MOB_WaitTillElementHasText","MOB_WaitTillElementIsEnabled",
+"MOB_WaitTillElementIsVisible","MOB_WaitTillElementContainsText","MOB_WaitTillAttributeOfElementContainsString","MOB_WaitTillAllElementsAreInvisible",
+"MOB_WaitTillElementIsInvisible","MOB_WaitTillElementIsClickable","MOB_WaitTillAttributeOfElementIsString","MOB_WaitTillPresenceOfAllElements",
+"MOB_WaitTillStalenessOfElement","MOB_WaitAndSendKeys","MOB_WaitAndClick","Sleep"`;
+
+
+    const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"]
+    const prompt = `
+    You are an intelligent assistant that extracts structured UI action data.
+Given the step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
+
+Select the perfect matching NLP name from the list that best fits the step's intent.
+
+Return **only valid JSON** in the following format:
+{
+  "attribute_value": "Fire-Flink-x",
+  "nlpName": "x",
+  "input_text":"x"
+  "keyword": "x",
+  "element_type": "x"
+}
+Rules:
+- nlpName must be exactly one from the provided list.
+- For steps like wait for some time or wait for 5 secondes. give input in number only and nlpName should be "Sleep".
+- Use context from the steps : ${priorAndNextSteps}, keyword and json to search for FF-inspecter.
+- **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the step.**
+- For some steps you can see two inputs but you need to uderstand step and send the requred input_text.
+        ex: Wait until the flights has the flight text
+        here keyword : "flights" , input : flight
+- Extract input_text from the step, if you can't find any input text in the step, return keyword as input_text.
+- Extract keyword from the step, if the step contains any keyword.
+- Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
+- Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
+- Respond with valid JSON only.
+`;
+    return prompt;
+}

package/src/ai/llmprompts/userPrompts/userPrompt.ts

@@ -1,9 +1,10 @@
 import { PromptType } from "../../../core/types/promptType.js";
 import { visionPromptMessage } from "../../../core/types/visionllmInputType.js";
+import { formatImageBase64 } from "../../../utils/helpers/imageUtils.js";
 
 export const userInputFormatters: {
     [K in PromptType]: (
-        userInput: Record<string, any>
+        userInput: Record<string, any>,
     ) => string | visionPromptMessage;
 } = {
     [PromptType.USER_STORY_TO_LIST]: (userInput) =>
@@ -12,13 +13,35 @@ export const userInputFormatters: {
     [PromptType.KEYWORD_EXTRACTOR]: (userInput) =>
         `Extract keywords from the following input:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
 
-    [PromptType.ERROR_DESCRIPTION]: (userInput) =>
-        `Analyze the following error context:\n\n${JSON.stringify(userInput.error, null, 2)}`,
+    [PromptType.ERROR_DESCRIPTION]: (error) =>
+        `Analyze the following error context:\n\n${JSON.stringify(error.error, null, 2)}`,
 
     [PromptType.FF_INSPECTOR]: (userInput) =>
         `Inspect the DOM for the following action context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
 
+    [PromptType.VERIFY_PROMPT]: (userInput) =>
+        `Inspect the DOM for the following verify context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
+
+    [PromptType.GET_PROMPT]: (userInput) =>
+        `Inspect the DOM for the following get context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
+
+    [PromptType.COMBINED_ACTION_EXTRACTOR_PROMPT]: (userInput) =>
+        `Inspect the DOM for the following wait context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
+
+    [PromptType.WAIT_ACTION_EXTRACTOR_PROMPT]: (userInput) =>
+        `Inspect the DOM for the following wait context:\n\n${JSON.stringify(userInput.currentStep, null, 2)}`,
+
     [PromptType.VISION_PROMPT]: (userInput) => {
+        let imageBase64: string | null = null;
+
+        if (userInput.annotatedScreenshot) {
+            imageBase64 = formatImageBase64(userInput.annotatedScreenshot);
+        }
+
+        if (!imageBase64) {
+            throw new Error("Vision image is missing. No annotatedScreenshot provided.");
+        }
+
         const content: visionPromptMessage = [
             {
                 type: "text",
@@ -31,10 +54,11 @@ export const userInputFormatters: {
             {
                 type: "image_url",
                 image_url: {
-                    url: String(userInput.imageBase64 ?? "")
+                    url: imageBase64
                 }
             }
         ];
+
         return content;
     }