ff-automationv2 2.2.14 → 2.2.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/dist/ai/llmprompts/systemPrompts/combinedActionExtractorPromptMob.js +4 -3
  2. package/dist/ai/llmprompts/systemPrompts/errorDescriptionPrompt.js +9 -11
  3. package/dist/ai/llmprompts/systemPrompts/fireflinkElementIndexExtactors.js +7 -7
  4. package/dist/ai/llmprompts/systemPrompts/fireflinkElementIndexExtractor_Mob.js +39 -0
  5. package/dist/ai/llmprompts/systemPrompts/mobileKeywordExtractor.js +2 -1
  6. package/dist/ai/llmprompts/systemPrompts/verifyActionExtractorPromptMob.js +2 -1
  7. package/dist/ai/llmprompts/systemPrompts/visionPrompt.js +3 -2
  8. package/dist/ai/llmprompts/systemPrompts/waitActionExtractorPrompt.js +1 -0
  9. package/dist/ai/llmprompts/systemPrompts/waitActionExtractorPromptMob.js +1 -0
  10. package/dist/automation/actions/executor.d.ts +19 -1
  11. package/dist/automation/actions/executor.js +303 -1
  12. package/dist/automation/actions/interaction/click/MOB_DoubleTapAtSpecifiedLocation.d.ts +2 -0
  13. package/dist/automation/actions/interaction/click/MOB_DoubleTapAtSpecifiedLocation.js +26 -0
  14. package/dist/automation/actions/interaction/elementLessAction/MobCheckIfAppIsClosed.d.ts +2 -0
  15. package/dist/automation/actions/interaction/elementLessAction/MobCheckIfAppIsClosed.js +20 -0
  16. package/dist/automation/actions/interaction/elementLessAction/MobRunAppInBackgroundInterface.d.ts +2 -0
  17. package/dist/automation/actions/interaction/elementLessAction/MobRunAppInBackgroundInterface.js +18 -0
  18. package/dist/automation/actions/interaction/enterActions/EnterInputIntoElementFromClipBoardInterface_mob.d.ts +2 -0
  19. package/dist/automation/actions/interaction/enterActions/EnterInputIntoElementFromClipBoardInterface_mob.js +26 -0
  20. package/dist/automation/actions/interaction/enterActions/enterInput.js +2 -2
  21. package/dist/automation/actions/interaction/enterActions/enterInputAndPress.js +1 -1
  22. package/dist/automation/actions/interaction/enterActions/enterusingJs.js +1 -1
  23. package/dist/automation/actions/interaction/enterActions/waitAndEnter.js +3 -3
  24. package/dist/automation/actions/interaction/find/MOB_FindElement.d.ts +2 -0
  25. package/dist/automation/actions/interaction/find/MOB_FindElement.js +34 -0
  26. package/dist/automation/actions/interaction/find/findElements.js +16 -2
  27. package/dist/automation/actions/interaction/get/getScreenshot.js +14 -16
  28. package/dist/automation/actions/interaction/get/getScreenshotAs.js +26 -16
  29. package/dist/automation/actions/interaction/pinch/PinchInByPercentMob.d.ts +2 -0
  30. package/dist/automation/actions/interaction/pinch/PinchInByPercentMob.js +27 -0
  31. package/dist/automation/actions/interaction/pinch/PinchOutByPercentMob.d.ts +2 -0
  32. package/dist/automation/actions/interaction/pinch/PinchOutByPercentMob.js +27 -0
  33. package/dist/automation/actions/interaction/press/mobPressBackSpaceKey.d.ts +2 -0
  34. package/dist/automation/actions/interaction/press/mobPressBackSpaceKey.js +16 -0
  35. package/dist/automation/actions/interaction/press/mobPressSpaceKey.d.ts +2 -0
  36. package/dist/automation/actions/interaction/press/mobPressSpaceKey.js +16 -0
  37. package/dist/automation/actions/interaction/swipe/swipeDirectionNTimes.d.ts +2 -0
  38. package/dist/automation/actions/interaction/swipe/swipeDirectionNTimes.js +27 -0
  39. package/dist/automation/actions/interaction/swipe/swipeDownToElement.d.ts +2 -0
  40. package/dist/automation/actions/interaction/swipe/swipeDownToElement.js +46 -0
  41. package/dist/automation/actions/interaction/swipe/swipeLeftToElement.d.ts +2 -0
  42. package/dist/automation/actions/interaction/swipe/swipeLeftToElement.js +46 -0
  43. package/dist/automation/actions/interaction/swipe/swipeRightToElement.d.ts +2 -0
  44. package/dist/automation/actions/interaction/swipe/swipeRightToElement.js +46 -0
  45. package/dist/automation/actions/interaction/swipe/swipeUpToElement.d.ts +2 -0
  46. package/dist/automation/actions/interaction/swipe/swipeUpToElement.js +49 -0
  47. package/dist/automation/actions/interaction/swipe/swipeUpToElementForMWeb.d.ts +2 -0
  48. package/dist/automation/actions/interaction/swipe/swipeUpToElementForMWeb.js +48 -0
  49. package/dist/automation/actions/interaction/swipe/swipeUsingReferenceElement.d.ts +2 -0
  50. package/dist/automation/actions/interaction/swipe/swipeUsingReferenceElement.js +140 -0
  51. package/dist/automation/actions/interaction/verify/VerifyElementNotContainsText.js +2 -2
  52. package/dist/automation/actions/interaction/wait/waitTillElementIsClickable.js +5 -1
  53. package/dist/automation/actions/interaction/wait/waitTillPresenceOfElement.js +1 -1
  54. package/dist/automation/actions/interface/clickActionInterface.d.ts +8 -0
  55. package/dist/automation/actions/interface/elelementLessActionInterface.d.ts +12 -0
  56. package/dist/automation/actions/interface/findActionInterface.d.ts +10 -2
  57. package/dist/automation/actions/interface/interactionActionInterface.d.ts +8 -0
  58. package/dist/automation/actions/interface/pinchActionInterface.d.ts +19 -0
  59. package/dist/automation/actions/interface/pressActionInterface.d.ts +10 -0
  60. package/dist/automation/actions/interface/swipeActionInterface.d.ts +89 -0
  61. package/dist/automation/actions/interface/swipeActionInterface.js +1 -0
  62. package/dist/automation/actions/interface/waitActionInterface.d.ts +1 -0
  63. package/dist/core/constants/allAction.js +1 -0
  64. package/dist/core/constants/supportedActions.js +9 -1
  65. package/dist/core/interfaces/actionInterface.d.ts +19 -1
  66. package/dist/core/interfaces/executionDetails.d.ts +1 -4
  67. package/dist/core/interfaces/fireflinkScriptPayloadInterface.d.ts +2 -3
  68. package/dist/core/main/actionHandlerFactory.js +62 -2
  69. package/dist/core/main/runAutomationScript.d.ts +0 -1
  70. package/dist/core/main/runAutomationScript.js +17 -13
  71. package/dist/service/fireflink.service.d.ts +2 -4
  72. package/dist/service/fireflink.service.js +4 -5
  73. package/dist/service/kafka/fireflinkKafka.service.d.ts +2 -2
  74. package/dist/service/kafka/fireflinkKafka.service.js +17 -3
  75. package/dist/utils/logger/logData.d.ts +1 -0
  76. package/dist/utils/logger/logData.js +49 -22
  77. package/dist/utils/swipe/domSearchHelper.d.ts +5 -0
  78. package/dist/utils/swipe/domSearchHelper.js +42 -0
  79. package/dist/utils/swipe/swipeMovement.d.ts +1 -0
  80. package/dist/utils/swipe/swipeMovement.js +81 -0
  81. package/package.json +3 -4
  82. package/dist/tests/Framework.d.ts +0 -0
  83. package/dist/tests/Framework.js +0 -62
  84. package/dist/tests/test1.d.ts +0 -7
  85. package/dist/tests/test1.js +0 -13
  86. package/dist/tests/test12.js +0 -123
  87. package/dist/tests/test3.js +0 -24
  88. /package/dist/{tests/test12.d.ts → automation/actions/interface/elelementLessActionInterface.js} +0 -0
  89. /package/dist/{tests/test3.d.ts → automation/actions/interface/pinchActionInterface.js} +0 -0
@@ -4,7 +4,7 @@ export async function combinedActionExtractorPromptMob({ extractedDomJson, prior
4
4
  "MOB_InstallAPK","MOB_SetClipBoardText","MOB_PressAnyKeyNTimes",
5
5
  "MOB_SetContext","MOB_TerminateAppUsingAppPackage","MOB_OpenAppWithApkFilePath","MOB_ActivateAppUsingAppPackage",
6
6
  "MOB_ExecuteJavaScript","MOB_LongPress","MOB_Clear","MOB_EnterInputIntoElementFromClipBoard",
7
- "MOB_FindElement","MOB_FindElements","MOB_OpenNotification","MOB_PressEnterKey","MOB_ResetApp","MOB_HideKeyboard",
7
+ "MOB_FindElement","FindElements","MOB_OpenNotification","MOB_PressEnterKey","MOB_ResetApp","MOB_HideKeyboard",
8
8
  "MOB_SetEmulatorPowerStateOff","MOB_OpenChromeBrowser","MOB_SetDeviceOrientationAsPortrait","MOB_SetDeviceOrientationAsLandscape",
9
9
  "MOB_SetEmulatorPowerStateOn","MOB_AirplaneModeSwitchOn","MOB_LockDevice","MOB_WifiSwitchOff",
10
10
  "Continue","MOB_GpsSwitchOn","MOB_GpsSwitchOff","MOB_RestartCurrentDevice","MOB_AirplaneModeSwitchOff","Break","MOB_WifiSwitchOn",
@@ -38,8 +38,9 @@ Rules:
38
38
  - Extract keyword from the step, if the step contains any keyword.
39
39
  - Extract keycode of keys if the step contains any key name like ENTER,SPACE,BACKSPACE etc and give keycode of the given key.
40
40
  - Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
41
- - Based on step give most relevant type of element. use this list to choose elementType: ${elementType} and Never change syntax of elementType, follow the syntax of elementType in list.if elementType is not there in list return 'link'.
42
- - Respond with valid JSON only.
41
+ - Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
42
+ - If the steps is to find elements then provide nlpName as findElements and if the step is to find element then provide nlpName as MOB_FindElement.
43
+ - Respond with valid json only.
43
44
  `;
44
45
  return prompt;
45
46
  }
@@ -3,16 +3,15 @@ export async function buildErrorDescriptionPrompt() {
3
3
  Analyze the automation failure and execution history and generate a structured diagnostic response.
4
4
 
5
5
  SYSTEM CONTEXT:
6
- - The automation is executed by FireFlink.
7
6
  - The underlying automation framework must NEVER be mentioned.
8
- - The explanation must sound like an official FireFlink platform diagnostic message.
9
7
  - The tone must be professional, clear, and user-friendly.
10
8
  - Do NOT expose internal technical details.
11
9
 
12
10
  ERROR NORMALIZATION:
13
11
  - Convert the failure into the closest equivalent Selenium-style exception class name.
14
12
  - If it clearly matches a known Selenium exception, return that exact class name.
15
- - If it does NOT clearly match any known Selenium exception, return: "WebDriverException".
13
+ - If it does NOT clearly match any known Selenium exception, return valid selenium exception name.
14
+ - If The exception for an invalid URL is the 'InvalidArgumentException'
16
15
  - Never return framework-specific error names.
17
16
  - Never include stack traces or extra text in the "error" field.
18
17
 
@@ -28,18 +27,17 @@ STRICT OUTPUT RULES:
28
27
  Return a JSON object with EXACTLY these keys:
29
28
 
30
29
  1. "error":
30
+ - A concise, single-line, selenium exception name or appium exception name (error).__name__ (e.g., "InvalidSessionIdException", "NoSuchElementException", "WebDriverException" etc) expamle: if error is element not found, return "NoSuchElementException"
31
31
  - Must contain ONLY a single Selenium-style exception class name.
32
32
  - Do NOT include additional words.
33
33
 
34
34
  2. "errorDescription":
35
- - May contain one or more sentences separated by newline characters (\n).
36
- - The tone must clearly reflect FireFlink as the acting system.
37
- - Use simple but sufficiently detailed language.
38
- - Do NOT mention Selenium, WebDriver, drivers, selectors, DOM, stack traces, or implementation details.
39
- - Do NOT use low-level debugging terminology.
40
- - Provide realistic and logically consistent possible causes based on the identified exception type.
41
- - Avoid repetition.
42
- - Avoid unrelated environment assumptions unless clearly supported by the failure context.
35
+ - Use simple, natural, human-friendly language.
36
+ - Do NOT mention system, Selenium, WebDriver, drivers, selectors, DOM, stack traces, or any technical/internal terms.
37
+ - Avoid words like "system", "framework", or "process".
38
+ - Describe the issue as a general situation, not from a system perspective.
39
+ - Provide realistic possible causes based on the exception type.
40
+ - Keep it concise and non-repetitive.
43
41
 
44
42
  Required JSON format:
45
43
 
@@ -133,12 +133,12 @@ Rules:
133
133
  - **Respond with valid JSON only. don't return any other text or don't return response in list format**
134
134
  `;
135
135
  }
136
- // ---------------- DEFAULT (CLICK / ENTER / UPLOAD etc.) ----------------
136
+ // ---------------- DEFAULT (CLICK / cleartext etc.) ----------------
137
137
  else if (stepAction === "click" || stepAction === "cleartext") {
138
138
  prompt = `
139
139
 
140
140
  -You are an intelligent assistant that extracts structured UI action data.
141
- -Given a structured UI JSON representation with uniquely identified elements (ff-inspect values like Fire-Flink-1, Fire-Flink-2, Fire-Flink-3... in DOM order),
141
+ -Given a structured UI JSON representation with uniquely identified elements (ff-inspect values like Fire-Flink-0, Fire-Flink-1, Fire-Flink-2, Fire-Flink-3... in DOM order),
142
142
  -locate the most appropriate element for an automation step by performing keyword-based matching using exact, partial, and fuzzy strategies, and
143
143
  return the identifier of the best match.
144
144
 
@@ -153,7 +153,7 @@ Return **only valid JSON** in the following format:
153
153
  Rules:
154
154
  You are an AI assistant. For the step, extract the element keyword from the step (the field name like 'Leaving from', 'Going To').
155
155
  - Use context from the ${JSON.stringify(priorAndNextSteps)}, keyword and filtered dom to search for FF-inspecter.
156
- - Simplified JSON: ${extractedDomJson} if this simplified json is empty return Fire-Flink-0 never return Fire-Flink-1 as attribute_value.
156
+ - Simplified JSON: ${extractedDomJson} if it is empty, null, or has no valid data, return strictly "attribute_value":"Fire-Flink-0" and never use any other Fire-Flink-x value.
157
157
  - **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the atep.**
158
158
  - If multiple matches exist, use the reference element and choose the closest match by ff-inspect distance.
159
159
  **important example: step: "click on login below user login"
@@ -176,12 +176,12 @@ You are an AI assistant. For the step, extract the element keyword from the step
176
176
 
177
177
  `;
178
178
  }
179
- // ---------------- DEFAULT (CLICK / ENTER / UPLOAD etc.) ----------------
179
+ // ---------------- DEFAULT (ENTER / UPLOAD etc.) ----------------
180
180
  else {
181
181
  prompt = `
182
182
 
183
183
  -You are an intelligent assistant that extracts structured UI action data.
184
- -Given a structured UI JSON representation with uniquely identified elements (ff-inspect values like Fire-Flink-1, Fire-Flink-2, Fire-Flink-3... in DOM order),
184
+ -Given a structured UI JSON representation with uniquely identified elements (ff-inspect values like Fire-Flink-0, Fire-Flink-1, Fire-Flink-2, Fire-Flink-3... in DOM order),
185
185
  -locate the most appropriate element for an automation step by performing keyword-based matching using exact, partial, and fuzzy strategies, and
186
186
  return the identifier of the best match.
187
187
 
@@ -196,7 +196,7 @@ Return **only valid JSON** in the following format:
196
196
  Rules:
197
197
  You are an AI assistant. For the step, extract the element keyword from the step (the field name like 'Leaving from', 'Going To').
198
198
  - Use context from the ${JSON.stringify(priorAndNextSteps)}, keyword and filtered dom to search for FF-inspecter.
199
- - Simplified JSON: ${extractedDomJson} if this simplified json is empty return Fire-Flink-0 never return Fire-Flink-1 as attribute_value.
199
+ - Simplified JSON: ${extractedDomJson} if it is empty, null, or has no valid data, return strictly "attribute_value":"Fire-Flink-0" and never use any other Fire-Flink-x value.
200
200
  - Select the closest semantic match to step and return only its attribute_value, else Fire-Flink-0.
201
201
  - **Find the FF-inspecter attribute value of the element in the Simplified JSON whose text or attributes best match the atep.**
202
202
  - If multiple matches exist, use the reference element and choose the closest match by ff-inspect distance.
@@ -208,7 +208,7 @@ login - ff-inspect-803
208
208
  then should return login - ff-inspect-803 witch is near to user reference element user login - ff-inspect-790
209
209
  - Respect direction (up/down), default is down.
210
210
  - Do not return elements far away from the reference.
211
- - Select the closest semantic match to step and return only its attribute_valu
211
+ - Select the closest semantic match to step and return only its attribute_value
212
212
  - Action:SendKeys for entering text, EnterDataAndPressKey for entering data and pressing the key, clearThenEnterInput for clear and Entering Input, 'upload' for uploading file.
213
213
  - Action:EnterDataAndPressKey if step is about entering data and pressing a key, then select this action.
214
214
  - Action:EnterInputUsingJS if step is about entering input using js,WaitAndSendKeys for waiting for some time and then entring.
@@ -1,4 +1,5 @@
1
1
  export async function ffInspectorNumExtractorMob({ stepAction, extractedDomJson, priorAndNextSteps, }) {
2
+ const SwipeActions = ["MOB_SwipeToElement", "MOB_SwipeUpToElement", "MOB_SwipeDownToElement", "MOB_SwipeLeftToElement", "MOB_SwipeRightToElement", "MOB_SwipeNTimes", "MOB_Builtin_Swipe", "MOB_SwipeDirectionNTimes", "MOB_SwipeToElementForMWeb", "MOB_SwipeUsingReferenceElement"];
2
3
  const Actions = ["MOB_Tap", "MOB_TapNTimes", "MOB_DoubleTap", "MOB_DoubleTapAtSpecifiedLocation", "MOB_TapOnSpecifiedLocation",
3
4
  "MOB_Enter", "MOB_EnterDataAndPressKey", "MOB_EnterInputIntoElementFromClipBoard", "MOB_EnterUrl", "MOB_ClearThenEnterInput",
4
5
  "MOB_LongPress", "MOB_PressEnterKey", "MOB_PressSpaceKey", "MOB_PressBackSpaceKey", "MOB_PressAnyKey", "MOB_PressAnyKeyNTimes", "MOB_EnterDataAndPressKey", "MOB_PressHomeKey", "MOB_PressBackKey"];
@@ -45,6 +46,44 @@ You are an AI assistant. For the step, extract the element keyword from the step
45
46
 
46
47
  filtered dom JSON: ${extractedDomJson}
47
48
  `;
49
+ }
50
+ else if (stepAction === "swipe") {
51
+ prompt = `
52
+ You are an intelligent assistant that extracts structured mobile scroll/swipe action data.
53
+ Given the step: ${JSON.stringify(priorAndNextSteps)}, the simplified DOM (for scroll context): ${extractedDomJson}, and the list of NLP names: ${SwipeActions}.
54
+
55
+ Select the perfect matching NLP name from the list that best fits the step's intent.
56
+
57
+ Return **only valid JSON** in the following format:
58
+ {{
59
+ "attribute_value": "Fire-Flink-x",
60
+ "nlpName": "x",
61
+ "input_text": "x",
62
+ "keyword": "x",
63
+ "type": "x",
64
+ "num_of_scrolls": "x",
65
+ "direction": "x"
66
+ }}
67
+
68
+ Rules:
69
+ - nlpName must be exactly one from the provided list.
70
+ - Use cont2ext from the steps : ${JSON.stringify(priorAndNextSteps)}, keyword and json to search for FF-inspecter.
71
+ - If direction is not specified, then match to "MOB_SwipeToElement".
72
+ - If direction is Not specified and element is not specified, then match to "MOB_SwipeNTimes".
73
+ - If direction is specified and element is not specified, then match to "MOB_SwipeDirectionNTimes".
74
+ - If step has a reference element, then match to "MOB_SwipeUsingReferenceElement". and give input_text: "reference element -> <next element>"
75
+ example: swipe up to "Contact Us" using "Settings" as reference.
76
+ input_text: "Settings -> Contact Us"
77
+ - If the step is saying 'up to' or 'till' element, then return the number of scrolls as 0. and also don't take 'up' as direction in this case, give direction as "none" ".
78
+ - **Extraction**:
79
+ - input_text: The target element in the step.e.g., "swipe to submit text",then input text must be "submit",Dont include its type in input_text like text,button,icon etc...Leave empty if just swiping N times.**
80
+ - keyword: Crucial keyword from step (e.g., "Settings").
81
+ - num_of_scrolls: Extract count if specified (e.g., "scroll 3 times"). Default to 0 if not specified but action implies generic scroll.
82
+ - direction: "up", "down", "left", "right". Infer from step (e.g., "Scroll down" -> "down"). If direction is not specified, default direction to "none".
83
+ - attribute_value: If an element is visible and matches the target, provide its FF-inspecter. Otherwise "Fire-Flink-2".
84
+ - type: Based on step give most relevant type of element. use this list to choose element_type: ${elementType} and Never change syntax of element_type, follow the syntax of element_type in list.if element_type is not there in list return 'link'.
85
+
86
+ - Respond with valid JSON only.`;
48
87
  }
49
88
  return prompt;
50
89
  }
@@ -20,7 +20,8 @@ If they mention package:com.app.android and activity:com.app.activity remove var
20
20
  -** element_name: extract name of the element that mentioned in the step not from keywords or other steps.(eg:tap on x -> element_name:x). always try to retuen short and meaning full element name from step**
21
21
  - action: openApp for opening or launching of app, tap for taping or selecting or clicking or pressing, enter for entering input, wait for waiting or sleeping, verify for verifying or checking,scroll for scrolling and swiping, get for getting,fetching element, closeApp for closing the app.
22
22
  - If step is press any key give action as tap.
23
- - action must be one of from this list ${allowedActions}.if not one of them, return action as 'combined'. if step about set or find or open chrome browser or open app with apk path or install apk or uninstall apk or activate or terminate action return action as 'combined'
23
+ - action must be one of from this list ${allowedActions}.if not one of them, return action as 'combined'. if step about set or find or open chrome browser or open app with apk path or install apk or uninstall apk or activate or terminate action pinch in or pinch out and running app in the backgroundaction return action as 'combined'
24
+ - if the step action is about finding then provide action as combined.
24
25
  Respond only with JSON using this format:
25
26
  {
26
27
  "keywords": [key1,key2,key3,key4,key5] if step is opening app then keywords should be like [app package,app activity],
@@ -8,7 +8,7 @@ export async function verifyActionExtractorPromptMob({ extractedDomJson, priorAn
8
8
  "MOB_VerifyIfAirplaneModeOff,MOB_VerifyIfGpsOff,MOB_VerifyHexCodeForGivenXYCoordinates,"
9
9
  "MOB_VerifyYLocationOfAndroidElement,MOB_VerifyIfGpsOn,MOB_VerifyIfAirplaneModeOn,"
10
10
  "MOB_VerifyWidthOfElement,MOB_VerifyElementIsNotDisplayed,MOB_VerifyElementNotContainsText,"
11
- "MOB_VerifyIfAppIsClosed"`;
11
+ "MOB_VerifyIfAppIsClosed, MOB_CheckIfElementIsDisplayed"`;
12
12
  const elementType = ["link", "textfield", "icon", "button", "radiobutton", "checkbox", "tab", "action overflow button", "hamburger menu", "toggle button", "steppers", "sliders"];
13
13
  const prompt = `You are an intelligent assistant that extracts structured UI action data.
14
14
  Given the step: step, the simplified DOM: ${extractedDomJson}, and the list of NLP names: ${nlpList}.
@@ -37,6 +37,7 @@ Rules:
37
37
  - If step mention verify text then only pick MOB_VerifyText nlp otherwise pick MOB_VerifyElementIsDisplayed.
38
38
  - If step mention verify x and y coordinate of element then only pick MOB_VerifyIfXAndYCoordinateOfElement.
39
39
  - Extract input_text from the step that is which is being verified in the {step}, if you can't find any input text in the step, return keyword as input_text.
40
+ - if the step is about checking the element is displayed then pick MOB_CheckIfElementIsDisplayed nlp.
40
41
  - For MOB_VerifyHeightOfElement just provide height mentioned in the step as the input_text don't add anything else
41
42
  - Extract keyword from the step, if the step contains any keyword.
42
43
  - Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
@@ -3,10 +3,11 @@ export async function visionPrompt({ priorAndNextSteps }) {
3
3
  You are a precise automation assistant.
4
4
  Context: ${priorAndNextSteps}
5
5
  - Use the context to understand the step and select the closest semantic match to Step and return only its index, else Fire-Flink-0.
6
- - **If u can't find the element, return Fire-Flink-0. dont return any other elements index.**
6
+ - **If u can't find the element and also If there is no element with number, return Fire-Flink-0. dont return any other elements index.**
7
7
  - Using the provided annotated screenshot and Step, identify the correct element index from the image.
8
8
  - Match the index to the text or action described in Step as accurately as possible.
9
- - Return only the Fire-Flink-x for the target element.
9
+ - Return only the Fire-Flink-x for the target element .
10
+ - **Never return Fire-Flink-1 only return Fire-Flink-0.**
10
11
  - Do not provide any additional text or explanation. The output must be strictly Fire-Flink-x.
11
12
  Respond only with JSON using this format:
12
13
  {fireflinkIndex: Fire-Flink-x | Fire-Flink-0}
@@ -46,6 +46,7 @@ Rules:
46
46
  - For some steps you can see two inputs but you need to uderstand step and send the requred input_text.
47
47
  ex: Wait until the flights has the flight text
48
48
  here keyword : "flights" , input : flight
49
+ - If nlpName is "WaitTillPresenceOfAllElements", the input_text MUST be formatted exactly as "xpath, [THE XPATH]". Do not return plain text.
49
50
  - Extract input_text from the step, if you can't find any input text in the step, return keyword as input_text.
50
51
  - Extract keyword from the step, if the step contains any keyword.
51
52
  - Use the closest semantic match for the step; return attribute_value as Fire-Flink-0, only if nothing is found.
@@ -26,6 +26,7 @@ Rules:
26
26
  - For some steps you can see two inputs but you need to uderstand step and send the requred input_text.
27
27
  ex: Wait until the flights has the flight text
28
28
  here keyword : "flights" , input : flight
29
+ - If nlpName is "MOB_WaitTillPresenceOfAllElements", the input_text MUST be formatted exactly as "xpath, [THE XPATH]". Do not return plain text.
29
30
  - **If the step has input in a string with comma take that string as it is in input_text.**
30
31
  - Extract input_text from the step, if you can't find any input text in the step, return keyword as input_text.
31
32
  - Extract keyword from the step, if the step contains any keyword.
@@ -247,13 +247,16 @@ export declare class ActionExecutor implements IActionExecutor {
247
247
  VerifyIfImageIsBroken(value: string): Promise<void>;
248
248
  VerifyAllBrokenImages(): Promise<void>;
249
249
  MOB_PressBackKey(): Promise<void>;
250
- WaitTillPresenceOfElement(pageDOM: string, selector: string, fireflinkIndex: string, elementName: string, elementType: string): Promise<void>;
250
+ MOB_PressBackSpaceKey(): Promise<void>;
251
+ MOB_PressSpaceKey(): Promise<void>;
252
+ WaitTillPresenceOfElement(pageDOM: string, selector: string, value: string, fireflinkIndex: string, elementName: string, elementType: string): Promise<void>;
251
253
  WaitTillPresenceOfAllElements(pageDOM: string, selector: string, value: string, fireflinkIndex: string, elementName: string, elementType: string): Promise<void>;
252
254
  WaitTillAttributeOfElementIsString(pageDOM: string, selector: string, value: string, fireflinkIndex: string, elementName: string, elementType: string): Promise<void>;
253
255
  EnterUrl(value: string): Promise<void>;
254
256
  SetClipBoardText(value: string): Promise<void>;
255
257
  PressAnyKeyNTimes(value: string): Promise<void>;
256
258
  FindElements(pageDOM: string, selector: string, value: string, fireflinkIndex: string, elementName: string, elementType: string): Promise<void>;
259
+ MOB_FindElement(selector: string, elementName: string, elementType: string): Promise<void>;
257
260
  WaitTillElementIsClickable(pageDOM: string, selector: string, fireflinkIndex: string, elementName: string, elementType: string): Promise<void>;
258
261
  CheckIfElementIsDisplayed(pageDOM: string, selector: string, value: string, fireflinkIndex: string, elementName: string, elementType: string): Promise<void>;
259
262
  PressAnyKey(value: string): Promise<void>;
@@ -343,4 +346,19 @@ export declare class ActionExecutor implements IActionExecutor {
343
346
  MOB_WaitTillAttributeOfElementContainsString(selector: string, elementName: string, attributeName: string, elementType: string, value: string): Promise<void>;
344
347
  MOB_SetContext(value: string): Promise<void>;
345
348
  MOB_SetEmulatorPowerPercentage(value: string): Promise<void>;
349
+ swipeToElement(value: string, elementName: string, elementType: string, num_of_scrolls: number): Promise<void>;
350
+ swipeUpToElement(value: string, elementName: string, elementType: string, num_of_scrolls?: number): Promise<void>;
351
+ swipeDownToElement(value: string, elementName: string, elementType: string, num_of_scrolls?: number): Promise<void>;
352
+ swipeLeftToElement(value: string, elementName: string, elementType: string, num_of_scrolls: number): Promise<void>;
353
+ swipeRightToElement(value: string, elementName: string, elementType: string, num_of_scrolls?: number): Promise<void>;
354
+ swipeNTimes(num_of_scrolls: string, direction?: string): Promise<void>;
355
+ swipeDirectionNTimes(num_of_scrolls: string, direction: string): Promise<void>;
356
+ swipeToElementForMWeb(value: string, elementName: string, elementType: string, num_of_scrolls?: number): Promise<void>;
357
+ swipeUsingReferenceElement(value: string, elementName: string, elementType: string, num_of_scrolls: string, direction: string): Promise<void>;
358
+ MOB_EnterInputIntoElementFromClipBoard(selector: string, elementName: string, elementType: string): Promise<void>;
359
+ MOB_CheckIfAppIsClosed(): Promise<void>;
360
+ MOB_RunAppInBackground(value: string): Promise<void>;
361
+ MOB_PinchInByPercent(selector: string, value: string, elementName: string, elementType: string): Promise<void>;
362
+ MOB_PinchOutByPercent(selector: string, value: string, elementName: string, elementType: string): Promise<void>;
363
+ MOB_DoubleTapAtSpecifiedLocation(value: string, elementName: string, elementType: string): Promise<void>;
346
364
  }
@@ -266,6 +266,19 @@ import { TerminateAppUsingAppPackage } from "./interaction/elementlessActions/te
266
266
  import { OpenAppWithApkFilePath } from "./interaction/elementlessActions/openAppWithApkFilePath.js";
267
267
  import { ActivateAppUsingAppPackage } from "./interaction/elementlessActions/activateAppUsingAppPackage.js";
268
268
  import { LongPress } from "./interaction/press/longPress.js";
269
+ import { swipeUpToElement } from "./interaction/swipe/swipeUpToElement.js";
270
+ import { swipeDownToElement } from "./interaction/swipe/swipeDownToElement.js";
271
+ import { swipeLeftToElement } from "./interaction/swipe/swipeLeftToElement.js";
272
+ import { swipeRightToElement } from "./interaction/swipe/swipeRightToElement.js";
273
+ import { swipeDirectionNTimes } from "./interaction/swipe/swipeDirectionNTimes.js";
274
+ import { swipeUsingReferenceElement } from "./interaction/swipe/swipeUsingReferenceElement.js";
275
+ import { swipeUpToElementForMWeb } from "./interaction/swipe/swipeUpToElementForMWeb.js";
276
+ import { MOB_EnterInputIntoElementFromClipBoard } from "./interaction/enterActions/EnterInputIntoElementFromClipBoardInterface_mob.js";
277
+ import { MOB_RunAppInBackground } from "./interaction/elementLessAction/MobRunAppInBackgroundInterface.js";
278
+ import { MOB_PinchInByPercent } from "./interaction/pinch/PinchInByPercentMob.js";
279
+ import { MOB_PinchOutByPercent } from "./interaction/pinch/PinchOutByPercentMob.js";
280
+ import { MOB_DoubleTapAtSpecifiedLocation } from "./interaction/click/MOB_DoubleTapAtSpecifiedLocation.js";
281
+ import { MOB_CheckIfAppIsClosed } from "./interaction/elementLessAction/MobCheckIfAppIsClosed.js";
269
282
  import { pressBackKey } from "./interaction/press/mobPressBackKey.js";
270
283
  import { waitTillPresenceOfElement } from "./interaction/wait/waitTillPresenceOfElement.js";
271
284
  import { waitTillPresenceOfAllElements } from "./interaction/wait/waitTillPresenceOfAllElements.js";
@@ -334,6 +347,9 @@ import { MOB_WaitTillStalenessOfElement } from "./interaction/wait/MOB_WaitTillS
334
347
  import { MOB_WaitTillElementIsInvisible } from "./interaction/wait/MOB_WaitTillElementIsInvisible.js";
335
348
  import { MOB_SetContext } from "./interaction/set/MOB_SetContext.js";
336
349
  import { MOB_SetEmulatorPowerPercentage } from "./interaction/set/MOB_SetEmulatorPowerPercentage.js";
350
+ import { pressBackSpaceKey } from "./interaction/press/mobPressBackSpaceKey.js";
351
+ import { pressSpaceKey } from "./interaction/press/mobPressSpaceKey.js";
352
+ import { MOB_FindElement } from "./interaction/find/MOB_FindElement.js";
337
353
  export class ActionExecutor {
338
354
  constructor(driver, scriptDataAppender, elementGetter, platform) {
339
355
  this.driver = driver;
@@ -4181,12 +4197,39 @@ export class ActionExecutor {
4181
4197
  throw error;
4182
4198
  }
4183
4199
  }
4184
- async WaitTillPresenceOfElement(pageDOM, selector, fireflinkIndex, elementName, elementType) {
4200
+ async MOB_PressBackSpaceKey() {
4201
+ try {
4202
+ await pressBackSpaceKey({
4203
+ driver: this.driver,
4204
+ scriptDataAppender: this.scriptDataAppender,
4205
+ platform: this.platform
4206
+ });
4207
+ }
4208
+ catch (error) {
4209
+ logger.error("Error in [ActionExecutor]", error);
4210
+ throw error;
4211
+ }
4212
+ }
4213
+ async MOB_PressSpaceKey() {
4214
+ try {
4215
+ await pressSpaceKey({
4216
+ driver: this.driver,
4217
+ scriptDataAppender: this.scriptDataAppender,
4218
+ platform: this.platform
4219
+ });
4220
+ }
4221
+ catch (error) {
4222
+ logger.error("Error in [ActionExecutor]", error);
4223
+ throw error;
4224
+ }
4225
+ }
4226
+ async WaitTillPresenceOfElement(pageDOM, selector, value, fireflinkIndex, elementName, elementType) {
4185
4227
  try {
4186
4228
  await waitTillPresenceOfElement({
4187
4229
  driver: this.driver,
4188
4230
  pageDOM,
4189
4231
  selector,
4232
+ value,
4190
4233
  fireflinkIndex,
4191
4234
  elementName,
4192
4235
  elementType,
@@ -4302,6 +4345,22 @@ export class ActionExecutor {
4302
4345
  throw error;
4303
4346
  }
4304
4347
  }
4348
+ async MOB_FindElement(selector, elementName, elementType) {
4349
+ try {
4350
+ await MOB_FindElement({
4351
+ driver: this.driver,
4352
+ selector,
4353
+ elementName,
4354
+ elementType,
4355
+ scriptDataAppender: this.scriptDataAppender,
4356
+ platform: this.platform,
4357
+ });
4358
+ }
4359
+ catch (error) {
4360
+ logger.error("Error in [ActionExecutor]", error);
4361
+ throw error;
4362
+ }
4363
+ }
4305
4364
  async WaitTillElementIsClickable(pageDOM, selector, fireflinkIndex, elementName, elementType) {
4306
4365
  try {
4307
4366
  await waitTillElementIsClickable({
@@ -5532,4 +5591,247 @@ export class ActionExecutor {
5532
5591
  throw error;
5533
5592
  }
5534
5593
  }
5594
+ async swipeToElement(value, elementName, elementType, num_of_scrolls) {
5595
+ try {
5596
+ await swipeUpToElement({
5597
+ driver: this.driver,
5598
+ value: value,
5599
+ scriptDataAppender: this.scriptDataAppender,
5600
+ elementName,
5601
+ elementType,
5602
+ num_of_scrolls,
5603
+ platform: this.platform
5604
+ });
5605
+ }
5606
+ catch (error) {
5607
+ logger.error("Error in [ActionExecutor]", error);
5608
+ throw error;
5609
+ }
5610
+ }
5611
+ async swipeUpToElement(value, elementName, elementType, num_of_scrolls = 1) {
5612
+ try {
5613
+ await swipeUpToElement({
5614
+ driver: this.driver,
5615
+ value: value,
5616
+ scriptDataAppender: this.scriptDataAppender,
5617
+ elementName,
5618
+ elementType,
5619
+ num_of_scrolls,
5620
+ platform: this.platform
5621
+ });
5622
+ }
5623
+ catch (error) {
5624
+ logger.error("Error in [ActionExecutor]", error);
5625
+ throw error;
5626
+ }
5627
+ }
5628
+ async swipeDownToElement(value, elementName, elementType, num_of_scrolls = 1) {
5629
+ try {
5630
+ await swipeDownToElement({
5631
+ driver: this.driver,
5632
+ value,
5633
+ scriptDataAppender: this.scriptDataAppender,
5634
+ elementName,
5635
+ elementType,
5636
+ num_of_scrolls,
5637
+ platform: this.platform
5638
+ });
5639
+ }
5640
+ catch (error) {
5641
+ logger.error("Error in [ActionExecutor]", error);
5642
+ throw error;
5643
+ }
5644
+ }
5645
+ async swipeLeftToElement(value, elementName, elementType, num_of_scrolls) {
5646
+ try {
5647
+ await swipeLeftToElement({
5648
+ driver: this.driver,
5649
+ value,
5650
+ scriptDataAppender: this.scriptDataAppender,
5651
+ elementName,
5652
+ elementType,
5653
+ num_of_scrolls,
5654
+ platform: this.platform
5655
+ });
5656
+ }
5657
+ catch (error) {
5658
+ logger.error("Error in [ActionExecutor]", error);
5659
+ throw error;
5660
+ }
5661
+ }
5662
+ async swipeRightToElement(value, elementName, elementType, num_of_scrolls = 1) {
5663
+ try {
5664
+ await swipeRightToElement({
5665
+ driver: this.driver,
5666
+ value,
5667
+ scriptDataAppender: this.scriptDataAppender,
5668
+ elementName,
5669
+ elementType,
5670
+ num_of_scrolls,
5671
+ platform: this.platform
5672
+ });
5673
+ }
5674
+ catch (error) {
5675
+ logger.error("Error in [ActionExecutor]", error);
5676
+ throw error;
5677
+ }
5678
+ }
5679
+ async swipeNTimes(num_of_scrolls, direction = 'down') {
5680
+ try {
5681
+ await swipeDirectionNTimes({
5682
+ driver: this.driver,
5683
+ num_of_scrolls,
5684
+ direction,
5685
+ scriptDataAppender: this.scriptDataAppender,
5686
+ platform: this.platform
5687
+ });
5688
+ }
5689
+ catch (error) {
5690
+ logger.error("Error in [ActionExecutor]", error);
5691
+ throw error;
5692
+ }
5693
+ }
5694
+ async swipeDirectionNTimes(num_of_scrolls, direction) {
5695
+ try {
5696
+ await swipeDirectionNTimes({
5697
+ driver: this.driver,
5698
+ num_of_scrolls,
5699
+ direction,
5700
+ scriptDataAppender: this.scriptDataAppender,
5701
+ platform: this.platform
5702
+ });
5703
+ }
5704
+ catch (error) {
5705
+ logger.error("Error in [ActionExecutor]", error);
5706
+ throw error;
5707
+ }
5708
+ }
5709
+ async swipeToElementForMWeb(value, elementName, elementType, num_of_scrolls = 1) {
5710
+ try {
5711
+ await swipeUpToElementForMWeb({
5712
+ driver: this.driver,
5713
+ value: value,
5714
+ scriptDataAppender: this.scriptDataAppender,
5715
+ elementName,
5716
+ elementType,
5717
+ num_of_scrolls,
5718
+ platform: this.platform
5719
+ });
5720
+ }
5721
+ catch (error) {
5722
+ logger.error("Error in [ActionExecutor]", error);
5723
+ throw error;
5724
+ }
5725
+ }
5726
+ async swipeUsingReferenceElement(value, elementName, elementType, num_of_scrolls, direction) {
5727
+ try {
5728
+ await swipeUsingReferenceElement({
5729
+ driver: this.driver,
5730
+ value: value,
5731
+ scriptDataAppender: this.scriptDataAppender,
5732
+ elementName,
5733
+ elementType,
5734
+ num_of_scrolls,
5735
+ direction,
5736
+ platform: this.platform
5737
+ });
5738
+ }
5739
+ catch (error) {
5740
+ logger.error("Error in [ActionExecutor]", error);
5741
+ throw error;
5742
+ }
5743
+ }
5744
+ async MOB_EnterInputIntoElementFromClipBoard(selector, elementName, elementType) {
5745
+ try {
5746
+ await MOB_EnterInputIntoElementFromClipBoard({
5747
+ driver: this.driver,
5748
+ selector,
5749
+ scriptDataAppender: this.scriptDataAppender,
5750
+ elementName,
5751
+ elementType,
5752
+ platform: this.platform
5753
+ });
5754
+ }
5755
+ catch (error) {
5756
+ logger.error("Error in [ActionExecutor]", error);
5757
+ throw error;
5758
+ }
5759
+ }
5760
+ async MOB_CheckIfAppIsClosed() {
5761
+ try {
5762
+ await MOB_CheckIfAppIsClosed({
5763
+ driver: this.driver,
5764
+ scriptDataAppender: this.scriptDataAppender,
5765
+ platform: this.platform
5766
+ });
5767
+ }
5768
+ catch (error) {
5769
+ logger.error("Error in [ActionExecutor]", error);
5770
+ throw error;
5771
+ }
5772
+ }
5773
+ async MOB_RunAppInBackground(value) {
5774
+ try {
5775
+ await MOB_RunAppInBackground({
5776
+ driver: this.driver,
5777
+ value,
5778
+ scriptDataAppender: this.scriptDataAppender,
5779
+ platform: this.platform
5780
+ });
5781
+ }
5782
+ catch (error) {
5783
+ logger.error("Error in [ActionExecutor]", error);
5784
+ throw error;
5785
+ }
5786
+ }
5787
+ async MOB_PinchInByPercent(selector, value, elementName, elementType) {
5788
+ try {
5789
+ await MOB_PinchInByPercent({
5790
+ driver: this.driver,
5791
+ selector,
5792
+ value,
5793
+ scriptDataAppender: this.scriptDataAppender,
5794
+ elementName,
5795
+ elementType,
5796
+ platform: this.platform
5797
+ });
5798
+ }
5799
+ catch (error) {
5800
+ logger.error("Error in [ActionExecutor]", error);
5801
+ throw error;
5802
+ }
5803
+ }
5804
+ async MOB_PinchOutByPercent(selector, value, elementName, elementType) {
5805
+ try {
5806
+ await MOB_PinchOutByPercent({
5807
+ driver: this.driver,
5808
+ selector,
5809
+ value,
5810
+ scriptDataAppender: this.scriptDataAppender,
5811
+ elementName,
5812
+ elementType,
5813
+ platform: this.platform
5814
+ });
5815
+ }
5816
+ catch (error) {
5817
+ logger.error("Error in [ActionExecutor]", error);
5818
+ throw error;
5819
+ }
5820
+ }
5821
+ async MOB_DoubleTapAtSpecifiedLocation(value, elementName, elementType) {
5822
+ try {
5823
+ await MOB_DoubleTapAtSpecifiedLocation({
5824
+ driver: this.driver,
5825
+ value,
5826
+ scriptDataAppender: this.scriptDataAppender,
5827
+ elementName,
5828
+ elementType,
5829
+ platform: this.platform
5830
+ });
5831
+ }
5832
+ catch (error) {
5833
+ logger.error("Error in [ActionExecutor]", error);
5834
+ throw error;
5835
+ }
5836
+ }
5535
5837
  }
@@ -0,0 +1,2 @@
1
+ import { MOB_DoubleTapAtSpecifiedLocation } from "../../interface/clickActionInterface.js";
2
+ export declare function MOB_DoubleTapAtSpecifiedLocation(args: MOB_DoubleTapAtSpecifiedLocation): Promise<void>;
@@ -0,0 +1,26 @@
1
+ export async function MOB_DoubleTapAtSpecifiedLocation(args) {
2
+ try {
3
+ const input = args.value || "";
4
+ const match = input.match(/(\d+)[,\s]+(\d+)/);
5
+ if (!match) {
6
+ throw new Error(`Could not extract coordinates from: ${input}`);
7
+ }
8
+ const x = Number(match[1]);
9
+ const y = Number(match[2]);
10
+ await args.driver.execute('mobile: doubleClickGesture', {
11
+ x: x,
12
+ y: y
13
+ });
14
+ args.scriptDataAppender.add(async () => {
15
+ return {
16
+ nlpName: 'MOB_DoubleTapAtSpecifiedLocation',
17
+ elementsData: [],
18
+ stepInputs: [x, y],
19
+ };
20
+ });
21
+ }
22
+ catch (error) {
23
+ args.scriptDataAppender.setErrorNLP("MOB_DoubleTapAtSpecifiedLocation");
24
+ throw new Error("MOB_DoubleTapAtSpecifiedLocation failed", { cause: error });
25
+ }
26
+ }