ff-automationv2 2.2.35 → 2.2.36-beta.2

package/dist/core/main/runAutomationScript.js

@@ -32,6 +32,7 @@ const stepActionInspector_js_1 = require("../helpers/stepActionInspector.js");
 const elementSelectorResolver_js_1 = require("../helpers/elementSelectorResolver.js");
 const runnerPostActionHelper_js_1 = require("../helpers/runnerPostActionHelper.js");
 const visionFallbackHandler_js_1 = require("../helpers/visionFallbackHandler.js");
+const mobileSwipeHelper_js_1 = require("../helpers/mobileSwipeHelper.js");
 class AutomationRunner {
     constructor() {
         this.KafkaPayload = {};
@@ -588,6 +589,48 @@ class AutomationRunner {
                                 stepResult = retryActionResolution.stepResult;
                                 stepAction = retryActionResolution.stepAction;
                             }
+                            const swipeVisionResult = await (0, mobileSwipeHelper_js_1.checkMobileSwipeSlider)({
+                                driver,
+                                currentPlatform,
+                                stepAction,
+                                step,
+                                domInfo,
+                                doc,
+                                stepProcessor,
+                                priorAndNextSteps,
+                                fallbackAttempt,
+                                driverContext,
+                            });
+                            domInfo = swipeVisionResult.domInfo;
+                            doc = swipeVisionResult.doc;
+                            if (swipeVisionResult.isSlider) {
+                                handler = actionHandlers["MOB_SwipeSlider"];
+                                await handler({
+                                    parentXpath: swipeVisionResult.parentXpath,
+                                    targetXpath: swipeVisionResult.targetXpath,
+                                    xpath: swipeVisionResult.targetXpath,
+                                    selector: swipeVisionResult.targetXpath,
+                                    parentIndex: swipeVisionResult.parentIndex,
+                                    targetIndex: swipeVisionResult.targetIndex,
+                                    fireflinkIndex: swipeVisionResult.targetIndex,
+                                    direction: swipeVisionResult.direction,
+                                    currentStep: step,
+                                    elementName: currentElementName || result.response.elementName,
+                                    elementType: stepResult.response.elementType || stepResult.response.type,
+                                    pageDOM: domInfo.dom,
+                                    currentPlatform,
+                                    driver,
+                                    driverContext,
+                                    priorAndNextSteps,
+                                    stepCount,
+                                    stepIndex: currentStepIndex,
+                                    manualStep: step,
+                                    stepsInputData,
+                                    returnTypeVariableName: currentReturnTypeVariableName,
+                                    ...(currentVariableName ? { variableName: currentVariableName } : {}),
+                                });
+                                break;
+                            }
                             if (stepAction === "Builtin_DragAndDropToElement") {
                                 handler = actionHandlers[stepAction];
                                 const fireflinkIndex1 = stepResult.response.drag;
@@ -647,7 +690,7 @@ class AutomationRunner {
                                 fireflinkIndex,
                                 pageDOM,
                                 elementName: currentElementName || result.response.elementName,
-                                elementType: stepResult.response.elementType,
+                                elementType: stepResult.response.elementType || stepResult.response.type,
                                 numOfScrolls: stepResult.response.numOfScrolls,
                                 direction: stepResult.response.direction,
                                 stepIndex: currentStepIndex,

package/dist/core/types/promptType.d.ts

@@ -5,6 +5,7 @@ export declare enum PromptType {
     ERROR_DESCRIPTION = "errorDescriptionPrompt",
     FF_INSPECTOR = "ffInspectorNumExtractor",
     VISION_PROMPT = "visionPrompt",
+    SWIPE_VISION_PROMPT = "swipe_vision_prompt",
     VERIFY_PROMPT = "verifyActionExtractorPrompt",
     GET_PROMPT = "getActionExtractorPrompt",
     WAIT_ACTION_EXTRACTOR_PROMPT = "waitActionExtractorPrompt",

package/dist/core/types/promptType.js

@@ -9,6 +9,7 @@ var PromptType;
     PromptType["ERROR_DESCRIPTION"] = "errorDescriptionPrompt";
     PromptType["FF_INSPECTOR"] = "ffInspectorNumExtractor";
     PromptType["VISION_PROMPT"] = "visionPrompt";
+    PromptType["SWIPE_VISION_PROMPT"] = "swipe_vision_prompt";
     PromptType["VERIFY_PROMPT"] = "verifyActionExtractorPrompt";
     PromptType["GET_PROMPT"] = "getActionExtractorPrompt";
     PromptType["WAIT_ACTION_EXTRACTOR_PROMPT"] = "waitActionExtractorPrompt";

package/dist/domAnalysis/searchBest.js

@@ -3,7 +3,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.DomSearcher = void 0;
 const fastest_levenshtein_1 = require("fastest-levenshtein");
 const fast_fuzzy_1 = require("fast-fuzzy");
-// import { logger } from "../utils/logger/logData";
 class DomSearcher {
     search(nodes, keywords, topN = 20, threshold = 40) {
         if (typeof keywords === "string") {

package/dist/utils/DomExtraction/jsForAttributeInjection.js

@@ -281,238 +281,238 @@ async function getReactAnnotatedDOM(browser, startIndex = 0) {
 // ==========================================================
 // ANGULAR INJECTION
 // ==========================================================
-exports.INJECT_AND_SERIALIZE_DOM_JS = `
-return (function (startIndex) {
-  let nodeIndex = typeof startIndex === "number" ? startIndex : 0;
-function getCoverageStatus(el) {
-    if (!el || !el.getBoundingClientRect) return "unknown";
-    const s = getComputedStyle(el);
-    if (s.pointerEvents === "none") return "false";
-    const rect = el.getBoundingClientRect();
-    if (rect.width === 0 || rect.height === 0) return "false";
-    // Off-screen element — check if something is blocking the viewport
-    const isOffScreen =
-        rect.bottom < 0 ||
-        rect.right < 0 ||
-        rect.top > window.innerHeight ||
-        rect.left > window.innerWidth;
-    if (isOffScreen) {
-        const centerEl = document.elementFromPoint(
-            window.innerWidth / 2,
-            window.innerHeight / 2
-        );
-        if (centerEl && centerEl !== document.body && centerEl !== document.documentElement) {
-            // Something is at center of viewport — check if el and centerEl
-            // share the same scrollable container
-            const elScrollParent = getScrollParent(el);
-            const centerScrollParent = getScrollParent(centerEl);
-            if (elScrollParent !== centerScrollParent) return "true";
-        }
-        return "false";
-    }
-    // On-screen element — use elementFromPoint
-    const testPoints = [
-        [rect.left + 5, rect.top + 5],
-        [rect.right - 5, rect.top + 5],
-        [rect.left + 5, rect.bottom - 5],
-        [rect.right - 5, rect.bottom - 5],
-        [rect.left + rect.width / 2, rect.top + rect.height / 2]
-    ];
-    for (const [x, y] of testPoints) {
-        if (x < 0 || y < 0 || x > window.innerWidth || y > window.innerHeight) continue;
-        const topEl = document.elementFromPoint(x, y);
-        if (!topEl) continue;
-        if (topEl === el || el.contains(topEl)) continue;
-        if (topEl.contains(el)) continue; // topEl is ancestor, not a cover
-        return "true";
-    }
-    return "false";
-}
-function getScrollParent(node) {
-    if (!node || node === document.body) return document.body;
-    const s = getComputedStyle(node);
-    if (
-        s.overflow === "auto" || s.overflow === "scroll" ||
-        s.overflowY === "auto" || s.overflowY === "scroll"
-    ) return node;
-    return getScrollParent(node.parentElement);
-}
-function isUsable(node) {
-    const s = getComputedStyle(node);
-    if (s.display === "none" || s.visibility === "hidden" || s.opacity === "0") return false;
-    const rect = node.getBoundingClientRect();
-    if (rect.width === 0 || rect.height === 0) return false;
-    return true;
-}
-
-function isSelectRelated(node) {
-    return (
-        node.matches?.("select, option") ||
-        node.closest?.("select, [role='listbox'], [role='combobox']")
-    );
-}
-  function annotate(node) {
-    const s = getComputedStyle(node);
-    node.setAttribute("element-visibility", s.visibility || "");
-    node.setAttribute("element-pointer-events", s.pointerEvents || "");
-    node.setAttribute("element-zindex", s.zIndex || "");
-     if (isSelectRelated(node)) {
-        node.setAttribute("element-covered", "false");
-        return;
-    }
-
-    if (!isUsable(node)) {
-        node.setAttribute("element-covered", "true");
-        return;
-    }
-     try {
-        node.setAttribute("element-covered", getCoverageStatus(node));
-        
-    } catch {
-        node.setAttribute("element-covered", "unknown");
-    }
-  }
-  function getXPath(node) {
-    if (!node || node.nodeType !== 1) return "";
-    let segments = [];
-    let current = node;
-    while (current && current.nodeType === 1) {
-        const parent = current.parentNode;
-        if (parent instanceof ShadowRoot) {
-            const host = parent.host;
-            const siblings = Array.from(parent.children).filter(n => n.tagName === current.tagName);
-            const index = siblings.indexOf(current) + 1;
-            segments.unshift("shadow/" + current.tagName.toLowerCase() + "[" + index + "]");
-            current = host;
-            continue;
-        }
-        let index = 1;
-        if (parent && parent.children) {
-            const siblings = Array.from(parent.children).filter(n => n.tagName === current.tagName);
-            index = siblings.indexOf(current) + 1;
-        }
-        segments.unshift(current.tagName.toLowerCase() + "[" + index + "]");
-        if (!parent || parent.nodeType !== 1) break;
-        current = parent;
-    }
-    let path = "/" + segments.join("/");
-    if (node.ownerDocument !== document) {
-        const frameEl = node.ownerDocument.defaultView && node.ownerDocument.defaultView.frameElement;
-        if (frameEl) {
-            let framePath = frameEl.getAttribute("ff-xpath");
-            if (!framePath) framePath = getXPath(frameEl);
-            path = framePath + path;
-        }
-    }
-    path = path.replace(/\/div\[0\]\/div\[0\]/g, "/div[1]").replace(/\/div\[0\]/g, "/div[1]");
-    return path;
-  }
-  function walk(root) {
-     const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT, null, false);
-    let node = walker.currentNode;
-    while (node) {
-        if (node.nodeType === 1 
-            && typeof node.setAttribute === 'function'
-            && !["META", "STYLE", "SCRIPT"].includes(node.tagName)) {
-            node.setAttribute("ff-inspect", "Fire-Flink-" + nodeIndex++);
-            node.setAttribute("ff-xpath", getXPath(node));
-            annotate(node);
-        }
-        if (node.shadowRoot) walk(node.shadowRoot);
-        if (node.tagName === "IFRAME") {
-            try {
-                const doc = node.contentDocument;
-                if (doc && doc.documentElement) {
-                    walk(doc.documentElement);
-                } else {
-                    node.setAttribute("ff-cross-origin", "true");
-                }
-            } catch {
-                node.setAttribute("ff-cross-origin", "true");
-            }
-        }
-        node = walker.nextNode();
-    }
-  }
-function cleanup(root) {
-    const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT, null, false);
-    let node = walker.currentNode;
-    while (node) {
-     if (node.nodeType === 1 && typeof node.removeAttribute === 'function') {
-        node.removeAttribute("ff-inspect");
-        node.removeAttribute("ff-xpath");
-        node.removeAttribute("element-visibility");
-        node.removeAttribute("element-pointer-events");
-        node.removeAttribute("element-zindex");
-        node.removeAttribute("element-covered");
-        node.removeAttribute("ff-cross-origin");
-}
-        if (node.shadowRoot) cleanup(node.shadowRoot);
-        node = walker.nextNode();
-    }
-}
-cleanup(document.documentElement);
-// Force browser to settle DOM after cleanup
-void document.documentElement.offsetHeight;
-walk(document.documentElement);
-  function serializeNode(node) {
-    if (node.nodeType === Node.TEXT_NODE) return node.textContent;
-    if (node.nodeType !== Node.ELEMENT_NODE) return "";
-    const tag = node.tagName.toLowerCase();
-    let html = "<" + tag;
-    for (const attr of node.attributes) {
-        if (attr.name === "srcdoc") continue;
-        html += " " + attr.name + "=\"" + attr.value.replace(/"/g, "&quot;") + "\"";
-    }
-    html += ">";
-    if (node.shadowRoot) {
-        html += "<!--#shadow-root-->";
-        node.shadowRoot.childNodes.forEach(child => { html += serializeNode(child); });
-        html += "<!--#/shadow-root-->";
-    }
-    if (tag === "iframe" && node.getAttribute("ff-cross-origin") !== "true") {
-        try {
-            const doc = node.contentDocument;
-            if (doc) html += serializeNode(doc.documentElement);
-        } catch {}
-    }
-    node.childNodes.forEach(child => { html += serializeNode(child); });
-    html += "</" + tag + ">";
-    return html;
-  }
-
-  const crossOriginIframes =
-      Array.from(
-          document.querySelectorAll("iframe")
-      )
-      .filter(
-          iframe =>
-              iframe.getAttribute(
-                  "ff-cross-origin"
-              ) === "true"
-      )
-      .map(iframe => ({
-          xpath:
-              iframe.getAttribute(
-                  "ff-xpath"
-              ),
-
-          src:
-              iframe.getAttribute("src"),
-
-          title:
-              iframe.getAttribute("title"),
-
-          name:
-              iframe.getAttribute("name")
-      }));
-
-  return {
-      dom: "<!DOCTYPE html>\n" + serializeNode(document.documentElement),
-      crossOriginIframes: crossOriginIframes,
-      nextNodeIndex: nodeIndex
-  };
-})(arguments[0]);
+exports.INJECT_AND_SERIALIZE_DOM_JS = `
+return (function (startIndex) {
+  let nodeIndex = typeof startIndex === "number" ? startIndex : 0;
+function getCoverageStatus(el) {
+    if (!el || !el.getBoundingClientRect) return "unknown";
+    const s = getComputedStyle(el);
+    if (s.pointerEvents === "none") return "false";
+    const rect = el.getBoundingClientRect();
+    if (rect.width === 0 || rect.height === 0) return "false";
+    // Off-screen element — check if something is blocking the viewport
+    const isOffScreen =
+        rect.bottom < 0 ||
+        rect.right < 0 ||
+        rect.top > window.innerHeight ||
+        rect.left > window.innerWidth;
+    if (isOffScreen) {
+        const centerEl = document.elementFromPoint(
+            window.innerWidth / 2,
+            window.innerHeight / 2
+        );
+        if (centerEl && centerEl !== document.body && centerEl !== document.documentElement) {
+            // Something is at center of viewport — check if el and centerEl
+            // share the same scrollable container
+            const elScrollParent = getScrollParent(el);
+            const centerScrollParent = getScrollParent(centerEl);
+            if (elScrollParent !== centerScrollParent) return "true";
+        }
+        return "false";
+    }
+    // On-screen element — use elementFromPoint
+    const testPoints = [
+        [rect.left + 5, rect.top + 5],
+        [rect.right - 5, rect.top + 5],
+        [rect.left + 5, rect.bottom - 5],
+        [rect.right - 5, rect.bottom - 5],
+        [rect.left + rect.width / 2, rect.top + rect.height / 2]
+    ];
+    for (const [x, y] of testPoints) {
+        if (x < 0 || y < 0 || x > window.innerWidth || y > window.innerHeight) continue;
+        const topEl = document.elementFromPoint(x, y);
+        if (!topEl) continue;
+        if (topEl === el || el.contains(topEl)) continue;
+        if (topEl.contains(el)) continue; // topEl is ancestor, not a cover
+        return "true";
+    }
+    return "false";
+}
+function getScrollParent(node) {
+    if (!node || node === document.body) return document.body;
+    const s = getComputedStyle(node);
+    if (
+        s.overflow === "auto" || s.overflow === "scroll" ||
+        s.overflowY === "auto" || s.overflowY === "scroll"
+    ) return node;
+    return getScrollParent(node.parentElement);
+}
+function isUsable(node) {
+    const s = getComputedStyle(node);
+    if (s.display === "none" || s.visibility === "hidden" || s.opacity === "0") return false;
+    const rect = node.getBoundingClientRect();
+    if (rect.width === 0 || rect.height === 0) return false;
+    return true;
+}
+
+function isSelectRelated(node) {
+    return (
+        node.matches?.("select, option") ||
+        node.closest?.("select, [role='listbox'], [role='combobox']")
+    );
+}
+  function annotate(node) {
+    const s = getComputedStyle(node);
+    node.setAttribute("element-visibility", s.visibility || "");
+    node.setAttribute("element-pointer-events", s.pointerEvents || "");
+    node.setAttribute("element-zindex", s.zIndex || "");
+     if (isSelectRelated(node)) {
+        node.setAttribute("element-covered", "false");
+        return;
+    }
+
+    if (!isUsable(node)) {
+        node.setAttribute("element-covered", "true");
+        return;
+    }
+     try {
+        node.setAttribute("element-covered", getCoverageStatus(node));
+        
+    } catch {
+        node.setAttribute("element-covered", "unknown");
+    }
+  }
+  function getXPath(node) {
+    if (!node || node.nodeType !== 1) return "";
+    let segments = [];
+    let current = node;
+    while (current && current.nodeType === 1) {
+        const parent = current.parentNode;
+        if (parent instanceof ShadowRoot) {
+            const host = parent.host;
+            const siblings = Array.from(parent.children).filter(n => n.tagName === current.tagName);
+            const index = siblings.indexOf(current) + 1;
+            segments.unshift("shadow/" + current.tagName.toLowerCase() + "[" + index + "]");
+            current = host;
+            continue;
+        }
+        let index = 1;
+        if (parent && parent.children) {
+            const siblings = Array.from(parent.children).filter(n => n.tagName === current.tagName);
+            index = siblings.indexOf(current) + 1;
+        }
+        segments.unshift(current.tagName.toLowerCase() + "[" + index + "]");
+        if (!parent || parent.nodeType !== 1) break;
+        current = parent;
+    }
+    let path = "/" + segments.join("/");
+    if (node.ownerDocument !== document) {
+        const frameEl = node.ownerDocument.defaultView && node.ownerDocument.defaultView.frameElement;
+        if (frameEl) {
+            let framePath = frameEl.getAttribute("ff-xpath");
+            if (!framePath) framePath = getXPath(frameEl);
+            path = framePath + path;
+        }
+    }
+    path = path.replace(/\/div\[0\]\/div\[0\]/g, "/div[1]").replace(/\/div\[0\]/g, "/div[1]");
+    return path;
+  }
+  function walk(root) {
+     const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT, null, false);
+    let node = walker.currentNode;
+    while (node) {
+        if (node.nodeType === 1 
+            && typeof node.setAttribute === 'function'
+            && !["META", "STYLE", "SCRIPT"].includes(node.tagName)) {
+            node.setAttribute("ff-inspect", "Fire-Flink-" + nodeIndex++);
+            node.setAttribute("ff-xpath", getXPath(node));
+            annotate(node);
+        }
+        if (node.shadowRoot) walk(node.shadowRoot);
+        if (node.tagName === "IFRAME") {
+            try {
+                const doc = node.contentDocument;
+                if (doc && doc.documentElement) {
+                    walk(doc.documentElement);
+                } else {
+                    node.setAttribute("ff-cross-origin", "true");
+                }
+            } catch {
+                node.setAttribute("ff-cross-origin", "true");
+            }
+        }
+        node = walker.nextNode();
+    }
+  }
+function cleanup(root) {
+    const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT, null, false);
+    let node = walker.currentNode;
+    while (node) {
+     if (node.nodeType === 1 && typeof node.removeAttribute === 'function') {
+        node.removeAttribute("ff-inspect");
+        node.removeAttribute("ff-xpath");
+        node.removeAttribute("element-visibility");
+        node.removeAttribute("element-pointer-events");
+        node.removeAttribute("element-zindex");
+        node.removeAttribute("element-covered");
+        node.removeAttribute("ff-cross-origin");
+}
+        if (node.shadowRoot) cleanup(node.shadowRoot);
+        node = walker.nextNode();
+    }
+}
+cleanup(document.documentElement);
+// Force browser to settle DOM after cleanup
+void document.documentElement.offsetHeight;
+walk(document.documentElement);
+  function serializeNode(node) {
+    if (node.nodeType === Node.TEXT_NODE) return node.textContent;
+    if (node.nodeType !== Node.ELEMENT_NODE) return "";
+    const tag = node.tagName.toLowerCase();
+    let html = "<" + tag;
+    for (const attr of node.attributes) {
+        if (attr.name === "srcdoc") continue;
+        html += " " + attr.name + "=\"" + attr.value.replace(/"/g, "&quot;") + "\"";
+    }
+    html += ">";
+    if (node.shadowRoot) {
+        html += "<!--#shadow-root-->";
+        node.shadowRoot.childNodes.forEach(child => { html += serializeNode(child); });
+        html += "<!--#/shadow-root-->";
+    }
+    if (tag === "iframe" && node.getAttribute("ff-cross-origin") !== "true") {
+        try {
+            const doc = node.contentDocument;
+            if (doc) html += serializeNode(doc.documentElement);
+        } catch {}
+    }
+    node.childNodes.forEach(child => { html += serializeNode(child); });
+    html += "</" + tag + ">";
+    return html;
+  }
+
+  const crossOriginIframes =
+      Array.from(
+          document.querySelectorAll("iframe")
+      )
+      .filter(
+          iframe =>
+              iframe.getAttribute(
+                  "ff-cross-origin"
+              ) === "true"
+      )
+      .map(iframe => ({
+          xpath:
+              iframe.getAttribute(
+                  "ff-xpath"
+              ),
+
+          src:
+              iframe.getAttribute("src"),
+
+          title:
+              iframe.getAttribute("title"),
+
+          name:
+              iframe.getAttribute("name")
+      }));
+
+  return {
+      dom: "<!DOCTYPE html>\n" + serializeNode(document.documentElement),
+      crossOriginIframes: crossOriginIframes,
+      nextNodeIndex: nodeIndex
+  };
+})(arguments[0]);
 `;
 async function injectAngular(browser, startIndex = 0) {
     const fn = new Function("startIndex", exports.INJECT_AND_SERIALIZE_DOM_JS);

package/dist/utils/helpers/xpathcreation.js

@@ -61,8 +61,24 @@ function getFFIndexAndXPath(annotatedDom, fireflinkIndex, platform) {
         else {
             xpathQuery = `//*[@FF-inspecter='${fireflinkIndex}']`;
         }
-        const nodes = xpath_1.default.select(xpathQuery, xmlDoc);
-        const element = nodes.length ? nodes[0] : null;
+        let nodes = xpath_1.default.select(xpathQuery, xmlDoc);
+        let element = nodes.length ? nodes[0] : null;
+        if (!element && isBounds) {
+            if (platform === "android") {
+                xpathQuery = `//*[@bounds='${fireflinkIndex}']`;
+                nodes = xpath_1.default.select(xpathQuery, xmlDoc);
+                element = nodes.length ? nodes[0] : null;
+            }
+            else {
+                const match = fireflinkIndex.match(/\[(\d+),(\d+)\]\[(\d+),(\d+)\]/);
+                if (match) {
+                    const [, x1, y1, x2, y2] = match.map(Number);
+                    xpathQuery = `//*[@x='${x1}' and @y='${y1}' and @width='${x2 - x1}' and @height='${y2 - y1}']`;
+                    nodes = xpath_1.default.select(xpathQuery, xmlDoc);
+                    element = nodes.length ? nodes[0] : null;
+                }
+            }
+        }
         if (!element) {
             return { xpathPath: "" };
         }