android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/utils/coordinates.py

@@ -0,0 +1,606 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import json
+import re
+from typing import Any
+
+
+def parse_swipe_parameters(
+    args: Any,
+    default_duration: int | None = 800,
+) -> tuple[str | None, str | list[int] | None, int | None]:
+    """Parses swipe/drag/scroll parameters from various possible input formats into a standardized tuple:
+
+    (kind, target, duration) where kind is 'direction' or 'coords' or None.
+    If 'coords', target is [start_x, start_y, end_x, end_y] in 0-1000 scale.
+    If 'direction', target is 'up' | 'down' | 'left' | 'right'.
+    """
+    if args is None:
+        return "direction", "up", default_duration
+
+    # Extract raw dictionary if args is a dict
+    if isinstance(args, dict):
+        direction = args.get("direction")
+        start = args.get("start")
+        end = args.get("end")
+        coordinates = args.get("coordinates") or args.get("normalized_coordinates")
+        action = args.get("action")
+        gesture = args.get("gesture")
+        duration = (
+            args.get("duration")
+            or args.get("duration_ms")
+            or args.get("time_in_ms")
+            or default_duration
+        )
+    else:
+        direction = None
+        start = None
+        end = None
+        coordinates = None
+        action = args
+        gesture = None
+        duration = default_duration
+
+    try:
+        duration_int = int(duration) if duration is not None else default_duration
+    except (ValueError, TypeError):
+        duration_int = default_duration
+
+    def _parse_pt(p: Any) -> list[int] | None:
+        if isinstance(p, (list, tuple)) and len(p) >= 2:
+            try:
+                return [int(float(p[0])), int(float(p[1]))]
+            except (ValueError, TypeError):
+                pass
+        if isinstance(p, str):
+            nums = re.findall(r"-?\d+(?:\.\d+)?", p)
+            if len(nums) >= 2:
+                return [int(float(nums[0])), int(float(nums[1]))]
+        return None
+
+    # 1. Check start and end
+    if start is not None and end is not None:
+        pt_s = _parse_pt(start)
+        pt_e = _parse_pt(end)
+        if pt_s and pt_e:
+            return "coords", [pt_s[0], pt_s[1], pt_e[0], pt_e[1]], duration_int
+
+    # 2. Check candidate values in priority order
+    candidates = [coordinates, action, gesture, direction]
+    for c in candidates:
+        if c is None:
+            continue
+        # If list of 4 numbers
+        if isinstance(c, (list, tuple)):
+            if len(c) == 4:
+                try:
+                    return "coords", [int(float(x)) for x in c], duration_int
+                except (ValueError, TypeError):
+                    pass
+            elif (
+                len(c) == 2 and isinstance(c[0], (list, tuple)) and isinstance(c[1], (list, tuple))
+            ):
+                pt1 = _parse_pt(c[0])
+                pt2 = _parse_pt(c[1])
+                if pt1 and pt2:
+                    return "coords", [pt1[0], pt1[1], pt2[0], pt2[1]], duration_int
+
+        # If string
+        if isinstance(c, str):
+            s = c.strip()
+            # Try JSON parse first
+            try:
+                parsed = json.loads(s)
+                if isinstance(parsed, (list, tuple)):
+                    if len(parsed) == 4:
+                        return "coords", [int(float(x)) for x in parsed], duration_int
+                    elif len(parsed) == 2:
+                        pt1 = _parse_pt(parsed[0])
+                        pt2 = _parse_pt(parsed[1])
+                        if pt1 and pt2:
+                            return "coords", [pt1[0], pt1[1], pt2[0], pt2[1]], duration_int
+            except (ValueError, TypeError):
+                pass
+
+            # Regex extract numbers
+            nums = re.findall(r"-?\d+(?:\.\d+)?", s)
+            if len(nums) == 4:
+                return "coords", [int(float(x)) for x in nums], duration_int
+
+            # Check direction string (only if no digits)
+            s_lower = s.lower()
+            if not any(char.isdigit() for char in s):
+                for d in ("up", "down", "left", "right"):
+                    if d in s_lower:
+                        return "direction", d, duration_int
+
+    return None, None, duration_int
+
+
+def _extract_bounds(val: Any) -> list[int] | None:
+    """Safely parses bounds representation into [left, top, right, bottom]."""
+    if val is None:
+        return None
+    if isinstance(val, (list, tuple)) and len(val) == 4:
+        try:
+            return [int(float(x)) for x in val]
+        except (ValueError, TypeError):
+            return None
+    if isinstance(val, str):
+        match = re.match(r"\[(\-?\d+),(\-?\d+)\]\[(\-?\d+),(\-?\d+)\]", val.strip())
+        if match:
+            return [int(x) for x in match.groups()]
+        nums = re.findall(r"-?\d+", val)
+        if len(nums) == 4:
+            return [int(x) for x in nums]
+    return None
+
+
+def resolve_scrollable_container_bounds(
+    target: Any = None,
+    indexed_elements: list[dict[str, Any]] | None = None,
+    ui_hierarchy: list[dict[str, Any]] | None = None,
+    screen_width: int = 1080,
+    screen_height: int = 2400,
+) -> list[int]:
+    """Resolves the best-effort bounding box [left, top, right, bottom] for scrolling.
+
+    Heuristic Priority:
+    1. Explicit target element bounds (if target index/element/bounds passed).
+    2. Primary active scrollable container (RecyclerView, ScrollView, ListView, WebView, ViewPager).
+    3. Fallback to full screen [0, 0, screen_width, screen_height].
+    """
+    # 1. Target element reference
+    if target is not None:
+        direct_b = _extract_bounds(target)
+        if direct_b and len(direct_b) == 4:
+            return direct_b
+
+        target_idx = None
+        if isinstance(target, (int, float)):
+            target_idx = int(target)
+        elif isinstance(target, str) and target.strip().isdigit():
+            target_idx = int(target.strip())
+
+        if target_idx is not None and indexed_elements:
+            if 1 <= target_idx <= len(indexed_elements):
+                elem = indexed_elements[target_idx - 1]
+                b = _extract_bounds(elem.get("bounds"))
+                if b and len(b) == 4:
+                    return b
+
+    # 2. Look for primary scrollable container in ui_hierarchy or indexed_elements
+    candidates: list[list[int]] = []
+
+    def scan_hierarchy(nodes: list[dict[str, Any]]):
+        for node in nodes:
+            if not isinstance(node, dict):
+                continue
+            is_scroll = (
+                node.get("scrollable") is True or str(node.get("scrollable", "")).lower() == "true"
+            )
+            cls_name = str(node.get("class") or node.get("className") or "").lower()
+            if not is_scroll:
+                for kw in ("scrollview", "recyclerview", "listview", "webview", "viewpager"):
+                    if kw in cls_name:
+                        is_scroll = True
+                        break
+            if is_scroll:
+                b = _extract_bounds(node.get("bounds"))
+                if b and len(b) == 4:
+                    w = b[2] - b[0]
+                    h = b[3] - b[1]
+                    if w >= int(screen_width * 0.25) and h >= int(screen_height * 0.15):
+                        candidates.append(b)
+            children = node.get("children")
+            if isinstance(children, list) and children:
+                scan_hierarchy(children)
+
+    if ui_hierarchy:
+        scan_hierarchy(ui_hierarchy)
+    elif indexed_elements:
+        for elem in indexed_elements:
+            b = _extract_bounds(elem.get("bounds"))
+            cls_name = str(elem.get("class") or elem.get("className") or "").lower()
+            if any(
+                kw in cls_name
+                for kw in ("scrollview", "recyclerview", "listview", "webview", "viewpager")
+            ):
+                if b and len(b) == 4:
+                    candidates.append(b)
+
+    if candidates:
+
+        def area(b: list[int]) -> int:
+            return max(0, b[2] - b[0]) * max(0, b[3] - b[1])
+
+        candidates.sort(key=area, reverse=True)
+        return candidates[0]
+
+    return [0, 0, screen_width, screen_height]
+
+
+def compute_smart_swipe_coordinates(
+    direction: str,
+    target: Any = None,
+    indexed_elements: list[dict[str, Any]] | None = None,
+    ui_hierarchy: list[dict[str, Any]] | None = None,
+    width: int = 1080,
+    height: int = 2400,
+    duration: int | None = None,
+) -> tuple[int, int, int, int, int]:
+    """Computes best-effort container-aware intelligent swipe coordinates and duration.
+
+    Returns: (start_x, start_y, end_x, end_y, duration_ms)
+    """
+    container = resolve_scrollable_container_bounds(
+        target=target,
+        indexed_elements=indexed_elements,
+        ui_hierarchy=ui_hierarchy,
+        screen_width=width,
+        screen_height=height,
+    )
+
+    c_left = max(0, min(width - 1, container[0]))
+    c_top = max(0, min(height - 1, container[1]))
+    c_right = max(c_left + 50, min(width, container[2]))
+    c_bottom = max(c_top + 50, min(height, container[3]))
+
+    c_width = c_right - c_left
+    c_height = c_bottom - c_top
+
+    dir_str = str(direction or "up").strip().lower()
+
+    if dir_str == "up":
+        # Drag upwards -> reveal content below
+        sx = ex = int(c_left + c_width * 0.60)
+        sy = int(c_top + c_height * 0.70)
+        ey = int(c_top + c_height * 0.30)
+    elif dir_str == "down":
+        # Drag downwards -> reveal content above
+        sx = ex = int(c_left + c_width * 0.60)
+        sy = int(c_top + c_height * 0.30)
+        ey = int(c_top + c_height * 0.70)
+    elif dir_str == "left":
+        # Drag leftwards -> page right
+        sy = ey = int(c_top + c_height * 0.50)
+        sx = int(c_left + c_width * 0.75)
+        ex = int(c_left + c_width * 0.25)
+    elif dir_str == "right":
+        # Drag rightwards -> page left
+        sy = ey = int(c_top + c_height * 0.50)
+        sx = int(c_left + c_width * 0.25)
+        ex = int(c_left + c_width * 0.75)
+    else:
+        # Default up
+        sx = ex = int(c_left + c_width * 0.60)
+        sy = int(c_top + c_height * 0.70)
+        ey = int(c_top + c_height * 0.30)
+
+    # Clamp coordinates to screen bounds
+    sx = max(0, min(width - 1, sx))
+    ex = max(0, min(width - 1, ex))
+    sy = max(0, min(height - 1, sy))
+    ey = max(0, min(height - 1, ey))
+
+    if duration is None:
+        dist = ((ex - sx) ** 2 + (ey - sy) ** 2) ** 0.5
+        if dist <= 300:
+            duration_ms = 350
+        elif dist <= 600:
+            duration_ms = 550
+        elif dist <= 1000:
+            duration_ms = 800
+        else:
+            duration_ms = 900
+    else:
+        duration_ms = int(duration)
+
+    return sx, sy, ex, ey, duration_ms
+
+
+def normalize_point(x: int, y: int, width: int, height: int) -> list[int]:
+    """Convert physical pixels to normalized 0-1000 coordinates."""
+    nx = int(round(x * 1000.0 / width))
+    ny = int(round(y * 1000.0 / height))
+    # Clamp to [0, 1000] to be safe
+    return [max(0, min(1000, nx)), max(0, min(1000, ny))]
+
+
+def denormalize_point(nx: int, ny: int, width: int, height: int) -> list[int]:
+    """Convert normalized 0-1000 coordinates to physical pixels."""
+    x = int(round(nx * width / 1000.0))
+    y = int(round(ny * height / 1000.0))
+    # Clamp to device bounds
+    return [max(0, min(width - 1, x)), max(0, min(height - 1, y))]
+
+
+#: Pro records physical pixels; Flash records normalized 0–1000 coordinates.
+#: The marker prevents normalization from being applied twice during rendering.
+COORDINATE_SPACE_KEY = "coordinate_space"
+COORDINATE_SPACE_PIXEL = "pixel"
+COORDINATE_SPACE_NORMALIZED = "normalized"
+
+_COORDINATE_FIELDS = ("coordinates", "start_coordinates", "end_coordinates", "target")
+
+
+def normalize_action_dict(action: dict[str, Any], width: int, height: int) -> dict[str, Any]:
+    """Normalize the coordinate fields of one recorded action dict (idempotent).
+
+    A dict already stamped ``coordinate_space="normalized"`` is returned as a
+    copy without any conversion; a pixel-space (or legacy unstamped) dict has
+    its ``coordinates`` / ``start_coordinates`` / ``end_coordinates`` /
+    ``target`` fields converted once and comes back stamped ``normalized``, so
+    a second pass over the same output is a no-op.
+    """
+    if not isinstance(action, dict):
+        return action
+
+    action_copy = copy.deepcopy(action)
+    if action_copy.get(COORDINATE_SPACE_KEY) == COORDINATE_SPACE_NORMALIZED:
+        return action_copy
+
+    converted = False
+    # Normalize 'coordinates' if present and valid
+    coords = action_copy.get("coordinates")
+    if coords and isinstance(coords, list):
+        if len(coords) == 2:
+            action_copy["coordinates"] = normalize_point(coords[0], coords[1], width, height)
+            converted = True
+        elif len(coords) == 4:
+            p1 = normalize_point(coords[0], coords[1], width, height)
+            p2 = normalize_point(coords[2], coords[3], width, height)
+            action_copy["coordinates"] = p1 + p2
+            converted = True
+
+    # Normalize 'start_coordinates' and 'end_coordinates' for generic formats
+    for key in _COORDINATE_FIELDS[1:]:
+        val = action_copy.get(key)
+        if val and isinstance(val, list) and len(val) == 2:
+            action_copy[key] = normalize_point(val[0], val[1], width, height)
+            converted = True
+
+    if converted or COORDINATE_SPACE_KEY in action_copy:
+        action_copy[COORDINATE_SPACE_KEY] = COORDINATE_SPACE_NORMALIZED
+    return action_copy
+
+
+def _is_action_item(obj: dict[str, Any]) -> bool:
+    """An action item names its action; a container that merely holds an
+    ``action`` dict (an execution incident) is recursed into instead."""
+    return isinstance(obj.get("action"), str) or isinstance(obj.get("name"), str)
+
+
+def normalize_any_structure(obj: Any, width: int, height: int) -> Any:
+    """Recursively searches lists and dicts to normalize any action coordinate schemas."""
+    if isinstance(obj, list):
+        return [normalize_any_structure(item, width, height) for item in obj]
+    elif isinstance(obj, dict):
+        # If it looks like a single action item
+        if _is_action_item(obj):
+            return normalize_action_dict(obj, width, height)
+        # Otherwise recurse down keys
+        return {k: normalize_any_structure(v, width, height) for k, v in obj.items()}
+    return obj
+
+
+def normalize_step_actions(step_dict: dict[str, Any]) -> dict[str, Any]:
+    """Normalizes step action representation between action_taken and generic_tools,
+
+    and ensures explicit normalized_coordinates (0-1000 scale) are injected for
+    UI rendering.
+    """
+    if not isinstance(step_dict, dict):
+        return step_dict
+
+    extra = step_dict.get("extra_metadata") or {}
+    width = extra.get("width") or 1080
+    height = extra.get("height") or 2400
+
+    def _enrich_action_item(item: dict[str, Any]) -> dict[str, Any]:
+        if not isinstance(item, dict):
+            return item
+        coords = item.get("coordinates")
+        act_name = str(item.get("action") or item.get("name") or "").lower()
+
+        if (
+            "swipe" in act_name or "scroll" in act_name or "drag" in act_name
+        ) and "normalized_coordinates" not in item:
+            kind, target, _ = parse_swipe_parameters(item)
+            if kind == "coords" and isinstance(target, list) and len(target) == 4:
+                item["normalized_coordinates"] = target
+                item["normalized_start_coordinates"] = target[:2]
+                item["normalized_end_coordinates"] = target[2:]
+                if not coords:
+                    item["coordinates"] = target
+            elif kind == "direction" and isinstance(target, str):
+                if target == "up":
+                    norm = [600, 700, 600, 300]
+                elif target == "down":
+                    norm = [600, 300, 600, 700]
+                elif target == "left":
+                    norm = [750, 500, 250, 500]
+                elif target == "right":
+                    norm = [250, 500, 750, 500]
+                else:
+                    norm = None
+                if norm:
+                    item["normalized_coordinates"] = norm
+                    item["normalized_start_coordinates"] = norm[:2]
+                    item["normalized_end_coordinates"] = norm[2:]
+
+        coords = item.get("coordinates")
+        if coords and isinstance(coords, list) and "normalized_coordinates" not in item:
+            # If coordinates look physical (e.g., target_bounds present or > 1000), normalize using width/height
+            if item.get("target_bounds") or any(
+                isinstance(c, (int, float)) and c > 1000 for c in coords
+            ):
+                if len(coords) == 2 and all(isinstance(c, (int, float)) for c in coords):
+                    item["normalized_coordinates"] = [
+                        int(round(coords[0] * 1000.0 / width)),
+                        int(round(coords[1] * 1000.0 / height)),
+                    ]
+                elif len(coords) == 4 and all(isinstance(c, (int, float)) for c in coords):
+                    nx1, ny1 = (
+                        int(round(coords[0] * 1000.0 / width)),
+                        int(round(coords[1] * 1000.0 / height)),
+                    )
+                    nx2, ny2 = (
+                        int(round(coords[2] * 1000.0 / width)),
+                        int(round(coords[3] * 1000.0 / height)),
+                    )
+                    item["normalized_coordinates"] = [nx1, ny1, nx2, ny2]
+                    item["normalized_start_coordinates"] = [nx1, ny1]
+                    item["normalized_end_coordinates"] = [nx2, ny2]
+            else:
+                item["normalized_coordinates"] = coords
+
+        seq = item.get("sequence") or item.get("targets")
+        if seq:
+            if isinstance(seq, str):
+                try:
+                    seq = json.loads(seq)
+                except Exception:
+                    try:
+                        import ast
+
+                        seq = ast.literal_eval(seq)
+                    except (ValueError, TypeError, SyntaxError):
+                        # Unparseable sequence string: leave it as-is; the
+                        # isinstance check below skips non-list values.
+                        pass
+                if isinstance(seq, (list, tuple)):
+                    item["sequence"] = list(seq)
+            if isinstance(item.get("sequence"), list) and "normalized_sequence" not in item:
+                norm_seq = []
+                for pt in item["sequence"]:
+                    if (
+                        isinstance(pt, (list, tuple))
+                        and len(pt) == 2
+                        and all(isinstance(c, (int, float)) for c in pt)
+                    ):
+                        if any(c > 1000 for c in pt) or item.get("target_bounds"):
+                            norm_seq.append(
+                                [
+                                    int(round(pt[0] * 1000.0 / width)),
+                                    int(round(pt[1] * 1000.0 / height)),
+                                ]
+                            )
+                        else:
+                            norm_seq.append([int(round(pt[0])), int(round(pt[1]))])
+                    elif isinstance(pt, int):
+                        norm_seq.append(pt)
+                if norm_seq:
+                    item["normalized_sequence"] = norm_seq
+
+        return item
+
+    action_taken = step_dict.get("action_taken")
+    if isinstance(action_taken, str):
+        try:
+            action_taken = json.loads(action_taken)
+            step_dict["action_taken"] = action_taken
+        except ValueError:
+            # Not JSON: keep the raw string; downstream isinstance checks skip it.
+            pass
+
+    if isinstance(action_taken, dict):
+        step_dict["action_taken"] = _enrich_action_item(action_taken)
+    elif isinstance(action_taken, list):
+        step_dict["action_taken"] = [
+            _enrich_action_item(x) if isinstance(x, dict) else x for x in action_taken
+        ]
+        action_taken = step_dict["action_taken"]
+
+    generic_tools = step_dict.get("generic_tools")
+    if not isinstance(generic_tools, list) or not generic_tools:
+        return step_dict
+
+    action_tool_names = {
+        "click",
+        "click_sequence",
+        "long_press",
+        "input_text",
+        "swipe",
+        "press_key",
+        "object_detection",
+        "manage_app",
+        "wait_for_delay",
+        "wait_for_text",
+        "report_failure_analysis",
+        "ask_explorer",
+        "run_adb_command",
+        "run_short_adb_command",
+    }
+
+    has_atomic_action_tool = False
+    for tool in generic_tools:
+        if not isinstance(tool, dict) or tool.get("type") == "llm_call":
+            continue
+        raw_name = str(tool.get("name", "")).lower().replace("_exec_", "").lstrip("_")
+        if raw_name in action_tool_names:
+            has_atomic_action_tool = True
+            if isinstance(action_taken, (dict, list)):
+                payload = tool.get("payload") or {}
+                if isinstance(payload, str):
+                    try:
+                        payload = json.loads(payload)
+                    except Exception:
+                        payload = {}
+                if not isinstance(payload, dict):
+                    payload = {}
+                args = payload.get("args") or {}
+                if isinstance(args, str):
+                    try:
+                        args = json.loads(args)
+                    except Exception:
+                        args = {}
+                if not isinstance(args, dict):
+                    args = {}
+
+                first_act = (
+                    action_taken[0]
+                    if isinstance(action_taken, list)
+                    and action_taken
+                    and isinstance(action_taken[0], dict)
+                    else (action_taken if isinstance(action_taken, dict) else {})
+                )
+
+                is_seq_tool = (
+                    raw_name in ("click_sequence", "tap_sequence")
+                    or "sequence" in args
+                    or "targets" in args
+                )
+
+                # Enrich missing coordinates/target/input from action_taken only if not a multi-point sequence action
+                if not is_seq_tool:
+                    if "target" not in args and "coordinates" not in args:
+                        if first_act.get("coordinates"):
+                            args["coordinates"] = first_act["coordinates"]
+                        if first_act.get("args", {}).get("target"):
+                            args["target"] = first_act["args"]["target"]
+                    if "normalized_coordinates" not in args and first_act.get(
+                        "normalized_coordinates"
+                    ):
+                        args["normalized_coordinates"] = first_act["normalized_coordinates"]
+                    if "text" not in args and first_act.get("text"):
+                        args["text"] = first_act["text"]
+
+                args = _enrich_action_item(args)
+                payload["args"] = args
+                tool["payload"] = payload
+
+    return step_dict