android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/utils/image_diff.py

@@ -0,0 +1,335 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import base64
+from collections.abc import Awaitable, Callable
+import logging
+import math
+import time
+
+import cv2
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+
+def check_ui_change(
+    img_before_bytes,
+    img_after_bytes,
+    action_item,
+    full_screen_threshold=0.0001,
+    roi_threshold=0.05,
+):
+    """Checks if the UI has changed after an action using image diff.
+
+    Args:
+        img_before_bytes: Image bytes before action.
+        img_after_bytes: Image bytes after action.
+        action_item: Dict containing 'action' and optionally 'coordinates'.
+        full_screen_threshold: Threshold for full screen change (fraction of
+          changed pixels).
+        roi_threshold: Threshold for ROI change (fraction of changed pixels in
+          ROI).
+
+    Returns:
+        bool: True if UI changed, False otherwise.
+    """
+    try:
+        # 1. Decode images
+        nparr_b = np.frombuffer(img_before_bytes, np.uint8)
+        nparr_a = np.frombuffer(img_after_bytes, np.uint8)
+        img_b = cv2.imdecode(nparr_b, cv2.IMREAD_GRAYSCALE)
+        img_a = cv2.imdecode(nparr_a, cv2.IMREAD_GRAYSCALE)
+
+        if img_b is None or img_a is None:
+            logger.error("Failed to decode images.")
+            return True  # Fallback to assuming change if we can't decode
+
+        h, w = img_b.shape
+        action_name = action_item.get("action")
+        coordinates = action_item.get("coordinates")
+
+        # 2. Masking: Ignore status bar (top 10%) and nav bar (bottom 5%)
+        top_crop = int(h * 0.10)
+        bottom_crop = int(h * 0.05)
+
+        content_b = img_b[top_crop : h - bottom_crop, :]
+        content_a = img_a[top_crop : h - bottom_crop, :]
+
+        # 3. Calculate absolute difference and threshold
+        diff = cv2.absdiff(content_b, content_a)
+        blurred = cv2.GaussianBlur(diff, (5, 5), 0)
+        _, thresh = cv2.threshold(blurred, 25, 255, cv2.THRESH_BINARY)
+
+        non_zero_count = cv2.countNonZero(thresh)
+        total_pixels = content_b.shape[0] * content_b.shape[1]
+        global_change_ratio = non_zero_count / total_pixels
+
+        logger.info(f"Action: {action_name} | Global UI change ratio: {global_change_ratio:.4f}")
+
+        # 4. Strategy based on action type
+        if action_name == "swipe":
+            # Swipe expects large scale movement
+            return global_change_ratio > full_screen_threshold
+
+        # For tap and others, check ROI if coordinates are available
+        if (
+            action_name
+            in [
+                "tap",
+                "long_press_on",
+                "focus_and_input_text",
+                "focus_and_clear_text",
+            ]
+            and coordinates
+            and len(coordinates) == 2
+        ):
+            cx, cy = coordinates
+            # Scale ROI size dynamically based on screen diagonal (reference is 1080x2400)
+            # and enforce a minimum of 15px to prevent collapsing on low-res screens
+            ref_diagonal = math.sqrt(1080**2 + 2400**2)
+            current_diagonal = math.sqrt(w**2 + h**2)
+            scale_factor = current_diagonal / ref_diagonal
+            roi_size = max(15, int(50 * scale_factor))
+
+            # Crop ROI from original images to avoid shifting issues with top_crop
+            y1 = max(0, cy - roi_size)
+            y2 = min(h, cy + roi_size)
+            x1 = max(0, cx - roi_size)
+            x2 = min(w, cx + roi_size)
+
+            roi_b = img_b[y1:y2, x1:x2]
+            roi_a = img_a[y1:y2, x1:x2]
+
+            roi_diff = cv2.absdiff(roi_b, roi_a)
+            roi_blurred = cv2.GaussianBlur(roi_diff, (3, 3), 0)
+            _, roi_thresh = cv2.threshold(roi_blurred, 20, 255, cv2.THRESH_BINARY)
+
+            roi_non_zero = cv2.countNonZero(roi_thresh)
+            roi_total = roi_b.shape[0] * roi_b.shape[1]
+            roi_change_ratio = roi_non_zero / roi_total
+
+            logger.info(f"ROI change ratio at [{cx}, {cy}]: {roi_change_ratio:.4f}")
+
+            # If ROI changed significantly, we assume action worked
+            if roi_change_ratio > roi_threshold:
+                return True
+
+        # Fallback to global check if no ROI check was triggered or passed
+        return global_change_ratio > full_screen_threshold
+
+    except Exception as e:
+        logger.error(f"Error during image diff: {e}")
+        return True  # Fallback to assuming change on error
+
+
+async def wait_for_screen_stability(
+    take_screenshot_fn: Callable[[], Awaitable[str]],
+    max_timeout: float = 1.5,
+    interval: float = 0.2,
+    stability_threshold: float = 0.001,
+) -> str:
+    """Wait for the screen to stabilize (nearly zero pixel changes between consecutive frames)
+
+    before returning the latest base64 screenshot.
+
+    Args:
+        take_screenshot_fn: Async callback function that captures a base64
+          screenshot.
+        max_timeout: Maximum time to wait for settling (in seconds) before
+          forcing exit.
+        interval: Polling time step between screenshot samples (in seconds).
+        stability_threshold: Pixel change ratio below which the screen is
+          considered settled (e.g. 0.001 for < 0.1%).
+
+    Returns:
+        str: The final settled base64 screenshot.
+    """
+    logger.info("Starting Dynamic Screen Settling...")
+
+    # 1. Capture the initial frame
+    try:
+        last_b64 = await take_screenshot_fn()
+        last_bytes = base64.b64decode(last_b64)
+    except Exception as e:
+        logger.error(f"Failed to capture initial frame for settling: {e}")
+        # Return fallback immediately on failure
+        return await take_screenshot_fn()
+
+    start_time = time.time()
+    while time.time() - start_time < max_timeout:
+        await asyncio.sleep(interval)
+
+        try:
+            curr_b64 = await take_screenshot_fn()
+            curr_bytes = base64.b64decode(curr_b64)
+        except Exception as e:
+            logger.warning(f"Failed to capture polling frame: {e}")
+            continue
+
+        # 2. Decode and compare using OpenCV
+        try:
+            nparr_b = np.frombuffer(last_bytes, np.uint8)
+            nparr_a = np.frombuffer(curr_bytes, np.uint8)
+            img_b = cv2.imdecode(nparr_b, cv2.IMREAD_GRAYSCALE)
+            img_a = cv2.imdecode(nparr_a, cv2.IMREAD_GRAYSCALE)
+
+            if img_b is None or img_a is None:
+                last_b64, last_bytes = curr_b64, curr_bytes
+                continue
+
+            h, w = img_b.shape
+
+            # Masking: crop status bar (top 10%) and navigation bar (bottom 5%)
+            # to avoid false positives from notification icons, clocks, or blinking elements
+            top_crop = int(h * 0.10)
+            bottom_crop = int(h * 0.05)
+
+            content_b = img_b[top_crop : h - bottom_crop, :]
+            content_a = img_a[top_crop : h - bottom_crop, :]
+
+            # Calculate absolute difference and filter noise
+            diff = cv2.absdiff(content_b, content_a)
+            blurred = cv2.GaussianBlur(diff, (5, 5), 0)
+            _, thresh = cv2.threshold(blurred, 25, 255, cv2.THRESH_BINARY)
+
+            non_zero_count = cv2.countNonZero(thresh)
+            total_pixels = content_b.shape[0] * content_b.shape[1]
+            change_ratio = non_zero_count / total_pixels
+
+            elapsed = time.time() - start_time
+            logger.info(
+                f"Screen settle poll: diff ratio = {change_ratio:.4f} (target <"
+                f" {stability_threshold}) | Elapsed: {elapsed:.2f}s"
+            )
+
+            # If the change ratio is below stability threshold, screen has stabilized!
+            if change_ratio < stability_threshold:
+                logger.info(f"Screen stabilized after {elapsed:.2f}s (diff: {change_ratio:.4f}).")
+                return curr_b64
+
+            last_b64, last_bytes = curr_b64, curr_bytes
+
+        except Exception as ex:
+            logger.error(f"Error during screen stability image diff calculation: {ex}")
+            last_b64, last_bytes = curr_b64, curr_bytes
+            continue
+
+    # 3. Timeout safety fallback (e.g., dynamic playing video/animated loaders)
+    logger.warning(
+        f"Screen settle timed out after {max_timeout}s without complete"
+        " stability. Forcing return of latest frame."
+    )
+    return last_b64
+
+
+async def wait_for_screen_data_stability(
+    get_screen_data_fn,
+    max_timeout: float = 1.5,
+    interval: float = 0.2,
+    stability_threshold: float = 0.001,
+):
+    """Wait for the screen to stabilize (nearly zero pixel changes between consecutive frames)
+
+    while fetching both screenshot and XML hierarchy directly
+    (ScreenDataResponse).
+
+    Args:
+        get_screen_data_fn: Async callback function that captures a
+          ScreenDataResponse.
+        max_timeout: Maximum time to wait for settling (in seconds) before
+          forcing exit.
+        interval: Polling time step between screen data samples (in seconds).
+        stability_threshold: Pixel change ratio below which the screen is
+          considered settled.
+
+    Returns:
+        ScreenDataResponse: The final settled ScreenDataResponse.
+    """
+    logger.info("Starting Dynamic Screen Data Settling...")
+
+    # 1. Capture the initial frame
+    try:
+        last_data = await get_screen_data_fn()
+        last_bytes = base64.b64decode(last_data.base64)
+    except Exception as e:
+        logger.error(f"Failed to capture initial frame for settling: {e}")
+        # Return fallback immediately on failure
+        return await get_screen_data_fn()
+
+    start_time = time.time()
+    while time.time() - start_time < max_timeout:
+        await asyncio.sleep(interval)
+
+        try:
+            curr_data = await get_screen_data_fn()
+            curr_bytes = base64.b64decode(curr_data.base64)
+        except Exception as e:
+            logger.warning(f"Failed to capture polling frame: {e}")
+            continue
+
+        # 2. Decode and compare using OpenCV
+        try:
+            nparr_b = np.frombuffer(last_bytes, np.uint8)
+            nparr_a = np.frombuffer(curr_bytes, np.uint8)
+            img_b = cv2.imdecode(nparr_b, cv2.IMREAD_GRAYSCALE)
+            img_a = cv2.imdecode(nparr_a, cv2.IMREAD_GRAYSCALE)
+
+            if img_b is None or img_a is None:
+                last_data, last_bytes = curr_data, curr_bytes
+                continue
+
+            h, w = img_b.shape
+
+            # Masking: crop status bar (top 10%) and navigation bar (bottom 5%)
+            top_crop = int(h * 0.10)
+            bottom_crop = int(h * 0.05)
+
+            content_b = img_b[top_crop : h - bottom_crop, :]
+            content_a = img_a[top_crop : h - bottom_crop, :]
+
+            # Calculate absolute difference and filter noise
+            diff = cv2.absdiff(content_b, content_a)
+            blurred = cv2.GaussianBlur(diff, (5, 5), 0)
+            _, thresh = cv2.threshold(blurred, 25, 255, cv2.THRESH_BINARY)
+
+            non_zero_count = cv2.countNonZero(thresh)
+            total_pixels = content_b.shape[0] * content_b.shape[1]
+            change_ratio = non_zero_count / total_pixels
+
+            elapsed = time.time() - start_time
+            logger.info(
+                f"Screen data settle poll: diff ratio = {change_ratio:.4f}"
+                f" (target < {stability_threshold}) | Elapsed: {elapsed:.2f}s"
+            )
+
+            # If the change ratio is below stability threshold, screen has stabilized!
+            if change_ratio < stability_threshold:
+                logger.info(f"Screen stabilized after {elapsed:.2f}s (diff: {change_ratio:.4f}).")
+                return curr_data
+
+            last_data, last_bytes = curr_data, curr_bytes
+
+        except Exception as ex:
+            logger.error(f"Error during screen stability image diff calculation: {ex}")
+            last_data, last_bytes = curr_data, curr_bytes
+            continue
+
+    # 3. Timeout safety fallback
+    logger.warning(
+        f"Screen data settle timed out after {max_timeout}s without complete"
+        " stability. Forcing return of latest frame."
+    )
+    return last_data

package/artemis-main/artemis/utils/image_hash.py

@@ -0,0 +1,80 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Pure-PIL perceptual hashing (dHash) for screen-similarity hints (M4).
+
+A 64-bit difference hash: grayscale, resize to 9x8, compare each pixel to its
+right neighbor. Robust to JPEG re-encoding and mild rendering noise, cheap to
+compute (one small resize), and comparable with an integer Hamming distance —
+no model call, no historical image bytes needed at compare time.
+
+Hashes are stored per step in ``extra_metadata`` at record time
+(``pre_image_dhash`` / ``post_image_dhash``, 16 hex chars) and compared
+against the current screenshot before prompt construction.
+"""
+
+import io
+
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+#: Grid width of the dHash (the hash carries HASH_SIZE * HASH_SIZE bits).
+HASH_SIZE = 8
+
+
+def dhash_hex(image_bytes: bytes | None, hash_size: int = HASH_SIZE) -> str | None:
+    """64-bit dHash of an encoded image, as a 16-char hex string.
+
+    Best-effort: returns ``None`` on any failure (missing/corrupt bytes,
+    Pillow unavailable) — perceptual hashing is an enhancement, never a
+    dependency.
+    """
+    if not image_bytes:
+        return None
+    try:
+        from PIL import Image
+
+        with Image.open(io.BytesIO(image_bytes)) as img:
+            # JPEG fast-path: DCT-scaled draft decode keeps the sync cost of
+            # hashing at record time to a few milliseconds. The draft is only
+            # an intermediate resolution; the final resize below fixes the
+            # grid, so hashes stay comparable across code paths.
+            try:
+                img.draft("L", (hash_size * 8, hash_size * 8))
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                # Draft decode is an optimization only; full decode below still works.
+                logger.debug(f"dhash draft decode skipped: {exc}", exc_info=True)
+            small = img.convert("L").resize((hash_size + 1, hash_size), Image.Resampling.LANCZOS)
+            pixels = list(small.getdata())
+        bits = 0
+        for row in range(hash_size):
+            for col in range(hash_size):
+                left = pixels[row * (hash_size + 1) + col]
+                right = pixels[row * (hash_size + 1) + col + 1]
+                bits = (bits << 1) | (1 if left > right else 0)
+        return f"{bits:0{hash_size * hash_size // 4}x}"
+    except Exception as e:
+        logger.debug(f"dhash_hex failed: {e}")
+        return None
+
+
+def hamming_distance_hex(hash_a: str | None, hash_b: str | None) -> int | None:
+    """Bit-level Hamming distance between two hex hashes; None if incomparable."""
+    if not hash_a or not hash_b or len(hash_a) != len(hash_b):
+        return None
+    try:
+        return bin(int(hash_a, 16) ^ int(hash_b, 16)).count("1")
+    except ValueError:
+        return None