android-midscene-automation 0.2.3 → 1.1.0

package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/DisplayUtils.java

@@ -0,0 +1,111 @@
+package com.artemis.helper;
+
+import android.accessibilityservice.AccessibilityService;
+import android.content.Context;
+import android.graphics.Point;
+import android.os.Build;
+import android.util.DisplayMetrics;
+import android.view.Display;
+import android.view.Surface;
+import android.view.WindowManager;
+
+/**
+ * Universal display and orientation utilities compatible across all Android versions (API 24 - 37+).
+ */
+public final class DisplayUtils {
+
+    private DisplayUtils() {}
+
+    public static class DisplayInfo {
+        public final int rotation;
+        public final int width;
+        public final int height;
+
+        public DisplayInfo(int rotation, int width, int height) {
+            this.rotation = rotation;
+            this.width = width;
+            this.height = height;
+        }
+    }
+
+    /**
+     * Obtains the display orientation (0, 1, 2, 3) and screen dimensions (width, height)
+     * using the most reliable API for the current Android runtime.
+     */
+    @SuppressWarnings("deprecation")
+    public static DisplayInfo getDisplayInfo(AccessibilityService service) {
+        int rotation = 0;
+        int width = 1080;
+        int height = 2400;
+
+        // Baseline initialization from resources metrics (handles tablets, TVs, emulators dynamically)
+        try {
+            DisplayMetrics resDm = service.getResources().getDisplayMetrics();
+            if (resDm != null && resDm.widthPixels > 0 && resDm.heightPixels > 0) {
+                width = resDm.widthPixels;
+                height = resDm.heightPixels;
+            }
+        } catch (Throwable ignored) {}
+
+        try {
+            // Modern API 30+ window metrics for full physical screen dimensions
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+                try {
+                    WindowManager wm = (WindowManager) service.getSystemService(Context.WINDOW_SERVICE);
+                    if (wm != null) {
+                        android.graphics.Rect bounds = wm.getMaximumWindowMetrics().getBounds();
+                        if (bounds.width() > 0 && bounds.height() > 0) {
+                            width = bounds.width();
+                            height = bounds.height();
+                        }
+                    }
+                } catch (Throwable ignored) {}
+            }
+
+            // Resolve Display instance for rotation and legacy metrics
+            Display display = null;
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+                try {
+                    display = service.getDisplay();
+                } catch (Throwable ignored) {}
+            }
+
+            if (display == null) {
+                WindowManager wm = (WindowManager) service.getSystemService(Context.WINDOW_SERVICE);
+                if (wm != null) {
+                    display = wm.getDefaultDisplay();
+                }
+            }
+
+            if (display != null) {
+                int r = display.getRotation();
+                switch (r) {
+                    case Surface.ROTATION_0:
+                        rotation = 0;
+                        break;
+                    case Surface.ROTATION_90:
+                        rotation = 1;
+                        break;
+                    case Surface.ROTATION_180:
+                        rotation = 2;
+                        break;
+                    case Surface.ROTATION_270:
+                        rotation = 3;
+                        break;
+                    default:
+                        rotation = 0;
+                        break;
+                }
+
+                if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
+                    DisplayMetrics dm = new DisplayMetrics();
+                    display.getRealMetrics(dm);
+                    width = dm.widthPixels;
+                    height = dm.heightPixels;
+                }
+            }
+        } catch (Throwable ignored) {}
+
+        return new DisplayInfo(rotation, width, height);
+    }
+}

package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java

@@ -0,0 +1,232 @@
+package com.artemis.helper;
+
+import android.accessibilityservice.AccessibilityService;
+import android.accessibilityservice.GestureDescription;
+import android.content.ClipData;
+import android.content.ClipboardManager;
+import android.content.Context;
+import android.graphics.Path;
+import android.os.Build;
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.Looper;
+import android.util.Log;
+import android.view.accessibility.AccessibilityNodeInfo;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Synchronous gesture and action executor using native AccessibilityService APIs.
+ * Works without requiring UiAutomation or root privileges.
+ */
+public final class GestureController {
+
+    private static final String TAG = "ArtemisGestureCtrl";
+    private static final Handler MAIN_HANDLER = new Handler(Looper.getMainLooper());
+
+    private GestureController() {}
+
+    public static boolean tap(AccessibilityService service, float x, float y, long timeoutMs) {
+        Path path = new Path();
+        path.moveTo(x, y);
+        GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription(path, 0L, 60L);
+        GestureDescription gesture = new GestureDescription.Builder().addStroke(stroke).build();
+        return dispatchSynchronous(service, gesture, timeoutMs);
+    }
+
+    public static boolean doubleTap(AccessibilityService service, float x, float y, long timeoutMs) {
+        if (!tap(service, x, y, timeoutMs)) return false;
+        try {
+            Thread.sleep(100L);
+        } catch (InterruptedException ignored) {}
+        return tap(service, x, y, timeoutMs);
+    }
+
+    public static boolean longPress(AccessibilityService service, float x, float y, long durationMs, long timeoutMs) {
+        Path path = new Path();
+        path.moveTo(x, y);
+        long pressDur = Math.max(500L, Math.min(durationMs, 5000L));
+        GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription(path, 0L, pressDur);
+        GestureDescription gesture = new GestureDescription.Builder().addStroke(stroke).build();
+        return dispatchSynchronous(service, gesture, timeoutMs + pressDur);
+    }
+
+    public static boolean swipe(
+            AccessibilityService service,
+            float x1, float y1, float x2, float y2,
+            long durationMs, long timeoutMs
+    ) {
+        Path path = new Path();
+        path.moveTo(x1, y1);
+        path.lineTo(x2, y2);
+        long safeDuration = Math.max(50L, Math.min(durationMs, 5000L));
+        GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription(path, 0L, safeDuration);
+        GestureDescription gesture = new GestureDescription.Builder().addStroke(stroke).build();
+        return dispatchSynchronous(service, gesture, timeoutMs + safeDuration);
+    }
+
+    public static boolean setText(AccessibilityService service, String text) {
+        return setText(service, text, false);
+    }
+
+    /**
+     * Sets (or, with {@code append}, extends) the text of the focused / first editable field.
+     * ACTION_SET_TEXT replaces the whole content; appending mirrors what typing through an
+     * IME does, which is the contract the host's send_text has always had.
+     */
+    public static boolean setText(AccessibilityService service, String text, boolean append) {
+        AccessibilityNodeInfo inputNode = HierarchyDumper.findInputNode(service);
+        if (inputNode == null) {
+            Log.w(TAG, "No editable/focused input node found for setText");
+            return false;
+        }
+        try {
+            String value = text == null ? "" : text;
+            if (append) {
+                CharSequence existing = inputNode.getText();
+                boolean showingHint = false;
+                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+                    try {
+                        showingHint = inputNode.isShowingHintText();
+                    } catch (Throwable ignored) {}
+                }
+                if (existing != null && existing.length() > 0 && !showingHint) {
+                    value = existing.toString() + value;
+                }
+            }
+            Bundle args = new Bundle();
+            args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, value);
+            return inputNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args);
+        } catch (Throwable t) {
+            Log.w(TAG, "performAction ACTION_SET_TEXT failed", t);
+            return false;
+        } finally {
+            HierarchyDumper.safeRecycle(inputNode);
+        }
+    }
+
+    public static boolean clearText(AccessibilityService service) {
+        return setText(service, "");
+    }
+
+    public static boolean performGlobalAction(AccessibilityService service, String actionName) {
+        if (actionName == null) return false;
+        int actionId;
+        String lower = actionName.toLowerCase();
+        switch (lower) {
+            case "back":
+                actionId = AccessibilityService.GLOBAL_ACTION_BACK;
+                break;
+            case "home":
+                actionId = AccessibilityService.GLOBAL_ACTION_HOME;
+                break;
+            case "recents":
+                actionId = AccessibilityService.GLOBAL_ACTION_RECENTS;
+                break;
+            case "notifications":
+                actionId = AccessibilityService.GLOBAL_ACTION_NOTIFICATIONS;
+                break;
+            case "quick_settings":
+                actionId = AccessibilityService.GLOBAL_ACTION_QUICK_SETTINGS;
+                break;
+            case "power_dialog":
+                actionId = AccessibilityService.GLOBAL_ACTION_POWER_DIALOG;
+                break;
+            case "toggle_split_screen":
+                actionId = AccessibilityService.GLOBAL_ACTION_TOGGLE_SPLIT_SCREEN;
+                break;
+            case "lock_screen":
+                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
+                    actionId = AccessibilityService.GLOBAL_ACTION_LOCK_SCREEN;
+                } else {
+                    return false;
+                }
+                break;
+            case "take_screenshot":
+                if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
+                    actionId = AccessibilityService.GLOBAL_ACTION_TAKE_SCREENSHOT;
+                } else {
+                    return false;
+                }
+                break;
+            default:
+                return false;
+        }
+        return service.performGlobalAction(actionId);
+    }
+
+    private static boolean dispatchSynchronous(
+            final AccessibilityService service,
+            final GestureDescription gesture,
+            long timeoutMs
+    ) {
+        final CountDownLatch latch = new CountDownLatch(1);
+        final AtomicBoolean result = new AtomicBoolean(false);
+
+        MAIN_HANDLER.post(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    service.dispatchGesture(gesture, new AccessibilityService.GestureResultCallback() {
+                        @Override
+                        public void onCompleted(GestureDescription gestureDescription) {
+                            result.set(true);
+                            latch.countDown();
+                        }
+
+                        @Override
+                        public void onCancelled(GestureDescription gestureDescription) {
+                            result.set(false);
+                            latch.countDown();
+                        }
+                    }, null);
+                } catch (Throwable t) {
+                    Log.w(TAG, "dispatchGesture threw exception", t);
+                    result.set(false);
+                    latch.countDown();
+                }
+            }
+        });
+
+        try {
+            return latch.await(timeoutMs, TimeUnit.MILLISECONDS) && result.get();
+        } catch (InterruptedException e) {
+            return false;
+        }
+    }
+
+    /**
+     * Write text to the system clipboard. Clipboard writes are allowed from any
+     * app on every API level (only reads are restricted since Android 10), so this
+     * is the IME-free path the host uses for multiline and non-ASCII input:
+     * set the clip here, then KEYCODE_PASTE over adb.
+     */
+    public static boolean setClipboard(final AccessibilityService service, final String text) {
+        final CountDownLatch latch = new CountDownLatch(1);
+        final AtomicBoolean ok = new AtomicBoolean(false);
+        MAIN_HANDLER.post(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    ClipboardManager cm = (ClipboardManager) service.getSystemService(Context.CLIPBOARD_SERVICE);
+                    if (cm != null) {
+                        cm.setPrimaryClip(ClipData.newPlainText("artemis", text == null ? "" : text));
+                        ok.set(true);
+                    }
+                } catch (Throwable t) {
+                    Log.w(TAG, "setClipboard failed: " + t.getMessage());
+                } finally {
+                    latch.countDown();
+                }
+            }
+        });
+        try {
+            latch.await(2000, TimeUnit.MILLISECONDS);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+        return ok.get();
+    }
+}