@11agents/cli 0.1.24 → 0.1.26

package/mobile-runtime/python/src/device_control/publishers/facebook_adb.py

@@ -0,0 +1,494 @@
+from __future__ import annotations
+
+import re
+import time
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+from device_control.adb import AdbClient
+from device_control.models import Device, PublishRecord, utc_now_iso
+from device_control.publishers import ui_helpers as ui
+from device_control.publishers.tiktok_adb import TikTokAdbPublisher
+from device_control.store import append_publish_record
+
+
+FACEBOOK_LITE_PACKAGE = "com.facebook.lite"
+
+
+class NeedsHumanError(RuntimeError):
+    pass
+
+
+@dataclass
+class FacebookAdbPublishResult:
+    device_id: str
+    status: str
+    record_id: str
+    started_at: str
+    ended_at: str
+    duration_seconds: int
+    post_type: str
+    text: str
+    screenshot_path: str
+    remote_media_path: str = ""
+    link_url: str = ""
+    error: str = ""
+
+
+@dataclass(frozen=True)
+class VisualPoint:
+    x: int
+    y: int
+    confidence: float
+    source: str
+
+
+class FacebookAdbPublisher:
+    """Facebook Lite text-post publisher for the D03 POC layout."""
+
+    def __init__(
+        self,
+        adb: AdbClient,
+        *,
+        records_path: str | Path = "data/publish_records.jsonl",
+        artifact_root: str | Path = "artifacts/screenshots",
+    ) -> None:
+        self.adb = adb
+        self.records_path = Path(records_path)
+        self.artifact_root = Path(artifact_root)
+        self.media_helper = TikTokAdbPublisher(adb, records_path=records_path, artifact_root=artifact_root)
+
+    def publish_text(
+        self,
+        device: Device,
+        *,
+        text: str,
+        account_id: str,
+        dry_run: bool = False,
+        app_package: str = FACEBOOK_LITE_PACKAGE,
+    ) -> FacebookAdbPublishResult:
+        return self.publish_post(
+            device,
+            post_type="text",
+            text=text,
+            account_id=account_id,
+            dry_run=dry_run,
+            app_package=app_package,
+        )
+
+    def publish_post(
+        self,
+        device: Device,
+        *,
+        post_type: str,
+        text: str = "",
+        account_id: str,
+        media_path: str | Path | None = None,
+        link_url: str = "",
+        dry_run: bool = False,
+        app_package: str = FACEBOOK_LITE_PACKAGE,
+    ) -> FacebookAdbPublishResult:
+        post_type = _normalize_post_type(post_type)
+        local: Path | None = Path(media_path).expanduser() if media_path else None
+        if post_type in {"image", "video"}:
+            if local is None or not local.exists():
+                raise FileNotFoundError(f"media not found: {media_path}")
+            if ui.media_kind_from_path(local) != post_type:
+                raise ValueError(f"post_type={post_type} does not match media file suffix: {local}")
+        if post_type == "link":
+            _require_adb_text_safe(link_url, "link_url")
+        if post_type == "text":
+            _require_adb_text_safe(text, "text")
+        else:
+            _require_adb_text_safe(text, "text", required=False)
+        publish_text = _facebook_text_body(text=text, link_url=link_url)
+
+        started_epoch = int(time.time())
+        started_at = _now_shanghai()
+        stamp = datetime.now(ZoneInfo("Asia/Shanghai")).strftime("%Y%m%d-%H%M%S")
+        run_dir = self.artifact_root / f"{device.device_id}-facebook-{stamp}"
+        run_dir.mkdir(parents=True, exist_ok=True)
+        record_id = f"pub_{stamp.replace('-', '')}_{device.device_id.lower()}_facebook"
+        remote = ""
+        if local is not None:
+            remote = f"/sdcard/DCIM/Camera/groupctl-{device.device_id}-facebook-{stamp}{local.suffix}"
+
+        try:
+            self._prepare_device(device)
+            if local is not None:
+                self.media_helper._push_media(device, local, remote)
+            self._open_home(device, app_package, run_dir)
+            self._open_composer(device, run_dir)
+            if publish_text:
+                self._input_text(device, publish_text, run_dir)
+            if local is not None:
+                self._attach_media(device, run_dir, post_type)
+
+            if dry_run:
+                final_shot = run_dir / "dry-run-post-form.png"
+                self.adb.screenshot(device.adb_serial, final_shot)
+                status = "dry_run"
+            else:
+                self._tap_post(device, run_dir)
+                self._wait_for_publish_complete(device, run_dir)
+                final_shot = run_dir / "after-post.png"
+                self.adb.screenshot(device.adb_serial, final_shot)
+                status = "published"
+                append_publish_record(
+                    self.records_path,
+                    PublishRecord(
+                        record_id=record_id,
+                        platform="facebook",
+                        account_id=account_id,
+                        device_id=device.device_id,
+                        post_type=post_type,
+                        local_media_path=str(local) if local else "",
+                        remote_media_path=remote,
+                        caption=publish_text,
+                        published_at=utc_now_iso(),
+                        result_screenshot_path=str(final_shot),
+                        status="published",
+                    ),
+                )
+
+            return FacebookAdbPublishResult(
+                device_id=device.device_id,
+                status=status,
+                record_id=record_id,
+                started_at=started_at,
+                ended_at=_now_shanghai(),
+                duration_seconds=int(time.time()) - started_epoch,
+                post_type=post_type,
+                text=publish_text,
+                remote_media_path=remote,
+                link_url=link_url,
+                screenshot_path=str(final_shot),
+            )
+        except NeedsHumanError as exc:
+            return self._failure_result(
+                device,
+                run_dir,
+                record_id,
+                started_at,
+                started_epoch,
+                publish_text,
+                post_type=post_type,
+                remote_media_path=remote,
+                link_url=link_url,
+                status="needs_human",
+                error=str(exc),
+            )
+        except Exception as exc:
+            return self._failure_result(
+                device,
+                run_dir,
+                record_id,
+                started_at,
+                started_epoch,
+                publish_text,
+                post_type=post_type,
+                remote_media_path=remote,
+                link_url=link_url,
+                status="failed",
+                error=str(exc),
+            )
+
+    def _failure_result(
+        self,
+        device: Device,
+        run_dir: Path,
+        record_id: str,
+        started_at: str,
+        started_epoch: int,
+        text: str,
+        *,
+        post_type: str,
+        remote_media_path: str = "",
+        link_url: str = "",
+        status: str,
+        error: str,
+    ) -> FacebookAdbPublishResult:
+        failure_shot = run_dir / f"{status}.png"
+        self.adb.screenshot(device.adb_serial, failure_shot)
+        return FacebookAdbPublishResult(
+            device_id=device.device_id,
+            status=status,
+            record_id=record_id,
+            started_at=started_at,
+            ended_at=_now_shanghai(),
+            duration_seconds=int(time.time()) - started_epoch,
+            post_type=post_type,
+            text=text,
+            remote_media_path=remote_media_path,
+            link_url=link_url,
+            screenshot_path=str(failure_shot),
+            error=error,
+        )
+
+    def _prepare_device(self, device: Device) -> None:
+        self._ok(self.adb.wake(device.adb_serial), "wake device")
+        time.sleep(1)
+        self.adb.swipe(device.adb_serial, 540, 1850, 540, 550, 350)
+        time.sleep(1)
+
+    def _open_home(self, device: Device, app_package: str, run_dir: Path) -> None:
+        ui.grant_publish_permissions(self.adb, device, app_package)
+        self.adb.shell(device.adb_serial, "am", "force-stop", app_package)
+        time.sleep(1)
+        self._ok(self.adb.launch_package(device.adb_serial, app_package), f"launch {app_package}")
+        time.sleep(7)
+        self.adb.screenshot(device.adb_serial, run_dir / "home.png")
+        if app_package not in self.adb.current_focus(device.adb_serial):
+            raise NeedsHumanError(f"Facebook is not foreground after launch; check install/login state for {app_package}")
+
+    def _open_composer(self, device: Device, run_dir: Path) -> None:
+        self._tap(device, 95, 180, 6)
+        home_shot = run_dir / "home-ready.png"
+        self.adb.screenshot(device.adb_serial, home_shot)
+        composer_entry = _locate_home_composer(home_shot) or VisualPoint(540, 480, 0.0, "fallback")
+        (run_dir / "composer-entry.txt").write_text(
+            f"x={composer_entry.x} y={composer_entry.y} confidence={composer_entry.confidence:.3f} source={composer_entry.source}\n",
+            encoding="utf-8",
+        )
+        self._tap(device, composer_entry.x, composer_entry.y, 4)
+        composer_shot = run_dir / "composer.png"
+        self.adb.screenshot(device.adb_serial, composer_shot)
+        if not self._is_compose_form(device, run_dir / "composer.xml") and not _locate_blue_post_button(composer_shot):
+            raise NeedsHumanError("Facebook composer not detected; check login, verification, prompts, or layout calibration")
+
+    def _input_text(self, device: Device, text: str, run_dir: Path) -> None:
+        self._tap(device, 170, 520, 1)
+        result = self.adb.input_text(device.adb_serial, text)
+        if not result.ok:
+            self.adb.screenshot(device.adb_serial, run_dir / "text-input-failed.png")
+            raise RuntimeError(f"text input failed: {result.stderr or result.stdout}")
+        time.sleep(1)
+        self.adb.keyevent(device.adb_serial, "KEYCODE_BACK")
+        time.sleep(1)
+        self.adb.screenshot(device.adb_serial, run_dir / "after-text.png")
+
+    def _attach_media(self, device: Device, run_dir: Path, post_type: str) -> None:
+        self._tap(device, 220, 1225, 3)
+        ui.tap_permission_prompt_if_present(self.adb, device, run_dir / "media-permission.xml")
+        picker_xml = run_dir / "media-picker.xml"
+        root = self._dump_optional(device, picker_xml)
+        if root is not None:
+            nodes = ui.sorted_visible_nodes(
+                ui.find_nodes(root, ids={"com.facebook.lite:id/gallery_item_image"}, id_suffixes={"gallery_item_image"})
+            )
+            if nodes:
+                ui.tap_node(self.adb, device, nodes[0], sleep_s=3)
+                self.adb.screenshot(device.adb_serial, run_dir / "after-media-select.png")
+                return
+        self.adb.screenshot(device.adb_serial, run_dir / "media-picker-not-found.png")
+        raise NeedsHumanError(f"Facebook {post_type} media picker did not expose a selectable gallery item")
+
+    def _tap_post(self, device: Device, run_dir: Path) -> None:
+        post_shot = run_dir / "before-post-locator.png"
+        self.adb.screenshot(device.adb_serial, post_shot)
+        post_button = _locate_blue_post_button(post_shot)
+        (run_dir / "post-button.txt").write_text(
+            (
+                f"x={post_button.x} y={post_button.y} confidence={post_button.confidence:.3f} source={post_button.source}\n"
+                if post_button
+                else "not_found\n"
+            ),
+            encoding="utf-8",
+        )
+        point = post_button or VisualPoint(540, 1845, 0.0, "fallback")
+        self._tap(device, point.x, point.y, 0)
+
+    def _wait_for_publish_complete(self, device: Device, run_dir: Path) -> None:
+        last_xml = run_dir / "after-post-poll.xml"
+        time.sleep(8)
+        for attempt in range(8):
+            time.sleep(3)
+            poll_shot = run_dir / f"after-post-poll-{attempt + 1}.png"
+            self.adb.screenshot(device.adb_serial, poll_shot)
+            if not self._is_compose_form(device, last_xml) and not _locate_blue_post_button(poll_shot):
+                return
+        raise RuntimeError("Facebook still appears to be on the composer after tapping POST")
+
+    def _is_compose_form(self, device: Device, xml_path: Path) -> bool:
+        dump = self.adb.dump_ui(device.adb_serial, xml_path)
+        if not dump.ok:
+            return False
+        try:
+            root = ET.parse(xml_path).getroot()
+        except ET.ParseError:
+            return False
+        for node in root.iter("node"):
+            if node.attrib.get("clickable") != "true":
+                continue
+            bounds = node.attrib.get("bounds", "")
+            match = re.fullmatch(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds)
+            if not match:
+                continue
+            left, top, right, bottom = [int(part) for part in match.groups()]
+            if left <= 70 and right >= 1000 and 1760 <= top <= 1820 and 1870 <= bottom <= 1925:
+                return True
+        return False
+
+    def _dump_optional(self, device: Device, xml_path: Path) -> ET.Element | None:
+        dump = self.adb.dump_ui(device.adb_serial, xml_path)
+        if not dump.ok:
+            return None
+        try:
+            return ET.parse(xml_path).getroot()
+        except ET.ParseError:
+            return None
+
+    def _tap(self, device: Device, x: int, y: int, sleep_s: int) -> None:
+        self._ok(self.adb.tap(device.adb_serial, x, y), f"tap {x},{y}")
+        if sleep_s:
+            time.sleep(sleep_s)
+
+    @staticmethod
+    def _ok(result, action: str) -> None:
+        if not result.ok:
+            raise RuntimeError(f"{action} failed: {result.stderr or result.stdout}")
+
+
+def _require_adb_text_safe(value: str, field_name: str, *, required: bool = True) -> None:
+    if required and not value.strip():
+        raise ValueError(f"{field_name} is required")
+    if not value:
+        return
+    if "\n" in value or "\r" in value or "\t" in value:
+        raise ValueError(f"{field_name} must be a single line for the current Facebook Lite POC")
+    if not value.isascii():
+        raise ValueError(f"{field_name} must be ASCII for the current Facebook Lite POC")
+
+
+def _normalize_post_type(value: str) -> str:
+    text = value.strip().lower().replace("_", "-")
+    if text in {"text", "link", "image", "photo", "video"}:
+        return "image" if text == "photo" else text
+    raise ValueError("Facebook post_type must be one of: text, link, image, video")
+
+
+def _facebook_text_body(*, text: str, link_url: str) -> str:
+    if text and link_url:
+        return f"{text} {link_url}"
+    return text or link_url
+
+
+def _locate_home_composer(image_path: Path) -> VisualPoint | None:
+    return _locate_wide_band(
+        image_path,
+        x_range=(0.12, 0.9),
+        y_range=(0.14, 0.32),
+        min_width_ratio=0.42,
+        color_predicate=_is_light_neutral_gray,
+        source="visual_home_composer",
+    )
+
+
+def _locate_blue_post_button(image_path: Path) -> VisualPoint | None:
+    return _locate_wide_band(
+        image_path,
+        x_range=(0.02, 0.98),
+        y_range=(0.6, 0.9),
+        min_width_ratio=0.58,
+        color_predicate=_is_facebook_blue,
+        source="visual_post_button",
+    )
+
+
+def _locate_wide_band(
+    image_path: Path,
+    *,
+    x_range: tuple[float, float],
+    y_range: tuple[float, float],
+    min_width_ratio: float,
+    color_predicate,
+    source: str,
+) -> VisualPoint | None:
+    try:
+        from PIL import Image
+    except Exception:
+        return None
+
+    try:
+        image = Image.open(image_path).convert("RGB")
+    except Exception:
+        return None
+
+    width, height = image.size
+    pixels = image.load()
+    x0 = max(0, int(width * x_range[0]))
+    x1 = min(width, int(width * x_range[1]))
+    y0 = max(0, int(height * y_range[0]))
+    y1 = min(height, int(height * y_range[1]))
+    min_width = int(width * min_width_ratio)
+
+    rows: list[tuple[int, int, int, int]] = []
+    for y in range(y0, y1, 2):
+        best_start = -1
+        best_end = -1
+        run_start = -1
+        for x in range(x0, x1):
+            if color_predicate(pixels[x, y]):
+                if run_start < 0:
+                    run_start = x
+                continue
+            if run_start >= 0:
+                if x - run_start > best_end - best_start:
+                    best_start = run_start
+                    best_end = x
+                run_start = -1
+        if run_start >= 0 and x1 - run_start > best_end - best_start:
+            best_start = run_start
+            best_end = x1
+        best_width = best_end - best_start
+        if best_width >= min_width:
+            rows.append((y, best_start, best_end, best_width))
+
+    if not rows:
+        return None
+
+    groups: list[list[tuple[int, int, int, int]]] = []
+    for row in rows:
+        if not groups or row[0] - groups[-1][-1][0] > 4:
+            groups.append([row])
+        else:
+            groups[-1].append(row)
+
+    best_group: list[tuple[int, int, int, int]] | None = None
+    best_score = 0
+    for group in groups:
+        band_height = group[-1][0] - group[0][0] + 2
+        avg_width = sum(row[3] for row in group) / len(group)
+        if band_height < 20:
+            continue
+        score = int(avg_width * band_height)
+        if score > best_score:
+            best_score = score
+            best_group = group
+
+    if not best_group:
+        return None
+
+    weighted_x = sum(((row[1] + row[2]) / 2) * row[3] for row in best_group)
+    total_width = sum(row[3] for row in best_group)
+    center_x = int(weighted_x / total_width)
+    center_y = int((best_group[0][0] + best_group[-1][0]) / 2)
+    confidence = min(1.0, best_score / max(1, (width * (y1 - y0) * 0.2)))
+    return VisualPoint(center_x, center_y, confidence, source)
+
+
+def _is_light_neutral_gray(rgb: tuple[int, int, int]) -> bool:
+    red, green, blue = rgb
+    return 220 <= red <= 248 and 220 <= green <= 248 and 220 <= blue <= 248 and max(rgb) - min(rgb) <= 10
+
+
+def _is_facebook_blue(rgb: tuple[int, int, int]) -> bool:
+    red, green, blue = rgb
+    return red <= 80 and 70 <= green <= 170 and blue >= 180 and blue - red >= 110 and blue - green >= 50
+
+
+def _now_shanghai() -> str:
+    return datetime.now(ZoneInfo("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S %z")