@11agents/cli 0.1.24 → 0.1.25

package/mobile-runtime/python/src/device_control/publishers/tiktok_appium.py

@@ -0,0 +1,259 @@
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+from device_control.adb import AdbClient
+from device_control.appium_client import AppiumClient
+from device_control.models import Device, PublishRecord, utc_now_iso
+from device_control.publishers import ui_helpers as ui
+from device_control.publishers.tiktok_adb import TIKTOK_PACKAGE, TikTokAdbPublisher
+from device_control.store import append_publish_record
+
+
+TIKTOK_ACTIVITY = "com.ss.android.ugc.aweme.splash.SplashActivity"
+
+
+@dataclass
+class TikTokAppiumPublishResult:
+    device_id: str
+    status: str
+    record_id: str
+    started_at: str
+    ended_at: str
+    duration_seconds: int
+    remote_media_path: str
+    screenshot_path: str
+    error: str = ""
+
+
+class TikTokAppiumPublisher:
+    """TikTok publisher that uses Appium for UI steps and ADB for device/media primitives."""
+
+    def __init__(
+        self,
+        adb: AdbClient,
+        *,
+        appium_server: str = "http://127.0.0.1:4723",
+        records_path: str | Path = "data/publish_records.jsonl",
+        artifact_root: str | Path = "artifacts/screenshots",
+    ) -> None:
+        self.adb = adb
+        self.appium_server = appium_server
+        self.records_path = Path(records_path)
+        self.artifact_root = Path(artifact_root)
+        self.media_helper = TikTokAdbPublisher(adb, records_path=records_path, artifact_root=artifact_root)
+
+    def publish_video(
+        self,
+        device: Device,
+        *,
+        video_path: str | Path,
+        account_id: str,
+        caption: str = "",
+        dry_run: bool = False,
+    ) -> TikTokAppiumPublishResult:
+        local = Path(video_path).expanduser()
+        if not local.exists():
+            raise FileNotFoundError(f"video not found: {local}")
+
+        started_epoch = int(time.time())
+        started_at = _now_shanghai()
+        stamp = datetime.now(ZoneInfo("Asia/Shanghai")).strftime("%Y%m%d-%H%M%S")
+        run_dir = self.artifact_root / f"{device.device_id}-tiktok-appium-{stamp}"
+        run_dir.mkdir(parents=True, exist_ok=True)
+        remote = f"/sdcard/DCIM/Camera/groupctl-{device.device_id}-tiktok-appium-{stamp}{local.suffix or '.mp4'}"
+        record_id = f"pub_{stamp.replace('-', '')}_{device.device_id.lower()}_tiktok_appium"
+        appium = AppiumClient(self.appium_server)
+
+        try:
+            self._prepare_device(device)
+            ui.grant_publish_permissions(self.adb, device, TIKTOK_PACKAGE)
+            self.media_helper._push_media(device, local, remote)
+            self.adb.shell(device.adb_serial, "am", "force-stop", TIKTOK_PACKAGE)
+            time.sleep(1)
+
+            appium.start_session(_capabilities(device.adb_serial))
+            time.sleep(5)
+
+            appium.source(run_dir / "home.xml")
+            self._click_first(appium, [("accessibility id", "创建"), ("id", f"{TIKTOK_PACKAGE}:id/neq")], "open create")
+            time.sleep(3)
+            self.adb.screenshot(device.adb_serial, run_dir / "create.png")
+            self._raise_if_live_surface(device, run_dir, "after opening TikTok create")
+
+            self._tap_album_entry(device)
+            time.sleep(5)
+            self._raise_if_live_surface(device, run_dir, "after opening TikTok gallery")
+            ui.tap_permission_prompt_if_present(self.adb, device, run_dir / "gallery-permission.xml")
+            self.adb.screenshot(device.adb_serial, run_dir / "gallery.png")
+            self._raise_if_live_surface(device, run_dir, "after opening TikTok gallery")
+
+            self._tap_first_gallery_video(device)
+            time.sleep(1)
+            self.adb.screenshot(device.adb_serial, run_dir / "gallery-selected.png")
+            self._raise_if_live_surface(device, run_dir, "after selecting TikTok media")
+
+            self._tap_gallery_next(device)
+            time.sleep(6)
+            self.adb.screenshot(device.adb_serial, run_dir / "editor.png")
+            self._raise_if_live_surface(device, run_dir, "after TikTok gallery next")
+
+            self._tap_editor_next(device)
+            time.sleep(7)
+            final_shot = run_dir / "post-form.png"
+            self.adb.screenshot(device.adb_serial, final_shot)
+            self._raise_if_live_surface(device, run_dir, "after TikTok editor next")
+
+            if caption:
+                self._set_caption(appium, device, caption, run_dir)
+
+            if dry_run:
+                status = "dry_run"
+            else:
+                self._tap_publish_button(device)
+                final_shot = self.media_helper._wait_for_publish_confirmation(device, run_dir)
+                status = "published"
+
+            ended_epoch = int(time.time())
+            if status == "published":
+                append_publish_record(
+                    self.records_path,
+                    PublishRecord(
+                        record_id=record_id,
+                        platform="tiktok",
+                        account_id=account_id,
+                        device_id=device.device_id,
+                        post_type="video",
+                        local_media_path=str(local),
+                        remote_media_path=remote,
+                        caption=caption,
+                        published_at=utc_now_iso(),
+                        result_screenshot_path=str(final_shot),
+                        status="published",
+                    ),
+                )
+            return TikTokAppiumPublishResult(
+                device_id=device.device_id,
+                status=status,
+                record_id=record_id,
+                started_at=started_at,
+                ended_at=_now_shanghai(),
+                duration_seconds=ended_epoch - started_epoch,
+                remote_media_path=remote,
+                screenshot_path=str(final_shot),
+            )
+        except Exception as exc:
+            failure_shot = run_dir / "failed.png"
+            try:
+                appium.screenshot(failure_shot)
+            except Exception:
+                self.adb.screenshot(device.adb_serial, failure_shot)
+            return TikTokAppiumPublishResult(
+                device_id=device.device_id,
+                status="failed",
+                record_id=record_id,
+                started_at=started_at,
+                ended_at=_now_shanghai(),
+                duration_seconds=int(time.time()) - started_epoch,
+                remote_media_path=remote,
+                screenshot_path=str(failure_shot),
+                error=str(exc),
+            )
+        finally:
+            try:
+                appium.delete_session()
+            except Exception:
+                pass
+
+    def _prepare_device(self, device: Device) -> None:
+        self._ok(self.adb.wake(device.adb_serial), "wake device")
+        time.sleep(1)
+        self.adb.swipe(device.adb_serial, 540, 1850, 540, 550, 350)
+        time.sleep(1)
+
+    def _click_first(self, appium: AppiumClient, locators: list[tuple[str, str]], action: str) -> None:
+        last_error = ""
+        for using, value in locators:
+            try:
+                appium.click(appium.find_element(using, value))
+                return
+            except Exception as exc:
+                last_error = str(exc)
+        raise RuntimeError(f"{action} failed: {last_error}")
+
+    def _set_caption(self, appium: AppiumClient, device: Device, caption: str, run_dir: Path) -> None:
+        try:
+            field = appium.find_element("id", f"{TIKTOK_PACKAGE}:id/gjp")
+            appium.click(field)
+            time.sleep(1)
+            appium.send_keys(field, caption)
+            time.sleep(1)
+            self.adb.keyevent(device.adb_serial, "KEYCODE_BACK")
+            time.sleep(1)
+            self.adb.screenshot(device.adb_serial, run_dir / "after-caption.png")
+        except Exception as exc:
+            self.adb.screenshot(device.adb_serial, run_dir / "caption-failed.png")
+            raise RuntimeError(f"caption input failed: {exc}") from exc
+
+    def _tap_album_entry(self, device: Device) -> None:
+        self._ok(self.adb.tap(device.adb_serial, 105, 2050), "open album")
+
+    def _tap_first_gallery_video(self, device: Device) -> None:
+        self._ok(self.adb.tap(device.adb_serial, 305, 455), "select first gallery video")
+
+    def _tap_gallery_next(self, device: Device) -> None:
+        self._ok(self.adb.tap(device.adb_serial, 790, 2208), "next from gallery")
+
+    def _tap_editor_next(self, device: Device) -> None:
+        self._ok(self.adb.tap(device.adb_serial, 795, 2234), "next from editor")
+
+    def _tap_publish_button(self, device: Device) -> None:
+        self._ok(self.adb.tap(device.adb_serial, 790, 2240), "publish")
+
+    def _raise_if_live_surface(self, device: Device, run_dir: Path, action: str) -> None:
+        try:
+            root = ui.dump_ui(self.adb, device, run_dir / f"{_slug(action)}.xml")
+        except Exception:
+            return
+        text = ui.all_text(root)
+        if (
+            ("访问你的麦克风" in text or "访问麦克风" in text)
+            and ("直播" in text or "LIVE" in text.upper())
+        ):
+            raise RuntimeError(f"{action} failed: TikTok opened the LIVE microphone permission surface, not video publish")
+        if "直播" in text and "发布" in text and "创作" in text and "访问麦克风" in text:
+            raise RuntimeError(f"{action} failed: TikTok is on the LIVE tab, not video publish")
+
+    @staticmethod
+    def _ok(result, action: str) -> None:
+        if not result.ok:
+            raise RuntimeError(f"{action} failed: {result.stderr or result.stdout}")
+
+
+def _capabilities(serial: str) -> dict:
+    return {
+        "platformName": "Android",
+        "appium:automationName": "UiAutomator2",
+        "appium:udid": serial,
+        "appium:noReset": True,
+        "appium:newCommandTimeout": 120,
+        "appium:disableWindowAnimation": True,
+        "appium:skipDeviceInitialization": True,
+        "appium:ignoreHiddenApiPolicyError": True,
+        "appium:disableSuppressAccessibilityService": True,
+        "appium:settings[enableNotificationListener]": False,
+        "appium:appPackage": TIKTOK_PACKAGE,
+        "appium:appActivity": TIKTOK_ACTIVITY,
+    }
+
+
+def _now_shanghai() -> str:
+    return datetime.now(ZoneInfo("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S %z")
+
+
+def _slug(value: str) -> str:
+    return "".join(char.lower() if char.isalnum() else "-" for char in value).strip("-")

package/mobile-runtime/python/src/device_control/publishers/ui_helpers.py

@@ -0,0 +1,372 @@
+from __future__ import annotations
+
+import re
+import time
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Iterable
+
+from device_control.adb import AdbClient
+from device_control.models import Device
+
+
+Bounds = tuple[int, int, int, int]
+PUBLISH_RUNTIME_PERMISSIONS = (
+    "android.permission.CAMERA",
+    "android.permission.RECORD_AUDIO",
+    "android.permission.READ_EXTERNAL_STORAGE",
+    "android.permission.WRITE_EXTERNAL_STORAGE",
+    "android.permission.READ_MEDIA_IMAGES",
+    "android.permission.READ_MEDIA_VIDEO",
+    "android.permission.READ_MEDIA_AUDIO",
+    "android.permission.READ_CONTACTS",
+    "android.permission.WRITE_CONTACTS",
+    "android.permission.GET_ACCOUNTS",
+    "android.permission.ACCESS_COARSE_LOCATION",
+    "android.permission.ACCESS_FINE_LOCATION",
+    "android.permission.POST_NOTIFICATIONS",
+)
+PUBLISH_APP_OPS = (
+    "CAMERA",
+    "RECORD_AUDIO",
+    "READ_EXTERNAL_STORAGE",
+    "WRITE_EXTERNAL_STORAGE",
+    "READ_MEDIA_IMAGES",
+    "READ_MEDIA_VIDEO",
+    "READ_MEDIA_AUDIO",
+    "READ_CONTACTS",
+    "WRITE_CONTACTS",
+    "GET_ACCOUNTS",
+    "ACCESS_COARSE_LOCATION",
+    "ACCESS_FINE_LOCATION",
+    "POST_NOTIFICATION",
+)
+PERMISSION_ALLOW_IDS = {
+    "com.android.permissioncontroller:id/permission_allow_button",
+    "com.android.permissioncontroller:id/permission_allow_foreground_only_button",
+    "com.android.permissioncontroller:id/permission_allow_one_time_button",
+    "com.android.permissioncontroller:id/permission_allow_all_button",
+}
+PERMISSION_ALLOW_ID_SUFFIXES = {
+    "permission_allow_button",
+    "permission_allow_foreground_only_button",
+    "permission_allow_one_time_button",
+    "permission_allow_all_button",
+}
+PERMISSION_ALLOW_TEXTS = {
+    "允许",
+    "确定",
+    "始终允许",
+    "仅在使用该应用时允许",
+    "仅限本次使用",
+    "全部照片和视频",
+    "允许访问所有照片和视频",
+    "允许访问所有照片",
+    "允许所有照片",
+    "Allow",
+    "OK",
+    "While using the app",
+    "Only this time",
+    "Allow all photos",
+    "Allow all photos and videos",
+}
+PERMISSION_ALLOW_PARTIALS = {
+    "始终允许",
+    "在使用该应用时允许",
+    "本次使用",
+    "全部照片",
+    "照片和视频",
+    "allow",
+    "while using",
+    "only this time",
+    "all photos",
+    "all photos and videos",
+}
+PERMISSION_DENY_PARTIALS = {
+    "不允许",
+    "拒绝",
+    "deny",
+    "don't allow",
+    "do not allow",
+    "not allow",
+}
+PERMISSION_LIMITED_MEDIA_PARTIALS = {
+    "选择照片",
+    "选择要允许",
+    "仅允许访问照片",
+    "仅允许访问部分",
+    "select photos",
+    "selected photos",
+    "select photos and videos",
+}
+
+
+def dump_ui(adb: AdbClient, device: Device, xml_path: Path) -> ET.Element:
+    result = adb.dump_ui(device.adb_serial, xml_path)
+    if not result.ok:
+        raise RuntimeError(f"dump ui failed: {result.stderr or result.stdout}")
+    return ET.parse(xml_path).getroot()
+
+
+def bounds(node: ET.Element) -> Bounds | None:
+    match = re.fullmatch(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", node.attrib.get("bounds", ""))
+    if not match:
+        return None
+    return tuple(int(part) for part in match.groups())
+
+
+def center(node_or_bounds: ET.Element | Bounds) -> tuple[int, int]:
+    value = bounds(node_or_bounds) if isinstance(node_or_bounds, ET.Element) else node_or_bounds
+    if not value:
+        raise RuntimeError("node has no valid bounds")
+    left, top, right, bottom = value
+    return (left + right) // 2, (top + bottom) // 2
+
+
+def is_visible(node: ET.Element) -> bool:
+    value = bounds(node)
+    if not value:
+        return False
+    left, top, right, bottom = value
+    return right > left and bottom > top
+
+
+def all_text(root: ET.Element) -> str:
+    parts: list[str] = []
+    for node in root.iter("node"):
+        for key in ("text", "content-desc", "resource-id"):
+            value = node.attrib.get(key, "")
+            if value:
+                parts.append(value)
+    return "\n".join(parts)
+
+
+def contains_any(text: str, needles: Iterable[str]) -> bool:
+    lowered = text.lower()
+    return any(needle.lower() in lowered for needle in needles)
+
+
+def find_node(
+    root: ET.Element,
+    *,
+    ids: set[str] | None = None,
+    id_suffixes: set[str] | None = None,
+    texts: set[str] | None = None,
+    descs: set[str] | None = None,
+    partial_texts: set[str] | None = None,
+    clickable_only: bool = False,
+    visible_only: bool = True,
+) -> ET.Element | None:
+    for node in root.iter("node"):
+        if clickable_only and node.attrib.get("clickable") != "true":
+            continue
+        if visible_only and not is_visible(node):
+            continue
+        resource_id = node.attrib.get("resource-id", "")
+        text = node.attrib.get("text", "")
+        desc = node.attrib.get("content-desc", "")
+        if ids and resource_id in ids:
+            return node
+        if id_suffixes and any(resource_id == suffix or resource_id.endswith(f":id/{suffix}") for suffix in id_suffixes):
+            return node
+        if texts and (text in texts or desc in texts):
+            return node
+        if descs and desc in descs:
+            return node
+        if partial_texts:
+            haystack = f"{text}\n{desc}".lower()
+            if any(part.lower() in haystack for part in partial_texts):
+                return node
+    return None
+
+
+def find_nodes(
+    root: ET.Element,
+    *,
+    ids: set[str] | None = None,
+    id_suffixes: set[str] | None = None,
+    partial_texts: set[str] | None = None,
+    visible_only: bool = True,
+) -> list[ET.Element]:
+    nodes: list[ET.Element] = []
+    for node in root.iter("node"):
+        if visible_only and not is_visible(node):
+            continue
+        resource_id = node.attrib.get("resource-id", "")
+        text = node.attrib.get("text", "")
+        desc = node.attrib.get("content-desc", "")
+        if ids and resource_id in ids:
+            nodes.append(node)
+            continue
+        if id_suffixes and any(resource_id == suffix or resource_id.endswith(f":id/{suffix}") for suffix in id_suffixes):
+            nodes.append(node)
+            continue
+        if partial_texts:
+            haystack = f"{text}\n{desc}".lower()
+            if any(part.lower() in haystack for part in partial_texts):
+                nodes.append(node)
+    return nodes
+
+
+def tap_node(adb: AdbClient, device: Device, node: ET.Element, *, sleep_s: float = 0) -> None:
+    x, y = center(node)
+    result = adb.tap(device.adb_serial, x, y)
+    if not result.ok:
+        raise RuntimeError(f"tap {x},{y} failed: {result.stderr or result.stdout}")
+    if sleep_s:
+        time.sleep(sleep_s)
+
+
+def tap_first(
+    adb: AdbClient,
+    device: Device,
+    root: ET.Element,
+    *,
+    ids: set[str] | None = None,
+    id_suffixes: set[str] | None = None,
+    texts: set[str] | None = None,
+    descs: set[str] | None = None,
+    partial_texts: set[str] | None = None,
+    sleep_s: float = 0,
+) -> bool:
+    node = find_node(
+        root,
+        ids=ids,
+        id_suffixes=id_suffixes,
+        texts=texts,
+        descs=descs,
+        partial_texts=partial_texts,
+        clickable_only=False,
+    )
+    if node is None:
+        return False
+    tap_node(adb, device, node, sleep_s=sleep_s)
+    return True
+
+
+def grant_publish_permissions(adb: AdbClient, device_or_serial: Device | str, package: str) -> None:
+    serial = device_or_serial.adb_serial if isinstance(device_or_serial, Device) else device_or_serial
+    for permission in PUBLISH_RUNTIME_PERMISSIONS:
+        adb.shell(serial, "pm", "grant", package, permission)
+    for op in PUBLISH_APP_OPS:
+        adb.shell(serial, "appops", "set", package, op, "allow")
+
+
+def tap_permission_prompt_if_present(
+    adb: AdbClient,
+    device: Device,
+    xml_path: Path,
+    *,
+    retries: int = 2,
+    sleep_s: float = 1.0,
+) -> bool:
+    tapped = False
+    for attempt in range(max(1, retries)):
+        try:
+            root = dump_ui(adb, device, _retry_path(xml_path, attempt))
+        except Exception:
+            return tapped
+        node = find_permission_allow_node(root)
+        if node is None:
+            return tapped
+        tap_node(adb, device, node, sleep_s=sleep_s)
+        tapped = True
+    return tapped
+
+
+def find_permission_allow_node(root: ET.Element) -> ET.Element | None:
+    candidates: list[tuple[int, ET.Element]] = []
+    for node in root.iter("node"):
+        if not is_visible(node):
+            continue
+        text = node.attrib.get("text", "")
+        desc = node.attrib.get("content-desc", "")
+        label = f"{text}\n{desc}".strip()
+        lowered = label.lower()
+        resource_id = node.attrib.get("resource-id", "")
+        if any(part in lowered for part in PERMISSION_DENY_PARTIALS):
+            continue
+        if any(part in lowered for part in PERMISSION_LIMITED_MEDIA_PARTIALS):
+            continue
+        if resource_id in PERMISSION_ALLOW_IDS:
+            candidates.append((300, node))
+            continue
+        if any(resource_id.endswith(f":id/{suffix}") for suffix in PERMISSION_ALLOW_ID_SUFFIXES):
+            candidates.append((280, node))
+            continue
+        if text in PERMISSION_ALLOW_TEXTS or desc in PERMISSION_ALLOW_TEXTS:
+            candidates.append((200, node))
+            continue
+        if any(part.lower() in lowered for part in PERMISSION_ALLOW_PARTIALS):
+            candidates.append((100, node))
+    if not candidates:
+        return None
+    return sorted(candidates, key=lambda item: (item[0], _node_bottom(item[1])), reverse=True)[0][1]
+
+
+def _retry_path(path: Path, attempt: int) -> Path:
+    if attempt == 0:
+        return path
+    return path.with_name(f"{path.stem}-{attempt + 1}{path.suffix}")
+
+
+def _node_bottom(node: ET.Element) -> int:
+    value = bounds(node)
+    if not value:
+        return 0
+    return value[3]
+
+
+def sorted_visible_nodes(nodes: list[ET.Element]) -> list[ET.Element]:
+    def key(node: ET.Element) -> tuple[int, int]:
+        left, top, _right, _bottom = bounds(node) or (10**9, 10**9, 10**9, 10**9)
+        return top, left
+
+    return sorted(nodes, key=key)
+
+
+def wait_for_node(
+    adb: AdbClient,
+    device: Device,
+    xml_path: Path,
+    *,
+    timeout_s: float,
+    interval_s: float = 1,
+    ids: set[str] | None = None,
+    id_suffixes: set[str] | None = None,
+    texts: set[str] | None = None,
+    partial_texts: set[str] | None = None,
+) -> ET.Element | None:
+    deadline = time.time() + timeout_s
+    while time.time() < deadline:
+        root = dump_ui(adb, device, xml_path)
+        node = find_node(root, ids=ids, id_suffixes=id_suffixes, texts=texts, partial_texts=partial_texts)
+        if node is not None:
+            return node
+        time.sleep(interval_s)
+    return None
+
+
+def media_kind_from_path(path: Path) -> str:
+    suffix = path.suffix.lower()
+    if suffix in {".jpg", ".jpeg", ".png", ".webp", ".heic", ".heif"}:
+        return "image"
+    if suffix in {".mp4", ".mov", ".m4v", ".3gp", ".webm"}:
+        return "video"
+    raise ValueError(f"unsupported media suffix: {path.suffix}")
+
+
+def adb_text_safe(text: str) -> bool:
+    allowed = set("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 .,!?:;'\"_@#%()+-/|&=")
+    return all(char in allowed for char in text)
+
+
+def require_adb_text_safe(value: str, field_name: str, *, required: bool = True) -> None:
+    if required and not value.strip():
+        raise ValueError(f"{field_name} is required")
+    if not value:
+        return
+    if "\n" in value or "\r" in value or "\t" in value:
+        raise ValueError(f"{field_name} must be a single line for the current POC")
+    if not value.isascii():
+        raise ValueError(f"{field_name} must be ASCII for the current POC")