@11agents/cli 0.1.24 → 0.1.25

package/mobile-runtime/python/src/device_control/publishers/tiktok_adb.py

@@ -0,0 +1,477 @@
+from __future__ import annotations
+
+import base64
+import shlex
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+from device_control.adb import AdbClient
+from device_control.models import Device, PublishRecord, new_id, utc_now_iso
+from device_control.publishers import ui_helpers as ui
+from device_control.store import append_publish_record
+
+
+TIKTOK_PACKAGE = "com.zhiliaoapp.musically"
+TIKTOK_BLOCKING_TEXTS = (
+    "你已达到每日上限",
+    "每日上限",
+    "想喝点水吗",
+    "暂时返回",
+    "daily limit",
+    "try again later",
+)
+TIKTOK_STORY_TEXTS = (
+    "你的想法",
+    "你的限时动态",
+    "24 小时内可见",
+    "24小时内可见",
+)
+TIKTOK_HOME_TEXTS = (
+    "首页",
+    "好友",
+    "收件箱",
+    "主页",
+)
+TIKTOK_PUBLISH_CONFIRMATION_TEXTS = (
+    "视频已发布",
+    "已发布",
+    "所有人可以查看",
+    "posted",
+    "published",
+)
+TIKTOK_NEXT_TEXTS = (
+    "下一步",
+    "Next",
+)
+TIKTOK_PREVIEW_TEXTS = (
+    "一键成片",
+    "AutoCut",
+)
+
+
+class TikTokNeedsHumanError(RuntimeError):
+    """Raised when TikTok blocks the publish flow with an account/platform state."""
+
+
+@dataclass
+class TikTokAdbPublishResult:
+    device_id: str
+    status: str
+    record_id: str
+    started_at: str
+    ended_at: str
+    duration_seconds: int
+    remote_media_path: str
+    screenshot_path: str
+    error: str = ""
+
+
+class TikTokAdbPublisher:
+    """ADB-only TikTok publisher for the Samsung SM-G9910 POC devices."""
+
+    def __init__(
+        self,
+        adb: AdbClient,
+        *,
+        records_path: str | Path = "data/publish_records.jsonl",
+        artifact_root: str | Path = "artifacts/screenshots",
+    ) -> None:
+        self.adb = adb
+        self.records_path = Path(records_path)
+        self.artifact_root = Path(artifact_root)
+
+    def publish_video(
+        self,
+        device: Device,
+        *,
+        video_path: str | Path,
+        account_id: str,
+        caption: str = "",
+        dry_run: bool = False,
+        allow_caption_fallback: bool = False,
+    ) -> TikTokAdbPublishResult:
+        local = Path(video_path).expanduser()
+        if not local.exists():
+            raise FileNotFoundError(f"video not found: {local}")
+
+        started_epoch = int(time.time())
+        started_at = _now_shanghai()
+        stamp = datetime.now(ZoneInfo("Asia/Shanghai")).strftime("%Y%m%d-%H%M%S")
+        run_dir = self.artifact_root / f"{device.device_id}-tiktok-{stamp}"
+        run_dir.mkdir(parents=True, exist_ok=True)
+        remote = f"/sdcard/DCIM/Camera/groupctl-{device.device_id}-tiktok-{stamp}{local.suffix or '.mp4'}"
+        record_id = f"pub_{stamp.replace('-', '')}_{device.device_id.lower()}_tiktok"
+
+        try:
+            self._prepare_device(device)
+            self._push_media(device, local, remote)
+            self._launch_tiktok(device)
+            self._open_create(device, run_dir)
+            self._open_gallery(device, run_dir)
+            self._select_first_video(device, run_dir)
+            self._next_from_gallery(device, run_dir)
+            self._next_from_editor(device, run_dir)
+            if caption:
+                self._try_caption(device, caption, run_dir, allow_fallback=allow_caption_fallback)
+            if dry_run:
+                final_shot = run_dir / "dry-run-post-form.png"
+                self.adb.screenshot(device.adb_serial, final_shot)
+                status = "dry_run"
+            else:
+                self._tap_publish(device, run_dir)
+                final_shot = self._wait_for_publish_confirmation(device, run_dir)
+                status = "published"
+
+            ended_epoch = int(time.time())
+            ended_at = _now_shanghai()
+            if status == "published":
+                record = PublishRecord(
+                    record_id=record_id,
+                    platform="tiktok",
+                    account_id=account_id,
+                    device_id=device.device_id,
+                    post_type="video",
+                    local_media_path=str(local),
+                    remote_media_path=remote,
+                    caption=caption,
+                    published_at=utc_now_iso(),
+                    result_screenshot_path=str(final_shot),
+                    status="published",
+                )
+                append_publish_record(self.records_path, record)
+            return TikTokAdbPublishResult(
+                device_id=device.device_id,
+                status=status,
+                record_id=record_id,
+                started_at=started_at,
+                ended_at=ended_at,
+                duration_seconds=ended_epoch - started_epoch,
+                remote_media_path=remote,
+                screenshot_path=str(final_shot),
+            )
+        except TikTokNeedsHumanError as exc:
+            ended_epoch = int(time.time())
+            failure_shot = run_dir / "needs-human.png"
+            self.adb.screenshot(device.adb_serial, failure_shot)
+            return TikTokAdbPublishResult(
+                device_id=device.device_id,
+                status="needs_human",
+                record_id=record_id,
+                started_at=started_at,
+                ended_at=_now_shanghai(),
+                duration_seconds=ended_epoch - started_epoch,
+                remote_media_path=remote,
+                screenshot_path=str(failure_shot),
+                error=str(exc),
+            )
+        except Exception as exc:
+            ended_epoch = int(time.time())
+            failure_shot = run_dir / "failed.png"
+            self.adb.screenshot(device.adb_serial, failure_shot)
+            return TikTokAdbPublishResult(
+                device_id=device.device_id,
+                status="failed",
+                record_id=record_id,
+                started_at=started_at,
+                ended_at=_now_shanghai(),
+                duration_seconds=ended_epoch - started_epoch,
+                remote_media_path=remote,
+                screenshot_path=str(failure_shot),
+                error=str(exc),
+            )
+
+    def _prepare_device(self, device: Device) -> None:
+        self._ok(self.adb.wake(device.adb_serial), "wake device")
+        time.sleep(1)
+        self.adb.swipe(device.adb_serial, 540, 1850, 540, 550, 350)
+        time.sleep(1)
+        self._ok(self.adb.keyevent(device.adb_serial, "KEYCODE_HOME"), "go home")
+        time.sleep(1)
+
+    def _push_media(self, device: Device, local: Path, remote: str) -> None:
+        self._ok(self.adb.shell(device.adb_serial, "mkdir", "-p", "/sdcard/DCIM/Camera"), "make camera dir")
+        if local.stat().st_size <= 64 * 1024 and self._push_media_via_base64(device, local, remote):
+            self.adb.media_scan(device.adb_serial, remote)
+            time.sleep(1)
+            return
+
+        pushed = self.adb.push(device.adb_serial, local, remote)
+        if not pushed.ok and not self._remote_file_matches(device, remote, local.stat().st_size):
+            streamed = self._push_media_via_base64(device, local, remote)
+            if not streamed:
+                self._ok(pushed, "push media")
+        self.adb.media_scan(device.adb_serial, remote)
+        time.sleep(1)
+
+    def _remote_file_matches(self, device: Device, remote: str, expected_size: int) -> bool:
+        result = self.adb.shell(device.adb_serial, "wc", "-c", remote)
+        if not result.ok:
+            return False
+        first = result.stdout.strip().split(maxsplit=1)[0]
+        return first.isdigit() and int(first) == expected_size
+
+    def _push_media_via_base64(self, device: Device, local: Path, remote: str) -> bool:
+        tmp = f"{remote}.b64"
+        expected_size = local.stat().st_size
+        encoded = base64.b64encode(local.read_bytes()).decode("ascii")
+        self.adb.shell(device.adb_serial, "rm", "-f", tmp, remote)
+
+        for i in range(0, len(encoded), 500):
+            chunk = encoded[i : i + 500]
+            result = self._remote_sh(
+                device,
+                f"printf '%s' {shlex.quote(chunk)} >> {shlex.quote(tmp)}",
+            )
+            if not result.ok:
+                return False
+
+        decode = self._remote_sh(device, f"base64 -d {shlex.quote(tmp)} > {shlex.quote(remote)}")
+        self.adb.shell(device.adb_serial, "rm", "-f", tmp)
+        return decode.ok and self._remote_file_matches(device, remote, expected_size)
+
+    def _remote_sh(self, device: Device, command: str):
+        return self.adb.run("-s", device.adb_serial, "shell", f"sh -c {shlex.quote(command)}")
+
+    def _launch_tiktok(self, device: Device) -> None:
+        ui.grant_publish_permissions(self.adb, device, TIKTOK_PACKAGE)
+        self.adb.shell(device.adb_serial, "am", "force-stop", TIKTOK_PACKAGE)
+        time.sleep(1)
+        self._ok(self.adb.launch_package(device.adb_serial, TIKTOK_PACKAGE), "launch TikTok")
+        time.sleep(5)
+
+    def _open_create(self, device: Device, run_dir: Path) -> None:
+        root = self._dump_and_check(device, run_dir, "before-create")
+        create_node = ui.find_node(
+            root,
+            id_suffixes={"neq"},
+            descs={"创建", "Create"},
+            clickable_only=True,
+        )
+        if create_node is None:
+            self._tap(device, 540, 2225, 0)
+        else:
+            ui.tap_node(self.adb, device, create_node)
+        time.sleep(5)
+        self.adb.screenshot(device.adb_serial, run_dir / "create.png")
+        root = self._dump_and_check(device, run_dir, "create")
+        text = ui.all_text(root)
+        if _contains_any(text, TIKTOK_HOME_TEXTS):
+            # Home text may remain underneath transient create overlays. Do not fail here unless
+            # a known TikTok account/platform block is present; the gallery check below is strict.
+            return
+
+    def _open_gallery(self, device: Device, run_dir: Path) -> None:
+        # D01 and D02 currently expose different create-page entry points.
+        # Try the D01 bottom-left upload first, then the D02 right-side album icon.
+        root = self._dump_and_check(device, run_dir, "before-gallery-tap")
+        upload_node = ui.find_node(root, id_suffixes={"upload_hot_area"}, clickable_only=True)
+        if upload_node is None:
+            self._tap(device, 105, 2050, 0)
+        else:
+            ui.tap_node(self.adb, device, upload_node)
+        time.sleep(2)
+        shot = run_dir / "after-gallery-tap-1.png"
+        self.adb.screenshot(device.adb_serial, shot)
+        self._assert_not_home_or_wrong_surface(
+            self._dump_and_check(device, run_dir, "after-gallery-tap-1"),
+            "open TikTok gallery",
+        )
+        ui.tap_permission_prompt_if_present(self.adb, device, run_dir / "gallery-permission.xml")
+        if device.device_id == "D02":
+            self._tap(device, 985, 610, 3)
+            self.adb.screenshot(device.adb_serial, run_dir / "after-gallery-tap-2.png")
+            self._assert_not_home_or_wrong_surface(
+                self._dump_and_check(device, run_dir, "after-gallery-tap-2"),
+                "open TikTok gallery on D02",
+            )
+        time.sleep(2)
+
+    def _select_first_video(self, device: Device, run_dir: Path) -> None:
+        self._tap(device, 175, 540, 1)
+        self.adb.screenshot(device.adb_serial, run_dir / "after-select-first.png")
+        self._dump_and_check(device, run_dir, "after-select-first")
+
+    def _next_from_gallery(self, device: Device, run_dir: Path) -> None:
+        self._tap_next_button(device, run_dir, "before-gallery-next", fallback=(790, 2208), sleep_s=5)
+        self.adb.screenshot(device.adb_serial, run_dir / "after-gallery-next.png")
+        root = self._dump_and_check(device, run_dir, "after-gallery-next")
+        if self._is_media_preview(root):
+            raise RuntimeError("next from TikTok media preview failed: still on the video selection preview")
+
+    def _next_from_editor(self, device: Device, run_dir: Path) -> None:
+        self._tap_next_button(device, run_dir, "before-editor-next", fallback=(795, 2234), sleep_s=6)
+        self.adb.screenshot(device.adb_serial, run_dir / "post-form.png")
+        root = self._dump_and_check(device, run_dir, "post-form")
+        self._assert_not_home_or_wrong_surface(root, "reach TikTok video post form")
+        if self._is_media_preview(root):
+            raise RuntimeError("next from TikTok editor failed: still on the video selection preview")
+
+    def _try_caption(self, device: Device, caption: str, run_dir: Path, *, allow_fallback: bool) -> None:
+        self._tap(device, 160, 275, 1)
+        result = self.adb.input_text(device.adb_serial, caption)
+        if not result.ok:
+            self.adb.screenshot(device.adb_serial, run_dir / "caption-input-failed.png")
+            (run_dir / "caption-warning.txt").write_text(
+                "ADB input text failed. Caption is recorded in PublishRecord but may not be visible in TikTok UI.\n"
+                f"caption={caption}\n"
+                f"stderr={result.stderr}\n"
+                f"stdout={result.stdout}\n",
+                encoding="utf-8",
+            )
+            if not allow_fallback:
+                raise RuntimeError(
+                    "caption input failed; stop before publish. "
+                    "Use ASCII caption, install a Unicode input bridge, or pass --allow-caption-fallback."
+                )
+        time.sleep(1)
+        self.adb.keyevent(device.adb_serial, "KEYCODE_BACK")
+        time.sleep(1)
+        self.adb.screenshot(device.adb_serial, run_dir / "after-caption.png")
+
+    def _tap_publish(self, device: Device, run_dir: Path) -> None:
+        root = self._dump_and_check(device, run_dir, "before-publish")
+        node = self._find_visible_text_node(root, ("发布", "Post"), clickable_only=True)
+        if node is None:
+            self._tap(device, 790, 2208, 0)
+        else:
+            ui.tap_node(self.adb, device, node)
+
+    def _tap_next_button(
+        self,
+        device: Device,
+        run_dir: Path,
+        dump_name: str,
+        *,
+        fallback: tuple[int, int],
+        sleep_s: int,
+    ) -> None:
+        root = self._dump_and_check(device, run_dir, dump_name)
+        node = self._find_visible_text_node(root, TIKTOK_NEXT_TEXTS, clickable_only=False)
+        if node is None:
+            self._tap(device, fallback[0], fallback[1], 0)
+        else:
+            ui.tap_node(self.adb, device, node)
+        if sleep_s:
+            time.sleep(sleep_s)
+
+    def _find_visible_text_node(self, root, texts: tuple[str, ...], *, clickable_only: bool):
+        candidates = []
+        for node in root.iter("node"):
+            if clickable_only and node.attrib.get("clickable") != "true":
+                continue
+            if not ui.is_visible(node):
+                continue
+            text = node.attrib.get("text", "")
+            desc = node.attrib.get("content-desc", "")
+            if text in texts or desc in texts:
+                bounds = ui.bounds(node)
+                if bounds:
+                    candidates.append((bounds, node))
+        if not candidates:
+            return None
+
+        def key(item):
+            left, top, right, bottom = item[0]
+            return bottom, right, top, left
+
+        return sorted(candidates, key=key, reverse=True)[0][1]
+
+    def _is_media_preview(self, root) -> bool:
+        text = ui.all_text(root)
+        return (
+            _contains_any(text, TIKTOK_NEXT_TEXTS)
+            and _contains_any(text, TIKTOK_PREVIEW_TEXTS)
+            and self._has_exact_text(root, ("选择", "Select"))
+        )
+
+    def _has_exact_text(self, root, texts: tuple[str, ...]) -> bool:
+        for node in root.iter("node"):
+            if node.attrib.get("text", "") in texts or node.attrib.get("content-desc", "") in texts:
+                return True
+        return False
+
+    def _is_published_video_surface(self, root) -> bool:
+        text = ui.all_text(root)
+        has_video_actions = _contains_any(text, ("点赞视频", "分享视频", "阅读或添加评论"))
+        has_recent_timestamp = _contains_any(text, ("秒前", "分钟前", "minute ago", "seconds ago"))
+        return has_video_actions and has_recent_timestamp
+
+    def _wait_for_publish_confirmation(self, device: Device, run_dir: Path) -> Path:
+        deadline = time.time() + 60
+        last_shot = run_dir / "after-post.png"
+        last_text = ""
+        while time.time() < deadline:
+            time.sleep(2)
+            self.adb.screenshot(device.adb_serial, last_shot)
+            root = self._dump_and_check(device, run_dir, "after-post")
+            last_text = ui.all_text(root)
+            if ui.tap_permission_prompt_if_present(self.adb, device, run_dir / "after-post-permission.xml"):
+                time.sleep(2)
+                self.adb.screenshot(device.adb_serial, last_shot)
+                continue
+            if _contains_any(last_text, TIKTOK_PUBLISH_CONFIRMATION_TEXTS):
+                return last_shot
+            if self._is_published_video_surface(root):
+                return last_shot
+            if _contains_any(last_text, TIKTOK_STORY_TEXTS):
+                raise RuntimeError("TikTok publish route is on story/thoughts surface, not the video post flow")
+        raise RuntimeError(
+            "TikTok publish completion was not confirmed within 60s; "
+            "not writing a published record. Last UI text: "
+            f"{_short_text(last_text)}"
+        )
+
+    def _dump_and_check(self, device: Device, run_dir: Path, name: str):
+        xml_path = run_dir / f"{name}.xml"
+        root = ui.dump_ui(self.adb, device, xml_path)
+        text = ui.all_text(root)
+        blocking = _first_contained(text, TIKTOK_BLOCKING_TEXTS)
+        if blocking:
+            raise TikTokNeedsHumanError(f"TikTok publish blocked by visible prompt: {blocking}")
+        if ("访问你的麦克风" in text or "访问麦克风" in text) and "直播" in text:
+            raise RuntimeError("TikTok opened the LIVE microphone permission surface, not the video publish flow")
+        return root
+
+    def _assert_not_home_or_wrong_surface(self, root, action: str) -> None:
+        text = ui.all_text(root)
+        story = _first_contained(text, TIKTOK_STORY_TEXTS)
+        if story and not _contains_any(text, TIKTOK_NEXT_TEXTS):
+            raise RuntimeError(f"{action} failed: TikTok opened story/thoughts surface ({story}), not video publish")
+        home_matches = [part for part in TIKTOK_HOME_TEXTS if part in text]
+        if len(home_matches) >= 3:
+            raise RuntimeError(f"{action} failed: TikTok is still on home/feed UI ({', '.join(home_matches)})")
+
+    def _tap(self, device: Device, x: int, y: int, sleep_s: int) -> None:
+        self._ok(self.adb.tap(device.adb_serial, x, y), f"tap {x},{y}")
+        if sleep_s:
+            time.sleep(sleep_s)
+
+    @staticmethod
+    def _ok(result, action: str) -> None:
+        if not result.ok:
+            raise RuntimeError(f"{action} failed: {result.stderr or result.stdout}")
+
+
+def _now_shanghai() -> str:
+    return datetime.now(ZoneInfo("Asia/Shanghai")).strftime("%Y-%m-%d %H:%M:%S %z")
+
+
+def _contains_any(text: str, needles: tuple[str, ...]) -> bool:
+    lowered = text.lower()
+    return any(needle.lower() in lowered for needle in needles)
+
+
+def _first_contained(text: str, needles: tuple[str, ...]) -> str:
+    lowered = text.lower()
+    for needle in needles:
+        if needle.lower() in lowered:
+            return needle
+    return ""
+
+
+def _short_text(text: str, limit: int = 240) -> str:
+    compact = " ".join(part.strip() for part in text.splitlines() if part.strip())
+    return compact[:limit]