@qubiqlabs/mobiflow 0.9.0 → 1.0.0

package/src/mobiflow/cloud/maestro_cloud.py

@@ -0,0 +1,141 @@
+"""First-party Maestro Cloud via `maestro cloud` CLI."""
+
+from __future__ import annotations
+
+import re
+import tempfile
+from pathlib import Path
+from typing import Callable
+
+from mobiflow.cloud.base import (
+    CloudProvider,
+    CloudRunRequest,
+    CloudRunResult,
+    resolve_credentials,
+)
+
+ProgressFn = Callable[[str], None] | None
+
+
+async def run_maestro_cloud(
+    req: CloudRunRequest,
+    *,
+    progress: ProgressFn = None,
+    artifact_dir: Path | None = None,
+) -> CloudRunResult:
+    """Upload app + flows with the official Maestro Cloud CLI."""
+    from mobiflow.maestro import _run_cmd, resolve_maestro_binary
+
+    binary = resolve_maestro_binary()
+    if not binary:
+        return CloudRunResult(
+            ok=False,
+            provider="maestro",
+            status="error",
+            error="maestro_not_installed",
+            stdout="",
+            stderr="Maestro CLI not found (needed for `maestro cloud`)",
+        )
+
+    try:
+        creds = resolve_credentials(
+            CloudProvider.MAESTRO,
+            username_env=req.username_env,
+            access_key_env=req.access_key_env,
+        )
+    except ValueError as e:
+        return CloudRunResult(
+            ok=False,
+            provider="maestro",
+            status="error",
+            error="credentials_missing",
+            stdout="",
+            stderr=str(e),
+        )
+
+    api_key = creds.access_key
+    app_file = (req.app_path or "").strip()
+    app_binary_id = (req.app_url or "").strip() if not app_file else ""
+
+    if not app_file and not app_binary_id:
+        return CloudRunResult(
+            ok=False,
+            provider="maestro",
+            status="error",
+            error="app_missing",
+            stdout="",
+            stderr="Set device.app_path (.apk/.ipa) or device.app_url (app binary id)",
+        )
+
+    with tempfile.TemporaryDirectory(prefix="mobiflow-maestro-cloud-") as tmp:
+        root = Path(tmp)
+        flow_path = root / (req.flow_name or "flow.yaml")
+        flow_path.write_text(req.flow_yaml, encoding="utf-8")
+        for rel, body in (req.scripts or {}).items():
+            sp = root / rel
+            sp.parent.mkdir(parents=True, exist_ok=True)
+            sp.write_text(body, encoding="utf-8")
+
+        args = [
+            binary,
+            "cloud",
+            "--api-key",
+            api_key,
+            "--flows",
+            str(flow_path),
+            "--format",
+            "JUNIT",
+            "--name",
+            req.build_name or req.project or "MobiFlow",
+        ]
+        if app_file:
+            args.extend(["--app-file", app_file])
+        elif app_binary_id:
+            args.extend(["--app-binary-id", app_binary_id])
+        for did in req.devices or []:
+            if did and ("-" in did or "_" in did):
+                args.extend(["--device-model", did])
+                break
+        out_xml = root / "report.xml"
+        args.extend(["--output", str(out_xml)])
+
+        if progress:
+            progress("Uploading to Maestro Cloud (`maestro cloud`)…")
+
+        result = await _run_cmd(
+            args, timeout=float(req.timeout_s or 1800), cwd=str(root)
+        )
+        stdout = str(result.get("stdout") or "")
+        stderr = str(result.get("stderr") or "")
+        ok = bool(result.get("ok"))
+
+        dashboard = ""
+        for line in (stdout + "\n" + stderr).splitlines():
+            m = re.search(r"https?://\S+", line)
+            if m and "maestro" in m.group(0).lower():
+                dashboard = m.group(0).rstrip(").,]")
+                break
+
+        if artifact_dir is not None:
+            artifact_dir.mkdir(parents=True, exist_ok=True)
+            if out_xml.is_file():
+                (artifact_dir / "maestro-junit.xml").write_text(
+                    out_xml.read_text(encoding="utf-8"), encoding="utf-8"
+                )
+            (artifact_dir / "maestro-cloud-stdout.txt").write_text(
+                stdout, encoding="utf-8"
+            )
+            (artifact_dir / "maestro-cloud-stderr.txt").write_text(
+                stderr, encoding="utf-8"
+            )
+
+        return CloudRunResult(
+            ok=ok,
+            provider="maestro",
+            status="passed" if ok else "failed",
+            error=None if ok else str(result.get("error") or "maestro_cloud_failed"),
+            stdout=stdout,
+            stderr=stderr,
+            dashboard_url=dashboard,
+            media_dir=str(artifact_dir) if artifact_dir else "",
+        )

package/src/mobiflow/cloud/media.py

@@ -0,0 +1,269 @@
+"""Download cloud-lab screenshots / video / logs into local run artifacts."""
+
+from __future__ import annotations
+
+import logging
+import re
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+import httpx
+
+from mobiflow.cloud.base import CloudCredentials
+
+logger = logging.getLogger(__name__)
+
+_URL_KEYS = {
+    "screenshots",
+    "screenshot",
+    "video",
+    "video_url",
+    "device_log",
+    "device_logs",
+    "devicelogs",
+    "maestro_log",
+    "maestro_logs",
+    "maestrologs",
+    "network_log",
+    "network_logs",
+    "instrumentation_log",
+}
+
+
+def collect_media_urls(payload: Any, *, limit: int = 40) -> list[dict[str, str]]:
+    """Walk nested JSON and collect media URL entries ``{kind, url}``."""
+    found: list[dict[str, str]] = []
+    seen: set[str] = set()
+
+    def _add(kind: str, url: str) -> None:
+        u = (url or "").strip()
+        if not u.startswith("http"):
+            return
+        # Strip video time anchors for download
+        clean = u.split("#", 1)[0]
+        if clean in seen:
+            return
+        seen.add(clean)
+        found.append({"kind": kind, "url": clean})
+
+    def _walk(node: Any) -> None:
+        if len(found) >= limit:
+            return
+        if isinstance(node, dict):
+            for key, val in node.items():
+                lk = str(key).lower()
+                if lk in _URL_KEYS and isinstance(val, str):
+                    kind = "screenshot" if "screenshot" in lk else (
+                        "video" if "video" in lk else "log"
+                    )
+                    _add(kind, val)
+                else:
+                    _walk(val)
+        elif isinstance(node, list):
+            for item in node:
+                _walk(item)
+
+    _walk(payload)
+    return found[:limit]
+
+
+def extract_session_ids(build_payload: dict[str, Any]) -> list[str]:
+    """Pull session ids from a BrowserStack Maestro build status payload."""
+    ids: list[str] = []
+    sessions = build_payload.get("sessions") or []
+    if isinstance(sessions, list):
+        for sess in sessions:
+            if isinstance(sess, dict):
+                sid = sess.get("id") or sess.get("session_id") or sess.get("sessionId")
+                if sid:
+                    ids.append(str(sid))
+            elif isinstance(sess, str):
+                ids.append(sess)
+    # Sometimes nested under devices
+    devices = build_payload.get("devices") or []
+    if isinstance(devices, list):
+        for dev in devices:
+            if not isinstance(dev, dict):
+                continue
+            for sess in dev.get("sessions") or []:
+                if isinstance(sess, dict):
+                    sid = sess.get("id") or sess.get("session_id")
+                    if sid:
+                        ids.append(str(sid))
+    # Deduplicate preserving order
+    out: list[str] = []
+    seen: set[str] = set()
+    for sid in ids:
+        if sid not in seen:
+            seen.add(sid)
+            out.append(sid)
+    return out
+
+
+def guess_filename(kind: str, url: str, index: int) -> str:
+    path = urlparse(url).path.rstrip("/")
+    leaf = Path(path).name or kind
+    # BrowserStack often ends with bare "screenshot" / "video"
+    if leaf in {"screenshot", "screenshots", "video", "devicelogs", "maestrologs"}:
+        ext = {
+            "screenshot": ".png",
+            "video": ".mp4",
+            "log": ".txt",
+        }.get(kind, ".bin")
+        return f"{index:02d}-{kind}{ext}"
+    if "." not in leaf:
+        ext = ".png" if kind == "screenshot" else (".mp4" if kind == "video" else ".txt")
+        return f"{index:02d}-{leaf}{ext}"
+    return f"{index:02d}-{leaf}"
+
+
+async def download_media_files(
+    client: httpx.AsyncClient,
+    items: list[dict[str, str]],
+    dest_dir: Path,
+    *,
+    auth: tuple[str, str] | None = None,
+    limit: int = 24,
+) -> list[str]:
+    """Download media URLs into dest_dir; return relative filenames written."""
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    written: list[str] = []
+    for i, item in enumerate(items[:limit], start=1):
+        url = item["url"]
+        kind = item.get("kind") or "file"
+        name = guess_filename(kind, url, i)
+        path = dest_dir / name
+        try:
+            resp = await client.get(url, auth=auth, timeout=120.0, follow_redirects=True)
+            if resp.status_code >= 400:
+                logger.warning("Cloud media HTTP %s for %s", resp.status_code, url)
+                continue
+            ctype = (resp.headers.get("content-type") or "").lower()
+            # If we guessed wrong extension for JSON error pages, skip
+            if (
+                "application/json" in ctype
+                and kind in {"screenshot", "video"}
+            ):
+                logger.debug("Skipping JSON media body for %s", url)
+                continue
+            path.write_bytes(resp.content)
+            written.append(name)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Cloud media download failed %s: %s", url, exc)
+    return written
+
+
+async def fetch_browserstack_session(
+    client: httpx.AsyncClient,
+    creds: CloudCredentials,
+    build_id: str,
+    session_id: str,
+) -> dict[str, Any]:
+    url = (
+        "https://api-cloud.browserstack.com/app-automate/maestro/v2/"
+        f"builds/{build_id}/sessions/{session_id}"
+    )
+    resp = await client.get(url, auth=(creds.username, creds.access_key), timeout=60.0)
+    resp.raise_for_status()
+    return resp.json()
+
+
+async def pull_browserstack_media(
+    creds: CloudCredentials,
+    build_id: str,
+    build_payload: dict[str, Any],
+    dest_dir: Path,
+    *,
+    progress: Any = None,
+) -> dict[str, Any]:
+    """Fetch session details + download screenshots/video/logs for a BS build."""
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    urls = collect_media_urls(build_payload)
+    session_payloads: list[dict[str, Any]] = []
+    async with httpx.AsyncClient() as client:
+        for sid in extract_session_ids(build_payload):
+            try:
+                sess = await fetch_browserstack_session(client, creds, build_id, sid)
+                session_payloads.append(sess)
+                urls.extend(collect_media_urls(sess))
+            except Exception as exc:  # noqa: BLE001
+                logger.warning("BrowserStack session %s: %s", sid, exc)
+        # Deduplicate urls
+        dedup: list[dict[str, str]] = []
+        seen: set[str] = set()
+        for item in urls:
+            if item["url"] in seen:
+                continue
+            seen.add(item["url"])
+            dedup.append(item)
+        if progress and dedup:
+            progress(f"Downloading {len(dedup)} cloud media file(s)…")
+        written = await download_media_files(
+            client,
+            dedup,
+            dest_dir,
+            auth=(creds.username, creds.access_key),
+        )
+    # Persist URL index for reports even when download fails
+    index = {
+        "provider": "browserstack",
+        "build_id": build_id,
+        "urls": dedup,
+        "files": written,
+        "sessions": len(session_payloads),
+    }
+    (dest_dir / "media-index.json").write_text(
+        __import__("json").dumps(index, indent=2) + "\n", encoding="utf-8"
+    )
+    return index
+
+
+_HTTP_RE = re.compile(r"https?://[^\s\"'<>]+")
+
+
+def extract_urls_from_text(text: str) -> list[dict[str, str]]:
+    """Best-effort media URLs from HyperExecute CLI stdout/stderr."""
+    out: list[dict[str, str]] = []
+    for match in _HTTP_RE.finditer(text or ""):
+        url = match.group(0).rstrip(".,);'\"")
+        lower = url.lower()
+        if any(x in lower for x in ("screenshot", "video", ".png", ".mp4", "artifact")):
+            kind = "video" if "video" in lower or lower.endswith(".mp4") else "screenshot"
+            out.append({"kind": kind, "url": url.split("#", 1)[0]})
+    return out
+
+
+async def pull_testmu_media(
+    stdout: str,
+    stderr: str,
+    dest_dir: Path,
+    *,
+    creds: CloudCredentials | None = None,
+    progress: Any = None,
+) -> dict[str, Any]:
+    """Download any media URLs found in TestMu/HyperExecute output."""
+    dest_dir.mkdir(parents=True, exist_ok=True)
+    urls = extract_urls_from_text(f"{stdout}\n{stderr}")
+    written: list[str] = []
+    if urls:
+        if progress:
+            progress(f"Downloading {len(urls)} TestMu media URL(s)…")
+        auth = (creds.username, creds.access_key) if creds else None
+        async with httpx.AsyncClient() as client:
+            written = await download_media_files(client, urls, dest_dir, auth=auth)
+    index = {
+        "provider": "testmu",
+        "urls": urls,
+        "files": written,
+        "note": (
+            "HyperExecute media is best-effort from CLI output; "
+            "open dashboard_url when empty."
+            if not written
+            else ""
+        ),
+    }
+    (dest_dir / "media-index.json").write_text(
+        __import__("json").dumps(index, indent=2) + "\n", encoding="utf-8"
+    )
+    return index

package/src/mobiflow/cloud/runner.py

@@ -0,0 +1,156 @@
+"""Dispatch Maestro runs to BrowserStack or TestMu."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+from typing import Any
+
+from mobiflow.cloud.base import (
+    CloudProvider,
+    CloudRunRequest,
+    CloudRunResult,
+    devices_from_config,
+    is_cloud_provider,
+    normalize_provider,
+    resolve_credentials,
+)
+from mobiflow.cloud.browserstack import run_browserstack
+from mobiflow.cloud.maestro_cloud import run_maestro_cloud
+from mobiflow.cloud.testmu import resolve_hyperexecute_binary, run_testmu
+
+ProgressFn = Callable[[str], None] | None
+
+
+def cloud_readiness(device_cfg: Any) -> dict[str, Any]:
+    """Report whether cloud credentials / tools are ready for device.provider."""
+    provider_raw = getattr(device_cfg, "provider", "local") or "local"
+    try:
+        provider = normalize_provider(provider_raw)
+    except ValueError as e:
+        return {
+            "provider": str(provider_raw),
+            "cloud": False,
+            "ready": False,
+            "message": str(e),
+        }
+    if not is_cloud_provider(provider):
+        return {
+            "provider": "local",
+            "cloud": False,
+            "ready": True,
+            "message": "Using local adb / simulators",
+        }
+
+    info: dict[str, Any] = {
+        "provider": provider.value,
+        "cloud": True,
+        "ready": False,
+        "app_path": getattr(device_cfg, "app_path", "") or "",
+        "app_url": getattr(device_cfg, "app_url", "") or "",
+        "device_id": getattr(device_cfg, "device_id", None),
+        "real_mobile": bool(getattr(device_cfg, "real_mobile", True)),
+    }
+    try:
+        creds = resolve_credentials(
+            provider,
+            username_env=getattr(device_cfg, "username_env", "") or "",
+            access_key_env=getattr(device_cfg, "access_key_env", "") or "",
+        )
+        info["username_env"] = creds.username_env
+        info["access_key_env"] = creds.access_key_env
+        info["credentials"] = True
+    except ValueError as e:
+        info["credentials"] = False
+        info["message"] = str(e)
+        return info
+
+    if provider == CloudProvider.TESTMU:
+        he = resolve_hyperexecute_binary()
+        info["hyperexecute"] = he or ""
+        # CLI can be auto-downloaded on first run
+        info["hyperexecute_installable"] = True
+    if provider == CloudProvider.MAESTRO:
+        from mobiflow.maestro import resolve_maestro_binary
+
+        info["maestro_cli"] = resolve_maestro_binary() or ""
+
+    has_app = bool(info["app_path"] or info["app_url"])
+    if not has_app:
+        info["message"] = "Set device.app_path or device.app_url for cloud runs"
+        return info
+
+    device_label = (info.get("device_id") or "").strip() or "(provider default)"
+    info["ready"] = True
+    info["message"] = f"{provider.value} credentials OK · device={device_label}"
+    return info
+
+
+def request_from_device_config(
+    device_cfg: Any,
+    *,
+    flow_yaml: str,
+    scripts: dict[str, str] | None = None,
+    platform: str | None = None,
+    device_id: str | None = None,
+    timeout_s: int | None = None,
+    flow_name: str = "flow.yaml",
+) -> CloudRunRequest:
+    provider = normalize_provider(getattr(device_cfg, "provider", "local"))
+    plat = (platform or getattr(device_cfg, "platform", "android") or "android").lower()
+    did = device_id if device_id is not None else getattr(device_cfg, "device_id", None)
+    devices = devices_from_config(did, platform=plat, provider=provider)
+    # Cloud runs can be long; default 30 min unless overridden
+    t = timeout_s if timeout_s is not None else 1800
+    run_timeout = getattr(device_cfg, "cloud_timeout_s", None)
+    if run_timeout:
+        t = int(run_timeout)
+    return CloudRunRequest(
+        provider=provider,
+        platform=plat,
+        flow_yaml=flow_yaml,
+        scripts=dict(scripts or {}),
+        devices=devices,
+        app_path=(getattr(device_cfg, "app_path", "") or "").strip(),
+        app_url=(getattr(device_cfg, "app_url", "") or "").strip(),
+        project=(getattr(device_cfg, "cloud_project", "") or "MobiFlow").strip()
+        or "MobiFlow",
+        build_name=(getattr(device_cfg, "cloud_build_name", "") or "").strip(),
+        real_mobile=bool(getattr(device_cfg, "real_mobile", True)),
+        username_env=(getattr(device_cfg, "username_env", "") or "").strip(),
+        access_key_env=(getattr(device_cfg, "access_key_env", "") or "").strip(),
+        timeout_s=int(t),
+        poll_interval_s=float(getattr(device_cfg, "poll_interval_s", 15.0) or 15.0),
+        local=bool(getattr(device_cfg, "browserstack_local", False)),
+        flow_name=flow_name,
+    )
+
+
+async def run_on_cloud(
+    request: CloudRunRequest,
+    *,
+    progress: ProgressFn = None,
+    artifact_dir: Any = None,
+) -> CloudRunResult:
+    if request.provider == CloudProvider.BROWSERSTACK:
+        return await run_browserstack(
+            request, progress=progress, artifact_dir=artifact_dir
+        )
+    if request.provider == CloudProvider.TESTMU:
+        return await run_testmu(
+            request, progress=progress, artifact_dir=artifact_dir
+        )
+    if request.provider == CloudProvider.MAESTRO:
+        return await run_maestro_cloud(
+            request, progress=progress, artifact_dir=artifact_dir
+        )
+    return CloudRunResult(
+        ok=False,
+        provider=str(request.provider),
+        error="not_cloud",
+        stderr=f"Provider {request.provider} is not a cloud lab",
+    )
+
+
+def env_present(name: str) -> bool:
+    return bool(os.environ.get(name, "").strip())