android-midscene-automation 0.2.3 → 1.1.0

package/artemis-main/apps/admin_console/routers/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

package/artemis-main/apps/admin_console/routers/media.py

@@ -0,0 +1,198 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+from pathlib import Path
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import FileResponse, HTMLResponse
+
+from artemis.config import IMAGES_DIR, TRACES_PATH, WORKSPACE_ROOT
+
+try:
+    from admin_console.database.repositories.session_repository import session_repo
+    from admin_console.services.media_service import media_service
+except ImportError:
+    from apps.admin_console.database.repositories.session_repository import session_repo
+    from apps.admin_console.services.media_service import media_service
+
+router = APIRouter(tags=["media"])
+
+_VIDEO_MEDIA_TYPES = {
+    ".mp4": "video/mp4",
+    ".webm": "video/webm",
+    ".mkv": "video/x-matroska",
+}
+
+
+def _allowed_media_roots() -> list[Path]:
+    """Directories the media endpoints are permitted to serve from."""
+    roots = []
+    for root in (WORKSPACE_ROOT, TRACES_PATH):
+        try:
+            roots.append(Path(root).resolve())
+        except OSError:
+            continue
+    return roots
+
+
+def _resolve_media_path(raw_path: str, allowed_suffixes: set[str]) -> Path:
+    """Resolve a client-supplied path strictly inside the allowed media roots.
+
+    Every candidate is fully resolved (symlinks, `..`, mixed separators) and
+    then re-checked against the allowed roots and an extension allowlist, so a
+    crafted URL can never address source code, databases, or dotfiles.
+    """
+    roots = _allowed_media_roots()
+    candidate = Path(raw_path)
+    attempts = [candidate] if candidate.is_absolute() else [root / candidate for root in roots]
+    for attempt in attempts:
+        try:
+            resolved = attempt.resolve(strict=True)
+        except (OSError, ValueError):
+            continue
+        if not resolved.is_file():
+            continue
+        if not any(resolved.is_relative_to(root) for root in roots):
+            continue
+        if resolved.suffix.lower() not in allowed_suffixes:
+            raise HTTPException(status_code=403, detail="File type is not served by the media API.")
+        return resolved
+    raise HTTPException(status_code=404, detail="Media file not found")
+
+
+@router.get("/admin", response_class=HTMLResponse)
+@router.get("/debug", response_class=HTMLResponse)
+async def get_admin_index():
+    admin_index = Path(__file__).resolve().parent.parent / "index.html"
+    if admin_index.exists():
+        return admin_index.read_text(encoding="utf-8")
+
+    admin_index_alt = WORKSPACE_ROOT / "apps" / "admin_console" / "index.html"
+    if admin_index_alt.exists():
+        return admin_index_alt.read_text(encoding="utf-8")
+
+    return "<h1>Admin Console index.html not found</h1>"
+
+
+@router.get("/images/{image_name}")
+@router.get("/api/images/{image_name}")
+async def get_image(image_name: str):
+    if not image_name.endswith(".jpg"):
+        image_name += ".jpg"
+    try:
+        images_root = IMAGES_DIR.resolve()
+        image_path = (images_root / image_name).resolve(strict=True)
+    except (OSError, ValueError):
+        raise HTTPException(status_code=404, detail="Image not found")
+    if not image_path.is_file() or not image_path.is_relative_to(images_root):
+        raise HTTPException(status_code=404, detail="Image not found")
+
+    return FileResponse(image_path, media_type="image/jpeg")
+
+
+@router.get("/videos/{video_path:path}")
+async def get_video(video_path: str):
+    path = _resolve_media_path(video_path, set(_VIDEO_MEDIA_TYPES))
+    return FileResponse(path, media_type=_VIDEO_MEDIA_TYPES[path.suffix.lower()])
+
+
+@router.get("/api/sessions/{session_id}/video")
+async def get_session_video(session_id: str):
+    # Blocking work (sqlite, filesystem scan, possible ffmpeg conversion) runs
+    # off the event loop.
+    return await asyncio.to_thread(_get_session_video_sync, session_id)
+
+
+def _get_session_video_sync(session_id: str):
+    video_rec_map = session_repo.get_video_recordings_map()
+    video_idx = media_service.build_video_index()
+    row = session_repo.get_session_by_id(session_id)
+    row_dict = dict(row) if row else {"session_id": session_id}
+    recording = session_repo.get_video_recording_for_session(session_id)
+    recording_status = str((recording or {}).get("status") or "")
+
+    if recording_status in ("recording", "finalizing"):
+        return {
+            "session_id": session_id,
+            "status": "processing",
+            "has_video": False,
+            "video_url": None,
+            "video_segments": [],
+            "retry_after_ms": 750,
+        }
+
+    if recording_status == "failed":
+        v_url = media_service.resolve_video_url(row_dict, video_rec_map, video_idx)
+        if not v_url:
+            return {
+                "session_id": session_id,
+                "status": "failed",
+                "has_video": False,
+                "video_url": None,
+                "video_segments": [],
+                "message": recording.get("error") or "Recording finalization failed",
+            }
+        # If a video was recovered/found, update DB to ready and proceed to serve it
+        session_repo.mark_recording_ready(session_id, v_url)
+
+    v_url = media_service.resolve_video_url(row_dict, video_rec_map, video_idx)
+    video_segments = media_service.resolve_video_segments(v_url)
+    if v_url:
+        version = int(
+            float((recording or {}).get("end_time") or row_dict.get("end_time") or 0) * 1000
+        )
+        separator = "&" if "?" in v_url else "?"
+        versioned_url = f"{v_url}{separator}v={version}" if version else v_url
+        for segment in video_segments:
+            segment_separator = "&" if "?" in segment["url"] else "?"
+            segment["url"] = (
+                f"{segment['url']}{segment_separator}v={version}" if version else segment["url"]
+            )
+        return {
+            "session_id": session_id,
+            "status": "ready",
+            "has_video": True,
+            "video_url": versioned_url,
+            "video_segments": video_segments,
+        }
+
+    return {
+        "session_id": session_id,
+        "status": "unavailable",
+        "has_video": False,
+        "video_url": None,
+        "video_segments": [],
+    }
+
+
+@router.get("/local_file")
+async def get_local_file(path: str):
+    p, media_type = media_service.get_safe_local_file(path)
+    return FileResponse(p, media_type=media_type)
+
+
+@router.get("/api/sessions/{session_id}/plan")
+async def get_task_plan(session_id: str):
+    return {"plan": media_service.get_task_plan_content(session_id)}
+
+
+@router.get("/api/sessions/{session_id}/notes")
+async def get_all_notes(session_id: str):
+    return {"notes": media_service.get_session_notes_content(session_id)}
+
+
+@router.get("/api/sessions/{session_id}/checks")
+async def get_session_checks(session_id: str):
+    """Checker verdict ledger + run outcome (backfill for the Checker panel)."""
+    return media_service.get_session_checks(session_id)

package/artemis-main/apps/admin_console/routers/replay.py

@@ -0,0 +1,91 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import traceback
+from fastapi import APIRouter, HTTPException
+
+from artemis.config import WORKSPACE_ROOT
+
+try:
+    from admin_console.schemas.task_schema import ReplayRequest
+    from admin_console.replay_manager import ReplayManager
+except ImportError:
+    from apps.admin_console.schemas.task_schema import ReplayRequest
+    from apps.admin_console.replay_manager import ReplayManager
+
+replay_manager = ReplayManager(WORKSPACE_ROOT)
+
+
+router = APIRouter(tags=["replay"])
+
+
+@router.get("/api/devices")
+async def list_devices():
+    """Dynamically queries the ADB server for connected Android devices."""
+    return replay_manager.list_devices()
+
+
+@router.get("/api/replay/tools")
+async def get_replay_tools():
+    """Returns the list of registered tools available for replay."""
+    return replay_manager.get_replay_tools()
+
+
+@router.get("/api/replay/config")
+async def get_replay_config(tool_name: str = "ask_explorer"):
+    """Returns the parameter list and types of the selected tool logic."""
+    try:
+        return replay_manager.get_replay_config(tool_name)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/api/sessions/{session_id}/replay_steps")
+async def get_replay_steps(session_id: str):
+    """Loads and formats metadata for all chunked steps in the session."""
+    try:
+        return replay_manager.get_replay_steps(session_id)
+    except FileNotFoundError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/api/sessions/{session_id}/steps/{step_number}/replay")
+async def trigger_step_replay_endpoint(session_id: str, step_number: int, req: ReplayRequest):
+    """Triggers sandbox execution of step replay and returns the resulting traces trees."""
+    try:
+        return await replay_manager.run_step_replay(
+            session_id=session_id,
+            step_number=step_number,
+            device_id=req.device_id,
+            user_submits=req.user_submits,
+            tool_name=req.tool_name,
+            replay_id=req.replay_id,
+        )
+    except Exception as e:
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/api/sessions/{session_id}/steps/{step_number}/replay_traces")
+async def get_step_replay_traces_endpoint(
+    session_id: str, step_number: int, tool_name: str = "ask_explorer"
+):
+    """Retrieves previously generated step replay traces if they exist."""
+    try:
+        return replay_manager.get_step_replay_traces(session_id, step_number, tool_name)
+    except Exception as e:
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=str(e))

package/artemis-main/apps/admin_console/routers/sessions.py

@@ -0,0 +1,239 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import json
+import time
+from uuid import UUID
+from fastapi import APIRouter, HTTPException
+
+from artemis.config import DB_PATH, TRACES_PATH
+from artemis.runtime import trace_store
+
+try:
+    from admin_console.core.state import state
+    from admin_console.database.repositories.session_repository import session_repo
+    from admin_console.database.repositories.trace_repository import trace_repo
+    from admin_console.services.media_service import media_service
+    from admin_console.services.model_service import model_service
+except ImportError:
+    from apps.admin_console.core.state import state
+    from apps.admin_console.database.repositories.session_repository import session_repo
+    from apps.admin_console.database.repositories.trace_repository import trace_repo
+    from apps.admin_console.services.media_service import media_service
+    from apps.admin_console.services.model_service import model_service
+
+
+router = APIRouter(tags=["sessions"])
+
+
+@router.get("/api/sessions")
+async def list_sessions():
+    # The body does blocking work (sqlite queries, filesystem scans, ffmpeg
+    # conversion on first sight of a new recording), so run it off the event
+    # loop — the frontend polls this endpoint and it must not stall other
+    # requests.
+    try:
+        return await asyncio.to_thread(_list_sessions_sync)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+def _list_sessions_sync():
+    rows = session_repo.get_all_sessions()
+    video_rec_map = session_repo.get_video_recordings_map()
+    latest_recordings = session_repo.get_latest_video_recordings_map()
+    agent_names_by_session = session_repo.get_agent_trace_names_map()
+    video_idx = media_service.build_video_index()
+    default_model_info = model_service.get_active_model_info()
+
+    orphaned_ids = []
+    unresolved_profiles = []
+    result = []
+
+    try:
+        from artemis.runtime import DeviceExecutionLock
+
+        active_owners = DeviceExecutionLock.get_active_owners()
+        active_owner_sids = {
+            str(owner.session_id) for owner in active_owners.values() if owner.session_id
+        }
+    except Exception:
+        active_owner_sids = set()
+
+    for row_dict in rows:
+        s_id = str(row_dict.get("session_id"))
+        recording = latest_recordings.get(s_id)
+        d_info_raw = row_dict.get("device_info")
+        device_id = None
+        if d_info_raw:
+            try:
+                d_info = json.loads(d_info_raw) if isinstance(d_info_raw, str) else d_info_raw
+                if isinstance(d_info, dict):
+                    device_id = d_info.get("device_id") or d_info.get("device_serial")
+            except (ValueError, TypeError):
+                # Malformed device_info JSON: fall back to the recording's device_id.
+                pass
+        if not device_id and recording:
+            device_id = recording.get("device_id")
+        row_dict["device_id"] = device_id
+        row_dict["device_serial"] = device_id
+
+        if row_dict.get("status") == "running":
+            is_active = (
+                s_id in active_owner_sids
+                or s_id in state.active_connections
+                or (state.is_running and s_id == str(state.active_session_id))
+            )
+            worker_is_alive = session_repo.process_is_alive(row_dict.get("pid"))
+            if not is_active and not worker_is_alive:
+                row_dict["status"] = "failed"
+                row_dict["end_time"] = time.time()
+                orphaned_ids.append(s_id)
+
+        recording_status = str((recording or {}).get("status") or "unavailable")
+        resolved_v_url = (
+            media_service.resolve_video_url(row_dict, video_rec_map, video_idx)
+            if recording_status != "recording"
+            else None
+        )
+        if resolved_v_url:
+            recording_status = "ready"
+            row_dict["video_url"] = resolved_v_url
+        else:
+            row_dict["video_url"] = None
+        row_dict["recording_status"] = recording_status
+
+        agent_names = agent_names_by_session.get(s_id, [])
+        sess_profile = model_service.resolve_session_profile(
+            row_dict, None, state.current_profile, agent_names=agent_names
+        )
+        row_dict["model_info"] = (
+            model_service.get_active_model_info(sess_profile)
+            if sess_profile
+            else default_model_info
+        )
+        if not sess_profile:
+            unresolved_profiles.append(row_dict)
+        result.append(row_dict)
+
+    # Most sessions carry a profile in device_info or an agent trace name.
+    # Only inspect the much larger LLM payload for legacy rows that remain
+    # ambiguous after those cheap checks.
+    if unresolved_profiles:
+        unresolved_ids = [str(row.get("session_id")) for row in unresolved_profiles]
+        llm_traces_by_session = session_repo.get_llm_traces_for_profiles_map(unresolved_ids)
+        for row_dict in unresolved_profiles:
+            s_id = str(row_dict.get("session_id"))
+            llm_traces = llm_traces_by_session.get(s_id)
+            if not llm_traces:
+                continue
+            sess_profile = model_service.resolve_session_profile(
+                row_dict, llm_traces, state.current_profile
+            )
+            if sess_profile:
+                row_dict["model_info"] = model_service.get_active_model_info(sess_profile)
+
+    if orphaned_ids:
+        try:
+            harvested = session_repo.harvest_orphaned_sessions(orphaned_ids)
+            print(f"[list_sessions] Auto-harvested {harvested} orphaned running session(s).")
+            try:
+                for o_id in orphaned_ids:
+                    if trace_store.read_status(str(o_id)):
+                        trace_store.update_trace_status(
+                            str(o_id),
+                            "failed",
+                            error="Process terminated unexpectedly (auto-harvested).",
+                        )
+            except OSError as e:
+                # read_status never raises; this guards the lock/write side
+                # of update_trace_status.
+                print(f"[list_sessions] Could not mark harvested traces failed: {e}")
+        except Exception as e:
+            print(f"[list_sessions] Failed to update orphaned sessions: {e}")
+
+    return result
+
+
+@router.get("/api/sessions/{session_id}")
+async def get_session_details(session_id: str):
+    """Retrieve details for a single automation session."""
+    row = session_repo.get_session_by_id(session_id)
+    if not row:
+        raise HTTPException(status_code=404, detail=f"Session {session_id} not found")
+    return dict(row)
+
+
+@router.get("/api/sessions/{session_id}/usage")
+async def get_session_usage(session_id: str):
+    """Session-wide LLM token totals, live executor context size and run tuning."""
+    try:
+        return await asyncio.to_thread(session_repo.get_session_usage, session_id)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/api/sessions/{session_id}/tree")
+async def get_tree(session_id: str):
+    try:
+        return trace_repo.get_trace_tree(session_id)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/api/sessions/{session_id}/background_tasks")
+async def get_session_background_tasks(session_id: str):
+    try:
+        return session_repo.get_background_tasks(session_id)
+    except Exception:
+        return []
+
+
+@router.get("/api/sessions/{session_id}/startup_progress")
+async def get_session_startup_progress(session_id: str):
+    try:
+        return state.get_startup_progress(session_id)
+    except Exception:
+        return []
+
+
+@router.post("/api/cleanup")
+async def cleanup_history_endpoint():
+    try:
+        from artemis.data_engine.storage import StorageManager
+
+        storage = StorageManager(DB_PATH, TRACES_PATH)
+        storage.clear_all_data()
+        return {
+            "status": "success",
+            "message": "History cleaned up successfully",
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/api/sessions/{session_id}/delete")
+async def delete_session_endpoint(session_id: str):
+    try:
+        from artemis.data_engine.storage import StorageManager
+
+        storage = StorageManager(DB_PATH, TRACES_PATH)
+        storage.delete_session(UUID(session_id))
+        return {
+            "status": "success",
+            "message": f"Session {session_id} deleted successfully",
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

package/artemis-main/apps/admin_console/routers/steps.py

@@ -0,0 +1,109 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import traceback
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import PlainTextResponse
+
+from artemis.config import TEST_OUTPUTS_DIR
+
+try:
+    from admin_console.database.repositories.step_repository import step_repo
+    from admin_console.database.repositories.trace_repository import trace_repo
+    from admin_console.services.media_service import media_service
+except ImportError:
+    from apps.admin_console.database.repositories.step_repository import step_repo
+    from apps.admin_console.database.repositories.trace_repository import trace_repo
+    from apps.admin_console.services.media_service import media_service
+
+
+router = APIRouter(tags=["steps"])
+
+
+@router.get("/api/sessions/{session_id}/steps")
+async def get_session_steps(session_id: str, client: str | None = None):
+    try:
+        return step_repo.get_session_steps(session_id, client=client)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/api/steps/{step_id}/traces")
+async def get_step_traces_endpoint(step_id: str):
+    try:
+        session_id = step_repo.get_step_session_id(step_id)
+        if not session_id:
+            raise HTTPException(status_code=404, detail="Step not found")
+        return trace_repo.get_step_traces_tree(session_id, step_id)
+    except HTTPException as e:
+        raise e
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+def _resolve_trace_db_path(session_id: str | None, step_number: int | None):
+    if session_id and step_number is not None:
+        sandbox_db_path = (
+            TEST_OUTPUTS_DIR
+            / f"{session_id}_step_{step_number:02d}"
+            / "temp_traces"
+            / "data_engine.db"
+        )
+        if sandbox_db_path.exists():
+            return sandbox_db_path
+    return None
+
+
+@router.get("/api/traces/{trace_id}")
+async def get_trace(trace_id: str, session_id: str = None, step_number: int = None):
+    try:
+        db_path = _resolve_trace_db_path(session_id, step_number)
+        trace_dict = trace_repo.get_trace_by_id(trace_id, db_path=db_path)
+        if not trace_dict:
+            raise HTTPException(status_code=404, detail="Trace not found")
+
+        if trace_dict.get("payload"):
+            try:
+                payload_obj = json.loads(trace_dict["payload"])
+                trace_dict["payload"] = media_service.unwrap_payload(payload_obj)
+            except (ValueError, TypeError, KeyError, AttributeError):
+                # Non-JSON or unexpectedly shaped payload: serve it raw.
+                pass
+
+        return trace_dict
+    except HTTPException as e:
+        raise e
+    except Exception as e:
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get("/api/traces/{trace_id}/download")
+async def download_trace(trace_id: str, session_id: str = None, step_number: int = None):
+    try:
+        db_path = _resolve_trace_db_path(session_id, step_number)
+        trace_dict = trace_repo.get_trace_by_id(trace_id, db_path=db_path)
+        if not trace_dict or not trace_dict.get("payload"):
+            raise HTTPException(status_code=404, detail="Payload not found")
+
+        return PlainTextResponse(
+            trace_dict["payload"],
+            headers={"Content-Disposition": f'attachment; filename="trace_{trace_id}.json"'},
+        )
+    except HTTPException as e:
+        raise e
+    except Exception as e:
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=str(e))