android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/graph/perception.py

@@ -0,0 +1,234 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import base64
+import hashlib
+import json
+from pathlib import Path
+
+from artemis.context import ArtemisContext
+from artemis.controllers.unified_controller import UnifiedMobileController
+from artemis.data_engine.trace import trace
+from artemis.graph.state import State
+from artemis.graph.visibility import strict_state
+from artemis.utils.ocr_api import is_ocr_configured, perform_ocr
+from artemis.utils.ocr_xml_fusion import (
+    _crop_image_remove_status_bar,
+    _detect_status_bar_height,
+    _map_coordinates_back,
+    fuse_ocr_with_xml,
+)
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def _should_skip_settling(state: State) -> bool:
+    """Determine if we should skip screen settling based on the last executed action.
+
+    Purely logical, non-UI-drawing, or wait operations don't require dynamic
+    settling checks.
+    """
+    if not state or not state.structured_decisions:
+        # First turn or no prior action, skip settling
+        return True
+
+    try:
+        actions = json.loads(state.structured_decisions)
+        if not actions or not isinstance(actions, list):
+            return True
+
+        last_action = actions[-1]
+        action_name = last_action.get("action")
+
+        # Actions that are known to have already completed full wait loops or don't render UI
+        skip_actions = {
+            "wait_for_delay",
+        }
+
+        if action_name in skip_actions:
+            logger.info(f"Last action was '{action_name}'. Skipping dynamic screen settling.")
+            return True
+
+        return False
+    except Exception as e:
+        logger.warning(
+            f"Failed to parse last action for settling heuristic: {e}."
+            " Defaulting to settling check."
+        )
+        return False
+
+
+@trace(type="agent", name="perception")
+def _check_injected_instruction_file(base_dir: str) -> dict | None:
+    """Consumes a pending injected-instruction payload, if any.
+
+    Returns the payload dict: `instruction` (str | None) plus the explicit
+    `release_loop` stop signal (bool) declared by the injecting caller.
+    """
+    instruction_path = Path(base_dir) / "injected_instruction.json"
+    if instruction_path.exists():
+        with open(instruction_path, encoding="utf-8") as f:
+            data = json.load(f)
+        instruction_path.unlink()
+        payload = {
+            "instruction": data.get("instruction"),
+            "release_loop": bool(data.get("release_loop")),
+        }
+        logger.info(f"Read and deleted injected instruction payload: {payload}")
+        return payload
+    return None
+
+
+def _unprotect_current_check_lines(ctx: ArtemisContext) -> None:
+    """Record the plan's current check lines as editable under user guidance."""
+    from artemis.utils.notes import get_note_file_path
+    from artemis.utils.plan_grammar import parse_plan
+
+    unprotected = getattr(ctx, "guidance_unprotected_checks", None)
+    if not isinstance(unprotected, set):
+        unprotected = set()
+    try:
+        plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
+        text = plan_path.read_text(encoding="utf-8") if plan_path.exists() else ""
+    except Exception as e:
+        logger.warning(f"Could not read task_plan for the guidance check-line waiver: {e}")
+        text = ""
+    added = {(ci.kind, ci.text) for ci in parse_plan(text).check_items}
+    unprotected |= added
+    ctx.guidance_unprotected_checks = unprotected
+    logger.info(
+        f"User guidance received: {len(added)} declared check line(s) may now be edited"
+        " by the Operator."
+    )
+
+
+async def perception_node(state: State, ctx: ArtemisContext) -> dict:
+    """Perception node (fast path): captures screenshot, extracts XML hierarchy, and performs OCR.
+
+    Always executes at the start of an operator cycle.
+    """
+    state = strict_state(state, "perception")
+    logger.info("Entering Perception Node...")
+
+    if not ctx.data_engine.current_step_id:
+        ctx.data_engine.allocate_step_id()
+        logger.info(f"Pre-allocated step ID in Perception Node: {ctx.data_engine.current_step_id}")
+
+    # Check for injected instruction without blocking event loop
+    injected_instruction = None
+    user_stop_requested = False
+    if ctx.data_engine and ctx.data_engine.base_dir:
+        injected_payload = await asyncio.to_thread(
+            _check_injected_instruction_file, str(ctx.data_engine.base_dir)
+        )
+        if injected_payload:
+            injected_instruction = injected_payload.get("instruction")
+            user_stop_requested = bool(injected_payload.get("release_loop"))
+            if isinstance(injected_instruction, str) and injected_instruction.strip():
+                # User guidance outranks the plan's declared standards: every
+                # check line that exists right now loses its machine
+                # restoration for the rest of the run (kept on the run context,
+                # not the per-turn State), so the Operator can drop or reword
+                # it in whichever later write it gets to it. Lines added after
+                # this moment stay protected until the next instruction.
+                _unprotect_current_check_lines(ctx)
+
+    controller = UnifiedMobileController(ctx)
+
+    # 1. Capture screenshot with hybrid settling strategy
+    if _should_skip_settling(state):
+        logger.info("Skipping settling. Retrieving screen data directly...")
+        device_data = await controller.get_screen_data()
+        latest_screenshot_b64 = device_data.base64
+        xml_hierarchy = device_data.elements
+    else:
+        logger.info("Screen settling required. Delaying 0.4s and capturing screen data...")
+        await asyncio.sleep(0.4)
+        device_data = await controller.get_screen_data()
+        latest_screenshot_b64 = device_data.base64
+        xml_hierarchy = device_data.elements
+
+    latest_screenshot_bytes = base64.b64decode(latest_screenshot_b64)
+
+    # Update context device dimensions dynamically
+    if ctx.device:
+        ctx.device.device_width = device_data.width
+        ctx.device.device_height = device_data.height
+        logger.info(f"Updated context device dimensions: {device_data.width}x{device_data.height}")
+
+    # 3. Perform OCR (if configured, else fallback to pure XML layout)
+    ocr_results = []
+    if is_ocr_configured():
+        screen_height = device_data.height
+        status_bar_height = _detect_status_bar_height(xml_hierarchy, screen_height)
+        try:
+            if status_bar_height > 0:
+                logger.info(
+                    f"Detected status bar height: {status_bar_height}. Cropping screenshot for OCR."
+                )
+                cropped_b64, _, _ = await asyncio.to_thread(
+                    _crop_image_remove_status_bar,
+                    latest_screenshot_b64,
+                    status_bar_height,
+                )
+                raw_ocr_results = await perform_ocr(cropped_b64)
+                ocr_results = _map_coordinates_back(raw_ocr_results, status_bar_height)
+            else:
+                logger.info("No status bar detected. Performing OCR on full screenshot.")
+                ocr_results = await perform_ocr(latest_screenshot_b64)
+        except Exception as ocr_err:
+            logger.warning(
+                f"Perception OCR execution encountered error: {ocr_err}. Proceeding with XML-only layout."
+            )
+            ocr_results = []
+    else:
+        logger.debug("OCR is not configured. Proceeding with XML-only layout.")
+
+    # 4. Fuse OCR with XML (if ocr_results is empty, returns original xml_hierarchy intact)
+    fused_xml = fuse_ocr_with_xml(xml_hierarchy, ocr_results)
+
+    # 5. Save to Data Engine in background thread
+    image_name = None
+    screenshot_path = None
+    if ctx.data_engine:
+        image_name = hashlib.sha256(latest_screenshot_bytes).hexdigest()
+        screenshot_path = str(ctx.data_engine.get_image_path(image_name))
+        asyncio.create_task(
+            asyncio.to_thread(
+                ctx.data_engine.get_or_create_image,
+                latest_screenshot_bytes,
+                ui_tree=xml_hierarchy,
+                ocr_result=ocr_results,
+            )
+        )
+        logger.info(f"Offloaded perception data saving to background with image_name: {image_name}")
+
+    # Return state update
+    update = {
+        "latest_screenshot": screenshot_path,
+        "latest_ui_hierarchy": fused_xml,
+        "operator_raw_data": {
+            "screenshot_b64": latest_screenshot_b64,
+            "xml_hierarchy": xml_hierarchy if xml_hierarchy is not None else [],
+            "ocr_results": ocr_results,
+            "width": device_data.width,
+            "height": device_data.height,
+        },
+        "injected_instruction": injected_instruction,
+        # sticky_or reducer: once latched True, later False updates keep it True
+        "user_stop_requested": user_stop_requested,
+    }
+    return update

package/artemis-main/artemis/graph/state.py

@@ -0,0 +1,151 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Annotated
+
+from pydantic import BaseModel, ConfigDict
+
+from third_party.mobile_use.graph import take_last
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+def sticky_or(a, b):
+    """Reducer that latches True: once set, the flag survives later updates."""
+    return bool(a) or bool(b)
+
+
+class State(BaseModel):
+    """Graph channel state.
+
+    Every field is explicit (``extra="forbid"``): an undeclared write or an
+    unknown constructor key fails loudly instead of becoming a silent ghost
+    key. Undeclared *reads* (``getattr(state, k, default)``) are guarded by the
+    node visibility manifest in ``artemis.graph.visibility`` instead.
+    """
+
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        extra="forbid",
+    )
+
+    # ── Control plane (cross-turn signals & routing) ─────────────────────
+    initial_goal: Annotated[str, "Initial goal given by the user"]
+    injected_instruction: Annotated[
+        str | None,
+        "Injected instruction from user for micro-adjustment",
+        take_last,
+    ] = None
+    user_stop_requested: Annotated[
+        bool,
+        "Latched True once the user externally signals stop (release_loop),"
+        " unlocking completion of [Loop:continuous] milestones",
+        sticky_or,
+    ] = False
+    checker_success: Annotated[
+        bool | None,
+        "True if checker succeeded or not triggered, False if failed",
+        take_last,
+    ] = None
+    operator_feedback: Annotated[
+        list[str] | None,
+        "Source-tagged findings ([checker]/[planner]/[final check]) injected"
+        " append-only into the Operator's next prompt (never switches the"
+        " prompt template)",
+        take_last,
+    ] = None
+    run_outcome: Annotated[
+        dict | None,
+        "Machine-readable run outcome (task_status + test summary), populated"
+        " by exit settlement before END",
+        take_last,
+    ] = None
+    exit_settlement_route: Annotated[
+        str | None,
+        "Routing decision produced by exit_settlement_node ('continue' | 'end')",
+        take_last,
+    ] = None
+
+    # ── Perception (moves behind PerceptionStore in Phase 2) ─────────────
+    latest_ui_hierarchy: Annotated[
+        list[dict] | None, "Latest UI hierarchy of the device", take_last
+    ] = None
+    latest_screenshot: Annotated[str | None, "Path to the latest screenshot", take_last] = None
+    operator_raw_data: Annotated[
+        dict | None,
+        "Raw perception data from operator (screenshot, xml, ocr)",
+        take_last,
+    ] = None
+    indexed_points: Annotated[
+        list[list[int]] | None,
+        "Active coordinate mappings matching indexed text elements",
+        take_last,
+    ] = None
+    indexed_elements: Annotated[
+        list[dict] | None,
+        "Active interactable elements matching indexed text elements,"
+        " containing text, bounds, class, and center coordinates.",
+        take_last,
+    ] = None
+
+    # ── Turn products (previous turn's decisions & results) ──────────────
+    current_step_id: Annotated[str | None, "Current step ID from data engine", take_last] = None
+    structured_decisions: Annotated[
+        str | None,
+        "Structured decisions made by the operator, for the validator to follow",
+        take_last,
+    ] = None
+    operator_raw_thinking: Annotated[
+        str | None,
+        "Raw thinking process of the operator",
+        take_last,
+    ] = None
+    operator_native_thinking: Annotated[
+        str | None,
+        "Native/implicit thinking process of the operator",
+        take_last,
+    ] = None
+    last_execution_result: Annotated[
+        dict | None, "Last execution result from validator", take_last
+    ] = None
+    open_incident: Annotated[
+        dict | None,
+        "Execution incident (blocked/failed terminal action) still awaiting the"
+        " Operator's resolution; None once a later action executes successfully",
+        take_last,
+    ] = None
+    last_closed_incident: Annotated[
+        dict | None,
+        "The incident the most recent successful terminal action closed (with"
+        " closed_at_step); rendered once so the Operator settles the original intent",
+        take_last,
+    ] = None
+    subagent_calls: Annotated[
+        list[str], "List of sub-agent calls in current session", take_last
+    ] = []
+    operator_tool_limit_exceeded: Annotated[
+        bool | None,
+        "True if operator exceeded tool call limit in the previous turn",
+        take_last,
+    ] = None
+
+    @classmethod
+    def initial(cls, goal: str) -> "State":
+        """Single source for the graph's initial state.
+
+        Every entrypoint (SDK, engine runners, tests) must construct the
+        first State through here so required fields stay in one place.
+        """
+        return cls(initial_goal=goal)

package/artemis-main/artemis/graph/visibility.py

@@ -0,0 +1,238 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Per-node State visibility manifests with test-time enforcement.
+
+``extra="forbid"`` on :class:`~artemis.graph.state.State` catches ghost
+*writes*; ghost *reads* (``getattr(state, key, default)`` silently returning
+the default) can only be caught by policing reads against a declared manifest.
+
+The manifest below is the declared visible domain of every graph node — the
+authoritative answer to "which State fields does this agent see / produce".
+Enforcement is zero-cost in production: :func:`strict_state` returns the state
+unchanged unless ``ARTEMIS_STRICT_STATE=1`` (set in tests), in which case field
+reads outside the node's manifest raise :class:`VisibilityError`.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any
+
+from artemis.graph.state import State
+
+
+class VisibilityError(Exception):
+    """A node touched a State field outside its declared visible domain."""
+
+
+@dataclass(frozen=True)
+class NodeVisibility:
+    reads: frozenset[str]
+    writes: frozenset[str]
+
+    @property
+    def visible(self) -> frozenset[str]:
+        return self.reads | self.writes
+
+
+def _vis(reads: set[str], writes: set[str]) -> NodeVisibility:
+    return NodeVisibility(reads=frozenset(reads), writes=frozenset(writes))
+
+
+#: Declared visible domain per node. Reads include fields touched by the tools
+#: a node invokes with ``InjectedState`` (they run inside the node's turn).
+#: Start generous, tighten over time — an entry here is a documented decision.
+NODE_VISIBILITY: dict[str, NodeVisibility] = {
+    "perception": _vis(
+        reads={"structured_decisions"},
+        writes={
+            "latest_screenshot",
+            "latest_ui_hierarchy",
+            "operator_raw_data",
+            "injected_instruction",
+            "user_stop_requested",
+        },
+    ),
+    "planner": _vis(
+        reads={"initial_goal", "latest_screenshot"},
+        # Mutates latest_screenshot in place when it captures its own frame
+        # (planner runs before perception on the first turn).
+        writes={"latest_screenshot"},
+    ),
+    "operator": _vis(
+        reads={
+            "initial_goal",
+            "injected_instruction",
+            "user_stop_requested",
+            "operator_feedback",
+            "operator_raw_data",
+            "latest_ui_hierarchy",
+            "latest_screenshot",
+            "indexed_points",
+            "indexed_elements",
+            "subagent_calls",
+            "current_step_id",
+            "operator_tool_limit_exceeded",
+            "structured_decisions",
+            "operator_raw_thinking",
+            "operator_native_thinking",
+            # Transcript path (M2): committing the previous turn into the
+            # ledger appends its validator result message.
+            "last_execution_result",
+            # Execution incident hand-off from the Validator (open block /
+            # one-turn CLOSED notice in the observation tail).
+            "open_incident",
+            "last_closed_incident",
+        },
+        writes={
+            "structured_decisions",
+            "operator_raw_thinking",
+            "operator_native_thinking",
+            "indexed_points",
+            "indexed_elements",
+            "current_step_id",
+            "subagent_calls",
+            "operator_tool_limit_exceeded",
+        },
+    ),
+    "execution_check": _vis(
+        reads={
+            "operator_raw_data",
+            "structured_decisions",
+            "operator_raw_thinking",
+            "operator_native_thinking",
+            "user_stop_requested",
+            "initial_goal",
+            # The turn's injected instruction is stamped verbatim onto the
+            # step record so the chunk ledger can preserve it (never-evict).
+            "injected_instruction",
+        },
+        writes={
+            "checker_success",
+            "structured_decisions",
+            "current_step_id",
+            "operator_feedback",
+            "subagent_calls",
+        },
+    ),
+    "validator": _vis(
+        reads={
+            "open_incident",
+            "initial_goal",
+            "structured_decisions",
+            "latest_screenshot",
+            "latest_ui_hierarchy",
+            "current_step_id",
+            "operator_raw_data",
+            "operator_raw_thinking",
+            "operator_native_thinking",
+            "indexed_points",
+            "indexed_elements",
+        },
+        writes={
+            "open_incident",
+            "last_closed_incident",
+            "last_execution_result",
+            # In-place turn-scoped mutations (formalized as TurnWorkspace in
+            # Phase 2): post-action screenshot and re-derived index mappings.
+            "latest_screenshot",
+            "indexed_points",
+            "indexed_elements",
+        },
+    ),
+    "summarizer": _vis(
+        reads={
+            "current_step_id",
+            "structured_decisions",
+            "operator_raw_thinking",
+            "operator_native_thinking",
+            "last_execution_result",
+        },
+        writes=set(),
+    ),
+    "convergence": _vis(
+        reads={"checker_success", "user_stop_requested"},
+        writes=set(),
+    ),
+    "exit_settlement": _vis(
+        reads={"user_stop_requested", "initial_goal"},
+        writes={"exit_settlement_route", "operator_feedback", "run_outcome"},
+    ),
+}
+
+_STATE_FIELDS = frozenset(State.model_fields.keys())
+
+
+def strict_enabled() -> bool:
+    return os.environ.get("ARTEMIS_STRICT_STATE", "") == "1"
+
+
+class StateView:
+    """Read-policing proxy handed to nodes under ``ARTEMIS_STRICT_STATE=1``.
+
+    Only declared State *fields* are policed; methods and pydantic internals
+    pass through. Attribute writes delegate to the underlying State (in-place
+    turn mutations are Phase 2 scope) but are still checked against the
+    node's write set.
+    """
+
+    __slots__ = ("_state", "_node", "_visibility")
+
+    def __init__(self, state: State, node: str):
+        object.__setattr__(self, "_state", state)
+        object.__setattr__(self, "_node", node)
+        object.__setattr__(self, "_visibility", NODE_VISIBILITY[node])
+
+    def __getattr__(self, name: str) -> Any:
+        if name in _STATE_FIELDS and name not in self._visibility.visible:
+            raise VisibilityError(
+                f"Node '{self._node}' read undeclared State field '{name}'."
+                " Declare it in NODE_VISIBILITY or stop reading it."
+            )
+        return getattr(self._state, name)
+
+    def __setattr__(self, name: str, value: Any) -> None:
+        if name in _STATE_FIELDS and name not in self._visibility.writes:
+            raise VisibilityError(
+                f"Node '{self._node}' mutated undeclared State field '{name}'."
+                " Declare it in NODE_VISIBILITY or stop writing it."
+            )
+        setattr(self._state, name, value)
+
+
+def strict_state(state: State, node: str) -> State | StateView:
+    """Wraps ``state`` in a policing view when strict mode is on (tests)."""
+    if strict_enabled() and isinstance(state, State):
+        return StateView(state, node)
+    return state
+
+
+def check_update(node: str, update: dict | None) -> None:
+    """Validates a node's returned update keys against its write manifest."""
+    if not update:
+        return
+    illegal = set(update) & _STATE_FIELDS - NODE_VISIBILITY[node].writes
+    if illegal:
+        raise VisibilityError(f"Node '{node}' returned undeclared update keys: {sorted(illegal)}.")
+
+
+def validate_manifest() -> list[str]:
+    """Returns problems in the manifest itself (ghost fields). Empty = clean."""
+    problems = []
+    for node, vis in NODE_VISIBILITY.items():
+        for field in vis.visible - _STATE_FIELDS:
+            problems.append(f"{node}: '{field}' is not a declared State field")
+    return problems

package/artemis-main/artemis/interfaces/cli/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""ARTEMIS CLI interface package."""
+
+from artemis.interfaces.cli.main import app, cli
+
+__all__ = ["app", "cli"]

package/artemis-main/artemis/interfaces/cli/commands/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""CLI command modules."""