android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/agents/flash/summarizer.py

@@ -0,0 +1,511 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Asynchronous Visual Step Summarizer for Artemis Flash profile.
+
+Processes Before/After screenshot pairs and action metadata in the background
+to generate high-density, strictly objective visual state transition summaries.
+
+Scheduling (zero-blocking dispatch, bounded retry, bounded flush, step_id
+keying with tool_call_id aliases) lives in the shared
+:class:`artemis.memory.step_memory.StepMemoryService`; this class contributes
+only the visual-transition lens: red action-marker overlay on the BEFORE
+frame, the ``flash_summarizer.md`` neutral-wording contract, and versioned
+``summary_status`` writes to the DataEngine.
+"""
+
+import asyncio
+import base64
+from pathlib import Path
+from typing import Any
+from uuid import UUID
+
+from jinja2 import Template
+from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
+
+from artemis.context import ArtemisContext
+from artemis.llm.parser import extract_thinking_tags
+from artemis.llm.structured import content_to_text
+from artemis.memory.step_memory import JobKey, StepMemoryService
+from artemis.services.llm import RobustChatModelWrapper, get_google_llm, get_llm
+from artemis.services.token_meter import record_llm_usage
+from artemis.utils.task_tree import format_actions_clean
+from artemis.utils.visualization import draw_action_overlay_on_image
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+# Degenerate-output guard (§5 echo validation): a summary that echoes the
+# input's section markers, or falls outside sane length bounds, fails the
+# attempt so the bounded service retry regenerates it.
+SUMMARY_ECHO_MARKER = "--- ["
+SUMMARY_MIN_CHARS = 15
+SUMMARY_MAX_CHARS = 1500
+
+
+def degenerate_summary_reason(text: str) -> str | None:
+    """Why a lens output is unusable (echo/length), or None when it is fine."""
+    if text.lower().startswith(("<think>", "<thought>", "<thinking>")):
+        return "unclosed thinking block; no final summary"
+    if SUMMARY_ECHO_MARKER in text or text.startswith("---"):
+        return "echoes an input section marker"
+    if len(text) < SUMMARY_MIN_CHARS:
+        return f"too short ({len(text)} chars < {SUMMARY_MIN_CHARS})"
+    if len(text) > SUMMARY_MAX_CHARS:
+        return f"too long ({len(text)} chars > {SUMMARY_MAX_CHARS})"
+    return None
+
+
+#: Header of the operator-focus block that leads the lens input. It shares
+#: the ``--- [`` shape of the frame headers so an echoed header trips the
+#: same degenerate-output guard.
+FOCUS_BLOCK_HEADER = "--- [0] OPERATOR FOCUS (context for attention, not evidence) ---"
+
+#: Cap on the operator reasoning excerpt carried in the focus block. The
+#: expectation statement closes the reasoning, so the tail is kept intact
+#: and the cut lands in the middle.
+FOCUS_INTENT_MAX_CHARS = 900
+FOCUS_INTENT_TAIL_CHARS = 600
+
+
+def _cap_intent(text: str) -> str:
+    if len(text) <= FOCUS_INTENT_MAX_CHARS:
+        return text
+    head_len = FOCUS_INTENT_MAX_CHARS - FOCUS_INTENT_TAIL_CHARS
+    head, tail = text[:head_len], text[-FOCUS_INTENT_TAIL_CHARS:]
+    cut = len(text) - len(head) - len(tail)
+    return f"{head}\n[… {cut} characters cut here …]\n{tail}"
+
+
+def build_focus_context(
+    *,
+    intent: str | None = None,
+    goal: str | None = None,
+    subgoal: str | None = None,
+    injected_instruction: str | None = None,
+) -> dict[str, str] | None:
+    """Collect operator context to guide the visual summary's focus.
+
+    Target descriptions remain on the action. Return None if all fields are empty.
+    """
+    focus: dict[str, str] = {}
+    if isinstance(goal, str) and goal.strip():
+        focus["goal"] = goal.strip()
+    if isinstance(subgoal, str) and subgoal.strip():
+        focus["subgoal"] = subgoal.strip()
+    if isinstance(injected_instruction, str) and injected_instruction.strip():
+        focus["injected_instruction"] = injected_instruction.strip()
+    if isinstance(intent, str) and intent.strip():
+        focus["intent"] = _cap_intent(intent.strip())
+    return focus or None
+
+
+def render_focus_block(focus: dict[str, str], *, dual: bool = True) -> str:
+    """Plain-text focus block (goal, sub-goal, instruction, then reasoning).
+
+    ``dual=False`` (a single decision frame, every Pro step) labels the
+    reasoning so its expectation is read as the operator's aim only: the
+    outcome is not in the frame and must not be described.
+    """
+    lines = [FOCUS_BLOCK_HEADER]
+    if focus.get("goal"):
+        lines.append(f"Task goal: {focus['goal']}")
+    if focus.get("subgoal"):
+        lines.append(f"Active sub-goal: {focus['subgoal']}")
+    if focus.get("injected_instruction"):
+        lines.append(f"User instruction at this step: {focus['injected_instruction']}")
+    if focus.get("intent"):
+        if dual:
+            lines.append(
+                "Operator reasoning for this step (why this target, what screen"
+                " change it expected):"
+            )
+        else:
+            lines.append(
+                "Operator reasoning for this step (why this target; the expected"
+                " outcome is NOT in this frame and must not be described):"
+            )
+        lines.append(focus["intent"])
+    return "\n".join(lines)
+
+
+class VisualStepSummarizer(StepMemoryService):
+    """Visual-transition lens on top of the shared step-memory runtime.
+
+    Key design properties (inherited from StepMemoryService):
+    1. Zero-blocking dispatch: The main Flash runner dispatches and proceeds immediately.
+    2. Lossless pending state: The compressor retains the source image until its summary is ready.
+    3. Independent bounded retry: Every action owns a retry loop capped at 1 + retry_limit attempts.
+    """
+
+    def __init__(
+        self,
+        ctx: ArtemisContext,
+        model_name: str | None = None,
+        retry_limit: int = 3,
+        *,
+        max_concurrency: int = 1,
+        flush_timeout_s: float = 30.0,
+    ):
+        super().__init__(
+            ctx,
+            max_concurrency=max_concurrency,
+            retry_limit=retry_limit,
+            flush_timeout_s=flush_timeout_s,
+        )
+
+        # Legacy lightweight Gemini names apply only to Google configurations.
+        target_model = model_name or "gemini-2.5-flash-lite"
+        self._model_name = target_model
+        if ctx.llm_config is None:
+            from artemis.config.llm import get_default_llm_config
+            ctx.llm_config = get_default_llm_config()
+        role = ctx.llm_config.get_agent("summarizer")
+        if role.provider != "google" or not model_name:
+            self._llm = get_llm(ctx, name="summarizer", temperature=0.0)
+            self._model_name = role.model
+        else:
+            self._llm = get_google_llm(model_name=target_model, temperature=0.0)
+        try:
+            configured = getattr(self._llm, "model", None) or getattr(self._llm, "model_name", None)
+            if isinstance(configured, str) and configured:
+                self._model_name = configured
+        except Exception as exc:
+            logger.debug(
+                "Could not read the configured summarizer model name; keeping %s: %s",
+                self._model_name,
+                exc,
+                exc_info=True,
+            )
+
+        # Load system prompt templates: the dual-frame transition prompt and
+        # the single-frame variant (§5 revision: describe whichever frames
+        # exist — a Pro step record never carries an independent after-frame,
+        # by design, and must not be prompted as if one were expected).
+        prompt_path = Path(__file__).parent / "flash_summarizer.md"
+        if prompt_path.exists():
+            self._prompt_template = prompt_path.read_text(encoding="utf-8")
+        else:
+            self._prompt_template = (
+                "You are the Step Summarizer for an Android UI automation agent.\n"
+                "Synthesize the physical action and visual delta between BEFORE and AFTER screens in exactly ONE "
+                "continuous first-person paragraph using 'I' (e.g., 'In Step {{ step_number }}, I tapped... and observed...').\n"
+                "Strictly avoid subjective validation words: successfully, completed, failed, achieved, navigated to."
+            )
+        single_path = Path(__file__).parent / "flash_summarizer_single.md"
+        if single_path.exists():
+            self._single_prompt_template = single_path.read_text(encoding="utf-8")
+        else:
+            self._single_prompt_template = (
+                "You are the Step Summarizer for an Android UI automation agent.\n"
+                "Exactly ONE screenshot (the decision frame) is available; there is NO after-action screenshot —"
+                " that only means no independent post-action evidence exists.\n"
+                "Describe strictly what THIS screen shows and where the action landed (red marker), in exactly ONE"
+                " continuous first-person paragraph using 'I' for Step {{ step_number }}. Never describe or guess"
+                " the post-action state.\n"
+                "Strictly avoid subjective validation words: successfully, completed, failed, achieved, navigated to."
+            )
+
+    def dispatch(
+        self,
+        step_number: int,
+        action_name: str,
+        action_args: dict[str, Any],
+        pre_img_bytes: bytes | None,
+        post_img_bytes: bytes | None,
+        exec_outcome: str,
+        *,
+        action_key: str | None = None,
+        data_engine_step_id: UUID | str | None = None,
+        focus: dict[str, str] | None = None,
+    ) -> None:
+        """Dispatches an asynchronous summarization task without blocking the caller.
+
+        Jobs are keyed by the DataEngine step id when one is available; the
+        tool_call_id (``action_key``) is retained as an alias so the message
+        compressor can keep querying by it. Callers with neither provide the
+        step ordinal, matching the legacy keying.
+
+        ``focus`` (:func:`build_focus_context`) is the operator's own context
+        for the step; the lens renders it ahead of the frames so the details
+        the operator was after are transcribed rather than summarized away.
+        """
+        key: JobKey
+        aliases: tuple[JobKey, ...] = ()
+        if data_engine_step_id is not None:
+            key = str(data_engine_step_id)
+            if action_key is not None:
+                aliases = (action_key,)
+        else:
+            key = action_key if action_key is not None else step_number
+
+        payload = {
+            "step_number": step_number,
+            "action_name": action_name,
+            "action_args": action_args,
+            "pre_img_bytes": pre_img_bytes,
+            "post_img_bytes": post_img_bytes,
+            "exec_outcome": exec_outcome,
+            "data_engine_step_id": data_engine_step_id,
+            "focus": focus,
+        }
+        self.submit(key, payload, aliases=aliases)
+
+    @staticmethod
+    def _action_phrase(action_name: str, action_args: dict[str, Any] | None) -> str:
+        """The action as every other history reader sees it (``format_actions_clean``).
+
+        ``action_args`` is the recorded action minus its verb (what the Pro
+        summarizer extracts from ``action_taken`` and what the Flash runner
+        builds from the same record shape). The action name always wins over
+        an argument of the same key: ``manage_app``'s own ``action="launch"``
+        argument becomes the ``intent`` the renderer reads, never the verb. A
+        described coordinate target renders as ``'play button'
+        (self-described)``, an observed element as ``'Play'``; a Pro burst
+        lists every member.
+        """
+        args = dict(action_args or {})
+        extra = args.pop("additional_actions", None)
+        own_verb = args.pop("action", None)
+        if own_verb is not None and own_verb != action_name:
+            args.setdefault("intent", own_verb)
+        # Flash's coordinate swipe names its points ``start``/``end``; the
+        # history renderer reads ``start_coordinates``/``end_coordinates``.
+        if action_name == "swipe":
+            raw = args.get("args") if isinstance(args.get("args"), dict) else {}
+            start = args.get("start", raw.get("start"))
+            end = args.get("end", raw.get("end"))
+            if start is not None and end is not None:
+                args.setdefault("start_coordinates", start)
+                args.setdefault("end_coordinates", end)
+        try:
+            first = {**args, "action": action_name}
+            actions = [first, *extra] if isinstance(extra, list) and extra else first
+            return format_actions_clean(actions)
+        except Exception as exc:
+            logger.debug(f"Action phrase rendering fell back to raw args: {exc}", exc_info=True)
+            return f"{action_name}({action_args})"
+
+    @staticmethod
+    def _verbatim_args(action_args: dict[str, Any] | None) -> dict[str, Any]:
+        """The agent-facing arguments of the action.
+
+        A Flash record keeps the tool call's arguments under ``args``; a Pro
+        record is the arguments themselves. Bookkeeping keys never count.
+        """
+        args = action_args or {}
+        raw = args.get("args")
+        if isinstance(raw, dict):
+            return dict(raw)
+        internal = {
+            "additional_actions",
+            "coordinate_space",
+            "normalized_coordinates",
+            "normalized_start_coordinates",
+            "normalized_end_coordinates",
+        }
+        return {k: v for k, v in args.items() if k not in internal}
+
+    @staticmethod
+    def _phrase_carries(phrase: str, args: dict[str, Any]) -> bool:
+        """Whether every argument value is visible in the rendered phrase.
+
+        Booleans and ``None`` never count (the phrase spells their effect, not
+        their value); everything else must appear verbatim, so an index
+        target, a repeat count or a duration the phrase folded away keeps the
+        verbatim argument line.
+        """
+        haystack = phrase.lower()
+
+        def _shown(value: Any) -> bool:
+            if isinstance(value, (list, tuple)):
+                # A point renders whole (``at [880, 410]``); a sequence or a
+                # description list renders member by member.
+                return str(list(value)).lower() in haystack or (
+                    bool(value) and all(_shown(v) for v in value)
+                )
+            return str(value).lower() in haystack
+
+        return all(
+            _shown(value)
+            for value in args.values()
+            if value is not None and not isinstance(value, bool)
+        )
+
+    def _meter_lens_call(self, response: Any) -> None:
+        """Meter one raw-model lens call as an ``llm_usage`` trace, best-effort.
+
+        Gateway-wrapped models already meter at the wrapper exit; only the raw
+        ``get_google_llm`` bypass needs explicit metering here. Lens prompts
+        are tiny and must not overwrite the session's ``last_prompt_tokens``
+        (the compaction thresholds' live context base), hence
+        ``update_last_prompt=False``.
+        """
+        if isinstance(self._llm, RobustChatModelWrapper):
+            return
+        engine = getattr(self.ctx, "data_engine", None) if self.ctx else None
+        record_llm_usage(
+            engine,
+            response,
+            source=f"lens:visual_transition:{self._model_name}",
+            update_last_prompt=False,
+        )
+
+    def _on_status(self, key: JobKey, status: str) -> None:
+        """Best-effort DataEngine summary-status write (pending/failed)."""
+        if not self.ctx.data_engine:
+            return
+        payload = self._step_inputs.get(key) or {}
+        target_step = payload.get("data_engine_step_id")
+        if not target_step:
+            return
+        try:
+            self.ctx.data_engine.update_step_summary(
+                target_step,
+                None,
+                status=status,
+                source="visual_transition",
+                model=self._model_name,
+            )
+        except Exception as de_err:
+            logger.debug(f"DataEngine summary status update skipped: {de_err}")
+
+    async def _attempt(self, key: JobKey) -> bool:
+        """Execute one lightweight VLM attempt for a step transition."""
+        input_data = self._step_inputs.get(key)
+        if not input_data:
+            return False
+
+        step_number = input_data["step_number"]
+        action_name = input_data["action_name"]
+        action_args = input_data["action_args"]
+        pre_bytes = input_data["pre_img_bytes"]
+        post_bytes = input_data["post_img_bytes"]
+        exec_outcome = input_data["exec_outcome"]
+
+        try:
+            # §5 revision: the lens input adapts to whichever frames exist.
+            # Both frames -> the classic BEFORE/AFTER transition prompt; a
+            # single frame (Pro steps never carry an independent after-frame,
+            # by design) -> the single-frame variant, which has no AFTER
+            # ACTION section and forbids synthesizing a transition.
+            dual = bool(pre_bytes) and bool(post_bytes)
+            template = self._prompt_template if dual else self._single_prompt_template
+            rendered_prompt = Template(template).render(step_number=step_number)
+            phrase = self._action_phrase(action_name, action_args)
+            lead_lines = [f"Step {step_number} Physical Action: {phrase}"]
+            # The verbatim arguments are worth a line only when the phrase
+            # folded some of them away (an index target, a repeat count, ...).
+            verbatim = self._verbatim_args(action_args)
+            if verbatim and not self._phrase_carries(phrase, verbatim):
+                lead_lines.append(f"Action arguments (verbatim): {action_name}({verbatim})")
+            lead_lines.append(f"Controller Outcome: {exec_outcome}")
+            focus = input_data.get("focus")
+            if isinstance(focus, dict) and focus:
+                lead_lines.extend(["", render_focus_block(focus, dual=dual)])
+            content_blocks: list[dict[str, Any]] = [{"type": "text", "text": "\n".join(lead_lines)}]
+
+            if pre_bytes:
+                # 🎨 Visually mark the exact action (tap ripple, sequence numbers, swipe arrow) on the BEFORE screenshot
+                annotated_pre_bytes = draw_action_overlay_on_image(
+                    image_bytes=pre_bytes,
+                    action_name=action_name,
+                    action_args=action_args,
+                )
+                b64_pre = base64.b64encode(annotated_pre_bytes).decode("utf-8")
+                content_blocks.append(
+                    {
+                        "type": "text",
+                        "text": (
+                            "--- [1] BEFORE ACTION SCREEN (Action Marked Visually in Red) ---"
+                            if dual
+                            else "--- [1] DECISION FRAME: SCREEN AT ACTION TIME"
+                            " (Action Marked Visually in Red) ---"
+                        ),
+                    }
+                )
+                content_blocks.append(
+                    {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64_pre}"}}
+                )
+
+            if post_bytes:
+                b64_post = base64.b64encode(post_bytes).decode("utf-8")
+                content_blocks.append(
+                    {
+                        "type": "text",
+                        "text": (
+                            "--- [2] AFTER ACTION SCREEN ---"
+                            if dual
+                            else "--- [1] SCREEN OBSERVED AFTER THE ACTION"
+                            " (no decision frame available) ---"
+                        ),
+                    }
+                )
+                content_blocks.append(
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/jpeg;base64,{b64_post}"},
+                    }
+                )
+
+            messages: list[BaseMessage] = [
+                SystemMessage(content=rendered_prompt),
+                HumanMessage(content=content_blocks),
+            ]
+
+            response = await asyncio.wait_for(self._llm.ainvoke(messages), timeout=25.0)
+            self._meter_lens_call(response)
+            # Only the final answer is a summary. Reasoning can be much longer
+            # than the summary limit and must not enter memory or trigger retries.
+            summary_text, _ = extract_thinking_tags(content_to_text(response.content))
+            degenerate = degenerate_summary_reason(summary_text) if summary_text else None
+            if degenerate:
+                logger.warning(
+                    f"VisualStepSummarizer: Discarding degenerate summary for"
+                    f" Step {step_number} ({degenerate}): {summary_text[:80]!r}"
+                )
+                return False
+            if summary_text:
+                self._summaries[key] = summary_text
+                logger.info(
+                    f"VisualStepSummarizer: Generated summary for Step {step_number}: {summary_text[:80]}..."
+                )
+
+                # Free binary image buffers (and the focus text) once the
+                # summary is secured; a landed job is never re-rendered.
+                if key in self._step_inputs:
+                    self._step_inputs[key]["pre_img_bytes"] = None
+                    self._step_inputs[key]["post_img_bytes"] = None
+                    self._step_inputs[key]["focus"] = None
+
+                # Update DataEngine telemetry if active
+                if self.ctx.data_engine:
+                    try:
+                        target_step = input_data.get("data_engine_step_id") or step_number
+                        self.ctx.data_engine.update_step_summary(
+                            target_step,
+                            summary_text,
+                            status="ready",
+                            source="visual_transition",
+                            model=self._model_name,
+                        )
+                    except Exception as de_err:
+                        logger.debug(f"DataEngine step summary update skipped: {de_err}")
+                return True
+
+        except Exception as e:
+            logger.warning(
+                f"VisualStepSummarizer: Error generating summary for step {step_number}: {e}"
+            )
+        return False

package/artemis-main/artemis/agents/history_analyzer/history_analyzer.md

@@ -0,0 +1,8 @@
+# IDENTITY
+You are a **History Analyzer**, expert in analyzing multi-agent session execution history to answer user queries.
+
+# MANDATE
+1. You are provided with the **Task Plan and Execution History** summarizing the high-level plan and history steps.
+2. For queries about specific step details (such as precise actions, coordinates, the reasoning behind a step, or its execution result), call `replay_steps` for the step or range in question; use `search_history` to locate the right steps by keyword when the step numbers are not obvious, and `get_step_screenshot` when you need to see what the screen showed.
+3. Never assume, guess, or invent details not visible in the Task Plan and Execution History. Always query `replay_steps` for specifics. If details are unavailable, state it clearly.
+4. Make good use of note tools to see what notes are available. When analyzing history records, open and cross-reference relevant notes to ensure you have grasped all necessary information.