android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/agents/explorer/tiers.py

@@ -0,0 +1,110 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Engine, turn budget, tools and caching settings for each Explorer tier.
+
+User configuration selects the tier. The calling agent sees the same
+``ask_explorer`` contract at every tier.
+"""
+
+from dataclasses import dataclass
+from typing import Literal
+
+from artemis.config.constants import ExplorerVersion
+
+ExplorerEngine = Literal["oneshot", "loop"]
+
+#: Tool that terminates a reasoning loop; always exposed by loop engines.
+SUBMIT_TOOL = "submit_answer"
+
+#: Every perception / transformation tool the loop engines can expose.
+PERCEPTION_TOOLS: frozenset[str] = frozenset(
+    {
+        "ask_perception_tool",
+        "detect_objects",
+        "get_ocr_list",
+        "inspect_region",
+        "ask_image_processor",
+    }
+)
+
+
+@dataclass(frozen=True)
+class ExplorerTier:
+    """Behavioral profile of one Explorer tier."""
+
+    name: ExplorerVersion
+    engine: ExplorerEngine
+    #: Reasoning turns for the loop engine (1 for one-shot detection).
+    max_turns: int
+    #: Perception tools exposed to the loop engine (``submit_answer`` is implicit).
+    tools: frozenset[str]
+    #: Default for Gemini explicit context caching when the user did not decide.
+    caching: bool
+    #: One-line summary used in logs and configuration help.
+    summary: str
+
+    @property
+    def is_oneshot(self) -> bool:
+        return self.engine == "oneshot"
+
+    @property
+    def hidden_tools(self) -> frozenset[str]:
+        """Perception tools this tier does not expose."""
+        return PERCEPTION_TOOLS - self.tools
+
+
+EXPLORER_TIERS: dict[ExplorerVersion, ExplorerTier] = {
+    "flash": ExplorerTier(
+        name="flash",
+        engine="oneshot",
+        max_turns=1,
+        tools=frozenset(),
+        caching=False,
+        summary="One-shot visual detection on the current screenshot; fastest, no reasoning loop.",
+    ),
+    "pro": ExplorerTier(
+        name="pro",
+        engine="loop",
+        max_turns=3,
+        tools=frozenset({"ask_perception_tool"}),
+        caching=False,
+        summary=(
+            "Short reasoning loop combining UI-tree search, coordinate audit and visual"
+            " detection; balanced default."
+        ),
+    ),
+    "ultra": ExplorerTier(
+        name="ultra",
+        engine="loop",
+        max_turns=8,
+        tools=PERCEPTION_TOOLS,
+        caching=True,
+        summary=(
+            "Deep reasoning loop with zooming, OCR and pixel-level image processing for"
+            " layout-critical searches; slowest."
+        ),
+    ),
+}
+
+DEFAULT_TIER: ExplorerVersion = "flash"
+
+
+def get_tier(version: str | None) -> ExplorerTier:
+    """Returns the tier for ``version``; unknown or empty names fall back to the default."""
+    key = str(version or "").strip().lower()
+    for tier in EXPLORER_TIERS.values():
+        if tier.name == key:
+            return tier
+    return EXPLORER_TIERS[DEFAULT_TIER]

package/artemis-main/artemis/agents/explorer/tool_declarations.py

@@ -0,0 +1,394 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Static tool declarations for the Explorer agent.
+
+Split out of ``artemis.agents.explorer.explorer`` as pure data:
+``UNIVERSAL_EXPLORER_TOOLS`` (OpenAI-style dict schemas for the universal
+LangChain path) and ``NATIVE_EXPLORER_TOOL_DECLARATIONS`` (google-genai
+``FunctionDeclaration`` objects for the native Gemini path).
+"""
+
+from google.genai import types
+
+UNIVERSAL_EXPLORER_TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "ask_perception_tool",
+            "description": (
+                "[Perception] Concurrently executes and awaits three sub-tasks in parallel,"
+                " including UI-tree text search, coordinate audit, and visual object detection."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "search_query": {
+                        "type": "string",
+                        "description": "Required. Text query to search for in UI elements.",
+                    },
+                    "nx": {
+                        "type": "integer",
+                        "description": "Required. Normalized X coordinate (0-1000 range).",
+                    },
+                    "ny": {
+                        "type": "integer",
+                        "description": "Required. Normalized Y coordinate (0-1000 range).",
+                    },
+                    "detect_queries": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "Required. List of query strings to visually detect.",
+                    },
+                },
+                "required": ["search_query", "nx", "ny", "detect_queries"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "detect_objects",
+            "description": (
+                "[Perception] Locates elements, shapes and more using a vision-language model."
+                " Returns their normalized coordinates (0-1000 scale) with label dots."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "target_image_id": {
+                        "type": "string",
+                        "description": "The ID of the image to process.",
+                    },
+                    "queries": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                        "description": "A list of single query strings to detect on the screen.",
+                    },
+                },
+                "required": ["queries", "target_image_id"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "get_ocr_list",
+            "description": (
+                "[Perception] Retrieves all text elements detected on the screen via OCR."
+            ),
+            "parameters": {"type": "object", "properties": {}},
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "ask_image_processor",
+            "description": ("[Transformation] A scripting tool for pixel-level image processing."),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "target_image_id": {
+                        "type": "string",
+                        "description": "The ID of the image in the Image Pool to start from.",
+                    },
+                    "instruction": {
+                        "type": "string",
+                        "description": "1-2 precise commands describing a simple task.",
+                    },
+                },
+                "required": ["instruction", "target_image_id"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "inspect_region",
+            "description": (
+                "[Perception] Crops and resizes a bounding box region of the original screenshot."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "x_min": {"type": "integer", "description": "Left X coordinate (0-1000)."},
+                    "y_min": {"type": "integer", "description": "Top Y coordinate (0-1000)."},
+                    "x_max": {"type": "integer", "description": "Right X coordinate (0-1000)."},
+                    "y_max": {"type": "integer", "description": "Bottom Y coordinate (0-1000)."},
+                    "zoom_factor": {
+                        "type": "number",
+                        "description": "Zoom factor to upscale the cropped region (1.0 to 4.0).",
+                    },
+                },
+                "required": ["x_min", "y_min", "x_max", "y_max", "zoom_factor"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "submit_answer",
+            "description": (
+                "[Finalization] Submits the final list of ranked candidate UI elements matching"
+                " the search query, or a fallback message if no elements were found."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "candidates": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "label": {
+                                    "type": "string",
+                                    "description": "Reference label (e.g. X1, O2, D3, or a number like 3)",
+                                },
+                                "description": {
+                                    "type": "string",
+                                    "description": "Brief description of this candidate element",
+                                },
+                                "coords": {
+                                    "type": "array",
+                                    "items": {"type": "integer"},
+                                    "description": (
+                                        "[nx, ny] coordinates in normalized 0-1000 scale"
+                                    ),
+                                },
+                            },
+                            "required": ["label", "coords"],
+                        },
+                        "description": "A ranked list of candidate UI elements matching the query.",
+                    },
+                    "fallback_message": {
+                        "type": "string",
+                        "description": (
+                            "Optional fallback explanation if target elements were not found."
+                        ),
+                    },
+                },
+                "required": ["candidates"],
+            },
+        },
+    },
+]
+
+
+NATIVE_EXPLORER_TOOL_DECLARATIONS = [
+    types.FunctionDeclaration(
+        name="ask_perception_tool",
+        description=(
+            "[Perception] Concurrently executes and awaits three sub-tasks"
+            " in parallel, including UI-tree text search, coordinate audit,"
+            " and visual object detection on the screen.\n\nOutput"
+            " Format:\nReturns:'text' (str): A single string consolidating"
+            " results with 0-1000 normalized coordinates, including UI"
+            " search matches labeled [X],"
+            " coordinate search matches labeled [X], and visually detected"
+            " object targets labeled [D].\n(Annotated screenshots are also"
+            " added to the context containing corresponding colored label"
+            " dots)."
+        ),
+        parameters=types.Schema(
+            type=types.Type.OBJECT,
+            properties={
+                "search_query": types.Schema(
+                    type=types.Type.STRING,
+                    description=("Required. Text query to search for in UI elements."),
+                ),
+                "nx": types.Schema(
+                    type=types.Type.INTEGER,
+                    description=(
+                        "Required. Normalized X coordinate (0-1000 range)"
+                        " to audit for overlapping elements."
+                    ),
+                ),
+                "ny": types.Schema(
+                    type=types.Type.INTEGER,
+                    description=(
+                        "Required. Normalized Y coordinate (0-1000 range)"
+                        " to audit for overlapping elements."
+                    ),
+                ),
+                "detect_queries": types.Schema(
+                    type=types.Type.ARRAY,
+                    items=types.Schema(type=types.Type.STRING),
+                    description=(
+                        "Required. List of query strings to visually detect on the screen."
+                    ),
+                ),
+            },
+            required=["search_query", "nx", "ny", "detect_queries"],
+        ),
+    ),
+    types.FunctionDeclaration(
+        name="detect_objects",
+        description=(
+            "[Perception] Locates elements, shapes and more using a"
+            " VLM. Returns their normalized coordinates (in a [0, 1000]"
+            " scale) labeled with [D1], [D2] etc. in the text output, along"
+            " with an annotated image containing corresponding visual label"
+            " dots."
+        ),
+        parameters=types.Schema(
+            type=types.Type.OBJECT,
+            properties={
+                "target_image_id": types.Schema(
+                    type=types.Type.STRING,
+                    description="The ID of the image to process.",
+                ),
+                "queries": types.Schema(
+                    type=types.Type.ARRAY,
+                    items=types.Schema(type=types.Type.STRING),
+                    description=("A list of single query strings to detect on the screen."),
+                ),
+            },
+            required=["queries", "target_image_id"],
+        ),
+    ),
+    types.FunctionDeclaration(
+        name="get_ocr_list",
+        description=(
+            "[Perception] Retrieves all text elements detected on the"
+            " original, unprocessed screen via OCR. Returns their"
+            " normalized coordinates (in a [0, 1000] scale) labeled with"
+            " [O1], [O2] etc. in the text output, along with an annotated"
+            " image containing corresponding visual label dots."
+        ),
+        parameters=types.Schema(
+            type=types.Type.OBJECT,
+            properties={},
+        ),
+    ),
+    types.FunctionDeclaration(
+        name="ask_image_processor",
+        description=(
+            "[Transformation] A scripting tool for pixel-level image"
+            " processing. CAUTION: This tool is slow and expensive. Returns"
+            " a summary of the operations performed and new image IDs,"
+            " along with new labels and screen coordinates."
+        ),
+        parameters=types.Schema(
+            type=types.Type.OBJECT,
+            properties={
+                "target_image_id": types.Schema(
+                    type=types.Type.STRING,
+                    description=(
+                        "The ID of the image in the Image Pool to start"
+                        " from (e.g. 'img_0', 'img_1')."
+                    ),
+                ),
+                "instruction": types.Schema(
+                    type=types.Type.STRING,
+                    description=(
+                        "1-2 precise commands describing a simple task."
+                        " Include all necessary coordinates and other"
+                        " parameters.\n"
+                    ),
+                ),
+            },
+            required=["instruction", "target_image_id"],
+        ),
+    ),
+    types.FunctionDeclaration(
+        name="submit_answer",
+        description=(
+            "[Finalization] Submits the final list of ranked candidate UI"
+            " elements matching the search query, or a fallback message if"
+            " no elements were found, to complete the search task. Fails"
+            " validation if: (1) candidates is empty and no"
+            " fallback_message is provided, (2) coordinates are not exactly"
+            " 2 integers [nx, ny], or (3) coordinates lie outside"
+            " normalized [0, 1000] range."
+        ),
+        parameters=types.Schema(
+            type=types.Type.OBJECT,
+            properties={
+                "candidates": types.Schema(
+                    type=types.Type.ARRAY,
+                    items=types.Schema(
+                        type=types.Type.OBJECT,
+                        properties={
+                            "label": types.Schema(
+                                type=types.Type.STRING,
+                                description=(
+                                    "Reference label (e.g. X1, O2, D3, or a number like 3)"
+                                ),
+                            ),
+                            "coords": types.Schema(
+                                type=types.Type.ARRAY,
+                                items=types.Schema(type=types.Type.INTEGER),
+                                description=("[nx, ny] coordinates in normalized 0-1000 scale"),
+                            ),
+                            "description": types.Schema(
+                                type=types.Type.STRING,
+                                description=("Brief description of this candidate element"),
+                            ),
+                        },
+                        required=["label", "coords"],
+                    ),
+                    description=(
+                        "A ranked list of candidate UI elements that match"
+                        " the query, sorted from highest to lowest"
+                        " confidence (maximum 10 candidates). Leave empty"
+                        " if no matching elements were found on the screen."
+                    ),
+                ),
+                "fallback_message": types.Schema(
+                    type=types.Type.STRING,
+                    description=(
+                        "Optional fallback explanation if target elements"
+                        " were not found or if there are special"
+                        " observations."
+                    ),
+                ),
+            },
+            required=["candidates"],
+        ),
+    ),
+    types.FunctionDeclaration(
+        name="inspect_region",
+        description=(
+            "[Perception] Crops and resizes a bounding box region of the"
+            " original screenshot. Returns the cropped and resized image"
+            " directly, without any image ID or coordinate transform."
+        ),
+        parameters=types.Schema(
+            type=types.Type.OBJECT,
+            properties={
+                "x_min": types.Schema(
+                    type=types.Type.INTEGER,
+                    description="Left X coordinate (0-1000).",
+                ),
+                "y_min": types.Schema(
+                    type=types.Type.INTEGER,
+                    description="Top Y coordinate (0-1000).",
+                ),
+                "x_max": types.Schema(
+                    type=types.Type.INTEGER,
+                    description="Right X coordinate (0-1000).",
+                ),
+                "y_max": types.Schema(
+                    type=types.Type.INTEGER,
+                    description="Bottom Y coordinate (0-1000).",
+                ),
+                "zoom_factor": types.Schema(
+                    type=types.Type.NUMBER,
+                    description=("Zoom factor to upscale the cropped region (1.0 to 4.0)."),
+                ),
+            },
+            required=["x_min", "y_min", "x_max", "y_max", "zoom_factor"],
+        ),
+    ),
+]