android-midscene-automation 0.2.4 → 1.1.0

package/artemis-main/artemis/llm/fake_model.py

@@ -0,0 +1,179 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Fake chat model for infrastructure testing (ARTEMIS_FAKE_LLM=1).
+
+Completes any task on the first LLM invocation so that queueing, entry
+points, device locking, and status propagation can be exercised without
+real model calls:
+
+- Flash (report_task_status bound): immediately reports status=completed.
+- Pro planner (save_note bound): saves a task plan whose only subgoal is
+  already checked, so convergence_gate ends the graph after planning.
+- Anything else: returns a plain "FAKE_LLM_OK" text message.
+
+ARTEMIS_FAKE_LLM_DELAY_S adds an artificial per-call delay so tasks stay
+alive long enough to observe queue/concurrency behavior.
+"""
+
+import asyncio
+import json
+import time
+from typing import Any, Iterator, AsyncIterator
+import uuid
+
+from langchain_core.callbacks import (
+    AsyncCallbackManagerForLLMRun,
+    CallbackManagerForLLMRun,
+)
+from langchain_core.language_models.chat_models import BaseChatModel
+from langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage
+from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
+
+FAKE_DONE_TEXT = "FAKE_LLM: task auto-completed for infrastructure testing."
+
+
+def _tool_name(tool: Any) -> str | None:
+    name = getattr(tool, "name", None)
+    if isinstance(name, str):
+        return name
+    if isinstance(tool, dict):
+        if isinstance(tool.get("name"), str):
+            return tool["name"]
+        fn = tool.get("function")
+        if isinstance(fn, dict) and isinstance(fn.get("name"), str):
+            return fn["name"]
+    return None
+
+
+class FakeChatModel(BaseChatModel):
+    """Deterministic fake model that finishes every task on the first call."""
+
+    bound_tool_names: list[str] = []
+    delay_s: float = 0.0
+
+    @property
+    def _llm_type(self) -> str:
+        return "artemis-fake"
+
+    def bind_tools(self, tools: Any, **kwargs: Any) -> "FakeChatModel":
+        names = [n for n in (_tool_name(t) for t in tools) if n]
+        return FakeChatModel(bound_tool_names=names, delay_s=self.delay_s)
+
+    def with_structured_output(self, schema: Any, **kwargs: Any) -> "FakeChatModel":
+        return self
+
+    def _decide(self, messages: list[BaseMessage]) -> AIMessage:
+        call_id = f"fake-{uuid.uuid4().hex[:12]}"
+        usage = {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
+
+        if "report_task_status" in self.bound_tool_names:
+            return AIMessage(
+                content="Task verified complete on first observation.",
+                tool_calls=[
+                    {
+                        "name": "report_task_status",
+                        "args": {"status": "completed", "explanation": FAKE_DONE_TEXT},
+                        "id": call_id,
+                    }
+                ],
+                usage_metadata=usage,
+            )
+
+        if "save_note" in self.bound_tool_names:
+            already_planned = any(
+                tc.get("name") == "save_note"
+                for m in messages
+                for tc in (getattr(m, "tool_calls", None) or [])
+            )
+            if already_planned:
+                return AIMessage(content="Plan saved. " + FAKE_DONE_TEXT, usage_metadata=usage)
+            return AIMessage(
+                content="Saving completed plan.",
+                tool_calls=[
+                    {
+                        "name": "save_note",
+                        "args": {
+                            "key": "task_plan",
+                            "content": "- [x] " + FAKE_DONE_TEXT,
+                        },
+                        "id": call_id,
+                    }
+                ],
+                usage_metadata=usage,
+            )
+
+        return AIMessage(content=FAKE_DONE_TEXT, usage_metadata=usage)
+
+    def _to_chunk(self, msg: AIMessage) -> ChatGenerationChunk:
+        tool_call_chunks = [
+            {
+                "name": tc["name"],
+                "args": json.dumps(tc["args"]),
+                "id": tc["id"],
+                "index": i,
+                "type": "tool_call_chunk",
+            }
+            for i, tc in enumerate(msg.tool_calls or [])
+        ]
+        chunk = AIMessageChunk(
+            content=msg.content,
+            tool_call_chunks=tool_call_chunks,
+            usage_metadata=msg.usage_metadata,
+        )
+        return ChatGenerationChunk(message=chunk)
+
+    def _generate(
+        self,
+        messages: list[BaseMessage],
+        stop: list[str] | None = None,
+        run_manager: CallbackManagerForLLMRun | None = None,
+        **kwargs: Any,
+    ) -> ChatResult:
+        if self.delay_s > 0:
+            time.sleep(self.delay_s)
+        return ChatResult(generations=[ChatGeneration(message=self._decide(messages))])
+
+    async def _agenerate(
+        self,
+        messages: list[BaseMessage],
+        stop: list[str] | None = None,
+        run_manager: AsyncCallbackManagerForLLMRun | None = None,
+        **kwargs: Any,
+    ) -> ChatResult:
+        if self.delay_s > 0:
+            await asyncio.sleep(self.delay_s)
+        return ChatResult(generations=[ChatGeneration(message=self._decide(messages))])
+
+    def _stream(
+        self,
+        messages: list[BaseMessage],
+        stop: list[str] | None = None,
+        run_manager: CallbackManagerForLLMRun | None = None,
+        **kwargs: Any,
+    ) -> Iterator[ChatGenerationChunk]:
+        if self.delay_s > 0:
+            time.sleep(self.delay_s)
+        yield self._to_chunk(self._decide(messages))
+
+    async def _astream(
+        self,
+        messages: list[BaseMessage],
+        stop: list[str] | None = None,
+        run_manager: AsyncCallbackManagerForLLMRun | None = None,
+        **kwargs: Any,
+    ) -> AsyncIterator[ChatGenerationChunk]:
+        if self.delay_s > 0:
+            await asyncio.sleep(self.delay_s)
+        yield self._to_chunk(self._decide(messages))

package/artemis-main/artemis/llm/google/__init__.py

@@ -0,0 +1,57 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Google / Gemini specific knowledge shared by every call site.
+
+Two concerns live here so they are defined exactly once:
+
+* :mod:`artemis.llm.google.provider` - "is this endpoint Google?" and the
+  Gemini model capability table (thinking levels, agentic video).
+* :mod:`artemis.llm.google.usage` - token-usage normalization across the
+  LangChain, ``generate_content`` and Interactions API response shapes.
+
+Both modules are dependency-free (no settings, no SDK client) so they can be
+imported from configuration code without creating import cycles.
+"""
+
+from artemis.llm.google.provider import (
+    VideoProcessing,
+    gemini_version,
+    is_agentic_video_auto_eligible,
+    is_gemini_model,
+    is_google_chat_model,
+    is_google_family_provider,
+    is_google_provider,
+    resolve_video_processing,
+    strip_provider_prefix,
+    supports_agentic_video,
+    supports_thinking_level,
+)
+from artemis.llm.google.usage import normalize_usage, usage_from_message
+
+__all__ = [
+    "VideoProcessing",
+    "gemini_version",
+    "is_agentic_video_auto_eligible",
+    "is_gemini_model",
+    "is_google_chat_model",
+    "is_google_family_provider",
+    "is_google_provider",
+    "normalize_usage",
+    "resolve_video_processing",
+    "strip_provider_prefix",
+    "supports_agentic_video",
+    "supports_thinking_level",
+    "usage_from_message",
+]

package/artemis-main/artemis/llm/google/provider.py

@@ -0,0 +1,185 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Google provider identity and the Gemini model capability table.
+
+Provider identity has two tiers:
+
+* :func:`is_google_provider` - the Gemini Developer API (``google`` /
+  ``gemini``). Native ``google-genai`` features (Files API, explicit caching,
+  the Interactions API) are only reachable here.
+* :func:`is_google_family_provider` - additionally Vertex AI, which shares
+  Google credentials but not the developer-API-only features.
+
+Model IDs use ``gemini-<major>.<minor>-<tier>[<suffix>]``. Release suffixes
+(``preview``, ``latest``, ``exp``, and numeric tokens) retain base-family
+capabilities. Variants such as ``image`` and ``tts`` are excluded from agentic
+video support, but their version still determines thinking-level support.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Literal, NamedTuple
+
+VideoProcessing = Literal["auto", "agentic", "static"]
+ResolvedVideoProcessing = Literal["agentic", "static"]
+
+_GOOGLE_API_PROVIDERS = frozenset({"google", "gemini"})
+_GOOGLE_FAMILY_PROVIDERS = _GOOGLE_API_PROVIDERS | frozenset({"vertexai", "vertex"})
+
+_GEMINI_MODEL_RE = re.compile(r"gemini-(\d+)\.(\d+)-(flash-lite|flash|pro)", re.IGNORECASE)
+_RELEASE_SUFFIX_TOKENS = frozenset({"preview", "latest", "exp"})
+
+
+class _GeminiModel(NamedTuple):
+    version: tuple[int, int]
+    tier: str
+    #: False for variant ids (``-image``, ``-tts``, ...) that carry their own
+    #: capabilities and must not inherit the family's answers.
+    is_base: bool
+
+
+# Agentic video understanding (Interactions API, ``processing: "agentic"``)
+# launched on Gemini 3.6/3.7/3.8 Flash and 3.5 Flash-Lite. Later releases in
+# the same families are assumed to keep it.
+_AGENTIC_VIDEO_MIN_FLASH = (3, 6)
+_AGENTIC_VIDEO_MIN_FLASH_LITE = (3, 5)
+
+
+def _normalize_provider(value: Any) -> str:
+    if value is None:
+        return ""
+    value = getattr(value, "value", value)
+    return str(value).lower().replace("_", "").replace("-", "").strip()
+
+
+def is_google_provider(value: Any) -> bool:
+    """True for the Gemini Developer API (``google`` / ``gemini``).
+
+    Accepts a provider name, a provider enum, or anything exposing ``.value``.
+    """
+    return _normalize_provider(value) in _GOOGLE_API_PROVIDERS
+
+
+def is_google_family_provider(value: Any) -> bool:
+    """True for every provider that authenticates with Google credentials."""
+    return _normalize_provider(value) in _GOOGLE_FAMILY_PROVIDERS
+
+
+def is_google_chat_model(model: Any) -> bool:
+    """True when a LangChain chat model is backed by Google (by class name).
+
+    Only used when no endpoint metadata is available; the class-name sniff is
+    the historical fallback and is kept in one place here.
+    """
+    cls = getattr(model, "__class__", None)
+    name = cls.__name__ if cls else ""
+    return "Google" in name or "Gemini" in name or "VertexAI" in name
+
+
+def strip_provider_prefix(model_name: Any) -> str:
+    """Drops a ``provider/`` prefix (``google/gemini-3.8-flash`` -> ``gemini-3.8-flash``)."""
+    name = str(model_name or "")
+    return name.rsplit("/", 1)[-1] if "/" in name else name
+
+
+def is_gemini_model(model_name: Any) -> bool:
+    """True when the model name refers to any Gemini model."""
+    return "gemini" in strip_provider_prefix(model_name).lower()
+
+
+def _is_release_suffix(rest: str) -> bool:
+    """Whether what follows the tier is a release suffix (see the module docstring)."""
+    if not rest:
+        return True
+    if not rest.startswith("-"):
+        return False
+    return all(token in _RELEASE_SUFFIX_TOKENS or token.isdigit() for token in rest[1:].split("-"))
+
+
+def _parse_gemini_model(model_name: Any) -> _GeminiModel | None:
+    name = strip_provider_prefix(model_name).lower()
+    match = _GEMINI_MODEL_RE.search(name)
+    if not match:
+        return None
+    return _GeminiModel(
+        version=(int(match.group(1)), int(match.group(2))),
+        tier=match.group(3),
+        is_base=_is_release_suffix(name[match.end() :]),
+    )
+
+
+def gemini_version(model_name: Any) -> tuple[int, int] | None:
+    """``(major, minor)`` parsed from a ``gemini-X.Y-*`` name, else ``None``."""
+    parsed = _parse_gemini_model(model_name)
+    return parsed.version if parsed else None
+
+
+def _gemini_tier(model_name: Any) -> str | None:
+    """The tier of a *base* model id; ``None`` for variants and non-Gemini names."""
+    parsed = _parse_gemini_model(model_name)
+    return parsed.tier if parsed and parsed.is_base else None
+
+
+def supports_thinking_level(model_name: Any) -> bool:
+    """Whether the model accepts ``thinking_level`` (Gemini 3 and later).
+
+    Gemini 1.x/2.x only understand ``thinking_budget``. Names that do not
+    carry a parsable version (e.g. the robotics ER models) are assumed to be
+    current and therefore accept it.
+    """
+    version = gemini_version(model_name)
+    return version is None or version[0] >= 3
+
+
+def supports_agentic_video(model_name: Any) -> bool:
+    """Whether the Gemini API accepts ``processing: "agentic"`` for this model.
+
+    Only base Flash / Flash-Lite ids qualify; variants (``-image``, ``-tts``,
+    ``-live``, ...) do not inherit the family's support.
+    """
+    parsed = _parse_gemini_model(model_name)
+    if parsed is None or not parsed.is_base:
+        return False
+    if parsed.tier == "flash":
+        return parsed.version >= _AGENTIC_VIDEO_MIN_FLASH
+    if parsed.tier == "flash-lite":
+        return parsed.version >= _AGENTIC_VIDEO_MIN_FLASH_LITE
+    return False
+
+
+def is_agentic_video_auto_eligible(model_name: Any) -> bool:
+    """Models that get agentic video processing under ``processing: "auto"``.
+
+    Flash-Lite is deliberately excluded: it accepts the mode but mis-reads
+    timestamps in practice, so it must be opted into explicitly.
+    """
+    return supports_agentic_video(model_name) and _gemini_tier(model_name) == "flash"
+
+
+def resolve_video_processing(configured: Any, model_name: Any) -> ResolvedVideoProcessing:
+    """Turns the configured video-processing knob into a concrete mode.
+
+    ``static`` is always honoured. ``agentic`` is honoured only for models
+    that support it and silently degrades otherwise. ``auto`` (and any
+    unrecognised value) enables agentic processing for the auto-eligible
+    models and keeps static processing for everything else.
+    """
+    mode = str(configured or "auto").strip().lower() if isinstance(configured, str) else "auto"
+    if mode == "static":
+        return "static"
+    if mode == "agentic":
+        return "agentic" if supports_agentic_video(model_name) else "static"
+    return "agentic" if is_agentic_video_auto_eligible(model_name) else "static"

package/artemis-main/artemis/llm/google/usage.py

@@ -0,0 +1,148 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Token-usage normalization across every response shape ARTEMIS sees.
+
+Google alone reports usage under three different vocabularies:
+
+* LangChain ``usage_metadata`` - ``input_tokens`` / ``output_tokens`` /
+  ``input_token_details.cache_read`` (OpenAI-style names appear via
+  ``response_metadata.token_usage``).
+* ``generate_content`` ``usage_metadata`` - ``prompt_token_count`` /
+  ``candidates_token_count`` / ``cached_content_token_count`` /
+  ``thoughts_token_count`` / ``tool_use_prompt_token_count``.
+* Interactions API ``usage`` - ``total_input_tokens`` /
+  ``total_output_tokens`` / ``total_cached_tokens`` / ``total_thought_tokens``
+  / ``total_tool_use_tokens``.
+
+:func:`normalize_usage` folds all of them into one dict so meters, traces and
+the UI read a single set of keys.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+_PROMPT_KEYS = ("prompt_tokens", "input_tokens", "prompt_token_count", "total_input_tokens")
+_COMPLETION_KEYS = (
+    "completion_tokens",
+    "output_tokens",
+    "candidates_token_count",
+    "total_output_tokens",
+)
+_TOTAL_KEYS = ("total_tokens", "total_token_count")
+_CACHED_KEYS = (
+    "cached_tokens",
+    "cached_content_token_count",
+    "cachedContentTokenCount",
+    "total_cached_tokens",
+)
+_THOUGHT_KEYS = (
+    "thought_tokens",
+    "thoughts_token_count",
+    "reasoning_tokens",
+    "total_thought_tokens",
+)
+_TOOL_USE_KEYS = ("tool_use_tokens", "tool_use_prompt_token_count", "total_tool_use_tokens")
+
+NORMALIZED_USAGE_KEYS = (
+    "prompt_tokens",
+    "completion_tokens",
+    "total_tokens",
+    "cached_tokens",
+    "thought_tokens",
+    "tool_use_tokens",
+)
+
+
+def _int(value: Any) -> int:
+    if isinstance(value, bool):
+        return 0
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return 0
+
+
+def _read(raw: Any, key: str) -> Any:
+    if isinstance(raw, dict):
+        return raw.get(key)
+    return getattr(raw, key, None)
+
+
+def _first(raw: Any, keys: tuple[str, ...]) -> int:
+    for key in keys:
+        value = _int(_read(raw, key))
+        if value:
+            return value
+    return 0
+
+
+def normalize_usage(raw: Any) -> dict[str, int] | None:
+    """Normalizes a usage payload (dict or object) into the shared key set.
+
+    Returns ``None`` when the payload carries no usable token counts, so
+    callers can distinguish "no usage reported" from an all-zero report.
+    """
+    if raw is None:
+        return None
+
+    prompt = _first(raw, _PROMPT_KEYS)
+    completion = _first(raw, _COMPLETION_KEYS)
+    total = _first(raw, _TOTAL_KEYS) or (prompt + completion)
+
+    cached = 0
+    details = _read(raw, "input_token_details")
+    if details is not None:
+        cached = _int(_read(details, "cache_read"))
+    if not cached:
+        cached = _first(raw, _CACHED_KEYS)
+
+    thought = 0
+    output_details = _read(raw, "output_token_details")
+    if output_details is not None:
+        thought = _int(_read(output_details, "reasoning"))
+    if not thought:
+        thought = _first(raw, _THOUGHT_KEYS)
+
+    tool_use = _first(raw, _TOOL_USE_KEYS)
+
+    if total <= 0:
+        return None
+    return {
+        "prompt_tokens": prompt,
+        "completion_tokens": completion,
+        "total_tokens": total,
+        "cached_tokens": cached,
+        "thought_tokens": thought,
+        "tool_use_tokens": tool_use,
+    }
+
+
+def usage_from_message(message: Any) -> dict[str, int] | None:
+    """Extracts normalized usage from a LangChain message, best-effort.
+
+    Prefers LangChain's ``usage_metadata``; falls back to the raw provider
+    payload stored under ``response_metadata`` (``usage_metadata`` for Google,
+    ``token_usage`` for OpenAI-compatible providers).
+    """
+    raw = getattr(message, "usage_metadata", None)
+    if isinstance(raw, dict) and raw:
+        return normalize_usage(raw)
+    meta = getattr(message, "response_metadata", None)
+    if isinstance(meta, dict):
+        candidate = meta.get("usage_metadata") or meta.get("token_usage")
+        if isinstance(candidate, dict) and candidate:
+            return normalize_usage(candidate)
+    return None

package/artemis-main/artemis/llm/parser.py

@@ -0,0 +1,32 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Structured output and thinking tag parser."""
+
+import re
+from artemis.llm.schemas import ThinkingBlock
+
+
+def extract_thinking_tags(text: str) -> tuple[str, list[ThinkingBlock]]:
+    """Extracts content inside <think>, <thought>, or <thinking> tags."""
+    thinking_blocks: list[ThinkingBlock] = []
+
+    pattern = r"<(think|thought|thinking)>(.*?)</\1>"
+    matches = re.findall(pattern, text, flags=re.DOTALL | re.IGNORECASE)
+
+    for _, m in matches:
+        thinking_blocks.append(ThinkingBlock(content=m.strip(), is_native=False))
+
+    cleaned_text = re.sub(pattern, "", text, flags=re.DOTALL | re.IGNORECASE).strip()
+    return cleaned_text, thinking_blocks