android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/artemis/agents/video_analyzer/audio_native.py

@@ -0,0 +1,543 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Native Gemini execution path for audio-only segment analysis.
+
+Extracted from ``video_analyzer.py`` as a pure structural split.  Patchable
+collaborators are looked up late through the facade module so
+``mock.patch("artemis.agents.video_analyzer.video_analyzer.<name>")`` targets
+keep working.
+"""
+
+import asyncio
+from pathlib import Path
+
+from google.genai import types
+
+from artemis.agents.video_analyzer import video_analyzer as _va
+from artemis.agents.video_analyzer.chunk_conversation import (
+    _MISSING_SUMMARY_ERROR,
+    _NO_TOOL_CALL_ERROR,
+    _SUBMIT_NOW_WARNING,
+    _WRONG_TOOL_ERROR,
+    SUB_AGENT_MAX_TURNS,
+    _append_rejection,
+    _clean_text,
+    _tool_name,
+)
+from artemis.agents.video_analyzer.reliability import (
+    SubAgentAnswerExhausted,
+    classify_video_failure,
+)
+from artemis.constants import SAFETY_SETTINGS_BLOCK_NONE
+from artemis.data_engine.trace import TraceSpan
+from artemis.llm.reliability import retry_policy_for
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+_BAD_CONFIDENCE_ERROR = (
+    "Error: 'confidence_score' is missing or out of bounds. It must be a float"
+    " between 0.0 and 1.0 inclusive. Fix the payload and call submit_answer again."
+)
+
+
+class _AudioMedia:
+    """Mutable holder for per-attempt audio artifacts shared with failure handling."""
+
+    __slots__ = ("audio_path", "actual_start", "prompt_with_context")
+
+    def __init__(self):
+        self.audio_path: Path | None = None
+        self.actual_start: float | None = None
+        self.prompt_with_context: str | None = None
+
+
+async def exec_analyze_audio_only(
+    analyzer, start_time: float, end_time: float | None, specific_query: str
+) -> str:
+    """Analyzes only the audio track of a segment, with claim/lease and retries."""
+    if not isinstance(start_time, (int, float)):
+        return "Error: start_time must be a numeric recording-relative timestamp."
+    if not str(specific_query or "").strip():
+        return "Error: specific_query must describe the audio evidence to find."
+    max_retries = analyzer.native_max_retries
+    current_start = round(start_time, 1) if isinstance(start_time, (int, float)) else start_time
+    current_end = round(end_time, 1) if isinstance(end_time, (int, float)) else end_time
+
+    if current_end is None:
+        controller = _va.get_controller(analyzer.ctx)
+        async with _va.TRANSCODE_SEMAPHORE:
+            metadata = await controller.extract_segment_metadata(current_start, None)
+        duration = getattr(metadata, "duration_seconds", None)
+        actual_start = getattr(metadata, "actual_start_relative_time", current_start)
+        if not metadata.success or not isinstance(duration, (int, float)) or duration <= 0:
+            return f"Error fetching audio segment metadata: {metadata.message}"
+        current_start = round(float(actual_start), 1)
+        current_end = round(current_start + float(duration), 1)
+        if metadata.video_path:
+            metadata_path = Path(metadata.video_path)
+            analyzer.local_files_to_cleanup.add(metadata_path)
+            analyzer.local_dirs_to_cleanup.add(metadata_path.parent)
+    if current_end <= current_start:
+        return "Error: end_time must be greater than start_time."
+
+    device_id = getattr(getattr(analyzer.ctx, "device", None), "device_id", None)
+    active_session = _va.get_active_session(device_id) if device_id else None
+    claim = analyzer.blackboard.claim_segment(
+        current_start,
+        current_end,
+        specific_query,
+        modality="audio",
+        model_name=analyzer.model_name,
+        source_generation=(active_session.generation if active_session is not None else None),
+    )
+    if claim.state == "cached":
+        result = (
+            f"[from {current_start:.1f}s to {current_end:.1f}s] Audio Summary: "
+            f"{claim.summary or 'No audio summary provided.'}"
+        )
+        if claim.analysis:
+            result += f" Analysis: {claim.analysis}"
+        return "CACHED AUDIO ANALYSIS: " + result
+    if claim.state == "in_progress":
+        return (
+            f"[from {current_start:.1f}s to {current_end:.1f}s] "
+            "Audio analysis is already in progress; do not duplicate it."
+        )
+    lease_owner = claim.lease_owner
+    if lease_owner is None:
+        raise RuntimeError("Video blackboard returned an audio claim without a lease owner")
+
+    media = _AudioMedia()
+
+    for attempt in range(max_retries + 1):
+        try:
+            logger.info(
+                f"Attempt {attempt + 1}/{max_retries + 1} for"
+                f" analyze_audio_only: start={current_start},"
+                f" end={current_end}"
+            )
+            result = await _prepare_audio_media(analyzer, media, current_start, current_end)
+            media.prompt_with_context = _build_audio_prompt(
+                analyzer, result, media.actual_start, start_time, end_time, specific_query
+            )
+
+            if not analyzer.use_native_gemini:
+                audio_result = await analyzer._exec_analyze_audio_universal(
+                    audio_path=media.audio_path,
+                    start_time=current_start,
+                    end_time=current_end,
+                    actual_start=media.actual_start,
+                    prompt_with_context=media.prompt_with_context,
+                    specific_query=specific_query,
+                )
+                summary, analysis = analyzer._parse_chunk_result(audio_result)
+                analyzer.blackboard.complete_segment(
+                    current_start,
+                    current_end,
+                    specific_query,
+                    lease_owner,
+                    summary,
+                    analysis,
+                    modality="audio",
+                )
+                return audio_result
+
+            return await _run_native_audio_conversation(
+                analyzer,
+                media,
+                current_start,
+                current_end,
+                start_time,
+                end_time,
+                specific_query,
+                lease_owner,
+            )
+
+        except Exception as e:
+            # Same contract as the video sub-agent loop: classification
+            # owns the retry decision and every retry restarts the attempt
+            # from scratch.
+            failure = classify_video_failure(e)
+            attempts_so_far = attempt + 1
+            if not failure.retryable:
+                logger.warning(
+                    f"Audio non-retryable failure [{failure.category.value}]"
+                    f" on attempt {attempts_so_far}: {e}"
+                )
+                return await _finish_failed_audio(
+                    analyzer,
+                    media,
+                    current_start,
+                    current_end,
+                    specific_query,
+                    lease_owner,
+                    e,
+                    retryable=False,
+                )
+            policy = retry_policy_for(failure.category)
+            allowed_attempts = min(max_retries + 1, policy.max_attempts)
+            if attempts_so_far >= allowed_attempts:
+                logger.warning(
+                    f"Audio analysis exhausted {attempts_so_far} attempt(s)"
+                    f" [{failure.category.value}]: {e}"
+                )
+                return await _finish_failed_audio(
+                    analyzer,
+                    media,
+                    current_start,
+                    current_end,
+                    specific_query,
+                    lease_owner,
+                    e,
+                )
+            delay = policy.delay_for(attempts_so_far)
+            logger.warning(
+                f"Audio retryable failure [{failure.category.value}]"
+                f" ({attempts_so_far}/{allowed_attempts}): {e}; retrying"
+                f" from scratch in {delay:.2f}s"
+            )
+            _va._record_llm_retry(
+                str(e),
+                delay,
+                attempt=attempts_so_far,
+                max_retries=allowed_attempts,
+                source="video_audio_agent",
+            )
+            await asyncio.sleep(delay)
+
+
+async def _finish_failed_audio(
+    analyzer,
+    media: _AudioMedia,
+    current_start: float,
+    current_end: float,
+    specific_query: str,
+    lease_owner: str,
+    error: Exception,
+    *,
+    retryable: bool | None = None,
+):
+    """Records terminal audio failure, attempting the universal fallback first."""
+    failure = classify_video_failure(error)
+    if (
+        analyzer.use_native_gemini
+        and failure.should_fallback
+        and media.audio_path is not None
+        and media.actual_start is not None
+        and media.prompt_with_context is not None
+    ):
+        try:
+            logger.warning(
+                f"Native audio analysis failed [{failure.category.value}]; using universal fallback"
+            )
+            _va._record_llm_event(
+                "llm_fallback",
+                {
+                    "reason": "native_audio_analysis_failed",
+                    "category": failure.category.value,
+                    "error": str(error)[:500],
+                },
+            )
+            fallback_result = await analyzer._exec_analyze_audio_universal(
+                audio_path=media.audio_path,
+                start_time=current_start,
+                end_time=current_end,
+                actual_start=media.actual_start,
+                prompt_with_context=media.prompt_with_context,
+                specific_query=specific_query,
+                force_fallback=True,
+            )
+            summary, analysis = analyzer._parse_chunk_result(fallback_result)
+            analyzer.blackboard.complete_segment(
+                current_start,
+                current_end,
+                specific_query,
+                lease_owner,
+                summary,
+                analysis,
+                modality="audio",
+            )
+            return fallback_result
+        except Exception as fallback_error:
+            error = fallback_error
+            failure = classify_video_failure(error)
+            retryable = None
+    analyzer.blackboard.fail_segment(
+        current_start,
+        current_end,
+        specific_query,
+        lease_owner,
+        str(error),
+        modality="audio",
+        retryable=failure.retryable if retryable is None else retryable,
+        error_category=failure.category.value,
+    )
+    raise error
+
+
+async def _prepare_audio_media(
+    analyzer, media: _AudioMedia, current_start: float, current_end: float
+):
+    """Extracts the segment and its audio track, tracking artifacts for cleanup."""
+    controller = _va.get_controller(analyzer.ctx)
+    async with _va.TRANSCODE_SEMAPHORE:
+        result = await controller.extract_segment_metadata(current_start, current_end)
+    if not result.success or not result.video_path:
+        raise Exception(f"Failed to get video segment: {result.message}")
+
+    path = Path(result.video_path)
+    analyzer.local_files_to_cleanup.add(path)
+    analyzer.local_dirs_to_cleanup.add(path.parent)
+
+    async with _va.TRANSCODE_SEMAPHORE:
+        audio_path = await _va.extract_audio_from_video(path)
+    analyzer.local_files_to_cleanup.add(audio_path)
+    analyzer.local_dirs_to_cleanup.add(audio_path.parent)
+    media.audio_path = audio_path
+    actual_start = getattr(result, "actual_start_relative_time", current_start)
+    if not isinstance(actual_start, (int, float)):
+        actual_start = current_start
+    media.actual_start = actual_start
+    return result
+
+
+def _build_audio_prompt(
+    analyzer,
+    result,
+    actual_start: float,
+    start_time: float,
+    end_time: float | None,
+    specific_query: str,
+) -> str:
+    """Assembles the audio sub-agent prompt with timing context, ledger, and warnings."""
+    warnings = analyzer.get_overlapping_warnings(start_time, end_time)
+    warning_block = ""
+    if warnings:
+        lines = ["WARNING: The following queries were already tried in this timeframe:\n"]
+        for w in warnings:
+            lines.append(f"Searched for: {w['target']} -> {w['summary']}")
+        warning_block = "\n".join(lines) + "\n\n"
+
+    duration_secs = getattr(result, "duration_seconds", None)
+    actual_end = actual_start + duration_secs if isinstance(duration_secs, (int, float)) else None
+    end_str = f" to {actual_end:.1f}s" if actual_end is not None else ""
+    warning_val = getattr(result, "warning", None)
+    truncation_note = (
+        f" (NOTE: {warning_val})" if isinstance(warning_val, str) and warning_val else ""
+    )
+
+    prompt_with_context = (
+        "IMPORTANT CONTEXT: This audio segment corresponds to"
+        " the test's relative time from"
+        f" {actual_start:.1f}s{end_str}{truncation_note}. When"
+        " you report timestamps, remember that your 00:00 is"
+        f" exactly {actual_start:.1f}s in the test relative"
+        " time. Please calculate and output all timestamps as"
+        " the exact test relative time (e.g., if you hear"
+        " something at 00:05, report"
+        f" {actual_start + 5:.1f}s).\n\n"
+    )
+    if analyzer.enable_ledger:
+        prompt_with_context += (
+            "BLACKBOARD"
+            f" LEDGER:\n{analyzer.blackboard_ledger}\nDirective:"
+            " Use the Blackboard to understand what happened"
+            " outside your assigned timeframe. Do not"
+            " duplicate findings already on the board.\n\n"
+        )
+    if warning_block:
+        prompt_with_context += (
+            f"{warning_block}Directive: Do not repeat the"
+            " failed searches listed above. Shift your"
+            " attention to the current task.\n\n"
+        )
+    prompt_with_context += f"{specific_query}"
+    return prompt_with_context
+
+
+async def _run_native_audio_conversation(
+    analyzer,
+    media: _AudioMedia,
+    current_start: float,
+    current_end: float,
+    start_time: float,
+    end_time: float | None,
+    specific_query: str,
+    lease_owner: str,
+) -> str:
+    """Uploads the audio and drives the native Gemini audio agent to a committed answer."""
+    file = None
+    try:
+        with TraceSpan(name="upload_audio_to_gemini") as span:
+            file = await analyzer.upload_and_poll_file(media.audio_path)
+            span.result = f"Uploaded {file.name}"
+        logger.info(f"Invoking Gemini for audio-only task with model {analyzer.model_name}...")
+
+        sub_agent_contents = [
+            types.Content(
+                role="user",
+                parts=[
+                    types.Part.from_uri(file_uri=file.uri, mime_type=file.mime_type)
+                    if hasattr(file, "uri")
+                    else file,
+                    types.Part.from_text(text=media.prompt_with_context),
+                ],
+            )
+        ]
+
+        (
+            final_summary,
+            final_analysis,
+            final_confidence_score,
+        ) = await _drive_audio_agent_loop(analyzer, sub_agent_contents)
+
+        end_val = end_time if end_time is not None else "unknown"
+        analyzer._record_blackboard_entry(
+            {
+                "start": start_time,
+                "end": end_val,
+                "target": specific_query,
+                "summary": final_summary,
+                "confidence_score": final_confidence_score,
+            },
+            modality="audio",
+        )
+
+        end_str = f"{end_time:.1f}s" if end_time is not None else "unknown"
+        audio_result = (
+            f"[from {start_time:.1f}s to {end_str}] Summary:"
+            f" {final_summary} Analysis: {final_analysis}"
+        )
+        analyzer.blackboard.complete_segment(
+            current_start,
+            current_end,
+            specific_query,
+            lease_owner,
+            final_summary,
+            final_analysis,
+            modality="audio",
+        )
+        return audio_result
+    finally:
+        if file:
+            logger.info(f"Cleaning up cloud file {file.name}...")
+            try:
+                await asyncio.wait_for(
+                    analyzer.client.aio.files.delete(name=file.name),
+                    timeout=30,
+                )
+                analyzer.cloud_files_to_cleanup.discard(file.name)
+            except Exception as ce:
+                logger.error(f"Failed to delete cloud file {file.name}: {ce}")
+
+
+async def _drive_audio_agent_loop(analyzer, sub_agent_contents: list) -> tuple[str, str, float]:
+    """Return a validated (summary, analysis, confidence_score).
+
+    Raise ``SubAgentAnswerExhausted`` if no answer is accepted within the turn limit.
+    """
+    rejection = _NO_TOOL_CALL_ERROR
+
+    for turn in range(1, SUB_AGENT_MAX_TURNS + 1):
+        if turn == SUB_AGENT_MAX_TURNS - 1:
+            sub_agent_contents.append(
+                types.Content(
+                    role="user",
+                    parts=[types.Part.from_text(text=_SUBMIT_NOW_WARNING)],
+                )
+            )
+
+        with TraceSpan(name="gemini_generate_audio_content") as span:
+            response = await asyncio.wait_for(
+                analyzer.client.aio.models.generate_content(
+                    model=analyzer.model_name,
+                    contents=sub_agent_contents,
+                    config=types.GenerateContentConfig(
+                        system_instruction=analyzer.audio_system_prompt,
+                        tools=[types.Tool(function_declarations=[analyzer.submit_answer_tool])],
+                        safety_settings=SAFETY_SETTINGS_BLOCK_NONE,
+                    ),
+                ),
+                timeout=analyzer.model_call_timeout_seconds,
+            )
+
+        full_text = response.text or ""
+        function_calls = response.function_calls or []
+
+        if function_calls:
+            answer, rejection = _handle_audio_function_calls(
+                sub_agent_contents, response, function_calls, full_text
+            )
+            if answer is not None:
+                return answer
+        else:
+            rejection = _NO_TOOL_CALL_ERROR
+            sub_agent_contents.append(
+                types.Content(
+                    role="model",
+                    parts=[types.Part.from_text(text=full_text)],
+                )
+            )
+
+    raise SubAgentAnswerExhausted(SUB_AGENT_MAX_TURNS, rejection)
+
+
+def _validate_audio_answer(summary: str, confidence_score) -> str | None:
+    """Return a validation error for the audio answer, or None if valid."""
+    if not summary:
+        return _MISSING_SUMMARY_ERROR
+    if (
+        not isinstance(confidence_score, (int, float))
+        or isinstance(confidence_score, bool)
+        or not (0.0 <= float(confidence_score) <= 1.0)
+    ):
+        return _BAD_CONFIDENCE_ERROR
+    return None
+
+
+def _handle_audio_function_calls(
+    sub_agent_contents: list,
+    response,
+    function_calls: list,
+    full_text: str,
+) -> tuple[tuple[str, str, float] | None, str | None]:
+    """Return (answer, rejection), appending any rejection to the conversation."""
+    original_parts = response.candidates[0].content.parts
+    for fc in function_calls:
+        if _tool_name(fc.name) != "submit_answer":
+            continue
+        args = dict(fc.args or {})
+        summary = _clean_text(args.get("summary"))
+        analysis = _clean_text(args.get("analysis"))
+
+        rejection = _validate_audio_answer(summary, args.get("confidence_score"))
+        if rejection is not None:
+            _append_rejection(
+                sub_agent_contents, original_parts, function_calls, full_text, fc.name, rejection
+            )
+            return None, rejection
+
+        return (summary, analysis or "No analysis provided.", float(args["confidence_score"])), None
+
+    _append_rejection(
+        sub_agent_contents,
+        original_parts,
+        function_calls,
+        full_text,
+        function_calls[0].name,
+        _WRONG_TOOL_ERROR,
+    )
+    return None, _WRONG_TOOL_ERROR