android-midscene-automation 0.2.3 → 1.1.0

package/artemis-main/artemis/agents/video_analyzer/chunk_native.py

@@ -0,0 +1,437 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Native Gemini execution path for a single video chunk sub-agent.
+
+Extracted from ``video_analyzer.py`` as a pure structural split.  Patchable
+collaborators (``get_controller``, ``compress_video_for_api``, semaphores,
+LLM event recorders, ...) are looked up late through the facade module so
+``mock.patch("artemis.agents.video_analyzer.video_analyzer.<name>")`` targets
+keep working.  The sub-agent conversation itself lives in
+``chunk_conversation``.
+"""
+
+import asyncio
+from pathlib import Path
+
+from artemis.agents.video_analyzer import video_analyzer as _va
+from artemis.agents.video_analyzer.chunk_agentic import run_agentic_chunk_conversation
+from artemis.agents.video_analyzer.chunk_conversation import (
+    _ChunkMedia,
+    _run_native_chunk_conversation,
+)
+from artemis.agents.video_analyzer.reliability import (
+    AgenticVideoDegraded,
+    classify_video_failure,
+    is_agentic_rejection,
+)
+from artemis.llm.reliability import retry_policy_for
+from third_party.mobile_use.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+async def exec_single_chunk(
+    analyzer, start_time: float, end_time: float | None, specific_query: str
+) -> str:
+    """Analyzes one closed chunk interval, with claim/lease and retry handling."""
+    max_retries = analyzer.native_max_retries
+    current_start = round(start_time, 1) if isinstance(start_time, (int, float)) else start_time
+    current_end = round(end_time, 1) if isinstance(end_time, (int, float)) else end_time
+
+    if not isinstance(current_start, (int, float)) or not isinstance(current_end, (int, float)):
+        raise ValueError("Child video analysis requires a closed numeric interval")
+
+    device_id = getattr(getattr(analyzer.ctx, "device", None), "device_id", None)
+    active_session = _va.get_active_session(device_id) if device_id else None
+    claim = analyzer.blackboard.claim_segment(
+        current_start,
+        current_end,
+        specific_query,
+        model_name=analyzer.model_name,
+        source_generation=(active_session.generation if active_session is not None else None),
+    )
+    if claim.state == "cached":
+        cached = (
+            f"[from {current_start:.1f}s to {current_end:.1f}s] Summary: "
+            f"{claim.summary or 'No summary provided.'}"
+        )
+        if claim.analysis:
+            cached += f" Analysis: {claim.analysis}"
+        return cached
+    if claim.state == "in_progress":
+        return (
+            f"[from {current_start:.1f}s to {current_end:.1f}s] "
+            "Analysis is already in progress in another video agent; do not duplicate it."
+        )
+    lease_owner = claim.lease_owner
+    if lease_owner is None:
+        raise RuntimeError("Video blackboard returned a claim without a lease owner")
+
+    media = _ChunkMedia()
+
+    attempt = 0
+    while attempt <= max_retries:
+        # The mode this attempt was launched in: a parallel chunk may degrade
+        # the run while this one is still in flight.
+        attempt_mode = analyzer.video_processing
+        try:
+            logger.info(
+                f"Attempt {attempt + 1}/{max_retries + 1} for sub-agent:"
+                f" start={current_start}, end={current_end}"
+            )
+            # A sibling may switch the run to static during queuing or media
+            # preparation. Check the size before preparation and before dispatch.
+            _ensure_fits_static_ceiling(analyzer, current_start, current_end)
+            result, slowdown_factor = await _prepare_chunk_media(
+                analyzer, media, current_start, current_end
+            )
+            media.prompt_with_context = _build_chunk_prompt(
+                analyzer,
+                result,
+                media.actual_start,
+                slowdown_factor,
+                start_time,
+                end_time,
+                specific_query,
+            )
+            _ensure_fits_static_ceiling(analyzer, current_start, current_end)
+
+            if not analyzer.use_native_gemini:
+                chunk_result = await analyzer._exec_single_chunk_universal(
+                    compressed_path=media.compressed_path,
+                    raw_path=media.path,
+                    start_time=current_start,
+                    end_time=current_end,
+                    actual_start=media.actual_start,
+                    prompt_with_context=media.prompt_with_context,
+                    specific_query=specific_query,
+                )
+                summary, analysis = analyzer._parse_chunk_result(chunk_result)
+                analyzer.blackboard.complete_segment(
+                    current_start,
+                    current_end,
+                    specific_query,
+                    lease_owner,
+                    summary,
+                    analysis,
+                )
+                return chunk_result
+
+            if analyzer.video_processing == "agentic":
+                return await run_agentic_chunk_conversation(
+                    analyzer,
+                    media,
+                    current_start,
+                    current_end,
+                    start_time,
+                    end_time,
+                    specific_query,
+                    lease_owner,
+                )
+
+            return await _run_native_chunk_conversation(
+                analyzer,
+                media,
+                current_start,
+                current_end,
+                start_time,
+                end_time,
+                specific_query,
+                lease_owner,
+            )
+
+        except Exception as e:
+            # Retry decisions are owned by the shared classification layer.
+            # Every attempt restarts from scratch (segment re-extracted,
+            # file re-uploaded, stream re-consumed); partially accumulated
+            # stream output from the failed attempt is discarded, never
+            # surfaced as a complete answer.
+            failure = classify_video_failure(e)
+            if attempt_mode == "agentic" and is_agentic_rejection(failure, e):
+                # Switch to static and let the coordinator resize the interval.
+                _degrade_agentic_to_static(analyzer, e, failure)
+                return await _finish_failed_chunk(
+                    analyzer,
+                    media,
+                    current_start,
+                    current_end,
+                    specific_query,
+                    lease_owner,
+                    AgenticVideoDegraded(
+                        f"Agentic video processing rejected ({e}); re-plan"
+                        f" {current_start:.1f}s-{current_end:.1f}s at the static chunk size"
+                    ),
+                    retryable=False,
+                )
+            attempts_so_far = attempt + 1
+            if not failure.retryable:
+                logger.warning(
+                    f"Sub-agent non-retryable failure [{failure.category.value}]"
+                    f" on attempt {attempts_so_far}: {e}"
+                )
+                return await _finish_failed_chunk(
+                    analyzer,
+                    media,
+                    current_start,
+                    current_end,
+                    specific_query,
+                    lease_owner,
+                    e,
+                    retryable=False,
+                )
+            policy = retry_policy_for(failure.category)
+            allowed_attempts = min(max_retries + 1, policy.max_attempts)
+            if attempts_so_far >= allowed_attempts:
+                logger.warning(
+                    f"Sub-agent exhausted {attempts_so_far} attempt(s)"
+                    f" [{failure.category.value}]: {e}"
+                )
+                return await _finish_failed_chunk(
+                    analyzer,
+                    media,
+                    current_start,
+                    current_end,
+                    specific_query,
+                    lease_owner,
+                    e,
+                )
+            delay = policy.delay_for(attempts_so_far)
+            logger.warning(
+                f"Sub-agent retryable failure [{failure.category.value}]"
+                f" ({attempts_so_far}/{allowed_attempts}): {e}; retrying"
+                f" from scratch in {delay:.2f}s (partial stream output"
+                " from the failed attempt is discarded)"
+            )
+            _va._record_llm_retry(
+                str(e),
+                delay,
+                attempt=attempts_so_far,
+                max_retries=allowed_attempts,
+                source="video_sub_agent",
+            )
+            await asyncio.sleep(delay)
+            attempt += 1
+
+    raise RuntimeError(
+        f"Video sub-agent exhausted retries for {current_start:.1f}s-{current_end:.1f}s"
+    )
+
+
+def _degrade_agentic_to_static(analyzer, error: Exception, failure) -> None:
+    """Switches the rest of this run to static video understanding (idempotent)."""
+    if analyzer.video_processing != "agentic":
+        return
+    logger.warning(
+        f"Agentic video processing rejected ({error}); the rest of this run"
+        " analyzes video in static mode"
+    )
+    _va._record_llm_event(
+        "llm_fallback",
+        {
+            "reason": "agentic_video_rejected",
+            "category": failure.category.value,
+            "error": str(error)[:500],
+        },
+    )
+    analyzer.video_processing = "static"
+    analyzer.chunk_size_seconds = analyzer.static_chunk_size_seconds
+
+
+def _ensure_fits_static_ceiling(analyzer, current_start: float, current_end: float) -> None:
+    """Refuses to analyze a clip sized for the agentic ceiling once the run is static.
+
+    Raises :class:`AgenticVideoDegraded`, which the attempt loop treats as a
+    non-retryable failure: the lease is released via ``fail_segment`` and the
+    coordinator re-plans the interval into static-sized pieces.
+    """
+    if analyzer.video_processing == "agentic":
+        return
+    if current_end - current_start <= analyzer.chunk_size_seconds + 1e-3:
+        return
+    raise AgenticVideoDegraded(
+        f"Run degraded to static video processing while {current_start:.1f}s-{current_end:.1f}s"
+        f" was pending; re-plan it at the static chunk size"
+        f" ({analyzer.chunk_size_seconds:.0f}s)"
+    )
+
+
+async def _finish_failed_chunk(
+    analyzer,
+    media: _ChunkMedia,
+    current_start: float,
+    current_end: float,
+    specific_query: str,
+    lease_owner: str,
+    error: Exception,
+    *,
+    retryable: bool | None = None,
+):
+    """Records terminal failure, attempting the universal fallback first when eligible."""
+    failure = classify_video_failure(error)
+    if (
+        analyzer.use_native_gemini
+        and failure.should_fallback
+        and media.path is not None
+        and media.compressed_path is not None
+        and media.actual_start is not None
+        and media.prompt_with_context is not None
+    ):
+        try:
+            logger.warning("Native video chunk exhausted retries; using universal fallback")
+            _va._record_llm_event(
+                "llm_fallback",
+                {
+                    "reason": "native_video_chunk_failed",
+                    "category": failure.category.value,
+                    "error": str(error)[:500],
+                },
+            )
+            fallback_result = await analyzer._exec_single_chunk_universal(
+                compressed_path=media.compressed_path,
+                raw_path=media.path,
+                start_time=current_start,
+                end_time=current_end,
+                actual_start=media.actual_start,
+                prompt_with_context=media.prompt_with_context,
+                specific_query=specific_query,
+                force_fallback=True,
+            )
+            summary, analysis = analyzer._parse_chunk_result(fallback_result)
+            analyzer.blackboard.complete_segment(
+                current_start,
+                current_end,
+                specific_query,
+                lease_owner,
+                summary,
+                analysis,
+            )
+            return fallback_result
+        except Exception as fallback_error:
+            error = fallback_error
+            failure = classify_video_failure(error)
+            retryable = None
+    analyzer.blackboard.fail_segment(
+        current_start,
+        current_end,
+        specific_query,
+        lease_owner,
+        str(error),
+        retryable=failure.retryable if retryable is None else retryable,
+        error_category=failure.category.value,
+    )
+    raise error
+
+
+async def _prepare_chunk_media(
+    analyzer, media: _ChunkMedia, current_start: float, current_end: float
+):
+    """Extracts and compresses the chunk segment, tracking artifacts for cleanup."""
+    controller = _va.get_controller(analyzer.ctx)
+    async with _va.TRANSCODE_SEMAPHORE:
+        result = await controller.extract_segment_metadata(current_start, current_end)
+    if not result.success or not result.video_path:
+        raise Exception(f"Failed to get video segment: {result.message}")
+
+    path = Path(result.video_path)
+    analyzer.local_files_to_cleanup.add(path)
+    analyzer.local_dirs_to_cleanup.add(path.parent)
+    media.path = path
+
+    actual_start = getattr(result, "actual_start_relative_time", current_start)
+    if not isinstance(actual_start, (int, float)):
+        actual_start = current_start
+    media.actual_start = actual_start
+
+    slowdown_factor = 1.0
+    if current_end is not None and (current_end - current_start) <= _va.SLOWDOWN_THRESHOLD_SECONDS:
+        slowdown_factor = _va.SLOWDOWN_FACTOR
+
+    async with _va.TRANSCODE_SEMAPHORE:
+        compressed_path = await _va.compress_video_for_api(
+            path,
+            force_compress=True,
+            start_offset_seconds=actual_start,
+            slowdown_factor=slowdown_factor,
+        )
+    if compressed_path != path:
+        analyzer.local_files_to_cleanup.add(compressed_path)
+        analyzer.local_dirs_to_cleanup.add(compressed_path.parent)
+    media.compressed_path = compressed_path
+
+    actual_start = getattr(result, "actual_start_relative_time", current_start)
+    if not isinstance(actual_start, (int, float)):
+        actual_start = current_start
+    media.actual_start = actual_start
+    return result, slowdown_factor
+
+
+def _build_chunk_prompt(
+    analyzer,
+    result,
+    actual_start: float,
+    slowdown_factor: float,
+    start_time: float,
+    end_time: float | None,
+    specific_query: str,
+) -> str:
+    """Assembles the sub-agent prompt with timing context, ledger, and warnings."""
+    warnings = analyzer.get_overlapping_warnings(start_time, end_time)
+    warning_block = ""
+    if warnings:
+        lines = ["WARNING: The following queries were already tried in this timeframe:\n"]
+        for w in warnings:
+            lines.append(f"Searched for: {w['target']} -> {w['summary']}")
+        warning_block = "\n".join(lines) + "\n\n"
+
+    duration_secs = getattr(result, "duration_seconds", None)
+    actual_end = actual_start + duration_secs if isinstance(duration_secs, (int, float)) else None
+    end_str = f" to {actual_end:.1f}s" if actual_end is not None else ""
+    warning_val = getattr(result, "warning", None)
+    truncation_note = (
+        f" (NOTE: {warning_val})" if isinstance(warning_val, str) and warning_val else ""
+    )
+
+    slowdown_note = ""
+    if slowdown_factor != 1.0:
+        slowdown_note = "WARNING: This video is slowed down to capture fast micro-actions.\n\n"
+
+    prompt_with_context = (
+        "IMPORTANT CONTEXT: This video segment corresponds to"
+        " the test's relative time from"
+        f" {actual_start:.1f}s{end_str}{truncation_note}. A"
+        " timestamp showing the exact test relative time"
+        " (formatted as '<seconds> s') is burned in the"
+        " top-right corner of the video. Please watch this"
+        " burned-in timestamp and use its value to report all"
+        " timestamps (e.g., if the burned timestamp shows '90"
+        f" s', report 90.0s).\n\n{slowdown_note}"
+    )
+    if analyzer.enable_ledger:
+        prompt_with_context += (
+            "BLACKBOARD"
+            f" LEDGER:\n{analyzer.blackboard_ledger}\nDirective:"
+            " Use the Blackboard to understand what happened"
+            " outside your assigned timeframe. Do not"
+            " duplicate findings already on the board.\n\n"
+        )
+    if warning_block:
+        prompt_with_context += (
+            f"{warning_block}Directive: Avoid repeating"
+            " searches that have already failed. Try to"
+            " approach the problem from a different angle or"
+            " use a different search strategy.\n\n"
+        )
+    prompt_with_context += f"{specific_query}"
+    return prompt_with_context

package/artemis-main/artemis/agents/video_analyzer/conflict_resolution.py

@@ -0,0 +1,137 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ConflictResolutionService:
+    @staticmethod
+    def calculate_1d_iou(a, b):
+        start = max(a[0], b[0])
+        end = min(a[1], b[1])
+        intersection = max(0.0, end - start)
+        union = (a[1] - a[0]) + (b[1] - b[0]) - intersection
+        return intersection / (union + 1e-6)
+
+    @staticmethod
+    def calculate_1d_iomin(a, b):
+        start = max(a[0], b[0])
+        end = min(a[1], b[1])
+        intersection = max(0.0, end - start)
+        min_len = min(a[1] - a[0], b[1] - b[0])
+        if min_len <= 0:
+            return 0.0
+        return intersection / min_len
+
+    @staticmethod
+    def sanitize_interval(interval):
+        start = interval.get("start", 0.0)
+        end = interval.get("end", "unknown")
+
+        if isinstance(start, (int, float)):
+            start = max(0.0, float(start))
+
+        if end != "unknown" and isinstance(end, (int, float)):
+            end = max(0.0, float(end))
+            if start > end:
+                start, end = end, start
+
+        interval["start"] = start
+        interval["end"] = end
+
+        if "confidence_score" not in interval or interval["confidence_score"] is None:
+            interval["confidence_score"] = 0.0
+
+        if end != "unknown":
+            interval["duration"] = float(end) - float(start)
+        else:
+            interval["duration"] = 0.0
+
+        return interval
+
+    @staticmethod
+    def group_by_semantic_class(proposals):
+        groups = {}
+        for p in proposals:
+            target = p.get("target", "unknown")
+            if target not in groups:
+                groups[target] = []
+            groups[target].append(p)
+        return groups
+
+    @staticmethod
+    def apply_1d_nms(group_proposals):
+        if len(group_proposals) <= 1:
+            return group_proposals
+
+        unknown_end_proposals = [p for p in group_proposals if p.get("end") == "unknown"]
+        valid_proposals = [p for p in group_proposals if p.get("end") != "unknown"]
+
+        valid_proposals.sort(
+            key=lambda x: (
+                x.get("confidence_score", 0.0),
+                -x.get("duration", 0.0),
+            ),
+            reverse=True,
+        )
+
+        n = len(valid_proposals)
+        suppressed_mask = [False] * n
+
+        for i in range(n):
+            if suppressed_mask[i]:
+                continue
+            a = (valid_proposals[i]["start"], valid_proposals[i]["end"])
+            for j in range(i + 1, n):
+                if suppressed_mask[j]:
+                    continue
+                b = (valid_proposals[j]["start"], valid_proposals[j]["end"])
+                iou = ConflictResolutionService.calculate_1d_iou(a, b)
+                iomin = ConflictResolutionService.calculate_1d_iomin(a, b)
+
+                if iou > 0.5 or iomin > 0.8:
+                    suppressed_mask[j] = True
+
+        kept_proposals = [valid_proposals[i] for i in range(n) if not suppressed_mask[i]]
+        return kept_proposals + unknown_end_proposals
+
+    @classmethod
+    def clean(cls, blackboard_entries):
+        try:
+            sanitized = [cls.sanitize_interval(dict(e)) for e in blackboard_entries]
+            groups = cls.group_by_semantic_class(sanitized)
+
+            cleaned_entries = []
+            for target, proposals in groups.items():
+                if len(proposals) <= 1:
+                    cleaned_entries.extend(proposals)
+                else:
+                    cleaned_entries.extend(cls.apply_1d_nms(proposals))
+
+            suppressed_count = len(blackboard_entries) - len(cleaned_entries)
+            if suppressed_count > 0:
+                logger.info(
+                    "NMS filtering telemetry: Successfully suppressed"
+                    f" {suppressed_count} redundant proposals."
+                )
+
+            # Sort chronologically by start time before returning so downstream systems read it in order
+            cleaned_entries.sort(key=lambda x: float(x.get("start", 0.0)))
+
+            return cleaned_entries
+        except Exception as e:
+            logger.error(f"NMS filter crashed: {e}. Falling back to raw ledger state.")
+            return blackboard_entries