android-midscene-automation 0.2.4 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +159 -29
- package/artemis-main/.dockerignore +15 -0
- package/artemis-main/.env.example +34 -0
- package/artemis-main/CONTRIBUTING.md +35 -0
- package/artemis-main/Dockerfile +68 -0
- package/artemis-main/LICENSE +202 -0
- package/artemis-main/Makefile +115 -0
- package/artemis-main/README.md +322 -0
- package/artemis-main/README_CN.md +320 -0
- package/artemis-main/apps/__init__.py +15 -0
- package/artemis-main/apps/admin_console/__init__.py +19 -0
- package/artemis-main/apps/admin_console/core/__init__.py +13 -0
- package/artemis-main/apps/admin_console/core/config.py +43 -0
- package/artemis-main/apps/admin_console/core/security.py +178 -0
- package/artemis-main/apps/admin_console/core/state.py +243 -0
- package/artemis-main/apps/admin_console/database/__init__.py +13 -0
- package/artemis-main/apps/admin_console/database/connection.py +60 -0
- package/artemis-main/apps/admin_console/database/repositories/__init__.py +13 -0
- package/artemis-main/apps/admin_console/database/repositories/session_repository.py +564 -0
- package/artemis-main/apps/admin_console/database/repositories/step_repository.py +512 -0
- package/artemis-main/apps/admin_console/database/repositories/trace_repository.py +133 -0
- package/artemis-main/apps/admin_console/index.html +3787 -0
- package/artemis-main/apps/admin_console/replay_manager.py +2585 -0
- package/artemis-main/apps/admin_console/routers/__init__.py +13 -0
- package/artemis-main/apps/admin_console/routers/media.py +198 -0
- package/artemis-main/apps/admin_console/routers/replay.py +91 -0
- package/artemis-main/apps/admin_console/routers/sessions.py +239 -0
- package/artemis-main/apps/admin_console/routers/steps.py +109 -0
- package/artemis-main/apps/admin_console/routers/stream.py +56 -0
- package/artemis-main/apps/admin_console/routers/system.py +761 -0
- package/artemis-main/apps/admin_console/routers/tasks.py +517 -0
- package/artemis-main/apps/admin_console/schemas/__init__.py +13 -0
- package/artemis-main/apps/admin_console/schemas/task_schema.py +47 -0
- package/artemis-main/apps/admin_console/server.py +508 -0
- package/artemis-main/apps/admin_console/services/__init__.py +13 -0
- package/artemis-main/apps/admin_console/services/device_stream_service.py +137 -0
- package/artemis-main/apps/admin_console/services/ipc_service.py +262 -0
- package/artemis-main/apps/admin_console/services/media_service.py +623 -0
- package/artemis-main/apps/admin_console/services/model_service.py +148 -0
- package/artemis-main/apps/admin_console/services/task_preset_catalog.py +514 -0
- package/artemis-main/apps/admin_console/services/task_queue_service.py +1719 -0
- package/artemis-main/apps/admin_console/services/worker_process_io.py +144 -0
- package/artemis-main/apps/showcase_ui/README.md +31 -0
- package/artemis-main/apps/showcase_ui/angular.json +132 -0
- package/artemis-main/apps/showcase_ui/package-lock.json +9472 -0
- package/artemis-main/apps/showcase_ui/package.json +39 -0
- package/artemis-main/apps/showcase_ui/proxy.conf.json +23 -0
- package/artemis-main/apps/showcase_ui/public/favicon.ico +0 -0
- package/artemis-main/apps/showcase_ui/public/logo.png +0 -0
- package/artemis-main/apps/showcase_ui/public/logo.svg +56 -0
- package/artemis-main/apps/showcase_ui/src/app/app.component.html +24 -0
- package/artemis-main/apps/showcase_ui/src/app/app.component.scss +49 -0
- package/artemis-main/apps/showcase_ui/src/app/app.component.spec.ts +42 -0
- package/artemis-main/apps/showcase_ui/src/app/app.component.ts +32 -0
- package/artemis-main/apps/showcase_ui/src/app/app.config.ts +29 -0
- package/artemis-main/apps/showcase_ui/src/app/app.routes.ts +27 -0
- package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream-startup.spec.ts +108 -0
- package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html +1436 -0
- package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss +3973 -0
- package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts +2235 -0
- package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.html +380 -0
- package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.scss +1171 -0
- package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.ts +270 -0
- package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html +433 -0
- package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss +987 -0
- package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts +632 -0
- package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html +130 -0
- package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.scss +406 -0
- package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.ts +74 -0
- package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.scss +115 -0
- package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.ts +52 -0
- package/artemis-main/apps/showcase_ui/src/app/core/data/smart-tasks.data.ts +334 -0
- package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.spec.ts +55 -0
- package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.ts +61 -0
- package/artemis-main/apps/showcase_ui/src/app/core/i18n/translate.pipe.ts +11 -0
- package/artemis-main/apps/showcase_ui/src/app/core/i18n/zh-cn.ts +651 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/markdown.model.ts +43 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.spec.ts +47 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.ts +165 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/session.model.ts +85 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/stream.model.ts +246 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/system.model.ts +118 -0
- package/artemis-main/apps/showcase_ui/src/app/core/services/task-recommendation.service.ts +111 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.html +2130 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.scss +5204 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.ts +1189 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.html +39 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.scss +151 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.ts +58 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.html +201 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.scss +886 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.ts +324 -0
- package/artemis-main/apps/showcase_ui/src/app/services/agent.service.spec.ts +418 -0
- package/artemis-main/apps/showcase_ui/src/app/services/agent.service.ts +2012 -0
- package/artemis-main/apps/showcase_ui/src/app/services/system.service.spec.ts +173 -0
- package/artemis-main/apps/showcase_ui/src/app/services/system.service.ts +620 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts +513 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.ts +1015 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.spec.ts +85 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.ts +662 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.spec.ts +70 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.ts +383 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.spec.ts +107 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.ts +123 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.spec.ts +92 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.ts +116 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts +640 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts +1021 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts +375 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.ts +1419 -0
- package/artemis-main/apps/showcase_ui/src/index.html +37 -0
- package/artemis-main/apps/showcase_ui/src/main.ts +22 -0
- package/artemis-main/apps/showcase_ui/src/styles.scss +60 -0
- package/artemis-main/apps/showcase_ui/tsconfig.app.json +23 -0
- package/artemis-main/apps/showcase_ui/tsconfig.json +27 -0
- package/artemis-main/apps/showcase_ui/tsconfig.spec.json +23 -0
- package/artemis-main/artemis/__init__.py +44 -0
- package/artemis-main/artemis/__main__.py +18 -0
- package/artemis-main/artemis/_legacy_imports.py +94 -0
- package/artemis-main/artemis/_version.py +30 -0
- package/artemis-main/artemis/agents/__init__.py +17 -0
- package/artemis-main/artemis/agents/checker/checker.json +9 -0
- package/artemis-main/artemis/agents/checker/checker.py +859 -0
- package/artemis-main/artemis/agents/diagnoser/diagnoser.json +23 -0
- package/artemis-main/artemis/agents/diagnoser/diagnoser.py +608 -0
- package/artemis-main/artemis/agents/explorer/constants.py +52 -0
- package/artemis-main/artemis/agents/explorer/explorer.json +57 -0
- package/artemis-main/artemis/agents/explorer/explorer.py +473 -0
- package/artemis-main/artemis/agents/explorer/geometry.py +73 -0
- package/artemis-main/artemis/agents/explorer/native_runner.py +878 -0
- package/artemis-main/artemis/agents/explorer/perception_tools.py +632 -0
- package/artemis-main/artemis/agents/explorer/run_setup.py +403 -0
- package/artemis-main/artemis/agents/explorer/screen_index.py +266 -0
- package/artemis-main/artemis/agents/explorer/tiers.py +110 -0
- package/artemis-main/artemis/agents/explorer/tool_declarations.py +394 -0
- package/artemis-main/artemis/agents/explorer/universal_runner.py +317 -0
- package/artemis-main/artemis/agents/flash/context_compressor.py +452 -0
- package/artemis-main/artemis/agents/flash/flash_runner.md +51 -0
- package/artemis-main/artemis/agents/flash/flash_summarizer.md +53 -0
- package/artemis-main/artemis/agents/flash/flash_summarizer_single.md +54 -0
- package/artemis-main/artemis/agents/flash/runner.py +1167 -0
- package/artemis-main/artemis/agents/flash/summarizer.py +511 -0
- package/artemis-main/artemis/agents/history_analyzer/history_analyzer.md +8 -0
- package/artemis-main/artemis/agents/history_analyzer/history_analyzer.py +182 -0
- package/artemis-main/artemis/agents/image_processor/image_processor.md +74 -0
- package/artemis-main/artemis/agents/image_processor/image_processor.py +322 -0
- package/artemis-main/artemis/agents/log_analyzer/log_analyzer.md +15 -0
- package/artemis-main/artemis/agents/log_analyzer/log_analyzer.py +403 -0
- package/artemis-main/artemis/agents/log_analyzer/log_reader_sub_agent.md +8 -0
- package/artemis-main/artemis/agents/log_analyzer/output_analyzer.md +11 -0
- package/artemis-main/artemis/agents/log_analyzer/output_analyzer.py +163 -0
- package/artemis-main/artemis/agents/object_detector/object_detector.json +9 -0
- package/artemis-main/artemis/agents/object_detector/object_detector.py +211 -0
- package/artemis-main/artemis/agents/operator/operator.json +3 -0
- package/artemis-main/artemis/agents/operator/operator.py +1394 -0
- package/artemis-main/artemis/agents/operator/prompts.py +1064 -0
- package/artemis-main/artemis/agents/outputter/outputter.md +36 -0
- package/artemis-main/artemis/agents/outputter/outputter.py +552 -0
- package/artemis-main/artemis/agents/planner/planner.json +32 -0
- package/artemis-main/artemis/agents/planner/planner.py +436 -0
- package/artemis-main/artemis/agents/prompt_assembly.py +79 -0
- package/artemis-main/artemis/agents/summarizer/summarizer.py +190 -0
- package/artemis-main/artemis/agents/validator/action_execution.py +98 -0
- package/artemis-main/artemis/agents/validator/categories.py +33 -0
- package/artemis-main/artemis/agents/validator/execution_loop.py +488 -0
- package/artemis-main/artemis/agents/validator/incidents.py +120 -0
- package/artemis-main/artemis/agents/validator/pixel_safety_net.md +15 -0
- package/artemis-main/artemis/agents/validator/precondition_pixel.py +320 -0
- package/artemis-main/artemis/agents/validator/precondition_xml.py +672 -0
- package/artemis-main/artemis/agents/validator/tool_declarations.py +332 -0
- package/artemis-main/artemis/agents/validator/validator.py +188 -0
- package/artemis-main/artemis/agents/video_analyzer/__init__.py +15 -0
- package/artemis-main/artemis/agents/video_analyzer/audio_native.py +543 -0
- package/artemis-main/artemis/agents/video_analyzer/blackboard.py +821 -0
- package/artemis-main/artemis/agents/video_analyzer/chunk_agentic.py +455 -0
- package/artemis-main/artemis/agents/video_analyzer/chunk_conversation.py +491 -0
- package/artemis-main/artemis/agents/video_analyzer/chunk_native.py +437 -0
- package/artemis-main/artemis/agents/video_analyzer/conflict_resolution.py +137 -0
- package/artemis-main/artemis/agents/video_analyzer/gemini_files.py +162 -0
- package/artemis-main/artemis/agents/video_analyzer/native_coordinator.py +541 -0
- package/artemis-main/artemis/agents/video_analyzer/reliability.py +128 -0
- package/artemis-main/artemis/agents/video_analyzer/universal_engine.py +487 -0
- package/artemis-main/artemis/agents/video_analyzer/universal_tools.py +323 -0
- package/artemis-main/artemis/agents/video_analyzer/video_analyzer.md +51 -0
- package/artemis-main/artemis/agents/video_analyzer/video_analyzer.py +875 -0
- package/artemis-main/artemis/agents/video_analyzer/video_sub_agent.md +15 -0
- package/artemis-main/artemis/agents/video_analyzer/video_sub_agent_audio.md +12 -0
- package/artemis-main/artemis/clients/accessibility_client.py +438 -0
- package/artemis-main/artemis/clients/screen_client_factory.py +403 -0
- package/artemis-main/artemis/config/__init__.py +309 -0
- package/artemis-main/artemis/config/agent.py +1098 -0
- package/artemis-main/artemis/config/constants.py +151 -0
- package/artemis-main/artemis/config/core/__init__.py +24 -0
- package/artemis-main/artemis/config/core/engine.py +118 -0
- package/artemis-main/artemis/config/core/interpolator.py +78 -0
- package/artemis-main/artemis/config/llm.py +267 -0
- package/artemis-main/artemis/config/paths.py +244 -0
- package/artemis-main/artemis/config/runtime.py +197 -0
- package/artemis-main/artemis/config/settings.py +315 -0
- package/artemis-main/artemis/constants.py +47 -0
- package/artemis-main/artemis/context.py +289 -0
- package/artemis-main/artemis/controllers/__init__.py +41 -0
- package/artemis-main/artemis/controllers/controller_factory.py +41 -0
- package/artemis-main/artemis/controllers/device_controller.py +29 -0
- package/artemis-main/artemis/controllers/types.py +37 -0
- package/artemis-main/artemis/controllers/unified_controller.py +649 -0
- package/artemis-main/artemis/core/__init__.py +21 -0
- package/artemis-main/artemis/core/diagnostics/__init__.py +90 -0
- package/artemis-main/artemis/core/diagnostics/adb_keys.py +270 -0
- package/artemis-main/artemis/core/diagnostics/adb_server_connection.py +330 -0
- package/artemis-main/artemis/core/diagnostics/device_smoke.py +364 -0
- package/artemis-main/artemis/core/diagnostics/emulator_manager.py +501 -0
- package/artemis-main/artemis/core/diagnostics/engine.py +438 -0
- package/artemis-main/artemis/core/diagnostics/hierarchy_parity.py +216 -0
- package/artemis-main/artemis/core/diagnostics/probes/adb_probe.py +887 -0
- package/artemis-main/artemis/core/diagnostics/probes/base.py +45 -0
- package/artemis-main/artemis/core/diagnostics/probes/credentials_probe.py +296 -0
- package/artemis-main/artemis/core/diagnostics/probes/host_probe.py +427 -0
- package/artemis-main/artemis/core/diagnostics/probes/runtime_probe.py +169 -0
- package/artemis-main/artemis/core/diagnostics/probes/toolchain_probe.py +139 -0
- package/artemis-main/artemis/core/diagnostics/readiness.py +83 -0
- package/artemis-main/artemis/core/diagnostics/schema.py +118 -0
- package/artemis-main/artemis/core/tool_declaration.py +43 -0
- package/artemis-main/artemis/core/tool_failure.py +56 -0
- package/artemis-main/artemis/data_engine/context_vars.py +24 -0
- package/artemis-main/artemis/data_engine/engine.py +1615 -0
- package/artemis-main/artemis/data_engine/history_reader.py +142 -0
- package/artemis-main/artemis/data_engine/models.py +131 -0
- package/artemis-main/artemis/data_engine/storage.py +1245 -0
- package/artemis-main/artemis/data_engine/trace.py +727 -0
- package/artemis-main/artemis/drivers/__init__.py +48 -0
- package/artemis-main/artemis/drivers/android/adb_driver.py +507 -0
- package/artemis-main/artemis/drivers/android/hierarchy.py +64 -0
- package/artemis-main/artemis/drivers/android/input_ime.py +120 -0
- package/artemis-main/artemis/drivers/android/recorder.py +57 -0
- package/artemis-main/artemis/drivers/base.py +290 -0
- package/artemis-main/artemis/drivers/cloud/cloud_driver.py +117 -0
- package/artemis-main/artemis/drivers/factory.py +80 -0
- package/artemis-main/artemis/drivers/mock/mock_driver.py +198 -0
- package/artemis-main/artemis/drivers/types.py +67 -0
- package/artemis-main/artemis/graph/checkpoints.py +1243 -0
- package/artemis-main/artemis/graph/graph.py +1055 -0
- package/artemis-main/artemis/graph/perception.py +234 -0
- package/artemis-main/artemis/graph/state.py +151 -0
- package/artemis-main/artemis/graph/visibility.py +238 -0
- package/artemis-main/artemis/interfaces/cli/__init__.py +19 -0
- package/artemis-main/artemis/interfaces/cli/commands/__init__.py +15 -0
- package/artemis-main/artemis/interfaces/cli/commands/batch.py +278 -0
- package/artemis-main/artemis/interfaces/cli/commands/doctor.py +468 -0
- package/artemis-main/artemis/interfaces/cli/commands/helper.py +226 -0
- package/artemis-main/artemis/interfaces/cli/commands/init.py +193 -0
- package/artemis-main/artemis/interfaces/cli/commands/mcp.py +862 -0
- package/artemis-main/artemis/interfaces/cli/commands/run.py +463 -0
- package/artemis-main/artemis/interfaces/cli/commands/server.py +151 -0
- package/artemis-main/artemis/interfaces/cli/commands/server_lifecycle.py +328 -0
- package/artemis-main/artemis/interfaces/cli/commands/trace.py +111 -0
- package/artemis-main/artemis/interfaces/cli/commands/ui.py +404 -0
- package/artemis-main/artemis/interfaces/cli/main.py +109 -0
- package/artemis-main/artemis/interfaces/sdk/__init__.py +31 -0
- package/artemis-main/artemis/interfaces/sdk/client.py +19 -0
- package/artemis-main/artemis/interfaces/sdk/task.py +72 -0
- package/artemis-main/artemis/llm/__init__.py +25 -0
- package/artemis-main/artemis/llm/fake_model.py +179 -0
- package/artemis-main/artemis/llm/google/__init__.py +57 -0
- package/artemis-main/artemis/llm/google/provider.py +185 -0
- package/artemis-main/artemis/llm/google/usage.py +148 -0
- package/artemis-main/artemis/llm/parser.py +32 -0
- package/artemis-main/artemis/llm/reliability.py +267 -0
- package/artemis-main/artemis/llm/router.py +389 -0
- package/artemis-main/artemis/llm/schemas.py +36 -0
- package/artemis-main/artemis/llm/structured.py +196 -0
- package/artemis-main/artemis/main.py +58 -0
- package/artemis-main/artemis/mcp/__init__.py +20 -0
- package/artemis-main/artemis/mcp/action_executor.py +760 -0
- package/artemis-main/artemis/mcp/action_manifest.py +320 -0
- package/artemis-main/artemis/mcp/action_names.py +172 -0
- package/artemis-main/artemis/mcp/action_server.py +202 -0
- package/artemis-main/artemis/mcp/action_session.py +245 -0
- package/artemis-main/artemis/mcp/action_specs.py +868 -0
- package/artemis-main/artemis/mcp/action_types.py +86 -0
- package/artemis-main/artemis/mcp/actuators/__init__.py +21 -0
- package/artemis-main/artemis/mcp/actuators/adb.py +414 -0
- package/artemis-main/artemis/mcp/actuators/base.py +113 -0
- package/artemis-main/artemis/mcp/actuators/mock.py +70 -0
- package/artemis-main/artemis/mcp/adb_server.py +490 -0
- package/artemis-main/artemis/mcp/observation.py +133 -0
- package/artemis-main/artemis/mcp/xml_search_server.py +297 -0
- package/artemis-main/artemis/memory/__init__.py +90 -0
- package/artemis-main/artemis/memory/chunking.py +1843 -0
- package/artemis-main/artemis/memory/context_policy.py +226 -0
- package/artemis-main/artemis/memory/step_capsule.md +73 -0
- package/artemis-main/artemis/memory/step_memory.py +281 -0
- package/artemis-main/artemis/memory/transcript.py +798 -0
- package/artemis-main/artemis/platform/__init__.py +61 -0
- package/artemis-main/artemis/platform/base.py +104 -0
- package/artemis-main/artemis/platform/darwin.py +184 -0
- package/artemis-main/artemis/platform/linux.py +207 -0
- package/artemis-main/artemis/platform/windows.py +193 -0
- package/artemis-main/artemis/resources/__init__.py +46 -0
- package/artemis-main/artemis/runtime/__init__.py +108 -0
- package/artemis-main/artemis/runtime/adb_endpoint.py +173 -0
- package/artemis-main/artemis/runtime/awake_lease.py +283 -0
- package/artemis-main/artemis/runtime/awake_service.py +343 -0
- package/artemis-main/artemis/runtime/cancel_requests.py +254 -0
- package/artemis-main/artemis/runtime/daemon_client.py +422 -0
- package/artemis-main/artemis/runtime/device_lock.py +1013 -0
- package/artemis-main/artemis/runtime/device_pool.py +478 -0
- package/artemis-main/artemis/runtime/helper_manager.py +1030 -0
- package/artemis-main/artemis/runtime/process_probe.py +98 -0
- package/artemis-main/artemis/runtime/server_lifecycle.py +435 -0
- package/artemis-main/artemis/runtime/supervisor.py +121 -0
- package/artemis-main/artemis/runtime/trace_store.py +375 -0
- package/artemis-main/artemis/sdk/__init__.py +26 -0
- package/artemis-main/artemis/sdk/agent.py +485 -0
- package/artemis-main/artemis/sdk/builders/__init__.py +29 -0
- package/artemis-main/artemis/sdk/builders/agent_config_builder.py +474 -0
- package/artemis-main/artemis/sdk/constants.py +15 -0
- package/artemis-main/artemis/sdk/run_outcome.py +58 -0
- package/artemis-main/artemis/sdk/types/__init__.py +61 -0
- package/artemis-main/artemis/sdk/types/agent.py +89 -0
- package/artemis-main/artemis/sdk/utils.py +28 -0
- package/artemis-main/artemis/services/llm.py +1040 -0
- package/artemis-main/artemis/services/token_meter.py +267 -0
- package/artemis-main/artemis/telemetry/__init__.py +35 -0
- package/artemis-main/artemis/telemetry/models.py +64 -0
- package/artemis-main/artemis/telemetry/replay.py +34 -0
- package/artemis-main/artemis/telemetry/storage/json_storage.py +47 -0
- package/artemis-main/artemis/telemetry/tracer.py +85 -0
- package/artemis-main/artemis/toolchain/__init__.py +52 -0
- package/artemis-main/artemis/toolchain/descriptors.py +92 -0
- package/artemis-main/artemis/toolchain/resolver.py +185 -0
- package/artemis-main/artemis/tools/base.py +232 -0
- package/artemis-main/artemis/tools/command_tool.py +1240 -0
- package/artemis-main/artemis/tools/committee_prompts.json +5 -0
- package/artemis-main/artemis/tools/committee_tool.py +422 -0
- package/artemis-main/artemis/tools/diagnoser_submit_answer_tool.py +87 -0
- package/artemis-main/artemis/tools/diagnostic_tool.py +120 -0
- package/artemis-main/artemis/tools/explorer_tool.py +581 -0
- package/artemis-main/artemis/tools/history/__init__.py +398 -0
- package/artemis-main/artemis/tools/history/replay.py +120 -0
- package/artemis-main/artemis/tools/history/screenshot.py +139 -0
- package/artemis-main/artemis/tools/history/search.py +563 -0
- package/artemis-main/artemis/tools/image_processor_tool.py +124 -0
- package/artemis-main/artemis/tools/index.py +47 -0
- package/artemis-main/artemis/tools/log_tool.py +108 -0
- package/artemis-main/artemis/tools/mobile/launch_app.py +132 -0
- package/artemis-main/artemis/tools/mobile/log_utils.py +138 -0
- package/artemis-main/artemis/tools/mobile/ocr.py +202 -0
- package/artemis-main/artemis/tools/mobile/read_hierarchy.py +108 -0
- package/artemis-main/artemis/tools/mobile/read_logs.py +116 -0
- package/artemis-main/artemis/tools/mobile/search_logs.py +260 -0
- package/artemis-main/artemis/tools/object_detection_tool.py +244 -0
- package/artemis-main/artemis/tools/scratchpad.py +819 -0
- package/artemis-main/artemis/tools/system/adb_shell.py +31 -0
- package/artemis-main/artemis/tools/tool_wrapper.py +270 -0
- package/artemis-main/artemis/tools/video_tool.py +261 -0
- package/artemis-main/artemis/tools/wait_tool.py +97 -0
- package/artemis-main/artemis/utils/app_launch_utils.py +207 -0
- package/artemis-main/artemis/utils/coordinates.py +606 -0
- package/artemis-main/artemis/utils/credentials_validator.py +201 -0
- package/artemis-main/artemis/utils/cv_canvas.py +189 -0
- package/artemis-main/artemis/utils/cython_compat.py +45 -0
- package/artemis-main/artemis/utils/element_hit_test.py +80 -0
- package/artemis-main/artemis/utils/image_diff.py +335 -0
- package/artemis-main/artemis/utils/image_hash.py +80 -0
- package/artemis-main/artemis/utils/notes.py +470 -0
- package/artemis-main/artemis/utils/ocr_api.py +105 -0
- package/artemis-main/artemis/utils/ocr_xml_fusion.py +318 -0
- package/artemis-main/artemis/utils/plan_grammar.py +513 -0
- package/artemis-main/artemis/utils/python_executor.py +213 -0
- package/artemis-main/artemis/utils/startup_progress.py +60 -0
- package/artemis-main/artemis/utils/task_tree.py +1174 -0
- package/artemis-main/artemis/utils/text.py +37 -0
- package/artemis-main/artemis/utils/ui_filter.py +651 -0
- package/artemis-main/artemis/utils/video.py +685 -0
- package/artemis-main/artemis/utils/visualization.py +802 -0
- package/artemis-main/config/artemis.example.jsonc +12 -0
- package/artemis-main/mcp_server/README.md +111 -0
- package/artemis-main/mcp_server/__init__.py +21 -0
- package/artemis-main/mcp_server/__main__.py +20 -0
- package/artemis-main/mcp_server/background/__init__.py +15 -0
- package/artemis-main/mcp_server/background/task_runner.py +453 -0
- package/artemis-main/mcp_server/base.py +49 -0
- package/artemis-main/mcp_server/notifiers/__init__.py +74 -0
- package/artemis-main/mcp_server/notifiers/agentapi.py +289 -0
- package/artemis-main/mcp_server/notifiers/base.py +45 -0
- package/artemis-main/mcp_server/notifiers/composite.py +87 -0
- package/artemis-main/mcp_server/notifiers/desktop.py +108 -0
- package/artemis-main/mcp_server/notifiers/file.py +69 -0
- package/artemis-main/mcp_server/notifiers/script.py +90 -0
- package/artemis-main/mcp_server/notifiers/webhook.py +87 -0
- package/artemis-main/mcp_server/rules.md +68 -0
- package/artemis-main/mcp_server/server.py +106 -0
- package/artemis-main/mcp_server/tools/__init__.py +29 -0
- package/artemis-main/mcp_server/tools/device_state.py +107 -0
- package/artemis-main/mcp_server/tools/diagnose.py +1249 -0
- package/artemis-main/mcp_server/tools/inspect_trace.py +394 -0
- package/artemis-main/mcp_server/tools/task_manager.py +627 -0
- package/artemis-main/mcp_server/tools/task_runner.py +548 -0
- package/artemis-main/mcp_server/utils/__init__.py +15 -0
- package/artemis-main/mcp_server/utils/device_utils.py +180 -0
- package/artemis-main/mcp_server/utils/env_utils.py +65 -0
- package/artemis-main/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk +0 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/build.gradle.kts +35 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml +49 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java +274 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ArtemisAccessibilityService.java +227 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java +463 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/DisplayUtils.java +111 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java +232 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/HierarchyDumper.java +832 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenReceiver.java +36 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenStore.java +38 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/XmlUtils.java +102 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/values/strings.xml +4 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/xml/accessibility_service_config.xml +10 -0
- package/artemis-main/packages/artemis-accessibility-helper/build.gradle.kts +3 -0
- package/artemis-main/packages/artemis-accessibility-helper/build_apk.sh +215 -0
- package/artemis-main/packages/artemis-accessibility-helper/debug.keystore +0 -0
- package/artemis-main/packages/artemis-accessibility-helper/gradle.properties +2 -0
- package/artemis-main/packages/artemis-accessibility-helper/helper_manifest.json +7 -0
- package/artemis-main/packages/artemis-accessibility-helper/settings.gradle.kts +17 -0
- package/artemis-main/packages/artemis-client/DESIGN.md +89 -0
- package/artemis-main/packages/artemis-client/README.md +112 -0
- package/artemis-main/packages/artemis-client/examples/basic.py +30 -0
- package/artemis-main/packages/artemis-client/pyproject.toml +36 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/__init__.py +45 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/client.py +401 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/errors.py +66 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/models.py +212 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/py.typed +1 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/transport.py +134 -0
- package/artemis-main/packages/artemis-client/tests/test_client.py +289 -0
- package/artemis-main/packages/artemis-client/tests/test_transport.py +85 -0
- package/artemis-main/pyproject.toml +259 -0
- package/artemis-main/pyright-core.json +17 -0
- package/artemis-main/scripts/check_dependency_sources.py +68 -0
- package/artemis-main/scripts/install_deps.ps1 +520 -0
- package/artemis-main/scripts/install_deps.sh +533 -0
- package/artemis-main/scripts/quality_ratchet.py +97 -0
- package/artemis-main/scripts/real_device_video_acceptance.py +483 -0
- package/artemis-main/scripts/run_edge_case_part.py +327 -0
- package/artemis-main/scripts/run_ui.py +242 -0
- package/artemis-main/scripts/start.ps1 +500 -0
- package/artemis-main/scripts/test_all_edge_cases.py +269 -0
- package/artemis-main/scripts/verify_concurrency_and_devices.py +385 -0
- package/artemis-main/setup.py +164 -0
- package/artemis-main/start.bat +35 -0
- package/artemis-main/start.sh +423 -0
- package/artemis-main/third_party/mobile_use/LICENSE +190 -0
- package/artemis-main/third_party/mobile_use/METADATA +27 -0
- package/artemis-main/third_party/mobile_use/NOTICE +12 -0
- package/artemis-main/third_party/mobile_use/__init__.py +19 -0
- package/artemis-main/third_party/mobile_use/agents/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/agents/hopper/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/agents/hopper/hopper.md +22 -0
- package/artemis-main/third_party/mobile_use/agents/hopper/hopper.py +80 -0
- package/artemis-main/third_party/mobile_use/clients/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/clients/ui_automator_client.py +482 -0
- package/artemis-main/third_party/mobile_use/config/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/config/llm.py +183 -0
- package/artemis-main/third_party/mobile_use/config/output.py +184 -0
- package/artemis-main/third_party/mobile_use/context.py +119 -0
- package/artemis-main/third_party/mobile_use/controllers/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/controllers/platform_specific_commands_controller.py +151 -0
- package/artemis-main/third_party/mobile_use/controllers/types.py +92 -0
- package/artemis-main/third_party/mobile_use/controllers/unified_controller.py +183 -0
- package/artemis-main/third_party/mobile_use/graph.py +54 -0
- package/artemis-main/third_party/mobile_use/llm_service.py +193 -0
- package/artemis-main/third_party/mobile_use/main.py +130 -0
- package/artemis-main/third_party/mobile_use/sdk/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/sdk/agent.py +1060 -0
- package/artemis-main/third_party/mobile_use/sdk/builders/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/sdk/builders/agent_config_builder.py +225 -0
- package/artemis-main/third_party/mobile_use/sdk/builders/index.py +33 -0
- package/artemis-main/third_party/mobile_use/sdk/builders/task_request_builder.py +243 -0
- package/artemis-main/third_party/mobile_use/sdk/types/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/sdk/types/agent.py +98 -0
- package/artemis-main/third_party/mobile_use/sdk/types/exceptions.py +109 -0
- package/artemis-main/third_party/mobile_use/sdk/types/task.py +234 -0
- package/artemis-main/third_party/mobile_use/testing.py +59 -0
- package/artemis-main/third_party/mobile_use/tools/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/tools/index.py +44 -0
- package/artemis-main/third_party/mobile_use/tools/mobile/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/tools/mobile/launch_app.py +89 -0
- package/artemis-main/third_party/mobile_use/tools/tool_wrapper.py +52 -0
- package/artemis-main/third_party/mobile_use/utils/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/utils/app_launch_utils.py +217 -0
- package/artemis-main/third_party/mobile_use/utils/cli_helpers.py +60 -0
- package/artemis-main/third_party/mobile_use/utils/decorators.py +158 -0
- package/artemis-main/third_party/mobile_use/utils/file.py +31 -0
- package/artemis-main/third_party/mobile_use/utils/logger.py +216 -0
- package/artemis-main/third_party/mobile_use/utils/media.py +213 -0
- package/artemis-main/third_party/mobile_use/utils/shell_utils.py +39 -0
- package/artemis-main/third_party/mobile_use/utils/video.py +398 -0
- package/artemis-main/uv.lock +4908 -0
- package/bin/android-midscene-automation.js +1 -5
- package/bin/artemis-config.mjs +15 -0
- package/bin/artemis-embedded.py +56 -0
- package/bin/start-artemis.mjs +30 -0
- package/bin/start-dev.mjs +112 -0
- package/bin/test_artemis_embedded.py +44 -0
- package/package.json +4 -2
- package/server/artemis.ts +34 -0
- package/server/http-api.ts +12 -0
- package/src/App.vue +10 -5
- package/src/api.ts +5 -0
- package/src/appium-recorder/AppiumPage.vue +1 -1
- package/src/appium-recorder/components/ComponentInspector.vue +1 -1
- package/src/appium-recorder/components/FlowNodeSearch.vue +3 -3
- package/src/appium-recorder/components/ImageCheckDialog.vue +3 -3
- package/src/appium-recorder/components/RecorderWorkspace.vue +4 -4
- package/src/appium-recorder/flow-appearance.ts +4 -4
- package/src/components/AppSidebar.vue +8 -7
- package/src/components/MidsceneMaintenanceNotice.vue +17 -0
- package/src/components/analytics/AnalysisCharts.vue +4 -4
- package/src/components/analytics/ReportAnalysis.vue +1 -1
- package/src/components/generator/TestCaseFileUpload.vue +1 -1
- package/src/components/help/HelpCenter.vue +56 -37
- package/src/pages/AiGeneratorPage.vue +1 -1
- package/src/pages/ArtemisPage.vue +72 -0
- package/src/pages/ConfigPage.vue +18 -19
- package/src/style.css +340 -121
- package/src/types.ts +1 -1
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"system_prompt": [
|
|
3
|
+
"# Diagnoser",
|
|
4
|
+
"",
|
|
5
|
+
"## Objective",
|
|
6
|
+
"You are an expert AI diagnoser. Your objective is to pinpoint the root cause of mobile execution failures by cross-referencing visual evidence (videos/screenshots) with system telemetry (logs). Precisely answer the `Diagnostic Query` based on the provided context.",
|
|
7
|
+
"",
|
|
8
|
+
"## Core Guidelines",
|
|
9
|
+
"- **Evidence-Based**: Base all conclusions strictly on facts from tools and context. Do not assume or hallucinate.",
|
|
10
|
+
"- **Laser Focus**: Focus exclusively on the `Diagnostic Query`. Do not investigate unrelated issues.",
|
|
11
|
+
"- **No Direct Operations**: Do not perform user actions or modify device state (e.g., tap, swipe, text input, BACK/HOME keys, or force-starting activities). Focus solely on analysis.",
|
|
12
|
+
"- **Tool Constraints**: Video analysis is expensive. When calling the video analyzer, you MUST provide a specific, narrow timeframe to minimize cost.",
|
|
13
|
+
"- **Leverage Note Tools**: Make good use of note tools to see what notes are available. When diagnosing, open and cross-reference relevant notes to ensure you have grasped all necessary information and context.",
|
|
14
|
+
"- **Loop & Monitoring Awareness**: When evaluating potential loops or repeating actions, cross-reference the task plan and notes to determine if the system is executing a legitimate `[Loop]` milestone (e.g., candidate iteration, periodic list polling with wait intervals, or continuous swipes for battery discharge). Do NOT misdiagnose planned iterations or paced polling checks as bugs or screen freezes.",
|
|
15
|
+
"- **Finalization Tool**: You MUST use the `submit_answer` tool to provide your final diagnosis."
|
|
16
|
+
],
|
|
17
|
+
"checker_feedback_template": [
|
|
18
|
+
"--- Checker Feedback ---",
|
|
19
|
+
"The Checker agent has provided feedback regarding the verification of the active subgoal:",
|
|
20
|
+
"{checker_feedback}",
|
|
21
|
+
""
|
|
22
|
+
]
|
|
23
|
+
}
|
|
@@ -0,0 +1,608 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import base64
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any
|
|
21
|
+
import uuid
|
|
22
|
+
|
|
23
|
+
from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, ToolMessage
|
|
24
|
+
|
|
25
|
+
from artemis.core.tool_failure import is_tool_failure
|
|
26
|
+
from artemis.context import ArtemisContext
|
|
27
|
+
from artemis.data_engine.trace import (
|
|
28
|
+
CURRENT_TRACE_ID,
|
|
29
|
+
TraceSpan,
|
|
30
|
+
trace,
|
|
31
|
+
trace_langchain_tool,
|
|
32
|
+
)
|
|
33
|
+
from artemis.graph.state import State
|
|
34
|
+
from artemis.llm.google import usage_from_message
|
|
35
|
+
from artemis.services.llm import acomplete, get_llm, invoke_llm_with_timeout_message
|
|
36
|
+
from artemis.tools.command_tool import get_run_short_adb_command_tool
|
|
37
|
+
from artemis.tools.diagnoser_submit_answer_tool import get_submit_answer_tool
|
|
38
|
+
from artemis.tools.history import get_history_tools
|
|
39
|
+
from artemis.tools.index import get_tool_by_name
|
|
40
|
+
from artemis.tools.log_tool import get_analyze_logs_tool
|
|
41
|
+
from artemis.tools.mobile.read_hierarchy import get_ui_hierarchy_tool
|
|
42
|
+
from artemis.tools.scratchpad import get_list_notes_tool, get_read_note_tool
|
|
43
|
+
from artemis.tools.tool_wrapper import (
|
|
44
|
+
get_tool_result_content,
|
|
45
|
+
invoke_tool_with_injection,
|
|
46
|
+
split_multimodal_result,
|
|
47
|
+
tool_result_messages,
|
|
48
|
+
)
|
|
49
|
+
from artemis.tools.video_tool import get_video_analyzer_tool
|
|
50
|
+
from artemis.tools.wait_tool import get_wait_tool
|
|
51
|
+
from artemis.memory.context_policy import build_history_for
|
|
52
|
+
from artemis.utils.task_tree import (
|
|
53
|
+
get_active_subgoal_hashes,
|
|
54
|
+
get_recent_subgoal_hashes,
|
|
55
|
+
)
|
|
56
|
+
from third_party.mobile_use.utils.logger import get_logger
|
|
57
|
+
|
|
58
|
+
logger = get_logger(__name__)
|
|
59
|
+
|
|
60
|
+
_MAX_ITERATIONS = 30
|
|
61
|
+
|
|
62
|
+
_FINAL_TURN_WARNING = (
|
|
63
|
+
"[WARNING] This is your final iteration. Call 'submit_answer' to provide your final diagnosis."
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class Diagnoser:
|
|
68
|
+
def __init__(self, ctx: ArtemisContext):
|
|
69
|
+
self.ctx = ctx
|
|
70
|
+
self.is_device_online = self._is_device_online()
|
|
71
|
+
#: The model of the current run; decides how screenshot results travel.
|
|
72
|
+
self._llm = None
|
|
73
|
+
|
|
74
|
+
def _is_device_online(self) -> bool:
|
|
75
|
+
try:
|
|
76
|
+
if self.ctx.adb_client is None:
|
|
77
|
+
return False
|
|
78
|
+
if getattr(self.ctx.adb_client, "is_online", True) is False:
|
|
79
|
+
return False
|
|
80
|
+
self.ctx.get_adb_client()
|
|
81
|
+
return True
|
|
82
|
+
except Exception:
|
|
83
|
+
return False
|
|
84
|
+
|
|
85
|
+
def _load_prompt_config(self) -> tuple[str, str]:
|
|
86
|
+
"""Loads the system prompt and checker-feedback template from disk."""
|
|
87
|
+
prompt_path = Path(__file__).parent.joinpath("diagnoser.json")
|
|
88
|
+
if not prompt_path.exists():
|
|
89
|
+
raise FileNotFoundError(f"Prompt file not found at {prompt_path}")
|
|
90
|
+
|
|
91
|
+
with open(prompt_path, encoding="utf-8") as f:
|
|
92
|
+
prompt_data = json.load(f)
|
|
93
|
+
|
|
94
|
+
system_prompt = prompt_data.get("system_prompt", "")
|
|
95
|
+
if isinstance(system_prompt, list):
|
|
96
|
+
system_prompt = "\n".join(system_prompt)
|
|
97
|
+
checker_feedback_template = prompt_data.get("checker_feedback_template", "")
|
|
98
|
+
if isinstance(checker_feedback_template, list):
|
|
99
|
+
checker_feedback_template = "\n".join(checker_feedback_template)
|
|
100
|
+
return system_prompt, checker_feedback_template
|
|
101
|
+
|
|
102
|
+
def _build_traced_tools(self) -> list:
|
|
103
|
+
"""Assembles the tool set (honoring the offline/video gates) and wraps it in tracing."""
|
|
104
|
+
all_tools = [
|
|
105
|
+
get_analyze_logs_tool(self.ctx),
|
|
106
|
+
get_read_note_tool(self.ctx),
|
|
107
|
+
get_list_notes_tool(self.ctx),
|
|
108
|
+
get_run_short_adb_command_tool(self.ctx),
|
|
109
|
+
get_wait_tool(self.ctx),
|
|
110
|
+
get_ui_hierarchy_tool(self.ctx),
|
|
111
|
+
*get_history_tools(self.ctx),
|
|
112
|
+
get_submit_answer_tool(self.ctx),
|
|
113
|
+
]
|
|
114
|
+
if not self.is_device_online:
|
|
115
|
+
logger.info(
|
|
116
|
+
"Diagnoser running in OFFLINE environment: stripping adb short"
|
|
117
|
+
" command and wait tools."
|
|
118
|
+
)
|
|
119
|
+
all_tools = [
|
|
120
|
+
t
|
|
121
|
+
for t in all_tools
|
|
122
|
+
if t.name not in ("run_adb_command", "wait")
|
|
123
|
+
and not t.name.endswith(":run_adb_command")
|
|
124
|
+
and not t.name.endswith(":wait")
|
|
125
|
+
]
|
|
126
|
+
|
|
127
|
+
if self.ctx.execution_setup and self.ctx.execution_setup.video_recording_tools_enabled:
|
|
128
|
+
all_tools.append(get_video_analyzer_tool(self.ctx, role="diagnoser"))
|
|
129
|
+
|
|
130
|
+
return [trace_langchain_tool(t, self.ctx) for t in all_tools]
|
|
131
|
+
|
|
132
|
+
def _build_plan_and_history(self) -> str:
|
|
133
|
+
"""Renders the task plan and recent step history from the DataEngine."""
|
|
134
|
+
task_plan = "No task plan yet."
|
|
135
|
+
steps = []
|
|
136
|
+
subgoal_hash = "default"
|
|
137
|
+
|
|
138
|
+
if self.ctx.data_engine:
|
|
139
|
+
notes_dir = Path(self.ctx.data_engine.base_dir) / "notes"
|
|
140
|
+
task_plan_path = notes_dir / "task_plan.md"
|
|
141
|
+
|
|
142
|
+
if task_plan_path.exists():
|
|
143
|
+
try:
|
|
144
|
+
task_plan = task_plan_path.read_text(encoding="utf-8")
|
|
145
|
+
|
|
146
|
+
subgoal_hash, _ = get_active_subgoal_hashes(task_plan)
|
|
147
|
+
except Exception as e:
|
|
148
|
+
logger.error(f"Failed to parse active subgoal in Diagnoser: {e}")
|
|
149
|
+
task_plan = f"Error reading task plan: {e}"
|
|
150
|
+
|
|
151
|
+
steps = self.ctx.data_engine.get_agent_friendly_steps()
|
|
152
|
+
|
|
153
|
+
plan_and_history = "No plan or history available."
|
|
154
|
+
if self.ctx.data_engine and task_plan != "No task plan yet.":
|
|
155
|
+
try:
|
|
156
|
+
keep_hashes = get_recent_subgoal_hashes(
|
|
157
|
+
steps,
|
|
158
|
+
subgoal_hash,
|
|
159
|
+
self.ctx.data_engine.base_dir,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
plan_and_history = build_history_for(
|
|
163
|
+
"diagnoser",
|
|
164
|
+
task_plan,
|
|
165
|
+
steps,
|
|
166
|
+
subgoal_hash,
|
|
167
|
+
keep_subgoal_hashes=keep_hashes,
|
|
168
|
+
engine=self.ctx.data_engine,
|
|
169
|
+
)
|
|
170
|
+
except Exception as e:
|
|
171
|
+
logger.error(f"Failed to build plan and history in Diagnoser: {e}")
|
|
172
|
+
plan_and_history = f"Error building plan and history: {e}"
|
|
173
|
+
return plan_and_history
|
|
174
|
+
|
|
175
|
+
def _collect_checker_feedback(self, state: State) -> str:
|
|
176
|
+
"""Renders verification findings injected by the checkpoint harvest, if any."""
|
|
177
|
+
findings = getattr(state, "operator_feedback", None)
|
|
178
|
+
if findings:
|
|
179
|
+
return "\n".join(f"- {f}" for f in findings)
|
|
180
|
+
return ""
|
|
181
|
+
|
|
182
|
+
def _read_screenshot_b64(self, state: State) -> str | None:
|
|
183
|
+
"""Reads the latest screenshot as base64, or ``None`` when unavailable."""
|
|
184
|
+
if state.latest_screenshot and os.path.exists(state.latest_screenshot):
|
|
185
|
+
try:
|
|
186
|
+
with open(state.latest_screenshot, "rb") as f:
|
|
187
|
+
return base64.b64encode(f.read()).decode("utf-8")
|
|
188
|
+
except Exception as e:
|
|
189
|
+
logger.error(f"Failed to read latest screenshot in Diagnoser: {e}")
|
|
190
|
+
return None
|
|
191
|
+
|
|
192
|
+
def _build_initial_messages(
|
|
193
|
+
self,
|
|
194
|
+
prompt: str,
|
|
195
|
+
state: State,
|
|
196
|
+
system_prompt: str,
|
|
197
|
+
checker_feedback_template: str,
|
|
198
|
+
plan_and_history: str,
|
|
199
|
+
checker_feedback: str,
|
|
200
|
+
minimal_list: str,
|
|
201
|
+
) -> list[BaseMessage]:
|
|
202
|
+
"""Build the initial prompt with history, device state and screenshot."""
|
|
203
|
+
content = []
|
|
204
|
+
content.append({"type": "text", "text": f"Initial Goal: {state.initial_goal}\n"})
|
|
205
|
+
content.append(
|
|
206
|
+
{
|
|
207
|
+
"type": "text",
|
|
208
|
+
"text": f"--- Plan & History ---\n{plan_and_history}\n",
|
|
209
|
+
}
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
if checker_feedback and checker_feedback_template:
|
|
213
|
+
formatted_feedback = checker_feedback_template.format(checker_feedback=checker_feedback)
|
|
214
|
+
content.append({"type": "text", "text": formatted_feedback})
|
|
215
|
+
|
|
216
|
+
if minimal_list:
|
|
217
|
+
content.append(
|
|
218
|
+
{
|
|
219
|
+
"type": "text",
|
|
220
|
+
"text": f"--- Visible UI Elements ---\n{minimal_list}\n",
|
|
221
|
+
}
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
latest_screenshot_b64 = self._read_screenshot_b64(state)
|
|
225
|
+
if latest_screenshot_b64:
|
|
226
|
+
content.append({"type": "text", "text": "--- Current Screenshot ---"})
|
|
227
|
+
content.append(
|
|
228
|
+
{
|
|
229
|
+
"type": "image_url",
|
|
230
|
+
"image_url": {"url": f"data:image/jpeg;base64,{latest_screenshot_b64}"},
|
|
231
|
+
}
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
content.append({"type": "text", "text": f"--- Diagnostic Query ---\n{prompt}"})
|
|
235
|
+
|
|
236
|
+
return [
|
|
237
|
+
SystemMessage(content=system_prompt),
|
|
238
|
+
HumanMessage(content=content),
|
|
239
|
+
]
|
|
240
|
+
|
|
241
|
+
async def _await_running_jobs(self) -> None:
|
|
242
|
+
"""Blocks until every still-running background job has settled."""
|
|
243
|
+
running_jobs = [
|
|
244
|
+
job for job in self.ctx.background_jobs.values() if job.get("status") == "running"
|
|
245
|
+
]
|
|
246
|
+
if running_jobs:
|
|
247
|
+
await asyncio.gather(
|
|
248
|
+
*(job["task"] for job in running_jobs if job.get("task") is not None),
|
|
249
|
+
return_exceptions=True,
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
def _drain_finished_jobs(self) -> list[ToolMessage]:
|
|
253
|
+
"""Converts finished, unconsumed background jobs into tool messages."""
|
|
254
|
+
new_tool_messages = []
|
|
255
|
+
for job in self.ctx.background_jobs.values():
|
|
256
|
+
if job.get("status") in (
|
|
257
|
+
"completed",
|
|
258
|
+
"failed",
|
|
259
|
+
) and not job.get("consumed", False):
|
|
260
|
+
tm_status = "success" if job["status"] == "completed" else "error"
|
|
261
|
+
tm = ToolMessage(
|
|
262
|
+
tool_call_id=job["tool_call_id"],
|
|
263
|
+
content=job["result"] or "",
|
|
264
|
+
status=tm_status,
|
|
265
|
+
)
|
|
266
|
+
new_tool_messages.append(tm)
|
|
267
|
+
job["consumed"] = True
|
|
268
|
+
return new_tool_messages
|
|
269
|
+
|
|
270
|
+
def _cancel_running_jobs(self) -> None:
|
|
271
|
+
"""Cancels any background job still running when the loop exits."""
|
|
272
|
+
for job in self.ctx.background_jobs.values():
|
|
273
|
+
if job.get("status") == "running" and job.get("task") is not None:
|
|
274
|
+
job["task"].cancel()
|
|
275
|
+
|
|
276
|
+
def _prepare_iteration_llm(
|
|
277
|
+
self,
|
|
278
|
+
base_llm,
|
|
279
|
+
traced_tools: list,
|
|
280
|
+
iteration: int,
|
|
281
|
+
current_messages: list[BaseMessage],
|
|
282
|
+
):
|
|
283
|
+
"""Appends the iteration notice and binds the tools available this turn.
|
|
284
|
+
|
|
285
|
+
The final iteration is restricted to ``submit_answer`` so the model
|
|
286
|
+
must conclude; earlier iterations get a progress nudge and the full
|
|
287
|
+
tool set.
|
|
288
|
+
"""
|
|
289
|
+
if iteration == _MAX_ITERATIONS - 1:
|
|
290
|
+
current_messages.append(HumanMessage(content=_FINAL_TURN_WARNING))
|
|
291
|
+
submit_tool = next(
|
|
292
|
+
(t for t in traced_tools if t.name == "submit_answer"),
|
|
293
|
+
None,
|
|
294
|
+
)
|
|
295
|
+
return base_llm.bind_tools(tools=[submit_tool] if submit_tool else [])
|
|
296
|
+
|
|
297
|
+
if iteration > 0:
|
|
298
|
+
current_messages.append(
|
|
299
|
+
HumanMessage(
|
|
300
|
+
content=(
|
|
301
|
+
"You have not completed the diagnosis yet"
|
|
302
|
+
f" (iteration {iteration + 1} of {_MAX_ITERATIONS})."
|
|
303
|
+
)
|
|
304
|
+
)
|
|
305
|
+
)
|
|
306
|
+
return base_llm.bind_tools(tools=traced_tools)
|
|
307
|
+
|
|
308
|
+
def _token_usage_payload(self, response) -> dict | None:
|
|
309
|
+
"""Extracts normalized token-usage counters from the response, if present."""
|
|
310
|
+
return usage_from_message(response)
|
|
311
|
+
|
|
312
|
+
async def _invoke_model(
|
|
313
|
+
self,
|
|
314
|
+
llm,
|
|
315
|
+
current_messages: list[BaseMessage],
|
|
316
|
+
context_components: dict,
|
|
317
|
+
):
|
|
318
|
+
"""Invokes the model inside a trace span carrying the context payload."""
|
|
319
|
+
with TraceSpan(name="gemini_diagnoser_call", ctx=self.ctx) as span:
|
|
320
|
+
span.payload = {"context_components": context_components}
|
|
321
|
+
response = await invoke_llm_with_timeout_message(acomplete(llm, current_messages))
|
|
322
|
+
span.result = response.content if hasattr(response, "content") else str(response)
|
|
323
|
+
|
|
324
|
+
token_usage = self._token_usage_payload(response)
|
|
325
|
+
if token_usage is not None:
|
|
326
|
+
if span.payload is None:
|
|
327
|
+
span.payload = {}
|
|
328
|
+
span.payload["token_usage"] = token_usage
|
|
329
|
+
return response
|
|
330
|
+
|
|
331
|
+
def _find_submit_call(self, tool_calls: list) -> dict | None:
|
|
332
|
+
"""Finds a ``submit_answer`` call among the response's tool calls."""
|
|
333
|
+
return next(
|
|
334
|
+
(
|
|
335
|
+
tc
|
|
336
|
+
for tc in tool_calls
|
|
337
|
+
if tc["name"] == "submit_answer" or tc["name"].endswith(":submit_answer")
|
|
338
|
+
),
|
|
339
|
+
None,
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
def _handle_submit_call(self, submit_call: dict, traced_tools: list) -> str:
|
|
343
|
+
"""Records the submitted answer and renders it as the run's outcome."""
|
|
344
|
+
args = submit_call["args"]
|
|
345
|
+
analysis = args.get("analysis", "No analysis provided.")
|
|
346
|
+
steps = args.get("actionable_steps", [])
|
|
347
|
+
|
|
348
|
+
submit_tool = next(
|
|
349
|
+
(
|
|
350
|
+
t
|
|
351
|
+
for t in traced_tools
|
|
352
|
+
if t.name == "submit_answer" or t.name.endswith(":submit_answer")
|
|
353
|
+
),
|
|
354
|
+
None,
|
|
355
|
+
)
|
|
356
|
+
if submit_tool:
|
|
357
|
+
submit_tool.invoke(args)
|
|
358
|
+
|
|
359
|
+
outcome = f"Analysis: {analysis}"
|
|
360
|
+
if steps:
|
|
361
|
+
numbered = "\n".join(f"{i}. {step}" for i, step in enumerate(steps, start=1))
|
|
362
|
+
outcome += f"\nActionable steps:\n{numbered}"
|
|
363
|
+
return outcome
|
|
364
|
+
|
|
365
|
+
def _max_iteration_outcome(self, response, outcome: str) -> str:
|
|
366
|
+
"""Renders the outcome when the loop cap is hit with tool calls pending."""
|
|
367
|
+
tool_names = ", ".join(tc["name"] for tc in response.tool_calls)
|
|
368
|
+
warning = (
|
|
369
|
+
"\n\n[Warning: Diagnoser reached the maximum iteration"
|
|
370
|
+
f" limit of {_MAX_ITERATIONS} and could not execute"
|
|
371
|
+
f" tool(s): {tool_names}]"
|
|
372
|
+
)
|
|
373
|
+
if outcome == "No result":
|
|
374
|
+
return (
|
|
375
|
+
"Diagnoser reached the maximum iteration limit of"
|
|
376
|
+
f" {_MAX_ITERATIONS} while attempting to call tools:"
|
|
377
|
+
f" {tool_names}"
|
|
378
|
+
)
|
|
379
|
+
return outcome + warning
|
|
380
|
+
|
|
381
|
+
async def _run_video_job(self, job_id: str, tc: dict, traced_tools: list, state: State) -> None:
|
|
382
|
+
"""Runs a ``video_analyzer`` call in the background and records its result."""
|
|
383
|
+
try:
|
|
384
|
+
tool_to_run = next(
|
|
385
|
+
(t for t in traced_tools if t.name == "video_analyzer"),
|
|
386
|
+
None,
|
|
387
|
+
)
|
|
388
|
+
if tool_to_run:
|
|
389
|
+
args = dict(tc["args"])
|
|
390
|
+
result_obj = await invoke_tool_with_injection(
|
|
391
|
+
tool=tool_to_run,
|
|
392
|
+
args=args,
|
|
393
|
+
tool_call_id=tc["id"],
|
|
394
|
+
state=state,
|
|
395
|
+
)
|
|
396
|
+
result_content = get_tool_result_content(result_obj)
|
|
397
|
+
self.ctx.background_jobs[job_id]["result"] = result_content
|
|
398
|
+
self.ctx.background_jobs[job_id]["status"] = "completed"
|
|
399
|
+
if self.ctx.data_engine:
|
|
400
|
+
self.ctx.data_engine.unregister_background_task(
|
|
401
|
+
task_id=job_id,
|
|
402
|
+
status="completed",
|
|
403
|
+
)
|
|
404
|
+
else:
|
|
405
|
+
self.ctx.background_jobs[job_id]["result"] = "Error: video_analyzer tool not found"
|
|
406
|
+
self.ctx.background_jobs[job_id]["status"] = "failed"
|
|
407
|
+
if self.ctx.data_engine:
|
|
408
|
+
self.ctx.data_engine.unregister_background_task(task_id=job_id, status="failed")
|
|
409
|
+
except asyncio.CancelledError:
|
|
410
|
+
logger.info(f"Background job {job_id} cancelled.")
|
|
411
|
+
raise
|
|
412
|
+
except Exception as e:
|
|
413
|
+
logger.error(f"Background job {job_id} failed: {e}")
|
|
414
|
+
self.ctx.background_jobs[job_id]["result"] = str(e)
|
|
415
|
+
self.ctx.background_jobs[job_id]["status"] = "failed"
|
|
416
|
+
if self.ctx.data_engine:
|
|
417
|
+
self.ctx.data_engine.unregister_background_task(task_id=job_id, status="failed")
|
|
418
|
+
|
|
419
|
+
def _start_video_job(self, tc: dict, traced_tools: list, state: State) -> ToolMessage:
|
|
420
|
+
"""Registers a background job for ``video_analyzer`` and acknowledges it."""
|
|
421
|
+
job_id = f"job_{uuid.uuid4().hex}"
|
|
422
|
+
self.ctx.background_jobs[job_id] = {
|
|
423
|
+
"tool_call_id": tc["id"],
|
|
424
|
+
"status": "running",
|
|
425
|
+
"result": None,
|
|
426
|
+
"consumed": False,
|
|
427
|
+
"task": None,
|
|
428
|
+
}
|
|
429
|
+
if self.ctx.data_engine:
|
|
430
|
+
self.ctx.data_engine.register_background_task(
|
|
431
|
+
task_id=job_id,
|
|
432
|
+
summary=(f"Diagnoser Video Analyzer Tool Call: {tc['id']}"),
|
|
433
|
+
)
|
|
434
|
+
|
|
435
|
+
task = asyncio.create_task(self._run_video_job(job_id, tc, traced_tools, state))
|
|
436
|
+
self.ctx.background_jobs[job_id]["task"] = task
|
|
437
|
+
|
|
438
|
+
return ToolMessage(
|
|
439
|
+
tool_call_id=tc["id"],
|
|
440
|
+
content=(
|
|
441
|
+
f"Video analyzer started. Job ID: {job_id}."
|
|
442
|
+
" [Warning] No background logs, status"
|
|
443
|
+
" updates, or errors will be logged."
|
|
444
|
+
),
|
|
445
|
+
status="success",
|
|
446
|
+
)
|
|
447
|
+
|
|
448
|
+
async def _run_tool(self, tc: dict, traced_tools: list, state: State) -> list[BaseMessage]:
|
|
449
|
+
"""Executes one requested tool call and wraps its result as messages.
|
|
450
|
+
|
|
451
|
+
Text results are one tool message; a step screenshot travels in the
|
|
452
|
+
carrier the model's provider accepts (``tool_result_messages``), so
|
|
453
|
+
base64 never lands in a text field.
|
|
454
|
+
"""
|
|
455
|
+
tool_name = tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]
|
|
456
|
+
logger.info(f"Diagnoser requested tool: {tool_name}")
|
|
457
|
+
|
|
458
|
+
result = ""
|
|
459
|
+
status = "success"
|
|
460
|
+
try:
|
|
461
|
+
if tool_name == "video_analyzer":
|
|
462
|
+
return [self._start_video_job(tc, traced_tools, state)]
|
|
463
|
+
|
|
464
|
+
if ":" in tool_name:
|
|
465
|
+
tool_to_run = get_tool_by_name(tool_name, traced_tools)
|
|
466
|
+
else:
|
|
467
|
+
tool_to_run = next(
|
|
468
|
+
(t for t in traced_tools if t.name == tool_name),
|
|
469
|
+
None,
|
|
470
|
+
)
|
|
471
|
+
if tool_to_run:
|
|
472
|
+
args = dict(tc["args"])
|
|
473
|
+
with TraceSpan(name=tool_name, ctx=self.ctx) as span:
|
|
474
|
+
span.payload = {"args": args}
|
|
475
|
+
try:
|
|
476
|
+
result_obj = await invoke_tool_with_injection(
|
|
477
|
+
tool=tool_to_run,
|
|
478
|
+
args=args,
|
|
479
|
+
tool_call_id=tc["id"],
|
|
480
|
+
state=state,
|
|
481
|
+
)
|
|
482
|
+
result = get_tool_result_content(result_obj)
|
|
483
|
+
result_str, _ = split_multimodal_result(result)
|
|
484
|
+
span.result = result
|
|
485
|
+
if is_tool_failure(result_obj):
|
|
486
|
+
status = "error"
|
|
487
|
+
span.status = "failed"
|
|
488
|
+
span.error = result
|
|
489
|
+
except Exception as e:
|
|
490
|
+
status = "error"
|
|
491
|
+
span.status = "failed"
|
|
492
|
+
span.error = str(e)
|
|
493
|
+
raise e
|
|
494
|
+
else:
|
|
495
|
+
result = f"Error: Tool {tool_name} not supported"
|
|
496
|
+
status = "error"
|
|
497
|
+
except Exception as e:
|
|
498
|
+
logger.error(f"Error running tool {tool_name}: {e}")
|
|
499
|
+
result = f"Error running tool {tool_name}: {e}"
|
|
500
|
+
status = "error"
|
|
501
|
+
|
|
502
|
+
return tool_result_messages(tc["id"], result, name=tool_name, status=status, llm=self._llm)
|
|
503
|
+
|
|
504
|
+
async def _execute_tool_calls(
|
|
505
|
+
self, tool_calls: list, traced_tools: list, state: State
|
|
506
|
+
) -> list[BaseMessage]:
|
|
507
|
+
"""Executes all requested tools in parallel, skipping ``google_search``."""
|
|
508
|
+
active_tool_calls = [
|
|
509
|
+
tc
|
|
510
|
+
for tc in tool_calls
|
|
511
|
+
if (tc["name"].split(":")[-1] if ":" in tc["name"] else tc["name"]) != "google_search"
|
|
512
|
+
]
|
|
513
|
+
batches = await asyncio.gather(
|
|
514
|
+
*(self._run_tool(tc, traced_tools, state) for tc in active_tool_calls)
|
|
515
|
+
)
|
|
516
|
+
return [message for batch in batches for message in batch]
|
|
517
|
+
|
|
518
|
+
@trace(type="agent", name="diagnoser")
|
|
519
|
+
async def run(self, prompt: str, state: State) -> str:
|
|
520
|
+
"""Runs the Diagnoser and returns its summary."""
|
|
521
|
+
|
|
522
|
+
system_prompt, checker_feedback_template = self._load_prompt_config()
|
|
523
|
+
|
|
524
|
+
base_llm = get_llm(ctx=self.ctx, name="diagnoser")
|
|
525
|
+
self._llm = base_llm
|
|
526
|
+
traced_tools = self._build_traced_tools()
|
|
527
|
+
|
|
528
|
+
plan_and_history = self._build_plan_and_history()
|
|
529
|
+
checker_feedback = self._collect_checker_feedback(state)
|
|
530
|
+
minimal_list = (
|
|
531
|
+
self._format_minimal_list(state.latest_ui_hierarchy)
|
|
532
|
+
if state.latest_ui_hierarchy
|
|
533
|
+
else ""
|
|
534
|
+
)
|
|
535
|
+
|
|
536
|
+
current_messages = self._build_initial_messages(
|
|
537
|
+
prompt,
|
|
538
|
+
state,
|
|
539
|
+
system_prompt,
|
|
540
|
+
checker_feedback_template,
|
|
541
|
+
plan_and_history,
|
|
542
|
+
checker_feedback,
|
|
543
|
+
minimal_list,
|
|
544
|
+
)
|
|
545
|
+
context_components = {
|
|
546
|
+
"system_prompt": system_prompt,
|
|
547
|
+
"plan_and_history": plan_and_history,
|
|
548
|
+
"checker_feedback": checker_feedback,
|
|
549
|
+
"visible_ui_elements": minimal_list,
|
|
550
|
+
"diagnostic_query": prompt,
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
self.ctx.background_jobs = {}
|
|
554
|
+
outcome = "No result"
|
|
555
|
+
|
|
556
|
+
try:
|
|
557
|
+
for i in range(_MAX_ITERATIONS):
|
|
558
|
+
# Include background results before asking for the final answer.
|
|
559
|
+
if i == _MAX_ITERATIONS - 1:
|
|
560
|
+
await self._await_running_jobs()
|
|
561
|
+
|
|
562
|
+
new_tool_messages = self._drain_finished_jobs()
|
|
563
|
+
if new_tool_messages:
|
|
564
|
+
current_messages.extend(new_tool_messages)
|
|
565
|
+
|
|
566
|
+
llm = self._prepare_iteration_llm(base_llm, traced_tools, i, current_messages)
|
|
567
|
+
|
|
568
|
+
response = await self._invoke_model(llm, current_messages, context_components)
|
|
569
|
+
|
|
570
|
+
if response.content:
|
|
571
|
+
outcome = response.content
|
|
572
|
+
|
|
573
|
+
if not response.tool_calls:
|
|
574
|
+
break
|
|
575
|
+
|
|
576
|
+
submit_call = self._find_submit_call(response.tool_calls)
|
|
577
|
+
if submit_call:
|
|
578
|
+
outcome = self._handle_submit_call(submit_call, traced_tools)
|
|
579
|
+
break
|
|
580
|
+
|
|
581
|
+
if i == _MAX_ITERATIONS - 1:
|
|
582
|
+
outcome = self._max_iteration_outcome(response, outcome)
|
|
583
|
+
break
|
|
584
|
+
|
|
585
|
+
current_messages.append(response)
|
|
586
|
+
current_messages.extend(
|
|
587
|
+
await self._execute_tool_calls(response.tool_calls, traced_tools, state)
|
|
588
|
+
)
|
|
589
|
+
finally:
|
|
590
|
+
self._cancel_running_jobs()
|
|
591
|
+
|
|
592
|
+
return outcome
|
|
593
|
+
|
|
594
|
+
def _format_minimal_list(self, fused_xml: list[dict[str, Any]]) -> str:
|
|
595
|
+
"""Formats fused XML into a minimal structured list."""
|
|
596
|
+
lines = []
|
|
597
|
+
for i, node in enumerate(fused_xml):
|
|
598
|
+
text = node.get("text") or node.get("content-desc") or ""
|
|
599
|
+
bounds = node.get("bounds")
|
|
600
|
+
|
|
601
|
+
ocr_elements = node.get("ocr_elements")
|
|
602
|
+
if ocr_elements:
|
|
603
|
+
for ocr in ocr_elements:
|
|
604
|
+
lines.append(f"[{i}] OCR Text: '{ocr['text']}' | Bounds: {ocr['bounds']}")
|
|
605
|
+
elif text.strip():
|
|
606
|
+
lines.append(f"[{i}] Text: '{text.strip()}' | Bounds: {bounds}")
|
|
607
|
+
|
|
608
|
+
return "\n".join(lines)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""The ``ask_explorer`` tool contract as seen by calling agents.
|
|
16
|
+
|
|
17
|
+
The contract is deliberately tier-agnostic: the Operator, Validator and Flash
|
|
18
|
+
runner receive the same tool name, description and argument schema whichever
|
|
19
|
+
Explorer tier the user configured. Tier behavior lives in
|
|
20
|
+
:mod:`artemis.agents.explorer.tiers`.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
ASK_EXPLORER_TOOL_NAME = "ask_explorer"
|
|
24
|
+
|
|
25
|
+
#: Model used when the LLM configuration carries no Explorer entry.
|
|
26
|
+
DEFAULT_EXPLORER_MODEL = "gemini-3.8-flash"
|
|
27
|
+
|
|
28
|
+
ASK_EXPLORER_DESCRIPTION = (
|
|
29
|
+
"[EXPLORER] Ask the UI Explorer to locate elements on the current screen"
|
|
30
|
+
" that are missing from the indexed element list or whose listed"
|
|
31
|
+
" coordinates look wrong. Every element it finds is appended to the"
|
|
32
|
+
" indexed list with a new index and returned with its normalized [x, y]"
|
|
33
|
+
" coordinate, so you can act on it right away."
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
ASK_EXPLORER_QUERY_DESCRIPTION = (
|
|
37
|
+
"What to find, described the way you see it on the screenshot: visible"
|
|
38
|
+
" text, icon shape, color, position, or nearby landmarks (e.g. 'gear icon"
|
|
39
|
+
" top-right', 'blue Send button below the message box'). Several targets"
|
|
40
|
+
" may be listed separated by ' | '."
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
ASK_EXPLORER_CONTEXT_FEEDBACK_DESCRIPTION = (
|
|
44
|
+
"Optional. What went wrong with earlier attempts or extra hints, e.g."
|
|
45
|
+
" which returned candidates were wrong and why."
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
#: Fed to the Explorer's own system prompt for loop engines.
|
|
49
|
+
EXECUTION_CONSTRAINT_TEMPLATE = (
|
|
50
|
+
"You are running with a maximum of {max_turns} turns: on your final turn"
|
|
51
|
+
" you will receive a warning and only `submit_answer` will be available."
|
|
52
|
+
)
|