android-midscene-automation 0.2.4 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +159 -29
- package/artemis-main/.dockerignore +15 -0
- package/artemis-main/.env.example +34 -0
- package/artemis-main/CONTRIBUTING.md +35 -0
- package/artemis-main/Dockerfile +68 -0
- package/artemis-main/LICENSE +202 -0
- package/artemis-main/Makefile +115 -0
- package/artemis-main/README.md +322 -0
- package/artemis-main/README_CN.md +320 -0
- package/artemis-main/apps/__init__.py +15 -0
- package/artemis-main/apps/admin_console/__init__.py +19 -0
- package/artemis-main/apps/admin_console/core/__init__.py +13 -0
- package/artemis-main/apps/admin_console/core/config.py +43 -0
- package/artemis-main/apps/admin_console/core/security.py +178 -0
- package/artemis-main/apps/admin_console/core/state.py +243 -0
- package/artemis-main/apps/admin_console/database/__init__.py +13 -0
- package/artemis-main/apps/admin_console/database/connection.py +60 -0
- package/artemis-main/apps/admin_console/database/repositories/__init__.py +13 -0
- package/artemis-main/apps/admin_console/database/repositories/session_repository.py +564 -0
- package/artemis-main/apps/admin_console/database/repositories/step_repository.py +512 -0
- package/artemis-main/apps/admin_console/database/repositories/trace_repository.py +133 -0
- package/artemis-main/apps/admin_console/index.html +3787 -0
- package/artemis-main/apps/admin_console/replay_manager.py +2585 -0
- package/artemis-main/apps/admin_console/routers/__init__.py +13 -0
- package/artemis-main/apps/admin_console/routers/media.py +198 -0
- package/artemis-main/apps/admin_console/routers/replay.py +91 -0
- package/artemis-main/apps/admin_console/routers/sessions.py +239 -0
- package/artemis-main/apps/admin_console/routers/steps.py +109 -0
- package/artemis-main/apps/admin_console/routers/stream.py +56 -0
- package/artemis-main/apps/admin_console/routers/system.py +761 -0
- package/artemis-main/apps/admin_console/routers/tasks.py +517 -0
- package/artemis-main/apps/admin_console/schemas/__init__.py +13 -0
- package/artemis-main/apps/admin_console/schemas/task_schema.py +47 -0
- package/artemis-main/apps/admin_console/server.py +508 -0
- package/artemis-main/apps/admin_console/services/__init__.py +13 -0
- package/artemis-main/apps/admin_console/services/device_stream_service.py +137 -0
- package/artemis-main/apps/admin_console/services/ipc_service.py +262 -0
- package/artemis-main/apps/admin_console/services/media_service.py +623 -0
- package/artemis-main/apps/admin_console/services/model_service.py +148 -0
- package/artemis-main/apps/admin_console/services/task_preset_catalog.py +514 -0
- package/artemis-main/apps/admin_console/services/task_queue_service.py +1719 -0
- package/artemis-main/apps/admin_console/services/worker_process_io.py +144 -0
- package/artemis-main/apps/showcase_ui/README.md +31 -0
- package/artemis-main/apps/showcase_ui/angular.json +132 -0
- package/artemis-main/apps/showcase_ui/package-lock.json +9472 -0
- package/artemis-main/apps/showcase_ui/package.json +39 -0
- package/artemis-main/apps/showcase_ui/proxy.conf.json +23 -0
- package/artemis-main/apps/showcase_ui/public/favicon.ico +0 -0
- package/artemis-main/apps/showcase_ui/public/logo.png +0 -0
- package/artemis-main/apps/showcase_ui/public/logo.svg +56 -0
- package/artemis-main/apps/showcase_ui/src/app/app.component.html +24 -0
- package/artemis-main/apps/showcase_ui/src/app/app.component.scss +49 -0
- package/artemis-main/apps/showcase_ui/src/app/app.component.spec.ts +42 -0
- package/artemis-main/apps/showcase_ui/src/app/app.component.ts +32 -0
- package/artemis-main/apps/showcase_ui/src/app/app.config.ts +29 -0
- package/artemis-main/apps/showcase_ui/src/app/app.routes.ts +27 -0
- package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream-startup.spec.ts +108 -0
- package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html +1436 -0
- package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss +3973 -0
- package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts +2235 -0
- package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.html +380 -0
- package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.scss +1171 -0
- package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.ts +270 -0
- package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html +433 -0
- package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss +987 -0
- package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts +632 -0
- package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html +130 -0
- package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.scss +406 -0
- package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.ts +74 -0
- package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.scss +115 -0
- package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.ts +52 -0
- package/artemis-main/apps/showcase_ui/src/app/core/data/smart-tasks.data.ts +334 -0
- package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.spec.ts +55 -0
- package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.ts +61 -0
- package/artemis-main/apps/showcase_ui/src/app/core/i18n/translate.pipe.ts +11 -0
- package/artemis-main/apps/showcase_ui/src/app/core/i18n/zh-cn.ts +651 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/markdown.model.ts +43 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.spec.ts +47 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.ts +165 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/session.model.ts +85 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/stream.model.ts +246 -0
- package/artemis-main/apps/showcase_ui/src/app/core/models/system.model.ts +118 -0
- package/artemis-main/apps/showcase_ui/src/app/core/services/task-recommendation.service.ts +111 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.html +2130 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.scss +5204 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.ts +1189 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.html +39 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.scss +151 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.ts +58 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.html +201 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.scss +886 -0
- package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.ts +324 -0
- package/artemis-main/apps/showcase_ui/src/app/services/agent.service.spec.ts +418 -0
- package/artemis-main/apps/showcase_ui/src/app/services/agent.service.ts +2012 -0
- package/artemis-main/apps/showcase_ui/src/app/services/system.service.spec.ts +173 -0
- package/artemis-main/apps/showcase_ui/src/app/services/system.service.ts +620 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts +513 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.ts +1015 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.spec.ts +85 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.ts +662 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.spec.ts +70 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.ts +383 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.spec.ts +107 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.ts +123 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.spec.ts +92 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.ts +116 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts +640 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts +1021 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts +375 -0
- package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.ts +1419 -0
- package/artemis-main/apps/showcase_ui/src/index.html +37 -0
- package/artemis-main/apps/showcase_ui/src/main.ts +22 -0
- package/artemis-main/apps/showcase_ui/src/styles.scss +60 -0
- package/artemis-main/apps/showcase_ui/tsconfig.app.json +23 -0
- package/artemis-main/apps/showcase_ui/tsconfig.json +27 -0
- package/artemis-main/apps/showcase_ui/tsconfig.spec.json +23 -0
- package/artemis-main/artemis/__init__.py +44 -0
- package/artemis-main/artemis/__main__.py +18 -0
- package/artemis-main/artemis/_legacy_imports.py +94 -0
- package/artemis-main/artemis/_version.py +30 -0
- package/artemis-main/artemis/agents/__init__.py +17 -0
- package/artemis-main/artemis/agents/checker/checker.json +9 -0
- package/artemis-main/artemis/agents/checker/checker.py +859 -0
- package/artemis-main/artemis/agents/diagnoser/diagnoser.json +23 -0
- package/artemis-main/artemis/agents/diagnoser/diagnoser.py +608 -0
- package/artemis-main/artemis/agents/explorer/constants.py +52 -0
- package/artemis-main/artemis/agents/explorer/explorer.json +57 -0
- package/artemis-main/artemis/agents/explorer/explorer.py +473 -0
- package/artemis-main/artemis/agents/explorer/geometry.py +73 -0
- package/artemis-main/artemis/agents/explorer/native_runner.py +878 -0
- package/artemis-main/artemis/agents/explorer/perception_tools.py +632 -0
- package/artemis-main/artemis/agents/explorer/run_setup.py +403 -0
- package/artemis-main/artemis/agents/explorer/screen_index.py +266 -0
- package/artemis-main/artemis/agents/explorer/tiers.py +110 -0
- package/artemis-main/artemis/agents/explorer/tool_declarations.py +394 -0
- package/artemis-main/artemis/agents/explorer/universal_runner.py +317 -0
- package/artemis-main/artemis/agents/flash/context_compressor.py +452 -0
- package/artemis-main/artemis/agents/flash/flash_runner.md +51 -0
- package/artemis-main/artemis/agents/flash/flash_summarizer.md +53 -0
- package/artemis-main/artemis/agents/flash/flash_summarizer_single.md +54 -0
- package/artemis-main/artemis/agents/flash/runner.py +1167 -0
- package/artemis-main/artemis/agents/flash/summarizer.py +511 -0
- package/artemis-main/artemis/agents/history_analyzer/history_analyzer.md +8 -0
- package/artemis-main/artemis/agents/history_analyzer/history_analyzer.py +182 -0
- package/artemis-main/artemis/agents/image_processor/image_processor.md +74 -0
- package/artemis-main/artemis/agents/image_processor/image_processor.py +322 -0
- package/artemis-main/artemis/agents/log_analyzer/log_analyzer.md +15 -0
- package/artemis-main/artemis/agents/log_analyzer/log_analyzer.py +403 -0
- package/artemis-main/artemis/agents/log_analyzer/log_reader_sub_agent.md +8 -0
- package/artemis-main/artemis/agents/log_analyzer/output_analyzer.md +11 -0
- package/artemis-main/artemis/agents/log_analyzer/output_analyzer.py +163 -0
- package/artemis-main/artemis/agents/object_detector/object_detector.json +9 -0
- package/artemis-main/artemis/agents/object_detector/object_detector.py +211 -0
- package/artemis-main/artemis/agents/operator/operator.json +3 -0
- package/artemis-main/artemis/agents/operator/operator.py +1394 -0
- package/artemis-main/artemis/agents/operator/prompts.py +1064 -0
- package/artemis-main/artemis/agents/outputter/outputter.md +36 -0
- package/artemis-main/artemis/agents/outputter/outputter.py +552 -0
- package/artemis-main/artemis/agents/planner/planner.json +32 -0
- package/artemis-main/artemis/agents/planner/planner.py +436 -0
- package/artemis-main/artemis/agents/prompt_assembly.py +79 -0
- package/artemis-main/artemis/agents/summarizer/summarizer.py +190 -0
- package/artemis-main/artemis/agents/validator/action_execution.py +98 -0
- package/artemis-main/artemis/agents/validator/categories.py +33 -0
- package/artemis-main/artemis/agents/validator/execution_loop.py +488 -0
- package/artemis-main/artemis/agents/validator/incidents.py +120 -0
- package/artemis-main/artemis/agents/validator/pixel_safety_net.md +15 -0
- package/artemis-main/artemis/agents/validator/precondition_pixel.py +320 -0
- package/artemis-main/artemis/agents/validator/precondition_xml.py +672 -0
- package/artemis-main/artemis/agents/validator/tool_declarations.py +332 -0
- package/artemis-main/artemis/agents/validator/validator.py +188 -0
- package/artemis-main/artemis/agents/video_analyzer/__init__.py +15 -0
- package/artemis-main/artemis/agents/video_analyzer/audio_native.py +543 -0
- package/artemis-main/artemis/agents/video_analyzer/blackboard.py +821 -0
- package/artemis-main/artemis/agents/video_analyzer/chunk_agentic.py +455 -0
- package/artemis-main/artemis/agents/video_analyzer/chunk_conversation.py +491 -0
- package/artemis-main/artemis/agents/video_analyzer/chunk_native.py +437 -0
- package/artemis-main/artemis/agents/video_analyzer/conflict_resolution.py +137 -0
- package/artemis-main/artemis/agents/video_analyzer/gemini_files.py +162 -0
- package/artemis-main/artemis/agents/video_analyzer/native_coordinator.py +541 -0
- package/artemis-main/artemis/agents/video_analyzer/reliability.py +128 -0
- package/artemis-main/artemis/agents/video_analyzer/universal_engine.py +487 -0
- package/artemis-main/artemis/agents/video_analyzer/universal_tools.py +323 -0
- package/artemis-main/artemis/agents/video_analyzer/video_analyzer.md +51 -0
- package/artemis-main/artemis/agents/video_analyzer/video_analyzer.py +875 -0
- package/artemis-main/artemis/agents/video_analyzer/video_sub_agent.md +15 -0
- package/artemis-main/artemis/agents/video_analyzer/video_sub_agent_audio.md +12 -0
- package/artemis-main/artemis/clients/accessibility_client.py +438 -0
- package/artemis-main/artemis/clients/screen_client_factory.py +403 -0
- package/artemis-main/artemis/config/__init__.py +309 -0
- package/artemis-main/artemis/config/agent.py +1098 -0
- package/artemis-main/artemis/config/constants.py +151 -0
- package/artemis-main/artemis/config/core/__init__.py +24 -0
- package/artemis-main/artemis/config/core/engine.py +118 -0
- package/artemis-main/artemis/config/core/interpolator.py +78 -0
- package/artemis-main/artemis/config/llm.py +267 -0
- package/artemis-main/artemis/config/paths.py +244 -0
- package/artemis-main/artemis/config/runtime.py +197 -0
- package/artemis-main/artemis/config/settings.py +315 -0
- package/artemis-main/artemis/constants.py +47 -0
- package/artemis-main/artemis/context.py +289 -0
- package/artemis-main/artemis/controllers/__init__.py +41 -0
- package/artemis-main/artemis/controllers/controller_factory.py +41 -0
- package/artemis-main/artemis/controllers/device_controller.py +29 -0
- package/artemis-main/artemis/controllers/types.py +37 -0
- package/artemis-main/artemis/controllers/unified_controller.py +649 -0
- package/artemis-main/artemis/core/__init__.py +21 -0
- package/artemis-main/artemis/core/diagnostics/__init__.py +90 -0
- package/artemis-main/artemis/core/diagnostics/adb_keys.py +270 -0
- package/artemis-main/artemis/core/diagnostics/adb_server_connection.py +330 -0
- package/artemis-main/artemis/core/diagnostics/device_smoke.py +364 -0
- package/artemis-main/artemis/core/diagnostics/emulator_manager.py +501 -0
- package/artemis-main/artemis/core/diagnostics/engine.py +438 -0
- package/artemis-main/artemis/core/diagnostics/hierarchy_parity.py +216 -0
- package/artemis-main/artemis/core/diagnostics/probes/adb_probe.py +887 -0
- package/artemis-main/artemis/core/diagnostics/probes/base.py +45 -0
- package/artemis-main/artemis/core/diagnostics/probes/credentials_probe.py +296 -0
- package/artemis-main/artemis/core/diagnostics/probes/host_probe.py +427 -0
- package/artemis-main/artemis/core/diagnostics/probes/runtime_probe.py +169 -0
- package/artemis-main/artemis/core/diagnostics/probes/toolchain_probe.py +139 -0
- package/artemis-main/artemis/core/diagnostics/readiness.py +83 -0
- package/artemis-main/artemis/core/diagnostics/schema.py +118 -0
- package/artemis-main/artemis/core/tool_declaration.py +43 -0
- package/artemis-main/artemis/core/tool_failure.py +56 -0
- package/artemis-main/artemis/data_engine/context_vars.py +24 -0
- package/artemis-main/artemis/data_engine/engine.py +1615 -0
- package/artemis-main/artemis/data_engine/history_reader.py +142 -0
- package/artemis-main/artemis/data_engine/models.py +131 -0
- package/artemis-main/artemis/data_engine/storage.py +1245 -0
- package/artemis-main/artemis/data_engine/trace.py +727 -0
- package/artemis-main/artemis/drivers/__init__.py +48 -0
- package/artemis-main/artemis/drivers/android/adb_driver.py +507 -0
- package/artemis-main/artemis/drivers/android/hierarchy.py +64 -0
- package/artemis-main/artemis/drivers/android/input_ime.py +120 -0
- package/artemis-main/artemis/drivers/android/recorder.py +57 -0
- package/artemis-main/artemis/drivers/base.py +290 -0
- package/artemis-main/artemis/drivers/cloud/cloud_driver.py +117 -0
- package/artemis-main/artemis/drivers/factory.py +80 -0
- package/artemis-main/artemis/drivers/mock/mock_driver.py +198 -0
- package/artemis-main/artemis/drivers/types.py +67 -0
- package/artemis-main/artemis/graph/checkpoints.py +1243 -0
- package/artemis-main/artemis/graph/graph.py +1055 -0
- package/artemis-main/artemis/graph/perception.py +234 -0
- package/artemis-main/artemis/graph/state.py +151 -0
- package/artemis-main/artemis/graph/visibility.py +238 -0
- package/artemis-main/artemis/interfaces/cli/__init__.py +19 -0
- package/artemis-main/artemis/interfaces/cli/commands/__init__.py +15 -0
- package/artemis-main/artemis/interfaces/cli/commands/batch.py +278 -0
- package/artemis-main/artemis/interfaces/cli/commands/doctor.py +468 -0
- package/artemis-main/artemis/interfaces/cli/commands/helper.py +226 -0
- package/artemis-main/artemis/interfaces/cli/commands/init.py +193 -0
- package/artemis-main/artemis/interfaces/cli/commands/mcp.py +862 -0
- package/artemis-main/artemis/interfaces/cli/commands/run.py +463 -0
- package/artemis-main/artemis/interfaces/cli/commands/server.py +151 -0
- package/artemis-main/artemis/interfaces/cli/commands/server_lifecycle.py +328 -0
- package/artemis-main/artemis/interfaces/cli/commands/trace.py +111 -0
- package/artemis-main/artemis/interfaces/cli/commands/ui.py +404 -0
- package/artemis-main/artemis/interfaces/cli/main.py +109 -0
- package/artemis-main/artemis/interfaces/sdk/__init__.py +31 -0
- package/artemis-main/artemis/interfaces/sdk/client.py +19 -0
- package/artemis-main/artemis/interfaces/sdk/task.py +72 -0
- package/artemis-main/artemis/llm/__init__.py +25 -0
- package/artemis-main/artemis/llm/fake_model.py +179 -0
- package/artemis-main/artemis/llm/google/__init__.py +57 -0
- package/artemis-main/artemis/llm/google/provider.py +185 -0
- package/artemis-main/artemis/llm/google/usage.py +148 -0
- package/artemis-main/artemis/llm/parser.py +32 -0
- package/artemis-main/artemis/llm/reliability.py +267 -0
- package/artemis-main/artemis/llm/router.py +389 -0
- package/artemis-main/artemis/llm/schemas.py +36 -0
- package/artemis-main/artemis/llm/structured.py +196 -0
- package/artemis-main/artemis/main.py +58 -0
- package/artemis-main/artemis/mcp/__init__.py +20 -0
- package/artemis-main/artemis/mcp/action_executor.py +760 -0
- package/artemis-main/artemis/mcp/action_manifest.py +320 -0
- package/artemis-main/artemis/mcp/action_names.py +172 -0
- package/artemis-main/artemis/mcp/action_server.py +202 -0
- package/artemis-main/artemis/mcp/action_session.py +245 -0
- package/artemis-main/artemis/mcp/action_specs.py +868 -0
- package/artemis-main/artemis/mcp/action_types.py +86 -0
- package/artemis-main/artemis/mcp/actuators/__init__.py +21 -0
- package/artemis-main/artemis/mcp/actuators/adb.py +414 -0
- package/artemis-main/artemis/mcp/actuators/base.py +113 -0
- package/artemis-main/artemis/mcp/actuators/mock.py +70 -0
- package/artemis-main/artemis/mcp/adb_server.py +490 -0
- package/artemis-main/artemis/mcp/observation.py +133 -0
- package/artemis-main/artemis/mcp/xml_search_server.py +297 -0
- package/artemis-main/artemis/memory/__init__.py +90 -0
- package/artemis-main/artemis/memory/chunking.py +1843 -0
- package/artemis-main/artemis/memory/context_policy.py +226 -0
- package/artemis-main/artemis/memory/step_capsule.md +73 -0
- package/artemis-main/artemis/memory/step_memory.py +281 -0
- package/artemis-main/artemis/memory/transcript.py +798 -0
- package/artemis-main/artemis/platform/__init__.py +61 -0
- package/artemis-main/artemis/platform/base.py +104 -0
- package/artemis-main/artemis/platform/darwin.py +184 -0
- package/artemis-main/artemis/platform/linux.py +207 -0
- package/artemis-main/artemis/platform/windows.py +193 -0
- package/artemis-main/artemis/resources/__init__.py +46 -0
- package/artemis-main/artemis/runtime/__init__.py +108 -0
- package/artemis-main/artemis/runtime/adb_endpoint.py +173 -0
- package/artemis-main/artemis/runtime/awake_lease.py +283 -0
- package/artemis-main/artemis/runtime/awake_service.py +343 -0
- package/artemis-main/artemis/runtime/cancel_requests.py +254 -0
- package/artemis-main/artemis/runtime/daemon_client.py +422 -0
- package/artemis-main/artemis/runtime/device_lock.py +1013 -0
- package/artemis-main/artemis/runtime/device_pool.py +478 -0
- package/artemis-main/artemis/runtime/helper_manager.py +1030 -0
- package/artemis-main/artemis/runtime/process_probe.py +98 -0
- package/artemis-main/artemis/runtime/server_lifecycle.py +435 -0
- package/artemis-main/artemis/runtime/supervisor.py +121 -0
- package/artemis-main/artemis/runtime/trace_store.py +375 -0
- package/artemis-main/artemis/sdk/__init__.py +26 -0
- package/artemis-main/artemis/sdk/agent.py +485 -0
- package/artemis-main/artemis/sdk/builders/__init__.py +29 -0
- package/artemis-main/artemis/sdk/builders/agent_config_builder.py +474 -0
- package/artemis-main/artemis/sdk/constants.py +15 -0
- package/artemis-main/artemis/sdk/run_outcome.py +58 -0
- package/artemis-main/artemis/sdk/types/__init__.py +61 -0
- package/artemis-main/artemis/sdk/types/agent.py +89 -0
- package/artemis-main/artemis/sdk/utils.py +28 -0
- package/artemis-main/artemis/services/llm.py +1040 -0
- package/artemis-main/artemis/services/token_meter.py +267 -0
- package/artemis-main/artemis/telemetry/__init__.py +35 -0
- package/artemis-main/artemis/telemetry/models.py +64 -0
- package/artemis-main/artemis/telemetry/replay.py +34 -0
- package/artemis-main/artemis/telemetry/storage/json_storage.py +47 -0
- package/artemis-main/artemis/telemetry/tracer.py +85 -0
- package/artemis-main/artemis/toolchain/__init__.py +52 -0
- package/artemis-main/artemis/toolchain/descriptors.py +92 -0
- package/artemis-main/artemis/toolchain/resolver.py +185 -0
- package/artemis-main/artemis/tools/base.py +232 -0
- package/artemis-main/artemis/tools/command_tool.py +1240 -0
- package/artemis-main/artemis/tools/committee_prompts.json +5 -0
- package/artemis-main/artemis/tools/committee_tool.py +422 -0
- package/artemis-main/artemis/tools/diagnoser_submit_answer_tool.py +87 -0
- package/artemis-main/artemis/tools/diagnostic_tool.py +120 -0
- package/artemis-main/artemis/tools/explorer_tool.py +581 -0
- package/artemis-main/artemis/tools/history/__init__.py +398 -0
- package/artemis-main/artemis/tools/history/replay.py +120 -0
- package/artemis-main/artemis/tools/history/screenshot.py +139 -0
- package/artemis-main/artemis/tools/history/search.py +563 -0
- package/artemis-main/artemis/tools/image_processor_tool.py +124 -0
- package/artemis-main/artemis/tools/index.py +47 -0
- package/artemis-main/artemis/tools/log_tool.py +108 -0
- package/artemis-main/artemis/tools/mobile/launch_app.py +132 -0
- package/artemis-main/artemis/tools/mobile/log_utils.py +138 -0
- package/artemis-main/artemis/tools/mobile/ocr.py +202 -0
- package/artemis-main/artemis/tools/mobile/read_hierarchy.py +108 -0
- package/artemis-main/artemis/tools/mobile/read_logs.py +116 -0
- package/artemis-main/artemis/tools/mobile/search_logs.py +260 -0
- package/artemis-main/artemis/tools/object_detection_tool.py +244 -0
- package/artemis-main/artemis/tools/scratchpad.py +819 -0
- package/artemis-main/artemis/tools/system/adb_shell.py +31 -0
- package/artemis-main/artemis/tools/tool_wrapper.py +270 -0
- package/artemis-main/artemis/tools/video_tool.py +261 -0
- package/artemis-main/artemis/tools/wait_tool.py +97 -0
- package/artemis-main/artemis/utils/app_launch_utils.py +207 -0
- package/artemis-main/artemis/utils/coordinates.py +606 -0
- package/artemis-main/artemis/utils/credentials_validator.py +201 -0
- package/artemis-main/artemis/utils/cv_canvas.py +189 -0
- package/artemis-main/artemis/utils/cython_compat.py +45 -0
- package/artemis-main/artemis/utils/element_hit_test.py +80 -0
- package/artemis-main/artemis/utils/image_diff.py +335 -0
- package/artemis-main/artemis/utils/image_hash.py +80 -0
- package/artemis-main/artemis/utils/notes.py +470 -0
- package/artemis-main/artemis/utils/ocr_api.py +105 -0
- package/artemis-main/artemis/utils/ocr_xml_fusion.py +318 -0
- package/artemis-main/artemis/utils/plan_grammar.py +513 -0
- package/artemis-main/artemis/utils/python_executor.py +213 -0
- package/artemis-main/artemis/utils/startup_progress.py +60 -0
- package/artemis-main/artemis/utils/task_tree.py +1174 -0
- package/artemis-main/artemis/utils/text.py +37 -0
- package/artemis-main/artemis/utils/ui_filter.py +651 -0
- package/artemis-main/artemis/utils/video.py +685 -0
- package/artemis-main/artemis/utils/visualization.py +802 -0
- package/artemis-main/config/artemis.example.jsonc +12 -0
- package/artemis-main/mcp_server/README.md +111 -0
- package/artemis-main/mcp_server/__init__.py +21 -0
- package/artemis-main/mcp_server/__main__.py +20 -0
- package/artemis-main/mcp_server/background/__init__.py +15 -0
- package/artemis-main/mcp_server/background/task_runner.py +453 -0
- package/artemis-main/mcp_server/base.py +49 -0
- package/artemis-main/mcp_server/notifiers/__init__.py +74 -0
- package/artemis-main/mcp_server/notifiers/agentapi.py +289 -0
- package/artemis-main/mcp_server/notifiers/base.py +45 -0
- package/artemis-main/mcp_server/notifiers/composite.py +87 -0
- package/artemis-main/mcp_server/notifiers/desktop.py +108 -0
- package/artemis-main/mcp_server/notifiers/file.py +69 -0
- package/artemis-main/mcp_server/notifiers/script.py +90 -0
- package/artemis-main/mcp_server/notifiers/webhook.py +87 -0
- package/artemis-main/mcp_server/rules.md +68 -0
- package/artemis-main/mcp_server/server.py +106 -0
- package/artemis-main/mcp_server/tools/__init__.py +29 -0
- package/artemis-main/mcp_server/tools/device_state.py +107 -0
- package/artemis-main/mcp_server/tools/diagnose.py +1249 -0
- package/artemis-main/mcp_server/tools/inspect_trace.py +394 -0
- package/artemis-main/mcp_server/tools/task_manager.py +627 -0
- package/artemis-main/mcp_server/tools/task_runner.py +548 -0
- package/artemis-main/mcp_server/utils/__init__.py +15 -0
- package/artemis-main/mcp_server/utils/device_utils.py +180 -0
- package/artemis-main/mcp_server/utils/env_utils.py +65 -0
- package/artemis-main/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk +0 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/build.gradle.kts +35 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml +49 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java +274 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ArtemisAccessibilityService.java +227 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java +463 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/DisplayUtils.java +111 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java +232 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/HierarchyDumper.java +832 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenReceiver.java +36 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenStore.java +38 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/XmlUtils.java +102 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/values/strings.xml +4 -0
- package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/xml/accessibility_service_config.xml +10 -0
- package/artemis-main/packages/artemis-accessibility-helper/build.gradle.kts +3 -0
- package/artemis-main/packages/artemis-accessibility-helper/build_apk.sh +215 -0
- package/artemis-main/packages/artemis-accessibility-helper/debug.keystore +0 -0
- package/artemis-main/packages/artemis-accessibility-helper/gradle.properties +2 -0
- package/artemis-main/packages/artemis-accessibility-helper/helper_manifest.json +7 -0
- package/artemis-main/packages/artemis-accessibility-helper/settings.gradle.kts +17 -0
- package/artemis-main/packages/artemis-client/DESIGN.md +89 -0
- package/artemis-main/packages/artemis-client/README.md +112 -0
- package/artemis-main/packages/artemis-client/examples/basic.py +30 -0
- package/artemis-main/packages/artemis-client/pyproject.toml +36 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/__init__.py +45 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/client.py +401 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/errors.py +66 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/models.py +212 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/py.typed +1 -0
- package/artemis-main/packages/artemis-client/src/artemis_client/transport.py +134 -0
- package/artemis-main/packages/artemis-client/tests/test_client.py +289 -0
- package/artemis-main/packages/artemis-client/tests/test_transport.py +85 -0
- package/artemis-main/pyproject.toml +259 -0
- package/artemis-main/pyright-core.json +17 -0
- package/artemis-main/scripts/check_dependency_sources.py +68 -0
- package/artemis-main/scripts/install_deps.ps1 +520 -0
- package/artemis-main/scripts/install_deps.sh +533 -0
- package/artemis-main/scripts/quality_ratchet.py +97 -0
- package/artemis-main/scripts/real_device_video_acceptance.py +483 -0
- package/artemis-main/scripts/run_edge_case_part.py +327 -0
- package/artemis-main/scripts/run_ui.py +242 -0
- package/artemis-main/scripts/start.ps1 +500 -0
- package/artemis-main/scripts/test_all_edge_cases.py +269 -0
- package/artemis-main/scripts/verify_concurrency_and_devices.py +385 -0
- package/artemis-main/setup.py +164 -0
- package/artemis-main/start.bat +35 -0
- package/artemis-main/start.sh +423 -0
- package/artemis-main/third_party/mobile_use/LICENSE +190 -0
- package/artemis-main/third_party/mobile_use/METADATA +27 -0
- package/artemis-main/third_party/mobile_use/NOTICE +12 -0
- package/artemis-main/third_party/mobile_use/__init__.py +19 -0
- package/artemis-main/third_party/mobile_use/agents/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/agents/hopper/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/agents/hopper/hopper.md +22 -0
- package/artemis-main/third_party/mobile_use/agents/hopper/hopper.py +80 -0
- package/artemis-main/third_party/mobile_use/clients/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/clients/ui_automator_client.py +482 -0
- package/artemis-main/third_party/mobile_use/config/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/config/llm.py +183 -0
- package/artemis-main/third_party/mobile_use/config/output.py +184 -0
- package/artemis-main/third_party/mobile_use/context.py +119 -0
- package/artemis-main/third_party/mobile_use/controllers/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/controllers/platform_specific_commands_controller.py +151 -0
- package/artemis-main/third_party/mobile_use/controllers/types.py +92 -0
- package/artemis-main/third_party/mobile_use/controllers/unified_controller.py +183 -0
- package/artemis-main/third_party/mobile_use/graph.py +54 -0
- package/artemis-main/third_party/mobile_use/llm_service.py +193 -0
- package/artemis-main/third_party/mobile_use/main.py +130 -0
- package/artemis-main/third_party/mobile_use/sdk/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/sdk/agent.py +1060 -0
- package/artemis-main/third_party/mobile_use/sdk/builders/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/sdk/builders/agent_config_builder.py +225 -0
- package/artemis-main/third_party/mobile_use/sdk/builders/index.py +33 -0
- package/artemis-main/third_party/mobile_use/sdk/builders/task_request_builder.py +243 -0
- package/artemis-main/third_party/mobile_use/sdk/types/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/sdk/types/agent.py +98 -0
- package/artemis-main/third_party/mobile_use/sdk/types/exceptions.py +109 -0
- package/artemis-main/third_party/mobile_use/sdk/types/task.py +234 -0
- package/artemis-main/third_party/mobile_use/testing.py +59 -0
- package/artemis-main/third_party/mobile_use/tools/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/tools/index.py +44 -0
- package/artemis-main/third_party/mobile_use/tools/mobile/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/tools/mobile/launch_app.py +89 -0
- package/artemis-main/third_party/mobile_use/tools/tool_wrapper.py +52 -0
- package/artemis-main/third_party/mobile_use/utils/__init__.py +13 -0
- package/artemis-main/third_party/mobile_use/utils/app_launch_utils.py +217 -0
- package/artemis-main/third_party/mobile_use/utils/cli_helpers.py +60 -0
- package/artemis-main/third_party/mobile_use/utils/decorators.py +158 -0
- package/artemis-main/third_party/mobile_use/utils/file.py +31 -0
- package/artemis-main/third_party/mobile_use/utils/logger.py +216 -0
- package/artemis-main/third_party/mobile_use/utils/media.py +213 -0
- package/artemis-main/third_party/mobile_use/utils/shell_utils.py +39 -0
- package/artemis-main/third_party/mobile_use/utils/video.py +398 -0
- package/artemis-main/uv.lock +4908 -0
- package/bin/android-midscene-automation.js +1 -5
- package/bin/artemis-config.mjs +15 -0
- package/bin/artemis-embedded.py +56 -0
- package/bin/start-artemis.mjs +30 -0
- package/bin/start-dev.mjs +112 -0
- package/bin/test_artemis_embedded.py +44 -0
- package/package.json +4 -2
- package/server/artemis.ts +34 -0
- package/server/http-api.ts +12 -0
- package/src/App.vue +10 -5
- package/src/api.ts +5 -0
- package/src/appium-recorder/AppiumPage.vue +1 -1
- package/src/appium-recorder/components/ComponentInspector.vue +1 -1
- package/src/appium-recorder/components/FlowNodeSearch.vue +3 -3
- package/src/appium-recorder/components/ImageCheckDialog.vue +3 -3
- package/src/appium-recorder/components/RecorderWorkspace.vue +4 -4
- package/src/appium-recorder/flow-appearance.ts +4 -4
- package/src/components/AppSidebar.vue +8 -7
- package/src/components/MidsceneMaintenanceNotice.vue +17 -0
- package/src/components/analytics/AnalysisCharts.vue +4 -4
- package/src/components/analytics/ReportAnalysis.vue +1 -1
- package/src/components/generator/TestCaseFileUpload.vue +1 -1
- package/src/components/help/HelpCenter.vue +56 -37
- package/src/pages/AiGeneratorPage.vue +1 -1
- package/src/pages/ArtemisPage.vue +72 -0
- package/src/pages/ConfigPage.vue +18 -19
- package/src/style.css +340 -121
- package/src/types.ts +1 -1
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
{
|
|
2
|
+
"planner_avatar": "You are the **Planner Avatar** (Meeting Host) in a committee resolving a task plan failure. Your role is to lead the meeting and advocate for a robust and correct high-level plan. You are aware of:\n- **History Analyzer**: The trajectory specialist who can analyze execution history and read notes. They have tools like `search_history`, `replay_steps`, `get_step_screenshot` and `read_note`.\n- **Diagnoser**: The forensics specialist who can analyze logs and videos to find why things failed. They have tools like `video_analyzer` and `analyze_logs`.\nYour master (Master Planner) has given you a specific directive for this meeting, which is at the top of the blackboard.\n**Your Task**:\n1. In the first round, you must open the meeting, state the problem based on the master's directive, and call on the other experts (Diagnoser or History Analyzer) to provide their analysis.\n2. In subsequent rounds, evaluate their suggestions and push for a concrete, revised plan of subgoals.\n3. In the final round, summarize the discussion and output the final recommended plan revisions and actions.",
|
|
3
|
+
"history_analyzer_expert": "You are the **History Analyzer** (Trajectory Specialist) in a committee resolving a task failure. Your role is to analyze the execution history of the current session, understand what has been tried, what succeeded, and what failed, and check persistent notes. You are aware of:\n- **Planner Avatar**: The meeting host who needs a concrete plan.\n- **Diagnoser**: The forensics specialist who analyzes logs and videos.\nYou have tools to search the execution history (`search_history`), replay steps in full (`replay_steps`), look at a step's stored screenshot (`get_step_screenshot`) and read notes. Wait for the Planner Avatar to call on you or respond to the current state of the blackboard. Be concise.",
|
|
4
|
+
"diagnoser_expert": "You are the **Diagnoser** (Forensics Specialist) in a committee resolving a UI automation failure. Your role is to analyze what went wrong in the past, look for error patterns in logs, and understand system state transitions. You are aware of:\n- **Planner Avatar**: The meeting host who needs a concrete plan.\n- **History Analyzer**: The trajectory specialist who analyzes history and notes.\nYou have tools to analyze video and logs. Wait for the Planner Avatar to call on you or respond to the current state of the blackboard. Be concise."
|
|
5
|
+
}
|
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
# pylint: disable=too-many-statements
|
|
2
|
+
# Copyright 2026 Google LLC
|
|
3
|
+
#
|
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
# you may not use this file except in compliance with the License.
|
|
6
|
+
# You may obtain a copy of the License at
|
|
7
|
+
#
|
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
#
|
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
# See the License for the specific language governing permissions and
|
|
14
|
+
# limitations under the License.
|
|
15
|
+
|
|
16
|
+
"""Committee tool for multi-agent deliberation and failure recovery."""
|
|
17
|
+
|
|
18
|
+
import asyncio
|
|
19
|
+
import base64
|
|
20
|
+
import json
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any
|
|
23
|
+
from uuid import uuid4
|
|
24
|
+
|
|
25
|
+
from langchain_core.messages import HumanMessage, SystemMessage
|
|
26
|
+
from langchain_core.tools import BaseTool
|
|
27
|
+
from pydantic import BaseModel, Field
|
|
28
|
+
|
|
29
|
+
from artemis.core.tool_failure import ToolFailure, is_tool_failure
|
|
30
|
+
from artemis.context import ArtemisContext
|
|
31
|
+
from artemis.data_engine.trace import (
|
|
32
|
+
CURRENT_TRACE_ID,
|
|
33
|
+
TraceSpan,
|
|
34
|
+
trace_langchain_tool,
|
|
35
|
+
)
|
|
36
|
+
from artemis.drivers.base import BaseDeviceDriver
|
|
37
|
+
from artemis.graph.state import State
|
|
38
|
+
from artemis.services.llm import get_llm, invoke_llm_with_timeout_message
|
|
39
|
+
from artemis.tools.base import ArtemisTool, ToolCategory
|
|
40
|
+
from artemis.tools.history import get_history_tools
|
|
41
|
+
from artemis.tools.log_tool import get_analyze_logs_tool
|
|
42
|
+
from artemis.tools.scratchpad import get_list_notes_tool, get_read_note_tool
|
|
43
|
+
from artemis.tools.tool_wrapper import (
|
|
44
|
+
ToolWrapper,
|
|
45
|
+
get_tool_result_content,
|
|
46
|
+
invoke_tool_with_injection,
|
|
47
|
+
tool_result_messages,
|
|
48
|
+
)
|
|
49
|
+
from artemis.utils.cython_compat import CyFunctionDetector
|
|
50
|
+
from artemis.tools.video_tool import get_video_analyzer_tool
|
|
51
|
+
from artemis.utils.notes import get_note_file_path, get_notes_dir
|
|
52
|
+
from artemis.memory.context_policy import build_history_for
|
|
53
|
+
from artemis.utils.task_tree import (
|
|
54
|
+
get_active_subgoal_hashes,
|
|
55
|
+
get_recent_subgoal_hashes,
|
|
56
|
+
)
|
|
57
|
+
from third_party.mobile_use.utils.logger import get_logger
|
|
58
|
+
|
|
59
|
+
logger = get_logger(__name__)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class AskCommitteeArgs(BaseModel):
|
|
63
|
+
"""Arguments schema for invoking the committee tool."""
|
|
64
|
+
|
|
65
|
+
model_config = {"ignored_types": (CyFunctionDetector,)}
|
|
66
|
+
avatar_directive: str = Field(
|
|
67
|
+
...,
|
|
68
|
+
description="Directive to frame and steer the debate for your avatar.",
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
ASK_COMMITTEE_DOCSTRING = (
|
|
73
|
+
"[COMMITTEE] Call this tool to summon a committee of sub-agents to debate"
|
|
74
|
+
" and resolve complex failures.\n\n"
|
|
75
|
+
"Your avatar will host the meeting and drive it based on your directive.\n"
|
|
76
|
+
"The result will be a synthesized recommendation."
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class AskCommitteeTool(ArtemisTool):
|
|
81
|
+
"""Universal tool for summoning a committee of sub-agents to debate and resolve failures."""
|
|
82
|
+
|
|
83
|
+
def __init__(self, category: ToolCategory = "custom"):
|
|
84
|
+
super().__init__(
|
|
85
|
+
name="ask_committee",
|
|
86
|
+
description=ASK_COMMITTEE_DOCSTRING,
|
|
87
|
+
args_schema=AskCommitteeArgs,
|
|
88
|
+
category=category,
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
|
92
|
+
async def execute(
|
|
93
|
+
self,
|
|
94
|
+
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
|
95
|
+
ctx: ArtemisContext | None = None,
|
|
96
|
+
avatar_directive: str = "",
|
|
97
|
+
tool_call_id: str | None = None, # pylint: disable=unused-argument
|
|
98
|
+
state: State | None = None,
|
|
99
|
+
**kwargs: Any,
|
|
100
|
+
) -> str:
|
|
101
|
+
directive = avatar_directive or kwargs.get("AvatarDirective") or ""
|
|
102
|
+
return await _run_committee_logic(
|
|
103
|
+
ctx=ctx,
|
|
104
|
+
state=state,
|
|
105
|
+
avatar_directive=directive,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# Universal tool instance & aliases
|
|
110
|
+
ask_committee = AskCommitteeTool()
|
|
111
|
+
AskCommittee = AskCommitteeTool
|
|
112
|
+
AskCommitteeToolAlias = AskCommitteeTool
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def get_ask_committee_tool(ctx: ArtemisContext) -> BaseTool:
|
|
116
|
+
"""Exports ask_committee as a LangChain BaseTool."""
|
|
117
|
+
return trace_langchain_tool(ask_committee.to_langchain_tool(ctx), ctx)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
|
|
121
|
+
async def _execute_committee(
|
|
122
|
+
ctx: ArtemisContext,
|
|
123
|
+
state: State | None,
|
|
124
|
+
avatar_directive: str,
|
|
125
|
+
) -> str:
|
|
126
|
+
"""Executes committee debate loop and returns the final conclusion."""
|
|
127
|
+
with TraceSpan(name="committee", trace_type="agent", ctx=ctx):
|
|
128
|
+
logger.info(f"Committee invoked with directive: {avatar_directive}")
|
|
129
|
+
|
|
130
|
+
# 1. Access or create blackboard file
|
|
131
|
+
if ctx.data_engine:
|
|
132
|
+
notes_dir = get_notes_dir(ctx.data_engine.base_dir)
|
|
133
|
+
else:
|
|
134
|
+
notes_dir = Path(".") / "notes"
|
|
135
|
+
notes_dir.mkdir(parents=True, exist_ok=True)
|
|
136
|
+
|
|
137
|
+
# Load latest screenshot as base64 to provide multimodal vision to the committee
|
|
138
|
+
screenshot_b64 = None
|
|
139
|
+
screenshot_path = getattr(state, "latest_screenshot", None) if state is not None else None
|
|
140
|
+
if screenshot_path and Path(screenshot_path).exists():
|
|
141
|
+
try:
|
|
142
|
+
with open(screenshot_path, "rb") as f:
|
|
143
|
+
screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
|
|
144
|
+
logger.info(f"Committee successfully loaded screenshot from: {screenshot_path}")
|
|
145
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
|
146
|
+
logger.error(f"Failed to read screenshot in committee: {e}")
|
|
147
|
+
|
|
148
|
+
trace_id = CURRENT_TRACE_ID.get() or str(uuid4())
|
|
149
|
+
blackboard_path = notes_dir / f"blackboard_{trace_id}.md"
|
|
150
|
+
|
|
151
|
+
# Initialize blackboard with goal and directive
|
|
152
|
+
initial_goal = (
|
|
153
|
+
getattr(state, "initial_goal", "Not specified.")
|
|
154
|
+
if state is not None
|
|
155
|
+
else "Not specified."
|
|
156
|
+
)
|
|
157
|
+
blackboard_content = (
|
|
158
|
+
"# Committee Discussion Blackboard\n\n"
|
|
159
|
+
f"## User's Initial Goal\n{initial_goal}\n\n"
|
|
160
|
+
f"## Master's Directive\n{avatar_directive}\n\n"
|
|
161
|
+
)
|
|
162
|
+
blackboard_path.write_text(blackboard_content, encoding="utf-8")
|
|
163
|
+
|
|
164
|
+
# 2. Extract history
|
|
165
|
+
history_summary = "## Previous Context\n"
|
|
166
|
+
if ctx.data_engine:
|
|
167
|
+
steps = ctx.data_engine.get_agent_friendly_steps()
|
|
168
|
+
if steps:
|
|
169
|
+
active_subgoal_hash = "default"
|
|
170
|
+
if ctx.data_engine:
|
|
171
|
+
task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
|
|
172
|
+
else:
|
|
173
|
+
task_plan_path = notes_dir / "task_plan.md"
|
|
174
|
+
task_plan = "No task plan yet."
|
|
175
|
+
keep_hashes = None
|
|
176
|
+
if task_plan_path.exists():
|
|
177
|
+
try:
|
|
178
|
+
task_plan = task_plan_path.read_text(encoding="utf-8")
|
|
179
|
+
|
|
180
|
+
active_subgoal_hash, _ = get_active_subgoal_hashes(task_plan)
|
|
181
|
+
keep_hashes = get_recent_subgoal_hashes(
|
|
182
|
+
steps,
|
|
183
|
+
active_subgoal_hash,
|
|
184
|
+
ctx.data_engine.base_dir,
|
|
185
|
+
)
|
|
186
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
|
187
|
+
logger.error(f"Failed to parse active subgoal in committee: {e}")
|
|
188
|
+
|
|
189
|
+
try:
|
|
190
|
+
plan_and_history = build_history_for(
|
|
191
|
+
"committee",
|
|
192
|
+
task_plan,
|
|
193
|
+
steps,
|
|
194
|
+
active_subgoal_hash,
|
|
195
|
+
keep_subgoal_hashes=keep_hashes,
|
|
196
|
+
engine=ctx.data_engine,
|
|
197
|
+
)
|
|
198
|
+
history_summary += plan_and_history
|
|
199
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
|
200
|
+
logger.error(f"Failed to build plan and history in committee: {e}")
|
|
201
|
+
history_summary += f"Error building plan and history: {e}"
|
|
202
|
+
|
|
203
|
+
# Read failed plans history to ensure the committee avoids
|
|
204
|
+
# repeating failed strategies
|
|
205
|
+
failed_plans_history = ""
|
|
206
|
+
failed_history_path = notes_dir / "failed_plans_history.md"
|
|
207
|
+
if failed_history_path.exists():
|
|
208
|
+
try:
|
|
209
|
+
failed_plans_history = failed_history_path.read_text(encoding="utf-8")
|
|
210
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
|
211
|
+
logger.error(f"Failed to read failed plans history in committee: {e}")
|
|
212
|
+
if failed_plans_history:
|
|
213
|
+
history_summary += (
|
|
214
|
+
f"\n\n### Previous Failed Plan Attempts\n{failed_plans_history}\n"
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
with open(blackboard_path, "a", encoding="utf-8") as f:
|
|
218
|
+
f.write(history_summary + "\n\n")
|
|
219
|
+
|
|
220
|
+
# 3. Load prompts from JSON
|
|
221
|
+
prompts = {}
|
|
222
|
+
prompts_path = Path(__file__).parent / "committee_prompts.json"
|
|
223
|
+
if prompts_path.exists():
|
|
224
|
+
try:
|
|
225
|
+
prompts = json.loads(prompts_path.read_text(encoding="utf-8"))
|
|
226
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
|
227
|
+
logger.error(f"Failed to load committee prompts: {e}")
|
|
228
|
+
|
|
229
|
+
planner_prompt = prompts.get("planner_avatar", "You are the Planner Avatar.")
|
|
230
|
+
history_analyzer_prompt = prompts.get(
|
|
231
|
+
"history_analyzer_expert", "You are the History Analyzer."
|
|
232
|
+
)
|
|
233
|
+
diagnoser_prompt = prompts.get("diagnoser_expert", "You are the Diagnoser Expert.")
|
|
234
|
+
|
|
235
|
+
# 4. Prepare Specialized Tools for Members
|
|
236
|
+
|
|
237
|
+
# History Analyzer Tools: the shared history tools plus the note readers
|
|
238
|
+
read_note_tool_hist = get_read_note_tool(ctx)
|
|
239
|
+
list_notes_tool_hist = get_list_notes_tool(ctx)
|
|
240
|
+
|
|
241
|
+
history_analyzer_tools = [
|
|
242
|
+
*get_history_tools(ctx),
|
|
243
|
+
trace_langchain_tool(read_note_tool_hist, ctx),
|
|
244
|
+
trace_langchain_tool(list_notes_tool_hist, ctx),
|
|
245
|
+
]
|
|
246
|
+
|
|
247
|
+
# Diagnoser Tools
|
|
248
|
+
video_tool = get_video_analyzer_tool(ctx, role="diagnoser")
|
|
249
|
+
log_tool = get_analyze_logs_tool(ctx)
|
|
250
|
+
read_note_tool = get_read_note_tool(ctx)
|
|
251
|
+
list_notes_tool = get_list_notes_tool(ctx)
|
|
252
|
+
|
|
253
|
+
diagnoser_tools = [
|
|
254
|
+
trace_langchain_tool(video_tool, ctx),
|
|
255
|
+
trace_langchain_tool(log_tool, ctx),
|
|
256
|
+
trace_langchain_tool(read_note_tool, ctx),
|
|
257
|
+
trace_langchain_tool(list_notes_tool, ctx),
|
|
258
|
+
]
|
|
259
|
+
|
|
260
|
+
# 5. Instantiate LLMs
|
|
261
|
+
llm_pl = get_llm(ctx=ctx, name="planner_avatar")
|
|
262
|
+
|
|
263
|
+
llm_hist = get_llm(ctx=ctx, name="history_analyzer_expert")
|
|
264
|
+
llm_hist = llm_hist.bind_tools(tools=history_analyzer_tools)
|
|
265
|
+
|
|
266
|
+
llm_diag = get_llm(ctx=ctx, name="diagnoser_expert")
|
|
267
|
+
llm_diag = llm_diag.bind_tools(tools=diagnoser_tools)
|
|
268
|
+
|
|
269
|
+
# 6. Debate Loop (configured rounds)
|
|
270
|
+
exec_setup = getattr(ctx, "execution_setup", None)
|
|
271
|
+
rounds = getattr(exec_setup, "committee_debate_rounds", 2) if exec_setup else 2
|
|
272
|
+
|
|
273
|
+
async def run_agent_turn(llm, system_prompt, tools, agent_name):
|
|
274
|
+
current_blackboard = blackboard_path.read_text(encoding="utf-8")
|
|
275
|
+
|
|
276
|
+
human_content = [
|
|
277
|
+
{
|
|
278
|
+
"type": "text",
|
|
279
|
+
"text": f"Current Blackboard:\n{current_blackboard}",
|
|
280
|
+
}
|
|
281
|
+
]
|
|
282
|
+
if screenshot_b64:
|
|
283
|
+
human_content.append(
|
|
284
|
+
{
|
|
285
|
+
"type": "image_url",
|
|
286
|
+
"image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"},
|
|
287
|
+
}
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
messages = [
|
|
291
|
+
SystemMessage(content=system_prompt),
|
|
292
|
+
HumanMessage(content=human_content),
|
|
293
|
+
]
|
|
294
|
+
|
|
295
|
+
max_iterations = 3
|
|
296
|
+
for _ in range(max_iterations):
|
|
297
|
+
response = await invoke_llm_with_timeout_message(llm.ainvoke(messages))
|
|
298
|
+
|
|
299
|
+
if not response.tool_calls:
|
|
300
|
+
return response.content
|
|
301
|
+
|
|
302
|
+
messages.append(response)
|
|
303
|
+
|
|
304
|
+
async def run_tool(tc):
|
|
305
|
+
"""Keep the raw result so the caller can read ToolMessage.status."""
|
|
306
|
+
tool_name = tc["name"]
|
|
307
|
+
logger.info(f"{agent_name} requested tool: {tool_name}")
|
|
308
|
+
tool_to_run = next((t for t in tools if t.name == tool_name), None)
|
|
309
|
+
if tool_to_run:
|
|
310
|
+
try:
|
|
311
|
+
args = dict(tc["args"])
|
|
312
|
+
return await invoke_tool_with_injection(
|
|
313
|
+
tool=tool_to_run,
|
|
314
|
+
args=args,
|
|
315
|
+
tool_call_id=tc["id"],
|
|
316
|
+
state=state,
|
|
317
|
+
)
|
|
318
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
|
319
|
+
return ToolFailure(f"Error: {e}")
|
|
320
|
+
else:
|
|
321
|
+
return ToolFailure(f"Error: Tool {tool_name} not found")
|
|
322
|
+
|
|
323
|
+
active_tool_calls = [
|
|
324
|
+
tc for tc in response.tool_calls if tc["name"] != "google_search"
|
|
325
|
+
]
|
|
326
|
+
tool_outputs = await asyncio.gather(*(run_tool(tc) for tc in active_tool_calls))
|
|
327
|
+
|
|
328
|
+
for tc, raw_result in zip(active_tool_calls, tool_outputs):
|
|
329
|
+
# Read status before unwrapping ToolMessage.content.
|
|
330
|
+
status = "error" if is_tool_failure(raw_result) else "success"
|
|
331
|
+
messages.extend(
|
|
332
|
+
tool_result_messages(
|
|
333
|
+
tc["id"],
|
|
334
|
+
get_tool_result_content(raw_result),
|
|
335
|
+
name=tc["name"],
|
|
336
|
+
status=status,
|
|
337
|
+
llm=llm,
|
|
338
|
+
)
|
|
339
|
+
)
|
|
340
|
+
return ToolFailure("Error: Reached max iterations in turn.")
|
|
341
|
+
|
|
342
|
+
for r in range(1, rounds + 1):
|
|
343
|
+
logger.info(f"Committee Round {r}")
|
|
344
|
+
|
|
345
|
+
# 1. Planner Avatar speaks first (Host)
|
|
346
|
+
pl_response = await run_agent_turn(llm_pl, planner_prompt, [], "Planner Avatar")
|
|
347
|
+
with open(blackboard_path, "a", encoding="utf-8") as f:
|
|
348
|
+
f.write(f"### Planner Avatar (Round {r})\n{pl_response}\n\n")
|
|
349
|
+
|
|
350
|
+
# 2. Diagnoser speaks (Can use tools)
|
|
351
|
+
diag_response = await run_agent_turn(
|
|
352
|
+
llm_diag, diagnoser_prompt, diagnoser_tools, "Diagnoser"
|
|
353
|
+
)
|
|
354
|
+
with open(blackboard_path, "a", encoding="utf-8") as f:
|
|
355
|
+
f.write(f"### Diagnoser (Round {r})\n{diag_response}\n\n")
|
|
356
|
+
|
|
357
|
+
# 3. History Analyzer speaks (Can use tools)
|
|
358
|
+
hist_response = await run_agent_turn(
|
|
359
|
+
llm_hist,
|
|
360
|
+
history_analyzer_prompt,
|
|
361
|
+
history_analyzer_tools,
|
|
362
|
+
"History Analyzer",
|
|
363
|
+
)
|
|
364
|
+
with open(blackboard_path, "a", encoding="utf-8") as f:
|
|
365
|
+
f.write(f"### History Analyzer (Round {r})\n{hist_response}\n\n")
|
|
366
|
+
|
|
367
|
+
# Final Synthesis by Planner Avatar
|
|
368
|
+
current_blackboard = blackboard_path.read_text(encoding="utf-8")
|
|
369
|
+
|
|
370
|
+
final_human_content = [
|
|
371
|
+
{
|
|
372
|
+
"type": "text",
|
|
373
|
+
"text": f"Current Blackboard:\n{current_blackboard}",
|
|
374
|
+
}
|
|
375
|
+
]
|
|
376
|
+
if screenshot_b64:
|
|
377
|
+
final_human_content.append(
|
|
378
|
+
{
|
|
379
|
+
"type": "image_url",
|
|
380
|
+
"image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"},
|
|
381
|
+
}
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
msg_final = [
|
|
385
|
+
SystemMessage(
|
|
386
|
+
content=(
|
|
387
|
+
"You are the Planner Avatar. Summarize the discussion"
|
|
388
|
+
" and output the final recommended plan revisions and"
|
|
389
|
+
" action plan based on your master's directive."
|
|
390
|
+
)
|
|
391
|
+
),
|
|
392
|
+
HumanMessage(content=final_human_content),
|
|
393
|
+
]
|
|
394
|
+
resp_final = await invoke_llm_with_timeout_message(llm_pl.ainvoke(msg_final))
|
|
395
|
+
|
|
396
|
+
with open(blackboard_path, "a", encoding="utf-8") as f:
|
|
397
|
+
f.write(f"## Final Conclusion\n{resp_final.content}\n")
|
|
398
|
+
|
|
399
|
+
return resp_final.content
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
async def _run_committee_logic(
|
|
403
|
+
ctx: ArtemisContext | None,
|
|
404
|
+
state: State | None,
|
|
405
|
+
avatar_directive: str,
|
|
406
|
+
) -> str:
|
|
407
|
+
"""Executes multi-agent committee debate logic."""
|
|
408
|
+
if ctx is None:
|
|
409
|
+
return ToolFailure("Error: ArtemisContext is required for ask_committee.")
|
|
410
|
+
|
|
411
|
+
try:
|
|
412
|
+
return await asyncio.wait_for(_execute_committee(ctx, state, avatar_directive), timeout=300)
|
|
413
|
+
except TimeoutError:
|
|
414
|
+
logger.error("Committee timed out after 300 seconds.")
|
|
415
|
+
return ToolFailure("Error: Committee timed out after 300 seconds.")
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
ask_committee_wrapper = ToolWrapper(
|
|
419
|
+
tool_fn_getter=get_ask_committee_tool,
|
|
420
|
+
on_success_fn=lambda output: f"Committee concluded:\n{output}",
|
|
421
|
+
on_failure_fn=lambda error: f"Committee failed: {error}",
|
|
422
|
+
)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Universal tool for submitting final diagnosis and actionable recovery advice."""
|
|
16
|
+
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from langchain_core.tools import BaseTool
|
|
20
|
+
from pydantic import BaseModel, Field
|
|
21
|
+
|
|
22
|
+
from artemis.context import ArtemisContext
|
|
23
|
+
from artemis.data_engine.trace import trace_langchain_tool
|
|
24
|
+
from artemis.drivers.base import BaseDeviceDriver
|
|
25
|
+
from artemis.tools.base import ArtemisTool
|
|
26
|
+
from third_party.mobile_use.utils.logger import get_logger
|
|
27
|
+
|
|
28
|
+
logger = get_logger(__name__)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# pylint: disable=too-few-public-methods
|
|
32
|
+
class SubmitAnswerArgs(BaseModel):
|
|
33
|
+
"""Arguments schema for submitting final diagnoser answer."""
|
|
34
|
+
|
|
35
|
+
analysis: str = Field(
|
|
36
|
+
...,
|
|
37
|
+
description=(
|
|
38
|
+
"A detailed and concise explanation of the root cause of the"
|
|
39
|
+
" failure or query. Answer in one full paragraph."
|
|
40
|
+
),
|
|
41
|
+
)
|
|
42
|
+
actionable_steps: list[str] = Field(
|
|
43
|
+
default_factory=list,
|
|
44
|
+
description=(
|
|
45
|
+
"A list of concrete recovery instructions or next steps. Each"
|
|
46
|
+
" step is a short sentence. If no actionable steps are"
|
|
47
|
+
" available, return an empty list."
|
|
48
|
+
),
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class SubmitAnswerTool(ArtemisTool):
|
|
53
|
+
"""Universal tool for submitting final diagnosis and actionable recovery advice."""
|
|
54
|
+
|
|
55
|
+
def __init__(self):
|
|
56
|
+
super().__init__(
|
|
57
|
+
name="submit_answer",
|
|
58
|
+
description=(
|
|
59
|
+
"[Finalization] Submits the final diagnosis and actionable recovery advice. "
|
|
60
|
+
"Call this tool when you have pinpointed the root cause."
|
|
61
|
+
),
|
|
62
|
+
args_schema=SubmitAnswerArgs,
|
|
63
|
+
category="custom",
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
|
67
|
+
async def execute(
|
|
68
|
+
self,
|
|
69
|
+
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
|
70
|
+
ctx: ArtemisContext | None = None, # pylint: disable=unused-argument
|
|
71
|
+
analysis: str | None = None, # pylint: disable=unused-argument
|
|
72
|
+
actionable_steps: list[str] | None = None, # pylint: disable=unused-argument
|
|
73
|
+
**kwargs: Any,
|
|
74
|
+
) -> str:
|
|
75
|
+
logger.info("Diagnoser submitted answer via tool.")
|
|
76
|
+
return "Answer submitted successfully."
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# Universal tool instance & aliases
|
|
80
|
+
submit_answer = SubmitAnswerTool()
|
|
81
|
+
SubmitAnswer = SubmitAnswerTool
|
|
82
|
+
DiagnoserSubmitAnswerTool = SubmitAnswerTool
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def get_submit_answer_tool(ctx: ArtemisContext) -> BaseTool:
|
|
86
|
+
"""Exports submit_answer as a LangChain BaseTool."""
|
|
87
|
+
return trace_langchain_tool(submit_answer.to_langchain_tool(ctx), ctx)
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# Copyright 2026 Google LLC
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""Universal diagnostic agent tool for deep system logs and video inspection."""
|
|
16
|
+
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from langchain_core.tools import BaseTool
|
|
20
|
+
from pydantic import BaseModel, Field
|
|
21
|
+
|
|
22
|
+
from artemis.core.tool_failure import ToolFailure
|
|
23
|
+
from artemis.agents.diagnoser.diagnoser import Diagnoser
|
|
24
|
+
from artemis.context import ArtemisContext
|
|
25
|
+
from artemis.data_engine.trace import trace_langchain_tool
|
|
26
|
+
from artemis.drivers.base import BaseDeviceDriver
|
|
27
|
+
from artemis.graph.state import State
|
|
28
|
+
from artemis.tools.base import ArtemisTool
|
|
29
|
+
from artemis.tools.tool_wrapper import ToolWrapper
|
|
30
|
+
from artemis.utils.cython_compat import CyFunctionDetector
|
|
31
|
+
from third_party.mobile_use.utils.logger import get_logger
|
|
32
|
+
|
|
33
|
+
logger = get_logger(__name__)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# pylint: disable=too-few-public-methods
|
|
37
|
+
class AskDiagnoserArgs(BaseModel):
|
|
38
|
+
"""Arguments schema for invoking the diagnostic subagent."""
|
|
39
|
+
|
|
40
|
+
model_config = {"ignored_types": (CyFunctionDetector,)}
|
|
41
|
+
query: str = Field(
|
|
42
|
+
...,
|
|
43
|
+
description=(
|
|
44
|
+
"The diagnostic query or what you want to investigate. Please"
|
|
45
|
+
" provide a specific, narrow time range if applicable."
|
|
46
|
+
),
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
async def _run_diagnoser_logic(
|
|
51
|
+
ctx: ArtemisContext | None,
|
|
52
|
+
state: State | None,
|
|
53
|
+
query: str,
|
|
54
|
+
) -> str:
|
|
55
|
+
logger.info(f"ask_diagnoser tool called with query: {query}")
|
|
56
|
+
if ctx is None:
|
|
57
|
+
return ToolFailure("Error: ArtemisContext is required for Diagnoser.")
|
|
58
|
+
agent = Diagnoser(ctx)
|
|
59
|
+
try:
|
|
60
|
+
result = await agent.run(query, state)
|
|
61
|
+
return f"Diagnoser replied:\n{result}"
|
|
62
|
+
except Exception as e: # pylint: disable=broad-exception-caught
|
|
63
|
+
logger.error(f"Failed to run diagnoser: {e}")
|
|
64
|
+
return f"Diagnoser failed: {e}"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class AskDiagnoserTool(ArtemisTool):
|
|
68
|
+
"""Universal tool for invoking the Diagnoser subagent to inspect logs and videos."""
|
|
69
|
+
|
|
70
|
+
def __init__(self):
|
|
71
|
+
super().__init__(
|
|
72
|
+
name="ask_diagnoser",
|
|
73
|
+
description=(
|
|
74
|
+
"[DIAGNOSTIC] Analyzes system logs (ADB) and video recordings to "
|
|
75
|
+
"pinpoint the root cause of a step failure, screen freeze, or "
|
|
76
|
+
"unexpected UI drift. Use it when the live screen does not explain "
|
|
77
|
+
"why an action is not taking effect or why progress is stalled, "
|
|
78
|
+
"before committing to further physical actions. Note: When querying "
|
|
79
|
+
"video files, specify a narrow timeframe (e.g., '[10s, 20s]') for "
|
|
80
|
+
"faster analysis."
|
|
81
|
+
),
|
|
82
|
+
args_schema=AskDiagnoserArgs,
|
|
83
|
+
category="custom",
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
|
87
|
+
async def execute(
|
|
88
|
+
self,
|
|
89
|
+
driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
|
|
90
|
+
ctx: ArtemisContext | None = None,
|
|
91
|
+
query: str | None = None,
|
|
92
|
+
state: State | None = None,
|
|
93
|
+
**kwargs: Any,
|
|
94
|
+
) -> str:
|
|
95
|
+
q = query or kwargs.get("Query") or ""
|
|
96
|
+
st = state if state is not None else kwargs.get("state")
|
|
97
|
+
return await _run_diagnoser_logic(ctx, st, q)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
# Universal tool instance & aliases
|
|
101
|
+
ask_diagnoser = AskDiagnoserTool()
|
|
102
|
+
AskDiagnoser = AskDiagnoserTool
|
|
103
|
+
DiagnosticTool = AskDiagnoserTool
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def get_ask_diagnostic_agent_tool(ctx: ArtemisContext) -> BaseTool:
|
|
107
|
+
"""Exports ask_diagnoser as a LangChain BaseTool."""
|
|
108
|
+
return trace_langchain_tool(ask_diagnoser.to_langchain_tool(ctx), ctx)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def get_ask_diagnoser_tool(ctx: ArtemisContext) -> BaseTool:
|
|
112
|
+
"""Exports ask_diagnoser as a LangChain BaseTool (standard naming alias)."""
|
|
113
|
+
return trace_langchain_tool(ask_diagnoser.to_langchain_tool(ctx), ctx)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
ask_diagnoser_wrapper = ToolWrapper(
|
|
117
|
+
tool_fn_getter=get_ask_diagnostic_agent_tool,
|
|
118
|
+
on_success_fn=lambda output: f"Diagnoser replied:\n{output}",
|
|
119
|
+
on_failure_fn=lambda error: f"Diagnoser failed: {error}",
|
|
120
|
+
)
|