android-midscene-automation 0.2.4 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (525) hide show
  1. package/CHANGELOG.md +7 -0
  2. package/README.md +35 -0
  3. package/artemis-main/.dockerignore +15 -0
  4. package/artemis-main/.env.example +34 -0
  5. package/artemis-main/CONTRIBUTING.md +35 -0
  6. package/artemis-main/Dockerfile +68 -0
  7. package/artemis-main/LICENSE +202 -0
  8. package/artemis-main/Makefile +115 -0
  9. package/artemis-main/README.md +322 -0
  10. package/artemis-main/README_CN.md +320 -0
  11. package/artemis-main/apps/__init__.py +15 -0
  12. package/artemis-main/apps/admin_console/__init__.py +19 -0
  13. package/artemis-main/apps/admin_console/core/__init__.py +13 -0
  14. package/artemis-main/apps/admin_console/core/config.py +43 -0
  15. package/artemis-main/apps/admin_console/core/security.py +178 -0
  16. package/artemis-main/apps/admin_console/core/state.py +243 -0
  17. package/artemis-main/apps/admin_console/database/__init__.py +13 -0
  18. package/artemis-main/apps/admin_console/database/connection.py +60 -0
  19. package/artemis-main/apps/admin_console/database/repositories/__init__.py +13 -0
  20. package/artemis-main/apps/admin_console/database/repositories/session_repository.py +564 -0
  21. package/artemis-main/apps/admin_console/database/repositories/step_repository.py +512 -0
  22. package/artemis-main/apps/admin_console/database/repositories/trace_repository.py +133 -0
  23. package/artemis-main/apps/admin_console/index.html +3787 -0
  24. package/artemis-main/apps/admin_console/replay_manager.py +2585 -0
  25. package/artemis-main/apps/admin_console/routers/__init__.py +13 -0
  26. package/artemis-main/apps/admin_console/routers/media.py +198 -0
  27. package/artemis-main/apps/admin_console/routers/replay.py +91 -0
  28. package/artemis-main/apps/admin_console/routers/sessions.py +239 -0
  29. package/artemis-main/apps/admin_console/routers/steps.py +109 -0
  30. package/artemis-main/apps/admin_console/routers/stream.py +56 -0
  31. package/artemis-main/apps/admin_console/routers/system.py +761 -0
  32. package/artemis-main/apps/admin_console/routers/tasks.py +517 -0
  33. package/artemis-main/apps/admin_console/schemas/__init__.py +13 -0
  34. package/artemis-main/apps/admin_console/schemas/task_schema.py +47 -0
  35. package/artemis-main/apps/admin_console/server.py +508 -0
  36. package/artemis-main/apps/admin_console/services/__init__.py +13 -0
  37. package/artemis-main/apps/admin_console/services/device_stream_service.py +137 -0
  38. package/artemis-main/apps/admin_console/services/ipc_service.py +262 -0
  39. package/artemis-main/apps/admin_console/services/media_service.py +623 -0
  40. package/artemis-main/apps/admin_console/services/model_service.py +148 -0
  41. package/artemis-main/apps/admin_console/services/task_preset_catalog.py +514 -0
  42. package/artemis-main/apps/admin_console/services/task_queue_service.py +1719 -0
  43. package/artemis-main/apps/admin_console/services/worker_process_io.py +144 -0
  44. package/artemis-main/apps/showcase_ui/README.md +31 -0
  45. package/artemis-main/apps/showcase_ui/angular.json +132 -0
  46. package/artemis-main/apps/showcase_ui/package-lock.json +9472 -0
  47. package/artemis-main/apps/showcase_ui/package.json +39 -0
  48. package/artemis-main/apps/showcase_ui/proxy.conf.json +23 -0
  49. package/artemis-main/apps/showcase_ui/public/favicon.ico +0 -0
  50. package/artemis-main/apps/showcase_ui/public/logo.png +0 -0
  51. package/artemis-main/apps/showcase_ui/public/logo.svg +56 -0
  52. package/artemis-main/apps/showcase_ui/src/app/app.component.html +24 -0
  53. package/artemis-main/apps/showcase_ui/src/app/app.component.scss +49 -0
  54. package/artemis-main/apps/showcase_ui/src/app/app.component.spec.ts +42 -0
  55. package/artemis-main/apps/showcase_ui/src/app/app.component.ts +32 -0
  56. package/artemis-main/apps/showcase_ui/src/app/app.config.ts +29 -0
  57. package/artemis-main/apps/showcase_ui/src/app/app.routes.ts +27 -0
  58. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream-startup.spec.ts +108 -0
  59. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html +1436 -0
  60. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss +3973 -0
  61. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts +2235 -0
  62. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.html +380 -0
  63. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.scss +1171 -0
  64. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.ts +270 -0
  65. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html +433 -0
  66. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss +987 -0
  67. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts +632 -0
  68. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html +130 -0
  69. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.scss +406 -0
  70. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.ts +74 -0
  71. package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.scss +115 -0
  72. package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.ts +52 -0
  73. package/artemis-main/apps/showcase_ui/src/app/core/data/smart-tasks.data.ts +334 -0
  74. package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.spec.ts +55 -0
  75. package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.ts +61 -0
  76. package/artemis-main/apps/showcase_ui/src/app/core/i18n/translate.pipe.ts +11 -0
  77. package/artemis-main/apps/showcase_ui/src/app/core/i18n/zh-cn.ts +651 -0
  78. package/artemis-main/apps/showcase_ui/src/app/core/models/markdown.model.ts +43 -0
  79. package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.spec.ts +47 -0
  80. package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.ts +165 -0
  81. package/artemis-main/apps/showcase_ui/src/app/core/models/session.model.ts +85 -0
  82. package/artemis-main/apps/showcase_ui/src/app/core/models/stream.model.ts +246 -0
  83. package/artemis-main/apps/showcase_ui/src/app/core/models/system.model.ts +118 -0
  84. package/artemis-main/apps/showcase_ui/src/app/core/services/task-recommendation.service.ts +111 -0
  85. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.html +2130 -0
  86. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.scss +5204 -0
  87. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.ts +1189 -0
  88. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.html +39 -0
  89. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.scss +151 -0
  90. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.ts +58 -0
  91. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.html +201 -0
  92. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.scss +886 -0
  93. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.ts +324 -0
  94. package/artemis-main/apps/showcase_ui/src/app/services/agent.service.spec.ts +418 -0
  95. package/artemis-main/apps/showcase_ui/src/app/services/agent.service.ts +2012 -0
  96. package/artemis-main/apps/showcase_ui/src/app/services/system.service.spec.ts +173 -0
  97. package/artemis-main/apps/showcase_ui/src/app/services/system.service.ts +620 -0
  98. package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts +513 -0
  99. package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.ts +1015 -0
  100. package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.spec.ts +85 -0
  101. package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.ts +662 -0
  102. package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.spec.ts +70 -0
  103. package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.ts +383 -0
  104. package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.spec.ts +107 -0
  105. package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.ts +123 -0
  106. package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.spec.ts +92 -0
  107. package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.ts +116 -0
  108. package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts +640 -0
  109. package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts +1021 -0
  110. package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts +375 -0
  111. package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.ts +1419 -0
  112. package/artemis-main/apps/showcase_ui/src/index.html +37 -0
  113. package/artemis-main/apps/showcase_ui/src/main.ts +22 -0
  114. package/artemis-main/apps/showcase_ui/src/styles.scss +60 -0
  115. package/artemis-main/apps/showcase_ui/tsconfig.app.json +23 -0
  116. package/artemis-main/apps/showcase_ui/tsconfig.json +27 -0
  117. package/artemis-main/apps/showcase_ui/tsconfig.spec.json +23 -0
  118. package/artemis-main/artemis/__init__.py +44 -0
  119. package/artemis-main/artemis/__main__.py +18 -0
  120. package/artemis-main/artemis/_legacy_imports.py +94 -0
  121. package/artemis-main/artemis/_version.py +30 -0
  122. package/artemis-main/artemis/agents/__init__.py +17 -0
  123. package/artemis-main/artemis/agents/checker/checker.json +9 -0
  124. package/artemis-main/artemis/agents/checker/checker.py +859 -0
  125. package/artemis-main/artemis/agents/diagnoser/diagnoser.json +23 -0
  126. package/artemis-main/artemis/agents/diagnoser/diagnoser.py +608 -0
  127. package/artemis-main/artemis/agents/explorer/constants.py +52 -0
  128. package/artemis-main/artemis/agents/explorer/explorer.json +57 -0
  129. package/artemis-main/artemis/agents/explorer/explorer.py +473 -0
  130. package/artemis-main/artemis/agents/explorer/geometry.py +73 -0
  131. package/artemis-main/artemis/agents/explorer/native_runner.py +878 -0
  132. package/artemis-main/artemis/agents/explorer/perception_tools.py +632 -0
  133. package/artemis-main/artemis/agents/explorer/run_setup.py +403 -0
  134. package/artemis-main/artemis/agents/explorer/screen_index.py +266 -0
  135. package/artemis-main/artemis/agents/explorer/tiers.py +110 -0
  136. package/artemis-main/artemis/agents/explorer/tool_declarations.py +394 -0
  137. package/artemis-main/artemis/agents/explorer/universal_runner.py +317 -0
  138. package/artemis-main/artemis/agents/flash/context_compressor.py +452 -0
  139. package/artemis-main/artemis/agents/flash/flash_runner.md +51 -0
  140. package/artemis-main/artemis/agents/flash/flash_summarizer.md +53 -0
  141. package/artemis-main/artemis/agents/flash/flash_summarizer_single.md +54 -0
  142. package/artemis-main/artemis/agents/flash/runner.py +1167 -0
  143. package/artemis-main/artemis/agents/flash/summarizer.py +511 -0
  144. package/artemis-main/artemis/agents/history_analyzer/history_analyzer.md +8 -0
  145. package/artemis-main/artemis/agents/history_analyzer/history_analyzer.py +182 -0
  146. package/artemis-main/artemis/agents/image_processor/image_processor.md +74 -0
  147. package/artemis-main/artemis/agents/image_processor/image_processor.py +322 -0
  148. package/artemis-main/artemis/agents/log_analyzer/log_analyzer.md +15 -0
  149. package/artemis-main/artemis/agents/log_analyzer/log_analyzer.py +403 -0
  150. package/artemis-main/artemis/agents/log_analyzer/log_reader_sub_agent.md +8 -0
  151. package/artemis-main/artemis/agents/log_analyzer/output_analyzer.md +11 -0
  152. package/artemis-main/artemis/agents/log_analyzer/output_analyzer.py +163 -0
  153. package/artemis-main/artemis/agents/object_detector/object_detector.json +9 -0
  154. package/artemis-main/artemis/agents/object_detector/object_detector.py +211 -0
  155. package/artemis-main/artemis/agents/operator/operator.json +3 -0
  156. package/artemis-main/artemis/agents/operator/operator.py +1394 -0
  157. package/artemis-main/artemis/agents/operator/prompts.py +1064 -0
  158. package/artemis-main/artemis/agents/outputter/outputter.md +36 -0
  159. package/artemis-main/artemis/agents/outputter/outputter.py +552 -0
  160. package/artemis-main/artemis/agents/planner/planner.json +32 -0
  161. package/artemis-main/artemis/agents/planner/planner.py +436 -0
  162. package/artemis-main/artemis/agents/prompt_assembly.py +79 -0
  163. package/artemis-main/artemis/agents/summarizer/summarizer.py +190 -0
  164. package/artemis-main/artemis/agents/validator/action_execution.py +98 -0
  165. package/artemis-main/artemis/agents/validator/categories.py +33 -0
  166. package/artemis-main/artemis/agents/validator/execution_loop.py +488 -0
  167. package/artemis-main/artemis/agents/validator/incidents.py +120 -0
  168. package/artemis-main/artemis/agents/validator/pixel_safety_net.md +15 -0
  169. package/artemis-main/artemis/agents/validator/precondition_pixel.py +320 -0
  170. package/artemis-main/artemis/agents/validator/precondition_xml.py +672 -0
  171. package/artemis-main/artemis/agents/validator/tool_declarations.py +332 -0
  172. package/artemis-main/artemis/agents/validator/validator.py +188 -0
  173. package/artemis-main/artemis/agents/video_analyzer/__init__.py +15 -0
  174. package/artemis-main/artemis/agents/video_analyzer/audio_native.py +543 -0
  175. package/artemis-main/artemis/agents/video_analyzer/blackboard.py +821 -0
  176. package/artemis-main/artemis/agents/video_analyzer/chunk_agentic.py +455 -0
  177. package/artemis-main/artemis/agents/video_analyzer/chunk_conversation.py +491 -0
  178. package/artemis-main/artemis/agents/video_analyzer/chunk_native.py +437 -0
  179. package/artemis-main/artemis/agents/video_analyzer/conflict_resolution.py +137 -0
  180. package/artemis-main/artemis/agents/video_analyzer/gemini_files.py +162 -0
  181. package/artemis-main/artemis/agents/video_analyzer/native_coordinator.py +541 -0
  182. package/artemis-main/artemis/agents/video_analyzer/reliability.py +128 -0
  183. package/artemis-main/artemis/agents/video_analyzer/universal_engine.py +487 -0
  184. package/artemis-main/artemis/agents/video_analyzer/universal_tools.py +323 -0
  185. package/artemis-main/artemis/agents/video_analyzer/video_analyzer.md +51 -0
  186. package/artemis-main/artemis/agents/video_analyzer/video_analyzer.py +875 -0
  187. package/artemis-main/artemis/agents/video_analyzer/video_sub_agent.md +15 -0
  188. package/artemis-main/artemis/agents/video_analyzer/video_sub_agent_audio.md +12 -0
  189. package/artemis-main/artemis/clients/accessibility_client.py +438 -0
  190. package/artemis-main/artemis/clients/screen_client_factory.py +403 -0
  191. package/artemis-main/artemis/config/__init__.py +309 -0
  192. package/artemis-main/artemis/config/agent.py +1098 -0
  193. package/artemis-main/artemis/config/constants.py +151 -0
  194. package/artemis-main/artemis/config/core/__init__.py +24 -0
  195. package/artemis-main/artemis/config/core/engine.py +118 -0
  196. package/artemis-main/artemis/config/core/interpolator.py +78 -0
  197. package/artemis-main/artemis/config/llm.py +267 -0
  198. package/artemis-main/artemis/config/paths.py +244 -0
  199. package/artemis-main/artemis/config/runtime.py +197 -0
  200. package/artemis-main/artemis/config/settings.py +315 -0
  201. package/artemis-main/artemis/constants.py +47 -0
  202. package/artemis-main/artemis/context.py +289 -0
  203. package/artemis-main/artemis/controllers/__init__.py +41 -0
  204. package/artemis-main/artemis/controllers/controller_factory.py +41 -0
  205. package/artemis-main/artemis/controllers/device_controller.py +29 -0
  206. package/artemis-main/artemis/controllers/types.py +37 -0
  207. package/artemis-main/artemis/controllers/unified_controller.py +649 -0
  208. package/artemis-main/artemis/core/__init__.py +21 -0
  209. package/artemis-main/artemis/core/diagnostics/__init__.py +90 -0
  210. package/artemis-main/artemis/core/diagnostics/adb_keys.py +270 -0
  211. package/artemis-main/artemis/core/diagnostics/adb_server_connection.py +330 -0
  212. package/artemis-main/artemis/core/diagnostics/device_smoke.py +364 -0
  213. package/artemis-main/artemis/core/diagnostics/emulator_manager.py +501 -0
  214. package/artemis-main/artemis/core/diagnostics/engine.py +438 -0
  215. package/artemis-main/artemis/core/diagnostics/hierarchy_parity.py +216 -0
  216. package/artemis-main/artemis/core/diagnostics/probes/adb_probe.py +887 -0
  217. package/artemis-main/artemis/core/diagnostics/probes/base.py +45 -0
  218. package/artemis-main/artemis/core/diagnostics/probes/credentials_probe.py +296 -0
  219. package/artemis-main/artemis/core/diagnostics/probes/host_probe.py +427 -0
  220. package/artemis-main/artemis/core/diagnostics/probes/runtime_probe.py +169 -0
  221. package/artemis-main/artemis/core/diagnostics/probes/toolchain_probe.py +139 -0
  222. package/artemis-main/artemis/core/diagnostics/readiness.py +83 -0
  223. package/artemis-main/artemis/core/diagnostics/schema.py +118 -0
  224. package/artemis-main/artemis/core/tool_declaration.py +43 -0
  225. package/artemis-main/artemis/core/tool_failure.py +56 -0
  226. package/artemis-main/artemis/data_engine/context_vars.py +24 -0
  227. package/artemis-main/artemis/data_engine/engine.py +1615 -0
  228. package/artemis-main/artemis/data_engine/history_reader.py +142 -0
  229. package/artemis-main/artemis/data_engine/models.py +131 -0
  230. package/artemis-main/artemis/data_engine/storage.py +1245 -0
  231. package/artemis-main/artemis/data_engine/trace.py +727 -0
  232. package/artemis-main/artemis/drivers/__init__.py +48 -0
  233. package/artemis-main/artemis/drivers/android/adb_driver.py +507 -0
  234. package/artemis-main/artemis/drivers/android/hierarchy.py +64 -0
  235. package/artemis-main/artemis/drivers/android/input_ime.py +120 -0
  236. package/artemis-main/artemis/drivers/android/recorder.py +57 -0
  237. package/artemis-main/artemis/drivers/base.py +290 -0
  238. package/artemis-main/artemis/drivers/cloud/cloud_driver.py +117 -0
  239. package/artemis-main/artemis/drivers/factory.py +80 -0
  240. package/artemis-main/artemis/drivers/mock/mock_driver.py +198 -0
  241. package/artemis-main/artemis/drivers/types.py +67 -0
  242. package/artemis-main/artemis/graph/checkpoints.py +1243 -0
  243. package/artemis-main/artemis/graph/graph.py +1055 -0
  244. package/artemis-main/artemis/graph/perception.py +234 -0
  245. package/artemis-main/artemis/graph/state.py +151 -0
  246. package/artemis-main/artemis/graph/visibility.py +238 -0
  247. package/artemis-main/artemis/interfaces/cli/__init__.py +19 -0
  248. package/artemis-main/artemis/interfaces/cli/commands/__init__.py +15 -0
  249. package/artemis-main/artemis/interfaces/cli/commands/batch.py +278 -0
  250. package/artemis-main/artemis/interfaces/cli/commands/doctor.py +468 -0
  251. package/artemis-main/artemis/interfaces/cli/commands/helper.py +226 -0
  252. package/artemis-main/artemis/interfaces/cli/commands/init.py +193 -0
  253. package/artemis-main/artemis/interfaces/cli/commands/mcp.py +862 -0
  254. package/artemis-main/artemis/interfaces/cli/commands/run.py +463 -0
  255. package/artemis-main/artemis/interfaces/cli/commands/server.py +151 -0
  256. package/artemis-main/artemis/interfaces/cli/commands/server_lifecycle.py +328 -0
  257. package/artemis-main/artemis/interfaces/cli/commands/trace.py +111 -0
  258. package/artemis-main/artemis/interfaces/cli/commands/ui.py +404 -0
  259. package/artemis-main/artemis/interfaces/cli/main.py +109 -0
  260. package/artemis-main/artemis/interfaces/sdk/__init__.py +31 -0
  261. package/artemis-main/artemis/interfaces/sdk/client.py +19 -0
  262. package/artemis-main/artemis/interfaces/sdk/task.py +72 -0
  263. package/artemis-main/artemis/llm/__init__.py +25 -0
  264. package/artemis-main/artemis/llm/fake_model.py +179 -0
  265. package/artemis-main/artemis/llm/google/__init__.py +57 -0
  266. package/artemis-main/artemis/llm/google/provider.py +185 -0
  267. package/artemis-main/artemis/llm/google/usage.py +148 -0
  268. package/artemis-main/artemis/llm/parser.py +32 -0
  269. package/artemis-main/artemis/llm/reliability.py +267 -0
  270. package/artemis-main/artemis/llm/router.py +389 -0
  271. package/artemis-main/artemis/llm/schemas.py +36 -0
  272. package/artemis-main/artemis/llm/structured.py +196 -0
  273. package/artemis-main/artemis/main.py +58 -0
  274. package/artemis-main/artemis/mcp/__init__.py +20 -0
  275. package/artemis-main/artemis/mcp/action_executor.py +760 -0
  276. package/artemis-main/artemis/mcp/action_manifest.py +320 -0
  277. package/artemis-main/artemis/mcp/action_names.py +172 -0
  278. package/artemis-main/artemis/mcp/action_server.py +202 -0
  279. package/artemis-main/artemis/mcp/action_session.py +245 -0
  280. package/artemis-main/artemis/mcp/action_specs.py +868 -0
  281. package/artemis-main/artemis/mcp/action_types.py +86 -0
  282. package/artemis-main/artemis/mcp/actuators/__init__.py +21 -0
  283. package/artemis-main/artemis/mcp/actuators/adb.py +414 -0
  284. package/artemis-main/artemis/mcp/actuators/base.py +113 -0
  285. package/artemis-main/artemis/mcp/actuators/mock.py +70 -0
  286. package/artemis-main/artemis/mcp/adb_server.py +490 -0
  287. package/artemis-main/artemis/mcp/observation.py +133 -0
  288. package/artemis-main/artemis/mcp/xml_search_server.py +297 -0
  289. package/artemis-main/artemis/memory/__init__.py +90 -0
  290. package/artemis-main/artemis/memory/chunking.py +1843 -0
  291. package/artemis-main/artemis/memory/context_policy.py +226 -0
  292. package/artemis-main/artemis/memory/step_capsule.md +73 -0
  293. package/artemis-main/artemis/memory/step_memory.py +281 -0
  294. package/artemis-main/artemis/memory/transcript.py +798 -0
  295. package/artemis-main/artemis/platform/__init__.py +61 -0
  296. package/artemis-main/artemis/platform/base.py +104 -0
  297. package/artemis-main/artemis/platform/darwin.py +184 -0
  298. package/artemis-main/artemis/platform/linux.py +207 -0
  299. package/artemis-main/artemis/platform/windows.py +193 -0
  300. package/artemis-main/artemis/resources/__init__.py +46 -0
  301. package/artemis-main/artemis/runtime/__init__.py +108 -0
  302. package/artemis-main/artemis/runtime/adb_endpoint.py +173 -0
  303. package/artemis-main/artemis/runtime/awake_lease.py +283 -0
  304. package/artemis-main/artemis/runtime/awake_service.py +343 -0
  305. package/artemis-main/artemis/runtime/cancel_requests.py +254 -0
  306. package/artemis-main/artemis/runtime/daemon_client.py +422 -0
  307. package/artemis-main/artemis/runtime/device_lock.py +1013 -0
  308. package/artemis-main/artemis/runtime/device_pool.py +478 -0
  309. package/artemis-main/artemis/runtime/helper_manager.py +1030 -0
  310. package/artemis-main/artemis/runtime/process_probe.py +98 -0
  311. package/artemis-main/artemis/runtime/server_lifecycle.py +435 -0
  312. package/artemis-main/artemis/runtime/supervisor.py +121 -0
  313. package/artemis-main/artemis/runtime/trace_store.py +375 -0
  314. package/artemis-main/artemis/sdk/__init__.py +26 -0
  315. package/artemis-main/artemis/sdk/agent.py +485 -0
  316. package/artemis-main/artemis/sdk/builders/__init__.py +29 -0
  317. package/artemis-main/artemis/sdk/builders/agent_config_builder.py +474 -0
  318. package/artemis-main/artemis/sdk/constants.py +15 -0
  319. package/artemis-main/artemis/sdk/run_outcome.py +58 -0
  320. package/artemis-main/artemis/sdk/types/__init__.py +61 -0
  321. package/artemis-main/artemis/sdk/types/agent.py +89 -0
  322. package/artemis-main/artemis/sdk/utils.py +28 -0
  323. package/artemis-main/artemis/services/llm.py +1040 -0
  324. package/artemis-main/artemis/services/token_meter.py +267 -0
  325. package/artemis-main/artemis/telemetry/__init__.py +35 -0
  326. package/artemis-main/artemis/telemetry/models.py +64 -0
  327. package/artemis-main/artemis/telemetry/replay.py +34 -0
  328. package/artemis-main/artemis/telemetry/storage/json_storage.py +47 -0
  329. package/artemis-main/artemis/telemetry/tracer.py +85 -0
  330. package/artemis-main/artemis/toolchain/__init__.py +52 -0
  331. package/artemis-main/artemis/toolchain/descriptors.py +92 -0
  332. package/artemis-main/artemis/toolchain/resolver.py +185 -0
  333. package/artemis-main/artemis/tools/base.py +232 -0
  334. package/artemis-main/artemis/tools/command_tool.py +1240 -0
  335. package/artemis-main/artemis/tools/committee_prompts.json +5 -0
  336. package/artemis-main/artemis/tools/committee_tool.py +422 -0
  337. package/artemis-main/artemis/tools/diagnoser_submit_answer_tool.py +87 -0
  338. package/artemis-main/artemis/tools/diagnostic_tool.py +120 -0
  339. package/artemis-main/artemis/tools/explorer_tool.py +581 -0
  340. package/artemis-main/artemis/tools/history/__init__.py +398 -0
  341. package/artemis-main/artemis/tools/history/replay.py +120 -0
  342. package/artemis-main/artemis/tools/history/screenshot.py +139 -0
  343. package/artemis-main/artemis/tools/history/search.py +563 -0
  344. package/artemis-main/artemis/tools/image_processor_tool.py +124 -0
  345. package/artemis-main/artemis/tools/index.py +47 -0
  346. package/artemis-main/artemis/tools/log_tool.py +108 -0
  347. package/artemis-main/artemis/tools/mobile/launch_app.py +132 -0
  348. package/artemis-main/artemis/tools/mobile/log_utils.py +138 -0
  349. package/artemis-main/artemis/tools/mobile/ocr.py +202 -0
  350. package/artemis-main/artemis/tools/mobile/read_hierarchy.py +108 -0
  351. package/artemis-main/artemis/tools/mobile/read_logs.py +116 -0
  352. package/artemis-main/artemis/tools/mobile/search_logs.py +260 -0
  353. package/artemis-main/artemis/tools/object_detection_tool.py +244 -0
  354. package/artemis-main/artemis/tools/scratchpad.py +819 -0
  355. package/artemis-main/artemis/tools/system/adb_shell.py +31 -0
  356. package/artemis-main/artemis/tools/tool_wrapper.py +270 -0
  357. package/artemis-main/artemis/tools/video_tool.py +261 -0
  358. package/artemis-main/artemis/tools/wait_tool.py +97 -0
  359. package/artemis-main/artemis/utils/app_launch_utils.py +207 -0
  360. package/artemis-main/artemis/utils/coordinates.py +606 -0
  361. package/artemis-main/artemis/utils/credentials_validator.py +201 -0
  362. package/artemis-main/artemis/utils/cv_canvas.py +189 -0
  363. package/artemis-main/artemis/utils/cython_compat.py +45 -0
  364. package/artemis-main/artemis/utils/element_hit_test.py +80 -0
  365. package/artemis-main/artemis/utils/image_diff.py +335 -0
  366. package/artemis-main/artemis/utils/image_hash.py +80 -0
  367. package/artemis-main/artemis/utils/notes.py +470 -0
  368. package/artemis-main/artemis/utils/ocr_api.py +105 -0
  369. package/artemis-main/artemis/utils/ocr_xml_fusion.py +318 -0
  370. package/artemis-main/artemis/utils/plan_grammar.py +513 -0
  371. package/artemis-main/artemis/utils/python_executor.py +213 -0
  372. package/artemis-main/artemis/utils/startup_progress.py +60 -0
  373. package/artemis-main/artemis/utils/task_tree.py +1174 -0
  374. package/artemis-main/artemis/utils/text.py +37 -0
  375. package/artemis-main/artemis/utils/ui_filter.py +651 -0
  376. package/artemis-main/artemis/utils/video.py +685 -0
  377. package/artemis-main/artemis/utils/visualization.py +802 -0
  378. package/artemis-main/config/artemis.example.jsonc +12 -0
  379. package/artemis-main/mcp_server/README.md +111 -0
  380. package/artemis-main/mcp_server/__init__.py +21 -0
  381. package/artemis-main/mcp_server/__main__.py +20 -0
  382. package/artemis-main/mcp_server/background/__init__.py +15 -0
  383. package/artemis-main/mcp_server/background/task_runner.py +453 -0
  384. package/artemis-main/mcp_server/base.py +49 -0
  385. package/artemis-main/mcp_server/notifiers/__init__.py +74 -0
  386. package/artemis-main/mcp_server/notifiers/agentapi.py +289 -0
  387. package/artemis-main/mcp_server/notifiers/base.py +45 -0
  388. package/artemis-main/mcp_server/notifiers/composite.py +87 -0
  389. package/artemis-main/mcp_server/notifiers/desktop.py +108 -0
  390. package/artemis-main/mcp_server/notifiers/file.py +69 -0
  391. package/artemis-main/mcp_server/notifiers/script.py +90 -0
  392. package/artemis-main/mcp_server/notifiers/webhook.py +87 -0
  393. package/artemis-main/mcp_server/rules.md +68 -0
  394. package/artemis-main/mcp_server/server.py +106 -0
  395. package/artemis-main/mcp_server/tools/__init__.py +29 -0
  396. package/artemis-main/mcp_server/tools/device_state.py +107 -0
  397. package/artemis-main/mcp_server/tools/diagnose.py +1249 -0
  398. package/artemis-main/mcp_server/tools/inspect_trace.py +394 -0
  399. package/artemis-main/mcp_server/tools/task_manager.py +627 -0
  400. package/artemis-main/mcp_server/tools/task_runner.py +548 -0
  401. package/artemis-main/mcp_server/utils/__init__.py +15 -0
  402. package/artemis-main/mcp_server/utils/device_utils.py +180 -0
  403. package/artemis-main/mcp_server/utils/env_utils.py +65 -0
  404. package/artemis-main/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk +0 -0
  405. package/artemis-main/packages/artemis-accessibility-helper/app/build.gradle.kts +35 -0
  406. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml +49 -0
  407. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java +274 -0
  408. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ArtemisAccessibilityService.java +227 -0
  409. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java +463 -0
  410. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/DisplayUtils.java +111 -0
  411. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java +232 -0
  412. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/HierarchyDumper.java +832 -0
  413. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenReceiver.java +36 -0
  414. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenStore.java +38 -0
  415. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/XmlUtils.java +102 -0
  416. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/values/strings.xml +4 -0
  417. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/xml/accessibility_service_config.xml +10 -0
  418. package/artemis-main/packages/artemis-accessibility-helper/build.gradle.kts +3 -0
  419. package/artemis-main/packages/artemis-accessibility-helper/build_apk.sh +215 -0
  420. package/artemis-main/packages/artemis-accessibility-helper/debug.keystore +0 -0
  421. package/artemis-main/packages/artemis-accessibility-helper/gradle.properties +2 -0
  422. package/artemis-main/packages/artemis-accessibility-helper/helper_manifest.json +7 -0
  423. package/artemis-main/packages/artemis-accessibility-helper/settings.gradle.kts +17 -0
  424. package/artemis-main/packages/artemis-client/DESIGN.md +89 -0
  425. package/artemis-main/packages/artemis-client/README.md +112 -0
  426. package/artemis-main/packages/artemis-client/examples/basic.py +30 -0
  427. package/artemis-main/packages/artemis-client/pyproject.toml +36 -0
  428. package/artemis-main/packages/artemis-client/src/artemis_client/__init__.py +45 -0
  429. package/artemis-main/packages/artemis-client/src/artemis_client/client.py +401 -0
  430. package/artemis-main/packages/artemis-client/src/artemis_client/errors.py +66 -0
  431. package/artemis-main/packages/artemis-client/src/artemis_client/models.py +212 -0
  432. package/artemis-main/packages/artemis-client/src/artemis_client/py.typed +1 -0
  433. package/artemis-main/packages/artemis-client/src/artemis_client/transport.py +134 -0
  434. package/artemis-main/packages/artemis-client/tests/test_client.py +289 -0
  435. package/artemis-main/packages/artemis-client/tests/test_transport.py +85 -0
  436. package/artemis-main/pyproject.toml +259 -0
  437. package/artemis-main/pyright-core.json +17 -0
  438. package/artemis-main/scripts/check_dependency_sources.py +68 -0
  439. package/artemis-main/scripts/install_deps.ps1 +520 -0
  440. package/artemis-main/scripts/install_deps.sh +533 -0
  441. package/artemis-main/scripts/quality_ratchet.py +97 -0
  442. package/artemis-main/scripts/real_device_video_acceptance.py +483 -0
  443. package/artemis-main/scripts/run_edge_case_part.py +327 -0
  444. package/artemis-main/scripts/run_ui.py +242 -0
  445. package/artemis-main/scripts/start.ps1 +500 -0
  446. package/artemis-main/scripts/test_all_edge_cases.py +269 -0
  447. package/artemis-main/scripts/verify_concurrency_and_devices.py +385 -0
  448. package/artemis-main/setup.py +164 -0
  449. package/artemis-main/start.bat +35 -0
  450. package/artemis-main/start.sh +423 -0
  451. package/artemis-main/third_party/mobile_use/LICENSE +190 -0
  452. package/artemis-main/third_party/mobile_use/METADATA +27 -0
  453. package/artemis-main/third_party/mobile_use/NOTICE +12 -0
  454. package/artemis-main/third_party/mobile_use/__init__.py +19 -0
  455. package/artemis-main/third_party/mobile_use/agents/__init__.py +13 -0
  456. package/artemis-main/third_party/mobile_use/agents/hopper/__init__.py +13 -0
  457. package/artemis-main/third_party/mobile_use/agents/hopper/hopper.md +22 -0
  458. package/artemis-main/third_party/mobile_use/agents/hopper/hopper.py +80 -0
  459. package/artemis-main/third_party/mobile_use/clients/__init__.py +13 -0
  460. package/artemis-main/third_party/mobile_use/clients/ui_automator_client.py +482 -0
  461. package/artemis-main/third_party/mobile_use/config/__init__.py +13 -0
  462. package/artemis-main/third_party/mobile_use/config/llm.py +183 -0
  463. package/artemis-main/third_party/mobile_use/config/output.py +184 -0
  464. package/artemis-main/third_party/mobile_use/context.py +119 -0
  465. package/artemis-main/third_party/mobile_use/controllers/__init__.py +13 -0
  466. package/artemis-main/third_party/mobile_use/controllers/platform_specific_commands_controller.py +151 -0
  467. package/artemis-main/third_party/mobile_use/controllers/types.py +92 -0
  468. package/artemis-main/third_party/mobile_use/controllers/unified_controller.py +183 -0
  469. package/artemis-main/third_party/mobile_use/graph.py +54 -0
  470. package/artemis-main/third_party/mobile_use/llm_service.py +193 -0
  471. package/artemis-main/third_party/mobile_use/main.py +130 -0
  472. package/artemis-main/third_party/mobile_use/sdk/__init__.py +13 -0
  473. package/artemis-main/third_party/mobile_use/sdk/agent.py +1060 -0
  474. package/artemis-main/third_party/mobile_use/sdk/builders/__init__.py +13 -0
  475. package/artemis-main/third_party/mobile_use/sdk/builders/agent_config_builder.py +225 -0
  476. package/artemis-main/third_party/mobile_use/sdk/builders/index.py +33 -0
  477. package/artemis-main/third_party/mobile_use/sdk/builders/task_request_builder.py +243 -0
  478. package/artemis-main/third_party/mobile_use/sdk/types/__init__.py +13 -0
  479. package/artemis-main/third_party/mobile_use/sdk/types/agent.py +98 -0
  480. package/artemis-main/third_party/mobile_use/sdk/types/exceptions.py +109 -0
  481. package/artemis-main/third_party/mobile_use/sdk/types/task.py +234 -0
  482. package/artemis-main/third_party/mobile_use/testing.py +59 -0
  483. package/artemis-main/third_party/mobile_use/tools/__init__.py +13 -0
  484. package/artemis-main/third_party/mobile_use/tools/index.py +44 -0
  485. package/artemis-main/third_party/mobile_use/tools/mobile/__init__.py +13 -0
  486. package/artemis-main/third_party/mobile_use/tools/mobile/launch_app.py +89 -0
  487. package/artemis-main/third_party/mobile_use/tools/tool_wrapper.py +52 -0
  488. package/artemis-main/third_party/mobile_use/utils/__init__.py +13 -0
  489. package/artemis-main/third_party/mobile_use/utils/app_launch_utils.py +217 -0
  490. package/artemis-main/third_party/mobile_use/utils/cli_helpers.py +60 -0
  491. package/artemis-main/third_party/mobile_use/utils/decorators.py +158 -0
  492. package/artemis-main/third_party/mobile_use/utils/file.py +31 -0
  493. package/artemis-main/third_party/mobile_use/utils/logger.py +216 -0
  494. package/artemis-main/third_party/mobile_use/utils/media.py +213 -0
  495. package/artemis-main/third_party/mobile_use/utils/shell_utils.py +39 -0
  496. package/artemis-main/third_party/mobile_use/utils/video.py +398 -0
  497. package/artemis-main/uv.lock +4908 -0
  498. package/bin/android-midscene-automation.js +1 -5
  499. package/bin/artemis-config.mjs +15 -0
  500. package/bin/artemis-embedded.py +56 -0
  501. package/bin/start-artemis.mjs +30 -0
  502. package/bin/start-dev.mjs +112 -0
  503. package/bin/test_artemis_embedded.py +44 -0
  504. package/package.json +4 -2
  505. package/server/artemis.ts +34 -0
  506. package/server/http-api.ts +12 -0
  507. package/src/App.vue +10 -5
  508. package/src/api.ts +5 -0
  509. package/src/appium-recorder/AppiumPage.vue +1 -1
  510. package/src/appium-recorder/components/ComponentInspector.vue +1 -1
  511. package/src/appium-recorder/components/FlowNodeSearch.vue +3 -3
  512. package/src/appium-recorder/components/ImageCheckDialog.vue +3 -3
  513. package/src/appium-recorder/components/RecorderWorkspace.vue +4 -4
  514. package/src/appium-recorder/flow-appearance.ts +4 -4
  515. package/src/components/AppSidebar.vue +8 -7
  516. package/src/components/MidsceneMaintenanceNotice.vue +17 -0
  517. package/src/components/analytics/AnalysisCharts.vue +4 -4
  518. package/src/components/analytics/ReportAnalysis.vue +1 -1
  519. package/src/components/generator/TestCaseFileUpload.vue +1 -1
  520. package/src/components/help/HelpCenter.vue +56 -37
  521. package/src/pages/AiGeneratorPage.vue +1 -1
  522. package/src/pages/ArtemisPage.vue +72 -0
  523. package/src/pages/ConfigPage.vue +18 -19
  524. package/src/style.css +340 -121
  525. package/src/types.ts +1 -1
@@ -0,0 +1,5 @@
1
+ {
2
+ "planner_avatar": "You are the **Planner Avatar** (Meeting Host) in a committee resolving a task plan failure. Your role is to lead the meeting and advocate for a robust and correct high-level plan. You are aware of:\n- **History Analyzer**: The trajectory specialist who can analyze execution history and read notes. They have tools like `search_history`, `replay_steps`, `get_step_screenshot` and `read_note`.\n- **Diagnoser**: The forensics specialist who can analyze logs and videos to find why things failed. They have tools like `video_analyzer` and `analyze_logs`.\nYour master (Master Planner) has given you a specific directive for this meeting, which is at the top of the blackboard.\n**Your Task**:\n1. In the first round, you must open the meeting, state the problem based on the master's directive, and call on the other experts (Diagnoser or History Analyzer) to provide their analysis.\n2. In subsequent rounds, evaluate their suggestions and push for a concrete, revised plan of subgoals.\n3. In the final round, summarize the discussion and output the final recommended plan revisions and actions.",
3
+ "history_analyzer_expert": "You are the **History Analyzer** (Trajectory Specialist) in a committee resolving a task failure. Your role is to analyze the execution history of the current session, understand what has been tried, what succeeded, and what failed, and check persistent notes. You are aware of:\n- **Planner Avatar**: The meeting host who needs a concrete plan.\n- **Diagnoser**: The forensics specialist who analyzes logs and videos.\nYou have tools to search the execution history (`search_history`), replay steps in full (`replay_steps`), look at a step's stored screenshot (`get_step_screenshot`) and read notes. Wait for the Planner Avatar to call on you or respond to the current state of the blackboard. Be concise.",
4
+ "diagnoser_expert": "You are the **Diagnoser** (Forensics Specialist) in a committee resolving a UI automation failure. Your role is to analyze what went wrong in the past, look for error patterns in logs, and understand system state transitions. You are aware of:\n- **Planner Avatar**: The meeting host who needs a concrete plan.\n- **History Analyzer**: The trajectory specialist who analyzes history and notes.\nYou have tools to analyze video and logs. Wait for the Planner Avatar to call on you or respond to the current state of the blackboard. Be concise."
5
+ }
@@ -0,0 +1,422 @@
1
+ # pylint: disable=too-many-statements
2
+ # Copyright 2026 Google LLC
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ """Committee tool for multi-agent deliberation and failure recovery."""
17
+
18
+ import asyncio
19
+ import base64
20
+ import json
21
+ from pathlib import Path
22
+ from typing import Any
23
+ from uuid import uuid4
24
+
25
+ from langchain_core.messages import HumanMessage, SystemMessage
26
+ from langchain_core.tools import BaseTool
27
+ from pydantic import BaseModel, Field
28
+
29
+ from artemis.core.tool_failure import ToolFailure, is_tool_failure
30
+ from artemis.context import ArtemisContext
31
+ from artemis.data_engine.trace import (
32
+ CURRENT_TRACE_ID,
33
+ TraceSpan,
34
+ trace_langchain_tool,
35
+ )
36
+ from artemis.drivers.base import BaseDeviceDriver
37
+ from artemis.graph.state import State
38
+ from artemis.services.llm import get_llm, invoke_llm_with_timeout_message
39
+ from artemis.tools.base import ArtemisTool, ToolCategory
40
+ from artemis.tools.history import get_history_tools
41
+ from artemis.tools.log_tool import get_analyze_logs_tool
42
+ from artemis.tools.scratchpad import get_list_notes_tool, get_read_note_tool
43
+ from artemis.tools.tool_wrapper import (
44
+ ToolWrapper,
45
+ get_tool_result_content,
46
+ invoke_tool_with_injection,
47
+ tool_result_messages,
48
+ )
49
+ from artemis.utils.cython_compat import CyFunctionDetector
50
+ from artemis.tools.video_tool import get_video_analyzer_tool
51
+ from artemis.utils.notes import get_note_file_path, get_notes_dir
52
+ from artemis.memory.context_policy import build_history_for
53
+ from artemis.utils.task_tree import (
54
+ get_active_subgoal_hashes,
55
+ get_recent_subgoal_hashes,
56
+ )
57
+ from third_party.mobile_use.utils.logger import get_logger
58
+
59
+ logger = get_logger(__name__)
60
+
61
+
62
+ class AskCommitteeArgs(BaseModel):
63
+ """Arguments schema for invoking the committee tool."""
64
+
65
+ model_config = {"ignored_types": (CyFunctionDetector,)}
66
+ avatar_directive: str = Field(
67
+ ...,
68
+ description="Directive to frame and steer the debate for your avatar.",
69
+ )
70
+
71
+
72
+ ASK_COMMITTEE_DOCSTRING = (
73
+ "[COMMITTEE] Call this tool to summon a committee of sub-agents to debate"
74
+ " and resolve complex failures.\n\n"
75
+ "Your avatar will host the meeting and drive it based on your directive.\n"
76
+ "The result will be a synthesized recommendation."
77
+ )
78
+
79
+
80
+ class AskCommitteeTool(ArtemisTool):
81
+ """Universal tool for summoning a committee of sub-agents to debate and resolve failures."""
82
+
83
+ def __init__(self, category: ToolCategory = "custom"):
84
+ super().__init__(
85
+ name="ask_committee",
86
+ description=ASK_COMMITTEE_DOCSTRING,
87
+ args_schema=AskCommitteeArgs,
88
+ category=category,
89
+ )
90
+
91
+ # pylint: disable=too-many-arguments,too-many-positional-arguments
92
+ async def execute(
93
+ self,
94
+ driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
95
+ ctx: ArtemisContext | None = None,
96
+ avatar_directive: str = "",
97
+ tool_call_id: str | None = None, # pylint: disable=unused-argument
98
+ state: State | None = None,
99
+ **kwargs: Any,
100
+ ) -> str:
101
+ directive = avatar_directive or kwargs.get("AvatarDirective") or ""
102
+ return await _run_committee_logic(
103
+ ctx=ctx,
104
+ state=state,
105
+ avatar_directive=directive,
106
+ )
107
+
108
+
109
+ # Universal tool instance & aliases
110
+ ask_committee = AskCommitteeTool()
111
+ AskCommittee = AskCommitteeTool
112
+ AskCommitteeToolAlias = AskCommitteeTool
113
+
114
+
115
+ def get_ask_committee_tool(ctx: ArtemisContext) -> BaseTool:
116
+ """Exports ask_committee as a LangChain BaseTool."""
117
+ return trace_langchain_tool(ask_committee.to_langchain_tool(ctx), ctx)
118
+
119
+
120
+ # pylint: disable=too-many-branches,too-many-locals,too-many-statements
121
+ async def _execute_committee(
122
+ ctx: ArtemisContext,
123
+ state: State | None,
124
+ avatar_directive: str,
125
+ ) -> str:
126
+ """Executes committee debate loop and returns the final conclusion."""
127
+ with TraceSpan(name="committee", trace_type="agent", ctx=ctx):
128
+ logger.info(f"Committee invoked with directive: {avatar_directive}")
129
+
130
+ # 1. Access or create blackboard file
131
+ if ctx.data_engine:
132
+ notes_dir = get_notes_dir(ctx.data_engine.base_dir)
133
+ else:
134
+ notes_dir = Path(".") / "notes"
135
+ notes_dir.mkdir(parents=True, exist_ok=True)
136
+
137
+ # Load latest screenshot as base64 to provide multimodal vision to the committee
138
+ screenshot_b64 = None
139
+ screenshot_path = getattr(state, "latest_screenshot", None) if state is not None else None
140
+ if screenshot_path and Path(screenshot_path).exists():
141
+ try:
142
+ with open(screenshot_path, "rb") as f:
143
+ screenshot_b64 = base64.b64encode(f.read()).decode("utf-8")
144
+ logger.info(f"Committee successfully loaded screenshot from: {screenshot_path}")
145
+ except Exception as e: # pylint: disable=broad-exception-caught
146
+ logger.error(f"Failed to read screenshot in committee: {e}")
147
+
148
+ trace_id = CURRENT_TRACE_ID.get() or str(uuid4())
149
+ blackboard_path = notes_dir / f"blackboard_{trace_id}.md"
150
+
151
+ # Initialize blackboard with goal and directive
152
+ initial_goal = (
153
+ getattr(state, "initial_goal", "Not specified.")
154
+ if state is not None
155
+ else "Not specified."
156
+ )
157
+ blackboard_content = (
158
+ "# Committee Discussion Blackboard\n\n"
159
+ f"## User's Initial Goal\n{initial_goal}\n\n"
160
+ f"## Master's Directive\n{avatar_directive}\n\n"
161
+ )
162
+ blackboard_path.write_text(blackboard_content, encoding="utf-8")
163
+
164
+ # 2. Extract history
165
+ history_summary = "## Previous Context\n"
166
+ if ctx.data_engine:
167
+ steps = ctx.data_engine.get_agent_friendly_steps()
168
+ if steps:
169
+ active_subgoal_hash = "default"
170
+ if ctx.data_engine:
171
+ task_plan_path = get_note_file_path(ctx.data_engine.base_dir, "task_plan")
172
+ else:
173
+ task_plan_path = notes_dir / "task_plan.md"
174
+ task_plan = "No task plan yet."
175
+ keep_hashes = None
176
+ if task_plan_path.exists():
177
+ try:
178
+ task_plan = task_plan_path.read_text(encoding="utf-8")
179
+
180
+ active_subgoal_hash, _ = get_active_subgoal_hashes(task_plan)
181
+ keep_hashes = get_recent_subgoal_hashes(
182
+ steps,
183
+ active_subgoal_hash,
184
+ ctx.data_engine.base_dir,
185
+ )
186
+ except Exception as e: # pylint: disable=broad-exception-caught
187
+ logger.error(f"Failed to parse active subgoal in committee: {e}")
188
+
189
+ try:
190
+ plan_and_history = build_history_for(
191
+ "committee",
192
+ task_plan,
193
+ steps,
194
+ active_subgoal_hash,
195
+ keep_subgoal_hashes=keep_hashes,
196
+ engine=ctx.data_engine,
197
+ )
198
+ history_summary += plan_and_history
199
+ except Exception as e: # pylint: disable=broad-exception-caught
200
+ logger.error(f"Failed to build plan and history in committee: {e}")
201
+ history_summary += f"Error building plan and history: {e}"
202
+
203
+ # Read failed plans history to ensure the committee avoids
204
+ # repeating failed strategies
205
+ failed_plans_history = ""
206
+ failed_history_path = notes_dir / "failed_plans_history.md"
207
+ if failed_history_path.exists():
208
+ try:
209
+ failed_plans_history = failed_history_path.read_text(encoding="utf-8")
210
+ except Exception as e: # pylint: disable=broad-exception-caught
211
+ logger.error(f"Failed to read failed plans history in committee: {e}")
212
+ if failed_plans_history:
213
+ history_summary += (
214
+ f"\n\n### Previous Failed Plan Attempts\n{failed_plans_history}\n"
215
+ )
216
+
217
+ with open(blackboard_path, "a", encoding="utf-8") as f:
218
+ f.write(history_summary + "\n\n")
219
+
220
+ # 3. Load prompts from JSON
221
+ prompts = {}
222
+ prompts_path = Path(__file__).parent / "committee_prompts.json"
223
+ if prompts_path.exists():
224
+ try:
225
+ prompts = json.loads(prompts_path.read_text(encoding="utf-8"))
226
+ except Exception as e: # pylint: disable=broad-exception-caught
227
+ logger.error(f"Failed to load committee prompts: {e}")
228
+
229
+ planner_prompt = prompts.get("planner_avatar", "You are the Planner Avatar.")
230
+ history_analyzer_prompt = prompts.get(
231
+ "history_analyzer_expert", "You are the History Analyzer."
232
+ )
233
+ diagnoser_prompt = prompts.get("diagnoser_expert", "You are the Diagnoser Expert.")
234
+
235
+ # 4. Prepare Specialized Tools for Members
236
+
237
+ # History Analyzer Tools: the shared history tools plus the note readers
238
+ read_note_tool_hist = get_read_note_tool(ctx)
239
+ list_notes_tool_hist = get_list_notes_tool(ctx)
240
+
241
+ history_analyzer_tools = [
242
+ *get_history_tools(ctx),
243
+ trace_langchain_tool(read_note_tool_hist, ctx),
244
+ trace_langchain_tool(list_notes_tool_hist, ctx),
245
+ ]
246
+
247
+ # Diagnoser Tools
248
+ video_tool = get_video_analyzer_tool(ctx, role="diagnoser")
249
+ log_tool = get_analyze_logs_tool(ctx)
250
+ read_note_tool = get_read_note_tool(ctx)
251
+ list_notes_tool = get_list_notes_tool(ctx)
252
+
253
+ diagnoser_tools = [
254
+ trace_langchain_tool(video_tool, ctx),
255
+ trace_langchain_tool(log_tool, ctx),
256
+ trace_langchain_tool(read_note_tool, ctx),
257
+ trace_langchain_tool(list_notes_tool, ctx),
258
+ ]
259
+
260
+ # 5. Instantiate LLMs
261
+ llm_pl = get_llm(ctx=ctx, name="planner_avatar")
262
+
263
+ llm_hist = get_llm(ctx=ctx, name="history_analyzer_expert")
264
+ llm_hist = llm_hist.bind_tools(tools=history_analyzer_tools)
265
+
266
+ llm_diag = get_llm(ctx=ctx, name="diagnoser_expert")
267
+ llm_diag = llm_diag.bind_tools(tools=diagnoser_tools)
268
+
269
+ # 6. Debate Loop (configured rounds)
270
+ exec_setup = getattr(ctx, "execution_setup", None)
271
+ rounds = getattr(exec_setup, "committee_debate_rounds", 2) if exec_setup else 2
272
+
273
+ async def run_agent_turn(llm, system_prompt, tools, agent_name):
274
+ current_blackboard = blackboard_path.read_text(encoding="utf-8")
275
+
276
+ human_content = [
277
+ {
278
+ "type": "text",
279
+ "text": f"Current Blackboard:\n{current_blackboard}",
280
+ }
281
+ ]
282
+ if screenshot_b64:
283
+ human_content.append(
284
+ {
285
+ "type": "image_url",
286
+ "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"},
287
+ }
288
+ )
289
+
290
+ messages = [
291
+ SystemMessage(content=system_prompt),
292
+ HumanMessage(content=human_content),
293
+ ]
294
+
295
+ max_iterations = 3
296
+ for _ in range(max_iterations):
297
+ response = await invoke_llm_with_timeout_message(llm.ainvoke(messages))
298
+
299
+ if not response.tool_calls:
300
+ return response.content
301
+
302
+ messages.append(response)
303
+
304
+ async def run_tool(tc):
305
+ """Keep the raw result so the caller can read ToolMessage.status."""
306
+ tool_name = tc["name"]
307
+ logger.info(f"{agent_name} requested tool: {tool_name}")
308
+ tool_to_run = next((t for t in tools if t.name == tool_name), None)
309
+ if tool_to_run:
310
+ try:
311
+ args = dict(tc["args"])
312
+ return await invoke_tool_with_injection(
313
+ tool=tool_to_run,
314
+ args=args,
315
+ tool_call_id=tc["id"],
316
+ state=state,
317
+ )
318
+ except Exception as e: # pylint: disable=broad-exception-caught
319
+ return ToolFailure(f"Error: {e}")
320
+ else:
321
+ return ToolFailure(f"Error: Tool {tool_name} not found")
322
+
323
+ active_tool_calls = [
324
+ tc for tc in response.tool_calls if tc["name"] != "google_search"
325
+ ]
326
+ tool_outputs = await asyncio.gather(*(run_tool(tc) for tc in active_tool_calls))
327
+
328
+ for tc, raw_result in zip(active_tool_calls, tool_outputs):
329
+ # Read status before unwrapping ToolMessage.content.
330
+ status = "error" if is_tool_failure(raw_result) else "success"
331
+ messages.extend(
332
+ tool_result_messages(
333
+ tc["id"],
334
+ get_tool_result_content(raw_result),
335
+ name=tc["name"],
336
+ status=status,
337
+ llm=llm,
338
+ )
339
+ )
340
+ return ToolFailure("Error: Reached max iterations in turn.")
341
+
342
+ for r in range(1, rounds + 1):
343
+ logger.info(f"Committee Round {r}")
344
+
345
+ # 1. Planner Avatar speaks first (Host)
346
+ pl_response = await run_agent_turn(llm_pl, planner_prompt, [], "Planner Avatar")
347
+ with open(blackboard_path, "a", encoding="utf-8") as f:
348
+ f.write(f"### Planner Avatar (Round {r})\n{pl_response}\n\n")
349
+
350
+ # 2. Diagnoser speaks (Can use tools)
351
+ diag_response = await run_agent_turn(
352
+ llm_diag, diagnoser_prompt, diagnoser_tools, "Diagnoser"
353
+ )
354
+ with open(blackboard_path, "a", encoding="utf-8") as f:
355
+ f.write(f"### Diagnoser (Round {r})\n{diag_response}\n\n")
356
+
357
+ # 3. History Analyzer speaks (Can use tools)
358
+ hist_response = await run_agent_turn(
359
+ llm_hist,
360
+ history_analyzer_prompt,
361
+ history_analyzer_tools,
362
+ "History Analyzer",
363
+ )
364
+ with open(blackboard_path, "a", encoding="utf-8") as f:
365
+ f.write(f"### History Analyzer (Round {r})\n{hist_response}\n\n")
366
+
367
+ # Final Synthesis by Planner Avatar
368
+ current_blackboard = blackboard_path.read_text(encoding="utf-8")
369
+
370
+ final_human_content = [
371
+ {
372
+ "type": "text",
373
+ "text": f"Current Blackboard:\n{current_blackboard}",
374
+ }
375
+ ]
376
+ if screenshot_b64:
377
+ final_human_content.append(
378
+ {
379
+ "type": "image_url",
380
+ "image_url": {"url": f"data:image/jpeg;base64,{screenshot_b64}"},
381
+ }
382
+ )
383
+
384
+ msg_final = [
385
+ SystemMessage(
386
+ content=(
387
+ "You are the Planner Avatar. Summarize the discussion"
388
+ " and output the final recommended plan revisions and"
389
+ " action plan based on your master's directive."
390
+ )
391
+ ),
392
+ HumanMessage(content=final_human_content),
393
+ ]
394
+ resp_final = await invoke_llm_with_timeout_message(llm_pl.ainvoke(msg_final))
395
+
396
+ with open(blackboard_path, "a", encoding="utf-8") as f:
397
+ f.write(f"## Final Conclusion\n{resp_final.content}\n")
398
+
399
+ return resp_final.content
400
+
401
+
402
+ async def _run_committee_logic(
403
+ ctx: ArtemisContext | None,
404
+ state: State | None,
405
+ avatar_directive: str,
406
+ ) -> str:
407
+ """Executes multi-agent committee debate logic."""
408
+ if ctx is None:
409
+ return ToolFailure("Error: ArtemisContext is required for ask_committee.")
410
+
411
+ try:
412
+ return await asyncio.wait_for(_execute_committee(ctx, state, avatar_directive), timeout=300)
413
+ except TimeoutError:
414
+ logger.error("Committee timed out after 300 seconds.")
415
+ return ToolFailure("Error: Committee timed out after 300 seconds.")
416
+
417
+
418
+ ask_committee_wrapper = ToolWrapper(
419
+ tool_fn_getter=get_ask_committee_tool,
420
+ on_success_fn=lambda output: f"Committee concluded:\n{output}",
421
+ on_failure_fn=lambda error: f"Committee failed: {error}",
422
+ )
@@ -0,0 +1,87 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Universal tool for submitting final diagnosis and actionable recovery advice."""
16
+
17
+ from typing import Any
18
+
19
+ from langchain_core.tools import BaseTool
20
+ from pydantic import BaseModel, Field
21
+
22
+ from artemis.context import ArtemisContext
23
+ from artemis.data_engine.trace import trace_langchain_tool
24
+ from artemis.drivers.base import BaseDeviceDriver
25
+ from artemis.tools.base import ArtemisTool
26
+ from third_party.mobile_use.utils.logger import get_logger
27
+
28
+ logger = get_logger(__name__)
29
+
30
+
31
+ # pylint: disable=too-few-public-methods
32
+ class SubmitAnswerArgs(BaseModel):
33
+ """Arguments schema for submitting final diagnoser answer."""
34
+
35
+ analysis: str = Field(
36
+ ...,
37
+ description=(
38
+ "A detailed and concise explanation of the root cause of the"
39
+ " failure or query. Answer in one full paragraph."
40
+ ),
41
+ )
42
+ actionable_steps: list[str] = Field(
43
+ default_factory=list,
44
+ description=(
45
+ "A list of concrete recovery instructions or next steps. Each"
46
+ " step is a short sentence. If no actionable steps are"
47
+ " available, return an empty list."
48
+ ),
49
+ )
50
+
51
+
52
+ class SubmitAnswerTool(ArtemisTool):
53
+ """Universal tool for submitting final diagnosis and actionable recovery advice."""
54
+
55
+ def __init__(self):
56
+ super().__init__(
57
+ name="submit_answer",
58
+ description=(
59
+ "[Finalization] Submits the final diagnosis and actionable recovery advice. "
60
+ "Call this tool when you have pinpointed the root cause."
61
+ ),
62
+ args_schema=SubmitAnswerArgs,
63
+ category="custom",
64
+ )
65
+
66
+ # pylint: disable=too-many-arguments,too-many-positional-arguments
67
+ async def execute(
68
+ self,
69
+ driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
70
+ ctx: ArtemisContext | None = None, # pylint: disable=unused-argument
71
+ analysis: str | None = None, # pylint: disable=unused-argument
72
+ actionable_steps: list[str] | None = None, # pylint: disable=unused-argument
73
+ **kwargs: Any,
74
+ ) -> str:
75
+ logger.info("Diagnoser submitted answer via tool.")
76
+ return "Answer submitted successfully."
77
+
78
+
79
+ # Universal tool instance & aliases
80
+ submit_answer = SubmitAnswerTool()
81
+ SubmitAnswer = SubmitAnswerTool
82
+ DiagnoserSubmitAnswerTool = SubmitAnswerTool
83
+
84
+
85
+ def get_submit_answer_tool(ctx: ArtemisContext) -> BaseTool:
86
+ """Exports submit_answer as a LangChain BaseTool."""
87
+ return trace_langchain_tool(submit_answer.to_langchain_tool(ctx), ctx)
@@ -0,0 +1,120 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Universal diagnostic agent tool for deep system logs and video inspection."""
16
+
17
+ from typing import Any
18
+
19
+ from langchain_core.tools import BaseTool
20
+ from pydantic import BaseModel, Field
21
+
22
+ from artemis.core.tool_failure import ToolFailure
23
+ from artemis.agents.diagnoser.diagnoser import Diagnoser
24
+ from artemis.context import ArtemisContext
25
+ from artemis.data_engine.trace import trace_langchain_tool
26
+ from artemis.drivers.base import BaseDeviceDriver
27
+ from artemis.graph.state import State
28
+ from artemis.tools.base import ArtemisTool
29
+ from artemis.tools.tool_wrapper import ToolWrapper
30
+ from artemis.utils.cython_compat import CyFunctionDetector
31
+ from third_party.mobile_use.utils.logger import get_logger
32
+
33
+ logger = get_logger(__name__)
34
+
35
+
36
+ # pylint: disable=too-few-public-methods
37
+ class AskDiagnoserArgs(BaseModel):
38
+ """Arguments schema for invoking the diagnostic subagent."""
39
+
40
+ model_config = {"ignored_types": (CyFunctionDetector,)}
41
+ query: str = Field(
42
+ ...,
43
+ description=(
44
+ "The diagnostic query or what you want to investigate. Please"
45
+ " provide a specific, narrow time range if applicable."
46
+ ),
47
+ )
48
+
49
+
50
+ async def _run_diagnoser_logic(
51
+ ctx: ArtemisContext | None,
52
+ state: State | None,
53
+ query: str,
54
+ ) -> str:
55
+ logger.info(f"ask_diagnoser tool called with query: {query}")
56
+ if ctx is None:
57
+ return ToolFailure("Error: ArtemisContext is required for Diagnoser.")
58
+ agent = Diagnoser(ctx)
59
+ try:
60
+ result = await agent.run(query, state)
61
+ return f"Diagnoser replied:\n{result}"
62
+ except Exception as e: # pylint: disable=broad-exception-caught
63
+ logger.error(f"Failed to run diagnoser: {e}")
64
+ return f"Diagnoser failed: {e}"
65
+
66
+
67
+ class AskDiagnoserTool(ArtemisTool):
68
+ """Universal tool for invoking the Diagnoser subagent to inspect logs and videos."""
69
+
70
+ def __init__(self):
71
+ super().__init__(
72
+ name="ask_diagnoser",
73
+ description=(
74
+ "[DIAGNOSTIC] Analyzes system logs (ADB) and video recordings to "
75
+ "pinpoint the root cause of a step failure, screen freeze, or "
76
+ "unexpected UI drift. Use it when the live screen does not explain "
77
+ "why an action is not taking effect or why progress is stalled, "
78
+ "before committing to further physical actions. Note: When querying "
79
+ "video files, specify a narrow timeframe (e.g., '[10s, 20s]') for "
80
+ "faster analysis."
81
+ ),
82
+ args_schema=AskDiagnoserArgs,
83
+ category="custom",
84
+ )
85
+
86
+ # pylint: disable=too-many-arguments,too-many-positional-arguments
87
+ async def execute(
88
+ self,
89
+ driver: BaseDeviceDriver | None = None, # pylint: disable=unused-argument
90
+ ctx: ArtemisContext | None = None,
91
+ query: str | None = None,
92
+ state: State | None = None,
93
+ **kwargs: Any,
94
+ ) -> str:
95
+ q = query or kwargs.get("Query") or ""
96
+ st = state if state is not None else kwargs.get("state")
97
+ return await _run_diagnoser_logic(ctx, st, q)
98
+
99
+
100
+ # Universal tool instance & aliases
101
+ ask_diagnoser = AskDiagnoserTool()
102
+ AskDiagnoser = AskDiagnoserTool
103
+ DiagnosticTool = AskDiagnoserTool
104
+
105
+
106
+ def get_ask_diagnostic_agent_tool(ctx: ArtemisContext) -> BaseTool:
107
+ """Exports ask_diagnoser as a LangChain BaseTool."""
108
+ return trace_langchain_tool(ask_diagnoser.to_langchain_tool(ctx), ctx)
109
+
110
+
111
+ def get_ask_diagnoser_tool(ctx: ArtemisContext) -> BaseTool:
112
+ """Exports ask_diagnoser as a LangChain BaseTool (standard naming alias)."""
113
+ return trace_langchain_tool(ask_diagnoser.to_langchain_tool(ctx), ctx)
114
+
115
+
116
+ ask_diagnoser_wrapper = ToolWrapper(
117
+ tool_fn_getter=get_ask_diagnostic_agent_tool,
118
+ on_success_fn=lambda output: f"Diagnoser replied:\n{output}",
119
+ on_failure_fn=lambda error: f"Diagnoser failed: {error}",
120
+ )