android-midscene-automation 0.2.4 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (525) hide show
  1. package/CHANGELOG.md +10 -0
  2. package/README.md +159 -29
  3. package/artemis-main/.dockerignore +15 -0
  4. package/artemis-main/.env.example +34 -0
  5. package/artemis-main/CONTRIBUTING.md +35 -0
  6. package/artemis-main/Dockerfile +68 -0
  7. package/artemis-main/LICENSE +202 -0
  8. package/artemis-main/Makefile +115 -0
  9. package/artemis-main/README.md +322 -0
  10. package/artemis-main/README_CN.md +320 -0
  11. package/artemis-main/apps/__init__.py +15 -0
  12. package/artemis-main/apps/admin_console/__init__.py +19 -0
  13. package/artemis-main/apps/admin_console/core/__init__.py +13 -0
  14. package/artemis-main/apps/admin_console/core/config.py +43 -0
  15. package/artemis-main/apps/admin_console/core/security.py +178 -0
  16. package/artemis-main/apps/admin_console/core/state.py +243 -0
  17. package/artemis-main/apps/admin_console/database/__init__.py +13 -0
  18. package/artemis-main/apps/admin_console/database/connection.py +60 -0
  19. package/artemis-main/apps/admin_console/database/repositories/__init__.py +13 -0
  20. package/artemis-main/apps/admin_console/database/repositories/session_repository.py +564 -0
  21. package/artemis-main/apps/admin_console/database/repositories/step_repository.py +512 -0
  22. package/artemis-main/apps/admin_console/database/repositories/trace_repository.py +133 -0
  23. package/artemis-main/apps/admin_console/index.html +3787 -0
  24. package/artemis-main/apps/admin_console/replay_manager.py +2585 -0
  25. package/artemis-main/apps/admin_console/routers/__init__.py +13 -0
  26. package/artemis-main/apps/admin_console/routers/media.py +198 -0
  27. package/artemis-main/apps/admin_console/routers/replay.py +91 -0
  28. package/artemis-main/apps/admin_console/routers/sessions.py +239 -0
  29. package/artemis-main/apps/admin_console/routers/steps.py +109 -0
  30. package/artemis-main/apps/admin_console/routers/stream.py +56 -0
  31. package/artemis-main/apps/admin_console/routers/system.py +761 -0
  32. package/artemis-main/apps/admin_console/routers/tasks.py +517 -0
  33. package/artemis-main/apps/admin_console/schemas/__init__.py +13 -0
  34. package/artemis-main/apps/admin_console/schemas/task_schema.py +47 -0
  35. package/artemis-main/apps/admin_console/server.py +508 -0
  36. package/artemis-main/apps/admin_console/services/__init__.py +13 -0
  37. package/artemis-main/apps/admin_console/services/device_stream_service.py +137 -0
  38. package/artemis-main/apps/admin_console/services/ipc_service.py +262 -0
  39. package/artemis-main/apps/admin_console/services/media_service.py +623 -0
  40. package/artemis-main/apps/admin_console/services/model_service.py +148 -0
  41. package/artemis-main/apps/admin_console/services/task_preset_catalog.py +514 -0
  42. package/artemis-main/apps/admin_console/services/task_queue_service.py +1719 -0
  43. package/artemis-main/apps/admin_console/services/worker_process_io.py +144 -0
  44. package/artemis-main/apps/showcase_ui/README.md +31 -0
  45. package/artemis-main/apps/showcase_ui/angular.json +132 -0
  46. package/artemis-main/apps/showcase_ui/package-lock.json +9472 -0
  47. package/artemis-main/apps/showcase_ui/package.json +39 -0
  48. package/artemis-main/apps/showcase_ui/proxy.conf.json +23 -0
  49. package/artemis-main/apps/showcase_ui/public/favicon.ico +0 -0
  50. package/artemis-main/apps/showcase_ui/public/logo.png +0 -0
  51. package/artemis-main/apps/showcase_ui/public/logo.svg +56 -0
  52. package/artemis-main/apps/showcase_ui/src/app/app.component.html +24 -0
  53. package/artemis-main/apps/showcase_ui/src/app/app.component.scss +49 -0
  54. package/artemis-main/apps/showcase_ui/src/app/app.component.spec.ts +42 -0
  55. package/artemis-main/apps/showcase_ui/src/app/app.component.ts +32 -0
  56. package/artemis-main/apps/showcase_ui/src/app/app.config.ts +29 -0
  57. package/artemis-main/apps/showcase_ui/src/app/app.routes.ts +27 -0
  58. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream-startup.spec.ts +108 -0
  59. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html +1436 -0
  60. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss +3973 -0
  61. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts +2235 -0
  62. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.html +380 -0
  63. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.scss +1171 -0
  64. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.ts +270 -0
  65. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html +433 -0
  66. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss +987 -0
  67. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts +632 -0
  68. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html +130 -0
  69. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.scss +406 -0
  70. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.ts +74 -0
  71. package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.scss +115 -0
  72. package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.ts +52 -0
  73. package/artemis-main/apps/showcase_ui/src/app/core/data/smart-tasks.data.ts +334 -0
  74. package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.spec.ts +55 -0
  75. package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.ts +61 -0
  76. package/artemis-main/apps/showcase_ui/src/app/core/i18n/translate.pipe.ts +11 -0
  77. package/artemis-main/apps/showcase_ui/src/app/core/i18n/zh-cn.ts +651 -0
  78. package/artemis-main/apps/showcase_ui/src/app/core/models/markdown.model.ts +43 -0
  79. package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.spec.ts +47 -0
  80. package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.ts +165 -0
  81. package/artemis-main/apps/showcase_ui/src/app/core/models/session.model.ts +85 -0
  82. package/artemis-main/apps/showcase_ui/src/app/core/models/stream.model.ts +246 -0
  83. package/artemis-main/apps/showcase_ui/src/app/core/models/system.model.ts +118 -0
  84. package/artemis-main/apps/showcase_ui/src/app/core/services/task-recommendation.service.ts +111 -0
  85. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.html +2130 -0
  86. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.scss +5204 -0
  87. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.ts +1189 -0
  88. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.html +39 -0
  89. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.scss +151 -0
  90. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.ts +58 -0
  91. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.html +201 -0
  92. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.scss +886 -0
  93. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.ts +324 -0
  94. package/artemis-main/apps/showcase_ui/src/app/services/agent.service.spec.ts +418 -0
  95. package/artemis-main/apps/showcase_ui/src/app/services/agent.service.ts +2012 -0
  96. package/artemis-main/apps/showcase_ui/src/app/services/system.service.spec.ts +173 -0
  97. package/artemis-main/apps/showcase_ui/src/app/services/system.service.ts +620 -0
  98. package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts +513 -0
  99. package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.ts +1015 -0
  100. package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.spec.ts +85 -0
  101. package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.ts +662 -0
  102. package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.spec.ts +70 -0
  103. package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.ts +383 -0
  104. package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.spec.ts +107 -0
  105. package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.ts +123 -0
  106. package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.spec.ts +92 -0
  107. package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.ts +116 -0
  108. package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts +640 -0
  109. package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts +1021 -0
  110. package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts +375 -0
  111. package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.ts +1419 -0
  112. package/artemis-main/apps/showcase_ui/src/index.html +37 -0
  113. package/artemis-main/apps/showcase_ui/src/main.ts +22 -0
  114. package/artemis-main/apps/showcase_ui/src/styles.scss +60 -0
  115. package/artemis-main/apps/showcase_ui/tsconfig.app.json +23 -0
  116. package/artemis-main/apps/showcase_ui/tsconfig.json +27 -0
  117. package/artemis-main/apps/showcase_ui/tsconfig.spec.json +23 -0
  118. package/artemis-main/artemis/__init__.py +44 -0
  119. package/artemis-main/artemis/__main__.py +18 -0
  120. package/artemis-main/artemis/_legacy_imports.py +94 -0
  121. package/artemis-main/artemis/_version.py +30 -0
  122. package/artemis-main/artemis/agents/__init__.py +17 -0
  123. package/artemis-main/artemis/agents/checker/checker.json +9 -0
  124. package/artemis-main/artemis/agents/checker/checker.py +859 -0
  125. package/artemis-main/artemis/agents/diagnoser/diagnoser.json +23 -0
  126. package/artemis-main/artemis/agents/diagnoser/diagnoser.py +608 -0
  127. package/artemis-main/artemis/agents/explorer/constants.py +52 -0
  128. package/artemis-main/artemis/agents/explorer/explorer.json +57 -0
  129. package/artemis-main/artemis/agents/explorer/explorer.py +473 -0
  130. package/artemis-main/artemis/agents/explorer/geometry.py +73 -0
  131. package/artemis-main/artemis/agents/explorer/native_runner.py +878 -0
  132. package/artemis-main/artemis/agents/explorer/perception_tools.py +632 -0
  133. package/artemis-main/artemis/agents/explorer/run_setup.py +403 -0
  134. package/artemis-main/artemis/agents/explorer/screen_index.py +266 -0
  135. package/artemis-main/artemis/agents/explorer/tiers.py +110 -0
  136. package/artemis-main/artemis/agents/explorer/tool_declarations.py +394 -0
  137. package/artemis-main/artemis/agents/explorer/universal_runner.py +317 -0
  138. package/artemis-main/artemis/agents/flash/context_compressor.py +452 -0
  139. package/artemis-main/artemis/agents/flash/flash_runner.md +51 -0
  140. package/artemis-main/artemis/agents/flash/flash_summarizer.md +53 -0
  141. package/artemis-main/artemis/agents/flash/flash_summarizer_single.md +54 -0
  142. package/artemis-main/artemis/agents/flash/runner.py +1167 -0
  143. package/artemis-main/artemis/agents/flash/summarizer.py +511 -0
  144. package/artemis-main/artemis/agents/history_analyzer/history_analyzer.md +8 -0
  145. package/artemis-main/artemis/agents/history_analyzer/history_analyzer.py +182 -0
  146. package/artemis-main/artemis/agents/image_processor/image_processor.md +74 -0
  147. package/artemis-main/artemis/agents/image_processor/image_processor.py +322 -0
  148. package/artemis-main/artemis/agents/log_analyzer/log_analyzer.md +15 -0
  149. package/artemis-main/artemis/agents/log_analyzer/log_analyzer.py +403 -0
  150. package/artemis-main/artemis/agents/log_analyzer/log_reader_sub_agent.md +8 -0
  151. package/artemis-main/artemis/agents/log_analyzer/output_analyzer.md +11 -0
  152. package/artemis-main/artemis/agents/log_analyzer/output_analyzer.py +163 -0
  153. package/artemis-main/artemis/agents/object_detector/object_detector.json +9 -0
  154. package/artemis-main/artemis/agents/object_detector/object_detector.py +211 -0
  155. package/artemis-main/artemis/agents/operator/operator.json +3 -0
  156. package/artemis-main/artemis/agents/operator/operator.py +1394 -0
  157. package/artemis-main/artemis/agents/operator/prompts.py +1064 -0
  158. package/artemis-main/artemis/agents/outputter/outputter.md +36 -0
  159. package/artemis-main/artemis/agents/outputter/outputter.py +552 -0
  160. package/artemis-main/artemis/agents/planner/planner.json +32 -0
  161. package/artemis-main/artemis/agents/planner/planner.py +436 -0
  162. package/artemis-main/artemis/agents/prompt_assembly.py +79 -0
  163. package/artemis-main/artemis/agents/summarizer/summarizer.py +190 -0
  164. package/artemis-main/artemis/agents/validator/action_execution.py +98 -0
  165. package/artemis-main/artemis/agents/validator/categories.py +33 -0
  166. package/artemis-main/artemis/agents/validator/execution_loop.py +488 -0
  167. package/artemis-main/artemis/agents/validator/incidents.py +120 -0
  168. package/artemis-main/artemis/agents/validator/pixel_safety_net.md +15 -0
  169. package/artemis-main/artemis/agents/validator/precondition_pixel.py +320 -0
  170. package/artemis-main/artemis/agents/validator/precondition_xml.py +672 -0
  171. package/artemis-main/artemis/agents/validator/tool_declarations.py +332 -0
  172. package/artemis-main/artemis/agents/validator/validator.py +188 -0
  173. package/artemis-main/artemis/agents/video_analyzer/__init__.py +15 -0
  174. package/artemis-main/artemis/agents/video_analyzer/audio_native.py +543 -0
  175. package/artemis-main/artemis/agents/video_analyzer/blackboard.py +821 -0
  176. package/artemis-main/artemis/agents/video_analyzer/chunk_agentic.py +455 -0
  177. package/artemis-main/artemis/agents/video_analyzer/chunk_conversation.py +491 -0
  178. package/artemis-main/artemis/agents/video_analyzer/chunk_native.py +437 -0
  179. package/artemis-main/artemis/agents/video_analyzer/conflict_resolution.py +137 -0
  180. package/artemis-main/artemis/agents/video_analyzer/gemini_files.py +162 -0
  181. package/artemis-main/artemis/agents/video_analyzer/native_coordinator.py +541 -0
  182. package/artemis-main/artemis/agents/video_analyzer/reliability.py +128 -0
  183. package/artemis-main/artemis/agents/video_analyzer/universal_engine.py +487 -0
  184. package/artemis-main/artemis/agents/video_analyzer/universal_tools.py +323 -0
  185. package/artemis-main/artemis/agents/video_analyzer/video_analyzer.md +51 -0
  186. package/artemis-main/artemis/agents/video_analyzer/video_analyzer.py +875 -0
  187. package/artemis-main/artemis/agents/video_analyzer/video_sub_agent.md +15 -0
  188. package/artemis-main/artemis/agents/video_analyzer/video_sub_agent_audio.md +12 -0
  189. package/artemis-main/artemis/clients/accessibility_client.py +438 -0
  190. package/artemis-main/artemis/clients/screen_client_factory.py +403 -0
  191. package/artemis-main/artemis/config/__init__.py +309 -0
  192. package/artemis-main/artemis/config/agent.py +1098 -0
  193. package/artemis-main/artemis/config/constants.py +151 -0
  194. package/artemis-main/artemis/config/core/__init__.py +24 -0
  195. package/artemis-main/artemis/config/core/engine.py +118 -0
  196. package/artemis-main/artemis/config/core/interpolator.py +78 -0
  197. package/artemis-main/artemis/config/llm.py +267 -0
  198. package/artemis-main/artemis/config/paths.py +244 -0
  199. package/artemis-main/artemis/config/runtime.py +197 -0
  200. package/artemis-main/artemis/config/settings.py +315 -0
  201. package/artemis-main/artemis/constants.py +47 -0
  202. package/artemis-main/artemis/context.py +289 -0
  203. package/artemis-main/artemis/controllers/__init__.py +41 -0
  204. package/artemis-main/artemis/controllers/controller_factory.py +41 -0
  205. package/artemis-main/artemis/controllers/device_controller.py +29 -0
  206. package/artemis-main/artemis/controllers/types.py +37 -0
  207. package/artemis-main/artemis/controllers/unified_controller.py +649 -0
  208. package/artemis-main/artemis/core/__init__.py +21 -0
  209. package/artemis-main/artemis/core/diagnostics/__init__.py +90 -0
  210. package/artemis-main/artemis/core/diagnostics/adb_keys.py +270 -0
  211. package/artemis-main/artemis/core/diagnostics/adb_server_connection.py +330 -0
  212. package/artemis-main/artemis/core/diagnostics/device_smoke.py +364 -0
  213. package/artemis-main/artemis/core/diagnostics/emulator_manager.py +501 -0
  214. package/artemis-main/artemis/core/diagnostics/engine.py +438 -0
  215. package/artemis-main/artemis/core/diagnostics/hierarchy_parity.py +216 -0
  216. package/artemis-main/artemis/core/diagnostics/probes/adb_probe.py +887 -0
  217. package/artemis-main/artemis/core/diagnostics/probes/base.py +45 -0
  218. package/artemis-main/artemis/core/diagnostics/probes/credentials_probe.py +296 -0
  219. package/artemis-main/artemis/core/diagnostics/probes/host_probe.py +427 -0
  220. package/artemis-main/artemis/core/diagnostics/probes/runtime_probe.py +169 -0
  221. package/artemis-main/artemis/core/diagnostics/probes/toolchain_probe.py +139 -0
  222. package/artemis-main/artemis/core/diagnostics/readiness.py +83 -0
  223. package/artemis-main/artemis/core/diagnostics/schema.py +118 -0
  224. package/artemis-main/artemis/core/tool_declaration.py +43 -0
  225. package/artemis-main/artemis/core/tool_failure.py +56 -0
  226. package/artemis-main/artemis/data_engine/context_vars.py +24 -0
  227. package/artemis-main/artemis/data_engine/engine.py +1615 -0
  228. package/artemis-main/artemis/data_engine/history_reader.py +142 -0
  229. package/artemis-main/artemis/data_engine/models.py +131 -0
  230. package/artemis-main/artemis/data_engine/storage.py +1245 -0
  231. package/artemis-main/artemis/data_engine/trace.py +727 -0
  232. package/artemis-main/artemis/drivers/__init__.py +48 -0
  233. package/artemis-main/artemis/drivers/android/adb_driver.py +507 -0
  234. package/artemis-main/artemis/drivers/android/hierarchy.py +64 -0
  235. package/artemis-main/artemis/drivers/android/input_ime.py +120 -0
  236. package/artemis-main/artemis/drivers/android/recorder.py +57 -0
  237. package/artemis-main/artemis/drivers/base.py +290 -0
  238. package/artemis-main/artemis/drivers/cloud/cloud_driver.py +117 -0
  239. package/artemis-main/artemis/drivers/factory.py +80 -0
  240. package/artemis-main/artemis/drivers/mock/mock_driver.py +198 -0
  241. package/artemis-main/artemis/drivers/types.py +67 -0
  242. package/artemis-main/artemis/graph/checkpoints.py +1243 -0
  243. package/artemis-main/artemis/graph/graph.py +1055 -0
  244. package/artemis-main/artemis/graph/perception.py +234 -0
  245. package/artemis-main/artemis/graph/state.py +151 -0
  246. package/artemis-main/artemis/graph/visibility.py +238 -0
  247. package/artemis-main/artemis/interfaces/cli/__init__.py +19 -0
  248. package/artemis-main/artemis/interfaces/cli/commands/__init__.py +15 -0
  249. package/artemis-main/artemis/interfaces/cli/commands/batch.py +278 -0
  250. package/artemis-main/artemis/interfaces/cli/commands/doctor.py +468 -0
  251. package/artemis-main/artemis/interfaces/cli/commands/helper.py +226 -0
  252. package/artemis-main/artemis/interfaces/cli/commands/init.py +193 -0
  253. package/artemis-main/artemis/interfaces/cli/commands/mcp.py +862 -0
  254. package/artemis-main/artemis/interfaces/cli/commands/run.py +463 -0
  255. package/artemis-main/artemis/interfaces/cli/commands/server.py +151 -0
  256. package/artemis-main/artemis/interfaces/cli/commands/server_lifecycle.py +328 -0
  257. package/artemis-main/artemis/interfaces/cli/commands/trace.py +111 -0
  258. package/artemis-main/artemis/interfaces/cli/commands/ui.py +404 -0
  259. package/artemis-main/artemis/interfaces/cli/main.py +109 -0
  260. package/artemis-main/artemis/interfaces/sdk/__init__.py +31 -0
  261. package/artemis-main/artemis/interfaces/sdk/client.py +19 -0
  262. package/artemis-main/artemis/interfaces/sdk/task.py +72 -0
  263. package/artemis-main/artemis/llm/__init__.py +25 -0
  264. package/artemis-main/artemis/llm/fake_model.py +179 -0
  265. package/artemis-main/artemis/llm/google/__init__.py +57 -0
  266. package/artemis-main/artemis/llm/google/provider.py +185 -0
  267. package/artemis-main/artemis/llm/google/usage.py +148 -0
  268. package/artemis-main/artemis/llm/parser.py +32 -0
  269. package/artemis-main/artemis/llm/reliability.py +267 -0
  270. package/artemis-main/artemis/llm/router.py +389 -0
  271. package/artemis-main/artemis/llm/schemas.py +36 -0
  272. package/artemis-main/artemis/llm/structured.py +196 -0
  273. package/artemis-main/artemis/main.py +58 -0
  274. package/artemis-main/artemis/mcp/__init__.py +20 -0
  275. package/artemis-main/artemis/mcp/action_executor.py +760 -0
  276. package/artemis-main/artemis/mcp/action_manifest.py +320 -0
  277. package/artemis-main/artemis/mcp/action_names.py +172 -0
  278. package/artemis-main/artemis/mcp/action_server.py +202 -0
  279. package/artemis-main/artemis/mcp/action_session.py +245 -0
  280. package/artemis-main/artemis/mcp/action_specs.py +868 -0
  281. package/artemis-main/artemis/mcp/action_types.py +86 -0
  282. package/artemis-main/artemis/mcp/actuators/__init__.py +21 -0
  283. package/artemis-main/artemis/mcp/actuators/adb.py +414 -0
  284. package/artemis-main/artemis/mcp/actuators/base.py +113 -0
  285. package/artemis-main/artemis/mcp/actuators/mock.py +70 -0
  286. package/artemis-main/artemis/mcp/adb_server.py +490 -0
  287. package/artemis-main/artemis/mcp/observation.py +133 -0
  288. package/artemis-main/artemis/mcp/xml_search_server.py +297 -0
  289. package/artemis-main/artemis/memory/__init__.py +90 -0
  290. package/artemis-main/artemis/memory/chunking.py +1843 -0
  291. package/artemis-main/artemis/memory/context_policy.py +226 -0
  292. package/artemis-main/artemis/memory/step_capsule.md +73 -0
  293. package/artemis-main/artemis/memory/step_memory.py +281 -0
  294. package/artemis-main/artemis/memory/transcript.py +798 -0
  295. package/artemis-main/artemis/platform/__init__.py +61 -0
  296. package/artemis-main/artemis/platform/base.py +104 -0
  297. package/artemis-main/artemis/platform/darwin.py +184 -0
  298. package/artemis-main/artemis/platform/linux.py +207 -0
  299. package/artemis-main/artemis/platform/windows.py +193 -0
  300. package/artemis-main/artemis/resources/__init__.py +46 -0
  301. package/artemis-main/artemis/runtime/__init__.py +108 -0
  302. package/artemis-main/artemis/runtime/adb_endpoint.py +173 -0
  303. package/artemis-main/artemis/runtime/awake_lease.py +283 -0
  304. package/artemis-main/artemis/runtime/awake_service.py +343 -0
  305. package/artemis-main/artemis/runtime/cancel_requests.py +254 -0
  306. package/artemis-main/artemis/runtime/daemon_client.py +422 -0
  307. package/artemis-main/artemis/runtime/device_lock.py +1013 -0
  308. package/artemis-main/artemis/runtime/device_pool.py +478 -0
  309. package/artemis-main/artemis/runtime/helper_manager.py +1030 -0
  310. package/artemis-main/artemis/runtime/process_probe.py +98 -0
  311. package/artemis-main/artemis/runtime/server_lifecycle.py +435 -0
  312. package/artemis-main/artemis/runtime/supervisor.py +121 -0
  313. package/artemis-main/artemis/runtime/trace_store.py +375 -0
  314. package/artemis-main/artemis/sdk/__init__.py +26 -0
  315. package/artemis-main/artemis/sdk/agent.py +485 -0
  316. package/artemis-main/artemis/sdk/builders/__init__.py +29 -0
  317. package/artemis-main/artemis/sdk/builders/agent_config_builder.py +474 -0
  318. package/artemis-main/artemis/sdk/constants.py +15 -0
  319. package/artemis-main/artemis/sdk/run_outcome.py +58 -0
  320. package/artemis-main/artemis/sdk/types/__init__.py +61 -0
  321. package/artemis-main/artemis/sdk/types/agent.py +89 -0
  322. package/artemis-main/artemis/sdk/utils.py +28 -0
  323. package/artemis-main/artemis/services/llm.py +1040 -0
  324. package/artemis-main/artemis/services/token_meter.py +267 -0
  325. package/artemis-main/artemis/telemetry/__init__.py +35 -0
  326. package/artemis-main/artemis/telemetry/models.py +64 -0
  327. package/artemis-main/artemis/telemetry/replay.py +34 -0
  328. package/artemis-main/artemis/telemetry/storage/json_storage.py +47 -0
  329. package/artemis-main/artemis/telemetry/tracer.py +85 -0
  330. package/artemis-main/artemis/toolchain/__init__.py +52 -0
  331. package/artemis-main/artemis/toolchain/descriptors.py +92 -0
  332. package/artemis-main/artemis/toolchain/resolver.py +185 -0
  333. package/artemis-main/artemis/tools/base.py +232 -0
  334. package/artemis-main/artemis/tools/command_tool.py +1240 -0
  335. package/artemis-main/artemis/tools/committee_prompts.json +5 -0
  336. package/artemis-main/artemis/tools/committee_tool.py +422 -0
  337. package/artemis-main/artemis/tools/diagnoser_submit_answer_tool.py +87 -0
  338. package/artemis-main/artemis/tools/diagnostic_tool.py +120 -0
  339. package/artemis-main/artemis/tools/explorer_tool.py +581 -0
  340. package/artemis-main/artemis/tools/history/__init__.py +398 -0
  341. package/artemis-main/artemis/tools/history/replay.py +120 -0
  342. package/artemis-main/artemis/tools/history/screenshot.py +139 -0
  343. package/artemis-main/artemis/tools/history/search.py +563 -0
  344. package/artemis-main/artemis/tools/image_processor_tool.py +124 -0
  345. package/artemis-main/artemis/tools/index.py +47 -0
  346. package/artemis-main/artemis/tools/log_tool.py +108 -0
  347. package/artemis-main/artemis/tools/mobile/launch_app.py +132 -0
  348. package/artemis-main/artemis/tools/mobile/log_utils.py +138 -0
  349. package/artemis-main/artemis/tools/mobile/ocr.py +202 -0
  350. package/artemis-main/artemis/tools/mobile/read_hierarchy.py +108 -0
  351. package/artemis-main/artemis/tools/mobile/read_logs.py +116 -0
  352. package/artemis-main/artemis/tools/mobile/search_logs.py +260 -0
  353. package/artemis-main/artemis/tools/object_detection_tool.py +244 -0
  354. package/artemis-main/artemis/tools/scratchpad.py +819 -0
  355. package/artemis-main/artemis/tools/system/adb_shell.py +31 -0
  356. package/artemis-main/artemis/tools/tool_wrapper.py +270 -0
  357. package/artemis-main/artemis/tools/video_tool.py +261 -0
  358. package/artemis-main/artemis/tools/wait_tool.py +97 -0
  359. package/artemis-main/artemis/utils/app_launch_utils.py +207 -0
  360. package/artemis-main/artemis/utils/coordinates.py +606 -0
  361. package/artemis-main/artemis/utils/credentials_validator.py +201 -0
  362. package/artemis-main/artemis/utils/cv_canvas.py +189 -0
  363. package/artemis-main/artemis/utils/cython_compat.py +45 -0
  364. package/artemis-main/artemis/utils/element_hit_test.py +80 -0
  365. package/artemis-main/artemis/utils/image_diff.py +335 -0
  366. package/artemis-main/artemis/utils/image_hash.py +80 -0
  367. package/artemis-main/artemis/utils/notes.py +470 -0
  368. package/artemis-main/artemis/utils/ocr_api.py +105 -0
  369. package/artemis-main/artemis/utils/ocr_xml_fusion.py +318 -0
  370. package/artemis-main/artemis/utils/plan_grammar.py +513 -0
  371. package/artemis-main/artemis/utils/python_executor.py +213 -0
  372. package/artemis-main/artemis/utils/startup_progress.py +60 -0
  373. package/artemis-main/artemis/utils/task_tree.py +1174 -0
  374. package/artemis-main/artemis/utils/text.py +37 -0
  375. package/artemis-main/artemis/utils/ui_filter.py +651 -0
  376. package/artemis-main/artemis/utils/video.py +685 -0
  377. package/artemis-main/artemis/utils/visualization.py +802 -0
  378. package/artemis-main/config/artemis.example.jsonc +12 -0
  379. package/artemis-main/mcp_server/README.md +111 -0
  380. package/artemis-main/mcp_server/__init__.py +21 -0
  381. package/artemis-main/mcp_server/__main__.py +20 -0
  382. package/artemis-main/mcp_server/background/__init__.py +15 -0
  383. package/artemis-main/mcp_server/background/task_runner.py +453 -0
  384. package/artemis-main/mcp_server/base.py +49 -0
  385. package/artemis-main/mcp_server/notifiers/__init__.py +74 -0
  386. package/artemis-main/mcp_server/notifiers/agentapi.py +289 -0
  387. package/artemis-main/mcp_server/notifiers/base.py +45 -0
  388. package/artemis-main/mcp_server/notifiers/composite.py +87 -0
  389. package/artemis-main/mcp_server/notifiers/desktop.py +108 -0
  390. package/artemis-main/mcp_server/notifiers/file.py +69 -0
  391. package/artemis-main/mcp_server/notifiers/script.py +90 -0
  392. package/artemis-main/mcp_server/notifiers/webhook.py +87 -0
  393. package/artemis-main/mcp_server/rules.md +68 -0
  394. package/artemis-main/mcp_server/server.py +106 -0
  395. package/artemis-main/mcp_server/tools/__init__.py +29 -0
  396. package/artemis-main/mcp_server/tools/device_state.py +107 -0
  397. package/artemis-main/mcp_server/tools/diagnose.py +1249 -0
  398. package/artemis-main/mcp_server/tools/inspect_trace.py +394 -0
  399. package/artemis-main/mcp_server/tools/task_manager.py +627 -0
  400. package/artemis-main/mcp_server/tools/task_runner.py +548 -0
  401. package/artemis-main/mcp_server/utils/__init__.py +15 -0
  402. package/artemis-main/mcp_server/utils/device_utils.py +180 -0
  403. package/artemis-main/mcp_server/utils/env_utils.py +65 -0
  404. package/artemis-main/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk +0 -0
  405. package/artemis-main/packages/artemis-accessibility-helper/app/build.gradle.kts +35 -0
  406. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml +49 -0
  407. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java +274 -0
  408. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ArtemisAccessibilityService.java +227 -0
  409. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java +463 -0
  410. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/DisplayUtils.java +111 -0
  411. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java +232 -0
  412. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/HierarchyDumper.java +832 -0
  413. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenReceiver.java +36 -0
  414. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenStore.java +38 -0
  415. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/XmlUtils.java +102 -0
  416. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/values/strings.xml +4 -0
  417. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/xml/accessibility_service_config.xml +10 -0
  418. package/artemis-main/packages/artemis-accessibility-helper/build.gradle.kts +3 -0
  419. package/artemis-main/packages/artemis-accessibility-helper/build_apk.sh +215 -0
  420. package/artemis-main/packages/artemis-accessibility-helper/debug.keystore +0 -0
  421. package/artemis-main/packages/artemis-accessibility-helper/gradle.properties +2 -0
  422. package/artemis-main/packages/artemis-accessibility-helper/helper_manifest.json +7 -0
  423. package/artemis-main/packages/artemis-accessibility-helper/settings.gradle.kts +17 -0
  424. package/artemis-main/packages/artemis-client/DESIGN.md +89 -0
  425. package/artemis-main/packages/artemis-client/README.md +112 -0
  426. package/artemis-main/packages/artemis-client/examples/basic.py +30 -0
  427. package/artemis-main/packages/artemis-client/pyproject.toml +36 -0
  428. package/artemis-main/packages/artemis-client/src/artemis_client/__init__.py +45 -0
  429. package/artemis-main/packages/artemis-client/src/artemis_client/client.py +401 -0
  430. package/artemis-main/packages/artemis-client/src/artemis_client/errors.py +66 -0
  431. package/artemis-main/packages/artemis-client/src/artemis_client/models.py +212 -0
  432. package/artemis-main/packages/artemis-client/src/artemis_client/py.typed +1 -0
  433. package/artemis-main/packages/artemis-client/src/artemis_client/transport.py +134 -0
  434. package/artemis-main/packages/artemis-client/tests/test_client.py +289 -0
  435. package/artemis-main/packages/artemis-client/tests/test_transport.py +85 -0
  436. package/artemis-main/pyproject.toml +259 -0
  437. package/artemis-main/pyright-core.json +17 -0
  438. package/artemis-main/scripts/check_dependency_sources.py +68 -0
  439. package/artemis-main/scripts/install_deps.ps1 +520 -0
  440. package/artemis-main/scripts/install_deps.sh +533 -0
  441. package/artemis-main/scripts/quality_ratchet.py +97 -0
  442. package/artemis-main/scripts/real_device_video_acceptance.py +483 -0
  443. package/artemis-main/scripts/run_edge_case_part.py +327 -0
  444. package/artemis-main/scripts/run_ui.py +242 -0
  445. package/artemis-main/scripts/start.ps1 +500 -0
  446. package/artemis-main/scripts/test_all_edge_cases.py +269 -0
  447. package/artemis-main/scripts/verify_concurrency_and_devices.py +385 -0
  448. package/artemis-main/setup.py +164 -0
  449. package/artemis-main/start.bat +35 -0
  450. package/artemis-main/start.sh +423 -0
  451. package/artemis-main/third_party/mobile_use/LICENSE +190 -0
  452. package/artemis-main/third_party/mobile_use/METADATA +27 -0
  453. package/artemis-main/third_party/mobile_use/NOTICE +12 -0
  454. package/artemis-main/third_party/mobile_use/__init__.py +19 -0
  455. package/artemis-main/third_party/mobile_use/agents/__init__.py +13 -0
  456. package/artemis-main/third_party/mobile_use/agents/hopper/__init__.py +13 -0
  457. package/artemis-main/third_party/mobile_use/agents/hopper/hopper.md +22 -0
  458. package/artemis-main/third_party/mobile_use/agents/hopper/hopper.py +80 -0
  459. package/artemis-main/third_party/mobile_use/clients/__init__.py +13 -0
  460. package/artemis-main/third_party/mobile_use/clients/ui_automator_client.py +482 -0
  461. package/artemis-main/third_party/mobile_use/config/__init__.py +13 -0
  462. package/artemis-main/third_party/mobile_use/config/llm.py +183 -0
  463. package/artemis-main/third_party/mobile_use/config/output.py +184 -0
  464. package/artemis-main/third_party/mobile_use/context.py +119 -0
  465. package/artemis-main/third_party/mobile_use/controllers/__init__.py +13 -0
  466. package/artemis-main/third_party/mobile_use/controllers/platform_specific_commands_controller.py +151 -0
  467. package/artemis-main/third_party/mobile_use/controllers/types.py +92 -0
  468. package/artemis-main/third_party/mobile_use/controllers/unified_controller.py +183 -0
  469. package/artemis-main/third_party/mobile_use/graph.py +54 -0
  470. package/artemis-main/third_party/mobile_use/llm_service.py +193 -0
  471. package/artemis-main/third_party/mobile_use/main.py +130 -0
  472. package/artemis-main/third_party/mobile_use/sdk/__init__.py +13 -0
  473. package/artemis-main/third_party/mobile_use/sdk/agent.py +1060 -0
  474. package/artemis-main/third_party/mobile_use/sdk/builders/__init__.py +13 -0
  475. package/artemis-main/third_party/mobile_use/sdk/builders/agent_config_builder.py +225 -0
  476. package/artemis-main/third_party/mobile_use/sdk/builders/index.py +33 -0
  477. package/artemis-main/third_party/mobile_use/sdk/builders/task_request_builder.py +243 -0
  478. package/artemis-main/third_party/mobile_use/sdk/types/__init__.py +13 -0
  479. package/artemis-main/third_party/mobile_use/sdk/types/agent.py +98 -0
  480. package/artemis-main/third_party/mobile_use/sdk/types/exceptions.py +109 -0
  481. package/artemis-main/third_party/mobile_use/sdk/types/task.py +234 -0
  482. package/artemis-main/third_party/mobile_use/testing.py +59 -0
  483. package/artemis-main/third_party/mobile_use/tools/__init__.py +13 -0
  484. package/artemis-main/third_party/mobile_use/tools/index.py +44 -0
  485. package/artemis-main/third_party/mobile_use/tools/mobile/__init__.py +13 -0
  486. package/artemis-main/third_party/mobile_use/tools/mobile/launch_app.py +89 -0
  487. package/artemis-main/third_party/mobile_use/tools/tool_wrapper.py +52 -0
  488. package/artemis-main/third_party/mobile_use/utils/__init__.py +13 -0
  489. package/artemis-main/third_party/mobile_use/utils/app_launch_utils.py +217 -0
  490. package/artemis-main/third_party/mobile_use/utils/cli_helpers.py +60 -0
  491. package/artemis-main/third_party/mobile_use/utils/decorators.py +158 -0
  492. package/artemis-main/third_party/mobile_use/utils/file.py +31 -0
  493. package/artemis-main/third_party/mobile_use/utils/logger.py +216 -0
  494. package/artemis-main/third_party/mobile_use/utils/media.py +213 -0
  495. package/artemis-main/third_party/mobile_use/utils/shell_utils.py +39 -0
  496. package/artemis-main/third_party/mobile_use/utils/video.py +398 -0
  497. package/artemis-main/uv.lock +4908 -0
  498. package/bin/android-midscene-automation.js +1 -5
  499. package/bin/artemis-config.mjs +15 -0
  500. package/bin/artemis-embedded.py +56 -0
  501. package/bin/start-artemis.mjs +30 -0
  502. package/bin/start-dev.mjs +112 -0
  503. package/bin/test_artemis_embedded.py +44 -0
  504. package/package.json +4 -2
  505. package/server/artemis.ts +34 -0
  506. package/server/http-api.ts +12 -0
  507. package/src/App.vue +10 -5
  508. package/src/api.ts +5 -0
  509. package/src/appium-recorder/AppiumPage.vue +1 -1
  510. package/src/appium-recorder/components/ComponentInspector.vue +1 -1
  511. package/src/appium-recorder/components/FlowNodeSearch.vue +3 -3
  512. package/src/appium-recorder/components/ImageCheckDialog.vue +3 -3
  513. package/src/appium-recorder/components/RecorderWorkspace.vue +4 -4
  514. package/src/appium-recorder/flow-appearance.ts +4 -4
  515. package/src/components/AppSidebar.vue +8 -7
  516. package/src/components/MidsceneMaintenanceNotice.vue +17 -0
  517. package/src/components/analytics/AnalysisCharts.vue +4 -4
  518. package/src/components/analytics/ReportAnalysis.vue +1 -1
  519. package/src/components/generator/TestCaseFileUpload.vue +1 -1
  520. package/src/components/help/HelpCenter.vue +56 -37
  521. package/src/pages/AiGeneratorPage.vue +1 -1
  522. package/src/pages/ArtemisPage.vue +72 -0
  523. package/src/pages/ConfigPage.vue +18 -19
  524. package/src/style.css +340 -121
  525. package/src/types.ts +1 -1
@@ -0,0 +1,403 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Select the UI-hierarchy backend for a device and build its screen client.
16
+
17
+ Three backends, chosen by ``ARTEMIS_HIERARCHY_BACKEND`` (environment or
18
+ ``.env``):
19
+
20
+ * ``auto`` (default): the Accessibility Helper, with UIAutomator2 as a
21
+ per-call fallback. UIAutomator2 is only constructed when the helper fails,
22
+ so a healthy helper never triggers ``u2.connect`` (which pushes its own APKs
23
+ and uninstalls Maestro).
24
+ * ``helper``: the helper only. Failures surface as errors; use it in CI to
25
+ prove the helper is really what serves the hierarchy.
26
+ * ``uiautomator``: the pre-helper path, unchanged.
27
+
28
+ Every entrypoint that used to build ``UIAutomatorClient`` directly (the SDK
29
+ agent, the MCP device controller, and through it the device smoke test) goes
30
+ through :func:`create_screen_client`, so the switch applies everywhere.
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ from collections.abc import Callable
36
+ from enum import Enum
37
+ import os
38
+ import subprocess
39
+ import time
40
+ from typing import Any
41
+
42
+ from PIL import Image
43
+
44
+ from artemis.clients.accessibility_client import AccessibilityClient, HelperUnavailable
45
+ from artemis.config.constants import ENV_ARTEMIS_HIERARCHY_BACKEND
46
+ from artemis.runtime.adb_endpoint import adb_command
47
+ from artemis.runtime.helper_manager import ProvisionEvent
48
+ from third_party.mobile_use.clients.ui_automator_client import (
49
+ UIAutomatorClient,
50
+ UIAutomatorScreenData,
51
+ )
52
+ from third_party.mobile_use.utils.logger import get_logger
53
+
54
+ logger = get_logger(__name__)
55
+
56
+ #: ``(previous_backend, new_backend, reason)``; ``previous_backend`` is None on first use.
57
+ BackendChangeListener = Callable[[str | None, str, str | None], None]
58
+
59
+
60
+ class DeviceOfflineError(RuntimeError):
61
+ """The device is not attached / authorized; no backend can serve it.
62
+
63
+ Raised instead of trying UIAutomator2 when the helper fails because the
64
+ phone itself is gone, so the failure names the real cause and does not
65
+ burn UIAutomator2's own multi-attempt connect timeout.
66
+ """
67
+
68
+ def __init__(self, device_id: str, state: str | None):
69
+ self.device_id = device_id
70
+ self.state = state
71
+ detail = f"adb reports state '{state}'" if state else "adb does not list it"
72
+ super().__init__(
73
+ f"Device {device_id} is not available ({detail}). Reconnect the device or "
74
+ "accept the USB debugging prompt, then retry."
75
+ )
76
+
77
+
78
+ def device_state(device_id: str, timeout: float = 5.0) -> str | None:
79
+ """``adb get-state`` for one serial: 'device', 'offline', 'unauthorized', or None."""
80
+ try:
81
+ result = subprocess.run(
82
+ adb_command(["-s", device_id, "get-state"]),
83
+ stdin=subprocess.DEVNULL,
84
+ capture_output=True,
85
+ text=True,
86
+ timeout=timeout,
87
+ check=False,
88
+ )
89
+ except (OSError, subprocess.SubprocessError):
90
+ return None
91
+ text = (result.stdout or "").strip()
92
+ if result.returncode != 0 or not text:
93
+ # "error: device unauthorized" / "device offline" arrive on stderr.
94
+ err = (result.stderr or "").lower()
95
+ for known in ("unauthorized", "offline"):
96
+ if known in err:
97
+ return known
98
+ return None
99
+ return text
100
+
101
+
102
+ class HierarchyBackend(str, Enum):
103
+ AUTO = "auto"
104
+ HELPER = "helper"
105
+ UIAUTOMATOR = "uiautomator"
106
+
107
+
108
+ def resolve_backend(value: str | HierarchyBackend | None = None) -> HierarchyBackend:
109
+ """Backend from an explicit value, else the environment, else settings, else ``auto``."""
110
+ if isinstance(value, HierarchyBackend):
111
+ return value
112
+ text = value
113
+ if text is None:
114
+ text = os.environ.get(ENV_ARTEMIS_HIERARCHY_BACKEND)
115
+ if text is None:
116
+ try:
117
+ from artemis.config import settings
118
+
119
+ text = getattr(settings, "ARTEMIS_HIERARCHY_BACKEND", None)
120
+ except (ImportError, ValueError):
121
+ text = None
122
+ normalized = (text or "auto").strip().lower()
123
+ try:
124
+ return HierarchyBackend(normalized)
125
+ except ValueError:
126
+ logger.warning(
127
+ f"Unknown hierarchy backend {text!r}; valid values are "
128
+ f"{', '.join(b.value for b in HierarchyBackend)}. Using 'auto'."
129
+ )
130
+ return HierarchyBackend.AUTO
131
+
132
+
133
+ class FallbackScreenClient:
134
+ """Helper first, UIAutomator2 on failure, with a retry window for the helper.
135
+
136
+ After a helper failure the client serves from UIAutomator2 for
137
+ ``retry_after`` seconds before trying the helper again, so one dead tunnel
138
+ does not cost a 5-second timeout on every step.
139
+ """
140
+
141
+ def __init__(
142
+ self,
143
+ device_id: str,
144
+ helper: AccessibilityClient | None = None,
145
+ uiautomator_factory: Callable[[], UIAutomatorClient] | None = None,
146
+ *,
147
+ retry_after: float = 30.0,
148
+ clock: Callable[[], float] = time.monotonic,
149
+ state_probe: Callable[[str], str | None] = device_state,
150
+ ) -> None:
151
+ self._device_id = device_id
152
+ self._helper = helper or AccessibilityClient(device_id)
153
+ self._uiautomator_factory = uiautomator_factory or (lambda: UIAutomatorClient(device_id))
154
+ self._uiautomator: UIAutomatorClient | None = None
155
+ self._retry_after = retry_after
156
+ self._clock = clock
157
+ self._state_probe = state_probe
158
+ self._helper_down_until: float | None = None
159
+ self._active_backend: str | None = None
160
+ self._last_failure: str | None = None
161
+ #: Every backend switch, oldest first: {time, backend, reason}.
162
+ self.backend_history: list[dict[str, Any]] = []
163
+ self._listeners: list[BackendChangeListener] = []
164
+
165
+ # ------------------------------------------------------------------ #
166
+
167
+ @property
168
+ def device_id(self) -> str:
169
+ return self._device_id
170
+
171
+ @property
172
+ def helper(self) -> AccessibilityClient:
173
+ return self._helper
174
+
175
+ @property
176
+ def active_backend(self) -> str | None:
177
+ """Backend that served the last call: ``"helper"``, ``"uiautomator"`` or ``None``."""
178
+ return self._active_backend
179
+
180
+ @property
181
+ def uiautomator(self) -> UIAutomatorClient:
182
+ if self._uiautomator is None:
183
+ self._uiautomator = self._uiautomator_factory()
184
+ return self._uiautomator
185
+
186
+ def add_backend_listener(self, listener: BackendChangeListener) -> None:
187
+ """Be told whenever the serving backend changes (first use included)."""
188
+ self._listeners.append(listener)
189
+
190
+ def remove_backend_listener(self, listener: BackendChangeListener) -> None:
191
+ if listener in self._listeners:
192
+ self._listeners.remove(listener)
193
+
194
+ def _set_active(self, backend: str, reason: str | None = None) -> None:
195
+ previous = self._active_backend
196
+ if previous == backend:
197
+ return
198
+ self._active_backend = backend
199
+ self.backend_history.append({"time": time.time(), "backend": backend, "reason": reason})
200
+ for listener in list(self._listeners):
201
+ try:
202
+ listener(previous, backend, reason)
203
+ except Exception as exc: # pylint: disable=broad-exception-caught
204
+ logger.debug(f"Backend listener failed: {exc}")
205
+
206
+ def _ensure_device_online(self) -> None:
207
+ """Fail fast with the real cause when the phone itself is gone."""
208
+ state = self._state_probe(self._device_id)
209
+ if state != "device":
210
+ raise DeviceOfflineError(self._device_id, state)
211
+
212
+ def _helper_allowed(self) -> bool:
213
+ if self._helper_down_until is None:
214
+ return True
215
+ if self._clock() >= self._helper_down_until:
216
+ self._helper_down_until = None
217
+ return True
218
+ return False
219
+
220
+ def _degrade(self, reason: BaseException) -> None:
221
+ # Never fall back to UIAutomator2 for a phone that is simply not there.
222
+ self._ensure_device_online()
223
+ self._last_failure = f"{reason.__class__.__name__}: {reason}"
224
+ if self._helper_down_until is None:
225
+ logger.warning(
226
+ f"Accessibility helper unavailable on {self._device_id} ({reason}); "
227
+ f"serving from UIAutomator2 and retrying the helper in {self._retry_after:g}s."
228
+ )
229
+ self._helper_down_until = self._clock() + self._retry_after
230
+
231
+ def _release_uiautomation(self) -> None:
232
+ """Stop our UIAutomator2 server before giving the helper another chance.
233
+
234
+ Its UiAutomation connection keeps every accessibility service unbound,
235
+ so while it lives the helper cannot answer and the retry would be a
236
+ guaranteed failure. UIAutomator2 restarts its server by itself on the
237
+ next call if the helper turns out to be still dead.
238
+ """
239
+ if self._uiautomator is not None and self._active_backend == "uiautomator":
240
+ self._uiautomator.stop_server()
241
+
242
+ def _call(self, name: str, *args: Any, **kwargs: Any) -> Any:
243
+ if self._helper_allowed():
244
+ self._release_uiautomation()
245
+ try:
246
+ result = getattr(self._helper, name)(*args, **kwargs)
247
+ except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as exc:
248
+ self._degrade(exc)
249
+ else:
250
+ self._set_active("helper", "helper recovered" if self._last_failure else None)
251
+ self._last_failure = None
252
+ return result
253
+ result = getattr(self.uiautomator, name)(*args, **kwargs)
254
+ self._set_active("uiautomator", self._last_failure)
255
+ return result
256
+
257
+ # ------------------------------------------------------------------ #
258
+ # Lifecycle
259
+ # ------------------------------------------------------------------ #
260
+
261
+ def connect(self, on_event: ProvisionEvent | None = None) -> None:
262
+ self._release_uiautomation()
263
+ self.backend_history.clear()
264
+ self._active_backend = None
265
+ if self._helper_allowed():
266
+ self._release_uiautomation()
267
+ try:
268
+ self._helper.connect(on_event=on_event)
269
+ except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as exc:
270
+ self._degrade(exc)
271
+ else:
272
+ self._set_active("helper")
273
+ return
274
+ self.uiautomator.connect()
275
+ self._set_active("uiautomator", self._last_failure)
276
+
277
+ def disconnect(self) -> None:
278
+ try:
279
+ self._helper.disconnect()
280
+ finally:
281
+ if self._uiautomator is not None:
282
+ # Release UiAutomation so the helper is back for the next task
283
+ # on this device (possibly served by another process).
284
+ self._uiautomator.disconnect(stop_server=True)
285
+ self._active_backend = None
286
+
287
+ # ------------------------------------------------------------------ #
288
+ # Screen data and input, same surface as UIAutomatorClient
289
+ # ------------------------------------------------------------------ #
290
+
291
+ def get_screen_data(self) -> UIAutomatorScreenData:
292
+ return self._call("get_screen_data")
293
+
294
+ def get_hierarchy(self) -> str:
295
+ return self._call("get_hierarchy")
296
+
297
+ def get_screenshot(self) -> Image.Image | None:
298
+ return self._call("get_screenshot")
299
+
300
+ def get_screenshot_base64(self) -> str | None:
301
+ return self._call("get_screenshot_base64")
302
+
303
+ def set_clipboard(self, text: str) -> bool:
304
+ return self._call("set_clipboard", text)
305
+
306
+ def send_text(self, text: str) -> Any:
307
+ return self._call("send_text", text)
308
+
309
+ def clear_text(self) -> bool:
310
+ return self._call("clear_text")
311
+
312
+ def press_key(self, key: str) -> Any:
313
+ return self._call("press_key", key)
314
+
315
+
316
+ ScreenClient = AccessibilityClient | UIAutomatorClient | FallbackScreenClient
317
+
318
+
319
+ def create_screen_client(
320
+ device_id: str, backend: str | HierarchyBackend | None = None
321
+ ) -> ScreenClient:
322
+ """Build the screen client for ``device_id`` according to the configured backend."""
323
+ resolved = resolve_backend(backend)
324
+ if resolved is HierarchyBackend.UIAUTOMATOR:
325
+ return UIAutomatorClient(device_id=device_id)
326
+ if resolved is HierarchyBackend.HELPER:
327
+ return AccessibilityClient(device_id)
328
+ return FallbackScreenClient(device_id)
329
+
330
+
331
+ def describe_backend(client: Any) -> str | None:
332
+ """Best-effort name of the backend a client is serving from, for diagnostics."""
333
+ active = getattr(client, "active_backend", None)
334
+ if active:
335
+ return str(active)
336
+ if isinstance(client, UIAutomatorClient):
337
+ return "uiautomator"
338
+ if isinstance(client, AccessibilityClient):
339
+ return "helper"
340
+ return None
341
+
342
+
343
+ def helper_version(client: Any) -> str | None:
344
+ """Version name of the helper a client is attached to, if any."""
345
+ helper = getattr(client, "helper", None) or client
346
+ session = getattr(helper, "session", None)
347
+ name = getattr(session, "version_name", None)
348
+ return str(name) if name else None
349
+
350
+
351
+ def hierarchy_backend_summary(client: Any) -> dict[str, Any] | None:
352
+ """Machine-readable account of which backend served a run, for reports."""
353
+ backend = describe_backend(client)
354
+ if backend is None:
355
+ return None
356
+ history = list(getattr(client, "backend_history", None) or [])
357
+ return {
358
+ "backend": backend,
359
+ "helper_version": helper_version(client),
360
+ "switches": history[1:] if len(history) > 1 else [],
361
+ }
362
+
363
+
364
+ _BACKEND_LABEL = {"helper": "the Artemis accessibility helper", "uiautomator": "UIAutomator2"}
365
+
366
+
367
+ def hierarchy_backend_sentence(
368
+ client: Any, relative_time: Callable[[float], str] | None = None
369
+ ) -> str | None:
370
+ """One plain sentence for reports: what served the hierarchy and whether it changed."""
371
+ summary = hierarchy_backend_summary(client)
372
+ if summary is None:
373
+ return None
374
+ history = getattr(client, "backend_history", None) or []
375
+ first = (history[0].get("backend") if history else None) or summary["backend"]
376
+ text = f"UI hierarchy source: {_BACKEND_LABEL.get(first, first)}"
377
+ if first == "helper" and summary.get("helper_version"):
378
+ text += f" v{summary['helper_version']}"
379
+ for switch in summary["switches"]:
380
+ when = relative_time(switch["time"]) if relative_time else None
381
+ text += f"; switched to {_BACKEND_LABEL.get(switch['backend'], switch['backend'])}"
382
+ if when:
383
+ text += f" at {when}"
384
+ if switch.get("reason"):
385
+ text += f" because {str(switch['reason']).rstrip('. ')}"
386
+ return text + "."
387
+
388
+
389
+ __all__ = [
390
+ "BackendChangeListener",
391
+ "DeviceOfflineError",
392
+ "FallbackScreenClient",
393
+ "HelperUnavailable",
394
+ "HierarchyBackend",
395
+ "ScreenClient",
396
+ "create_screen_client",
397
+ "describe_backend",
398
+ "device_state",
399
+ "helper_version",
400
+ "hierarchy_backend_sentence",
401
+ "hierarchy_backend_summary",
402
+ "resolve_backend",
403
+ ]
@@ -0,0 +1,309 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Unified configuration package and single entry point for ARTEMIS settings, paths, and models."""
16
+
17
+ from artemis.config.agent import (
18
+ AgentGlobalConfig,
19
+ CheckerConfig,
20
+ CommitteeConfig,
21
+ DEFAULT_VERIFICATION_LEVEL,
22
+ ExplorerConfig,
23
+ FlashProfileConfig,
24
+ MemoryChunkingConfig,
25
+ MemoryConfig,
26
+ MemoryRecallConfig,
27
+ MemoryReplayConfig,
28
+ MemoryRuntimeConfig,
29
+ MemoryTranscriptConfig,
30
+ OutputterConfig,
31
+ PlannerValidationConfig,
32
+ ProProfileConfig,
33
+ StepSummarizerConfig,
34
+ VideoAnalyzerConfig,
35
+ load_agent_config,
36
+ resolve_explorer_version,
37
+ VERIFICATION_LEVEL_PRESETS,
38
+ VerificationLevel,
39
+ checker_overrides_for_level,
40
+ run_tuning_for_profile,
41
+ verification_level_for_checker,
42
+ )
43
+ from artemis.config.constants import (
44
+ AGENT_CONFIG_FILENAME,
45
+ DATA_ENGINE_DB_FILENAME,
46
+ DEFAULT_ACTION_TIMEOUT,
47
+ DEFAULT_ADB_HOST,
48
+ DEFAULT_ADB_PORT,
49
+ DEFAULT_EDGE_MONITOR_PORT,
50
+ DEFAULT_EXPLORER_VERSION,
51
+ DEFAULT_GATEWAY_PORT,
52
+ DEFAULT_MODEL,
53
+ DEFAULT_PROFILE,
54
+ DEFAULT_STREAM_PING_INTERVAL,
55
+ DOTENV_FILENAME,
56
+ ENV_ADB_DEVICE_SERIAL,
57
+ ENV_ADB_HOST,
58
+ ENV_ADB_PORT,
59
+ ENV_ADB_SERVER_SOCKET,
60
+ ENV_ANTHROPIC_API_KEY,
61
+ ENV_ANTIGRAVITY_APP_DIR,
62
+ ENV_ANTIGRAVITY_LS_ADDRESS,
63
+ ENV_DATA_ENGINE_DB_PATH,
64
+ ENV_EVENTS_OUTPUT_PATH,
65
+ ENV_GCP_API_KEY,
66
+ ENV_GEMINI_API_KEY,
67
+ ENV_GOOGLE_API_KEY,
68
+ ENV_KEEP_VIDEOS,
69
+ ENV_ARTEMIS_APP_DIR,
70
+ ENV_ARTEMIS_CLOUD_GATEWAY_URL,
71
+ ENV_ARTEMIS_CLOUD_MODE,
72
+ ENV_ARTEMIS_CLOUD_SESSION_ID,
73
+ ENV_ARTEMIS_DEBUG,
74
+ ENV_ARTEMIS_DEFAULT_MODEL,
75
+ ENV_ARTEMIS_DEFAULT_PROFILE,
76
+ ENV_ARTEMIS_DEVICE_ID,
77
+ ENV_ARTEMIS_EDGE_PORT,
78
+ ENV_ARTEMIS_EXPLORER_VERSION,
79
+ ENV_ARTEMIS_IPC_PORT,
80
+ ENV_ARTEMIS_MCP_SERVER,
81
+ ENV_ARTEMIS_TENANT_ID,
82
+ ENV_ARTEMIS_TENANT_TOKEN,
83
+ ENV_ARTEMIS_TRACES_DIR,
84
+ ENV_ARTEMIS_USE_FILE_API,
85
+ ENV_ARTEMIS_USE_USER_DIR,
86
+ ENV_OPEN_ROUTER_API_KEY,
87
+ ENV_OPENAI_API_KEY,
88
+ ENV_OPENAI_BASE_URL,
89
+ ENV_RESULTS_OUTPUT_PATH,
90
+ ENV_XAI_API_KEY,
91
+ IPC_PORT_FILENAME,
92
+ LLM_CONFIG_FILENAME,
93
+ LLM_CONFIG_OVERRIDE_FILENAME,
94
+ LS_ADDRESS_FILENAME,
95
+ AgentNode,
96
+ ExplorerVersion,
97
+ LLMProvider,
98
+ LLMUtilsNode,
99
+ )
100
+ from artemis.config.llm import (
101
+ CyFunctionDetector,
102
+ LLM,
103
+ AgentNodeWithFallback,
104
+ LLMUtilsNodeWithFallback,
105
+ LLMConfig,
106
+ LLMConfigUtils,
107
+ LLMWithFallback,
108
+ deep_merge_llm_config,
109
+ get_default_llm_config,
110
+ initialize_llm_config,
111
+ load_llm_config_override,
112
+ parse_llm_config,
113
+ validate_vertex_ai_credentials,
114
+ )
115
+ from artemis.config.paths import (
116
+ CONFIG_DIR,
117
+ DB_PATH,
118
+ GLOBAL_APP_DIR,
119
+ GLOBAL_JETSKI_DIR,
120
+ IMAGES_DIR,
121
+ PAUSE_FILE,
122
+ REPLAY_BASE_DIR,
123
+ ROOT_DIR,
124
+ TEST_DATA_DIR,
125
+ TEST_OUTPUTS_DIR,
126
+ TRACES_PATH,
127
+ WORKSPACE_ROOT,
128
+ get_app_dir,
129
+ get_config_path,
130
+ get_data_engine_db_path,
131
+ get_default_traces_path,
132
+ get_env_file,
133
+ get_images_dir,
134
+ get_ipc_port_file,
135
+ get_ls_address_file,
136
+ get_pause_file,
137
+ get_replay_dir,
138
+ get_temp_dir,
139
+ get_test_data_dir,
140
+ get_test_outputs_dir,
141
+ get_traces_dir,
142
+ is_frozen_bundle,
143
+ is_source_checkout,
144
+ )
145
+ from artemis.config.runtime import (
146
+ cleanup_temp_dir,
147
+ clear_ipc_port,
148
+ clear_ls_address,
149
+ init_ls_address,
150
+ read_ipc_port,
151
+ read_ls_address,
152
+ write_ipc_port,
153
+ write_ls_address,
154
+ )
155
+ from artemis.config.settings import (
156
+ Settings,
157
+ settings,
158
+ )
159
+ from third_party.mobile_use.config.output import (
160
+ OutputConfig,
161
+ prepare_output_files,
162
+ record_events,
163
+ )
164
+
165
+ __all__ = [
166
+ # Settings & Singleton
167
+ "Settings",
168
+ "settings",
169
+ # Paths & Dirs
170
+ "ROOT_DIR",
171
+ "WORKSPACE_ROOT",
172
+ "CONFIG_DIR",
173
+ "GLOBAL_APP_DIR",
174
+ "GLOBAL_JETSKI_DIR",
175
+ "TRACES_PATH",
176
+ "DB_PATH",
177
+ "PAUSE_FILE",
178
+ "REPLAY_BASE_DIR",
179
+ "TEST_DATA_DIR",
180
+ "TEST_OUTPUTS_DIR",
181
+ "IMAGES_DIR",
182
+ "get_app_dir",
183
+ "get_traces_dir",
184
+ "get_default_traces_path",
185
+ "get_env_file",
186
+ "get_temp_dir",
187
+ "get_config_path",
188
+ "get_data_engine_db_path",
189
+ "get_pause_file",
190
+ "is_source_checkout",
191
+ "get_replay_dir",
192
+ "get_test_data_dir",
193
+ "get_test_outputs_dir",
194
+ "get_images_dir",
195
+ "get_ipc_port_file",
196
+ "get_ls_address_file",
197
+ "is_frozen_bundle",
198
+ # LLM Config & Models
199
+ "CyFunctionDetector",
200
+ "LLM",
201
+ "LLMWithFallback",
202
+ "LLMConfigUtils",
203
+ "LLMConfig",
204
+ "parse_llm_config",
205
+ "initialize_llm_config",
206
+ "get_default_llm_config",
207
+ "deep_merge_llm_config",
208
+ "load_llm_config_override",
209
+ "validate_vertex_ai_credentials",
210
+ # Agent & Tool Config
211
+ "AgentGlobalConfig",
212
+ "CheckerConfig",
213
+ "CommitteeConfig",
214
+ "DEFAULT_VERIFICATION_LEVEL",
215
+ "ExplorerConfig",
216
+ "FlashProfileConfig",
217
+ "MemoryChunkingConfig",
218
+ "MemoryConfig",
219
+ "MemoryRecallConfig",
220
+ "MemoryReplayConfig",
221
+ "MemoryRuntimeConfig",
222
+ "MemoryTranscriptConfig",
223
+ "OutputterConfig",
224
+ "PlannerValidationConfig",
225
+ "ProProfileConfig",
226
+ "StepSummarizerConfig",
227
+ "VideoAnalyzerConfig",
228
+ "load_agent_config",
229
+ "resolve_explorer_version",
230
+ "VERIFICATION_LEVEL_PRESETS",
231
+ "VerificationLevel",
232
+ "checker_overrides_for_level",
233
+ "run_tuning_for_profile",
234
+ "verification_level_for_checker",
235
+ # Output Config
236
+ "OutputConfig",
237
+ "prepare_output_files",
238
+ "record_events",
239
+ # Runtime & IPC State
240
+ "read_ipc_port",
241
+ "write_ipc_port",
242
+ "clear_ipc_port",
243
+ "read_ls_address",
244
+ "write_ls_address",
245
+ "clear_ls_address",
246
+ "init_ls_address",
247
+ "cleanup_temp_dir",
248
+ # Types & Literals
249
+ "LLMProvider",
250
+ "ExplorerVersion",
251
+ "LLMUtilsNode",
252
+ "LLMUtilsNodeWithFallback",
253
+ "AgentNode",
254
+ "AgentNodeWithFallback",
255
+ # Filenames
256
+ "LLM_CONFIG_FILENAME",
257
+ "LLM_CONFIG_OVERRIDE_FILENAME",
258
+ "AGENT_CONFIG_FILENAME",
259
+ "DATA_ENGINE_DB_FILENAME",
260
+ "IPC_PORT_FILENAME",
261
+ "LS_ADDRESS_FILENAME",
262
+ "DOTENV_FILENAME",
263
+ # Defaults
264
+ "DEFAULT_ADB_HOST",
265
+ "DEFAULT_ADB_PORT",
266
+ "DEFAULT_GATEWAY_PORT",
267
+ "DEFAULT_EDGE_MONITOR_PORT",
268
+ "DEFAULT_ACTION_TIMEOUT",
269
+ "DEFAULT_STREAM_PING_INTERVAL",
270
+ "DEFAULT_PROFILE",
271
+ "DEFAULT_MODEL",
272
+ "DEFAULT_EXPLORER_VERSION",
273
+ # Environment Variables
274
+ "ENV_ARTEMIS_APP_DIR",
275
+ "ENV_ANTIGRAVITY_APP_DIR",
276
+ "ENV_ARTEMIS_USE_USER_DIR",
277
+ "ENV_ARTEMIS_TRACES_DIR",
278
+ "ENV_DATA_ENGINE_DB_PATH",
279
+ "ENV_ARTEMIS_CLOUD_MODE",
280
+ "ENV_ARTEMIS_CLOUD_SESSION_ID",
281
+ "ENV_ARTEMIS_CLOUD_GATEWAY_URL",
282
+ "ENV_ARTEMIS_TENANT_ID",
283
+ "ENV_ARTEMIS_TENANT_TOKEN",
284
+ "ENV_ARTEMIS_EDGE_PORT",
285
+ "ENV_ARTEMIS_IPC_PORT",
286
+ "ENV_ANTIGRAVITY_LS_ADDRESS",
287
+ "ENV_ARTEMIS_MCP_SERVER",
288
+ "ENV_ADB_DEVICE_SERIAL",
289
+ "ENV_ARTEMIS_DEVICE_ID",
290
+ "ENV_ADB_HOST",
291
+ "ENV_ADB_PORT",
292
+ "ENV_ADB_SERVER_SOCKET",
293
+ "ENV_GOOGLE_API_KEY",
294
+ "ENV_GEMINI_API_KEY",
295
+ "ENV_GCP_API_KEY",
296
+ "ENV_OPENAI_API_KEY",
297
+ "ENV_OPENAI_BASE_URL",
298
+ "ENV_ANTHROPIC_API_KEY",
299
+ "ENV_OPEN_ROUTER_API_KEY",
300
+ "ENV_XAI_API_KEY",
301
+ "ENV_ARTEMIS_EXPLORER_VERSION",
302
+ "ENV_ARTEMIS_DEFAULT_PROFILE",
303
+ "ENV_ARTEMIS_DEFAULT_MODEL",
304
+ "ENV_ARTEMIS_USE_FILE_API",
305
+ "ENV_KEEP_VIDEOS",
306
+ "ENV_ARTEMIS_DEBUG",
307
+ "ENV_EVENTS_OUTPUT_PATH",
308
+ "ENV_RESULTS_OUTPUT_PATH",
309
+ ]