android-midscene-automation 0.2.4 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (525) hide show
  1. package/CHANGELOG.md +7 -0
  2. package/README.md +35 -0
  3. package/artemis-main/.dockerignore +15 -0
  4. package/artemis-main/.env.example +34 -0
  5. package/artemis-main/CONTRIBUTING.md +35 -0
  6. package/artemis-main/Dockerfile +68 -0
  7. package/artemis-main/LICENSE +202 -0
  8. package/artemis-main/Makefile +115 -0
  9. package/artemis-main/README.md +322 -0
  10. package/artemis-main/README_CN.md +320 -0
  11. package/artemis-main/apps/__init__.py +15 -0
  12. package/artemis-main/apps/admin_console/__init__.py +19 -0
  13. package/artemis-main/apps/admin_console/core/__init__.py +13 -0
  14. package/artemis-main/apps/admin_console/core/config.py +43 -0
  15. package/artemis-main/apps/admin_console/core/security.py +178 -0
  16. package/artemis-main/apps/admin_console/core/state.py +243 -0
  17. package/artemis-main/apps/admin_console/database/__init__.py +13 -0
  18. package/artemis-main/apps/admin_console/database/connection.py +60 -0
  19. package/artemis-main/apps/admin_console/database/repositories/__init__.py +13 -0
  20. package/artemis-main/apps/admin_console/database/repositories/session_repository.py +564 -0
  21. package/artemis-main/apps/admin_console/database/repositories/step_repository.py +512 -0
  22. package/artemis-main/apps/admin_console/database/repositories/trace_repository.py +133 -0
  23. package/artemis-main/apps/admin_console/index.html +3787 -0
  24. package/artemis-main/apps/admin_console/replay_manager.py +2585 -0
  25. package/artemis-main/apps/admin_console/routers/__init__.py +13 -0
  26. package/artemis-main/apps/admin_console/routers/media.py +198 -0
  27. package/artemis-main/apps/admin_console/routers/replay.py +91 -0
  28. package/artemis-main/apps/admin_console/routers/sessions.py +239 -0
  29. package/artemis-main/apps/admin_console/routers/steps.py +109 -0
  30. package/artemis-main/apps/admin_console/routers/stream.py +56 -0
  31. package/artemis-main/apps/admin_console/routers/system.py +761 -0
  32. package/artemis-main/apps/admin_console/routers/tasks.py +517 -0
  33. package/artemis-main/apps/admin_console/schemas/__init__.py +13 -0
  34. package/artemis-main/apps/admin_console/schemas/task_schema.py +47 -0
  35. package/artemis-main/apps/admin_console/server.py +508 -0
  36. package/artemis-main/apps/admin_console/services/__init__.py +13 -0
  37. package/artemis-main/apps/admin_console/services/device_stream_service.py +137 -0
  38. package/artemis-main/apps/admin_console/services/ipc_service.py +262 -0
  39. package/artemis-main/apps/admin_console/services/media_service.py +623 -0
  40. package/artemis-main/apps/admin_console/services/model_service.py +148 -0
  41. package/artemis-main/apps/admin_console/services/task_preset_catalog.py +514 -0
  42. package/artemis-main/apps/admin_console/services/task_queue_service.py +1719 -0
  43. package/artemis-main/apps/admin_console/services/worker_process_io.py +144 -0
  44. package/artemis-main/apps/showcase_ui/README.md +31 -0
  45. package/artemis-main/apps/showcase_ui/angular.json +132 -0
  46. package/artemis-main/apps/showcase_ui/package-lock.json +9472 -0
  47. package/artemis-main/apps/showcase_ui/package.json +39 -0
  48. package/artemis-main/apps/showcase_ui/proxy.conf.json +23 -0
  49. package/artemis-main/apps/showcase_ui/public/favicon.ico +0 -0
  50. package/artemis-main/apps/showcase_ui/public/logo.png +0 -0
  51. package/artemis-main/apps/showcase_ui/public/logo.svg +56 -0
  52. package/artemis-main/apps/showcase_ui/src/app/app.component.html +24 -0
  53. package/artemis-main/apps/showcase_ui/src/app/app.component.scss +49 -0
  54. package/artemis-main/apps/showcase_ui/src/app/app.component.spec.ts +42 -0
  55. package/artemis-main/apps/showcase_ui/src/app/app.component.ts +32 -0
  56. package/artemis-main/apps/showcase_ui/src/app/app.config.ts +29 -0
  57. package/artemis-main/apps/showcase_ui/src/app/app.routes.ts +27 -0
  58. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream-startup.spec.ts +108 -0
  59. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html +1436 -0
  60. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss +3973 -0
  61. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts +2235 -0
  62. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.html +380 -0
  63. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.scss +1171 -0
  64. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.ts +270 -0
  65. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html +433 -0
  66. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss +987 -0
  67. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts +632 -0
  68. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html +130 -0
  69. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.scss +406 -0
  70. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.ts +74 -0
  71. package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.scss +115 -0
  72. package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.ts +52 -0
  73. package/artemis-main/apps/showcase_ui/src/app/core/data/smart-tasks.data.ts +334 -0
  74. package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.spec.ts +55 -0
  75. package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.ts +61 -0
  76. package/artemis-main/apps/showcase_ui/src/app/core/i18n/translate.pipe.ts +11 -0
  77. package/artemis-main/apps/showcase_ui/src/app/core/i18n/zh-cn.ts +651 -0
  78. package/artemis-main/apps/showcase_ui/src/app/core/models/markdown.model.ts +43 -0
  79. package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.spec.ts +47 -0
  80. package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.ts +165 -0
  81. package/artemis-main/apps/showcase_ui/src/app/core/models/session.model.ts +85 -0
  82. package/artemis-main/apps/showcase_ui/src/app/core/models/stream.model.ts +246 -0
  83. package/artemis-main/apps/showcase_ui/src/app/core/models/system.model.ts +118 -0
  84. package/artemis-main/apps/showcase_ui/src/app/core/services/task-recommendation.service.ts +111 -0
  85. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.html +2130 -0
  86. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.scss +5204 -0
  87. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.ts +1189 -0
  88. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.html +39 -0
  89. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.scss +151 -0
  90. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.ts +58 -0
  91. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.html +201 -0
  92. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.scss +886 -0
  93. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.ts +324 -0
  94. package/artemis-main/apps/showcase_ui/src/app/services/agent.service.spec.ts +418 -0
  95. package/artemis-main/apps/showcase_ui/src/app/services/agent.service.ts +2012 -0
  96. package/artemis-main/apps/showcase_ui/src/app/services/system.service.spec.ts +173 -0
  97. package/artemis-main/apps/showcase_ui/src/app/services/system.service.ts +620 -0
  98. package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts +513 -0
  99. package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.ts +1015 -0
  100. package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.spec.ts +85 -0
  101. package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.ts +662 -0
  102. package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.spec.ts +70 -0
  103. package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.ts +383 -0
  104. package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.spec.ts +107 -0
  105. package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.ts +123 -0
  106. package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.spec.ts +92 -0
  107. package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.ts +116 -0
  108. package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts +640 -0
  109. package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts +1021 -0
  110. package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts +375 -0
  111. package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.ts +1419 -0
  112. package/artemis-main/apps/showcase_ui/src/index.html +37 -0
  113. package/artemis-main/apps/showcase_ui/src/main.ts +22 -0
  114. package/artemis-main/apps/showcase_ui/src/styles.scss +60 -0
  115. package/artemis-main/apps/showcase_ui/tsconfig.app.json +23 -0
  116. package/artemis-main/apps/showcase_ui/tsconfig.json +27 -0
  117. package/artemis-main/apps/showcase_ui/tsconfig.spec.json +23 -0
  118. package/artemis-main/artemis/__init__.py +44 -0
  119. package/artemis-main/artemis/__main__.py +18 -0
  120. package/artemis-main/artemis/_legacy_imports.py +94 -0
  121. package/artemis-main/artemis/_version.py +30 -0
  122. package/artemis-main/artemis/agents/__init__.py +17 -0
  123. package/artemis-main/artemis/agents/checker/checker.json +9 -0
  124. package/artemis-main/artemis/agents/checker/checker.py +859 -0
  125. package/artemis-main/artemis/agents/diagnoser/diagnoser.json +23 -0
  126. package/artemis-main/artemis/agents/diagnoser/diagnoser.py +608 -0
  127. package/artemis-main/artemis/agents/explorer/constants.py +52 -0
  128. package/artemis-main/artemis/agents/explorer/explorer.json +57 -0
  129. package/artemis-main/artemis/agents/explorer/explorer.py +473 -0
  130. package/artemis-main/artemis/agents/explorer/geometry.py +73 -0
  131. package/artemis-main/artemis/agents/explorer/native_runner.py +878 -0
  132. package/artemis-main/artemis/agents/explorer/perception_tools.py +632 -0
  133. package/artemis-main/artemis/agents/explorer/run_setup.py +403 -0
  134. package/artemis-main/artemis/agents/explorer/screen_index.py +266 -0
  135. package/artemis-main/artemis/agents/explorer/tiers.py +110 -0
  136. package/artemis-main/artemis/agents/explorer/tool_declarations.py +394 -0
  137. package/artemis-main/artemis/agents/explorer/universal_runner.py +317 -0
  138. package/artemis-main/artemis/agents/flash/context_compressor.py +452 -0
  139. package/artemis-main/artemis/agents/flash/flash_runner.md +51 -0
  140. package/artemis-main/artemis/agents/flash/flash_summarizer.md +53 -0
  141. package/artemis-main/artemis/agents/flash/flash_summarizer_single.md +54 -0
  142. package/artemis-main/artemis/agents/flash/runner.py +1167 -0
  143. package/artemis-main/artemis/agents/flash/summarizer.py +511 -0
  144. package/artemis-main/artemis/agents/history_analyzer/history_analyzer.md +8 -0
  145. package/artemis-main/artemis/agents/history_analyzer/history_analyzer.py +182 -0
  146. package/artemis-main/artemis/agents/image_processor/image_processor.md +74 -0
  147. package/artemis-main/artemis/agents/image_processor/image_processor.py +322 -0
  148. package/artemis-main/artemis/agents/log_analyzer/log_analyzer.md +15 -0
  149. package/artemis-main/artemis/agents/log_analyzer/log_analyzer.py +403 -0
  150. package/artemis-main/artemis/agents/log_analyzer/log_reader_sub_agent.md +8 -0
  151. package/artemis-main/artemis/agents/log_analyzer/output_analyzer.md +11 -0
  152. package/artemis-main/artemis/agents/log_analyzer/output_analyzer.py +163 -0
  153. package/artemis-main/artemis/agents/object_detector/object_detector.json +9 -0
  154. package/artemis-main/artemis/agents/object_detector/object_detector.py +211 -0
  155. package/artemis-main/artemis/agents/operator/operator.json +3 -0
  156. package/artemis-main/artemis/agents/operator/operator.py +1394 -0
  157. package/artemis-main/artemis/agents/operator/prompts.py +1064 -0
  158. package/artemis-main/artemis/agents/outputter/outputter.md +36 -0
  159. package/artemis-main/artemis/agents/outputter/outputter.py +552 -0
  160. package/artemis-main/artemis/agents/planner/planner.json +32 -0
  161. package/artemis-main/artemis/agents/planner/planner.py +436 -0
  162. package/artemis-main/artemis/agents/prompt_assembly.py +79 -0
  163. package/artemis-main/artemis/agents/summarizer/summarizer.py +190 -0
  164. package/artemis-main/artemis/agents/validator/action_execution.py +98 -0
  165. package/artemis-main/artemis/agents/validator/categories.py +33 -0
  166. package/artemis-main/artemis/agents/validator/execution_loop.py +488 -0
  167. package/artemis-main/artemis/agents/validator/incidents.py +120 -0
  168. package/artemis-main/artemis/agents/validator/pixel_safety_net.md +15 -0
  169. package/artemis-main/artemis/agents/validator/precondition_pixel.py +320 -0
  170. package/artemis-main/artemis/agents/validator/precondition_xml.py +672 -0
  171. package/artemis-main/artemis/agents/validator/tool_declarations.py +332 -0
  172. package/artemis-main/artemis/agents/validator/validator.py +188 -0
  173. package/artemis-main/artemis/agents/video_analyzer/__init__.py +15 -0
  174. package/artemis-main/artemis/agents/video_analyzer/audio_native.py +543 -0
  175. package/artemis-main/artemis/agents/video_analyzer/blackboard.py +821 -0
  176. package/artemis-main/artemis/agents/video_analyzer/chunk_agentic.py +455 -0
  177. package/artemis-main/artemis/agents/video_analyzer/chunk_conversation.py +491 -0
  178. package/artemis-main/artemis/agents/video_analyzer/chunk_native.py +437 -0
  179. package/artemis-main/artemis/agents/video_analyzer/conflict_resolution.py +137 -0
  180. package/artemis-main/artemis/agents/video_analyzer/gemini_files.py +162 -0
  181. package/artemis-main/artemis/agents/video_analyzer/native_coordinator.py +541 -0
  182. package/artemis-main/artemis/agents/video_analyzer/reliability.py +128 -0
  183. package/artemis-main/artemis/agents/video_analyzer/universal_engine.py +487 -0
  184. package/artemis-main/artemis/agents/video_analyzer/universal_tools.py +323 -0
  185. package/artemis-main/artemis/agents/video_analyzer/video_analyzer.md +51 -0
  186. package/artemis-main/artemis/agents/video_analyzer/video_analyzer.py +875 -0
  187. package/artemis-main/artemis/agents/video_analyzer/video_sub_agent.md +15 -0
  188. package/artemis-main/artemis/agents/video_analyzer/video_sub_agent_audio.md +12 -0
  189. package/artemis-main/artemis/clients/accessibility_client.py +438 -0
  190. package/artemis-main/artemis/clients/screen_client_factory.py +403 -0
  191. package/artemis-main/artemis/config/__init__.py +309 -0
  192. package/artemis-main/artemis/config/agent.py +1098 -0
  193. package/artemis-main/artemis/config/constants.py +151 -0
  194. package/artemis-main/artemis/config/core/__init__.py +24 -0
  195. package/artemis-main/artemis/config/core/engine.py +118 -0
  196. package/artemis-main/artemis/config/core/interpolator.py +78 -0
  197. package/artemis-main/artemis/config/llm.py +267 -0
  198. package/artemis-main/artemis/config/paths.py +244 -0
  199. package/artemis-main/artemis/config/runtime.py +197 -0
  200. package/artemis-main/artemis/config/settings.py +315 -0
  201. package/artemis-main/artemis/constants.py +47 -0
  202. package/artemis-main/artemis/context.py +289 -0
  203. package/artemis-main/artemis/controllers/__init__.py +41 -0
  204. package/artemis-main/artemis/controllers/controller_factory.py +41 -0
  205. package/artemis-main/artemis/controllers/device_controller.py +29 -0
  206. package/artemis-main/artemis/controllers/types.py +37 -0
  207. package/artemis-main/artemis/controllers/unified_controller.py +649 -0
  208. package/artemis-main/artemis/core/__init__.py +21 -0
  209. package/artemis-main/artemis/core/diagnostics/__init__.py +90 -0
  210. package/artemis-main/artemis/core/diagnostics/adb_keys.py +270 -0
  211. package/artemis-main/artemis/core/diagnostics/adb_server_connection.py +330 -0
  212. package/artemis-main/artemis/core/diagnostics/device_smoke.py +364 -0
  213. package/artemis-main/artemis/core/diagnostics/emulator_manager.py +501 -0
  214. package/artemis-main/artemis/core/diagnostics/engine.py +438 -0
  215. package/artemis-main/artemis/core/diagnostics/hierarchy_parity.py +216 -0
  216. package/artemis-main/artemis/core/diagnostics/probes/adb_probe.py +887 -0
  217. package/artemis-main/artemis/core/diagnostics/probes/base.py +45 -0
  218. package/artemis-main/artemis/core/diagnostics/probes/credentials_probe.py +296 -0
  219. package/artemis-main/artemis/core/diagnostics/probes/host_probe.py +427 -0
  220. package/artemis-main/artemis/core/diagnostics/probes/runtime_probe.py +169 -0
  221. package/artemis-main/artemis/core/diagnostics/probes/toolchain_probe.py +139 -0
  222. package/artemis-main/artemis/core/diagnostics/readiness.py +83 -0
  223. package/artemis-main/artemis/core/diagnostics/schema.py +118 -0
  224. package/artemis-main/artemis/core/tool_declaration.py +43 -0
  225. package/artemis-main/artemis/core/tool_failure.py +56 -0
  226. package/artemis-main/artemis/data_engine/context_vars.py +24 -0
  227. package/artemis-main/artemis/data_engine/engine.py +1615 -0
  228. package/artemis-main/artemis/data_engine/history_reader.py +142 -0
  229. package/artemis-main/artemis/data_engine/models.py +131 -0
  230. package/artemis-main/artemis/data_engine/storage.py +1245 -0
  231. package/artemis-main/artemis/data_engine/trace.py +727 -0
  232. package/artemis-main/artemis/drivers/__init__.py +48 -0
  233. package/artemis-main/artemis/drivers/android/adb_driver.py +507 -0
  234. package/artemis-main/artemis/drivers/android/hierarchy.py +64 -0
  235. package/artemis-main/artemis/drivers/android/input_ime.py +120 -0
  236. package/artemis-main/artemis/drivers/android/recorder.py +57 -0
  237. package/artemis-main/artemis/drivers/base.py +290 -0
  238. package/artemis-main/artemis/drivers/cloud/cloud_driver.py +117 -0
  239. package/artemis-main/artemis/drivers/factory.py +80 -0
  240. package/artemis-main/artemis/drivers/mock/mock_driver.py +198 -0
  241. package/artemis-main/artemis/drivers/types.py +67 -0
  242. package/artemis-main/artemis/graph/checkpoints.py +1243 -0
  243. package/artemis-main/artemis/graph/graph.py +1055 -0
  244. package/artemis-main/artemis/graph/perception.py +234 -0
  245. package/artemis-main/artemis/graph/state.py +151 -0
  246. package/artemis-main/artemis/graph/visibility.py +238 -0
  247. package/artemis-main/artemis/interfaces/cli/__init__.py +19 -0
  248. package/artemis-main/artemis/interfaces/cli/commands/__init__.py +15 -0
  249. package/artemis-main/artemis/interfaces/cli/commands/batch.py +278 -0
  250. package/artemis-main/artemis/interfaces/cli/commands/doctor.py +468 -0
  251. package/artemis-main/artemis/interfaces/cli/commands/helper.py +226 -0
  252. package/artemis-main/artemis/interfaces/cli/commands/init.py +193 -0
  253. package/artemis-main/artemis/interfaces/cli/commands/mcp.py +862 -0
  254. package/artemis-main/artemis/interfaces/cli/commands/run.py +463 -0
  255. package/artemis-main/artemis/interfaces/cli/commands/server.py +151 -0
  256. package/artemis-main/artemis/interfaces/cli/commands/server_lifecycle.py +328 -0
  257. package/artemis-main/artemis/interfaces/cli/commands/trace.py +111 -0
  258. package/artemis-main/artemis/interfaces/cli/commands/ui.py +404 -0
  259. package/artemis-main/artemis/interfaces/cli/main.py +109 -0
  260. package/artemis-main/artemis/interfaces/sdk/__init__.py +31 -0
  261. package/artemis-main/artemis/interfaces/sdk/client.py +19 -0
  262. package/artemis-main/artemis/interfaces/sdk/task.py +72 -0
  263. package/artemis-main/artemis/llm/__init__.py +25 -0
  264. package/artemis-main/artemis/llm/fake_model.py +179 -0
  265. package/artemis-main/artemis/llm/google/__init__.py +57 -0
  266. package/artemis-main/artemis/llm/google/provider.py +185 -0
  267. package/artemis-main/artemis/llm/google/usage.py +148 -0
  268. package/artemis-main/artemis/llm/parser.py +32 -0
  269. package/artemis-main/artemis/llm/reliability.py +267 -0
  270. package/artemis-main/artemis/llm/router.py +389 -0
  271. package/artemis-main/artemis/llm/schemas.py +36 -0
  272. package/artemis-main/artemis/llm/structured.py +196 -0
  273. package/artemis-main/artemis/main.py +58 -0
  274. package/artemis-main/artemis/mcp/__init__.py +20 -0
  275. package/artemis-main/artemis/mcp/action_executor.py +760 -0
  276. package/artemis-main/artemis/mcp/action_manifest.py +320 -0
  277. package/artemis-main/artemis/mcp/action_names.py +172 -0
  278. package/artemis-main/artemis/mcp/action_server.py +202 -0
  279. package/artemis-main/artemis/mcp/action_session.py +245 -0
  280. package/artemis-main/artemis/mcp/action_specs.py +868 -0
  281. package/artemis-main/artemis/mcp/action_types.py +86 -0
  282. package/artemis-main/artemis/mcp/actuators/__init__.py +21 -0
  283. package/artemis-main/artemis/mcp/actuators/adb.py +414 -0
  284. package/artemis-main/artemis/mcp/actuators/base.py +113 -0
  285. package/artemis-main/artemis/mcp/actuators/mock.py +70 -0
  286. package/artemis-main/artemis/mcp/adb_server.py +490 -0
  287. package/artemis-main/artemis/mcp/observation.py +133 -0
  288. package/artemis-main/artemis/mcp/xml_search_server.py +297 -0
  289. package/artemis-main/artemis/memory/__init__.py +90 -0
  290. package/artemis-main/artemis/memory/chunking.py +1843 -0
  291. package/artemis-main/artemis/memory/context_policy.py +226 -0
  292. package/artemis-main/artemis/memory/step_capsule.md +73 -0
  293. package/artemis-main/artemis/memory/step_memory.py +281 -0
  294. package/artemis-main/artemis/memory/transcript.py +798 -0
  295. package/artemis-main/artemis/platform/__init__.py +61 -0
  296. package/artemis-main/artemis/platform/base.py +104 -0
  297. package/artemis-main/artemis/platform/darwin.py +184 -0
  298. package/artemis-main/artemis/platform/linux.py +207 -0
  299. package/artemis-main/artemis/platform/windows.py +193 -0
  300. package/artemis-main/artemis/resources/__init__.py +46 -0
  301. package/artemis-main/artemis/runtime/__init__.py +108 -0
  302. package/artemis-main/artemis/runtime/adb_endpoint.py +173 -0
  303. package/artemis-main/artemis/runtime/awake_lease.py +283 -0
  304. package/artemis-main/artemis/runtime/awake_service.py +343 -0
  305. package/artemis-main/artemis/runtime/cancel_requests.py +254 -0
  306. package/artemis-main/artemis/runtime/daemon_client.py +422 -0
  307. package/artemis-main/artemis/runtime/device_lock.py +1013 -0
  308. package/artemis-main/artemis/runtime/device_pool.py +478 -0
  309. package/artemis-main/artemis/runtime/helper_manager.py +1030 -0
  310. package/artemis-main/artemis/runtime/process_probe.py +98 -0
  311. package/artemis-main/artemis/runtime/server_lifecycle.py +435 -0
  312. package/artemis-main/artemis/runtime/supervisor.py +121 -0
  313. package/artemis-main/artemis/runtime/trace_store.py +375 -0
  314. package/artemis-main/artemis/sdk/__init__.py +26 -0
  315. package/artemis-main/artemis/sdk/agent.py +485 -0
  316. package/artemis-main/artemis/sdk/builders/__init__.py +29 -0
  317. package/artemis-main/artemis/sdk/builders/agent_config_builder.py +474 -0
  318. package/artemis-main/artemis/sdk/constants.py +15 -0
  319. package/artemis-main/artemis/sdk/run_outcome.py +58 -0
  320. package/artemis-main/artemis/sdk/types/__init__.py +61 -0
  321. package/artemis-main/artemis/sdk/types/agent.py +89 -0
  322. package/artemis-main/artemis/sdk/utils.py +28 -0
  323. package/artemis-main/artemis/services/llm.py +1040 -0
  324. package/artemis-main/artemis/services/token_meter.py +267 -0
  325. package/artemis-main/artemis/telemetry/__init__.py +35 -0
  326. package/artemis-main/artemis/telemetry/models.py +64 -0
  327. package/artemis-main/artemis/telemetry/replay.py +34 -0
  328. package/artemis-main/artemis/telemetry/storage/json_storage.py +47 -0
  329. package/artemis-main/artemis/telemetry/tracer.py +85 -0
  330. package/artemis-main/artemis/toolchain/__init__.py +52 -0
  331. package/artemis-main/artemis/toolchain/descriptors.py +92 -0
  332. package/artemis-main/artemis/toolchain/resolver.py +185 -0
  333. package/artemis-main/artemis/tools/base.py +232 -0
  334. package/artemis-main/artemis/tools/command_tool.py +1240 -0
  335. package/artemis-main/artemis/tools/committee_prompts.json +5 -0
  336. package/artemis-main/artemis/tools/committee_tool.py +422 -0
  337. package/artemis-main/artemis/tools/diagnoser_submit_answer_tool.py +87 -0
  338. package/artemis-main/artemis/tools/diagnostic_tool.py +120 -0
  339. package/artemis-main/artemis/tools/explorer_tool.py +581 -0
  340. package/artemis-main/artemis/tools/history/__init__.py +398 -0
  341. package/artemis-main/artemis/tools/history/replay.py +120 -0
  342. package/artemis-main/artemis/tools/history/screenshot.py +139 -0
  343. package/artemis-main/artemis/tools/history/search.py +563 -0
  344. package/artemis-main/artemis/tools/image_processor_tool.py +124 -0
  345. package/artemis-main/artemis/tools/index.py +47 -0
  346. package/artemis-main/artemis/tools/log_tool.py +108 -0
  347. package/artemis-main/artemis/tools/mobile/launch_app.py +132 -0
  348. package/artemis-main/artemis/tools/mobile/log_utils.py +138 -0
  349. package/artemis-main/artemis/tools/mobile/ocr.py +202 -0
  350. package/artemis-main/artemis/tools/mobile/read_hierarchy.py +108 -0
  351. package/artemis-main/artemis/tools/mobile/read_logs.py +116 -0
  352. package/artemis-main/artemis/tools/mobile/search_logs.py +260 -0
  353. package/artemis-main/artemis/tools/object_detection_tool.py +244 -0
  354. package/artemis-main/artemis/tools/scratchpad.py +819 -0
  355. package/artemis-main/artemis/tools/system/adb_shell.py +31 -0
  356. package/artemis-main/artemis/tools/tool_wrapper.py +270 -0
  357. package/artemis-main/artemis/tools/video_tool.py +261 -0
  358. package/artemis-main/artemis/tools/wait_tool.py +97 -0
  359. package/artemis-main/artemis/utils/app_launch_utils.py +207 -0
  360. package/artemis-main/artemis/utils/coordinates.py +606 -0
  361. package/artemis-main/artemis/utils/credentials_validator.py +201 -0
  362. package/artemis-main/artemis/utils/cv_canvas.py +189 -0
  363. package/artemis-main/artemis/utils/cython_compat.py +45 -0
  364. package/artemis-main/artemis/utils/element_hit_test.py +80 -0
  365. package/artemis-main/artemis/utils/image_diff.py +335 -0
  366. package/artemis-main/artemis/utils/image_hash.py +80 -0
  367. package/artemis-main/artemis/utils/notes.py +470 -0
  368. package/artemis-main/artemis/utils/ocr_api.py +105 -0
  369. package/artemis-main/artemis/utils/ocr_xml_fusion.py +318 -0
  370. package/artemis-main/artemis/utils/plan_grammar.py +513 -0
  371. package/artemis-main/artemis/utils/python_executor.py +213 -0
  372. package/artemis-main/artemis/utils/startup_progress.py +60 -0
  373. package/artemis-main/artemis/utils/task_tree.py +1174 -0
  374. package/artemis-main/artemis/utils/text.py +37 -0
  375. package/artemis-main/artemis/utils/ui_filter.py +651 -0
  376. package/artemis-main/artemis/utils/video.py +685 -0
  377. package/artemis-main/artemis/utils/visualization.py +802 -0
  378. package/artemis-main/config/artemis.example.jsonc +12 -0
  379. package/artemis-main/mcp_server/README.md +111 -0
  380. package/artemis-main/mcp_server/__init__.py +21 -0
  381. package/artemis-main/mcp_server/__main__.py +20 -0
  382. package/artemis-main/mcp_server/background/__init__.py +15 -0
  383. package/artemis-main/mcp_server/background/task_runner.py +453 -0
  384. package/artemis-main/mcp_server/base.py +49 -0
  385. package/artemis-main/mcp_server/notifiers/__init__.py +74 -0
  386. package/artemis-main/mcp_server/notifiers/agentapi.py +289 -0
  387. package/artemis-main/mcp_server/notifiers/base.py +45 -0
  388. package/artemis-main/mcp_server/notifiers/composite.py +87 -0
  389. package/artemis-main/mcp_server/notifiers/desktop.py +108 -0
  390. package/artemis-main/mcp_server/notifiers/file.py +69 -0
  391. package/artemis-main/mcp_server/notifiers/script.py +90 -0
  392. package/artemis-main/mcp_server/notifiers/webhook.py +87 -0
  393. package/artemis-main/mcp_server/rules.md +68 -0
  394. package/artemis-main/mcp_server/server.py +106 -0
  395. package/artemis-main/mcp_server/tools/__init__.py +29 -0
  396. package/artemis-main/mcp_server/tools/device_state.py +107 -0
  397. package/artemis-main/mcp_server/tools/diagnose.py +1249 -0
  398. package/artemis-main/mcp_server/tools/inspect_trace.py +394 -0
  399. package/artemis-main/mcp_server/tools/task_manager.py +627 -0
  400. package/artemis-main/mcp_server/tools/task_runner.py +548 -0
  401. package/artemis-main/mcp_server/utils/__init__.py +15 -0
  402. package/artemis-main/mcp_server/utils/device_utils.py +180 -0
  403. package/artemis-main/mcp_server/utils/env_utils.py +65 -0
  404. package/artemis-main/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk +0 -0
  405. package/artemis-main/packages/artemis-accessibility-helper/app/build.gradle.kts +35 -0
  406. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml +49 -0
  407. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java +274 -0
  408. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ArtemisAccessibilityService.java +227 -0
  409. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java +463 -0
  410. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/DisplayUtils.java +111 -0
  411. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java +232 -0
  412. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/HierarchyDumper.java +832 -0
  413. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenReceiver.java +36 -0
  414. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenStore.java +38 -0
  415. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/XmlUtils.java +102 -0
  416. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/values/strings.xml +4 -0
  417. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/xml/accessibility_service_config.xml +10 -0
  418. package/artemis-main/packages/artemis-accessibility-helper/build.gradle.kts +3 -0
  419. package/artemis-main/packages/artemis-accessibility-helper/build_apk.sh +215 -0
  420. package/artemis-main/packages/artemis-accessibility-helper/debug.keystore +0 -0
  421. package/artemis-main/packages/artemis-accessibility-helper/gradle.properties +2 -0
  422. package/artemis-main/packages/artemis-accessibility-helper/helper_manifest.json +7 -0
  423. package/artemis-main/packages/artemis-accessibility-helper/settings.gradle.kts +17 -0
  424. package/artemis-main/packages/artemis-client/DESIGN.md +89 -0
  425. package/artemis-main/packages/artemis-client/README.md +112 -0
  426. package/artemis-main/packages/artemis-client/examples/basic.py +30 -0
  427. package/artemis-main/packages/artemis-client/pyproject.toml +36 -0
  428. package/artemis-main/packages/artemis-client/src/artemis_client/__init__.py +45 -0
  429. package/artemis-main/packages/artemis-client/src/artemis_client/client.py +401 -0
  430. package/artemis-main/packages/artemis-client/src/artemis_client/errors.py +66 -0
  431. package/artemis-main/packages/artemis-client/src/artemis_client/models.py +212 -0
  432. package/artemis-main/packages/artemis-client/src/artemis_client/py.typed +1 -0
  433. package/artemis-main/packages/artemis-client/src/artemis_client/transport.py +134 -0
  434. package/artemis-main/packages/artemis-client/tests/test_client.py +289 -0
  435. package/artemis-main/packages/artemis-client/tests/test_transport.py +85 -0
  436. package/artemis-main/pyproject.toml +259 -0
  437. package/artemis-main/pyright-core.json +17 -0
  438. package/artemis-main/scripts/check_dependency_sources.py +68 -0
  439. package/artemis-main/scripts/install_deps.ps1 +520 -0
  440. package/artemis-main/scripts/install_deps.sh +533 -0
  441. package/artemis-main/scripts/quality_ratchet.py +97 -0
  442. package/artemis-main/scripts/real_device_video_acceptance.py +483 -0
  443. package/artemis-main/scripts/run_edge_case_part.py +327 -0
  444. package/artemis-main/scripts/run_ui.py +242 -0
  445. package/artemis-main/scripts/start.ps1 +500 -0
  446. package/artemis-main/scripts/test_all_edge_cases.py +269 -0
  447. package/artemis-main/scripts/verify_concurrency_and_devices.py +385 -0
  448. package/artemis-main/setup.py +164 -0
  449. package/artemis-main/start.bat +35 -0
  450. package/artemis-main/start.sh +423 -0
  451. package/artemis-main/third_party/mobile_use/LICENSE +190 -0
  452. package/artemis-main/third_party/mobile_use/METADATA +27 -0
  453. package/artemis-main/third_party/mobile_use/NOTICE +12 -0
  454. package/artemis-main/third_party/mobile_use/__init__.py +19 -0
  455. package/artemis-main/third_party/mobile_use/agents/__init__.py +13 -0
  456. package/artemis-main/third_party/mobile_use/agents/hopper/__init__.py +13 -0
  457. package/artemis-main/third_party/mobile_use/agents/hopper/hopper.md +22 -0
  458. package/artemis-main/third_party/mobile_use/agents/hopper/hopper.py +80 -0
  459. package/artemis-main/third_party/mobile_use/clients/__init__.py +13 -0
  460. package/artemis-main/third_party/mobile_use/clients/ui_automator_client.py +482 -0
  461. package/artemis-main/third_party/mobile_use/config/__init__.py +13 -0
  462. package/artemis-main/third_party/mobile_use/config/llm.py +183 -0
  463. package/artemis-main/third_party/mobile_use/config/output.py +184 -0
  464. package/artemis-main/third_party/mobile_use/context.py +119 -0
  465. package/artemis-main/third_party/mobile_use/controllers/__init__.py +13 -0
  466. package/artemis-main/third_party/mobile_use/controllers/platform_specific_commands_controller.py +151 -0
  467. package/artemis-main/third_party/mobile_use/controllers/types.py +92 -0
  468. package/artemis-main/third_party/mobile_use/controllers/unified_controller.py +183 -0
  469. package/artemis-main/third_party/mobile_use/graph.py +54 -0
  470. package/artemis-main/third_party/mobile_use/llm_service.py +193 -0
  471. package/artemis-main/third_party/mobile_use/main.py +130 -0
  472. package/artemis-main/third_party/mobile_use/sdk/__init__.py +13 -0
  473. package/artemis-main/third_party/mobile_use/sdk/agent.py +1060 -0
  474. package/artemis-main/third_party/mobile_use/sdk/builders/__init__.py +13 -0
  475. package/artemis-main/third_party/mobile_use/sdk/builders/agent_config_builder.py +225 -0
  476. package/artemis-main/third_party/mobile_use/sdk/builders/index.py +33 -0
  477. package/artemis-main/third_party/mobile_use/sdk/builders/task_request_builder.py +243 -0
  478. package/artemis-main/third_party/mobile_use/sdk/types/__init__.py +13 -0
  479. package/artemis-main/third_party/mobile_use/sdk/types/agent.py +98 -0
  480. package/artemis-main/third_party/mobile_use/sdk/types/exceptions.py +109 -0
  481. package/artemis-main/third_party/mobile_use/sdk/types/task.py +234 -0
  482. package/artemis-main/third_party/mobile_use/testing.py +59 -0
  483. package/artemis-main/third_party/mobile_use/tools/__init__.py +13 -0
  484. package/artemis-main/third_party/mobile_use/tools/index.py +44 -0
  485. package/artemis-main/third_party/mobile_use/tools/mobile/__init__.py +13 -0
  486. package/artemis-main/third_party/mobile_use/tools/mobile/launch_app.py +89 -0
  487. package/artemis-main/third_party/mobile_use/tools/tool_wrapper.py +52 -0
  488. package/artemis-main/third_party/mobile_use/utils/__init__.py +13 -0
  489. package/artemis-main/third_party/mobile_use/utils/app_launch_utils.py +217 -0
  490. package/artemis-main/third_party/mobile_use/utils/cli_helpers.py +60 -0
  491. package/artemis-main/third_party/mobile_use/utils/decorators.py +158 -0
  492. package/artemis-main/third_party/mobile_use/utils/file.py +31 -0
  493. package/artemis-main/third_party/mobile_use/utils/logger.py +216 -0
  494. package/artemis-main/third_party/mobile_use/utils/media.py +213 -0
  495. package/artemis-main/third_party/mobile_use/utils/shell_utils.py +39 -0
  496. package/artemis-main/third_party/mobile_use/utils/video.py +398 -0
  497. package/artemis-main/uv.lock +4908 -0
  498. package/bin/android-midscene-automation.js +1 -5
  499. package/bin/artemis-config.mjs +15 -0
  500. package/bin/artemis-embedded.py +56 -0
  501. package/bin/start-artemis.mjs +30 -0
  502. package/bin/start-dev.mjs +112 -0
  503. package/bin/test_artemis_embedded.py +44 -0
  504. package/package.json +4 -2
  505. package/server/artemis.ts +34 -0
  506. package/server/http-api.ts +12 -0
  507. package/src/App.vue +10 -5
  508. package/src/api.ts +5 -0
  509. package/src/appium-recorder/AppiumPage.vue +1 -1
  510. package/src/appium-recorder/components/ComponentInspector.vue +1 -1
  511. package/src/appium-recorder/components/FlowNodeSearch.vue +3 -3
  512. package/src/appium-recorder/components/ImageCheckDialog.vue +3 -3
  513. package/src/appium-recorder/components/RecorderWorkspace.vue +4 -4
  514. package/src/appium-recorder/flow-appearance.ts +4 -4
  515. package/src/components/AppSidebar.vue +8 -7
  516. package/src/components/MidsceneMaintenanceNotice.vue +17 -0
  517. package/src/components/analytics/AnalysisCharts.vue +4 -4
  518. package/src/components/analytics/ReportAnalysis.vue +1 -1
  519. package/src/components/generator/TestCaseFileUpload.vue +1 -1
  520. package/src/components/help/HelpCenter.vue +56 -37
  521. package/src/pages/AiGeneratorPage.vue +1 -1
  522. package/src/pages/ArtemisPage.vue +72 -0
  523. package/src/pages/ConfigPage.vue +18 -19
  524. package/src/style.css +340 -121
  525. package/src/types.ts +1 -1
@@ -0,0 +1,227 @@
1
+ package com.artemis.helper;
2
+
3
+ import android.accessibilityservice.AccessibilityService;
4
+ import android.accessibilityservice.AccessibilityServiceInfo;
5
+ import android.app.Notification;
6
+ import android.app.NotificationChannel;
7
+ import android.app.NotificationManager;
8
+ import android.content.Context;
9
+ import android.content.Intent;
10
+ import android.content.pm.PackageInfo;
11
+ import android.content.pm.ServiceInfo;
12
+ import android.os.Build;
13
+ import android.util.Log;
14
+ import android.view.accessibility.AccessibilityEvent;
15
+
16
+ /**
17
+ * Artemis non-exclusive Accessibility Service with foreground keep-alive support.
18
+ *
19
+ * Runs concurrently with any Android UI testing framework (Mobly, Appium, Espresso)
20
+ * without monopolizing Android's singleton UiAutomationService connection.
21
+ *
22
+ * The service subscribes to window-state changes only: that is the single event it
23
+ * uses (to remember the foreground package / activity), and anything wider costs
24
+ * CPU and battery on a device that is idle between tasks.
25
+ */
26
+ public class ArtemisAccessibilityService extends AccessibilityService {
27
+
28
+ private static final String TAG = "ArtemisA11yService";
29
+ public static final int DEFAULT_PORT = 18888;
30
+ /**
31
+ * Bumped whenever the HTTP contract changes in a way an older host cannot use.
32
+ * 2: token authentication, byte-accurate bodies, visible-only dumps, fields=.
33
+ */
34
+ public static final int PROTOCOL_VERSION = 2;
35
+ // Channel importance is frozen by Android once a channel exists, so a quieter
36
+ // channel needs a new id; the pre-1.1.3 channel is deleted on start.
37
+ private static final String LEGACY_CHANNEL_ID = "artemis_helper_channel";
38
+ private static final String CHANNEL_ID = "artemis_helper_quiet";
39
+ private static final int NOTIFICATION_ID = 18888;
40
+
41
+ private static volatile ArtemisAccessibilityService instance;
42
+
43
+ private volatile String currentPackageName = "";
44
+ private volatile String currentActivityName = "";
45
+ private CommandServer server;
46
+
47
+ public static ArtemisAccessibilityService getInstance() {
48
+ return instance;
49
+ }
50
+
51
+ public String getCurrentPackageName() {
52
+ return currentPackageName;
53
+ }
54
+
55
+ public String getCurrentActivityName() {
56
+ return currentActivityName;
57
+ }
58
+
59
+ /** Installed helper versionCode, or -1 when the package manager cannot answer. */
60
+ @SuppressWarnings("deprecation")
61
+ public long getVersionCode() {
62
+ try {
63
+ PackageInfo info = getPackageManager().getPackageInfo(getPackageName(), 0);
64
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
65
+ return info.getLongVersionCode();
66
+ }
67
+ return info.versionCode;
68
+ } catch (Throwable t) {
69
+ return -1L;
70
+ }
71
+ }
72
+
73
+ public String getVersionName() {
74
+ try {
75
+ PackageInfo info = getPackageManager().getPackageInfo(getPackageName(), 0);
76
+ return info.versionName == null ? "" : info.versionName;
77
+ } catch (Throwable t) {
78
+ return "";
79
+ }
80
+ }
81
+
82
+ @Override
83
+ public void onServiceConnected() {
84
+ super.onServiceConnected();
85
+ instance = this;
86
+ Log.i(TAG, "ArtemisAccessibilityService connected");
87
+
88
+ // 1. Start foreground service to resist low-memory killer on aggressive custom ROMs
89
+ startForegroundNotification();
90
+
91
+ // 2. Dynamically enforce flags to guarantee compatibility across custom OEM ROMs
92
+ try {
93
+ AccessibilityServiceInfo info = getServiceInfo();
94
+ if (info == null) {
95
+ info = new AccessibilityServiceInfo();
96
+ }
97
+ info.eventTypes = AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED;
98
+ info.feedbackType = AccessibilityServiceInfo.FEEDBACK_GENERIC;
99
+ info.notificationTimeout = 100;
100
+ info.flags |= AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
101
+ | AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS;
102
+ // Not-important views are what UIAutomator's compressed dump leaves out;
103
+ // keep them out here too so both backends describe the same tree.
104
+ info.flags &= ~AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS;
105
+ setServiceInfo(info);
106
+ Log.i(TAG, "AccessibilityServiceInfo flags dynamically enforced");
107
+ } catch (Throwable t) {
108
+ Log.w(TAG, "Failed to dynamically configure AccessibilityServiceInfo", t);
109
+ }
110
+
111
+ // 3. Start local loopback command server
112
+ if (server != null) {
113
+ server.shutdown();
114
+ server = null;
115
+ }
116
+
117
+ server = new CommandServer(this, DEFAULT_PORT);
118
+ server.setDaemon(true);
119
+ server.start();
120
+ Log.i(TAG, "CommandServer started on port " + DEFAULT_PORT);
121
+ }
122
+
123
+ @Override
124
+ public int onStartCommand(Intent intent, int flags, int startId) {
125
+ return START_STICKY;
126
+ }
127
+
128
+ @SuppressWarnings("deprecation")
129
+ private void startForegroundNotification() {
130
+ try {
131
+ NotificationManager nm = (NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE);
132
+ if (nm == null) return;
133
+
134
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
135
+ try {
136
+ nm.deleteNotificationChannel(LEGACY_CHANNEL_ID);
137
+ } catch (Throwable ignored) {}
138
+ // Ask for IMPORTANCE_MIN (no sound, collapsed in the shade). Android raises
139
+ // foreground-service channels to LOW at most, which is still silent. The
140
+ // notification exists only to keep the service alive on aggressive ROMs;
141
+ // it must not look like something the person needs to act on.
142
+ NotificationChannel channel = new NotificationChannel(
143
+ CHANNEL_ID,
144
+ "Artemis test helper",
145
+ NotificationManager.IMPORTANCE_MIN
146
+ );
147
+ channel.setDescription("Shown while the Artemis test helper is installed on this device");
148
+ channel.setShowBadge(false);
149
+ nm.createNotificationChannel(channel);
150
+ }
151
+
152
+ Notification.Builder builder;
153
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
154
+ builder = new Notification.Builder(this, CHANNEL_ID);
155
+ } else {
156
+ builder = new Notification.Builder(this);
157
+ }
158
+
159
+ builder.setContentTitle("Artemis test helper is running")
160
+ .setContentText("Lets Artemis read this screen during automated tests. "
161
+ + "Remove with: artemis helper uninstall")
162
+ .setStyle(new Notification.BigTextStyle().bigText(
163
+ "Installed by Artemis to read the screen layout during automated "
164
+ + "tests. It answers only on this device (127.0.0.1:" + DEFAULT_PORT
165
+ + "), only to the computer that is connected over USB debugging and "
166
+ + "holds the session token, and it sends nothing anywhere. Remove it "
167
+ + "any time with `artemis helper uninstall` or from Settings > Apps."))
168
+ .setSmallIcon(android.R.drawable.stat_notify_sync)
169
+ .setOngoing(true);
170
+ if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
171
+ builder.setPriority(Notification.PRIORITY_MIN);
172
+ }
173
+
174
+ if (Build.VERSION.SDK_INT >= 34) {
175
+ startForeground(NOTIFICATION_ID, builder.build(), ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE);
176
+ } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
177
+ startForeground(NOTIFICATION_ID, builder.build(), 0);
178
+ } else {
179
+ startForeground(NOTIFICATION_ID, builder.build());
180
+ }
181
+ Log.i(TAG, "Foreground keep-alive notification active");
182
+ } catch (Throwable t) {
183
+ Log.w(TAG, "Foreground notification start skipped or deferred: " + t.getMessage());
184
+ }
185
+ }
186
+
187
+ @SuppressWarnings("deprecation")
188
+ private void stopForegroundNotification() {
189
+ try {
190
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
191
+ stopForeground(STOP_FOREGROUND_REMOVE);
192
+ } else {
193
+ stopForeground(true);
194
+ }
195
+ } catch (Throwable ignored) {}
196
+ }
197
+
198
+ @Override
199
+ public void onAccessibilityEvent(AccessibilityEvent event) {
200
+ if (event == null) return;
201
+ if (event.getEventType() == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) {
202
+ CharSequence pkg = event.getPackageName();
203
+ CharSequence cls = event.getClassName();
204
+ if (pkg != null) currentPackageName = pkg.toString();
205
+ if (cls != null) currentActivityName = cls.toString();
206
+ }
207
+ }
208
+
209
+ @Override
210
+ public void onInterrupt() {
211
+ Log.w(TAG, "ArtemisAccessibilityService interrupted");
212
+ }
213
+
214
+ @Override
215
+ public void onDestroy() {
216
+ super.onDestroy();
217
+ Log.i(TAG, "ArtemisAccessibilityService destroyed");
218
+ stopForegroundNotification();
219
+ if (server != null) {
220
+ server.shutdown();
221
+ server = null;
222
+ }
223
+ if (instance == this) {
224
+ instance = null;
225
+ }
226
+ }
227
+ }
@@ -0,0 +1,463 @@
1
+ package com.artemis.helper;
2
+
3
+ import android.util.Log;
4
+ import org.json.JSONObject;
5
+
6
+ import java.io.BufferedInputStream;
7
+ import java.io.ByteArrayOutputStream;
8
+ import java.io.Closeable;
9
+ import java.io.InputStream;
10
+ import java.io.OutputStream;
11
+ import java.net.InetAddress;
12
+ import java.net.InetSocketAddress;
13
+ import java.net.ServerSocket;
14
+ import java.net.Socket;
15
+ import java.net.URLDecoder;
16
+ import java.nio.charset.StandardCharsets;
17
+ import java.util.HashMap;
18
+ import java.util.Locale;
19
+ import java.util.Map;
20
+ import java.util.concurrent.ExecutorService;
21
+ import java.util.concurrent.Executors;
22
+
23
+ /**
24
+ * Multi-threaded command server bound strictly to loopback (127.0.0.1:18888).
25
+ *
26
+ * Protocols:
27
+ * - HTTP (/ping, /dump, /dump_xml, /hierarchy.xml, /snapshot, /action)
28
+ * - Line-delimited JSON-RPC over a raw TCP socket
29
+ *
30
+ * Every endpoint except {@code /ping} requires the session token the host pushed
31
+ * through {@link TokenReceiver} (header {@code X-Artemis-Token}, query parameter
32
+ * {@code token}, or JSON field {@code token}). Loopback is reachable by every app
33
+ * on the device, so without the token this service would hand any local app a
34
+ * full-screen reader and a gesture injector.
35
+ *
36
+ * Request bodies are read as bytes (Content-Length is a byte count), so UTF-8
37
+ * payloads of any script are handled exactly.
38
+ */
39
+ public class CommandServer extends Thread {
40
+
41
+ private static final String TAG = "ArtemisCommandServer";
42
+ private static final int MAX_HEADER_LINE = 16 * 1024;
43
+ private static final int MAX_BODY = 4 * 1024 * 1024;
44
+ private static final String TOKEN_HEADER = "x-artemis-token";
45
+
46
+ private final ArtemisAccessibilityService service;
47
+ private final int port;
48
+ private volatile boolean isRunning = true;
49
+ private ServerSocket serverSocket;
50
+ private final ExecutorService clientExecutor = Executors.newCachedThreadPool();
51
+
52
+ public CommandServer(ArtemisAccessibilityService service, int port) {
53
+ super("ArtemisCommandServer");
54
+ this.service = service;
55
+ this.port = port;
56
+ }
57
+
58
+ @Override
59
+ public void run() {
60
+ try {
61
+ serverSocket = new ServerSocket();
62
+ serverSocket.setReuseAddress(true);
63
+ serverSocket.bind(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), port), 50);
64
+ Log.i(TAG, "CommandServer listening on 127.0.0.1:" + port);
65
+
66
+ while (isRunning) {
67
+ final Socket socket;
68
+ try {
69
+ socket = serverSocket.accept();
70
+ } catch (Exception e) {
71
+ if (!isRunning) break;
72
+ continue;
73
+ }
74
+
75
+ clientExecutor.execute(new Runnable() {
76
+ @Override
77
+ public void run() {
78
+ handleClient(socket);
79
+ }
80
+ });
81
+ }
82
+ } catch (Exception e) {
83
+ Log.e(TAG, "Server error", e);
84
+ } finally {
85
+ closeQuietly(serverSocket);
86
+ }
87
+ }
88
+
89
+ // ------------------------------------------------------------------ //
90
+ // Byte-accurate request reading
91
+ // ------------------------------------------------------------------ //
92
+
93
+ /** Reads one line (without CR/LF) as raw bytes; null at end of stream. */
94
+ private static byte[] readLineBytes(InputStream in) throws Exception {
95
+ ByteArrayOutputStream line = new ByteArrayOutputStream(256);
96
+ int b;
97
+ while ((b = in.read()) >= 0) {
98
+ if (b == '\n') {
99
+ break;
100
+ }
101
+ if (b != '\r') {
102
+ line.write(b);
103
+ }
104
+ if (line.size() > MAX_HEADER_LINE) {
105
+ throw new IllegalStateException("Header line too long");
106
+ }
107
+ }
108
+ if (b < 0 && line.size() == 0) {
109
+ return null;
110
+ }
111
+ return line.toByteArray();
112
+ }
113
+
114
+ private static byte[] readExactly(InputStream in, int length) throws Exception {
115
+ byte[] buf = new byte[length];
116
+ int total = 0;
117
+ while (total < length) {
118
+ int r = in.read(buf, total, length - total);
119
+ if (r < 0) break;
120
+ total += r;
121
+ }
122
+ if (total < length) {
123
+ byte[] shorter = new byte[total];
124
+ System.arraycopy(buf, 0, shorter, 0, total);
125
+ return shorter;
126
+ }
127
+ return buf;
128
+ }
129
+
130
+ private void handleClient(Socket socket) {
131
+ try {
132
+ socket.setSoTimeout(10000);
133
+ InputStream in = new BufferedInputStream(socket.getInputStream());
134
+ OutputStream output = socket.getOutputStream();
135
+
136
+ byte[] firstLine = readLineBytes(in);
137
+ if (firstLine == null) return;
138
+ String ascii = new String(firstLine, StandardCharsets.ISO_8859_1).trim();
139
+
140
+ if (ascii.startsWith("GET ") || ascii.startsWith("POST ")) {
141
+ handleHttp(ascii, in, output);
142
+ } else if (ascii.startsWith("{")) {
143
+ handleJsonRpc(new String(firstLine, StandardCharsets.UTF_8).trim(), output);
144
+ } else {
145
+ JSONObject err = new JSONObject();
146
+ err.put("success", false);
147
+ err.put("error", "Unsupported protocol");
148
+ sendJson(output, 400, err.toString());
149
+ }
150
+ } catch (Exception e) {
151
+ Log.w(TAG, "Client handling error: " + e.getMessage());
152
+ } finally {
153
+ closeQuietly(socket);
154
+ }
155
+ }
156
+
157
+ // ------------------------------------------------------------------ //
158
+ // HTTP
159
+ // ------------------------------------------------------------------ //
160
+
161
+ private static Map<String, String> parseQuery(String query) {
162
+ Map<String, String> params = new HashMap<>();
163
+ if (query == null || query.isEmpty()) return params;
164
+ for (String pair : query.split("&")) {
165
+ if (pair.isEmpty()) continue;
166
+ int eq = pair.indexOf('=');
167
+ try {
168
+ String key = URLDecoder.decode(eq < 0 ? pair : pair.substring(0, eq), "UTF-8");
169
+ String value = eq < 0 ? "" : URLDecoder.decode(pair.substring(eq + 1), "UTF-8");
170
+ params.put(key, value);
171
+ } catch (Exception ignored) {}
172
+ }
173
+ return params;
174
+ }
175
+
176
+ private void handleHttp(String requestLine, InputStream in, OutputStream output) {
177
+ try {
178
+ String[] parts = requestLine.split(" ");
179
+ String target = parts.length > 1 ? parts[1] : "/";
180
+ String path = target;
181
+ Map<String, String> query;
182
+ int q = target.indexOf('?');
183
+ if (q >= 0) {
184
+ path = target.substring(0, q);
185
+ query = parseQuery(target.substring(q + 1));
186
+ } else {
187
+ query = new HashMap<>();
188
+ }
189
+
190
+ int contentLength = 0;
191
+ String headerToken = null;
192
+ byte[] lineBytes;
193
+ while ((lineBytes = readLineBytes(in)) != null) {
194
+ if (lineBytes.length == 0) break;
195
+ String line = new String(lineBytes, StandardCharsets.ISO_8859_1);
196
+ int colon = line.indexOf(':');
197
+ if (colon < 0) continue;
198
+ String name = line.substring(0, colon).trim().toLowerCase(Locale.ROOT);
199
+ String value = line.substring(colon + 1).trim();
200
+ if (name.equals("content-length")) {
201
+ try {
202
+ contentLength = Integer.parseInt(value);
203
+ } catch (Exception ignored) {}
204
+ } else if (name.equals(TOKEN_HEADER)) {
205
+ headerToken = value;
206
+ }
207
+ }
208
+ if (contentLength < 0 || contentLength > MAX_BODY) {
209
+ sendJson(output, 413, errorJson("Body too large").toString());
210
+ return;
211
+ }
212
+
213
+ String body = "";
214
+ if (contentLength > 0) {
215
+ body = new String(readExactly(in, contentLength), StandardCharsets.UTF_8);
216
+ }
217
+
218
+ String token = headerToken != null ? headerToken : query.get("token");
219
+ boolean authed = TokenStore.matches(token);
220
+
221
+ if (path.equals("/ping") || path.equals("/")) {
222
+ sendJson(output, 200, buildPing(authed).toString());
223
+ return;
224
+ }
225
+ if (!authed) {
226
+ sendJson(output, 401, unauthorizedJson().toString());
227
+ return;
228
+ }
229
+
230
+ // Route endpoints
231
+ if (path.equals("/snapshot")) {
232
+ HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forSnapshot().apply(query);
233
+ JSONObject json = HierarchyDumper.dumpAtomicSnapshot(service, options);
234
+ sendJson(output, 200, json.toString());
235
+ } else if (path.equals("/dump_xml") || path.equals("/hierarchy.xml")
236
+ || (path.equals("/dump") && "xml".equals(query.get("format")))) {
237
+ HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forSnapshot().apply(query);
238
+ sendXml(output, HierarchyDumper.dumpXml(service, options));
239
+ } else if (path.equals("/dump") || path.equals("/hierarchy")) {
240
+ HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forDump().apply(query);
241
+ sendJson(output, 200, HierarchyDumper.dump(service, options).toString());
242
+ } else if (path.equals("/action") || path.equals("/rpc")) {
243
+ JSONObject json = body.isEmpty() ? new JSONObject() : new JSONObject(body);
244
+ JSONObject resp = executeCommand(json.optString("cmd", ""), json);
245
+ sendJson(output, 200, resp.toString());
246
+ } else {
247
+ sendJson(output, 404, errorJson("Unknown endpoint: " + path).toString());
248
+ }
249
+
250
+ } catch (Exception e) {
251
+ Log.e(TAG, "HTTP error", e);
252
+ try {
253
+ sendJson(output, 500, errorJson("Internal error: " + e.getMessage()).toString());
254
+ } catch (Throwable ignored) {}
255
+ }
256
+ }
257
+
258
+ // ------------------------------------------------------------------ //
259
+ // Raw JSON-RPC
260
+ // ------------------------------------------------------------------ //
261
+
262
+ private void handleJsonRpc(String line, OutputStream output) {
263
+ try {
264
+ JSONObject json = new JSONObject(line);
265
+ String cmd = json.optString("cmd", "");
266
+ boolean authed = TokenStore.matches(json.optString("token", null));
267
+ JSONObject resp;
268
+ if (cmd.equalsIgnoreCase("ping")) {
269
+ resp = buildPing(authed);
270
+ } else if (!authed) {
271
+ resp = unauthorizedJson();
272
+ } else {
273
+ resp = executeCommand(cmd, json);
274
+ }
275
+ byte[] bytes = resp.toString().getBytes(StandardCharsets.UTF_8);
276
+ output.write(bytes);
277
+ output.write('\n');
278
+ output.flush();
279
+ } catch (Exception e) {
280
+ try {
281
+ JSONObject err = errorJson("JSON parse error: " + e.getMessage());
282
+ output.write(err.toString().getBytes(StandardCharsets.UTF_8));
283
+ output.write('\n');
284
+ output.flush();
285
+ } catch (Exception ignored) {}
286
+ }
287
+ }
288
+
289
+ // ------------------------------------------------------------------ //
290
+ // Commands
291
+ // ------------------------------------------------------------------ //
292
+
293
+ private JSONObject executeCommand(String cmd, JSONObject params) {
294
+ JSONObject resp = new JSONObject();
295
+ try {
296
+ switch (cmd.toLowerCase(Locale.ROOT)) {
297
+ case "ping":
298
+ return buildPing(true);
299
+ case "dump":
300
+ case "dump_ui":
301
+ return HierarchyDumper.dump(service, HierarchyDumper.DumpOptions.forDump());
302
+ case "snapshot":
303
+ return HierarchyDumper.dumpAtomicSnapshot(service, HierarchyDumper.DumpOptions.forSnapshot());
304
+ case "dump_xml":
305
+ resp.put("success", true);
306
+ resp.put("xml", HierarchyDumper.dumpXml(service));
307
+ break;
308
+ case "tap":
309
+ float x = (float) params.optDouble("x", -1.0);
310
+ float y = (float) params.optDouble("y", -1.0);
311
+ long tapTimeout = params.optLong("timeout", 1500L);
312
+ if (x < 0 || y < 0) {
313
+ resp.put("success", false);
314
+ resp.put("error", "Invalid coordinates");
315
+ } else {
316
+ resp.put("success", GestureController.tap(service, x, y, tapTimeout));
317
+ }
318
+ break;
319
+ case "double_tap":
320
+ float dtx = (float) params.optDouble("x", -1.0);
321
+ float dty = (float) params.optDouble("y", -1.0);
322
+ long dtTimeout = params.optLong("timeout", 2000L);
323
+ if (dtx < 0 || dty < 0) {
324
+ resp.put("success", false);
325
+ resp.put("error", "Invalid coordinates");
326
+ } else {
327
+ resp.put("success", GestureController.doubleTap(service, dtx, dty, dtTimeout));
328
+ }
329
+ break;
330
+ case "long_press":
331
+ float lpx = (float) params.optDouble("x", -1.0);
332
+ float lpy = (float) params.optDouble("y", -1.0);
333
+ long lpDuration = params.optLong("duration", 1000L);
334
+ if (lpx < 0 || lpy < 0) {
335
+ resp.put("success", false);
336
+ resp.put("error", "Invalid coordinates");
337
+ } else {
338
+ resp.put("success", GestureController.longPress(service, lpx, lpy, lpDuration, 2500L));
339
+ }
340
+ break;
341
+ case "swipe":
342
+ float x1 = (float) params.optDouble("x1", -1.0);
343
+ float y1 = (float) params.optDouble("y1", -1.0);
344
+ float x2 = (float) params.optDouble("x2", -1.0);
345
+ float y2 = (float) params.optDouble("y2", -1.0);
346
+ long dur = params.optLong("duration", 300L);
347
+ resp.put("success", GestureController.swipe(service, x1, y1, x2, y2, dur, 3000L));
348
+ break;
349
+ case "type":
350
+ String text = params.optString("text", "");
351
+ boolean append = params.optBoolean("append", false);
352
+ resp.put("success", GestureController.setText(service, text, append));
353
+ break;
354
+ case "clear":
355
+ resp.put("success", GestureController.clearText(service));
356
+ break;
357
+ case "clipboard":
358
+ resp.put("success", GestureController.setClipboard(service, params.optString("text", "")));
359
+ break;
360
+ case "global":
361
+ String action = params.optString("action", "");
362
+ resp.put("success", GestureController.performGlobalAction(service, action));
363
+ break;
364
+ default:
365
+ resp.put("success", false);
366
+ resp.put("error", "Unknown command: " + cmd);
367
+ break;
368
+ }
369
+ } catch (Exception e) {
370
+ try {
371
+ resp.put("success", false);
372
+ resp.put("error", e.getMessage());
373
+ } catch (Exception ignored) {}
374
+ }
375
+ return resp;
376
+ }
377
+
378
+ /**
379
+ * Health payload shared by GET /ping and the "ping" RPC. The host compares
380
+ * version_code / protocol_version against the bundled APK to decide whether
381
+ * to upgrade, and token_set to decide whether to push its token. The
382
+ * foreground package and activity are only disclosed to an authenticated caller.
383
+ */
384
+ private JSONObject buildPing(boolean authed) throws Exception {
385
+ JSONObject r = new JSONObject();
386
+ r.put("success", true);
387
+ r.put("service", "ArtemisAccessibilityService");
388
+ r.put("version_code", service.getVersionCode());
389
+ r.put("version_name", service.getVersionName());
390
+ r.put("protocol_version", ArtemisAccessibilityService.PROTOCOL_VERSION);
391
+ r.put("port", port);
392
+ r.put("auth_required", true);
393
+ r.put("token_set", TokenStore.isSet());
394
+ r.put("authenticated", authed);
395
+ if (authed) {
396
+ r.put("package", service.getCurrentPackageName());
397
+ r.put("activity", service.getCurrentActivityName());
398
+ }
399
+ return r;
400
+ }
401
+
402
+ private static JSONObject errorJson(String message) throws Exception {
403
+ JSONObject r = new JSONObject();
404
+ r.put("success", false);
405
+ r.put("error", message);
406
+ return r;
407
+ }
408
+
409
+ private static JSONObject unauthorizedJson() throws Exception {
410
+ JSONObject r = errorJson(TokenStore.isSet()
411
+ ? "unauthorized: wrong or missing X-Artemis-Token"
412
+ : "unauthorized: no session token has been pushed to the helper yet");
413
+ r.put("token_set", TokenStore.isSet());
414
+ return r;
415
+ }
416
+
417
+ // ------------------------------------------------------------------ //
418
+ // Responses
419
+ // ------------------------------------------------------------------ //
420
+
421
+ private static String reason(int status) {
422
+ switch (status) {
423
+ case 200: return "OK";
424
+ case 400: return "Bad Request";
425
+ case 401: return "Unauthorized";
426
+ case 404: return "Not Found";
427
+ case 413: return "Payload Too Large";
428
+ default: return "Internal Server Error";
429
+ }
430
+ }
431
+
432
+ private static void send(OutputStream output, int status, String contentType, byte[] bytes) {
433
+ try {
434
+ String header = "HTTP/1.1 " + status + " " + reason(status) + "\r\n" +
435
+ "Content-Type: " + contentType + "; charset=utf-8\r\n" +
436
+ "Content-Length: " + bytes.length + "\r\n" +
437
+ "Connection: close\r\n\r\n";
438
+ output.write(header.getBytes(StandardCharsets.UTF_8));
439
+ output.write(bytes);
440
+ output.flush();
441
+ } catch (Exception ignored) {}
442
+ }
443
+
444
+ private static void sendJson(OutputStream output, int status, String jsonStr) {
445
+ send(output, status, "application/json", jsonStr.getBytes(StandardCharsets.UTF_8));
446
+ }
447
+
448
+ private static void sendXml(OutputStream output, String xmlStr) {
449
+ send(output, 200, "application/xml", xmlStr.getBytes(StandardCharsets.UTF_8));
450
+ }
451
+
452
+ public void shutdown() {
453
+ isRunning = false;
454
+ closeQuietly(serverSocket);
455
+ clientExecutor.shutdownNow();
456
+ }
457
+
458
+ private static void closeQuietly(Closeable c) {
459
+ if (c != null) {
460
+ try { c.close(); } catch (Throwable ignored) {}
461
+ }
462
+ }
463
+ }