android-midscene-automation 0.2.3 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (525) hide show
  1. package/CHANGELOG.md +11 -1
  2. package/README.md +63 -1
  3. package/artemis-main/.dockerignore +15 -0
  4. package/artemis-main/.env.example +34 -0
  5. package/artemis-main/CONTRIBUTING.md +35 -0
  6. package/artemis-main/Dockerfile +68 -0
  7. package/artemis-main/LICENSE +202 -0
  8. package/artemis-main/Makefile +115 -0
  9. package/artemis-main/README.md +322 -0
  10. package/artemis-main/README_CN.md +320 -0
  11. package/artemis-main/apps/__init__.py +15 -0
  12. package/artemis-main/apps/admin_console/__init__.py +19 -0
  13. package/artemis-main/apps/admin_console/core/__init__.py +13 -0
  14. package/artemis-main/apps/admin_console/core/config.py +43 -0
  15. package/artemis-main/apps/admin_console/core/security.py +178 -0
  16. package/artemis-main/apps/admin_console/core/state.py +243 -0
  17. package/artemis-main/apps/admin_console/database/__init__.py +13 -0
  18. package/artemis-main/apps/admin_console/database/connection.py +60 -0
  19. package/artemis-main/apps/admin_console/database/repositories/__init__.py +13 -0
  20. package/artemis-main/apps/admin_console/database/repositories/session_repository.py +564 -0
  21. package/artemis-main/apps/admin_console/database/repositories/step_repository.py +512 -0
  22. package/artemis-main/apps/admin_console/database/repositories/trace_repository.py +133 -0
  23. package/artemis-main/apps/admin_console/index.html +3787 -0
  24. package/artemis-main/apps/admin_console/replay_manager.py +2585 -0
  25. package/artemis-main/apps/admin_console/routers/__init__.py +13 -0
  26. package/artemis-main/apps/admin_console/routers/media.py +198 -0
  27. package/artemis-main/apps/admin_console/routers/replay.py +91 -0
  28. package/artemis-main/apps/admin_console/routers/sessions.py +239 -0
  29. package/artemis-main/apps/admin_console/routers/steps.py +109 -0
  30. package/artemis-main/apps/admin_console/routers/stream.py +56 -0
  31. package/artemis-main/apps/admin_console/routers/system.py +761 -0
  32. package/artemis-main/apps/admin_console/routers/tasks.py +517 -0
  33. package/artemis-main/apps/admin_console/schemas/__init__.py +13 -0
  34. package/artemis-main/apps/admin_console/schemas/task_schema.py +47 -0
  35. package/artemis-main/apps/admin_console/server.py +508 -0
  36. package/artemis-main/apps/admin_console/services/__init__.py +13 -0
  37. package/artemis-main/apps/admin_console/services/device_stream_service.py +137 -0
  38. package/artemis-main/apps/admin_console/services/ipc_service.py +262 -0
  39. package/artemis-main/apps/admin_console/services/media_service.py +623 -0
  40. package/artemis-main/apps/admin_console/services/model_service.py +148 -0
  41. package/artemis-main/apps/admin_console/services/task_preset_catalog.py +514 -0
  42. package/artemis-main/apps/admin_console/services/task_queue_service.py +1719 -0
  43. package/artemis-main/apps/admin_console/services/worker_process_io.py +144 -0
  44. package/artemis-main/apps/showcase_ui/README.md +31 -0
  45. package/artemis-main/apps/showcase_ui/angular.json +132 -0
  46. package/artemis-main/apps/showcase_ui/package-lock.json +9472 -0
  47. package/artemis-main/apps/showcase_ui/package.json +39 -0
  48. package/artemis-main/apps/showcase_ui/proxy.conf.json +23 -0
  49. package/artemis-main/apps/showcase_ui/public/favicon.ico +0 -0
  50. package/artemis-main/apps/showcase_ui/public/logo.png +0 -0
  51. package/artemis-main/apps/showcase_ui/public/logo.svg +56 -0
  52. package/artemis-main/apps/showcase_ui/src/app/app.component.html +24 -0
  53. package/artemis-main/apps/showcase_ui/src/app/app.component.scss +49 -0
  54. package/artemis-main/apps/showcase_ui/src/app/app.component.spec.ts +42 -0
  55. package/artemis-main/apps/showcase_ui/src/app/app.component.ts +32 -0
  56. package/artemis-main/apps/showcase_ui/src/app/app.config.ts +29 -0
  57. package/artemis-main/apps/showcase_ui/src/app/app.routes.ts +27 -0
  58. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream-startup.spec.ts +108 -0
  59. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.html +1436 -0
  60. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.scss +3973 -0
  61. package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream.component.ts +2235 -0
  62. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.html +380 -0
  63. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.scss +1171 -0
  64. package/artemis-main/apps/showcase_ui/src/app/components/chat-interface/chat-interface.component.ts +270 -0
  65. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html +433 -0
  66. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.scss +987 -0
  67. package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.ts +632 -0
  68. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.html +130 -0
  69. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.scss +406 -0
  70. package/artemis-main/apps/showcase_ui/src/app/components/legacy-agent-stream/legacy-agent-stream.component.ts +74 -0
  71. package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.scss +115 -0
  72. package/artemis-main/apps/showcase_ui/src/app/components/nav-switcher/nav-switcher.component.ts +52 -0
  73. package/artemis-main/apps/showcase_ui/src/app/core/data/smart-tasks.data.ts +334 -0
  74. package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.spec.ts +55 -0
  75. package/artemis-main/apps/showcase_ui/src/app/core/i18n/language.service.ts +61 -0
  76. package/artemis-main/apps/showcase_ui/src/app/core/i18n/translate.pipe.ts +11 -0
  77. package/artemis-main/apps/showcase_ui/src/app/core/i18n/zh-cn.ts +651 -0
  78. package/artemis-main/apps/showcase_ui/src/app/core/models/markdown.model.ts +43 -0
  79. package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.spec.ts +47 -0
  80. package/artemis-main/apps/showcase_ui/src/app/core/models/pro-tuning.model.ts +165 -0
  81. package/artemis-main/apps/showcase_ui/src/app/core/models/session.model.ts +85 -0
  82. package/artemis-main/apps/showcase_ui/src/app/core/models/stream.model.ts +246 -0
  83. package/artemis-main/apps/showcase_ui/src/app/core/models/system.model.ts +118 -0
  84. package/artemis-main/apps/showcase_ui/src/app/core/services/task-recommendation.service.ts +111 -0
  85. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.html +2130 -0
  86. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.scss +5204 -0
  87. package/artemis-main/apps/showcase_ui/src/app/pages/home/home.component.ts +1189 -0
  88. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.html +39 -0
  89. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.scss +151 -0
  90. package/artemis-main/apps/showcase_ui/src/app/pages/legacy-workspace/legacy-workspace.component.ts +58 -0
  91. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.html +201 -0
  92. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.scss +886 -0
  93. package/artemis-main/apps/showcase_ui/src/app/pages/workspace/workspace.component.ts +324 -0
  94. package/artemis-main/apps/showcase_ui/src/app/services/agent.service.spec.ts +418 -0
  95. package/artemis-main/apps/showcase_ui/src/app/services/agent.service.ts +2012 -0
  96. package/artemis-main/apps/showcase_ui/src/app/services/system.service.spec.ts +173 -0
  97. package/artemis-main/apps/showcase_ui/src/app/services/system.service.ts +620 -0
  98. package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.spec.ts +513 -0
  99. package/artemis-main/apps/showcase_ui/src/app/utils/action-formatter.util.ts +1015 -0
  100. package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.spec.ts +85 -0
  101. package/artemis-main/apps/showcase_ui/src/app/utils/image-overlay.util.ts +662 -0
  102. package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.spec.ts +70 -0
  103. package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.ts +383 -0
  104. package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.spec.ts +107 -0
  105. package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.ts +123 -0
  106. package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.spec.ts +92 -0
  107. package/artemis-main/apps/showcase_ui/src/app/utils/run-info.util.ts +116 -0
  108. package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.spec.ts +640 -0
  109. package/artemis-main/apps/showcase_ui/src/app/utils/stream-aggregator.util.ts +1021 -0
  110. package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.spec.ts +375 -0
  111. package/artemis-main/apps/showcase_ui/src/app/utils/tool-formatter.util.ts +1419 -0
  112. package/artemis-main/apps/showcase_ui/src/index.html +37 -0
  113. package/artemis-main/apps/showcase_ui/src/main.ts +22 -0
  114. package/artemis-main/apps/showcase_ui/src/styles.scss +60 -0
  115. package/artemis-main/apps/showcase_ui/tsconfig.app.json +23 -0
  116. package/artemis-main/apps/showcase_ui/tsconfig.json +27 -0
  117. package/artemis-main/apps/showcase_ui/tsconfig.spec.json +23 -0
  118. package/artemis-main/artemis/__init__.py +44 -0
  119. package/artemis-main/artemis/__main__.py +18 -0
  120. package/artemis-main/artemis/_legacy_imports.py +94 -0
  121. package/artemis-main/artemis/_version.py +30 -0
  122. package/artemis-main/artemis/agents/__init__.py +17 -0
  123. package/artemis-main/artemis/agents/checker/checker.json +9 -0
  124. package/artemis-main/artemis/agents/checker/checker.py +859 -0
  125. package/artemis-main/artemis/agents/diagnoser/diagnoser.json +23 -0
  126. package/artemis-main/artemis/agents/diagnoser/diagnoser.py +608 -0
  127. package/artemis-main/artemis/agents/explorer/constants.py +52 -0
  128. package/artemis-main/artemis/agents/explorer/explorer.json +57 -0
  129. package/artemis-main/artemis/agents/explorer/explorer.py +473 -0
  130. package/artemis-main/artemis/agents/explorer/geometry.py +73 -0
  131. package/artemis-main/artemis/agents/explorer/native_runner.py +878 -0
  132. package/artemis-main/artemis/agents/explorer/perception_tools.py +632 -0
  133. package/artemis-main/artemis/agents/explorer/run_setup.py +403 -0
  134. package/artemis-main/artemis/agents/explorer/screen_index.py +266 -0
  135. package/artemis-main/artemis/agents/explorer/tiers.py +110 -0
  136. package/artemis-main/artemis/agents/explorer/tool_declarations.py +394 -0
  137. package/artemis-main/artemis/agents/explorer/universal_runner.py +317 -0
  138. package/artemis-main/artemis/agents/flash/context_compressor.py +452 -0
  139. package/artemis-main/artemis/agents/flash/flash_runner.md +51 -0
  140. package/artemis-main/artemis/agents/flash/flash_summarizer.md +53 -0
  141. package/artemis-main/artemis/agents/flash/flash_summarizer_single.md +54 -0
  142. package/artemis-main/artemis/agents/flash/runner.py +1167 -0
  143. package/artemis-main/artemis/agents/flash/summarizer.py +511 -0
  144. package/artemis-main/artemis/agents/history_analyzer/history_analyzer.md +8 -0
  145. package/artemis-main/artemis/agents/history_analyzer/history_analyzer.py +182 -0
  146. package/artemis-main/artemis/agents/image_processor/image_processor.md +74 -0
  147. package/artemis-main/artemis/agents/image_processor/image_processor.py +322 -0
  148. package/artemis-main/artemis/agents/log_analyzer/log_analyzer.md +15 -0
  149. package/artemis-main/artemis/agents/log_analyzer/log_analyzer.py +403 -0
  150. package/artemis-main/artemis/agents/log_analyzer/log_reader_sub_agent.md +8 -0
  151. package/artemis-main/artemis/agents/log_analyzer/output_analyzer.md +11 -0
  152. package/artemis-main/artemis/agents/log_analyzer/output_analyzer.py +163 -0
  153. package/artemis-main/artemis/agents/object_detector/object_detector.json +9 -0
  154. package/artemis-main/artemis/agents/object_detector/object_detector.py +211 -0
  155. package/artemis-main/artemis/agents/operator/operator.json +3 -0
  156. package/artemis-main/artemis/agents/operator/operator.py +1394 -0
  157. package/artemis-main/artemis/agents/operator/prompts.py +1064 -0
  158. package/artemis-main/artemis/agents/outputter/outputter.md +36 -0
  159. package/artemis-main/artemis/agents/outputter/outputter.py +552 -0
  160. package/artemis-main/artemis/agents/planner/planner.json +32 -0
  161. package/artemis-main/artemis/agents/planner/planner.py +436 -0
  162. package/artemis-main/artemis/agents/prompt_assembly.py +79 -0
  163. package/artemis-main/artemis/agents/summarizer/summarizer.py +190 -0
  164. package/artemis-main/artemis/agents/validator/action_execution.py +98 -0
  165. package/artemis-main/artemis/agents/validator/categories.py +33 -0
  166. package/artemis-main/artemis/agents/validator/execution_loop.py +488 -0
  167. package/artemis-main/artemis/agents/validator/incidents.py +120 -0
  168. package/artemis-main/artemis/agents/validator/pixel_safety_net.md +15 -0
  169. package/artemis-main/artemis/agents/validator/precondition_pixel.py +320 -0
  170. package/artemis-main/artemis/agents/validator/precondition_xml.py +672 -0
  171. package/artemis-main/artemis/agents/validator/tool_declarations.py +332 -0
  172. package/artemis-main/artemis/agents/validator/validator.py +188 -0
  173. package/artemis-main/artemis/agents/video_analyzer/__init__.py +15 -0
  174. package/artemis-main/artemis/agents/video_analyzer/audio_native.py +543 -0
  175. package/artemis-main/artemis/agents/video_analyzer/blackboard.py +821 -0
  176. package/artemis-main/artemis/agents/video_analyzer/chunk_agentic.py +455 -0
  177. package/artemis-main/artemis/agents/video_analyzer/chunk_conversation.py +491 -0
  178. package/artemis-main/artemis/agents/video_analyzer/chunk_native.py +437 -0
  179. package/artemis-main/artemis/agents/video_analyzer/conflict_resolution.py +137 -0
  180. package/artemis-main/artemis/agents/video_analyzer/gemini_files.py +162 -0
  181. package/artemis-main/artemis/agents/video_analyzer/native_coordinator.py +541 -0
  182. package/artemis-main/artemis/agents/video_analyzer/reliability.py +128 -0
  183. package/artemis-main/artemis/agents/video_analyzer/universal_engine.py +487 -0
  184. package/artemis-main/artemis/agents/video_analyzer/universal_tools.py +323 -0
  185. package/artemis-main/artemis/agents/video_analyzer/video_analyzer.md +51 -0
  186. package/artemis-main/artemis/agents/video_analyzer/video_analyzer.py +875 -0
  187. package/artemis-main/artemis/agents/video_analyzer/video_sub_agent.md +15 -0
  188. package/artemis-main/artemis/agents/video_analyzer/video_sub_agent_audio.md +12 -0
  189. package/artemis-main/artemis/clients/accessibility_client.py +438 -0
  190. package/artemis-main/artemis/clients/screen_client_factory.py +403 -0
  191. package/artemis-main/artemis/config/__init__.py +309 -0
  192. package/artemis-main/artemis/config/agent.py +1098 -0
  193. package/artemis-main/artemis/config/constants.py +151 -0
  194. package/artemis-main/artemis/config/core/__init__.py +24 -0
  195. package/artemis-main/artemis/config/core/engine.py +118 -0
  196. package/artemis-main/artemis/config/core/interpolator.py +78 -0
  197. package/artemis-main/artemis/config/llm.py +267 -0
  198. package/artemis-main/artemis/config/paths.py +244 -0
  199. package/artemis-main/artemis/config/runtime.py +197 -0
  200. package/artemis-main/artemis/config/settings.py +315 -0
  201. package/artemis-main/artemis/constants.py +47 -0
  202. package/artemis-main/artemis/context.py +289 -0
  203. package/artemis-main/artemis/controllers/__init__.py +41 -0
  204. package/artemis-main/artemis/controllers/controller_factory.py +41 -0
  205. package/artemis-main/artemis/controllers/device_controller.py +29 -0
  206. package/artemis-main/artemis/controllers/types.py +37 -0
  207. package/artemis-main/artemis/controllers/unified_controller.py +649 -0
  208. package/artemis-main/artemis/core/__init__.py +21 -0
  209. package/artemis-main/artemis/core/diagnostics/__init__.py +90 -0
  210. package/artemis-main/artemis/core/diagnostics/adb_keys.py +270 -0
  211. package/artemis-main/artemis/core/diagnostics/adb_server_connection.py +330 -0
  212. package/artemis-main/artemis/core/diagnostics/device_smoke.py +364 -0
  213. package/artemis-main/artemis/core/diagnostics/emulator_manager.py +501 -0
  214. package/artemis-main/artemis/core/diagnostics/engine.py +438 -0
  215. package/artemis-main/artemis/core/diagnostics/hierarchy_parity.py +216 -0
  216. package/artemis-main/artemis/core/diagnostics/probes/adb_probe.py +887 -0
  217. package/artemis-main/artemis/core/diagnostics/probes/base.py +45 -0
  218. package/artemis-main/artemis/core/diagnostics/probes/credentials_probe.py +296 -0
  219. package/artemis-main/artemis/core/diagnostics/probes/host_probe.py +427 -0
  220. package/artemis-main/artemis/core/diagnostics/probes/runtime_probe.py +169 -0
  221. package/artemis-main/artemis/core/diagnostics/probes/toolchain_probe.py +139 -0
  222. package/artemis-main/artemis/core/diagnostics/readiness.py +83 -0
  223. package/artemis-main/artemis/core/diagnostics/schema.py +118 -0
  224. package/artemis-main/artemis/core/tool_declaration.py +43 -0
  225. package/artemis-main/artemis/core/tool_failure.py +56 -0
  226. package/artemis-main/artemis/data_engine/context_vars.py +24 -0
  227. package/artemis-main/artemis/data_engine/engine.py +1615 -0
  228. package/artemis-main/artemis/data_engine/history_reader.py +142 -0
  229. package/artemis-main/artemis/data_engine/models.py +131 -0
  230. package/artemis-main/artemis/data_engine/storage.py +1245 -0
  231. package/artemis-main/artemis/data_engine/trace.py +727 -0
  232. package/artemis-main/artemis/drivers/__init__.py +48 -0
  233. package/artemis-main/artemis/drivers/android/adb_driver.py +507 -0
  234. package/artemis-main/artemis/drivers/android/hierarchy.py +64 -0
  235. package/artemis-main/artemis/drivers/android/input_ime.py +120 -0
  236. package/artemis-main/artemis/drivers/android/recorder.py +57 -0
  237. package/artemis-main/artemis/drivers/base.py +290 -0
  238. package/artemis-main/artemis/drivers/cloud/cloud_driver.py +117 -0
  239. package/artemis-main/artemis/drivers/factory.py +80 -0
  240. package/artemis-main/artemis/drivers/mock/mock_driver.py +198 -0
  241. package/artemis-main/artemis/drivers/types.py +67 -0
  242. package/artemis-main/artemis/graph/checkpoints.py +1243 -0
  243. package/artemis-main/artemis/graph/graph.py +1055 -0
  244. package/artemis-main/artemis/graph/perception.py +234 -0
  245. package/artemis-main/artemis/graph/state.py +151 -0
  246. package/artemis-main/artemis/graph/visibility.py +238 -0
  247. package/artemis-main/artemis/interfaces/cli/__init__.py +19 -0
  248. package/artemis-main/artemis/interfaces/cli/commands/__init__.py +15 -0
  249. package/artemis-main/artemis/interfaces/cli/commands/batch.py +278 -0
  250. package/artemis-main/artemis/interfaces/cli/commands/doctor.py +468 -0
  251. package/artemis-main/artemis/interfaces/cli/commands/helper.py +226 -0
  252. package/artemis-main/artemis/interfaces/cli/commands/init.py +193 -0
  253. package/artemis-main/artemis/interfaces/cli/commands/mcp.py +862 -0
  254. package/artemis-main/artemis/interfaces/cli/commands/run.py +463 -0
  255. package/artemis-main/artemis/interfaces/cli/commands/server.py +151 -0
  256. package/artemis-main/artemis/interfaces/cli/commands/server_lifecycle.py +328 -0
  257. package/artemis-main/artemis/interfaces/cli/commands/trace.py +111 -0
  258. package/artemis-main/artemis/interfaces/cli/commands/ui.py +404 -0
  259. package/artemis-main/artemis/interfaces/cli/main.py +109 -0
  260. package/artemis-main/artemis/interfaces/sdk/__init__.py +31 -0
  261. package/artemis-main/artemis/interfaces/sdk/client.py +19 -0
  262. package/artemis-main/artemis/interfaces/sdk/task.py +72 -0
  263. package/artemis-main/artemis/llm/__init__.py +25 -0
  264. package/artemis-main/artemis/llm/fake_model.py +179 -0
  265. package/artemis-main/artemis/llm/google/__init__.py +57 -0
  266. package/artemis-main/artemis/llm/google/provider.py +185 -0
  267. package/artemis-main/artemis/llm/google/usage.py +148 -0
  268. package/artemis-main/artemis/llm/parser.py +32 -0
  269. package/artemis-main/artemis/llm/reliability.py +267 -0
  270. package/artemis-main/artemis/llm/router.py +389 -0
  271. package/artemis-main/artemis/llm/schemas.py +36 -0
  272. package/artemis-main/artemis/llm/structured.py +196 -0
  273. package/artemis-main/artemis/main.py +58 -0
  274. package/artemis-main/artemis/mcp/__init__.py +20 -0
  275. package/artemis-main/artemis/mcp/action_executor.py +760 -0
  276. package/artemis-main/artemis/mcp/action_manifest.py +320 -0
  277. package/artemis-main/artemis/mcp/action_names.py +172 -0
  278. package/artemis-main/artemis/mcp/action_server.py +202 -0
  279. package/artemis-main/artemis/mcp/action_session.py +245 -0
  280. package/artemis-main/artemis/mcp/action_specs.py +868 -0
  281. package/artemis-main/artemis/mcp/action_types.py +86 -0
  282. package/artemis-main/artemis/mcp/actuators/__init__.py +21 -0
  283. package/artemis-main/artemis/mcp/actuators/adb.py +414 -0
  284. package/artemis-main/artemis/mcp/actuators/base.py +113 -0
  285. package/artemis-main/artemis/mcp/actuators/mock.py +70 -0
  286. package/artemis-main/artemis/mcp/adb_server.py +490 -0
  287. package/artemis-main/artemis/mcp/observation.py +133 -0
  288. package/artemis-main/artemis/mcp/xml_search_server.py +297 -0
  289. package/artemis-main/artemis/memory/__init__.py +90 -0
  290. package/artemis-main/artemis/memory/chunking.py +1843 -0
  291. package/artemis-main/artemis/memory/context_policy.py +226 -0
  292. package/artemis-main/artemis/memory/step_capsule.md +73 -0
  293. package/artemis-main/artemis/memory/step_memory.py +281 -0
  294. package/artemis-main/artemis/memory/transcript.py +798 -0
  295. package/artemis-main/artemis/platform/__init__.py +61 -0
  296. package/artemis-main/artemis/platform/base.py +104 -0
  297. package/artemis-main/artemis/platform/darwin.py +184 -0
  298. package/artemis-main/artemis/platform/linux.py +207 -0
  299. package/artemis-main/artemis/platform/windows.py +193 -0
  300. package/artemis-main/artemis/resources/__init__.py +46 -0
  301. package/artemis-main/artemis/runtime/__init__.py +108 -0
  302. package/artemis-main/artemis/runtime/adb_endpoint.py +173 -0
  303. package/artemis-main/artemis/runtime/awake_lease.py +283 -0
  304. package/artemis-main/artemis/runtime/awake_service.py +343 -0
  305. package/artemis-main/artemis/runtime/cancel_requests.py +254 -0
  306. package/artemis-main/artemis/runtime/daemon_client.py +422 -0
  307. package/artemis-main/artemis/runtime/device_lock.py +1013 -0
  308. package/artemis-main/artemis/runtime/device_pool.py +478 -0
  309. package/artemis-main/artemis/runtime/helper_manager.py +1030 -0
  310. package/artemis-main/artemis/runtime/process_probe.py +98 -0
  311. package/artemis-main/artemis/runtime/server_lifecycle.py +435 -0
  312. package/artemis-main/artemis/runtime/supervisor.py +121 -0
  313. package/artemis-main/artemis/runtime/trace_store.py +375 -0
  314. package/artemis-main/artemis/sdk/__init__.py +26 -0
  315. package/artemis-main/artemis/sdk/agent.py +485 -0
  316. package/artemis-main/artemis/sdk/builders/__init__.py +29 -0
  317. package/artemis-main/artemis/sdk/builders/agent_config_builder.py +474 -0
  318. package/artemis-main/artemis/sdk/constants.py +15 -0
  319. package/artemis-main/artemis/sdk/run_outcome.py +58 -0
  320. package/artemis-main/artemis/sdk/types/__init__.py +61 -0
  321. package/artemis-main/artemis/sdk/types/agent.py +89 -0
  322. package/artemis-main/artemis/sdk/utils.py +28 -0
  323. package/artemis-main/artemis/services/llm.py +1040 -0
  324. package/artemis-main/artemis/services/token_meter.py +267 -0
  325. package/artemis-main/artemis/telemetry/__init__.py +35 -0
  326. package/artemis-main/artemis/telemetry/models.py +64 -0
  327. package/artemis-main/artemis/telemetry/replay.py +34 -0
  328. package/artemis-main/artemis/telemetry/storage/json_storage.py +47 -0
  329. package/artemis-main/artemis/telemetry/tracer.py +85 -0
  330. package/artemis-main/artemis/toolchain/__init__.py +52 -0
  331. package/artemis-main/artemis/toolchain/descriptors.py +92 -0
  332. package/artemis-main/artemis/toolchain/resolver.py +185 -0
  333. package/artemis-main/artemis/tools/base.py +232 -0
  334. package/artemis-main/artemis/tools/command_tool.py +1240 -0
  335. package/artemis-main/artemis/tools/committee_prompts.json +5 -0
  336. package/artemis-main/artemis/tools/committee_tool.py +422 -0
  337. package/artemis-main/artemis/tools/diagnoser_submit_answer_tool.py +87 -0
  338. package/artemis-main/artemis/tools/diagnostic_tool.py +120 -0
  339. package/artemis-main/artemis/tools/explorer_tool.py +581 -0
  340. package/artemis-main/artemis/tools/history/__init__.py +398 -0
  341. package/artemis-main/artemis/tools/history/replay.py +120 -0
  342. package/artemis-main/artemis/tools/history/screenshot.py +139 -0
  343. package/artemis-main/artemis/tools/history/search.py +563 -0
  344. package/artemis-main/artemis/tools/image_processor_tool.py +124 -0
  345. package/artemis-main/artemis/tools/index.py +47 -0
  346. package/artemis-main/artemis/tools/log_tool.py +108 -0
  347. package/artemis-main/artemis/tools/mobile/launch_app.py +132 -0
  348. package/artemis-main/artemis/tools/mobile/log_utils.py +138 -0
  349. package/artemis-main/artemis/tools/mobile/ocr.py +202 -0
  350. package/artemis-main/artemis/tools/mobile/read_hierarchy.py +108 -0
  351. package/artemis-main/artemis/tools/mobile/read_logs.py +116 -0
  352. package/artemis-main/artemis/tools/mobile/search_logs.py +260 -0
  353. package/artemis-main/artemis/tools/object_detection_tool.py +244 -0
  354. package/artemis-main/artemis/tools/scratchpad.py +819 -0
  355. package/artemis-main/artemis/tools/system/adb_shell.py +31 -0
  356. package/artemis-main/artemis/tools/tool_wrapper.py +270 -0
  357. package/artemis-main/artemis/tools/video_tool.py +261 -0
  358. package/artemis-main/artemis/tools/wait_tool.py +97 -0
  359. package/artemis-main/artemis/utils/app_launch_utils.py +207 -0
  360. package/artemis-main/artemis/utils/coordinates.py +606 -0
  361. package/artemis-main/artemis/utils/credentials_validator.py +201 -0
  362. package/artemis-main/artemis/utils/cv_canvas.py +189 -0
  363. package/artemis-main/artemis/utils/cython_compat.py +45 -0
  364. package/artemis-main/artemis/utils/element_hit_test.py +80 -0
  365. package/artemis-main/artemis/utils/image_diff.py +335 -0
  366. package/artemis-main/artemis/utils/image_hash.py +80 -0
  367. package/artemis-main/artemis/utils/notes.py +470 -0
  368. package/artemis-main/artemis/utils/ocr_api.py +105 -0
  369. package/artemis-main/artemis/utils/ocr_xml_fusion.py +318 -0
  370. package/artemis-main/artemis/utils/plan_grammar.py +513 -0
  371. package/artemis-main/artemis/utils/python_executor.py +213 -0
  372. package/artemis-main/artemis/utils/startup_progress.py +60 -0
  373. package/artemis-main/artemis/utils/task_tree.py +1174 -0
  374. package/artemis-main/artemis/utils/text.py +37 -0
  375. package/artemis-main/artemis/utils/ui_filter.py +651 -0
  376. package/artemis-main/artemis/utils/video.py +685 -0
  377. package/artemis-main/artemis/utils/visualization.py +802 -0
  378. package/artemis-main/config/artemis.example.jsonc +12 -0
  379. package/artemis-main/mcp_server/README.md +111 -0
  380. package/artemis-main/mcp_server/__init__.py +21 -0
  381. package/artemis-main/mcp_server/__main__.py +20 -0
  382. package/artemis-main/mcp_server/background/__init__.py +15 -0
  383. package/artemis-main/mcp_server/background/task_runner.py +453 -0
  384. package/artemis-main/mcp_server/base.py +49 -0
  385. package/artemis-main/mcp_server/notifiers/__init__.py +74 -0
  386. package/artemis-main/mcp_server/notifiers/agentapi.py +289 -0
  387. package/artemis-main/mcp_server/notifiers/base.py +45 -0
  388. package/artemis-main/mcp_server/notifiers/composite.py +87 -0
  389. package/artemis-main/mcp_server/notifiers/desktop.py +108 -0
  390. package/artemis-main/mcp_server/notifiers/file.py +69 -0
  391. package/artemis-main/mcp_server/notifiers/script.py +90 -0
  392. package/artemis-main/mcp_server/notifiers/webhook.py +87 -0
  393. package/artemis-main/mcp_server/rules.md +68 -0
  394. package/artemis-main/mcp_server/server.py +106 -0
  395. package/artemis-main/mcp_server/tools/__init__.py +29 -0
  396. package/artemis-main/mcp_server/tools/device_state.py +107 -0
  397. package/artemis-main/mcp_server/tools/diagnose.py +1249 -0
  398. package/artemis-main/mcp_server/tools/inspect_trace.py +394 -0
  399. package/artemis-main/mcp_server/tools/task_manager.py +627 -0
  400. package/artemis-main/mcp_server/tools/task_runner.py +548 -0
  401. package/artemis-main/mcp_server/utils/__init__.py +15 -0
  402. package/artemis-main/mcp_server/utils/device_utils.py +180 -0
  403. package/artemis-main/mcp_server/utils/env_utils.py +65 -0
  404. package/artemis-main/packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk +0 -0
  405. package/artemis-main/packages/artemis-accessibility-helper/app/build.gradle.kts +35 -0
  406. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml +49 -0
  407. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java +274 -0
  408. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ArtemisAccessibilityService.java +227 -0
  409. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java +463 -0
  410. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/DisplayUtils.java +111 -0
  411. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java +232 -0
  412. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/HierarchyDumper.java +832 -0
  413. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenReceiver.java +36 -0
  414. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenStore.java +38 -0
  415. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/XmlUtils.java +102 -0
  416. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/values/strings.xml +4 -0
  417. package/artemis-main/packages/artemis-accessibility-helper/app/src/main/res/xml/accessibility_service_config.xml +10 -0
  418. package/artemis-main/packages/artemis-accessibility-helper/build.gradle.kts +3 -0
  419. package/artemis-main/packages/artemis-accessibility-helper/build_apk.sh +215 -0
  420. package/artemis-main/packages/artemis-accessibility-helper/debug.keystore +0 -0
  421. package/artemis-main/packages/artemis-accessibility-helper/gradle.properties +2 -0
  422. package/artemis-main/packages/artemis-accessibility-helper/helper_manifest.json +7 -0
  423. package/artemis-main/packages/artemis-accessibility-helper/settings.gradle.kts +17 -0
  424. package/artemis-main/packages/artemis-client/DESIGN.md +89 -0
  425. package/artemis-main/packages/artemis-client/README.md +112 -0
  426. package/artemis-main/packages/artemis-client/examples/basic.py +30 -0
  427. package/artemis-main/packages/artemis-client/pyproject.toml +36 -0
  428. package/artemis-main/packages/artemis-client/src/artemis_client/__init__.py +45 -0
  429. package/artemis-main/packages/artemis-client/src/artemis_client/client.py +401 -0
  430. package/artemis-main/packages/artemis-client/src/artemis_client/errors.py +66 -0
  431. package/artemis-main/packages/artemis-client/src/artemis_client/models.py +212 -0
  432. package/artemis-main/packages/artemis-client/src/artemis_client/py.typed +1 -0
  433. package/artemis-main/packages/artemis-client/src/artemis_client/transport.py +134 -0
  434. package/artemis-main/packages/artemis-client/tests/test_client.py +289 -0
  435. package/artemis-main/packages/artemis-client/tests/test_transport.py +85 -0
  436. package/artemis-main/pyproject.toml +259 -0
  437. package/artemis-main/pyright-core.json +17 -0
  438. package/artemis-main/scripts/check_dependency_sources.py +68 -0
  439. package/artemis-main/scripts/install_deps.ps1 +520 -0
  440. package/artemis-main/scripts/install_deps.sh +533 -0
  441. package/artemis-main/scripts/quality_ratchet.py +97 -0
  442. package/artemis-main/scripts/real_device_video_acceptance.py +483 -0
  443. package/artemis-main/scripts/run_edge_case_part.py +327 -0
  444. package/artemis-main/scripts/run_ui.py +242 -0
  445. package/artemis-main/scripts/start.ps1 +500 -0
  446. package/artemis-main/scripts/test_all_edge_cases.py +269 -0
  447. package/artemis-main/scripts/verify_concurrency_and_devices.py +385 -0
  448. package/artemis-main/setup.py +164 -0
  449. package/artemis-main/start.bat +35 -0
  450. package/artemis-main/start.sh +423 -0
  451. package/artemis-main/third_party/mobile_use/LICENSE +190 -0
  452. package/artemis-main/third_party/mobile_use/METADATA +27 -0
  453. package/artemis-main/third_party/mobile_use/NOTICE +12 -0
  454. package/artemis-main/third_party/mobile_use/__init__.py +19 -0
  455. package/artemis-main/third_party/mobile_use/agents/__init__.py +13 -0
  456. package/artemis-main/third_party/mobile_use/agents/hopper/__init__.py +13 -0
  457. package/artemis-main/third_party/mobile_use/agents/hopper/hopper.md +22 -0
  458. package/artemis-main/third_party/mobile_use/agents/hopper/hopper.py +80 -0
  459. package/artemis-main/third_party/mobile_use/clients/__init__.py +13 -0
  460. package/artemis-main/third_party/mobile_use/clients/ui_automator_client.py +482 -0
  461. package/artemis-main/third_party/mobile_use/config/__init__.py +13 -0
  462. package/artemis-main/third_party/mobile_use/config/llm.py +183 -0
  463. package/artemis-main/third_party/mobile_use/config/output.py +184 -0
  464. package/artemis-main/third_party/mobile_use/context.py +119 -0
  465. package/artemis-main/third_party/mobile_use/controllers/__init__.py +13 -0
  466. package/artemis-main/third_party/mobile_use/controllers/platform_specific_commands_controller.py +151 -0
  467. package/artemis-main/third_party/mobile_use/controllers/types.py +92 -0
  468. package/artemis-main/third_party/mobile_use/controllers/unified_controller.py +183 -0
  469. package/artemis-main/third_party/mobile_use/graph.py +54 -0
  470. package/artemis-main/third_party/mobile_use/llm_service.py +193 -0
  471. package/artemis-main/third_party/mobile_use/main.py +130 -0
  472. package/artemis-main/third_party/mobile_use/sdk/__init__.py +13 -0
  473. package/artemis-main/third_party/mobile_use/sdk/agent.py +1060 -0
  474. package/artemis-main/third_party/mobile_use/sdk/builders/__init__.py +13 -0
  475. package/artemis-main/third_party/mobile_use/sdk/builders/agent_config_builder.py +225 -0
  476. package/artemis-main/third_party/mobile_use/sdk/builders/index.py +33 -0
  477. package/artemis-main/third_party/mobile_use/sdk/builders/task_request_builder.py +243 -0
  478. package/artemis-main/third_party/mobile_use/sdk/types/__init__.py +13 -0
  479. package/artemis-main/third_party/mobile_use/sdk/types/agent.py +98 -0
  480. package/artemis-main/third_party/mobile_use/sdk/types/exceptions.py +109 -0
  481. package/artemis-main/third_party/mobile_use/sdk/types/task.py +234 -0
  482. package/artemis-main/third_party/mobile_use/testing.py +59 -0
  483. package/artemis-main/third_party/mobile_use/tools/__init__.py +13 -0
  484. package/artemis-main/third_party/mobile_use/tools/index.py +44 -0
  485. package/artemis-main/third_party/mobile_use/tools/mobile/__init__.py +13 -0
  486. package/artemis-main/third_party/mobile_use/tools/mobile/launch_app.py +89 -0
  487. package/artemis-main/third_party/mobile_use/tools/tool_wrapper.py +52 -0
  488. package/artemis-main/third_party/mobile_use/utils/__init__.py +13 -0
  489. package/artemis-main/third_party/mobile_use/utils/app_launch_utils.py +217 -0
  490. package/artemis-main/third_party/mobile_use/utils/cli_helpers.py +60 -0
  491. package/artemis-main/third_party/mobile_use/utils/decorators.py +158 -0
  492. package/artemis-main/third_party/mobile_use/utils/file.py +31 -0
  493. package/artemis-main/third_party/mobile_use/utils/logger.py +216 -0
  494. package/artemis-main/third_party/mobile_use/utils/media.py +213 -0
  495. package/artemis-main/third_party/mobile_use/utils/shell_utils.py +39 -0
  496. package/artemis-main/third_party/mobile_use/utils/video.py +398 -0
  497. package/artemis-main/uv.lock +4908 -0
  498. package/bin/android-midscene-automation.js +1 -5
  499. package/bin/artemis-config.mjs +15 -0
  500. package/bin/artemis-embedded.py +56 -0
  501. package/bin/start-artemis.mjs +30 -0
  502. package/bin/start-dev.mjs +112 -0
  503. package/bin/test_artemis_embedded.py +44 -0
  504. package/package.json +10 -8
  505. package/server/artemis.ts +34 -0
  506. package/server/http-api.ts +13 -1
  507. package/src/App.vue +72 -23
  508. package/src/api.ts +5 -0
  509. package/src/appium-recorder/AppiumPage.vue +1 -1
  510. package/src/appium-recorder/components/ComponentInspector.vue +1 -1
  511. package/src/appium-recorder/components/FlowNodeSearch.vue +3 -3
  512. package/src/appium-recorder/components/ImageCheckDialog.vue +3 -3
  513. package/src/appium-recorder/components/RecorderWorkspace.vue +4 -4
  514. package/src/appium-recorder/flow-appearance.ts +4 -4
  515. package/src/components/AppSidebar.vue +8 -7
  516. package/src/components/MidsceneMaintenanceNotice.vue +17 -0
  517. package/src/components/analytics/AnalysisCharts.vue +4 -4
  518. package/src/components/analytics/ReportAnalysis.vue +1 -1
  519. package/src/components/generator/TestCaseFileUpload.vue +1 -1
  520. package/src/components/help/HelpCenter.vue +56 -37
  521. package/src/pages/AiGeneratorPage.vue +1 -1
  522. package/src/pages/ArtemisPage.vue +72 -0
  523. package/src/pages/ConfigPage.vue +18 -19
  524. package/src/style.css +369 -186
  525. package/src/types.ts +1 -1
@@ -0,0 +1,821 @@
1
+ # Copyright 2026 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Persistent, video-scoped memory for video analysis agents."""
16
+
17
+ from __future__ import annotations
18
+
19
+ from contextlib import contextmanager
20
+ from dataclasses import dataclass
21
+ import hashlib
22
+ import json
23
+ from pathlib import Path
24
+ import shutil
25
+ import sqlite3
26
+ import threading
27
+ import time
28
+ from typing import Any, Iterator, Literal
29
+ from uuid import uuid4
30
+
31
+ from artemis.context import ArtemisContext
32
+ from third_party.mobile_use.utils.logger import get_logger
33
+ from third_party.mobile_use.utils.video import get_active_session
34
+
35
+ logger = get_logger(__name__)
36
+
37
+
38
+ ClaimState = Literal["claimed", "cached", "in_progress"]
39
+
40
+
41
+ @dataclass(frozen=True)
42
+ class SegmentClaim:
43
+ """Result of atomically attempting to acquire one analysis segment."""
44
+
45
+ state: ClaimState
46
+ lease_owner: str | None = None
47
+ summary: str | None = None
48
+ analysis: str | None = None
49
+
50
+
51
+ def _normalise_query(query: str | None) -> str:
52
+ return " ".join(str(query or "").strip().lower().split())
53
+
54
+
55
+ def _query_key(query: str | None) -> str:
56
+ return hashlib.sha256(_normalise_query(query).encode("utf-8")).hexdigest()
57
+
58
+
59
+ def _to_ms(value: float | int) -> int:
60
+ return int(round(float(value) * 1000.0))
61
+
62
+
63
+ def _to_seconds(value: int) -> float:
64
+ return round(value / 1000.0, 3)
65
+
66
+
67
+ class VideoBlackboard:
68
+ """SQLite-backed blackboard with a context-local in-memory fallback."""
69
+
70
+ def __init__(
71
+ self,
72
+ board_key: str,
73
+ *,
74
+ db_path: Path | None = None,
75
+ evidence_dir: Path | None = None,
76
+ lease_seconds: float = 300.0,
77
+ ) -> None:
78
+ self.board_key = board_key
79
+ self.db_path = Path(db_path) if db_path is not None else None
80
+ self.evidence_dir = Path(evidence_dir) if evidence_dir is not None else None
81
+ self.lease_seconds = max(0.01, float(lease_seconds))
82
+ self._lock = threading.RLock()
83
+ self._memory_observations: dict[str, dict[str, Any]] = {}
84
+ self._memory_segments: dict[tuple[str, str, int, int], dict[str, Any]] = {}
85
+
86
+ if self.db_path is not None:
87
+ self.db_path.parent.mkdir(parents=True, exist_ok=True)
88
+ self._init_db()
89
+ if self.evidence_dir is not None:
90
+ self.evidence_dir.mkdir(parents=True, exist_ok=True)
91
+
92
+ @contextmanager
93
+ def _connection(self) -> Iterator[sqlite3.Connection]:
94
+ if self.db_path is None:
95
+ raise RuntimeError("SQLite connection requested for in-memory blackboard")
96
+ with self._lock:
97
+ conn = sqlite3.connect(self.db_path, timeout=30.0)
98
+ conn.row_factory = sqlite3.Row
99
+ try:
100
+ yield conn
101
+ finally:
102
+ conn.close()
103
+
104
+ def _init_db(self) -> None:
105
+ with self._connection() as conn:
106
+ conn.execute("PRAGMA journal_mode=WAL")
107
+ conn.execute(
108
+ """
109
+ CREATE TABLE IF NOT EXISTS video_analysis_segments (
110
+ board_key TEXT NOT NULL,
111
+ query_key TEXT NOT NULL,
112
+ query_text TEXT NOT NULL,
113
+ modality TEXT NOT NULL,
114
+ start_ms INTEGER NOT NULL,
115
+ end_ms INTEGER NOT NULL,
116
+ status TEXT NOT NULL,
117
+ lease_owner TEXT,
118
+ lease_expires_at REAL,
119
+ attempt_count INTEGER NOT NULL DEFAULT 0,
120
+ summary TEXT,
121
+ analysis TEXT,
122
+ error TEXT,
123
+ error_category TEXT,
124
+ model_name TEXT,
125
+ source_generation INTEGER,
126
+ created_at REAL NOT NULL,
127
+ updated_at REAL NOT NULL,
128
+ PRIMARY KEY (board_key, query_key, modality, start_ms, end_ms)
129
+ )
130
+ """
131
+ )
132
+ conn.execute(
133
+ """
134
+ CREATE TABLE IF NOT EXISTS video_analysis_observations (
135
+ observation_id TEXT PRIMARY KEY,
136
+ board_key TEXT NOT NULL,
137
+ start_ms INTEGER NOT NULL,
138
+ end_ms INTEGER,
139
+ query_text TEXT NOT NULL,
140
+ summary TEXT NOT NULL,
141
+ confidence REAL,
142
+ modality TEXT NOT NULL,
143
+ screenshot_path TEXT,
144
+ extra_json TEXT,
145
+ created_at REAL NOT NULL
146
+ )
147
+ """
148
+ )
149
+ columns = {
150
+ row["name"]
151
+ for row in conn.execute("PRAGMA table_info(video_analysis_segments)").fetchall()
152
+ }
153
+ for name, sql_type in (
154
+ ("error_category", "TEXT"),
155
+ ("model_name", "TEXT"),
156
+ ("source_generation", "INTEGER"),
157
+ ):
158
+ if name not in columns:
159
+ try:
160
+ conn.execute(
161
+ f"ALTER TABLE video_analysis_segments ADD COLUMN {name} {sql_type}"
162
+ )
163
+ except sqlite3.OperationalError as error:
164
+ if "duplicate column" not in str(error).lower():
165
+ raise
166
+ conn.execute(
167
+ """
168
+ CREATE INDEX IF NOT EXISTS idx_video_analysis_segments_lookup
169
+ ON video_analysis_segments (board_key, query_key, modality, status, start_ms, end_ms)
170
+ """
171
+ )
172
+ conn.execute(
173
+ """
174
+ CREATE INDEX IF NOT EXISTS idx_video_analysis_observations_board
175
+ ON video_analysis_observations (board_key, start_ms, end_ms)
176
+ """
177
+ )
178
+ conn.commit()
179
+
180
+ def _persist_evidence(self, entry: dict[str, Any], observation_id: str) -> str | None:
181
+ candidate = entry.get("screenshot") or entry.get("screenshot_path")
182
+ if not candidate:
183
+ return None
184
+ source = Path(str(candidate))
185
+ if not source.exists() or not source.is_file() or self.evidence_dir is None:
186
+ return str(source) if source.exists() else None
187
+ try:
188
+ if source.stat().st_size <= 0:
189
+ return None
190
+ except OSError:
191
+ return None
192
+
193
+ suffix = source.suffix.lower() or ".jpg"
194
+ destination = self.evidence_dir / f"{observation_id}{suffix}"
195
+ if source.resolve() != destination.resolve() and not destination.exists():
196
+ try:
197
+ shutil.copy2(source, destination)
198
+ except OSError as exc:
199
+ logger.warning(f"Failed to persist video evidence {source}: {exc}")
200
+ return str(source)
201
+ return str(destination)
202
+
203
+ def add_observation(self, entry: dict[str, Any], *, modality: str = "video") -> dict[str, Any]:
204
+ """Idempotently persist one evidence-backed observation."""
205
+
206
+ start = float(entry.get("start", 0.0))
207
+ raw_end = entry.get("end")
208
+ end = None if raw_end in (None, "unknown") else float(raw_end)
209
+ query_text = str(entry.get("target", ""))
210
+ summary = str(entry.get("summary", "")).strip()
211
+ identity = json.dumps(
212
+ {
213
+ "board_key": self.board_key,
214
+ "start_ms": _to_ms(start),
215
+ "end_ms": _to_ms(end) if end is not None else None,
216
+ "query": _normalise_query(query_text),
217
+ "summary": summary,
218
+ "modality": modality,
219
+ },
220
+ sort_keys=True,
221
+ separators=(",", ":"),
222
+ )
223
+ observation_id = hashlib.sha256(identity.encode("utf-8")).hexdigest()
224
+ screenshot_path = self._persist_evidence(entry, observation_id)
225
+
226
+ stored = dict(entry)
227
+ if screenshot_path:
228
+ stored["screenshot"] = screenshot_path
229
+ stored["modality"] = modality
230
+ stored["observation_id"] = observation_id
231
+
232
+ excluded = {
233
+ "start",
234
+ "end",
235
+ "target",
236
+ "summary",
237
+ "confidence_score",
238
+ "screenshot",
239
+ "screenshot_path",
240
+ "modality",
241
+ "observation_id",
242
+ }
243
+ extra = {key: value for key, value in stored.items() if key not in excluded}
244
+ now = time.time()
245
+
246
+ if self.db_path is None:
247
+ with self._lock:
248
+ self._memory_observations.setdefault(observation_id, stored)
249
+ return dict(self._memory_observations[observation_id])
250
+
251
+ with self._connection() as conn:
252
+ conn.execute(
253
+ """
254
+ INSERT OR IGNORE INTO video_analysis_observations (
255
+ observation_id, board_key, start_ms, end_ms, query_text,
256
+ summary, confidence, modality, screenshot_path, extra_json, created_at
257
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
258
+ """,
259
+ (
260
+ observation_id,
261
+ self.board_key,
262
+ _to_ms(start),
263
+ _to_ms(end) if end is not None else None,
264
+ query_text,
265
+ summary,
266
+ float(entry.get("confidence_score", 0.0)),
267
+ modality,
268
+ screenshot_path,
269
+ json.dumps(extra, default=str),
270
+ now,
271
+ ),
272
+ )
273
+ conn.commit()
274
+ return stored
275
+
276
+ def list_observations(self) -> list[dict[str, Any]]:
277
+ """Return all observations for this recording in timeline order."""
278
+
279
+ if self.db_path is None:
280
+ with self._lock:
281
+ entries = [dict(value) for value in self._memory_observations.values()]
282
+ return sorted(entries, key=lambda value: float(value.get("start", 0.0)))
283
+
284
+ with self._connection() as conn:
285
+ rows = conn.execute(
286
+ """
287
+ SELECT * FROM video_analysis_observations
288
+ WHERE board_key = ?
289
+ ORDER BY start_ms, created_at
290
+ """,
291
+ (self.board_key,),
292
+ ).fetchall()
293
+
294
+ entries: list[dict[str, Any]] = []
295
+ for row in rows:
296
+ entry: dict[str, Any] = {
297
+ "start": _to_seconds(row["start_ms"]),
298
+ "end": _to_seconds(row["end_ms"]) if row["end_ms"] is not None else "unknown",
299
+ "target": row["query_text"],
300
+ "summary": row["summary"],
301
+ "confidence_score": row["confidence"],
302
+ "modality": row["modality"],
303
+ "observation_id": row["observation_id"],
304
+ }
305
+ if row["screenshot_path"]:
306
+ entry["screenshot"] = row["screenshot_path"]
307
+ if row["extra_json"]:
308
+ try:
309
+ entry.update(json.loads(row["extra_json"]))
310
+ except (TypeError, json.JSONDecodeError):
311
+ pass
312
+ entries.append(entry)
313
+ return entries
314
+
315
+ def list_ledger_entries(self) -> list[dict[str, Any]]:
316
+ """Return observations plus successful query-specific segment summaries."""
317
+
318
+ entries = self.list_observations()
319
+ if self.db_path is None:
320
+ with self._lock:
321
+ for (query_key, modality, start_ms, end_ms), value in self._memory_segments.items():
322
+ if value.get("status") != "succeeded":
323
+ continue
324
+ entries.append(
325
+ {
326
+ "start": _to_seconds(start_ms),
327
+ "end": _to_seconds(end_ms),
328
+ "target": value.get("query_text", ""),
329
+ "summary": value.get("summary") or "No summary provided.",
330
+ "confidence_score": 0.5,
331
+ "modality": modality,
332
+ "kind": "segment_result",
333
+ "query_key": query_key,
334
+ }
335
+ )
336
+ else:
337
+ with self._connection() as conn:
338
+ rows = conn.execute(
339
+ """
340
+ SELECT query_key, query_text, modality, start_ms, end_ms, summary
341
+ FROM video_analysis_segments
342
+ WHERE board_key = ? AND status = 'succeeded'
343
+ ORDER BY start_ms, updated_at
344
+ """,
345
+ (self.board_key,),
346
+ ).fetchall()
347
+ entries.extend(
348
+ {
349
+ "start": _to_seconds(row["start_ms"]),
350
+ "end": _to_seconds(row["end_ms"]),
351
+ "target": row["query_text"],
352
+ "summary": row["summary"] or "No summary provided.",
353
+ "confidence_score": 0.5,
354
+ "modality": row["modality"],
355
+ "kind": "segment_result",
356
+ "query_key": row["query_key"],
357
+ }
358
+ for row in rows
359
+ )
360
+ return sorted(entries, key=lambda value: float(value.get("start", 0.0)))
361
+
362
+ def claim_segment(
363
+ self,
364
+ start: float,
365
+ end: float,
366
+ query: str,
367
+ *,
368
+ modality: str = "video",
369
+ model_name: str | None = None,
370
+ source_generation: int | None = None,
371
+ ) -> SegmentClaim:
372
+ """Atomically acquire work, or return a cached/in-progress result."""
373
+
374
+ start_ms = _to_ms(start)
375
+ end_ms = _to_ms(end)
376
+ if end_ms <= start_ms:
377
+ raise ValueError("Video analysis segment end must be greater than start")
378
+ q_key = _query_key(query)
379
+ key = (q_key, modality, start_ms, end_ms)
380
+ now = time.time()
381
+ owner = str(uuid4())
382
+ expires = now + self.lease_seconds
383
+
384
+ if self.db_path is None:
385
+ with self._lock:
386
+ existing = self._memory_segments.get(key)
387
+ if existing and existing["status"] == "succeeded":
388
+ return SegmentClaim(
389
+ "cached", summary=existing.get("summary"), analysis=existing.get("analysis")
390
+ )
391
+ if (
392
+ existing
393
+ and existing["status"] == "running"
394
+ and float(existing.get("lease_expires_at") or 0.0) > now
395
+ ):
396
+ return SegmentClaim("in_progress")
397
+ attempts = int(existing.get("attempt_count", 0)) + 1 if existing else 1
398
+ self._memory_segments[key] = {
399
+ "query_text": query,
400
+ "status": "running",
401
+ "lease_owner": owner,
402
+ "lease_expires_at": expires,
403
+ "attempt_count": attempts,
404
+ "model_name": model_name,
405
+ "source_generation": source_generation,
406
+ "start_ms": start_ms,
407
+ "end_ms": end_ms,
408
+ "updated_at": now,
409
+ }
410
+ return SegmentClaim("claimed", lease_owner=owner)
411
+
412
+ with self._connection() as conn:
413
+ conn.execute("BEGIN IMMEDIATE")
414
+ row = conn.execute(
415
+ """
416
+ SELECT * FROM video_analysis_segments
417
+ WHERE board_key = ? AND query_key = ? AND modality = ?
418
+ AND start_ms = ? AND end_ms = ?
419
+ """,
420
+ (self.board_key, q_key, modality, start_ms, end_ms),
421
+ ).fetchone()
422
+ if row is not None and row["status"] == "succeeded":
423
+ conn.commit()
424
+ return SegmentClaim("cached", summary=row["summary"], analysis=row["analysis"])
425
+ if (
426
+ row is not None
427
+ and row["status"] == "running"
428
+ and float(row["lease_expires_at"] or 0.0) > now
429
+ ):
430
+ conn.commit()
431
+ return SegmentClaim("in_progress")
432
+
433
+ if row is None:
434
+ conn.execute(
435
+ """
436
+ INSERT INTO video_analysis_segments (
437
+ board_key, query_key, query_text, modality, start_ms, end_ms,
438
+ status, lease_owner, lease_expires_at, attempt_count,
439
+ model_name, source_generation, created_at, updated_at
440
+ ) VALUES (?, ?, ?, ?, ?, ?, 'running', ?, ?, 1, ?, ?, ?, ?)
441
+ """,
442
+ (
443
+ self.board_key,
444
+ q_key,
445
+ query,
446
+ modality,
447
+ start_ms,
448
+ end_ms,
449
+ owner,
450
+ expires,
451
+ model_name,
452
+ source_generation,
453
+ now,
454
+ now,
455
+ ),
456
+ )
457
+ else:
458
+ conn.execute(
459
+ """
460
+ UPDATE video_analysis_segments
461
+ SET status = 'running', lease_owner = ?, lease_expires_at = ?,
462
+ attempt_count = attempt_count + 1, error = NULL,
463
+ error_category = NULL, model_name = ?, source_generation = ?,
464
+ updated_at = ?
465
+ WHERE board_key = ? AND query_key = ? AND modality = ?
466
+ AND start_ms = ? AND end_ms = ?
467
+ """,
468
+ (
469
+ owner,
470
+ expires,
471
+ model_name,
472
+ source_generation,
473
+ now,
474
+ self.board_key,
475
+ q_key,
476
+ modality,
477
+ start_ms,
478
+ end_ms,
479
+ ),
480
+ )
481
+ conn.commit()
482
+ return SegmentClaim("claimed", lease_owner=owner)
483
+
484
+ def complete_segment(
485
+ self,
486
+ start: float,
487
+ end: float,
488
+ query: str,
489
+ lease_owner: str,
490
+ summary: str,
491
+ analysis: str | None = None,
492
+ *,
493
+ modality: str = "video",
494
+ ) -> None:
495
+ """Commit a complete result while preserving lease ownership."""
496
+
497
+ key = (_query_key(query), modality, _to_ms(start), _to_ms(end))
498
+ now = time.time()
499
+ if self.db_path is None:
500
+ with self._lock:
501
+ row = self._memory_segments.get(key)
502
+ if row is None or row.get("lease_owner") != lease_owner:
503
+ return
504
+ row.update(
505
+ {
506
+ "status": "succeeded",
507
+ "summary": summary,
508
+ "analysis": analysis,
509
+ "lease_expires_at": None,
510
+ "updated_at": now,
511
+ }
512
+ )
513
+ return
514
+
515
+ with self._connection() as conn:
516
+ conn.execute(
517
+ """
518
+ UPDATE video_analysis_segments
519
+ SET status = 'succeeded', summary = ?, analysis = ?, error = NULL,
520
+ lease_expires_at = NULL, updated_at = ?
521
+ WHERE board_key = ? AND query_key = ? AND modality = ?
522
+ AND start_ms = ? AND end_ms = ? AND lease_owner = ?
523
+ """,
524
+ (
525
+ summary,
526
+ analysis,
527
+ now,
528
+ self.board_key,
529
+ key[0],
530
+ modality,
531
+ key[2],
532
+ key[3],
533
+ lease_owner,
534
+ ),
535
+ )
536
+ conn.commit()
537
+
538
+ def fail_segment(
539
+ self,
540
+ start: float,
541
+ end: float,
542
+ query: str,
543
+ lease_owner: str,
544
+ error: str,
545
+ *,
546
+ modality: str = "video",
547
+ retryable: bool = True,
548
+ error_category: str | None = None,
549
+ ) -> None:
550
+ """Persist a failed attempt without deleting prior observations."""
551
+
552
+ key = (_query_key(query), modality, _to_ms(start), _to_ms(end))
553
+ status = "retryable_failed" if retryable else "permanent_failed"
554
+ now = time.time()
555
+ if self.db_path is None:
556
+ with self._lock:
557
+ row = self._memory_segments.get(key)
558
+ if row is None or row.get("lease_owner") != lease_owner:
559
+ return
560
+ row.update(
561
+ {
562
+ "status": status,
563
+ "error": str(error),
564
+ "error_category": error_category,
565
+ "lease_expires_at": None,
566
+ "updated_at": now,
567
+ }
568
+ )
569
+ return
570
+
571
+ with self._connection() as conn:
572
+ conn.execute(
573
+ """
574
+ UPDATE video_analysis_segments
575
+ SET status = ?, error = ?, error_category = ?,
576
+ lease_expires_at = NULL, updated_at = ?
577
+ WHERE board_key = ? AND query_key = ? AND modality = ?
578
+ AND start_ms = ? AND end_ms = ? AND lease_owner = ?
579
+ """,
580
+ (
581
+ status,
582
+ str(error),
583
+ error_category,
584
+ now,
585
+ self.board_key,
586
+ key[0],
587
+ modality,
588
+ key[2],
589
+ key[3],
590
+ lease_owner,
591
+ ),
592
+ )
593
+ conn.commit()
594
+
595
+ def metrics(self) -> dict[str, Any]:
596
+ """Return a compact operational snapshot for tracing and diagnostics."""
597
+
598
+ if self.db_path is None:
599
+ with self._lock:
600
+ rows = [dict(value) for value in self._memory_segments.values()]
601
+ observation_count = len(self._memory_observations)
602
+ else:
603
+ with self._connection() as conn:
604
+ rows = [
605
+ dict(row)
606
+ for row in conn.execute(
607
+ """
608
+ SELECT status, attempt_count, start_ms, end_ms, error_category
609
+ FROM video_analysis_segments WHERE board_key = ?
610
+ """,
611
+ (self.board_key,),
612
+ ).fetchall()
613
+ ]
614
+ observation_count = int(
615
+ conn.execute(
616
+ "SELECT COUNT(*) FROM video_analysis_observations WHERE board_key = ?",
617
+ (self.board_key,),
618
+ ).fetchone()[0]
619
+ )
620
+
621
+ statuses: dict[str, int] = {}
622
+ failure_categories: dict[str, int] = {}
623
+ successful_seconds = 0.0
624
+ attempts = 0
625
+ for row in rows:
626
+ status = str(row.get("status", "unknown"))
627
+ statuses[status] = statuses.get(status, 0) + 1
628
+ attempts += int(row.get("attempt_count", 0) or 0)
629
+ category = row.get("error_category")
630
+ if category:
631
+ failure_categories[str(category)] = failure_categories.get(str(category), 0) + 1
632
+ if status == "succeeded" and row.get("start_ms") is not None:
633
+ successful_seconds += max(
634
+ 0.0,
635
+ (float(row.get("end_ms", 0)) - float(row.get("start_ms", 0))) / 1000.0,
636
+ )
637
+ return {
638
+ "board_key": self.board_key,
639
+ "segments": statuses,
640
+ "attempts": attempts,
641
+ "failure_categories": failure_categories,
642
+ "successful_seconds": round(successful_seconds, 3),
643
+ "observations": observation_count,
644
+ }
645
+
646
+ def successful_segments(
647
+ self, start: float, end: float, query: str, *, modality: str = "video"
648
+ ) -> list[dict[str, Any]]:
649
+ """Return successful segments overlapping a requested interval."""
650
+
651
+ start_ms, end_ms = _to_ms(start), _to_ms(end)
652
+ q_key = _query_key(query)
653
+ if self.db_path is None:
654
+ with self._lock:
655
+ rows = []
656
+ for (
657
+ row_query,
658
+ row_modality,
659
+ row_start,
660
+ row_end,
661
+ ), value in self._memory_segments.items():
662
+ if (
663
+ row_query == q_key
664
+ and row_modality == modality
665
+ and value.get("status") == "succeeded"
666
+ and row_end > start_ms
667
+ and row_start < end_ms
668
+ ):
669
+ rows.append(
670
+ {
671
+ **value,
672
+ "start": _to_seconds(row_start),
673
+ "end": _to_seconds(row_end),
674
+ }
675
+ )
676
+ return sorted(rows, key=lambda value: value["start"])
677
+
678
+ with self._connection() as conn:
679
+ db_rows = conn.execute(
680
+ """
681
+ SELECT start_ms, end_ms, summary, analysis, status
682
+ FROM video_analysis_segments
683
+ WHERE board_key = ? AND query_key = ? AND modality = ?
684
+ AND status = 'succeeded' AND end_ms > ? AND start_ms < ?
685
+ ORDER BY start_ms
686
+ """,
687
+ (self.board_key, q_key, modality, start_ms, end_ms),
688
+ ).fetchall()
689
+ return [
690
+ {
691
+ "start": _to_seconds(row["start_ms"]),
692
+ "end": _to_seconds(row["end_ms"]),
693
+ "summary": row["summary"],
694
+ "analysis": row["analysis"],
695
+ "status": row["status"],
696
+ }
697
+ for row in db_rows
698
+ ]
699
+
700
+ def missing_intervals(
701
+ self, start: float, end: float, query: str, *, modality: str = "video"
702
+ ) -> list[tuple[float, float]]:
703
+ """Subtract successful exact-query coverage from a requested interval."""
704
+
705
+ start_ms, end_ms = _to_ms(start), _to_ms(end)
706
+ if end_ms <= start_ms:
707
+ return []
708
+ covered = []
709
+ for row in self.successful_segments(start, end, query, modality=modality):
710
+ covered.append((max(start_ms, _to_ms(row["start"])), min(end_ms, _to_ms(row["end"]))))
711
+ covered = sorted((left, right) for left, right in covered if right > left)
712
+
713
+ merged: list[tuple[int, int]] = []
714
+ for left, right in covered:
715
+ if not merged or left > merged[-1][1]:
716
+ merged.append((left, right))
717
+ else:
718
+ merged[-1] = (merged[-1][0], max(merged[-1][1], right))
719
+
720
+ missing: list[tuple[float, float]] = []
721
+ cursor = start_ms
722
+ for left, right in merged:
723
+ if left > cursor:
724
+ missing.append((_to_seconds(cursor), _to_seconds(left)))
725
+ cursor = max(cursor, right)
726
+ if cursor < end_ms:
727
+ missing.append((_to_seconds(cursor), _to_seconds(end_ms)))
728
+ return missing
729
+
730
+ def format_cached_segments(
731
+ self, start: float, end: float, query: str, *, modality: str = "video"
732
+ ) -> list[str]:
733
+ """Format cached segment results in the child-agent response shape."""
734
+
735
+ results = []
736
+ for row in self.successful_segments(start, end, query, modality=modality):
737
+ text = (
738
+ f"[from {row['start']:.1f}s to {row['end']:.1f}s] "
739
+ f"Summary: {row.get('summary') or 'No summary provided.'}"
740
+ )
741
+ if row.get("analysis"):
742
+ text += f" Analysis: {row['analysis']}"
743
+ results.append(text)
744
+ return results
745
+
746
+
747
+ def _safe_path(value: Any) -> Path | None:
748
+ if isinstance(value, Path):
749
+ return value
750
+ if isinstance(value, str) and value:
751
+ return Path(value)
752
+ return None
753
+
754
+
755
+ def _latest_recorded_video_key(db_path: Path | None, session_id: Any) -> str | None:
756
+ """Resolve a stopped recording so a restarted analyzer keeps its memory key."""
757
+ if db_path is None or not session_id or not db_path.exists():
758
+ return None
759
+ try:
760
+ conn = sqlite3.connect(db_path, timeout=5.0)
761
+ try:
762
+ row = conn.execute(
763
+ """
764
+ SELECT video_id FROM video_recordings
765
+ WHERE session_id = ?
766
+ ORDER BY start_time DESC LIMIT 1
767
+ """,
768
+ (str(session_id),),
769
+ ).fetchone()
770
+ finally:
771
+ conn.close()
772
+ except sqlite3.Error:
773
+ return None
774
+ return f"video:{row[0]}" if row and row[0] else None
775
+
776
+
777
+ def get_video_blackboard(ctx: ArtemisContext) -> VideoBlackboard:
778
+ """Return the blackboard shared by every analyzer in an Artemis context."""
779
+
780
+ existing = getattr(ctx, "_video_blackboard", None)
781
+ if isinstance(existing, VideoBlackboard):
782
+ return existing
783
+
784
+ device_id = str(getattr(getattr(ctx, "device", None), "device_id", "default"))
785
+ data_engine = getattr(ctx, "data_engine", None)
786
+ db_path = None
787
+ base_dir = None
788
+ storage = getattr(data_engine, "storage", None)
789
+ if storage is not None:
790
+ db_path = _safe_path(getattr(storage, "db_path", None))
791
+ if data_engine is not None:
792
+ base_dir = _safe_path(getattr(data_engine, "base_dir", None))
793
+
794
+ if base_dir is None:
795
+ execution_setup = getattr(ctx, "execution_setup", None)
796
+ traces_path = _safe_path(getattr(execution_setup, "traces_path", None))
797
+ trace_name = getattr(execution_setup, "trace_name", None)
798
+ if traces_path is not None:
799
+ base_dir = traces_path / str(trace_name) if trace_name else traces_path
800
+
801
+ if db_path is None and base_dir is not None:
802
+ db_path = base_dir / "video_blackboard.db"
803
+ evidence_dir = base_dir / "video_blackboard" / "evidence" if base_dir else None
804
+
805
+ session = get_active_session(device_id)
806
+ current_session_id = getattr(data_engine, "current_session_id", None)
807
+ if session is not None:
808
+ board_key = f"video:{session.video_id}"
809
+ else:
810
+ board_key = _latest_recorded_video_key(db_path, current_session_id) or (
811
+ f"session:{current_session_id}" if current_session_id else f"device:{device_id}:runtime"
812
+ )
813
+
814
+ board = VideoBlackboard(board_key, db_path=db_path, evidence_dir=evidence_dir)
815
+ try:
816
+ setattr(ctx, "_video_blackboard", board)
817
+ except (AttributeError, ValueError):
818
+ logger.debug(
819
+ "Context does not allow caching video_blackboard; SQLite sharing remains active"
820
+ )
821
+ return board