@moretea-labs/forge 1.5.1 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (434) hide show
  1. package/AGENTS.md +2 -0
  2. package/CHANGELOG.md +46 -0
  3. package/CLAUDE.md +2 -0
  4. package/DESIGN.md +7 -2
  5. package/README.md +4 -4
  6. package/README.zh-CN.md +2 -2
  7. package/SKILL.md +2 -2
  8. package/THIRD_PARTY_NOTICES.md +1 -1
  9. package/agents/openai.yaml +1 -1
  10. package/assets/hooks/hook-input.sh +11 -8
  11. package/assets/hooks/lib/session-state.sh +10 -7
  12. package/assets/hooks/lib/workflow-state.sh +13 -177
  13. package/assets/hooks/post-tool-observer.sh +8 -2
  14. package/assets/hooks/pre-edit-guard.sh +8 -39
  15. package/assets/hooks/prompt-guard.sh +7 -83
  16. package/assets/hooks/session-start-context.sh +56 -1
  17. package/assets/partials/05-workflow.partial.md +1 -1
  18. package/assets/partials-agents/03-orchestration.partial.md +1 -1
  19. package/assets/plugin-registry.v1.json +54 -3
  20. package/assets/reference-configs/document-generation.md +1 -1
  21. package/assets/reference-configs/harness-overview.md +1 -1
  22. package/assets/reference-configs/hook-operations.md +1 -1
  23. package/assets/reference-configs/sprint-contracts.md +3 -3
  24. package/assets/skill-version.json +2 -2
  25. package/assets/templates/contract.template.md +2 -7
  26. package/assets/templates/helpers/capture-plan.sh +2 -2
  27. package/assets/templates/helpers/check-task-workflow.sh +4 -4
  28. package/assets/templates/helpers/contract-worktree.sh +1 -21
  29. package/assets/templates/helpers/ensure-task-workflow.sh +13 -38
  30. package/assets/templates/helpers/migrate-project-template.sh +22 -45
  31. package/assets/templates/helpers/new-plan.sh +1 -1
  32. package/assets/templates/helpers/plan-to-todo.sh +12 -37
  33. package/assets/templates/helpers/ship-worktrees.sh +2 -29
  34. package/assets/templates/helpers/sprint-backlog.sh +6 -6
  35. package/assets/templates/helpers/verify-contract.sh +1 -111
  36. package/assets/templates/helpers/verify-sprint.sh +3 -180
  37. package/assets/templates/plan.template.md +1 -1
  38. package/assets/templates/review.template.md +9 -34
  39. package/assets/workflow-contract.v1.json +10 -19
  40. package/docs/README.md +10 -10
  41. package/docs/architecture/CURRENT.md +170 -0
  42. package/docs/architecture/EVOLUTION.md +42 -0
  43. package/docs/architecture/current/README.md +4 -156
  44. package/docs/architecture/decisions/20260803-release-identity-binding-and-exit-policy.md +2 -2
  45. package/docs/architecture/decisions/20260823-immediate-controller-relay.md +53 -0
  46. package/docs/architecture/decisions/20260826-browser-runtime-v2.md +73 -0
  47. package/docs/architecture/history/README.md +7 -0
  48. package/docs/architecture/{chatgpt-handoff-facade.md → history/chatgpt-handoff-facade.md} +4 -0
  49. package/docs/architecture/{global-hook-runtime.md → history/global-hook-runtime.md} +7 -3
  50. package/docs/architecture/{ios-semantic-automation-provider-v2.md → history/ios-semantic-automation-provider-v2.md} +6 -2
  51. package/docs/architecture/history.md +3 -34
  52. package/docs/architecture/index.md +15 -98
  53. package/docs/architecture/modules/controller-runtime/transport-lifecycle.md +1 -1
  54. package/docs/architecture/reviews/2026-08-22-semantic-harness-convergence-acceptance.md +39 -0
  55. package/docs/architecture/versions/1.6.md +38 -0
  56. package/docs/architecture/versions/1.7.md +41 -0
  57. package/docs/forge-chatgpt-browser-engine.md +2 -2
  58. package/docs/forge-chatgpt-mcp-setup.md +4 -28
  59. package/docs/forge-chatgpt-supervised-automation.md +3 -3
  60. package/docs/forge-github-issue-launcher.md +1 -1
  61. package/docs/forge-runtime-self-healing-loop.md +1 -1
  62. package/docs/operations/controller-browser-plugin.md +38 -22
  63. package/docs/operations/controller-performance-and-502.md +124 -0
  64. package/docs/operations/controller-reliability-runbook.md +1 -1
  65. package/docs/operations/features.md +3 -3
  66. package/docs/operations/github-repository.md +1 -1
  67. package/docs/operations/homebrew.md +2 -2
  68. package/docs/operations/mcp-tool-exposure.md +5 -3
  69. package/docs/tutorials/01-install-and-start.md +1 -1
  70. package/docs/tutorials/01-install-and-start.zh-CN.md +1 -1
  71. package/docs/wiki/Architecture.md +6 -6
  72. package/docs/wiki/Implementation.md +2 -2
  73. package/docs/wiki/Installation.md +1 -1
  74. package/docs/wiki/Integrations.md +2 -2
  75. package/docs/wiki/Quick-Start.md +1 -1
  76. package/docs/wiki/Work-Lifecycle.md +1 -1
  77. package/package.json +29 -10
  78. package/scripts/activate-source-baseline.ts +17 -13
  79. package/scripts/benchmark-background-check-overlap.ts +228 -0
  80. package/scripts/benchmark-mcp-host.sh +117 -0
  81. package/scripts/benchmark-mcp-transport.ts +170 -0
  82. package/scripts/benchmark-quality-harness.ts +146 -0
  83. package/scripts/benchmark-route-session-concurrency.ts +207 -6
  84. package/scripts/benchmark-thin-harness-mcp-e2e.ts +7 -1
  85. package/scripts/benchmark-typescript-navigation.ts +186 -0
  86. package/scripts/benchmark-work-efficiency.ts +334 -0
  87. package/scripts/capture-plan.sh +3 -3
  88. package/scripts/check-agent-tooling.sh +8 -35
  89. package/scripts/check-architecture-sync.sh +17 -86
  90. package/scripts/check-mcp-compatibility.ts +14 -0
  91. package/scripts/check-public-docs.sh +14 -1
  92. package/scripts/check-runtime-architecture.mjs +107 -64
  93. package/scripts/check-stable-baseline.ts +92 -0
  94. package/scripts/check-task-workflow.sh +4 -4
  95. package/scripts/contract-worktree.sh +1 -21
  96. package/scripts/ensure-task-workflow.sh +22 -55
  97. package/scripts/host-temporary-cloud-mcp-e2e.sh +265 -0
  98. package/scripts/install-standalone-recovery.ts +24 -0
  99. package/scripts/lib/project-init-lib.sh +15 -44
  100. package/scripts/migrate-chatgpt-automation-schedules.ts +64 -0
  101. package/scripts/migrate-project-template.sh +23 -1179
  102. package/scripts/new-plan.sh +1 -1
  103. package/scripts/plan-to-todo.sh +11 -36
  104. package/scripts/run-governed-gate.ts +2 -1
  105. package/scripts/ship-worktrees.sh +2 -29
  106. package/scripts/smoke-ios-physical-device.ts +4 -2
  107. package/scripts/smoke-runtime-recovery.ts +1 -35
  108. package/scripts/smoke-schedule-engine.ts +1 -20
  109. package/scripts/sprint-backlog.sh +1 -1
  110. package/scripts/verify-contract.sh +1 -111
  111. package/scripts/verify-forge-runtime.sh +0 -1
  112. package/scripts/verify-sprint.sh +3 -180
  113. package/scripts/workflow-contract.ts +1 -5
  114. package/src/cli/commands/chatgpt.ts +31 -2
  115. package/src/cli/commands/init-hook.ts +3 -3
  116. package/src/cli/commands/mcp.ts +9 -5
  117. package/src/cli/commands/plugin.ts +37 -10
  118. package/src/cli/commands/recovery.ts +25 -3
  119. package/src/cli/commands/runtime.ts +55 -3
  120. package/src/cli/commands/security.ts +12 -1
  121. package/src/cli/commands/setup-profile.ts +31 -21
  122. package/src/cli/controller/check-runner.ts +106 -3
  123. package/src/cli/controller/composite-result.ts +1 -1
  124. package/src/cli/controller/context/focus.ts +51 -0
  125. package/src/cli/controller/context/instruction-context.ts +114 -0
  126. package/src/cli/controller/context/known-paths.ts +76 -0
  127. package/src/cli/controller/context/query-planning.ts +110 -0
  128. package/src/cli/controller/context/rule-resolution.ts +532 -0
  129. package/src/cli/controller/context/source-materializer.ts +278 -0
  130. package/src/cli/controller/context/structural.ts +16 -0
  131. package/src/cli/controller/context/types.ts +286 -0
  132. package/src/cli/controller/context-pack.ts +898 -430
  133. package/src/cli/controller/launch-agents.ts +77 -2
  134. package/src/cli/controller/operational-plan.ts +2 -2
  135. package/src/cli/controller/work-mode.ts +6 -4
  136. package/src/cli/editing/edit-session.ts +60 -7
  137. package/src/cli/github/github.ts +133 -0
  138. package/src/cli/hook/prompt-guard-decision.ts +17 -18
  139. package/src/cli/hook/prompt-intents.ts +1 -1
  140. package/src/cli/installer/managed-entries.ts +15 -1
  141. package/src/cli/local-bridge/console-automations.ts +225 -67
  142. package/src/cli/local-bridge/console-view-models.ts +3 -35
  143. package/src/cli/local-bridge/facade-api.ts +2 -217
  144. package/src/cli/local-bridge/frontend/src/App.tsx +3 -3
  145. package/src/cli/local-bridge/frontend/src/features/automations/AutomationsView.tsx +153 -3
  146. package/src/cli/local-bridge/frontend/src/features/capabilities/CapabilitiesView.tsx +37 -4
  147. package/src/cli/local-bridge/frontend/src/features/overview/OverviewView.tsx +183 -3
  148. package/src/cli/local-bridge/frontend/src/features/repositories/RepositoriesView.tsx +1 -1
  149. package/src/cli/local-bridge/frontend/src/features/work/WorkView.tsx +4 -3
  150. package/src/cli/local-bridge/frontend/src/lib/api.ts +18 -11
  151. package/src/cli/local-bridge/frontend/src/lib/routes.ts +1 -1
  152. package/src/cli/local-bridge/frontend/src/shell/CommandBar.tsx +1 -1
  153. package/src/cli/local-bridge/frontend/src/shell/Sidebar.tsx +4 -3
  154. package/src/cli/local-bridge/frontend/src/styles.css +27 -11
  155. package/src/cli/local-bridge/frontend/src/types.ts +8 -7
  156. package/src/cli/local-bridge/job-store.ts +1 -490
  157. package/src/cli/local-bridge/server.ts +29 -532
  158. package/src/cli/local-bridge/ui-dist/app.css +1 -1
  159. package/src/cli/local-bridge/ui-dist/app.js +5101 -4967
  160. package/src/cli/mcp/auth.ts +5 -5
  161. package/src/cli/mcp/instructions.ts +3 -2
  162. package/src/cli/mcp/legacy-tool-service.ts +0 -189
  163. package/src/cli/mcp/multi-repository.ts +16 -5
  164. package/src/cli/mcp/oauth.ts +175 -26
  165. package/src/cli/mcp/paths.ts +8 -2
  166. package/src/cli/mcp/repository-tools.ts +356 -101
  167. package/src/cli/mcp/server.ts +625 -54
  168. package/src/cli/mcp/tool-capability-cache.ts +1 -1
  169. package/src/cli/mcp/toolset-selection.ts +1 -1
  170. package/src/cli/mcp/toolset.ts +42 -5
  171. package/src/cli/mcp/transports/http.ts +125 -15
  172. package/src/cli/mcp/transports/session-registry.ts +67 -5
  173. package/src/cli/mcp/types.ts +3 -3
  174. package/src/cli/repositories/command-classifier.ts +209 -14
  175. package/src/cli/repositories/command-executor.ts +55 -521
  176. package/src/cli/repositories/command-process.ts +108 -0
  177. package/src/cli/repositories/command-scope.ts +99 -41
  178. package/src/cli/repositories/controller-home.ts +7 -8
  179. package/src/cli/repositories/git-command-executor.ts +2 -0
  180. package/src/cli/repositories/locks.ts +53 -16
  181. package/src/cli/repositories/repository-snapshot.ts +121 -0
  182. package/src/cli/repositories/safe-patch.ts +6 -4
  183. package/src/cli/repositories/selected-path-actions.ts +17 -9
  184. package/src/cli/repositories/structured-git.ts +110 -10
  185. package/src/cli/repository/inspector.ts +565 -62
  186. package/src/cli/repository/session-cache.ts +58 -17
  187. package/src/cli/v81-entry.ts +1 -4
  188. package/src/effects/fs-transaction.ts +85 -17
  189. package/src/runtime/context/codegraph-read-provider.ts +233 -2
  190. package/src/runtime/context/codegraph-sidecar.cjs +74 -40
  191. package/src/runtime/context/generic-lsp-provider.ts +355 -0
  192. package/src/runtime/context/lsp-client.ts +283 -0
  193. package/src/runtime/context/semantic-navigation.ts +275 -0
  194. package/src/runtime/context/swift-navigation.ts +565 -0
  195. package/src/runtime/context/typescript-navigation.ts +226 -0
  196. package/src/runtime/control-plane/dispatch-priority.ts +28 -0
  197. package/src/runtime/control-plane/execution/direct-edit-work-completion.ts +196 -2
  198. package/src/runtime/control-plane/execution/edit-validation-coordinator.ts +86 -24
  199. package/src/runtime/control-plane/execution/execution-identity.ts +65 -14
  200. package/src/runtime/control-plane/execution/session-store.ts +18 -0
  201. package/src/runtime/control-plane/execution/validation.ts +15 -2
  202. package/src/runtime/control-plane/execution/verification-evidence.ts +58 -3
  203. package/src/runtime/control-plane/execution/work-handle-store.ts +27 -2
  204. package/src/runtime/control-plane/execution/work-lifecycle-audit.ts +258 -0
  205. package/src/runtime/control-plane/execution/work-remote-delivery.ts +215 -0
  206. package/src/runtime/control-plane/execution/work-task-receipt.ts +54 -11
  207. package/src/runtime/control-plane/execution/work-terminal-cleanup.ts +175 -8
  208. package/src/runtime/control-plane/execution/work-validation-reconciler.ts +28 -21
  209. package/src/runtime/control-plane/execution/work-verification-snapshot.ts +87 -9
  210. package/src/runtime/control-plane/facade/capability-registry.ts +129 -25
  211. package/src/runtime/control-plane/facade/controller-round-relay.ts +991 -0
  212. package/src/runtime/control-plane/facade/controller-session-store.ts +144 -14
  213. package/src/runtime/control-plane/facade/goal-workloop-access.ts +9 -2
  214. package/src/runtime/control-plane/facade/goal-workloop.ts +885 -105
  215. package/src/runtime/control-plane/facade/index.ts +1 -0
  216. package/src/runtime/control-plane/facade/plan-contract-store.ts +211 -26
  217. package/src/runtime/control-plane/facade/requirement-board.ts +24 -8
  218. package/src/runtime/control-plane/facade/semantic-admission.ts +205 -0
  219. package/src/runtime/control-plane/facade/suggested-actions.ts +5 -1
  220. package/src/runtime/control-plane/facade/types.ts +55 -11
  221. package/src/runtime/control-plane/facade/work-contract-store.ts +204 -73
  222. package/src/runtime/control-plane/generated-cache-retention.ts +271 -0
  223. package/src/runtime/control-plane/global-scheduler/config.ts +44 -0
  224. package/src/runtime/control-plane/global-scheduler/dispatch-capacity.ts +80 -0
  225. package/src/runtime/control-plane/global-scheduler/durable-admission.ts +46 -0
  226. package/src/runtime/control-plane/global-scheduler/maintenance.ts +145 -0
  227. package/src/runtime/control-plane/global-scheduler/projection-refresh.ts +84 -0
  228. package/src/runtime/control-plane/global-scheduler/reconciliation.ts +3 -3
  229. package/src/runtime/control-plane/global-scheduler/scheduler.ts +273 -542
  230. package/src/runtime/control-plane/global-scheduler/source-scan.ts +59 -0
  231. package/src/runtime/control-plane/global-scheduler/state.ts +88 -0
  232. package/src/runtime/control-plane/global-scheduler/worker-attachment.ts +57 -0
  233. package/src/runtime/control-plane/global-scheduler/worker-exit-decision.ts +38 -0
  234. package/src/runtime/control-plane/global-scheduler/worker-exit-reconciler.ts +175 -0
  235. package/src/runtime/control-plane/global-scheduler/worker-launch.ts +111 -0
  236. package/src/runtime/control-plane/global-scheduler/worker-lifecycle-store.ts +64 -0
  237. package/src/runtime/control-plane/global-scheduler/worker-lifecycle.ts +150 -0
  238. package/src/runtime/control-plane/global-scheduler/worker-process.ts +119 -0
  239. package/src/runtime/control-plane/global-scheduler/worker-stderr.ts +60 -0
  240. package/src/runtime/control-plane/launcher/chatgpt-work-binding-store.ts +50 -0
  241. package/src/runtime/control-plane/launcher/chatgpt-work-continuation.ts +642 -51
  242. package/src/runtime/control-plane/launcher/launch-reservation-store.ts +40 -0
  243. package/src/runtime/control-plane/launcher/thin-launcher.ts +194 -10
  244. package/src/runtime/control-plane/persistence/controller-home-migration.ts +487 -0
  245. package/src/runtime/control-plane/persistence/requirement-store.ts +15 -11
  246. package/src/runtime/control-plane/persistence/sqlite-store.ts +33 -2
  247. package/src/runtime/control-plane/release-retention.ts +62 -2
  248. package/src/runtime/control-plane/routing/route-policy.ts +7 -16
  249. package/src/runtime/control-plane/runtime-status-client.ts +0 -2
  250. package/src/runtime/diagnostics/mcp-timing.ts +19 -0
  251. package/src/runtime/diagnostics/mcp-transport-benchmark.ts +129 -0
  252. package/src/runtime/diagnostics/performance.ts +24 -1
  253. package/src/runtime/evidence/event-ledger.ts +1 -1
  254. package/src/runtime/execution/jobs/store.ts +58 -4
  255. package/src/runtime/execution/jobs/types.ts +1 -1
  256. package/src/runtime/execution/managed-workspace.ts +203 -1
  257. package/src/runtime/execution/process-runtime/check-facade.ts +52 -14
  258. package/src/runtime/execution/process-runtime/check-result.ts +63 -2
  259. package/src/runtime/execution/process-runtime/check-scheduling.ts +114 -0
  260. package/src/runtime/execution/process-runtime/command-facade.ts +346 -40
  261. package/src/runtime/execution/process-runtime/gc.ts +98 -12
  262. package/src/runtime/execution/process-runtime/lightweight-managed.ts +1184 -0
  263. package/src/runtime/execution/process-runtime/persisted-check.ts +57 -6
  264. package/src/runtime/execution/process-runtime/resource-claims.ts +97 -8
  265. package/src/runtime/execution/process-runtime/runtime.ts +59 -4
  266. package/src/runtime/execution/process-runtime/store.ts +7 -0
  267. package/src/runtime/execution/process-runtime/types.ts +2 -0
  268. package/src/runtime/execution/thin-harness/async-process.ts +12 -2
  269. package/src/runtime/execution/thin-harness/search-worker.ts +54 -5
  270. package/src/runtime/execution/workers/executor.ts +0 -48
  271. package/src/runtime/gateway/mcp/execution-tools.ts +1219 -126
  272. package/src/runtime/gateway/mcp/process-tools.ts +83 -37
  273. package/src/runtime/gateway/mcp/router.ts +185 -83
  274. package/src/runtime/gateway/mcp/runtime-tool-definitions.ts +546 -0
  275. package/src/runtime/gateway/mcp/runtime-tools.ts +2555 -1616
  276. package/src/runtime/model-clients/deepseek-adapter.ts +0 -32
  277. package/src/runtime/plugins/app-store-connect-adapter.ts +123 -19
  278. package/src/runtime/plugins/browser-adapter.ts +2925 -318
  279. package/src/runtime/plugins/browser-automation-service.ts +10 -0
  280. package/src/runtime/plugins/browser-handoff-host.ts +6 -31
  281. package/src/runtime/plugins/browser-handoff.ts +92 -5
  282. package/src/runtime/plugins/browser-macos-bridge.ts +586 -45
  283. package/src/runtime/plugins/browser-node-bridge.ts +6 -34
  284. package/src/runtime/plugins/browser-provider-registry.ts +109 -0
  285. package/src/runtime/plugins/browser-runtime-contract.ts +152 -0
  286. package/src/runtime/plugins/browser-runtime.ts +183 -0
  287. package/src/runtime/plugins/browser-session-authority.ts +332 -0
  288. package/src/runtime/plugins/capability-authorization-grants.ts +354 -0
  289. package/src/runtime/plugins/desktop-operator-registration.ts +15 -12
  290. package/src/runtime/plugins/external-adapter.ts +324 -3
  291. package/src/runtime/plugins/external-registration.ts +84 -5
  292. package/src/runtime/plugins/external-unix-socket.ts +7 -1
  293. package/src/runtime/plugins/first-party-registry.ts +4 -2
  294. package/src/runtime/plugins/github-adapter.ts +125 -0
  295. package/src/runtime/plugins/gmail-adapter.ts +28 -2
  296. package/src/runtime/plugins/ios/agent-device-provider.ts +1 -1
  297. package/src/runtime/plugins/ios/agent-device-typed-provider.ts +52 -9
  298. package/src/runtime/plugins/ios/remote-xpc-hid.ts +1171 -0
  299. package/src/runtime/plugins/ios-adapter.ts +106 -16
  300. package/src/runtime/plugins/ios-agent-device.ts +234 -64
  301. package/src/runtime/plugins/ios-physical-device.ts +1143 -392
  302. package/src/runtime/plugins/lightweight-action.ts +153 -0
  303. package/src/runtime/plugins/local-system-adapter.ts +125 -8
  304. package/src/runtime/plugins/macos-capability-broker.ts +55 -9
  305. package/src/runtime/plugins/plugin-action-sidecar.ts +59 -0
  306. package/src/runtime/plugins/plugin-management-adapter.ts +181 -0
  307. package/src/runtime/plugins/resend-adapter.ts +1 -1
  308. package/src/runtime/plugins/store.ts +366 -24
  309. package/src/runtime/plugins/types.ts +39 -2
  310. package/src/runtime/plugins/xiaohongshu-publish.ts +515 -0
  311. package/src/runtime/projections/git-status-sampler.ts +16 -16
  312. package/src/runtime/projections/materialized-view.ts +28 -10
  313. package/src/runtime/recovery/actions.ts +0 -10
  314. package/src/runtime/recovery/classifier.ts +0 -9
  315. package/src/runtime/recovery/maintenance-executor.ts +415 -18
  316. package/src/runtime/recovery/probe.ts +0 -1
  317. package/src/runtime/recovery/types.ts +0 -1
  318. package/src/runtime/root/mcp-transport.ts +180 -10
  319. package/src/runtime/root/package-connector-service.ts +186 -8
  320. package/src/runtime/root/package-runtime-release.ts +143 -17
  321. package/src/runtime/root/package-runtime-service.ts +430 -22
  322. package/src/runtime/root/release-manifest.ts +7 -2
  323. package/src/runtime/root/release-materialize.ts +162 -19
  324. package/src/runtime/root/release-store.ts +48 -11
  325. package/src/runtime/root/runtime.ts +23 -5
  326. package/src/runtime/root/service.ts +100 -27
  327. package/src/runtime/root/types.ts +2 -0
  328. package/src/runtime/safe-tooling/index.ts +0 -1
  329. package/src/runtime/safe-tooling/ios-development.ts +321 -63
  330. package/src/runtime/safe-tooling/types.ts +0 -30
  331. package/src/runtime/shared/trusted-executable.ts +62 -0
  332. package/src/runtime/shared/trusted-node-executable.ts +48 -0
  333. package/src/runtime/standalone-recovery/core.ts +702 -94
  334. package/src/runtime/standalone-recovery/entry.ts +99 -23
  335. package/src/runtime/standalone-recovery/installer.ts +19 -3
  336. package/src/runtime/standalone-recovery/service-labels.ts +2 -0
  337. package/src/runtime/standalone-recovery/watchdog-heartbeat.ts +160 -0
  338. package/src/runtime/workflow/schedules/browser-probe.ts +258 -0
  339. package/src/runtime/workflow/schedules/chatgpt-automation-migration.ts +40 -0
  340. package/src/runtime/workflow/schedules/engine.ts +509 -80
  341. package/src/runtime/workflow/schedules/github-issue-watch.ts +158 -0
  342. package/src/runtime/workflow/schedules/settlement.ts +48 -4
  343. package/src/runtime/workflow/schedules/store.ts +131 -10
  344. package/src/runtime/workflow/schedules/types.ts +6 -3
  345. package/src/runtime/workflow/schedules/work-continuation.ts +120 -34
  346. package/src/testing/test-governance.ts +8 -40
  347. package/assets/reference-configs/heartbeat-triage.md +0 -57
  348. package/assets/templates/helpers/harness-trace-grade.sh +0 -150
  349. package/assets/templates/helpers/heartbeat-triage.sh +0 -361
  350. package/assets/templates/helpers/maintenance-triage.sh +0 -167
  351. package/bin/forge-desktop-helper.mjs +0 -204
  352. package/docs/architecture/current/approved-target-architecture.zh-CN.md +0 -1501
  353. package/docs/architecture/current/architecture-invariants.md +0 -331
  354. package/docs/architecture/current/automation-and-schedule-engine.md +0 -453
  355. package/docs/architecture/current/codegraph-context-provider.md +0 -226
  356. package/docs/architecture/current/control-plane-authority-inventory.md +0 -94
  357. package/docs/architecture/current/controller-repository-decoupling.md +0 -148
  358. package/docs/architecture/current/dispatch-and-agent-strategy.md +0 -354
  359. package/docs/architecture/current/entity-model.md +0 -592
  360. package/docs/architecture/current/external-plugin-governance.md +0 -396
  361. package/docs/architecture/current/failure-recovery.md +0 -447
  362. package/docs/architecture/current/governance.md +0 -213
  363. package/docs/architecture/current/human-interaction-plane.md +0 -87
  364. package/docs/architecture/current/implementation-status.md +0 -128
  365. package/docs/architecture/current/job-and-run-lifecycle.md +0 -413
  366. package/docs/architecture/current/legacy-governance.md +0 -58
  367. package/docs/architecture/current/local-workspace-target-grants.md +0 -70
  368. package/docs/architecture/current/migration-roadmap.md +0 -105
  369. package/docs/architecture/current/multi-repository-execution.md +0 -371
  370. package/docs/architecture/current/operations-runbook.md +0 -45
  371. package/docs/architecture/current/peer-supercontroller-runtime.md +0 -32
  372. package/docs/architecture/current/personal-assistant-plugin-baseline.md +0 -206
  373. package/docs/architecture/current/repository-access-modes.md +0 -102
  374. package/docs/architecture/current/route-policy.md +0 -15
  375. package/docs/architecture/current/route-session-concurrency.md +0 -85
  376. package/docs/architecture/current/runtime-architecture-simplification.md +0 -214
  377. package/docs/architecture/current/runtime-directory-map.md +0 -49
  378. package/docs/architecture/current/runtime-health-and-resource-lifecycle.md +0 -85
  379. package/docs/architecture/current/scheduler-and-resource-claims.md +0 -422
  380. package/docs/architecture/current/session-aware-execution-and-authorization.md +0 -77
  381. package/docs/architecture/current/system-overview.md +0 -144
  382. package/docs/architecture/current/target-requirements-traceability.md +0 -202
  383. package/docs/architecture/current/thin-harness-governance-policy.md +0 -32
  384. package/docs/architecture/current/thin-harness-v1.md +0 -311
  385. package/docs/architecture/current/verification-and-release-gates.md +0 -391
  386. package/docs/forge-autonomous-goal-loop.md +0 -9
  387. package/docs/forge-provider-configuration.md +0 -7
  388. package/docs/operations/chatgpt-personal-assistant-runtime.md +0 -83
  389. package/docs/operations/controller-desktop-plugin.md +0 -112
  390. package/docs/operations/gmail-assistant-routines.md +0 -186
  391. package/docs/operations/personal-assistant-operator-guide.md +0 -61
  392. package/docs/operations/provider-configuration.md +0 -31
  393. package/scripts/harness-trace-grade.sh +0 -150
  394. package/scripts/heartbeat-triage.sh +0 -361
  395. package/scripts/maintenance-triage.sh +0 -167
  396. package/src/cli/local-bridge/frontend/src/features/settings/SettingsView.tsx +0 -2
  397. package/src/cli/repositories/goal-registry.ts +0 -239
  398. package/src/runtime/assistant/action-proposals.ts +0 -279
  399. package/src/runtime/assistant/intent.ts +0 -395
  400. package/src/runtime/assistant/model-provider.ts +0 -460
  401. package/src/runtime/assistant/openapi.ts +0 -223
  402. package/src/runtime/assistant/policy.ts +0 -89
  403. package/src/runtime/assistant/readiness.ts +0 -258
  404. package/src/runtime/assistant/routine-runtime.ts +0 -477
  405. package/src/runtime/assistant/schedule-binding.ts +0 -171
  406. package/src/runtime/assistant/standing-grants.ts +0 -375
  407. package/src/runtime/assistant/store.ts +0 -243
  408. package/src/runtime/assistant/types.ts +0 -135
  409. package/src/runtime/control-plane/goal-loop/config-facade.ts +0 -700
  410. package/src/runtime/control-plane/goal-loop/config-store.ts +0 -570
  411. package/src/runtime/control-plane/goal-loop/config-types.ts +0 -213
  412. package/src/runtime/control-plane/goal-loop/executor-router.ts +0 -109
  413. package/src/runtime/control-plane/goal-loop/failure-taxonomy.ts +0 -149
  414. package/src/runtime/control-plane/goal-loop/goal-contract-store.ts +0 -304
  415. package/src/runtime/control-plane/goal-loop/goal-loop-engine.ts +0 -1001
  416. package/src/runtime/control-plane/goal-loop/handoff-packet.ts +0 -160
  417. package/src/runtime/control-plane/goal-loop/index.ts +0 -13
  418. package/src/runtime/control-plane/goal-loop/local-tools.ts +0 -290
  419. package/src/runtime/control-plane/goal-loop/policy-gates.ts +0 -203
  420. package/src/runtime/control-plane/goal-loop/provider-dispatch.ts +0 -93
  421. package/src/runtime/control-plane/goal-loop/provider-registry.ts +0 -545
  422. package/src/runtime/control-plane/goal-loop/types.ts +0 -378
  423. package/src/runtime/control-plane/startup-recovery.ts +0 -198
  424. package/src/runtime/gateway/mcp/resource-policy.ts +0 -186
  425. package/src/runtime/personal-assistant/gmail-triage-manager.ts +0 -231
  426. package/src/runtime/personal-assistant/reporting-runtime.ts +0 -344
  427. package/src/runtime/personal-assistant/triage-runtime.ts +0 -748
  428. package/src/runtime/plugins/desktop-adapter.ts +0 -374
  429. package/src/runtime/safe-tooling/browser-safe.ts +0 -133
  430. package/src/runtime/workflow/findings/store.ts +0 -204
  431. package/src/runtime/workflow/findings/types.ts +0 -58
  432. package/src/runtime/workflow/portfolio/engine.ts +0 -20
  433. package/src/runtime/workflow/portfolio/store.ts +0 -185
  434. package/src/runtime/workflow/portfolio/types.ts +0 -29
@@ -1,8 +1,10 @@
1
- import { randomUUID } from 'crypto';
2
- import { existsSync, mkdirSync } from 'fs';
1
+ import { createHash, randomUUID } from 'crypto';
2
+ import { existsSync, mkdirSync, readFileSync } from 'fs';
3
3
  import { join } from 'path';
4
4
  import { spawnSync } from 'child_process';
5
+ import { performance } from 'perf_hooks';
5
6
  import { repositoryControllerRoot } from '../../cli/repositories/controller-home';
7
+ import { runBoundedProcess } from '../execution/thin-harness/async-process';
6
8
  import { readJsonFile, writeJsonAtomic } from '../shared/json-files';
7
9
  import { AssistantPluginError, toAssistantPluginError } from './errors';
8
10
  import {
@@ -19,15 +21,22 @@ import {
19
21
  import type {
20
22
  AssistantPluginActionDescriptor,
21
23
  AssistantPluginActionExecutionInput,
24
+ AssistantPluginAuthorizationContext,
25
+ AssistantPluginAuthorizationTarget,
22
26
  AssistantPluginCapability,
23
27
  } from './types';
28
+ import { executeRemoteXpcHidInput, prewarmRemoteXpcHid, remoteXpcHidStatus, stopRemoteXpcHidForDevice } from './ios/remote-xpc-hid';
24
29
 
25
30
  const PROVIDER = 'ios-device' as const;
26
31
  const SESSION_EXPIRY_MS = 2 * 60 * 60_000;
32
+ const CORE_DEVICE_READY_CACHE_MS = 5 * 60_000;
33
+ const INPUT_UNLOCK_CACHE_MS = 60_000;
34
+ const INPUT_DISPLAY_CACHE_MS = 5 * 60_000;
35
+ const INPUT_FOREGROUND_OBSERVATION_TTL_MS = 30_000;
27
36
  const MAX_JSON_BYTES = 64 * 1024;
28
37
  const MAX_EVENTS = 200;
29
- const LOCAL_RUNNER_HOSTS = new Set(['127.0.0.1', 'localhost', '::1', '[::1]']);
30
- const SENSITIVE_SEMANTICS = /secure\s*text|securetextfield|password|passcode|verification|one[ -]?time|otp|2fa|验证码|校验码|短信码|生物识别|biometric|face\s?id|touch\s?id|支付|付款|购买|下单|提交订单|结算|checkout|payment|purchase|confirm\s+order|bank|card|cvv|身份证/i;
38
+ const MAX_BATCH_STEPS = 20;
39
+ const MAX_BATCH_WAIT_MS = 5_000;
31
40
 
32
41
  interface CommandResult {
33
42
  ok: boolean;
@@ -41,69 +50,69 @@ interface CommandOptions {
41
50
  cwd?: string;
42
51
  timeoutMs?: number;
43
52
  env?: NodeJS.ProcessEnv;
44
- }
45
-
46
- export interface RunnerHttpResult {
47
- ok: boolean;
48
- status: number;
49
- body?: unknown;
50
- text: string;
53
+ signal?: AbortSignal;
51
54
  }
52
55
 
53
56
  export interface IosPhysicalDeviceRuntimeHooks {
54
57
  platform(): NodeJS.Platform;
55
58
  now(): Date;
56
59
  runCommand(command: string, args: string[], options?: CommandOptions): CommandResult;
57
- requestJson(method: string, url: string, body?: unknown, timeoutMs?: number): Promise<RunnerHttpResult>;
60
+ runCommandAsync(command: string, args: string[], options?: CommandOptions): Promise<CommandResult>;
61
+ }
62
+
63
+ function runCommandSync(command: string, args: string[], options: CommandOptions = {}): CommandResult {
64
+ const result = spawnSync(command, args, {
65
+ cwd: options.cwd,
66
+ env: options.env,
67
+ encoding: 'utf8',
68
+ timeout: options.timeoutMs ?? 30_000,
69
+ maxBuffer: 4 * 1024 * 1024,
70
+ });
71
+ return {
72
+ ok: result.status === 0,
73
+ status: result.status,
74
+ stdout: String(result.stdout ?? ''),
75
+ stderr: String(result.stderr ?? result.error?.message ?? ''),
76
+ command: [command, ...args],
77
+ };
78
+ }
79
+
80
+ async function runCommandAsync(command: string, args: string[], options: CommandOptions = {}): Promise<CommandResult> {
81
+ const result = await runBoundedProcess(command, args, {
82
+ cwd: options.cwd ?? process.cwd(),
83
+ env: options.env,
84
+ timeoutMs: options.timeoutMs ?? 30_000,
85
+ maxOutputBytes: 4 * 1024 * 1024,
86
+ signal: options.signal,
87
+ });
88
+ return {
89
+ ok: result.ok,
90
+ status: result.exitCode,
91
+ stdout: result.stdout,
92
+ stderr: result.stderr,
93
+ command: [command, ...args],
94
+ };
58
95
  }
59
96
 
60
97
  const defaultHooks: IosPhysicalDeviceRuntimeHooks = {
61
98
  platform: () => process.platform,
62
99
  now: () => new Date(),
63
- runCommand: (command, args, options = {}) => {
64
- const result = spawnSync(command, args, {
65
- cwd: options.cwd,
66
- env: options.env,
67
- encoding: 'utf8',
68
- timeout: options.timeoutMs ?? 30_000,
69
- maxBuffer: 4 * 1024 * 1024,
70
- });
71
- return {
72
- ok: result.status === 0,
73
- status: result.status,
74
- stdout: String(result.stdout ?? ''),
75
- stderr: String(result.stderr ?? result.error?.message ?? ''),
76
- command: [command, ...args],
77
- };
78
- },
79
- requestJson: async (method, url, body, timeoutMs = 30_000) => {
80
- const controller = new AbortController();
81
- const timer = setTimeout(() => controller.abort(), timeoutMs);
82
- try {
83
- const response = await fetch(url, {
84
- method,
85
- headers: body === undefined ? undefined : { 'content-type': 'application/json' },
86
- body: body === undefined ? undefined : JSON.stringify(body),
87
- signal: controller.signal,
88
- });
89
- const text = await response.text();
90
- let parsed: unknown;
91
- try { parsed = text ? JSON.parse(text) : undefined; } catch { parsed = undefined; }
92
- return { ok: response.ok, status: response.status, body: parsed, text };
93
- } finally {
94
- clearTimeout(timer);
95
- }
96
- },
100
+ runCommand: runCommandSync,
101
+ runCommandAsync,
97
102
  };
98
103
 
99
104
  let hooks: IosPhysicalDeviceRuntimeHooks = { ...defaultHooks };
100
105
 
101
106
  export function setIosPhysicalDeviceRuntimeHooksForTest(overrides: Partial<IosPhysicalDeviceRuntimeHooks>): void {
102
107
  hooks = { ...defaultHooks, ...overrides };
108
+ if (overrides.runCommand && !overrides.runCommandAsync) {
109
+ hooks.runCommandAsync = async (command, args, options = {}) => overrides.runCommand!(command, args, options);
110
+ }
103
111
  }
104
112
 
105
113
  export function resetIosPhysicalDeviceRuntimeHooksForTest(): void {
106
114
  hooks = { ...defaultHooks };
115
+ coreDeviceReadyCache = undefined;
107
116
  }
108
117
 
109
118
  interface PhysicalDevice {
@@ -124,6 +133,19 @@ interface PhysicalDevice {
124
133
  connected: boolean;
125
134
  }
126
135
 
136
+ interface PhysicalDeviceCapabilitySummary {
137
+ applicationControl: boolean;
138
+ screenshot: boolean;
139
+ displayInfo: boolean;
140
+ lockState: boolean;
141
+ viewDeviceScreen: boolean;
142
+ hidDigitizer: boolean;
143
+ hidKeyboard: boolean;
144
+ hidScroll: boolean;
145
+ hidButton: boolean;
146
+ universalHid: boolean;
147
+ }
148
+
127
149
  interface InstalledApp {
128
150
  name: string;
129
151
  bundleIdentifier: string;
@@ -138,25 +160,85 @@ interface PhysicalEvent {
138
160
  details?: unknown;
139
161
  }
140
162
 
163
+ interface DisplayGeometry {
164
+ width: number;
165
+ height: number;
166
+ pointScale?: number;
167
+ }
168
+
169
+ interface CachedDisplayGeometry extends DisplayGeometry {
170
+ observedAt: string;
171
+ }
172
+
173
+ interface PhysicalScreenshotObservation {
174
+ observationId: string;
175
+ observedAt: string;
176
+ label: string;
177
+ }
178
+
179
+ interface PhysicalForegroundObservation {
180
+ observationId: string;
181
+ screenshotObservedAt: string;
182
+ confirmedAt: string;
183
+ bundleId: string;
184
+ }
185
+
141
186
  interface PhysicalSessionState {
142
187
  schemaVersion: 1;
143
188
  interactionId: string;
144
189
  device: PhysicalDevice;
145
190
  bundleId: string;
146
- runner: {
147
- configured: boolean;
148
- endpoint?: string;
149
- ready: boolean;
150
- sessionId?: string;
151
- error?: string;
152
- };
191
+ display?: CachedDisplayGeometry;
192
+ unlockVerifiedAt?: string;
193
+ lastScreenshot?: PhysicalScreenshotObservation;
194
+ foregroundObservation?: PhysicalForegroundObservation;
153
195
  events: PhysicalEvent[];
154
196
  }
155
197
 
198
+ interface CoreDeviceStatus {
199
+ available: boolean;
200
+ platform: NodeJS.Platform;
201
+ coreDeviceReady: boolean;
202
+ devicectlVersion?: string;
203
+ reason?: string;
204
+ probed?: boolean;
205
+ }
206
+
207
+ type TimingStages = Record<string, { ms: number; cached?: boolean }>;
208
+ let coreDeviceReadyCache: { checkedAtMs: number; status: CoreDeviceStatus } | undefined;
209
+
156
210
  function timestamp(): string {
157
211
  return hooks.now().toISOString();
158
212
  }
159
213
 
214
+ function elapsedMs(startedAt: number): number {
215
+ return Math.round((performance.now() - startedAt) * 100) / 100;
216
+ }
217
+
218
+ function recordTiming(stages: TimingStages, name: string, startedAt: number, cached?: boolean): void {
219
+ stages[name] = { ms: elapsedMs(startedAt), ...(cached === undefined ? {} : { cached }) };
220
+ }
221
+
222
+ function finishTimedResult(
223
+ actionStartedAt: number,
224
+ stages: TimingStages,
225
+ result: Record<string, unknown>,
226
+ ): Record<string, unknown> {
227
+ return {
228
+ ...result,
229
+ timing: {
230
+ totalMs: elapsedMs(actionStartedAt),
231
+ stages,
232
+ },
233
+ };
234
+ }
235
+
236
+ function cacheAgeMs(observedAt: string | undefined): number | undefined {
237
+ if (!observedAt) return undefined;
238
+ const parsed = Date.parse(observedAt);
239
+ return Number.isFinite(parsed) ? Math.max(0, hooks.now().getTime() - parsed) : undefined;
240
+ }
241
+
160
242
  function sanitize(value: string): string {
161
243
  return value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '') || 'ios-device';
162
244
  }
@@ -235,6 +317,11 @@ function optionalString(value: unknown): string | undefined {
235
317
  return typeof value === 'string' && value.trim() ? value.trim() : undefined;
236
318
  }
237
319
 
320
+ function requireFiniteNumber(value: unknown, name: string): number {
321
+ if (typeof value === 'number' && Number.isFinite(value)) return value;
322
+ throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `${name} must be a finite number.`, { retryable: false });
323
+ }
324
+
238
325
  function requireBundleId(value: unknown): string {
239
326
  const bundleId = requireString(value, 'bundle_id');
240
327
  if (!/^[A-Za-z0-9][A-Za-z0-9.-]+$/.test(bundleId) || !bundleId.includes('.') || /^[a-z][a-z0-9+.-]*:\/\//i.test(bundleId)) {
@@ -255,18 +342,19 @@ function commandFailure(result: CommandResult, code: string, fallback: string):
255
342
  });
256
343
  }
257
344
 
258
- function runCoreJson(
345
+ async function runCoreJson(
259
346
  input: AssistantPluginActionExecutionInput,
260
347
  args: string[],
261
348
  code: string,
262
349
  timeoutMs = 30_000,
263
- ): Record<string, unknown> {
350
+ ): Promise<Record<string, unknown>> {
264
351
  if (hooks.platform() !== 'darwin') {
265
352
  throw new AssistantPluginError('PLUGIN_DEPENDENCY_MISSING', 'Physical iOS device support requires macOS and Xcode CoreDevice.', { retryable: false });
266
353
  }
267
- const result = hooks.runCommand('xcrun', ['devicectl', ...args, '--json-output', '-'], {
354
+ const result = await hooks.runCommandAsync('xcrun', ['devicectl', ...args, '--json-output', '-'], {
268
355
  cwd: input.repoRoot,
269
356
  timeoutMs,
357
+ signal: input.signal,
270
358
  });
271
359
  if (!result.ok) return commandFailure(result, code, 'CoreDevice command failed.');
272
360
  let parsed: Record<string, unknown> | undefined;
@@ -289,8 +377,30 @@ function stringField(value: Record<string, unknown>, key: string): string | unde
289
377
  return typeof value[key] === 'string' && String(value[key]).trim() ? String(value[key]).trim() : undefined;
290
378
  }
291
379
 
292
- function physicalDevices(input: AssistantPluginActionExecutionInput): PhysicalDevice[] {
293
- const response = runCoreJson(input, ['list', 'devices'], 'IOS_DEVICE_LIST_FAILED');
380
+ function capabilityNames(value: unknown): string[] {
381
+ return (Array.isArray(value) ? value : [])
382
+ .map((entry) => stringField(objectValue(entry), 'name'))
383
+ .filter((entry): entry is string => Boolean(entry));
384
+ }
385
+
386
+ function capabilitySummary(names: string[]): PhysicalDeviceCapabilitySummary {
387
+ const features = new Set(names);
388
+ return {
389
+ applicationControl: features.has('Application Control') || features.has('Launch Application'),
390
+ screenshot: features.has('Capture Screenshot'),
391
+ displayInfo: features.has('Get Display Information'),
392
+ lockState: features.has('Get Lock State'),
393
+ viewDeviceScreen: features.has('View Device Screen'),
394
+ hidDigitizer: features.has('HID Digitizer'),
395
+ hidKeyboard: features.has('HID Keyboard'),
396
+ hidScroll: features.has('HID Scroll'),
397
+ hidButton: features.has('HID Button'),
398
+ universalHid: features.has('Universal HID Service Pool') || features.has('UniversalHIDService'),
399
+ };
400
+ }
401
+
402
+ async function physicalDevices(input: AssistantPluginActionExecutionInput): Promise<PhysicalDevice[]> {
403
+ const response = await runCoreJson(input, ['list', 'devices'], 'IOS_DEVICE_LIST_FAILED');
294
404
  const result = objectValue(response.result);
295
405
  const entries = Array.isArray(result.devices) ? result.devices : [];
296
406
  return entries
@@ -299,8 +409,7 @@ function physicalDevices(input: AssistantPluginActionExecutionInput): PhysicalDe
299
409
  const hardware = objectValue(entry.hardwareProperties);
300
410
  const device = objectValue(entry.deviceProperties);
301
411
  const connection = objectValue(entry.connectionProperties);
302
- const capabilities = Array.isArray(entry.capabilities) ? entry.capabilities : [];
303
- const capabilityNames = capabilities.map((item) => stringField(objectValue(item), 'name')).filter(Boolean);
412
+ const capabilities = capabilityNames(entry.capabilities);
304
413
  const pairingState = stringField(connection, 'pairingState');
305
414
  const tunnelState = stringField(connection, 'tunnelState');
306
415
  const bootState = stringField(device, 'bootState');
@@ -318,13 +427,13 @@ function physicalDevices(input: AssistantPluginActionExecutionInput): PhysicalDe
318
427
  bootState,
319
428
  developerMode: stringField(device, 'developerModeStatus'),
320
429
  ddiServicesAvailable: device.ddiServicesAvailable === true,
321
- screenshotAvailable: capabilityNames.includes('Capture Screenshot'),
430
+ screenshotAvailable: capabilities.includes('Capture Screenshot'),
322
431
  connected: pairingState === 'paired'
323
432
  && bootState !== 'shutdown'
324
433
  && (tunnelState === 'connected'
325
434
  || device.ddiServicesAvailable === true
326
- || capabilityNames.includes('Application Control')
327
- || capabilityNames.includes('Launch Application')
435
+ || capabilities.includes('Application Control')
436
+ || capabilities.includes('Launch Application')
328
437
  || connection.transportType === 'usb'
329
438
  || connection.transportType === 'wired'),
330
439
  reality: stringField(hardware, 'reality'),
@@ -337,9 +446,9 @@ function physicalDevices(input: AssistantPluginActionExecutionInput): PhysicalDe
337
446
  .map(({ reality: _reality, platform: _platform, ...entry }) => entry);
338
447
  }
339
448
 
340
- function selectDevice(input: AssistantPluginActionExecutionInput, selectorValue: unknown): PhysicalDevice {
449
+ async function selectDevice(input: AssistantPluginActionExecutionInput, selectorValue: unknown): Promise<PhysicalDevice> {
341
450
  const selector = requireString(selectorValue, 'device');
342
- const inventory = physicalDevices(input);
451
+ const inventory = await physicalDevices(input);
343
452
  const matches = inventory.filter((entry) => entry.identifier === selector || entry.udid === selector || entry.name === selector);
344
453
  if (matches.length === 0) {
345
454
  throw new AssistantPluginError('IOS_DEVICE_NOT_FOUND', 'No paired physical iPhone matches the exact device selector.', {
@@ -360,14 +469,130 @@ function selectDevice(input: AssistantPluginActionExecutionInput, selectorValue:
360
469
  return selected;
361
470
  }
362
471
 
363
- function installedApps(
472
+ async function physicalDeviceLockState(input: AssistantPluginActionExecutionInput, device: PhysicalDevice): Promise<Record<string, unknown>> {
473
+ const response = await runCoreJson(input, [
474
+ 'device', 'info', 'lockState', '--device', device.identifier,
475
+ ], 'IOS_DEVICE_LOCK_STATE_FAILED', 30_000);
476
+ return objectValue(response.result);
477
+ }
478
+
479
+ async function requirePhysicalDeviceUnlocked(input: AssistantPluginActionExecutionInput, device: PhysicalDevice): Promise<void> {
480
+ const lockState = await physicalDeviceLockState(input, device);
481
+ if (lockState.passcodeRequired !== true) return;
482
+ throw new AssistantPluginError(
483
+ 'IOS_DEVICE_LOCKED',
484
+ `The selected iPhone ${device.name} is locked. Unlock it before launching an app through CoreDevice.`,
485
+ {
486
+ retryable: true,
487
+ details: {
488
+ device: { identifier: device.identifier, name: device.name },
489
+ lockState: bounded(lockState),
490
+ },
491
+ },
492
+ );
493
+ }
494
+
495
+ async function physicalDisplayGeometry(input: AssistantPluginActionExecutionInput, device: PhysicalDevice): Promise<DisplayGeometry> {
496
+ const displayResponse = await runCoreJson(input, [
497
+ 'device', 'info', 'displays', '--device', device.identifier,
498
+ ], 'IOS_DEVICE_DISPLAY_INFO_FAILED', 30_000);
499
+ const result = objectValue(displayResponse.result);
500
+ const displays = Array.isArray(result.displays) ? result.displays : [];
501
+ const rows = displays.filter((entry): entry is Record<string, unknown> => Boolean(entry) && typeof entry === 'object' && !Array.isArray(entry));
502
+ const primary = rows.find((entry) => entry.primary === true) ?? rows[0];
503
+ if (!primary) {
504
+ throw new AssistantPluginError('IOS_HID_DISPLAY_INVALID', 'CoreDevice did not report a usable iPhone display for input coordinate mapping.', { retryable: true });
505
+ }
506
+ const bounds = Array.isArray(primary.bounds) ? primary.bounds : undefined;
507
+ const size = bounds && Array.isArray(bounds[1]) ? bounds[1] : Array.isArray(primary.nativeSize) ? primary.nativeSize : undefined;
508
+ const width = Number(size?.[0]);
509
+ const height = Number(size?.[1]);
510
+ if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 1 || height <= 1) {
511
+ throw new AssistantPluginError('IOS_HID_DISPLAY_INVALID', 'CoreDevice returned invalid display dimensions for input coordinate mapping.', {
512
+ retryable: true,
513
+ details: { display: bounded(primary) },
514
+ });
515
+ }
516
+ return {
517
+ width: Math.trunc(width),
518
+ height: Math.trunc(height),
519
+ pointScale: typeof primary.pointScale === 'number' && Number.isFinite(primary.pointScale) ? primary.pointScale : undefined,
520
+ };
521
+ }
522
+
523
+ function pngDisplayGeometry(path: string, pointScale?: number): DisplayGeometry | undefined {
524
+ try {
525
+ const header = readFileSync(path).subarray(0, 24);
526
+ const signature = header.subarray(0, 8).toString('hex');
527
+ if (header.length < 24 || signature !== '89504e470d0a1a0a') return undefined;
528
+ const width = header.readUInt32BE(16);
529
+ const height = header.readUInt32BE(20);
530
+ if (width <= 1 || height <= 1) return undefined;
531
+ return { width, height, pointScale };
532
+ } catch {
533
+ return undefined;
534
+ }
535
+ }
536
+
537
+ async function sessionDisplayGeometry(
538
+ input: AssistantPluginActionExecutionInput,
539
+ state: PhysicalSessionState,
540
+ ): Promise<{ display: DisplayGeometry; cached: boolean }> {
541
+ const age = cacheAgeMs(state.display?.observedAt);
542
+ if (state.display && age !== undefined && age <= INPUT_DISPLAY_CACHE_MS) {
543
+ const { observedAt: _observedAt, ...display } = state.display;
544
+ return { display, cached: true };
545
+ }
546
+ const display = await physicalDisplayGeometry(input, state.device);
547
+ state.display = { ...display, observedAt: timestamp() };
548
+ writeState(input, state);
549
+ return { display, cached: false };
550
+ }
551
+
552
+ async function requireSessionUnlocked(
553
+ input: AssistantPluginActionExecutionInput,
554
+ state: PhysicalSessionState,
555
+ ): Promise<{ cached: boolean }> {
556
+ const age = cacheAgeMs(state.unlockVerifiedAt);
557
+ if (age !== undefined && age <= INPUT_UNLOCK_CACHE_MS) return { cached: true };
558
+ await requirePhysicalDeviceUnlocked(input, state.device);
559
+ state.unlockVerifiedAt = timestamp();
560
+ writeState(input, state);
561
+ return { cached: false };
562
+ }
563
+
564
+ async function physicalDeviceInfo(input: AssistantPluginActionExecutionInput, device: PhysicalDevice): Promise<Record<string, unknown>> {
565
+ const detailsResponse = await runCoreJson(input, [
566
+ 'device', 'info', 'details', '--device', device.identifier,
567
+ ], 'IOS_DEVICE_DETAILS_FAILED', 60_000);
568
+ const details = objectValue(detailsResponse.result);
569
+ const deviceProperties = objectValue(details.deviceProperties);
570
+ const names = capabilityNames(details.capabilities);
571
+
572
+ const lockState = await physicalDeviceLockState(input, device);
573
+ const displayResponse = await runCoreJson(input, [
574
+ 'device', 'info', 'displays', '--device', device.identifier,
575
+ ], 'IOS_DEVICE_DISPLAY_INFO_FAILED', 30_000);
576
+
577
+ return {
578
+ device,
579
+ capabilities: capabilitySummary(names),
580
+ capabilityCount: names.length,
581
+ screenViewingURL: stringField(deviceProperties, 'screenViewingURL'),
582
+ lockState: bounded(lockState),
583
+ displays: bounded(displayResponse.result),
584
+ inputBackend: remoteXpcHidStatus(input.controllerHome),
585
+ };
586
+ }
587
+
588
+ async function installedApps(
364
589
  input: AssistantPluginActionExecutionInput,
365
590
  device: PhysicalDevice,
366
591
  bundleId: string,
367
- ): InstalledApp[] {
592
+ ): Promise<InstalledApp[]> {
368
593
  let response: Record<string, unknown>;
369
594
  try {
370
- response = runCoreJson(input, [
595
+ response = await runCoreJson(input, [
371
596
  'device', 'info', 'apps', '--device', device.identifier,
372
597
  '--include-all-apps', '--bundle-id', bundleId,
373
598
  ], 'IOS_DEVICE_APPS_FAILED', 60_000);
@@ -394,110 +619,23 @@ function installedApps(
394
619
  .filter((entry) => entry.bundleIdentifier === bundleId);
395
620
  }
396
621
 
397
- function configuredRunner(): { configured: boolean; endpoint?: string; error?: string } {
398
- const raw = process.env.FORGE_IOS_DEVICE_RUNNER_URL?.trim();
399
- if (!raw) return { configured: false, error: 'Set FORGE_IOS_DEVICE_RUNNER_URL to a trusted localhost WDA-compatible endpoint.' };
400
- try {
401
- const parsed = new URL(raw);
402
- if (parsed.protocol !== 'http:' || !LOCAL_RUNNER_HOSTS.has(parsed.hostname) || parsed.username || parsed.password) {
403
- return { configured: false, error: 'The iOS UI runner URL must be an unauthenticated localhost HTTP endpoint.' };
404
- }
405
- parsed.pathname = parsed.pathname.replace(/\/$/, '');
406
- parsed.search = '';
407
- parsed.hash = '';
408
- return { configured: true, endpoint: parsed.toString().replace(/\/$/, '') };
409
- } catch {
410
- return { configured: false, error: 'FORGE_IOS_DEVICE_RUNNER_URL is not a valid URL.' };
411
- }
412
- }
413
-
414
- function runnerUrl(endpoint: string, path: string): string {
415
- return `${endpoint}${path.startsWith('/') ? path : `/${path}`}`;
416
- }
417
-
418
- function responseValue(body: unknown): unknown {
419
- const record = objectValue(body);
420
- return 'value' in record ? record.value : body;
421
- }
422
-
423
- async function runnerRequest(
424
- method: string,
425
- endpoint: string,
426
- path: string,
427
- body: unknown,
428
- code: string,
429
- sensitive = false,
430
- ): Promise<unknown> {
431
- let response: RunnerHttpResult;
432
- try {
433
- response = await hooks.requestJson(method, runnerUrl(endpoint, path), body, 30_000);
434
- } catch (error) {
435
- const normalized = toAssistantPluginError(error, {
436
- code,
437
- message: 'The local iOS UI runner could not be reached.',
438
- retryable: true,
439
- });
440
- throw new AssistantPluginError(normalized.code, normalized.message, {
441
- retryable: normalized.retryable,
442
- details: sensitive ? { endpoint, response: '<redacted>' } : { endpoint },
443
- });
444
- }
445
- if (!response.ok) {
446
- throw new AssistantPluginError(code, sensitive ? 'The local iOS UI runner rejected a sensitive-input action.' : `The local iOS UI runner returned HTTP ${response.status}.`, {
447
- retryable: response.status >= 500,
448
- details: sensitive ? { status: response.status, response: '<redacted>' } : { status: response.status, response: bounded(response.body ?? response.text) },
622
+ function expirePhysicalDeviceSessions(input: AssistantPluginActionExecutionInput): number {
623
+ const nowMs = hooks.now().getTime();
624
+ let expired = 0;
625
+ for (const record of listInteractionSessions(input.repoRoot, PROVIDER)) {
626
+ if (!isInteractionSessionActive(record.status)) continue;
627
+ const expiresAtMs = Date.parse(record.expiresAt);
628
+ if (!Number.isFinite(expiresAtMs) || nowMs < expiresAtMs) continue;
629
+ const state = readState(input, record.interactionId);
630
+ patchInteractionSession(input.repoRoot, PROVIDER, record.interactionId, {
631
+ status: 'failed',
632
+ error: { code: 'IOS_DEVICE_SESSION_EXPIRED', message: 'The physical iOS interaction expired.' },
449
633
  });
634
+ appendEvent(input, record.interactionId, 'session_expired');
635
+ if (state?.device.identifier) stopRemoteXpcHidForDevice(state.device.identifier);
636
+ expired += 1;
450
637
  }
451
- return responseValue(response.body);
452
- }
453
-
454
- async function probeRunner(): Promise<{ configured: boolean; endpoint?: string; ready: boolean; error?: string; status?: unknown }> {
455
- const configured = configuredRunner();
456
- if (!configured.configured || !configured.endpoint) return { ...configured, ready: false };
457
- try {
458
- const status = await runnerRequest('GET', configured.endpoint, '/status', undefined, 'IOS_DEVICE_UI_RUNNER_UNAVAILABLE');
459
- const record = objectValue(status);
460
- return {
461
- configured: true,
462
- endpoint: configured.endpoint,
463
- ready: record.ready !== false,
464
- status: bounded(status),
465
- };
466
- } catch (error) {
467
- return {
468
- configured: true,
469
- endpoint: configured.endpoint,
470
- ready: false,
471
- error: toAssistantPluginError(error, {
472
- code: 'IOS_DEVICE_UI_RUNNER_UNAVAILABLE',
473
- message: 'The local iOS UI runner is unavailable.',
474
- retryable: true,
475
- }).message,
476
- };
477
- }
478
- }
479
-
480
- async function createRunnerSession(endpoint: string, bundleId: string): Promise<string> {
481
- const value = await runnerRequest('POST', endpoint, '/session', {
482
- capabilities: { alwaysMatch: { bundleId } },
483
- desiredCapabilities: { bundleId },
484
- }, 'IOS_DEVICE_UI_SESSION_FAILED');
485
- const record = objectValue(value);
486
- const sessionId = stringField(record, 'sessionId') ?? stringField(objectValue(objectValue(value).capabilities), 'sessionId');
487
- if (!sessionId) {
488
- throw new AssistantPluginError('IOS_DEVICE_UI_SESSION_FAILED', 'The UI runner did not return a WebDriver session id.', { retryable: false });
489
- }
490
- return sessionId;
491
- }
492
-
493
- async function closeRunner(state: PhysicalSessionState): Promise<boolean> {
494
- if (!state.runner.endpoint || !state.runner.sessionId) return true;
495
- try {
496
- await runnerRequest('DELETE', state.runner.endpoint, `/session/${encodeURIComponent(state.runner.sessionId)}`, undefined, 'IOS_DEVICE_UI_CLOSE_FAILED');
497
- return true;
498
- } catch {
499
- return false;
500
- }
638
+ return expired;
501
639
  }
502
640
 
503
641
  function requireRecord(input: AssistantPluginActionExecutionInput, allowTerminal = false): { record: InteractionSessionRecord; state: PhysicalSessionState } {
@@ -511,10 +649,7 @@ function requireRecord(input: AssistantPluginActionExecutionInput, allowTerminal
511
649
  throw new AssistantPluginError(
512
650
  'PLUGIN_ACTION_ARGUMENT_INVALID',
513
651
  `Physical iOS interaction ${interactionId} belongs to ${engine ?? 'an unknown engine'}, not the CoreDevice engine.`,
514
- {
515
- retryable: false,
516
- details: { interactionId, expectedEngine: 'coredevice', actualEngine: engine },
517
- },
652
+ { retryable: false, details: { interactionId, expectedEngine: 'coredevice', actualEngine: engine } },
518
653
  );
519
654
  }
520
655
  const state = readState(input, interactionId);
@@ -523,8 +658,7 @@ function requireRecord(input: AssistantPluginActionExecutionInput, allowTerminal
523
658
  }
524
659
  if (!isInteractionSessionActive(record.status) && !allowTerminal) {
525
660
  throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `Physical iOS interaction is ${record.status}.`, {
526
- retryable: false,
527
- details: { interactionId, status: record.status },
661
+ retryable: false, details: { interactionId, status: record.status },
528
662
  });
529
663
  }
530
664
  if (isInteractionSessionActive(record.status) && hooks.now().getTime() >= Date.parse(record.expiresAt)) {
@@ -532,103 +666,343 @@ function requireRecord(input: AssistantPluginActionExecutionInput, allowTerminal
532
666
  status: 'failed',
533
667
  error: { code: 'IOS_DEVICE_SESSION_EXPIRED', message: 'The physical iOS interaction expired.' },
534
668
  });
669
+ appendEvent(input, interactionId, 'session_expired');
670
+ stopRemoteXpcHidForDevice(state.device.identifier);
535
671
  throw new AssistantPluginError('IOS_DEVICE_SESSION_EXPIRED', 'The physical iOS interaction expired; open a new session.', { retryable: false });
536
672
  }
537
673
  return { record, state };
538
674
  }
539
675
 
540
- function requireRunner(state: PhysicalSessionState): { endpoint: string; sessionId: string } {
541
- if (!state.runner.ready || !state.runner.endpoint || !state.runner.sessionId) {
542
- throw new AssistantPluginError('IOS_DEVICE_UI_RUNNER_UNAVAILABLE', state.runner.error ?? 'No signed WDA-compatible UI runner is attached to this physical-device session.', {
543
- retryable: true,
544
- details: {
545
- configured: state.runner.configured,
546
- prerequisites: [
547
- 'Enable Developer Mode and trust/pair the iPhone with Xcode.',
548
- 'Build and run a correctly signed WebDriverAgent-compatible runner for this device.',
549
- 'Forward its HTTP endpoint to localhost and set FORGE_IOS_DEVICE_RUNNER_URL.',
550
- ],
551
- },
552
- });
676
+ function physicalDeviceStatusFromResult(result: CommandResult, platform: NodeJS.Platform): CoreDeviceStatus {
677
+ return {
678
+ available: result.ok,
679
+ platform,
680
+ coreDeviceReady: result.ok,
681
+ devicectlVersion: result.ok ? result.stdout.trim() : undefined,
682
+ reason: result.ok ? undefined : (result.stderr || result.stdout || 'xcrun devicectl is unavailable.'),
683
+ probed: true,
684
+ };
685
+ }
686
+
687
+ export function iosPhysicalDeviceStatus(): CoreDeviceStatus {
688
+ const platform = hooks.platform();
689
+ if (platform !== 'darwin') {
690
+ coreDeviceReadyCache = undefined;
691
+ return {
692
+ available: false,
693
+ platform,
694
+ coreDeviceReady: false,
695
+ reason: 'Physical iOS device support requires macOS and Xcode.',
696
+ probed: true,
697
+ };
698
+ }
699
+ const cached = coreDeviceReadyCache;
700
+ if (cached && hooks.now().getTime() - cached.checkedAtMs <= CORE_DEVICE_READY_CACHE_MS) return cached.status;
701
+ return {
702
+ available: false,
703
+ platform,
704
+ coreDeviceReady: false,
705
+ reason: 'CoreDevice readiness has not been probed by an explicit asynchronous action in this runtime yet.',
706
+ probed: false,
707
+ };
708
+ }
709
+
710
+ async function iosPhysicalDeviceActionStatus(input: AssistantPluginActionExecutionInput): Promise<CoreDeviceStatus> {
711
+ const platform = hooks.platform();
712
+ if (platform !== 'darwin') return iosPhysicalDeviceStatus();
713
+ const result = await hooks.runCommandAsync('xcrun', ['devicectl', '--version'], {
714
+ cwd: input.repoRoot,
715
+ timeoutMs: Math.min(5_000, input.timeoutMs ?? 5_000),
716
+ signal: input.signal,
717
+ });
718
+ const status = physicalDeviceStatusFromResult(result, platform);
719
+ coreDeviceReadyCache = status.coreDeviceReady ? { checkedAtMs: hooks.now().getTime(), status } : undefined;
720
+ return status;
721
+ }
722
+
723
+ async function cachedIosPhysicalDeviceActionStatus(input: AssistantPluginActionExecutionInput): Promise<{ status: CoreDeviceStatus; cached: boolean }> {
724
+ const cached = coreDeviceReadyCache;
725
+ if (cached && hooks.now().getTime() - cached.checkedAtMs <= CORE_DEVICE_READY_CACHE_MS) {
726
+ return { status: cached.status, cached: true };
553
727
  }
554
- return { endpoint: state.runner.endpoint, sessionId: state.runner.sessionId };
728
+ return { status: await iosPhysicalDeviceActionStatus(input), cached: false };
555
729
  }
556
730
 
557
- function rejectSensitiveSemantic(value: string, action: string): void {
558
- if (SENSITIVE_SEMANTICS.test(value)) {
559
- throw new AssistantPluginError('IOS_DEVICE_HUMAN_ACTION_REQUIRED', `${action} targets a credential, verification, biometric, purchase, checkout, or payment flow and must be completed manually.`, {
560
- retryable: false,
561
- details: { humanHandoffRequired: true },
562
- });
731
+ function recentScreenshotObservation(
732
+ state: PhysicalSessionState,
733
+ observationId: string,
734
+ ): { observation: PhysicalScreenshotObservation; ageMs: number } {
735
+ const observation = state.lastScreenshot;
736
+ const ageMs = cacheAgeMs(observation?.observedAt);
737
+ if (!observation || observation.observationId !== observationId || ageMs === undefined || ageMs > INPUT_FOREGROUND_OBSERVATION_TTL_MS) {
738
+ throw new AssistantPluginError(
739
+ 'IOS_DEVICE_SCREEN_OBSERVATION_REQUIRED',
740
+ 'A fresh matching CoreDevice screenshot observation is required before screen-relative physical input.',
741
+ {
742
+ retryable: true,
743
+ details: {
744
+ deviceIdentifier: state.device.identifier,
745
+ expectedObservationId: observation?.observationId,
746
+ providedObservationId: observationId,
747
+ observationAgeMs: ageMs,
748
+ observationTtlMs: INPUT_FOREGROUND_OBSERVATION_TTL_MS,
749
+ hidMutationDispatched: false,
750
+ },
751
+ },
752
+ );
563
753
  }
754
+ return { observation, ageMs };
564
755
  }
565
756
 
566
- function locator(target: string): { using: string; value: string } {
567
- if (target.startsWith('xpath:')) return { using: 'xpath', value: target.slice(6) };
568
- if (target.startsWith('id:')) return { using: 'accessibility id', value: target.slice(3) };
569
- return { using: 'accessibility id', value: target };
570
- }
571
-
572
- async function elementId(endpoint: string, sessionId: string, target: string): Promise<string> {
573
- const value = await runnerRequest('POST', endpoint, `/session/${encodeURIComponent(sessionId)}/element`, locator(target), 'IOS_DEVICE_ELEMENT_NOT_FOUND');
574
- const record = objectValue(value);
575
- const id = stringField(record, 'element-6066-11e4-a52e-4f735466cecf') ?? stringField(record, 'ELEMENT');
576
- if (!id) throw new AssistantPluginError('IOS_DEVICE_ELEMENT_NOT_FOUND', 'The UI runner did not return an element id.', { retryable: false });
577
- return id;
578
- }
579
-
580
- async function inspectElementSemantics(
581
- endpoint: string,
582
- sessionId: string,
583
- elementIdValue: string,
584
- target: string,
585
- action: string,
586
- ): Promise<void> {
587
- const attributes = ['type', 'name', 'label', 'placeholderValue'];
588
- const values: string[] = [target];
589
- for (const attribute of attributes) {
590
- const value = await runnerRequest(
591
- 'GET',
592
- endpoint,
593
- `/session/${encodeURIComponent(sessionId)}/element/${encodeURIComponent(elementIdValue)}/attribute/${encodeURIComponent(attribute)}`,
594
- undefined,
595
- 'IOS_DEVICE_ELEMENT_INSPECTION_FAILED',
757
+ function requireObservedPhysicalTargetForeground(state: PhysicalSessionState): PhysicalForegroundObservation {
758
+ const confirmation = state.foregroundObservation;
759
+ const screenshot = state.lastScreenshot;
760
+ const screenshotAgeMs = cacheAgeMs(screenshot?.observedAt);
761
+ const confirmationAgeMs = cacheAgeMs(confirmation?.confirmedAt);
762
+ const valid = Boolean(
763
+ confirmation
764
+ && screenshot
765
+ && confirmation.bundleId === state.bundleId
766
+ && confirmation.observationId === screenshot.observationId
767
+ && confirmation.screenshotObservedAt === screenshot.observedAt
768
+ && screenshotAgeMs !== undefined
769
+ && screenshotAgeMs <= INPUT_FOREGROUND_OBSERVATION_TTL_MS
770
+ && confirmationAgeMs !== undefined
771
+ && confirmationAgeMs <= INPUT_FOREGROUND_OBSERVATION_TTL_MS
772
+ );
773
+ if (!valid || !confirmation) {
774
+ throw new AssistantPluginError(
775
+ 'IOS_DEVICE_FOREGROUND_OBSERVATION_REQUIRED',
776
+ `A recent screenshot explicitly confirmed by the Controller as ${state.bundleId} foreground is required before HID input.`,
777
+ {
778
+ retryable: true,
779
+ details: {
780
+ bundleId: state.bundleId,
781
+ deviceIdentifier: state.device.identifier,
782
+ screenshotObservationId: screenshot?.observationId,
783
+ screenshotAgeMs,
784
+ confirmationObservationId: confirmation?.observationId,
785
+ confirmationAgeMs,
786
+ observationTtlMs: INPUT_FOREGROUND_OBSERVATION_TTL_MS,
787
+ hidMutationDispatched: false,
788
+ },
789
+ },
596
790
  );
597
- if (typeof value === 'string') values.push(value);
598
791
  }
599
- rejectSensitiveSemantic(values.join(' '), action);
792
+ return confirmation;
600
793
  }
601
794
 
602
- export function iosPhysicalDeviceStatus() {
603
- if (hooks.platform() !== 'darwin') {
795
+ function resolvePhysicalBatchForegroundObservation(
796
+ state: PhysicalSessionState,
797
+ observationId: string | undefined,
798
+ ): { foregroundObservation: PhysicalForegroundObservation; source: 'atomic_screenshot' | 'explicit_confirmation'; screenshotAgeMs?: number } {
799
+ if (observationId) {
800
+ const { observation, ageMs } = recentScreenshotObservation(state, observationId);
604
801
  return {
605
- available: false,
606
- platform: hooks.platform(),
607
- coreDeviceReady: false,
608
- uiRunner: { ...configuredRunner(), ready: false },
609
- reason: 'Physical iOS device support requires macOS and Xcode.',
802
+ foregroundObservation: {
803
+ observationId: observation.observationId,
804
+ screenshotObservedAt: observation.observedAt,
805
+ confirmedAt: timestamp(),
806
+ bundleId: state.bundleId,
807
+ },
808
+ source: 'atomic_screenshot',
809
+ screenshotAgeMs: ageMs,
610
810
  };
611
811
  }
612
- const result = hooks.runCommand('xcrun', ['devicectl', '--version'], { timeoutMs: 5_000 });
613
812
  return {
614
- available: result.ok,
615
- platform: hooks.platform(),
616
- coreDeviceReady: result.ok,
617
- devicectlVersion: result.ok ? result.stdout.trim() : undefined,
618
- uiRunner: { ...configuredRunner(), ready: false, readinessCheckedOnAction: true },
619
- reason: result.ok ? undefined : (result.stderr || result.stdout || 'xcrun devicectl is unavailable.'),
813
+ foregroundObservation: requireObservedPhysicalTargetForeground(state),
814
+ source: 'explicit_confirmation',
620
815
  };
621
816
  }
622
817
 
818
+ function consumePhysicalScreenObservation(input: AssistantPluginActionExecutionInput, state: PhysicalSessionState): void {
819
+ state.lastScreenshot = undefined;
820
+ state.foregroundObservation = undefined;
821
+ writeState(input, state);
822
+ }
823
+
824
+ type PhysicalInputBatchKind = 'tap' | 'swipe' | 'type' | 'wait';
825
+
826
+ interface PhysicalInputBatchStep {
827
+ kind: PhysicalInputBatchKind;
828
+ x?: number;
829
+ y?: number;
830
+ toX?: number;
831
+ toY?: number;
832
+ durationMs?: number;
833
+ text?: string;
834
+ textMode?: 'auto' | 'keys' | 'pasteboard';
835
+ replaceExisting?: boolean;
836
+ }
837
+
838
+ function physicalInputBatchSteps(value: unknown): PhysicalInputBatchStep[] {
839
+ if (!Array.isArray(value) || value.length === 0 || value.length > MAX_BATCH_STEPS) {
840
+ throw new AssistantPluginError(
841
+ 'PLUGIN_ACTION_ARGUMENT_INVALID',
842
+ `steps must contain between 1 and ${MAX_BATCH_STEPS} physical input steps.`,
843
+ { retryable: false },
844
+ );
845
+ }
846
+ return value.map((entry, index) => {
847
+ if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
848
+ throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `steps[${index}] must be an object.`, { retryable: false });
849
+ }
850
+ const row = entry as Record<string, unknown>;
851
+ const kind = requireString(row.kind, `steps[${index}].kind`) as PhysicalInputBatchKind;
852
+ if (!['tap', 'swipe', 'type', 'wait'].includes(kind)) {
853
+ throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `Unsupported steps[${index}].kind: ${kind}.`, { retryable: false });
854
+ }
855
+ if (kind === 'tap') {
856
+ return {
857
+ kind,
858
+ x: requireFiniteNumber(row.x, `steps[${index}].x`),
859
+ y: requireFiniteNumber(row.y, `steps[${index}].y`),
860
+ };
861
+ }
862
+ if (kind === 'swipe') {
863
+ const durationMs = typeof row.duration_ms === 'number' && Number.isFinite(row.duration_ms)
864
+ ? Math.max(80, Math.min(3_000, Math.trunc(row.duration_ms)))
865
+ : 250;
866
+ return {
867
+ kind,
868
+ x: requireFiniteNumber(row.x, `steps[${index}].x`),
869
+ y: requireFiniteNumber(row.y, `steps[${index}].y`),
870
+ toX: requireFiniteNumber(row.to_x, `steps[${index}].to_x`),
871
+ toY: requireFiniteNumber(row.to_y, `steps[${index}].to_y`),
872
+ durationMs,
873
+ };
874
+ }
875
+ if (kind === 'type') {
876
+ const textMode = optionalString(row.input_mode) ?? 'auto';
877
+ if (!['auto', 'keys', 'pasteboard'].includes(textMode)) {
878
+ throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `Unsupported steps[${index}].input_mode: ${textMode}.`, { retryable: false });
879
+ }
880
+ const text = requireString(row.text, `steps[${index}].text`);
881
+ if (text.length > 2048) {
882
+ throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `steps[${index}].text exceeds the 2048 character input limit.`, { retryable: false });
883
+ }
884
+ if (textMode === 'keys' && [...text].some((character) => character.charCodeAt(0) > 0x7f)) {
885
+ throw new AssistantPluginError('IOS_HID_UNICODE_TEXT_UNSUPPORTED', `steps[${index}] requests keys mode for Unicode text; use auto or pasteboard mode.`, { retryable: false });
886
+ }
887
+ return {
888
+ kind,
889
+ text,
890
+ textMode: textMode as 'auto' | 'keys' | 'pasteboard',
891
+ replaceExisting: row.replace_existing === true,
892
+ };
893
+ }
894
+ const durationMs = typeof row.duration_ms === 'number' && Number.isFinite(row.duration_ms)
895
+ ? Math.max(0, Math.min(MAX_BATCH_WAIT_MS, Math.trunc(row.duration_ms)))
896
+ : 120;
897
+ return { kind, durationMs };
898
+ });
899
+ }
900
+
901
+ function validatePhysicalInputBatchCoordinates(steps: PhysicalInputBatchStep[], display: DisplayGeometry | undefined): void {
902
+ const touchSteps = steps.filter((step) => step.kind === 'tap' || step.kind === 'swipe');
903
+ if (touchSteps.length === 0) return;
904
+ if (!display) {
905
+ throw new AssistantPluginError('IOS_HID_DISPLAY_INVALID', 'Physical input batch contains touch steps without usable display geometry.', { retryable: true });
906
+ }
907
+ const points: Array<{ stepIndex: number; label: string; x: number | undefined; y: number | undefined }> = [];
908
+ steps.forEach((step, stepIndex) => {
909
+ if (step.kind === 'tap' || step.kind === 'swipe') points.push({ stepIndex, label: 'from', x: step.x, y: step.y });
910
+ if (step.kind === 'swipe') points.push({ stepIndex, label: 'to', x: step.toX, y: step.toY });
911
+ });
912
+ for (const point of points) {
913
+ if (point.x === undefined || point.y === undefined || point.x < 0 || point.y < 0 || point.x > display.width - 1 || point.y > display.height - 1) {
914
+ throw new AssistantPluginError(
915
+ 'IOS_HID_COORDINATE_OUT_OF_BOUNDS',
916
+ `steps[${point.stepIndex}] ${point.label} coordinate is outside the current CoreDevice display.`,
917
+ { retryable: false, details: { stepIndex: point.stepIndex, label: point.label, x: point.x, y: point.y, width: display.width, height: display.height, mutationDispatched: false } },
918
+ );
919
+ }
920
+ }
921
+ }
922
+
923
+ async function waitPhysicalInputBatch(durationMs: number, signal?: AbortSignal): Promise<void> {
924
+ if (durationMs <= 0) return;
925
+ await new Promise<void>((resolve, reject) => {
926
+ let settled = false;
927
+ let timer: NodeJS.Timeout | undefined;
928
+ const finish = (error?: Error) => {
929
+ if (settled) return;
930
+ settled = true;
931
+ if (timer) clearTimeout(timer);
932
+ signal?.removeEventListener('abort', abort);
933
+ if (error) reject(error);
934
+ else resolve();
935
+ };
936
+ const abort = () => finish(new AssistantPluginError('PLUGIN_ACTION_CANCELLED', 'Physical iOS input batch was cancelled during a wait step.', { retryable: true }));
937
+ timer = setTimeout(() => finish(), durationMs);
938
+ if (signal?.aborted) abort();
939
+ else signal?.addEventListener('abort', abort, { once: true });
940
+ });
941
+ }
942
+
623
943
  export function isIosPhysicalDeviceAction(actionId: string): boolean {
624
944
  return actionId.startsWith('physical_device_');
625
945
  }
626
946
 
947
+ function physicalDeviceAuthorizationTarget(device: PhysicalDevice): AssistantPluginAuthorizationTarget {
948
+ const identityFingerprint = createHash('sha256')
949
+ .update(`${device.identifier}\0${device.udid ?? ''}`)
950
+ .digest('hex');
951
+ return {
952
+ kind: 'ios-physical-device',
953
+ id: device.identifier,
954
+ identityFingerprint,
955
+ };
956
+ }
957
+
958
+ /**
959
+ * Resolve authorization against the exact CoreDevice identity. Interaction ids are
960
+ * intentionally never grant identities: follow-up input actions must recover and
961
+ * verify the immutable device target stored when the session was opened.
962
+ */
963
+ export async function resolveIosPhysicalDeviceAuthorizationContext(
964
+ input: AssistantPluginActionExecutionInput,
965
+ ): Promise<AssistantPluginAuthorizationContext | undefined> {
966
+ const action = iosPhysicalDeviceActions().find((entry) => entry.actionId === input.actionId);
967
+ if (!action || action.confirmation !== 'authorization') return undefined;
968
+
969
+ if (input.actionId === 'physical_device_open') {
970
+ const selected = await selectDevice(input, input.args.device);
971
+ return {
972
+ target: physicalDeviceAuthorizationTarget(selected),
973
+ expiresInMinutes: 30 * 24 * 60,
974
+ };
975
+ }
976
+
977
+ const { record, state } = requireRecord(input, input.actionId === 'physical_device_close');
978
+ const aliases = new Set([record.targetId, ...(record.targetAliases ?? [])]);
979
+ if (record.targetId !== state.device.identifier
980
+ || !aliases.has(state.device.identifier)
981
+ || (state.device.udid && !aliases.has(state.device.udid))) {
982
+ throw new AssistantPluginError(
983
+ 'IOS_DEVICE_AUTHORIZATION_TARGET_MISMATCH',
984
+ 'The persisted physical iOS session no longer matches its exact device authorization identity.',
985
+ {
986
+ retryable: false,
987
+ details: {
988
+ interactionId: record.interactionId,
989
+ sessionTargetId: record.targetId,
990
+ stateDeviceIdentifier: state.device.identifier,
991
+ },
992
+ },
993
+ );
994
+ }
995
+ return {
996
+ target: physicalDeviceAuthorizationTarget(state.device),
997
+ expiresInMinutes: 30 * 24 * 60,
998
+ };
999
+ }
1000
+
627
1001
  export function iosPhysicalDeviceCapabilities(): AssistantPluginCapability[] {
628
1002
  return [{
629
1003
  capabilityId: 'ios-physical-device',
630
1004
  title: 'Physical iOS Computer Use',
631
- description: 'Bounded CoreDevice discovery, installed-app launch and screenshot, with optional signed localhost WDA-compatible UI automation.',
1005
+ description: 'CoreDevice lifecycle/observation plus runnerless RemoteXPC HID input for physical iPhones. XCTest remains only an explicit semantic fallback.',
632
1006
  scopes: ['ios.discover', 'ios.device'],
633
1007
  actions: iosPhysicalDeviceActions().map((action) => action.actionId),
634
1008
  }];
@@ -641,10 +1015,10 @@ export function iosPhysicalDeviceActions(): AssistantPluginActionDescriptor[] {
641
1015
  { resource: 'repo-state' as const, mode: 'write' as const },
642
1016
  ];
643
1017
  const interactionProperty = { interaction_id: { type: 'string' } };
644
- return [
1018
+ const actions: AssistantPluginActionDescriptor[] = [
645
1019
  {
646
1020
  actionId: 'physical_device_status', title: 'Physical iOS device status',
647
- description: 'Report CoreDevice readiness and probe an explicitly configured localhost UI runner.',
1021
+ description: 'Report CoreDevice readiness. This provider has no XCTest/WDA Runner lifecycle.',
648
1022
  readOnly: true, risk: 'readonly', confirmation: 'none', defaultTimeoutMs: 30_000, cancellable: true, idempotent: true,
649
1023
  scopes: ['ios.discover'], resourceClaims: [], argumentsSchema: { type: 'object', properties: {}, additionalProperties: false },
650
1024
  },
@@ -654,6 +1028,13 @@ export function iosPhysicalDeviceActions(): AssistantPluginActionDescriptor[] {
654
1028
  readOnly: true, risk: 'readonly', confirmation: 'none', defaultTimeoutMs: 30_000, cancellable: true, idempotent: true,
655
1029
  scopes: ['ios.discover'], resourceClaims: [], argumentsSchema: { type: 'object', properties: {}, additionalProperties: false },
656
1030
  },
1031
+ {
1032
+ actionId: 'physical_device_info', title: 'Inspect physical iPhone capabilities',
1033
+ description: 'Read CoreDevice details, lock state, display metadata, View Device Screen URL, and HID capability availability without starting a Runner.',
1034
+ readOnly: true, risk: 'readonly', confirmation: 'none', defaultTimeoutMs: 90_000, cancellable: true, idempotent: true,
1035
+ scopes: ['ios.device'], resourceClaims: [],
1036
+ argumentsSchema: { type: 'object', properties: { device: { type: 'string' } }, required: ['device'], additionalProperties: false },
1037
+ },
657
1038
  {
658
1039
  actionId: 'physical_device_apps', title: 'Find installed physical-device app',
659
1040
  description: 'Verify one exact bundle identifier is installed on an exact paired iPhone without reading its data container.',
@@ -663,45 +1044,107 @@ export function iosPhysicalDeviceActions(): AssistantPluginActionDescriptor[] {
663
1044
  },
664
1045
  {
665
1046
  actionId: 'physical_device_open', title: 'Open physical iOS app session',
666
- description: 'Launch one installed third-party app on an exact paired iPhone and create a bounded interaction session. URLs and deep links are rejected.',
1047
+ description: 'Request activation of one installed third-party app through CoreDevice and create a bounded interaction session. CoreDevice launch metadata is diagnostic only: capture a screenshot and explicitly confirm the observed foreground before any HID input. Set prewarm_input=true when runnerless input will follow; Forge then waits up to 4 seconds for a non-mutating RemoteXPC readiness result so the first HID action does not need a blind retry.',
667
1048
  readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 2 * 60_000, cancellable: true, idempotent: false,
668
1049
  scopes: ['ios.device'], resourceClaims: mutationClaims,
669
- argumentsSchema: { type: 'object', properties: { device: { type: 'string' }, bundle_id: { type: 'string' }, relaunch: { type: 'boolean' } }, required: ['device', 'bundle_id'], additionalProperties: false },
1050
+ argumentsSchema: { type: 'object', properties: { device: { type: 'string' }, bundle_id: { type: 'string' }, relaunch: { type: 'boolean' }, prewarm_input: { type: 'boolean' } }, required: ['device', 'bundle_id'], additionalProperties: false },
670
1051
  },
671
1052
  {
672
- actionId: 'physical_device_snapshot', title: 'Snapshot physical iOS UI',
673
- description: 'Read bounded UI source through the signed localhost runner. Fails closed when the runner is unavailable.',
674
- readOnly: true, risk: 'readonly', confirmation: 'none', defaultTimeoutMs: 60_000, cancellable: true, idempotent: true,
1053
+ actionId: 'physical_device_screenshot', title: 'Capture physical iOS screenshot',
1054
+ description: 'Capture the exact paired iPhone display through CoreDevice into Controller-owned bounded artifact storage and return a short-lived observation ID. A screenshot alone never authorizes input.',
1055
+ readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 60_000, cancellable: true, idempotent: false,
1056
+ scopes: ['ios.device'], resourceClaims: mutationClaims,
1057
+ argumentsSchema: { type: 'object', properties: { ...interactionProperty, label: { type: 'string' } }, required: ['interaction_id'], additionalProperties: false },
1058
+ },
1059
+ {
1060
+ actionId: 'physical_device_confirm_foreground', title: 'Confirm observed physical iPhone foreground',
1061
+ description: 'Bind the latest fresh CoreDevice screenshot observation to the session target bundle after the Controller has visually verified that exact app is foreground. This only writes the short-lived observation fence and sends no device input.',
1062
+ readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 30_000, cancellable: true, idempotent: true,
675
1063
  scopes: ['ios.device'], resourceClaims: stateClaim,
676
- argumentsSchema: { type: 'object', properties: { ...interactionProperty }, required: ['interaction_id'], additionalProperties: false },
1064
+ argumentsSchema: {
1065
+ type: 'object',
1066
+ properties: { ...interactionProperty, observation_id: { type: 'string' }, bundle_id: { type: 'string' } },
1067
+ required: ['interaction_id', 'observation_id', 'bundle_id'], additionalProperties: false,
1068
+ },
677
1069
  },
678
1070
  {
679
- actionId: 'physical_device_press', title: 'Press physical iOS target',
680
- description: 'Press a non-sensitive accessibility target or coordinate through the signed runner. Purchase, payment, biometric and credential semantics are blocked.',
681
- readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 60_000, cancellable: true, idempotent: false,
1071
+ actionId: 'physical_device_observed_tap', title: 'Tap a freshly observed physical iPhone screen',
1072
+ description: 'One-shot screen-relative tap bound to the latest fresh screenshot observation. It does not reactivate any app and consumes the observation before dispatch, so it can safely navigate system UI such as Spotlight into the target app when CoreDevice activation did not actually foreground it.',
1073
+ readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 30_000, cancellable: true, idempotent: false,
682
1074
  scopes: ['ios.device'], resourceClaims: mutationClaims,
683
- argumentsSchema: { type: 'object', properties: { ...interactionProperty, target: { type: 'string' }, x: { type: 'number' }, y: { type: 'number' }, purpose: { type: 'string' } }, required: ['interaction_id'], additionalProperties: false },
1075
+ argumentsSchema: {
1076
+ type: 'object',
1077
+ properties: { ...interactionProperty, observation_id: { type: 'string' }, x: { type: 'number', minimum: 0 }, y: { type: 'number', minimum: 0 } },
1078
+ required: ['interaction_id', 'observation_id', 'x', 'y'], additionalProperties: false,
1079
+ },
684
1080
  },
685
1081
  {
686
- actionId: 'physical_device_fill', title: 'Fill physical iOS target',
687
- description: 'Replace non-sensitive text through the signed runner. Text is redacted and passwords, codes, biometrics, checkout and payment targets are blocked.',
1082
+ actionId: 'physical_device_batch', title: 'Run fast physical iPhone input batch',
1083
+ description: 'Preferred fast path for a visually confirmed target app: perform up to 20 tap, swipe, text, or bounded-wait steps under one fresh screenshot foreground fence, one unlock/display readiness pass, and one plugin round-trip. Pass observation_id to atomically bind and consume the latest visually verified screenshot without a separate confirm_foreground round-trip; omitting it preserves the explicit confirm_foreground compatibility path. Stops on the first failed step and never replays a partially-mutated batch.',
688
1084
  readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 60_000, cancellable: true, idempotent: false,
689
1085
  scopes: ['ios.device'], resourceClaims: mutationClaims,
690
- argumentsSchema: { type: 'object', properties: { ...interactionProperty, target: { type: 'string' }, text: { type: 'string' } }, required: ['interaction_id', 'target', 'text'], additionalProperties: false },
1086
+ argumentsSchema: {
1087
+ type: 'object',
1088
+ properties: {
1089
+ ...interactionProperty,
1090
+ observation_id: { type: 'string' },
1091
+ steps: {
1092
+ type: 'array', minItems: 1, maxItems: MAX_BATCH_STEPS,
1093
+ items: {
1094
+ type: 'object',
1095
+ properties: {
1096
+ kind: { type: 'string', enum: ['tap', 'swipe', 'type', 'wait'] },
1097
+ x: { type: 'number', minimum: 0 }, y: { type: 'number', minimum: 0 },
1098
+ to_x: { type: 'number', minimum: 0 }, to_y: { type: 'number', minimum: 0 },
1099
+ duration_ms: { type: 'number', minimum: 0, maximum: MAX_BATCH_WAIT_MS },
1100
+ text: { type: 'string', maxLength: 2048 },
1101
+ input_mode: { type: 'string', enum: ['auto', 'keys', 'pasteboard'] },
1102
+ replace_existing: { type: 'boolean' },
1103
+ },
1104
+ required: ['kind'], additionalProperties: false,
1105
+ },
1106
+ },
1107
+ },
1108
+ required: ['interaction_id', 'steps'], additionalProperties: false,
1109
+ },
691
1110
  },
692
1111
  {
693
- actionId: 'physical_device_scroll', title: 'Scroll physical iOS session',
694
- description: 'Scroll an active physical-device UI session through the signed runner.',
695
- readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 60_000, cancellable: true, idempotent: false,
1112
+ actionId: 'physical_device_tap', title: 'Tap physical iPhone coordinate',
1113
+ description: 'Send one runnerless touch through RemoteXPC only while a fresh screenshot observation has been explicitly confirmed as the target app foreground. The observation is consumed before dispatch. For system-screen navigation use physical_device_observed_tap; for known form steps prefer physical_device_batch.',
1114
+ readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 30_000, cancellable: true, idempotent: false,
696
1115
  scopes: ['ios.device'], resourceClaims: mutationClaims,
697
- argumentsSchema: { type: 'object', properties: { ...interactionProperty, direction: { type: 'string', enum: ['up', 'down', 'left', 'right', 'top', 'bottom'] }, amount: { type: 'number' } }, required: ['interaction_id', 'direction'], additionalProperties: false },
1116
+ argumentsSchema: {
1117
+ type: 'object',
1118
+ properties: { ...interactionProperty, x: { type: 'number', minimum: 0 }, y: { type: 'number', minimum: 0 } },
1119
+ required: ['interaction_id', 'x', 'y'], additionalProperties: false,
1120
+ },
698
1121
  },
699
1122
  {
700
- actionId: 'physical_device_screenshot', title: 'Capture physical iOS screenshot',
701
- description: 'Capture the exact paired iPhone display through CoreDevice into Controller-owned bounded artifact storage.',
702
- readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 60_000, cancellable: true, idempotent: false,
1123
+ actionId: 'physical_device_swipe', title: 'Swipe physical iPhone',
1124
+ description: 'Send one runnerless touchscreen swipe through RemoteXPC HID only while a fresh screenshot observation has been explicitly confirmed as the target app foreground; the observation is consumed before dispatch.',
1125
+ readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 30_000, cancellable: true, idempotent: false,
703
1126
  scopes: ['ios.device'], resourceClaims: mutationClaims,
704
- argumentsSchema: { type: 'object', properties: { ...interactionProperty, label: { type: 'string' } }, required: ['interaction_id'], additionalProperties: false },
1127
+ argumentsSchema: {
1128
+ type: 'object',
1129
+ properties: {
1130
+ ...interactionProperty,
1131
+ x: { type: 'number', minimum: 0 }, y: { type: 'number', minimum: 0 },
1132
+ to_x: { type: 'number', minimum: 0 }, to_y: { type: 'number', minimum: 0 },
1133
+ duration_ms: { type: 'number', minimum: 80, maximum: 3000 },
1134
+ },
1135
+ required: ['interaction_id', 'x', 'y', 'to_x', 'to_y'], additionalProperties: false,
1136
+ },
1137
+ },
1138
+ {
1139
+ actionId: 'physical_device_type_text', title: 'Type physical iPhone text',
1140
+ description: 'Type bounded text through reusable runnerless RemoteXPC input only while a fresh screenshot observation has been explicitly confirmed as the target app foreground. Auto mode uses direct HID keys for short ASCII and clipboard-preserving CoreDevice pasteboard + Command-V for Unicode; the observation is consumed before dispatch.',
1141
+ readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 30_000, cancellable: true, idempotent: false,
1142
+ scopes: ['ios.device'], resourceClaims: mutationClaims,
1143
+ argumentsSchema: {
1144
+ type: 'object',
1145
+ properties: { ...interactionProperty, text: { type: 'string', maxLength: 2048 }, input_mode: { type: 'string', enum: ['auto', 'keys', 'pasteboard'] }, replace_existing: { type: 'boolean' } },
1146
+ required: ['interaction_id', 'text'], additionalProperties: false,
1147
+ },
705
1148
  },
706
1149
  {
707
1150
  actionId: 'physical_device_events', title: 'Read physical iOS events',
@@ -712,40 +1155,68 @@ export function iosPhysicalDeviceActions(): AssistantPluginActionDescriptor[] {
712
1155
  },
713
1156
  {
714
1157
  actionId: 'physical_device_close', title: 'Close physical iOS session',
715
- description: 'Close the optional UI-runner session and release Controller ownership without shutting down or modifying the iPhone.',
1158
+ description: 'Release Controller ownership without shutting down or modifying the iPhone. No XCTest/WDA Runner is owned by this provider.',
716
1159
  readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 60_000, cancellable: true, idempotent: true,
717
1160
  scopes: ['ios.device'], resourceClaims: mutationClaims,
718
1161
  argumentsSchema: { type: 'object', properties: { ...interactionProperty }, required: ['interaction_id'], additionalProperties: false },
719
1162
  },
720
1163
  ];
1164
+ return actions.map((action) => action.readOnly || action.actionId === 'physical_device_confirm_foreground'
1165
+ ? action
1166
+ : { ...action, executionMode: 'lightweight_process' });
721
1167
  }
722
1168
 
723
1169
  export async function executeIosPhysicalDeviceAction(input: AssistantPluginActionExecutionInput): Promise<Record<string, unknown>> {
1170
+ const actionStartedAt = performance.now();
1171
+ const timingStages: TimingStages = {};
724
1172
  if (input.actionId === 'physical_device_status') {
725
- return { provider: 'coredevice', ...iosPhysicalDeviceStatus(), uiRunner: await probeRunner() };
1173
+ const startedAt = performance.now();
1174
+ const status = await iosPhysicalDeviceActionStatus(input);
1175
+ recordTiming(timingStages, 'coreDeviceStatus', startedAt, false);
1176
+ return finishTimedResult(actionStartedAt, timingStages, { provider: 'coredevice', ...status });
726
1177
  }
727
- const status = iosPhysicalDeviceStatus();
1178
+ const statusStartedAt = performance.now();
1179
+ const readiness = await cachedIosPhysicalDeviceActionStatus(input);
1180
+ recordTiming(timingStages, 'coreDeviceStatus', statusStartedAt, readiness.cached);
1181
+ const status = readiness.status;
728
1182
  if (!status.coreDeviceReady) {
729
- throw new AssistantPluginError('PLUGIN_DEPENDENCY_MISSING', status.reason ?? 'Xcode CoreDevice is unavailable.', { retryable: false, details: status });
1183
+ throw new AssistantPluginError('PLUGIN_DEPENDENCY_MISSING', status.reason ?? 'Xcode CoreDevice is unavailable.', { retryable: false, details: { ...status } });
730
1184
  }
731
1185
  if (input.actionId === 'physical_device_list') {
732
- return { provider: 'coredevice', devices: physicalDevices(input) };
1186
+ return { provider: 'coredevice', devices: await physicalDevices(input) };
1187
+ }
1188
+ if (input.actionId === 'physical_device_info') {
1189
+ const selected = await selectDevice(input, input.args.device);
1190
+ return { provider: 'coredevice', ...await physicalDeviceInfo(input, selected) };
733
1191
  }
734
1192
  if (input.actionId === 'physical_device_apps') {
735
- const selected = selectDevice(input, input.args.device);
1193
+ const selected = await selectDevice(input, input.args.device);
736
1194
  const bundleId = requireBundleId(input.args.bundle_id);
737
- return { provider: 'coredevice', device: selected, apps: installedApps(input, selected, bundleId) };
1195
+ return { provider: 'coredevice', device: selected, apps: await installedApps(input, selected, bundleId) };
738
1196
  }
739
1197
  if (input.actionId === 'physical_device_open') {
740
- const selected = selectDevice(input, input.args.device);
1198
+ const selectStartedAt = performance.now();
1199
+ const selected = await selectDevice(input, input.args.device);
1200
+ recordTiming(timingStages, 'deviceSelection', selectStartedAt, false);
741
1201
  const bundleId = requireBundleId(input.args.bundle_id);
742
- const apps = installedApps(input, selected, bundleId);
1202
+ const appsStartedAt = performance.now();
1203
+ const lockStartedAt = performance.now();
1204
+ const appsPromise = installedApps(input, selected, bundleId)
1205
+ .finally(() => recordTiming(timingStages, 'installedAppLookup', appsStartedAt, false));
1206
+ const lockPromise = requirePhysicalDeviceUnlocked(input, selected)
1207
+ .finally(() => recordTiming(timingStages, 'lockState', lockStartedAt, false));
1208
+ const [apps] = await Promise.all([appsPromise, lockPromise]);
743
1209
  if (apps.length !== 1) {
744
1210
  throw new AssistantPluginError('IOS_DEVICE_APP_NOT_INSTALLED', `The app ${bundleId} is not installed on the selected iPhone.`, { retryable: false });
745
1211
  }
746
1212
  const selectedAliases = [selected.identifier, selected.udid];
1213
+ const sessionLookupStartedAt = performance.now();
1214
+ const expiredSessionCount = expirePhysicalDeviceSessions(input);
1215
+ pruneInteractionSessions(input.repoRoot, PROVIDER, 100);
747
1216
  const conflict = listInteractionSessions(input.repoRoot, PROVIDER).find((entry) =>
748
1217
  isInteractionSessionActive(entry.status) && interactionMayOwnTarget(entry, selectedAliases));
1218
+ recordTiming(timingStages, 'sessionLookup', sessionLookupStartedAt, false);
1219
+ if (expiredSessionCount > 0) timingStages.sessionReconcile = { ms: timingStages.sessionLookup.ms, cached: false };
749
1220
  if (conflict) {
750
1221
  throw new AssistantPluginError('PLUGIN_RESOURCE_BUSY', 'The selected iPhone already has an active forge interaction.', {
751
1222
  retryable: true,
@@ -757,7 +1228,6 @@ export async function executeIosPhysicalDeviceAction(input: AssistantPluginActio
757
1228
  },
758
1229
  });
759
1230
  }
760
- pruneInteractionSessions(input.repoRoot, PROVIDER, 100);
761
1231
  const interactionId = `ios_device_${randomUUID()}`;
762
1232
  const createdAt = timestamp();
763
1233
  const record: InteractionSessionRecord = {
@@ -777,55 +1247,93 @@ export async function executeIosPhysicalDeviceAction(input: AssistantPluginActio
777
1247
  expiresAt: new Date(hooks.now().getTime() + SESSION_EXPIRY_MS).toISOString(),
778
1248
  };
779
1249
  writeInteractionSession(input.repoRoot, record);
780
- const runnerProbe = await probeRunner();
781
1250
  const state: PhysicalSessionState = {
782
1251
  schemaVersion: 1,
783
1252
  interactionId,
784
1253
  device: selected,
785
1254
  bundleId,
786
- runner: {
787
- configured: runnerProbe.configured,
788
- endpoint: runnerProbe.endpoint,
789
- ready: false,
790
- error: runnerProbe.error,
791
- },
1255
+ unlockVerifiedAt: createdAt,
792
1256
  events: [{ at: createdAt, type: 'session_created', details: { bundleId, deviceIdentifier: selected.identifier } }],
793
1257
  };
1258
+ const statePersistStartedAt = performance.now();
794
1259
  writeState(input, state);
1260
+ recordTiming(timingStages, 'eventPersistence', statePersistStartedAt, false);
795
1261
  try {
796
- const args = ['device', 'process', 'launch', '--device', selected.identifier];
1262
+ const args = ['device', 'process', 'launch', '--device', selected.identifier, '--activate'];
797
1263
  if (input.args.relaunch === true) args.push('--terminate-existing');
798
1264
  args.push(bundleId);
799
- const launch = runCoreJson(input, args, 'IOS_DEVICE_LAUNCH_FAILED', 60_000);
1265
+ const prewarmStartedAt = performance.now();
1266
+ const prewarmRequested = input.args.prewarm_input === true && Boolean(selected.udid);
1267
+ const inputPrewarm: Record<string, unknown> = prewarmRequested && selected.udid
1268
+ ? prewarmRemoteXpcHid({
1269
+ controllerHome: input.controllerHome,
1270
+ deviceIdentifier: selected.identifier,
1271
+ udid: selected.udid,
1272
+ })
1273
+ : { backend: 'remote-xpc-hid', state: 'not_requested', runnerOwned: false };
1274
+ recordTiming(
1275
+ timingStages,
1276
+ 'hidPrewarm',
1277
+ prewarmStartedAt,
1278
+ inputPrewarm.state === 'ready' || inputPrewarm.state === 'test' || input.args.prewarm_input !== true,
1279
+ );
1280
+ const displayStartedAt = performance.now();
1281
+ const display = input.args.prewarm_input === true
1282
+ ? await physicalDisplayGeometry(input, selected).then((result) => {
1283
+ recordTiming(timingStages, 'displayInfo', displayStartedAt, false);
1284
+ return result;
1285
+ }, (error) => {
1286
+ recordTiming(timingStages, 'displayInfo', displayStartedAt, false);
1287
+ throw error;
1288
+ })
1289
+ : undefined;
1290
+ if (input.args.prewarm_input !== true) recordTiming(timingStages, 'displayInfo', displayStartedAt, true);
1291
+ if (display) {
1292
+ state.display = { ...display, observedAt: createdAt };
1293
+ writeState(input, state);
1294
+ }
1295
+ const launchStartedAt = performance.now();
1296
+ const launchPromise = runCoreJson(input, args, 'IOS_DEVICE_LAUNCH_FAILED', 60_000)
1297
+ .finally(() => recordTiming(timingStages, 'activationRequest', launchStartedAt, false));
1298
+ const launch = await launchPromise;
1299
+ const launchEventStartedAt = performance.now();
800
1300
  appendEvent(input, interactionId, 'app_launched', { bundleId, relaunch: input.args.relaunch === true });
801
- if (runnerProbe.ready && runnerProbe.endpoint) {
802
- try {
803
- state.runner.sessionId = await createRunnerSession(runnerProbe.endpoint, bundleId);
804
- state.runner.ready = true;
805
- state.runner.error = undefined;
806
- writeState(input, state);
807
- appendEvent(input, interactionId, 'ui_runner_attached', { endpoint: runnerProbe.endpoint });
808
- } catch (error) {
809
- state.runner.ready = false;
810
- state.runner.error = toAssistantPluginError(error, {
811
- code: 'IOS_DEVICE_UI_SESSION_FAILED',
812
- message: 'The app launched, but the UI runner session could not be created.',
813
- retryable: true,
814
- }).message;
815
- writeState(input, state);
816
- appendEvent(input, interactionId, 'ui_runner_unavailable', { error: state.runner.error });
817
- }
1301
+ recordTiming(timingStages, 'eventPersistence', launchEventStartedAt, false);
1302
+ if (input.args.prewarm_input === true) {
1303
+ appendEvent(input, interactionId, 'input_prewarm', {
1304
+ state: inputPrewarm.state,
1305
+ ...(typeof inputPrewarm.waitMs === 'number' ? { waitMs: inputPrewarm.waitMs } : {}),
1306
+ ...(typeof inputPrewarm.errorCode === 'string' ? { errorCode: inputPrewarm.errorCode } : {}),
1307
+ ...(typeof inputPrewarm.phase === 'string' ? { phase: inputPrewarm.phase } : {}),
1308
+ });
818
1309
  }
1310
+ // CoreDevice is the default physical-device substrate. Never probe, start,
1311
+ // build, install, or attach an XCTest/WDA runner during ordinary app open.
1312
+ // Semantic automation is an explicit opt-in action so repeated computer-use
1313
+ // workflows cannot accidentally create fresh Runner lifecycles.
819
1314
  const active = patchInteractionSession(input.repoRoot, PROVIDER, interactionId, { status: 'waiting_for_user' }) ?? record;
820
- return {
1315
+ return finishTimedResult(actionStartedAt, timingStages, {
821
1316
  provider: 'coredevice',
822
1317
  interaction: active,
823
1318
  device: selected,
824
1319
  app: apps[0],
825
1320
  launch: bounded(launch),
826
- uiAutomation: bounded(readState(input, interactionId)?.runner),
827
- };
1321
+ foregroundVerification: {
1322
+ authority: 'screenshot_observation',
1323
+ required: true,
1324
+ coreDeviceActivatedWhenStartedHint: objectValue(objectValue(launch.result).launchOptions).activatedWhenStarted === true,
1325
+ },
1326
+ inputPrewarm,
1327
+ controlPlane: {
1328
+ lifecycle: 'coredevice',
1329
+ observation: 'coredevice',
1330
+ input: 'remote-xpc-hid',
1331
+ semanticFallback: 'agent-device',
1332
+ runnerOwned: false,
1333
+ },
1334
+ });
828
1335
  } catch (error) {
1336
+ stopRemoteXpcHidForDevice(selected.identifier);
829
1337
  const normalized = toAssistantPluginError(error, {
830
1338
  code: 'IOS_DEVICE_OPEN_FAILED',
831
1339
  message: 'The physical iOS app could not be opened.',
@@ -840,114 +1348,357 @@ export async function executeIosPhysicalDeviceAction(input: AssistantPluginActio
840
1348
  }
841
1349
  }
842
1350
 
1351
+ const sessionLookupStartedAt = performance.now();
843
1352
  const { record, state } = requireRecord(input, input.actionId === 'physical_device_close');
1353
+ recordTiming(timingStages, 'sessionLookup', sessionLookupStartedAt, false);
844
1354
  switch (input.actionId) {
845
- case 'physical_device_snapshot': {
846
- const runner = requireRunner(state);
847
- const value = await runnerRequest('GET', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/source`, undefined, 'IOS_DEVICE_SNAPSHOT_FAILED');
848
- appendEvent(input, record.interactionId, 'snapshot');
849
- return { provider: 'coredevice', interaction: record, source: bounded(value) };
850
- }
851
- case 'physical_device_press': {
852
- const runner = requireRunner(state);
853
- const target = optionalString(input.args.target);
854
- const hasPoint = typeof input.args.x === 'number' && typeof input.args.y === 'number';
855
- if (!target && !hasPoint) throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', 'target or x/y is required.', { retryable: false });
856
- if (target) {
857
- rejectSensitiveSemantic(target, 'Press');
858
- const id = await elementId(runner.endpoint, runner.sessionId, target);
859
- await inspectElementSemantics(runner.endpoint, runner.sessionId, id, target, 'Press');
860
- await runnerRequest('POST', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/element/${encodeURIComponent(id)}/click`, {}, 'IOS_DEVICE_PRESS_FAILED');
861
- appendEvent(input, record.interactionId, 'press', { target });
862
- } else {
863
- const purpose = requireString(input.args.purpose, 'purpose');
864
- rejectSensitiveSemantic(purpose, 'Coordinate press');
865
- await runnerRequest('POST', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/wda/tap`, {
866
- x: Number(input.args.x), y: Number(input.args.y),
867
- }, 'IOS_DEVICE_PRESS_FAILED');
868
- appendEvent(input, record.interactionId, 'press_coordinate', { x: input.args.x, y: input.args.y, purpose });
869
- }
870
- return { provider: 'coredevice', interaction: record, success: true };
871
- }
872
- case 'physical_device_fill': {
873
- const runner = requireRunner(state);
874
- const target = requireString(input.args.target, 'target');
875
- rejectSensitiveSemantic(target, 'Fill');
876
- const text = typeof input.args.text === 'string' ? input.args.text : undefined;
877
- if (text === undefined) throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', 'text is required.', { retryable: false });
878
- const id = await elementId(runner.endpoint, runner.sessionId, target);
879
- await inspectElementSemantics(runner.endpoint, runner.sessionId, id, target, 'Fill');
880
- await runnerRequest('POST', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/element/${encodeURIComponent(id)}/clear`, {}, 'IOS_DEVICE_FILL_FAILED', true);
881
- await runnerRequest('POST', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/element/${encodeURIComponent(id)}/value`, {
882
- text, value: Array.from(text),
883
- }, 'IOS_DEVICE_FILL_FAILED', true);
884
- appendEvent(input, record.interactionId, 'fill', { target, text: '<redacted>' });
885
- return { provider: 'coredevice', interaction: record, success: true, text: '<redacted>' };
886
- }
887
- case 'physical_device_scroll': {
888
- const runner = requireRunner(state);
889
- const direction = requireString(input.args.direction, 'direction');
890
- if (!['up', 'down', 'left', 'right', 'top', 'bottom'].includes(direction)) {
891
- throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', 'Unsupported scroll direction.', { retryable: false });
892
- }
893
- const sizeValue = await runnerRequest('GET', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/window/size`, undefined, 'IOS_DEVICE_SCROLL_FAILED');
894
- const size = objectValue(sizeValue);
895
- const width = Number(size.width ?? 0);
896
- const height = Number(size.height ?? 0);
897
- if (!(width > 0 && height > 0)) throw new AssistantPluginError('IOS_DEVICE_SCROLL_FAILED', 'The UI runner returned an invalid window size.', { retryable: false });
898
- const amount = Math.max(20, Math.min(80, typeof input.args.amount === 'number' ? Math.trunc(input.args.amount) : 60)) / 100;
899
- let fromX = width / 2; let toX = width / 2; let fromY = height / 2; let toY = height / 2;
900
- if (direction === 'down' || direction === 'bottom') { fromY = height * 0.8; toY = height * (0.8 - amount); }
901
- if (direction === 'up' || direction === 'top') { fromY = height * 0.2; toY = height * (0.2 + amount); }
902
- if (direction === 'right') { fromX = width * 0.8; toX = width * (0.8 - amount); }
903
- if (direction === 'left') { fromX = width * 0.2; toX = width * (0.2 + amount); }
904
- await runnerRequest('POST', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/wda/dragfromtoforduration`, {
905
- fromX, fromY, toX, toY, duration: 0.25,
906
- }, 'IOS_DEVICE_SCROLL_FAILED');
907
- appendEvent(input, record.interactionId, 'scroll', { direction, amount });
908
- return { provider: 'coredevice', interaction: record, success: true };
909
- }
910
1355
  case 'physical_device_screenshot': {
911
1356
  const label = sanitize(optionalString(input.args.label) ?? 'screenshot');
912
1357
  const path = join(artifactDir(input, record.interactionId), `${label}-${hooks.now().getTime()}.png`);
913
- const result = runCoreJson(input, [
1358
+ const result = await runCoreJson(input, [
914
1359
  'device', 'capture', 'screenshot', '--device', state.device.identifier, '--destination', path,
915
1360
  ], 'IOS_DEVICE_SCREENSHOT_FAILED', 60_000);
916
1361
  if (!existsSync(path)) {
917
1362
  throw new AssistantPluginError('IOS_DEVICE_SCREENSHOT_MISSING', 'CoreDevice succeeded without creating the requested screenshot.', { retryable: false });
918
1363
  }
919
- appendEvent(input, record.interactionId, 'screenshot', { label });
920
- return {
1364
+ const observedAt = timestamp();
1365
+ const observationId = randomUUID();
1366
+ const screenshotDisplay = pngDisplayGeometry(path, state.display?.pointScale);
1367
+ if (screenshotDisplay) state.display = { ...screenshotDisplay, observedAt };
1368
+ state.lastScreenshot = { observationId, observedAt, label };
1369
+ state.foregroundObservation = undefined;
1370
+ writeState(input, state);
1371
+ const eventStartedAt = performance.now();
1372
+ appendEvent(input, record.interactionId, 'screenshot', { label, observationId });
1373
+ recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
1374
+ return finishTimedResult(actionStartedAt, timingStages, {
921
1375
  provider: 'coredevice', interaction: record, result: bounded(result),
1376
+ observation: { observationId, observedAt, label, ttlMs: INPUT_FOREGROUND_OBSERVATION_TTL_MS },
1377
+ fastPath: {
1378
+ actionId: 'physical_device_batch',
1379
+ arguments: { interaction_id: record.interactionId, observation_id: observationId },
1380
+ requires: ['steps'],
1381
+ ttlMs: INPUT_FOREGROUND_OBSERVATION_TTL_MS,
1382
+ oneShot: true,
1383
+ },
922
1384
  artifactCandidates: [{ kind: 'ios_physical_device_screenshot', mediaType: 'image/png', path }],
1385
+ });
1386
+ }
1387
+ case 'physical_device_confirm_foreground': {
1388
+ const observationId = requireString(input.args.observation_id, 'observation_id');
1389
+ const bundleId = requireString(input.args.bundle_id, 'bundle_id');
1390
+ if (bundleId !== state.bundleId) {
1391
+ throw new AssistantPluginError('IOS_DEVICE_FOREGROUND_OBSERVATION_MISMATCH', `Foreground confirmation bundle ${bundleId} does not match session target ${state.bundleId}.`, {
1392
+ retryable: false,
1393
+ details: { expectedBundleId: state.bundleId, providedBundleId: bundleId, hidMutationDispatched: false },
1394
+ });
1395
+ }
1396
+ const { observation, ageMs } = recentScreenshotObservation(state, observationId);
1397
+ const confirmedAt = timestamp();
1398
+ state.foregroundObservation = {
1399
+ observationId: observation.observationId,
1400
+ screenshotObservedAt: observation.observedAt,
1401
+ confirmedAt,
1402
+ bundleId,
1403
+ };
1404
+ writeState(input, state);
1405
+ appendEvent(input, record.interactionId, 'foreground_confirmed', { observationId, bundleId, screenshotAgeMs: ageMs });
1406
+ return {
1407
+ provider: 'coredevice',
1408
+ interaction: record,
1409
+ foregroundObservation: { observationId, bundleId, screenshotObservedAt: observation.observedAt, confirmedAt, ttlMs: INPUT_FOREGROUND_OBSERVATION_TTL_MS },
1410
+ deviceUnmodified: true,
923
1411
  };
924
1412
  }
1413
+ case 'physical_device_observed_tap': {
1414
+ const lockStartedAt = performance.now();
1415
+ const unlock = await requireSessionUnlocked(input, state);
1416
+ recordTiming(timingStages, 'lockState', lockStartedAt, unlock.cached);
1417
+ if (!state.device.udid) throw new AssistantPluginError('IOS_HID_UDID_MISSING', 'The selected iPhone does not expose a hardware UDID for RemoteXPC HID.', { retryable: false });
1418
+ const observationId = requireString(input.args.observation_id, 'observation_id');
1419
+ const observationStartedAt = performance.now();
1420
+ const { observation, ageMs } = recentScreenshotObservation(state, observationId);
1421
+ recordTiming(timingStages, 'screenObservation', observationStartedAt, true);
1422
+ const displayStartedAt = performance.now();
1423
+ const geometry = await sessionDisplayGeometry(input, state);
1424
+ recordTiming(timingStages, 'displayInfo', displayStartedAt, geometry.cached);
1425
+ const display = geometry.display;
1426
+ const x = requireFiniteNumber(input.args.x, 'x');
1427
+ const y = requireFiniteNumber(input.args.y, 'y');
1428
+ consumePhysicalScreenObservation(input, state);
1429
+ const hidStartedAt = performance.now();
1430
+ const result = await executeRemoteXpcHidInput({
1431
+ controllerHome: input.controllerHome, deviceIdentifier: state.device.identifier, udid: state.device.udid,
1432
+ width: display.width, height: display.height, action: 'tap', x, y,
1433
+ });
1434
+ timingStages.hidWorkerReady = { ms: result.timings.workerStartupMs + result.timings.workerReadyMs, cached: result.reusedWorker };
1435
+ timingStages.hidWriteAck = { ms: result.timings.hidMs ?? result.timings.requestMs, cached: false };
1436
+ timingStages.hidRequestTotal = { ms: elapsedMs(hidStartedAt), cached: result.reusedWorker };
1437
+ appendEvent(input, record.interactionId, 'observed_tap', { requestId: input.requestId, observationId: observation.observationId, observationAgeMs: ageMs, x, y, reusedWorker: result.reusedWorker });
1438
+ return finishTimedResult(actionStartedAt, timingStages, { provider: 'coredevice', inputBackend: 'remote-xpc-hid', interaction: record, display, result: bounded(result), observationConsumed: true });
1439
+ }
1440
+ case 'physical_device_batch': {
1441
+ const lockStartedAt = performance.now();
1442
+ const unlock = await requireSessionUnlocked(input, state);
1443
+ recordTiming(timingStages, 'lockState', lockStartedAt, unlock.cached);
1444
+ if (!state.device.udid) throw new AssistantPluginError('IOS_HID_UDID_MISSING', 'The selected iPhone does not expose a hardware UDID for RemoteXPC HID.', { retryable: false });
1445
+ const steps = physicalInputBatchSteps(input.args.steps);
1446
+ let display: DisplayGeometry | undefined;
1447
+ if (steps.some((step) => step.kind === 'tap' || step.kind === 'swipe')) {
1448
+ const displayStartedAt = performance.now();
1449
+ const geometry = await sessionDisplayGeometry(input, state);
1450
+ recordTiming(timingStages, 'displayInfo', displayStartedAt, geometry.cached);
1451
+ display = geometry.display;
1452
+ }
1453
+ validatePhysicalInputBatchCoordinates(steps, display);
1454
+ const foregroundStartedAt = performance.now();
1455
+ const observationId = input.args.observation_id === undefined
1456
+ ? undefined
1457
+ : requireString(input.args.observation_id, 'observation_id');
1458
+ const foreground = resolvePhysicalBatchForegroundObservation(state, observationId);
1459
+ const foregroundObservation = foreground.foregroundObservation;
1460
+ recordTiming(timingStages, 'foregroundObservation', foregroundStartedAt, true);
1461
+ consumePhysicalScreenObservation(input, state);
1462
+ const batchStartedAt = performance.now();
1463
+ const completed: Array<Record<string, unknown>> = [];
1464
+ let completedMutations = 0;
1465
+ for (let index = 0; index < steps.length; index += 1) {
1466
+ const step = steps[index]!;
1467
+ try {
1468
+ if (step.kind === 'wait') {
1469
+ await waitPhysicalInputBatch(step.durationMs ?? 0, input.signal);
1470
+ completed.push({ index, kind: step.kind, durationMs: step.durationMs ?? 0 });
1471
+ continue;
1472
+ }
1473
+ let result: Awaited<ReturnType<typeof executeRemoteXpcHidInput>>;
1474
+ if (step.kind === 'tap') {
1475
+ result = await executeRemoteXpcHidInput({
1476
+ controllerHome: input.controllerHome,
1477
+ deviceIdentifier: state.device.identifier,
1478
+ udid: state.device.udid,
1479
+ width: display?.width,
1480
+ height: display?.height,
1481
+ action: 'tap', x: step.x, y: step.y,
1482
+ });
1483
+ } else if (step.kind === 'swipe') {
1484
+ result = await executeRemoteXpcHidInput({
1485
+ controllerHome: input.controllerHome,
1486
+ deviceIdentifier: state.device.identifier,
1487
+ udid: state.device.udid,
1488
+ width: display?.width,
1489
+ height: display?.height,
1490
+ action: 'swipe', x: step.x, y: step.y, x2: step.toX, y2: step.toY, durationMs: step.durationMs,
1491
+ });
1492
+ } else {
1493
+ result = await executeRemoteXpcHidInput({
1494
+ controllerHome: input.controllerHome,
1495
+ deviceIdentifier: state.device.identifier,
1496
+ udid: state.device.udid,
1497
+ action: 'type', text: step.text, textMode: step.textMode, replaceExisting: step.replaceExisting,
1498
+ });
1499
+ }
1500
+ completedMutations += 1;
1501
+ completed.push({
1502
+ index,
1503
+ kind: step.kind,
1504
+ reusedWorker: result.reusedWorker,
1505
+ requestMs: result.timings.requestMs,
1506
+ hidMs: result.timings.hidMs,
1507
+ ...(step.kind === 'type' ? { inputMode: objectValue(result.result).inputMode } : {}),
1508
+ });
1509
+ } catch (error) {
1510
+ const normalized = toAssistantPluginError(error, {
1511
+ code: 'IOS_HID_BATCH_FAILED',
1512
+ message: `Physical iOS input batch failed at step ${index}.`,
1513
+ retryable: false,
1514
+ });
1515
+ const eventStartedAt = performance.now();
1516
+ const causeDetails = normalized.details && typeof normalized.details === 'object' && !Array.isArray(normalized.details)
1517
+ ? normalized.details as Record<string, unknown>
1518
+ : {};
1519
+ const retryWholeBatch = completedMutations === 0 && causeDetails.mutationDispatched === false;
1520
+ const causePhase = typeof causeDetails.phase === 'string' ? causeDetails.phase : undefined;
1521
+ const causeMutationDispatched = typeof causeDetails.mutationDispatched === 'boolean'
1522
+ ? causeDetails.mutationDispatched
1523
+ : undefined;
1524
+ const causeMessage = normalized.message.startsWith(`${normalized.code}: `)
1525
+ ? normalized.message.slice(normalized.code.length + 2, normalized.code.length + 514)
1526
+ : normalized.message.slice(0, 512);
1527
+ appendEvent(input, record.interactionId, 'batch_failed', {
1528
+ requestId: input.requestId,
1529
+ failedStepIndex: index,
1530
+ failedStepKind: step.kind,
1531
+ completedSteps: completed.length,
1532
+ completedMutations,
1533
+ retryWholeBatch,
1534
+ causeCode: normalized.code,
1535
+ ...(causePhase ? { causePhase } : {}),
1536
+ ...(causeMutationDispatched === undefined ? {} : { mutationDispatched: causeMutationDispatched }),
1537
+ causeMessage,
1538
+ });
1539
+ recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
1540
+ throw new AssistantPluginError(
1541
+ 'IOS_HID_BATCH_FAILED',
1542
+ retryWholeBatch
1543
+ ? `Physical iOS input batch stopped at step ${index} before any HID mutation was sent; retrying the whole batch is safe.`
1544
+ : `Physical iOS input batch stopped at step ${index}; do not retry the whole batch because an earlier or in-flight step may already have mutated the app.`,
1545
+ {
1546
+ retryable: retryWholeBatch,
1547
+ details: {
1548
+ failedStepIndex: index,
1549
+ failedStepKind: step.kind,
1550
+ completedSteps: completed.length,
1551
+ completedMutations,
1552
+ retryWholeBatch,
1553
+ causeCode: normalized.code,
1554
+ ...(causePhase ? { causePhase } : {}),
1555
+ ...(causeMutationDispatched === undefined ? {} : { mutationDispatched: causeMutationDispatched }),
1556
+ causeMessage,
1557
+ },
1558
+ },
1559
+ );
1560
+ }
1561
+ }
1562
+ recordTiming(timingStages, 'hidBatchTotal', batchStartedAt, false);
1563
+ const eventStartedAt = performance.now();
1564
+ appendEvent(input, record.interactionId, 'batch_input', {
1565
+ requestId: input.requestId,
1566
+ stepCount: steps.length,
1567
+ mutationCount: completedMutations,
1568
+ kinds: steps.map((step) => step.kind),
1569
+ });
1570
+ recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
1571
+ return finishTimedResult(actionStartedAt, timingStages, {
1572
+ provider: 'coredevice',
1573
+ inputBackend: 'remote-xpc-hid',
1574
+ interaction: record,
1575
+ ...(display ? { display } : {}),
1576
+ completed,
1577
+ foregroundObservation: {
1578
+ ...foregroundObservation,
1579
+ consumed: true,
1580
+ source: foreground.source,
1581
+ ...(foreground.screenshotAgeMs === undefined ? {} : { screenshotAgeMs: foreground.screenshotAgeMs }),
1582
+ },
1583
+ executionPlan: {
1584
+ foregroundActivations: 0,
1585
+ foregroundObservationChecks: 1,
1586
+ foregroundObservationSource: foreground.source,
1587
+ unlockChecks: unlock.cached ? 0 : 1,
1588
+ displayLookups: display ? 1 : 0,
1589
+ pluginRoundTrips: 1,
1590
+ runnerOwned: false,
1591
+ },
1592
+ });
1593
+ }
1594
+ case 'physical_device_tap': {
1595
+ const lockStartedAt = performance.now();
1596
+ const unlock = await requireSessionUnlocked(input, state);
1597
+ recordTiming(timingStages, 'lockState', lockStartedAt, unlock.cached);
1598
+ if (!state.device.udid) throw new AssistantPluginError('IOS_HID_UDID_MISSING', 'The selected iPhone does not expose a hardware UDID for RemoteXPC HID.', { retryable: false });
1599
+ const displayStartedAt = performance.now();
1600
+ const geometry = await sessionDisplayGeometry(input, state);
1601
+ recordTiming(timingStages, 'displayInfo', displayStartedAt, geometry.cached);
1602
+ const display = geometry.display;
1603
+ const x = requireFiniteNumber(input.args.x, 'x');
1604
+ const y = requireFiniteNumber(input.args.y, 'y');
1605
+ const foregroundStartedAt = performance.now();
1606
+ const foregroundObservation = requireObservedPhysicalTargetForeground(state);
1607
+ recordTiming(timingStages, 'foregroundObservation', foregroundStartedAt, true);
1608
+ consumePhysicalScreenObservation(input, state);
1609
+ const hidStartedAt = performance.now();
1610
+ const result = await executeRemoteXpcHidInput({
1611
+ controllerHome: input.controllerHome,
1612
+ deviceIdentifier: state.device.identifier,
1613
+ udid: state.device.udid,
1614
+ width: display.width,
1615
+ height: display.height,
1616
+ action: 'tap', x, y,
1617
+ });
1618
+ timingStages.hidWorkerReady = { ms: result.timings.workerStartupMs + result.timings.workerReadyMs, cached: result.reusedWorker };
1619
+ timingStages.hidWriteAck = { ms: result.timings.hidMs ?? result.timings.requestMs, cached: false };
1620
+ timingStages.hidRequestTotal = { ms: elapsedMs(hidStartedAt), cached: result.reusedWorker };
1621
+ const eventStartedAt = performance.now();
1622
+ appendEvent(input, record.interactionId, 'tap', { requestId: input.requestId, x, y, width: display.width, height: display.height, reusedWorker: result.reusedWorker });
1623
+ recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
1624
+ return finishTimedResult(actionStartedAt, timingStages, { provider: 'coredevice', inputBackend: 'remote-xpc-hid', interaction: record, display, result: bounded(result), foregroundObservation: { ...foregroundObservation, consumed: true } });
1625
+ }
1626
+ case 'physical_device_swipe': {
1627
+ const lockStartedAt = performance.now();
1628
+ const unlock = await requireSessionUnlocked(input, state);
1629
+ recordTiming(timingStages, 'lockState', lockStartedAt, unlock.cached);
1630
+ if (!state.device.udid) throw new AssistantPluginError('IOS_HID_UDID_MISSING', 'The selected iPhone does not expose a hardware UDID for RemoteXPC HID.', { retryable: false });
1631
+ const displayStartedAt = performance.now();
1632
+ const geometry = await sessionDisplayGeometry(input, state);
1633
+ recordTiming(timingStages, 'displayInfo', displayStartedAt, geometry.cached);
1634
+ const display = geometry.display;
1635
+ const x = requireFiniteNumber(input.args.x, 'x');
1636
+ const y = requireFiniteNumber(input.args.y, 'y');
1637
+ const x2 = requireFiniteNumber(input.args.to_x, 'to_x');
1638
+ const y2 = requireFiniteNumber(input.args.to_y, 'to_y');
1639
+ const durationMs = typeof input.args.duration_ms === 'number' ? Math.trunc(input.args.duration_ms) : 250;
1640
+ const foregroundStartedAt = performance.now();
1641
+ const foregroundObservation = requireObservedPhysicalTargetForeground(state);
1642
+ recordTiming(timingStages, 'foregroundObservation', foregroundStartedAt, true);
1643
+ consumePhysicalScreenObservation(input, state);
1644
+ const hidStartedAt = performance.now();
1645
+ const result = await executeRemoteXpcHidInput({
1646
+ controllerHome: input.controllerHome,
1647
+ deviceIdentifier: state.device.identifier,
1648
+ udid: state.device.udid,
1649
+ width: display.width,
1650
+ height: display.height,
1651
+ action: 'swipe', x, y, x2, y2, durationMs,
1652
+ });
1653
+ timingStages.hidWorkerReady = { ms: result.timings.workerStartupMs + result.timings.workerReadyMs, cached: result.reusedWorker };
1654
+ timingStages.hidWriteAck = { ms: result.timings.hidMs ?? result.timings.requestMs, cached: false };
1655
+ timingStages.hidRequestTotal = { ms: elapsedMs(hidStartedAt), cached: result.reusedWorker };
1656
+ const eventStartedAt = performance.now();
1657
+ appendEvent(input, record.interactionId, 'swipe', { requestId: input.requestId, from: [x, y], to: [x2, y2], durationMs, reusedWorker: result.reusedWorker });
1658
+ recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
1659
+ return finishTimedResult(actionStartedAt, timingStages, { provider: 'coredevice', inputBackend: 'remote-xpc-hid', interaction: record, display, result: bounded(result), foregroundObservation: { ...foregroundObservation, consumed: true } });
1660
+ }
1661
+ case 'physical_device_type_text': {
1662
+ const lockStartedAt = performance.now();
1663
+ const unlock = await requireSessionUnlocked(input, state);
1664
+ recordTiming(timingStages, 'lockState', lockStartedAt, unlock.cached);
1665
+ if (!state.device.udid) throw new AssistantPluginError('IOS_HID_UDID_MISSING', 'The selected iPhone does not expose a hardware UDID for RemoteXPC HID.', { retryable: false });
1666
+ const text = requireString(input.args.text, 'text');
1667
+ const textMode = optionalString(input.args.input_mode) ?? 'auto';
1668
+ if (!['auto', 'keys', 'pasteboard'].includes(textMode)) {
1669
+ throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `Unsupported input_mode: ${textMode}.`, { retryable: false });
1670
+ }
1671
+ const foregroundStartedAt = performance.now();
1672
+ const foregroundObservation = requireObservedPhysicalTargetForeground(state);
1673
+ recordTiming(timingStages, 'foregroundObservation', foregroundStartedAt, true);
1674
+ consumePhysicalScreenObservation(input, state);
1675
+ const hidStartedAt = performance.now();
1676
+ const result = await executeRemoteXpcHidInput({
1677
+ controllerHome: input.controllerHome,
1678
+ deviceIdentifier: state.device.identifier,
1679
+ udid: state.device.udid,
1680
+ action: 'type', text, textMode: textMode as 'auto' | 'keys' | 'pasteboard', replaceExisting: input.args.replace_existing === true,
1681
+ });
1682
+ timingStages.hidWorkerReady = { ms: result.timings.workerStartupMs + result.timings.workerReadyMs, cached: result.reusedWorker };
1683
+ timingStages.hidWriteAck = { ms: result.timings.hidMs ?? result.timings.requestMs, cached: false };
1684
+ timingStages.hidRequestTotal = { ms: elapsedMs(hidStartedAt), cached: result.reusedWorker };
1685
+ const eventStartedAt = performance.now();
1686
+ appendEvent(input, record.interactionId, 'type_text', { requestId: input.requestId, length: text.length, reusedWorker: result.reusedWorker });
1687
+ recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
1688
+ return finishTimedResult(actionStartedAt, timingStages, { provider: 'coredevice', inputBackend: 'remote-xpc-hid', interaction: record, result: bounded(result), text: '<redacted>', foregroundObservation: { ...foregroundObservation, consumed: true } });
1689
+ }
925
1690
  case 'physical_device_events': {
926
1691
  const limit = Math.max(1, Math.min(MAX_EVENTS, typeof input.args.limit === 'number' ? Math.trunc(input.args.limit) : 50));
927
1692
  return { provider: 'coredevice', interaction: record, events: bounded(state.events.slice(-limit)) };
928
1693
  }
929
1694
  case 'physical_device_close': {
1695
+ const inputWorkerRelease = stopRemoteXpcHidForDevice(state.device.identifier);
930
1696
  const wasActive = isInteractionSessionActive(record.status);
931
- const runnerNeedsCleanup = Boolean(state.runner.endpoint && state.runner.sessionId);
932
- if (!wasActive && !runnerNeedsCleanup) return { provider: 'coredevice', interaction: record, alreadyClosed: true };
933
- if (wasActive) patchInteractionSession(input.repoRoot, PROVIDER, record.interactionId, { status: 'closing' });
934
- const closed = await closeRunner(state);
935
- if (!closed) {
936
- patchInteractionSession(input.repoRoot, PROVIDER, record.interactionId, {
937
- status: 'closing',
938
- error: { code: 'IOS_DEVICE_UI_CLOSE_FAILED', message: 'The UI runner session could not be closed; ownership remains fenced.' },
939
- });
940
- throw new AssistantPluginError('IOS_DEVICE_UI_CLOSE_FAILED', 'The UI runner session could not be closed; retry physical_device_close.', { retryable: true });
941
- }
942
- state.runner.ready = false;
943
- state.runner.sessionId = undefined;
944
- writeState(input, state);
945
- appendEvent(input, record.interactionId, 'session_closed');
946
- if (!wasActive) {
947
- return { provider: 'coredevice', interaction: record, alreadyClosed: true, runnerCleaned: runnerNeedsCleanup, deviceUnmodified: true };
948
- }
1697
+ if (!wasActive) return { provider: 'coredevice', interaction: record, alreadyClosed: true, deviceUnmodified: true, inputWorkerRelease };
1698
+ patchInteractionSession(input.repoRoot, PROVIDER, record.interactionId, { status: 'closing' });
1699
+ appendEvent(input, record.interactionId, 'session_closed', { inputWorkerRelease });
949
1700
  const finalRecord = patchInteractionSession(input.repoRoot, PROVIDER, record.interactionId, { status: 'closed', error: undefined }) ?? record;
950
- return { provider: 'coredevice', interaction: finalRecord, deviceUnmodified: true };
1701
+ return { provider: 'coredevice', interaction: finalRecord, deviceUnmodified: true, inputWorkerRelease };
951
1702
  }
952
1703
  default:
953
1704
  throw new AssistantPluginError('PLUGIN_ACTION_NOT_SUPPORTED', `ios/${input.actionId} is not supported.`, { retryable: false });