machine-bridge-mcp 2.0.0 → 3.0.0-beta.102

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (473) hide show
  1. package/CHANGELOG.md +1142 -0
  2. package/CONTRIBUTING.md +23 -25
  3. package/GOVERNANCE.md +2 -2
  4. package/README.md +58 -20
  5. package/SECURITY.md +161 -120
  6. package/SUPPORT.md +1 -1
  7. package/browser-extension/broker-auth.js +83 -0
  8. package/browser-extension/broker-liveness.js +35 -0
  9. package/browser-extension/browser-error-boundary.js +96 -0
  10. package/browser-extension/browser-operations.js +1456 -101
  11. package/browser-extension/devtools-input.js +195 -34
  12. package/browser-extension/devtools-observation.js +429 -0
  13. package/browser-extension/devtools-session.js +48 -0
  14. package/browser-extension/manifest.json +4 -3
  15. package/browser-extension/page-automation.js +578 -118
  16. package/browser-extension/pairing-bootstrap.js +42 -0
  17. package/browser-extension/pairing.js +28 -12
  18. package/browser-extension/service-worker.js +126 -106
  19. package/docs/AGENT_CONTEXT.md +13 -10
  20. package/docs/ARCHITECTURE.md +159 -67
  21. package/docs/AUDIT.md +1302 -4
  22. package/docs/CLIENTS.md +19 -6
  23. package/docs/COMPUTER_USE.md +297 -0
  24. package/docs/ENGINEERING.md +72 -13
  25. package/docs/GETTING_STARTED.md +18 -20
  26. package/docs/LOCAL_AUTHORIZATION.md +115 -72
  27. package/docs/LOCAL_AUTOMATION.md +27 -21
  28. package/docs/LOGGING.md +40 -19
  29. package/docs/MANAGED_JOBS.md +37 -34
  30. package/docs/MULTI_ACCOUNT.md +98 -37
  31. package/docs/OPERATIONS.md +145 -51
  32. package/docs/OVERVIEW.md +14 -11
  33. package/docs/POLICY_REFERENCE.md +71 -3
  34. package/docs/PRIVACY.md +41 -2
  35. package/docs/PROJECT_STANDARDS.md +29 -6
  36. package/docs/RELEASING.md +238 -64
  37. package/docs/TESTING.md +150 -39
  38. package/docs/THREAT_MODEL.md +164 -67
  39. package/docs/TOOL_REFERENCE.md +653 -60
  40. package/docs/UPGRADING.md +78 -18
  41. package/native/macos/MachineBridgeBackgroundInput.swift +1051 -0
  42. package/native/macos/MachineBridgeBackgroundInputSmokeFixture.swift +295 -0
  43. package/native/macos/MachineBridgeTrustBroker.swift +233 -0
  44. package/package.json +94 -16
  45. package/scripts/accepted-candidate-tarball.mjs +77 -0
  46. package/scripts/candidate-runtime-store.mjs +93 -0
  47. package/scripts/check-plan.mjs +73 -3
  48. package/scripts/check-runner.mjs +129 -17
  49. package/scripts/commit-message-check.mjs +5 -2
  50. package/scripts/consumer-package-security.mjs +293 -0
  51. package/scripts/coverage-check.mjs +251 -15
  52. package/scripts/coverage-generation.mjs +41 -0
  53. package/scripts/coverage-range-merge.mjs +41 -0
  54. package/scripts/foreground-daemon-recovery.mjs +88 -0
  55. package/scripts/generate-policy-reference.mjs +28 -5
  56. package/scripts/generate-tool-reference.mjs +1 -1
  57. package/scripts/generate-worker-types.mjs +21 -11
  58. package/scripts/github-backlog.mjs +11 -10
  59. package/scripts/github-push.mjs +62 -16
  60. package/scripts/github-release-asset.mjs +58 -0
  61. package/scripts/github-release.mjs +241 -132
  62. package/scripts/global-package-installation.mjs +68 -0
  63. package/scripts/hardened-npm-session.mjs +42 -0
  64. package/scripts/install-published-prerelease.mjs +198 -0
  65. package/scripts/local-release-acceptance.mjs +103 -45
  66. package/scripts/macos-background-input-smoke.mjs +515 -0
  67. package/scripts/network-retry.mjs +8 -1
  68. package/scripts/npm-global-prefix.mjs +27 -0
  69. package/scripts/npm-publication-policy.mjs +17 -0
  70. package/scripts/official-mcp-conformance.mjs +267 -0
  71. package/scripts/persistent-activation-process.mjs +101 -0
  72. package/scripts/prepare-pinned-npm.mjs +12 -69
  73. package/scripts/prerelease-activation.mjs +160 -0
  74. package/scripts/privacy-check.mjs +14 -6
  75. package/scripts/promotion-digest.mjs +128 -0
  76. package/scripts/publish-npm.mjs +260 -0
  77. package/scripts/published-release.mjs +132 -0
  78. package/scripts/release-acceptance.mjs +62 -54
  79. package/scripts/release-browser-extension-store.mjs +172 -0
  80. package/scripts/release-candidate-manifest.mjs +52 -0
  81. package/scripts/release-channel.mjs +106 -0
  82. package/scripts/release-diagnostic.mjs +41 -0
  83. package/scripts/release-impact-check.mjs +12 -22
  84. package/scripts/release-oauth-canary-core.mjs +325 -0
  85. package/scripts/release-oauth-canary-evidence.mjs +100 -0
  86. package/scripts/release-oauth-canary.mjs +127 -0
  87. package/scripts/release-publication-guard.mjs +67 -0
  88. package/scripts/release-soak.mjs +284 -0
  89. package/scripts/release-state.mjs +1 -1
  90. package/scripts/run-checks.mjs +44 -7
  91. package/scripts/run-worker-dry-run.mjs +21 -0
  92. package/scripts/sarif-security-gate.mjs +2 -1
  93. package/scripts/sbom-check.mjs +106 -0
  94. package/scripts/start-release-candidate.mjs +267 -44
  95. package/scripts/syntax-check.mjs +36 -14
  96. package/scripts/verification-generation-guard.mjs +15 -0
  97. package/scripts/verification-idle-sleep-guard.mjs +37 -0
  98. package/scripts/verification-state.mjs +108 -0
  99. package/scripts/wrangler-command-lifecycle.mjs +115 -0
  100. package/src/local/account-access.mjs +13 -1
  101. package/src/local/account-admin-response.mjs +92 -0
  102. package/src/local/account-admin.mjs +52 -19
  103. package/src/local/agent-context-projection.mjs +26 -7
  104. package/src/local/agent-context.mjs +88 -49
  105. package/src/local/agent-skill-discovery.mjs +16 -5
  106. package/src/local/app-automation-macos-jxa.mjs +350 -0
  107. package/src/local/app-automation.mjs +873 -163
  108. package/src/local/application-capability-projection.mjs +24 -0
  109. package/src/local/authority-context.mjs +106 -0
  110. package/src/local/autostart-log-maintenance.mjs +36 -0
  111. package/src/local/browser-bridge-http.mjs +5 -3
  112. package/src/local/browser-bridge.mjs +109 -31
  113. package/src/local/browser-broker-auth-http.mjs +67 -0
  114. package/src/local/browser-broker-auth.mjs +98 -0
  115. package/src/local/browser-broker-routes.mjs +63 -20
  116. package/src/local/browser-broker-server.mjs +17 -7
  117. package/src/local/browser-command.mjs +79 -30
  118. package/src/local/browser-computer-observation-service.mjs +266 -0
  119. package/src/local/browser-extension-identity.mjs +50 -0
  120. package/src/local/browser-extension-path.mjs +22 -0
  121. package/src/local/browser-extension-protocol.mjs +50 -11
  122. package/src/local/browser-operation-service.mjs +147 -117
  123. package/src/local/browser-pairing-grant.mjs +109 -0
  124. package/src/local/browser-pairing-http.mjs +36 -0
  125. package/src/local/browser-pairing-launch.mjs +58 -0
  126. package/src/local/browser-pairing-store.mjs +85 -48
  127. package/src/local/browser-request-registry.mjs +40 -14
  128. package/src/local/browser-request-settlement.mjs +78 -0
  129. package/src/local/browser-resource-input.mjs +74 -0
  130. package/src/local/browser-trusted-input-health.mjs +63 -0
  131. package/src/local/call-authority.mjs +21 -0
  132. package/src/local/call-capacity.mjs +45 -0
  133. package/src/local/call-registry-drain.mjs +54 -0
  134. package/src/local/call-registry.mjs +30 -15
  135. package/src/local/capability-observer.mjs +5 -0
  136. package/src/local/capability-ranking.mjs +3 -2
  137. package/src/local/child-process-settlement.mjs +132 -0
  138. package/src/local/cli-account-admin.mjs +40 -6
  139. package/src/local/cli-activate.mjs +162 -0
  140. package/src/local/cli-local-admin.mjs +20 -45
  141. package/src/local/cli-options.mjs +12 -13
  142. package/src/local/cli-ready-output.mjs +52 -0
  143. package/src/local/cli-service.mjs +91 -11
  144. package/src/local/cli.mjs +232 -150
  145. package/src/local/computer-use-application-observation.mjs +224 -0
  146. package/src/local/computer-use-arguments.mjs +270 -0
  147. package/src/local/computer-use-deadline.mjs +50 -0
  148. package/src/local/computer-use-dispatch-settlement.mjs +40 -0
  149. package/src/local/computer-use-expectation.mjs +171 -0
  150. package/src/local/computer-use-observation.mjs +769 -0
  151. package/src/local/computer-use-recovery.mjs +82 -0
  152. package/src/local/computer-use-result-budget.mjs +145 -0
  153. package/src/local/computer-use-snapshot-store.mjs +67 -0
  154. package/src/local/computer-use.mjs +2255 -0
  155. package/src/local/daemon-http-relay-auth.mjs +26 -0
  156. package/src/local/daemon-http-relay-connection.mjs +298 -0
  157. package/src/local/daemon-http-relay-request.mjs +64 -0
  158. package/src/local/daemon-http-relay-sequence.mjs +49 -0
  159. package/src/local/daemon-process.mjs +40 -7
  160. package/src/local/delegated-process-sandbox.mjs +148 -0
  161. package/src/local/device-identity.mjs +156 -49
  162. package/src/local/device-root-provider.mjs +81 -0
  163. package/src/local/directory-metadata.mjs +47 -0
  164. package/src/local/doctor-reporting.mjs +23 -0
  165. package/src/local/durable-process-spec.mjs +80 -0
  166. package/src/local/errors.mjs +25 -15
  167. package/src/local/exclusive-file.mjs +74 -26
  168. package/src/local/execution-limits.mjs +9 -0
  169. package/src/local/execution-routing.mjs +276 -0
  170. package/src/local/execution-surface.mjs +23 -0
  171. package/src/local/file-mutation-coordinator.mjs +47 -0
  172. package/src/local/file-snapshot-preservation.mjs +24 -0
  173. package/src/local/filesystem-identity.mjs +54 -0
  174. package/src/local/fixed-process-environment.mjs +21 -0
  175. package/src/local/full-access-test.mjs +31 -13
  176. package/src/local/git-commit.mjs +69 -0
  177. package/src/local/git-config-safety.mjs +55 -0
  178. package/src/local/git-log-parser.mjs +41 -0
  179. package/src/local/git-metadata-boundary.mjs +77 -0
  180. package/src/local/git-metadata-tree-safety.mjs +59 -0
  181. package/src/local/git-operation-state.mjs +19 -0
  182. package/src/local/git-service.mjs +165 -30
  183. package/src/local/hardened-npm-download-timeout.mjs +46 -0
  184. package/src/local/hardened-npm-download.mjs +109 -0
  185. package/src/local/hardened-npm-extract.mjs +20 -0
  186. package/src/local/hardened-npm-verification.mjs +104 -0
  187. package/src/local/hardened-npm.mjs +199 -0
  188. package/src/local/job-runner.mjs +244 -89
  189. package/src/local/lifecycle.mjs +6 -0
  190. package/src/local/log.mjs +19 -45
  191. package/src/local/macos-background-input.mjs +398 -0
  192. package/src/local/macos-idle-sleep-assertion.mjs +65 -0
  193. package/src/local/macos-trust-broker.mjs +404 -0
  194. package/src/local/managed-job-cancellation.mjs +42 -0
  195. package/src/local/managed-job-capacity.mjs +16 -0
  196. package/src/local/managed-job-directory-generation.mjs +60 -0
  197. package/src/local/managed-job-directory.mjs +75 -0
  198. package/src/local/managed-job-durable-process.mjs +49 -0
  199. package/src/local/managed-job-lock.mjs +46 -14
  200. package/src/local/managed-job-plan.mjs +38 -14
  201. package/src/local/managed-job-projection.mjs +23 -15
  202. package/src/local/managed-job-retention.mjs +128 -0
  203. package/src/local/managed-job-runner-claim.mjs +70 -0
  204. package/src/local/managed-job-runner.mjs +35 -10
  205. package/src/local/managed-job-storage.mjs +22 -6
  206. package/src/local/managed-job-terminal-maintenance.mjs +40 -0
  207. package/src/local/managed-job-terminal.mjs +150 -0
  208. package/src/local/managed-jobs.mjs +424 -287
  209. package/src/local/npm-cli.mjs +42 -0
  210. package/src/local/npm-environment.mjs +37 -0
  211. package/src/local/operation-authorization.mjs +68 -324
  212. package/src/local/operation-risk.mjs +51 -4
  213. package/src/local/owner-state-lock.mjs +118 -0
  214. package/src/local/package-identity.mjs +9 -0
  215. package/src/local/patch.mjs +20 -18
  216. package/src/local/path-inspection.mjs +23 -0
  217. package/src/local/policy.mjs +19 -0
  218. package/src/local/private-toolchain-integrity.mjs +54 -0
  219. package/src/local/process-execution.mjs +87 -75
  220. package/src/local/process-foreground-timeout.mjs +50 -0
  221. package/src/local/process-identity.mjs +41 -7
  222. package/src/local/process-nonreplayable-settlement.mjs +50 -0
  223. package/src/local/process-result-projection.mjs +22 -0
  224. package/src/local/process-session-events.mjs +47 -0
  225. package/src/local/process-session-remote-activity.mjs +10 -0
  226. package/src/local/process-session-termination.mjs +73 -0
  227. package/src/local/process-sessions.mjs +144 -112
  228. package/src/local/process-tracker.mjs +140 -13
  229. package/src/local/process-tree-ownership-types.d.ts +37 -0
  230. package/src/local/process-tree-ownership.mjs +57 -0
  231. package/src/local/process-tree-signal.mjs +34 -0
  232. package/src/local/process-tree-snapshot.mjs +77 -0
  233. package/src/local/process-tree-supervisor.mjs +69 -0
  234. package/src/local/process-tree.mjs +11 -53
  235. package/src/local/project-metadata.mjs +19 -6
  236. package/src/local/relay-call-recovery.mjs +58 -38
  237. package/src/local/relay-connection-classification.mjs +131 -0
  238. package/src/local/relay-connection-support.mjs +91 -0
  239. package/src/local/relay-connection.mjs +179 -272
  240. package/src/local/relay-diagnostics.mjs +91 -0
  241. package/src/local/relay-heartbeat.mjs +119 -0
  242. package/src/local/relay-liveness.mjs +109 -0
  243. package/src/local/relay-peer-diagnostics.mjs +29 -0
  244. package/src/local/release-runtime-lock.mjs +18 -0
  245. package/src/local/remote-activity-idle-sleep-guard.mjs +77 -0
  246. package/src/local/remote-configuration.mjs +48 -0
  247. package/src/local/resilient-relay-connection.mjs +178 -0
  248. package/src/local/resource-admission-diagnostics.mjs +52 -0
  249. package/src/local/resource-admission-policy.mjs +115 -0
  250. package/src/local/resource-admission.mjs +339 -0
  251. package/src/local/resource-build-root.mjs +83 -0
  252. package/src/local/resource-cargo-concurrency.mjs +38 -0
  253. package/src/local/resource-cmake-concurrency.mjs +42 -0
  254. package/src/local/resource-command-concurrency.mjs +37 -0
  255. package/src/local/resource-command-profile.mjs +176 -0
  256. package/src/local/resource-coordinator-accounting.mjs +21 -0
  257. package/src/local/resource-cpu-window.mjs +20 -0
  258. package/src/local/resource-disk-headroom.mjs +13 -0
  259. package/src/local/resource-elastic-memory.mjs +19 -0
  260. package/src/local/resource-elastic-request.mjs +36 -0
  261. package/src/local/resource-foreground-wait.mjs +29 -0
  262. package/src/local/resource-go-concurrency.mjs +43 -0
  263. package/src/local/resource-gradle-concurrency.mjs +57 -0
  264. package/src/local/resource-host-cache.mjs +39 -0
  265. package/src/local/resource-host-darwin.mjs +54 -0
  266. package/src/local/resource-host-linux.mjs +35 -0
  267. package/src/local/resource-host-sample-file.mjs +23 -0
  268. package/src/local/resource-host-snapshot.mjs +99 -0
  269. package/src/local/resource-lease-accounting.mjs +119 -0
  270. package/src/local/resource-light-command.mjs +6 -0
  271. package/src/local/resource-make-concurrency.mjs +42 -0
  272. package/src/local/resource-maven-concurrency.mjs +38 -0
  273. package/src/local/resource-ninja-command-concurrency.mjs +20 -0
  274. package/src/local/resource-ninja-concurrency.mjs +28 -0
  275. package/src/local/resource-operations.mjs +26 -10
  276. package/src/local/resource-pressure.mjs +67 -0
  277. package/src/local/resource-probe-command.mjs +35 -0
  278. package/src/local/resource-process-admission.mjs +59 -0
  279. package/src/local/resource-process-ancestry-cache.mjs +43 -0
  280. package/src/local/resource-process-ancestry.mjs +32 -0
  281. package/src/local/resource-process-priority.mjs +15 -0
  282. package/src/local/resource-project-key.mjs +53 -0
  283. package/src/local/resource-pytest-concurrency.mjs +33 -0
  284. package/src/local/resource-release-control-classification.mjs +14 -0
  285. package/src/local/resource-release-control-executable.mjs +70 -0
  286. package/src/local/resource-release-control-workspace.mjs +41 -0
  287. package/src/local/resource-request-contract.mjs +30 -0
  288. package/src/local/resource-script-classification.mjs +44 -0
  289. package/src/local/resource-shell-analysis.mjs +58 -0
  290. package/src/local/resource-staging-recovery.mjs +97 -0
  291. package/src/local/resource-swift-concurrency.mjs +31 -0
  292. package/src/local/resource-transaction-lock.mjs +189 -0
  293. package/src/local/resource-wait.mjs +48 -0
  294. package/src/local/resource-waiters.mjs +139 -0
  295. package/src/local/resource-xcode-command.mjs +7 -0
  296. package/src/local/resource-xcode-concurrency.mjs +34 -0
  297. package/src/local/resource-xcode-non-build.mjs +19 -0
  298. package/src/local/runtime-activation.mjs +700 -0
  299. package/src/local/runtime-activity-projection.mjs +48 -0
  300. package/src/local/runtime-capabilities.mjs +48 -15
  301. package/src/local/runtime-diagnostic-state.mjs +14 -0
  302. package/src/local/runtime-diagnostics.mjs +40 -13
  303. package/src/local/runtime-info-projection.mjs +60 -0
  304. package/src/local/runtime-info-relay-projection.mjs +21 -0
  305. package/src/local/runtime-paths.mjs +2 -1
  306. package/src/local/runtime-process-routing.mjs +33 -0
  307. package/src/local/runtime-relay-connection-options.mjs +44 -0
  308. package/src/local/runtime-relay-control.mjs +98 -0
  309. package/src/local/runtime-relay.mjs +26 -50
  310. package/src/local/runtime-reporting.mjs +37 -21
  311. package/src/local/runtime-resource-service.mjs +77 -0
  312. package/src/local/runtime-tool-handlers.mjs +13 -10
  313. package/src/local/runtime.mjs +244 -208
  314. package/src/local/secure-file.mjs +73 -20
  315. package/src/local/security-audit-dispatch.mjs +35 -0
  316. package/src/local/security-audit-log.mjs +215 -0
  317. package/src/local/security-audit-state.mjs +139 -0
  318. package/src/local/security-audit-storage.mjs +135 -0
  319. package/src/local/security-audit-warning.mjs +31 -0
  320. package/src/local/security-audit-worker.mjs +87 -0
  321. package/src/local/service-convergence.mjs +42 -1
  322. package/src/local/service-definition.mjs +24 -0
  323. package/src/local/service-environment.mjs +18 -10
  324. package/src/local/service-owner.mjs +146 -0
  325. package/src/local/service-ownership.mjs +17 -0
  326. package/src/local/service-restart-handoff.mjs +64 -0
  327. package/src/local/service-restart-scheduler.mjs +49 -0
  328. package/src/local/service-runtime-convergence.mjs +52 -0
  329. package/src/local/service-runtime.mjs +105 -0
  330. package/src/local/service-status.mjs +60 -0
  331. package/src/local/service.mjs +385 -92
  332. package/src/local/shell.mjs +39 -19
  333. package/src/local/ssh-key.mjs +158 -42
  334. package/src/local/state-inventory.mjs +50 -24
  335. package/src/local/state-owner-lock-inventory.mjs +23 -0
  336. package/src/local/state-root-owned-namespaces.mjs +83 -0
  337. package/src/local/state-root-retirement.mjs +89 -0
  338. package/src/local/state.mjs +386 -96
  339. package/src/local/stdio.mjs +96 -66
  340. package/src/local/support-state-projection.mjs +30 -0
  341. package/src/local/system-network-route.mjs +76 -0
  342. package/src/local/systemd-removal.mjs +20 -0
  343. package/src/local/tool-executor.mjs +85 -20
  344. package/src/local/tool-result-boundary.mjs +51 -0
  345. package/src/local/toolchain-operation-lock.mjs +41 -0
  346. package/src/local/tools.mjs +7 -5
  347. package/src/local/trusted-executable.mjs +53 -0
  348. package/src/local/trusted-git-executable.mjs +34 -0
  349. package/src/local/trusted-github-cli.mjs +24 -0
  350. package/src/local/windows-service-convergence.mjs +49 -0
  351. package/src/local/windows-service.mjs +52 -45
  352. package/src/local/worker-deployment-fingerprint.mjs +102 -0
  353. package/src/local/worker-deployment.mjs +48 -60
  354. package/src/local/worker-health.mjs +12 -4
  355. package/src/local/worker-secret-file.mjs +37 -17
  356. package/src/local/workspace-file-service.mjs +156 -239
  357. package/src/local/workspace-file-transaction.mjs +189 -0
  358. package/src/local/workspace-search.mjs +55 -0
  359. package/src/local/wrangler-toolchain/package-lock.json +1505 -0
  360. package/src/local/wrangler-toolchain/package.json +23 -0
  361. package/src/local/wrangler-toolchain-verification.mjs +135 -0
  362. package/src/local/wrangler-toolchain.mjs +188 -0
  363. package/src/shared/access-contract.json +7 -1
  364. package/src/shared/activation-recovery.mjs +49 -0
  365. package/src/shared/admin-auth.d.mts +2 -1
  366. package/src/shared/admin-auth.mjs +3 -3
  367. package/src/shared/authority-revocation.d.mts +9 -0
  368. package/src/shared/authority-revocation.mjs +28 -0
  369. package/src/shared/daemon-auth.d.mts +11 -0
  370. package/src/shared/daemon-auth.mjs +17 -0
  371. package/src/shared/device-session-auth.d.mts +20 -0
  372. package/src/shared/device-session-auth.mjs +51 -0
  373. package/src/shared/foreground-timeout.d.mts +15 -0
  374. package/src/shared/foreground-timeout.mjs +70 -0
  375. package/src/shared/log-redaction.d.mts +9 -0
  376. package/src/shared/log-redaction.mjs +51 -0
  377. package/src/shared/mcp-protocol.d.mts +27 -0
  378. package/src/shared/mcp-protocol.mjs +275 -0
  379. package/src/shared/project-overview-projection.d.mts +4 -0
  380. package/src/shared/project-overview-projection.mjs +96 -0
  381. package/src/shared/relay-contract.json +28 -0
  382. package/src/shared/result-projection.d.mts +2 -1
  383. package/src/shared/result-projection.mjs +13 -2
  384. package/src/shared/server-metadata.json +15 -9
  385. package/src/shared/tool-argument-validation.d.mts +17 -0
  386. package/src/shared/tool-argument-validation.mjs +325 -0
  387. package/src/shared/tool-call-capacity.d.mts +41 -0
  388. package/src/shared/tool-call-capacity.mjs +59 -0
  389. package/src/shared/tool-catalog.json +626 -57
  390. package/src/worker/access.ts +12 -0
  391. package/src/worker/account-admin.ts +55 -33
  392. package/src/worker/authority-revocations.ts +174 -0
  393. package/src/worker/authority.ts +52 -6
  394. package/src/worker/daemon-auth.ts +49 -68
  395. package/src/worker/daemon-channel.ts +36 -0
  396. package/src/worker/daemon-http-auth.ts +63 -0
  397. package/src/worker/daemon-http-channel.ts +120 -0
  398. package/src/worker/daemon-http-controller.ts +163 -0
  399. package/src/worker/daemon-http-protocol.ts +62 -0
  400. package/src/worker/daemon-http-registry.ts +82 -0
  401. package/src/worker/daemon-last-observation.ts +30 -0
  402. package/src/worker/daemon-liveness.ts +8 -6
  403. package/src/worker/daemon-ready-dispatch.ts +17 -0
  404. package/src/worker/daemon-ready-messages.ts +73 -0
  405. package/src/worker/daemon-ready-waiters.ts +79 -0
  406. package/src/worker/daemon-recovery-budget.ts +13 -0
  407. package/src/worker/daemon-registry.ts +58 -0
  408. package/src/worker/daemon-relay-diagnostics.ts +127 -0
  409. package/src/worker/daemon-socket-attachment.ts +55 -0
  410. package/src/worker/daemon-sockets.ts +72 -62
  411. package/src/worker/daemon-status.ts +38 -0
  412. package/src/worker/device-session-verifier.ts +129 -0
  413. package/src/worker/dpop.ts +157 -0
  414. package/src/worker/durable-process-timeout.ts +29 -0
  415. package/src/worker/errors.ts +57 -0
  416. package/src/worker/http.ts +112 -26
  417. package/src/worker/index.ts +446 -440
  418. package/src/worker/mcp-access.ts +57 -0
  419. package/src/worker/mcp-controller.ts +188 -0
  420. package/src/worker/mcp-http-accept.ts +11 -0
  421. package/src/worker/mcp-http-contract.ts +322 -0
  422. package/src/worker/mcp-initialization-compat.ts +141 -0
  423. package/src/worker/mcp-jsonrpc.ts +11 -29
  424. package/src/worker/mcp-removed-protocol.ts +17 -0
  425. package/src/worker/mcp-response-proxy.ts +135 -0
  426. package/src/worker/mcp-response-stream.ts +65 -0
  427. package/src/worker/mcp-stale-schema-compat.ts +57 -0
  428. package/src/worker/mcp-stream-proxy-contract.ts +48 -0
  429. package/src/worker/mcp-tool-call-input.ts +23 -0
  430. package/src/worker/nonce-store.ts +33 -35
  431. package/src/worker/oauth-authorization-page.ts +21 -1
  432. package/src/worker/oauth-client-admin.ts +52 -0
  433. package/src/worker/oauth-client-contract.ts +38 -0
  434. package/src/worker/oauth-controller.ts +107 -47
  435. package/src/worker/oauth-field-contract.ts +31 -0
  436. package/src/worker/oauth-record-contract.ts +7 -0
  437. package/src/worker/oauth-refresh-authority.ts +27 -0
  438. package/src/worker/oauth-refresh-exchange.ts +162 -0
  439. package/src/worker/oauth-refresh-families.ts +130 -27
  440. package/src/worker/oauth-refresh-persistence.ts +116 -0
  441. package/src/worker/oauth-state.ts +33 -18
  442. package/src/worker/oauth-store-validation.ts +131 -0
  443. package/src/worker/oauth-token-derivation.ts +33 -0
  444. package/src/worker/oauth-token-issuance.ts +119 -0
  445. package/src/worker/oauth-tokens.ts +56 -190
  446. package/src/worker/observability.ts +38 -5
  447. package/src/worker/pending-admission.ts +15 -0
  448. package/src/worker/pending-call-capacity.ts +108 -0
  449. package/src/worker/pending-call-contract.ts +42 -5
  450. package/src/worker/pending-call-deadlines.ts +75 -0
  451. package/src/worker/pending-calls.ts +131 -125
  452. package/src/worker/runtime-alarm-storage.ts +37 -0
  453. package/src/worker/runtime-alarm.ts +136 -0
  454. package/src/worker/server-info-activity.ts +36 -0
  455. package/src/worker/server-info-tool-delivery.ts +15 -0
  456. package/src/worker/server-info.ts +168 -0
  457. package/src/worker/tool-catalog.ts +47 -1
  458. package/src/worker/tool-timeout.ts +75 -14
  459. package/src/worker/websocket-protocol.ts +26 -2
  460. package/src/worker/worker-edge-guard.ts +90 -0
  461. package/src/worker/worker-edge-log.ts +65 -0
  462. package/src/worker/worker-entry.ts +52 -0
  463. package/src/worker/worker-mcp-config.ts +23 -0
  464. package/src/worker/worker-metadata.ts +48 -0
  465. package/src/worker/worker-rate-limit-key.ts +30 -0
  466. package/src/worker/worker-runtime-config.ts +20 -0
  467. package/src/worker/worker-static-routes.ts +54 -0
  468. package/src/worker/worker-task-lifetime.ts +18 -0
  469. package/tsconfig.local.json +17 -1
  470. package/wrangler.jsonc +13 -0
  471. package/src/local/cli-approval.mjs +0 -117
  472. package/src/local/operation-state-lock.mjs +0 -92
  473. package/src/worker/mcp-session.ts +0 -72
@@ -0,0 +1,2255 @@
1
+ import { createHash, randomBytes } from "node:crypto";
2
+ import { performance } from "node:perf_hooks";
3
+ import { BridgeError, errorCode } from "./errors.mjs";
4
+ import {
5
+ browserScrollDelta, clampInt, computerScreenshotFormat, normalizeApplicationAction, normalizeBrowserAction,
6
+ normalizeInputMode, normalizeNavigationWait, normalizePostObservationDetail, normalizePostScreenshotPolicy,
7
+ optionalApplicationFocusQuery, optionalBoolean, optionalPositiveInt, requiredResource, requiredSnapshotId,
8
+ requiredString, requiredStringAllowEmpty, requiredSurface, requiredTargetRef, shouldIncludePostScreenshot,
9
+ validateActionDispatchArguments, validateDragTargets, validateObserveArgs, validateSurfaceActionArgs,
10
+ } from "./computer-use-arguments.mjs";
11
+ import { applicationStateActionTargetSupported, applicationVerificationTarget, normalizeExpectation, validateApplicationStateActionTarget, validateExpectationPrerequisites } from "./computer-use-expectation.mjs";
12
+ import { ComputerUseSnapshotStore } from "./computer-use-snapshot-store.mjs";
13
+ import { computerActPostObservationTimeoutError, computerActRemainingTimeoutSeconds, computerActVerificationTimeoutProbe, publicPostObservationError, requiredComputerActRemainingTimeoutSeconds, requiredComputerObserveRemainingTimeoutSeconds } from "./computer-use-deadline.mjs";
14
+ import { settleComputerUseDispatch } from "./computer-use-dispatch-settlement.mjs";
15
+ import { createMonotonicDeadline } from "./monotonic-deadline.mjs";
16
+ import { buildBrowserObservation, buildContinuation, extractBrowserPrivateBindings, observationDiff, projectPostObservation } from "./computer-use-observation.mjs";
17
+ import { buildRetryGuidance } from "./computer-use-recovery.mjs";
18
+ import { applicationElementSupportsPointClick, applicationMatchesSelector, prepareApplicationObservationElements } from "./computer-use-application-observation.mjs";
19
+ import {
20
+ assertObservationResultFits, fitActionResultToBudget, isResultLimitExceeded, RESULT_BUDGET_OMISSION_REASON,
21
+ observationResult, omitApplicationScreenshotForResultBudget, omitBrowserScreenshotForResultBudget,
22
+ } from "./computer-use-result-budget.mjs";
23
+
24
+ const BROWSER_TARGETLESS_ACTIONS = new Set(["navigate", "reload", "back", "forward"]);
25
+ const BROWSER_DOCUMENT_BOUND_TARGETLESS_ACTIONS = new Set(["reload", "back", "forward"]);
26
+ const BROWSER_POINT_ACTIONS = new Set(["click", "double_click", "hover", "drag", "scroll"]);
27
+ const BROWSER_BACKEND_TRUSTED_ACTIONS = new Set(["click", "double_click", "hover", "press", "type_text", "fill", "check", "uncheck", "submit"]);
28
+ const BROWSER_BACKEND_TEXT_ACTIONS = new Set(["type_text", "fill"]);
29
+ const BROWSER_SNAPSHOT_IDENTITY_FIELDS = Object.freeze([
30
+ "tag", "type", "role", "name", "id", "field_name", "label", "placeholder", "href", "sensitive", "in_shadow_dom",
31
+ ]);
32
+ const BROWSER_SNAPSHOT_BOOLEAN_IDENTITY_FIELDS = new Set(["sensitive", "in_shadow_dom"]);
33
+ const APPLICATION_VERIFY_POLL_MS = 100;
34
+ const APPLICATION_VERIFY_MAX_CAPTURES = 9;
35
+ const MAX_APPLICATION_OBSERVATION_ELEMENTS = 500;
36
+ const MAX_APPLICATION_SCREENSHOT_BYTES = 32 * 1024 * 1024;
37
+
38
+ export class ComputerUseManager {
39
+ constructor({
40
+ authorizeTool,
41
+ browserBridgeManager,
42
+ appAutomationManager,
43
+ throwIfCancelled = () => {},
44
+ now = () => performance.now(),
45
+ sleep = defaultSleep,
46
+ createId = () => `cu_${randomBytes(18).toString("base64url")}`,
47
+ }) {
48
+ this.authorizeTool = authorizeTool;
49
+ this.browser = browserBridgeManager;
50
+ this.applications = appAutomationManager;
51
+ this.throwIfCancelled = throwIfCancelled;
52
+ this.now = now;
53
+ this.sleep = sleep;
54
+ this.snapshots = new ComputerUseSnapshotStore({ now, createId });
55
+ }
56
+
57
+ async observe(args = {}, context = {}) {
58
+ this.authorizeTool("computer_observe");
59
+ const capture = await this.capture(args, context);
60
+ return observationResult(capture.observation, capture.imageContent);
61
+ }
62
+
63
+ async act(args = {}, context = {}) {
64
+ this.authorizeTool("computer_act");
65
+ const surface = requiredSurface(args.surface);
66
+ const snapshotId = requiredSnapshotId(args.snapshot_id);
67
+ const snapshot = this.snapshots.get(snapshotId);
68
+ if (snapshot.observation.surface !== surface) {
69
+ throw new BridgeError("conflict", "computer snapshot surface does not match this action", {
70
+ details: { reason: "snapshot_surface_mismatch", expected: snapshot.observation.surface, received: surface },
71
+ });
72
+ }
73
+
74
+ const action = surface === "browser" ? normalizeBrowserAction(args.action) : normalizeApplicationAction(args.action);
75
+ validateSurfaceActionArgs(surface, action, args);
76
+ const actionTimeoutSeconds = clampInt(args.timeout_seconds, 30, 1, 60);
77
+ const actionDeadline = createMonotonicDeadline(actionTimeoutSeconds * 1000, this.now);
78
+ const target = this.resolveTarget(snapshot.observation, snapshot.privateState, surface, action, args.target);
79
+ const destination = action === "drag"
80
+ ? this.resolveTarget(snapshot.observation, snapshot.privateState, surface, action, args.destination)
81
+ : null;
82
+ if (action === "drag") validateDragTargets(surface, target, destination);
83
+ validateActionDispatchArguments(surface, action, target, args);
84
+ const expectation = normalizeExpectation(surface, action, target, args.expect);
85
+ const verifyTimeoutSeconds = clampInt(args.verify_timeout_seconds, 5, 1, 60);
86
+ const postScreenshotPolicy = normalizePostScreenshotPolicy(args, surface);
87
+ validateExpectationPrerequisites(expectation, snapshot.observation, postScreenshotPolicy);
88
+ const postObservationDetail = normalizePostObservationDetail(args.post_observation_detail, surface);
89
+ await this.preflight(snapshot.observation, snapshot.privateState, surface, action, target, context, actionDeadline);
90
+ if (destination) await this.preflight(snapshot.observation, snapshot.privateState, surface, action, destination, context, actionDeadline);
91
+ const dispatchTimeoutSeconds = requiredComputerActRemainingTimeoutSeconds(actionDeadline, actionTimeoutSeconds);
92
+ this.snapshots.claim(snapshotId, snapshot);
93
+
94
+ const dispatchArgs = { ...args, timeout_seconds: dispatchTimeoutSeconds };
95
+ const { dispatchStatus, dispatchResult, dispatchError } = await settleComputerUseDispatch(surface, async () => (
96
+ surface === "browser"
97
+ ? await this.dispatchBrowser(snapshot.observation, snapshot.privateState, action, target, dispatchArgs, context, destination)
98
+ : await this.dispatchApplication(snapshot.observation, snapshot.privateState, action, target, dispatchArgs, context, destination)
99
+ ));
100
+
101
+ let verificationProbe = { requested: Boolean(expectation), matched: false, reason: expectation ? "not_checked" : "not_requested" };
102
+ if (surface === "browser" && expectation) {
103
+ const browserVerifyTimeoutSeconds = computerActRemainingTimeoutSeconds(actionDeadline, verifyTimeoutSeconds);
104
+ verificationProbe = browserVerifyTimeoutSeconds > 0
105
+ ? await this.verifyBrowserExpectation(
106
+ snapshot.observation,
107
+ target,
108
+ expectation,
109
+ browserVerifyTimeoutSeconds,
110
+ context,
111
+ )
112
+ : computerActVerificationTimeoutProbe();
113
+ }
114
+
115
+ const includePostScreenshot = shouldIncludePostScreenshot(postScreenshotPolicy, { surface, target, expectation, dispatchStatus });
116
+ const initialPostCaptureTimeoutSeconds = computerActRemainingTimeoutSeconds(actionDeadline, actionTimeoutSeconds);
117
+ const applicationPostArgs = surface === "application" ? {
118
+ surface,
119
+ application: snapshot.observation.target.application,
120
+ include_screenshot: includePostScreenshot,
121
+ max_elements: clampInt(args.post_max_elements, 200, 1, 500),
122
+ max_depth: clampInt(args.post_max_depth, 6, 1, 12),
123
+ include_values: false,
124
+ include_menus: args.include_menus === true,
125
+ focus_query: applicationPostFocusQuery(target),
126
+ timeout_seconds: Math.max(1, initialPostCaptureTimeoutSeconds),
127
+ } : null;
128
+ let postCapture = null;
129
+ let postCaptureError = "";
130
+ let observedDiff = null;
131
+ if (surface === "application" && expectation) {
132
+ const applicationVerifyTimeoutSeconds = computerActRemainingTimeoutSeconds(actionDeadline, verifyTimeoutSeconds);
133
+ if (applicationVerifyTimeoutSeconds > 0) {
134
+ const verified = await this.verifyApplicationPostAction({
135
+ beforeObservation: snapshot.observation,
136
+ beforePrivateState: snapshot.privateState,
137
+ target,
138
+ expectation,
139
+ args,
140
+ dispatchResult,
141
+ captureArgs: applicationPostArgs,
142
+ timeoutSeconds: applicationVerifyTimeoutSeconds,
143
+ operationDeadline: actionDeadline,
144
+ }, context);
145
+ postCapture = verified.postCapture;
146
+ postCaptureError = verified.postCaptureError;
147
+ observedDiff = verified.observedDiff;
148
+ verificationProbe = verified.verificationProbe;
149
+ } else {
150
+ verificationProbe = computerActVerificationTimeoutProbe();
151
+ postCaptureError = computerActPostObservationTimeoutError(surface);
152
+ }
153
+ } else {
154
+ const postCaptureTimeoutSeconds = computerActRemainingTimeoutSeconds(actionDeadline, actionTimeoutSeconds);
155
+ if (postCaptureTimeoutSeconds > 0) {
156
+ try {
157
+ postCapture = await this.capture(surface === "browser" ? {
158
+ surface,
159
+ tab_id: snapshot.observation.target.tab_id,
160
+ include_screenshot: includePostScreenshot,
161
+ max_elements: clampInt(args.post_max_elements, 180, 1, 1000),
162
+ max_ax_nodes: clampInt(args.post_max_ax_nodes, 180, 1, 2000),
163
+ max_frames: 16,
164
+ ax_depth: 10,
165
+ include_values: false,
166
+ focus_query: browserPostFocusQuery(target),
167
+ timeout_seconds: postCaptureTimeoutSeconds,
168
+ } : { ...applicationPostArgs, timeout_seconds: postCaptureTimeoutSeconds }, context, surface === "application" ? actionDeadline : null);
169
+ } catch (error) {
170
+ postCaptureError = publicPostObservationError(surface, error);
171
+ }
172
+ } else {
173
+ postCaptureError = computerActPostObservationTimeoutError(surface);
174
+ }
175
+ observedDiff = postCapture
176
+ ? observationDiff(snapshot.observation, postCapture.observation, snapshot.privateState, postCapture.privateState)
177
+ : null;
178
+ if (surface === "browser" && expectation && postCapture) {
179
+ verificationProbe = combineBrowserPostChecks(
180
+ verificationProbe,
181
+ observedDiff,
182
+ expectation,
183
+ snapshot.observation,
184
+ postCapture.observation,
185
+ target,
186
+ );
187
+ }
188
+ }
189
+
190
+ const effectStatus = classifyEffectStatus({ dispatchStatus, expectation, verificationProbe, postCapture });
191
+ const continuation = buildContinuation(
192
+ snapshot.observation,
193
+ postCapture?.observation || null,
194
+ target,
195
+ observedDiff,
196
+ snapshot.privateState,
197
+ postCapture?.privateState || null,
198
+ );
199
+ const retryGuidance = buildRetryGuidance({
200
+ dispatchStatus,
201
+ effectStatus,
202
+ expectationRequested: Boolean(expectation),
203
+ postObservation: postCapture?.observation || null,
204
+ continuation,
205
+ });
206
+ const postScreenshotOmittedForBudget = postCapture?.observation?.capture?.screenshot_omitted_reason === RESULT_BUDGET_OMISSION_REASON;
207
+ const result = {
208
+ surface,
209
+ action,
210
+ snapshot_id: snapshotId,
211
+ dispatch_status: dispatchStatus,
212
+ effect_status: effectStatus,
213
+ verification: verificationProbe,
214
+ observed_diff: observedDiff,
215
+ post_snapshot_id: postCapture?.observation.snapshot_id || null,
216
+ post_observation_detail: postObservationDetail,
217
+ post_screenshot_policy: postScreenshotPolicy,
218
+ post_screenshot_included: Boolean(postCapture?.imageContent?.length),
219
+ ...(postScreenshotOmittedForBudget ? {
220
+ post_screenshot_omitted_reason: RESULT_BUDGET_OMISSION_REASON,
221
+ result_budget_compacted: true,
222
+ } : {}),
223
+ post_observation: postCapture ? projectPostObservation(postCapture.observation, postObservationDetail) : null,
224
+ continuation,
225
+ retry_guidance: retryGuidance,
226
+ dispatch: sanitizeDispatchResult(surface, dispatchResult),
227
+ ...(dispatchError ? { dispatch_error: dispatchError } : {}),
228
+ ...(postCaptureError ? { post_observation_error: postCaptureError } : {}),
229
+ recovery: retryGuidance.message,
230
+ };
231
+ return fitActionResultToBudget({
232
+ result,
233
+ imageContent: postCapture?.imageContent || [],
234
+ beforeObservation: snapshot.observation,
235
+ beforePrivateState: snapshot.privateState,
236
+ postCapture,
237
+ target,
238
+ effectStatus,
239
+ expectationRequested: Boolean(expectation),
240
+ postObservationDetail,
241
+ });
242
+ }
243
+
244
+ async capture(args, context, operationDeadline = null) {
245
+ const surface = requiredSurface(args.surface);
246
+ validateObserveArgs(surface, args);
247
+ if (surface === "browser") return this.captureBrowser(args, context);
248
+ return this.captureApplication(args, context, operationDeadline);
249
+ }
250
+
251
+ async captureApplicationScreenshot({ application, screenshotRequested, timeoutSeconds, captureDeadline }, context) {
252
+ let screenshot = null;
253
+ let screenshotError = "";
254
+ if (screenshotRequested && typeof this.applications.captureApplication === "function") {
255
+ try {
256
+ screenshot = await this.applications.captureApplication({
257
+ application,
258
+ timeout_seconds: requiredComputerObserveRemainingTimeoutSeconds(captureDeadline, timeoutSeconds),
259
+ }, context);
260
+ } catch (error) {
261
+ screenshotError = applicationScreenshotError(error);
262
+ }
263
+ }
264
+ const screenshotPayload = screenshot?.screenshot;
265
+ const screenshotBytes = screenshotPayload?.mime_type === "image/png" ? applicationScreenshotBytes(screenshotPayload?.data) : null;
266
+ const image = screenshotBytes ? { type: "image", data: screenshotPayload.data, mimeType: "image/png" } : null;
267
+ const screenshotSource = image && typeof screenshotPayload?.source === "string" ? screenshotPayload.source : image ? "unknown" : "none";
268
+ const screenshotSha256 = screenshotBytes ? createHash("sha256").update(screenshotBytes).digest("hex") : "";
269
+ return {
270
+ screenshot,
271
+ screenshotError,
272
+ image,
273
+ screenshotSource,
274
+ screenshotSha256,
275
+ windowBinding: applicationWindowBinding(screenshot, screenshotSha256),
276
+ screenshotProcessId: optionalPrivateApplicationProcessId(screenshot?._machine_process_id),
277
+ screenshotProcessGeneration: optionalPrivateApplicationProcessGeneration(screenshot?._machine_process_generation),
278
+ };
279
+ }
280
+
281
+ async captureBrowser(args, context) {
282
+ const captured = await this.browser.observeComputer(browserObservationArgs(args), context);
283
+ this.throwIfCancelled(context);
284
+ const privateState = extractBrowserPrivateBindings(captured);
285
+ validateBrowserObservationForSnapshot(captured);
286
+ const observation = buildBrowserObservation(captured, args);
287
+ let imageContent = Array.isArray(captured.imageContent) ? captured.imageContent : [];
288
+ try {
289
+ assertObservationResultFits(observation, imageContent);
290
+ } catch (error) {
291
+ if (!imageContent.length || !isResultLimitExceeded(error)) throw error;
292
+ omitBrowserScreenshotForResultBudget(observation);
293
+ imageContent = [];
294
+ assertObservationResultFits(observation, imageContent);
295
+ }
296
+ observation.snapshot_id = this.snapshots.add(observation, privateState);
297
+ return { observation, privateState, imageContent };
298
+ }
299
+
300
+ async captureApplication(args, context, operationDeadline = null) {
301
+ const application = requiredString(args.application, "application", 300);
302
+ const maxElements = clampInt(args.max_elements, 200, 1, 500);
303
+ const maxDepth = clampInt(args.max_depth, 6, 1, 12);
304
+ const includeValues = optionalBoolean(args.include_values, "include_values", false);
305
+ const includeMenus = optionalBoolean(args.include_menus, "include_menus", false);
306
+ const focusQuery = optionalApplicationFocusQuery(args.focus_query);
307
+ const timeoutSeconds = clampInt(args.timeout_seconds, 30, 1, 60);
308
+ const captureDeadline = operationDeadline || createMonotonicDeadline(timeoutSeconds * 1000, this.now);
309
+ const screenshotRequested = optionalBoolean(args.include_screenshot, "include_screenshot", true);
310
+ const {
311
+ screenshotError, image, screenshotSource, screenshotSha256, windowBinding,
312
+ screenshotProcessId, screenshotProcessGeneration,
313
+ } = await this.captureApplicationScreenshot({ application, screenshotRequested, timeoutSeconds, captureDeadline }, context);
314
+ const inspected = await this.applications.inspectApplication({
315
+ application,
316
+ include_process_id: true,
317
+ ...(screenshotProcessId ? { expected_process_id: screenshotProcessId } : {}),
318
+ ...(screenshotProcessGeneration ? { expected_process_generation: screenshotProcessGeneration } : {}),
319
+ max_depth: maxDepth,
320
+ max_elements: focusQuery ? 500 : maxElements,
321
+ include_values: includeValues,
322
+ include_menus: includeMenus,
323
+ include_geometry: true,
324
+ include_window_state: Boolean(windowBinding),
325
+ timeout_seconds: requiredComputerObserveRemainingTimeoutSeconds(captureDeadline, timeoutSeconds),
326
+ }, context);
327
+ validateApplicationInspectionEvidence(inspected);
328
+ const applicationProcessId = requiredPrivateApplicationProcessId(inspected?._machine_process_id);
329
+ const applicationProcessGeneration = requiredPrivateApplicationProcessGeneration(inspected?._machine_process_generation);
330
+ if (screenshotProcessId && screenshotProcessId !== applicationProcessId) {
331
+ throw staleSnapshot("application process changed between screenshot and Accessibility inspection");
332
+ }
333
+ if (screenshotProcessGeneration && screenshotProcessGeneration !== applicationProcessGeneration) {
334
+ throw staleSnapshot("application process generation changed between screenshot and Accessibility inspection");
335
+ }
336
+ const { windowCoherent, windowRevalidationError } = await this.revalidateApplicationWindow({
337
+ application,
338
+ windowBinding,
339
+ inspected,
340
+ applicationProcessId,
341
+ applicationProcessGeneration,
342
+ timeoutSeconds,
343
+ captureDeadline,
344
+ context,
345
+ });
346
+ const visualPointCapability = typeof this.applications.visualPointCapability === "function"
347
+ ? this.applications.visualPointCapability()
348
+ : { available: typeof this.applications.pointApplication === "function", backend: "test-or-legacy", experimental: false };
349
+ const prepared = prepareApplicationObservationElements(inspected.elements, {
350
+ maxElements,
351
+ focusQuery,
352
+ windowBounds: windowCoherent === true ? windowBinding?.bounds || null : null,
353
+ requireWindowOwnership: windowCoherent === true,
354
+ sourceTruncated: inspected.truncated === true,
355
+ });
356
+ const semanticPointCoverageComplete = inspected.truncated === false && prepared.truncated === false;
357
+ const semanticPointGeometry = windowCoherent === true && semanticPointCoverageComplete && prepared.elements.some((element) => (
358
+ element?.bounding_box && prepared.bindings.has(element.ref) && applicationElementSupportsPointClick(element)
359
+ ));
360
+ const captureCoherence = applicationCaptureCoherence({ screenshotRequested, hasImage: Boolean(image), windowCoherent });
361
+ const observation = {
362
+ snapshot_id: "",
363
+ surface: "application",
364
+ captured_at: new Date().toISOString(),
365
+ target: {
366
+ application,
367
+ process_name: inspected.process_name,
368
+ frontmost: inspected.frontmost === true,
369
+ },
370
+ capture: {
371
+ semantic: true,
372
+ screenshot_requested: screenshotRequested,
373
+ screenshot: Boolean(image),
374
+ atomic: !image,
375
+ coherence: captureCoherence,
376
+ window_coherent: windowCoherent,
377
+ screenshot_source: screenshotSource,
378
+ screenshot_sha256: screenshotSha256,
379
+ ...(screenshotError ? { screenshot_error: screenshotError } : {}),
380
+ ...(windowRevalidationError ? { window_revalidation_error: windowRevalidationError } : {}),
381
+ },
382
+ capabilities: {
383
+ snapshot_bound_refs: true,
384
+ stable_backend_refs: false,
385
+ screenshot: typeof this.applications.captureApplication === "function",
386
+ action_verification: true,
387
+ snapshot_bound_visual_points: screenshotSource === "macos_window" && Boolean(windowBinding) && windowCoherent !== false && visualPointCapability.available === true,
388
+ snapshot_bound_semantic_points: semanticPointGeometry,
389
+ accessibility_geometry: prepared.elements.some((element) => element.bounding_box),
390
+ accessibility_geometry_coherent: windowCoherent === true,
391
+ visual_grounding: false,
392
+ visual_point_backend: visualPointCapability.backend || "disabled",
393
+ visual_point_configured: visualPointCapability.configured === true,
394
+ visual_point_probed: visualPointCapability.probed === true,
395
+ visual_point_experimental: visualPointCapability.experimental === true,
396
+ visual_point_error_class: typeof visualPointCapability.error_class === "string"
397
+ ? visualPointCapability.error_class.slice(0, 200) : "",
398
+ },
399
+ semantic: {
400
+ kind: "accessibility",
401
+ elements: prepared.elements,
402
+ truncated: inspected.truncated === true || prepared.truncated,
403
+ menus_included: inspected.menus_included === true,
404
+ selection: prepared.selection,
405
+ },
406
+ };
407
+ const privateState = {
408
+ application_process_id: applicationProcessId,
409
+ application_process_generation: applicationProcessGeneration,
410
+ application_ref_bindings: prepared.bindings,
411
+ application_inspection: { max_depth: maxDepth, include_menus: includeMenus },
412
+ };
413
+ if (windowBinding) privateState.application_window_binding = windowBinding;
414
+ let imageContent = image ? [image] : [];
415
+ try {
416
+ assertObservationResultFits(observation, imageContent);
417
+ } catch (error) {
418
+ if (!imageContent.length || !isResultLimitExceeded(error)) throw error;
419
+ omitApplicationScreenshotForResultBudget(observation);
420
+ imageContent = [];
421
+ assertObservationResultFits(observation, imageContent);
422
+ }
423
+ this.throwIfCancelled(context);
424
+ observation.snapshot_id = this.snapshots.add(observation, privateState);
425
+ return { observation, privateState, imageContent };
426
+ }
427
+
428
+ async revalidateApplicationWindow({
429
+ application, windowBinding, inspected, applicationProcessId, applicationProcessGeneration, timeoutSeconds, captureDeadline, context,
430
+ }) {
431
+ if (!windowBinding) return { windowCoherent: null, windowRevalidationError: "" };
432
+ if (inspected?._machine_window_state_checked === true) {
433
+ const currentWindow = inspected?._machine_window || null;
434
+ if (!currentWindow) {
435
+ return { windowCoherent: null, windowRevalidationError: "application window identity could not be revalidated during capture" };
436
+ }
437
+ const currentProcessId = optionalPrivateApplicationProcessId(currentWindow.process_id);
438
+ const currentProcessGeneration = optionalPrivateApplicationProcessGeneration(currentWindow.process_generation);
439
+ if (currentProcessId !== applicationProcessId || currentProcessGeneration !== applicationProcessGeneration) {
440
+ throw staleSnapshot("application process instance changed during Accessibility window revalidation");
441
+ }
442
+ return { windowCoherent: sameApplicationWindowIdentity(windowBinding, currentWindow), windowRevalidationError: "" };
443
+ }
444
+ if (typeof this.applications.inspectApplicationWindow !== "function") {
445
+ return { windowCoherent: null, windowRevalidationError: "" };
446
+ }
447
+ const revalidationTimeoutSeconds = requiredComputerObserveRemainingTimeoutSeconds(captureDeadline, timeoutSeconds);
448
+ try {
449
+ const currentWindow = await this.applications.inspectApplicationWindow({
450
+ application,
451
+ expected_process_id: applicationProcessId,
452
+ expected_process_generation: applicationProcessGeneration,
453
+ timeout_seconds: revalidationTimeoutSeconds,
454
+ }, context);
455
+ return { windowCoherent: sameApplicationWindowIdentity(windowBinding, currentWindow), windowRevalidationError: "" };
456
+ } catch (error) {
457
+ return { windowCoherent: null, windowRevalidationError: applicationWindowRevalidationError(error) };
458
+ }
459
+ }
460
+
461
+ resolveTarget(observation, privateState, surface, action, rawTarget) {
462
+ if (surface === "browser") {
463
+ if (BROWSER_TARGETLESS_ACTIONS.has(action)) {
464
+ if (rawTarget !== undefined && rawTarget !== null) throw new BridgeError("invalid_request", `${action} does not accept a target`);
465
+ return null;
466
+ }
467
+ if (isVisualPointTarget(rawTarget)) return browserPointTarget(observation, action, rawTarget);
468
+ const ref = requiredTargetRef(rawTarget);
469
+ const matches = [];
470
+ for (const frame of observation.semantic.frames || []) {
471
+ for (const element of frame.elements || []) if (element?.ref === ref) matches.push({ frame_id: frame.frame_id, element });
472
+ }
473
+ if (matches.length !== 1) {
474
+ throw new BridgeError("conflict", "target ref does not belong uniquely to this computer snapshot", {
475
+ details: { reason: matches.length ? "ambiguous_snapshot_ref" : "unknown_snapshot_ref", ref },
476
+ });
477
+ }
478
+ return {
479
+ kind: "ref",
480
+ ref,
481
+ frame_id: matches[0].frame_id,
482
+ element: matches[0].element,
483
+ trusted_binding: privateState?.browser_ref_bindings?.get?.(ref) || null,
484
+ };
485
+ }
486
+
487
+ if (action === "activate") {
488
+ if (rawTarget !== undefined && rawTarget !== null) throw new BridgeError("invalid_request", "activate does not accept a target");
489
+ return null;
490
+ }
491
+ if (isVisualPointTarget(rawTarget)) return applicationPointTarget(observation, privateState, action, rawTarget);
492
+ if (action === "scroll") {
493
+ throw new BridgeError("invalid_request", "application scroll requires a normalized point anchor from the same window snapshot");
494
+ }
495
+ if (action === "double_click") {
496
+ throw new BridgeError("invalid_request", "application double_click requires a normalized point from the same window snapshot");
497
+ }
498
+ const ref = requiredTargetRef(rawTarget);
499
+ const element = (observation.semantic.elements || []).find((candidate) => candidate?.ref === ref) || null;
500
+ const binding = privateState?.application_ref_bindings?.get?.(ref) || null;
501
+ if (!element || !binding) {
502
+ throw new BridgeError("conflict", "target ref does not belong to an addressable element in this computer snapshot", {
503
+ details: { reason: element ? "unaddressable_accessibility_element" : "unknown_snapshot_ref", ref },
504
+ });
505
+ }
506
+ validateApplicationStateActionTarget(action, element);
507
+ return {
508
+ kind: "ref", ref, element, selector: binding.selector, occurrence: binding.occurrence,
509
+ inspection: privateState?.application_inspection || null,
510
+ owner_window_bounds: binding.owner_window_bounds || null,
511
+ screen_box: binding.screen_box || null,
512
+ };
513
+ }
514
+
515
+ async preflight(observation, privateState, surface, action, target, context, actionDeadline = null) {
516
+ if (surface === "browser") {
517
+ return this.preflightBrowser(observation, privateState, action, target, context, actionDeadline);
518
+ }
519
+
520
+ const expectedProcessId = requiredPrivateApplicationProcessId(privateState?.application_process_id);
521
+ const expectedProcessGeneration = requiredPrivateApplicationProcessGeneration(privateState?.application_process_generation);
522
+ if (action === "activate") return;
523
+ if (target?.kind === "point") {
524
+ if (target.semantic_delivery) {
525
+ await this.preflightApplicationSemanticPoint(
526
+ observation, target, expectedProcessId, expectedProcessGeneration, context, actionDeadline,
527
+ );
528
+ return;
529
+ }
530
+ const visualPointMethod = action === "drag"
531
+ ? this.applications.dragApplication
532
+ : action === "scroll" ? this.applications.scrollApplication : this.applications.pointApplication;
533
+ if (typeof visualPointMethod !== "function") {
534
+ throw new BridgeError("unavailable", "application visual point dispatch is unavailable", {
535
+ details: { reason: "application_visual_preflight_unavailable" },
536
+ });
537
+ }
538
+ return;
539
+ }
540
+ const inspection = applicationTargetInspection(target);
541
+ let current;
542
+ try {
543
+ current = await this.applications.inspectApplication({
544
+ application: observation.target.application,
545
+ include_process_id: true,
546
+ expected_process_id: expectedProcessId,
547
+ expected_process_generation: expectedProcessGeneration,
548
+ max_depth: inspection.max_depth,
549
+ max_elements: 500,
550
+ include_values: false,
551
+ include_menus: inspection.include_menus,
552
+ include_geometry: applicationTargetHasGeometry(target),
553
+ timeout_seconds: requiredComputerActRemainingTimeoutSeconds(actionDeadline, 10),
554
+ }, context);
555
+ } catch (error) {
556
+ if (applicationProcessIdentityError(error)) {
557
+ throw staleSnapshot("application process instance changed after the snapshot");
558
+ }
559
+ throw error;
560
+ }
561
+ try { validateApplicationInspectionEvidence(current); }
562
+ catch { throw staleSnapshot("application Accessibility evidence became invalid after the snapshot"); }
563
+ if (!applicationProcessIdentityMatches(current, expectedProcessId, expectedProcessGeneration)) {
564
+ throw staleSnapshot("application process instance changed after the snapshot");
565
+ }
566
+ const matched = findApplicationMatch(current.elements, target.selector, target.occurrence);
567
+ if (!matched) {
568
+ throw staleSnapshot("application target is no longer present with the observed identity");
569
+ }
570
+ if (!applicationTargetGeometryMatches(matched, target)) {
571
+ throw staleSnapshot("application target window or geometry changed after the snapshot");
572
+ }
573
+ if ((action === "check" || action === "uncheck") && !applicationStateActionTargetSupported(action, matched)) {
574
+ throw staleSnapshot("application checked-state target changed after the snapshot");
575
+ }
576
+ }
577
+
578
+ async preflightBrowser(observation, privateState, action, target, context, actionDeadline = null) {
579
+ const tabs = await this.browser.listTabs({
580
+ include_pinned: true,
581
+ timeout_seconds: requiredComputerActRemainingTimeoutSeconds(actionDeadline, 5),
582
+ }, context);
583
+ const tab = (tabs.tabs || []).find((item) => item.id === observation.target.tab_id || item.tab_id === observation.target.tab_id);
584
+ if (!tab) throw staleSnapshot("browser tab no longer exists");
585
+ const tabUrl = typeof tab.url === "string" ? tab.url : "";
586
+ const snapshotUrl = typeof observation.target.url === "string" ? observation.target.url : "";
587
+ if (!tabUrl || !snapshotUrl || tabUrl !== snapshotUrl) throw staleSnapshot("browser tab navigated after the snapshot");
588
+ const semanticEpoch = typeof observation.capture?.semantic_epoch === "string" ? observation.capture.semantic_epoch : "";
589
+ let currentDocument = null;
590
+ let documentStateError = null;
591
+ if (semanticEpoch && typeof this.browser.documentState === "function") {
592
+ const documentStateTimeoutSeconds = requiredComputerActRemainingTimeoutSeconds(actionDeadline, 5);
593
+ try {
594
+ currentDocument = await this.browser.documentState({
595
+ tab_id: observation.target.tab_id,
596
+ timeout_seconds: documentStateTimeoutSeconds,
597
+ }, context);
598
+ if (currentDocument?.document_epoch !== undefined && currentDocument?.document_epoch !== null
599
+ && (typeof currentDocument.document_epoch !== "string" || currentDocument.document_epoch !== semanticEpoch)) {
600
+ throw staleSnapshot("browser document was replaced after the snapshot");
601
+ }
602
+ } catch (error) {
603
+ if (error instanceof BridgeError && error.details?.reason === "stale_snapshot") throw error;
604
+ documentStateError = error;
605
+ }
606
+ }
607
+ enforceBrowserHistoryDocumentPreflight(
608
+ action, semanticEpoch, privateBrowserHistoryEntryKey(privateState?.browser_history_entry_key), currentDocument, documentStateError,
609
+ );
610
+ if (target?.kind === "point") {
611
+ if (!currentDocument) {
612
+ throw new BridgeError("unavailable", "cannot validate the visual snapshot before point dispatch; observe again or use a semantic ref", {
613
+ details: { reason: "visual_preflight_unavailable", ...(documentStateError ? { error_class: errorCode(documentStateError) } : {}) },
614
+ });
615
+ }
616
+ if (!sameVisualViewport(target.viewport, currentDocument.viewport)) throw staleSnapshot("browser viewport changed after the screenshot");
617
+ } else if (target) {
618
+ const targetWaitTimeoutSeconds = requiredComputerActRemainingTimeoutSeconds(actionDeadline, 1);
619
+ try {
620
+ await this.browser.wait({
621
+ tab_id: observation.target.tab_id,
622
+ frame_id: target.frame_id,
623
+ selector: { ref: target.ref },
624
+ state: "attached",
625
+ timeout_seconds: targetWaitTimeoutSeconds,
626
+ }, context);
627
+ } catch {
628
+ throw staleSnapshot("browser target ref is no longer attached");
629
+ }
630
+ }
631
+ }
632
+
633
+ async preflightApplicationSemanticPoint(
634
+ observation, target, expectedProcessId, expectedProcessGeneration, context, actionDeadline = null,
635
+ ) {
636
+ if (typeof this.applications.captureApplication !== "function") {
637
+ throw new BridgeError("unavailable", "cannot revalidate the application screenshot before semantic point delivery", {
638
+ details: { reason: "application_visual_preflight_unavailable" },
639
+ });
640
+ }
641
+ const screenshotTimeoutSeconds = requiredComputerActRemainingTimeoutSeconds(actionDeadline, 10);
642
+ let captured;
643
+ try {
644
+ captured = await this.applications.captureApplication({
645
+ application: observation.target.application,
646
+ expected_process_id: expectedProcessId,
647
+ expected_process_generation: expectedProcessGeneration,
648
+ timeout_seconds: screenshotTimeoutSeconds,
649
+ }, context);
650
+ } catch (error) {
651
+ if (applicationProcessIdentityError(error)) {
652
+ throw staleSnapshot("application process instance changed before semantic point screenshot revalidation");
653
+ }
654
+ throw new BridgeError("unavailable", "cannot recapture the application window before semantic point delivery", {
655
+ details: { reason: "application_visual_preflight_unavailable", detail: applicationScreenshotError(error) },
656
+ });
657
+ }
658
+ const screenshot = captured?.screenshot;
659
+ const data = screenshot?.source === "macos_window" && screenshot?.mime_type === "image/png" && typeof screenshot?.data === "string"
660
+ ? screenshot.data : "";
661
+ const digest = data ? createHash("sha256").update(Buffer.from(data, "base64")).digest("hex") : "";
662
+ const currentBinding = applicationWindowBinding(captured, digest);
663
+ if (!sameApplicationWindowBinding(target.window_binding, currentBinding)) {
664
+ throw staleSnapshot("application window identity, bounds, or screenshot changed before semantic point delivery");
665
+ }
666
+ const semantic = target.semantic_delivery;
667
+ const inspection = applicationTargetInspection(target);
668
+ let current;
669
+ try {
670
+ current = await this.applications.inspectApplication({
671
+ application: observation.target.application,
672
+ include_process_id: true,
673
+ expected_process_id: expectedProcessId,
674
+ expected_process_generation: expectedProcessGeneration,
675
+ max_depth: inspection.max_depth,
676
+ max_elements: 500,
677
+ include_values: false,
678
+ include_menus: inspection.include_menus,
679
+ include_geometry: applicationTargetHasGeometry(target),
680
+ timeout_seconds: requiredComputerActRemainingTimeoutSeconds(actionDeadline, 10),
681
+ }, context);
682
+ } catch (error) {
683
+ if (applicationProcessIdentityError(error)) {
684
+ throw staleSnapshot("application process instance changed during semantic point preflight");
685
+ }
686
+ throw error;
687
+ }
688
+ try { validateApplicationInspectionEvidence(current); }
689
+ catch { throw staleSnapshot("application Accessibility evidence became invalid during semantic point preflight"); }
690
+ if (!applicationProcessIdentityMatches(current, expectedProcessId, expectedProcessGeneration)) {
691
+ throw staleSnapshot("application process instance changed during semantic point preflight");
692
+ }
693
+ const matched = findApplicationMatch(current.elements, semantic.selector, semantic.occurrence);
694
+ if (!matched) {
695
+ throw staleSnapshot("application semantic point target is no longer present with the observed identity");
696
+ }
697
+ if (!applicationTargetGeometryMatches(matched, target)) {
698
+ throw staleSnapshot("application semantic point target window or geometry changed before dispatch");
699
+ }
700
+ if (current.truncated === true) {
701
+ throw staleSnapshot("application semantic point Accessibility coverage became incomplete before dispatch");
702
+ }
703
+ const livePointCandidates = currentApplicationSemanticPointCandidates(current.elements, target);
704
+ if (livePointCandidates.length !== 1 || livePointCandidates[0] !== matched) {
705
+ throw staleSnapshot("application semantic point no longer resolves uniquely to the same Accessibility target");
706
+ }
707
+ if (typeof this.applications.inspectApplicationWindow === "function") {
708
+ const currentWindow = await this.applications.inspectApplicationWindow({
709
+ application: observation.target.application,
710
+ expected_process_id: expectedProcessId,
711
+ expected_process_generation: expectedProcessGeneration,
712
+ timeout_seconds: requiredComputerActRemainingTimeoutSeconds(actionDeadline, 10),
713
+ }, context).catch(() => null);
714
+ if (!sameApplicationWindowIdentity(target.window_binding, currentWindow)) {
715
+ throw staleSnapshot("application window changed during semantic point preflight");
716
+ }
717
+ }
718
+ }
719
+
720
+ async dispatchBrowserDrag(observation, target, destination, args, context) {
721
+ const inputMode = normalizeInputMode(args.input_mode);
722
+ if (inputMode === "dom") throw new BridgeError("invalid_request", "snapshot-bound drag requires trusted browser input");
723
+ if (normalizeNavigationWait(args.wait_for) !== "none") {
724
+ throw new BridgeError("invalid_request", "wait_for is not supported for drag; use expect.* post-conditions instead");
725
+ }
726
+ if (target?.kind === "point") {
727
+ try {
728
+ return await this.browser.pointAction({
729
+ tab_id: observation.target.tab_id,
730
+ action: "drag",
731
+ normalized_x: target.normalized_x,
732
+ normalized_y: target.normalized_y,
733
+ destination_normalized_x: destination.normalized_x,
734
+ destination_normalized_y: destination.normalized_y,
735
+ document_epoch: browserObservationSemanticEpoch(observation),
736
+ viewport: target.viewport,
737
+ screenshot_sha256: target.screenshot_sha256,
738
+ screenshot_format: target.screenshot_format,
739
+ screenshot_quality: target.screenshot_quality,
740
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
741
+ }, context);
742
+ } catch (error) {
743
+ const message = String(error?.message || error);
744
+ if (message.includes("visual_snapshot_changed_before_dispatch")
745
+ || message.includes("visual snapshot changed before trusted input; observe again")) {
746
+ throw staleSnapshot("browser screenshot changed after the visual snapshot");
747
+ }
748
+ throw error;
749
+ }
750
+ }
751
+ if (!target?.trusted_binding || !destination?.trusted_binding) {
752
+ throw new BridgeError("unavailable", "snapshot-bound ref drag requires trusted backend bindings for both endpoints; observe with a screenshot and use point targets if needed", {
753
+ details: { reason: "snapshot_backend_drag_unavailable" },
754
+ });
755
+ }
756
+ try {
757
+ return await this.browser.backendNodeAction({
758
+ tab_id: observation.target.tab_id,
759
+ action: "drag",
760
+ backend_node_id: target.trusted_binding.backend_node_id,
761
+ extension_frame_id: target.trusted_binding.extension_frame_id,
762
+ frame_document_epoch: target.trusted_binding.extension_frame_epoch,
763
+ frame_url: target.trusted_binding.extension_frame_url,
764
+ extension_ref: target.ref,
765
+ expected_ref_identity: browserSnapshotIdentity(target.element),
766
+ destination_backend_node_id: destination.trusted_binding.backend_node_id,
767
+ destination_extension_frame_id: destination.trusted_binding.extension_frame_id,
768
+ destination_frame_document_epoch: destination.trusted_binding.extension_frame_epoch,
769
+ destination_frame_url: destination.trusted_binding.extension_frame_url,
770
+ destination_extension_ref: destination.ref,
771
+ destination_expected_ref_identity: browserSnapshotIdentity(destination.element),
772
+ document_epoch: browserObservationSemanticEpoch(observation),
773
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
774
+ }, context);
775
+ } catch (error) {
776
+ const message = String(error?.message || error);
777
+ if (message.includes("snapshot backend target changed before trusted input; observe again")
778
+ || message.includes("snapshot_backend_target_changed_before_dispatch")) {
779
+ throw staleSnapshot("browser drag endpoint identity changed before trusted input");
780
+ }
781
+ if (message.includes("outcome is unknown") || message.includes("partially dispatched")) throw error;
782
+ throw new BridgeError("unavailable", "snapshot-bound trusted drag is unavailable before dispatch", {
783
+ cause: error instanceof Error ? error : undefined,
784
+ details: { reason: "snapshot_backend_drag_unavailable" },
785
+ });
786
+ }
787
+ }
788
+
789
+ async dispatchBrowserScroll(observation, target, args, context) {
790
+ const inputMode = normalizeInputMode(args.input_mode);
791
+ if (inputMode === "dom") throw new BridgeError("invalid_request", "snapshot-bound scroll requires trusted browser input");
792
+ if (normalizeNavigationWait(args.wait_for) !== "none") {
793
+ throw new BridgeError("invalid_request", "wait_for is not supported for scroll; use expect.* post-conditions instead");
794
+ }
795
+ const deltaX = browserScrollDelta(args.delta_x, "delta_x");
796
+ const deltaY = browserScrollDelta(args.delta_y, "delta_y");
797
+ if (deltaX === 0 && deltaY === 0) throw new BridgeError("invalid_request", "browser scroll requires a non-zero delta_x or delta_y");
798
+ if (target?.kind === "point") {
799
+ try {
800
+ return await this.browser.pointAction({
801
+ tab_id: observation.target.tab_id,
802
+ action: "scroll",
803
+ normalized_x: target.normalized_x,
804
+ normalized_y: target.normalized_y,
805
+ delta_x: deltaX,
806
+ delta_y: deltaY,
807
+ document_epoch: browserObservationSemanticEpoch(observation),
808
+ viewport: target.viewport,
809
+ screenshot_sha256: target.screenshot_sha256,
810
+ screenshot_format: target.screenshot_format,
811
+ screenshot_quality: target.screenshot_quality,
812
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
813
+ }, context);
814
+ } catch (error) {
815
+ const message = String(error?.message || error);
816
+ if (message.includes("visual_snapshot_changed_before_dispatch")
817
+ || message.includes("visual snapshot changed before trusted input; observe again")) {
818
+ throw staleSnapshot("browser screenshot changed after the visual snapshot");
819
+ }
820
+ throw error;
821
+ }
822
+ }
823
+ if (!target?.trusted_binding) {
824
+ throw new BridgeError("unavailable", "snapshot-bound ref scroll requires a trusted backend binding; observe with a screenshot and use a point target if needed", {
825
+ details: { reason: "snapshot_backend_scroll_unavailable" },
826
+ });
827
+ }
828
+ try {
829
+ return await this.browser.backendNodeAction({
830
+ tab_id: observation.target.tab_id,
831
+ action: "scroll",
832
+ backend_node_id: target.trusted_binding.backend_node_id,
833
+ extension_frame_id: target.trusted_binding.extension_frame_id,
834
+ frame_document_epoch: target.trusted_binding.extension_frame_epoch,
835
+ frame_url: target.trusted_binding.extension_frame_url,
836
+ extension_ref: target.ref,
837
+ expected_ref_identity: browserSnapshotIdentity(target.element),
838
+ document_epoch: browserObservationSemanticEpoch(observation),
839
+ delta_x: deltaX,
840
+ delta_y: deltaY,
841
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
842
+ }, context);
843
+ } catch (error) {
844
+ const message = String(error?.message || error);
845
+ if (message.includes("snapshot backend target changed before trusted input; observe again")
846
+ || message.includes("snapshot_backend_target_changed_before_dispatch")) {
847
+ throw staleSnapshot("browser scroll anchor identity changed before trusted input");
848
+ }
849
+ if (message.includes("outcome is unknown") || message.includes("partially dispatched")) throw error;
850
+ throw new BridgeError("unavailable", "snapshot-bound trusted scroll is unavailable before dispatch", {
851
+ cause: error instanceof Error ? error : undefined,
852
+ details: { reason: "snapshot_backend_scroll_unavailable" },
853
+ });
854
+ }
855
+ }
856
+
857
+ async dispatchBrowser(observation, privateState, action, target, args, context, destination = null) {
858
+ if (action === "drag") return this.dispatchBrowserDrag(observation, target, destination, args, context);
859
+ if (action === "scroll") return this.dispatchBrowserScroll(observation, target, args, context);
860
+ if (target?.kind === "point") {
861
+ const inputMode = normalizeInputMode(args.input_mode);
862
+ if (inputMode === "dom") throw new BridgeError("invalid_request", "snapshot-bound visual point actions require trusted input");
863
+ if (normalizeNavigationWait(args.wait_for) !== "none") {
864
+ throw new BridgeError("invalid_request", "wait_for is not supported for visual point dispatch; use expect.load_state or another post-condition");
865
+ }
866
+ try {
867
+ return await this.browser.pointAction({
868
+ tab_id: observation.target.tab_id,
869
+ action,
870
+ normalized_x: target.normalized_x,
871
+ normalized_y: target.normalized_y,
872
+ document_epoch: browserObservationSemanticEpoch(observation),
873
+ viewport: target.viewport,
874
+ screenshot_sha256: target.screenshot_sha256,
875
+ screenshot_format: target.screenshot_format,
876
+ screenshot_quality: target.screenshot_quality,
877
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
878
+ }, context);
879
+ } catch (error) {
880
+ const message = String(error?.message || error);
881
+ if (message.includes("visual_snapshot_changed_before_dispatch")
882
+ || message.includes("visual snapshot changed before trusted input; observe again")) {
883
+ throw staleSnapshot("browser screenshot changed after the visual snapshot");
884
+ }
885
+ throw error;
886
+ }
887
+ }
888
+ const inputMode = normalizeInputMode(args.input_mode);
889
+ const navigationWait = normalizeNavigationWait(args.wait_for);
890
+ const boundResult = await trySnapshotBoundBackendAction({
891
+ browser: this.browser,
892
+ observation,
893
+ action,
894
+ target,
895
+ args,
896
+ context,
897
+ inputMode,
898
+ navigationWait,
899
+ });
900
+ if (boundResult) return boundResult;
901
+ const call = {
902
+ tab_id: observation.target.tab_id,
903
+ action,
904
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
905
+ wait_for: navigationWait,
906
+ input_mode: inputMode,
907
+ element_timeout_seconds: clampInt(args.element_timeout_seconds, 10, 1, 60),
908
+ };
909
+ if (target) {
910
+ call.frame_id = target.frame_id;
911
+ call.selector = { ref: target.ref };
912
+ call.expected_ref_identity = browserSnapshotIdentity(target.element);
913
+ }
914
+ if (BROWSER_TARGETLESS_ACTIONS.has(action)) {
915
+ const expectedTabUrl = typeof observation.target?.url === "string" ? observation.target.url : "";
916
+ if (expectedTabUrl) call.expected_tab_url = expectedTabUrl;
917
+ }
918
+ if (BROWSER_DOCUMENT_BOUND_TARGETLESS_ACTIONS.has(action)) {
919
+ const semanticEpoch = browserObservationSemanticEpoch(observation);
920
+ const historyEntryKey = privateBrowserHistoryEntryKey(privateState?.browser_history_entry_key);
921
+ if (semanticEpoch) call.expected_document_epoch = semanticEpoch;
922
+ if (historyEntryKey) call.expected_history_entry_key = historyEntryKey;
923
+ }
924
+ if (action === "navigate") call.url = requiredString(args.url, "url", 32768);
925
+ if (args.value !== undefined) call.value = requiredStringAllowEmpty(args.value, "value", 131072);
926
+ if (args.value_resource !== undefined) call.value_resource = requiredResource(args.value_resource);
927
+ if (args.key !== undefined) call.key = requiredString(args.key, "key", 100);
928
+ try {
929
+ return await this.browser.act(call, context);
930
+ } catch (error) {
931
+ throw browserSnapshotDispatchError(error);
932
+ }
933
+ }
934
+
935
+ async dispatchApplication(observation, privateState, action, target, args, context, destination = null) {
936
+ const expectedProcessId = requiredPrivateApplicationProcessId(privateState?.application_process_id);
937
+ const expectedProcessGeneration = requiredPrivateApplicationProcessGeneration(privateState?.application_process_generation);
938
+ if (target?.kind === "point") {
939
+ if (action === "drag") {
940
+ try {
941
+ return await this.applications.dragApplication({
942
+ application: observation.target.application,
943
+ expected_process_id: expectedProcessId,
944
+ expected_process_generation: expectedProcessGeneration,
945
+ normalized_x: target.normalized_x,
946
+ normalized_y: target.normalized_y,
947
+ destination_normalized_x: destination.normalized_x,
948
+ destination_normalized_y: destination.normalized_y,
949
+ window_id: target.window_binding.window_id,
950
+ bounds: target.window_binding.bounds,
951
+ screenshot_sha256: target.window_binding.screenshot_sha256,
952
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
953
+ }, context);
954
+ } catch (error) {
955
+ const message = String(error?.message || error);
956
+ if (message.includes("application visual snapshot is stale") || message.includes("application visual snapshot changed before dispatch")) {
957
+ throw staleSnapshot("application window identity, bounds, or screenshot changed immediately before visual drag");
958
+ }
959
+ if (applicationProcessIdentityError(error)) {
960
+ throw staleSnapshot("application process changed immediately before visual drag");
961
+ }
962
+ if (message.includes("application visual snapshot unavailable before dispatch") || message.includes("application visual input unavailable before dispatch")) {
963
+ throw new BridgeError("unavailable", "application visual drag could not be validated before dispatch", {
964
+ details: { reason: "application_visual_preflight_unavailable" },
965
+ });
966
+ }
967
+ throw error;
968
+ }
969
+ }
970
+ if (action === "scroll") {
971
+ try {
972
+ return await this.applications.scrollApplication({
973
+ application: observation.target.application,
974
+ expected_process_id: expectedProcessId,
975
+ expected_process_generation: expectedProcessGeneration,
976
+ normalized_x: target.normalized_x,
977
+ normalized_y: target.normalized_y,
978
+ delta_x: args.delta_x,
979
+ delta_y: args.delta_y,
980
+ window_id: target.window_binding.window_id,
981
+ bounds: target.window_binding.bounds,
982
+ screenshot_sha256: target.window_binding.screenshot_sha256,
983
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
984
+ }, context);
985
+ } catch (error) {
986
+ const message = String(error?.message || error);
987
+ if (message.includes("application visual snapshot is stale") || message.includes("application visual snapshot changed before dispatch")) {
988
+ throw staleSnapshot("application window identity, bounds, or screenshot changed immediately before visual scroll");
989
+ }
990
+ if (applicationProcessIdentityError(error)) {
991
+ throw staleSnapshot("application process changed immediately before visual scroll");
992
+ }
993
+ if (message.includes("application visual snapshot unavailable before dispatch") || message.includes("application visual input unavailable before dispatch")) {
994
+ throw new BridgeError("unavailable", "application visual scroll could not be validated before dispatch", {
995
+ details: { reason: "application_visual_preflight_unavailable" },
996
+ });
997
+ }
998
+ throw error;
999
+ }
1000
+ }
1001
+ if (target.semantic_delivery) {
1002
+ const semantic = target.semantic_delivery;
1003
+ const selector = semantic.occurrence > 0 ? { ...semantic.selector, index: semantic.occurrence } : semantic.selector;
1004
+ const inspection = applicationTargetInspection(target);
1005
+ const call = applyApplicationTargetGeometry({
1006
+ application: observation.target.application,
1007
+ expected_process_id: expectedProcessId,
1008
+ expected_process_generation: expectedProcessGeneration,
1009
+ action: "click",
1010
+ selector,
1011
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
1012
+ include_menus: inspection.include_menus,
1013
+ max_depth: inspection.max_depth,
1014
+ }, target);
1015
+ const result = await this.operateApplicationBound(call, context);
1016
+ return {
1017
+ ...result,
1018
+ coordinate_source: "accessibility_point_resolution",
1019
+ window_bound: true,
1020
+ screenshot_revalidated: true,
1021
+ normalized_point: { x: target.normalized_x, y: target.normalized_y },
1022
+ semantic_point_candidates: target.semantic_point_candidates,
1023
+ };
1024
+ }
1025
+ try {
1026
+ const result = await this.applications.pointApplication({
1027
+ application: observation.target.application,
1028
+ expected_process_id: expectedProcessId,
1029
+ expected_process_generation: expectedProcessGeneration,
1030
+ normalized_x: target.normalized_x,
1031
+ normalized_y: target.normalized_y,
1032
+ ...(action === "double_click" ? { click_count: 2 } : {}),
1033
+ window_id: target.window_binding.window_id,
1034
+ bounds: target.window_binding.bounds,
1035
+ screenshot_sha256: target.window_binding.screenshot_sha256,
1036
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
1037
+ }, context);
1038
+ return { ...result, semantic_point_candidates: target.semantic_point_candidates };
1039
+ } catch (error) {
1040
+ const message = String(error?.message || error);
1041
+ if (message.includes("application visual snapshot is stale") || message.includes("application visual snapshot changed before dispatch")) {
1042
+ throw staleSnapshot("application window identity, bounds, or screenshot changed immediately before visual dispatch");
1043
+ }
1044
+ if (applicationProcessIdentityError(error)) {
1045
+ throw staleSnapshot("application process changed immediately before visual dispatch");
1046
+ }
1047
+ if (message.includes("application visual snapshot unavailable before dispatch") || message.includes("application visual input unavailable before dispatch")) {
1048
+ throw new BridgeError("unavailable", "application visual input could not be validated before dispatch", {
1049
+ details: { reason: "application_visual_preflight_unavailable" },
1050
+ });
1051
+ }
1052
+ throw error;
1053
+ }
1054
+ }
1055
+ const inspection = applicationTargetInspection(target);
1056
+ const call = {
1057
+ application: observation.target.application,
1058
+ expected_process_id: expectedProcessId,
1059
+ expected_process_generation: expectedProcessGeneration,
1060
+ action,
1061
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
1062
+ include_menus: inspection.include_menus,
1063
+ max_depth: inspection.max_depth,
1064
+ };
1065
+ if (target) call.selector = target.occurrence > 0 ? { ...target.selector, index: target.occurrence } : target.selector;
1066
+ if (args.value !== undefined) call.value = requiredStringAllowEmpty(args.value, "value", 4000);
1067
+ if (args.value_resource !== undefined) call.value_resource = requiredResource(args.value_resource);
1068
+ if (args.key !== undefined) call.key = requiredString(args.key, "key", 100);
1069
+ if (action === "set_value" && args.value_resource !== undefined && target?.element?.sensitive !== true) {
1070
+ call.retain_value_verification = true;
1071
+ }
1072
+ return this.operateApplicationBound(applyApplicationTargetGeometry(call, target), context);
1073
+ }
1074
+
1075
+ async operateApplicationBound(call, context) {
1076
+ try {
1077
+ return await this.applications.operateApplication(call, context);
1078
+ } catch (error) {
1079
+ const message = String(error?.message || error);
1080
+ if (message.includes("application target window changed before dispatch")
1081
+ || message.includes("application target geometry changed before dispatch")
1082
+ || applicationProcessIdentityError(error)) {
1083
+ throw staleSnapshot("application target window or geometry changed immediately before Accessibility dispatch");
1084
+ }
1085
+ if (message.includes("application target checked state is unavailable before dispatch")
1086
+ || message.includes("application check target is not an Accessibility checkbox or radio button")
1087
+ || message.includes("application uncheck target is not an Accessibility checkbox")) {
1088
+ throw staleSnapshot("application checked-state target changed immediately before Accessibility dispatch");
1089
+ }
1090
+ if (message.includes("application target does not expose AXPress before checked-state dispatch")) {
1091
+ throw new BridgeError("unavailable", "application checked-state input is unavailable before dispatch", {
1092
+ details: { reason: "application_checked_state_input_unavailable" },
1093
+ });
1094
+ }
1095
+ throw error;
1096
+ }
1097
+ }
1098
+
1099
+ async verifyApplicationPostAction({
1100
+ beforeObservation,
1101
+ beforePrivateState,
1102
+ target,
1103
+ expectation,
1104
+ args,
1105
+ dispatchResult,
1106
+ captureArgs,
1107
+ timeoutSeconds,
1108
+ operationDeadline = null,
1109
+ }, context) {
1110
+ const deadline = createMonotonicDeadline(timeoutSeconds * 1000, this.now);
1111
+ const retainedHandle = typeof dispatchResult?._machine_value_verification_handle === "string"
1112
+ ? dispatchResult._machine_value_verification_handle : "";
1113
+ let postCapture = null;
1114
+ let postCaptureError = "";
1115
+ let observedDiff = null;
1116
+ let captureAttempts = 0;
1117
+ let verificationProbe = { requested: true, matched: false, reason: "not_checked" };
1118
+ try {
1119
+ for (;;) {
1120
+ const remainingBeforeCaptureMs = deadline.remainingMs();
1121
+ if (remainingBeforeCaptureMs <= 0 || captureAttempts >= APPLICATION_VERIFY_MAX_CAPTURES) {
1122
+ if (postCaptureError) verificationProbe = { ...verificationProbe, matched: false, inconclusive: true, reason: "post_conditions_inconclusive" };
1123
+ break;
1124
+ }
1125
+ try { this.throwIfCancelled(context); }
1126
+ catch (error) {
1127
+ postCaptureError = publicPostObservationError("application", error);
1128
+ verificationProbe = { ...verificationProbe, matched: false, inconclusive: true, reason: "post_conditions_inconclusive" };
1129
+ break;
1130
+ }
1131
+ let captured = null;
1132
+ let captureError = "";
1133
+ const configuredCaptureTimeoutSeconds = clampInt(captureArgs.timeout_seconds, 30, 1, 60);
1134
+ const operationTimeoutSeconds = computerActRemainingTimeoutSeconds(operationDeadline, configuredCaptureTimeoutSeconds);
1135
+ const captureTimeoutSeconds = Math.min(
1136
+ Math.max(1, Math.min(configuredCaptureTimeoutSeconds, Math.ceil(remainingBeforeCaptureMs / 1000))),
1137
+ operationTimeoutSeconds,
1138
+ );
1139
+ if (operationTimeoutSeconds <= 0) {
1140
+ if (!postCapture) {
1141
+ postCaptureError = computerActPostObservationTimeoutError("application");
1142
+ verificationProbe = computerActVerificationTimeoutProbe();
1143
+ }
1144
+ break;
1145
+ }
1146
+ captureAttempts += 1;
1147
+ try { captured = await this.capture({ ...captureArgs, timeout_seconds: captureTimeoutSeconds }, context, operationDeadline); }
1148
+ catch (error) {
1149
+ captureError = publicPostObservationError("application", error);
1150
+ if (errorCode(error) === "cancelled") {
1151
+ postCaptureError = captureError;
1152
+ verificationProbe = { ...verificationProbe, matched: false, inconclusive: true, reason: "post_conditions_inconclusive" };
1153
+ break;
1154
+ }
1155
+ }
1156
+ if (captured) {
1157
+ if (postCapture) this.snapshots.discard(postCapture.observation.snapshot_id);
1158
+ postCapture = captured;
1159
+ postCaptureError = "";
1160
+ observedDiff = observationDiff(beforeObservation, captured.observation, beforePrivateState, captured.privateState);
1161
+ const processChanged = applicationProcessChanged(beforePrivateState, captured.privateState) === true;
1162
+ let valueProbe = null;
1163
+ if (expectation.target_value_matches !== undefined) {
1164
+ const remainingValueVerificationMs = deadline.remainingMs();
1165
+ const operationValueTimeoutSeconds = computerActRemainingTimeoutSeconds(operationDeadline, 60);
1166
+ const valueTimeoutSeconds = Math.min(Math.max(1, Math.ceil(remainingValueVerificationMs / 1000)), operationValueTimeoutSeconds);
1167
+ valueProbe = processChanged
1168
+ ? { available: false, matched: false, reason: "application_process_changed" }
1169
+ : remainingValueVerificationMs <= 0 || operationValueTimeoutSeconds <= 0
1170
+ ? { available: false, matched: false, reason: "verification_deadline_elapsed" }
1171
+ : await this.verifyApplicationValueExpectation(
1172
+ captured, target, args, dispatchResult, context,
1173
+ valueTimeoutSeconds,
1174
+ );
1175
+ }
1176
+ verificationProbe = verifyApplicationExpectation(
1177
+ beforeObservation,
1178
+ captured.observation,
1179
+ target,
1180
+ expectation,
1181
+ observedDiff,
1182
+ beforePrivateState,
1183
+ captured.privateState,
1184
+ valueProbe,
1185
+ );
1186
+ if (verificationProbe.matched === true || processChanged) break;
1187
+ if (expectation.target_value_matches !== undefined
1188
+ && valueProbe?.reason !== "post_target_binding_unavailable"
1189
+ && !(valueProbe?.reason === "value_readback_failed" && args.value !== undefined)) break;
1190
+ } else {
1191
+ postCaptureError = captureError;
1192
+ }
1193
+ const remainingMs = Math.min(
1194
+ deadline.remainingMs(),
1195
+ operationDeadline?.remainingMs?.() ?? Number.POSITIVE_INFINITY,
1196
+ );
1197
+ if (remainingMs <= 0 || captureAttempts >= APPLICATION_VERIFY_MAX_CAPTURES) {
1198
+ if (postCaptureError) verificationProbe = { ...verificationProbe, matched: false, inconclusive: true, reason: "post_conditions_inconclusive" };
1199
+ break;
1200
+ }
1201
+ const retryDelayMs = APPLICATION_VERIFY_POLL_MS * (2 ** Math.min(Math.max(captureAttempts - 1, 0), 3));
1202
+ try { await this.sleep(Math.min(retryDelayMs, remainingMs)); }
1203
+ catch (error) {
1204
+ postCaptureError = publicPostObservationError("application", error);
1205
+ verificationProbe = { ...verificationProbe, matched: false, inconclusive: true, reason: "post_conditions_inconclusive" };
1206
+ break;
1207
+ }
1208
+ }
1209
+ } catch (error) {
1210
+ postCaptureError = publicPostObservationError("application", error);
1211
+ verificationProbe = { ...verificationProbe, matched: false, inconclusive: true, reason: "post_conditions_inconclusive" };
1212
+ } finally {
1213
+ this.discardApplicationValueVerification(retainedHandle);
1214
+ }
1215
+ return { postCapture, postCaptureError, observedDiff, verificationProbe };
1216
+ }
1217
+
1218
+ async verifyApplicationValueExpectation(postCapture, target, args, dispatchResult, context, remainingTimeoutSeconds) {
1219
+ if (typeof this.applications.verifyApplicationValue !== "function") {
1220
+ return { available: false, matched: false, reason: "value_readback_backend_unavailable" };
1221
+ }
1222
+ const verificationTarget = applicationVerificationTarget(target);
1223
+ const postTarget = verificationTarget
1224
+ ? findPostApplicationTargetBinding(postCapture.observation, postCapture.privateState, verificationTarget)
1225
+ : null;
1226
+ if (!postTarget) return { available: false, matched: false, reason: "post_target_binding_unavailable" };
1227
+ if (postTarget.element?.sensitive === true) return { available: false, matched: false, reason: "sensitive_target" };
1228
+ const inspection = postCapture.privateState?.application_inspection || null;
1229
+ const selector = postTarget.binding.occurrence > 0
1230
+ ? { ...postTarget.binding.selector, index: postTarget.binding.occurrence }
1231
+ : postTarget.binding.selector;
1232
+ const call = applyApplicationTargetGeometry({
1233
+ application: postCapture.observation.target.application,
1234
+ expected_process_id: requiredPrivateApplicationProcessId(postCapture.privateState?.application_process_id),
1235
+ expected_process_generation: requiredPrivateApplicationProcessGeneration(postCapture.privateState?.application_process_generation),
1236
+ selector,
1237
+ max_depth: clampInt(inspection?.max_depth, 6, 1, 12),
1238
+ include_menus: inspection?.include_menus === true,
1239
+ timeout_seconds: Math.min(
1240
+ clampInt(args.timeout_seconds, 30, 1, 60),
1241
+ clampInt(remainingTimeoutSeconds, 1, 1, 60),
1242
+ ),
1243
+ }, {
1244
+ owner_window_bounds: postTarget.binding.owner_window_bounds || null,
1245
+ screen_box: postTarget.binding.screen_box || null,
1246
+ });
1247
+ const retainedHandle = typeof dispatchResult?._machine_value_verification_handle === "string"
1248
+ ? dispatchResult._machine_value_verification_handle : "";
1249
+ if (retainedHandle) call.value_verification_handle = retainedHandle;
1250
+ else if (args.value !== undefined) call.value = requiredStringAllowEmpty(args.value, "value", 4000);
1251
+ else if (args.value_resource !== undefined) {
1252
+ return { available: false, matched: false, reason: "exact_dispatched_resource_value_unavailable" };
1253
+ }
1254
+ try {
1255
+ const result = await this.applications.verifyApplicationValue(call, context);
1256
+ if (result?.supported !== true) return { available: false, matched: false, reason: String(result?.reason || "value_readback_unsupported") };
1257
+ return { available: true, matched: result.matched === true, reason: "accessibility_value_compared" };
1258
+ } catch {
1259
+ return { available: false, matched: false, reason: "value_readback_failed" };
1260
+ }
1261
+ }
1262
+
1263
+ discardApplicationValueVerification(handle) {
1264
+ if (!handle || typeof this.applications.discardApplicationValueVerification !== "function") return;
1265
+ try { this.applications.discardApplicationValueVerification(handle); }
1266
+ catch { /* TTL remains the fallback if best-effort cleanup cannot run. */ }
1267
+ }
1268
+
1269
+ async verifyBrowserExpectation(observation, target, expectation, timeoutSeconds, context) {
1270
+ const waitArgs = { tab_id: observation.target.tab_id, timeout_seconds: timeoutSeconds };
1271
+ let hasWaitCondition = false;
1272
+ const postCheckPending = browserExpectationNeedsPostObservation(expectation);
1273
+ if (expectation.url_contains) { waitArgs.url_contains = expectation.url_contains; hasWaitCondition = true; }
1274
+ if (expectation.text) { waitArgs.text = expectation.text; hasWaitCondition = true; }
1275
+ if (expectation.load_state) { waitArgs.load_state = expectation.load_state; hasWaitCondition = true; }
1276
+ if (expectation.target_state) {
1277
+ if (!target || target.kind !== "ref") return { requested: true, matched: false, reason: "target_state_requires_ref_target" };
1278
+ waitArgs.frame_id = target.frame_id;
1279
+ waitArgs.selector = { ref: target.ref };
1280
+ waitArgs.state = expectation.target_state;
1281
+ hasWaitCondition = true;
1282
+ }
1283
+ if (!hasWaitCondition) {
1284
+ return postCheckPending
1285
+ ? { requested: true, matched: false, wait_matched: true, post_check_pending: true, reason: "post_conditions_pending" }
1286
+ : { requested: true, matched: true, wait_matched: true, reason: "no_wait_conditions" };
1287
+ }
1288
+ try {
1289
+ await this.browser.wait(waitArgs, context);
1290
+ return postCheckPending
1291
+ ? { requested: true, matched: false, wait_matched: true, post_check_pending: true, reason: "post_conditions_pending" }
1292
+ : { requested: true, matched: true, wait_matched: true, reason: "wait_conditions_satisfied" };
1293
+ } catch (error) {
1294
+ return {
1295
+ requested: true,
1296
+ matched: false,
1297
+ wait_matched: false,
1298
+ post_check_pending: postCheckPending,
1299
+ reason: "wait_conditions_not_observed",
1300
+ error_class: errorCode(error),
1301
+ };
1302
+ }
1303
+ }
1304
+ }
1305
+
1306
+ function combineBrowserPostChecks(probe, diff, expectation, before, after, target) {
1307
+ let matched = probe.post_check_pending === true ? probe.wait_matched === true : probe.matched === true;
1308
+ const checks = [];
1309
+ if (expectation.url_changed !== undefined) {
1310
+ const ok = diff?.url_changed === expectation.url_changed;
1311
+ matched = matched && ok;
1312
+ checks.push({ condition: "url_changed", matched: ok, evidence_source: "browser_tab_state" });
1313
+ }
1314
+ if (expectation.semantic_change !== undefined) {
1315
+ const ok = diff?.semantic_changed === expectation.semantic_change;
1316
+ matched = matched && ok;
1317
+ checks.push({ condition: "semantic_change", matched: ok, evidence_source: "browser_semantic" });
1318
+ }
1319
+ if (expectation.visual_change !== undefined) {
1320
+ const observed = diff?.screenshot_changed;
1321
+ const ok = typeof observed === "boolean" && observed === expectation.visual_change;
1322
+ matched = matched && ok;
1323
+ checks.push({ condition: "visual_change", matched: ok, observed: typeof observed === "boolean" ? observed : null, evidence_source: "screenshot" });
1324
+ }
1325
+ if (expectation.target_state) {
1326
+ for (const check of browserPostTargetStateChecks(before, after, target, expectation.target_state)) {
1327
+ if (check.inconclusive === true) matched = false;
1328
+ else matched = matched && check.matched === true;
1329
+ checks.push({ evidence_source: "browser_semantic", ...check });
1330
+ }
1331
+ }
1332
+ const definitiveFailure = probe.wait_matched === false || checks.some((check) => check.matched === false && check.inconclusive !== true);
1333
+ const inconclusive = !definitiveFailure && checks.some((check) => check.inconclusive === true);
1334
+ return {
1335
+ ...probe,
1336
+ matched: matched && !inconclusive,
1337
+ inconclusive,
1338
+ post_check_pending: false,
1339
+ reason: matched && !inconclusive
1340
+ ? "post_conditions_satisfied"
1341
+ : inconclusive
1342
+ ? "post_conditions_inconclusive"
1343
+ : probe.wait_matched === false ? probe.reason : "post_conditions_not_observed",
1344
+ post_checks: checks,
1345
+ };
1346
+ }
1347
+
1348
+ function browserExpectationNeedsPostObservation(expectation) {
1349
+ return expectation?.url_changed !== undefined
1350
+ || expectation?.semantic_change !== undefined
1351
+ || expectation?.visual_change !== undefined
1352
+ || Boolean(expectation?.target_state);
1353
+ }
1354
+
1355
+ function browserPostTargetStateChecks(before, after, target, state) {
1356
+ if (!target || target.kind !== "ref") return [{ condition: "target_state", matched: false, reason: "target_state_requires_ref_target" }];
1357
+ const frameEpochStatus = browserTargetFrameEpochStatus(before, after, target);
1358
+ if (frameEpochStatus === "unknown") return [{
1359
+ condition: "target_state",
1360
+ matched: false,
1361
+ observed: null,
1362
+ inconclusive: true,
1363
+ reason: "post_target_frame_coverage_incomplete",
1364
+ }];
1365
+ if (frameEpochStatus === "changed") {
1366
+ const goneMatches = state === "detached" || state === "hidden";
1367
+ return [{
1368
+ condition: "target_state",
1369
+ matched: goneMatches,
1370
+ observed: goneMatches ? state : null,
1371
+ reason: "target_frame_epoch_changed",
1372
+ }];
1373
+ }
1374
+ const frame = (after.semantic?.frames || []).find((candidate) => candidate?.frame_id === target.frame_id) || null;
1375
+ const complete = after.semantic?.frames_truncated === false && Boolean(frame) && frame.truncated === false;
1376
+ const element = (frame?.elements || []).find((candidate) => candidate?.ref === target.ref) || null;
1377
+ if (!element) {
1378
+ if (!complete) return [{
1379
+ condition: "target_state",
1380
+ matched: false,
1381
+ observed: null,
1382
+ inconclusive: true,
1383
+ reason: "post_target_coverage_incomplete",
1384
+ }];
1385
+ const absentMatches = state === "detached" || state === "hidden";
1386
+ return [{ condition: "target_state", matched: absentMatches, observed: absentMatches ? state : "detached" }];
1387
+ }
1388
+ const identityMatched = browserSnapshotIdentityMatches(browserSnapshotIdentity(target.element), element);
1389
+ const checks = [{ condition: "target_identity", matched: identityMatched }];
1390
+ if (!identityMatched) {
1391
+ checks.push({ condition: "target_state", matched: false, observed: null, reason: "target_semantic_identity_changed" });
1392
+ return checks;
1393
+ }
1394
+ const observed = browserObservedTargetState(element, state);
1395
+ if (observed === null) {
1396
+ checks.push({ condition: "target_state", matched: false, observed: null, inconclusive: true, reason: "target_state_unavailable" });
1397
+ return checks;
1398
+ }
1399
+ checks.push({ condition: "target_state", matched: observed, observed });
1400
+ return checks;
1401
+ }
1402
+
1403
+ function browserTargetFrameEpochStatus(before, after, target) {
1404
+ if (!Number.isInteger(target?.frame_id)) return "unknown";
1405
+ const beforeFrame = (before.semantic?.frames || []).find((frame) => frame?.frame_id === target.frame_id) || null;
1406
+ const afterFrame = (after.semantic?.frames || []).find((frame) => frame?.frame_id === target.frame_id) || null;
1407
+ if (!beforeFrame) return "unknown";
1408
+ if (!afterFrame) return after.semantic?.frames_truncated === true ? "unknown" : "changed";
1409
+ const beforeEpoch = typeof beforeFrame.document?.epoch === "string" ? beforeFrame.document.epoch : "";
1410
+ const afterEpoch = typeof afterFrame.document?.epoch === "string" ? afterFrame.document.epoch : "";
1411
+ const beforeUrl = typeof beforeFrame.document?.url === "string" ? beforeFrame.document.url : "";
1412
+ const afterUrl = typeof afterFrame.document?.url === "string" ? afterFrame.document.url : "";
1413
+ if (!beforeEpoch || !afterEpoch || !beforeUrl || !afterUrl) return "unknown";
1414
+ return beforeEpoch === afterEpoch && beforeUrl === afterUrl ? "same" : "changed";
1415
+ }
1416
+
1417
+ function browserObservedTargetState(element, state) {
1418
+ if (state === "attached") return true;
1419
+ if (state === "detached") return false;
1420
+ if (state === "visible") return typeof element?.visible === "boolean" ? element.visible : null;
1421
+ if (state === "hidden") return typeof element?.visible === "boolean" ? !element.visible : null;
1422
+ if (state === "enabled") return typeof element?.enabled === "boolean" ? element.enabled : null;
1423
+ if (state === "editable") return typeof element?.editable === "boolean" ? element.editable : null;
1424
+ if (state === "checked") return typeof element?.checked === "boolean" ? element.checked : null;
1425
+ if (state === "unchecked") return typeof element?.checked === "boolean" ? !element.checked : null;
1426
+ return null;
1427
+ }
1428
+
1429
+ function applicationPostTargetCheck(condition, match, expected, field, coverageComplete) {
1430
+ if (!match && !coverageComplete) return {
1431
+ condition, matched: false, observed: null, inconclusive: true, reason: "post_target_coverage_incomplete",
1432
+ };
1433
+ const observed = field ? match?.[field] : Boolean(match);
1434
+ return { condition, matched: field ? Boolean(match) && observed === expected : observed === expected };
1435
+ }
1436
+
1437
+ function applicationVerificationEvidenceSource(condition) {
1438
+ if (condition === "process_identity") return "application_process";
1439
+ if (condition === "frontmost") return "application_workspace";
1440
+ if (condition === "visual_change") return "screenshot";
1441
+ return "application_accessibility";
1442
+ }
1443
+
1444
+ function verifyApplicationExpectation(before, after, target, expectation, diff, beforePrivateState, postPrivateState, valueProbe = null) {
1445
+ const checks = [];
1446
+ if (applicationProcessChanged(beforePrivateState, postPrivateState) === true) {
1447
+ checks.push({ condition: "process_identity", matched: false });
1448
+ }
1449
+ if (expectation.frontmost !== undefined) checks.push({
1450
+ condition: "frontmost",
1451
+ matched: after.target.frontmost === expectation.frontmost,
1452
+ });
1453
+ if (expectation.semantic_change !== undefined) checks.push({
1454
+ condition: "semantic_change",
1455
+ matched: diff?.semantic_changed === expectation.semantic_change,
1456
+ });
1457
+ if (expectation.visual_change !== undefined) {
1458
+ const observed = diff?.screenshot_changed;
1459
+ checks.push({ condition: "visual_change", matched: typeof observed === "boolean" && observed === expectation.visual_change, observed: typeof observed === "boolean" ? observed : null });
1460
+ }
1461
+ const verificationTarget = applicationVerificationTarget(target);
1462
+ if (verificationTarget) {
1463
+ const match = findPostApplicationTarget(after, postPrivateState, verificationTarget);
1464
+ const targetCoverageComplete = after.semantic?.truncated === false;
1465
+ if (expectation.target_exists !== undefined) checks.push(applicationPostTargetCheck(
1466
+ "target_exists", match, expectation.target_exists, null, targetCoverageComplete));
1467
+ if (expectation.target_enabled !== undefined) checks.push(applicationPostTargetCheck(
1468
+ "target_enabled", match, expectation.target_enabled, "enabled", targetCoverageComplete));
1469
+ if (expectation.target_focused !== undefined) checks.push(applicationPostTargetCheck(
1470
+ "target_focused", match, expectation.target_focused, "focused", targetCoverageComplete));
1471
+ if (expectation.target_checked !== undefined) checks.push(applicationPostTargetCheck(
1472
+ "target_checked", match, expectation.target_checked, "checked", targetCoverageComplete));
1473
+ if (expectation.target_selected !== undefined) checks.push(applicationPostTargetCheck(
1474
+ "target_selected", match, expectation.target_selected, "selected", targetCoverageComplete));
1475
+ if (expectation.target_expanded !== undefined) checks.push(applicationPostTargetCheck(
1476
+ "target_expanded", match, expectation.target_expanded, "expanded", targetCoverageComplete));
1477
+ if (expectation.target_value_matches !== undefined) {
1478
+ const available = valueProbe?.available === true;
1479
+ checks.push({
1480
+ condition: "target_value_matches",
1481
+ matched: available && valueProbe.matched === expectation.target_value_matches,
1482
+ observed: available ? valueProbe.matched === true : null,
1483
+ ...(available ? {} : { inconclusive: true, reason: String(valueProbe?.reason || "value_readback_unavailable") }),
1484
+ });
1485
+ }
1486
+ }
1487
+ const postChecks = checks.map((check) => ({ evidence_source: applicationVerificationEvidenceSource(check.condition), ...check }));
1488
+ const definitiveFailure = postChecks.some((check) => check.matched === false && check.inconclusive !== true);
1489
+ const inconclusive = !definitiveFailure && postChecks.some((check) => check.inconclusive === true);
1490
+ return {
1491
+ requested: true,
1492
+ matched: postChecks.length > 0 && postChecks.every((check) => check.matched),
1493
+ inconclusive,
1494
+ reason: postChecks.every((check) => check.matched)
1495
+ ? "post_conditions_satisfied"
1496
+ : inconclusive
1497
+ ? "post_conditions_inconclusive"
1498
+ : "post_conditions_not_observed",
1499
+ post_checks: postChecks,
1500
+ };
1501
+ }
1502
+
1503
+ function classifyEffectStatus({ dispatchStatus, expectation, verificationProbe, postCapture }) {
1504
+ if (!expectation) return "unknown";
1505
+ if (verificationProbe.matched === true) return "confirmed";
1506
+ if (verificationProbe.inconclusive === true) return "unknown";
1507
+ if (dispatchStatus === "unknown" || !postCapture) return "unknown";
1508
+ return "not_observed";
1509
+ }
1510
+
1511
+ function browserPostFocusQuery(target) {
1512
+ if (!target || target.kind !== "ref") return undefined;
1513
+ const element = target.element || {};
1514
+ const value = element.name || element.label || element.placeholder || element.text || element.id || "";
1515
+ return String(value).trim().slice(0, 1000) || undefined;
1516
+ }
1517
+
1518
+ function validateApplicationInspectionEvidence(value) {
1519
+ if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("application observation is invalid");
1520
+ applicationObservationString(value.process_name, 1000, false);
1521
+ if (!Array.isArray(value.elements) || value.elements.length > MAX_APPLICATION_OBSERVATION_ELEMENTS) {
1522
+ throw new Error("application observation elements are invalid");
1523
+ }
1524
+ for (const element of value.elements) {
1525
+ if (!element || typeof element !== "object" || Array.isArray(element)) throw new Error("application observation element is invalid");
1526
+ }
1527
+ for (const field of ["frontmost", "truncated", "menus_included"]) {
1528
+ if (typeof value[field] !== "boolean") throw new Error(`application observation ${field} is invalid`);
1529
+ }
1530
+ }
1531
+
1532
+ function applicationObservationString(value, maxLength, allowEmpty) {
1533
+ if (typeof value !== "string" || (!allowEmpty && !value) || value.length > maxLength || value.includes("\0")) {
1534
+ throw new Error("application observation string is invalid");
1535
+ }
1536
+ return value;
1537
+ }
1538
+
1539
+ function validateBrowserObservationForSnapshot(captured) {
1540
+ if (!captured || typeof captured !== "object" || Array.isArray(captured)) throw new Error("browser observation is invalid");
1541
+ if (!Number.isSafeInteger(captured.tab_id) || captured.tab_id < 1) throw new Error("browser observation tab id is invalid");
1542
+ browserObservationAuthorityString(captured.url, 32768, false);
1543
+ browserObservationString(captured.title, 32768, true);
1544
+ if (captured.document_epoch !== undefined) browserObservationAuthorityString(captured.document_epoch, 9000, true);
1545
+ if (captured.capture !== undefined) {
1546
+ if (!captured.capture || typeof captured.capture !== "object" || Array.isArray(captured.capture)) throw new Error("browser observation capture is invalid");
1547
+ if (captured.capture.semantic_epoch !== undefined) browserObservationAuthorityString(captured.capture.semantic_epoch, 9000, true);
1548
+ if (captured.capture.cdp_epoch !== undefined) browserObservationAuthorityString(captured.capture.cdp_epoch, 9000, true);
1549
+ }
1550
+ if (captured.semantic === undefined) return;
1551
+ if (!captured.semantic || typeof captured.semantic !== "object" || Array.isArray(captured.semantic)) {
1552
+ throw new Error("browser observation semantic payload is invalid");
1553
+ }
1554
+ if (captured.semantic.tab_id !== undefined) {
1555
+ if (!Number.isSafeInteger(captured.semantic.tab_id) || captured.semantic.tab_id < 1) throw new Error("browser observation tab id is invalid");
1556
+ }
1557
+ if (captured.semantic.url !== undefined) browserObservationAuthorityString(captured.semantic.url, 32768, false);
1558
+ if (captured.semantic.title !== undefined) browserObservationString(captured.semantic.title, 32768, true);
1559
+ if (typeof captured.semantic.frames_truncated !== "boolean") {
1560
+ throw new Error("browser observation truncation evidence is invalid");
1561
+ }
1562
+ if (captured.semantic.frames === undefined) return;
1563
+ if (!Array.isArray(captured.semantic.frames)) throw new Error("browser observation semantic frames are invalid");
1564
+ for (const frame of captured.semantic.frames) {
1565
+ if (!frame || typeof frame !== "object" || Array.isArray(frame) || !Number.isSafeInteger(frame.frame_id) || frame.frame_id < 0) {
1566
+ throw new Error("browser observation frame authority is invalid");
1567
+ }
1568
+ if (typeof frame.truncated !== "boolean") {
1569
+ throw new Error("browser observation truncation evidence is invalid");
1570
+ }
1571
+ if (frame.document === undefined) continue;
1572
+ if (!frame.document || typeof frame.document !== "object" || Array.isArray(frame.document)) {
1573
+ throw new Error("browser observation frame authority is invalid");
1574
+ }
1575
+ if (frame.document.epoch !== undefined) browserObservationAuthorityString(frame.document.epoch, 9000, true);
1576
+ if (frame.document.url !== undefined) browserObservationAuthorityString(frame.document.url, 32768, false);
1577
+ }
1578
+ }
1579
+
1580
+ function browserObservationAuthorityString(value, maxLength, allowEmpty) {
1581
+ if (typeof value !== "string" || (!allowEmpty && !value) || value.length > maxLength || value.includes("\0")) {
1582
+ throw new Error("browser observation authority string is invalid");
1583
+ }
1584
+ return value;
1585
+ }
1586
+
1587
+ function browserObservationString(value, maxLength, allowEmpty) {
1588
+ if (typeof value !== "string" || (!allowEmpty && !value) || value.length > maxLength || value.includes("\0")) {
1589
+ throw new Error("browser observation string is invalid");
1590
+ }
1591
+ return value;
1592
+ }
1593
+
1594
+ function browserObservationArgs(args) {
1595
+ return {
1596
+ tab_id: optionalPositiveInt(args.tab_id, "tab_id"),
1597
+ max_elements: clampInt(args.max_elements, 300, 1, 1000),
1598
+ max_ax_nodes: clampInt(args.max_ax_nodes, 600, 1, 2000),
1599
+ max_frames: clampInt(args.max_frames, 32, 1, 64),
1600
+ ax_depth: clampInt(args.ax_depth, 12, 1, 16),
1601
+ include_values: optionalBoolean(args.include_values, "include_values", false),
1602
+ all_frames: optionalBoolean(args.all_frames, "all_frames", true),
1603
+ include_screenshot: optionalBoolean(args.include_screenshot, "include_screenshot", true),
1604
+ screenshot_format: computerScreenshotFormat(args.screenshot_format),
1605
+ screenshot_quality: clampInt(args.screenshot_quality, 90, 1, 100),
1606
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
1607
+ focus_query: args.focus_query === undefined ? undefined : requiredStringAllowEmpty(args.focus_query, "focus_query", 1000),
1608
+ };
1609
+ }
1610
+
1611
+ function findApplicationMatch(elements, selector, occurrence) {
1612
+ const matches = elements.filter((element) => applicationMatchesSelector(element, selector));
1613
+ return matches[occurrence] || null;
1614
+ }
1615
+
1616
+ function findPostApplicationTarget(observation, privateState, target) {
1617
+ const record = findPostApplicationTargetBinding(observation, privateState, target);
1618
+ if (record) return record.element;
1619
+ if (privateState?.application_ref_bindings?.entries) return null;
1620
+ return findApplicationMatch(observation.semantic?.elements || [], target.selector, target.occurrence);
1621
+ }
1622
+
1623
+ function findPostApplicationTargetBinding(observation, privateState, target) {
1624
+ const bindings = privateState?.application_ref_bindings;
1625
+ if (bindings?.entries) {
1626
+ const expectedSelector = JSON.stringify(target.selector || {});
1627
+ for (const [ref, binding] of bindings.entries()) {
1628
+ if (binding?.occurrence !== target.occurrence || JSON.stringify(binding?.selector || {}) !== expectedSelector) continue;
1629
+ if (!applicationPostOwnerWindowContinues(target?.owner_window_bounds, binding?.owner_window_bounds)) return null;
1630
+ const element = (observation.semantic?.elements || []).find((candidate) => candidate?.ref === ref) || null;
1631
+ if (element) return { ref, element, binding };
1632
+ }
1633
+ }
1634
+ return null;
1635
+ }
1636
+
1637
+ function applicationPostOwnerWindowContinues(beforeBounds, afterBounds) {
1638
+ const before = normalizedWindowBounds(beforeBounds);
1639
+ const after = normalizedWindowBounds(afterBounds);
1640
+ if (!before && !after) return true;
1641
+ if (!before || !after) return false;
1642
+ return ["x", "y", "width", "height"].every((key) => Math.abs(before[key] - after[key]) <= 1);
1643
+ }
1644
+
1645
+ function sanitizeDispatchResult(surface, value) {
1646
+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;
1647
+ const point = value.point ? sanitizeDispatchPoint(value.point) : null;
1648
+ const destinationPoint = value.destination_point ? sanitizeDispatchPoint(value.destination_point) : null;
1649
+ const scrollDelta = sanitizeDispatchScrollDelta(value.scroll_delta);
1650
+ if (surface === "browser") {
1651
+ return {
1652
+ ok: value.ok === true,
1653
+ input_mode: typeof value.input_mode === "string" ? value.input_mode : "",
1654
+ trusted_input_fallback: value.trusted_input_fallback === true,
1655
+ coordinate_source: typeof value.coordinate_source === "string" ? value.coordinate_source : "",
1656
+ cross_frame_trusted: value.cross_frame_trusted === true,
1657
+ no_input_required: value.no_input_required === true,
1658
+ tab_id: Number.isSafeInteger(value.tab_id) && value.tab_id > 0 ? value.tab_id : null,
1659
+ url: typeof value.url === "string" ? value.url : "",
1660
+ title: typeof value.title === "string" ? value.title : "",
1661
+ tab_metadata_verified: typeof value.tab_metadata_verified === "boolean" ? value.tab_metadata_verified : null,
1662
+ ...(point ? { point } : {}),
1663
+ ...(destinationPoint ? { destination_point: destinationPoint } : {}),
1664
+ ...(scrollDelta ? { scroll_delta: scrollDelta } : {}),
1665
+ ...(value.hit ? { hit: sanitizeVisualHit(value.hit) } : {}),
1666
+ ...(value.destination_hit ? { destination_hit: sanitizeVisualHit(value.destination_hit) } : {}),
1667
+ };
1668
+ }
1669
+ const normalizedPoint = sanitizeNormalizedResultPoint(value.normalized_point);
1670
+ const destinationNormalizedPoint = sanitizeNormalizedResultPoint(value.destination_normalized_point);
1671
+ return {
1672
+ ok: value.ok === true,
1673
+ matched: Number.isSafeInteger(value.matched) && value.matched >= 0 ? value.matched : 0,
1674
+ selected_index: Number.isSafeInteger(value.selected_index) && value.selected_index >= 0 ? value.selected_index : null,
1675
+ no_input_required: value.no_input_required === true,
1676
+ checked_before: typeof value.checked_before === "boolean" ? value.checked_before : null,
1677
+ checked_after: typeof value.checked_after === "boolean" ? value.checked_after : null,
1678
+ coordinate_source: typeof value.coordinate_source === "string" ? value.coordinate_source : "",
1679
+ input_transport: typeof value.input_transport === "string" ? value.input_transport : "",
1680
+ focus_prepared: value.focus_prepared === true,
1681
+ window_bound: value.window_bound === true,
1682
+ screenshot_revalidated: value.screenshot_revalidated === true,
1683
+ experimental_backend: value.experimental_backend === true,
1684
+ focus_without_raise: value.focus_without_raise === true,
1685
+ front_window_validated: value.front_window_validated === true,
1686
+ cursor_preserved: typeof value.cursor_preserved === "boolean" ? value.cursor_preserved : null,
1687
+ frontmost_restored: value.frontmost_restored === true,
1688
+ ...(normalizedPoint ? { normalized_point: normalizedPoint } : {}),
1689
+ ...(destinationNormalizedPoint ? { destination_normalized_point: destinationNormalizedPoint } : {}),
1690
+ ...(scrollDelta ? { scroll_delta: scrollDelta } : {}),
1691
+ ...(Array.isArray(value.semantic_point_candidates) ? { semantic_point_candidates: value.semantic_point_candidates.slice(0, 4).map(sanitizeApplicationPointCandidate) } : {}),
1692
+ };
1693
+ }
1694
+
1695
+ function sanitizeDispatchScrollDelta(value) {
1696
+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;
1697
+ const deltaX = value.delta_x;
1698
+ const deltaY = value.delta_y;
1699
+ if (typeof deltaX !== "number" || !Number.isFinite(deltaX) || typeof deltaY !== "number" || !Number.isFinite(deltaY)) return null;
1700
+ return { delta_x: Object.is(deltaX, -0) ? 0 : deltaX, delta_y: Object.is(deltaY, -0) ? 0 : deltaY };
1701
+ }
1702
+
1703
+ function sanitizeNormalizedResultPoint(value) {
1704
+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;
1705
+ const x = value.x;
1706
+ const y = value.y;
1707
+ if (typeof x !== "number" || !Number.isFinite(x) || x < 0 || x >= 1
1708
+ || typeof y !== "number" || !Number.isFinite(y) || y < 0 || y >= 1) return null;
1709
+ return { x: Object.is(x, -0) ? 0 : x, y: Object.is(y, -0) ? 0 : y };
1710
+ }
1711
+
1712
+ async function trySnapshotBoundBackendAction({ browser, observation, action, target, args, context, inputMode, navigationWait }) {
1713
+ const backendTextEligible = !BROWSER_BACKEND_TEXT_ACTIONS.has(action) || target?.element?.editable !== false;
1714
+ if (target?.kind !== "ref" || !target.trusted_binding || !BROWSER_BACKEND_TRUSTED_ACTIONS.has(action) || !backendTextEligible || inputMode === "dom") return null;
1715
+ try {
1716
+ const result = await browser.backendNodeAction({
1717
+ tab_id: observation.target.tab_id,
1718
+ action,
1719
+ backend_node_id: target.trusted_binding.backend_node_id,
1720
+ extension_frame_id: target.trusted_binding.extension_frame_id,
1721
+ frame_document_epoch: target.trusted_binding.extension_frame_epoch,
1722
+ frame_url: target.trusted_binding.extension_frame_url,
1723
+ extension_ref: target.ref,
1724
+ expected_ref_identity: browserSnapshotIdentity(target.element),
1725
+ document_epoch: browserObservationSemanticEpoch(observation),
1726
+ value: args.value,
1727
+ value_resource: args.value_resource,
1728
+ key: args.key,
1729
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
1730
+ }, context);
1731
+ if (navigationWait !== "none") {
1732
+ try {
1733
+ await browser.wait({
1734
+ tab_id: observation.target.tab_id,
1735
+ load_state: navigationWait,
1736
+ timeout_seconds: clampInt(args.timeout_seconds, 30, 1, 60),
1737
+ }, context);
1738
+ } catch (error) {
1739
+ throw browserPostDispatchWaitUnknown(error);
1740
+ }
1741
+ }
1742
+ return result;
1743
+ } catch (error) {
1744
+ const message = String(error?.message || error);
1745
+ if (message.includes("snapshot backend target changed before trusted input; observe again")
1746
+ || message.includes("snapshot_backend_target_changed_before_dispatch")) {
1747
+ throw staleSnapshot("browser semantic target changed before trusted input");
1748
+ }
1749
+ if (message.includes("outcome is unknown")) throw error;
1750
+ if (message.includes("trusted browser input may have been partially dispatched")) throw error;
1751
+ if (inputMode === "trusted") throw backendTrustedUnavailable();
1752
+ return null;
1753
+ }
1754
+ }
1755
+
1756
+ function browserSnapshotDispatchError(error) {
1757
+ const message = String(error?.message || error);
1758
+ if (message.includes("snapshot ref identity changed before dispatch; observe again")
1759
+ || message.includes("snapshot_ref_identity_changed_before_dispatch")) {
1760
+ return staleSnapshot("browser semantic target identity changed before input");
1761
+ }
1762
+ if (message.includes("snapshot browser tab changed before navigation dispatch; observe again")) {
1763
+ return staleSnapshot("browser tab URL changed immediately before the navigation action");
1764
+ }
1765
+ if (message.includes("snapshot browser tab could not be verified before navigation dispatch; observe again")) {
1766
+ return new BridgeError("unavailable", "cannot validate the browser tab immediately before this navigation action; observe again", {
1767
+ details: { reason: "browser_navigation_last_hop_unavailable" },
1768
+ });
1769
+ }
1770
+ if (message.includes("snapshot history document changed before dispatch; observe again")) {
1771
+ return staleSnapshot("browser document changed immediately before the history action");
1772
+ }
1773
+ if (message.includes("snapshot history document could not be verified before dispatch; observe again")) {
1774
+ return new BridgeError("unavailable", "cannot validate the browser document immediately before this history action; observe again", {
1775
+ details: { reason: "browser_document_last_hop_unavailable" },
1776
+ });
1777
+ }
1778
+ if (message.includes("snapshot history entry changed before dispatch; observe again")) {
1779
+ return staleSnapshot("browser history entry changed immediately before the history action");
1780
+ }
1781
+ if (message.includes("snapshot history entry could not be verified before dispatch; observe again")) {
1782
+ return new BridgeError("unavailable", "cannot validate the browser history entry immediately before this history action; observe again", {
1783
+ details: { reason: "browser_history_last_hop_unavailable" },
1784
+ });
1785
+ }
1786
+ if (message.includes("snapshot browser history has no back entry before dispatch; observe again")
1787
+ || message.includes("snapshot browser history has no forward entry before dispatch; observe again")) {
1788
+ return new BridgeError("unavailable", "the requested browser history direction is unavailable in this snapshot; observe again", {
1789
+ details: { reason: "browser_history_direction_unavailable" },
1790
+ });
1791
+ }
1792
+ if (message.includes("snapshot history mutation API is unavailable before dispatch; observe again")) {
1793
+ return new BridgeError("unavailable", "the snapshot-bound browser history mutation API is unavailable; observe again", {
1794
+ details: { reason: "browser_history_mutation_api_unavailable" },
1795
+ });
1796
+ }
1797
+ return error instanceof Error ? error : new Error(message);
1798
+ }
1799
+
1800
+ function backendTrustedUnavailable() {
1801
+ return new BridgeError("unavailable", "snapshot-bound trusted input is unavailable for this semantic target", {
1802
+ details: { reason: "snapshot_backend_trusted_input_unavailable" },
1803
+ });
1804
+ }
1805
+
1806
+ function browserPostDispatchWaitUnknown(error) {
1807
+ return new BridgeError(errorCode(error), "browser action may have been dispatched; the action outcome is unknown because post-dispatch wait failed. Inspect the page before retrying.", {
1808
+ cause: error instanceof Error ? error : undefined,
1809
+ });
1810
+ }
1811
+
1812
+ function sanitizeDispatchPoint(point) {
1813
+ if (!point || typeof point !== "object" || Array.isArray(point)) return null;
1814
+ if (typeof point.normalized_x === "number" && Number.isFinite(point.normalized_x) && point.normalized_x >= 0 && point.normalized_x < 1
1815
+ && typeof point.normalized_y === "number" && Number.isFinite(point.normalized_y) && point.normalized_y >= 0 && point.normalized_y < 1
1816
+ && typeof point.css_x === "number" && Number.isFinite(point.css_x)
1817
+ && typeof point.css_y === "number" && Number.isFinite(point.css_y)) {
1818
+ return { normalized_x: point.normalized_x, normalized_y: point.normalized_y, css_x: point.css_x, css_y: point.css_y };
1819
+ }
1820
+ if (typeof point.x === "number" && Number.isFinite(point.x) && typeof point.y === "number" && Number.isFinite(point.y)) {
1821
+ return { css_x: point.x, css_y: point.y };
1822
+ }
1823
+ return null;
1824
+ }
1825
+
1826
+ function staleSnapshot(detail) {
1827
+ return new BridgeError("conflict", `computer snapshot is stale: ${detail}; observe again before acting`, {
1828
+ details: { reason: "stale_snapshot" },
1829
+ });
1830
+ }
1831
+
1832
+ function optionalPrivateApplicationProcessId(value) {
1833
+ return Number.isSafeInteger(value) && value > 0 ? value : null;
1834
+ }
1835
+
1836
+ function requiredPrivateApplicationProcessId(value) {
1837
+ const processId = optionalPrivateApplicationProcessId(value);
1838
+ if (!processId) {
1839
+ throw new BridgeError("unavailable", "application process identity is unavailable for snapshot-bound Computer Use", {
1840
+ details: { reason: "application_process_identity_unavailable" },
1841
+ });
1842
+ }
1843
+ return processId;
1844
+ }
1845
+
1846
+ function optionalPrivateApplicationProcessGeneration(value) {
1847
+ if (typeof value !== "string" || !value || value.length > 2048 || /[\r\n\0]/.test(value)) return null;
1848
+ return value;
1849
+ }
1850
+
1851
+ function requiredPrivateApplicationProcessGeneration(value) {
1852
+ const generation = optionalPrivateApplicationProcessGeneration(value);
1853
+ if (!generation) {
1854
+ throw new BridgeError("unavailable", "application process generation is unavailable for snapshot-bound Computer Use", {
1855
+ details: { reason: "application_process_identity_unavailable" },
1856
+ });
1857
+ }
1858
+ return generation;
1859
+ }
1860
+
1861
+ function applicationProcessChanged(beforePrivateState, afterPrivateState) {
1862
+ const beforeId = optionalPrivateApplicationProcessId(beforePrivateState?.application_process_id);
1863
+ const afterId = optionalPrivateApplicationProcessId(afterPrivateState?.application_process_id);
1864
+ const beforeGeneration = optionalPrivateApplicationProcessGeneration(beforePrivateState?.application_process_generation);
1865
+ const afterGeneration = optionalPrivateApplicationProcessGeneration(afterPrivateState?.application_process_generation);
1866
+ if (!beforeId || !afterId || !beforeGeneration || !afterGeneration) return null;
1867
+ return beforeId !== afterId || beforeGeneration !== afterGeneration;
1868
+ }
1869
+
1870
+ function applicationProcessIdentityError(error) {
1871
+ const message = String(error?.message || error);
1872
+ return message.includes("application process changed before operation")
1873
+ || message.includes("application process generation changed before operation");
1874
+ }
1875
+
1876
+ function applicationProcessIdentityMatches(value, expectedProcessId, expectedProcessGeneration) {
1877
+ return requiredPrivateApplicationProcessId(value?._machine_process_id) === expectedProcessId
1878
+ && requiredPrivateApplicationProcessGeneration(value?._machine_process_generation) === expectedProcessGeneration;
1879
+ }
1880
+
1881
+ function isVisualPointTarget(value) {
1882
+ return Boolean(value && typeof value === "object" && !Array.isArray(value) && Object.hasOwn(value, "point"));
1883
+ }
1884
+
1885
+ function browserPointTarget(observation, action, value) {
1886
+ if (!BROWSER_POINT_ACTIONS.has(action)) throw new BridgeError("invalid_request", `visual point target is not valid for browser ${action}`);
1887
+ const keys = Object.keys(value || {});
1888
+ if (keys.length !== 1 || keys[0] !== "point") throw new BridgeError("invalid_request", "visual target accepts only point");
1889
+ const point = value.point;
1890
+ if (!point || typeof point !== "object" || Array.isArray(point)) throw new BridgeError("invalid_request", "target.point must be an object");
1891
+ const pointKeys = Object.keys(point);
1892
+ if (pointKeys.some((key) => !["x", "y", "space"].includes(key)) || !pointKeys.includes("x") || !pointKeys.includes("y")) {
1893
+ throw new BridgeError("invalid_request", "target.point requires only x, y, and optional space");
1894
+ }
1895
+ const space = point.space === undefined ? "normalized_viewport" : point.space;
1896
+ if (typeof space !== "string" || space !== "normalized_viewport") throw new BridgeError("invalid_request", "target.point.space must be normalized_viewport");
1897
+ const x = normalizedPointCoordinate(point.x, "target.point.x");
1898
+ const y = normalizedPointCoordinate(point.y, "target.point.y");
1899
+ if (observation.capture?.screenshot !== true || observation.capture?.screenshot_source !== "cdp_surface" || observation.capture?.navigation_coherent !== true) {
1900
+ throw new BridgeError("conflict", "visual point action requires a navigation-coherent CDP screenshot from this snapshot", {
1901
+ details: { reason: "visual_snapshot_not_actionable" },
1902
+ });
1903
+ }
1904
+ const screenshotDigest = observation.capture?.screenshot_sha256;
1905
+ const screenshotSha256 = typeof screenshotDigest === "string" ? screenshotDigest.toLowerCase() : "";
1906
+ if (!/^[a-f0-9]{64}$/.test(screenshotSha256)) {
1907
+ throw new BridgeError("conflict", "visual point action requires the screenshot fingerprint from this snapshot", {
1908
+ details: { reason: "visual_snapshot_fingerprint_missing" },
1909
+ });
1910
+ }
1911
+ const viewport = observation.semantic?.viewport;
1912
+ if (!positiveNativeViewport(viewport)) {
1913
+ throw new BridgeError("conflict", "visual point action requires viewport metadata from the screenshot snapshot", {
1914
+ details: { reason: "visual_viewport_missing" },
1915
+ });
1916
+ }
1917
+ return {
1918
+ kind: "point",
1919
+ normalized_x: x,
1920
+ normalized_y: y,
1921
+ viewport: { width: viewport.width, height: viewport.height, scale: viewport.scale },
1922
+ screenshot_sha256: screenshotSha256,
1923
+ screenshot_format: observation.capture?.screenshot_format === "jpeg" ? "jpeg" : "png",
1924
+ screenshot_quality: validPointScreenshotQuality(observation.capture?.screenshot_quality),
1925
+ };
1926
+ }
1927
+
1928
+ function applicationPointTarget(observation, privateState, action, value) {
1929
+ if (!["click", "double_click", "drag", "scroll"].includes(action)) throw new BridgeError("invalid_request", `visual point target is not valid for application ${action}`);
1930
+ const keys = Object.keys(value || {});
1931
+ if (keys.length !== 1 || keys[0] !== "point") throw new BridgeError("invalid_request", "visual target accepts only point");
1932
+ const point = value.point;
1933
+ if (!point || typeof point !== "object" || Array.isArray(point)) throw new BridgeError("invalid_request", "target.point must be an object");
1934
+ const pointKeys = Object.keys(point);
1935
+ if (pointKeys.some((key) => !["x", "y", "space"].includes(key)) || !pointKeys.includes("x") || !pointKeys.includes("y")) {
1936
+ throw new BridgeError("invalid_request", "target.point requires only x, y, and optional space");
1937
+ }
1938
+ const space = point.space === undefined ? "normalized_viewport" : point.space;
1939
+ if (typeof space !== "string" || space !== "normalized_viewport") throw new BridgeError("invalid_request", "target.point.space must be normalized_viewport");
1940
+ const x = normalizedPointCoordinate(point.x, "target.point.x");
1941
+ const y = normalizedPointCoordinate(point.y, "target.point.y");
1942
+ const binding = privateState?.application_window_binding || null;
1943
+ if (observation.capture?.screenshot !== true || observation.capture?.screenshot_source !== "macos_window" || !binding) {
1944
+ throw new BridgeError("conflict", "application visual point action requires the exact macOS window screenshot from this snapshot", {
1945
+ details: { reason: "application_visual_snapshot_not_actionable" },
1946
+ });
1947
+ }
1948
+ const candidates = applicationPointCandidates(observation, binding, x, y);
1949
+ const semanticDelivery = action === "click" && observation.capabilities?.snapshot_bound_semantic_points === true
1950
+ ? applicationSemanticPointDelivery(observation, privateState, candidates)
1951
+ : null;
1952
+ if (!semanticDelivery && observation.capabilities?.snapshot_bound_visual_points !== true) {
1953
+ throw new BridgeError("conflict", "application point did not resolve uniquely to an actionable Accessibility element and no snapshot-bound pixel backend is available", {
1954
+ details: { reason: "application_visual_snapshot_not_actionable" },
1955
+ });
1956
+ }
1957
+ return {
1958
+ kind: "point",
1959
+ normalized_x: x,
1960
+ normalized_y: y,
1961
+ window_binding: binding,
1962
+ semantic_delivery: semanticDelivery,
1963
+ semantic_point_candidates: candidates,
1964
+ };
1965
+ }
1966
+
1967
+ function applicationSemanticPointDelivery(observation, privateState, candidates) {
1968
+ const bindings = privateState?.application_ref_bindings;
1969
+ if (!bindings?.get) return null;
1970
+ const matches = [];
1971
+ for (const candidate of candidates) {
1972
+ const ref = typeof candidate?.ref === "string" ? candidate.ref : "";
1973
+ const element = (observation.semantic?.elements || []).find((item) => item?.ref === ref) || null;
1974
+ const binding = ref ? bindings.get(ref) || null : null;
1975
+ if (!element || !binding || !applicationElementSupportsPointClick(element)) continue;
1976
+ matches.push({
1977
+ ref,
1978
+ element,
1979
+ selector: binding.selector,
1980
+ occurrence: binding.occurrence,
1981
+ inspection: privateState?.application_inspection || null,
1982
+ owner_window_bounds: binding.owner_window_bounds || null,
1983
+ screen_box: binding.screen_box || null,
1984
+ });
1985
+ }
1986
+ return matches.length === 1 ? matches[0] : null;
1987
+ }
1988
+
1989
+ function applicationPointCandidates(observation, binding, normalizedX, normalizedY) {
1990
+ const windowBounds = normalizedWindowBounds(binding?.bounds);
1991
+ if (!windowBounds) return [];
1992
+ const x = normalizedX * windowBounds.width;
1993
+ const y = normalizedY * windowBounds.height;
1994
+ const candidates = [];
1995
+ for (const element of observation.semantic?.elements || []) {
1996
+ const box = normalizedWindowBounds(element?.bounding_box);
1997
+ if (!box || x < box.x || y < box.y || x > box.x + box.width || y > box.y + box.height) continue;
1998
+ candidates.push({
1999
+ ref: typeof element.ref === "string" ? element.ref : "",
2000
+ role: typeof element.role === "string" ? element.role : "",
2001
+ name: typeof element.name === "string" ? element.name.slice(0, 500)
2002
+ : typeof element.title === "string" ? element.title.slice(0, 500)
2003
+ : typeof element.description === "string" ? element.description.slice(0, 500) : "",
2004
+ sensitive: element.sensitive === true,
2005
+ bounding_box: box,
2006
+ area: box.width * box.height,
2007
+ });
2008
+ }
2009
+ candidates.sort((left, right) => left.area - right.area || left.ref.localeCompare(right.ref));
2010
+ return candidates.slice(0, 4).map(({ area: _area, ...candidate }) => candidate);
2011
+ }
2012
+
2013
+ function currentApplicationSemanticPointCandidates(elements, target) {
2014
+ const windowBounds = normalizedWindowBounds(target?.window_binding?.bounds);
2015
+ const normalizedX = target?.normalized_x;
2016
+ const normalizedY = target?.normalized_y;
2017
+ if (!windowBounds || typeof normalizedX !== "number" || !Number.isFinite(normalizedX)
2018
+ || typeof normalizedY !== "number" || !Number.isFinite(normalizedY)) return [];
2019
+ const screenX = windowBounds.x + normalizedX * windowBounds.width;
2020
+ const screenY = windowBounds.y + normalizedY * windowBounds.height;
2021
+ const candidates = [];
2022
+ for (const element of elements || []) {
2023
+ if (!applicationElementSupportsPointClick(element)) continue;
2024
+ const ownerBounds = normalizedWindowBounds(element?.window_screen_box);
2025
+ if (!ownerBounds || !sameApplicationBounds(ownerBounds, windowBounds)) continue;
2026
+ const box = normalizedWindowBounds(element?.screen_box);
2027
+ if (!box || screenX < box.x || screenY < box.y || screenX > box.x + box.width || screenY > box.y + box.height) continue;
2028
+ candidates.push(element);
2029
+ }
2030
+ return candidates;
2031
+ }
2032
+
2033
+ function applicationScreenshotBytes(value) {
2034
+ if (typeof value !== "string" || !value || value.length > Math.ceil(MAX_APPLICATION_SCREENSHOT_BYTES / 3) * 4) return null;
2035
+ const bytes = Buffer.from(value, "base64");
2036
+ if (bytes.length < 8 || bytes.length > MAX_APPLICATION_SCREENSHOT_BYTES || bytes.toString("base64") !== value) return null;
2037
+ return bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47
2038
+ && bytes[4] === 0x0d && bytes[5] === 0x0a && bytes[6] === 0x1a && bytes[7] === 0x0a ? bytes : null;
2039
+ }
2040
+
2041
+ function applicationWindowBinding(captured, screenshotSha256) {
2042
+ const screenshot = captured?.screenshot;
2043
+ const windowId = captured?.window?.id;
2044
+ const bounds = normalizedWindowBounds(captured?.window?.bounds || screenshot?.bounds);
2045
+ const digest = typeof screenshotSha256 === "string" ? screenshotSha256.toLowerCase() : "";
2046
+ if (screenshot?.source !== "macos_window" || screenshot?.mime_type !== "image/png" || typeof screenshot?.data !== "string"
2047
+ || !Number.isSafeInteger(windowId) || windowId < 1 || !bounds || !/^[a-f0-9]{64}$/.test(digest)) return null;
2048
+ return { window_id: windowId, bounds, screenshot_sha256: digest };
2049
+ }
2050
+
2051
+ function sameApplicationWindowBinding(expected, current) {
2052
+ if (!expected || !current || expected.window_id !== current.window_id || expected.screenshot_sha256 !== current.screenshot_sha256) return false;
2053
+ return sameApplicationBounds(normalizedWindowBounds(expected.bounds), normalizedWindowBounds(current.bounds));
2054
+ }
2055
+
2056
+ function sameApplicationWindowIdentity(expected, current) {
2057
+ const windowId = current?.window?.id ?? current?.id;
2058
+ const bounds = normalizedWindowBounds(current?.window?.bounds ?? current?.bounds);
2059
+ if (!expected || !Number.isSafeInteger(windowId) || windowId !== expected.window_id || !bounds) return false;
2060
+ return sameApplicationBounds(normalizedWindowBounds(expected.bounds), bounds);
2061
+ }
2062
+
2063
+ function applicationCaptureCoherence({ screenshotRequested, hasImage, windowCoherent }) {
2064
+ if (!screenshotRequested) return "single_accessibility_inspection";
2065
+ if (!hasImage) return "accessibility_only_after_screenshot_failure";
2066
+ if (windowCoherent === true) return "window_screenshot_then_accessibility_window_stable";
2067
+ if (windowCoherent === false) return "window_changed_during_capture";
2068
+ return "window_screenshot_then_accessibility_unverified";
2069
+ }
2070
+
2071
+ function applicationTargetInspection(target) {
2072
+ const inspection = target?.semantic_delivery?.inspection || target?.inspection || null;
2073
+ return {
2074
+ max_depth: clampInt(inspection?.max_depth, 6, 1, 12),
2075
+ include_menus: inspection?.include_menus === true,
2076
+ };
2077
+ }
2078
+
2079
+ function applicationTargetGeometry(target) {
2080
+ const source = target?.semantic_delivery || target || {};
2081
+ return {
2082
+ owner_window_bounds: normalizedWindowBounds(source.owner_window_bounds),
2083
+ screen_box: normalizedWindowBounds(source.screen_box),
2084
+ };
2085
+ }
2086
+
2087
+ function applicationTargetHasGeometry(target) {
2088
+ const geometry = applicationTargetGeometry(target);
2089
+ return Boolean(geometry.owner_window_bounds || geometry.screen_box);
2090
+ }
2091
+
2092
+ function applicationTargetGeometryMatches(element, target) {
2093
+ const expected = applicationTargetGeometry(target);
2094
+ if (expected.owner_window_bounds && !sameApplicationBounds(expected.owner_window_bounds, normalizedWindowBounds(element?.window_screen_box))) return false;
2095
+ if (expected.screen_box && !sameApplicationBounds(expected.screen_box, normalizedWindowBounds(element?.screen_box))) return false;
2096
+ return true;
2097
+ }
2098
+
2099
+ function applyApplicationTargetGeometry(call, target) {
2100
+ const geometry = applicationTargetGeometry(target);
2101
+ if (geometry.owner_window_bounds) call.expected_window_bounds = geometry.owner_window_bounds;
2102
+ if (geometry.screen_box) call.expected_element_bounds = geometry.screen_box;
2103
+ return call;
2104
+ }
2105
+
2106
+ function sameApplicationBounds(left, right) {
2107
+ if (!left || !right) return false;
2108
+ return ["x", "y", "width", "height"].every((key) => {
2109
+ const a = left[key];
2110
+ const b = right[key];
2111
+ return typeof a === "number" && Number.isFinite(a) && typeof b === "number" && Number.isFinite(b) && Math.abs(a - b) <= 1;
2112
+ });
2113
+ }
2114
+
2115
+ function normalizedWindowBounds(value) {
2116
+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;
2117
+ const bounds = {};
2118
+ for (const key of ["x", "y", "width", "height"]) {
2119
+ const number = value[key];
2120
+ if (typeof number !== "number" || !Number.isFinite(number)) return null;
2121
+ bounds[key] = number;
2122
+ }
2123
+ return bounds.width > 0 && bounds.height > 0 ? bounds : null;
2124
+ }
2125
+
2126
+ function normalizedPointCoordinate(value, label) {
2127
+ if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value >= 1) {
2128
+ throw new BridgeError("invalid_request", `${label} must be from 0 (inclusive) to 1 (exclusive)`);
2129
+ }
2130
+ return value;
2131
+ }
2132
+
2133
+ function validPointScreenshotQuality(value) {
2134
+ return Number.isInteger(value) && value >= 1 && value <= 100 ? value : 90;
2135
+ }
2136
+
2137
+ function positiveNativeViewport(value) {
2138
+ return Boolean(value && typeof value === "object" && !Array.isArray(value)
2139
+ && typeof value.width === "number" && Number.isFinite(value.width) && value.width > 0
2140
+ && typeof value.height === "number" && Number.isFinite(value.height) && value.height > 0
2141
+ && typeof value.scale === "number" && Number.isFinite(value.scale) && value.scale > 0);
2142
+ }
2143
+
2144
+ function sameVisualViewport(expected, current) {
2145
+ if (!positiveNativeViewport(expected) || !positiveNativeViewport(current)) return false;
2146
+ for (const key of ["width", "height", "scale"]) {
2147
+ if (Math.abs(expected[key] - current[key]) > 0.5) return false;
2148
+ }
2149
+ const offsetLeft = current.offset_left === undefined || current.offset_left === null ? 0 : current.offset_left;
2150
+ const offsetTop = current.offset_top === undefined || current.offset_top === null ? 0 : current.offset_top;
2151
+ if (typeof offsetLeft !== "number" || !Number.isFinite(offsetLeft) || typeof offsetTop !== "number" || !Number.isFinite(offsetTop)) return false;
2152
+ return Math.abs(current.scale - 1) <= 0.001 && Math.abs(offsetLeft) <= 0.5 && Math.abs(offsetTop) <= 0.5;
2153
+ }
2154
+
2155
+ function sanitizeApplicationPointCandidate(value) {
2156
+ return {
2157
+ ref: typeof value?.ref === "string" ? value.ref : "",
2158
+ role: typeof value?.role === "string" ? value.role : "",
2159
+ name: typeof value?.name === "string" ? value.name.slice(0, 500) : "",
2160
+ sensitive: value?.sensitive === true,
2161
+ bounding_box: normalizedWindowBounds(value?.bounding_box),
2162
+ };
2163
+ }
2164
+
2165
+ function sanitizeVisualHit(value) {
2166
+ return {
2167
+ ref: typeof value.ref === "string" ? value.ref : "",
2168
+ tag: typeof value.tag === "string" ? value.tag : "",
2169
+ role: typeof value.role === "string" ? value.role : "",
2170
+ name: typeof value.name === "string" ? value.name : "",
2171
+ sensitive: value.sensitive === true,
2172
+ bounding_box: normalizedWindowBounds(value.bounding_box),
2173
+ };
2174
+ }
2175
+
2176
+ function applicationPostFocusQuery(target) {
2177
+ if (!target) return undefined;
2178
+ const element = target.kind === "ref" ? target.element || {} : target.semantic_delivery?.element || {};
2179
+ const value = element.name || element.title || element.description || element.identifier || element.role || "";
2180
+ return String(value).trim().slice(0, 1000) || undefined;
2181
+ }
2182
+
2183
+ function browserSnapshotIdentity(element) {
2184
+ const output = {};
2185
+ for (const field of BROWSER_SNAPSHOT_IDENTITY_FIELDS) {
2186
+ if (!Object.hasOwn(element || {}, field)) continue;
2187
+ const value = element[field];
2188
+ const valid = BROWSER_SNAPSHOT_BOOLEAN_IDENTITY_FIELDS.has(field) ? typeof value === "boolean" : typeof value === "string";
2189
+ if (!valid) throw staleSnapshot("browser snapshot target identity is malformed");
2190
+ output[field] = value;
2191
+ }
2192
+ return Object.keys(output).length ? output : null;
2193
+ }
2194
+
2195
+ function browserSnapshotIdentityMatches(expected, element) {
2196
+ if (!expected || !element) return false;
2197
+ return Object.entries(expected).every(([field, value]) => {
2198
+ if (!Object.hasOwn(element, field)) return false;
2199
+ const actual = element[field];
2200
+ return typeof value === "boolean" ? typeof actual === "boolean" && actual === value
2201
+ : typeof actual === "string" && actual === value;
2202
+ });
2203
+ }
2204
+
2205
+ function applicationScreenshotError(error) {
2206
+ const message = String(error?.message || error || "").toLowerCase();
2207
+ if (message.includes("screen recording") || message.includes("permission") || message.includes("not authorized")) {
2208
+ return "application window screenshot unavailable; Screen Recording permission may be required";
2209
+ }
2210
+ if (message.includes("no capturable on-screen window")) return "application has no capturable on-screen window";
2211
+ return "application window screenshot unavailable";
2212
+ }
2213
+
2214
+ function applicationWindowRevalidationError(error) {
2215
+ const message = String(error?.message || error || "").toLowerCase();
2216
+ if (message.includes("accessibility") || message.includes("permission") || message.includes("not authorized")) {
2217
+ return "application window identity could not be revalidated; Accessibility permission may be required";
2218
+ }
2219
+ if (message.includes("no capturable on-screen window")) return "application window identity changed or became unavailable during capture";
2220
+ return "application window identity could not be revalidated during capture";
2221
+ }
2222
+
2223
+ function browserObservationSemanticEpoch(observation) {
2224
+ const value = observation?.capture?.semantic_epoch;
2225
+ return typeof value === "string" && value && value.length <= 9000 && !value.includes("\0") ? value : "";
2226
+ }
2227
+
2228
+ function privateBrowserHistoryEntryKey(value) {
2229
+ return typeof value === "string" && value && value.length <= 512 && !value.includes("\0") ? value : "";
2230
+ }
2231
+
2232
+ function enforceBrowserHistoryDocumentPreflight(action, semanticEpoch, expectedHistoryEntryKey, currentDocument, documentStateError) {
2233
+ if (!BROWSER_DOCUMENT_BOUND_TARGETLESS_ACTIONS.has(action)) return;
2234
+ if (!semanticEpoch || !currentDocument?.document_epoch) {
2235
+ throw new BridgeError("unavailable", "cannot validate the browser document before this snapshot-bound history action; observe again", {
2236
+ details: { reason: "browser_document_preflight_unavailable", ...(documentStateError ? { error_class: errorCode(documentStateError) } : {}) },
2237
+ });
2238
+ }
2239
+ if (!expectedHistoryEntryKey) {
2240
+ throw new BridgeError("unavailable", "cannot bind this snapshot to a browser history entry; observe again", {
2241
+ details: { reason: "browser_history_preflight_unavailable" },
2242
+ });
2243
+ }
2244
+ const currentHistoryEntryKey = privateBrowserHistoryEntryKey(currentDocument?._machine_history_entry_key);
2245
+ if (!currentHistoryEntryKey) {
2246
+ throw new BridgeError("unavailable", "cannot validate the browser history entry before this snapshot-bound history action; observe again", {
2247
+ details: { reason: "browser_history_preflight_unavailable" },
2248
+ });
2249
+ }
2250
+ if (currentHistoryEntryKey !== expectedHistoryEntryKey) throw staleSnapshot("browser history entry changed after the snapshot");
2251
+ }
2252
+
2253
+ function defaultSleep(milliseconds) {
2254
+ return new Promise((resolvePromise) => { setTimeout(resolvePromise, Math.max(0, Number(milliseconds) || 0)); });
2255
+ }