agent-web-interface 4.3.0 → 4.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (251) hide show
  1. package/dist/src/browser/connection-utils.d.ts +48 -0
  2. package/dist/src/browser/connection-utils.d.ts.map +1 -0
  3. package/dist/src/browser/connection-utils.js +129 -0
  4. package/dist/src/browser/connection-utils.js.map +1 -0
  5. package/dist/src/browser/index.d.ts +3 -1
  6. package/dist/src/browser/index.d.ts.map +1 -1
  7. package/dist/src/browser/index.js +2 -1
  8. package/dist/src/browser/index.js.map +1 -1
  9. package/dist/src/browser/session-manager.d.ts +1 -89
  10. package/dist/src/browser/session-manager.d.ts.map +1 -1
  11. package/dist/src/browser/session-manager.js +1 -116
  12. package/dist/src/browser/session-manager.js.map +1 -1
  13. package/dist/src/browser/session-manager.types.d.ts +90 -0
  14. package/dist/src/browser/session-manager.types.d.ts.map +1 -0
  15. package/dist/src/browser/session-manager.types.js +7 -0
  16. package/dist/src/browser/session-manager.types.js.map +1 -0
  17. package/dist/src/form/constraint-extraction.d.ts +31 -0
  18. package/dist/src/form/constraint-extraction.d.ts.map +1 -0
  19. package/dist/src/form/constraint-extraction.js +110 -0
  20. package/dist/src/form/constraint-extraction.js.map +1 -0
  21. package/dist/src/form/field-extractor.d.ts.map +1 -1
  22. package/dist/src/form/field-extractor.js +3 -444
  23. package/dist/src/form/field-extractor.js.map +1 -1
  24. package/dist/src/form/field-state-extractor.d.ts +22 -0
  25. package/dist/src/form/field-state-extractor.d.ts.map +1 -0
  26. package/dist/src/form/field-state-extractor.js +55 -0
  27. package/dist/src/form/field-state-extractor.js.map +1 -0
  28. package/dist/src/form/form-actions.d.ts +45 -0
  29. package/dist/src/form/form-actions.d.ts.map +1 -0
  30. package/dist/src/form/form-actions.js +108 -0
  31. package/dist/src/form/form-actions.js.map +1 -0
  32. package/dist/src/form/form-detector.d.ts +0 -36
  33. package/dist/src/form/form-detector.d.ts.map +1 -1
  34. package/dist/src/form/form-detector.js +11 -376
  35. package/dist/src/form/form-detector.js.map +1 -1
  36. package/dist/src/form/input-clustering.d.ts +15 -0
  37. package/dist/src/form/input-clustering.d.ts.map +1 -0
  38. package/dist/src/form/input-clustering.js +61 -0
  39. package/dist/src/form/input-clustering.js.map +1 -0
  40. package/dist/src/form/intent-inference.d.ts +28 -0
  41. package/dist/src/form/intent-inference.d.ts.map +1 -0
  42. package/dist/src/form/intent-inference.js +137 -0
  43. package/dist/src/form/intent-inference.js.map +1 -0
  44. package/dist/src/form/purpose-inference.d.ts +50 -0
  45. package/dist/src/form/purpose-inference.d.ts.map +1 -0
  46. package/dist/src/form/purpose-inference.js +313 -0
  47. package/dist/src/form/purpose-inference.js.map +1 -0
  48. package/dist/src/form/submit-detection.d.ts +36 -0
  49. package/dist/src/form/submit-detection.d.ts.map +1 -0
  50. package/dist/src/form/submit-detection.js +101 -0
  51. package/dist/src/form/submit-detection.js.map +1 -0
  52. package/dist/src/form/types.d.ts +2 -2
  53. package/dist/src/index.js +52 -47
  54. package/dist/src/index.js.map +1 -1
  55. package/dist/src/observation/observation-accumulator.d.ts +1 -1
  56. package/dist/src/observation/observation-accumulator.js +1 -1
  57. package/dist/src/observation/observer-script.d.ts +1 -1
  58. package/dist/src/observation/observer-script.d.ts.map +1 -1
  59. package/dist/src/observation/observer-script.js +129 -7
  60. package/dist/src/observation/observer-script.js.map +1 -1
  61. package/dist/src/query/disambiguation.d.ts +18 -0
  62. package/dist/src/query/disambiguation.d.ts.map +1 -0
  63. package/dist/src/query/disambiguation.js +123 -0
  64. package/dist/src/query/disambiguation.js.map +1 -0
  65. package/dist/src/query/fuzzy-match.d.ts +17 -0
  66. package/dist/src/query/fuzzy-match.d.ts.map +1 -0
  67. package/dist/src/query/fuzzy-match.js +34 -0
  68. package/dist/src/query/fuzzy-match.js.map +1 -0
  69. package/dist/src/query/index.d.ts +3 -0
  70. package/dist/src/query/index.d.ts.map +1 -1
  71. package/dist/src/query/index.js +6 -0
  72. package/dist/src/query/index.js.map +1 -1
  73. package/dist/src/query/query-engine.d.ts +0 -35
  74. package/dist/src/query/query-engine.d.ts.map +1 -1
  75. package/dist/src/query/query-engine.js +9 -309
  76. package/dist/src/query/query-engine.js.map +1 -1
  77. package/dist/src/query/scoring.d.ts +52 -0
  78. package/dist/src/query/scoring.d.ts.map +1 -0
  79. package/dist/src/query/scoring.js +162 -0
  80. package/dist/src/query/scoring.js.map +1 -0
  81. package/dist/src/snapshot/element-resolver.d.ts +24 -13
  82. package/dist/src/snapshot/element-resolver.d.ts.map +1 -1
  83. package/dist/src/snapshot/element-resolver.js +117 -87
  84. package/dist/src/snapshot/element-resolver.js.map +1 -1
  85. package/dist/src/snapshot/extractors/ax-extractor.d.ts +1 -1
  86. package/dist/src/snapshot/extractors/ax-extractor.d.ts.map +1 -1
  87. package/dist/src/snapshot/extractors/ax-extractor.js +4 -1
  88. package/dist/src/snapshot/extractors/ax-extractor.js.map +1 -1
  89. package/dist/src/snapshot/extractors/index.d.ts +1 -1
  90. package/dist/src/snapshot/extractors/index.d.ts.map +1 -1
  91. package/dist/src/snapshot/extractors/index.js +1 -1
  92. package/dist/src/snapshot/extractors/index.js.map +1 -1
  93. package/dist/src/snapshot/extractors/region-resolver.d.ts.map +1 -1
  94. package/dist/src/snapshot/extractors/region-resolver.js +8 -0
  95. package/dist/src/snapshot/extractors/region-resolver.js.map +1 -1
  96. package/dist/src/snapshot/extractors/types.d.ts +8 -0
  97. package/dist/src/snapshot/extractors/types.d.ts.map +1 -1
  98. package/dist/src/snapshot/extractors/types.js +16 -0
  99. package/dist/src/snapshot/extractors/types.js.map +1 -1
  100. package/dist/src/snapshot/frame-context.d.ts +68 -0
  101. package/dist/src/snapshot/frame-context.d.ts.map +1 -0
  102. package/dist/src/snapshot/frame-context.js +131 -0
  103. package/dist/src/snapshot/frame-context.js.map +1 -0
  104. package/dist/src/snapshot/heading-index.d.ts +28 -0
  105. package/dist/src/snapshot/heading-index.d.ts.map +1 -0
  106. package/dist/src/snapshot/heading-index.js +108 -0
  107. package/dist/src/snapshot/heading-index.js.map +1 -0
  108. package/dist/src/snapshot/index.d.ts +5 -3
  109. package/dist/src/snapshot/index.d.ts.map +1 -1
  110. package/dist/src/snapshot/index.js +3 -2
  111. package/dist/src/snapshot/index.js.map +1 -1
  112. package/dist/src/snapshot/kind-mapping.d.ts +30 -0
  113. package/dist/src/snapshot/kind-mapping.d.ts.map +1 -0
  114. package/dist/src/snapshot/kind-mapping.js +114 -0
  115. package/dist/src/snapshot/kind-mapping.js.map +1 -0
  116. package/dist/src/snapshot/node-filter.d.ts +31 -0
  117. package/dist/src/snapshot/node-filter.d.ts.map +1 -0
  118. package/dist/src/snapshot/node-filter.js +137 -0
  119. package/dist/src/snapshot/node-filter.js.map +1 -0
  120. package/dist/src/snapshot/node-synthesizer.d.ts +62 -0
  121. package/dist/src/snapshot/node-synthesizer.d.ts.map +1 -0
  122. package/dist/src/snapshot/node-synthesizer.js +185 -0
  123. package/dist/src/snapshot/node-synthesizer.js.map +1 -0
  124. package/dist/src/snapshot/snapshot-compiler.d.ts +2 -36
  125. package/dist/src/snapshot/snapshot-compiler.d.ts.map +1 -1
  126. package/dist/src/snapshot/snapshot-compiler.js +25 -547
  127. package/dist/src/snapshot/snapshot-compiler.js.map +1 -1
  128. package/dist/src/snapshot/snapshot.types.d.ts +7 -2
  129. package/dist/src/snapshot/snapshot.types.d.ts.map +1 -1
  130. package/dist/src/snapshot/snapshot.types.js +8 -0
  131. package/dist/src/snapshot/snapshot.types.js.map +1 -1
  132. package/dist/src/state/actionables-filter.d.ts +5 -0
  133. package/dist/src/state/actionables-filter.d.ts.map +1 -1
  134. package/dist/src/state/actionables-filter.js +21 -3
  135. package/dist/src/state/actionables-filter.js.map +1 -1
  136. package/dist/src/state/diff-engine.js +3 -3
  137. package/dist/src/state/diff-engine.js.map +1 -1
  138. package/dist/src/state/element-registry.d.ts.map +1 -1
  139. package/dist/src/state/element-registry.js +6 -4
  140. package/dist/src/state/element-registry.js.map +1 -1
  141. package/dist/src/state/hash-utils.d.ts +24 -0
  142. package/dist/src/state/hash-utils.d.ts.map +1 -0
  143. package/dist/src/state/hash-utils.js +41 -0
  144. package/dist/src/state/hash-utils.js.map +1 -0
  145. package/dist/src/state/layer-detector.d.ts.map +1 -1
  146. package/dist/src/state/layer-detector.js +15 -286
  147. package/dist/src/state/layer-detector.js.map +1 -1
  148. package/dist/src/state/layer-detectors/drawer-detector.d.ts +32 -0
  149. package/dist/src/state/layer-detectors/drawer-detector.d.ts.map +1 -0
  150. package/dist/src/state/layer-detectors/drawer-detector.js +96 -0
  151. package/dist/src/state/layer-detectors/drawer-detector.js.map +1 -0
  152. package/dist/src/state/layer-detectors/index.d.ts +10 -0
  153. package/dist/src/state/layer-detectors/index.d.ts.map +1 -0
  154. package/dist/src/state/layer-detectors/index.js +10 -0
  155. package/dist/src/state/layer-detectors/index.js.map +1 -0
  156. package/dist/src/state/layer-detectors/modal-detector.d.ts +30 -0
  157. package/dist/src/state/layer-detectors/modal-detector.d.ts.map +1 -0
  158. package/dist/src/state/layer-detectors/modal-detector.js +127 -0
  159. package/dist/src/state/layer-detectors/modal-detector.js.map +1 -0
  160. package/dist/src/state/layer-detectors/popover-detector.d.ts +20 -0
  161. package/dist/src/state/layer-detectors/popover-detector.d.ts.map +1 -0
  162. package/dist/src/state/layer-detectors/popover-detector.js +76 -0
  163. package/dist/src/state/layer-detectors/popover-detector.js.map +1 -0
  164. package/dist/src/state/layer-detectors/toast-detector.d.ts +24 -0
  165. package/dist/src/state/layer-detectors/toast-detector.d.ts.map +1 -0
  166. package/dist/src/state/layer-detectors/toast-detector.js +48 -0
  167. package/dist/src/state/layer-detectors/toast-detector.js.map +1 -0
  168. package/dist/src/state/region-mapping.d.ts +13 -0
  169. package/dist/src/state/region-mapping.d.ts.map +1 -0
  170. package/dist/src/state/region-mapping.js +25 -0
  171. package/dist/src/state/region-mapping.js.map +1 -0
  172. package/dist/src/state/state-manager.d.ts.map +1 -1
  173. package/dist/src/state/state-manager.js +8 -192
  174. package/dist/src/state/state-manager.js.map +1 -1
  175. package/dist/src/state/state-renderer.d.ts.map +1 -1
  176. package/dist/src/state/state-renderer.js +14 -2
  177. package/dist/src/state/state-renderer.js.map +1 -1
  178. package/dist/src/state/types.d.ts +8 -4
  179. package/dist/src/state/types.d.ts.map +1 -1
  180. package/dist/src/state/url-sanitization.d.ts +22 -0
  181. package/dist/src/state/url-sanitization.d.ts.map +1 -0
  182. package/dist/src/state/url-sanitization.js +60 -0
  183. package/dist/src/state/url-sanitization.js.map +1 -0
  184. package/dist/src/state/value-masking.d.ts +36 -0
  185. package/dist/src/state/value-masking.d.ts.map +1 -0
  186. package/dist/src/state/value-masking.js +86 -0
  187. package/dist/src/state/value-masking.js.map +1 -0
  188. package/dist/src/tools/action-context.d.ts +60 -0
  189. package/dist/src/tools/action-context.d.ts.map +1 -0
  190. package/dist/src/tools/action-context.js +78 -0
  191. package/dist/src/tools/action-context.js.map +1 -0
  192. package/dist/src/tools/action-stabilization.d.ts +48 -0
  193. package/dist/src/tools/action-stabilization.d.ts.map +1 -0
  194. package/dist/src/tools/action-stabilization.js +87 -0
  195. package/dist/src/tools/action-stabilization.js.map +1 -0
  196. package/dist/src/tools/browser-tools.d.ts +8 -167
  197. package/dist/src/tools/browser-tools.d.ts.map +1 -1
  198. package/dist/src/tools/browser-tools.js +13 -651
  199. package/dist/src/tools/browser-tools.js.map +1 -1
  200. package/dist/src/tools/effect-tracker.d.ts +25 -0
  201. package/dist/src/tools/effect-tracker.d.ts.map +1 -0
  202. package/dist/src/tools/effect-tracker.js +69 -0
  203. package/dist/src/tools/effect-tracker.js.map +1 -0
  204. package/dist/src/tools/execute-action.d.ts +1 -31
  205. package/dist/src/tools/execute-action.d.ts.map +1 -1
  206. package/dist/src/tools/execute-action.js +7 -276
  207. package/dist/src/tools/execute-action.js.map +1 -1
  208. package/dist/src/tools/form-tools.d.ts +2 -2
  209. package/dist/src/tools/form-tools.js +4 -4
  210. package/dist/src/tools/form-tools.js.map +1 -1
  211. package/dist/src/tools/index.d.ts +2 -2
  212. package/dist/src/tools/index.d.ts.map +1 -1
  213. package/dist/src/tools/index.js +10 -8
  214. package/dist/src/tools/index.js.map +1 -1
  215. package/dist/src/tools/interaction-tools.d.ts +46 -0
  216. package/dist/src/tools/interaction-tools.d.ts.map +1 -0
  217. package/dist/src/tools/interaction-tools.js +138 -0
  218. package/dist/src/tools/interaction-tools.js.map +1 -0
  219. package/dist/src/tools/navigation-detection.d.ts +31 -0
  220. package/dist/src/tools/navigation-detection.d.ts.map +1 -0
  221. package/dist/src/tools/navigation-detection.js +46 -0
  222. package/dist/src/tools/navigation-detection.js.map +1 -0
  223. package/dist/src/tools/navigation-tools.d.ts +57 -0
  224. package/dist/src/tools/navigation-tools.d.ts.map +1 -0
  225. package/dist/src/tools/navigation-tools.js +178 -0
  226. package/dist/src/tools/navigation-tools.js.map +1 -0
  227. package/dist/src/tools/observation-tools.d.ts +53 -0
  228. package/dist/src/tools/observation-tools.d.ts.map +1 -0
  229. package/dist/src/tools/observation-tools.js +247 -0
  230. package/dist/src/tools/observation-tools.js.map +1 -0
  231. package/dist/src/tools/response-builder.js +2 -2
  232. package/dist/src/tools/response-builder.js.map +1 -1
  233. package/dist/src/tools/stale-element-retry.d.ts +37 -0
  234. package/dist/src/tools/stale-element-retry.d.ts.map +1 -0
  235. package/dist/src/tools/stale-element-retry.js +68 -0
  236. package/dist/src/tools/stale-element-retry.js.map +1 -0
  237. package/dist/src/tools/state-manager-registry.d.ts +26 -0
  238. package/dist/src/tools/state-manager-registry.d.ts.map +1 -0
  239. package/dist/src/tools/state-manager-registry.js +39 -0
  240. package/dist/src/tools/state-manager-registry.js.map +1 -0
  241. package/dist/src/tools/tool-context.js +1 -1
  242. package/dist/src/tools/tool-context.js.map +1 -1
  243. package/dist/src/tools/tool-schemas.d.ts +106 -21
  244. package/dist/src/tools/tool-schemas.d.ts.map +1 -1
  245. package/dist/src/tools/tool-schemas.js +59 -18
  246. package/dist/src/tools/tool-schemas.js.map +1 -1
  247. package/dist/src/tools/viewport-tools.d.ts +36 -0
  248. package/dist/src/tools/viewport-tools.d.ts.map +1 -0
  249. package/dist/src/tools/viewport-tools.js +105 -0
  250. package/dist/src/tools/viewport-tools.js.map +1 -0
  251. package/package.json +1 -1
@@ -1,655 +1,17 @@
1
1
  /**
2
2
  * Browser Tools
3
3
  *
4
- * MCP tool handlers for browser automation.
5
- */
6
- import { clickByBackendNodeId, clickAtCoordinates, clickAtElementOffset, getElementTopLeft, dragBetweenCoordinates, typeByBackendNodeId, pressKey, selectOption, hoverByBackendNodeId, scrollIntoView, scrollPage as scrollPageByAmount, } from '../snapshot/index.js';
7
- import { observationAccumulator } from '../observation/index.js';
8
- import { ATTACHMENT_SIGNIFICANCE_THRESHOLD } from '../observation/observation.types.js';
9
- import { ClosePageInputSchema, CloseSessionInputSchema, NavigateInputSchema, GoBackInputSchema, GoForwardInputSchema, ReloadInputSchema, CaptureSnapshotInputSchema, FindElementsInputSchema, GetNodeDetailsInputSchema, ScrollElementIntoViewInputSchema, ScrollPageInputSchema, ClickInputSchema, TypeInputSchema, PressInputSchema, SelectInputSchema, HoverInputSchema, TakeScreenshotInputSchema, DragInputSchema, } from './tool-schemas.js';
10
- import { captureScreenshot, getElementBoundingBox } from '../screenshot/index.js';
11
- import { cleanupTempFiles } from '../lib/temp-file.js';
12
- import { QueryEngine } from '../query/query-engine.js';
13
- import { isReadableNode, isStructuralNode } from '../snapshot/snapshot.types.js';
14
- import { computeEid } from '../state/element-identity.js';
15
- import { captureWithStabilization, determineHealthCode, } from '../snapshot/snapshot-health.js';
16
- import { executeAction, executeActionWithRetry, executeActionWithOutcome, stabilizeAfterNavigation, getStateManager, removeStateManager, clearAllStateManagers, } from './execute-action.js';
17
- import { buildClosePageResponse, buildCloseSessionResponse, buildListPagesResponse, buildFindElementsResponse, buildGetElementDetailsResponse, } from './response-builder.js';
18
- import { getDependencyTracker } from '../form/index.js';
19
- import { initializeToolContext, getSessionManager, getSnapshotStore, resolveExistingPage, ensureCdpSession, requireSnapshot, resolveElementByEid, } from './tool-context.js';
20
- // Re-export for backward compatibility (external consumers import from browser-tools)
21
- export { getSnapshotStore } from './tool-context.js';
22
- /**
23
- * Initialize tools with a session manager instance.
24
- * Must be called before using any tool handlers.
25
- */
26
- export function initializeTools(manager) {
27
- initializeToolContext(manager);
28
- }
29
- // Convenience alias for module-internal use
30
- const snapshotStore = getSnapshotStore();
31
- /**
32
- * Build runtime health details from a capture attempt.
33
- */
34
- function buildRuntimeHealth(cdpHealth, result) {
35
- const code = determineHealthCode(result);
36
- return {
37
- cdp: cdpHealth,
38
- snapshot: {
39
- ok: code === 'HEALTHY',
40
- code,
41
- attempts: result.attempts,
42
- message: result.health.message,
43
- },
44
- };
45
- }
46
- /**
47
- * Capture a snapshot with stabilization and CDP recovery when empty.
48
- */
49
- async function captureSnapshotWithRecovery(session, handle, pageId) {
50
- const ensureResult = await ensureCdpSession(session, handle);
51
- handle = ensureResult.handle;
52
- let result = await captureWithStabilization(handle.cdp, handle.page, pageId);
53
- let runtime_health = buildRuntimeHealth(ensureResult.runtime_health.cdp, result);
54
- if (!result.health.valid) {
55
- const healthCode = determineHealthCode(result);
56
- console.warn(`[RECOVERY] Empty snapshot for ${pageId} (${healthCode}); rebinding CDP session`);
57
- handle = await session.rebindCdpSession(pageId);
58
- result = await captureWithStabilization(handle.cdp, handle.page, pageId, { maxRetries: 1 });
59
- runtime_health = buildRuntimeHealth({ ok: true, recovered: true, recovery_method: 'rebind' }, result);
60
- }
61
- return { snapshot: result.snapshot, handle, runtime_health };
62
- }
63
- /**
64
- * Create a capture function that keeps the handle updated after recovery.
65
- */
66
- function createActionCapture(session, handleRef, pageId) {
67
- return async () => {
68
- const captureResult = await captureSnapshotWithRecovery(session, handleRef.current, pageId);
69
- handleRef.current = captureResult.handle;
70
- return {
71
- snapshot: captureResult.snapshot,
72
- runtime_health: captureResult.runtime_health,
73
- };
74
- };
75
- }
76
- /**
77
- * Prepare context for action execution.
78
- * Resolves page, ensures CDP session health, and creates capture function.
79
- *
80
- * @param pageId - Optional page ID to resolve
81
- * @returns Action context with handle, capture function, and session
82
- */
83
- async function prepareActionContext(pageId) {
84
- const session = getSessionManager();
85
- const handleRef = { current: resolveExistingPage(session, pageId) };
86
- const resolvedPageId = handleRef.current.page_id;
87
- handleRef.current = (await ensureCdpSession(session, handleRef.current)).handle;
88
- const captureSnapshot = createActionCapture(session, handleRef, resolvedPageId);
89
- return { handleRef, pageId: resolvedPageId, captureSnapshot, session };
90
- }
91
- /**
92
- * Execute a navigation action with snapshot capture.
93
- * Consolidates goBack, goForward, and reload handlers.
94
- *
95
- * Waits for both DOM stabilization and network idle after navigation
96
- * to ensure the page is fully loaded before capturing snapshot.
97
- *
98
- * @param pageId - Optional page ID
99
- * @param action - Navigation action to execute
100
- * @returns State response after navigation
101
- */
102
- async function executeNavigationAction(pageId, action) {
103
- const session = getSessionManager();
104
- let handle = await session.resolvePageOrCreate(pageId);
105
- const page_id = handle.page_id;
106
- session.touchPage(page_id);
107
- // Clear dependency tracker before navigation (old dependencies no longer valid)
108
- getDependencyTracker().clearPage(page_id);
109
- // Execute navigation
110
- switch (action) {
111
- case 'back':
112
- await handle.page.goBack();
113
- break;
114
- case 'forward':
115
- await handle.page.goForward();
116
- break;
117
- case 'reload':
118
- await handle.page.reload();
119
- break;
120
- }
121
- // Wait for page to stabilize (DOM + network idle)
122
- await stabilizeAfterNavigation(handle.page);
123
- // Re-inject observation accumulator (new document context after navigation)
124
- await observationAccumulator.inject(handle.page);
125
- // Auto-capture snapshot after navigation
126
- const captureResult = await captureSnapshotWithRecovery(session, handle, page_id);
127
- handle = captureResult.handle;
128
- const snapshot = captureResult.snapshot;
129
- snapshotStore.store(page_id, snapshot);
130
- // Return XML state response (trimmed for navigation snapshots)
131
- const stateManager = getStateManager(page_id);
132
- return stateManager.generateResponse(snapshot, { trimRegions: true });
133
- }
134
- // ============================================================================
135
- // SIMPLIFIED API - Tool handlers with clearer contracts
136
- // ============================================================================
137
- /**
138
- * List all open browser pages with their metadata.
139
- *
140
- * Syncs with browser context to ensure all tabs are registered,
141
- * including tabs opened externally or after reconnection.
142
- *
143
- * @returns XML result with page list
144
- */
145
- export async function listPages() {
146
- const session = getSessionManager();
147
- // Sync to pick up any unregistered browser tabs
148
- const pages = await session.syncPages();
149
- const pageInfos = pages.map((h) => ({
150
- page_id: h.page_id,
151
- url: h.url ?? '',
152
- title: h.title ?? '',
153
- }));
154
- return buildListPagesResponse(pageInfos);
155
- }
156
- /**
157
- * Close a specific page.
158
- *
159
- * @param rawInput - Close options (will be validated)
160
- * @returns Close result
161
- */
162
- export async function closePage(rawInput) {
163
- const input = ClosePageInputSchema.parse(rawInput);
164
- const session = getSessionManager();
165
- await session.closePage(input.page_id);
166
- snapshotStore.removeByPageId(input.page_id);
167
- removeStateManager(input.page_id); // Clean up state manager
168
- getDependencyTracker().clearPage(input.page_id); // Clean up dependencies
169
- return buildClosePageResponse(input.page_id);
170
- }
171
- /**
172
- * Close the entire browser session.
173
- *
174
- * @param rawInput - Close options (will be validated)
175
- * @returns Close result
176
- */
177
- export async function closeSession(rawInput) {
178
- CloseSessionInputSchema.parse(rawInput);
179
- const session = getSessionManager();
180
- await session.shutdown();
181
- snapshotStore.clear();
182
- clearAllStateManagers(); // Clean up all state managers
183
- getDependencyTracker().clearAll(); // Clean up all dependencies
184
- await cleanupTempFiles(); // Clean up screenshot temp files
185
- return buildCloseSessionResponse();
186
- }
187
- /**
188
- * Navigate to a URL.
189
- *
190
- * @param rawInput - Navigation options (will be validated)
191
- * @returns Navigation result with snapshot data
192
- */
193
- export async function navigate(rawInput) {
194
- const input = NavigateInputSchema.parse(rawInput);
195
- const session = getSessionManager();
196
- let handle = await session.resolvePageOrCreate(input.page_id);
197
- const page_id = handle.page_id;
198
- session.touchPage(page_id);
199
- // Clear dependency tracker before navigation (old dependencies no longer valid)
200
- getDependencyTracker().clearPage(page_id);
201
- await session.navigateTo(page_id, input.url);
202
- // Auto-capture snapshot after navigation
203
- const captureResult = await captureSnapshotWithRecovery(session, handle, page_id);
204
- handle = captureResult.handle;
205
- const snapshot = captureResult.snapshot;
206
- snapshotStore.store(page_id, snapshot);
207
- // Return XML state response directly (trimmed for navigation snapshots)
208
- const stateManager = getStateManager(page_id);
209
- return stateManager.generateResponse(snapshot, { trimRegions: true });
210
- }
211
- /**
212
- * Go back in browser history.
213
- *
214
- * @param rawInput - Navigation options (will be validated)
215
- * @returns Navigation result with snapshot data
216
- */
217
- export async function goBack(rawInput) {
218
- const input = GoBackInputSchema.parse(rawInput);
219
- return executeNavigationAction(input.page_id, 'back');
220
- }
221
- /**
222
- * Go forward in browser history.
223
- *
224
- * @param rawInput - Navigation options (will be validated)
225
- * @returns Navigation result with snapshot data
226
- */
227
- export async function goForward(rawInput) {
228
- const input = GoForwardInputSchema.parse(rawInput);
229
- return executeNavigationAction(input.page_id, 'forward');
230
- }
231
- /**
232
- * Reload the current page.
233
- *
234
- * @param rawInput - Navigation options (will be validated)
235
- * @returns Navigation result with snapshot data
236
- */
237
- export async function reload(rawInput) {
238
- const input = ReloadInputSchema.parse(rawInput);
239
- return executeNavigationAction(input.page_id, 'reload');
240
- }
241
- /**
242
- * Capture a fresh snapshot of the current page.
243
- *
244
- * @param rawInput - Capture options (will be validated)
245
- * @returns Snapshot data for the current page
246
- */
247
- export async function captureSnapshot(rawInput) {
248
- const input = CaptureSnapshotInputSchema.parse(rawInput);
249
- const session = getSessionManager();
250
- let handle = resolveExistingPage(session, input.page_id);
251
- const page_id = handle.page_id;
252
- // Capture any accumulated observations (no action window)
253
- const observations = await observationAccumulator.getAccumulatedObservations(handle.page);
254
- const captureResult = await captureSnapshotWithRecovery(session, handle, page_id);
255
- handle = captureResult.handle;
256
- const snapshot = captureResult.snapshot;
257
- // Filter observations to reduce noise (threshold 5 requires semantic signals)
258
- const filteredObservations = observationAccumulator.filterBySignificance(observations, ATTACHMENT_SIGNIFICANCE_THRESHOLD);
259
- // Attach accumulated observations to snapshot if any
260
- if (filteredObservations.sincePrevious.length > 0) {
261
- snapshot.observations = filteredObservations;
262
- }
263
- snapshotStore.store(page_id, snapshot);
264
- // Return XML state response directly (trimmed for observation snapshots)
265
- const stateManager = getStateManager(page_id);
266
- return stateManager.generateResponse(snapshot, { trimRegions: true });
267
- }
268
- /**
269
- * Map schema kind values to internal NodeKind values.
270
- *
271
- * The find_elements schema uses user-friendly names that don't always match
272
- * internal NodeKind values. For example, 'textbox' in the schema maps to
273
- * both 'input' and 'textarea' internally.
274
- *
275
- * @param schemaKind - Kind value from the find_elements schema
276
- * @returns Matching NodeKind value(s)
277
- */
278
- export function mapSchemaKindToNodeKind(schemaKind) {
279
- switch (schemaKind) {
280
- case 'textbox':
281
- return ['input', 'textarea'];
282
- default:
283
- return schemaKind;
284
- }
285
- }
286
- /**
287
- * Find elements by semantic criteria.
288
- *
289
- * @param rawInput - Query filters (will be validated)
290
- * @returns Matched nodes
291
- */
292
- export function findElements(rawInput) {
293
- const input = FindElementsInputSchema.parse(rawInput);
294
- const session = getSessionManager();
295
- const handle = resolveExistingPage(session, input.page_id);
296
- const page_id = handle.page_id;
297
- const snap = snapshotStore.getByPageId(page_id);
298
- if (!snap) {
299
- throw new Error(`No snapshot for page ${page_id} - capture a snapshot first`);
300
- }
301
- // Build query request from input
302
- const request = {
303
- limit: input.limit,
304
- };
305
- if (input.kind) {
306
- request.kind = mapSchemaKindToNodeKind(input.kind);
307
- }
308
- if (input.label) {
309
- request.label = { text: input.label, mode: 'contains', caseSensitive: false };
310
- }
311
- if (input.region) {
312
- request.region = input.region;
313
- }
314
- const engine = new QueryEngine(snap);
315
- const response = engine.find(request);
316
- // Get registry and state manager for EID lookup
317
- const stateManager = getStateManager(page_id);
318
- const registry = stateManager.getElementRegistry();
319
- const activeLayer = stateManager.getActiveLayer();
320
- const matches = response.matches.map((m) => {
321
- // Check if this is a readable/structural (non-interactive) node
322
- const isNonInteractive = isReadableNode(m.node) || isStructuralNode(m.node);
323
- // Look up EID from registry (for interactive nodes)
324
- const registryEid = registry.getEidBySnapshotAndBackendNodeId(snap.snapshot_id, m.node.backend_node_id);
325
- // Determine EID:
326
- // - Interactive nodes: use registry EID
327
- // - Non-interactive nodes with include_readable: compute rd-* ID on-demand
328
- // - Non-interactive nodes without include_readable: use unknown-* fallback
329
- let eid;
330
- if (registryEid) {
331
- eid = registryEid;
332
- }
333
- else if (isNonInteractive && input.include_readable) {
334
- // Compute on-demand semantic ID for readable content with rd- prefix
335
- eid = `rd-${computeEid(m.node, activeLayer).substring(0, 10)}`;
336
- }
337
- else {
338
- eid = `unknown-${m.node.backend_node_id}`;
339
- }
340
- const match = {
341
- eid,
342
- kind: m.node.kind,
343
- label: m.node.label,
344
- selector: m.node.find?.primary ?? '',
345
- region: m.node.where.region,
346
- };
347
- // Include state if present (type-safe assignment via NodeState interface)
348
- if (m.node.state) {
349
- match.state = m.node.state;
350
- }
351
- // Include attributes if present (filter to common ones)
352
- if (m.node.attributes) {
353
- const attrs = {};
354
- if (m.node.attributes.input_type)
355
- attrs.input_type = m.node.attributes.input_type;
356
- if (m.node.attributes.placeholder)
357
- attrs.placeholder = m.node.attributes.placeholder;
358
- if (m.node.attributes.value)
359
- attrs.value = m.node.attributes.value;
360
- if (m.node.attributes.href)
361
- attrs.href = m.node.attributes.href;
362
- if (m.node.attributes.alt)
363
- attrs.alt = m.node.attributes.alt;
364
- if (m.node.attributes.src)
365
- attrs.src = m.node.attributes.src;
366
- if (Object.keys(attrs).length > 0) {
367
- match.attributes = attrs;
368
- }
369
- }
370
- return match;
371
- });
372
- return buildFindElementsResponse(page_id, snap.snapshot_id, matches);
373
- }
374
- /**
375
- * Get full details for a specific node.
376
- *
377
- * @param rawInput - Node details request (will be validated)
378
- * @returns Full node details
379
- */
380
- export function getNodeDetails(rawInput) {
381
- const input = GetNodeDetailsInputSchema.parse(rawInput);
382
- const session = getSessionManager();
383
- const handle = resolveExistingPage(session, input.page_id);
384
- const page_id = handle.page_id;
385
- const snap = snapshotStore.getByPageId(page_id);
386
- if (!snap) {
387
- throw new Error(`No snapshot for page ${page_id} - capture a snapshot first`);
388
- }
389
- // Look up element by EID from registry
390
- const stateManager = getStateManager(page_id);
391
- const elementRef = stateManager.getElementRegistry().getByEid(input.eid);
392
- if (!elementRef) {
393
- throw new Error(`Element with eid ${input.eid} not found in registry`);
394
- }
395
- // Find the node by backend_node_id
396
- const node = snap.nodes.find((n) => n.backend_node_id === elementRef.ref.backend_node_id);
397
- if (!node) {
398
- throw new Error(`Element with eid ${input.eid} has stale reference`);
399
- }
400
- const details = {
401
- eid: input.eid,
402
- kind: node.kind,
403
- label: node.label,
404
- where: {
405
- region: node.where.region,
406
- group_id: node.where.group_id,
407
- group_path: node.where.group_path,
408
- heading_context: node.where.heading_context,
409
- },
410
- layout: {
411
- bbox: node.layout.bbox,
412
- display: node.layout.display,
413
- screen_zone: node.layout.screen_zone,
414
- },
415
- };
416
- if (node.state) {
417
- details.state = { ...node.state };
418
- }
419
- if (node.find) {
420
- details.find = { primary: node.find.primary, alternates: node.find.alternates };
421
- }
422
- if (node.attributes) {
423
- details.attributes = { ...node.attributes };
424
- }
425
- return buildGetElementDetailsResponse(page_id, snap.snapshot_id, details);
426
- }
427
- /**
428
- * Scroll an element into view.
429
- *
430
- * @param rawInput - Scroll options (will be validated)
431
- * @returns Scroll result with delta
432
- */
433
- export async function scrollElementIntoView(rawInput) {
434
- const input = ScrollElementIntoViewInputSchema.parse(rawInput);
435
- const { handleRef, pageId, captureSnapshot } = await prepareActionContext(input.page_id);
436
- const snap = requireSnapshot(pageId);
437
- const node = resolveElementByEid(pageId, input.eid, snap);
438
- // Execute action with automatic retry on stale elements
439
- const result = await executeActionWithRetry(handleRef.current, node, async (backendNodeId) => {
440
- await scrollIntoView(handleRef.current.cdp, backendNodeId);
441
- }, snapshotStore, captureSnapshot);
442
- // Store snapshot for future queries
443
- snapshotStore.store(pageId, result.snapshot);
444
- // Return XML state response directly
445
- return result.state_response;
446
- }
447
- /**
448
- * Scroll the page up or down.
449
- *
450
- * @param rawInput - Scroll options (will be validated)
451
- * @returns Scroll result with delta
452
- */
453
- export async function scrollPage(rawInput) {
454
- const input = ScrollPageInputSchema.parse(rawInput);
455
- const { handleRef, pageId, captureSnapshot } = await prepareActionContext(input.page_id);
456
- // Execute action with new simplified wrapper
457
- const result = await executeAction(handleRef.current, async () => {
458
- await scrollPageByAmount(handleRef.current.cdp, input.direction, input.amount);
459
- }, captureSnapshot);
460
- // Store snapshot for future queries
461
- snapshotStore.store(pageId, result.snapshot);
462
- // Return XML state response directly
463
- return result.state_response;
464
- }
465
- /**
466
- * Click an element or at viewport coordinates.
467
- *
468
- * Three modes:
469
- * 1. eid only → click element center (existing behavior)
470
- * 2. eid + x/y → click at offset relative to element top-left
471
- * 3. x/y only → click at absolute viewport coordinates
472
- *
473
- * @param rawInput - Click options (will be validated)
474
- * @returns Click result with navigation-aware outcome
475
- */
476
- export async function click(rawInput) {
477
- const input = ClickInputSchema.parse(rawInput);
478
- const hasEid = input.eid !== undefined;
479
- const hasCoords = input.x !== undefined && input.y !== undefined;
480
- if (!hasEid && !hasCoords) {
481
- throw new Error('Either eid or both x and y coordinates must be provided.');
482
- }
483
- if ((input.x !== undefined) !== (input.y !== undefined)) {
484
- throw new Error('Both x and y coordinates must be provided together.');
485
- }
486
- const { handleRef, pageId, captureSnapshot } = await prepareActionContext(input.page_id);
487
- let result;
488
- if (hasEid) {
489
- // Mode 1 & 2: element-based click (center or offset)
490
- const snap = requireSnapshot(pageId);
491
- const node = resolveElementByEid(pageId, input.eid, snap);
492
- result = await executeActionWithOutcome(handleRef.current, node, async (backendNodeId) => {
493
- if (hasCoords) {
494
- await clickAtElementOffset(handleRef.current.cdp, backendNodeId, input.x, input.y);
495
- }
496
- else {
497
- await clickByBackendNodeId(handleRef.current.cdp, backendNodeId);
498
- }
499
- }, snapshotStore, captureSnapshot);
500
- }
501
- else {
502
- // Mode 3: x/y only → absolute viewport click
503
- result = await executeAction(handleRef.current, async () => {
504
- await clickAtCoordinates(handleRef.current.cdp, input.x, input.y);
505
- }, captureSnapshot);
506
- }
507
- snapshotStore.store(pageId, result.snapshot);
508
- return result.state_response;
509
- }
510
- /**
511
- * Type text into an element.
512
- *
513
- * @param rawInput - Type options (will be validated)
514
- * @returns Type result with delta
515
- */
516
- export async function type(rawInput) {
517
- const input = TypeInputSchema.parse(rawInput);
518
- const { handleRef, pageId, captureSnapshot } = await prepareActionContext(input.page_id);
519
- const snap = requireSnapshot(pageId);
520
- const node = resolveElementByEid(pageId, input.eid, snap);
521
- // Execute action with automatic retry on stale elements
522
- const result = await executeActionWithRetry(handleRef.current, node, async (backendNodeId) => {
523
- await typeByBackendNodeId(handleRef.current.cdp, backendNodeId, input.text, {
524
- clear: input.clear,
525
- });
526
- }, snapshotStore, captureSnapshot);
527
- // Store snapshot for future queries
528
- snapshotStore.store(pageId, result.snapshot);
529
- // Return XML state response directly
530
- return result.state_response;
531
- }
532
- /**
533
- * Press a keyboard key (no agent_version).
534
- *
535
- * @param rawInput - Press options (will be validated)
536
- * @returns Press result with delta
537
- */
538
- export async function press(rawInput) {
539
- const input = PressInputSchema.parse(rawInput);
540
- const { handleRef, pageId, captureSnapshot } = await prepareActionContext(input.page_id);
541
- // Execute action with new simplified wrapper
542
- const result = await executeAction(handleRef.current, async () => {
543
- await pressKey(handleRef.current.cdp, input.key, input.modifiers);
544
- }, captureSnapshot);
545
- // Store snapshot for future queries
546
- snapshotStore.store(pageId, result.snapshot);
547
- // Return XML state response directly
548
- return result.state_response;
549
- }
550
- /**
551
- * Select a dropdown option.
552
- *
553
- * @param rawInput - Select options (will be validated)
554
- * @returns Select result with delta
555
- */
556
- export async function select(rawInput) {
557
- const input = SelectInputSchema.parse(rawInput);
558
- const { handleRef, pageId, captureSnapshot } = await prepareActionContext(input.page_id);
559
- const snap = requireSnapshot(pageId);
560
- const node = resolveElementByEid(pageId, input.eid, snap);
561
- // Execute action with automatic retry on stale elements
562
- const result = await executeActionWithRetry(handleRef.current, node, async (backendNodeId) => {
563
- await selectOption(handleRef.current.cdp, backendNodeId, input.value);
564
- }, snapshotStore, captureSnapshot);
565
- // Store snapshot for future queries
566
- snapshotStore.store(pageId, result.snapshot);
567
- // Return XML state response directly
568
- return result.state_response;
569
- }
570
- /**
571
- * Hover over an element.
572
- *
573
- * @param rawInput - Hover options (will be validated)
574
- * @returns Hover result with delta
575
- */
576
- export async function hover(rawInput) {
577
- const input = HoverInputSchema.parse(rawInput);
578
- const { handleRef, pageId, captureSnapshot } = await prepareActionContext(input.page_id);
579
- const snap = requireSnapshot(pageId);
580
- const node = resolveElementByEid(pageId, input.eid, snap);
581
- // Execute action with automatic retry on stale elements
582
- const result = await executeActionWithRetry(handleRef.current, node, async (backendNodeId) => {
583
- await hoverByBackendNodeId(handleRef.current.cdp, backendNodeId);
584
- }, snapshotStore, captureSnapshot);
585
- // Store snapshot for future queries
586
- snapshotStore.store(pageId, result.snapshot);
587
- // Return XML state response directly
588
- return result.state_response;
589
- }
590
- /**
591
- * Drag from one point to another.
592
- *
593
- * If eid is provided, coordinates are relative to the element's top-left corner.
594
- * Otherwise, coordinates are absolute viewport coordinates.
595
- *
596
- * @param rawInput - Drag options (will be validated)
597
- * @returns Drag result with updated snapshot
598
- */
599
- export async function drag(rawInput) {
600
- const input = DragInputSchema.parse(rawInput);
601
- const { handleRef, pageId, captureSnapshot } = await prepareActionContext(input.page_id);
602
- let sourceX = input.source_x;
603
- let sourceY = input.source_y;
604
- let targetX = input.target_x;
605
- let targetY = input.target_y;
606
- if (input.eid) {
607
- const snap = requireSnapshot(pageId);
608
- const node = resolveElementByEid(pageId, input.eid, snap);
609
- const { x, y } = await getElementTopLeft(handleRef.current.cdp, node.backend_node_id);
610
- sourceX = x + input.source_x;
611
- sourceY = y + input.source_y;
612
- targetX = x + input.target_x;
613
- targetY = y + input.target_y;
614
- }
615
- const result = await executeAction(handleRef.current, async () => {
616
- await dragBetweenCoordinates(handleRef.current.cdp, sourceX, sourceY, targetX, targetY);
617
- }, captureSnapshot);
618
- snapshotStore.store(pageId, result.snapshot);
619
- return result.state_response;
620
- }
621
- /**
622
- * Take a screenshot of the page or a specific element.
623
- *
624
- * Observation tool - does not mutate page state.
625
- * Returns inline image (<2MB) or temp file path (>=2MB).
626
- *
627
- * @param rawInput - Screenshot options (will be validated)
628
- * @returns ImageResult or FileResult
629
- */
630
- export async function takeScreenshot(rawInput) {
631
- const input = TakeScreenshotInputSchema.parse(rawInput);
632
- if (input.eid && input.fullPage) {
633
- throw new Error("Cannot use both 'eid' and 'fullPage'. Use eid for element screenshots OR fullPage for full-page capture.");
634
- }
635
- const session = getSessionManager();
636
- let handle = resolveExistingPage(session, input.page_id);
637
- const pageId = handle.page_id;
638
- // Ensure CDP session is healthy (auto-repair if needed)
639
- const ensureResult = await ensureCdpSession(session, handle);
640
- handle = ensureResult.handle;
641
- let clip;
642
- if (input.eid) {
643
- const snapshot = requireSnapshot(pageId);
644
- const node = resolveElementByEid(pageId, input.eid, snapshot);
645
- clip = await getElementBoundingBox(handle.cdp, node.backend_node_id);
646
- }
647
- return captureScreenshot(handle.cdp, {
648
- format: input.format ?? 'png',
649
- quality: input.quality,
650
- clip,
651
- captureBeyondViewport: input.fullPage ?? false,
652
- });
653
- }
654
- // Canvas inspection: see canvas-tools.ts
4
+ * Re-export barrel for all browser automation tool handlers.
5
+ * The actual implementations live in category-specific modules.
6
+ */
7
+ // Action context & initialization
8
+ export { initializeTools, prepareActionContext, captureSnapshotWithRecovery, createActionCapture, buildRuntimeHealth, } from './action-context.js';
9
+ // Navigation & session tools
10
+ export { listPages, closePage, closeSession, navigate, goBack, goForward, reload, } from './navigation-tools.js';
11
+ // Observation tools
12
+ export { captureSnapshot, findElements, getNodeDetails, scrollElementIntoView, scrollPage, mapSchemaKindToNodeKind, } from './observation-tools.js';
13
+ // Interaction tools
14
+ export { click, type, press, select, hover } from './interaction-tools.js';
15
+ // Viewport tools
16
+ export { drag, wheel, takeScreenshot } from './viewport-tools.js';
655
17
  //# sourceMappingURL=browser-tools.js.map