@testdriverai/agent 7.8.0-test.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (528) hide show
  1. package/.claude/settings.local.json +7 -0
  2. package/.env.example +4 -0
  3. package/.prettierignore +4 -0
  4. package/.prettierrc +1 -0
  5. package/CHANGELOG.md +953 -0
  6. package/README.md +81 -0
  7. package/agent/events.js +135 -0
  8. package/agent/index.js +2450 -0
  9. package/agent/interface.js +35 -0
  10. package/agent/lib/analytics.js +22 -0
  11. package/agent/lib/censorship.js +75 -0
  12. package/agent/lib/commander.js +246 -0
  13. package/agent/lib/commands.js +1684 -0
  14. package/agent/lib/config.js +60 -0
  15. package/agent/lib/generator.js +91 -0
  16. package/agent/lib/http.js +144 -0
  17. package/agent/lib/logger.js +56 -0
  18. package/agent/lib/outputs.js +29 -0
  19. package/agent/lib/parser.js +209 -0
  20. package/agent/lib/redraw.js +386 -0
  21. package/agent/lib/resources/cursor-2.png +0 -0
  22. package/agent/lib/sandbox.js +1104 -0
  23. package/agent/lib/sdk.js +633 -0
  24. package/agent/lib/session.js +25 -0
  25. package/agent/lib/source-mapper.js +342 -0
  26. package/agent/lib/subimage/index.js +77 -0
  27. package/agent/lib/subimage/opencv.js +69 -0
  28. package/agent/lib/system.js +204 -0
  29. package/agent/lib/theme.js +14 -0
  30. package/agent/lib/valid-version.js +21 -0
  31. package/agent/lib/validation.js +169 -0
  32. package/ai/.claude-plugin/plugin.json +9 -0
  33. package/ai/agents/testdriver.md +638 -0
  34. package/ai/skills/testdriver-ai/SKILL.md +204 -0
  35. package/ai/skills/testdriver-assert/SKILL.md +315 -0
  36. package/ai/skills/testdriver-aws-setup/SKILL.md +448 -0
  37. package/ai/skills/testdriver-cache/SKILL.md +221 -0
  38. package/ai/skills/testdriver-caching/SKILL.md +124 -0
  39. package/ai/skills/testdriver-captcha/SKILL.md +158 -0
  40. package/ai/skills/testdriver-ci-cd/SKILL.md +602 -0
  41. package/ai/skills/testdriver-click/SKILL.md +286 -0
  42. package/ai/skills/testdriver-client/SKILL.md +477 -0
  43. package/ai/skills/testdriver-cloud/SKILL.md +119 -0
  44. package/ai/skills/testdriver-customizing-devices/SKILL.md +319 -0
  45. package/ai/skills/testdriver-dashcam/SKILL.md +418 -0
  46. package/ai/skills/testdriver-debugging-with-screenshots/SKILL.md +401 -0
  47. package/ai/skills/testdriver-device-config/SKILL.md +317 -0
  48. package/ai/skills/testdriver-double-click/SKILL.md +102 -0
  49. package/ai/skills/testdriver-elements/SKILL.md +605 -0
  50. package/ai/skills/testdriver-enterprise/SKILL.md +114 -0
  51. package/ai/skills/testdriver-errors/SKILL.md +246 -0
  52. package/ai/skills/testdriver-events/SKILL.md +356 -0
  53. package/ai/skills/testdriver-examples/SKILL.md +7 -0
  54. package/ai/skills/testdriver-exec/SKILL.md +317 -0
  55. package/ai/skills/testdriver-find/SKILL.md +829 -0
  56. package/ai/skills/testdriver-focus-application/SKILL.md +293 -0
  57. package/ai/skills/testdriver-generating-tests/SKILL.md +36 -0
  58. package/ai/skills/testdriver-hover/SKILL.md +278 -0
  59. package/ai/skills/testdriver-locating-elements/SKILL.md +71 -0
  60. package/ai/skills/testdriver-making-assertions/SKILL.md +32 -0
  61. package/ai/skills/testdriver-mcp/SKILL.md +7 -0
  62. package/ai/skills/testdriver-mcp-workflow/SKILL.md +410 -0
  63. package/ai/skills/testdriver-mouse-down/SKILL.md +161 -0
  64. package/ai/skills/testdriver-mouse-up/SKILL.md +164 -0
  65. package/ai/skills/testdriver-parse/SKILL.md +236 -0
  66. package/ai/skills/testdriver-performing-actions/SKILL.md +54 -0
  67. package/ai/skills/testdriver-press-keys/SKILL.md +348 -0
  68. package/ai/skills/testdriver-provision/SKILL.md +331 -0
  69. package/ai/skills/testdriver-quickstart/SKILL.md +144 -0
  70. package/ai/skills/testdriver-redraw/SKILL.md +214 -0
  71. package/ai/skills/testdriver-reusable-code/SKILL.md +249 -0
  72. package/ai/skills/testdriver-right-click/SKILL.md +123 -0
  73. package/ai/skills/testdriver-running-tests/SKILL.md +185 -0
  74. package/ai/skills/testdriver-screenshot/SKILL.md +248 -0
  75. package/ai/skills/testdriver-screenshots/SKILL.md +184 -0
  76. package/ai/skills/testdriver-scroll/SKILL.md +335 -0
  77. package/ai/skills/testdriver-secrets/SKILL.md +115 -0
  78. package/ai/skills/testdriver-self-hosted/SKILL.md +65 -0
  79. package/ai/skills/testdriver-test-writer/SKILL.md +448 -0
  80. package/ai/skills/testdriver-testdriver/SKILL.md +628 -0
  81. package/ai/skills/testdriver-testdriver-mechanic/SKILL.md +165 -0
  82. package/ai/skills/testdriver-type/SKILL.md +357 -0
  83. package/ai/skills/testdriver-variables/SKILL.md +111 -0
  84. package/ai/skills/testdriver-wait/SKILL.md +50 -0
  85. package/ai/skills/testdriver-waiting-for-elements/SKILL.md +90 -0
  86. package/ai/skills/testdriver-what-is-testdriver/SKILL.md +54 -0
  87. package/bin/testdriverai.js +22 -0
  88. package/debugger/bg.png +0 -0
  89. package/debugger/icon.png +0 -0
  90. package/debugger/index.html +469 -0
  91. package/debugger/td.png +0 -0
  92. package/debugger/tray-buffered.png +0 -0
  93. package/debugger/tray.png +0 -0
  94. package/docs/GITHUB_COMMENTS.md +330 -0
  95. package/docs/GITHUB_COMMENTS_ANNOUNCEMENT.md +167 -0
  96. package/docs/QUICK-START-GITHUB-COMMENTS.md +84 -0
  97. package/docs/TEST-GITHUB-COMMENTS.md +129 -0
  98. package/docs/_data/examples-manifest.json +177 -0
  99. package/docs/_data/examples-manifest.schema.json +41 -0
  100. package/docs/_scripts/extract-example-urls.js +165 -0
  101. package/docs/_scripts/generate-examples.js +560 -0
  102. package/docs/_scripts/generate-skills.js +154 -0
  103. package/docs/_scripts/link-replacer.js +164 -0
  104. package/docs/_scripts/upload-docs-to-openai.js +284 -0
  105. package/docs/changelog.mdx +161 -0
  106. package/docs/claude-mcp-plugin.mdx +160 -0
  107. package/docs/docs.json +442 -0
  108. package/docs/github-integration-setup.md +266 -0
  109. package/docs/guide/best-practices-polling.mdx +174 -0
  110. package/docs/images/content/account/newprojectsettings.png +0 -0
  111. package/docs/images/content/account/projectpage.png +0 -0
  112. package/docs/images/content/account/projectreplays.png +0 -0
  113. package/docs/images/content/account/team-manage.png +0 -0
  114. package/docs/images/content/account/teampage.png +0 -0
  115. package/docs/images/content/extension/cursor.svg +1 -0
  116. package/docs/images/content/extension/vscode.svg +57 -0
  117. package/docs/images/content/extension/windsurf.svg +3 -0
  118. package/docs/images/content/parse/output.png +0 -0
  119. package/docs/images/content/self-hosted/launchtemplateid.png +0 -0
  120. package/docs/images/content/side-by-side.png +0 -0
  121. package/docs/images/content/vscode/ide-full.png +0 -0
  122. package/docs/images/content/vscode/running.png +0 -0
  123. package/docs/images/content/vscode/v7-chat.png +0 -0
  124. package/docs/images/content/vscode/v7-choose-agent.png +0 -0
  125. package/docs/images/content/vscode/v7-full.png +0 -0
  126. package/docs/images/content/vscode/v7-onboarding.png +0 -0
  127. package/docs/images/content/vscode/vscode-2-assert.png +0 -0
  128. package/docs/images/content/vscode/vscode-agent-preview.png +0 -0
  129. package/docs/images/content/vscode/vscode-copilot-ask.png +0 -0
  130. package/docs/images/content/vscode/vscode-file-creation.png +0 -0
  131. package/docs/images/content/vscode/vscode-install.png +0 -0
  132. package/docs/images/content/vscode/vscode-overview.png +0 -0
  133. package/docs/images/content/vscode/vscode-setup-walkthrough.png +0 -0
  134. package/docs/images/content/vscode/vscode-stopchat.png +0 -0
  135. package/docs/images/content/vscode/vscode-stoptest.png +0 -0
  136. package/docs/images/content/vscode/vscode-tdservice.png +0 -0
  137. package/docs/images/content/vscode/vscode-test-output.png +0 -0
  138. package/docs/images/content/vscode/vscode-testhistory.png +0 -0
  139. package/docs/images/content/vscode/vscode-testpane-runtests.png +0 -0
  140. package/docs/images/content/vscode/vscode-testpane.png +0 -0
  141. package/docs/images/template/dark.png +0 -0
  142. package/docs/images/template/icon.png +0 -0
  143. package/docs/images/template/light.png +0 -0
  144. package/docs/snippets/calendar-link.mdx +4 -0
  145. package/docs/snippets/gitignore-warning.mdx +7 -0
  146. package/docs/snippets/lifecycle-warning.mdx +6 -0
  147. package/docs/snippets/test-prereqs.mdx +12 -0
  148. package/docs/snippets/tests/assert-replay.mdx +7 -0
  149. package/docs/snippets/tests/assert-yaml.mdx +8 -0
  150. package/docs/snippets/tests/exec-js-replay.mdx +7 -0
  151. package/docs/snippets/tests/exec-js-yaml.mdx +32 -0
  152. package/docs/snippets/tests/exec-shell-replay.mdx +7 -0
  153. package/docs/snippets/tests/exec-shell-yaml.mdx +15 -0
  154. package/docs/snippets/tests/hover-image-replay.mdx +7 -0
  155. package/docs/snippets/tests/hover-image-yaml.mdx +17 -0
  156. package/docs/snippets/tests/hover-text-replay.mdx +7 -0
  157. package/docs/snippets/tests/hover-text-with-description-replay.mdx +7 -0
  158. package/docs/snippets/tests/hover-text-with-description-yaml.mdx +24 -0
  159. package/docs/snippets/tests/hover-text-yaml.mdx +14 -0
  160. package/docs/snippets/tests/match-image-replay.mdx +7 -0
  161. package/docs/snippets/tests/match-image-yaml.mdx +17 -0
  162. package/docs/snippets/tests/press-keys-replay.mdx +7 -0
  163. package/docs/snippets/tests/press-keys-yaml.mdx +36 -0
  164. package/docs/snippets/tests/remember-replay.mdx +7 -0
  165. package/docs/snippets/tests/remember-yaml.mdx +28 -0
  166. package/docs/snippets/tests/scroll-replay.mdx +7 -0
  167. package/docs/snippets/tests/scroll-until-image-replay.mdx +7 -0
  168. package/docs/snippets/tests/scroll-until-image-yaml.mdx +14 -0
  169. package/docs/snippets/tests/scroll-until-text-replay.mdx +7 -0
  170. package/docs/snippets/tests/scroll-until-text-yaml.mdx +17 -0
  171. package/docs/snippets/tests/scroll-yaml.mdx +30 -0
  172. package/docs/snippets/tests/type-repeated-replay.mdx +7 -0
  173. package/docs/snippets/tests/type-repeated-yaml.mdx +22 -0
  174. package/docs/snippets/tests/type-replay.mdx +7 -0
  175. package/docs/snippets/tests/type-yaml.mdx +28 -0
  176. package/docs/snippets/tests/wait-for-image-replay.mdx +7 -0
  177. package/docs/snippets/tests/wait-for-image-yaml.mdx +18 -0
  178. package/docs/snippets/tests/wait-for-text-replay.mdx +7 -0
  179. package/docs/snippets/tests/wait-for-text-yaml.mdx +18 -0
  180. package/docs/snippets/tests/wait-replay.mdx +7 -0
  181. package/docs/snippets/tests/wait-yaml.mdx +13 -0
  182. package/docs/styles.css +65 -0
  183. package/docs/v6/account/dashboard.mdx +16 -0
  184. package/docs/v6/account/enterprise.mdx +110 -0
  185. package/docs/v6/account/pricing.mdx +33 -0
  186. package/docs/v6/account/projects.mdx +33 -0
  187. package/docs/v6/account/team.mdx +35 -0
  188. package/docs/v6/action/ami.mdx +109 -0
  189. package/docs/v6/action/performance.mdx +105 -0
  190. package/docs/v6/action/secrets.mdx +93 -0
  191. package/docs/v6/apps/chrome-extensions.mdx +48 -0
  192. package/docs/v6/apps/desktop-apps.mdx +93 -0
  193. package/docs/v6/apps/mobile-apps.mdx +26 -0
  194. package/docs/v6/apps/static-websites.mdx +54 -0
  195. package/docs/v6/apps/tauri-apps.mdx +361 -0
  196. package/docs/v6/bugs/jira.mdx +232 -0
  197. package/docs/v6/cli/overview.mdx +66 -0
  198. package/docs/v6/commands/assert.mdx +45 -0
  199. package/docs/v6/commands/exec.mdx +276 -0
  200. package/docs/v6/commands/focus-application.mdx +44 -0
  201. package/docs/v6/commands/hover-image.mdx +69 -0
  202. package/docs/v6/commands/hover-text.mdx +47 -0
  203. package/docs/v6/commands/if.mdx +53 -0
  204. package/docs/v6/commands/match-image.mdx +67 -0
  205. package/docs/v6/commands/press-keys.mdx +87 -0
  206. package/docs/v6/commands/remember.mdx +49 -0
  207. package/docs/v6/commands/run.mdx +44 -0
  208. package/docs/v6/commands/scroll-until-image.mdx +66 -0
  209. package/docs/v6/commands/scroll-until-text.mdx +60 -0
  210. package/docs/v6/commands/scroll.mdx +69 -0
  211. package/docs/v6/commands/type.mdx +45 -0
  212. package/docs/v6/commands/wait-for-image.mdx +54 -0
  213. package/docs/v6/commands/wait-for-text.mdx +48 -0
  214. package/docs/v6/commands/wait.mdx +45 -0
  215. package/docs/v6/exporting/junit.mdx +218 -0
  216. package/docs/v6/exporting/playwright.mdx +197 -0
  217. package/docs/v6/features/auto-healing.mdx +144 -0
  218. package/docs/v6/features/generation.mdx +116 -0
  219. package/docs/v6/features/parallel-testing.mdx +151 -0
  220. package/docs/v6/features/reusable-snippets.mdx +131 -0
  221. package/docs/v6/features/selectorless.mdx +80 -0
  222. package/docs/v6/features/visual-assertions.mdx +139 -0
  223. package/docs/v6/getting-started/ci.mdx +146 -0
  224. package/docs/v6/getting-started/cli.mdx +91 -0
  225. package/docs/v6/getting-started/editing.mdx +100 -0
  226. package/docs/v6/getting-started/playwright.mdx +342 -0
  227. package/docs/v6/getting-started/running.mdx +48 -0
  228. package/docs/v6/getting-started/self-hosting.mdx +408 -0
  229. package/docs/v6/getting-started/vscode.mdx +88 -0
  230. package/docs/v6/guide/assertions.mdx +189 -0
  231. package/docs/v6/guide/authentication.mdx +136 -0
  232. package/docs/v6/guide/code.mdx +65 -0
  233. package/docs/v6/guide/dashcam.mdx +118 -0
  234. package/docs/v6/guide/environment-variables.mdx +26 -0
  235. package/docs/v6/guide/lifecycle.mdx +242 -0
  236. package/docs/v6/guide/locating.mdx +141 -0
  237. package/docs/v6/guide/protips.mdx +43 -0
  238. package/docs/v6/guide/variables.mdx +143 -0
  239. package/docs/v6/guide/waiting.mdx +130 -0
  240. package/docs/v6/importing/csv.mdx +196 -0
  241. package/docs/v6/importing/gherkin.mdx +143 -0
  242. package/docs/v6/importing/jira.mdx +164 -0
  243. package/docs/v6/importing/testrail.mdx +162 -0
  244. package/docs/v6/integrations/electron.mdx +146 -0
  245. package/docs/v6/integrations/netlify.mdx +100 -0
  246. package/docs/v6/integrations/vercel.mdx +125 -0
  247. package/docs/v6/interactive/explore.mdx +99 -0
  248. package/docs/v6/interactive/run.mdx +52 -0
  249. package/docs/v6/interactive/save.mdx +63 -0
  250. package/docs/v6/overview/comparison.mdx +101 -0
  251. package/docs/v6/overview/faq.mdx +162 -0
  252. package/docs/v6/overview/performance.mdx +52 -0
  253. package/docs/v6/overview/quickstart.mdx +137 -0
  254. package/docs/v6/overview/what-is-testdriver.mdx +85 -0
  255. package/docs/v6/scenarios/ai-chatbot.mdx +28 -0
  256. package/docs/v6/scenarios/cookie-banner.mdx +32 -0
  257. package/docs/v6/scenarios/file-upload.mdx +33 -0
  258. package/docs/v6/scenarios/form-filling.mdx +32 -0
  259. package/docs/v6/scenarios/log-in.mdx +75 -0
  260. package/docs/v6/scenarios/pdf-generation.mdx +25 -0
  261. package/docs/v6/scenarios/spell-check.mdx +22 -0
  262. package/docs/v6/security/action.mdx +84 -0
  263. package/docs/v6/security/agent.mdx +73 -0
  264. package/docs/v6/security/platform.mdx +77 -0
  265. package/docs/v6/tutorials/advanced-test.mdx +81 -0
  266. package/docs/v6/tutorials/basic-test.mdx +45 -0
  267. package/docs/v7/_drafts/agents.mdx +843 -0
  268. package/docs/v7/_drafts/architecture.mdx +399 -0
  269. package/docs/v7/_drafts/auto-cache-key.mdx +167 -0
  270. package/docs/v7/_drafts/awesome-logs-quick-ref.mdx +100 -0
  271. package/docs/v7/_drafts/best-practices.mdx +486 -0
  272. package/docs/v7/_drafts/caching-ai.mdx +215 -0
  273. package/docs/v7/_drafts/caching-selectors.mdx +424 -0
  274. package/docs/v7/_drafts/caching.mdx +366 -0
  275. package/docs/v7/_drafts/cli-to-sdk-migration.mdx +425 -0
  276. package/docs/v7/_drafts/commands/assert.mdx +45 -0
  277. package/docs/v7/_drafts/commands/exec.mdx +276 -0
  278. package/docs/v7/_drafts/commands/focus-application.mdx +44 -0
  279. package/docs/v7/_drafts/commands/hover-image.mdx +69 -0
  280. package/docs/v7/_drafts/commands/hover-text.mdx +47 -0
  281. package/docs/v7/_drafts/commands/if.mdx +53 -0
  282. package/docs/v7/_drafts/commands/match-image.mdx +67 -0
  283. package/docs/v7/_drafts/commands/press-keys.mdx +87 -0
  284. package/docs/v7/_drafts/commands/remember.mdx +49 -0
  285. package/docs/v7/_drafts/commands/run.mdx +44 -0
  286. package/docs/v7/_drafts/commands/scroll-until-image.mdx +66 -0
  287. package/docs/v7/_drafts/commands/scroll-until-text.mdx +60 -0
  288. package/docs/v7/_drafts/commands/scroll.mdx +69 -0
  289. package/docs/v7/_drafts/commands/type.mdx +45 -0
  290. package/docs/v7/_drafts/commands/wait-for-image.mdx +54 -0
  291. package/docs/v7/_drafts/commands/wait-for-text.mdx +48 -0
  292. package/docs/v7/_drafts/commands/wait.mdx +45 -0
  293. package/docs/v7/_drafts/configuration.mdx +378 -0
  294. package/docs/v7/_drafts/contributing.mdx +174 -0
  295. package/docs/v7/_drafts/core.mdx +458 -0
  296. package/docs/v7/_drafts/dashcam-title-feature.mdx +89 -0
  297. package/docs/v7/_drafts/debugging.mdx +349 -0
  298. package/docs/v7/_drafts/error-handling.mdx +501 -0
  299. package/docs/v7/_drafts/faq.mdx +393 -0
  300. package/docs/v7/_drafts/hooks.mdx +360 -0
  301. package/docs/v7/_drafts/init-command.mdx +95 -0
  302. package/docs/v7/_drafts/installation.mdx +420 -0
  303. package/docs/v7/_drafts/migration.mdx +562 -0
  304. package/docs/v7/_drafts/observable.mdx +604 -0
  305. package/docs/v7/_drafts/playwright.mdx +342 -0
  306. package/docs/v7/_drafts/plugin-migration.mdx +220 -0
  307. package/docs/v7/_drafts/powerful.mdx +419 -0
  308. package/docs/v7/_drafts/presets.mdx +210 -0
  309. package/docs/v7/_drafts/progressive-disclosure.mdx +230 -0
  310. package/docs/v7/_drafts/prompt-cache.mdx +200 -0
  311. package/docs/v7/_drafts/provision.mdx +390 -0
  312. package/docs/v7/_drafts/quick-start-test-recording.mdx +214 -0
  313. package/docs/v7/_drafts/readme.mdx +135 -0
  314. package/docs/v7/_drafts/reports.mdx +414 -0
  315. package/docs/v7/_drafts/scalable.mdx +763 -0
  316. package/docs/v7/_drafts/screenshot.mdx +155 -0
  317. package/docs/v7/_drafts/sdk-awesome-logs.mdx +468 -0
  318. package/docs/v7/_drafts/sdk-browser-rendering.mdx +167 -0
  319. package/docs/v7/_drafts/sdk-migration.mdx +474 -0
  320. package/docs/v7/_drafts/sdk-v7-complete.mdx +345 -0
  321. package/docs/v7/_drafts/self-hosting.mdx +369 -0
  322. package/docs/v7/_drafts/test-recording.mdx +382 -0
  323. package/docs/v7/_drafts/troubleshooting.mdx +526 -0
  324. package/docs/v7/_drafts/vitest-plugin.mdx +477 -0
  325. package/docs/v7/_drafts/vitest.mdx +535 -0
  326. package/docs/v7/_drafts/writing-tests.mdx +25 -0
  327. package/docs/v7/ai.mdx +205 -0
  328. package/docs/v7/assert.mdx +316 -0
  329. package/docs/v7/aws-setup.mdx +449 -0
  330. package/docs/v7/cache.mdx +223 -0
  331. package/docs/v7/caching.mdx +128 -0
  332. package/docs/v7/captcha.mdx +159 -0
  333. package/docs/v7/ci-cd.mdx +603 -0
  334. package/docs/v7/click.mdx +287 -0
  335. package/docs/v7/client.mdx +478 -0
  336. package/docs/v7/copilot/auto-healing.mdx +265 -0
  337. package/docs/v7/copilot/creating-tests.mdx +156 -0
  338. package/docs/v7/copilot/github.mdx +143 -0
  339. package/docs/v7/copilot/running-tests.mdx +149 -0
  340. package/docs/v7/copilot/setup.mdx +143 -0
  341. package/docs/v7/customizing-devices.mdx +319 -0
  342. package/docs/v7/dashcam.mdx +419 -0
  343. package/docs/v7/debugging-with-screenshots.mdx +402 -0
  344. package/docs/v7/device-config.mdx +317 -0
  345. package/docs/v7/double-click.mdx +102 -0
  346. package/docs/v7/elements.mdx +606 -0
  347. package/docs/v7/enterprise.mdx +9 -0
  348. package/docs/v7/errors.mdx +248 -0
  349. package/docs/v7/events.mdx +358 -0
  350. package/docs/v7/examples/ai.mdx +72 -0
  351. package/docs/v7/examples/assert.mdx +72 -0
  352. package/docs/v7/examples/captcha-api.mdx +92 -0
  353. package/docs/v7/examples/chrome-extension.mdx +132 -0
  354. package/docs/v7/examples/drag-and-drop.mdx +100 -0
  355. package/docs/v7/examples/element-not-found.mdx +67 -0
  356. package/docs/v7/examples/exec-output.mdx +85 -0
  357. package/docs/v7/examples/exec-pwsh.mdx +83 -0
  358. package/docs/v7/examples/focus-window.mdx +62 -0
  359. package/docs/v7/examples/hover-image.mdx +94 -0
  360. package/docs/v7/examples/hover-text.mdx +69 -0
  361. package/docs/v7/examples/installer.mdx +91 -0
  362. package/docs/v7/examples/launch-vscode-linux.mdx +101 -0
  363. package/docs/v7/examples/match-image.mdx +96 -0
  364. package/docs/v7/examples/press-keys.mdx +92 -0
  365. package/docs/v7/examples/scroll-keyboard.mdx +79 -0
  366. package/docs/v7/examples/scroll-until-image.mdx +81 -0
  367. package/docs/v7/examples/scroll-until-text.mdx +109 -0
  368. package/docs/v7/examples/scroll.mdx +81 -0
  369. package/docs/v7/examples/type.mdx +92 -0
  370. package/docs/v7/examples/windows-installer.mdx +89 -0
  371. package/docs/v7/exec.mdx +318 -0
  372. package/docs/v7/find.mdx +830 -0
  373. package/docs/v7/focus-application.mdx +294 -0
  374. package/docs/v7/generating-tests.mdx +36 -0
  375. package/docs/v7/hosted.mdx +158 -0
  376. package/docs/v7/hover.mdx +279 -0
  377. package/docs/v7/locating-elements.mdx +71 -0
  378. package/docs/v7/making-assertions.mdx +32 -0
  379. package/docs/v7/mcp.mdx +9 -0
  380. package/docs/v7/mouse-down.mdx +161 -0
  381. package/docs/v7/mouse-up.mdx +164 -0
  382. package/docs/v7/parse.mdx +237 -0
  383. package/docs/v7/performing-actions.mdx +54 -0
  384. package/docs/v7/press-keys.mdx +349 -0
  385. package/docs/v7/provision.mdx +333 -0
  386. package/docs/v7/quickstart.mdx +173 -0
  387. package/docs/v7/redraw.mdx +216 -0
  388. package/docs/v7/reusable-code.mdx +249 -0
  389. package/docs/v7/right-click.mdx +123 -0
  390. package/docs/v7/running-tests.mdx +185 -0
  391. package/docs/v7/screenshot.mdx +249 -0
  392. package/docs/v7/screenshots.mdx +186 -0
  393. package/docs/v7/scroll.mdx +336 -0
  394. package/docs/v7/secrets.mdx +115 -0
  395. package/docs/v7/self-hosted.mdx +149 -0
  396. package/docs/v7/type.mdx +358 -0
  397. package/docs/v7/variables.mdx +111 -0
  398. package/docs/v7/wait.mdx +52 -0
  399. package/docs/v7/waiting-for-elements.mdx +90 -0
  400. package/docs/v7/what-is-testdriver.mdx +54 -0
  401. package/eslint.config.js +67 -0
  402. package/examples/ai.test.mjs +31 -0
  403. package/examples/assert.test.mjs +47 -0
  404. package/examples/chrome-extension.test.mjs +97 -0
  405. package/examples/config.mjs +5 -0
  406. package/examples/element-not-found.test.mjs +27 -0
  407. package/examples/exec-output.test.mjs +60 -0
  408. package/examples/exec-pwsh.test.mjs +58 -0
  409. package/examples/findall-coffee-icons.test.mjs +42 -0
  410. package/examples/focus-window.test.mjs +37 -0
  411. package/examples/formatted-logging.test.mjs +27 -0
  412. package/examples/hover-image.test.mjs +53 -0
  413. package/examples/hover-text-with-description.test.mjs +57 -0
  414. package/examples/hover-text.test.mjs +28 -0
  415. package/examples/installer.test.mjs +50 -0
  416. package/examples/launch-vscode-linux.test.mjs +55 -0
  417. package/examples/match-image.test.mjs +55 -0
  418. package/examples/parse.test.mjs +19 -0
  419. package/examples/press-keys.test.mjs +44 -0
  420. package/examples/prompt.test.mjs +34 -0
  421. package/examples/scroll-keyboard.test.mjs +38 -0
  422. package/examples/scroll-until-image.test.mjs +40 -0
  423. package/examples/scroll.test.mjs +42 -0
  424. package/examples/type.test.mjs +46 -0
  425. package/examples/windows-installer.test.mjs +54 -0
  426. package/index.js +2 -0
  427. package/interfaces/cli/commands/init.js +438 -0
  428. package/interfaces/cli/commands/setup.js +382 -0
  429. package/interfaces/cli/lib/base.js +285 -0
  430. package/interfaces/cli.js +20 -0
  431. package/interfaces/junit-reporter.js +290 -0
  432. package/interfaces/logger.js +388 -0
  433. package/interfaces/readline.js +234 -0
  434. package/interfaces/shared-test-state.mjs +64 -0
  435. package/interfaces/vitest-plugin.d.ts +115 -0
  436. package/interfaces/vitest-plugin.mjs +1698 -0
  437. package/lib/captcha/solver.js +358 -0
  438. package/lib/core/Dashcam.js +533 -0
  439. package/lib/core/index.d.ts +172 -0
  440. package/lib/core/index.js +12 -0
  441. package/lib/environments.json +18 -0
  442. package/lib/github-comment-formatter.js +263 -0
  443. package/lib/github-comment.mjs +452 -0
  444. package/lib/init-project.js +575 -0
  445. package/lib/presets/index.mjs +331 -0
  446. package/lib/resolve-channel.js +46 -0
  447. package/lib/sentry.js +417 -0
  448. package/lib/vitest/hooks.d.ts +57 -0
  449. package/lib/vitest/hooks.mjs +674 -0
  450. package/lib/vitest/setup-aws.mjs +247 -0
  451. package/lib/vitest/setup-self-hosted.mjs +151 -0
  452. package/lib/vitest/setup.mjs +46 -0
  453. package/manual/captcha-api.test.mjs +51 -0
  454. package/manual/drag-and-drop.test.mjs +59 -0
  455. package/manual/flake-diffthreshold-001.test.mjs +9 -0
  456. package/manual/flake-diffthreshold-01.test.mjs +9 -0
  457. package/manual/flake-diffthreshold-05.test.mjs +9 -0
  458. package/manual/flake-noredraw-cache.test.mjs +9 -0
  459. package/manual/flake-noredraw-nocache.test.mjs +9 -0
  460. package/manual/flake-redraw-cache.test.mjs +9 -0
  461. package/manual/flake-redraw-nocache.test.mjs +9 -0
  462. package/manual/flake-rocket-match.test.mjs +30 -0
  463. package/manual/flake-shared.mjs +51 -0
  464. package/manual/no-provision.test.mjs +31 -0
  465. package/manual/packer-hover-image.test.mjs +176 -0
  466. package/manual/scroll-until-text.test.mjs +68 -0
  467. package/manual/test-init-command.js +223 -0
  468. package/mcp-server/README.md +322 -0
  469. package/mcp-server/dist/codegen.d.ts +9 -0
  470. package/mcp-server/dist/codegen.js +165 -0
  471. package/mcp-server/dist/mcp-app.html +114 -0
  472. package/mcp-server/dist/package.json +1 -0
  473. package/mcp-server/dist/provision-types.d.ts +290 -0
  474. package/mcp-server/dist/provision-types.js +174 -0
  475. package/mcp-server/dist/server.d.ts +6 -0
  476. package/mcp-server/dist/server.mjs +1925 -0
  477. package/mcp-server/dist/session.d.ts +85 -0
  478. package/mcp-server/dist/session.js +152 -0
  479. package/mcp-server/mcp-app.html +28 -0
  480. package/mcp-server/mcp-config.example.json +19 -0
  481. package/mcp-server/package-lock.json +4027 -0
  482. package/mcp-server/package.json +31 -0
  483. package/mcp-server/src/codegen.ts +189 -0
  484. package/mcp-server/src/mcp-app.css +360 -0
  485. package/mcp-server/src/mcp-app.ts +547 -0
  486. package/mcp-server/src/provision-types.ts +209 -0
  487. package/mcp-server/src/server.ts +2391 -0
  488. package/mcp-server/src/session.ts +194 -0
  489. package/mcp-server/tsconfig.json +16 -0
  490. package/mcp-server/vite.config.ts +23 -0
  491. package/package.json +158 -0
  492. package/schema.json +1046 -0
  493. package/scripts/generate-skills.js +94 -0
  494. package/sdk-log-formatter.js +1157 -0
  495. package/sdk.d.ts +1486 -0
  496. package/sdk.js +4336 -0
  497. package/setup/aws/cloudformation.yaml +463 -0
  498. package/setup/aws/disable-defender.sh +42 -0
  499. package/setup/aws/install-dev-runner.sh +79 -0
  500. package/setup/aws/spawn-runner.sh +289 -0
  501. package/test/captcha-solver.test.mjs +152 -0
  502. package/test/chrome-remote-debugging.test.mjs +66 -0
  503. package/test/duckduckgo/experiment.test.mjs +28 -0
  504. package/test/duckduckgo/setup.test.mjs +29 -0
  505. package/test/manual/debug-locate-response.js +82 -0
  506. package/test/manual/reconnect-provision.test.mjs +49 -0
  507. package/test/manual/test-console-logs.test.mjs +42 -0
  508. package/test/manual/test-find-api.js +73 -0
  509. package/test/manual/test-init.sh +54 -0
  510. package/test/manual/test-prompt-cache.js +97 -0
  511. package/test/manual/test-provision-auth.mjs +22 -0
  512. package/test/manual/test-sandbox-render.js +29 -0
  513. package/test/manual/test-sdk-methods.js +15 -0
  514. package/test/manual/test-sdk-refactor.js +53 -0
  515. package/test/manual/test-stack-trace.mjs +57 -0
  516. package/test/manual/verify-element-api.js +89 -0
  517. package/test/manual/verify-types.js +0 -0
  518. package/test/manual-unawaited-promise.test.mjs +31 -0
  519. package/vitest.config.mjs +58 -0
  520. package/vitest.runner.config.mjs +33 -0
  521. package/vscode-extension/.vscodeignore +12 -0
  522. package/vscode-extension/README.md +94 -0
  523. package/vscode-extension/media/icon.png +0 -0
  524. package/vscode-extension/package-lock.json +4126 -0
  525. package/vscode-extension/package.json +86 -0
  526. package/vscode-extension/src/extension.ts +829 -0
  527. package/vscode-extension/testdriverai-0.1.0.vsix +0 -0
  528. package/vscode-extension/tsconfig.json +16 -0
@@ -0,0 +1,2391 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * TestDriver MCP Server
4
+ * Enables AI agents to iteratively build tests with visual feedback
5
+ */
6
+
7
+ // Configure logger to use stderr to avoid corrupting MCP JSON-RPC stream on stdout
8
+ process.env.TD_STDIO = "stderr";
9
+ // Enable debug mode to preserve croppedImage in SDK responses (needed for MCP App visuals)
10
+ process.env.TD_DEBUG = "true";
11
+
12
+ import { registerAppResource, registerAppTool, RESOURCE_MIME_TYPE } from "@modelcontextprotocol/ext-apps/server";
13
+ import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
14
+ import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
15
+ import type { Variables } from "@modelcontextprotocol/sdk/shared/uriTemplate.js";
16
+ import type { CallToolResult, ReadResourceResult } from "@modelcontextprotocol/sdk/types.js";
17
+ import * as Sentry from "@sentry/node";
18
+ import * as fs from "fs";
19
+ import * as os from "os";
20
+ import * as path from "path";
21
+ import { fileURLToPath, pathToFileURL } from "url";
22
+ import { z } from "zod";
23
+
24
+ import { generateActionCode } from "./codegen.js";
25
+ import { getProvisionOptions, SessionStartInputSchema, type SessionStartInput } from "./provision-types.js";
26
+ import { sessionManager, type SessionState } from "./session.js";
27
+
28
+ // =============================================================================
29
+ // Sentry
30
+ // =============================================================================
31
+
32
+ // Read version from main package.json (../../package.json from mcp-server/dist/)
33
+ const sdkRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
34
+ const packageJson = JSON.parse(fs.readFileSync(path.join(sdkRoot, "package.json"), "utf-8"));
35
+ const version = packageJson.version || "1.0.0";
36
+
37
+ // Derive release channel from package version prerelease tag (e.g. "7.6.0-test.5" → "test")
38
+ import semver from "semver";
39
+ const KNOWN_CHANNELS = new Set(["dev", "test", "canary", "latest"]);
40
+ function resolveReleaseChannel(ver: string): string {
41
+ if (process.env.TD_CHANNEL && KNOWN_CHANNELS.has(process.env.TD_CHANNEL)) return process.env.TD_CHANNEL;
42
+ const pre = semver.prerelease(ver);
43
+ if (pre && pre.length > 0 && KNOWN_CHANNELS.has(String(pre[0]))) return String(pre[0]);
44
+ return "latest";
45
+ }
46
+ const releaseChannel = resolveReleaseChannel(version);
47
+
48
+ const isSentryEnabled = () => {
49
+ if (process.env.TD_TELEMETRY === "false") {
50
+ return false;
51
+ }
52
+ return true;
53
+ };
54
+
55
+ if (isSentryEnabled()) {
56
+ console.error("Analytics enabled. Set TD_TELEMETRY=false to disable.");
57
+ Sentry.init({
58
+ dsn:
59
+ process.env.SENTRY_DSN ||
60
+ "https://452bd5a00dbd83a38ee8813e11c57694@o4510262629236736.ingest.us.sentry.io/4510480443637760",
61
+ environment: releaseChannel,
62
+ release: version,
63
+ sampleRate: 1.0,
64
+ tracesSampleRate: 1.0,
65
+ sendDefaultPii: true,
66
+ integrations: [Sentry.httpIntegration(), Sentry.nodeContextIntegration()],
67
+ initialScope: {
68
+ tags: {
69
+ platform: os.platform(),
70
+ arch: os.arch(),
71
+ nodeVersion: process.version,
72
+ },
73
+ },
74
+ // Filter out expected test/element failures - only report actual exceptions and crashes
75
+ beforeSend(event, hint) {
76
+ const error = hint.originalException;
77
+
78
+ if (error && typeof error === "object" && "message" in error) {
79
+ const msg = (error as { message: string }).message;
80
+
81
+ // Don't send user-initiated exits
82
+ if (msg.includes("User cancelled")) {
83
+ return null;
84
+ }
85
+
86
+ // Don't send expected test/element failures - these are normal test outcomes, not crashes
87
+ if (
88
+ msg.includes("Element not found") ||
89
+ msg.includes("No elements found") ||
90
+ msg.includes("No element found") ||
91
+ msg.includes("Assertion failed") ||
92
+ msg.includes("assertion failed")
93
+ ) {
94
+ return null;
95
+ }
96
+ }
97
+
98
+ // Filter out TestFailure errors (test failures, not crashes)
99
+ if (error && typeof error === "object" && "name" in error && (error as { name: string }).name === "TestFailure") {
100
+ return null;
101
+ }
102
+
103
+ // Filter out ElementNotFoundError - expected test outcome, not a crash
104
+ if (error && typeof error === "object" && "name" in error && (error as { name: string }).name === "ElementNotFoundError") {
105
+ return null;
106
+ }
107
+
108
+ return event;
109
+ },
110
+ });
111
+ }
112
+
113
+ function captureException(error: Error, context: { tags?: Record<string, string>; extra?: Record<string, unknown> } = {}) {
114
+ if (!isSentryEnabled()) return;
115
+
116
+ Sentry.withScope((scope) => {
117
+ if (context.tags) {
118
+ Object.entries(context.tags).forEach(([key, value]) => {
119
+ scope.setTag(key, value);
120
+ });
121
+ }
122
+ if (context.extra) {
123
+ Object.entries(context.extra).forEach(([key, value]) => {
124
+ scope.setExtra(key, value);
125
+ });
126
+ }
127
+ Sentry.captureException(error);
128
+ });
129
+ }
130
+
131
+ function setSessionContext(sessionId: string, sandboxId?: string) {
132
+ if (!isSentryEnabled()) return;
133
+
134
+ Sentry.setTag("session", sessionId);
135
+ if (sandboxId) {
136
+ Sentry.setTag("sandbox", sandboxId);
137
+ }
138
+ Sentry.setContext("session", {
139
+ sessionId,
140
+ sandboxId,
141
+ });
142
+ }
143
+
144
+ async function flushSentry(timeout = 2000) {
145
+ if (!isSentryEnabled()) return;
146
+ await Sentry.flush(timeout);
147
+ }
148
+
149
+ // =============================================================================
150
+ // Logging
151
+ // =============================================================================
152
+
153
+ const LOG_LEVELS = {
154
+ DEBUG: 0,
155
+ INFO: 1,
156
+ WARN: 2,
157
+ ERROR: 3,
158
+ } as const;
159
+
160
+ type LogLevel = keyof typeof LOG_LEVELS;
161
+
162
+ // Set via TD_LOG_LEVEL env var (default: INFO)
163
+ const currentLogLevel = LOG_LEVELS[(process.env.TD_LOG_LEVEL?.toUpperCase() as LogLevel) || "INFO"] ?? LOG_LEVELS.INFO;
164
+
165
+ function log(level: LogLevel, message: string, data?: Record<string, unknown>) {
166
+ if (LOG_LEVELS[level] < currentLogLevel) return;
167
+
168
+ const timestamp = new Date().toISOString();
169
+ const prefix = `[${timestamp}] [${level}]`;
170
+ const dataStr = data ? ` ${JSON.stringify(data)}` : "";
171
+ console.error(`${prefix} ${message}${dataStr}`);
172
+ }
173
+
174
+ const logger = {
175
+ debug: (msg: string, data?: Record<string, unknown>) => log("DEBUG", msg, data),
176
+ info: (msg: string, data?: Record<string, unknown>) => log("INFO", msg, data),
177
+ warn: (msg: string, data?: Record<string, unknown>) => log("WARN", msg, data),
178
+ error: (msg: string, data?: Record<string, unknown>) => log("ERROR", msg, data),
179
+ };
180
+
181
+ // Get directory for UI files - works both from source and compiled
182
+ const __filename = fileURLToPath(import.meta.url);
183
+ const __dirname = path.dirname(__filename);
184
+ const DIST_DIR = __filename.endsWith(".ts")
185
+ ? path.join(__dirname, "..", "dist")
186
+ : __dirname;
187
+
188
+ // Resource URI for the screenshot result UI
189
+ const RESOURCE_URI = "ui://testdriver/mcp-app.html";
190
+
191
+ // Resource URI base for serving screenshot blobs (with dynamic IDs)
192
+ const SCREENSHOT_RESOURCE_BASE = "screenshot://testdriver/screenshot";
193
+ const CROPPED_IMAGE_RESOURCE_BASE = "screenshot://testdriver/cropped";
194
+
195
+ // SDK instance (will be initialized on session start)
196
+ let sdk: any = null;
197
+
198
+ // Last screenshot base64 for check comparisons
199
+ let lastScreenshotBase64: string | null = null;
200
+
201
+ // =============================================================================
202
+ // Image Store - Stores images with unique IDs for reload persistence
203
+ // =============================================================================
204
+
205
+ interface StoredImage {
206
+ data: string; // base64 image data
207
+ type: "screenshot" | "cropped";
208
+ timestamp: number;
209
+ }
210
+
211
+ // Map of image ID -> image data
212
+ const imageStore = new Map<string, StoredImage>();
213
+
214
+ // Counter for generating unique image IDs
215
+ let imageIdCounter = 0;
216
+
217
+ // Maximum number of images to store (to prevent memory leaks)
218
+ const MAX_STORED_IMAGES = 100;
219
+
220
+ /**
221
+ * Store an image and return its unique resource URI
222
+ */
223
+ function storeImage(data: string, type: "screenshot" | "cropped"): string {
224
+ const id = `${type}-${++imageIdCounter}`;
225
+
226
+ // Clean up old images if we exceed the limit
227
+ if (imageStore.size >= MAX_STORED_IMAGES) {
228
+ // Remove oldest images (first entries in the map)
229
+ const entriesToRemove = Math.floor(MAX_STORED_IMAGES / 4);
230
+ const keys = Array.from(imageStore.keys()).slice(0, entriesToRemove);
231
+ for (const key of keys) {
232
+ imageStore.delete(key);
233
+ }
234
+ logger.debug("storeImage: Cleaned up old images", { removed: entriesToRemove, remaining: imageStore.size });
235
+ }
236
+
237
+ imageStore.set(id, {
238
+ data,
239
+ type,
240
+ timestamp: Date.now(),
241
+ });
242
+
243
+ logger.debug("storeImage: Stored image", { id, type, dataLength: data.length });
244
+
245
+ const base = type === "screenshot" ? SCREENSHOT_RESOURCE_BASE : CROPPED_IMAGE_RESOURCE_BASE;
246
+ return `${base}/${id}`;
247
+ }
248
+
249
+ /**
250
+ * Get an image by its ID
251
+ */
252
+ function getStoredImage(id: string): StoredImage | undefined {
253
+ return imageStore.get(id);
254
+ }
255
+
256
+ /**
257
+ * Get session info for structured content
258
+ */
259
+ function getSessionData(session: SessionState | null) {
260
+ if (!session) return { id: null, expiresIn: 0 };
261
+ return {
262
+ id: session.sessionId,
263
+ expiresIn: sessionManager.getTimeRemaining(session.sessionId),
264
+ };
265
+ }
266
+
267
+ /**
268
+ * Check if session is ready for use - returns error result if not
269
+ * This helper provides clear, actionable error messages for the AI
270
+ *
271
+ * Auto-extends the session on each successful check to prevent expiry during active use
272
+ */
273
+ function requireActiveSession(): { valid: true } | { valid: false; error: CallToolResult } {
274
+ const session = sessionManager.getCurrentSession();
275
+
276
+ // No session ever created
277
+ if (!sdk || !session) {
278
+ return {
279
+ valid: false,
280
+ error: createToolResult(
281
+ false,
282
+ "ERROR: No active session. You must call session_start first to create a sandbox before using any other tools.",
283
+ {
284
+ error: "NO_SESSION",
285
+ action: "session_start",
286
+ message: "No sandbox session exists. Call session_start to create one."
287
+ }
288
+ )
289
+ };
290
+ }
291
+
292
+ // Session exists but has expired
293
+ if (!sessionManager.isSessionValid(session.sessionId)) {
294
+ // Clear the SDK reference since the sandbox is no longer available
295
+ sdk = null;
296
+ return {
297
+ valid: false,
298
+ error: createToolResult(
299
+ false,
300
+ "ERROR: Session has expired or timed out. The sandbox is no longer available. You must call session_start again to create a new sandbox session before continuing.",
301
+ {
302
+ error: "SESSION_EXPIRED",
303
+ action: "session_start",
304
+ message: "The previous sandbox session has expired. Call session_start to create a new one.",
305
+ expiredSessionId: session.sessionId
306
+ }
307
+ )
308
+ };
309
+ }
310
+
311
+ // Auto-extend session on each command to prevent expiry during active use
312
+ // This resets the expiry timer back to the original keepAlive duration
313
+ sessionManager.refreshSession(session.sessionId);
314
+
315
+ return { valid: true };
316
+ }
317
+
318
+ /**
319
+ * Create tool result with structured content for MCP App
320
+ * Images: imageUrl (data URL) goes to structuredContent for UI to display
321
+ * The croppedImage from find() is small (~10KB) so it's acceptable as data URL
322
+ *
323
+ * If generatedCode is provided, it's appended to the text response with instructions
324
+ * for the agent to write it to the test file.
325
+ */
326
+ function createToolResult(
327
+ success: boolean,
328
+ textContent: string,
329
+ structuredData: Record<string, unknown>,
330
+ generatedCode?: string
331
+ ): CallToolResult {
332
+ // Build text content - append generated code if provided with directive instructions
333
+ let fullText = textContent;
334
+ if (generatedCode && success) {
335
+ // Get the test file from the current session
336
+ const session = sessionManager.getCurrentSession();
337
+ const testFile = session?.testFile;
338
+
339
+ if (testFile) {
340
+ fullText += `\n\n⚠️ ACTION REQUIRED: Append this code to ${testFile}:\n\`\`\`javascript\n${generatedCode}\n\`\`\``;
341
+ } else {
342
+ fullText += `\n\n⚠️ ACTION REQUIRED: Append this code to the test file:\n\`\`\`javascript\n${generatedCode}\n\`\`\``;
343
+ }
344
+ }
345
+
346
+ const content: CallToolResult["content"] = [{ type: "text", text: fullText }];
347
+
348
+ logger.debug("createToolResult", {
349
+ success,
350
+ action: structuredData.action,
351
+ hasImage: !!structuredData.imageUrl,
352
+ duration: structuredData.duration,
353
+ hasGeneratedCode: !!generatedCode
354
+ });
355
+
356
+ // structuredContent goes to UI (includes imageUrl for display)
357
+ // Always include success flag so UI can display correct status indicator
358
+ // Include generatedCode and testFile in structured data so agents can programmatically handle it
359
+ const session = sessionManager.getCurrentSession();
360
+ return {
361
+ content,
362
+ structuredContent: {
363
+ ...structuredData,
364
+ success,
365
+ generatedCode: generatedCode && success ? generatedCode : undefined,
366
+ testFile: session?.testFile || undefined,
367
+ },
368
+ };
369
+ }
370
+
371
+ // Create MCP server wrapped with Sentry for automatic tracing
372
+ const server = isSentryEnabled()
373
+ ? Sentry.wrapMcpServerWithSentry(
374
+ new McpServer({
375
+ name: "testdriver",
376
+ version: version,
377
+ })
378
+ )
379
+ : new McpServer({
380
+ name: "testdriver",
381
+ version: version,
382
+ });
383
+
384
+ // Element reference storage (for click/hover after find)
385
+ // Stores actual Element instances - no raw coordinates as input
386
+ const elementRefs = new Map<string, { element: any; description: string; coords: { x: number; y: number; centerX: number; centerY: number } }>();
387
+
388
+ // =============================================================================
389
+ // Register UI Resource
390
+ // =============================================================================
391
+
392
+ registerAppResource(
393
+ server,
394
+ RESOURCE_URI,
395
+ RESOURCE_URI,
396
+ { mimeType: RESOURCE_MIME_TYPE, description: "TestDriver Screenshot Viewer UI" },
397
+ async (): Promise<ReadResourceResult> => {
398
+ const htmlPath = path.join(DIST_DIR, "mcp-app.html");
399
+
400
+ if (!fs.existsSync(htmlPath)) {
401
+ throw new Error(`UI file not found: ${htmlPath}`);
402
+ }
403
+
404
+ const html = fs.readFileSync(htmlPath, "utf-8");
405
+ return {
406
+ contents: [{ uri: RESOURCE_URI, mimeType: RESOURCE_MIME_TYPE, text: html }],
407
+ };
408
+ }
409
+ );
410
+
411
+ // Register screenshot resource template for serving binary blobs by ID
412
+ server.registerResource(
413
+ "Screenshot",
414
+ new ResourceTemplate(`${SCREENSHOT_RESOURCE_BASE}/{imageId}`, { list: undefined }),
415
+ {
416
+ description: "Screenshot from TestDriver session served as base64 blob",
417
+ mimeType: "image/png",
418
+ },
419
+ async (uri: URL, variables: Variables): Promise<ReadResourceResult> => {
420
+ const imageId = variables.imageId as string;
421
+ const image = getStoredImage(imageId);
422
+
423
+ if (!image) {
424
+ throw new Error(`Screenshot not found: ${imageId}. It may have been cleaned up.`);
425
+ }
426
+
427
+ logger.debug("screenshot resource: Serving screenshot blob", {
428
+ imageId,
429
+ blobLength: image.data.length
430
+ });
431
+
432
+ return {
433
+ contents: [{
434
+ uri: uri.href,
435
+ mimeType: "image/png",
436
+ blob: image.data,
437
+ }],
438
+ };
439
+ }
440
+ );
441
+
442
+ // Register cropped image resource template for serving find operation results by ID
443
+ server.registerResource(
444
+ "CroppedImage",
445
+ new ResourceTemplate(`${CROPPED_IMAGE_RESOURCE_BASE}/{imageId}`, { list: undefined }),
446
+ {
447
+ description: "Cropped image from find operations served as base64 blob",
448
+ mimeType: "image/png",
449
+ },
450
+ async (uri: URL, variables: Variables): Promise<ReadResourceResult> => {
451
+ const imageId = variables.imageId as string;
452
+ const image = getStoredImage(imageId);
453
+
454
+ if (!image) {
455
+ throw new Error(`Cropped image not found: ${imageId}. It may have been cleaned up.`);
456
+ }
457
+
458
+ logger.debug("cropped image resource: Serving cropped image blob", {
459
+ imageId,
460
+ blobLength: image.data.length
461
+ });
462
+
463
+ return {
464
+ contents: [{
465
+ uri: uri.href,
466
+ mimeType: "image/png",
467
+ blob: image.data,
468
+ }],
469
+ };
470
+ }
471
+ );
472
+
473
+ // =============================================================================
474
+ // Tools
475
+ // =============================================================================
476
+
477
+ // Session Start
478
+ registerAppTool(
479
+ server,
480
+ "session_start",
481
+ {
482
+ title: "Session Start",
483
+ description: `Start a new TestDriver session and provision a sandbox with browser or app.
484
+
485
+ ⚠️ IMPORTANT - Test File Parameter:
486
+ When 'testFile' is provided, you MUST append the generated code to that file after EVERY successful action.
487
+ Each tool response includes "ACTION REQUIRED: Append this code..." - you must write that code to the test file immediately.
488
+
489
+ Provision types:
490
+ - chrome: Launch Chrome browser (default). Use 'url' for starting page.
491
+ - chromeExtension: Launch Chrome with an extension. Use 'extensionPath' or 'extensionId'.
492
+ - vscode: Launch VS Code. Use 'workspace' and optional 'extensions'.
493
+ - installer: Download and install an app. Use 'installerUrl' (required).
494
+ - electron: Launch an Electron app. Use 'appPath' (required).
495
+
496
+ Self-hosted mode:
497
+ - Provide 'ip' parameter to connect directly to a self-hosted Windows instance
498
+ - Set 'os' to 'windows' when connecting to Windows instances
499
+ - The IP can be from an AWS EC2 instance spawned via CloudFormation
500
+ - See https://docs.testdriver.ai/v7/aws-setup for AWS setup guide
501
+
502
+ Debug mode (connect to existing sandbox):
503
+ - Provide 'sandboxId' to connect to an existing sandbox (e.g., from a failed test with debugOnFailure: true)
504
+ - Skips provisioning - connects to sandbox in its current state
505
+ - Use this to interactively debug failed tests without re-running from scratch`,
506
+ inputSchema: SessionStartInputSchema as any,
507
+ _meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
508
+ },
509
+ async (params: SessionStartInput): Promise<CallToolResult> => {
510
+ const startTime = Date.now();
511
+ logger.info("session_start: Starting", {
512
+ type: params.type,
513
+ url: params.url,
514
+ os: params.os,
515
+ reconnect: params.reconnect,
516
+ sandboxId: params.sandboxId,
517
+ });
518
+
519
+ try {
520
+ // Validate required fields for specific provision types (unless connecting to existing sandbox)
521
+ if (!params.sandboxId) {
522
+ if (params.type === "installer" && !params.installerUrl) {
523
+ return createToolResult(false, "installer type requires 'installerUrl' parameter", { error: "Missing required parameter: installerUrl" });
524
+ }
525
+ if (params.type === "electron" && !params.appPath) {
526
+ return createToolResult(false, "electron type requires 'appPath' parameter", { error: "Missing required parameter: appPath" });
527
+ }
528
+ }
529
+
530
+ // Create new session
531
+ const newSession = sessionManager.createSession({
532
+ os: params.os,
533
+ keepAlive: params.keepAlive,
534
+ testFile: params.testFile,
535
+ });
536
+ logger.debug("session_start: Session created", { sessionId: newSession.sessionId });
537
+
538
+ // Determine API root
539
+ const apiRoot = params.apiRoot || process.env.TD_API_ROOT || "https://api.testdriver.ai";
540
+ logger.debug("session_start: Using API root", { apiRoot });
541
+
542
+ // Initialize SDK
543
+ logger.debug("session_start: Initializing SDK");
544
+ const TestDriverSDK = (await import("../../sdk.js")).default;
545
+
546
+ // Determine preview mode from environment variable
547
+ // TD_PREVIEW can be "ide", "browser", or "none"
548
+ // Default to "ide" so the live preview shows within the IDE
549
+ const previewMode = process.env.TD_PREVIEW || "ide";
550
+ logger.debug("session_start: Preview mode", { preview: previewMode });
551
+
552
+ // Get IP from params or environment (for self-hosted instances)
553
+ const instanceIp = params.ip || process.env.TD_IP;
554
+
555
+ // Get API key - check multiple sources for GitHub Copilot coding agent compatibility
556
+ // 1. TD_API_KEY (standard environment variable)
557
+ // 2. COPILOT_MCP_TD_API_KEY (fallback for GitHub Copilot coding agent)
558
+ const apiKey = process.env.TD_API_KEY || process.env.COPILOT_MCP_TD_API_KEY || "";
559
+
560
+ if (!apiKey) {
561
+ logger.error("session_start: No API key found", {
562
+ hasTD_API_KEY: !!process.env.TD_API_KEY,
563
+ hasCOPILOT_MCP_TD_API_KEY: !!process.env.COPILOT_MCP_TD_API_KEY,
564
+ availableEnvVars: Object.keys(process.env).filter(k => k.includes('TD') || k.includes('COPILOT_MCP'))
565
+ });
566
+ return createToolResult(false, "No API key found. Please set TD_API_KEY or COPILOT_MCP_TD_API_KEY environment variable.", {
567
+ error: "Missing API key",
568
+ hint: "For GitHub Copilot coding agent, create a Copilot environment secret named COPILOT_MCP_TD_API_KEY"
569
+ });
570
+ }
571
+
572
+ logger.debug("session_start: API key found", {
573
+ source: process.env.TD_API_KEY ? "TD_API_KEY" : "COPILOT_MCP_TD_API_KEY",
574
+ keyPrefix: apiKey.substring(0, 7) + "..."
575
+ });
576
+
577
+ sdk = new TestDriverSDK(apiKey, {
578
+ os: params.os,
579
+ logging: false,
580
+ apiRoot,
581
+ preview: previewMode as "browser" | "ide" | "none",
582
+ ip: instanceIp,
583
+ });
584
+
585
+ // Handle sandboxId mode - connect to existing sandbox (debug-on-failure mode)
586
+ if (params.sandboxId) {
587
+ logger.info("session_start: Connecting to existing sandbox (debug mode)", { sandboxId: params.sandboxId });
588
+ await sdk.connect({
589
+ sandboxId: params.sandboxId,
590
+ keepAlive: params.keepAlive,
591
+ });
592
+
593
+ // Get sandbox ID
594
+ const instance = sdk.getInstance();
595
+ logger.info("session_start: Connected to existing sandbox", { instanceId: instance?.instanceId });
596
+ sessionManager.activateSession(newSession.sessionId, instance?.instanceId || params.sandboxId);
597
+
598
+ // Set Sentry context for error tracking
599
+ setSessionContext(newSession.sessionId, instance?.instanceId);
600
+
601
+ // Capture screenshot of current state
602
+ logger.debug("session_start: Capturing screenshot of existing sandbox");
603
+ const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
604
+
605
+ let screenshotResourceUri: string | undefined;
606
+ if (screenshotBase64) {
607
+ screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
608
+ lastScreenshotBase64 = screenshotBase64;
609
+ }
610
+
611
+ const duration = Date.now() - startTime;
612
+ logger.info("session_start: Connected to existing sandbox", { duration, sessionId: newSession.sessionId, sandboxId: params.sandboxId });
613
+
614
+ return createToolResult(
615
+ true,
616
+ `Connected to existing sandbox (debug mode)
617
+ Session: ${newSession.sessionId}
618
+ Sandbox: ${params.sandboxId}
619
+ Expires in: ${Math.round(params.keepAlive / 1000)}s
620
+
621
+ You are now connected to the sandbox in its current state. Use find, click, type, etc. to interact.`,
622
+ {
623
+ action: "session_start",
624
+ sessionId: newSession.sessionId,
625
+ sandboxId: params.sandboxId,
626
+ debugMode: true,
627
+ screenshotResourceUri,
628
+ duration
629
+ },
630
+ "// Connected to existing sandbox - no provision code needed"
631
+ );
632
+ }
633
+
634
+ // Connect to sandbox
635
+ if (instanceIp) {
636
+ logger.info("session_start: Connecting to self-hosted instance...", { ip: instanceIp });
637
+ } else {
638
+ logger.info("session_start: Connecting to cloud sandbox...");
639
+ }
640
+ await sdk.connect({
641
+ reconnect: params.reconnect,
642
+ keepAlive: params.keepAlive,
643
+ ip: instanceIp,
644
+ });
645
+
646
+ // Get sandbox ID
647
+ const instance = sdk.getInstance();
648
+ logger.info("session_start: Connected to sandbox", { instanceId: instance?.instanceId });
649
+ sessionManager.activateSession(newSession.sessionId, instance?.instanceId || "unknown");
650
+
651
+ // Set Sentry context for error tracking
652
+ setSessionContext(newSession.sessionId, instance?.instanceId);
653
+
654
+ // Get provision-specific options
655
+ const provisionOptions = getProvisionOptions(params);
656
+ let provisionCmd = "";
657
+
658
+ // Provision based on type
659
+ switch (params.type) {
660
+ case "chrome": {
661
+ const chromeOpts = provisionOptions as { url: string; maximized?: boolean; guest?: boolean };
662
+ logger.info("session_start: Provisioning Chrome", { url: chromeOpts.url });
663
+ await sdk.provision.chrome(chromeOpts);
664
+ provisionCmd = "provision.chrome";
665
+ logger.debug("session_start: Chrome provisioned");
666
+ break;
667
+ }
668
+
669
+ case "chromeExtension": {
670
+ const extOpts = provisionOptions as { extensionPath?: string; extensionId?: string; maximized?: boolean };
671
+ logger.info("session_start: Provisioning Chrome Extension", { extensionPath: extOpts.extensionPath, extensionId: extOpts.extensionId });
672
+ await sdk.provision.chromeExtension(extOpts);
673
+ provisionCmd = "provision.chromeExtension";
674
+ logger.debug("session_start: Chrome Extension provisioned");
675
+ break;
676
+ }
677
+
678
+ case "vscode": {
679
+ const vscodeOpts = provisionOptions as { workspace?: string; extensions?: string[] };
680
+ logger.info("session_start: Provisioning VS Code", { workspace: vscodeOpts.workspace });
681
+ await sdk.provision.vscode(vscodeOpts);
682
+ provisionCmd = "provision.vscode";
683
+ logger.debug("session_start: VS Code provisioned");
684
+ break;
685
+ }
686
+
687
+ case "installer": {
688
+ const installerOpts = provisionOptions as { url: string; filename?: string; appName?: string; launch?: boolean };
689
+ logger.info("session_start: Provisioning installer", { url: installerOpts.url });
690
+ await sdk.provision.installer(installerOpts);
691
+ provisionCmd = "provision.installer";
692
+ logger.debug("session_start: Installer provisioned");
693
+ break;
694
+ }
695
+
696
+ case "electron": {
697
+ const electronOpts = provisionOptions as { appPath: string; args?: string[] };
698
+ logger.info("session_start: Provisioning Electron", { appPath: electronOpts.appPath });
699
+ await sdk.provision.electron(electronOpts);
700
+ provisionCmd = "provision.electron";
701
+ logger.debug("session_start: Electron app provisioned");
702
+ break;
703
+ }
704
+ }
705
+
706
+ // Capture initial screenshot after provisioning
707
+ logger.debug("session_start: Capturing initial screenshot");
708
+ const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
709
+
710
+ let screenshotResourceUri: string | undefined;
711
+ if (screenshotBase64) {
712
+ screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
713
+ lastScreenshotBase64 = screenshotBase64;
714
+ }
715
+
716
+ const duration = Date.now() - startTime;
717
+ logger.info("session_start: Completed", { duration, sessionId: newSession.sessionId, selfHosted: !!instanceIp });
718
+
719
+ // Generate the code for this provision action
720
+ const generatedCode = generateActionCode(provisionCmd, provisionOptions);
721
+
722
+ // Build debugger URL for the session
723
+ const debuggerUrl = instance?.debuggerUrl || (instanceIp ? `http://${instanceIp}:9222` : null);
724
+
725
+ const connectionType = instanceIp ? `Self-hosted (${instanceIp})` : "Cloud";
726
+ return createToolResult(
727
+ true,
728
+ `Session started: ${newSession.sessionId}\nConnection: ${connectionType}\nType: ${params.type}\nSandbox: ${instance?.instanceId}\nExpires in: ${Math.round(params.keepAlive / 1000)}s
729
+
730
+ IMPORTANT - If creating a new test project, use these EXACT dependencies in package.json:
731
+ {
732
+ "type": "module",
733
+ "devDependencies": {
734
+ "testdriverai": "beta",
735
+ "vitest": "^4.0.0"
736
+ },
737
+ "scripts": {
738
+ "test": "vitest"
739
+ }
740
+ }`,
741
+ {
742
+ action: "session_start",
743
+ sessionId: newSession.sessionId,
744
+ provisionType: params.type,
745
+ selfHosted: !!instanceIp,
746
+ instanceIp: instanceIp || undefined,
747
+ debuggerUrl,
748
+ screenshotResourceUri,
749
+ duration
750
+ },
751
+ generatedCode
752
+ );
753
+ } catch (error) {
754
+ logger.error("session_start: Failed", { error: String(error) });
755
+ captureException(error as Error, { tags: { tool: "session_start" }, extra: { params } });
756
+ throw error;
757
+ }
758
+ }
759
+ );
760
+
761
+ // Session Status
762
+ server.registerTool(
763
+ "session_status",
764
+ {
765
+ description: "Check the current session status and time remaining",
766
+ inputSchema: z.object({}),
767
+ },
768
+ async (): Promise<CallToolResult> => {
769
+ const startTime = Date.now();
770
+ logger.info("session_status: Checking");
771
+ const session = sessionManager.getCurrentSession();
772
+
773
+ if (!session) {
774
+ logger.warn("session_status: No active session");
775
+ return createToolResult(false, "No active session", { error: "No active session. Call session_start first." });
776
+ }
777
+
778
+ const summary = sessionManager.getSessionSummary(session.sessionId);
779
+ const duration = Date.now() - startTime;
780
+ logger.info("session_status: Completed", {
781
+ sessionId: session.sessionId,
782
+ status: session.status,
783
+ timeRemaining: summary?.timeRemaining,
784
+ duration
785
+ });
786
+
787
+ return createToolResult(
788
+ true,
789
+ `Session: ${session.sessionId}\nStatus: ${session.status}\nTime remaining: ${Math.round((summary?.timeRemaining || 0) / 1000)}s`,
790
+ { action: "session_status", ...summary, sessionId: session.sessionId, status: session.status, duration }
791
+ );
792
+ }
793
+ );
794
+
795
+ // Session Extend
796
+ server.registerTool(
797
+ "session_extend",
798
+ {
799
+ description: "Extend the session keepAlive time",
800
+ inputSchema: z.object({
801
+ additionalMs: z.number().default(60000).describe("Additional time in ms"),
802
+ }),
803
+ },
804
+ async (params) => {
805
+ logger.info("session_extend: Extending", { additionalMs: params.additionalMs });
806
+ const session = sessionManager.getCurrentSession();
807
+
808
+ if (!session) {
809
+ logger.warn("session_extend: No active session");
810
+ return { content: [{ type: "text" as const, text: "No active session" }] };
811
+ }
812
+
813
+ sessionManager.extendSession(session.sessionId, params.additionalMs);
814
+ const newExpiry = sessionManager.getTimeRemaining(session.sessionId);
815
+ logger.info("session_extend: Extended", { sessionId: session.sessionId, newExpiry });
816
+
817
+ return {
818
+ content: [
819
+ {
820
+ type: "text" as const,
821
+ text: `Session extended by ${params.additionalMs / 1000}s. New expiry: ${Math.round(newExpiry / 1000)}s`,
822
+ },
823
+ ],
824
+ };
825
+ }
826
+ );
827
+
828
+ // Find Element
829
+ registerAppTool(
830
+ server,
831
+ "find",
832
+ {
833
+ title: "Find Element",
834
+ description: "Find an element on screen by natural language description",
835
+ inputSchema: z.object({
836
+ description: z.string().describe("Natural language description of the element"),
837
+ timeout: z.number().optional().describe("Timeout in ms for polling"),
838
+ }) as any,
839
+ _meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
840
+ },
841
+ async (params: { description: string; timeout?: number }): Promise<CallToolResult> => {
842
+ const startTime = Date.now();
843
+ logger.info("find: Starting", { description: params.description, timeout: params.timeout });
844
+
845
+ const sessionCheck = requireActiveSession();
846
+ if (!sessionCheck.valid) {
847
+ logger.warn("find: No active session");
848
+ return sessionCheck.error;
849
+ }
850
+
851
+ try {
852
+ logger.debug("find: Calling SDK find");
853
+ const element = await sdk.find(params.description, params.timeout ? { timeout: params.timeout } : undefined);
854
+ const found = element.found();
855
+ const coords = element.getCoordinates();
856
+
857
+ // Store element ref for later use (stores actual Element instance)
858
+ const elementRef = `el-${Date.now()}`;
859
+ if (found && coords) {
860
+ elementRefs.set(elementRef, {
861
+ element: element, // Store the actual Element instance
862
+ description: params.description,
863
+ coords: {
864
+ x: coords.x,
865
+ y: coords.y,
866
+ centerX: coords.centerX,
867
+ centerY: coords.centerY,
868
+ },
869
+ });
870
+ logger.info("find: Element found", {
871
+ description: params.description,
872
+ coords: { x: coords.centerX, y: coords.centerY },
873
+ confidence: element.confidence,
874
+ elementRef
875
+ });
876
+ } else {
877
+ logger.warn("find: Element not found", { description: params.description });
878
+ }
879
+
880
+ // Return raw SDK response directly
881
+ const rawResponse = element._response || {};
882
+ const duration = Date.now() - startTime;
883
+
884
+ // Store cropped image for resource serving (instead of inline data URL)
885
+ let croppedImageResourceUri: string | undefined;
886
+ let screenshotResourceUri: string | undefined;
887
+ const croppedImage = rawResponse.croppedImage;
888
+ if (croppedImage) {
889
+ const imageData = croppedImage.startsWith('data:')
890
+ ? croppedImage.replace(/^data:image\/\w+;base64,/, '')
891
+ : croppedImage;
892
+ croppedImageResourceUri = storeImage(imageData, "cropped");
893
+ // Remove croppedImage from response to avoid context bloat
894
+ delete rawResponse.croppedImage;
895
+ } else if (!found) {
896
+ // Element not found and no cropped image - capture a fresh screenshot
897
+ // so the user can see what's currently visible on screen
898
+ try {
899
+ const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
900
+ if (screenshotBase64) {
901
+ screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
902
+ logger.debug("find: Captured screenshot for not-found state");
903
+ }
904
+ } catch (e) {
905
+ logger.warn("find: Failed to capture screenshot for not-found state", { error: String(e) });
906
+ }
907
+ }
908
+
909
+ // Remove extractedText and pixelDiffImage from response to reduce context bloat
910
+ delete rawResponse.extractedText;
911
+ delete rawResponse.pixelDiffImage;
912
+
913
+ // Generate code for this find action
914
+ const generatedCode = found ? generateActionCode("find", { description: params.description }) : undefined;
915
+
916
+ // Build element info for display (cropped image is always centered on element)
917
+ const elementInfo = found ? {
918
+ description: params.description,
919
+ centerX: coords?.centerX,
920
+ centerY: coords?.centerY,
921
+ confidence: element.confidence,
922
+ ref: elementRef,
923
+ } : undefined;
924
+
925
+ return createToolResult(
926
+ found,
927
+ found
928
+ ? `Found: "${params.description}" at (${rawResponse.coordinates?.x}, ${rawResponse.coordinates?.y})\nRef: ${elementRef}`
929
+ : `Element not found: "${params.description}"`,
930
+ {
931
+ ...rawResponse,
932
+ action: "find",
933
+ element: elementInfo,
934
+ ref: elementRef,
935
+ croppedImageResourceUri,
936
+ screenshotResourceUri,
937
+ duration,
938
+ },
939
+ generatedCode
940
+ );
941
+ } catch (error) {
942
+ logger.error("find: Failed", { error: String(error), description: params.description });
943
+ captureException(error as Error, { tags: { tool: "find" }, extra: { description: params.description } });
944
+ throw error;
945
+ }
946
+ }
947
+ );
948
+
949
+ // Find All Elements
950
+ registerAppTool(
951
+ server,
952
+ "findall",
953
+ {
954
+ title: "Find All Elements",
955
+ description: "Find all elements on screen matching a natural language description. Returns an array of element references.",
956
+ inputSchema: z.object({
957
+ description: z.string().describe("Natural language description of the elements to find"),
958
+ timeout: z.number().optional().describe("Timeout in ms for polling"),
959
+ }) as any,
960
+ _meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
961
+ },
962
+ async (params: { description: string; timeout?: number }): Promise<CallToolResult> => {
963
+ const startTime = Date.now();
964
+ logger.info("findall: Starting", { description: params.description, timeout: params.timeout });
965
+
966
+ const sessionCheck = requireActiveSession();
967
+ if (!sessionCheck.valid) {
968
+ logger.warn("findall: No active session");
969
+ return sessionCheck.error;
970
+ }
971
+
972
+ try {
973
+ logger.debug("findall: Calling SDK findAll");
974
+ const elements = await sdk.findAll(params.description, params.timeout ? { timeout: params.timeout } : undefined);
975
+ const count = elements.length;
976
+
977
+ // Store element refs for later use
978
+ const refs: string[] = [];
979
+ const elementInfos: Array<{ ref: string; x: number; y: number; centerX: number; centerY: number; confidence: number }> = [];
980
+
981
+ for (let i = 0; i < elements.length; i++) {
982
+ const element = elements[i];
983
+ const coords = element.getCoordinates();
984
+ const elementRef = `el-${Date.now()}-${i}`;
985
+
986
+ if (coords) {
987
+ elementRefs.set(elementRef, {
988
+ element: element,
989
+ description: `${params.description} [${i}]`,
990
+ coords: {
991
+ x: coords.x,
992
+ y: coords.y,
993
+ centerX: coords.centerX,
994
+ centerY: coords.centerY,
995
+ },
996
+ });
997
+ refs.push(elementRef);
998
+ elementInfos.push({
999
+ ref: elementRef,
1000
+ x: coords.x,
1001
+ y: coords.y,
1002
+ centerX: coords.centerX,
1003
+ centerY: coords.centerY,
1004
+ confidence: element.confidence,
1005
+ });
1006
+ }
1007
+ }
1008
+
1009
+ logger.info("findall: Elements found", {
1010
+ description: params.description,
1011
+ count,
1012
+ refs
1013
+ });
1014
+
1015
+ // Get the first element's response for the image (shows all highlights)
1016
+ const rawResponse = elements[0]?._response || {};
1017
+ const duration = Date.now() - startTime;
1018
+
1019
+ // Store cropped image for resource serving (instead of inline data URL)
1020
+ let croppedImageResourceUri: string | undefined;
1021
+ let screenshotResourceUri: string | undefined;
1022
+ const croppedImage = rawResponse.croppedImage;
1023
+ if (croppedImage) {
1024
+ const imageData = croppedImage.startsWith('data:')
1025
+ ? croppedImage.replace(/^data:image\/\w+;base64,/, '')
1026
+ : croppedImage;
1027
+ croppedImageResourceUri = storeImage(imageData, "cropped");
1028
+ // Remove croppedImage from response to avoid context bloat
1029
+ delete rawResponse.croppedImage;
1030
+ } else if (count === 0) {
1031
+ // No elements found and no cropped image - capture a fresh screenshot
1032
+ // so the user can see what's currently visible on screen
1033
+ try {
1034
+ const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
1035
+ if (screenshotBase64) {
1036
+ screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
1037
+ logger.debug("findall: Captured screenshot for not-found state");
1038
+ }
1039
+ } catch (e) {
1040
+ logger.warn("findall: Failed to capture screenshot for not-found state", { error: String(e) });
1041
+ }
1042
+ }
1043
+
1044
+ // Remove extractedText and pixelDiffImage from response to reduce context bloat
1045
+ delete rawResponse.extractedText;
1046
+ delete rawResponse.pixelDiffImage;
1047
+
1048
+ // Generate code for this findall action
1049
+ const generatedCode = count > 0 ? generateActionCode("findall", { description: params.description }) : undefined;
1050
+
1051
+ // Build refs list for text output
1052
+ const refsList = refs.map((ref, i) => ` [${i}] ${ref}`).join('\n');
1053
+
1054
+ return createToolResult(
1055
+ count > 0,
1056
+ count > 0
1057
+ ? `Found ${count} elements matching "${params.description}":\n${refsList}`
1058
+ : `No elements found matching: "${params.description}"`,
1059
+ {
1060
+ ...rawResponse,
1061
+ count,
1062
+ refs,
1063
+ elements: elementInfos,
1064
+ croppedImageResourceUri,
1065
+ screenshotResourceUri,
1066
+ duration,
1067
+ },
1068
+ generatedCode
1069
+ );
1070
+ } catch (error) {
1071
+ logger.error("findall: Failed", { error: String(error), description: params.description });
1072
+ captureException(error as Error, { tags: { tool: "findall" }, extra: { description: params.description } });
1073
+ throw error;
1074
+ }
1075
+ }
1076
+ );
1077
+
1078
+ // Click
1079
+ registerAppTool(
1080
+ server,
1081
+ "click",
1082
+ {
1083
+ title: "Click Element",
1084
+ description: "Click on a previously found element. Use 'find' first to locate the element.",
1085
+ inputSchema: z.object({
1086
+ elementRef: z.string().describe("Reference to previously found element (required). Get this from a 'find' call."),
1087
+ action: z.enum(["click", "double-click", "right-click"]).default("click"),
1088
+ }) as any,
1089
+ _meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
1090
+ },
1091
+ async (params: { elementRef: string; action: "click" | "double-click" | "right-click" }): Promise<CallToolResult> => {
1092
+ const startTime = Date.now();
1093
+ logger.info("click: Starting", { elementRef: params.elementRef, action: params.action });
1094
+
1095
+ const sessionCheck = requireActiveSession();
1096
+ if (!sessionCheck.valid) {
1097
+ logger.warn("click: No active session");
1098
+ return sessionCheck.error;
1099
+ }
1100
+
1101
+ // Look up the element reference
1102
+ const ref = elementRefs.get(params.elementRef);
1103
+ if (!ref) {
1104
+ logger.warn("click: Element reference not found", { elementRef: params.elementRef });
1105
+ return createToolResult(false, `Element reference "${params.elementRef}" not found. Use 'find' first to locate the element.`, { error: "Element reference not found" });
1106
+ }
1107
+
1108
+ const { element, description, coords } = ref;
1109
+
1110
+ try {
1111
+ logger.debug("click: Executing click on element", { description, action: params.action });
1112
+
1113
+ // Use the Element's click method instead of raw coordinates
1114
+ if (params.action === "click") {
1115
+ await element.click();
1116
+ } else if (params.action === "double-click") {
1117
+ await element.doubleClick();
1118
+ } else if (params.action === "right-click") {
1119
+ await element.rightClick();
1120
+ }
1121
+
1122
+ // Capture screenshot after click to show result
1123
+ logger.debug("click: Capturing screenshot after click");
1124
+ const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
1125
+
1126
+ let screenshotResourceUri: string | undefined;
1127
+ if (screenshotBase64) {
1128
+ screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
1129
+ lastScreenshotBase64 = screenshotBase64;
1130
+ }
1131
+
1132
+ const rawResponse = element._response || {};
1133
+ // Remove large data from response to reduce context bloat
1134
+ delete rawResponse.croppedImage;
1135
+ delete rawResponse.extractedText;
1136
+ delete rawResponse.pixelDiffImage;
1137
+
1138
+ const duration = Date.now() - startTime;
1139
+ logger.info("click: Completed", { description, duration });
1140
+
1141
+ // Generate code for this click action
1142
+ const generatedCode = generateActionCode("click", { action: params.action });
1143
+
1144
+ return createToolResult(
1145
+ true,
1146
+ `Clicked on "${description}"`,
1147
+ {
1148
+ ...rawResponse,
1149
+ action: "click",
1150
+ clickAction: params.action,
1151
+ clickPosition: coords,
1152
+ screenshotResourceUri,
1153
+ duration
1154
+ },
1155
+ generatedCode
1156
+ );
1157
+ } catch (error) {
1158
+ logger.error("click: Failed", { error: String(error), description });
1159
+ captureException(error as Error, { tags: { tool: "click" }, extra: { elementRef: params.elementRef, action: params.action } });
1160
+ throw error;
1161
+ }
1162
+ }
1163
+ );
1164
+
1165
+ // Hover
1166
+ registerAppTool(
1167
+ server,
1168
+ "hover",
1169
+ {
1170
+ title: "Hover Element",
1171
+ description: "Hover over a previously found element. Use 'find' first to locate the element.",
1172
+ inputSchema: z.object({
1173
+ elementRef: z.string().describe("Reference to previously found element (required). Get this from a 'find' call."),
1174
+ }) as any,
1175
+ _meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
1176
+ },
1177
+ async (params: { elementRef: string }): Promise<CallToolResult> => {
1178
+ const startTime = Date.now();
1179
+ logger.info("hover: Starting", { elementRef: params.elementRef });
1180
+
1181
+ const sessionCheck = requireActiveSession();
1182
+ if (!sessionCheck.valid) {
1183
+ logger.warn("hover: No active session");
1184
+ return sessionCheck.error;
1185
+ }
1186
+
1187
+ // Look up the element reference
1188
+ const ref = elementRefs.get(params.elementRef);
1189
+ if (!ref) {
1190
+ logger.warn("hover: Element reference not found", { elementRef: params.elementRef });
1191
+ return createToolResult(false, `Element reference "${params.elementRef}" not found. Use 'find' first to locate the element.`, { error: "Element reference not found" });
1192
+ }
1193
+
1194
+ const { element, description, coords } = ref;
1195
+
1196
+ try {
1197
+ logger.debug("hover: Executing hover on element", { description });
1198
+ await element.hover();
1199
+
1200
+ // Capture screenshot after hover to show result
1201
+ logger.debug("hover: Capturing screenshot after hover");
1202
+ const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
1203
+
1204
+ let screenshotResourceUri: string | undefined;
1205
+ if (screenshotBase64) {
1206
+ screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
1207
+ lastScreenshotBase64 = screenshotBase64;
1208
+ }
1209
+
1210
+ const rawResponse = element._response || {};
1211
+ // Remove large data from response to reduce context bloat
1212
+ delete rawResponse.croppedImage;
1213
+ delete rawResponse.extractedText;
1214
+ delete rawResponse.pixelDiffImage;
1215
+
1216
+ const duration = Date.now() - startTime;
1217
+ logger.info("hover: Completed", { description, duration });
1218
+
1219
+ // Generate code for this hover action
1220
+ const generatedCode = generateActionCode("hover", {});
1221
+
1222
+ return createToolResult(
1223
+ true,
1224
+ `Hovered over "${description}"`,
1225
+ {
1226
+ ...rawResponse,
1227
+ action: "hover",
1228
+ screenshotResourceUri,
1229
+ duration
1230
+ },
1231
+ generatedCode
1232
+ );
1233
+ } catch (error) {
1234
+ logger.error("hover: Failed", { error: String(error), description });
1235
+ captureException(error as Error, { tags: { tool: "hover" }, extra: { elementRef: params.elementRef } });
1236
+ throw error;
1237
+ }
1238
+ }
1239
+ );
1240
+
1241
+ // Wait
1242
+ server.registerTool(
1243
+ "wait",
1244
+ {
1245
+ description: "Wait for a specified amount of time",
1246
+ inputSchema: z.object({
1247
+ timeout: z.number().default(3000).describe("Time to wait in milliseconds (default: 3000)"),
1248
+ }),
1249
+ },
1250
+ async (params): Promise<CallToolResult> => {
1251
+ const startTime = Date.now();
1252
+ logger.info("wait: Starting", { timeout: params.timeout });
1253
+
1254
+ const sessionCheck = requireActiveSession();
1255
+ if (!sessionCheck.valid) {
1256
+ logger.warn("wait: No active session");
1257
+ return sessionCheck.error;
1258
+ }
1259
+
1260
+ try {
1261
+ logger.debug("wait: Waiting", { timeout: params.timeout });
1262
+ await sdk.wait(params.timeout);
1263
+
1264
+ const duration = Date.now() - startTime;
1265
+ logger.info("wait: Completed", { timeout: params.timeout, duration });
1266
+
1267
+ // Generate code for this wait action
1268
+ const generatedCode = generateActionCode("wait", { timeout: params.timeout });
1269
+
1270
+ return createToolResult(
1271
+ true,
1272
+ `Waited for ${params.timeout}ms`,
1273
+ { action: "wait", timeout: params.timeout, duration },
1274
+ generatedCode
1275
+ );
1276
+ } catch (error) {
1277
+ logger.error("wait: Failed", { error: String(error) });
1278
+ captureException(error as Error, { tags: { tool: "wait" }, extra: { timeout: params.timeout } });
1279
+ throw error;
1280
+ }
1281
+ }
1282
+ );
1283
+
1284
+ // Focus Application
1285
+ server.registerTool(
1286
+ "focus_application",
1287
+ {
1288
+ description: "Bring an application window to the foreground",
1289
+ inputSchema: z.object({
1290
+ name: z.string().describe("Name of the application to focus (e.g., 'Google Chrome', 'Visual Studio Code')"),
1291
+ }),
1292
+ },
1293
+ async (params): Promise<CallToolResult> => {
1294
+ const startTime = Date.now();
1295
+ logger.info("focus_application: Starting", { name: params.name });
1296
+
1297
+ const sessionCheck = requireActiveSession();
1298
+ if (!sessionCheck.valid) {
1299
+ logger.warn("focus_application: No active session");
1300
+ return sessionCheck.error;
1301
+ }
1302
+
1303
+ try {
1304
+ logger.debug("focus_application: Focusing", { name: params.name });
1305
+ await sdk.focusApplication(params.name);
1306
+
1307
+ const duration = Date.now() - startTime;
1308
+ logger.info("focus_application: Completed", { name: params.name, duration });
1309
+
1310
+ // Generate code for this focus action
1311
+ const generatedCode = generateActionCode("focus_application", { name: params.name });
1312
+
1313
+ return createToolResult(
1314
+ true,
1315
+ `Focused application: "${params.name}"`,
1316
+ { action: "focus", name: params.name, duration },
1317
+ generatedCode
1318
+ );
1319
+ } catch (error) {
1320
+ logger.error("focus_application: Failed", { error: String(error), name: params.name });
1321
+ captureException(error as Error, { tags: { tool: "focus_application" }, extra: { name: params.name } });
1322
+ throw error;
1323
+ }
1324
+ }
1325
+ );
1326
+
1327
+ // Find and Click
1328
+ registerAppTool(
1329
+ server,
1330
+ "find_and_click",
1331
+ {
1332
+ title: "Find and Click",
1333
+ description: "Find an element and click it in one action",
1334
+ inputSchema: z.object({
1335
+ description: z.string().describe("Natural language description of element"),
1336
+ action: z.enum(["click", "double-click", "right-click"]).default("click"),
1337
+ }) as any,
1338
+ _meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
1339
+ },
1340
+ async (params: { description: string; action: "click" | "double-click" | "right-click" }): Promise<CallToolResult> => {
1341
+ const startTime = Date.now();
1342
+ logger.info("find_and_click: Starting", { description: params.description, action: params.action });
1343
+
1344
+ const sessionCheck = requireActiveSession();
1345
+ if (!sessionCheck.valid) {
1346
+ logger.warn("find_and_click: No active session");
1347
+ return sessionCheck.error;
1348
+ }
1349
+
1350
+ try {
1351
+ logger.debug("find_and_click: Finding element");
1352
+ const element = await sdk.find(params.description);
1353
+ const found = element.found();
1354
+
1355
+ if (!found) {
1356
+ logger.warn("find_and_click: Element not found", { description: params.description });
1357
+
1358
+ // Capture screenshot to show current state even when element not found
1359
+ const rawResponse = element._response || {};
1360
+ const duration = Date.now() - startTime;
1361
+
1362
+ // Store cropped image (screenshot) for resource serving
1363
+ let croppedImageResourceUri: string | undefined;
1364
+ let screenshotResourceUri: string | undefined;
1365
+ const croppedImage = rawResponse.croppedImage;
1366
+ if (croppedImage) {
1367
+ const imageData = croppedImage.startsWith('data:')
1368
+ ? croppedImage.replace(/^data:image\/\w+;base64,/, '')
1369
+ : croppedImage;
1370
+ croppedImageResourceUri = storeImage(imageData, "screenshot");
1371
+ delete rawResponse.croppedImage;
1372
+ } else {
1373
+ // No cropped image - capture a fresh screenshot so the user can see
1374
+ // what's currently visible on screen when element was not found
1375
+ try {
1376
+ const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
1377
+ if (screenshotBase64) {
1378
+ screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
1379
+ logger.debug("find_and_click: Captured screenshot for not-found state");
1380
+ }
1381
+ } catch (e) {
1382
+ logger.warn("find_and_click: Failed to capture screenshot for not-found state", { error: String(e) });
1383
+ }
1384
+ }
1385
+
1386
+ // Remove extractedText and pixelDiffImage from response to reduce context bloat
1387
+ delete rawResponse.extractedText;
1388
+ delete rawResponse.pixelDiffImage;
1389
+
1390
+ return createToolResult(
1391
+ false,
1392
+ `Element not found: "${params.description}"`,
1393
+ {
1394
+ ...rawResponse,
1395
+ action: "find_and_click",
1396
+ error: "Element not found",
1397
+ croppedImageResourceUri,
1398
+ screenshotResourceUri,
1399
+ duration
1400
+ }
1401
+ );
1402
+ }
1403
+
1404
+ const coords = element.getCoordinates();
1405
+
1406
+ // Store element ref for later use (in case user wants to interact again)
1407
+ const elementRef = `el-${Date.now()}`;
1408
+ if (coords) {
1409
+ elementRefs.set(elementRef, {
1410
+ element: element,
1411
+ description: params.description,
1412
+ coords: {
1413
+ x: coords.x,
1414
+ y: coords.y,
1415
+ centerX: coords.centerX,
1416
+ centerY: coords.centerY,
1417
+ },
1418
+ });
1419
+ }
1420
+
1421
+ logger.debug("find_and_click: Element found, clicking", { action: params.action, elementRef });
1422
+ if (params.action === "click") {
1423
+ await element.click();
1424
+ } else if (params.action === "double-click") {
1425
+ await element.doubleClick();
1426
+ } else if (params.action === "right-click") {
1427
+ await element.rightClick();
1428
+ }
1429
+
1430
+ // Return raw SDK response directly
1431
+ const rawResponse = element._response || {};
1432
+ const duration = Date.now() - startTime;
1433
+
1434
+ // Store cropped image for resource serving (instead of inline data URL)
1435
+ let croppedImageResourceUri: string | undefined;
1436
+ const croppedImage = rawResponse.croppedImage;
1437
+ if (croppedImage) {
1438
+ const imageData = croppedImage.startsWith('data:')
1439
+ ? croppedImage.replace(/^data:image\/\w+;base64,/, '')
1440
+ : croppedImage;
1441
+ croppedImageResourceUri = storeImage(imageData, "cropped");
1442
+ // Remove croppedImage from response to avoid context bloat
1443
+ delete rawResponse.croppedImage;
1444
+ }
1445
+
1446
+ // Remove extractedText and pixelDiffImage from response to reduce context bloat
1447
+ delete rawResponse.extractedText;
1448
+ delete rawResponse.pixelDiffImage;
1449
+
1450
+ // Generate code for this find_and_click action
1451
+ const generatedCode = generateActionCode("find_and_click", { description: params.description, action: params.action });
1452
+
1453
+ // Build element info for display (match find action format)
1454
+ const elementInfo = coords ? {
1455
+ description: params.description,
1456
+ centerX: coords.centerX,
1457
+ centerY: coords.centerY,
1458
+ confidence: element.confidence,
1459
+ ref: elementRef,
1460
+ } : undefined;
1461
+
1462
+ return createToolResult(
1463
+ true,
1464
+ `Found and clicked: "${params.description}" at (${rawResponse.coordinates?.x}, ${rawResponse.coordinates?.y})\nRef: ${elementRef}`,
1465
+ {
1466
+ ...rawResponse,
1467
+ action: "find_and_click",
1468
+ element: elementInfo,
1469
+ ref: elementRef,
1470
+ clickAction: params.action,
1471
+ clickPosition: coords ? { x: coords.centerX, y: coords.centerY } : undefined,
1472
+ croppedImageResourceUri,
1473
+ duration,
1474
+ },
1475
+ generatedCode
1476
+ );
1477
+ } catch (error) {
1478
+ logger.error("find_and_click: Failed", { error: String(error), description: params.description });
1479
+ captureException(error as Error, { tags: { tool: "find_and_click" }, extra: { description: params.description, action: params.action } });
1480
+ throw error;
1481
+ }
1482
+ }
1483
+ );
1484
+
1485
+ // Type
1486
+ server.registerTool(
1487
+ "type",
1488
+ {
1489
+ description: "Type text into the currently focused field",
1490
+ inputSchema: z.object({
1491
+ text: z.string().describe("Text to type"),
1492
+ secret: z.boolean().default(false).describe("Whether this is sensitive data"),
1493
+ delay: z.number().optional().describe("Delay between keystrokes in ms"),
1494
+ }),
1495
+ },
1496
+ async (params): Promise<CallToolResult> => {
1497
+ const startTime = Date.now();
1498
+ logger.info("type: Starting", { textLength: params.text.length, secret: params.secret });
1499
+
1500
+ const sessionCheck = requireActiveSession();
1501
+ if (!sessionCheck.valid) {
1502
+ logger.warn("type: No active session");
1503
+ return sessionCheck.error;
1504
+ }
1505
+
1506
+ try {
1507
+ logger.debug("type: Typing text");
1508
+ await sdk.type(params.text, { secret: params.secret, delay: params.delay });
1509
+
1510
+ const duration = Date.now() - startTime;
1511
+ logger.info("type: Completed", { duration });
1512
+
1513
+ // Generate code for this type action
1514
+ const generatedCode = generateActionCode("type", { text: params.text, secret: params.secret });
1515
+
1516
+ return createToolResult(
1517
+ true,
1518
+ `Typed: ${params.secret ? "[secret text]" : `"${params.text}"`}`,
1519
+ { action: "type", text: params.secret ? "[SECRET]" : params.text, duration },
1520
+ generatedCode
1521
+ );
1522
+ } catch (error) {
1523
+ logger.error("type: Failed", { error: String(error) });
1524
+ captureException(error as Error, { tags: { tool: "type" }, extra: { textLength: params.text.length, secret: params.secret } });
1525
+ throw error;
1526
+ }
1527
+ }
1528
+ );
1529
+
1530
+ // Press Keys
1531
+ server.registerTool(
1532
+ "press_keys",
1533
+ {
1534
+ description: "Press keyboard keys or shortcuts",
1535
+ inputSchema: z.object({
1536
+ keys: z.array(z.string()).describe("Array of keys to press (e.g., ['ctrl', 'a'])"),
1537
+ }),
1538
+ },
1539
+ async (params): Promise<CallToolResult> => {
1540
+ const startTime = Date.now();
1541
+ logger.info("press_keys: Starting", { keys: params.keys });
1542
+
1543
+ const sessionCheck = requireActiveSession();
1544
+ if (!sessionCheck.valid) {
1545
+ logger.warn("press_keys: No active session");
1546
+ return sessionCheck.error;
1547
+ }
1548
+
1549
+ try {
1550
+ logger.debug("press_keys: Pressing keys");
1551
+ await sdk.pressKeys(params.keys);
1552
+
1553
+ const duration = Date.now() - startTime;
1554
+ logger.info("press_keys: Completed", { keys: params.keys, duration });
1555
+
1556
+ // Generate code for this press_keys action
1557
+ const generatedCode = generateActionCode("press_keys", { keys: params.keys });
1558
+
1559
+ return createToolResult(
1560
+ true,
1561
+ `Pressed keys: ${params.keys.join(" + ")}`,
1562
+ { action: "press_keys", keys: params.keys, duration },
1563
+ generatedCode
1564
+ );
1565
+ } catch (error) {
1566
+ logger.error("press_keys: Failed", { error: String(error), keys: params.keys });
1567
+ captureException(error as Error, { tags: { tool: "press_keys" }, extra: { keys: params.keys } });
1568
+ throw error;
1569
+ }
1570
+ }
1571
+ );
1572
+
1573
+ // Scroll
1574
+ server.registerTool(
1575
+ "scroll",
1576
+ {
1577
+ description: "Scroll the page or element",
1578
+ inputSchema: z.object({
1579
+ direction: z.enum(["up", "down", "left", "right"]).default("down"),
1580
+ amount: z.number().optional().describe("Amount to scroll in pixels"),
1581
+ }),
1582
+ },
1583
+ async (params): Promise<CallToolResult> => {
1584
+ const startTime = Date.now();
1585
+ logger.info("scroll: Starting", { direction: params.direction, amount: params.amount });
1586
+
1587
+ const sessionCheck = requireActiveSession();
1588
+ if (!sessionCheck.valid) {
1589
+ logger.warn("scroll: No active session");
1590
+ return sessionCheck.error;
1591
+ }
1592
+
1593
+ try {
1594
+ logger.debug("scroll: Scrolling");
1595
+ await sdk.scroll(params.direction, params.amount ? { amount: params.amount } : undefined);
1596
+
1597
+ const duration = Date.now() - startTime;
1598
+ logger.info("scroll: Completed", { direction: params.direction, duration });
1599
+
1600
+ // Generate code for this scroll action
1601
+ const generatedCode = generateActionCode("scroll", { direction: params.direction, amount: params.amount });
1602
+
1603
+ return createToolResult(
1604
+ true,
1605
+ `Scrolled ${params.direction}${params.amount ? ` by ${params.amount}px` : ""}`,
1606
+ { action: "scroll", scrollDirection: params.direction, direction: params.direction, amount: params.amount, duration },
1607
+ generatedCode
1608
+ );
1609
+ } catch (error) {
1610
+ logger.error("scroll: Failed", { error: String(error), direction: params.direction });
1611
+ captureException(error as Error, { tags: { tool: "scroll" }, extra: { direction: params.direction, amount: params.amount } });
1612
+ throw error;
1613
+ }
1614
+ }
1615
+ );
1616
+
1617
+ // Assert - generates code for test files
1618
+ server.registerTool(
1619
+ "assert",
1620
+ {
1621
+ description: `Make an AI-powered assertion about the current screen state. GENERATES CODE for the test file.
1622
+
1623
+ Use this when you want a verification step recorded in the generated test. This will add code like:
1624
+ const assertResult = await testdriver.assert("your assertion");
1625
+ expect(assertResult).toBeTruthy();
1626
+
1627
+ Unlike 'check' which is for your understanding during development, 'assert' creates verification code that runs in CI/CD.`,
1628
+ inputSchema: z.object({
1629
+ assertion: z.string().describe("Natural language assertion to verify"),
1630
+ }),
1631
+ },
1632
+ async (params): Promise<CallToolResult> => {
1633
+ const startTime = Date.now();
1634
+ logger.info("assert: Starting", { assertion: params.assertion });
1635
+
1636
+ const sessionCheck = requireActiveSession();
1637
+ if (!sessionCheck.valid) {
1638
+ logger.warn("assert: No active session");
1639
+ return sessionCheck.error;
1640
+ }
1641
+
1642
+ try {
1643
+ logger.debug("assert: Running assertion");
1644
+ const result = await sdk.assert(params.assertion);
1645
+
1646
+ const duration = Date.now() - startTime;
1647
+ logger.info("assert: Completed", { assertion: params.assertion, passed: result, duration });
1648
+
1649
+ // Generate code for this assert action
1650
+ const generatedCode = generateActionCode("assert", { assertion: params.assertion });
1651
+
1652
+ return createToolResult(
1653
+ result,
1654
+ result ? `✓ Assertion passed: "${params.assertion}"` : `✗ Assertion failed: "${params.assertion}"`,
1655
+ { action: "assert", assertion: params.assertion, passed: result, success: result, duration },
1656
+ generatedCode
1657
+ );
1658
+ } catch (error) {
1659
+ logger.error("assert: Failed", { error: String(error), assertion: params.assertion });
1660
+ captureException(error as Error, { tags: { tool: "assert" }, extra: { assertion: params.assertion } });
1661
+ throw error;
1662
+ }
1663
+ }
1664
+ );
1665
+
1666
+ // Check - AI uses this to understand the screen state (DOES NOT generate code)
1667
+ registerAppTool(
1668
+ server,
1669
+ "check",
1670
+ {
1671
+ title: "Check Screen State",
1672
+ description: `👁️ THIS IS HOW YOU SEE THE SCREEN. Use this tool whenever you need to understand what's currently displayed.
1673
+
1674
+ This tool captures a screenshot and returns AI analysis to YOU. Use it to:
1675
+ - See what's on the screen right now
1676
+ - Verify if your last action worked
1677
+ - Understand the current application state
1678
+ - Check if elements are visible or if navigation completed
1679
+
1680
+ Examples:
1681
+ - "What is currently on the screen?"
1682
+ - "Did the button click work?"
1683
+ - "Is the login form visible?"
1684
+ - "Did the page navigate to the dashboard?"
1685
+
1686
+ ⚠️ Do NOT use 'screenshot' to see the screen - that only shows the user, not you.
1687
+
1688
+ Note: This tool does NOT generate test code. Use 'assert' when you want to add a verification step to the test file.
1689
+
1690
+ You can optionally provide a reference image URI to compare against a previous state.`,
1691
+ inputSchema: z.object({
1692
+ task: z.string().describe("The task or condition to verify (e.g., 'Did the login succeed?', 'Is the modal visible?')"),
1693
+ referenceImageUri: z.string().optional().describe("Optional screenshot resource URI (e.g., 'screenshot://testdriver/screenshot/screenshot-1') to compare against instead of the automatically captured 'before' screenshot. Use a screenshotResourceUri from a previous action."),
1694
+ }) as any,
1695
+ _meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
1696
+ },
1697
+ async (params: { task: string; referenceImageUri?: string }): Promise<CallToolResult> => {
1698
+ const startTime = Date.now();
1699
+ logger.info("check: Starting", { task: params.task, hasReferenceImageUri: !!params.referenceImageUri });
1700
+
1701
+ const sessionCheck = requireActiveSession();
1702
+ if (!sessionCheck.valid) {
1703
+ logger.warn("check: No active session");
1704
+ return sessionCheck.error;
1705
+ }
1706
+
1707
+ try {
1708
+ // Capture current screenshot
1709
+ logger.debug("check: Capturing current screenshot");
1710
+ const currentScreenshot = await sdk.agent.system.captureScreenBase64(1, false, true);
1711
+
1712
+ // Use provided reference image URI, last screenshot as "before" state, or current if no previous screenshot
1713
+ let beforeScreenshot: string;
1714
+ if (params.referenceImageUri) {
1715
+ // Extract image ID from URI (e.g., "screenshot://testdriver/screenshot/screenshot-1" -> "screenshot-1")
1716
+ const uriParts = params.referenceImageUri.split('/');
1717
+ const imageId = uriParts[uriParts.length - 1];
1718
+
1719
+ logger.info("check: Looking up reference image", {
1720
+ referenceImageUri: params.referenceImageUri,
1721
+ extractedImageId: imageId,
1722
+ imageStoreSize: imageStore.size,
1723
+ availableKeys: Array.from(imageStore.keys())
1724
+ });
1725
+
1726
+ const storedImage = getStoredImage(imageId);
1727
+
1728
+ if (storedImage) {
1729
+ logger.info("check: Found reference image", {
1730
+ imageId,
1731
+ dataLength: storedImage.data?.length,
1732
+ type: storedImage.type,
1733
+ hasData: !!storedImage.data
1734
+ });
1735
+ beforeScreenshot = storedImage.data;
1736
+ } else {
1737
+ logger.warn("check: Reference image NOT found in store, falling back to last screenshot", {
1738
+ referenceImageUri: params.referenceImageUri,
1739
+ imageId,
1740
+ imageStoreSize: imageStore.size,
1741
+ availableKeys: Array.from(imageStore.keys())
1742
+ });
1743
+ beforeScreenshot = lastScreenshotBase64 || currentScreenshot;
1744
+ }
1745
+ } else {
1746
+ beforeScreenshot = lastScreenshotBase64 || currentScreenshot;
1747
+ }
1748
+
1749
+ // Update last screenshot for next check
1750
+ lastScreenshotBase64 = currentScreenshot;
1751
+
1752
+ // Get system state
1753
+ const mousePosition = await sdk.agent.system.getMousePosition();
1754
+ const activeWindow = await sdk.agent.system.activeWin();
1755
+
1756
+ // Call the check endpoint
1757
+ logger.info("check: Calling check API endpoint", {
1758
+ hasLastScreenshot: beforeScreenshot !== currentScreenshot,
1759
+ usingReferenceImageUri: !!params.referenceImageUri,
1760
+ beforeScreenshotLength: beforeScreenshot?.length || 0,
1761
+ currentScreenshotLength: currentScreenshot?.length || 0,
1762
+ beforeScreenshotPreview: beforeScreenshot?.substring(0, 50),
1763
+ currentScreenshotPreview: currentScreenshot?.substring(0, 50)
1764
+ });
1765
+ const response = await sdk.agent.sdk.req("check", {
1766
+ tasks: [params.task],
1767
+ images: [beforeScreenshot, currentScreenshot],
1768
+ mousePosition,
1769
+ activeWindow,
1770
+ });
1771
+
1772
+ const aiResponse = response.data;
1773
+
1774
+ // Store screenshot for resource serving
1775
+ let screenshotResourceUri: string | undefined;
1776
+ if (currentScreenshot) {
1777
+ screenshotResourceUri = storeImage(currentScreenshot, "screenshot");
1778
+ }
1779
+
1780
+ // Determine if the check passed based on the AI response
1781
+ // The AI typically returns markdown with its analysis
1782
+ // We consider it "complete" if the response doesn't contain code blocks (indicating more work needed)
1783
+ const hasCodeBlocks = aiResponse && (
1784
+ aiResponse.includes("```yml") ||
1785
+ aiResponse.includes("```yaml") ||
1786
+ aiResponse.includes("- command:")
1787
+ );
1788
+ const isComplete = !hasCodeBlocks;
1789
+
1790
+ const duration = Date.now() - startTime;
1791
+ logger.info("check: Completed", { task: params.task, complete: isComplete, duration });
1792
+
1793
+ // Note: check doesn't generate code - it's for AI understanding, not test recording
1794
+ return createToolResult(
1795
+ isComplete,
1796
+ isComplete
1797
+ ? `✓ Task appears complete: "${params.task}"\n\nAI Analysis:\n${aiResponse}`
1798
+ : `⚠ Task may not be complete: "${params.task}"\n\nAI Analysis:\n${aiResponse}`,
1799
+ {
1800
+ action: "check",
1801
+ task: params.task,
1802
+ complete: isComplete,
1803
+ success: isComplete,
1804
+ aiResponse,
1805
+ screenshotResourceUri,
1806
+ duration
1807
+ }
1808
+ );
1809
+ } catch (error) {
1810
+ logger.error("check: Failed", { error: String(error), task: params.task });
1811
+ captureException(error as Error, { tags: { tool: "check" }, extra: { task: params.task } });
1812
+ throw error;
1813
+ }
1814
+ }
1815
+ );
1816
+
1817
+ // Exec
1818
+ server.registerTool(
1819
+ "exec",
1820
+ {
1821
+ description: "Execute shell or PowerShell commands in the sandbox",
1822
+ inputSchema: z.object({
1823
+ language: z.enum(["sh", "pwsh"]).default("sh"),
1824
+ code: z.string().describe("Code to execute"),
1825
+ timeout: z.number().default(30000).describe("Timeout in ms"),
1826
+ }),
1827
+ },
1828
+ async (params): Promise<CallToolResult> => {
1829
+ const startTime = Date.now();
1830
+ logger.info("exec: Starting", { language: params.language, codeLength: params.code.length, timeout: params.timeout });
1831
+
1832
+ const sessionCheck = requireActiveSession();
1833
+ if (!sessionCheck.valid) {
1834
+ logger.warn("exec: No active session");
1835
+ return sessionCheck.error;
1836
+ }
1837
+
1838
+ try {
1839
+ logger.debug("exec: Executing code", { language: params.language });
1840
+ const output = await sdk.exec(params.language, params.code, params.timeout);
1841
+
1842
+ const duration = Date.now() - startTime;
1843
+ logger.info("exec: Completed", { language: params.language, outputLength: output?.length || 0, duration });
1844
+
1845
+ // Generate code for this exec action
1846
+ const generatedCode = generateActionCode("exec", { language: params.language, code: params.code, timeout: params.timeout });
1847
+
1848
+ return createToolResult(
1849
+ true,
1850
+ `Executed ${params.language} code:\n${output || "(no output)"}`,
1851
+ { action: "exec", language: params.language, output, duration },
1852
+ generatedCode
1853
+ );
1854
+ } catch (error) {
1855
+ logger.error("exec: Failed", { error: String(error), language: params.language });
1856
+ captureException(error as Error, { tags: { tool: "exec" }, extra: { language: params.language, codeLength: params.code.length } });
1857
+ throw error;
1858
+ }
1859
+ }
1860
+ );
1861
+
1862
+ // Parse auto-screenshot filename format: <seq>-<action>-<phase>-L<line>-<description>.png
1863
+ // Example: 001-click-before-L42-submit-button.png
1864
+ // Example: 003-click-error-L42-submit-button.png (error phase when action fails)
1865
+ interface ParsedScreenshotInfo {
1866
+ sequence?: number;
1867
+ action?: string;
1868
+ phase?: "before" | "after" | "error";
1869
+ lineNumber?: number;
1870
+ description?: string;
1871
+ }
1872
+
1873
+ function parseScreenshotFilename(filename: string): ParsedScreenshotInfo {
1874
+ // Match pattern: 001-click-before-L42-submit-button.png or 001-click-error-L42-submit-button.png
1875
+ const match = filename.match(/^(\d+)-([a-z]+)-(before|after|error)-L(\d+)-(.+)\.png$/i);
1876
+ if (match) {
1877
+ return {
1878
+ sequence: parseInt(match[1], 10),
1879
+ action: match[2].toLowerCase(),
1880
+ phase: match[3].toLowerCase() as "before" | "after" | "error",
1881
+ lineNumber: parseInt(match[4], 10),
1882
+ description: match[5],
1883
+ };
1884
+ }
1885
+ return {};
1886
+ }
1887
+
1888
+ // List Local Screenshots - lists screenshots saved to .testdriver directory
1889
+ server.registerTool(
1890
+ "list_local_screenshots",
1891
+ {
1892
+ description: `List and filter screenshots saved in the .testdriver directory.
1893
+
1894
+ Screenshots from auto-screenshot feature use the format: <seq>-<action>-<phase>-L<line>-<description>.png
1895
+ Example: 001-click-before-L42-submit-button.png
1896
+
1897
+ This tool supports powerful filtering to find specific screenshots:
1898
+ - By test file (directory)
1899
+ - By line number or range
1900
+ - By action type (click, find, type, assert, etc.)
1901
+ - By phase (before/after/error - error screenshots are captured when actions fail)
1902
+ - By regex pattern on filename
1903
+ - By sequence number range
1904
+
1905
+ Returns a list of screenshot paths that can be viewed with the 'view_local_screenshot' tool.`,
1906
+ inputSchema: z.object({
1907
+ directory: z.string().optional().describe("Test file or subdirectory to search (e.g., 'login.test', 'mcp-screenshots'). If not provided, searches all."),
1908
+ line: z.number().optional().describe("Filter by exact line number from test file (e.g., 42 matches L42)"),
1909
+ lineRange: z.object({
1910
+ start: z.number().describe("Start line number (inclusive)"),
1911
+ end: z.number().describe("End line number (inclusive)"),
1912
+ }).optional().describe("Filter by line number range (e.g., { start: 10, end: 20 })"),
1913
+ action: z.string().optional().describe("Filter by action type: click, find, type, assert, provision, scroll, hover, etc."),
1914
+ phase: z.enum(["before", "after", "error"]).optional().describe("Filter by phase: 'before' (pre-action), 'after' (post-action), or 'error' (when action fails)"),
1915
+ pattern: z.string().optional().describe("Regex pattern to match against filename (e.g., 'submit|login' or 'button.*click')"),
1916
+ sequence: z.number().optional().describe("Filter by exact sequence number"),
1917
+ sequenceRange: z.object({
1918
+ start: z.number().describe("Start sequence (inclusive)"),
1919
+ end: z.number().describe("End sequence (inclusive)"),
1920
+ }).optional().describe("Filter by sequence range (e.g., { start: 1, end: 10 })"),
1921
+ limit: z.number().optional().describe("Maximum number of results to return (default: 50)"),
1922
+ sortBy: z.enum(["modified", "sequence", "line"]).optional().describe("Sort by: 'modified' (newest first), 'sequence' (execution order), or 'line' (line number). Default: 'modified'"),
1923
+ }),
1924
+ },
1925
+ async (params): Promise<CallToolResult> => {
1926
+ const startTime = Date.now();
1927
+ logger.info("list_local_screenshots: Starting", { ...params });
1928
+
1929
+ try {
1930
+ // Find .testdriver directory - check current working directory and common locations
1931
+ const possiblePaths = [
1932
+ path.join(process.cwd(), ".testdriver"),
1933
+ path.join(os.homedir(), ".testdriver"),
1934
+ ];
1935
+
1936
+ let testdriverDir: string | null = null;
1937
+ for (const p of possiblePaths) {
1938
+ if (fs.existsSync(p)) {
1939
+ testdriverDir = p;
1940
+ break;
1941
+ }
1942
+ }
1943
+
1944
+ if (!testdriverDir) {
1945
+ logger.warn("list_local_screenshots: .testdriver directory not found");
1946
+ return createToolResult(false, "No .testdriver directory found. Screenshots are saved here during test runs.", { error: "Directory not found" });
1947
+ }
1948
+
1949
+ interface ScreenshotInfo {
1950
+ path: string;
1951
+ name: string;
1952
+ modified: Date;
1953
+ size: number;
1954
+ parsed: ParsedScreenshotInfo;
1955
+ }
1956
+
1957
+ const screenshots: ScreenshotInfo[] = [];
1958
+
1959
+ // Compile regex pattern if provided
1960
+ let regexPattern: RegExp | null = null;
1961
+ if (params.pattern) {
1962
+ try {
1963
+ regexPattern = new RegExp(params.pattern, "i");
1964
+ } catch {
1965
+ return createToolResult(false, `Invalid regex pattern: ${params.pattern}`, { error: "Invalid regex" });
1966
+ }
1967
+ }
1968
+
1969
+ // Function to recursively find PNG files
1970
+ const findPngFiles = (dir: string) => {
1971
+ if (!fs.existsSync(dir)) return;
1972
+
1973
+ const entries = fs.readdirSync(dir, { withFileTypes: true });
1974
+ for (const entry of entries) {
1975
+ const fullPath = path.join(dir, entry.name);
1976
+ if (entry.isDirectory()) {
1977
+ // If a specific directory was requested, only search that one
1978
+ if (!params.directory || entry.name === params.directory || dir !== testdriverDir) {
1979
+ findPngFiles(fullPath);
1980
+ }
1981
+ } else if (entry.isFile() && entry.name.toLowerCase().endsWith(".png")) {
1982
+ const parsed = parseScreenshotFilename(entry.name);
1983
+
1984
+ // Apply filters
1985
+ if (params.line !== undefined && parsed.lineNumber !== params.line) continue;
1986
+ if (params.lineRange && (
1987
+ parsed.lineNumber === undefined ||
1988
+ parsed.lineNumber < params.lineRange.start ||
1989
+ parsed.lineNumber > params.lineRange.end
1990
+ )) continue;
1991
+ if (params.action && parsed.action !== params.action.toLowerCase()) continue;
1992
+ if (params.phase && parsed.phase !== params.phase) continue;
1993
+ if (params.sequence !== undefined && parsed.sequence !== params.sequence) continue;
1994
+ if (params.sequenceRange && (
1995
+ parsed.sequence === undefined ||
1996
+ parsed.sequence < params.sequenceRange.start ||
1997
+ parsed.sequence > params.sequenceRange.end
1998
+ )) continue;
1999
+ if (regexPattern && !regexPattern.test(entry.name)) continue;
2000
+
2001
+ const stats = fs.statSync(fullPath);
2002
+ screenshots.push({
2003
+ path: fullPath,
2004
+ name: entry.name,
2005
+ modified: stats.mtime,
2006
+ size: stats.size,
2007
+ parsed,
2008
+ });
2009
+ }
2010
+ }
2011
+ };
2012
+
2013
+ findPngFiles(testdriverDir);
2014
+
2015
+ // Sort based on sortBy parameter
2016
+ const sortBy = params.sortBy || "modified";
2017
+ if (sortBy === "modified") {
2018
+ screenshots.sort((a, b) => b.modified.getTime() - a.modified.getTime());
2019
+ } else if (sortBy === "sequence") {
2020
+ screenshots.sort((a, b) => (a.parsed.sequence ?? Infinity) - (b.parsed.sequence ?? Infinity));
2021
+ } else if (sortBy === "line") {
2022
+ screenshots.sort((a, b) => (a.parsed.lineNumber ?? Infinity) - (b.parsed.lineNumber ?? Infinity));
2023
+ }
2024
+
2025
+ const duration = Date.now() - startTime;
2026
+ logger.info("list_local_screenshots: Completed", { count: screenshots.length, duration });
2027
+
2028
+ if (screenshots.length === 0) {
2029
+ const filters = [];
2030
+ if (params.directory) filters.push(`directory=${params.directory}`);
2031
+ if (params.line) filters.push(`line=${params.line}`);
2032
+ if (params.lineRange) filters.push(`lineRange=${params.lineRange.start}-${params.lineRange.end}`);
2033
+ if (params.action) filters.push(`action=${params.action}`);
2034
+ if (params.phase) filters.push(`phase=${params.phase}`);
2035
+ if (params.pattern) filters.push(`pattern=${params.pattern}`);
2036
+ if (params.sequence) filters.push(`sequence=${params.sequence}`);
2037
+ if (params.sequenceRange) filters.push(`sequenceRange=${params.sequenceRange.start}-${params.sequenceRange.end}`);
2038
+
2039
+ const filterMsg = filters.length > 0 ? ` with filters: ${filters.join(", ")}` : "";
2040
+ return createToolResult(true, `No screenshots found in .testdriver directory${filterMsg}.`, {
2041
+ action: "list_local_screenshots",
2042
+ count: 0,
2043
+ directory: testdriverDir,
2044
+ filters: params,
2045
+ duration
2046
+ });
2047
+ }
2048
+
2049
+ const limit = params.limit || 50;
2050
+ const limitedScreenshots = screenshots.slice(0, limit);
2051
+
2052
+ // Format the list for display with parsed info
2053
+ const screenshotList = limitedScreenshots.map((s, i) => {
2054
+ const relativePath = path.relative(testdriverDir!, s.path);
2055
+ const sizeKB = Math.round(s.size / 1024);
2056
+ const timeAgo = formatTimeAgo(s.modified);
2057
+
2058
+ // Add parsed info if available
2059
+ const parts = [`${i + 1}. ${relativePath}`];
2060
+ const meta = [];
2061
+ if (s.parsed.lineNumber) meta.push(`L${s.parsed.lineNumber}`);
2062
+ if (s.parsed.action) meta.push(s.parsed.action);
2063
+ if (s.parsed.phase) meta.push(s.parsed.phase);
2064
+ meta.push(`${sizeKB}KB`);
2065
+ meta.push(timeAgo);
2066
+ parts.push(`(${meta.join(", ")})`);
2067
+
2068
+ return parts.join(" ");
2069
+ }).join("\n");
2070
+
2071
+ const message = screenshots.length > limit
2072
+ ? `Found ${screenshots.length} screenshots (showing ${limit} results, sorted by ${sortBy}):\n\n${screenshotList}`
2073
+ : `Found ${screenshots.length} screenshot(s) (sorted by ${sortBy}):\n\n${screenshotList}`;
2074
+
2075
+ return createToolResult(true, message, {
2076
+ action: "list_local_screenshots",
2077
+ count: screenshots.length,
2078
+ returned: limitedScreenshots.length,
2079
+ directory: testdriverDir,
2080
+ filters: params,
2081
+ sortBy,
2082
+ screenshots: limitedScreenshots.map(s => ({
2083
+ path: s.path,
2084
+ relativePath: path.relative(testdriverDir!, s.path),
2085
+ name: s.name,
2086
+ modified: s.modified.toISOString(),
2087
+ sizeBytes: s.size,
2088
+ sequence: s.parsed.sequence,
2089
+ action: s.parsed.action,
2090
+ phase: s.parsed.phase,
2091
+ lineNumber: s.parsed.lineNumber,
2092
+ description: s.parsed.description,
2093
+ })),
2094
+ duration
2095
+ });
2096
+ } catch (error) {
2097
+ logger.error("list_local_screenshots: Failed", { error: String(error) });
2098
+ captureException(error as Error, { tags: { tool: "list_local_screenshots" } });
2099
+ throw error;
2100
+ }
2101
+ }
2102
+ );
2103
+
2104
+ // Helper to format time ago
2105
+ function formatTimeAgo(date: Date): string {
2106
+ const seconds = Math.floor((Date.now() - date.getTime()) / 1000);
2107
+ if (seconds < 60) return `${seconds}s ago`;
2108
+ const minutes = Math.floor(seconds / 60);
2109
+ if (minutes < 60) return `${minutes}m ago`;
2110
+ const hours = Math.floor(minutes / 60);
2111
+ if (hours < 24) return `${hours}h ago`;
2112
+ const days = Math.floor(hours / 24);
2113
+ return `${days}d ago`;
2114
+ }
2115
+
2116
+ // View Local Screenshot - view a screenshot from .testdriver directory
2117
+ // Returns the image so AI clients that support images can see it
2118
+ // Also displays to the user via MCP App
2119
+ registerAppTool(
2120
+ server,
2121
+ "view_local_screenshot",
2122
+ {
2123
+ title: "View Local Screenshot",
2124
+ description: `View a screenshot from the .testdriver directory.
2125
+
2126
+ Use 'list_local_screenshots' first to see available screenshots, then use this tool to view one.
2127
+
2128
+ This tool returns the image content so AI clients that support images can see it directly.
2129
+ The image is also displayed to the user via the MCP App UI.
2130
+
2131
+ Useful for:
2132
+ - Reviewing screenshots from previous test runs
2133
+ - Debugging test failures by examining saved screenshots
2134
+ - Comparing current screen state to saved screenshots`,
2135
+ inputSchema: z.object({
2136
+ path: z.string().describe("Full path to the screenshot file (from list_local_screenshots)"),
2137
+ }) as any,
2138
+ _meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
2139
+ },
2140
+ async (params: { path: string }): Promise<CallToolResult> => {
2141
+ const startTime = Date.now();
2142
+ logger.info("view_local_screenshot: Starting", { path: params.path });
2143
+
2144
+ try {
2145
+ // Validate the path exists and is a PNG
2146
+ if (!fs.existsSync(params.path)) {
2147
+ logger.warn("view_local_screenshot: File not found", { path: params.path });
2148
+ return createToolResult(false, `Screenshot not found: ${params.path}`, { error: "File not found" });
2149
+ }
2150
+
2151
+ if (!params.path.toLowerCase().endsWith(".png")) {
2152
+ logger.warn("view_local_screenshot: Not a PNG file", { path: params.path });
2153
+ return createToolResult(false, "Only PNG files are supported", { error: "Invalid file type" });
2154
+ }
2155
+
2156
+ // Security check - only allow files from .testdriver directory
2157
+ const normalizedPath = path.resolve(params.path);
2158
+ if (!normalizedPath.includes(".testdriver")) {
2159
+ logger.warn("view_local_screenshot: Path not in .testdriver", { path: normalizedPath });
2160
+ return createToolResult(false, "Can only view screenshots from .testdriver directory", { error: "Security: path not allowed" });
2161
+ }
2162
+
2163
+ // Read the file
2164
+ const imageBuffer = fs.readFileSync(params.path);
2165
+ const imageBase64 = imageBuffer.toString("base64");
2166
+
2167
+ // Store image for MCP App UI display
2168
+ const screenshotResourceUri = storeImage(imageBase64, "screenshot");
2169
+
2170
+ const stats = fs.statSync(params.path);
2171
+ const sizeKB = Math.round(stats.size / 1024);
2172
+ const fileName = path.basename(params.path);
2173
+
2174
+ const duration = Date.now() - startTime;
2175
+ logger.info("view_local_screenshot: Completed", { path: params.path, sizeKB, duration });
2176
+
2177
+ // Return the image content for AI clients that support images
2178
+ // The content array includes both text and image for maximum compatibility
2179
+ const content: CallToolResult["content"] = [
2180
+ { type: "text", text: `Screenshot: ${fileName} (${sizeKB}KB)` },
2181
+ {
2182
+ type: "image",
2183
+ data: imageBase64,
2184
+ mimeType: "image/png"
2185
+ },
2186
+ ];
2187
+
2188
+ return {
2189
+ content,
2190
+ structuredContent: {
2191
+ action: "view_local_screenshot",
2192
+ success: true,
2193
+ path: params.path,
2194
+ fileName,
2195
+ sizeBytes: stats.size,
2196
+ modified: stats.mtime.toISOString(),
2197
+ screenshotResourceUri,
2198
+ duration
2199
+ },
2200
+ };
2201
+ } catch (error) {
2202
+ logger.error("view_local_screenshot: Failed", { error: String(error), path: params.path });
2203
+ captureException(error as Error, { tags: { tool: "view_local_screenshot" }, extra: { path: params.path } });
2204
+ throw error;
2205
+ }
2206
+ }
2207
+ );
2208
+
2209
+ // Screenshot - captures full screen to show user the current state
2210
+ // NOTE: This is for SHOWING the user the screen, not for AI understanding.
2211
+ // Use 'check' tool for AI to understand screen state.
2212
+ registerAppTool(
2213
+ server,
2214
+ "screenshot",
2215
+ {
2216
+ title: "Screenshot",
2217
+ description: `Display a screenshot to the user. This tool does NOT return the image to you (the AI).
2218
+
2219
+ ⚠️ IMPORTANT: Do NOT use this tool to understand the screen state. The screenshot is ONLY displayed to the human user - you will NOT receive the image or any analysis.
2220
+
2221
+ If you need to:
2222
+ - See what's on screen → use 'check' instead
2223
+ - Verify an action worked → use 'check' instead
2224
+ - Understand the current state → use 'check' instead
2225
+
2226
+ Only use 'screenshot' when you explicitly want to show something to the human user without needing to see it yourself.`,
2227
+ inputSchema: z.object({}),
2228
+ _meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
2229
+ },
2230
+ async (): Promise<CallToolResult> => {
2231
+ const startTime = Date.now();
2232
+ logger.info("screenshot: Starting");
2233
+
2234
+ const sessionCheck = requireActiveSession();
2235
+ if (!sessionCheck.valid) {
2236
+ logger.warn("screenshot: No active session");
2237
+ return sessionCheck.error;
2238
+ }
2239
+
2240
+ try {
2241
+ // Capture full screen screenshot
2242
+ const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
2243
+
2244
+ let screenshotResourceUri: string | undefined;
2245
+ if (screenshotBase64) {
2246
+ // Store raw base64 for the resource blob with unique ID
2247
+ screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
2248
+ }
2249
+
2250
+ const duration = Date.now() - startTime;
2251
+ logger.info("screenshot: Completed", { duration, hasImage: !!screenshotBase64 });
2252
+
2253
+ // Only send the resource URI - the MCP app will fetch the image via resources/read
2254
+ // This keeps the base64 image data OUT of AI context
2255
+ return createToolResult(
2256
+ true,
2257
+ "Screenshot captured and displayed to user",
2258
+ {
2259
+ action: "screenshot",
2260
+ screenshotResourceUri,
2261
+ duration
2262
+ }
2263
+ );
2264
+ } catch (error) {
2265
+ logger.error("screenshot: Failed", { error: String(error) });
2266
+ return createToolResult(false, `Screenshot failed: ${error}`, { error: String(error) });
2267
+ }
2268
+ }
2269
+ );
2270
+
2271
+ // Init - Initialize a new TestDriver project
2272
+ server.registerTool(
2273
+ "init",
2274
+ {
2275
+ description: `Initialize a new TestDriver project with Vitest SDK examples.
2276
+
2277
+ This creates:
2278
+ - package.json with proper dependencies
2279
+ - Example test files (tests/example.test.js, tests/login.js)
2280
+ - vitest.config.js
2281
+ - .gitignore
2282
+ - GitHub Actions workflow (.github/workflows/testdriver.yml)
2283
+ - VSCode MCP config (.vscode/mcp.json)
2284
+ - VSCode extensions recommendations (.vscode/extensions.json)
2285
+ - TestDriver skills (.github/skills/)
2286
+ - TestDriver agents (.github/agents/)
2287
+ - .env file with API key (if provided)
2288
+
2289
+ API Key: The apiKey parameter is optional. If not provided, you'll need to manually add TD_API_KEY to the .env file after initialization. The project structure will still be created successfully.`,
2290
+ inputSchema: z.object({
2291
+ directory: z.string().optional().describe("Target directory (defaults to current working directory)"),
2292
+ apiKey: z.string().optional().describe("TestDriver API key (will be saved to .env)"),
2293
+ skipInstall: z.boolean().default(false).describe("Skip npm install step"),
2294
+ }),
2295
+ },
2296
+ async (params): Promise<CallToolResult> => {
2297
+ const startTime = Date.now();
2298
+ const targetDir = params.directory ? path.resolve(params.directory) : process.cwd();
2299
+
2300
+ logger.info("init: Starting", { targetDir, hasApiKey: !!params.apiKey, skipInstall: params.skipInstall });
2301
+
2302
+ try {
2303
+ // Import the shared init logic (dynamic import for ESM/CJS compatibility)
2304
+ const initProjectPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "lib", "init-project.js");
2305
+ const { initProject } = await import(pathToFileURL(initProjectPath).href);
2306
+
2307
+ // Run the shared init logic
2308
+ const result = await initProject({
2309
+ targetDir,
2310
+ apiKey: params.apiKey,
2311
+ skipInstall: params.skipInstall,
2312
+ });
2313
+
2314
+ const duration = Date.now() - startTime;
2315
+ logger.info("init: Completed", { targetDir, duration, success: result.success });
2316
+
2317
+ const nextSteps = `
2318
+
2319
+ 📚 Next steps:
2320
+
2321
+ 1. Run your tests:
2322
+ vitest run
2323
+
2324
+ 2. Use AI agents to write tests:
2325
+ Open VSCode/Cursor and use @testdriver agent
2326
+
2327
+ 3. MCP server configured:
2328
+ TestDriver tools available via MCP in .vscode/mcp.json
2329
+
2330
+ 4. For CI/CD, add TD_API_KEY to your GitHub repository secrets:
2331
+ Settings → Secrets → Actions → New repository secret
2332
+
2333
+ Learn more at https://docs.testdriver.ai/v7/getting-started/
2334
+ `;
2335
+
2336
+ const allMessages = [...result.results, ...result.errors.map((e: string) => `⚠️ ${e}`)];
2337
+
2338
+ return createToolResult(
2339
+ result.success,
2340
+ result.success
2341
+ ? `✅ TestDriver project initialized successfully!\n\n${allMessages.join("\n")}${nextSteps}`
2342
+ : `⚠️ TestDriver project initialization completed with errors:\n\n${allMessages.join("\n")}`,
2343
+ {
2344
+ action: "init",
2345
+ targetDir,
2346
+ filesCreated: result.results.length,
2347
+ hasApiKey: !!params.apiKey,
2348
+ errors: result.errors,
2349
+ duration
2350
+ }
2351
+ );
2352
+ } catch (error) {
2353
+ logger.error("init: Failed", { error: String(error), targetDir });
2354
+ captureException(error as Error, { tags: { tool: "init" }, extra: { targetDir } });
2355
+ throw error;
2356
+ }
2357
+ }
2358
+ );
2359
+
2360
+
2361
+ // Start the server
2362
+ async function main() {
2363
+ logger.info("Starting TestDriver MCP Server", {
2364
+ version,
2365
+ logLevel: process.env.TD_LOG_LEVEL || "INFO",
2366
+ distDir: DIST_DIR,
2367
+ sentryEnabled: isSentryEnabled(),
2368
+ });
2369
+
2370
+ const transport = new StdioServerTransport();
2371
+ await server.connect(transport);
2372
+
2373
+ logger.info("TestDriver MCP Server running on stdio");
2374
+
2375
+ // Handle graceful shutdown
2376
+ const shutdown = async () => {
2377
+ logger.info("Shutting down MCP Server");
2378
+ await flushSentry();
2379
+ process.exit(0);
2380
+ };
2381
+
2382
+ process.on("SIGINT", shutdown);
2383
+ process.on("SIGTERM", shutdown);
2384
+ }
2385
+
2386
+ main().catch(async (error) => {
2387
+ logger.error("Server failed to start", { error: String(error) });
2388
+ captureException(error as Error, { tags: { phase: "startup" } });
2389
+ await flushSentry();
2390
+ process.exit(1);
2391
+ });