@testdriverai/agent 7.8.0-canary.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +7 -0
- package/.env.example +4 -0
- package/.prettierignore +4 -0
- package/.prettierrc +1 -0
- package/CHANGELOG.md +953 -0
- package/README.md +81 -0
- package/agent/events.js +135 -0
- package/agent/index.js +2450 -0
- package/agent/interface.js +35 -0
- package/agent/lib/analytics.js +22 -0
- package/agent/lib/censorship.js +75 -0
- package/agent/lib/commander.js +246 -0
- package/agent/lib/commands.js +1684 -0
- package/agent/lib/config.js +60 -0
- package/agent/lib/generator.js +91 -0
- package/agent/lib/http.js +144 -0
- package/agent/lib/logger.js +56 -0
- package/agent/lib/outputs.js +29 -0
- package/agent/lib/parser.js +209 -0
- package/agent/lib/redraw.js +386 -0
- package/agent/lib/resources/cursor-2.png +0 -0
- package/agent/lib/sandbox.js +1104 -0
- package/agent/lib/sdk.js +633 -0
- package/agent/lib/session.js +25 -0
- package/agent/lib/source-mapper.js +342 -0
- package/agent/lib/subimage/index.js +77 -0
- package/agent/lib/subimage/opencv.js +69 -0
- package/agent/lib/system.js +204 -0
- package/agent/lib/theme.js +14 -0
- package/agent/lib/valid-version.js +21 -0
- package/agent/lib/validation.js +169 -0
- package/ai/.claude-plugin/plugin.json +9 -0
- package/ai/agents/testdriver.md +638 -0
- package/ai/skills/testdriver-ai/SKILL.md +204 -0
- package/ai/skills/testdriver-assert/SKILL.md +315 -0
- package/ai/skills/testdriver-aws-setup/SKILL.md +448 -0
- package/ai/skills/testdriver-cache/SKILL.md +221 -0
- package/ai/skills/testdriver-caching/SKILL.md +124 -0
- package/ai/skills/testdriver-captcha/SKILL.md +158 -0
- package/ai/skills/testdriver-ci-cd/SKILL.md +602 -0
- package/ai/skills/testdriver-click/SKILL.md +286 -0
- package/ai/skills/testdriver-client/SKILL.md +477 -0
- package/ai/skills/testdriver-cloud/SKILL.md +119 -0
- package/ai/skills/testdriver-customizing-devices/SKILL.md +319 -0
- package/ai/skills/testdriver-dashcam/SKILL.md +418 -0
- package/ai/skills/testdriver-debugging-with-screenshots/SKILL.md +401 -0
- package/ai/skills/testdriver-device-config/SKILL.md +317 -0
- package/ai/skills/testdriver-double-click/SKILL.md +102 -0
- package/ai/skills/testdriver-elements/SKILL.md +605 -0
- package/ai/skills/testdriver-enterprise/SKILL.md +114 -0
- package/ai/skills/testdriver-errors/SKILL.md +246 -0
- package/ai/skills/testdriver-events/SKILL.md +356 -0
- package/ai/skills/testdriver-examples/SKILL.md +7 -0
- package/ai/skills/testdriver-exec/SKILL.md +317 -0
- package/ai/skills/testdriver-find/SKILL.md +829 -0
- package/ai/skills/testdriver-focus-application/SKILL.md +293 -0
- package/ai/skills/testdriver-generating-tests/SKILL.md +36 -0
- package/ai/skills/testdriver-hover/SKILL.md +278 -0
- package/ai/skills/testdriver-locating-elements/SKILL.md +71 -0
- package/ai/skills/testdriver-making-assertions/SKILL.md +32 -0
- package/ai/skills/testdriver-mcp/SKILL.md +7 -0
- package/ai/skills/testdriver-mcp-workflow/SKILL.md +410 -0
- package/ai/skills/testdriver-mouse-down/SKILL.md +161 -0
- package/ai/skills/testdriver-mouse-up/SKILL.md +164 -0
- package/ai/skills/testdriver-parse/SKILL.md +236 -0
- package/ai/skills/testdriver-performing-actions/SKILL.md +54 -0
- package/ai/skills/testdriver-press-keys/SKILL.md +348 -0
- package/ai/skills/testdriver-provision/SKILL.md +331 -0
- package/ai/skills/testdriver-quickstart/SKILL.md +144 -0
- package/ai/skills/testdriver-redraw/SKILL.md +214 -0
- package/ai/skills/testdriver-reusable-code/SKILL.md +249 -0
- package/ai/skills/testdriver-right-click/SKILL.md +123 -0
- package/ai/skills/testdriver-running-tests/SKILL.md +185 -0
- package/ai/skills/testdriver-screenshot/SKILL.md +248 -0
- package/ai/skills/testdriver-screenshots/SKILL.md +184 -0
- package/ai/skills/testdriver-scroll/SKILL.md +335 -0
- package/ai/skills/testdriver-secrets/SKILL.md +115 -0
- package/ai/skills/testdriver-self-hosted/SKILL.md +65 -0
- package/ai/skills/testdriver-test-writer/SKILL.md +448 -0
- package/ai/skills/testdriver-testdriver/SKILL.md +628 -0
- package/ai/skills/testdriver-testdriver-mechanic/SKILL.md +165 -0
- package/ai/skills/testdriver-type/SKILL.md +357 -0
- package/ai/skills/testdriver-variables/SKILL.md +111 -0
- package/ai/skills/testdriver-wait/SKILL.md +50 -0
- package/ai/skills/testdriver-waiting-for-elements/SKILL.md +90 -0
- package/ai/skills/testdriver-what-is-testdriver/SKILL.md +54 -0
- package/bin/testdriverai.js +22 -0
- package/debugger/bg.png +0 -0
- package/debugger/icon.png +0 -0
- package/debugger/index.html +469 -0
- package/debugger/td.png +0 -0
- package/debugger/tray-buffered.png +0 -0
- package/debugger/tray.png +0 -0
- package/docs/GITHUB_COMMENTS.md +330 -0
- package/docs/GITHUB_COMMENTS_ANNOUNCEMENT.md +167 -0
- package/docs/QUICK-START-GITHUB-COMMENTS.md +84 -0
- package/docs/TEST-GITHUB-COMMENTS.md +129 -0
- package/docs/_data/examples-manifest.json +177 -0
- package/docs/_data/examples-manifest.schema.json +41 -0
- package/docs/_scripts/extract-example-urls.js +165 -0
- package/docs/_scripts/generate-examples.js +560 -0
- package/docs/_scripts/generate-skills.js +154 -0
- package/docs/_scripts/link-replacer.js +164 -0
- package/docs/_scripts/upload-docs-to-openai.js +284 -0
- package/docs/changelog.mdx +161 -0
- package/docs/claude-mcp-plugin.mdx +160 -0
- package/docs/docs.json +442 -0
- package/docs/github-integration-setup.md +266 -0
- package/docs/guide/best-practices-polling.mdx +174 -0
- package/docs/images/content/account/newprojectsettings.png +0 -0
- package/docs/images/content/account/projectpage.png +0 -0
- package/docs/images/content/account/projectreplays.png +0 -0
- package/docs/images/content/account/team-manage.png +0 -0
- package/docs/images/content/account/teampage.png +0 -0
- package/docs/images/content/extension/cursor.svg +1 -0
- package/docs/images/content/extension/vscode.svg +57 -0
- package/docs/images/content/extension/windsurf.svg +3 -0
- package/docs/images/content/parse/output.png +0 -0
- package/docs/images/content/self-hosted/launchtemplateid.png +0 -0
- package/docs/images/content/side-by-side.png +0 -0
- package/docs/images/content/vscode/ide-full.png +0 -0
- package/docs/images/content/vscode/running.png +0 -0
- package/docs/images/content/vscode/v7-chat.png +0 -0
- package/docs/images/content/vscode/v7-choose-agent.png +0 -0
- package/docs/images/content/vscode/v7-full.png +0 -0
- package/docs/images/content/vscode/v7-onboarding.png +0 -0
- package/docs/images/content/vscode/vscode-2-assert.png +0 -0
- package/docs/images/content/vscode/vscode-agent-preview.png +0 -0
- package/docs/images/content/vscode/vscode-copilot-ask.png +0 -0
- package/docs/images/content/vscode/vscode-file-creation.png +0 -0
- package/docs/images/content/vscode/vscode-install.png +0 -0
- package/docs/images/content/vscode/vscode-overview.png +0 -0
- package/docs/images/content/vscode/vscode-setup-walkthrough.png +0 -0
- package/docs/images/content/vscode/vscode-stopchat.png +0 -0
- package/docs/images/content/vscode/vscode-stoptest.png +0 -0
- package/docs/images/content/vscode/vscode-tdservice.png +0 -0
- package/docs/images/content/vscode/vscode-test-output.png +0 -0
- package/docs/images/content/vscode/vscode-testhistory.png +0 -0
- package/docs/images/content/vscode/vscode-testpane-runtests.png +0 -0
- package/docs/images/content/vscode/vscode-testpane.png +0 -0
- package/docs/images/template/dark.png +0 -0
- package/docs/images/template/icon.png +0 -0
- package/docs/images/template/light.png +0 -0
- package/docs/snippets/calendar-link.mdx +4 -0
- package/docs/snippets/gitignore-warning.mdx +7 -0
- package/docs/snippets/lifecycle-warning.mdx +6 -0
- package/docs/snippets/test-prereqs.mdx +12 -0
- package/docs/snippets/tests/assert-replay.mdx +7 -0
- package/docs/snippets/tests/assert-yaml.mdx +8 -0
- package/docs/snippets/tests/exec-js-replay.mdx +7 -0
- package/docs/snippets/tests/exec-js-yaml.mdx +32 -0
- package/docs/snippets/tests/exec-shell-replay.mdx +7 -0
- package/docs/snippets/tests/exec-shell-yaml.mdx +15 -0
- package/docs/snippets/tests/hover-image-replay.mdx +7 -0
- package/docs/snippets/tests/hover-image-yaml.mdx +17 -0
- package/docs/snippets/tests/hover-text-replay.mdx +7 -0
- package/docs/snippets/tests/hover-text-with-description-replay.mdx +7 -0
- package/docs/snippets/tests/hover-text-with-description-yaml.mdx +24 -0
- package/docs/snippets/tests/hover-text-yaml.mdx +14 -0
- package/docs/snippets/tests/match-image-replay.mdx +7 -0
- package/docs/snippets/tests/match-image-yaml.mdx +17 -0
- package/docs/snippets/tests/press-keys-replay.mdx +7 -0
- package/docs/snippets/tests/press-keys-yaml.mdx +36 -0
- package/docs/snippets/tests/remember-replay.mdx +7 -0
- package/docs/snippets/tests/remember-yaml.mdx +28 -0
- package/docs/snippets/tests/scroll-replay.mdx +7 -0
- package/docs/snippets/tests/scroll-until-image-replay.mdx +7 -0
- package/docs/snippets/tests/scroll-until-image-yaml.mdx +14 -0
- package/docs/snippets/tests/scroll-until-text-replay.mdx +7 -0
- package/docs/snippets/tests/scroll-until-text-yaml.mdx +17 -0
- package/docs/snippets/tests/scroll-yaml.mdx +30 -0
- package/docs/snippets/tests/type-repeated-replay.mdx +7 -0
- package/docs/snippets/tests/type-repeated-yaml.mdx +22 -0
- package/docs/snippets/tests/type-replay.mdx +7 -0
- package/docs/snippets/tests/type-yaml.mdx +28 -0
- package/docs/snippets/tests/wait-for-image-replay.mdx +7 -0
- package/docs/snippets/tests/wait-for-image-yaml.mdx +18 -0
- package/docs/snippets/tests/wait-for-text-replay.mdx +7 -0
- package/docs/snippets/tests/wait-for-text-yaml.mdx +18 -0
- package/docs/snippets/tests/wait-replay.mdx +7 -0
- package/docs/snippets/tests/wait-yaml.mdx +13 -0
- package/docs/styles.css +65 -0
- package/docs/v6/account/dashboard.mdx +16 -0
- package/docs/v6/account/enterprise.mdx +110 -0
- package/docs/v6/account/pricing.mdx +33 -0
- package/docs/v6/account/projects.mdx +33 -0
- package/docs/v6/account/team.mdx +35 -0
- package/docs/v6/action/ami.mdx +109 -0
- package/docs/v6/action/performance.mdx +105 -0
- package/docs/v6/action/secrets.mdx +93 -0
- package/docs/v6/apps/chrome-extensions.mdx +48 -0
- package/docs/v6/apps/desktop-apps.mdx +93 -0
- package/docs/v6/apps/mobile-apps.mdx +26 -0
- package/docs/v6/apps/static-websites.mdx +54 -0
- package/docs/v6/apps/tauri-apps.mdx +361 -0
- package/docs/v6/bugs/jira.mdx +232 -0
- package/docs/v6/cli/overview.mdx +66 -0
- package/docs/v6/commands/assert.mdx +45 -0
- package/docs/v6/commands/exec.mdx +276 -0
- package/docs/v6/commands/focus-application.mdx +44 -0
- package/docs/v6/commands/hover-image.mdx +69 -0
- package/docs/v6/commands/hover-text.mdx +47 -0
- package/docs/v6/commands/if.mdx +53 -0
- package/docs/v6/commands/match-image.mdx +67 -0
- package/docs/v6/commands/press-keys.mdx +87 -0
- package/docs/v6/commands/remember.mdx +49 -0
- package/docs/v6/commands/run.mdx +44 -0
- package/docs/v6/commands/scroll-until-image.mdx +66 -0
- package/docs/v6/commands/scroll-until-text.mdx +60 -0
- package/docs/v6/commands/scroll.mdx +69 -0
- package/docs/v6/commands/type.mdx +45 -0
- package/docs/v6/commands/wait-for-image.mdx +54 -0
- package/docs/v6/commands/wait-for-text.mdx +48 -0
- package/docs/v6/commands/wait.mdx +45 -0
- package/docs/v6/exporting/junit.mdx +218 -0
- package/docs/v6/exporting/playwright.mdx +197 -0
- package/docs/v6/features/auto-healing.mdx +144 -0
- package/docs/v6/features/generation.mdx +116 -0
- package/docs/v6/features/parallel-testing.mdx +151 -0
- package/docs/v6/features/reusable-snippets.mdx +131 -0
- package/docs/v6/features/selectorless.mdx +80 -0
- package/docs/v6/features/visual-assertions.mdx +139 -0
- package/docs/v6/getting-started/ci.mdx +146 -0
- package/docs/v6/getting-started/cli.mdx +91 -0
- package/docs/v6/getting-started/editing.mdx +100 -0
- package/docs/v6/getting-started/playwright.mdx +342 -0
- package/docs/v6/getting-started/running.mdx +48 -0
- package/docs/v6/getting-started/self-hosting.mdx +408 -0
- package/docs/v6/getting-started/vscode.mdx +88 -0
- package/docs/v6/guide/assertions.mdx +189 -0
- package/docs/v6/guide/authentication.mdx +136 -0
- package/docs/v6/guide/code.mdx +65 -0
- package/docs/v6/guide/dashcam.mdx +118 -0
- package/docs/v6/guide/environment-variables.mdx +26 -0
- package/docs/v6/guide/lifecycle.mdx +242 -0
- package/docs/v6/guide/locating.mdx +141 -0
- package/docs/v6/guide/protips.mdx +43 -0
- package/docs/v6/guide/variables.mdx +143 -0
- package/docs/v6/guide/waiting.mdx +130 -0
- package/docs/v6/importing/csv.mdx +196 -0
- package/docs/v6/importing/gherkin.mdx +143 -0
- package/docs/v6/importing/jira.mdx +164 -0
- package/docs/v6/importing/testrail.mdx +162 -0
- package/docs/v6/integrations/electron.mdx +146 -0
- package/docs/v6/integrations/netlify.mdx +100 -0
- package/docs/v6/integrations/vercel.mdx +125 -0
- package/docs/v6/interactive/explore.mdx +99 -0
- package/docs/v6/interactive/run.mdx +52 -0
- package/docs/v6/interactive/save.mdx +63 -0
- package/docs/v6/overview/comparison.mdx +101 -0
- package/docs/v6/overview/faq.mdx +162 -0
- package/docs/v6/overview/performance.mdx +52 -0
- package/docs/v6/overview/quickstart.mdx +137 -0
- package/docs/v6/overview/what-is-testdriver.mdx +85 -0
- package/docs/v6/scenarios/ai-chatbot.mdx +28 -0
- package/docs/v6/scenarios/cookie-banner.mdx +32 -0
- package/docs/v6/scenarios/file-upload.mdx +33 -0
- package/docs/v6/scenarios/form-filling.mdx +32 -0
- package/docs/v6/scenarios/log-in.mdx +75 -0
- package/docs/v6/scenarios/pdf-generation.mdx +25 -0
- package/docs/v6/scenarios/spell-check.mdx +22 -0
- package/docs/v6/security/action.mdx +84 -0
- package/docs/v6/security/agent.mdx +73 -0
- package/docs/v6/security/platform.mdx +77 -0
- package/docs/v6/tutorials/advanced-test.mdx +81 -0
- package/docs/v6/tutorials/basic-test.mdx +45 -0
- package/docs/v7/_drafts/agents.mdx +843 -0
- package/docs/v7/_drafts/architecture.mdx +399 -0
- package/docs/v7/_drafts/auto-cache-key.mdx +167 -0
- package/docs/v7/_drafts/awesome-logs-quick-ref.mdx +100 -0
- package/docs/v7/_drafts/best-practices.mdx +486 -0
- package/docs/v7/_drafts/caching-ai.mdx +215 -0
- package/docs/v7/_drafts/caching-selectors.mdx +424 -0
- package/docs/v7/_drafts/caching.mdx +366 -0
- package/docs/v7/_drafts/cli-to-sdk-migration.mdx +425 -0
- package/docs/v7/_drafts/commands/assert.mdx +45 -0
- package/docs/v7/_drafts/commands/exec.mdx +276 -0
- package/docs/v7/_drafts/commands/focus-application.mdx +44 -0
- package/docs/v7/_drafts/commands/hover-image.mdx +69 -0
- package/docs/v7/_drafts/commands/hover-text.mdx +47 -0
- package/docs/v7/_drafts/commands/if.mdx +53 -0
- package/docs/v7/_drafts/commands/match-image.mdx +67 -0
- package/docs/v7/_drafts/commands/press-keys.mdx +87 -0
- package/docs/v7/_drafts/commands/remember.mdx +49 -0
- package/docs/v7/_drafts/commands/run.mdx +44 -0
- package/docs/v7/_drafts/commands/scroll-until-image.mdx +66 -0
- package/docs/v7/_drafts/commands/scroll-until-text.mdx +60 -0
- package/docs/v7/_drafts/commands/scroll.mdx +69 -0
- package/docs/v7/_drafts/commands/type.mdx +45 -0
- package/docs/v7/_drafts/commands/wait-for-image.mdx +54 -0
- package/docs/v7/_drafts/commands/wait-for-text.mdx +48 -0
- package/docs/v7/_drafts/commands/wait.mdx +45 -0
- package/docs/v7/_drafts/configuration.mdx +378 -0
- package/docs/v7/_drafts/contributing.mdx +174 -0
- package/docs/v7/_drafts/core.mdx +458 -0
- package/docs/v7/_drafts/dashcam-title-feature.mdx +89 -0
- package/docs/v7/_drafts/debugging.mdx +349 -0
- package/docs/v7/_drafts/error-handling.mdx +501 -0
- package/docs/v7/_drafts/faq.mdx +393 -0
- package/docs/v7/_drafts/hooks.mdx +360 -0
- package/docs/v7/_drafts/init-command.mdx +95 -0
- package/docs/v7/_drafts/installation.mdx +420 -0
- package/docs/v7/_drafts/migration.mdx +562 -0
- package/docs/v7/_drafts/observable.mdx +604 -0
- package/docs/v7/_drafts/playwright.mdx +342 -0
- package/docs/v7/_drafts/plugin-migration.mdx +220 -0
- package/docs/v7/_drafts/powerful.mdx +419 -0
- package/docs/v7/_drafts/presets.mdx +210 -0
- package/docs/v7/_drafts/progressive-disclosure.mdx +230 -0
- package/docs/v7/_drafts/prompt-cache.mdx +200 -0
- package/docs/v7/_drafts/provision.mdx +390 -0
- package/docs/v7/_drafts/quick-start-test-recording.mdx +214 -0
- package/docs/v7/_drafts/readme.mdx +135 -0
- package/docs/v7/_drafts/reports.mdx +414 -0
- package/docs/v7/_drafts/scalable.mdx +763 -0
- package/docs/v7/_drafts/screenshot.mdx +155 -0
- package/docs/v7/_drafts/sdk-awesome-logs.mdx +468 -0
- package/docs/v7/_drafts/sdk-browser-rendering.mdx +167 -0
- package/docs/v7/_drafts/sdk-migration.mdx +474 -0
- package/docs/v7/_drafts/sdk-v7-complete.mdx +345 -0
- package/docs/v7/_drafts/self-hosting.mdx +369 -0
- package/docs/v7/_drafts/test-recording.mdx +382 -0
- package/docs/v7/_drafts/troubleshooting.mdx +526 -0
- package/docs/v7/_drafts/vitest-plugin.mdx +477 -0
- package/docs/v7/_drafts/vitest.mdx +535 -0
- package/docs/v7/_drafts/writing-tests.mdx +25 -0
- package/docs/v7/ai.mdx +205 -0
- package/docs/v7/assert.mdx +316 -0
- package/docs/v7/aws-setup.mdx +449 -0
- package/docs/v7/cache.mdx +223 -0
- package/docs/v7/caching.mdx +128 -0
- package/docs/v7/captcha.mdx +159 -0
- package/docs/v7/ci-cd.mdx +603 -0
- package/docs/v7/click.mdx +287 -0
- package/docs/v7/client.mdx +478 -0
- package/docs/v7/copilot/auto-healing.mdx +265 -0
- package/docs/v7/copilot/creating-tests.mdx +156 -0
- package/docs/v7/copilot/github.mdx +143 -0
- package/docs/v7/copilot/running-tests.mdx +149 -0
- package/docs/v7/copilot/setup.mdx +143 -0
- package/docs/v7/customizing-devices.mdx +319 -0
- package/docs/v7/dashcam.mdx +419 -0
- package/docs/v7/debugging-with-screenshots.mdx +402 -0
- package/docs/v7/device-config.mdx +317 -0
- package/docs/v7/double-click.mdx +102 -0
- package/docs/v7/elements.mdx +606 -0
- package/docs/v7/enterprise.mdx +9 -0
- package/docs/v7/errors.mdx +248 -0
- package/docs/v7/events.mdx +358 -0
- package/docs/v7/examples/ai.mdx +72 -0
- package/docs/v7/examples/assert.mdx +72 -0
- package/docs/v7/examples/captcha-api.mdx +92 -0
- package/docs/v7/examples/chrome-extension.mdx +132 -0
- package/docs/v7/examples/drag-and-drop.mdx +100 -0
- package/docs/v7/examples/element-not-found.mdx +67 -0
- package/docs/v7/examples/exec-output.mdx +85 -0
- package/docs/v7/examples/exec-pwsh.mdx +83 -0
- package/docs/v7/examples/focus-window.mdx +62 -0
- package/docs/v7/examples/hover-image.mdx +94 -0
- package/docs/v7/examples/hover-text.mdx +69 -0
- package/docs/v7/examples/installer.mdx +91 -0
- package/docs/v7/examples/launch-vscode-linux.mdx +101 -0
- package/docs/v7/examples/match-image.mdx +96 -0
- package/docs/v7/examples/press-keys.mdx +92 -0
- package/docs/v7/examples/scroll-keyboard.mdx +79 -0
- package/docs/v7/examples/scroll-until-image.mdx +81 -0
- package/docs/v7/examples/scroll-until-text.mdx +109 -0
- package/docs/v7/examples/scroll.mdx +81 -0
- package/docs/v7/examples/type.mdx +92 -0
- package/docs/v7/examples/windows-installer.mdx +89 -0
- package/docs/v7/exec.mdx +318 -0
- package/docs/v7/find.mdx +830 -0
- package/docs/v7/focus-application.mdx +294 -0
- package/docs/v7/generating-tests.mdx +36 -0
- package/docs/v7/hosted.mdx +158 -0
- package/docs/v7/hover.mdx +279 -0
- package/docs/v7/locating-elements.mdx +71 -0
- package/docs/v7/making-assertions.mdx +32 -0
- package/docs/v7/mcp.mdx +9 -0
- package/docs/v7/mouse-down.mdx +161 -0
- package/docs/v7/mouse-up.mdx +164 -0
- package/docs/v7/parse.mdx +237 -0
- package/docs/v7/performing-actions.mdx +54 -0
- package/docs/v7/press-keys.mdx +349 -0
- package/docs/v7/provision.mdx +333 -0
- package/docs/v7/quickstart.mdx +173 -0
- package/docs/v7/redraw.mdx +216 -0
- package/docs/v7/reusable-code.mdx +249 -0
- package/docs/v7/right-click.mdx +123 -0
- package/docs/v7/running-tests.mdx +185 -0
- package/docs/v7/screenshot.mdx +249 -0
- package/docs/v7/screenshots.mdx +186 -0
- package/docs/v7/scroll.mdx +336 -0
- package/docs/v7/secrets.mdx +115 -0
- package/docs/v7/self-hosted.mdx +149 -0
- package/docs/v7/type.mdx +358 -0
- package/docs/v7/variables.mdx +111 -0
- package/docs/v7/wait.mdx +52 -0
- package/docs/v7/waiting-for-elements.mdx +90 -0
- package/docs/v7/what-is-testdriver.mdx +54 -0
- package/eslint.config.js +67 -0
- package/examples/ai.test.mjs +31 -0
- package/examples/assert.test.mjs +47 -0
- package/examples/chrome-extension.test.mjs +97 -0
- package/examples/config.mjs +5 -0
- package/examples/element-not-found.test.mjs +27 -0
- package/examples/exec-output.test.mjs +60 -0
- package/examples/exec-pwsh.test.mjs +58 -0
- package/examples/findall-coffee-icons.test.mjs +42 -0
- package/examples/focus-window.test.mjs +37 -0
- package/examples/formatted-logging.test.mjs +27 -0
- package/examples/hover-image.test.mjs +53 -0
- package/examples/hover-text-with-description.test.mjs +57 -0
- package/examples/hover-text.test.mjs +28 -0
- package/examples/installer.test.mjs +50 -0
- package/examples/launch-vscode-linux.test.mjs +55 -0
- package/examples/match-image.test.mjs +55 -0
- package/examples/parse.test.mjs +19 -0
- package/examples/press-keys.test.mjs +44 -0
- package/examples/prompt.test.mjs +34 -0
- package/examples/scroll-keyboard.test.mjs +38 -0
- package/examples/scroll-until-image.test.mjs +40 -0
- package/examples/scroll.test.mjs +42 -0
- package/examples/type.test.mjs +46 -0
- package/examples/windows-installer.test.mjs +54 -0
- package/index.js +2 -0
- package/interfaces/cli/commands/init.js +438 -0
- package/interfaces/cli/commands/setup.js +382 -0
- package/interfaces/cli/lib/base.js +285 -0
- package/interfaces/cli.js +20 -0
- package/interfaces/junit-reporter.js +290 -0
- package/interfaces/logger.js +388 -0
- package/interfaces/readline.js +234 -0
- package/interfaces/shared-test-state.mjs +64 -0
- package/interfaces/vitest-plugin.d.ts +115 -0
- package/interfaces/vitest-plugin.mjs +1698 -0
- package/lib/captcha/solver.js +358 -0
- package/lib/core/Dashcam.js +533 -0
- package/lib/core/index.d.ts +172 -0
- package/lib/core/index.js +12 -0
- package/lib/environments.json +18 -0
- package/lib/github-comment-formatter.js +263 -0
- package/lib/github-comment.mjs +452 -0
- package/lib/init-project.js +575 -0
- package/lib/presets/index.mjs +331 -0
- package/lib/resolve-channel.js +46 -0
- package/lib/sentry.js +417 -0
- package/lib/vitest/hooks.d.ts +57 -0
- package/lib/vitest/hooks.mjs +674 -0
- package/lib/vitest/setup-aws.mjs +247 -0
- package/lib/vitest/setup-self-hosted.mjs +151 -0
- package/lib/vitest/setup.mjs +46 -0
- package/manual/captcha-api.test.mjs +51 -0
- package/manual/drag-and-drop.test.mjs +59 -0
- package/manual/flake-diffthreshold-001.test.mjs +9 -0
- package/manual/flake-diffthreshold-01.test.mjs +9 -0
- package/manual/flake-diffthreshold-05.test.mjs +9 -0
- package/manual/flake-noredraw-cache.test.mjs +9 -0
- package/manual/flake-noredraw-nocache.test.mjs +9 -0
- package/manual/flake-redraw-cache.test.mjs +9 -0
- package/manual/flake-redraw-nocache.test.mjs +9 -0
- package/manual/flake-rocket-match.test.mjs +30 -0
- package/manual/flake-shared.mjs +51 -0
- package/manual/no-provision.test.mjs +31 -0
- package/manual/packer-hover-image.test.mjs +176 -0
- package/manual/scroll-until-text.test.mjs +68 -0
- package/manual/test-init-command.js +223 -0
- package/mcp-server/README.md +322 -0
- package/mcp-server/dist/codegen.d.ts +9 -0
- package/mcp-server/dist/codegen.js +165 -0
- package/mcp-server/dist/mcp-app.html +114 -0
- package/mcp-server/dist/package.json +1 -0
- package/mcp-server/dist/provision-types.d.ts +290 -0
- package/mcp-server/dist/provision-types.js +174 -0
- package/mcp-server/dist/server.d.ts +6 -0
- package/mcp-server/dist/server.mjs +1925 -0
- package/mcp-server/dist/session.d.ts +85 -0
- package/mcp-server/dist/session.js +152 -0
- package/mcp-server/mcp-app.html +28 -0
- package/mcp-server/mcp-config.example.json +19 -0
- package/mcp-server/package-lock.json +4027 -0
- package/mcp-server/package.json +31 -0
- package/mcp-server/src/codegen.ts +189 -0
- package/mcp-server/src/mcp-app.css +360 -0
- package/mcp-server/src/mcp-app.ts +547 -0
- package/mcp-server/src/provision-types.ts +209 -0
- package/mcp-server/src/server.ts +2391 -0
- package/mcp-server/src/session.ts +194 -0
- package/mcp-server/tsconfig.json +16 -0
- package/mcp-server/vite.config.ts +23 -0
- package/package.json +158 -0
- package/schema.json +1046 -0
- package/scripts/generate-skills.js +94 -0
- package/sdk-log-formatter.js +1157 -0
- package/sdk.d.ts +1486 -0
- package/sdk.js +4336 -0
- package/setup/aws/cloudformation.yaml +463 -0
- package/setup/aws/disable-defender.sh +42 -0
- package/setup/aws/install-dev-runner.sh +79 -0
- package/setup/aws/spawn-runner.sh +289 -0
- package/test/captcha-solver.test.mjs +152 -0
- package/test/chrome-remote-debugging.test.mjs +66 -0
- package/test/duckduckgo/experiment.test.mjs +28 -0
- package/test/duckduckgo/setup.test.mjs +29 -0
- package/test/manual/debug-locate-response.js +82 -0
- package/test/manual/reconnect-provision.test.mjs +49 -0
- package/test/manual/test-console-logs.test.mjs +42 -0
- package/test/manual/test-find-api.js +73 -0
- package/test/manual/test-init.sh +54 -0
- package/test/manual/test-prompt-cache.js +97 -0
- package/test/manual/test-provision-auth.mjs +22 -0
- package/test/manual/test-sandbox-render.js +29 -0
- package/test/manual/test-sdk-methods.js +15 -0
- package/test/manual/test-sdk-refactor.js +53 -0
- package/test/manual/test-stack-trace.mjs +57 -0
- package/test/manual/verify-element-api.js +89 -0
- package/test/manual/verify-types.js +0 -0
- package/test/manual-unawaited-promise.test.mjs +31 -0
- package/vitest.config.mjs +58 -0
- package/vitest.runner.config.mjs +33 -0
- package/vscode-extension/.vscodeignore +12 -0
- package/vscode-extension/README.md +94 -0
- package/vscode-extension/media/icon.png +0 -0
- package/vscode-extension/package-lock.json +4126 -0
- package/vscode-extension/package.json +86 -0
- package/vscode-extension/src/extension.ts +829 -0
- package/vscode-extension/testdriverai-0.1.0.vsix +0 -0
- package/vscode-extension/tsconfig.json +16 -0
|
@@ -0,0 +1,2391 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* TestDriver MCP Server
|
|
4
|
+
* Enables AI agents to iteratively build tests with visual feedback
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
// Configure logger to use stderr to avoid corrupting MCP JSON-RPC stream on stdout
|
|
8
|
+
process.env.TD_STDIO = "stderr";
|
|
9
|
+
// Enable debug mode to preserve croppedImage in SDK responses (needed for MCP App visuals)
|
|
10
|
+
process.env.TD_DEBUG = "true";
|
|
11
|
+
|
|
12
|
+
import { registerAppResource, registerAppTool, RESOURCE_MIME_TYPE } from "@modelcontextprotocol/ext-apps/server";
|
|
13
|
+
import { McpServer, ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
14
|
+
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
15
|
+
import type { Variables } from "@modelcontextprotocol/sdk/shared/uriTemplate.js";
|
|
16
|
+
import type { CallToolResult, ReadResourceResult } from "@modelcontextprotocol/sdk/types.js";
|
|
17
|
+
import * as Sentry from "@sentry/node";
|
|
18
|
+
import * as fs from "fs";
|
|
19
|
+
import * as os from "os";
|
|
20
|
+
import * as path from "path";
|
|
21
|
+
import { fileURLToPath, pathToFileURL } from "url";
|
|
22
|
+
import { z } from "zod";
|
|
23
|
+
|
|
24
|
+
import { generateActionCode } from "./codegen.js";
|
|
25
|
+
import { getProvisionOptions, SessionStartInputSchema, type SessionStartInput } from "./provision-types.js";
|
|
26
|
+
import { sessionManager, type SessionState } from "./session.js";
|
|
27
|
+
|
|
28
|
+
// =============================================================================
|
|
29
|
+
// Sentry
|
|
30
|
+
// =============================================================================
|
|
31
|
+
|
|
32
|
+
// Read version from main package.json (../../package.json from mcp-server/dist/)
|
|
33
|
+
const sdkRoot = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
34
|
+
const packageJson = JSON.parse(fs.readFileSync(path.join(sdkRoot, "package.json"), "utf-8"));
|
|
35
|
+
const version = packageJson.version || "1.0.0";
|
|
36
|
+
|
|
37
|
+
// Derive release channel from package version prerelease tag (e.g. "7.6.0-test.5" → "test")
|
|
38
|
+
import semver from "semver";
|
|
39
|
+
const KNOWN_CHANNELS = new Set(["dev", "test", "canary", "latest"]);
|
|
40
|
+
function resolveReleaseChannel(ver: string): string {
|
|
41
|
+
if (process.env.TD_CHANNEL && KNOWN_CHANNELS.has(process.env.TD_CHANNEL)) return process.env.TD_CHANNEL;
|
|
42
|
+
const pre = semver.prerelease(ver);
|
|
43
|
+
if (pre && pre.length > 0 && KNOWN_CHANNELS.has(String(pre[0]))) return String(pre[0]);
|
|
44
|
+
return "latest";
|
|
45
|
+
}
|
|
46
|
+
const releaseChannel = resolveReleaseChannel(version);
|
|
47
|
+
|
|
48
|
+
const isSentryEnabled = () => {
|
|
49
|
+
if (process.env.TD_TELEMETRY === "false") {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
return true;
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
if (isSentryEnabled()) {
|
|
56
|
+
console.error("Analytics enabled. Set TD_TELEMETRY=false to disable.");
|
|
57
|
+
Sentry.init({
|
|
58
|
+
dsn:
|
|
59
|
+
process.env.SENTRY_DSN ||
|
|
60
|
+
"https://452bd5a00dbd83a38ee8813e11c57694@o4510262629236736.ingest.us.sentry.io/4510480443637760",
|
|
61
|
+
environment: releaseChannel,
|
|
62
|
+
release: version,
|
|
63
|
+
sampleRate: 1.0,
|
|
64
|
+
tracesSampleRate: 1.0,
|
|
65
|
+
sendDefaultPii: true,
|
|
66
|
+
integrations: [Sentry.httpIntegration(), Sentry.nodeContextIntegration()],
|
|
67
|
+
initialScope: {
|
|
68
|
+
tags: {
|
|
69
|
+
platform: os.platform(),
|
|
70
|
+
arch: os.arch(),
|
|
71
|
+
nodeVersion: process.version,
|
|
72
|
+
},
|
|
73
|
+
},
|
|
74
|
+
// Filter out expected test/element failures - only report actual exceptions and crashes
|
|
75
|
+
beforeSend(event, hint) {
|
|
76
|
+
const error = hint.originalException;
|
|
77
|
+
|
|
78
|
+
if (error && typeof error === "object" && "message" in error) {
|
|
79
|
+
const msg = (error as { message: string }).message;
|
|
80
|
+
|
|
81
|
+
// Don't send user-initiated exits
|
|
82
|
+
if (msg.includes("User cancelled")) {
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Don't send expected test/element failures - these are normal test outcomes, not crashes
|
|
87
|
+
if (
|
|
88
|
+
msg.includes("Element not found") ||
|
|
89
|
+
msg.includes("No elements found") ||
|
|
90
|
+
msg.includes("No element found") ||
|
|
91
|
+
msg.includes("Assertion failed") ||
|
|
92
|
+
msg.includes("assertion failed")
|
|
93
|
+
) {
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Filter out TestFailure errors (test failures, not crashes)
|
|
99
|
+
if (error && typeof error === "object" && "name" in error && (error as { name: string }).name === "TestFailure") {
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Filter out ElementNotFoundError - expected test outcome, not a crash
|
|
104
|
+
if (error && typeof error === "object" && "name" in error && (error as { name: string }).name === "ElementNotFoundError") {
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return event;
|
|
109
|
+
},
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function captureException(error: Error, context: { tags?: Record<string, string>; extra?: Record<string, unknown> } = {}) {
|
|
114
|
+
if (!isSentryEnabled()) return;
|
|
115
|
+
|
|
116
|
+
Sentry.withScope((scope) => {
|
|
117
|
+
if (context.tags) {
|
|
118
|
+
Object.entries(context.tags).forEach(([key, value]) => {
|
|
119
|
+
scope.setTag(key, value);
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
if (context.extra) {
|
|
123
|
+
Object.entries(context.extra).forEach(([key, value]) => {
|
|
124
|
+
scope.setExtra(key, value);
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
Sentry.captureException(error);
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function setSessionContext(sessionId: string, sandboxId?: string) {
|
|
132
|
+
if (!isSentryEnabled()) return;
|
|
133
|
+
|
|
134
|
+
Sentry.setTag("session", sessionId);
|
|
135
|
+
if (sandboxId) {
|
|
136
|
+
Sentry.setTag("sandbox", sandboxId);
|
|
137
|
+
}
|
|
138
|
+
Sentry.setContext("session", {
|
|
139
|
+
sessionId,
|
|
140
|
+
sandboxId,
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async function flushSentry(timeout = 2000) {
|
|
145
|
+
if (!isSentryEnabled()) return;
|
|
146
|
+
await Sentry.flush(timeout);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// =============================================================================
|
|
150
|
+
// Logging
|
|
151
|
+
// =============================================================================
|
|
152
|
+
|
|
153
|
+
const LOG_LEVELS = {
|
|
154
|
+
DEBUG: 0,
|
|
155
|
+
INFO: 1,
|
|
156
|
+
WARN: 2,
|
|
157
|
+
ERROR: 3,
|
|
158
|
+
} as const;
|
|
159
|
+
|
|
160
|
+
type LogLevel = keyof typeof LOG_LEVELS;
|
|
161
|
+
|
|
162
|
+
// Set via TD_LOG_LEVEL env var (default: INFO)
|
|
163
|
+
const currentLogLevel = LOG_LEVELS[(process.env.TD_LOG_LEVEL?.toUpperCase() as LogLevel) || "INFO"] ?? LOG_LEVELS.INFO;
|
|
164
|
+
|
|
165
|
+
function log(level: LogLevel, message: string, data?: Record<string, unknown>) {
|
|
166
|
+
if (LOG_LEVELS[level] < currentLogLevel) return;
|
|
167
|
+
|
|
168
|
+
const timestamp = new Date().toISOString();
|
|
169
|
+
const prefix = `[${timestamp}] [${level}]`;
|
|
170
|
+
const dataStr = data ? ` ${JSON.stringify(data)}` : "";
|
|
171
|
+
console.error(`${prefix} ${message}${dataStr}`);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
const logger = {
|
|
175
|
+
debug: (msg: string, data?: Record<string, unknown>) => log("DEBUG", msg, data),
|
|
176
|
+
info: (msg: string, data?: Record<string, unknown>) => log("INFO", msg, data),
|
|
177
|
+
warn: (msg: string, data?: Record<string, unknown>) => log("WARN", msg, data),
|
|
178
|
+
error: (msg: string, data?: Record<string, unknown>) => log("ERROR", msg, data),
|
|
179
|
+
};
|
|
180
|
+
|
|
181
|
+
// Get directory for UI files - works both from source and compiled
|
|
182
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
183
|
+
const __dirname = path.dirname(__filename);
|
|
184
|
+
const DIST_DIR = __filename.endsWith(".ts")
|
|
185
|
+
? path.join(__dirname, "..", "dist")
|
|
186
|
+
: __dirname;
|
|
187
|
+
|
|
188
|
+
// Resource URI for the screenshot result UI
|
|
189
|
+
const RESOURCE_URI = "ui://testdriver/mcp-app.html";
|
|
190
|
+
|
|
191
|
+
// Resource URI base for serving screenshot blobs (with dynamic IDs)
|
|
192
|
+
const SCREENSHOT_RESOURCE_BASE = "screenshot://testdriver/screenshot";
|
|
193
|
+
const CROPPED_IMAGE_RESOURCE_BASE = "screenshot://testdriver/cropped";
|
|
194
|
+
|
|
195
|
+
// SDK instance (will be initialized on session start)
|
|
196
|
+
let sdk: any = null;
|
|
197
|
+
|
|
198
|
+
// Last screenshot base64 for check comparisons
|
|
199
|
+
let lastScreenshotBase64: string | null = null;
|
|
200
|
+
|
|
201
|
+
// =============================================================================
|
|
202
|
+
// Image Store - Stores images with unique IDs for reload persistence
|
|
203
|
+
// =============================================================================
|
|
204
|
+
|
|
205
|
+
interface StoredImage {
|
|
206
|
+
data: string; // base64 image data
|
|
207
|
+
type: "screenshot" | "cropped";
|
|
208
|
+
timestamp: number;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// Map of image ID -> image data
|
|
212
|
+
const imageStore = new Map<string, StoredImage>();
|
|
213
|
+
|
|
214
|
+
// Counter for generating unique image IDs
|
|
215
|
+
let imageIdCounter = 0;
|
|
216
|
+
|
|
217
|
+
// Maximum number of images to store (to prevent memory leaks)
|
|
218
|
+
const MAX_STORED_IMAGES = 100;
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Store an image and return its unique resource URI
|
|
222
|
+
*/
|
|
223
|
+
function storeImage(data: string, type: "screenshot" | "cropped"): string {
|
|
224
|
+
const id = `${type}-${++imageIdCounter}`;
|
|
225
|
+
|
|
226
|
+
// Clean up old images if we exceed the limit
|
|
227
|
+
if (imageStore.size >= MAX_STORED_IMAGES) {
|
|
228
|
+
// Remove oldest images (first entries in the map)
|
|
229
|
+
const entriesToRemove = Math.floor(MAX_STORED_IMAGES / 4);
|
|
230
|
+
const keys = Array.from(imageStore.keys()).slice(0, entriesToRemove);
|
|
231
|
+
for (const key of keys) {
|
|
232
|
+
imageStore.delete(key);
|
|
233
|
+
}
|
|
234
|
+
logger.debug("storeImage: Cleaned up old images", { removed: entriesToRemove, remaining: imageStore.size });
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
imageStore.set(id, {
|
|
238
|
+
data,
|
|
239
|
+
type,
|
|
240
|
+
timestamp: Date.now(),
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
logger.debug("storeImage: Stored image", { id, type, dataLength: data.length });
|
|
244
|
+
|
|
245
|
+
const base = type === "screenshot" ? SCREENSHOT_RESOURCE_BASE : CROPPED_IMAGE_RESOURCE_BASE;
|
|
246
|
+
return `${base}/${id}`;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Get an image by its ID
|
|
251
|
+
*/
|
|
252
|
+
function getStoredImage(id: string): StoredImage | undefined {
|
|
253
|
+
return imageStore.get(id);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Get session info for structured content
|
|
258
|
+
*/
|
|
259
|
+
function getSessionData(session: SessionState | null) {
|
|
260
|
+
if (!session) return { id: null, expiresIn: 0 };
|
|
261
|
+
return {
|
|
262
|
+
id: session.sessionId,
|
|
263
|
+
expiresIn: sessionManager.getTimeRemaining(session.sessionId),
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Check if session is ready for use - returns error result if not
|
|
269
|
+
* This helper provides clear, actionable error messages for the AI
|
|
270
|
+
*
|
|
271
|
+
* Auto-extends the session on each successful check to prevent expiry during active use
|
|
272
|
+
*/
|
|
273
|
+
function requireActiveSession(): { valid: true } | { valid: false; error: CallToolResult } {
|
|
274
|
+
const session = sessionManager.getCurrentSession();
|
|
275
|
+
|
|
276
|
+
// No session ever created
|
|
277
|
+
if (!sdk || !session) {
|
|
278
|
+
return {
|
|
279
|
+
valid: false,
|
|
280
|
+
error: createToolResult(
|
|
281
|
+
false,
|
|
282
|
+
"ERROR: No active session. You must call session_start first to create a sandbox before using any other tools.",
|
|
283
|
+
{
|
|
284
|
+
error: "NO_SESSION",
|
|
285
|
+
action: "session_start",
|
|
286
|
+
message: "No sandbox session exists. Call session_start to create one."
|
|
287
|
+
}
|
|
288
|
+
)
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// Session exists but has expired
|
|
293
|
+
if (!sessionManager.isSessionValid(session.sessionId)) {
|
|
294
|
+
// Clear the SDK reference since the sandbox is no longer available
|
|
295
|
+
sdk = null;
|
|
296
|
+
return {
|
|
297
|
+
valid: false,
|
|
298
|
+
error: createToolResult(
|
|
299
|
+
false,
|
|
300
|
+
"ERROR: Session has expired or timed out. The sandbox is no longer available. You must call session_start again to create a new sandbox session before continuing.",
|
|
301
|
+
{
|
|
302
|
+
error: "SESSION_EXPIRED",
|
|
303
|
+
action: "session_start",
|
|
304
|
+
message: "The previous sandbox session has expired. Call session_start to create a new one.",
|
|
305
|
+
expiredSessionId: session.sessionId
|
|
306
|
+
}
|
|
307
|
+
)
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// Auto-extend session on each command to prevent expiry during active use
|
|
312
|
+
// This resets the expiry timer back to the original keepAlive duration
|
|
313
|
+
sessionManager.refreshSession(session.sessionId);
|
|
314
|
+
|
|
315
|
+
return { valid: true };
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Create tool result with structured content for MCP App
|
|
320
|
+
* Images: imageUrl (data URL) goes to structuredContent for UI to display
|
|
321
|
+
* The croppedImage from find() is small (~10KB) so it's acceptable as data URL
|
|
322
|
+
*
|
|
323
|
+
* If generatedCode is provided, it's appended to the text response with instructions
|
|
324
|
+
* for the agent to write it to the test file.
|
|
325
|
+
*/
|
|
326
|
+
function createToolResult(
|
|
327
|
+
success: boolean,
|
|
328
|
+
textContent: string,
|
|
329
|
+
structuredData: Record<string, unknown>,
|
|
330
|
+
generatedCode?: string
|
|
331
|
+
): CallToolResult {
|
|
332
|
+
// Build text content - append generated code if provided with directive instructions
|
|
333
|
+
let fullText = textContent;
|
|
334
|
+
if (generatedCode && success) {
|
|
335
|
+
// Get the test file from the current session
|
|
336
|
+
const session = sessionManager.getCurrentSession();
|
|
337
|
+
const testFile = session?.testFile;
|
|
338
|
+
|
|
339
|
+
if (testFile) {
|
|
340
|
+
fullText += `\n\n⚠️ ACTION REQUIRED: Append this code to ${testFile}:\n\`\`\`javascript\n${generatedCode}\n\`\`\``;
|
|
341
|
+
} else {
|
|
342
|
+
fullText += `\n\n⚠️ ACTION REQUIRED: Append this code to the test file:\n\`\`\`javascript\n${generatedCode}\n\`\`\``;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
const content: CallToolResult["content"] = [{ type: "text", text: fullText }];
|
|
347
|
+
|
|
348
|
+
logger.debug("createToolResult", {
|
|
349
|
+
success,
|
|
350
|
+
action: structuredData.action,
|
|
351
|
+
hasImage: !!structuredData.imageUrl,
|
|
352
|
+
duration: structuredData.duration,
|
|
353
|
+
hasGeneratedCode: !!generatedCode
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
// structuredContent goes to UI (includes imageUrl for display)
|
|
357
|
+
// Always include success flag so UI can display correct status indicator
|
|
358
|
+
// Include generatedCode and testFile in structured data so agents can programmatically handle it
|
|
359
|
+
const session = sessionManager.getCurrentSession();
|
|
360
|
+
return {
|
|
361
|
+
content,
|
|
362
|
+
structuredContent: {
|
|
363
|
+
...structuredData,
|
|
364
|
+
success,
|
|
365
|
+
generatedCode: generatedCode && success ? generatedCode : undefined,
|
|
366
|
+
testFile: session?.testFile || undefined,
|
|
367
|
+
},
|
|
368
|
+
};
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// Create MCP server wrapped with Sentry for automatic tracing
|
|
372
|
+
const server = isSentryEnabled()
|
|
373
|
+
? Sentry.wrapMcpServerWithSentry(
|
|
374
|
+
new McpServer({
|
|
375
|
+
name: "testdriver",
|
|
376
|
+
version: version,
|
|
377
|
+
})
|
|
378
|
+
)
|
|
379
|
+
: new McpServer({
|
|
380
|
+
name: "testdriver",
|
|
381
|
+
version: version,
|
|
382
|
+
});
|
|
383
|
+
|
|
384
|
+
// Element reference storage (for click/hover after find)
|
|
385
|
+
// Stores actual Element instances - no raw coordinates as input
|
|
386
|
+
const elementRefs = new Map<string, { element: any; description: string; coords: { x: number; y: number; centerX: number; centerY: number } }>();
|
|
387
|
+
|
|
388
|
+
// =============================================================================
|
|
389
|
+
// Register UI Resource
|
|
390
|
+
// =============================================================================
|
|
391
|
+
|
|
392
|
+
registerAppResource(
|
|
393
|
+
server,
|
|
394
|
+
RESOURCE_URI,
|
|
395
|
+
RESOURCE_URI,
|
|
396
|
+
{ mimeType: RESOURCE_MIME_TYPE, description: "TestDriver Screenshot Viewer UI" },
|
|
397
|
+
async (): Promise<ReadResourceResult> => {
|
|
398
|
+
const htmlPath = path.join(DIST_DIR, "mcp-app.html");
|
|
399
|
+
|
|
400
|
+
if (!fs.existsSync(htmlPath)) {
|
|
401
|
+
throw new Error(`UI file not found: ${htmlPath}`);
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const html = fs.readFileSync(htmlPath, "utf-8");
|
|
405
|
+
return {
|
|
406
|
+
contents: [{ uri: RESOURCE_URI, mimeType: RESOURCE_MIME_TYPE, text: html }],
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
);
|
|
410
|
+
|
|
411
|
+
// Register screenshot resource template for serving binary blobs by ID
|
|
412
|
+
server.registerResource(
|
|
413
|
+
"Screenshot",
|
|
414
|
+
new ResourceTemplate(`${SCREENSHOT_RESOURCE_BASE}/{imageId}`, { list: undefined }),
|
|
415
|
+
{
|
|
416
|
+
description: "Screenshot from TestDriver session served as base64 blob",
|
|
417
|
+
mimeType: "image/png",
|
|
418
|
+
},
|
|
419
|
+
async (uri: URL, variables: Variables): Promise<ReadResourceResult> => {
|
|
420
|
+
const imageId = variables.imageId as string;
|
|
421
|
+
const image = getStoredImage(imageId);
|
|
422
|
+
|
|
423
|
+
if (!image) {
|
|
424
|
+
throw new Error(`Screenshot not found: ${imageId}. It may have been cleaned up.`);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
logger.debug("screenshot resource: Serving screenshot blob", {
|
|
428
|
+
imageId,
|
|
429
|
+
blobLength: image.data.length
|
|
430
|
+
});
|
|
431
|
+
|
|
432
|
+
return {
|
|
433
|
+
contents: [{
|
|
434
|
+
uri: uri.href,
|
|
435
|
+
mimeType: "image/png",
|
|
436
|
+
blob: image.data,
|
|
437
|
+
}],
|
|
438
|
+
};
|
|
439
|
+
}
|
|
440
|
+
);
|
|
441
|
+
|
|
442
|
+
// Register cropped image resource template for serving find operation results by ID
|
|
443
|
+
server.registerResource(
|
|
444
|
+
"CroppedImage",
|
|
445
|
+
new ResourceTemplate(`${CROPPED_IMAGE_RESOURCE_BASE}/{imageId}`, { list: undefined }),
|
|
446
|
+
{
|
|
447
|
+
description: "Cropped image from find operations served as base64 blob",
|
|
448
|
+
mimeType: "image/png",
|
|
449
|
+
},
|
|
450
|
+
async (uri: URL, variables: Variables): Promise<ReadResourceResult> => {
|
|
451
|
+
const imageId = variables.imageId as string;
|
|
452
|
+
const image = getStoredImage(imageId);
|
|
453
|
+
|
|
454
|
+
if (!image) {
|
|
455
|
+
throw new Error(`Cropped image not found: ${imageId}. It may have been cleaned up.`);
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
logger.debug("cropped image resource: Serving cropped image blob", {
|
|
459
|
+
imageId,
|
|
460
|
+
blobLength: image.data.length
|
|
461
|
+
});
|
|
462
|
+
|
|
463
|
+
return {
|
|
464
|
+
contents: [{
|
|
465
|
+
uri: uri.href,
|
|
466
|
+
mimeType: "image/png",
|
|
467
|
+
blob: image.data,
|
|
468
|
+
}],
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
);
|
|
472
|
+
|
|
473
|
+
// =============================================================================
|
|
474
|
+
// Tools
|
|
475
|
+
// =============================================================================
|
|
476
|
+
|
|
477
|
+
// Session Start
|
|
478
|
+
registerAppTool(
|
|
479
|
+
server,
|
|
480
|
+
"session_start",
|
|
481
|
+
{
|
|
482
|
+
title: "Session Start",
|
|
483
|
+
description: `Start a new TestDriver session and provision a sandbox with browser or app.
|
|
484
|
+
|
|
485
|
+
⚠️ IMPORTANT - Test File Parameter:
|
|
486
|
+
When 'testFile' is provided, you MUST append the generated code to that file after EVERY successful action.
|
|
487
|
+
Each tool response includes "ACTION REQUIRED: Append this code..." - you must write that code to the test file immediately.
|
|
488
|
+
|
|
489
|
+
Provision types:
|
|
490
|
+
- chrome: Launch Chrome browser (default). Use 'url' for starting page.
|
|
491
|
+
- chromeExtension: Launch Chrome with an extension. Use 'extensionPath' or 'extensionId'.
|
|
492
|
+
- vscode: Launch VS Code. Use 'workspace' and optional 'extensions'.
|
|
493
|
+
- installer: Download and install an app. Use 'installerUrl' (required).
|
|
494
|
+
- electron: Launch an Electron app. Use 'appPath' (required).
|
|
495
|
+
|
|
496
|
+
Self-hosted mode:
|
|
497
|
+
- Provide 'ip' parameter to connect directly to a self-hosted Windows instance
|
|
498
|
+
- Set 'os' to 'windows' when connecting to Windows instances
|
|
499
|
+
- The IP can be from an AWS EC2 instance spawned via CloudFormation
|
|
500
|
+
- See https://docs.testdriver.ai/v7/aws-setup for AWS setup guide
|
|
501
|
+
|
|
502
|
+
Debug mode (connect to existing sandbox):
|
|
503
|
+
- Provide 'sandboxId' to connect to an existing sandbox (e.g., from a failed test with debugOnFailure: true)
|
|
504
|
+
- Skips provisioning - connects to sandbox in its current state
|
|
505
|
+
- Use this to interactively debug failed tests without re-running from scratch`,
|
|
506
|
+
inputSchema: SessionStartInputSchema as any,
|
|
507
|
+
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
508
|
+
},
|
|
509
|
+
async (params: SessionStartInput): Promise<CallToolResult> => {
|
|
510
|
+
const startTime = Date.now();
|
|
511
|
+
logger.info("session_start: Starting", {
|
|
512
|
+
type: params.type,
|
|
513
|
+
url: params.url,
|
|
514
|
+
os: params.os,
|
|
515
|
+
reconnect: params.reconnect,
|
|
516
|
+
sandboxId: params.sandboxId,
|
|
517
|
+
});
|
|
518
|
+
|
|
519
|
+
try {
|
|
520
|
+
// Validate required fields for specific provision types (unless connecting to existing sandbox)
|
|
521
|
+
if (!params.sandboxId) {
|
|
522
|
+
if (params.type === "installer" && !params.installerUrl) {
|
|
523
|
+
return createToolResult(false, "installer type requires 'installerUrl' parameter", { error: "Missing required parameter: installerUrl" });
|
|
524
|
+
}
|
|
525
|
+
if (params.type === "electron" && !params.appPath) {
|
|
526
|
+
return createToolResult(false, "electron type requires 'appPath' parameter", { error: "Missing required parameter: appPath" });
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
// Create new session
|
|
531
|
+
const newSession = sessionManager.createSession({
|
|
532
|
+
os: params.os,
|
|
533
|
+
keepAlive: params.keepAlive,
|
|
534
|
+
testFile: params.testFile,
|
|
535
|
+
});
|
|
536
|
+
logger.debug("session_start: Session created", { sessionId: newSession.sessionId });
|
|
537
|
+
|
|
538
|
+
// Determine API root
|
|
539
|
+
const apiRoot = params.apiRoot || process.env.TD_API_ROOT || "https://api.testdriver.ai";
|
|
540
|
+
logger.debug("session_start: Using API root", { apiRoot });
|
|
541
|
+
|
|
542
|
+
// Initialize SDK
|
|
543
|
+
logger.debug("session_start: Initializing SDK");
|
|
544
|
+
const TestDriverSDK = (await import("../../sdk.js")).default;
|
|
545
|
+
|
|
546
|
+
// Determine preview mode from environment variable
|
|
547
|
+
// TD_PREVIEW can be "ide", "browser", or "none"
|
|
548
|
+
// Default to "ide" so the live preview shows within the IDE
|
|
549
|
+
const previewMode = process.env.TD_PREVIEW || "ide";
|
|
550
|
+
logger.debug("session_start: Preview mode", { preview: previewMode });
|
|
551
|
+
|
|
552
|
+
// Get IP from params or environment (for self-hosted instances)
|
|
553
|
+
const instanceIp = params.ip || process.env.TD_IP;
|
|
554
|
+
|
|
555
|
+
// Get API key - check multiple sources for GitHub Copilot coding agent compatibility
|
|
556
|
+
// 1. TD_API_KEY (standard environment variable)
|
|
557
|
+
// 2. COPILOT_MCP_TD_API_KEY (fallback for GitHub Copilot coding agent)
|
|
558
|
+
const apiKey = process.env.TD_API_KEY || process.env.COPILOT_MCP_TD_API_KEY || "";
|
|
559
|
+
|
|
560
|
+
if (!apiKey) {
|
|
561
|
+
logger.error("session_start: No API key found", {
|
|
562
|
+
hasTD_API_KEY: !!process.env.TD_API_KEY,
|
|
563
|
+
hasCOPILOT_MCP_TD_API_KEY: !!process.env.COPILOT_MCP_TD_API_KEY,
|
|
564
|
+
availableEnvVars: Object.keys(process.env).filter(k => k.includes('TD') || k.includes('COPILOT_MCP'))
|
|
565
|
+
});
|
|
566
|
+
return createToolResult(false, "No API key found. Please set TD_API_KEY or COPILOT_MCP_TD_API_KEY environment variable.", {
|
|
567
|
+
error: "Missing API key",
|
|
568
|
+
hint: "For GitHub Copilot coding agent, create a Copilot environment secret named COPILOT_MCP_TD_API_KEY"
|
|
569
|
+
});
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
logger.debug("session_start: API key found", {
|
|
573
|
+
source: process.env.TD_API_KEY ? "TD_API_KEY" : "COPILOT_MCP_TD_API_KEY",
|
|
574
|
+
keyPrefix: apiKey.substring(0, 7) + "..."
|
|
575
|
+
});
|
|
576
|
+
|
|
577
|
+
sdk = new TestDriverSDK(apiKey, {
|
|
578
|
+
os: params.os,
|
|
579
|
+
logging: false,
|
|
580
|
+
apiRoot,
|
|
581
|
+
preview: previewMode as "browser" | "ide" | "none",
|
|
582
|
+
ip: instanceIp,
|
|
583
|
+
});
|
|
584
|
+
|
|
585
|
+
// Handle sandboxId mode - connect to existing sandbox (debug-on-failure mode)
|
|
586
|
+
if (params.sandboxId) {
|
|
587
|
+
logger.info("session_start: Connecting to existing sandbox (debug mode)", { sandboxId: params.sandboxId });
|
|
588
|
+
await sdk.connect({
|
|
589
|
+
sandboxId: params.sandboxId,
|
|
590
|
+
keepAlive: params.keepAlive,
|
|
591
|
+
});
|
|
592
|
+
|
|
593
|
+
// Get sandbox ID
|
|
594
|
+
const instance = sdk.getInstance();
|
|
595
|
+
logger.info("session_start: Connected to existing sandbox", { instanceId: instance?.instanceId });
|
|
596
|
+
sessionManager.activateSession(newSession.sessionId, instance?.instanceId || params.sandboxId);
|
|
597
|
+
|
|
598
|
+
// Set Sentry context for error tracking
|
|
599
|
+
setSessionContext(newSession.sessionId, instance?.instanceId);
|
|
600
|
+
|
|
601
|
+
// Capture screenshot of current state
|
|
602
|
+
logger.debug("session_start: Capturing screenshot of existing sandbox");
|
|
603
|
+
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
604
|
+
|
|
605
|
+
let screenshotResourceUri: string | undefined;
|
|
606
|
+
if (screenshotBase64) {
|
|
607
|
+
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
608
|
+
lastScreenshotBase64 = screenshotBase64;
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
const duration = Date.now() - startTime;
|
|
612
|
+
logger.info("session_start: Connected to existing sandbox", { duration, sessionId: newSession.sessionId, sandboxId: params.sandboxId });
|
|
613
|
+
|
|
614
|
+
return createToolResult(
|
|
615
|
+
true,
|
|
616
|
+
`Connected to existing sandbox (debug mode)
|
|
617
|
+
Session: ${newSession.sessionId}
|
|
618
|
+
Sandbox: ${params.sandboxId}
|
|
619
|
+
Expires in: ${Math.round(params.keepAlive / 1000)}s
|
|
620
|
+
|
|
621
|
+
You are now connected to the sandbox in its current state. Use find, click, type, etc. to interact.`,
|
|
622
|
+
{
|
|
623
|
+
action: "session_start",
|
|
624
|
+
sessionId: newSession.sessionId,
|
|
625
|
+
sandboxId: params.sandboxId,
|
|
626
|
+
debugMode: true,
|
|
627
|
+
screenshotResourceUri,
|
|
628
|
+
duration
|
|
629
|
+
},
|
|
630
|
+
"// Connected to existing sandbox - no provision code needed"
|
|
631
|
+
);
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
// Connect to sandbox
|
|
635
|
+
if (instanceIp) {
|
|
636
|
+
logger.info("session_start: Connecting to self-hosted instance...", { ip: instanceIp });
|
|
637
|
+
} else {
|
|
638
|
+
logger.info("session_start: Connecting to cloud sandbox...");
|
|
639
|
+
}
|
|
640
|
+
await sdk.connect({
|
|
641
|
+
reconnect: params.reconnect,
|
|
642
|
+
keepAlive: params.keepAlive,
|
|
643
|
+
ip: instanceIp,
|
|
644
|
+
});
|
|
645
|
+
|
|
646
|
+
// Get sandbox ID
|
|
647
|
+
const instance = sdk.getInstance();
|
|
648
|
+
logger.info("session_start: Connected to sandbox", { instanceId: instance?.instanceId });
|
|
649
|
+
sessionManager.activateSession(newSession.sessionId, instance?.instanceId || "unknown");
|
|
650
|
+
|
|
651
|
+
// Set Sentry context for error tracking
|
|
652
|
+
setSessionContext(newSession.sessionId, instance?.instanceId);
|
|
653
|
+
|
|
654
|
+
// Get provision-specific options
|
|
655
|
+
const provisionOptions = getProvisionOptions(params);
|
|
656
|
+
let provisionCmd = "";
|
|
657
|
+
|
|
658
|
+
// Provision based on type
|
|
659
|
+
switch (params.type) {
|
|
660
|
+
case "chrome": {
|
|
661
|
+
const chromeOpts = provisionOptions as { url: string; maximized?: boolean; guest?: boolean };
|
|
662
|
+
logger.info("session_start: Provisioning Chrome", { url: chromeOpts.url });
|
|
663
|
+
await sdk.provision.chrome(chromeOpts);
|
|
664
|
+
provisionCmd = "provision.chrome";
|
|
665
|
+
logger.debug("session_start: Chrome provisioned");
|
|
666
|
+
break;
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
case "chromeExtension": {
|
|
670
|
+
const extOpts = provisionOptions as { extensionPath?: string; extensionId?: string; maximized?: boolean };
|
|
671
|
+
logger.info("session_start: Provisioning Chrome Extension", { extensionPath: extOpts.extensionPath, extensionId: extOpts.extensionId });
|
|
672
|
+
await sdk.provision.chromeExtension(extOpts);
|
|
673
|
+
provisionCmd = "provision.chromeExtension";
|
|
674
|
+
logger.debug("session_start: Chrome Extension provisioned");
|
|
675
|
+
break;
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
case "vscode": {
|
|
679
|
+
const vscodeOpts = provisionOptions as { workspace?: string; extensions?: string[] };
|
|
680
|
+
logger.info("session_start: Provisioning VS Code", { workspace: vscodeOpts.workspace });
|
|
681
|
+
await sdk.provision.vscode(vscodeOpts);
|
|
682
|
+
provisionCmd = "provision.vscode";
|
|
683
|
+
logger.debug("session_start: VS Code provisioned");
|
|
684
|
+
break;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
case "installer": {
|
|
688
|
+
const installerOpts = provisionOptions as { url: string; filename?: string; appName?: string; launch?: boolean };
|
|
689
|
+
logger.info("session_start: Provisioning installer", { url: installerOpts.url });
|
|
690
|
+
await sdk.provision.installer(installerOpts);
|
|
691
|
+
provisionCmd = "provision.installer";
|
|
692
|
+
logger.debug("session_start: Installer provisioned");
|
|
693
|
+
break;
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
case "electron": {
|
|
697
|
+
const electronOpts = provisionOptions as { appPath: string; args?: string[] };
|
|
698
|
+
logger.info("session_start: Provisioning Electron", { appPath: electronOpts.appPath });
|
|
699
|
+
await sdk.provision.electron(electronOpts);
|
|
700
|
+
provisionCmd = "provision.electron";
|
|
701
|
+
logger.debug("session_start: Electron app provisioned");
|
|
702
|
+
break;
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
// Capture initial screenshot after provisioning
|
|
707
|
+
logger.debug("session_start: Capturing initial screenshot");
|
|
708
|
+
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
709
|
+
|
|
710
|
+
let screenshotResourceUri: string | undefined;
|
|
711
|
+
if (screenshotBase64) {
|
|
712
|
+
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
713
|
+
lastScreenshotBase64 = screenshotBase64;
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
const duration = Date.now() - startTime;
|
|
717
|
+
logger.info("session_start: Completed", { duration, sessionId: newSession.sessionId, selfHosted: !!instanceIp });
|
|
718
|
+
|
|
719
|
+
// Generate the code for this provision action
|
|
720
|
+
const generatedCode = generateActionCode(provisionCmd, provisionOptions);
|
|
721
|
+
|
|
722
|
+
// Build debugger URL for the session
|
|
723
|
+
const debuggerUrl = instance?.debuggerUrl || (instanceIp ? `http://${instanceIp}:9222` : null);
|
|
724
|
+
|
|
725
|
+
const connectionType = instanceIp ? `Self-hosted (${instanceIp})` : "Cloud";
|
|
726
|
+
return createToolResult(
|
|
727
|
+
true,
|
|
728
|
+
`Session started: ${newSession.sessionId}\nConnection: ${connectionType}\nType: ${params.type}\nSandbox: ${instance?.instanceId}\nExpires in: ${Math.round(params.keepAlive / 1000)}s
|
|
729
|
+
|
|
730
|
+
IMPORTANT - If creating a new test project, use these EXACT dependencies in package.json:
|
|
731
|
+
{
|
|
732
|
+
"type": "module",
|
|
733
|
+
"devDependencies": {
|
|
734
|
+
"testdriverai": "beta",
|
|
735
|
+
"vitest": "^4.0.0"
|
|
736
|
+
},
|
|
737
|
+
"scripts": {
|
|
738
|
+
"test": "vitest"
|
|
739
|
+
}
|
|
740
|
+
}`,
|
|
741
|
+
{
|
|
742
|
+
action: "session_start",
|
|
743
|
+
sessionId: newSession.sessionId,
|
|
744
|
+
provisionType: params.type,
|
|
745
|
+
selfHosted: !!instanceIp,
|
|
746
|
+
instanceIp: instanceIp || undefined,
|
|
747
|
+
debuggerUrl,
|
|
748
|
+
screenshotResourceUri,
|
|
749
|
+
duration
|
|
750
|
+
},
|
|
751
|
+
generatedCode
|
|
752
|
+
);
|
|
753
|
+
} catch (error) {
|
|
754
|
+
logger.error("session_start: Failed", { error: String(error) });
|
|
755
|
+
captureException(error as Error, { tags: { tool: "session_start" }, extra: { params } });
|
|
756
|
+
throw error;
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
);
|
|
760
|
+
|
|
761
|
+
// Session Status
|
|
762
|
+
server.registerTool(
|
|
763
|
+
"session_status",
|
|
764
|
+
{
|
|
765
|
+
description: "Check the current session status and time remaining",
|
|
766
|
+
inputSchema: z.object({}),
|
|
767
|
+
},
|
|
768
|
+
async (): Promise<CallToolResult> => {
|
|
769
|
+
const startTime = Date.now();
|
|
770
|
+
logger.info("session_status: Checking");
|
|
771
|
+
const session = sessionManager.getCurrentSession();
|
|
772
|
+
|
|
773
|
+
if (!session) {
|
|
774
|
+
logger.warn("session_status: No active session");
|
|
775
|
+
return createToolResult(false, "No active session", { error: "No active session. Call session_start first." });
|
|
776
|
+
}
|
|
777
|
+
|
|
778
|
+
const summary = sessionManager.getSessionSummary(session.sessionId);
|
|
779
|
+
const duration = Date.now() - startTime;
|
|
780
|
+
logger.info("session_status: Completed", {
|
|
781
|
+
sessionId: session.sessionId,
|
|
782
|
+
status: session.status,
|
|
783
|
+
timeRemaining: summary?.timeRemaining,
|
|
784
|
+
duration
|
|
785
|
+
});
|
|
786
|
+
|
|
787
|
+
return createToolResult(
|
|
788
|
+
true,
|
|
789
|
+
`Session: ${session.sessionId}\nStatus: ${session.status}\nTime remaining: ${Math.round((summary?.timeRemaining || 0) / 1000)}s`,
|
|
790
|
+
{ action: "session_status", ...summary, sessionId: session.sessionId, status: session.status, duration }
|
|
791
|
+
);
|
|
792
|
+
}
|
|
793
|
+
);
|
|
794
|
+
|
|
795
|
+
// Session Extend
|
|
796
|
+
server.registerTool(
|
|
797
|
+
"session_extend",
|
|
798
|
+
{
|
|
799
|
+
description: "Extend the session keepAlive time",
|
|
800
|
+
inputSchema: z.object({
|
|
801
|
+
additionalMs: z.number().default(60000).describe("Additional time in ms"),
|
|
802
|
+
}),
|
|
803
|
+
},
|
|
804
|
+
async (params) => {
|
|
805
|
+
logger.info("session_extend: Extending", { additionalMs: params.additionalMs });
|
|
806
|
+
const session = sessionManager.getCurrentSession();
|
|
807
|
+
|
|
808
|
+
if (!session) {
|
|
809
|
+
logger.warn("session_extend: No active session");
|
|
810
|
+
return { content: [{ type: "text" as const, text: "No active session" }] };
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
sessionManager.extendSession(session.sessionId, params.additionalMs);
|
|
814
|
+
const newExpiry = sessionManager.getTimeRemaining(session.sessionId);
|
|
815
|
+
logger.info("session_extend: Extended", { sessionId: session.sessionId, newExpiry });
|
|
816
|
+
|
|
817
|
+
return {
|
|
818
|
+
content: [
|
|
819
|
+
{
|
|
820
|
+
type: "text" as const,
|
|
821
|
+
text: `Session extended by ${params.additionalMs / 1000}s. New expiry: ${Math.round(newExpiry / 1000)}s`,
|
|
822
|
+
},
|
|
823
|
+
],
|
|
824
|
+
};
|
|
825
|
+
}
|
|
826
|
+
);
|
|
827
|
+
|
|
828
|
+
// Find Element
|
|
829
|
+
registerAppTool(
|
|
830
|
+
server,
|
|
831
|
+
"find",
|
|
832
|
+
{
|
|
833
|
+
title: "Find Element",
|
|
834
|
+
description: "Find an element on screen by natural language description",
|
|
835
|
+
inputSchema: z.object({
|
|
836
|
+
description: z.string().describe("Natural language description of the element"),
|
|
837
|
+
timeout: z.number().optional().describe("Timeout in ms for polling"),
|
|
838
|
+
}) as any,
|
|
839
|
+
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
840
|
+
},
|
|
841
|
+
async (params: { description: string; timeout?: number }): Promise<CallToolResult> => {
|
|
842
|
+
const startTime = Date.now();
|
|
843
|
+
logger.info("find: Starting", { description: params.description, timeout: params.timeout });
|
|
844
|
+
|
|
845
|
+
const sessionCheck = requireActiveSession();
|
|
846
|
+
if (!sessionCheck.valid) {
|
|
847
|
+
logger.warn("find: No active session");
|
|
848
|
+
return sessionCheck.error;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
try {
|
|
852
|
+
logger.debug("find: Calling SDK find");
|
|
853
|
+
const element = await sdk.find(params.description, params.timeout ? { timeout: params.timeout } : undefined);
|
|
854
|
+
const found = element.found();
|
|
855
|
+
const coords = element.getCoordinates();
|
|
856
|
+
|
|
857
|
+
// Store element ref for later use (stores actual Element instance)
|
|
858
|
+
const elementRef = `el-${Date.now()}`;
|
|
859
|
+
if (found && coords) {
|
|
860
|
+
elementRefs.set(elementRef, {
|
|
861
|
+
element: element, // Store the actual Element instance
|
|
862
|
+
description: params.description,
|
|
863
|
+
coords: {
|
|
864
|
+
x: coords.x,
|
|
865
|
+
y: coords.y,
|
|
866
|
+
centerX: coords.centerX,
|
|
867
|
+
centerY: coords.centerY,
|
|
868
|
+
},
|
|
869
|
+
});
|
|
870
|
+
logger.info("find: Element found", {
|
|
871
|
+
description: params.description,
|
|
872
|
+
coords: { x: coords.centerX, y: coords.centerY },
|
|
873
|
+
confidence: element.confidence,
|
|
874
|
+
elementRef
|
|
875
|
+
});
|
|
876
|
+
} else {
|
|
877
|
+
logger.warn("find: Element not found", { description: params.description });
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
// Return raw SDK response directly
|
|
881
|
+
const rawResponse = element._response || {};
|
|
882
|
+
const duration = Date.now() - startTime;
|
|
883
|
+
|
|
884
|
+
// Store cropped image for resource serving (instead of inline data URL)
|
|
885
|
+
let croppedImageResourceUri: string | undefined;
|
|
886
|
+
let screenshotResourceUri: string | undefined;
|
|
887
|
+
const croppedImage = rawResponse.croppedImage;
|
|
888
|
+
if (croppedImage) {
|
|
889
|
+
const imageData = croppedImage.startsWith('data:')
|
|
890
|
+
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
891
|
+
: croppedImage;
|
|
892
|
+
croppedImageResourceUri = storeImage(imageData, "cropped");
|
|
893
|
+
// Remove croppedImage from response to avoid context bloat
|
|
894
|
+
delete rawResponse.croppedImage;
|
|
895
|
+
} else if (!found) {
|
|
896
|
+
// Element not found and no cropped image - capture a fresh screenshot
|
|
897
|
+
// so the user can see what's currently visible on screen
|
|
898
|
+
try {
|
|
899
|
+
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
900
|
+
if (screenshotBase64) {
|
|
901
|
+
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
902
|
+
logger.debug("find: Captured screenshot for not-found state");
|
|
903
|
+
}
|
|
904
|
+
} catch (e) {
|
|
905
|
+
logger.warn("find: Failed to capture screenshot for not-found state", { error: String(e) });
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
910
|
+
delete rawResponse.extractedText;
|
|
911
|
+
delete rawResponse.pixelDiffImage;
|
|
912
|
+
|
|
913
|
+
// Generate code for this find action
|
|
914
|
+
const generatedCode = found ? generateActionCode("find", { description: params.description }) : undefined;
|
|
915
|
+
|
|
916
|
+
// Build element info for display (cropped image is always centered on element)
|
|
917
|
+
const elementInfo = found ? {
|
|
918
|
+
description: params.description,
|
|
919
|
+
centerX: coords?.centerX,
|
|
920
|
+
centerY: coords?.centerY,
|
|
921
|
+
confidence: element.confidence,
|
|
922
|
+
ref: elementRef,
|
|
923
|
+
} : undefined;
|
|
924
|
+
|
|
925
|
+
return createToolResult(
|
|
926
|
+
found,
|
|
927
|
+
found
|
|
928
|
+
? `Found: "${params.description}" at (${rawResponse.coordinates?.x}, ${rawResponse.coordinates?.y})\nRef: ${elementRef}`
|
|
929
|
+
: `Element not found: "${params.description}"`,
|
|
930
|
+
{
|
|
931
|
+
...rawResponse,
|
|
932
|
+
action: "find",
|
|
933
|
+
element: elementInfo,
|
|
934
|
+
ref: elementRef,
|
|
935
|
+
croppedImageResourceUri,
|
|
936
|
+
screenshotResourceUri,
|
|
937
|
+
duration,
|
|
938
|
+
},
|
|
939
|
+
generatedCode
|
|
940
|
+
);
|
|
941
|
+
} catch (error) {
|
|
942
|
+
logger.error("find: Failed", { error: String(error), description: params.description });
|
|
943
|
+
captureException(error as Error, { tags: { tool: "find" }, extra: { description: params.description } });
|
|
944
|
+
throw error;
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
);
|
|
948
|
+
|
|
949
|
+
// Find All Elements
|
|
950
|
+
registerAppTool(
|
|
951
|
+
server,
|
|
952
|
+
"findall",
|
|
953
|
+
{
|
|
954
|
+
title: "Find All Elements",
|
|
955
|
+
description: "Find all elements on screen matching a natural language description. Returns an array of element references.",
|
|
956
|
+
inputSchema: z.object({
|
|
957
|
+
description: z.string().describe("Natural language description of the elements to find"),
|
|
958
|
+
timeout: z.number().optional().describe("Timeout in ms for polling"),
|
|
959
|
+
}) as any,
|
|
960
|
+
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
961
|
+
},
|
|
962
|
+
async (params: { description: string; timeout?: number }): Promise<CallToolResult> => {
|
|
963
|
+
const startTime = Date.now();
|
|
964
|
+
logger.info("findall: Starting", { description: params.description, timeout: params.timeout });
|
|
965
|
+
|
|
966
|
+
const sessionCheck = requireActiveSession();
|
|
967
|
+
if (!sessionCheck.valid) {
|
|
968
|
+
logger.warn("findall: No active session");
|
|
969
|
+
return sessionCheck.error;
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
try {
|
|
973
|
+
logger.debug("findall: Calling SDK findAll");
|
|
974
|
+
const elements = await sdk.findAll(params.description, params.timeout ? { timeout: params.timeout } : undefined);
|
|
975
|
+
const count = elements.length;
|
|
976
|
+
|
|
977
|
+
// Store element refs for later use
|
|
978
|
+
const refs: string[] = [];
|
|
979
|
+
const elementInfos: Array<{ ref: string; x: number; y: number; centerX: number; centerY: number; confidence: number }> = [];
|
|
980
|
+
|
|
981
|
+
for (let i = 0; i < elements.length; i++) {
|
|
982
|
+
const element = elements[i];
|
|
983
|
+
const coords = element.getCoordinates();
|
|
984
|
+
const elementRef = `el-${Date.now()}-${i}`;
|
|
985
|
+
|
|
986
|
+
if (coords) {
|
|
987
|
+
elementRefs.set(elementRef, {
|
|
988
|
+
element: element,
|
|
989
|
+
description: `${params.description} [${i}]`,
|
|
990
|
+
coords: {
|
|
991
|
+
x: coords.x,
|
|
992
|
+
y: coords.y,
|
|
993
|
+
centerX: coords.centerX,
|
|
994
|
+
centerY: coords.centerY,
|
|
995
|
+
},
|
|
996
|
+
});
|
|
997
|
+
refs.push(elementRef);
|
|
998
|
+
elementInfos.push({
|
|
999
|
+
ref: elementRef,
|
|
1000
|
+
x: coords.x,
|
|
1001
|
+
y: coords.y,
|
|
1002
|
+
centerX: coords.centerX,
|
|
1003
|
+
centerY: coords.centerY,
|
|
1004
|
+
confidence: element.confidence,
|
|
1005
|
+
});
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
logger.info("findall: Elements found", {
|
|
1010
|
+
description: params.description,
|
|
1011
|
+
count,
|
|
1012
|
+
refs
|
|
1013
|
+
});
|
|
1014
|
+
|
|
1015
|
+
// Get the first element's response for the image (shows all highlights)
|
|
1016
|
+
const rawResponse = elements[0]?._response || {};
|
|
1017
|
+
const duration = Date.now() - startTime;
|
|
1018
|
+
|
|
1019
|
+
// Store cropped image for resource serving (instead of inline data URL)
|
|
1020
|
+
let croppedImageResourceUri: string | undefined;
|
|
1021
|
+
let screenshotResourceUri: string | undefined;
|
|
1022
|
+
const croppedImage = rawResponse.croppedImage;
|
|
1023
|
+
if (croppedImage) {
|
|
1024
|
+
const imageData = croppedImage.startsWith('data:')
|
|
1025
|
+
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
1026
|
+
: croppedImage;
|
|
1027
|
+
croppedImageResourceUri = storeImage(imageData, "cropped");
|
|
1028
|
+
// Remove croppedImage from response to avoid context bloat
|
|
1029
|
+
delete rawResponse.croppedImage;
|
|
1030
|
+
} else if (count === 0) {
|
|
1031
|
+
// No elements found and no cropped image - capture a fresh screenshot
|
|
1032
|
+
// so the user can see what's currently visible on screen
|
|
1033
|
+
try {
|
|
1034
|
+
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1035
|
+
if (screenshotBase64) {
|
|
1036
|
+
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1037
|
+
logger.debug("findall: Captured screenshot for not-found state");
|
|
1038
|
+
}
|
|
1039
|
+
} catch (e) {
|
|
1040
|
+
logger.warn("findall: Failed to capture screenshot for not-found state", { error: String(e) });
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
1045
|
+
delete rawResponse.extractedText;
|
|
1046
|
+
delete rawResponse.pixelDiffImage;
|
|
1047
|
+
|
|
1048
|
+
// Generate code for this findall action
|
|
1049
|
+
const generatedCode = count > 0 ? generateActionCode("findall", { description: params.description }) : undefined;
|
|
1050
|
+
|
|
1051
|
+
// Build refs list for text output
|
|
1052
|
+
const refsList = refs.map((ref, i) => ` [${i}] ${ref}`).join('\n');
|
|
1053
|
+
|
|
1054
|
+
return createToolResult(
|
|
1055
|
+
count > 0,
|
|
1056
|
+
count > 0
|
|
1057
|
+
? `Found ${count} elements matching "${params.description}":\n${refsList}`
|
|
1058
|
+
: `No elements found matching: "${params.description}"`,
|
|
1059
|
+
{
|
|
1060
|
+
...rawResponse,
|
|
1061
|
+
count,
|
|
1062
|
+
refs,
|
|
1063
|
+
elements: elementInfos,
|
|
1064
|
+
croppedImageResourceUri,
|
|
1065
|
+
screenshotResourceUri,
|
|
1066
|
+
duration,
|
|
1067
|
+
},
|
|
1068
|
+
generatedCode
|
|
1069
|
+
);
|
|
1070
|
+
} catch (error) {
|
|
1071
|
+
logger.error("findall: Failed", { error: String(error), description: params.description });
|
|
1072
|
+
captureException(error as Error, { tags: { tool: "findall" }, extra: { description: params.description } });
|
|
1073
|
+
throw error;
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
);
|
|
1077
|
+
|
|
1078
|
+
// Click
|
|
1079
|
+
registerAppTool(
|
|
1080
|
+
server,
|
|
1081
|
+
"click",
|
|
1082
|
+
{
|
|
1083
|
+
title: "Click Element",
|
|
1084
|
+
description: "Click on a previously found element. Use 'find' first to locate the element.",
|
|
1085
|
+
inputSchema: z.object({
|
|
1086
|
+
elementRef: z.string().describe("Reference to previously found element (required). Get this from a 'find' call."),
|
|
1087
|
+
action: z.enum(["click", "double-click", "right-click"]).default("click"),
|
|
1088
|
+
}) as any,
|
|
1089
|
+
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
1090
|
+
},
|
|
1091
|
+
async (params: { elementRef: string; action: "click" | "double-click" | "right-click" }): Promise<CallToolResult> => {
|
|
1092
|
+
const startTime = Date.now();
|
|
1093
|
+
logger.info("click: Starting", { elementRef: params.elementRef, action: params.action });
|
|
1094
|
+
|
|
1095
|
+
const sessionCheck = requireActiveSession();
|
|
1096
|
+
if (!sessionCheck.valid) {
|
|
1097
|
+
logger.warn("click: No active session");
|
|
1098
|
+
return sessionCheck.error;
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
// Look up the element reference
|
|
1102
|
+
const ref = elementRefs.get(params.elementRef);
|
|
1103
|
+
if (!ref) {
|
|
1104
|
+
logger.warn("click: Element reference not found", { elementRef: params.elementRef });
|
|
1105
|
+
return createToolResult(false, `Element reference "${params.elementRef}" not found. Use 'find' first to locate the element.`, { error: "Element reference not found" });
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
const { element, description, coords } = ref;
|
|
1109
|
+
|
|
1110
|
+
try {
|
|
1111
|
+
logger.debug("click: Executing click on element", { description, action: params.action });
|
|
1112
|
+
|
|
1113
|
+
// Use the Element's click method instead of raw coordinates
|
|
1114
|
+
if (params.action === "click") {
|
|
1115
|
+
await element.click();
|
|
1116
|
+
} else if (params.action === "double-click") {
|
|
1117
|
+
await element.doubleClick();
|
|
1118
|
+
} else if (params.action === "right-click") {
|
|
1119
|
+
await element.rightClick();
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
// Capture screenshot after click to show result
|
|
1123
|
+
logger.debug("click: Capturing screenshot after click");
|
|
1124
|
+
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1125
|
+
|
|
1126
|
+
let screenshotResourceUri: string | undefined;
|
|
1127
|
+
if (screenshotBase64) {
|
|
1128
|
+
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1129
|
+
lastScreenshotBase64 = screenshotBase64;
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
const rawResponse = element._response || {};
|
|
1133
|
+
// Remove large data from response to reduce context bloat
|
|
1134
|
+
delete rawResponse.croppedImage;
|
|
1135
|
+
delete rawResponse.extractedText;
|
|
1136
|
+
delete rawResponse.pixelDiffImage;
|
|
1137
|
+
|
|
1138
|
+
const duration = Date.now() - startTime;
|
|
1139
|
+
logger.info("click: Completed", { description, duration });
|
|
1140
|
+
|
|
1141
|
+
// Generate code for this click action
|
|
1142
|
+
const generatedCode = generateActionCode("click", { action: params.action });
|
|
1143
|
+
|
|
1144
|
+
return createToolResult(
|
|
1145
|
+
true,
|
|
1146
|
+
`Clicked on "${description}"`,
|
|
1147
|
+
{
|
|
1148
|
+
...rawResponse,
|
|
1149
|
+
action: "click",
|
|
1150
|
+
clickAction: params.action,
|
|
1151
|
+
clickPosition: coords,
|
|
1152
|
+
screenshotResourceUri,
|
|
1153
|
+
duration
|
|
1154
|
+
},
|
|
1155
|
+
generatedCode
|
|
1156
|
+
);
|
|
1157
|
+
} catch (error) {
|
|
1158
|
+
logger.error("click: Failed", { error: String(error), description });
|
|
1159
|
+
captureException(error as Error, { tags: { tool: "click" }, extra: { elementRef: params.elementRef, action: params.action } });
|
|
1160
|
+
throw error;
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
);
|
|
1164
|
+
|
|
1165
|
+
// Hover
|
|
1166
|
+
registerAppTool(
|
|
1167
|
+
server,
|
|
1168
|
+
"hover",
|
|
1169
|
+
{
|
|
1170
|
+
title: "Hover Element",
|
|
1171
|
+
description: "Hover over a previously found element. Use 'find' first to locate the element.",
|
|
1172
|
+
inputSchema: z.object({
|
|
1173
|
+
elementRef: z.string().describe("Reference to previously found element (required). Get this from a 'find' call."),
|
|
1174
|
+
}) as any,
|
|
1175
|
+
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
1176
|
+
},
|
|
1177
|
+
async (params: { elementRef: string }): Promise<CallToolResult> => {
|
|
1178
|
+
const startTime = Date.now();
|
|
1179
|
+
logger.info("hover: Starting", { elementRef: params.elementRef });
|
|
1180
|
+
|
|
1181
|
+
const sessionCheck = requireActiveSession();
|
|
1182
|
+
if (!sessionCheck.valid) {
|
|
1183
|
+
logger.warn("hover: No active session");
|
|
1184
|
+
return sessionCheck.error;
|
|
1185
|
+
}
|
|
1186
|
+
|
|
1187
|
+
// Look up the element reference
|
|
1188
|
+
const ref = elementRefs.get(params.elementRef);
|
|
1189
|
+
if (!ref) {
|
|
1190
|
+
logger.warn("hover: Element reference not found", { elementRef: params.elementRef });
|
|
1191
|
+
return createToolResult(false, `Element reference "${params.elementRef}" not found. Use 'find' first to locate the element.`, { error: "Element reference not found" });
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
const { element, description, coords } = ref;
|
|
1195
|
+
|
|
1196
|
+
try {
|
|
1197
|
+
logger.debug("hover: Executing hover on element", { description });
|
|
1198
|
+
await element.hover();
|
|
1199
|
+
|
|
1200
|
+
// Capture screenshot after hover to show result
|
|
1201
|
+
logger.debug("hover: Capturing screenshot after hover");
|
|
1202
|
+
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1203
|
+
|
|
1204
|
+
let screenshotResourceUri: string | undefined;
|
|
1205
|
+
if (screenshotBase64) {
|
|
1206
|
+
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1207
|
+
lastScreenshotBase64 = screenshotBase64;
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
const rawResponse = element._response || {};
|
|
1211
|
+
// Remove large data from response to reduce context bloat
|
|
1212
|
+
delete rawResponse.croppedImage;
|
|
1213
|
+
delete rawResponse.extractedText;
|
|
1214
|
+
delete rawResponse.pixelDiffImage;
|
|
1215
|
+
|
|
1216
|
+
const duration = Date.now() - startTime;
|
|
1217
|
+
logger.info("hover: Completed", { description, duration });
|
|
1218
|
+
|
|
1219
|
+
// Generate code for this hover action
|
|
1220
|
+
const generatedCode = generateActionCode("hover", {});
|
|
1221
|
+
|
|
1222
|
+
return createToolResult(
|
|
1223
|
+
true,
|
|
1224
|
+
`Hovered over "${description}"`,
|
|
1225
|
+
{
|
|
1226
|
+
...rawResponse,
|
|
1227
|
+
action: "hover",
|
|
1228
|
+
screenshotResourceUri,
|
|
1229
|
+
duration
|
|
1230
|
+
},
|
|
1231
|
+
generatedCode
|
|
1232
|
+
);
|
|
1233
|
+
} catch (error) {
|
|
1234
|
+
logger.error("hover: Failed", { error: String(error), description });
|
|
1235
|
+
captureException(error as Error, { tags: { tool: "hover" }, extra: { elementRef: params.elementRef } });
|
|
1236
|
+
throw error;
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
);
|
|
1240
|
+
|
|
1241
|
+
// Wait
|
|
1242
|
+
server.registerTool(
|
|
1243
|
+
"wait",
|
|
1244
|
+
{
|
|
1245
|
+
description: "Wait for a specified amount of time",
|
|
1246
|
+
inputSchema: z.object({
|
|
1247
|
+
timeout: z.number().default(3000).describe("Time to wait in milliseconds (default: 3000)"),
|
|
1248
|
+
}),
|
|
1249
|
+
},
|
|
1250
|
+
async (params): Promise<CallToolResult> => {
|
|
1251
|
+
const startTime = Date.now();
|
|
1252
|
+
logger.info("wait: Starting", { timeout: params.timeout });
|
|
1253
|
+
|
|
1254
|
+
const sessionCheck = requireActiveSession();
|
|
1255
|
+
if (!sessionCheck.valid) {
|
|
1256
|
+
logger.warn("wait: No active session");
|
|
1257
|
+
return sessionCheck.error;
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
try {
|
|
1261
|
+
logger.debug("wait: Waiting", { timeout: params.timeout });
|
|
1262
|
+
await sdk.wait(params.timeout);
|
|
1263
|
+
|
|
1264
|
+
const duration = Date.now() - startTime;
|
|
1265
|
+
logger.info("wait: Completed", { timeout: params.timeout, duration });
|
|
1266
|
+
|
|
1267
|
+
// Generate code for this wait action
|
|
1268
|
+
const generatedCode = generateActionCode("wait", { timeout: params.timeout });
|
|
1269
|
+
|
|
1270
|
+
return createToolResult(
|
|
1271
|
+
true,
|
|
1272
|
+
`Waited for ${params.timeout}ms`,
|
|
1273
|
+
{ action: "wait", timeout: params.timeout, duration },
|
|
1274
|
+
generatedCode
|
|
1275
|
+
);
|
|
1276
|
+
} catch (error) {
|
|
1277
|
+
logger.error("wait: Failed", { error: String(error) });
|
|
1278
|
+
captureException(error as Error, { tags: { tool: "wait" }, extra: { timeout: params.timeout } });
|
|
1279
|
+
throw error;
|
|
1280
|
+
}
|
|
1281
|
+
}
|
|
1282
|
+
);
|
|
1283
|
+
|
|
1284
|
+
// Focus Application
|
|
1285
|
+
server.registerTool(
|
|
1286
|
+
"focus_application",
|
|
1287
|
+
{
|
|
1288
|
+
description: "Bring an application window to the foreground",
|
|
1289
|
+
inputSchema: z.object({
|
|
1290
|
+
name: z.string().describe("Name of the application to focus (e.g., 'Google Chrome', 'Visual Studio Code')"),
|
|
1291
|
+
}),
|
|
1292
|
+
},
|
|
1293
|
+
async (params): Promise<CallToolResult> => {
|
|
1294
|
+
const startTime = Date.now();
|
|
1295
|
+
logger.info("focus_application: Starting", { name: params.name });
|
|
1296
|
+
|
|
1297
|
+
const sessionCheck = requireActiveSession();
|
|
1298
|
+
if (!sessionCheck.valid) {
|
|
1299
|
+
logger.warn("focus_application: No active session");
|
|
1300
|
+
return sessionCheck.error;
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
try {
|
|
1304
|
+
logger.debug("focus_application: Focusing", { name: params.name });
|
|
1305
|
+
await sdk.focusApplication(params.name);
|
|
1306
|
+
|
|
1307
|
+
const duration = Date.now() - startTime;
|
|
1308
|
+
logger.info("focus_application: Completed", { name: params.name, duration });
|
|
1309
|
+
|
|
1310
|
+
// Generate code for this focus action
|
|
1311
|
+
const generatedCode = generateActionCode("focus_application", { name: params.name });
|
|
1312
|
+
|
|
1313
|
+
return createToolResult(
|
|
1314
|
+
true,
|
|
1315
|
+
`Focused application: "${params.name}"`,
|
|
1316
|
+
{ action: "focus", name: params.name, duration },
|
|
1317
|
+
generatedCode
|
|
1318
|
+
);
|
|
1319
|
+
} catch (error) {
|
|
1320
|
+
logger.error("focus_application: Failed", { error: String(error), name: params.name });
|
|
1321
|
+
captureException(error as Error, { tags: { tool: "focus_application" }, extra: { name: params.name } });
|
|
1322
|
+
throw error;
|
|
1323
|
+
}
|
|
1324
|
+
}
|
|
1325
|
+
);
|
|
1326
|
+
|
|
1327
|
+
// Find and Click
|
|
1328
|
+
registerAppTool(
|
|
1329
|
+
server,
|
|
1330
|
+
"find_and_click",
|
|
1331
|
+
{
|
|
1332
|
+
title: "Find and Click",
|
|
1333
|
+
description: "Find an element and click it in one action",
|
|
1334
|
+
inputSchema: z.object({
|
|
1335
|
+
description: z.string().describe("Natural language description of element"),
|
|
1336
|
+
action: z.enum(["click", "double-click", "right-click"]).default("click"),
|
|
1337
|
+
}) as any,
|
|
1338
|
+
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
1339
|
+
},
|
|
1340
|
+
async (params: { description: string; action: "click" | "double-click" | "right-click" }): Promise<CallToolResult> => {
|
|
1341
|
+
const startTime = Date.now();
|
|
1342
|
+
logger.info("find_and_click: Starting", { description: params.description, action: params.action });
|
|
1343
|
+
|
|
1344
|
+
const sessionCheck = requireActiveSession();
|
|
1345
|
+
if (!sessionCheck.valid) {
|
|
1346
|
+
logger.warn("find_and_click: No active session");
|
|
1347
|
+
return sessionCheck.error;
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
try {
|
|
1351
|
+
logger.debug("find_and_click: Finding element");
|
|
1352
|
+
const element = await sdk.find(params.description);
|
|
1353
|
+
const found = element.found();
|
|
1354
|
+
|
|
1355
|
+
if (!found) {
|
|
1356
|
+
logger.warn("find_and_click: Element not found", { description: params.description });
|
|
1357
|
+
|
|
1358
|
+
// Capture screenshot to show current state even when element not found
|
|
1359
|
+
const rawResponse = element._response || {};
|
|
1360
|
+
const duration = Date.now() - startTime;
|
|
1361
|
+
|
|
1362
|
+
// Store cropped image (screenshot) for resource serving
|
|
1363
|
+
let croppedImageResourceUri: string | undefined;
|
|
1364
|
+
let screenshotResourceUri: string | undefined;
|
|
1365
|
+
const croppedImage = rawResponse.croppedImage;
|
|
1366
|
+
if (croppedImage) {
|
|
1367
|
+
const imageData = croppedImage.startsWith('data:')
|
|
1368
|
+
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
1369
|
+
: croppedImage;
|
|
1370
|
+
croppedImageResourceUri = storeImage(imageData, "screenshot");
|
|
1371
|
+
delete rawResponse.croppedImage;
|
|
1372
|
+
} else {
|
|
1373
|
+
// No cropped image - capture a fresh screenshot so the user can see
|
|
1374
|
+
// what's currently visible on screen when element was not found
|
|
1375
|
+
try {
|
|
1376
|
+
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1377
|
+
if (screenshotBase64) {
|
|
1378
|
+
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
1379
|
+
logger.debug("find_and_click: Captured screenshot for not-found state");
|
|
1380
|
+
}
|
|
1381
|
+
} catch (e) {
|
|
1382
|
+
logger.warn("find_and_click: Failed to capture screenshot for not-found state", { error: String(e) });
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1385
|
+
|
|
1386
|
+
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
1387
|
+
delete rawResponse.extractedText;
|
|
1388
|
+
delete rawResponse.pixelDiffImage;
|
|
1389
|
+
|
|
1390
|
+
return createToolResult(
|
|
1391
|
+
false,
|
|
1392
|
+
`Element not found: "${params.description}"`,
|
|
1393
|
+
{
|
|
1394
|
+
...rawResponse,
|
|
1395
|
+
action: "find_and_click",
|
|
1396
|
+
error: "Element not found",
|
|
1397
|
+
croppedImageResourceUri,
|
|
1398
|
+
screenshotResourceUri,
|
|
1399
|
+
duration
|
|
1400
|
+
}
|
|
1401
|
+
);
|
|
1402
|
+
}
|
|
1403
|
+
|
|
1404
|
+
const coords = element.getCoordinates();
|
|
1405
|
+
|
|
1406
|
+
// Store element ref for later use (in case user wants to interact again)
|
|
1407
|
+
const elementRef = `el-${Date.now()}`;
|
|
1408
|
+
if (coords) {
|
|
1409
|
+
elementRefs.set(elementRef, {
|
|
1410
|
+
element: element,
|
|
1411
|
+
description: params.description,
|
|
1412
|
+
coords: {
|
|
1413
|
+
x: coords.x,
|
|
1414
|
+
y: coords.y,
|
|
1415
|
+
centerX: coords.centerX,
|
|
1416
|
+
centerY: coords.centerY,
|
|
1417
|
+
},
|
|
1418
|
+
});
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1421
|
+
logger.debug("find_and_click: Element found, clicking", { action: params.action, elementRef });
|
|
1422
|
+
if (params.action === "click") {
|
|
1423
|
+
await element.click();
|
|
1424
|
+
} else if (params.action === "double-click") {
|
|
1425
|
+
await element.doubleClick();
|
|
1426
|
+
} else if (params.action === "right-click") {
|
|
1427
|
+
await element.rightClick();
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
// Return raw SDK response directly
|
|
1431
|
+
const rawResponse = element._response || {};
|
|
1432
|
+
const duration = Date.now() - startTime;
|
|
1433
|
+
|
|
1434
|
+
// Store cropped image for resource serving (instead of inline data URL)
|
|
1435
|
+
let croppedImageResourceUri: string | undefined;
|
|
1436
|
+
const croppedImage = rawResponse.croppedImage;
|
|
1437
|
+
if (croppedImage) {
|
|
1438
|
+
const imageData = croppedImage.startsWith('data:')
|
|
1439
|
+
? croppedImage.replace(/^data:image\/\w+;base64,/, '')
|
|
1440
|
+
: croppedImage;
|
|
1441
|
+
croppedImageResourceUri = storeImage(imageData, "cropped");
|
|
1442
|
+
// Remove croppedImage from response to avoid context bloat
|
|
1443
|
+
delete rawResponse.croppedImage;
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
// Remove extractedText and pixelDiffImage from response to reduce context bloat
|
|
1447
|
+
delete rawResponse.extractedText;
|
|
1448
|
+
delete rawResponse.pixelDiffImage;
|
|
1449
|
+
|
|
1450
|
+
// Generate code for this find_and_click action
|
|
1451
|
+
const generatedCode = generateActionCode("find_and_click", { description: params.description, action: params.action });
|
|
1452
|
+
|
|
1453
|
+
// Build element info for display (match find action format)
|
|
1454
|
+
const elementInfo = coords ? {
|
|
1455
|
+
description: params.description,
|
|
1456
|
+
centerX: coords.centerX,
|
|
1457
|
+
centerY: coords.centerY,
|
|
1458
|
+
confidence: element.confidence,
|
|
1459
|
+
ref: elementRef,
|
|
1460
|
+
} : undefined;
|
|
1461
|
+
|
|
1462
|
+
return createToolResult(
|
|
1463
|
+
true,
|
|
1464
|
+
`Found and clicked: "${params.description}" at (${rawResponse.coordinates?.x}, ${rawResponse.coordinates?.y})\nRef: ${elementRef}`,
|
|
1465
|
+
{
|
|
1466
|
+
...rawResponse,
|
|
1467
|
+
action: "find_and_click",
|
|
1468
|
+
element: elementInfo,
|
|
1469
|
+
ref: elementRef,
|
|
1470
|
+
clickAction: params.action,
|
|
1471
|
+
clickPosition: coords ? { x: coords.centerX, y: coords.centerY } : undefined,
|
|
1472
|
+
croppedImageResourceUri,
|
|
1473
|
+
duration,
|
|
1474
|
+
},
|
|
1475
|
+
generatedCode
|
|
1476
|
+
);
|
|
1477
|
+
} catch (error) {
|
|
1478
|
+
logger.error("find_and_click: Failed", { error: String(error), description: params.description });
|
|
1479
|
+
captureException(error as Error, { tags: { tool: "find_and_click" }, extra: { description: params.description, action: params.action } });
|
|
1480
|
+
throw error;
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
);
|
|
1484
|
+
|
|
1485
|
+
// Type
|
|
1486
|
+
server.registerTool(
|
|
1487
|
+
"type",
|
|
1488
|
+
{
|
|
1489
|
+
description: "Type text into the currently focused field",
|
|
1490
|
+
inputSchema: z.object({
|
|
1491
|
+
text: z.string().describe("Text to type"),
|
|
1492
|
+
secret: z.boolean().default(false).describe("Whether this is sensitive data"),
|
|
1493
|
+
delay: z.number().optional().describe("Delay between keystrokes in ms"),
|
|
1494
|
+
}),
|
|
1495
|
+
},
|
|
1496
|
+
async (params): Promise<CallToolResult> => {
|
|
1497
|
+
const startTime = Date.now();
|
|
1498
|
+
logger.info("type: Starting", { textLength: params.text.length, secret: params.secret });
|
|
1499
|
+
|
|
1500
|
+
const sessionCheck = requireActiveSession();
|
|
1501
|
+
if (!sessionCheck.valid) {
|
|
1502
|
+
logger.warn("type: No active session");
|
|
1503
|
+
return sessionCheck.error;
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1506
|
+
try {
|
|
1507
|
+
logger.debug("type: Typing text");
|
|
1508
|
+
await sdk.type(params.text, { secret: params.secret, delay: params.delay });
|
|
1509
|
+
|
|
1510
|
+
const duration = Date.now() - startTime;
|
|
1511
|
+
logger.info("type: Completed", { duration });
|
|
1512
|
+
|
|
1513
|
+
// Generate code for this type action
|
|
1514
|
+
const generatedCode = generateActionCode("type", { text: params.text, secret: params.secret });
|
|
1515
|
+
|
|
1516
|
+
return createToolResult(
|
|
1517
|
+
true,
|
|
1518
|
+
`Typed: ${params.secret ? "[secret text]" : `"${params.text}"`}`,
|
|
1519
|
+
{ action: "type", text: params.secret ? "[SECRET]" : params.text, duration },
|
|
1520
|
+
generatedCode
|
|
1521
|
+
);
|
|
1522
|
+
} catch (error) {
|
|
1523
|
+
logger.error("type: Failed", { error: String(error) });
|
|
1524
|
+
captureException(error as Error, { tags: { tool: "type" }, extra: { textLength: params.text.length, secret: params.secret } });
|
|
1525
|
+
throw error;
|
|
1526
|
+
}
|
|
1527
|
+
}
|
|
1528
|
+
);
|
|
1529
|
+
|
|
1530
|
+
// Press Keys
|
|
1531
|
+
server.registerTool(
|
|
1532
|
+
"press_keys",
|
|
1533
|
+
{
|
|
1534
|
+
description: "Press keyboard keys or shortcuts",
|
|
1535
|
+
inputSchema: z.object({
|
|
1536
|
+
keys: z.array(z.string()).describe("Array of keys to press (e.g., ['ctrl', 'a'])"),
|
|
1537
|
+
}),
|
|
1538
|
+
},
|
|
1539
|
+
async (params): Promise<CallToolResult> => {
|
|
1540
|
+
const startTime = Date.now();
|
|
1541
|
+
logger.info("press_keys: Starting", { keys: params.keys });
|
|
1542
|
+
|
|
1543
|
+
const sessionCheck = requireActiveSession();
|
|
1544
|
+
if (!sessionCheck.valid) {
|
|
1545
|
+
logger.warn("press_keys: No active session");
|
|
1546
|
+
return sessionCheck.error;
|
|
1547
|
+
}
|
|
1548
|
+
|
|
1549
|
+
try {
|
|
1550
|
+
logger.debug("press_keys: Pressing keys");
|
|
1551
|
+
await sdk.pressKeys(params.keys);
|
|
1552
|
+
|
|
1553
|
+
const duration = Date.now() - startTime;
|
|
1554
|
+
logger.info("press_keys: Completed", { keys: params.keys, duration });
|
|
1555
|
+
|
|
1556
|
+
// Generate code for this press_keys action
|
|
1557
|
+
const generatedCode = generateActionCode("press_keys", { keys: params.keys });
|
|
1558
|
+
|
|
1559
|
+
return createToolResult(
|
|
1560
|
+
true,
|
|
1561
|
+
`Pressed keys: ${params.keys.join(" + ")}`,
|
|
1562
|
+
{ action: "press_keys", keys: params.keys, duration },
|
|
1563
|
+
generatedCode
|
|
1564
|
+
);
|
|
1565
|
+
} catch (error) {
|
|
1566
|
+
logger.error("press_keys: Failed", { error: String(error), keys: params.keys });
|
|
1567
|
+
captureException(error as Error, { tags: { tool: "press_keys" }, extra: { keys: params.keys } });
|
|
1568
|
+
throw error;
|
|
1569
|
+
}
|
|
1570
|
+
}
|
|
1571
|
+
);
|
|
1572
|
+
|
|
1573
|
+
// Scroll
|
|
1574
|
+
server.registerTool(
|
|
1575
|
+
"scroll",
|
|
1576
|
+
{
|
|
1577
|
+
description: "Scroll the page or element",
|
|
1578
|
+
inputSchema: z.object({
|
|
1579
|
+
direction: z.enum(["up", "down", "left", "right"]).default("down"),
|
|
1580
|
+
amount: z.number().optional().describe("Amount to scroll in pixels"),
|
|
1581
|
+
}),
|
|
1582
|
+
},
|
|
1583
|
+
async (params): Promise<CallToolResult> => {
|
|
1584
|
+
const startTime = Date.now();
|
|
1585
|
+
logger.info("scroll: Starting", { direction: params.direction, amount: params.amount });
|
|
1586
|
+
|
|
1587
|
+
const sessionCheck = requireActiveSession();
|
|
1588
|
+
if (!sessionCheck.valid) {
|
|
1589
|
+
logger.warn("scroll: No active session");
|
|
1590
|
+
return sessionCheck.error;
|
|
1591
|
+
}
|
|
1592
|
+
|
|
1593
|
+
try {
|
|
1594
|
+
logger.debug("scroll: Scrolling");
|
|
1595
|
+
await sdk.scroll(params.direction, params.amount ? { amount: params.amount } : undefined);
|
|
1596
|
+
|
|
1597
|
+
const duration = Date.now() - startTime;
|
|
1598
|
+
logger.info("scroll: Completed", { direction: params.direction, duration });
|
|
1599
|
+
|
|
1600
|
+
// Generate code for this scroll action
|
|
1601
|
+
const generatedCode = generateActionCode("scroll", { direction: params.direction, amount: params.amount });
|
|
1602
|
+
|
|
1603
|
+
return createToolResult(
|
|
1604
|
+
true,
|
|
1605
|
+
`Scrolled ${params.direction}${params.amount ? ` by ${params.amount}px` : ""}`,
|
|
1606
|
+
{ action: "scroll", scrollDirection: params.direction, direction: params.direction, amount: params.amount, duration },
|
|
1607
|
+
generatedCode
|
|
1608
|
+
);
|
|
1609
|
+
} catch (error) {
|
|
1610
|
+
logger.error("scroll: Failed", { error: String(error), direction: params.direction });
|
|
1611
|
+
captureException(error as Error, { tags: { tool: "scroll" }, extra: { direction: params.direction, amount: params.amount } });
|
|
1612
|
+
throw error;
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
);
|
|
1616
|
+
|
|
1617
|
+
// Assert - generates code for test files
|
|
1618
|
+
server.registerTool(
|
|
1619
|
+
"assert",
|
|
1620
|
+
{
|
|
1621
|
+
description: `Make an AI-powered assertion about the current screen state. GENERATES CODE for the test file.
|
|
1622
|
+
|
|
1623
|
+
Use this when you want a verification step recorded in the generated test. This will add code like:
|
|
1624
|
+
const assertResult = await testdriver.assert("your assertion");
|
|
1625
|
+
expect(assertResult).toBeTruthy();
|
|
1626
|
+
|
|
1627
|
+
Unlike 'check' which is for your understanding during development, 'assert' creates verification code that runs in CI/CD.`,
|
|
1628
|
+
inputSchema: z.object({
|
|
1629
|
+
assertion: z.string().describe("Natural language assertion to verify"),
|
|
1630
|
+
}),
|
|
1631
|
+
},
|
|
1632
|
+
async (params): Promise<CallToolResult> => {
|
|
1633
|
+
const startTime = Date.now();
|
|
1634
|
+
logger.info("assert: Starting", { assertion: params.assertion });
|
|
1635
|
+
|
|
1636
|
+
const sessionCheck = requireActiveSession();
|
|
1637
|
+
if (!sessionCheck.valid) {
|
|
1638
|
+
logger.warn("assert: No active session");
|
|
1639
|
+
return sessionCheck.error;
|
|
1640
|
+
}
|
|
1641
|
+
|
|
1642
|
+
try {
|
|
1643
|
+
logger.debug("assert: Running assertion");
|
|
1644
|
+
const result = await sdk.assert(params.assertion);
|
|
1645
|
+
|
|
1646
|
+
const duration = Date.now() - startTime;
|
|
1647
|
+
logger.info("assert: Completed", { assertion: params.assertion, passed: result, duration });
|
|
1648
|
+
|
|
1649
|
+
// Generate code for this assert action
|
|
1650
|
+
const generatedCode = generateActionCode("assert", { assertion: params.assertion });
|
|
1651
|
+
|
|
1652
|
+
return createToolResult(
|
|
1653
|
+
result,
|
|
1654
|
+
result ? `✓ Assertion passed: "${params.assertion}"` : `✗ Assertion failed: "${params.assertion}"`,
|
|
1655
|
+
{ action: "assert", assertion: params.assertion, passed: result, success: result, duration },
|
|
1656
|
+
generatedCode
|
|
1657
|
+
);
|
|
1658
|
+
} catch (error) {
|
|
1659
|
+
logger.error("assert: Failed", { error: String(error), assertion: params.assertion });
|
|
1660
|
+
captureException(error as Error, { tags: { tool: "assert" }, extra: { assertion: params.assertion } });
|
|
1661
|
+
throw error;
|
|
1662
|
+
}
|
|
1663
|
+
}
|
|
1664
|
+
);
|
|
1665
|
+
|
|
1666
|
+
// Check - AI uses this to understand the screen state (DOES NOT generate code)
|
|
1667
|
+
registerAppTool(
|
|
1668
|
+
server,
|
|
1669
|
+
"check",
|
|
1670
|
+
{
|
|
1671
|
+
title: "Check Screen State",
|
|
1672
|
+
description: `👁️ THIS IS HOW YOU SEE THE SCREEN. Use this tool whenever you need to understand what's currently displayed.
|
|
1673
|
+
|
|
1674
|
+
This tool captures a screenshot and returns AI analysis to YOU. Use it to:
|
|
1675
|
+
- See what's on the screen right now
|
|
1676
|
+
- Verify if your last action worked
|
|
1677
|
+
- Understand the current application state
|
|
1678
|
+
- Check if elements are visible or if navigation completed
|
|
1679
|
+
|
|
1680
|
+
Examples:
|
|
1681
|
+
- "What is currently on the screen?"
|
|
1682
|
+
- "Did the button click work?"
|
|
1683
|
+
- "Is the login form visible?"
|
|
1684
|
+
- "Did the page navigate to the dashboard?"
|
|
1685
|
+
|
|
1686
|
+
⚠️ Do NOT use 'screenshot' to see the screen - that only shows the user, not you.
|
|
1687
|
+
|
|
1688
|
+
Note: This tool does NOT generate test code. Use 'assert' when you want to add a verification step to the test file.
|
|
1689
|
+
|
|
1690
|
+
You can optionally provide a reference image URI to compare against a previous state.`,
|
|
1691
|
+
inputSchema: z.object({
|
|
1692
|
+
task: z.string().describe("The task or condition to verify (e.g., 'Did the login succeed?', 'Is the modal visible?')"),
|
|
1693
|
+
referenceImageUri: z.string().optional().describe("Optional screenshot resource URI (e.g., 'screenshot://testdriver/screenshot/screenshot-1') to compare against instead of the automatically captured 'before' screenshot. Use a screenshotResourceUri from a previous action."),
|
|
1694
|
+
}) as any,
|
|
1695
|
+
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
1696
|
+
},
|
|
1697
|
+
async (params: { task: string; referenceImageUri?: string }): Promise<CallToolResult> => {
|
|
1698
|
+
const startTime = Date.now();
|
|
1699
|
+
logger.info("check: Starting", { task: params.task, hasReferenceImageUri: !!params.referenceImageUri });
|
|
1700
|
+
|
|
1701
|
+
const sessionCheck = requireActiveSession();
|
|
1702
|
+
if (!sessionCheck.valid) {
|
|
1703
|
+
logger.warn("check: No active session");
|
|
1704
|
+
return sessionCheck.error;
|
|
1705
|
+
}
|
|
1706
|
+
|
|
1707
|
+
try {
|
|
1708
|
+
// Capture current screenshot
|
|
1709
|
+
logger.debug("check: Capturing current screenshot");
|
|
1710
|
+
const currentScreenshot = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
1711
|
+
|
|
1712
|
+
// Use provided reference image URI, last screenshot as "before" state, or current if no previous screenshot
|
|
1713
|
+
let beforeScreenshot: string;
|
|
1714
|
+
if (params.referenceImageUri) {
|
|
1715
|
+
// Extract image ID from URI (e.g., "screenshot://testdriver/screenshot/screenshot-1" -> "screenshot-1")
|
|
1716
|
+
const uriParts = params.referenceImageUri.split('/');
|
|
1717
|
+
const imageId = uriParts[uriParts.length - 1];
|
|
1718
|
+
|
|
1719
|
+
logger.info("check: Looking up reference image", {
|
|
1720
|
+
referenceImageUri: params.referenceImageUri,
|
|
1721
|
+
extractedImageId: imageId,
|
|
1722
|
+
imageStoreSize: imageStore.size,
|
|
1723
|
+
availableKeys: Array.from(imageStore.keys())
|
|
1724
|
+
});
|
|
1725
|
+
|
|
1726
|
+
const storedImage = getStoredImage(imageId);
|
|
1727
|
+
|
|
1728
|
+
if (storedImage) {
|
|
1729
|
+
logger.info("check: Found reference image", {
|
|
1730
|
+
imageId,
|
|
1731
|
+
dataLength: storedImage.data?.length,
|
|
1732
|
+
type: storedImage.type,
|
|
1733
|
+
hasData: !!storedImage.data
|
|
1734
|
+
});
|
|
1735
|
+
beforeScreenshot = storedImage.data;
|
|
1736
|
+
} else {
|
|
1737
|
+
logger.warn("check: Reference image NOT found in store, falling back to last screenshot", {
|
|
1738
|
+
referenceImageUri: params.referenceImageUri,
|
|
1739
|
+
imageId,
|
|
1740
|
+
imageStoreSize: imageStore.size,
|
|
1741
|
+
availableKeys: Array.from(imageStore.keys())
|
|
1742
|
+
});
|
|
1743
|
+
beforeScreenshot = lastScreenshotBase64 || currentScreenshot;
|
|
1744
|
+
}
|
|
1745
|
+
} else {
|
|
1746
|
+
beforeScreenshot = lastScreenshotBase64 || currentScreenshot;
|
|
1747
|
+
}
|
|
1748
|
+
|
|
1749
|
+
// Update last screenshot for next check
|
|
1750
|
+
lastScreenshotBase64 = currentScreenshot;
|
|
1751
|
+
|
|
1752
|
+
// Get system state
|
|
1753
|
+
const mousePosition = await sdk.agent.system.getMousePosition();
|
|
1754
|
+
const activeWindow = await sdk.agent.system.activeWin();
|
|
1755
|
+
|
|
1756
|
+
// Call the check endpoint
|
|
1757
|
+
logger.info("check: Calling check API endpoint", {
|
|
1758
|
+
hasLastScreenshot: beforeScreenshot !== currentScreenshot,
|
|
1759
|
+
usingReferenceImageUri: !!params.referenceImageUri,
|
|
1760
|
+
beforeScreenshotLength: beforeScreenshot?.length || 0,
|
|
1761
|
+
currentScreenshotLength: currentScreenshot?.length || 0,
|
|
1762
|
+
beforeScreenshotPreview: beforeScreenshot?.substring(0, 50),
|
|
1763
|
+
currentScreenshotPreview: currentScreenshot?.substring(0, 50)
|
|
1764
|
+
});
|
|
1765
|
+
const response = await sdk.agent.sdk.req("check", {
|
|
1766
|
+
tasks: [params.task],
|
|
1767
|
+
images: [beforeScreenshot, currentScreenshot],
|
|
1768
|
+
mousePosition,
|
|
1769
|
+
activeWindow,
|
|
1770
|
+
});
|
|
1771
|
+
|
|
1772
|
+
const aiResponse = response.data;
|
|
1773
|
+
|
|
1774
|
+
// Store screenshot for resource serving
|
|
1775
|
+
let screenshotResourceUri: string | undefined;
|
|
1776
|
+
if (currentScreenshot) {
|
|
1777
|
+
screenshotResourceUri = storeImage(currentScreenshot, "screenshot");
|
|
1778
|
+
}
|
|
1779
|
+
|
|
1780
|
+
// Determine if the check passed based on the AI response
|
|
1781
|
+
// The AI typically returns markdown with its analysis
|
|
1782
|
+
// We consider it "complete" if the response doesn't contain code blocks (indicating more work needed)
|
|
1783
|
+
const hasCodeBlocks = aiResponse && (
|
|
1784
|
+
aiResponse.includes("```yml") ||
|
|
1785
|
+
aiResponse.includes("```yaml") ||
|
|
1786
|
+
aiResponse.includes("- command:")
|
|
1787
|
+
);
|
|
1788
|
+
const isComplete = !hasCodeBlocks;
|
|
1789
|
+
|
|
1790
|
+
const duration = Date.now() - startTime;
|
|
1791
|
+
logger.info("check: Completed", { task: params.task, complete: isComplete, duration });
|
|
1792
|
+
|
|
1793
|
+
// Note: check doesn't generate code - it's for AI understanding, not test recording
|
|
1794
|
+
return createToolResult(
|
|
1795
|
+
isComplete,
|
|
1796
|
+
isComplete
|
|
1797
|
+
? `✓ Task appears complete: "${params.task}"\n\nAI Analysis:\n${aiResponse}`
|
|
1798
|
+
: `⚠ Task may not be complete: "${params.task}"\n\nAI Analysis:\n${aiResponse}`,
|
|
1799
|
+
{
|
|
1800
|
+
action: "check",
|
|
1801
|
+
task: params.task,
|
|
1802
|
+
complete: isComplete,
|
|
1803
|
+
success: isComplete,
|
|
1804
|
+
aiResponse,
|
|
1805
|
+
screenshotResourceUri,
|
|
1806
|
+
duration
|
|
1807
|
+
}
|
|
1808
|
+
);
|
|
1809
|
+
} catch (error) {
|
|
1810
|
+
logger.error("check: Failed", { error: String(error), task: params.task });
|
|
1811
|
+
captureException(error as Error, { tags: { tool: "check" }, extra: { task: params.task } });
|
|
1812
|
+
throw error;
|
|
1813
|
+
}
|
|
1814
|
+
}
|
|
1815
|
+
);
|
|
1816
|
+
|
|
1817
|
+
// Exec
|
|
1818
|
+
server.registerTool(
|
|
1819
|
+
"exec",
|
|
1820
|
+
{
|
|
1821
|
+
description: "Execute shell or PowerShell commands in the sandbox",
|
|
1822
|
+
inputSchema: z.object({
|
|
1823
|
+
language: z.enum(["sh", "pwsh"]).default("sh"),
|
|
1824
|
+
code: z.string().describe("Code to execute"),
|
|
1825
|
+
timeout: z.number().default(30000).describe("Timeout in ms"),
|
|
1826
|
+
}),
|
|
1827
|
+
},
|
|
1828
|
+
async (params): Promise<CallToolResult> => {
|
|
1829
|
+
const startTime = Date.now();
|
|
1830
|
+
logger.info("exec: Starting", { language: params.language, codeLength: params.code.length, timeout: params.timeout });
|
|
1831
|
+
|
|
1832
|
+
const sessionCheck = requireActiveSession();
|
|
1833
|
+
if (!sessionCheck.valid) {
|
|
1834
|
+
logger.warn("exec: No active session");
|
|
1835
|
+
return sessionCheck.error;
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
try {
|
|
1839
|
+
logger.debug("exec: Executing code", { language: params.language });
|
|
1840
|
+
const output = await sdk.exec(params.language, params.code, params.timeout);
|
|
1841
|
+
|
|
1842
|
+
const duration = Date.now() - startTime;
|
|
1843
|
+
logger.info("exec: Completed", { language: params.language, outputLength: output?.length || 0, duration });
|
|
1844
|
+
|
|
1845
|
+
// Generate code for this exec action
|
|
1846
|
+
const generatedCode = generateActionCode("exec", { language: params.language, code: params.code, timeout: params.timeout });
|
|
1847
|
+
|
|
1848
|
+
return createToolResult(
|
|
1849
|
+
true,
|
|
1850
|
+
`Executed ${params.language} code:\n${output || "(no output)"}`,
|
|
1851
|
+
{ action: "exec", language: params.language, output, duration },
|
|
1852
|
+
generatedCode
|
|
1853
|
+
);
|
|
1854
|
+
} catch (error) {
|
|
1855
|
+
logger.error("exec: Failed", { error: String(error), language: params.language });
|
|
1856
|
+
captureException(error as Error, { tags: { tool: "exec" }, extra: { language: params.language, codeLength: params.code.length } });
|
|
1857
|
+
throw error;
|
|
1858
|
+
}
|
|
1859
|
+
}
|
|
1860
|
+
);
|
|
1861
|
+
|
|
1862
|
+
// Parse auto-screenshot filename format: <seq>-<action>-<phase>-L<line>-<description>.png
|
|
1863
|
+
// Example: 001-click-before-L42-submit-button.png
|
|
1864
|
+
// Example: 003-click-error-L42-submit-button.png (error phase when action fails)
|
|
1865
|
+
interface ParsedScreenshotInfo {
|
|
1866
|
+
sequence?: number;
|
|
1867
|
+
action?: string;
|
|
1868
|
+
phase?: "before" | "after" | "error";
|
|
1869
|
+
lineNumber?: number;
|
|
1870
|
+
description?: string;
|
|
1871
|
+
}
|
|
1872
|
+
|
|
1873
|
+
function parseScreenshotFilename(filename: string): ParsedScreenshotInfo {
|
|
1874
|
+
// Match pattern: 001-click-before-L42-submit-button.png or 001-click-error-L42-submit-button.png
|
|
1875
|
+
const match = filename.match(/^(\d+)-([a-z]+)-(before|after|error)-L(\d+)-(.+)\.png$/i);
|
|
1876
|
+
if (match) {
|
|
1877
|
+
return {
|
|
1878
|
+
sequence: parseInt(match[1], 10),
|
|
1879
|
+
action: match[2].toLowerCase(),
|
|
1880
|
+
phase: match[3].toLowerCase() as "before" | "after" | "error",
|
|
1881
|
+
lineNumber: parseInt(match[4], 10),
|
|
1882
|
+
description: match[5],
|
|
1883
|
+
};
|
|
1884
|
+
}
|
|
1885
|
+
return {};
|
|
1886
|
+
}
|
|
1887
|
+
|
|
1888
|
+
// List Local Screenshots - lists screenshots saved to .testdriver directory
|
|
1889
|
+
server.registerTool(
|
|
1890
|
+
"list_local_screenshots",
|
|
1891
|
+
{
|
|
1892
|
+
description: `List and filter screenshots saved in the .testdriver directory.
|
|
1893
|
+
|
|
1894
|
+
Screenshots from auto-screenshot feature use the format: <seq>-<action>-<phase>-L<line>-<description>.png
|
|
1895
|
+
Example: 001-click-before-L42-submit-button.png
|
|
1896
|
+
|
|
1897
|
+
This tool supports powerful filtering to find specific screenshots:
|
|
1898
|
+
- By test file (directory)
|
|
1899
|
+
- By line number or range
|
|
1900
|
+
- By action type (click, find, type, assert, etc.)
|
|
1901
|
+
- By phase (before/after/error - error screenshots are captured when actions fail)
|
|
1902
|
+
- By regex pattern on filename
|
|
1903
|
+
- By sequence number range
|
|
1904
|
+
|
|
1905
|
+
Returns a list of screenshot paths that can be viewed with the 'view_local_screenshot' tool.`,
|
|
1906
|
+
inputSchema: z.object({
|
|
1907
|
+
directory: z.string().optional().describe("Test file or subdirectory to search (e.g., 'login.test', 'mcp-screenshots'). If not provided, searches all."),
|
|
1908
|
+
line: z.number().optional().describe("Filter by exact line number from test file (e.g., 42 matches L42)"),
|
|
1909
|
+
lineRange: z.object({
|
|
1910
|
+
start: z.number().describe("Start line number (inclusive)"),
|
|
1911
|
+
end: z.number().describe("End line number (inclusive)"),
|
|
1912
|
+
}).optional().describe("Filter by line number range (e.g., { start: 10, end: 20 })"),
|
|
1913
|
+
action: z.string().optional().describe("Filter by action type: click, find, type, assert, provision, scroll, hover, etc."),
|
|
1914
|
+
phase: z.enum(["before", "after", "error"]).optional().describe("Filter by phase: 'before' (pre-action), 'after' (post-action), or 'error' (when action fails)"),
|
|
1915
|
+
pattern: z.string().optional().describe("Regex pattern to match against filename (e.g., 'submit|login' or 'button.*click')"),
|
|
1916
|
+
sequence: z.number().optional().describe("Filter by exact sequence number"),
|
|
1917
|
+
sequenceRange: z.object({
|
|
1918
|
+
start: z.number().describe("Start sequence (inclusive)"),
|
|
1919
|
+
end: z.number().describe("End sequence (inclusive)"),
|
|
1920
|
+
}).optional().describe("Filter by sequence range (e.g., { start: 1, end: 10 })"),
|
|
1921
|
+
limit: z.number().optional().describe("Maximum number of results to return (default: 50)"),
|
|
1922
|
+
sortBy: z.enum(["modified", "sequence", "line"]).optional().describe("Sort by: 'modified' (newest first), 'sequence' (execution order), or 'line' (line number). Default: 'modified'"),
|
|
1923
|
+
}),
|
|
1924
|
+
},
|
|
1925
|
+
async (params): Promise<CallToolResult> => {
|
|
1926
|
+
const startTime = Date.now();
|
|
1927
|
+
logger.info("list_local_screenshots: Starting", { ...params });
|
|
1928
|
+
|
|
1929
|
+
try {
|
|
1930
|
+
// Find .testdriver directory - check current working directory and common locations
|
|
1931
|
+
const possiblePaths = [
|
|
1932
|
+
path.join(process.cwd(), ".testdriver"),
|
|
1933
|
+
path.join(os.homedir(), ".testdriver"),
|
|
1934
|
+
];
|
|
1935
|
+
|
|
1936
|
+
let testdriverDir: string | null = null;
|
|
1937
|
+
for (const p of possiblePaths) {
|
|
1938
|
+
if (fs.existsSync(p)) {
|
|
1939
|
+
testdriverDir = p;
|
|
1940
|
+
break;
|
|
1941
|
+
}
|
|
1942
|
+
}
|
|
1943
|
+
|
|
1944
|
+
if (!testdriverDir) {
|
|
1945
|
+
logger.warn("list_local_screenshots: .testdriver directory not found");
|
|
1946
|
+
return createToolResult(false, "No .testdriver directory found. Screenshots are saved here during test runs.", { error: "Directory not found" });
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
interface ScreenshotInfo {
|
|
1950
|
+
path: string;
|
|
1951
|
+
name: string;
|
|
1952
|
+
modified: Date;
|
|
1953
|
+
size: number;
|
|
1954
|
+
parsed: ParsedScreenshotInfo;
|
|
1955
|
+
}
|
|
1956
|
+
|
|
1957
|
+
const screenshots: ScreenshotInfo[] = [];
|
|
1958
|
+
|
|
1959
|
+
// Compile regex pattern if provided
|
|
1960
|
+
let regexPattern: RegExp | null = null;
|
|
1961
|
+
if (params.pattern) {
|
|
1962
|
+
try {
|
|
1963
|
+
regexPattern = new RegExp(params.pattern, "i");
|
|
1964
|
+
} catch {
|
|
1965
|
+
return createToolResult(false, `Invalid regex pattern: ${params.pattern}`, { error: "Invalid regex" });
|
|
1966
|
+
}
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
// Function to recursively find PNG files
|
|
1970
|
+
const findPngFiles = (dir: string) => {
|
|
1971
|
+
if (!fs.existsSync(dir)) return;
|
|
1972
|
+
|
|
1973
|
+
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
1974
|
+
for (const entry of entries) {
|
|
1975
|
+
const fullPath = path.join(dir, entry.name);
|
|
1976
|
+
if (entry.isDirectory()) {
|
|
1977
|
+
// If a specific directory was requested, only search that one
|
|
1978
|
+
if (!params.directory || entry.name === params.directory || dir !== testdriverDir) {
|
|
1979
|
+
findPngFiles(fullPath);
|
|
1980
|
+
}
|
|
1981
|
+
} else if (entry.isFile() && entry.name.toLowerCase().endsWith(".png")) {
|
|
1982
|
+
const parsed = parseScreenshotFilename(entry.name);
|
|
1983
|
+
|
|
1984
|
+
// Apply filters
|
|
1985
|
+
if (params.line !== undefined && parsed.lineNumber !== params.line) continue;
|
|
1986
|
+
if (params.lineRange && (
|
|
1987
|
+
parsed.lineNumber === undefined ||
|
|
1988
|
+
parsed.lineNumber < params.lineRange.start ||
|
|
1989
|
+
parsed.lineNumber > params.lineRange.end
|
|
1990
|
+
)) continue;
|
|
1991
|
+
if (params.action && parsed.action !== params.action.toLowerCase()) continue;
|
|
1992
|
+
if (params.phase && parsed.phase !== params.phase) continue;
|
|
1993
|
+
if (params.sequence !== undefined && parsed.sequence !== params.sequence) continue;
|
|
1994
|
+
if (params.sequenceRange && (
|
|
1995
|
+
parsed.sequence === undefined ||
|
|
1996
|
+
parsed.sequence < params.sequenceRange.start ||
|
|
1997
|
+
parsed.sequence > params.sequenceRange.end
|
|
1998
|
+
)) continue;
|
|
1999
|
+
if (regexPattern && !regexPattern.test(entry.name)) continue;
|
|
2000
|
+
|
|
2001
|
+
const stats = fs.statSync(fullPath);
|
|
2002
|
+
screenshots.push({
|
|
2003
|
+
path: fullPath,
|
|
2004
|
+
name: entry.name,
|
|
2005
|
+
modified: stats.mtime,
|
|
2006
|
+
size: stats.size,
|
|
2007
|
+
parsed,
|
|
2008
|
+
});
|
|
2009
|
+
}
|
|
2010
|
+
}
|
|
2011
|
+
};
|
|
2012
|
+
|
|
2013
|
+
findPngFiles(testdriverDir);
|
|
2014
|
+
|
|
2015
|
+
// Sort based on sortBy parameter
|
|
2016
|
+
const sortBy = params.sortBy || "modified";
|
|
2017
|
+
if (sortBy === "modified") {
|
|
2018
|
+
screenshots.sort((a, b) => b.modified.getTime() - a.modified.getTime());
|
|
2019
|
+
} else if (sortBy === "sequence") {
|
|
2020
|
+
screenshots.sort((a, b) => (a.parsed.sequence ?? Infinity) - (b.parsed.sequence ?? Infinity));
|
|
2021
|
+
} else if (sortBy === "line") {
|
|
2022
|
+
screenshots.sort((a, b) => (a.parsed.lineNumber ?? Infinity) - (b.parsed.lineNumber ?? Infinity));
|
|
2023
|
+
}
|
|
2024
|
+
|
|
2025
|
+
const duration = Date.now() - startTime;
|
|
2026
|
+
logger.info("list_local_screenshots: Completed", { count: screenshots.length, duration });
|
|
2027
|
+
|
|
2028
|
+
if (screenshots.length === 0) {
|
|
2029
|
+
const filters = [];
|
|
2030
|
+
if (params.directory) filters.push(`directory=${params.directory}`);
|
|
2031
|
+
if (params.line) filters.push(`line=${params.line}`);
|
|
2032
|
+
if (params.lineRange) filters.push(`lineRange=${params.lineRange.start}-${params.lineRange.end}`);
|
|
2033
|
+
if (params.action) filters.push(`action=${params.action}`);
|
|
2034
|
+
if (params.phase) filters.push(`phase=${params.phase}`);
|
|
2035
|
+
if (params.pattern) filters.push(`pattern=${params.pattern}`);
|
|
2036
|
+
if (params.sequence) filters.push(`sequence=${params.sequence}`);
|
|
2037
|
+
if (params.sequenceRange) filters.push(`sequenceRange=${params.sequenceRange.start}-${params.sequenceRange.end}`);
|
|
2038
|
+
|
|
2039
|
+
const filterMsg = filters.length > 0 ? ` with filters: ${filters.join(", ")}` : "";
|
|
2040
|
+
return createToolResult(true, `No screenshots found in .testdriver directory${filterMsg}.`, {
|
|
2041
|
+
action: "list_local_screenshots",
|
|
2042
|
+
count: 0,
|
|
2043
|
+
directory: testdriverDir,
|
|
2044
|
+
filters: params,
|
|
2045
|
+
duration
|
|
2046
|
+
});
|
|
2047
|
+
}
|
|
2048
|
+
|
|
2049
|
+
const limit = params.limit || 50;
|
|
2050
|
+
const limitedScreenshots = screenshots.slice(0, limit);
|
|
2051
|
+
|
|
2052
|
+
// Format the list for display with parsed info
|
|
2053
|
+
const screenshotList = limitedScreenshots.map((s, i) => {
|
|
2054
|
+
const relativePath = path.relative(testdriverDir!, s.path);
|
|
2055
|
+
const sizeKB = Math.round(s.size / 1024);
|
|
2056
|
+
const timeAgo = formatTimeAgo(s.modified);
|
|
2057
|
+
|
|
2058
|
+
// Add parsed info if available
|
|
2059
|
+
const parts = [`${i + 1}. ${relativePath}`];
|
|
2060
|
+
const meta = [];
|
|
2061
|
+
if (s.parsed.lineNumber) meta.push(`L${s.parsed.lineNumber}`);
|
|
2062
|
+
if (s.parsed.action) meta.push(s.parsed.action);
|
|
2063
|
+
if (s.parsed.phase) meta.push(s.parsed.phase);
|
|
2064
|
+
meta.push(`${sizeKB}KB`);
|
|
2065
|
+
meta.push(timeAgo);
|
|
2066
|
+
parts.push(`(${meta.join(", ")})`);
|
|
2067
|
+
|
|
2068
|
+
return parts.join(" ");
|
|
2069
|
+
}).join("\n");
|
|
2070
|
+
|
|
2071
|
+
const message = screenshots.length > limit
|
|
2072
|
+
? `Found ${screenshots.length} screenshots (showing ${limit} results, sorted by ${sortBy}):\n\n${screenshotList}`
|
|
2073
|
+
: `Found ${screenshots.length} screenshot(s) (sorted by ${sortBy}):\n\n${screenshotList}`;
|
|
2074
|
+
|
|
2075
|
+
return createToolResult(true, message, {
|
|
2076
|
+
action: "list_local_screenshots",
|
|
2077
|
+
count: screenshots.length,
|
|
2078
|
+
returned: limitedScreenshots.length,
|
|
2079
|
+
directory: testdriverDir,
|
|
2080
|
+
filters: params,
|
|
2081
|
+
sortBy,
|
|
2082
|
+
screenshots: limitedScreenshots.map(s => ({
|
|
2083
|
+
path: s.path,
|
|
2084
|
+
relativePath: path.relative(testdriverDir!, s.path),
|
|
2085
|
+
name: s.name,
|
|
2086
|
+
modified: s.modified.toISOString(),
|
|
2087
|
+
sizeBytes: s.size,
|
|
2088
|
+
sequence: s.parsed.sequence,
|
|
2089
|
+
action: s.parsed.action,
|
|
2090
|
+
phase: s.parsed.phase,
|
|
2091
|
+
lineNumber: s.parsed.lineNumber,
|
|
2092
|
+
description: s.parsed.description,
|
|
2093
|
+
})),
|
|
2094
|
+
duration
|
|
2095
|
+
});
|
|
2096
|
+
} catch (error) {
|
|
2097
|
+
logger.error("list_local_screenshots: Failed", { error: String(error) });
|
|
2098
|
+
captureException(error as Error, { tags: { tool: "list_local_screenshots" } });
|
|
2099
|
+
throw error;
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
2102
|
+
);
|
|
2103
|
+
|
|
2104
|
+
// Helper to format time ago
|
|
2105
|
+
function formatTimeAgo(date: Date): string {
|
|
2106
|
+
const seconds = Math.floor((Date.now() - date.getTime()) / 1000);
|
|
2107
|
+
if (seconds < 60) return `${seconds}s ago`;
|
|
2108
|
+
const minutes = Math.floor(seconds / 60);
|
|
2109
|
+
if (minutes < 60) return `${minutes}m ago`;
|
|
2110
|
+
const hours = Math.floor(minutes / 60);
|
|
2111
|
+
if (hours < 24) return `${hours}h ago`;
|
|
2112
|
+
const days = Math.floor(hours / 24);
|
|
2113
|
+
return `${days}d ago`;
|
|
2114
|
+
}
|
|
2115
|
+
|
|
2116
|
+
// View Local Screenshot - view a screenshot from .testdriver directory
|
|
2117
|
+
// Returns the image so AI clients that support images can see it
|
|
2118
|
+
// Also displays to the user via MCP App
|
|
2119
|
+
registerAppTool(
|
|
2120
|
+
server,
|
|
2121
|
+
"view_local_screenshot",
|
|
2122
|
+
{
|
|
2123
|
+
title: "View Local Screenshot",
|
|
2124
|
+
description: `View a screenshot from the .testdriver directory.
|
|
2125
|
+
|
|
2126
|
+
Use 'list_local_screenshots' first to see available screenshots, then use this tool to view one.
|
|
2127
|
+
|
|
2128
|
+
This tool returns the image content so AI clients that support images can see it directly.
|
|
2129
|
+
The image is also displayed to the user via the MCP App UI.
|
|
2130
|
+
|
|
2131
|
+
Useful for:
|
|
2132
|
+
- Reviewing screenshots from previous test runs
|
|
2133
|
+
- Debugging test failures by examining saved screenshots
|
|
2134
|
+
- Comparing current screen state to saved screenshots`,
|
|
2135
|
+
inputSchema: z.object({
|
|
2136
|
+
path: z.string().describe("Full path to the screenshot file (from list_local_screenshots)"),
|
|
2137
|
+
}) as any,
|
|
2138
|
+
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
2139
|
+
},
|
|
2140
|
+
async (params: { path: string }): Promise<CallToolResult> => {
|
|
2141
|
+
const startTime = Date.now();
|
|
2142
|
+
logger.info("view_local_screenshot: Starting", { path: params.path });
|
|
2143
|
+
|
|
2144
|
+
try {
|
|
2145
|
+
// Validate the path exists and is a PNG
|
|
2146
|
+
if (!fs.existsSync(params.path)) {
|
|
2147
|
+
logger.warn("view_local_screenshot: File not found", { path: params.path });
|
|
2148
|
+
return createToolResult(false, `Screenshot not found: ${params.path}`, { error: "File not found" });
|
|
2149
|
+
}
|
|
2150
|
+
|
|
2151
|
+
if (!params.path.toLowerCase().endsWith(".png")) {
|
|
2152
|
+
logger.warn("view_local_screenshot: Not a PNG file", { path: params.path });
|
|
2153
|
+
return createToolResult(false, "Only PNG files are supported", { error: "Invalid file type" });
|
|
2154
|
+
}
|
|
2155
|
+
|
|
2156
|
+
// Security check - only allow files from .testdriver directory
|
|
2157
|
+
const normalizedPath = path.resolve(params.path);
|
|
2158
|
+
if (!normalizedPath.includes(".testdriver")) {
|
|
2159
|
+
logger.warn("view_local_screenshot: Path not in .testdriver", { path: normalizedPath });
|
|
2160
|
+
return createToolResult(false, "Can only view screenshots from .testdriver directory", { error: "Security: path not allowed" });
|
|
2161
|
+
}
|
|
2162
|
+
|
|
2163
|
+
// Read the file
|
|
2164
|
+
const imageBuffer = fs.readFileSync(params.path);
|
|
2165
|
+
const imageBase64 = imageBuffer.toString("base64");
|
|
2166
|
+
|
|
2167
|
+
// Store image for MCP App UI display
|
|
2168
|
+
const screenshotResourceUri = storeImage(imageBase64, "screenshot");
|
|
2169
|
+
|
|
2170
|
+
const stats = fs.statSync(params.path);
|
|
2171
|
+
const sizeKB = Math.round(stats.size / 1024);
|
|
2172
|
+
const fileName = path.basename(params.path);
|
|
2173
|
+
|
|
2174
|
+
const duration = Date.now() - startTime;
|
|
2175
|
+
logger.info("view_local_screenshot: Completed", { path: params.path, sizeKB, duration });
|
|
2176
|
+
|
|
2177
|
+
// Return the image content for AI clients that support images
|
|
2178
|
+
// The content array includes both text and image for maximum compatibility
|
|
2179
|
+
const content: CallToolResult["content"] = [
|
|
2180
|
+
{ type: "text", text: `Screenshot: ${fileName} (${sizeKB}KB)` },
|
|
2181
|
+
{
|
|
2182
|
+
type: "image",
|
|
2183
|
+
data: imageBase64,
|
|
2184
|
+
mimeType: "image/png"
|
|
2185
|
+
},
|
|
2186
|
+
];
|
|
2187
|
+
|
|
2188
|
+
return {
|
|
2189
|
+
content,
|
|
2190
|
+
structuredContent: {
|
|
2191
|
+
action: "view_local_screenshot",
|
|
2192
|
+
success: true,
|
|
2193
|
+
path: params.path,
|
|
2194
|
+
fileName,
|
|
2195
|
+
sizeBytes: stats.size,
|
|
2196
|
+
modified: stats.mtime.toISOString(),
|
|
2197
|
+
screenshotResourceUri,
|
|
2198
|
+
duration
|
|
2199
|
+
},
|
|
2200
|
+
};
|
|
2201
|
+
} catch (error) {
|
|
2202
|
+
logger.error("view_local_screenshot: Failed", { error: String(error), path: params.path });
|
|
2203
|
+
captureException(error as Error, { tags: { tool: "view_local_screenshot" }, extra: { path: params.path } });
|
|
2204
|
+
throw error;
|
|
2205
|
+
}
|
|
2206
|
+
}
|
|
2207
|
+
);
|
|
2208
|
+
|
|
2209
|
+
// Screenshot - captures full screen to show user the current state
|
|
2210
|
+
// NOTE: This is for SHOWING the user the screen, not for AI understanding.
|
|
2211
|
+
// Use 'check' tool for AI to understand screen state.
|
|
2212
|
+
registerAppTool(
|
|
2213
|
+
server,
|
|
2214
|
+
"screenshot",
|
|
2215
|
+
{
|
|
2216
|
+
title: "Screenshot",
|
|
2217
|
+
description: `Display a screenshot to the user. This tool does NOT return the image to you (the AI).
|
|
2218
|
+
|
|
2219
|
+
⚠️ IMPORTANT: Do NOT use this tool to understand the screen state. The screenshot is ONLY displayed to the human user - you will NOT receive the image or any analysis.
|
|
2220
|
+
|
|
2221
|
+
If you need to:
|
|
2222
|
+
- See what's on screen → use 'check' instead
|
|
2223
|
+
- Verify an action worked → use 'check' instead
|
|
2224
|
+
- Understand the current state → use 'check' instead
|
|
2225
|
+
|
|
2226
|
+
Only use 'screenshot' when you explicitly want to show something to the human user without needing to see it yourself.`,
|
|
2227
|
+
inputSchema: z.object({}),
|
|
2228
|
+
_meta: { ui: { resourceUri: RESOURCE_URI, expanded: true } },
|
|
2229
|
+
},
|
|
2230
|
+
async (): Promise<CallToolResult> => {
|
|
2231
|
+
const startTime = Date.now();
|
|
2232
|
+
logger.info("screenshot: Starting");
|
|
2233
|
+
|
|
2234
|
+
const sessionCheck = requireActiveSession();
|
|
2235
|
+
if (!sessionCheck.valid) {
|
|
2236
|
+
logger.warn("screenshot: No active session");
|
|
2237
|
+
return sessionCheck.error;
|
|
2238
|
+
}
|
|
2239
|
+
|
|
2240
|
+
try {
|
|
2241
|
+
// Capture full screen screenshot
|
|
2242
|
+
const screenshotBase64 = await sdk.agent.system.captureScreenBase64(1, false, true);
|
|
2243
|
+
|
|
2244
|
+
let screenshotResourceUri: string | undefined;
|
|
2245
|
+
if (screenshotBase64) {
|
|
2246
|
+
// Store raw base64 for the resource blob with unique ID
|
|
2247
|
+
screenshotResourceUri = storeImage(screenshotBase64, "screenshot");
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2250
|
+
const duration = Date.now() - startTime;
|
|
2251
|
+
logger.info("screenshot: Completed", { duration, hasImage: !!screenshotBase64 });
|
|
2252
|
+
|
|
2253
|
+
// Only send the resource URI - the MCP app will fetch the image via resources/read
|
|
2254
|
+
// This keeps the base64 image data OUT of AI context
|
|
2255
|
+
return createToolResult(
|
|
2256
|
+
true,
|
|
2257
|
+
"Screenshot captured and displayed to user",
|
|
2258
|
+
{
|
|
2259
|
+
action: "screenshot",
|
|
2260
|
+
screenshotResourceUri,
|
|
2261
|
+
duration
|
|
2262
|
+
}
|
|
2263
|
+
);
|
|
2264
|
+
} catch (error) {
|
|
2265
|
+
logger.error("screenshot: Failed", { error: String(error) });
|
|
2266
|
+
return createToolResult(false, `Screenshot failed: ${error}`, { error: String(error) });
|
|
2267
|
+
}
|
|
2268
|
+
}
|
|
2269
|
+
);
|
|
2270
|
+
|
|
2271
|
+
// Init - Initialize a new TestDriver project
|
|
2272
|
+
server.registerTool(
|
|
2273
|
+
"init",
|
|
2274
|
+
{
|
|
2275
|
+
description: `Initialize a new TestDriver project with Vitest SDK examples.
|
|
2276
|
+
|
|
2277
|
+
This creates:
|
|
2278
|
+
- package.json with proper dependencies
|
|
2279
|
+
- Example test files (tests/example.test.js, tests/login.js)
|
|
2280
|
+
- vitest.config.js
|
|
2281
|
+
- .gitignore
|
|
2282
|
+
- GitHub Actions workflow (.github/workflows/testdriver.yml)
|
|
2283
|
+
- VSCode MCP config (.vscode/mcp.json)
|
|
2284
|
+
- VSCode extensions recommendations (.vscode/extensions.json)
|
|
2285
|
+
- TestDriver skills (.github/skills/)
|
|
2286
|
+
- TestDriver agents (.github/agents/)
|
|
2287
|
+
- .env file with API key (if provided)
|
|
2288
|
+
|
|
2289
|
+
API Key: The apiKey parameter is optional. If not provided, you'll need to manually add TD_API_KEY to the .env file after initialization. The project structure will still be created successfully.`,
|
|
2290
|
+
inputSchema: z.object({
|
|
2291
|
+
directory: z.string().optional().describe("Target directory (defaults to current working directory)"),
|
|
2292
|
+
apiKey: z.string().optional().describe("TestDriver API key (will be saved to .env)"),
|
|
2293
|
+
skipInstall: z.boolean().default(false).describe("Skip npm install step"),
|
|
2294
|
+
}),
|
|
2295
|
+
},
|
|
2296
|
+
async (params): Promise<CallToolResult> => {
|
|
2297
|
+
const startTime = Date.now();
|
|
2298
|
+
const targetDir = params.directory ? path.resolve(params.directory) : process.cwd();
|
|
2299
|
+
|
|
2300
|
+
logger.info("init: Starting", { targetDir, hasApiKey: !!params.apiKey, skipInstall: params.skipInstall });
|
|
2301
|
+
|
|
2302
|
+
try {
|
|
2303
|
+
// Import the shared init logic (dynamic import for ESM/CJS compatibility)
|
|
2304
|
+
const initProjectPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "lib", "init-project.js");
|
|
2305
|
+
const { initProject } = await import(pathToFileURL(initProjectPath).href);
|
|
2306
|
+
|
|
2307
|
+
// Run the shared init logic
|
|
2308
|
+
const result = await initProject({
|
|
2309
|
+
targetDir,
|
|
2310
|
+
apiKey: params.apiKey,
|
|
2311
|
+
skipInstall: params.skipInstall,
|
|
2312
|
+
});
|
|
2313
|
+
|
|
2314
|
+
const duration = Date.now() - startTime;
|
|
2315
|
+
logger.info("init: Completed", { targetDir, duration, success: result.success });
|
|
2316
|
+
|
|
2317
|
+
const nextSteps = `
|
|
2318
|
+
|
|
2319
|
+
📚 Next steps:
|
|
2320
|
+
|
|
2321
|
+
1. Run your tests:
|
|
2322
|
+
vitest run
|
|
2323
|
+
|
|
2324
|
+
2. Use AI agents to write tests:
|
|
2325
|
+
Open VSCode/Cursor and use @testdriver agent
|
|
2326
|
+
|
|
2327
|
+
3. MCP server configured:
|
|
2328
|
+
TestDriver tools available via MCP in .vscode/mcp.json
|
|
2329
|
+
|
|
2330
|
+
4. For CI/CD, add TD_API_KEY to your GitHub repository secrets:
|
|
2331
|
+
Settings → Secrets → Actions → New repository secret
|
|
2332
|
+
|
|
2333
|
+
Learn more at https://docs.testdriver.ai/v7/getting-started/
|
|
2334
|
+
`;
|
|
2335
|
+
|
|
2336
|
+
const allMessages = [...result.results, ...result.errors.map((e: string) => `⚠️ ${e}`)];
|
|
2337
|
+
|
|
2338
|
+
return createToolResult(
|
|
2339
|
+
result.success,
|
|
2340
|
+
result.success
|
|
2341
|
+
? `✅ TestDriver project initialized successfully!\n\n${allMessages.join("\n")}${nextSteps}`
|
|
2342
|
+
: `⚠️ TestDriver project initialization completed with errors:\n\n${allMessages.join("\n")}`,
|
|
2343
|
+
{
|
|
2344
|
+
action: "init",
|
|
2345
|
+
targetDir,
|
|
2346
|
+
filesCreated: result.results.length,
|
|
2347
|
+
hasApiKey: !!params.apiKey,
|
|
2348
|
+
errors: result.errors,
|
|
2349
|
+
duration
|
|
2350
|
+
}
|
|
2351
|
+
);
|
|
2352
|
+
} catch (error) {
|
|
2353
|
+
logger.error("init: Failed", { error: String(error), targetDir });
|
|
2354
|
+
captureException(error as Error, { tags: { tool: "init" }, extra: { targetDir } });
|
|
2355
|
+
throw error;
|
|
2356
|
+
}
|
|
2357
|
+
}
|
|
2358
|
+
);
|
|
2359
|
+
|
|
2360
|
+
|
|
2361
|
+
// Start the server
|
|
2362
|
+
async function main() {
|
|
2363
|
+
logger.info("Starting TestDriver MCP Server", {
|
|
2364
|
+
version,
|
|
2365
|
+
logLevel: process.env.TD_LOG_LEVEL || "INFO",
|
|
2366
|
+
distDir: DIST_DIR,
|
|
2367
|
+
sentryEnabled: isSentryEnabled(),
|
|
2368
|
+
});
|
|
2369
|
+
|
|
2370
|
+
const transport = new StdioServerTransport();
|
|
2371
|
+
await server.connect(transport);
|
|
2372
|
+
|
|
2373
|
+
logger.info("TestDriver MCP Server running on stdio");
|
|
2374
|
+
|
|
2375
|
+
// Handle graceful shutdown
|
|
2376
|
+
const shutdown = async () => {
|
|
2377
|
+
logger.info("Shutting down MCP Server");
|
|
2378
|
+
await flushSentry();
|
|
2379
|
+
process.exit(0);
|
|
2380
|
+
};
|
|
2381
|
+
|
|
2382
|
+
process.on("SIGINT", shutdown);
|
|
2383
|
+
process.on("SIGTERM", shutdown);
|
|
2384
|
+
}
|
|
2385
|
+
|
|
2386
|
+
main().catch(async (error) => {
|
|
2387
|
+
logger.error("Server failed to start", { error: String(error) });
|
|
2388
|
+
captureException(error as Error, { tags: { phase: "startup" } });
|
|
2389
|
+
await flushSentry();
|
|
2390
|
+
process.exit(1);
|
|
2391
|
+
});
|