@testdriverai/agent 7.11.240-test → 7.11.241-canary
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ai/skills/testdriver-agent/SKILL.md +174 -91
- package/ai/skills/testdriver-assert/SKILL.md +73 -269
- package/ai/skills/testdriver-aws-setup/SKILL.md +194 -312
- package/ai/skills/testdriver-bring-your-own-machine/SKILL.md +137 -204
- package/ai/skills/testdriver-cache/SKILL.md +142 -149
- package/ai/skills/testdriver-caching/SKILL.md +97 -87
- package/ai/skills/testdriver-captcha/SKILL.md +126 -111
- package/ai/skills/testdriver-ci-cd/SKILL.md +153 -542
- package/ai/skills/testdriver-click/SKILL.md +74 -229
- package/ai/skills/testdriver-client/SKILL.md +175 -229
- package/ai/skills/testdriver-customizing-devices/SKILL.md +72 -250
- package/ai/skills/testdriver-dashcam/SKILL.md +71 -404
- package/ai/skills/testdriver-debugging-with-screenshots/SKILL.md +103 -369
- package/ai/skills/testdriver-double-click/SKILL.md +46 -67
- package/ai/skills/testdriver-errors/SKILL.md +102 -176
- package/ai/skills/testdriver-events/SKILL.md +170 -327
- package/ai/skills/testdriver-exec/SKILL.md +92 -256
- package/ai/skills/testdriver-extract/SKILL.md +69 -170
- package/ai/skills/testdriver-find/SKILL.md +140 -736
- package/ai/skills/testdriver-focus-application/SKILL.md +69 -246
- package/ai/skills/testdriver-generating-tests/SKILL.md +102 -136
- package/ai/skills/testdriver-hosted/SKILL.md +79 -143
- package/ai/skills/testdriver-hover/SKILL.md +62 -235
- package/ai/skills/testdriver-locating-elements/SKILL.md +71 -366
- package/ai/skills/testdriver-making-assertions/SKILL.md +114 -72
- package/ai/skills/testdriver-mcp-setup/SKILL.md +79 -62
- package/ai/skills/testdriver-mouse-down/SKILL.md +46 -124
- package/ai/skills/testdriver-mouse-up/SKILL.md +45 -128
- package/ai/skills/testdriver-options/SKILL.md +73 -46
- package/ai/skills/testdriver-parse/SKILL.md +95 -184
- package/ai/skills/testdriver-performing-actions/SKILL.md +101 -91
- package/ai/skills/testdriver-press-keys/SKILL.md +59 -299
- package/ai/skills/testdriver-provision/SKILL.md +135 -179
- package/ai/skills/testdriver-quickstart/SKILL.md +24 -8
- package/ai/skills/testdriver-quickstart-claude/SKILL.md +84 -0
- package/ai/skills/testdriver-quickstart-github/SKILL.md +62 -32
- package/ai/skills/testdriver-quickstart-manual/SKILL.md +59 -32
- package/ai/skills/testdriver-quickstart-vscode/SKILL.md +96 -0
- package/ai/skills/testdriver-redraw/SKILL.md +100 -164
- package/ai/skills/testdriver-reusable-code/SKILL.md +95 -199
- package/ai/skills/testdriver-right-click/SKILL.md +45 -88
- package/ai/skills/testdriver-screenshot/SKILL.md +47 -201
- package/ai/skills/testdriver-screenshots/SKILL.md +86 -136
- package/ai/skills/testdriver-scroll/SKILL.md +71 -154
- package/ai/skills/testdriver-secrets/SKILL.md +94 -90
- package/ai/skills/testdriver-self-hosted/SKILL.md +82 -140
- package/ai/skills/testdriver-test-results-json/SKILL.md +165 -202
- package/ai/skills/testdriver-type/SKILL.md +56 -304
- package/ai/skills/testdriver-variables/SKILL.md +119 -80
- package/ai/skills/testdriver-wait/SKILL.md +31 -29
- package/ai/skills/testdriver-what-is-testdriver/SKILL.md +80 -32
- package/package.json +1 -1
|
@@ -1,32 +1,46 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: testdriver:options
|
|
3
|
-
description: Every option you can pass to
|
|
3
|
+
description: Every option you can pass to a TestDriver client, with its default.
|
|
4
4
|
---
|
|
5
5
|
<!-- Generated from options.mdx. DO NOT EDIT. -->
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
import { EnterpriseBadge, EnterpriseCta } from '/snippets/enterprise.mdx';
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
Options control the sandbox, the live preview, the AI, the cache, and redraw detection. A [quick reference](#quick-reference) table at the end lists every option and its default.
|
|
10
|
+
|
|
11
|
+
## Where to set options
|
|
12
|
+
|
|
13
|
+
Pass options in any of three places:
|
|
10
14
|
|
|
11
15
|
```javascript
|
|
12
|
-
// 1. Vitest hook (most common)
|
|
13
|
-
const testdriver = TestDriver(context, { os:
|
|
16
|
+
// 1. The Vitest hook (most common)
|
|
17
|
+
const testdriver = TestDriver(context, { os: 'windows' });
|
|
14
18
|
|
|
15
|
-
// 2.
|
|
16
|
-
const testdriver = new TestDriver(
|
|
19
|
+
// 2. The client constructor
|
|
20
|
+
const testdriver = new TestDriver({ os: 'windows' });
|
|
21
|
+
```
|
|
17
22
|
|
|
18
|
-
|
|
19
|
-
|
|
23
|
+
```javascript vitest.config.mjs
|
|
24
|
+
// 3. The Vitest plugin, for every test in the project
|
|
25
|
+
import { defineConfig } from 'vitest/config';
|
|
26
|
+
import TestDriver from 'testdriverai/vitest';
|
|
27
|
+
|
|
28
|
+
export default defineConfig({
|
|
29
|
+
test: {
|
|
30
|
+
reporters: ['default', TestDriver({ os: 'windows' })],
|
|
31
|
+
setupFiles: ['testdriverai/vitest/setup'],
|
|
32
|
+
},
|
|
33
|
+
});
|
|
20
34
|
```
|
|
21
35
|
|
|
22
|
-
|
|
36
|
+
When the same option is set in more than one place, the most specific wins:
|
|
23
37
|
|
|
24
|
-
1.
|
|
25
|
-
2.
|
|
26
|
-
3.
|
|
27
|
-
4.
|
|
38
|
+
1. Options you pass to the hook or the constructor
|
|
39
|
+
2. Plugin options in `vitest.config`
|
|
40
|
+
3. Environment variables: `TD_OS`, `TD_API_ROOT`, and `TD_E2B_TEMPLATE_ID`
|
|
41
|
+
4. Defaults
|
|
28
42
|
|
|
29
|
-
Options
|
|
43
|
+
Options you pass to [`connect()`](/client#connect-options) override the client's values for that connection only.
|
|
30
44
|
|
|
31
45
|
## Sandbox
|
|
32
46
|
|
|
@@ -34,36 +48,42 @@ These options control the virtual machine that runs your test.
|
|
|
34
48
|
|
|
35
49
|
<ParamField path="os" type="'linux' | 'windows'" default="linux">
|
|
36
50
|
The operating system of the sandbox. You can also set this with the `TD_OS` environment variable.
|
|
51
|
+
|
|
52
|
+
<EnterpriseBadge /> `'windows'` is available on Enterprise plans.
|
|
37
53
|
</ParamField>
|
|
38
54
|
|
|
39
55
|
<ParamField path="resolution" type="string" default="1366x768">
|
|
40
|
-
The screen resolution of the sandbox
|
|
56
|
+
The screen resolution of the sandbox, such as `'1920x1080'`.
|
|
57
|
+
|
|
58
|
+
<EnterpriseBadge /> Custom resolutions are available on Enterprise plans.
|
|
41
59
|
</ParamField>
|
|
42
60
|
|
|
43
61
|
<ParamField path="newSandbox" type="boolean" default="true">
|
|
44
|
-
Create a new sandbox for this run. Set to `false` to reuse a recent sandbox when one is available.
|
|
62
|
+
Create a new sandbox for this run. Set to `false` to reuse a recent sandbox when one is available. Defaults to `false` when you set `sandboxId` or `reconnect`.
|
|
45
63
|
</ParamField>
|
|
46
64
|
|
|
47
65
|
<ParamField path="reconnect" type="boolean" default="false">
|
|
48
|
-
Reattach to the last used
|
|
66
|
+
Reattach to the last sandbox this project used. The SDK records it in `.testdriver/last-sandbox` on every successful connect.
|
|
49
67
|
|
|
50
|
-
|
|
68
|
+
While `reconnect` is `true`, provision methods such as `chrome()` and `installer()` are skipped, because the app is already running. The sandbox must still be alive. See [`keepAlive`](#param-keep-alive) and [Reuse a sandbox while you iterate](/copilot/running-tests#reuse-a-sandbox-while-you-iterate).
|
|
51
69
|
</ParamField>
|
|
52
70
|
|
|
53
71
|
<ParamField path="sandboxId" type="string">
|
|
54
|
-
|
|
72
|
+
Attach to a specific sandbox by its id, for example to pin a chain of tests to one machine. This sets `newSandbox` to `false`.
|
|
73
|
+
|
|
74
|
+
Unlike `reconnect`, `sandboxId` alone doesn't skip provision calls. Set `reconnect: true` as well if the app is already running.
|
|
55
75
|
</ParamField>
|
|
56
76
|
|
|
57
77
|
<ParamField path="keepAlive" type="number" default="60000">
|
|
58
|
-
The number of milliseconds to keep the sandbox
|
|
78
|
+
The number of milliseconds to keep the sandbox running after `disconnect()`, so you can reconnect to the same sandbox for debugging. Values above 30 minutes are capped at 30 minutes, and `0` uses the default.
|
|
59
79
|
</ParamField>
|
|
60
80
|
|
|
61
81
|
<ParamField path="debugOnFailure" type="boolean" default="false">
|
|
62
|
-
Keep the sandbox
|
|
82
|
+
Keep the sandbox running when a test fails, and print its id so you can reconnect with `sandboxId` and debug. Available only in the Vitest hook.
|
|
63
83
|
</ParamField>
|
|
64
84
|
|
|
65
85
|
<ParamField path="environment" type="object">
|
|
66
|
-
|
|
86
|
+
Extra `TD_*` settings for the SDK itself, as key-value pairs. These configure TestDriver on your machine, not environment variables inside the sandbox. To set variables in the sandbox, run a command with [`exec()`](/exec).
|
|
67
87
|
</ParamField>
|
|
68
88
|
|
|
69
89
|
### Example
|
|
@@ -85,7 +105,7 @@ These options control what you see while the test runs.
|
|
|
85
105
|
The preview mode for live test visualization.
|
|
86
106
|
|
|
87
107
|
- `"browser"` opens the debugger in your default browser.
|
|
88
|
-
- `"ide"` opens the preview in
|
|
108
|
+
- `"ide"` opens the preview in an editor panel. This needs the [TestDriver extension](/quickstart-vscode) for VS Code or Cursor.
|
|
89
109
|
- `"none"` runs without a visual preview. Use this in CI.
|
|
90
110
|
</ParamField>
|
|
91
111
|
|
|
@@ -98,13 +118,11 @@ These options control what you see while the test runs.
|
|
|
98
118
|
</ParamField>
|
|
99
119
|
|
|
100
120
|
<ParamField path="autoScreenshots" type="boolean" default="false">
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
The file name format is `<seq>-<action>-<phase>-L<line>-<description>.png`. Example: `001-click-before-L42-submit-button.png`. See [Debugging with screenshots](/debugging-with-screenshots).
|
|
121
|
+
Save a screenshot before and after each `testdriver` method call and each `find()`, and when one fails, to `.testdriver/screenshots/<test file>/`. Actions on an element, like `element.click()`, aren't captured. Files are named like `001-find-before-L42-submit-button.png`. See [Screenshots](/screenshots).
|
|
104
122
|
</ParamField>
|
|
105
123
|
|
|
106
124
|
<ParamField path="logging" type="boolean" default="true">
|
|
107
|
-
Write SDK logs to the console. You can change this at runtime with [`setLogging()`](/client#setlogging).
|
|
125
|
+
Write SDK logs to the console. You can change this at runtime with [`setLogging()`](/client#setlogging-enabled).
|
|
108
126
|
</ParamField>
|
|
109
127
|
|
|
110
128
|
<ParamField path="analytics" type="boolean" default="true">
|
|
@@ -125,13 +143,21 @@ const testdriver = TestDriver(context, {
|
|
|
125
143
|
These options control how the AI locates elements and checks assertions.
|
|
126
144
|
|
|
127
145
|
<ParamField path="verify" type="boolean" default="false">
|
|
128
|
-
|
|
146
|
+
Double-check every `find()` result with a second AI call. Off by default, because it makes each call slower.
|
|
129
147
|
|
|
130
|
-
|
|
148
|
+
When on, after the AI locates an element:
|
|
149
|
+
|
|
150
|
+
1. A second AI looks at that spot and confirms it's the element you described.
|
|
151
|
+
2. If it isn't, TestDriver locates the element again with a hint about what was wrong, then checks once more.
|
|
152
|
+
3. If that check also fails, `find()` returns an element whose `found()` is `false`.
|
|
153
|
+
|
|
154
|
+
Results that fail the check are never cached. Results served from the cache aren't re-checked.
|
|
155
|
+
|
|
156
|
+
Override it for one call with `find(description, { verify: true })`. See [find()](/find).
|
|
131
157
|
</ParamField>
|
|
132
158
|
|
|
133
159
|
<ParamField path="ai" type="object">
|
|
134
|
-
Global AI sampling configuration. These values apply to `find()` verification and to `assert()`. You can
|
|
160
|
+
Global AI sampling configuration. These values apply to `find()` verification and to `assert()`. You can override them for one `assert()` call with its `ai` option. `find()` ignores a per-call `ai` option, so set it here.
|
|
135
161
|
|
|
136
162
|
<Expandable title="properties">
|
|
137
163
|
<ParamField path="temperature" type="number">
|
|
@@ -165,16 +191,14 @@ const testdriver = TestDriver(context, {
|
|
|
165
191
|
|
|
166
192
|
## Caching
|
|
167
193
|
|
|
168
|
-
These options control the element cache
|
|
194
|
+
These options control the element cache, which reuses element positions from a passing run when the screen hasn't changed. See [Caching](/caching).
|
|
169
195
|
|
|
170
196
|
<ParamField path="cache" type="boolean | object" default="true">
|
|
171
|
-
|
|
197
|
+
Pass `false` to turn the cache off, or an object to set the match thresholds.
|
|
172
198
|
|
|
173
|
-
|
|
174
|
-
<ParamField path="enabled" type="boolean" default="true">
|
|
175
|
-
Enable or disable the cache.
|
|
176
|
-
</ParamField>
|
|
199
|
+
`cache: { enabled: false }` does not turn the cache off. Use `cache: false` or `TD_NO_CACHE=true`.
|
|
177
200
|
|
|
201
|
+
<Expandable title="advanced config">
|
|
178
202
|
<ParamField path="thresholds" type="object">
|
|
179
203
|
Fine-tune the cache matching.
|
|
180
204
|
|
|
@@ -202,7 +226,7 @@ These options control the element cache. The cache stores the position of an ele
|
|
|
202
226
|
</ParamField>
|
|
203
227
|
|
|
204
228
|
<ParamField path="cacheKey" type="string">
|
|
205
|
-
The
|
|
229
|
+
The key the cache is stored under. By default, the SDK uses a hash of the test file, so editing the file starts a fresh cache. Set your own key to keep the cache across edits, or to share it between files.
|
|
206
230
|
</ParamField>
|
|
207
231
|
|
|
208
232
|
<ParamField path="cacheThreshold" type="object">
|
|
@@ -214,7 +238,6 @@ These options control the element cache. The cache stores the position of an ele
|
|
|
214
238
|
```javascript
|
|
215
239
|
const testdriver = TestDriver(context, {
|
|
216
240
|
cache: {
|
|
217
|
-
enabled: true,
|
|
218
241
|
thresholds: {
|
|
219
242
|
find: { screen: 0.05, element: 0.8 },
|
|
220
243
|
assert: 0.05,
|
|
@@ -226,14 +249,14 @@ const testdriver = TestDriver(context, {
|
|
|
226
249
|
|
|
227
250
|
## Redraw detection
|
|
228
251
|
|
|
229
|
-
These options
|
|
252
|
+
These options make the SDK wait for the screen to settle after each action. Redraw detection is off by default. See [Redraw](/redraw).
|
|
230
253
|
|
|
231
|
-
<ParamField path="redraw" type="boolean | object" default="
|
|
232
|
-
|
|
254
|
+
<ParamField path="redraw" type="boolean | object" default="false">
|
|
255
|
+
Turn on screen-change detection with `true`, or pass an object to set the thresholds.
|
|
233
256
|
|
|
234
257
|
<Expandable title="advanced config">
|
|
235
258
|
<ParamField path="enabled" type="boolean" default="true">
|
|
236
|
-
|
|
259
|
+
Whether redraw detection is on when you pass an object.
|
|
237
260
|
</ParamField>
|
|
238
261
|
|
|
239
262
|
<ParamField path="thresholds" type="object">
|
|
@@ -241,7 +264,7 @@ These options control how the SDK waits for the screen to settle after an action
|
|
|
241
264
|
|
|
242
265
|
<Expandable title="properties">
|
|
243
266
|
<ParamField path="screen" type="number | false" default="0.05">
|
|
244
|
-
|
|
267
|
+
Set to `false` to turn off screen-change detection and wait only for the network. Any number keeps it on; the number itself isn't used, because the change threshold is fixed.
|
|
245
268
|
</ParamField>
|
|
246
269
|
|
|
247
270
|
<ParamField path="network" type="boolean" default="false">
|
|
@@ -268,7 +291,9 @@ const testdriver = TestDriver(context, {
|
|
|
268
291
|
|
|
269
292
|
## Self-hosted and infrastructure
|
|
270
293
|
|
|
271
|
-
Most
|
|
294
|
+
Most projects don't need these. They're for self-hosted or custom deployments.
|
|
295
|
+
|
|
296
|
+
<EnterpriseCta feature="Self-hosting" />
|
|
272
297
|
|
|
273
298
|
<ParamField path="apiRoot" type="string">
|
|
274
299
|
The API endpoint URL. The default depends on the release channel. You can also set this with the `TD_API_ROOT` environment variable.
|
|
@@ -276,6 +301,8 @@ Most users do not need these options. Use them for self-hosted or custom deploym
|
|
|
276
301
|
|
|
277
302
|
<ParamField path="ip" type="string">
|
|
278
303
|
The IP address of a running sandbox. The SDK connects to this address directly.
|
|
304
|
+
|
|
305
|
+
<EnterpriseBadge /> Connecting to your own machines is available on Enterprise plans.
|
|
279
306
|
</ParamField>
|
|
280
307
|
|
|
281
308
|
<ParamField path="sandboxAmi" type="string">
|
|
@@ -311,7 +338,7 @@ Most users do not need these options. Use them for self-hosted or custom deploym
|
|
|
311
338
|
| `ai` | `object` | — |
|
|
312
339
|
| `cache` | `boolean \| object` | `true` |
|
|
313
340
|
| `cacheKey` | `string` | — |
|
|
314
|
-
| `redraw` | `boolean \| object` | `
|
|
341
|
+
| `redraw` | `boolean \| object` | `false` |
|
|
315
342
|
| `apiRoot` | `string` | channel default |
|
|
316
343
|
| `ip` | `string` | — |
|
|
317
344
|
| `sandboxAmi` | `string` | — |
|
|
@@ -1,236 +1,147 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: testdriver:parse
|
|
3
|
-
description:
|
|
3
|
+
description: List every UI element on screen, with its type, text, and position.
|
|
4
4
|
---
|
|
5
5
|
<!-- Generated from parse.mdx. DO NOT EDIT. -->
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
Detect every element on the current screen and get back a list with each one's type, text, and position.
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
```javascript
|
|
10
|
+
const { elements } = await testdriver.parse();
|
|
11
|
+
console.log(`Found ${elements.length} elements`);
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
`parse()` uses [OmniParser v2](https://github.com/microsoft/OmniParser). It works from the screenshot alone, so it doesn't need a DOM or accessibility tree.
|
|
10
15
|
|
|
11
|
-
|
|
12
|
-
- To understand the full UI layout of a screen
|
|
13
|
-
- To find all clickable or interactive elements
|
|
14
|
-
- To build custom element logic
|
|
15
|
-
- To debug which elements TestDriver can find
|
|
16
|
-
- To do an accessibility audit
|
|
16
|
+
To act on one specific element, use [`find()`](/find) instead. Use `parse()` when you need an inventory of the whole screen.
|
|
17
17
|
|
|
18
18
|
<Note>
|
|
19
|
-
|
|
19
|
+
`parse()` requires an Enterprise or self-hosted plan.
|
|
20
20
|
</Note>
|
|
21
21
|
|
|
22
|
-
##
|
|
22
|
+
## Signature
|
|
23
23
|
|
|
24
|
-
```
|
|
25
|
-
|
|
24
|
+
```typescript
|
|
25
|
+
testdriver.parse(): Promise<ParseResult>
|
|
26
26
|
```
|
|
27
27
|
|
|
28
28
|
## Parameters
|
|
29
29
|
|
|
30
|
-
None.
|
|
30
|
+
None. `parse()` analyzes the screen as it is when you call it.
|
|
31
31
|
|
|
32
32
|
## Returns
|
|
33
33
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
34
|
+
<ResponseField name="ParseResult" type="object">
|
|
35
|
+
<Expandable title="properties">
|
|
36
|
+
<ResponseField name="elements" type="ParsedElement[]">
|
|
37
|
+
One entry per detected element.
|
|
38
|
+
<Expandable title="ParsedElement">
|
|
39
|
+
<ResponseField name="index" type="number">
|
|
40
|
+
The element's position in the list, starting at `0`.
|
|
41
|
+
</ResponseField>
|
|
42
|
+
<ResponseField name="type" type="string">
|
|
43
|
+
What OmniParser detected, such as `'text'` or `'icon'`.
|
|
44
|
+
</ResponseField>
|
|
45
|
+
<ResponseField name="content" type="string">
|
|
46
|
+
The element's text, or a short description of an icon.
|
|
47
|
+
</ResponseField>
|
|
48
|
+
<ResponseField name="interactivity" type="boolean">
|
|
49
|
+
`true` when OmniParser thinks the element can be clicked.
|
|
50
|
+
</ResponseField>
|
|
51
|
+
<ResponseField name="bbox" type="object">
|
|
52
|
+
The corners in pixels: `{ x0, y0, x1, y1 }`, where `(x0, y0)` is top-left and `(x1, y1)` is bottom-right.
|
|
53
|
+
</ResponseField>
|
|
54
|
+
<ResponseField name="boundingBox" type="object">
|
|
55
|
+
The same box as `{ left, top, width, height }`, in pixels.
|
|
56
|
+
</ResponseField>
|
|
57
|
+
</Expandable>
|
|
58
|
+
</ResponseField>
|
|
59
|
+
<ResponseField name="annotatedImageUrl" type="string | null">
|
|
60
|
+
A URL of the screenshot with every detected element outlined.
|
|
61
|
+
</ResponseField>
|
|
62
|
+
<ResponseField name="imageWidth" type="number">
|
|
63
|
+
The width of the analyzed screenshot, in pixels.
|
|
64
|
+
</ResponseField>
|
|
65
|
+
<ResponseField name="imageHeight" type="number">
|
|
66
|
+
The height of the analyzed screenshot, in pixels.
|
|
67
|
+
</ResponseField>
|
|
68
|
+
</Expandable>
|
|
69
|
+
</ResponseField>
|
|
70
|
+
|
|
71
|
+
<Warning>
|
|
72
|
+
`interactivity` is a boolean, although the TypeScript types in `sdk.d.ts` declare it as a string. Compare it with `true`, not with a string like `'clickable'`.
|
|
73
|
+
</Warning>
|
|
74
|
+
|
|
75
|
+
## Errors
|
|
76
|
+
|
|
77
|
+
`parse()` throws if the screen analysis fails. The error message comes from the TestDriver API.
|
|
55
78
|
|
|
56
79
|
## Examples
|
|
57
80
|
|
|
58
|
-
###
|
|
81
|
+
### List what's on screen
|
|
59
82
|
|
|
60
83
|
```javascript
|
|
61
|
-
const
|
|
62
|
-
console.log(`Found ${result.elements.length} elements`);
|
|
84
|
+
const { elements } = await testdriver.parse();
|
|
63
85
|
|
|
64
|
-
|
|
65
|
-
console.log(
|
|
66
|
-
}
|
|
86
|
+
for (const element of elements) {
|
|
87
|
+
console.log(`[${element.type}] "${element.content}" clickable=${element.interactivity}`);
|
|
88
|
+
}
|
|
67
89
|
```
|
|
68
90
|
|
|
69
|
-
### Find
|
|
91
|
+
### Find the clickable elements
|
|
70
92
|
|
|
71
93
|
```javascript
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
const clickable = result.elements.filter(e => e.interactivity === 'clickable');
|
|
75
|
-
console.log(`Found ${clickable.length} clickable elements`);
|
|
76
|
-
|
|
77
|
-
clickable.forEach(el => {
|
|
78
|
-
console.log(`- "${el.content}" at (${el.bbox.x0}, ${el.bbox.y0})`);
|
|
79
|
-
});
|
|
94
|
+
const { elements } = await testdriver.parse();
|
|
95
|
+
const clickable = elements.filter((element) => element.interactivity);
|
|
80
96
|
```
|
|
81
97
|
|
|
82
|
-
###
|
|
98
|
+
### Click an element by its text
|
|
83
99
|
|
|
84
100
|
```javascript
|
|
85
|
-
const
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
);
|
|
91
|
-
|
|
92
|
-
if (submitBtn) {
|
|
93
|
-
// Calculate center of the bounding box
|
|
94
|
-
const x = Math.round((submitBtn.bbox.x0 + submitBtn.bbox.x1) / 2);
|
|
95
|
-
const y = Math.round((submitBtn.bbox.y0 + submitBtn.bbox.y1) / 2);
|
|
96
|
-
|
|
101
|
+
const { elements } = await testdriver.parse();
|
|
102
|
+
const signIn = elements.find((element) => element.content.includes('Sign in'));
|
|
103
|
+
|
|
104
|
+
if (signIn) {
|
|
105
|
+
const x = Math.round((signIn.bbox.x0 + signIn.bbox.x1) / 2);
|
|
106
|
+
const y = Math.round((signIn.bbox.y0 + signIn.bbox.y1) / 2);
|
|
97
107
|
await testdriver.click({ x, y });
|
|
98
108
|
}
|
|
99
109
|
```
|
|
100
110
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
```javascript
|
|
104
|
-
const result = await testdriver.parse();
|
|
105
|
-
|
|
106
|
-
// Get all text elements
|
|
107
|
-
const textElements = result.elements.filter(e => e.type === 'text');
|
|
108
|
-
textElements.forEach(e => console.log(`Text: "${e.content}"`));
|
|
111
|
+
Clicking by coordinates breaks when the layout changes. Prefer `find()` for clicks.
|
|
109
112
|
|
|
110
|
-
|
|
111
|
-
const icons = result.elements.filter(e => e.type === 'icon');
|
|
112
|
-
console.log(`Found ${icons.length} icons`);
|
|
113
|
+
### See what was detected
|
|
113
114
|
|
|
114
|
-
|
|
115
|
-
const
|
|
116
|
-
console.log(
|
|
115
|
+
```javascript
|
|
116
|
+
const { annotatedImageUrl } = await testdriver.parse();
|
|
117
|
+
console.log(annotatedImageUrl);
|
|
117
118
|
```
|
|
118
119
|
|
|
119
|
-
|
|
120
|
+
## Complete example
|
|
120
121
|
|
|
121
|
-
```javascript
|
|
122
|
-
import { describe, expect, it } from
|
|
123
|
-
import { TestDriver } from
|
|
122
|
+
```javascript tests/login-form.test.mjs
|
|
123
|
+
import { describe, expect, it } from 'vitest';
|
|
124
|
+
import { TestDriver } from 'testdriverai/vitest/hooks';
|
|
124
125
|
|
|
125
|
-
describe(
|
|
126
|
-
it(
|
|
126
|
+
describe('Login page', () => {
|
|
127
|
+
it('shows the username and password fields', async (context) => {
|
|
127
128
|
const testdriver = TestDriver(context);
|
|
128
|
-
|
|
129
|
-
await testdriver.provision.chrome({
|
|
130
|
-
url: 'https://myapp.com/login',
|
|
131
|
-
});
|
|
132
|
-
|
|
133
|
-
const result = await testdriver.parse();
|
|
134
|
-
|
|
135
|
-
// Assert expected elements exist
|
|
136
|
-
const textContent = result.elements.map(e => e.content.toLowerCase());
|
|
137
|
-
expect(textContent).toContain('email');
|
|
138
|
-
expect(textContent).toContain('password');
|
|
139
|
-
|
|
140
|
-
// Assert there are clickable elements
|
|
141
|
-
const clickable = result.elements.filter(e => e.interactivity === 'clickable');
|
|
142
|
-
expect(clickable.length).toBeGreaterThan(0);
|
|
143
|
-
});
|
|
144
|
-
});
|
|
145
|
-
```
|
|
129
|
+
await testdriver.provision.chrome({ url: 'https://testdriver-sandbox.vercel.app/login' });
|
|
146
130
|
|
|
147
|
-
|
|
131
|
+
const { elements } = await testdriver.parse();
|
|
132
|
+
const text = elements.map((element) => element.content.toLowerCase()).join(' ');
|
|
148
133
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
// Pixel coordinates
|
|
154
|
-
console.log(`Element "${el.content}":`);
|
|
155
|
-
console.log(` bbox: (${el.bbox.x0}, ${el.bbox.y0}) to (${el.bbox.x1}, ${el.bbox.y1})`);
|
|
156
|
-
console.log(` size: ${el.boundingBox.width}x${el.boundingBox.height}`);
|
|
157
|
-
console.log(` position: left=${el.boundingBox.left}, top=${el.boundingBox.top}`);
|
|
134
|
+
expect(text).toContain('username');
|
|
135
|
+
expect(text).toContain('password');
|
|
136
|
+
expect(elements.some((element) => element.interactivity)).toBe(true);
|
|
137
|
+
});
|
|
158
138
|
});
|
|
159
139
|
```
|
|
160
140
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
```javascript
|
|
164
|
-
const result = await testdriver.parse();
|
|
165
|
-
|
|
166
|
-
// The annotated image shows all detected elements with bounding boxes
|
|
167
|
-
console.log('Annotated screenshot:', result.annotatedImageUrl);
|
|
168
|
-
console.log(`Image dimensions: ${result.imageWidth}x${result.imageHeight}`);
|
|
169
|
-
```
|
|
170
|
-
|
|
171
|
-
## How It Works
|
|
172
|
-
|
|
173
|
-
1. TestDriver captures a screenshot of the current screen
|
|
174
|
-
2. The image is sent to the TestDriver API
|
|
175
|
-
3. OmniParser v2 analyzes the image to detect all UI elements
|
|
176
|
-
4. Each element is classified by type (text, icon, button, etc.) and interactivity
|
|
177
|
-
5. Bounding box coordinates are returned in pixel coordinates matching the screen resolution
|
|
178
|
-
|
|
179
|
-
<Note>
|
|
180
|
-
OmniParser detects elements visually — it works with any UI framework, native apps, and even non-standard interfaces. It does not rely on DOM or accessibility trees.
|
|
181
|
-
</Note>
|
|
182
|
-
|
|
183
|
-
## Best Practices
|
|
184
|
-
|
|
185
|
-
<AccordionGroup>
|
|
186
|
-
<Accordion title="Use find() for targeting specific elements">
|
|
187
|
-
For locating and interacting with a specific element, prefer `find()` which uses AI vision. Use `parse()` when you need a complete inventory of all elements on screen.
|
|
188
|
-
|
|
189
|
-
```javascript
|
|
190
|
-
// Prefer this for clicking a specific element
|
|
191
|
-
await testdriver.find("Submit button").click();
|
|
192
|
-
|
|
193
|
-
// Use parse() for full UI analysis
|
|
194
|
-
const result = await testdriver.parse();
|
|
195
|
-
const allButtons = result.elements.filter(e => e.type === 'button');
|
|
196
|
-
```
|
|
197
|
-
</Accordion>
|
|
198
|
-
|
|
199
|
-
<Accordion title="Filter by interactivity">
|
|
200
|
-
Use the `interactivity` field to distinguish between clickable and non-interactive elements.
|
|
201
|
-
|
|
202
|
-
```javascript
|
|
203
|
-
const result = await testdriver.parse();
|
|
204
|
-
const interactive = result.elements.filter(e => e.interactivity === 'clickable');
|
|
205
|
-
const static_ = result.elements.filter(e => e.interactivity === 'non-interactive');
|
|
206
|
-
```
|
|
207
|
-
</Accordion>
|
|
208
|
-
|
|
209
|
-
<Accordion title="Wait for content to load">
|
|
210
|
-
If elements aren't being detected, the page may not be fully loaded. Add a wait first.
|
|
211
|
-
|
|
212
|
-
```javascript
|
|
213
|
-
// Wait for page to stabilize
|
|
214
|
-
await testdriver.wait(2000);
|
|
215
|
-
|
|
216
|
-
// Then parse
|
|
217
|
-
const result = await testdriver.parse();
|
|
218
|
-
```
|
|
219
|
-
</Accordion>
|
|
220
|
-
|
|
221
|
-
<Accordion title="Use the annotated image for debugging">
|
|
222
|
-
The `annotatedImageUrl` provides a visual overlay showing all detected elements with their bounding boxes — great for debugging.
|
|
223
|
-
|
|
224
|
-
```javascript
|
|
225
|
-
const result = await testdriver.parse();
|
|
226
|
-
console.log('View annotated screenshot:', result.annotatedImageUrl);
|
|
227
|
-
```
|
|
228
|
-
</Accordion>
|
|
229
|
-
</AccordionGroup>
|
|
141
|
+
See the [parse example](/examples/parse) for another complete test.
|
|
230
142
|
|
|
231
143
|
## Related
|
|
232
144
|
|
|
233
|
-
- [find()](/find)
|
|
234
|
-
- [
|
|
235
|
-
- [
|
|
236
|
-
- [Elements Reference](/elements) - Complete Element API
|
|
145
|
+
- [find()](/find): locate one element and act on it
|
|
146
|
+
- [extract()](/extract): read one value off the screen
|
|
147
|
+
- [click()](/click): click a point by its coordinates
|