@testdriverai/agent 7.9.77-test → 7.9.78-test
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/_data/examples-manifest.json +52 -52
- package/docs/changelog.mdx +1 -1
- package/docs/v7/_drafts/best-practices.mdx +0 -12
- package/docs/v7/_drafts/configuration.mdx +0 -12
- package/docs/v7/_drafts/faq.mdx +2 -10
- package/docs/v7/_drafts/troubleshooting.mdx +1 -4
- package/docs/v7/examples/ai.mdx +1 -1
- package/docs/v7/examples/assert.mdx +1 -1
- package/docs/v7/examples/chrome-extension.mdx +1 -1
- package/docs/v7/examples/element-not-found.mdx +1 -1
- package/docs/v7/examples/findall-coffee-icons.mdx +1 -1
- package/docs/v7/examples/hover-image.mdx +1 -1
- package/docs/v7/examples/hover-text-with-description.mdx +1 -1
- package/docs/v7/examples/hover-text.mdx +1 -1
- package/docs/v7/examples/installer.mdx +1 -1
- package/docs/v7/examples/launch-vscode-linux.mdx +1 -1
- package/docs/v7/examples/parse.mdx +1 -1
- package/docs/v7/examples/press-keys.mdx +1 -1
- package/docs/v7/examples/scroll-keyboard.mdx +1 -1
- package/docs/v7/examples/scroll.mdx +1 -1
- package/docs/v7/examples/type.mdx +1 -1
- package/package.json +1 -1
- package/docs/v7/_drafts/caching-ai.mdx +0 -215
- package/docs/v7/_drafts/caching.mdx +0 -366
- package/docs/v7/_drafts/prompt-cache.mdx +0 -200
|
@@ -2,18 +2,18 @@
|
|
|
2
2
|
"$schema": "./examples-manifest.schema.json",
|
|
3
3
|
"examples": {
|
|
4
4
|
"assert.test.mjs": {
|
|
5
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
6
|
-
"replayUrl": "https://console-test.testdriver.ai/replay/
|
|
7
|
-
"embedUrl": "https://console-test.testdriver.ai/replay/
|
|
8
|
-
"lastUpdated": "2026-05-
|
|
5
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a62d3fb50e9fd352c6501",
|
|
6
|
+
"replayUrl": "https://console-test.testdriver.ai/replay/6a0a69f85e84df6215f782f4?share=AaQZE9gfcazcIyAwZ54Dtg",
|
|
7
|
+
"embedUrl": "https://console-test.testdriver.ai/replay/6a0a69f85e84df6215f782f4?share=AaQZE9gfcazcIyAwZ54Dtg&embed=true",
|
|
8
|
+
"lastUpdated": "2026-05-18T01:23:09.954Z"
|
|
9
9
|
},
|
|
10
10
|
"drag-and-drop.test.mjs": {
|
|
11
11
|
"url": "https://console.testdriver.ai/runs/69a62b3aaa712ecd3dea730a/69a62b42fc0ac3cc632a918b",
|
|
12
12
|
"lastUpdated": "2026-03-03T00:32:25.275Z"
|
|
13
13
|
},
|
|
14
14
|
"exec-pwsh.test.mjs": {
|
|
15
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
16
|
-
"lastUpdated": "2026-05-
|
|
15
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a61f2fb50e9fd352c6475",
|
|
16
|
+
"lastUpdated": "2026-05-18T01:02:42.315Z"
|
|
17
17
|
},
|
|
18
18
|
"match-image.test.mjs": {
|
|
19
19
|
"url": "https://console-test.testdriver.ai/runs/69c8738614b73310c7839412/69c8738c14b73310c783941d",
|
|
@@ -24,92 +24,92 @@
|
|
|
24
24
|
"lastUpdated": "2026-03-03T00:32:25.282Z"
|
|
25
25
|
},
|
|
26
26
|
"hover-text-with-description.test.mjs": {
|
|
27
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
28
|
-
"lastUpdated": "2026-05-
|
|
27
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a6208fb50e9fd352c64ab",
|
|
28
|
+
"lastUpdated": "2026-05-18T00:49:12.155Z"
|
|
29
29
|
},
|
|
30
30
|
"windows-installer.test.mjs": {
|
|
31
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
32
|
-
"lastUpdated": "2026-05-
|
|
31
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a6234fb50e9fd352c64f2",
|
|
32
|
+
"lastUpdated": "2026-05-18T01:03:48.466Z"
|
|
33
33
|
},
|
|
34
34
|
"exec-output.test.mjs": {
|
|
35
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
36
|
-
"lastUpdated": "2026-05-
|
|
35
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a6235fb50e9fd352c64f4",
|
|
36
|
+
"lastUpdated": "2026-05-18T01:03:49.902Z"
|
|
37
37
|
},
|
|
38
38
|
"chrome-extension.test.mjs": {
|
|
39
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
40
|
-
"lastUpdated": "2026-05-
|
|
39
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a61f0fb50e9fd352c646e",
|
|
40
|
+
"lastUpdated": "2026-05-18T00:55:44.331Z"
|
|
41
41
|
},
|
|
42
42
|
"launch-vscode-linux.test.mjs": {
|
|
43
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
44
|
-
"replayUrl": "https://console-test.testdriver.ai/replay/
|
|
45
|
-
"embedUrl": "https://console-test.testdriver.ai/replay/
|
|
46
|
-
"lastUpdated": "2026-05-
|
|
43
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a621dfb50e9fd352c64d5",
|
|
44
|
+
"replayUrl": "https://console-test.testdriver.ai/replay/6a0a67d35e84df6215f782e4?share=BZj28mmTsITtFqfMHy74Hw",
|
|
45
|
+
"embedUrl": "https://console-test.testdriver.ai/replay/6a0a67d35e84df6215f782e4?share=BZj28mmTsITtFqfMHy74Hw&embed=true",
|
|
46
|
+
"lastUpdated": "2026-05-18T01:14:01.776Z"
|
|
47
47
|
},
|
|
48
48
|
"hover-image.test.mjs": {
|
|
49
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
50
|
-
"lastUpdated": "2026-05-
|
|
49
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a624bfb50e9fd352c64f7",
|
|
50
|
+
"lastUpdated": "2026-05-18T00:57:17.705Z"
|
|
51
51
|
},
|
|
52
52
|
"installer.test.mjs": {
|
|
53
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
54
|
-
"lastUpdated": "2026-05-
|
|
53
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a6264fb50e9fd352c64fc",
|
|
54
|
+
"lastUpdated": "2026-05-18T00:50:44.205Z"
|
|
55
55
|
},
|
|
56
56
|
"type.test.mjs": {
|
|
57
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
58
|
-
"lastUpdated": "2026-05-
|
|
57
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a62a5fb50e9fd352c64ff",
|
|
58
|
+
"lastUpdated": "2026-05-18T00:58:43.676Z"
|
|
59
59
|
},
|
|
60
60
|
"press-keys.test.mjs": {
|
|
61
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
62
|
-
"replayUrl": "https://console-test.testdriver.ai/replay/
|
|
63
|
-
"embedUrl": "https://console-test.testdriver.ai/replay/
|
|
64
|
-
"lastUpdated": "2026-05-
|
|
61
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a62bcfb50e9fd352c6500",
|
|
62
|
+
"replayUrl": "https://console-test.testdriver.ai/replay/6a0a69b95e84df6215f782f2?share=YxG4LqDBIszM8ed0axBXCw",
|
|
63
|
+
"embedUrl": "https://console-test.testdriver.ai/replay/6a0a69b95e84df6215f782f2?share=YxG4LqDBIszM8ed0axBXCw&embed=true",
|
|
64
|
+
"lastUpdated": "2026-05-18T01:22:07.432Z"
|
|
65
65
|
},
|
|
66
66
|
"scroll-keyboard.test.mjs": {
|
|
67
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
68
|
-
"lastUpdated": "2026-05-
|
|
67
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a6302fb50e9fd352c6503",
|
|
68
|
+
"lastUpdated": "2026-05-18T01:07:11.127Z"
|
|
69
69
|
},
|
|
70
70
|
"scroll.test.mjs": {
|
|
71
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
72
|
-
"replayUrl": "https://console-test.testdriver.ai/replay/
|
|
73
|
-
"embedUrl": "https://console-test.testdriver.ai/replay/
|
|
74
|
-
"lastUpdated": "2026-05-
|
|
71
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a631efb50e9fd352c6507",
|
|
72
|
+
"replayUrl": "https://console-test.testdriver.ai/replay/6a0a6afe5e84df6215f782fa?share=OuQYCYU4nb44QRLS6JCejQ",
|
|
73
|
+
"embedUrl": "https://console-test.testdriver.ai/replay/6a0a6afe5e84df6215f782fa?share=OuQYCYU4nb44QRLS6JCejQ&embed=true",
|
|
74
|
+
"lastUpdated": "2026-05-18T01:27:32.768Z"
|
|
75
75
|
},
|
|
76
76
|
"scroll-until-image.test.mjs": {
|
|
77
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
78
|
-
"lastUpdated": "2026-05-
|
|
77
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a6304fb50e9fd352c6504",
|
|
78
|
+
"lastUpdated": "2026-05-18T00:53:24.479Z"
|
|
79
79
|
},
|
|
80
80
|
"prompt.test.mjs": {
|
|
81
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
82
|
-
"lastUpdated": "2026-05-
|
|
81
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a6305fb50e9fd352c6505",
|
|
82
|
+
"lastUpdated": "2026-05-18T01:07:14.174Z"
|
|
83
83
|
},
|
|
84
84
|
"focus-window.test.mjs": {
|
|
85
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
86
|
-
"lastUpdated": "2026-05-
|
|
85
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a6307fb50e9fd352c6506",
|
|
86
|
+
"lastUpdated": "2026-05-18T00:53:27.493Z"
|
|
87
87
|
},
|
|
88
88
|
"captcha-api.test.mjs": {
|
|
89
89
|
"url": "https://console.testdriver.ai/runs/698f7df69e27ce1528d7d087/698f7fb0d3b320ad547d9d44",
|
|
90
90
|
"lastUpdated": "2026-02-13T19:55:05.951Z"
|
|
91
91
|
},
|
|
92
92
|
"element-not-found.test.mjs": {
|
|
93
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
94
|
-
"lastUpdated": "2026-05-
|
|
93
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a6334fb50e9fd352c6508",
|
|
94
|
+
"lastUpdated": "2026-05-18T01:08:01.612Z"
|
|
95
95
|
},
|
|
96
96
|
"formatted-logging.test.mjs": {
|
|
97
97
|
"url": "https://console-test.testdriver.ai/runs/69c8738614b73310c7839412/69c873a714b73310c7839450",
|
|
98
98
|
"lastUpdated": "2026-03-29T00:36:10.628Z"
|
|
99
99
|
},
|
|
100
100
|
"hover-text.test.mjs": {
|
|
101
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
102
|
-
"lastUpdated": "2026-05-
|
|
101
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a6364fb50e9fd352c650a",
|
|
102
|
+
"lastUpdated": "2026-05-18T00:55:00.064Z"
|
|
103
103
|
},
|
|
104
104
|
"no-provision.test.mjs": {
|
|
105
105
|
"url": "https://console.testdriver.ai/runs/69a62b3aaa712ecd3dea730a/69a62b7706a177a05bccd1cf",
|
|
106
106
|
"lastUpdated": "2026-03-03T00:32:25.279Z"
|
|
107
107
|
},
|
|
108
108
|
"ai.test.mjs": {
|
|
109
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
110
|
-
"replayUrl": "https://console-test.testdriver.ai/replay/
|
|
111
|
-
"embedUrl": "https://console-test.testdriver.ai/replay/
|
|
112
|
-
"lastUpdated": "2026-05-
|
|
109
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a634dfb50e9fd352c6509",
|
|
110
|
+
"replayUrl": "https://console-test.testdriver.ai/replay/6a0a6bbf5e84df6215f782fe?share=EdDcVTwBw6TWT2QyoDOxYg",
|
|
111
|
+
"embedUrl": "https://console-test.testdriver.ai/replay/6a0a6bbf5e84df6215f782fe?share=EdDcVTwBw6TWT2QyoDOxYg&embed=true",
|
|
112
|
+
"lastUpdated": "2026-05-18T01:30:45.341Z"
|
|
113
113
|
},
|
|
114
114
|
"popup-loading.test.mjs": {
|
|
115
115
|
"url": "https://console.testdriver.ai/runs/698bc89f7140c3fa7daaca8d/698bca7f7140c3fa7daacbf7",
|
|
@@ -144,12 +144,12 @@
|
|
|
144
144
|
"lastUpdated": "2026-02-13T19:55:05.953Z"
|
|
145
145
|
},
|
|
146
146
|
"findall-coffee-icons.test.mjs": {
|
|
147
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
148
|
-
"lastUpdated": "2026-05-
|
|
147
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a62ebfb50e9fd352c6502",
|
|
148
|
+
"lastUpdated": "2026-05-18T00:59:52.578Z"
|
|
149
149
|
},
|
|
150
150
|
"parse.test.mjs": {
|
|
151
|
-
"url": "https://console-test.testdriver.ai/runs/
|
|
152
|
-
"lastUpdated": "2026-05-
|
|
151
|
+
"url": "https://console-test.testdriver.ai/runs/6a0a61dafb50e9fd352c6464/6a0a6379fb50e9fd352c650b",
|
|
152
|
+
"lastUpdated": "2026-05-18T01:09:07.859Z"
|
|
153
153
|
},
|
|
154
154
|
"flake-diffthreshold-001.test.mjs": {
|
|
155
155
|
"url": "https://console.testdriver.ai/runs/69a62b3aaa712ecd3dea730a/69a62bcafc0ac3cc632a91aa",
|
package/docs/changelog.mdx
CHANGED
|
@@ -110,7 +110,7 @@ This release includes all changes from v7.8.0-canary.4 with version bumps.
|
|
|
110
110
|
✨ New features
|
|
111
111
|
|
|
112
112
|
- **[Cache API](/v7/cache)** — Speed up repeated test runs with screenshot-based caching. The system compares screenshots to cached results and reuses element positions when the screen hasn't changed, reducing AI calls.
|
|
113
|
-
- **[Custom error classes](/v7/errors)** — New `ElementNotFoundError`
|
|
113
|
+
- **[Custom error classes](/v7/errors)** — New `ElementNotFoundError` class with rich debugging info including screenshot paths, pixel diffs, and detailed messages.
|
|
114
114
|
- **[Events system](/v7/events)** — Listen to SDK lifecycle events with wildcard support via `testdriver.emitter`. Uses colon-delimited namespaces (e.g., `command:start`, `log:*`).
|
|
115
115
|
- **[Provision API](/v7/provision)** — Launch browsers, desktop apps, VS Code, and Chrome extensions in your sandbox before tests run. Access via `testdriver.provision.*`.
|
|
116
116
|
- **[Redraw detection](/v7/redraw)** — Wait for screens to stabilize after interactions using two-phase detection with pixel comparison and z-score analysis.
|
|
@@ -117,18 +117,6 @@ await button.click();
|
|
|
117
117
|
|
|
118
118
|
## Actions
|
|
119
119
|
|
|
120
|
-
### Use Descriptive Prompts
|
|
121
|
-
|
|
122
|
-
```javascript
|
|
123
|
-
// ❌ Generic
|
|
124
|
-
await testdriver.ai('login');
|
|
125
|
-
|
|
126
|
-
// ✅ Specific steps
|
|
127
|
-
await testdriver.ai('click the username field and type user@example.com');
|
|
128
|
-
await testdriver.ai('click the password field and type password123');
|
|
129
|
-
await testdriver.ai('click the blue submit button');
|
|
130
|
-
```
|
|
131
|
-
|
|
132
120
|
### Chain Actions
|
|
133
121
|
|
|
134
122
|
```javascript
|
|
@@ -205,18 +205,6 @@ const { testdriver, dashcam } = await electron(context, {
|
|
|
205
205
|
|
|
206
206
|
## Cache Configuration
|
|
207
207
|
|
|
208
|
-
### Prompt Cache
|
|
209
|
-
|
|
210
|
-
Configure AI prompt caching:
|
|
211
|
-
|
|
212
|
-
```javascript
|
|
213
|
-
// Disable prompt cache globally
|
|
214
|
-
process.env.TD_NO_PROMPT_CACHE = 'true';
|
|
215
|
-
|
|
216
|
-
// Per call
|
|
217
|
-
await testdriver.ai('click button', false); // bypass cache
|
|
218
|
-
```
|
|
219
|
-
|
|
220
208
|
### Selector Cache
|
|
221
209
|
|
|
222
210
|
Configure element location caching:
|
package/docs/v7/_drafts/faq.mdx
CHANGED
|
@@ -116,21 +116,13 @@ Features:
|
|
|
116
116
|
|
|
117
117
|
### How does caching work?
|
|
118
118
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
**1. Prompt Cache** - Caches AI-generated commands locally
|
|
122
|
-
```javascript
|
|
123
|
-
await testdriver.ai('click button'); // AI call
|
|
124
|
-
await testdriver.ai('click button'); // Uses cache
|
|
125
|
-
```
|
|
126
|
-
|
|
127
|
-
**2. Selector Cache** - Caches element locations on server
|
|
119
|
+
**Selector Cache** - Caches element locations on server
|
|
128
120
|
```javascript
|
|
129
121
|
await testdriver.find('button'); // AI vision
|
|
130
122
|
await testdriver.find('button'); // Cache hit (95%+ similar UI)
|
|
131
123
|
```
|
|
132
124
|
|
|
133
|
-
See [caching guide](/v7/
|
|
125
|
+
See [caching guide](/v7/caching) for details.
|
|
134
126
|
|
|
135
127
|
### Can I test responsive designs?
|
|
136
128
|
|
|
@@ -331,11 +331,8 @@ test('slow test', async () => {
|
|
|
331
331
|
|
|
332
332
|
**2. Optimize test:**
|
|
333
333
|
```javascript
|
|
334
|
-
// Use
|
|
334
|
+
// Use explicit actions
|
|
335
335
|
await testdriver.find('button').then(el => el.click());
|
|
336
|
-
|
|
337
|
-
// Instead of
|
|
338
|
-
await testdriver.ai('click the button');
|
|
339
336
|
```
|
|
340
337
|
|
|
341
338
|
### Tests Fail in CI
|
package/docs/v7/examples/ai.mdx
CHANGED
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* ai.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://console-test.testdriver.ai/replay/
|
|
15
|
+
src="https://console-test.testdriver.ai/replay/6a0a6bbf5e84df6215f782fe?share=EdDcVTwBw6TWT2QyoDOxYg&embed=true"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* assert.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://console-test.testdriver.ai/replay/
|
|
15
|
+
src="https://console-test.testdriver.ai/replay/6a0a69f85e84df6215f782f4?share=AaQZE9gfcazcIyAwZ54Dtg&embed=true"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* chrome-extension.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/6a0a61f0fb50e9fd352c646e/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* element-not-found.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/6a0a6334fb50e9fd352c6508/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -18,7 +18,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
18
18
|
|
|
19
19
|
{/* findall-coffee-icons.test.mjs output */}
|
|
20
20
|
<iframe
|
|
21
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
21
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/6a0a62ebfb50e9fd352c6502/replay"
|
|
22
22
|
width="100%"
|
|
23
23
|
height="600"
|
|
24
24
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* hover-image.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/6a0a624bfb50e9fd352c64f7/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -18,7 +18,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
18
18
|
|
|
19
19
|
{/* hover-text-with-description.test.mjs output */}
|
|
20
20
|
<iframe
|
|
21
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
21
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/6a0a6208fb50e9fd352c64ab/replay"
|
|
22
22
|
width="100%"
|
|
23
23
|
height="600"
|
|
24
24
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* hover-text.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/6a0a6364fb50e9fd352c650a/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* installer.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/6a0a6264fb50e9fd352c64fc/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* launch-vscode-linux.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://console-test.testdriver.ai/replay/
|
|
15
|
+
src="https://console-test.testdriver.ai/replay/6a0a67d35e84df6215f782e4?share=BZj28mmTsITtFqfMHy74Hw&embed=true"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -18,7 +18,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
18
18
|
|
|
19
19
|
{/* parse.test.mjs output */}
|
|
20
20
|
<iframe
|
|
21
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
21
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/6a0a6379fb50e9fd352c650b/replay"
|
|
22
22
|
width="100%"
|
|
23
23
|
height="600"
|
|
24
24
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* press-keys.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://console-test.testdriver.ai/replay/
|
|
15
|
+
src="https://console-test.testdriver.ai/replay/6a0a69b95e84df6215f782f2?share=YxG4LqDBIszM8ed0axBXCw&embed=true"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* scroll-keyboard.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/6a0a6302fb50e9fd352c6503/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* scroll.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://console-test.testdriver.ai/replay/
|
|
15
|
+
src="https://console-test.testdriver.ai/replay/6a0a6afe5e84df6215f782fa?share=OuQYCYU4nb44QRLS6JCejQ&embed=true"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
|
@@ -12,7 +12,7 @@ Watch this test execute in a real sandbox environment:
|
|
|
12
12
|
|
|
13
13
|
{/* type.test.mjs output */}
|
|
14
14
|
<iframe
|
|
15
|
-
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/
|
|
15
|
+
src="https://api-test.testdriver.ai/api/v1/testdriver/testcase/6a0a62a5fb50e9fd352c64ff/replay"
|
|
16
16
|
width="100%"
|
|
17
17
|
height="390"
|
|
18
18
|
style={{ border: "1px solid #333", borderRadius: "8px" }}
|
package/package.json
CHANGED
|
@@ -1,215 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
title: "AI Prompt Caching"
|
|
3
|
-
sidebarTitle: "AI Caching"
|
|
4
|
-
description: "How TestDriver caches AI-generated YAML commands for faster tests"
|
|
5
|
-
icon: "bolt"
|
|
6
|
-
---
|
|
7
|
-
|
|
8
|
-
## Overview
|
|
9
|
-
|
|
10
|
-
The Prompt Cache stores AI-generated YAML commands locally, so repeated `.ai()` calls with the same prompt skip the AI entirely.
|
|
11
|
-
|
|
12
|
-
This provides:
|
|
13
|
-
- ⚡ **Instant execution** - No AI call needed
|
|
14
|
-
- 💰 **Cost savings** - Reduces API usage
|
|
15
|
-
- 🔌 **Offline testing** - Works without network
|
|
16
|
-
- 🎯 **Deterministic** - Same prompt = same commands
|
|
17
|
-
|
|
18
|
-
## How It Works
|
|
19
|
-
|
|
20
|
-
<Steps>
|
|
21
|
-
<Step title="First Call">
|
|
22
|
-
```javascript
|
|
23
|
-
await testdriver.ai('click the submit button');
|
|
24
|
-
```
|
|
25
|
-
|
|
26
|
-
- Sends prompt + screenshot to AI
|
|
27
|
-
- Receives YAML commands
|
|
28
|
-
- Saves to `.testdriver/.cache/{prompt-hash}.yaml`
|
|
29
|
-
</Step>
|
|
30
|
-
|
|
31
|
-
<Step title="Subsequent Calls">
|
|
32
|
-
```javascript
|
|
33
|
-
await testdriver.ai('click the submit button');
|
|
34
|
-
```
|
|
35
|
-
|
|
36
|
-
- Checks cache first
|
|
37
|
-
- Finds matching cached YAML
|
|
38
|
-
- Uses cached commands (no AI call)
|
|
39
|
-
- Shows `(using cached response)` in output
|
|
40
|
-
</Step>
|
|
41
|
-
</Steps>
|
|
42
|
-
|
|
43
|
-
## Cache Location
|
|
44
|
-
|
|
45
|
-
Cached prompts are stored locally in your project:
|
|
46
|
-
|
|
47
|
-
```
|
|
48
|
-
.testdriver/
|
|
49
|
-
.cache/
|
|
50
|
-
click-the-submit-button-a1b2c3d4.yaml
|
|
51
|
-
find-login-form-e5f6a7b8.yaml
|
|
52
|
-
verify-dashboard-c9d0e1f2.yaml
|
|
53
|
-
```
|
|
54
|
-
|
|
55
|
-
Files are named using:
|
|
56
|
-
- Sanitized prompt (first 50 chars, alphanumeric)
|
|
57
|
-
- MD5 hash of full prompt for uniqueness
|
|
58
|
-
|
|
59
|
-
## Cache Matching
|
|
60
|
-
|
|
61
|
-
The prompt cache uses **exact text matching**:
|
|
62
|
-
- Case-insensitive comparison
|
|
63
|
-
- Whitespace trimmed
|
|
64
|
-
- No screenshot comparison
|
|
65
|
-
|
|
66
|
-
```javascript
|
|
67
|
-
// These all match the same cache entry:
|
|
68
|
-
await testdriver.ai('click the submit button');
|
|
69
|
-
await testdriver.ai('CLICK THE SUBMIT BUTTON');
|
|
70
|
-
await testdriver.ai(' click the submit button ');
|
|
71
|
-
|
|
72
|
-
// This creates a new cache entry:
|
|
73
|
-
await testdriver.ai('click the submit btn'); // Different text
|
|
74
|
-
```
|
|
75
|
-
|
|
76
|
-
## Disabling Prompt Cache
|
|
77
|
-
|
|
78
|
-
Bypass the cache for a specific call:
|
|
79
|
-
|
|
80
|
-
```javascript
|
|
81
|
-
// Force fresh AI call, bypass cache
|
|
82
|
-
await testdriver.ai('click the submit button', false);
|
|
83
|
-
|
|
84
|
-
// These use cache (default)
|
|
85
|
-
await testdriver.ai('click the submit button');
|
|
86
|
-
await testdriver.ai('click the submit button', true);
|
|
87
|
-
```
|
|
88
|
-
|
|
89
|
-
## Clearing Prompt Cache
|
|
90
|
-
|
|
91
|
-
Clear all cached prompts:
|
|
92
|
-
|
|
93
|
-
```bash
|
|
94
|
-
rm -rf .testdriver/.cache/*.yaml
|
|
95
|
-
```
|
|
96
|
-
|
|
97
|
-
Or programmatically:
|
|
98
|
-
|
|
99
|
-
```javascript
|
|
100
|
-
const promptCache = require('testdriverai/agent/lib/cache.js');
|
|
101
|
-
promptCache.clearCache();
|
|
102
|
-
```
|
|
103
|
-
|
|
104
|
-
## Usage Examples
|
|
105
|
-
|
|
106
|
-
### Basic Caching
|
|
107
|
-
|
|
108
|
-
```javascript
|
|
109
|
-
import { test } from 'vitest';
|
|
110
|
-
import { chrome } from 'testdriverai/presets';
|
|
111
|
-
|
|
112
|
-
test('login flow', async (context) => {
|
|
113
|
-
const { testdriver } = await chrome(context, {
|
|
114
|
-
url: 'https://myapp.com/login'
|
|
115
|
-
});
|
|
116
|
-
|
|
117
|
-
// First call: AI generates commands, saves to cache
|
|
118
|
-
await testdriver.ai('click the login button');
|
|
119
|
-
|
|
120
|
-
// Run test again - uses cache (instant)
|
|
121
|
-
// Look for: "(using cached response)" in output
|
|
122
|
-
});
|
|
123
|
-
```
|
|
124
|
-
|
|
125
|
-
### Bypassing Cache
|
|
126
|
-
|
|
127
|
-
```javascript
|
|
128
|
-
test('always fresh', async (context) => {
|
|
129
|
-
const { testdriver } = await chrome(context, { url });
|
|
130
|
-
|
|
131
|
-
// Always get fresh AI response
|
|
132
|
-
await testdriver.ai('analyze the current state', false);
|
|
133
|
-
});
|
|
134
|
-
```
|
|
135
|
-
|
|
136
|
-
## Best Practices
|
|
137
|
-
|
|
138
|
-
### 1. Use Consistent Prompts
|
|
139
|
-
|
|
140
|
-
```javascript
|
|
141
|
-
// ✅ Good - consistent prompt
|
|
142
|
-
await testdriver.ai('fill out the login form');
|
|
143
|
-
await testdriver.ai('fill out the login form'); // Cache hit
|
|
144
|
-
|
|
145
|
-
// ❌ Bad - different prompts
|
|
146
|
-
await testdriver.ai('fill out the login form');
|
|
147
|
-
await testdriver.ai('complete the login form'); // Cache miss
|
|
148
|
-
```
|
|
149
|
-
|
|
150
|
-
### 2. Clear Cache When Test Logic Changes
|
|
151
|
-
|
|
152
|
-
If you update your test prompts, clear the cache:
|
|
153
|
-
|
|
154
|
-
```bash
|
|
155
|
-
rm -rf .testdriver/.cache/*.yaml
|
|
156
|
-
```
|
|
157
|
-
|
|
158
|
-
### 3. Don't Commit Cache to Git
|
|
159
|
-
|
|
160
|
-
Add to `.gitignore`:
|
|
161
|
-
|
|
162
|
-
```gitignore
|
|
163
|
-
.testdriver/.cache/
|
|
164
|
-
```
|
|
165
|
-
|
|
166
|
-
### 4. Version Control Consideration
|
|
167
|
-
|
|
168
|
-
While you _can_ commit cached prompts for team consistency, it's generally not recommended because:
|
|
169
|
-
- AI responses may improve over time
|
|
170
|
-
- Different team members may need different cached results
|
|
171
|
-
- Cache files can become stale
|
|
172
|
-
|
|
173
|
-
## Cache Storage Details
|
|
174
|
-
|
|
175
|
-
| Property | Value |
|
|
176
|
-
|----------|-------|
|
|
177
|
-
| Location | Local (`.testdriver/.cache/`) |
|
|
178
|
-
| Persistence | Until manually cleared |
|
|
179
|
-
| Scope | Per-project |
|
|
180
|
-
| Matching | Exact prompt text (case-insensitive) |
|
|
181
|
-
| Expiration | Never |
|
|
182
|
-
|
|
183
|
-
## Troubleshooting
|
|
184
|
-
|
|
185
|
-
### Cache Not Working
|
|
186
|
-
|
|
187
|
-
Check:
|
|
188
|
-
1. Prompts match exactly (case-insensitive)
|
|
189
|
-
2. `.testdriver/.cache/` directory exists and is writable
|
|
190
|
-
3. `TD_NO_PROMPT_CACHE` environment variable is not set
|
|
191
|
-
|
|
192
|
-
### Stale Cache Data
|
|
193
|
-
|
|
194
|
-
If AI responses seem outdated:
|
|
195
|
-
|
|
196
|
-
```bash
|
|
197
|
-
# Clear all cached prompts
|
|
198
|
-
rm -rf .testdriver/.cache/*.yaml
|
|
199
|
-
```
|
|
200
|
-
|
|
201
|
-
Or clear specific prompts:
|
|
202
|
-
|
|
203
|
-
```bash
|
|
204
|
-
# Find cache files
|
|
205
|
-
ls -la .testdriver/.cache/
|
|
206
|
-
|
|
207
|
-
# Delete specific cache file
|
|
208
|
-
rm .testdriver/.cache/click-the-submit-button-*.yaml
|
|
209
|
-
```
|
|
210
|
-
|
|
211
|
-
## See Also
|
|
212
|
-
|
|
213
|
-
- [Selector Caching](/v7/guides/caching-selectors) - Cache element locations
|
|
214
|
-
- [`.ai()` Method](/v7/api/ai) - AI command generation
|
|
215
|
-
- [Vitest Integration](/v7/guides/vitest) - Testing with TestDriver
|
|
@@ -1,366 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
title: "Caching Prompts"
|
|
3
|
-
sidebarTitle: "Caching Prompts"
|
|
4
|
-
description: "How TestDriver caches AI responses and element locations for faster tests"
|
|
5
|
-
icon: "bolt"
|
|
6
|
-
---
|
|
7
|
-
|
|
8
|
-
## Overview
|
|
9
|
-
|
|
10
|
-
TestDriver uses two types of caching to dramatically speed up your tests and reduce AI costs:
|
|
11
|
-
|
|
12
|
-
1. **Prompt Cache** - Caches AI-generated commands (`.ai()` responses)
|
|
13
|
-
2. **Selector Cache** - Caches element locations (`.find()` results)
|
|
14
|
-
|
|
15
|
-
Both caches are enabled by default and work automatically.
|
|
16
|
-
|
|
17
|
-
## Prompt Cache
|
|
18
|
-
|
|
19
|
-
The Prompt Cache stores AI-generated YAML commands locally, so repeated `.ai()` calls with the same prompt skip the AI entirely.
|
|
20
|
-
|
|
21
|
-
### How It Works
|
|
22
|
-
|
|
23
|
-
<Steps>
|
|
24
|
-
<Step title="First Call">
|
|
25
|
-
```javascript
|
|
26
|
-
await testdriver.ai('click the submit button');
|
|
27
|
-
```
|
|
28
|
-
|
|
29
|
-
- Sends prompt + screenshot to AI
|
|
30
|
-
- Receives YAML commands
|
|
31
|
-
- Saves to `.testdriver/.cache/{prompt-hash}.yaml`
|
|
32
|
-
</Step>
|
|
33
|
-
|
|
34
|
-
<Step title="Subsequent Calls">
|
|
35
|
-
```javascript
|
|
36
|
-
await testdriver.ai('click the submit button');
|
|
37
|
-
```
|
|
38
|
-
|
|
39
|
-
- Checks cache first
|
|
40
|
-
- Finds matching cached YAML
|
|
41
|
-
- Uses cached commands (no AI call)
|
|
42
|
-
- Shows `(using cached response)` in output
|
|
43
|
-
</Step>
|
|
44
|
-
</Steps>
|
|
45
|
-
|
|
46
|
-
### Cache Location
|
|
47
|
-
|
|
48
|
-
Cached prompts are stored locally:
|
|
49
|
-
|
|
50
|
-
```
|
|
51
|
-
.testdriver/
|
|
52
|
-
.cache/
|
|
53
|
-
click-the-submit-button-a1b2c3d4.yaml
|
|
54
|
-
find-login-form-e5f6a7b8.yaml
|
|
55
|
-
verify-dashboard-c9d0e1f2.yaml
|
|
56
|
-
```
|
|
57
|
-
|
|
58
|
-
Files are named using:
|
|
59
|
-
- Sanitized prompt (first 50 chars, alphanumeric)
|
|
60
|
-
- MD5 hash of full prompt for uniqueness
|
|
61
|
-
|
|
62
|
-
### Disabling Prompt Cache
|
|
63
|
-
|
|
64
|
-
Bypass the cache for a specific call:
|
|
65
|
-
|
|
66
|
-
```javascript
|
|
67
|
-
// Force fresh AI call, bypass cache
|
|
68
|
-
await testdriver.ai('click the submit button', false);
|
|
69
|
-
|
|
70
|
-
// These use cache (default)
|
|
71
|
-
await testdriver.ai('click the submit button');
|
|
72
|
-
await testdriver.ai('click the submit button', true);
|
|
73
|
-
```
|
|
74
|
-
|
|
75
|
-
### Clearing Prompt Cache
|
|
76
|
-
|
|
77
|
-
Clear all cached prompts:
|
|
78
|
-
|
|
79
|
-
```bash
|
|
80
|
-
rm -rf .testdriver/.cache/*.yaml
|
|
81
|
-
```
|
|
82
|
-
|
|
83
|
-
Or programmatically:
|
|
84
|
-
|
|
85
|
-
```javascript
|
|
86
|
-
const promptCache = require('testdriverai/agent/lib/cache.js');
|
|
87
|
-
promptCache.clearCache();
|
|
88
|
-
```
|
|
89
|
-
|
|
90
|
-
### Benefits
|
|
91
|
-
|
|
92
|
-
- ⚡ **Instant execution** - No AI call needed
|
|
93
|
-
- 💰 **Cost savings** - Reduces API usage
|
|
94
|
-
- 🔌 **Offline testing** - Works without network
|
|
95
|
-
- 🎯 **Deterministic** - Same prompt = same commands
|
|
96
|
-
|
|
97
|
-
## Selector Cache
|
|
98
|
-
|
|
99
|
-
The Selector Cache stores element locations on the server, so `.find()` calls can skip the AI vision analysis.
|
|
100
|
-
|
|
101
|
-
### How It Works
|
|
102
|
-
|
|
103
|
-
```mermaid
|
|
104
|
-
sequenceDiagram
|
|
105
|
-
participant Test as Your Test
|
|
106
|
-
participant SDK as TestDriver SDK
|
|
107
|
-
participant API as TestDriver API
|
|
108
|
-
participant AI as Claude Vision
|
|
109
|
-
participant Cache as Selector Cache DB
|
|
110
|
-
|
|
111
|
-
Test->>SDK: testdriver.find('submit button')
|
|
112
|
-
SDK->>API: POST /locate (screenshot + prompt)
|
|
113
|
-
|
|
114
|
-
API->>Cache: Check for matching cache entry
|
|
115
|
-
|
|
116
|
-
alt Cache Hit (95%+ similar screenshot)
|
|
117
|
-
Cache-->>API: Return cached coordinates
|
|
118
|
-
API-->>SDK: {x, y, cacheHit: true}
|
|
119
|
-
SDK-->>Test: Element found (instant)
|
|
120
|
-
else Cache Miss
|
|
121
|
-
API->>AI: Analyze screenshot
|
|
122
|
-
AI-->>API: Element coordinates
|
|
123
|
-
API->>Cache: Save new cache entry
|
|
124
|
-
API-->>SDK: {x, y, cacheHit: false}
|
|
125
|
-
SDK-->>Test: Element found
|
|
126
|
-
end
|
|
127
|
-
```
|
|
128
|
-
|
|
129
|
-
### Cache Matching Strategy
|
|
130
|
-
|
|
131
|
-
The selector cache uses a three-tier matching system:
|
|
132
|
-
|
|
133
|
-
1. **Exact Hash Match** (Fastest)
|
|
134
|
-
- Perceptual hash comparison
|
|
135
|
-
- Same screenshot = instant match
|
|
136
|
-
- 0% difference threshold
|
|
137
|
-
|
|
138
|
-
2. **Pixel Diff Match** (Fast)
|
|
139
|
-
- 80%+ perceptual hash similarity
|
|
140
|
-
- Pixel-by-pixel comparison
|
|
141
|
-
- Default 5% difference threshold (95% similarity)
|
|
142
|
-
- Configurable per call
|
|
143
|
-
|
|
144
|
-
3. **Template Match** (Fallback)
|
|
145
|
-
- Edge detection + template matching
|
|
146
|
-
- Finds visually similar UI elements
|
|
147
|
-
- 75%+ confidence threshold
|
|
148
|
-
|
|
149
|
-
### Viewing Cache Entries
|
|
150
|
-
|
|
151
|
-
View all cached selectors at [console.testdriver.ai](https://console.testdriver.ai)
|
|
152
|
-
|
|
153
|
-
The console shows:
|
|
154
|
-
- Cached screenshots with green circles on found elements
|
|
155
|
-
- Original prompts
|
|
156
|
-
- Hit count (how many times cache was used)
|
|
157
|
-
- Similarity scores
|
|
158
|
-
- Cache age and last accessed time
|
|
159
|
-
|
|
160
|
-
### Controlling Cache Threshold
|
|
161
|
-
|
|
162
|
-
Adjust similarity threshold per call:
|
|
163
|
-
|
|
164
|
-
```javascript
|
|
165
|
-
// Default: 95% similarity (5% difference allowed)
|
|
166
|
-
await testdriver.find('submit button');
|
|
167
|
-
|
|
168
|
-
// Stricter: 99% similarity (1% difference allowed)
|
|
169
|
-
await testdriver.find('submit button', { threshold: 0.01 });
|
|
170
|
-
|
|
171
|
-
// More lenient: 90% similarity (10% difference allowed)
|
|
172
|
-
await testdriver.find('submit button', { threshold: 0.10 });
|
|
173
|
-
|
|
174
|
-
// Disable cache: force fresh AI analysis
|
|
175
|
-
await testdriver.find('submit button', { threshold: -1 });
|
|
176
|
-
```
|
|
177
|
-
|
|
178
|
-
### Cache Filtering
|
|
179
|
-
|
|
180
|
-
The selector cache automatically filters by:
|
|
181
|
-
|
|
182
|
-
- **Prompt** - Exact text match (case-insensitive)
|
|
183
|
-
- **Team** - Your team ID
|
|
184
|
-
- **OS** - Operating system (if specified)
|
|
185
|
-
- **Resolution** - Screen resolution (if specified)
|
|
186
|
-
- **Time Window** - Last 7 days by default
|
|
187
|
-
|
|
188
|
-
### Cache Statistics
|
|
189
|
-
|
|
190
|
-
Each cache entry tracks:
|
|
191
|
-
|
|
192
|
-
- **Hit Count** - Number of times cache was used
|
|
193
|
-
- **Last Hit** - When cache was last accessed
|
|
194
|
-
- **Similarity** - Percentage match to original
|
|
195
|
-
- **Created At** - When entry was created
|
|
196
|
-
- **Element Type** - button, input, link, etc.
|
|
197
|
-
- **Element Bounds** - Bounding box coordinates
|
|
198
|
-
|
|
199
|
-
### Benefits
|
|
200
|
-
|
|
201
|
-
- ⚡ **Up to 10x faster** - Skip AI vision analysis
|
|
202
|
-
- 💰 **Lower AI costs** - Fewer vision API calls
|
|
203
|
-
- 🎯 **Consistent results** - Same UI = same coordinates
|
|
204
|
-
- 📊 **Metrics tracking** - See cache hit rates in console
|
|
205
|
-
|
|
206
|
-
## Cache Behavior Examples
|
|
207
|
-
|
|
208
|
-
### Prompt Cache Example
|
|
209
|
-
|
|
210
|
-
```javascript
|
|
211
|
-
import { test } from 'vitest';
|
|
212
|
-
import { chrome } from 'testdriverai/presets';
|
|
213
|
-
|
|
214
|
-
test('login flow', async (context) => {
|
|
215
|
-
const { testdriver } = await chrome(context, {
|
|
216
|
-
url: 'https://myapp.com/login'
|
|
217
|
-
});
|
|
218
|
-
|
|
219
|
-
// First call: AI generates commands, saves to cache
|
|
220
|
-
await testdriver.ai('click the login button');
|
|
221
|
-
|
|
222
|
-
// Run test again - uses cache (instant)
|
|
223
|
-
// Look for: "(using cached response)" in output
|
|
224
|
-
});
|
|
225
|
-
```
|
|
226
|
-
|
|
227
|
-
### Selector Cache Example
|
|
228
|
-
|
|
229
|
-
```javascript
|
|
230
|
-
import { test } from 'vitest';
|
|
231
|
-
import { chrome } from 'testdriverai/presets';
|
|
232
|
-
|
|
233
|
-
test('find element', async (context) => {
|
|
234
|
-
const { testdriver } = await chrome(context, {
|
|
235
|
-
url: 'https://example.com'
|
|
236
|
-
});
|
|
237
|
-
|
|
238
|
-
// First call: AI vision analysis, saves to cache
|
|
239
|
-
const button = await testdriver.find('More information link');
|
|
240
|
-
console.log('Cache hit:', button.cacheHit); // false
|
|
241
|
-
|
|
242
|
-
// Second call: uses cache (instant)
|
|
243
|
-
const button2 = await testdriver.find('More information link');
|
|
244
|
-
console.log('Cache hit:', button2.cacheHit); // true
|
|
245
|
-
});
|
|
246
|
-
```
|
|
247
|
-
|
|
248
|
-
### Dynamic Threshold Example
|
|
249
|
-
|
|
250
|
-
```javascript
|
|
251
|
-
test('strict vs lenient matching', async (context) => {
|
|
252
|
-
const { testdriver } = await chrome(context, { url });
|
|
253
|
-
|
|
254
|
-
// Strict: 99% similarity required
|
|
255
|
-
const elem1 = await testdriver.find('button', { threshold: 0.01 });
|
|
256
|
-
|
|
257
|
-
// Lenient: 90% similarity acceptable
|
|
258
|
-
const elem2 = await testdriver.find('button', { threshold: 0.10 });
|
|
259
|
-
|
|
260
|
-
// Bypass cache entirely
|
|
261
|
-
const elem3 = await testdriver.find('button', { threshold: -1 });
|
|
262
|
-
});
|
|
263
|
-
```
|
|
264
|
-
|
|
265
|
-
## Best Practices
|
|
266
|
-
|
|
267
|
-
### 1. Use Consistent Prompts
|
|
268
|
-
|
|
269
|
-
```javascript
|
|
270
|
-
// ✅ Good - consistent prompt
|
|
271
|
-
await testdriver.find('submit button');
|
|
272
|
-
await testdriver.find('submit button'); // Cache hit
|
|
273
|
-
|
|
274
|
-
// ❌ Bad - different prompts
|
|
275
|
-
await testdriver.find('submit button');
|
|
276
|
-
await testdriver.find('the submit button'); // Cache miss
|
|
277
|
-
```
|
|
278
|
-
|
|
279
|
-
### 2. Clear Cache When UI Changes
|
|
280
|
-
|
|
281
|
-
If your UI changes significantly, clear the cache:
|
|
282
|
-
|
|
283
|
-
```bash
|
|
284
|
-
# Clear prompt cache
|
|
285
|
-
rm -rf .testdriver/.cache/*.yaml
|
|
286
|
-
|
|
287
|
-
# Clear selector cache
|
|
288
|
-
# Use console.testdriver.ai to delete entries
|
|
289
|
-
```
|
|
290
|
-
|
|
291
|
-
### 3. Use Appropriate Thresholds
|
|
292
|
-
|
|
293
|
-
```javascript
|
|
294
|
-
// Stable UI: strict threshold
|
|
295
|
-
await testdriver.find('logo', { threshold: 0.01 });
|
|
296
|
-
|
|
297
|
-
// Dynamic UI: lenient threshold
|
|
298
|
-
await testdriver.find('news feed item', { threshold: 0.10 });
|
|
299
|
-
|
|
300
|
-
// Always fresh: disable cache
|
|
301
|
-
await testdriver.find('timestamp', { threshold: -1 });
|
|
302
|
-
```
|
|
303
|
-
|
|
304
|
-
### 4. Monitor Cache Performance
|
|
305
|
-
|
|
306
|
-
Check [console.testdriver.ai](https://console.testdriver.ai) regularly to:
|
|
307
|
-
- See cache hit rates
|
|
308
|
-
- Identify frequently used selectors
|
|
309
|
-
- Remove stale cache entries
|
|
310
|
-
- Optimize threshold settings
|
|
311
|
-
|
|
312
|
-
## Cache Storage
|
|
313
|
-
|
|
314
|
-
| Cache Type | Location | Persistence |
|
|
315
|
-
|------------|----------|-------------|
|
|
316
|
-
| Prompt Cache | Local (`.testdriver/.cache/`) | Until manually cleared |
|
|
317
|
-
| Selector Cache | Server (MongoDB + S3) | 7 days default |
|
|
318
|
-
|
|
319
|
-
## Important Notes
|
|
320
|
-
|
|
321
|
-
### Prompt Cache
|
|
322
|
-
|
|
323
|
-
- Cache keys are **prompt text only** (no screenshot comparison)
|
|
324
|
-
- Case-insensitive, trimmed matching
|
|
325
|
-
- Safe to commit to version control (but not recommended)
|
|
326
|
-
- Never expires automatically
|
|
327
|
-
|
|
328
|
-
### Selector Cache
|
|
329
|
-
|
|
330
|
-
- Cache matches by **screenshot similarity** + prompt
|
|
331
|
-
- Platform-specific (OS, resolution)
|
|
332
|
-
- Team-scoped (your cache entries only)
|
|
333
|
-
- 7-day rolling window by default
|
|
334
|
-
|
|
335
|
-
## Troubleshooting
|
|
336
|
-
|
|
337
|
-
### Prompt Cache Not Working
|
|
338
|
-
|
|
339
|
-
Check:
|
|
340
|
-
1. Prompts match exactly (case-insensitive)
|
|
341
|
-
2. `.testdriver/.cache/` directory exists and is writable
|
|
342
|
-
3. `TD_NO_PROMPT_CACHE` environment variable is not set
|
|
343
|
-
|
|
344
|
-
### Selector Cache Not Working
|
|
345
|
-
|
|
346
|
-
Check:
|
|
347
|
-
1. Threshold isn't too strict (try 0.10 for 90% similarity)
|
|
348
|
-
2. Screenshot hasn't changed significantly
|
|
349
|
-
3. OS/resolution matches cached entry
|
|
350
|
-
4. Cache entry isn't older than 7 days
|
|
351
|
-
|
|
352
|
-
### Stale Cache Data
|
|
353
|
-
|
|
354
|
-
Clear prompt cache:
|
|
355
|
-
```bash
|
|
356
|
-
rm -rf .testdriver/.cache/*.yaml
|
|
357
|
-
```
|
|
358
|
-
|
|
359
|
-
Delete selector cache entries at [console.testdriver.ai](https://console.testdriver.ai)
|
|
360
|
-
|
|
361
|
-
## See Also
|
|
362
|
-
|
|
363
|
-
- [Console Dashboard](https://console.testdriver.ai) - View and manage selector cache
|
|
364
|
-
- [`.ai()` Method](/v7/api/ai) - AI command generation
|
|
365
|
-
- [`.find()` Method](/v7/api/find) - Element location
|
|
366
|
-
- [Vitest Integration](/v7/guides/vitest) - Testing with TestDriver
|
|
@@ -1,200 +0,0 @@
|
|
|
1
|
-
# Prompt Cache System
|
|
2
|
-
|
|
3
|
-
## Overview
|
|
4
|
-
|
|
5
|
-
The TestDriver SDK includes automatic caching of `.ai()` responses. When you use the SDK to execute natural language prompts, the YAML commands generated by the AI are cached locally to improve performance and reduce API calls.
|
|
6
|
-
|
|
7
|
-
**Caching is enabled by default** and can be controlled per-call using the `cache` parameter.
|
|
8
|
-
|
|
9
|
-
## How It Works
|
|
10
|
-
|
|
11
|
-
1. **First Call**: When you call `.ai('click the submit button')` for the first time:
|
|
12
|
-
|
|
13
|
-
- The SDK sends the prompt and screenshot to the API
|
|
14
|
-
- The API returns YAML commands in markdown format
|
|
15
|
-
- The response is cached to `.testdriver/.cache/{prompt-hash}.yaml`
|
|
16
|
-
|
|
17
|
-
2. **Subsequent Calls**: When you call `.ai('click the submit button')` again:
|
|
18
|
-
|
|
19
|
-
- The SDK checks the cache first
|
|
20
|
-
- If found, it uses the cached YAML instead of calling the API
|
|
21
|
-
- You'll see `(using cached response)` in the output
|
|
22
|
-
|
|
23
|
-
3. **Disable Caching**: Pass `false` as the second parameter:
|
|
24
|
-
- `.ai('click the submit button', false)` bypasses the cache
|
|
25
|
-
- Forces a fresh API call and updates the cache
|
|
26
|
-
|
|
27
|
-
## Cache Location
|
|
28
|
-
|
|
29
|
-
Cached responses are stored in:
|
|
30
|
-
|
|
31
|
-
```
|
|
32
|
-
.testdriver/.cache/
|
|
33
|
-
```
|
|
34
|
-
|
|
35
|
-
Files are named using a combination of:
|
|
36
|
-
|
|
37
|
-
- Sanitized prompt (first 50 chars, alphanumeric only)
|
|
38
|
-
- MD5 hash of the full prompt for uniqueness
|
|
39
|
-
|
|
40
|
-
Example:
|
|
41
|
-
|
|
42
|
-
```
|
|
43
|
-
.testdriver/.cache/click-the-submit-button-a1b2c3d4e5f6.yaml
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
## Benefits
|
|
47
|
-
|
|
48
|
-
- **Faster execution**: No API call needed for repeated prompts
|
|
49
|
-
- **Cost savings**: Reduces API usage for repeated operations
|
|
50
|
-
- **Offline testing**: Can rerun tests without network access
|
|
51
|
-
- **Deterministic tests**: Same prompt always produces same commands
|
|
52
|
-
|
|
53
|
-
## Disabling Cache
|
|
54
|
-
|
|
55
|
-
To disable caching for a specific `.ai()` call, pass `false` as the second parameter:
|
|
56
|
-
|
|
57
|
-
```javascript
|
|
58
|
-
// Force fresh API call, bypass cache
|
|
59
|
-
await client.ai("click the submit button", false);
|
|
60
|
-
```
|
|
61
|
-
|
|
62
|
-
Caching is **enabled by default** (equivalent to passing `true`):
|
|
63
|
-
|
|
64
|
-
```javascript
|
|
65
|
-
// These are equivalent
|
|
66
|
-
await client.ai("click the submit button");
|
|
67
|
-
await client.ai("click the submit button", true);
|
|
68
|
-
```
|
|
69
|
-
|
|
70
|
-
## Clearing Cache
|
|
71
|
-
|
|
72
|
-
To clear all cached prompts:
|
|
73
|
-
|
|
74
|
-
```bash
|
|
75
|
-
rm -rf .testdriver/.cache/*.yaml
|
|
76
|
-
```
|
|
77
|
-
|
|
78
|
-
Or programmatically:
|
|
79
|
-
|
|
80
|
-
```javascript
|
|
81
|
-
const promptCache = require("testdriverai/agent/lib/cache.js");
|
|
82
|
-
promptCache.clearCache();
|
|
83
|
-
```
|
|
84
|
-
|
|
85
|
-
## Usage Examples
|
|
86
|
-
|
|
87
|
-
### Basic Usage (Automatic Caching)
|
|
88
|
-
|
|
89
|
-
```javascript
|
|
90
|
-
const TestDriver = require("testdriverai");
|
|
91
|
-
|
|
92
|
-
const client = new TestDriver(process.env.TD_API_KEY);
|
|
93
|
-
await client.connect();
|
|
94
|
-
|
|
95
|
-
// First call - makes API request and caches
|
|
96
|
-
await client.ai("click the login button");
|
|
97
|
-
|
|
98
|
-
// Second call - uses cache (instant)
|
|
99
|
-
await client.ai("click the login button");
|
|
100
|
-
|
|
101
|
-
// Force fresh API call, bypass cache
|
|
102
|
-
await client.ai("click the login button", false);
|
|
103
|
-
```
|
|
104
|
-
|
|
105
|
-
### Check Cache Status
|
|
106
|
-
|
|
107
|
-
```javascript
|
|
108
|
-
const promptCache = require("testdriverai/agent/lib/cache.js");
|
|
109
|
-
|
|
110
|
-
// Check if a prompt is cached
|
|
111
|
-
if (promptCache.hasCache("click the login button")) {
|
|
112
|
-
console.log("This prompt is cached");
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
// Get cache statistics
|
|
116
|
-
const stats = promptCache.getCacheStats();
|
|
117
|
-
console.log(`Cached prompts: ${stats.count}`);
|
|
118
|
-
console.log(`Files: ${stats.files}`);
|
|
119
|
-
```
|
|
120
|
-
|
|
121
|
-
### Read Cached YAML
|
|
122
|
-
|
|
123
|
-
```javascript
|
|
124
|
-
const promptCache = require("testdriverai/agent/lib/cache.js");
|
|
125
|
-
|
|
126
|
-
const yaml = promptCache.readCache("click the login button");
|
|
127
|
-
if (yaml) {
|
|
128
|
-
console.log("Cached YAML:", yaml);
|
|
129
|
-
}
|
|
130
|
-
```
|
|
131
|
-
|
|
132
|
-
### Manual Cache Control
|
|
133
|
-
|
|
134
|
-
```javascript
|
|
135
|
-
const promptCache = require("testdriverai/agent/lib/cache.js");
|
|
136
|
-
|
|
137
|
-
// Write to cache manually
|
|
138
|
-
promptCache.writeCache("custom prompt", yamlContent);
|
|
139
|
-
|
|
140
|
-
// Clear entire cache
|
|
141
|
-
promptCache.clearCache();
|
|
142
|
-
|
|
143
|
-
// Get cache path for a prompt
|
|
144
|
-
const path = promptCache.getCachePath("my prompt");
|
|
145
|
-
console.log(`Cache file: ${path}`);
|
|
146
|
-
```
|
|
147
|
-
|
|
148
|
-
## Important Notes
|
|
149
|
-
|
|
150
|
-
1. **Prompt Matching**: Cache keys are based on the exact prompt text (case-insensitive, trimmed)
|
|
151
|
-
|
|
152
|
-
- `'Click the button'` and `'click the button'` will match
|
|
153
|
-
- `'Click the button'` and `'Click the button '` will match (trailing space trimmed)
|
|
154
|
-
- `'Click button'` and `'Click the button'` will NOT match
|
|
155
|
-
|
|
156
|
-
2. **No Screenshot Comparison**: The cache does NOT compare screenshots
|
|
157
|
-
|
|
158
|
-
- Cached responses assume the UI state is similar
|
|
159
|
-
- Use different prompts if the UI state has changed significantly
|
|
160
|
-
|
|
161
|
-
3. **Manual Cache Management**: The cache never expires automatically
|
|
162
|
-
|
|
163
|
-
- Clear it periodically if needed
|
|
164
|
-
- Delete specific files if a prompt's behavior should change
|
|
165
|
-
|
|
166
|
-
4. **Version Compatibility**: Cache files are plain YAML
|
|
167
|
-
- Compatible across TestDriver versions
|
|
168
|
-
- Safe to commit to version control (though not recommended)
|
|
169
|
-
|
|
170
|
-
## Testing
|
|
171
|
-
|
|
172
|
-
Run the test suite to verify caching:
|
|
173
|
-
|
|
174
|
-
```bash
|
|
175
|
-
node test-prompt-cache.js
|
|
176
|
-
```
|
|
177
|
-
|
|
178
|
-
This will:
|
|
179
|
-
|
|
180
|
-
- Make an initial `.ai()` call that caches the response
|
|
181
|
-
- Make a second `.ai()` call that uses the cache
|
|
182
|
-
- Show cache statistics and file locations
|
|
183
|
-
|
|
184
|
-
## Troubleshooting
|
|
185
|
-
|
|
186
|
-
### Cache not being used
|
|
187
|
-
|
|
188
|
-
- Check that prompts match exactly (case-insensitive)
|
|
189
|
-
- Verify `TD_NO_PROMPT_CACHE` is not set
|
|
190
|
-
- Check `.testdriver/.cache/` directory exists and is writable
|
|
191
|
-
|
|
192
|
-
### Stale cache data
|
|
193
|
-
|
|
194
|
-
- Clear the cache directory
|
|
195
|
-
- Or delete specific cached prompt files
|
|
196
|
-
|
|
197
|
-
### Permission errors
|
|
198
|
-
|
|
199
|
-
- Ensure `.testdriver/.cache/` is writable
|
|
200
|
-
- Check file system permissions
|