@testdriverai/agent 7.8.0-canary.15 → 7.8.0-canary.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,39 +1,33 @@
1
1
  #!/usr/bin/env node
2
2
 
3
3
  /**
4
- * Extract Example URLs from CI Logs
5
- *
6
- * Parses vitest output to extract TESTDRIVER_EXAMPLE_URL lines
7
- * and updates the examples-manifest.json file.
8
- *
4
+ * Extract Example URLs from Test Result JSON Files
5
+ *
6
+ * Reads per-test-case JSON result files written by the vitest plugin
7
+ * to .testdriver/results/ and updates examples-manifest.json.
8
+ *
9
9
  * Usage:
10
- * cat ci-log.txt | node extract-example-urls.js
11
- * node extract-example-urls.js < ci-log.txt
12
- * node extract-example-urls.js --file=ci-log.txt
10
+ * node extract-example-urls.js --results-dir=.testdriver/results
13
11
  */
14
12
 
15
13
  const fs = require("fs");
16
14
  const path = require("path");
17
- const readline = require("readline");
18
15
 
19
16
  const MANIFEST_PATH = path.join(__dirname, "../_data/examples-manifest.json");
20
17
 
21
- // Regex to match TESTDRIVER_EXAMPLE_URL::filename::url (handles optional timestamp prefix from CI logs)
22
- const URL_PATTERN = /TESTDRIVER_EXAMPLE_URL::([^:]+)::(.+)$/;
23
-
24
18
  // Parse command line arguments
25
19
  function parseArgs() {
26
20
  const args = process.argv.slice(2);
27
21
  const options = {
28
- file: null,
22
+ resultsDir: null,
29
23
  help: false,
30
24
  };
31
25
 
32
26
  for (const arg of args) {
33
27
  if (arg === "--help" || arg === "-h") {
34
28
  options.help = true;
35
- } else if (arg.startsWith("--file=")) {
36
- options.file = arg.slice(7);
29
+ } else if (arg.startsWith("--results-dir=")) {
30
+ options.resultsDir = arg.slice(14);
37
31
  }
38
32
  }
39
33
 
@@ -62,40 +56,51 @@ function saveManifest(manifest) {
62
56
  fs.writeFileSync(MANIFEST_PATH, JSON.stringify(manifest, null, 2) + "\n", "utf-8");
63
57
  }
64
58
 
65
- // Process a single line and extract URL if present
66
- function processLine(line, manifest, stats) {
67
- const match = line.match(URL_PATTERN);
68
- if (match) {
69
- const [, filename, url] = match;
70
- const isNew = !manifest.examples[filename];
71
-
72
- manifest.examples[filename] = {
73
- url: url.trim(),
74
- lastUpdated: new Date().toISOString(),
75
- };
76
-
77
- if (isNew) {
78
- stats.added++;
79
- } else {
80
- stats.updated++;
81
- }
82
-
83
- console.log(`${isNew ? "āž•" : "šŸ”„"} ${filename}: ${url}`);
84
- }
85
- }
86
-
87
- // Process input stream
88
- async function processInput(inputStream) {
59
+ // Process JSON result files from .testdriver/results/
60
+ function processResultsDir(resultsDir) {
89
61
  const manifest = loadManifest();
90
62
  const stats = { added: 0, updated: 0 };
91
63
 
92
- const rl = readline.createInterface({
93
- input: inputStream,
94
- crlfDelay: Infinity,
95
- });
64
+ // Look for JSON files under examples/ subdirectories
65
+ const examplesDir = path.join(resultsDir, "examples");
66
+ if (!fs.existsSync(examplesDir)) {
67
+ console.log(`\nāš ļø No examples results found in ${examplesDir}`);
68
+ return stats;
69
+ }
96
70
 
97
- for await (const line of rl) {
98
- processLine(line, manifest, stats);
71
+ // Walk example test directories (e.g., examples/assert.test.mjs/)
72
+ const testDirs = fs.readdirSync(examplesDir, { withFileTypes: true });
73
+ for (const entry of testDirs) {
74
+ if (!entry.isDirectory()) continue;
75
+ const testDir = path.join(examplesDir, entry.name);
76
+ const jsonFiles = fs.readdirSync(testDir).filter(f => f.endsWith(".json"));
77
+
78
+ for (const jsonFile of jsonFiles) {
79
+ try {
80
+ const content = fs.readFileSync(path.join(testDir, jsonFile), "utf-8");
81
+ const result = JSON.parse(content);
82
+ const testFileName = path.basename(result.test?.file || result.testFile || entry.name);
83
+ const url = result.urls?.testRun || result.testRunLink;
84
+
85
+ if (!url) continue;
86
+
87
+ const isNew = !manifest.examples[testFileName];
88
+ manifest.examples[testFileName] = {
89
+ url: url,
90
+ lastUpdated: result.date || new Date().toISOString(),
91
+ };
92
+
93
+ if (isNew) {
94
+ stats.added++;
95
+ } else {
96
+ stats.updated++;
97
+ }
98
+
99
+ console.log(`${isNew ? "āž•" : "šŸ”„"} ${testFileName}: ${url}`);
100
+ } catch (err) {
101
+ console.warn(`āš ļø Failed to read ${jsonFile}: ${err.message}`);
102
+ }
103
+ }
99
104
  }
100
105
 
101
106
  if (stats.added > 0 || stats.updated > 0) {
@@ -103,7 +108,7 @@ async function processInput(inputStream) {
103
108
  console.log(`\n✨ Manifest updated: ${stats.added} added, ${stats.updated} updated`);
104
109
  console.log(`šŸ“‚ Written to: ${MANIFEST_PATH}`);
105
110
  } else {
106
- console.log("\nāš ļø No TESTDRIVER_EXAMPLE_URL entries found in input");
111
+ console.log("\nāš ļø No example URLs found in result files");
107
112
  }
108
113
 
109
114
  return stats;
@@ -112,28 +117,24 @@ async function processInput(inputStream) {
112
117
  // Show help
113
118
  function showHelp() {
114
119
  console.log(`
115
- Extract Example URLs from CI Logs
120
+ Extract Example URLs from Test Result JSON Files
116
121
 
117
122
  Usage:
118
- cat ci-log.txt | node extract-example-urls.js
119
- node extract-example-urls.js < ci-log.txt
120
- node extract-example-urls.js --file=ci-log.txt
123
+ node extract-example-urls.js --results-dir=.testdriver/results
121
124
 
122
125
  Options:
123
- --file=<path> Read from file instead of stdin
124
- --help, -h Show this help message
126
+ --results-dir=<path> Path to .testdriver/results directory (required)
127
+ --help, -h Show this help message
125
128
 
126
129
  Description:
127
- Parses CI log output looking for lines matching:
128
- TESTDRIVER_EXAMPLE_URL::<filename>::<url>
129
-
130
- Updates docs/_data/examples-manifest.json with the extracted URLs.
130
+ Reads per-test-case JSON result files from .testdriver/results/examples/
131
+ and updates docs/_data/examples-manifest.json with the extracted URLs.
131
132
  Existing entries are updated, new entries are added.
132
133
  `);
133
134
  }
134
135
 
135
136
  // Main function
136
- async function main() {
137
+ function main() {
137
138
  const options = parseArgs();
138
139
 
139
140
  if (options.help) {
@@ -141,25 +142,19 @@ async function main() {
141
142
  process.exit(0);
142
143
  }
143
144
 
144
- console.log("šŸ” Extracting example URLs from input...\n");
145
-
146
- let inputStream;
147
- if (options.file) {
148
- if (!fs.existsSync(options.file)) {
149
- console.error(`āŒ File not found: ${options.file}`);
150
- process.exit(1);
151
- }
152
- inputStream = fs.createReadStream(options.file);
153
- } else {
154
- inputStream = process.stdin;
145
+ if (!options.resultsDir) {
146
+ console.error("āŒ --results-dir is required. Example: --results-dir=.testdriver/results");
147
+ process.exit(1);
155
148
  }
156
149
 
157
- try {
158
- await processInput(inputStream);
159
- } catch (error) {
160
- console.error(`āŒ Error: ${error.message}`);
150
+ console.log("šŸ” Reading example URLs from JSON result files...\n");
151
+
152
+ if (!fs.existsSync(options.resultsDir)) {
153
+ console.error(`āŒ Results directory not found: ${options.resultsDir}`);
161
154
  process.exit(1);
162
155
  }
156
+
157
+ processResultsDir(options.resultsDir);
163
158
  }
164
159
 
165
160
  main();
package/docs/docs.json CHANGED
@@ -87,7 +87,8 @@
87
87
  "pages": [
88
88
  "/v7/running-tests",
89
89
  "/v7/caching",
90
- "/v7/ci-cd"
90
+ "/v7/ci-cd",
91
+ "/v7/test-results-json"
91
92
  ]
92
93
  },
93
94
  {
package/docs/v7/find.mdx CHANGED
@@ -50,8 +50,8 @@ const element = await testdriver.find(description, options)
50
50
  - `"any"` — No wrapping, uses the description as-is (default behavior)
51
51
  </ParamField>
52
52
 
53
- <ParamField path="zoom" type="boolean" default={false}>
54
- Enable two-phase zoom mode for better precision in crowded UIs with many similar elements.
53
+ <ParamField path="zoom" type="boolean" default={true}>
54
+ Two-phase zoom mode for better precision in crowded UIs with many similar elements. Enabled by default.
55
55
  </ParamField>
56
56
 
57
57
  <ParamField path="ai" type="object">
@@ -333,14 +333,19 @@ The `timeout` option:
333
333
  - Returns the element (check `element.found()` if not throwing on failure)
334
334
  - Set to `0` to disable polling and make a single attempt
335
335
 
336
- ## Zoom Mode for Crowded UIs
336
+ ## Zoom Mode
337
337
 
338
- When dealing with many similar icons or elements clustered together (like browser toolbars), enable `zoom` mode for better precision:
338
+ Zoom mode is **enabled by default**. It uses a two-phase approach for better precision when locating elements, especially in crowded UIs with many similar elements.
339
+
340
+ To disable zoom for a specific find call, pass `zoom: false`:
339
341
 
340
342
  ```javascript
341
- // Enable zoom for better precision in crowded UIs
342
- const extensionsBtn = await testdriver.find('extensions puzzle icon in Chrome toolbar', { zoom: true });
343
+ // Zoom is on by default — no option needed
344
+ const extensionsBtn = await testdriver.find('extensions puzzle icon in Chrome toolbar');
343
345
  await extensionsBtn.click();
346
+
347
+ // Disable zoom for a specific call if needed
348
+ const largeButton = await testdriver.find('big hero button', { zoom: false });
344
349
  ```
345
350
 
346
351
  ### How Zoom Mode Works
@@ -353,22 +358,11 @@ await extensionsBtn.click();
353
358
  This two-phase approach gives the AI a higher-resolution view of the target area, improving accuracy when multiple similar elements are close together.
354
359
 
355
360
  <Tip>
356
- Use `zoom: true` when:
357
- - Clicking small icons in toolbars
358
- - Selecting from a grid of similar items
359
- - Targeting elements in dense UI areas
360
- - The default locate is clicking the wrong similar element
361
- - You get an AI verification rejection like "The crosshair is located in the empty space of the browser's tab bar/title bar area" — this means the initial locate was imprecise and zoom will help the AI pinpoint the correct element
361
+ You may want to disable zoom with `zoom: false` when:
362
+ - Targeting large, isolated elements where the extra precision isn't needed
363
+ - You want to speed up find calls in simple UIs
362
364
  </Tip>
363
365
 
364
- ```javascript
365
- // Without zoom - may click wrong icon in toolbar
366
- const icon = await testdriver.find('settings icon');
367
-
368
- // With zoom - better precision for crowded areas
369
- const icon = await testdriver.find('settings icon', { zoom: true });
370
- ```
371
-
372
366
  ## Cache Options
373
367
 
374
368
  Control caching behavior to optimize performance, especially when using dynamic variables in prompts.
@@ -0,0 +1,258 @@
1
+ ---
2
+ title: "Test Result JSON"
3
+ sidebarTitle: "Test Result JSON"
4
+ description: "Per-test JSON result files with metadata, versions, and infrastructure details"
5
+ icon: "file-code"
6
+ ---
7
+
8
+ ## Overview
9
+
10
+ TestDriver automatically writes a JSON result file for each test case after it finishes. These files contain comprehensive metadata about the test run, including SDK and runner versions, infrastructure details, interaction statistics, and links to recordings.
11
+
12
+ Result files are written to:
13
+
14
+ ```
15
+ .testdriver/results/<testFile>/<testName>.json
16
+ ```
17
+
18
+ For example, a test file `tests/login.test.mjs` with a test named `"should log in"` produces:
19
+
20
+ ```
21
+ .testdriver/results/tests/login.test.mjs/should_log_in.json
22
+ ```
23
+
24
+ <Note>
25
+ Test names are sanitized for filesystem use — special characters are replaced with underscores and names are truncated to 200 characters.
26
+ </Note>
27
+
28
+ ## Enabling
29
+
30
+ No configuration is required. The JSON files are written automatically by the TestDriver Vitest reporter plugin whenever tests run.
31
+
32
+ ## JSON Schema
33
+
34
+ Each result file is organized into logical groups:
35
+
36
+ ### `versions`
37
+
38
+ | Field | Type | Description |
39
+ |---|---|---|
40
+ | `versions.sdk` | `string \| null` | TestDriver SDK version (e.g. `"7.8.0"`) |
41
+ | `versions.vitest` | `string \| null` | Vitest version used to run the test |
42
+ | `versions.api` | `string \| null` | TestDriver API server version |
43
+ | `versions.runnerBefore` | `string \| null` | Runner version at sandbox start |
44
+ | `versions.runnerAfter` | `string \| null` | Runner version after auto-update |
45
+ | `versions.runnerWasUpdated` | `boolean` | Whether the runner was auto-updated during provisioning |
46
+
47
+ ### `test`
48
+
49
+ | Field | Type | Description |
50
+ |---|---|---|
51
+ | `test.file` | `string \| null` | Relative path to the test file |
52
+ | `test.name` | `string \| null` | Name of the test case |
53
+ | `test.suite` | `string \| null` | Name of the parent `describe` block |
54
+ | `test.passed` | `boolean` | Whether the test passed |
55
+ | `test.caseId` | `string \| null` | Database ID for this test case |
56
+ | `test.runId` | `string \| null` | Database ID for the overall test run |
57
+ | `test.error` | `string \| null` | Error message if the test failed |
58
+ | `test.errorStack` | `string \| null` | Error stack trace if the test failed |
59
+
60
+ ### `urls`
61
+
62
+ | Field | Type | Description |
63
+ |---|---|---|
64
+ | `urls.api` | `string \| null` | API root URL used for this test |
65
+ | `urls.console` | `string \| null` | TestDriver console base URL |
66
+ | `urls.vnc` | `string \| null` | VNC URL for the sandbox |
67
+ | `urls.testRun` | `string \| null` | Direct link to this test case in the console |
68
+
69
+ ### `replay`
70
+
71
+ The `replay` object contains the recording replay URL and derived embed links. The `gifUrl` and `embedUrl` are generated automatically from the replay URL.
72
+
73
+ | Field | Type | Description |
74
+ |---|---|---|
75
+ | `replay.url` | `string \| null` | Recording replay URL |
76
+ | `replay.gifUrl` | `string \| null` | Animated GIF thumbnail of the recording |
77
+ | `replay.embedUrl` | `string \| null` | Embeddable replay URL (appends `&embed=true`) |
78
+ | `replay.markdown` | `string \| null` | Ready-to-use Markdown embed with GIF linking to the replay |
79
+
80
+ The `replay.markdown` field produces a clickable GIF badge you can paste directly into PR comments, README files, or issue descriptions:
81
+
82
+ ```markdown
83
+ [![Test Recording](https://api.testdriver.ai/replay/abc123/gif?shareKey=xyz)](https://console.testdriver.ai/replay/abc123?share=xyz)
84
+ ```
85
+
86
+ ### `date`
87
+
88
+ | Field | Type | Description |
89
+ |---|---|---|
90
+ | `date` | `string` | ISO 8601 timestamp when the test finished |
91
+
92
+ ### `team`
93
+
94
+ | Field | Type | Description |
95
+ |---|---|---|
96
+ | `team.id` | `string \| null` | Team ID from the sandbox |
97
+ | `team.sessionId` | `string \| null` | SDK session ID |
98
+
99
+ ### `infrastructure`
100
+
101
+ | Field | Type | Description |
102
+ |---|---|---|
103
+ | `infrastructure.sandboxId` | `string \| null` | Sandbox instance ID |
104
+ | `infrastructure.instanceId` | `string \| null` | Instance ID |
105
+ | `infrastructure.os` | `string \| null` | Operating system of the sandbox (`"linux"` or `"windows"`) |
106
+ | `infrastructure.amiId` | `string \| null` | AWS AMI ID used for provisioning |
107
+ | `infrastructure.e2bTemplateId` | `string \| null` | E2B template ID used for provisioning |
108
+ | `infrastructure.imageVersion` | `string \| null` | Sandbox image version |
109
+
110
+ ### `realtime`
111
+
112
+ | Field | Type | Description |
113
+ |---|---|---|
114
+ | `realtime.channel` | `string \| null` | Ably channel name used for communication |
115
+ | `realtime.messageCount` | `number` | Number of messages published to the realtime channel |
116
+
117
+ ### `interactions`
118
+
119
+ | Field | Type | Description |
120
+ |---|---|---|
121
+ | `interactions.total` | `number` | Total number of interactions recorded |
122
+ | `interactions.cached` | `number` | Number of interactions served from cache |
123
+ | `interactions.byType` | `object` | Breakdown of interactions by type (e.g. `find`, `click`, `assert`) |
124
+
125
+ ## Example Output
126
+
127
+ ```json
128
+ {
129
+ "sdkVersion": "7.8.0",
130
+ "vitestVersion": "4.0.0",
131
+ "apiVersion": "1.45.0",
132
+ "runnerVersionBefore": "2.1.0",
133
+ "runnerVersionAfter": "2.1.1",
134
+ "wasUpdated": true,
135
+ "apiUrl": "https://api.testdriver.ai",
136
+ "consoleUrl": "https://console.testdriver.ai",
137
+ "testRunLink": "https://console.testdriver.ai/runs/abc123/def456",
138
+ "dashcamUrl": "https://app.dashcam.io/replay/abc123",
139
+ "vncUrl": "wss://sandbox-123.testdriver.ai/vnc",
140
+ "date": "2025-01-15T14:30:00.000Z",
141
+ "team": {
142
+ "id": "team_abc123",
143
+ "sessionId": "sess_xyz789"
144
+ },
145
+ "infrastructure": {
146
+ "sandboxId": "sandbox-123",
147
+ "instanceId": "i-abc123",
148
+ "os": "linux",
149
+ "amiId": "ami-0abc123",
150
+ "e2bTemplateId": null,
151
+ "imageVersion": "v2.1.0"
152
+ },
153
+ "realtime": {
154
+ "channel": "sandbox:sandbox-123",
155
+ "messageCount": 42
156
+ },
157
+ "interactions": {
158
+ "total": 15,
159
+ "cached": 3,
160
+ "byType": {
161
+ "find": 8,
162
+ "click": 5,
163
+ "assert": 2
164
+ }
165
+ }
166
+ }
167
+ ```
168
+
169
+ ## Using Result Files in CI
170
+
171
+ Result files are useful for extracting test metadata in CI pipelines without parsing log output.
172
+
173
+ ### GitHub Actions Example
174
+
175
+ Use `fromJSON` to parse a result file into a GitHub Actions expression you can reference in subsequent steps:
176
+
177
+ ```yaml
178
+ - name: Run tests
179
+ run: npx vitest run tests/login.test.mjs
180
+
181
+ - name: Parse result
182
+ id: result
183
+ run: |
184
+ # Read the first JSON result file
185
+ FILE=$(find .testdriver/results -name '*.json' | head -n 1)
186
+ echo "json=$(cat "$FILE")" >> "$GITHUB_OUTPUT"
187
+
188
+ - name: Comment on PR
189
+ if: fromJSON(steps.result.outputs.json).test.passed == false
190
+ uses: actions/github-script@v7
191
+ with:
192
+ script: |
193
+ const result = ${{ steps.result.outputs.json }};
194
+ await github.rest.issues.createComment({
195
+ owner: context.repo.owner,
196
+ repo: context.repo.repo,
197
+ issue_number: context.issue.number,
198
+ body: [
199
+ `āŒ **${result.test.name}** failed`,
200
+ ``,
201
+ `Error: ${result.test.error}`,
202
+ ``,
203
+ result.replay.markdown,
204
+ ``,
205
+ `[View full recording](${result.urls.testRun})`
206
+ ].join('\n')
207
+ });
208
+ ```
209
+
210
+ You can also load all results into a matrix or iterate over them:
211
+
212
+ ```yaml
213
+ - name: Run tests
214
+ run: npx vitest run tests/*.test.mjs
215
+
216
+ - name: Collect results
217
+ id: results
218
+ run: |
219
+ # Merge all result files into a JSON array
220
+ echo "json=$(find .testdriver/results -name '*.json' -exec cat {} + | jq -s '.')" >> "$GITHUB_OUTPUT"
221
+
222
+ - name: Summary
223
+ run: |
224
+ echo '## Test Results' >> $GITHUB_STEP_SUMMARY
225
+ RESULTS='${{ steps.results.outputs.json }}'
226
+ echo "$RESULTS" | jq -r '.[] | "| \(.test.name) | \(if .test.passed then "āœ…" else "āŒ" end) | \(.urls.testRun) |"' >> $GITHUB_STEP_SUMMARY
227
+ ```
228
+
229
+ ### Reading Results Programmatically
230
+
231
+ ```javascript
232
+ import fs from "fs";
233
+ import path from "path";
234
+
235
+ const resultsDir = ".testdriver/results";
236
+
237
+ function readResults(dir) {
238
+ const results = [];
239
+ for (const testDir of fs.readdirSync(dir, { recursive: true })) {
240
+ const fullPath = path.join(dir, testDir);
241
+ if (fullPath.endsWith(".json") && fs.statSync(fullPath).isFile()) {
242
+ results.push(JSON.parse(fs.readFileSync(fullPath, "utf-8")));
243
+ }
244
+ }
245
+ return results;
246
+ }
247
+
248
+ const results = readResults(resultsDir);
249
+ const passed = results.filter(r => r.test.passed);
250
+ const failed = results.filter(r => !r.test.passed);
251
+
252
+ console.log(`${passed.length} passed, ${failed.length} failed`);
253
+ for (const r of failed) {
254
+ console.log(` FAIL: ${r.test.name} — ${r.test.error}`);
255
+ console.log(` Recording: ${r.urls.testRun}`);
256
+ console.log(` Embed: ${r.replay.markdown}`);
257
+ }
258
+ ```
@@ -16,7 +16,7 @@ describe("Scroll Keyboard Test", () => {
16
16
  // Navigate to https://www.webhamster.com/
17
17
  await testdriver.focusApplication("Google Chrome");
18
18
  const urlBar = await testdriver.find(
19
- "testdriver-sandbox.vercel.app/login, the URL in the omnibox showing the current page", {zoom: true}
19
+ "the URL in the omnibox", {zoom: true}
20
20
  );
21
21
  await urlBar.click();
22
22
  await testdriver.pressKeys(["ctrl", "a"]);
@@ -1218,6 +1218,122 @@ class TestDriverReporter {
1218
1218
  );
1219
1219
  console.log("");
1220
1220
 
1221
+ // Write per-test-case JSON result file
1222
+ {
1223
+ const testResult = meta.testResult || {};
1224
+
1225
+ // Parse replay URL to extract replayId and shareKey for embed links
1226
+ let replayUrl = dashcamUrl || null;
1227
+ let replayGifUrl = null;
1228
+ let replayEmbedUrl = null;
1229
+ let replayMarkdown = null;
1230
+ const replayMatch = dashcamUrl && dashcamUrl.match(/\/replay\/([a-f0-9]+)\?share=([^&\s]+)/);
1231
+ if (replayMatch) {
1232
+ const [, replayId, shareKey] = replayMatch;
1233
+ const apiRoot = pluginState.apiRoot;
1234
+ replayGifUrl = `${apiRoot}/replay/${replayId}/gif?shareKey=${shareKey}`;
1235
+ replayEmbedUrl = `${consoleUrl}/replay/${replayId}?share=${shareKey}&embed=true`;
1236
+ replayMarkdown = `[![Test Recording](${replayGifUrl})](${replayUrl})`;
1237
+ }
1238
+
1239
+ const resultData = {
1240
+ // Versions
1241
+ versions: {
1242
+ sdk: testResult.sdkVersion || null,
1243
+ vitest: resolveVitestVersion() || null,
1244
+ api: testResult.apiVersion || null,
1245
+ runnerBefore: testResult.runnerVersionBefore || null,
1246
+ runnerAfter: testResult.runnerVersionAfter || null,
1247
+ runnerWasUpdated: testResult.wasUpdated || false,
1248
+ },
1249
+
1250
+ // Test info
1251
+ test: {
1252
+ file: testResult.testFile || null,
1253
+ name: testResult.testName || null,
1254
+ suite: testResult.suiteName || null,
1255
+ passed: status === "passed",
1256
+ caseId: testCaseDbId || null,
1257
+ runId: testRunDbId || null,
1258
+ error: errorMessage || testResult.error || null,
1259
+ errorStack: errorStack || testResult.errorStack || null,
1260
+ },
1261
+
1262
+ // URLs
1263
+ urls: {
1264
+ api: testResult.apiUrl || null,
1265
+ console: consoleUrl || null,
1266
+ vnc: testResult.vncUrl || null,
1267
+ testRun: testCaseDbId ? `${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}` : null,
1268
+ },
1269
+
1270
+ // Recording replay
1271
+ replay: {
1272
+ url: replayUrl,
1273
+ gifUrl: replayGifUrl,
1274
+ embedUrl: replayEmbedUrl,
1275
+ markdown: replayMarkdown,
1276
+ },
1277
+
1278
+ // Timing
1279
+ date: testResult.date || new Date().toISOString(),
1280
+
1281
+ // Team & session
1282
+ team: {
1283
+ id: testResult.teamId || null,
1284
+ sessionId: testResult.sessionId || null,
1285
+ },
1286
+
1287
+ // Infrastructure
1288
+ infrastructure: {
1289
+ sandboxId: testResult.sandboxId || null,
1290
+ instanceId: testResult.instanceId || null,
1291
+ os: testResult.os || null,
1292
+ amiId: testResult.amiId || null,
1293
+ e2bTemplateId: testResult.e2bTemplateId || null,
1294
+ imageVersion: testResult.imageVersion || null,
1295
+ },
1296
+
1297
+ // Realtime
1298
+ realtime: {
1299
+ channel: testResult.realtimeChannel || null,
1300
+ messageCount: testResult.realtimeMessageCount || 0,
1301
+ },
1302
+
1303
+ // Interactions
1304
+ interactions: testResult.interactions || { total: 0, cached: 0, byType: {} },
1305
+ };
1306
+
1307
+ // Sanitize testName for filesystem use
1308
+ const safeName = (test.name || "unknown").replace(/[^a-zA-Z0-9_.-]/g, "_").substring(0, 200);
1309
+ const resultDir = path.join(process.cwd(), ".testdriver", "results", testFile);
1310
+ fs.mkdirSync(resultDir, { recursive: true });
1311
+
1312
+ // Include a stable unique suffix in the filename to avoid collisions
1313
+ // when multiple tests in the same file share the same name.
1314
+ const hashSourceParts = [];
1315
+ if (test.id) {
1316
+ hashSourceParts.push(String(test.id));
1317
+ }
1318
+ if (Array.isArray(test.suitePath)) {
1319
+ hashSourceParts.push(test.suitePath.join(" > "));
1320
+ }
1321
+ if (test.file && (test.file.name || test.file.path)) {
1322
+ hashSourceParts.push(test.file.name || test.file.path);
1323
+ }
1324
+ // Fallback to the test name if no other identifiers are available.
1325
+ if (hashSourceParts.length === 0) {
1326
+ hashSourceParts.push(test.name || "unknown");
1327
+ }
1328
+ const hashSource = hashSourceParts.join(" | ");
1329
+ const uniqueHash = crypto.createHash("sha256").update(hashSource).digest("hex").slice(0, 8);
1330
+
1331
+ fs.writeFileSync(
1332
+ path.join(resultDir, `${safeName}-${uniqueHash}.json`),
1333
+ JSON.stringify(resultData, null, 2),
1334
+ );
1335
+ }
1336
+
1221
1337
  // If there were retries, list all per-attempt dashcam URLs for debugging
1222
1338
  if (hasRetries) {
1223
1339
  const validAttempts = dashcamUrls.filter(a => a.url);
@@ -1229,13 +1345,6 @@ class TestDriverReporter {
1229
1345
  }
1230
1346
  }
1231
1347
 
1232
- // Output parseable format for docs generation (examples only)
1233
- if (testFile.startsWith("examples/")) {
1234
- const testFileName = path.basename(testFile);
1235
- console.log(
1236
- `TESTDRIVER_EXAMPLE_URL::${testFileName}::${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}`,
1237
- );
1238
- }
1239
1348
  } catch (error) {
1240
1349
  logger.error("Failed to report test case:", error.message);
1241
1350
  }