@testdriverai/mcp 7.10.0-canary.0 → 7.10.1-test

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/.claude-plugin/marketplace.json +17 -0
  2. package/agent/index.js +116 -71
  3. package/agent/interface.js +13 -1
  4. package/agent/lib/commands.js +35 -15
  5. package/agent/lib/provision-commands.js +92 -11
  6. package/agent/lib/sandbox.js +263 -30
  7. package/agent/lib/sdk.js +16 -12
  8. package/agent/lib/system.js +107 -31
  9. package/ai/.claude-plugin/plugin.json +89 -4
  10. package/ai/agents/testdriver.md +4 -8
  11. package/ai/skills/testdriver-assert/SKILL.md +0 -1
  12. package/ai/skills/testdriver-aws-setup/SKILL.md +1 -1
  13. package/ai/skills/testdriver-caching/SKILL.md +13 -6
  14. package/ai/skills/testdriver-ci-cd/SKILL.md +57 -14
  15. package/ai/skills/testdriver-client/SKILL.md +10 -6
  16. package/ai/skills/testdriver-customizing-devices/SKILL.md +7 -5
  17. package/ai/skills/testdriver-dashcam/SKILL.md +36 -3
  18. package/ai/skills/testdriver-debugging-with-screenshots/SKILL.md +28 -4
  19. package/ai/skills/testdriver-device-config/SKILL.md +5 -5
  20. package/ai/skills/testdriver-find/SKILL.md +50 -11
  21. package/ai/skills/testdriver-generating-tests/SKILL.md +163 -8
  22. package/ai/skills/testdriver-hosted/SKILL.md +1 -1
  23. package/ai/skills/testdriver-machine-setup/SKILL.md +331 -0
  24. package/ai/skills/testdriver-making-assertions/SKILL.md +78 -2
  25. package/ai/skills/testdriver-mcp-workflow/SKILL.md +2 -2
  26. package/ai/skills/testdriver-parse/SKILL.md +1 -1
  27. package/ai/skills/testdriver-performing-actions/SKILL.md +97 -3
  28. package/ai/skills/testdriver-provision/SKILL.md +0 -20
  29. package/ai/skills/testdriver-quickstart/SKILL.md +299 -21
  30. package/ai/skills/testdriver-reusable-code/SKILL.md +3 -3
  31. package/ai/skills/testdriver-running-tests/SKILL.md +1 -1
  32. package/ai/skills/testdriver-screenshot/SKILL.md +5 -5
  33. package/ai/skills/testdriver-scroll/SKILL.md +30 -160
  34. package/ai/skills/testdriver-test-writer/SKILL.md +3 -5
  35. package/ai/skills/testdriver-testdriver/SKILL.md +3 -7
  36. package/ai/skills/testdriver-wait/SKILL.md +1 -1
  37. package/{docs/v7/ai.mdx → ai/skills/testdriver:ai/SKILL.md} +3 -4
  38. package/ai/skills/testdriver:assert/SKILL.md +315 -0
  39. package/ai/skills/testdriver:aws-setup/SKILL.md +448 -0
  40. package/ai/skills/testdriver:cache/SKILL.md +221 -0
  41. package/ai/skills/testdriver:caching/SKILL.md +124 -0
  42. package/ai/skills/testdriver:captcha/SKILL.md +158 -0
  43. package/ai/skills/testdriver:ci-cd/SKILL.md +659 -0
  44. package/ai/skills/testdriver:click/SKILL.md +286 -0
  45. package/ai/skills/testdriver:client/SKILL.md +477 -0
  46. package/ai/skills/testdriver:customizing-devices/SKILL.md +319 -0
  47. package/ai/skills/testdriver:dashcam/SKILL.md +451 -0
  48. package/ai/skills/testdriver:debugging-with-screenshots/SKILL.md +415 -0
  49. package/{docs/v7/device-config.mdx → ai/skills/testdriver:device-config/SKILL.md} +8 -8
  50. package/ai/skills/testdriver:double-click/SKILL.md +102 -0
  51. package/ai/skills/testdriver:elements/SKILL.md +605 -0
  52. package/ai/skills/testdriver:enterprise/SKILL.md +7 -0
  53. package/ai/skills/testdriver:errors/SKILL.md +246 -0
  54. package/ai/skills/testdriver:events/SKILL.md +356 -0
  55. package/ai/skills/testdriver:exec/SKILL.md +317 -0
  56. package/ai/skills/testdriver:find/SKILL.md +860 -0
  57. package/ai/skills/testdriver:focus-application/SKILL.md +293 -0
  58. package/ai/skills/testdriver:generating-tests/SKILL.md +25 -0
  59. package/ai/skills/testdriver:hosted/SKILL.md +156 -0
  60. package/ai/skills/testdriver:hover/SKILL.md +278 -0
  61. package/{docs/v7/locating-elements.mdx → ai/skills/testdriver:locating-elements/SKILL.md} +3 -3
  62. package/ai/skills/testdriver:making-assertions/SKILL.md +32 -0
  63. package/ai/skills/testdriver:mcp/SKILL.md +7 -0
  64. package/ai/skills/testdriver:mouse-down/SKILL.md +161 -0
  65. package/ai/skills/testdriver:mouse-up/SKILL.md +164 -0
  66. package/ai/skills/testdriver:parse/SKILL.md +236 -0
  67. package/ai/skills/testdriver:performing-actions/SKILL.md +53 -0
  68. package/ai/skills/testdriver:press-keys/SKILL.md +348 -0
  69. package/ai/skills/testdriver:provision/SKILL.md +331 -0
  70. package/ai/skills/testdriver:quickstart/SKILL.md +172 -0
  71. package/ai/skills/testdriver:redraw/SKILL.md +214 -0
  72. package/ai/skills/testdriver:reusable-code/SKILL.md +249 -0
  73. package/ai/skills/testdriver:right-click/SKILL.md +123 -0
  74. package/{docs/v7/running-tests.mdx → ai/skills/testdriver:running-tests/SKILL.md} +4 -4
  75. package/ai/skills/testdriver:screenshot/SKILL.md +248 -0
  76. package/ai/skills/testdriver:screenshots/SKILL.md +184 -0
  77. package/ai/skills/testdriver:scroll/SKILL.md +205 -0
  78. package/ai/skills/testdriver:secrets/SKILL.md +115 -0
  79. package/ai/skills/testdriver:self-hosted/SKILL.md +147 -0
  80. package/ai/skills/testdriver:test-results-json/SKILL.md +257 -0
  81. package/ai/skills/testdriver:testdriver/SKILL.md +624 -0
  82. package/ai/skills/testdriver:type/SKILL.md +357 -0
  83. package/ai/skills/testdriver:variables/SKILL.md +111 -0
  84. package/ai/skills/testdriver:wait/SKILL.md +50 -0
  85. package/{docs/v7/waiting-for-elements.mdx → ai/skills/testdriver:waiting-for-elements/SKILL.md} +3 -3
  86. package/ai/skills/testdriver:what-is-testdriver/SKILL.md +54 -0
  87. package/debugger/index.html +12 -0
  88. package/docs/_data/examples-manifest.json +54 -46
  89. package/docs/_data/examples-manifest.schema.json +12 -2
  90. package/docs/_scripts/extract-example-urls.js +21 -3
  91. package/docs/_scripts/generate-examples.js +64 -16
  92. package/docs/changelog.mdx +29 -2
  93. package/docs/claude-mcp-plugin.mdx +65 -18
  94. package/docs/docs.json +16 -33
  95. package/docs/snippets/tests/scroll-yaml.mdx +2 -2
  96. package/docs/v6/commands/scroll-until-image.mdx +1 -1
  97. package/docs/v6/commands/scroll-until-text.mdx +2 -2
  98. package/docs/v6/commands/scroll.mdx +2 -2
  99. package/docs/v7/_drafts/agents.mdx +0 -4
  100. package/docs/v7/_drafts/best-practices.mdx +0 -12
  101. package/docs/v7/_drafts/commands/scroll-until-image.mdx +1 -1
  102. package/docs/v7/_drafts/commands/scroll-until-text.mdx +2 -2
  103. package/docs/v7/_drafts/commands/scroll.mdx +1 -1
  104. package/docs/v7/_drafts/configuration.mdx +0 -12
  105. package/docs/v7/_drafts/faq.mdx +2 -10
  106. package/docs/v7/_drafts/readme.mdx +0 -1
  107. package/docs/v7/_drafts/troubleshooting.mdx +1 -4
  108. package/docs/v7/assert.mdx +0 -1
  109. package/docs/v7/aws-setup.mdx +1 -1
  110. package/docs/v7/caching.mdx +14 -10
  111. package/docs/v7/ci-cd.mdx +98 -14
  112. package/docs/v7/client.mdx +10 -6
  113. package/docs/v7/copilot/auto-healing.mdx +167 -18
  114. package/docs/v7/copilot/running-tests.mdx +915 -54
  115. package/docs/v7/customizing-devices.mdx +7 -5
  116. package/docs/v7/dashcam.mdx +36 -3
  117. package/docs/v7/debugging-with-screenshots.mdx +31 -7
  118. package/docs/v7/examples/ai.mdx +13 -8
  119. package/docs/v7/examples/assert.mdx +1 -1
  120. package/docs/v7/examples/chrome-extension.mdx +30 -13
  121. package/docs/v7/examples/element-not-found.mdx +1 -1
  122. package/docs/v7/examples/findall-coffee-icons.mdx +3 -2
  123. package/docs/v7/examples/formatted-logging.mdx +74 -0
  124. package/docs/v7/examples/hover-image.mdx +1 -1
  125. package/docs/v7/examples/hover-text-with-description.mdx +2 -2
  126. package/docs/v7/examples/hover-text.mdx +1 -1
  127. package/docs/v7/examples/installer.mdx +1 -1
  128. package/docs/v7/examples/launch-vscode-linux.mdx +27 -27
  129. package/docs/v7/examples/match-image.mdx +1 -1
  130. package/docs/v7/examples/parse.mdx +3 -2
  131. package/docs/v7/examples/press-keys.mdx +2 -2
  132. package/docs/v7/examples/scroll-keyboard.mdx +2 -2
  133. package/docs/v7/examples/scroll-until-text.mdx +1 -1
  134. package/docs/v7/examples/scroll.mdx +1 -1
  135. package/docs/v7/examples/type.mdx +1 -1
  136. package/docs/v7/find.mdx +50 -11
  137. package/docs/v7/generating-tests.mdx +166 -10
  138. package/docs/v7/hosted.mdx +1 -1
  139. package/docs/v7/making-assertions.mdx +81 -4
  140. package/docs/v7/parse.mdx +1 -1
  141. package/docs/v7/performing-actions.mdx +100 -5
  142. package/docs/v7/provision.mdx +0 -20
  143. package/docs/v7/quickstart.mdx +299 -21
  144. package/docs/v7/reusable-code.mdx +3 -3
  145. package/docs/v7/screenshot.mdx +5 -5
  146. package/docs/v7/scroll.mdx +30 -160
  147. package/docs/v7/wait.mdx +1 -1
  148. package/examples/chrome-extension.test.mjs +29 -12
  149. package/examples/config.mjs +1 -1
  150. package/examples/launch-vscode-linux.test.mjs +25 -25
  151. package/interfaces/cli/commands/init.js +84 -2
  152. package/interfaces/vitest-plugin.mjs +51 -61
  153. package/lib/core/Dashcam.js +43 -2
  154. package/lib/core/index.d.ts +13 -1
  155. package/lib/github-comment.mjs +11 -5
  156. package/lib/init-project.js +98 -44
  157. package/lib/install-clients.js +470 -0
  158. package/lib/provision.js +749 -0
  159. package/lib/resolve-channel.js +2 -2
  160. package/lib/sentry.js +15 -2
  161. package/lib/vitest/hooks.mjs +44 -34
  162. package/mcp-server/README.md +32 -0
  163. package/mcp-server/dist/core/actions.d.ts +176 -0
  164. package/mcp-server/dist/core/actions.js +753 -0
  165. package/mcp-server/dist/env-utils.d.ts +45 -0
  166. package/mcp-server/dist/env-utils.js +63 -0
  167. package/mcp-server/dist/provision-types.d.ts +6 -2
  168. package/mcp-server/dist/provision-types.js +3 -1
  169. package/mcp-server/dist/server.mjs +449 -751
  170. package/mcp-server/package-lock.json +384 -1
  171. package/mcp-server/package.json +5 -2
  172. package/mcp-server/src/core/actions.ts +912 -0
  173. package/mcp-server/src/env-utils.test.ts +82 -0
  174. package/mcp-server/src/env-utils.ts +77 -0
  175. package/mcp-server/src/provision-types.ts +4 -1
  176. package/mcp-server/src/server.ts +533 -954
  177. package/mcp-server/tsconfig.json +1 -1
  178. package/mcp-server/vitest.config.ts +7 -0
  179. package/package.json +15 -3
  180. package/sdk.d.ts +11 -9
  181. package/sdk.js +116 -746
  182. package/setup/aws/spawn-runner.sh +24 -3
  183. package/docs/v7/_drafts/caching-ai.mdx +0 -215
  184. package/docs/v7/_drafts/caching.mdx +0 -366
  185. package/docs/v7/_drafts/prompt-cache.mdx +0 -200
  186. package/docs/v7/copilot/creating-tests.mdx +0 -156
  187. package/docs/v7/copilot/github.mdx +0 -143
  188. package/docs/v7/copilot/setup.mdx +0 -143
  189. package/docs/v7/examples/exec-output.mdx +0 -84
  190. package/docs/v7/examples/exec-pwsh.mdx +0 -82
  191. package/docs/v7/examples/focus-window.mdx +0 -61
  192. package/docs/v7/examples/scroll-until-image.mdx +0 -82
  193. package/docs/v7/examples/windows-installer.mdx +0 -95
  194. package/examples/formatted-logging.test.mjs +0 -27
  195. package/examples/match-image.test.mjs +0 -55
  196. package/manual/exec-stream-logs.test.mjs +0 -25
@@ -10,6 +10,7 @@ import { setTestRunInfo } from "./shared-test-state.mjs";
10
10
  // Use createRequire to import CommonJS modules without esbuild processing
11
11
  const require = createRequire(import.meta.url);
12
12
  const channelConfig = require("../lib/resolve-channel.js");
13
+ const environments = require("../lib/environments.json");
13
14
 
14
15
  // Import Sentry for error reporting
15
16
  const Sentry = require("@sentry/node");
@@ -39,8 +40,8 @@ function initializeSentry() {
39
40
  "https://452bd5a00dbd83a38ee8813e11c57694@o4510262629236736.ingest.us.sentry.io/4510480443637760",
40
41
  environment: channelConfig.sentryEnvironment,
41
42
  release: version,
42
- sampleRate: 1.0,
43
- tracesSampleRate: 1.0,
43
+ sampleRate: 0.01,
44
+ tracesSampleRate: 0.01,
44
45
  enableLogs: true,
45
46
  integrations: [Sentry.httpIntegration(), Sentry.nodeContextIntegration()],
46
47
  initialScope: {
@@ -820,10 +821,6 @@ class TestDriverReporter {
820
821
  // Initialize Sentry for error reporting
821
822
  initializeSentry();
822
823
 
823
- // Store project root for making file paths relative
824
- pluginState.projectRoot = ctx.config.root || process.cwd();
825
- logger.debug("Project root:", pluginState.projectRoot);
826
-
827
824
  // NOW read the API key and API root (after setupFiles have run, including dotenv/config)
828
825
  pluginState.apiKey = this.options.apiKey || process.env.TD_API_KEY;
829
826
  pluginState.apiRoot =
@@ -875,7 +872,7 @@ class TestDriverReporter {
875
872
  // Create test run via direct API call
876
873
  const testRunData = {
877
874
  runId: pluginState.testRunId,
878
- suiteName: getSuiteName(),
875
+ suiteName: getSuiteName(this.ctx),
879
876
  ...pluginState.gitInfo,
880
877
  };
881
878
 
@@ -1094,27 +1091,25 @@ class TestDriverReporter {
1094
1091
  const sessionId = meta.sessionId || null;
1095
1092
  const platform = meta.platform || null;
1096
1093
  const sandboxId = meta.sandboxId || null;
1097
- let testFile = meta.testFile || "unknown";
1098
- const testOrder = meta.testOrder !== undefined ? meta.testOrder : 0;
1099
-
1100
- // If testFile not in meta, fallback to test object properties
1101
- if (testFile === "unknown") {
1102
- const absolutePath =
1103
- test.module?.task?.filepath ||
1104
- test.module?.file?.filepath ||
1105
- test.module?.file?.name ||
1106
- test.file?.filepath ||
1107
- test.file?.name ||
1108
- test.suite?.file?.filepath ||
1109
- test.suite?.file?.name ||
1110
- test.location?.file ||
1111
- "unknown";
1112
- testFile =
1113
- pluginState.projectRoot && absolutePath !== "unknown"
1114
- ? path.relative(pluginState.projectRoot, absolutePath)
1115
- : absolutePath;
1116
- logger.debug(`Resolved testFile from fallback: ${testFile}`);
1094
+
1095
+ // Test identity from Vitest's standard reporter API (TestCase in v4+).
1096
+ const projectName = test.project?.name || null;
1097
+ const projectRoot = test.project?.config?.root || process.cwd();
1098
+ const moduleId = test.module?.moduleId || null;
1099
+ let testFile =
1100
+ moduleId && !moduleId.includes("://") ? path.relative(projectRoot, moduleId) : "unknown";
1101
+ if (path.isAbsolute(testFile) || testFile.startsWith("..")) testFile = "unknown";
1102
+
1103
+ // Full nested describe path (top-level first), joined with " > ".
1104
+ const suitePath = [];
1105
+ {
1106
+ let cursor = test.parent;
1107
+ while (cursor && cursor.type === "suite") {
1108
+ suitePath.unshift(cursor.name);
1109
+ cursor = cursor.parent;
1110
+ }
1117
1111
  }
1112
+ const suiteName = suitePath.length > 0 ? suitePath.join(" > ") : null;
1118
1113
 
1119
1114
  // Update test run platform from first test that reports it
1120
1115
  if (platform && !pluginState.detectedPlatform) {
@@ -1150,7 +1145,6 @@ class TestDriverReporter {
1150
1145
  // We only want actual SDK crashes and exceptions reported to Sentry.
1151
1146
  }
1152
1147
 
1153
- const suiteName = test.suite?.name;
1154
1148
  const startTime = Date.now() - duration; // Calculate start time from duration
1155
1149
  // In Vitest v4, retryCount is on diagnostic(), not result()
1156
1150
  // result() only returns { state, errors }, while diagnostic() has retryCount, duration, etc.
@@ -1160,12 +1154,14 @@ class TestDriverReporter {
1160
1154
  const consoleUrl = getConsoleUrl(pluginState.apiRoot);
1161
1155
  const hasRetries = retryCount > 0 && dashcamUrls.length > 1;
1162
1156
 
1163
- // Record a single test case with all metadata
1157
+ // Record a single test case with all metadata.
1158
+ // Field names match the current API contract; `projectName` is a
1159
+ // forward-compatible addition (ignored by the API today).
1164
1160
  const testCaseData = {
1165
1161
  runId: testRunId,
1166
1162
  testName: test.name,
1167
1163
  testFile: testFile,
1168
- testOrder: testOrder,
1164
+ projectName: projectName,
1169
1165
  status,
1170
1166
  startTime: startTime,
1171
1167
  endTime: Date.now(),
@@ -1186,7 +1182,13 @@ class TestDriverReporter {
1186
1182
  // Include per-attempt replay URLs for retry visibility
1187
1183
  if (dashcamUrls.length > 0) {
1188
1184
  const attemptUrls = dashcamUrls
1189
- .map(a => ({ attempt: a.attempt, url: a.url || null, sessionId: a.sessionId || null }));
1185
+ .map(a => ({
1186
+ attempt: a.attempt,
1187
+ url: a.url || null,
1188
+ sessionId: a.sessionId || null,
1189
+ errorMessage: a.errorMessage || null,
1190
+ errorStack: a.errorStack || null,
1191
+ }));
1190
1192
  testCaseData.replayUrls = attemptUrls;
1191
1193
  }
1192
1194
 
@@ -1195,7 +1197,7 @@ class TestDriverReporter {
1195
1197
  if (errorStack) testCaseData.errorStack = errorStack;
1196
1198
 
1197
1199
  logger.debug(
1198
- `Recording test case: ${test.name} (${status}) with testFile: ${testFile}, testOrder: ${testOrder}, duration: ${duration}ms, replay: ${dashcamUrl ? "yes" : "no"}`,
1200
+ `Recording test case: ${test.name} (${status}) with testFile: ${testFile}, project: ${projectName}, suite: ${suiteName || "(none)"}, duration: ${duration}ms, replay: ${dashcamUrl ? "yes" : "no"}`,
1199
1201
  );
1200
1202
 
1201
1203
  const testCaseResponse = await recordTestCaseDirect(
@@ -1304,32 +1306,13 @@ class TestDriverReporter {
1304
1306
  interactions: testResult.interactions || { total: 0, cached: 0, byType: {} },
1305
1307
  };
1306
1308
 
1307
- // Sanitize testName for filesystem use
1308
- const safeName = (test.name || "unknown").replace(/[^a-zA-Z0-9_.-]/g, "_").substring(0, 200);
1309
+ // Use Vitest's stable per-test id as the filename — it's unique within
1310
+ // a run and avoids the name-collision problem entirely.
1309
1311
  const resultDir = path.join(process.cwd(), ".testdriver", "results", testFile);
1310
1312
  fs.mkdirSync(resultDir, { recursive: true });
1311
-
1312
- // Include a stable unique suffix in the filename to avoid collisions
1313
- // when multiple tests in the same file share the same name.
1314
- const hashSourceParts = [];
1315
- if (test.id) {
1316
- hashSourceParts.push(String(test.id));
1317
- }
1318
- if (Array.isArray(test.suitePath)) {
1319
- hashSourceParts.push(test.suitePath.join(" > "));
1320
- }
1321
- if (test.file && (test.file.name || test.file.path)) {
1322
- hashSourceParts.push(test.file.name || test.file.path);
1323
- }
1324
- // Fallback to the test name if no other identifiers are available.
1325
- if (hashSourceParts.length === 0) {
1326
- hashSourceParts.push(test.name || "unknown");
1327
- }
1328
- const hashSource = hashSourceParts.join(" | ");
1329
- const uniqueHash = crypto.createHash("sha256").update(hashSource).digest("hex").slice(0, 8);
1330
-
1313
+ const safeId = (test.id || "unknown").replace(/[^\w.-]/g, "_");
1331
1314
  fs.writeFileSync(
1332
- path.join(resultDir, `${safeName}-${uniqueHash}.json`),
1315
+ path.join(resultDir, `${safeId}.json`),
1333
1316
  JSON.stringify(resultData, null, 2),
1334
1317
  );
1335
1318
  }
@@ -1376,11 +1359,11 @@ function getConsoleUrl(apiRoot) {
1376
1359
  return `https://${flyMatch[1]}-web.fly.dev`;
1377
1360
  }
1378
1361
 
1379
- // Known channel API URLs -> console equivalents
1380
- // e.g. https://api-canary.testdriver.ai -> https://console-canary.testdriver.ai
1381
- for (const url of Object.values(channelConfig.channels)) {
1382
- if (url === apiRoot) {
1383
- return url.replace("api", "console").replace("1337", "3001");
1362
+ // Known channel API URLs -> console equivalents (from environments.json)
1363
+ // e.g. https://api-dev.testdriver.ai -> https://console-dev.testdriver.ai
1364
+ for (const env of Object.values(environments)) {
1365
+ if (env.apiRoot === apiRoot && env.consoleUrl) {
1366
+ return env.consoleUrl;
1384
1367
  }
1385
1368
  }
1386
1369
 
@@ -1404,7 +1387,14 @@ function generateRunId() {
1404
1387
  return `${Date.now()}-${crypto.randomBytes(4).toString("hex")}`;
1405
1388
  }
1406
1389
 
1407
- function getSuiteName() {
1390
+ function getSuiteName(ctx) {
1391
+ // Derived from Vitest's configured projects (joined when multiple).
1392
+ // Must always return a string: the test-run-create API requires suiteName.
1393
+ const projects = ctx?.projects;
1394
+ if (Array.isArray(projects) && projects.length > 0) {
1395
+ const names = projects.map((p) => p?.name).filter(Boolean);
1396
+ if (names.length > 0) return names.join(",");
1397
+ }
1408
1398
  return process.env.npm_package_name || path.basename(process.cwd());
1409
1399
  }
1410
1400
 
@@ -98,7 +98,7 @@ class Dashcam {
98
98
  if (!apiRoot) return "https://console.testdriver.ai";
99
99
 
100
100
  // Dash-separated environments: api-{env}.testdriver.ai -> console-{env}.testdriver.ai
101
- const envMatch = apiRoot.match(/^https:\/\/api-(test|canary)\.testdriver\.ai/);
101
+ const envMatch = apiRoot.match(/^https:\/\/api-(dev|test|canary)\.testdriver\.ai/);
102
102
  if (envMatch) {
103
103
  return `https://console-${envMatch[1]}.testdriver.ai`;
104
104
  }
@@ -150,6 +150,39 @@ class Dashcam {
150
150
 
151
151
  }
152
152
 
153
+ /**
154
+ * Workaround for dashcam <1.5.0: webLogsDaemon.js writes its state to a
155
+ * `.dashcam` directory relative to process.cwd(). When launched by the
156
+ * runner, cwd is typically a root-owned global install path (for example
157
+ * /usr/lib/node_modules/@testdriverai/runner or
158
+ * /usr/local/lib/node_modules/@testdriverai/runner),
159
+ * so the daemon can't mkdir its state dir as user `user` and silently
160
+ * dies — recordings never start (or `dashcam logs --add` fails with EACCES).
161
+ * Pre-create world-writable .dashcam dirs at both the current cwd and the
162
+ * known global-install path so the daemon survives regardless of which path
163
+ * dashcam ends up using. Must run before ANY dashcam invocation, not just
164
+ * auth(), because addWebLog() is called before start()/auth() in provision.
165
+ * TODO: remove once dashcam >=1.5.0 (which uses $HOME) is bundled.
166
+ * @private
167
+ */
168
+ async _ensureDashcamStateDirs() {
169
+ if (this.client.os === "windows") return;
170
+ if (!this._dashcamDirsReady) {
171
+ this._dashcamDirsReady = this.client.exec(
172
+ this._getShell(),
173
+ `GLOBAL_NPM_ROOT="$(npm root -g 2>/dev/null || true)"; ` +
174
+ `PWD_DIR="$(pwd)"; REAL_PWD_DIR="$(pwd -P 2>/dev/null || pwd)"; ` +
175
+ `DIRS="$PWD_DIR/.dashcam $REAL_PWD_DIR/.dashcam /usr/lib/node_modules/@testdriverai/runner/.dashcam /usr/local/lib/node_modules/@testdriverai/runner/.dashcam"; ` +
176
+ `if [ -n "$GLOBAL_NPM_ROOT" ]; then DIRS="$DIRS $GLOBAL_NPM_ROOT/@testdriverai/runner/.dashcam"; fi; ` +
177
+ `for d in $DIRS; do sudo mkdir -p "$d" 2>/dev/null || true; sudo chmod 0777 "$d" 2>/dev/null || true; done; ` +
178
+ `true`,
179
+ 10000,
180
+ process.env.TD_DEBUG == "true" ? false : true,
181
+ );
182
+ }
183
+ await this._dashcamDirsReady;
184
+ }
185
+
153
186
  /**
154
187
  * Authenticate dashcam with API key
155
188
  * @param {string} [apiKey] - Override API key
@@ -174,6 +207,8 @@ class Dashcam {
174
207
  this._log("debug", "Auth output:", authOutput);
175
208
  } else {
176
209
  // Linux/Mac authentication with TD_API_ROOT
210
+ await this._ensureDashcamStateDirs();
211
+
177
212
  const authOutput = await this.client.exec(
178
213
  shell,
179
214
  `TD_API_ROOT="${apiRoot}" dashcam auth ${key}`,
@@ -215,6 +250,7 @@ class Dashcam {
215
250
  );
216
251
  this._log("debug", "Add log tracking output:", addLogOutput);
217
252
  } else {
253
+ await this._ensureDashcamStateDirs();
218
254
  // Create log file
219
255
  await this.client.exec(
220
256
  shell,
@@ -254,6 +290,7 @@ class Dashcam {
254
290
  );
255
291
  this._log("debug", "Add application log tracking output:", addLogOutput);
256
292
  } else {
293
+ await this._ensureDashcamStateDirs();
257
294
  const addLogOutput = await this.client.exec(
258
295
  shell,
259
296
  `TD_API_ROOT="${apiRoot}" dashcam logs --add --type=application --application="${application}" --name="${name}"`,
@@ -288,6 +325,7 @@ class Dashcam {
288
325
  this._log("warn", "Add web log tracking failed:", err.message);
289
326
  }
290
327
  } else {
328
+ await this._ensureDashcamStateDirs();
291
329
  const addLogOutput = await this.client.exec(
292
330
  shell,
293
331
  `TD_API_ROOT="${apiRoot}" dashcam logs --add --type=web --pattern="${pattern}" --name="${name}"`,
@@ -368,6 +406,7 @@ class Dashcam {
368
406
  this._log("debug", "Dashcam recording started");
369
407
  } else {
370
408
  // Linux/Mac with TD_API_ROOT
409
+ await this._ensureDashcamStateDirs();
371
410
  this._log("debug", "Starting dashcam recording on Linux/Mac...");
372
411
  const dashcamPath = await this._getDashcamPath();
373
412
  const titleArg = this.title
@@ -430,10 +469,12 @@ class Dashcam {
430
469
  this._log("debug", "Dashcam stop command output:", output);
431
470
  } else {
432
471
  // Linux/Mac with TD_API_ROOT
472
+ // Use 2>&1 to merge stderr into stdout so the replay URL is captured
473
+ // regardless of which stream the dashcam CLI writes it to.
433
474
  const dashcamPath = await this._getDashcamPath();
434
475
  output = await this.client.exec(
435
476
  shell,
436
- `TD_API_ROOT="${apiRoot}" "${dashcamPath}" stop`,
477
+ `TD_API_ROOT="${apiRoot}" "${dashcamPath}" stop 2>&1`,
437
478
  300000,
438
479
  process.env.TD_DEBUG == "true" ? false : true,
439
480
  );
@@ -38,6 +38,13 @@ export class Dashcam {
38
38
  */
39
39
  addApplicationLog(application: string, name: string): Promise<void>;
40
40
 
41
+ /**
42
+ * Add a web log to Dashcam
43
+ * @param pattern - URL pattern to match (e.g., "*example.com*")
44
+ * @param name - Name/description for the log entry
45
+ */
46
+ addWebLog(pattern: string, name: string): Promise<void>;
47
+
41
48
  /**
42
49
  * Start recording
43
50
  * @returns Promise that resolves when recording starts
@@ -68,7 +75,7 @@ export interface LogConfig {
68
75
  /**
69
76
  * Type of log entry
70
77
  */
71
- type: 'file' | 'application';
78
+ type: 'file' | 'application' | 'web';
72
79
 
73
80
  /**
74
81
  * Path to file (for file logs)
@@ -80,6 +87,11 @@ export interface LogConfig {
80
87
  */
81
88
  application?: string;
82
89
 
90
+ /**
91
+ * URL pattern (for web logs, e.g., "*example.com*")
92
+ */
93
+ pattern?: string;
94
+
83
95
  /**
84
96
  * Name/description for the log entry
85
97
  */
@@ -8,7 +8,13 @@
8
8
  * - Links to test runs
9
9
  */
10
10
 
11
- import { Octokit } from '@octokit/rest';
11
+ /**
12
+ * Lazily load Octokit to avoid hard failure when @octokit/rest is not installed
13
+ */
14
+ async function getOctokit(token) {
15
+ const { Octokit } = await import('@octokit/rest');
16
+ return new Octokit({ auth: token });
17
+ }
12
18
 
13
19
  /**
14
20
  * Format test duration in human-readable format
@@ -334,7 +340,7 @@ export async function postGitHubComment(options) {
334
340
  throw new Error('Either prNumber or commitSha must be provided');
335
341
  }
336
342
 
337
- const octokit = new Octokit({ auth: token });
343
+ const octokit = await getOctokit(token);
338
344
 
339
345
  if (prNumber) {
340
346
  // Comment on PR
@@ -374,7 +380,7 @@ export async function updateGitHubComment(options) {
374
380
  throw new Error('Token, owner, repo, and commentId are required');
375
381
  }
376
382
 
377
- const octokit = new Octokit({ auth: token });
383
+ const octokit = await getOctokit(token);
378
384
 
379
385
  const response = await octokit.rest.issues.updateComment({
380
386
  owner,
@@ -402,7 +408,7 @@ export async function findExistingComment(options) {
402
408
  return null;
403
409
  }
404
410
 
405
- const octokit = new Octokit({ auth: token });
411
+ const octokit = await getOctokit(token);
406
412
 
407
413
  const comments = await octokit.rest.issues.listComments({
408
414
  owner,
@@ -435,7 +441,7 @@ export async function postOrUpdateTestResults(testRunData, testCases, githubOpti
435
441
 
436
442
  if (existingComment) {
437
443
  // Delete the old comment
438
- const octokit = new Octokit({ auth: githubOptions.token });
444
+ const octokit = await getOctokit(githubOptions.token);
439
445
  await octokit.rest.issues.deleteComment({
440
446
  owner: githubOptions.owner,
441
447
  repo: githubOptions.repo,
@@ -1,6 +1,7 @@
1
1
  const fs = require("fs");
2
2
  const path = require("path");
3
3
  const { execSync, spawn } = require("child_process");
4
+ const { active: ACTIVE_CHANNEL } = require("./resolve-channel.js");
4
5
 
5
6
  /**
6
7
  * Run an npm install command with an animated progress bar
@@ -38,12 +39,18 @@ function runInstall(cmd, args, cwd, label) {
38
39
  * @param {string} [options.targetDir] - Target directory (defaults to current working directory)
39
40
  * @param {string} [options.apiKey] - TestDriver API key (will be saved to .env)
40
41
  * @param {boolean} [options.skipInstall=false] - Skip npm install step
42
+ * @param {boolean} [options.skipSampleTest=false] - Skip scaffolding the example test files (tests/example.test.js + tests/login.js). The tests/ dir, config, workflow, etc. are still created.
41
43
  * @param {boolean} [options.interactive=false] - Whether to prompt for missing values (CLI mode)
44
+ * @param {string[]} [options.clients] - AI client ids to wire up (e.g. ["claude-code","cursor"] or ["all"]). When set, native per-client config is written instead of the interactive add-mcp flow.
45
+ * @param {string} [options.channel] - Release channel (dev|test|canary|stable) the generated GitHub workflow should pin the action to. Defaults to the SDK's own active channel.
42
46
  * @param {function} [options.onProgress] - Callback for progress updates (receives message string)
43
47
  * @returns {Promise<{success: boolean, results: string[], errors: string[]}>}
44
48
  */
45
49
  async function initProject(options = {}) {
46
50
  const targetDir = options.targetDir || process.cwd();
51
+ // Which TestDriver channel the generated workflow should target. A -test SDK
52
+ // scaffolds a test-pinned workflow, -canary -> canary, clean semver -> stable.
53
+ const channel = options.channel || ACTIVE_CHANNEL;
47
54
  const results = [];
48
55
  const errors = [];
49
56
 
@@ -94,6 +101,11 @@ async function initProject(options = {}) {
94
101
  fs.mkdirSync(testDir, { recursive: true });
95
102
  }
96
103
 
104
+ // Sample test files (login snippet + example test). Skipped when the caller
105
+ // opts out (e.g. /eve, which scaffolds the project but writes its own tests).
106
+ if (options.skipSampleTest) {
107
+ progress("⊘ Skipped sample test files (tests/example.test.js, tests/login.js)");
108
+ } else {
97
109
  // Create login snippet file
98
110
  const loginSnippetFile = path.join(testDir, "login.js");
99
111
  if (!fs.existsSync(loginSnippetFile)) {
@@ -170,6 +182,7 @@ test('should login and add item to cart', async (context) => {
170
182
  fs.writeFileSync(testFile, vitestContent);
171
183
  progress("✓ Created test file: tests/example.test.js");
172
184
  }
185
+ }
173
186
 
174
187
  // 3. Create vitest.config.js
175
188
  const configFile = path.join(targetDir, "vitest.config.js");
@@ -234,6 +247,10 @@ export default defineConfig({
234
247
 
235
248
  const workflowFile = path.join(workflowDir, "testdriver.yml");
236
249
  if (!fs.existsSync(workflowFile)) {
250
+ // The published action is tagged test/canary/stable. `dev` runs locally and
251
+ // isn't published, so pin its `uses:` ref to the test action while still
252
+ // passing channel: dev so the action targets the dev API root.
253
+ const actionRef = channel === "dev" ? "test" : channel;
237
254
  const workflowContent = `name: TestDriver.ai Tests
238
255
 
239
256
  on:
@@ -241,10 +258,14 @@ on:
241
258
  branches: [ main, master ]
242
259
  pull_request:
243
260
  branches: [ main, master ]
261
+ workflow_dispatch:
244
262
 
245
263
  jobs:
246
264
  test:
247
265
  runs-on: ubuntu-latest
266
+ permissions:
267
+ id-token: write # required to mint the OIDC token for auth
268
+ contents: read
248
269
 
249
270
  steps:
250
271
  - uses: actions/checkout@v4
@@ -258,9 +279,16 @@ jobs:
258
279
  - name: Install dependencies
259
280
  run: npm ci
260
281
 
282
+ # Authenticate to TestDriver via GitHub OIDC (no stored secret needed once the
283
+ # TestDriver GitHub App is authorized for your org). Falls back to a TD_API_KEY
284
+ # secret if you set one. Exports TD_API_KEY for the test step below.
285
+ - name: Authenticate to TestDriver
286
+ uses: testdriverai/action@${actionRef}
287
+ with:
288
+ channel: ${channel}
289
+ api-key: \${{ secrets.TD_API_KEY }}
290
+
261
291
  - name: Run TestDriver.ai tests
262
- env:
263
- TD_API_KEY: \${{ secrets.TD_API_KEY }}
264
292
  run: npx vitest run
265
293
 
266
294
  - name: Upload test results
@@ -277,12 +305,64 @@ jobs:
277
305
  progress("⊘ GitHub workflow already exists");
278
306
  }
279
307
 
308
+ // Resolve the bundled agent/skills source dirs once; reused by the
309
+ // client-aware installer (step 6) and the legacy copy steps (8 & 9).
310
+ const resolveSource = (...candidates) =>
311
+ candidates.find((c) => fs.existsSync(c)) || null;
312
+ const skillsSourceDir = resolveSource(
313
+ path.join(targetDir, "node_modules", "testdriverai", "ai", "skills"),
314
+ path.join(__dirname, "..", "ai", "skills"),
315
+ );
316
+ const agentsSourceDir = resolveSource(
317
+ path.join(targetDir, "node_modules", "testdriverai", "ai", "agents"),
318
+ path.join(__dirname, "..", "ai", "agents"),
319
+ );
320
+
280
321
  // 6. Setup MCP configuration
281
- // When triggered from VS Code extension, create .vscode/mcp.json silently
282
- // When triggered from CLI, use interactive add-mcp for user to select their MCP client
322
+ // Priority:
323
+ // a) options.clients set -> write native config per selected client
324
+ // b) TD_INIT_SOURCE=vscode -> create .vscode/mcp.json silently
325
+ // c) otherwise (CLI) -> interactive add-mcp client picker
283
326
  const isVscodeInit = process.env.TD_INIT_SOURCE === "vscode";
284
-
285
- if (isVscodeInit) {
327
+ const requestedClients =
328
+ Array.isArray(options.clients) && options.clients.length
329
+ ? options.clients
330
+ : null;
331
+
332
+ if (requestedClients) {
333
+ const {
334
+ installClient,
335
+ resolveClientIds,
336
+ } = require("./install-clients.js");
337
+ const { valid, unknown } = resolveClientIds(requestedClients);
338
+ for (const id of unknown) {
339
+ progress(`⚠ Unknown client "${id}" - skipping`);
340
+ }
341
+ for (const id of valid) {
342
+ try {
343
+ const result = installClient(id, {
344
+ targetDir,
345
+ apiKey: options.apiKey,
346
+ agentSource: agentsSourceDir,
347
+ skillsSource: skillsSourceDir,
348
+ });
349
+ progress(`🔧 ${result.label}:`);
350
+ for (const step of result.steps) {
351
+ if (step.status === "manual") {
352
+ progress(` ℹ ${step.message}`);
353
+ } else if (step.status === "error") {
354
+ progress(` ⚠ ${step.message}`);
355
+ } else if (step.status === "skipped") {
356
+ progress(` ⊘ ${step.message || step.file}`);
357
+ } else {
358
+ progress(` ✓ ${step.message || step.file}`);
359
+ }
360
+ }
361
+ } catch (err) {
362
+ progress(`⚠ Failed to configure ${id}: ${err.message}`);
363
+ }
364
+ }
365
+ } else if (isVscodeInit) {
286
366
  // VS Code extension: create .vscode/mcp.json directly
287
367
  const vscodeDir = path.join(targetDir, ".vscode");
288
368
  if (!fs.existsSync(vscodeDir)) {
@@ -365,18 +445,6 @@ jobs:
365
445
 
366
446
  // 8. Copy TestDriver skills
367
447
  const skillsDestDir = path.join(targetDir, ".github", "skills");
368
- const possibleSkillsSources = [
369
- path.join(targetDir, "node_modules", "testdriverai", "ai", "skills"),
370
- path.join(__dirname, "..", "ai", "skills"),
371
- ];
372
-
373
- let skillsSourceDir = null;
374
- for (const source of possibleSkillsSources) {
375
- if (fs.existsSync(source)) {
376
- skillsSourceDir = source;
377
- break;
378
- }
379
- }
380
448
 
381
449
  if (skillsSourceDir) {
382
450
  if (!fs.existsSync(skillsDestDir)) {
@@ -412,18 +480,6 @@ jobs:
412
480
 
413
481
  // 9. Copy TestDriver agents
414
482
  const agentsDestDir = path.join(targetDir, ".github", "agents");
415
- const possibleAgentsSources = [
416
- path.join(targetDir, "node_modules", "testdriverai", "ai", "agents"),
417
- path.join(__dirname, "..", "ai", "agents"),
418
- ];
419
-
420
- let agentsSourceDir = null;
421
- for (const source of possibleAgentsSources) {
422
- if (fs.existsSync(source)) {
423
- agentsSourceDir = source;
424
- break;
425
- }
426
- }
427
483
 
428
484
  if (agentsSourceDir) {
429
485
  if (!fs.existsSync(agentsDestDir)) {
@@ -447,16 +503,6 @@ jobs:
447
503
  if (copiedCount > 0) {
448
504
  progress(`✓ Copied ${copiedCount} agent(s) to .github/agents/`);
449
505
  }
450
-
451
- // Also set testdriver.md as copilot-instructions.md if it doesn't already exist
452
- const copilotInstructionsPath = path.join(targetDir, ".github", "copilot-instructions.md");
453
- const testdriverAgentSource = path.join(agentsSourceDir, "testdriver.md");
454
- if (!fs.existsSync(copilotInstructionsPath) && fs.existsSync(testdriverAgentSource)) {
455
- fs.copyFileSync(testdriverAgentSource, copilotInstructionsPath);
456
- progress("✓ Created .github/copilot-instructions.md");
457
- } else if (fs.existsSync(copilotInstructionsPath)) {
458
- progress("⊘ copilot-instructions.md already exists, skipping");
459
- }
460
506
  } else {
461
507
  progress("⚠ Agents directory not found (will be available after npm install)");
462
508
  }
@@ -485,21 +531,29 @@ jobs:
485
531
  }
486
532
 
487
533
  // 11. Install dependencies (unless skipped)
534
+ // Pin testdriverai to the SAME channel this CLI is running from (the npm
535
+ // dist-tags match the channel names: test | canary | stable). Otherwise a
536
+ // bare `testdriverai` resolves to `latest` (= stable) even when init was run
537
+ // via `npx testdriverai@test`, so the scaffolded project — and the vitest
538
+ // run in the generated GitHub action — would target stable instead. `dev`
539
+ // isn't a published tag, so fall back to `test` (matches the actionRef logic above).
540
+ const installTag = channel === "dev" ? "test" : channel;
541
+ const testdriveraiSpec = `testdriverai@${installTag}`;
488
542
  if (!options.skipInstall) {
489
543
  progress("\n📦 Installing dependencies...");
490
544
  try {
491
- await runInstall("npm", ["install", "-D", "vitest", "testdriverai"], targetDir, "vitest testdriverai");
492
- progress("✓ Installed vitest, testdriverai");
545
+ await runInstall("npm", ["install", "-D", "vitest", testdriveraiSpec], targetDir, `vitest ${testdriveraiSpec}`);
546
+ progress(`✓ Installed vitest, ${testdriveraiSpec}`);
493
547
  await runInstall("npm", ["install", "dotenv"], targetDir, "dotenv");
494
548
  progress("✓ Installed dotenv");
495
549
  } catch (error) {
496
550
  errors.push("Failed to install dependencies. Run manually:");
497
- errors.push(" npm install -D vitest testdriverai");
551
+ errors.push(` npm install -D vitest ${testdriveraiSpec}`);
498
552
  errors.push(" npm install dotenv");
499
553
  }
500
554
  } else {
501
555
  progress("\nℹ Skipped dependency installation. Run manually:");
502
- progress(" npm install -D vitest testdriverai");
556
+ progress(` npm install -D vitest ${testdriveraiSpec}`);
503
557
  progress(" npm install dotenv");
504
558
  }
505
559