digital-ai-testing-mcp 1.2.7 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (231) hide show
  1. package/README.md +161 -56
  2. package/dist/api/access-level.d.ts +46 -0
  3. package/dist/api/access-level.d.ts.map +1 -0
  4. package/dist/api/access-level.js +183 -0
  5. package/dist/api/access-level.js.map +1 -0
  6. package/dist/api/agents.js +1 -1
  7. package/dist/api/agents.js.map +1 -1
  8. package/dist/api/browser-inspection.d.ts.map +1 -1
  9. package/dist/api/browser-inspection.js +18 -17
  10. package/dist/api/browser-inspection.js.map +1 -1
  11. package/dist/api/client.d.ts +14 -2
  12. package/dist/api/client.d.ts.map +1 -1
  13. package/dist/api/client.js +60 -43
  14. package/dist/api/client.js.map +1 -1
  15. package/dist/api/device-groups.js +1 -1
  16. package/dist/api/device-groups.js.map +1 -1
  17. package/dist/api/license.js +1 -1
  18. package/dist/api/license.js.map +1 -1
  19. package/dist/api/nv-servers.js +1 -1
  20. package/dist/api/nv-servers.js.map +1 -1
  21. package/dist/api/projects.d.ts +16 -3
  22. package/dist/api/projects.d.ts.map +1 -1
  23. package/dist/api/projects.js +60 -18
  24. package/dist/api/projects.js.map +1 -1
  25. package/dist/api/regions.js +1 -1
  26. package/dist/api/regions.js.map +1 -1
  27. package/dist/api/reporter-projects.d.ts.map +1 -1
  28. package/dist/api/reporter-projects.js +2 -1
  29. package/dist/api/reporter-projects.js.map +1 -1
  30. package/dist/api/reporting.d.ts +91 -0
  31. package/dist/api/reporting.d.ts.map +1 -1
  32. package/dist/api/reporting.js +220 -22
  33. package/dist/api/reporting.js.map +1 -1
  34. package/dist/api/sessions.js +1 -1
  35. package/dist/api/sessions.js.map +1 -1
  36. package/dist/api/test-orchestrator.d.ts +59 -0
  37. package/dist/api/test-orchestrator.d.ts.map +1 -0
  38. package/dist/api/test-orchestrator.js +126 -0
  39. package/dist/api/test-orchestrator.js.map +1 -0
  40. package/dist/api/test-runs.d.ts +80 -0
  41. package/dist/api/test-runs.d.ts.map +1 -0
  42. package/dist/api/test-runs.js +209 -0
  43. package/dist/api/test-runs.js.map +1 -0
  44. package/dist/api/test-views.d.ts.map +1 -1
  45. package/dist/api/test-views.js +5 -1
  46. package/dist/api/test-views.js.map +1 -1
  47. package/dist/api/transactions.d.ts +6 -0
  48. package/dist/api/transactions.d.ts.map +1 -1
  49. package/dist/api/transactions.js +19 -3
  50. package/dist/api/transactions.js.map +1 -1
  51. package/dist/api/usage-reports.d.ts.map +1 -1
  52. package/dist/api/usage-reports.js +3 -3
  53. package/dist/api/usage-reports.js.map +1 -1
  54. package/dist/api/webdriver.d.ts +39 -1
  55. package/dist/api/webdriver.d.ts.map +1 -1
  56. package/dist/api/webdriver.js +177 -71
  57. package/dist/api/webdriver.js.map +1 -1
  58. package/dist/index.js +82 -33
  59. package/dist/index.js.map +1 -1
  60. package/dist/tools/agent-tools.js +2 -2
  61. package/dist/tools/agent-tools.js.map +1 -1
  62. package/dist/tools/application-tools.d.ts.map +1 -1
  63. package/dist/tools/application-tools.js +17 -10
  64. package/dist/tools/application-tools.js.map +1 -1
  65. package/dist/tools/boilerplate-tools.d.ts +3 -0
  66. package/dist/tools/boilerplate-tools.d.ts.map +1 -1
  67. package/dist/tools/boilerplate-tools.js +302 -21
  68. package/dist/tools/boilerplate-tools.js.map +1 -1
  69. package/dist/tools/browser-tools.d.ts.map +1 -1
  70. package/dist/tools/browser-tools.js +8 -5
  71. package/dist/tools/browser-tools.js.map +1 -1
  72. package/dist/tools/debug-tools.d.ts.map +1 -1
  73. package/dist/tools/debug-tools.js +6 -3
  74. package/dist/tools/debug-tools.js.map +1 -1
  75. package/dist/tools/device-group-tools.js +5 -5
  76. package/dist/tools/device-group-tools.js.map +1 -1
  77. package/dist/tools/device-tools.d.ts.map +1 -1
  78. package/dist/tools/device-tools.js +5 -4
  79. package/dist/tools/device-tools.js.map +1 -1
  80. package/dist/tools/health-tools.d.ts.map +1 -1
  81. package/dist/tools/health-tools.js +6 -5
  82. package/dist/tools/health-tools.js.map +1 -1
  83. package/dist/tools/inspection-tools.d.ts.map +1 -1
  84. package/dist/tools/inspection-tools.js +122 -22
  85. package/dist/tools/inspection-tools.js.map +1 -1
  86. package/dist/tools/meta-tools.d.ts +4 -1
  87. package/dist/tools/meta-tools.d.ts.map +1 -1
  88. package/dist/tools/meta-tools.js +87 -58
  89. package/dist/tools/meta-tools.js.map +1 -1
  90. package/dist/tools/performance-tools.d.ts.map +1 -1
  91. package/dist/tools/performance-tools.js +2 -8
  92. package/dist/tools/performance-tools.js.map +1 -1
  93. package/dist/tools/project-tools.d.ts.map +1 -1
  94. package/dist/tools/project-tools.js +38 -19
  95. package/dist/tools/project-tools.js.map +1 -1
  96. package/dist/tools/provisioning-profile-tools.d.ts.map +1 -1
  97. package/dist/tools/provisioning-profile-tools.js +5 -4
  98. package/dist/tools/provisioning-profile-tools.js.map +1 -1
  99. package/dist/tools/remediation-tools.d.ts +7 -0
  100. package/dist/tools/remediation-tools.d.ts.map +1 -0
  101. package/dist/tools/remediation-tools.js +50 -0
  102. package/dist/tools/remediation-tools.js.map +1 -0
  103. package/dist/tools/reporting-tools.d.ts.map +1 -1
  104. package/dist/tools/reporting-tools.js +393 -147
  105. package/dist/tools/reporting-tools.js.map +1 -1
  106. package/dist/tools/repository-tools.d.ts.map +1 -1
  107. package/dist/tools/repository-tools.js +5 -4
  108. package/dist/tools/repository-tools.js.map +1 -1
  109. package/dist/tools/reservation-tools.js +2 -2
  110. package/dist/tools/reservation-tools.js.map +1 -1
  111. package/dist/tools/resources.js +2 -2
  112. package/dist/tools/resources.js.map +1 -1
  113. package/dist/tools/test-run-tools.d.ts +3 -0
  114. package/dist/tools/test-run-tools.d.ts.map +1 -0
  115. package/dist/tools/test-run-tools.js +284 -0
  116. package/dist/tools/test-run-tools.js.map +1 -0
  117. package/dist/tools/test-view-tools.d.ts.map +1 -1
  118. package/dist/tools/test-view-tools.js +11 -11
  119. package/dist/tools/test-view-tools.js.map +1 -1
  120. package/dist/tools/toolset-tools.d.ts +8 -0
  121. package/dist/tools/toolset-tools.d.ts.map +1 -0
  122. package/dist/tools/toolset-tools.js +40 -0
  123. package/dist/tools/toolset-tools.js.map +1 -0
  124. package/dist/tools/transaction-tools.d.ts.map +1 -1
  125. package/dist/tools/transaction-tools.js +17 -7
  126. package/dist/tools/transaction-tools.js.map +1 -1
  127. package/dist/tools/usage-report-tools.d.ts.map +1 -1
  128. package/dist/tools/usage-report-tools.js +43 -15
  129. package/dist/tools/usage-report-tools.js.map +1 -1
  130. package/dist/tools/user-tools.js +2 -2
  131. package/dist/tools/user-tools.js.map +1 -1
  132. package/dist/tools/web-inspection-tools.d.ts.map +1 -1
  133. package/dist/tools/web-inspection-tools.js +45 -33
  134. package/dist/tools/web-inspection-tools.js.map +1 -1
  135. package/dist/tools/workflow-tools.js +1 -1
  136. package/dist/types/digital-ai.d.ts +19 -1
  137. package/dist/types/digital-ai.d.ts.map +1 -1
  138. package/dist/utils/access-level.d.ts +39 -0
  139. package/dist/utils/access-level.d.ts.map +1 -0
  140. package/dist/utils/access-level.js +111 -0
  141. package/dist/utils/access-level.js.map +1 -0
  142. package/dist/utils/browser-name.d.ts +2 -0
  143. package/dist/utils/browser-name.d.ts.map +1 -0
  144. package/dist/utils/browser-name.js +19 -0
  145. package/dist/utils/browser-name.js.map +1 -0
  146. package/dist/utils/command-payload.d.ts +19 -0
  147. package/dist/utils/command-payload.d.ts.map +1 -0
  148. package/dist/utils/command-payload.js +13 -0
  149. package/dist/utils/command-payload.js.map +1 -0
  150. package/dist/utils/csv.d.ts +6 -0
  151. package/dist/utils/csv.d.ts.map +1 -1
  152. package/dist/utils/csv.js +28 -2
  153. package/dist/utils/csv.js.map +1 -1
  154. package/dist/utils/download-command.d.ts.map +1 -1
  155. package/dist/utils/download-command.js +5 -4
  156. package/dist/utils/download-command.js.map +1 -1
  157. package/dist/utils/locality.d.ts +7 -0
  158. package/dist/utils/locality.d.ts.map +1 -1
  159. package/dist/utils/locality.js +19 -0
  160. package/dist/utils/locality.js.map +1 -1
  161. package/dist/utils/maestro-flow.d.ts +38 -0
  162. package/dist/utils/maestro-flow.d.ts.map +1 -0
  163. package/dist/utils/maestro-flow.js +114 -0
  164. package/dist/utils/maestro-flow.js.map +1 -0
  165. package/dist/utils/metric-semantics.d.ts +22 -0
  166. package/dist/utils/metric-semantics.d.ts.map +1 -0
  167. package/dist/utils/metric-semantics.js +24 -0
  168. package/dist/utils/metric-semantics.js.map +1 -0
  169. package/dist/utils/pagination.d.ts +11 -0
  170. package/dist/utils/pagination.d.ts.map +1 -1
  171. package/dist/utils/pagination.js +16 -0
  172. package/dist/utils/pagination.js.map +1 -1
  173. package/dist/utils/path-guard.js +1 -1
  174. package/dist/utils/path-guard.js.map +1 -1
  175. package/dist/utils/profile-loader.d.ts +4 -2
  176. package/dist/utils/profile-loader.d.ts.map +1 -1
  177. package/dist/utils/profile-loader.js +5 -4
  178. package/dist/utils/profile-loader.js.map +1 -1
  179. package/dist/utils/remediation.d.ts +113 -0
  180. package/dist/utils/remediation.d.ts.map +1 -0
  181. package/dist/utils/remediation.js +392 -0
  182. package/dist/utils/remediation.js.map +1 -0
  183. package/dist/utils/report-ref.d.ts +23 -0
  184. package/dist/utils/report-ref.d.ts.map +1 -0
  185. package/dist/utils/report-ref.js +31 -0
  186. package/dist/utils/report-ref.js.map +1 -0
  187. package/dist/utils/response-formatter.d.ts +2 -1
  188. package/dist/utils/response-formatter.d.ts.map +1 -1
  189. package/dist/utils/response-formatter.js +32 -21
  190. package/dist/utils/response-formatter.js.map +1 -1
  191. package/dist/utils/test-orchestrator.d.ts +102 -0
  192. package/dist/utils/test-orchestrator.d.ts.map +1 -0
  193. package/dist/utils/test-orchestrator.js +236 -0
  194. package/dist/utils/test-orchestrator.js.map +1 -0
  195. package/dist/utils/test-status.d.ts +48 -0
  196. package/dist/utils/test-status.d.ts.map +1 -0
  197. package/dist/utils/test-status.js +92 -0
  198. package/dist/utils/test-status.js.map +1 -0
  199. package/dist/utils/tool-catalog.d.ts +6 -0
  200. package/dist/utils/tool-catalog.d.ts.map +1 -0
  201. package/dist/utils/tool-catalog.js +15 -0
  202. package/dist/utils/tool-catalog.js.map +1 -0
  203. package/dist/utils/toolsets.d.ts +79 -0
  204. package/dist/utils/toolsets.d.ts.map +1 -0
  205. package/dist/utils/toolsets.js +156 -0
  206. package/dist/utils/toolsets.js.map +1 -0
  207. package/dist/utils/upload-command.d.ts.map +1 -1
  208. package/dist/utils/upload-command.js +5 -4
  209. package/dist/utils/upload-command.js.map +1 -1
  210. package/dist/utils/usage-report-guard.d.ts +7 -0
  211. package/dist/utils/usage-report-guard.d.ts.map +1 -1
  212. package/dist/utils/usage-report-guard.js +18 -1
  213. package/dist/utils/usage-report-guard.js.map +1 -1
  214. package/dist/utils/usage-report-summary.d.ts +12 -0
  215. package/dist/utils/usage-report-summary.d.ts.map +1 -1
  216. package/dist/utils/usage-report-summary.js +25 -6
  217. package/dist/utils/usage-report-summary.js.map +1 -1
  218. package/package.json +40 -4
  219. package/resources/boilerplate/Android-Native/Java JUnit5/gradle +6 -1
  220. package/resources/boilerplate/Android-Native/Java JUnit5/gradle-oss +6 -1
  221. package/resources/boilerplate/Android-Native/java TestNG/AndroidNative-oss.java +2 -2
  222. package/resources/boilerplate/Android-Native/java TestNG/AndroidNative.java +2 -2
  223. package/resources/boilerplate/Android-Native/java TestNG/gradle +1 -1
  224. package/resources/boilerplate/Android-Native/java TestNG/gradle-oss +1 -1
  225. package/resources/boilerplate/iOS-Native/java-JUnit5/gradle +6 -1
  226. package/resources/boilerplate/iOS-Native/java-JUnit5/gradle-oss +6 -1
  227. package/resources/boilerplate/iOS-Native/java-TestNG/gradle +1 -1
  228. package/resources/boilerplate/iOS-Native/java-TestNG/gradle-oss +1 -1
  229. package/resources/boilerplate/iOS-Native/java-TestNG/iOSNative-oss.java +2 -2
  230. package/resources/boilerplate/iOS-Native/java-TestNG/iOSNative.java +2 -2
  231. package/resources/test-orchestrator/agent.json +19 -0
package/README.md CHANGED
@@ -6,7 +6,7 @@
6
6
 
7
7
  # Digital.ai Testing — MCP Server
8
8
 
9
- An MCP (Model Context Protocol) server that connects AI assistants to a Digital.ai Testing device farm. The server exposes **191 tools**, **2 resources**, and **7 prompts** covering 26 capability areas: device management, test execution, app lifecycle, reporting, analytics, performance, project administration, interactive mobile inspection, interactive browser inspection, and more.
9
+ An MCP (Model Context Protocol) server that connects AI assistants to a Digital.ai Testing device farm. The server exposes **202 tools**, **2 resources**, and **7 prompts** covering 27 capability areas: device management, test execution (including Espresso, XCUITest and Maestro test runs), app lifecycle, reporting and root-cause analysis, analytics, performance, project administration, interactive mobile inspection, interactive browser inspection, and more.
10
10
 
11
11
  ---
12
12
 
@@ -53,6 +53,8 @@ Then ask: *"Show me the overall health of the device farm."*
53
53
  - [Access Keys](#access-keys)
54
54
  - [Installation](#installation)
55
55
  - [Configuration](#configuration)
56
+ - [Toolsets](#toolsets-reducing-context-size)
57
+ - [Debug mode](#debug-mode-remediation-notes)
56
58
  - [Connecting AI Clients](#connecting-ai-clients)
57
59
  - [Claude Desktop](#claude-desktop)
58
60
  - [Claude Code (VS Code)](#claude-code-vs-code)
@@ -66,11 +68,13 @@ Then ask: *"Show me the overall health of the device farm."*
66
68
  - [Project Lifecycle](#project-lifecycle)
67
69
  - [Boilerplate Generation](#boilerplate-generation)
68
70
  - [Recommended agent guardrails](docs/recommended-agent-guardrails.md)
71
+ - [Performance Comparison Reports](#performance-comparison-reports)
69
72
  - [Reference](#reference)
70
73
  - [Response Format](#response-format)
71
74
  - [Understanding maxResults](#understanding-maxresults)
72
75
  - [List Filters & Sorting](#list-filters--sorting)
73
76
  - [Test Reporting Schema](#test-reporting-schema)
77
+ - [Local File Transfer](#local-file-transfer-docker--remote-deployments-only)
74
78
  - [Safety Guards](#safety-guards)
75
79
  - [Development](#development)
76
80
  - [Troubleshooting](#troubleshooting)
@@ -91,13 +95,15 @@ Then ask: *"Show me the overall health of the device farm."*
91
95
 
92
96
  Your Digital.ai access key determines what the MCP server can do on your behalf. There are three access levels.
93
97
 
94
- | Access level | Key format | Access |
95
- |---|---|---|
96
- | **Cloud Admin** | `eyJ…` (long base-64 string) | All tools: device management, user provisioning, project administration, infrastructure, performance data |
97
- | **Project Admin** | `aut_1_…` | Scoped to one project. Can manage device tags, view project admin settings, list users. v2 API tools (agents, regions, license data) return 403. |
98
- | **Project User** | `aut_1_…` | Scoped to one project. Read/test operations only. Cannot manage tags, access admin settings, or delete reports. |
98
+ | Access level | Access |
99
+ |---|---|
100
+ | **Cloud Admin** | All tools: device management, user provisioning, project administration, infrastructure, performance data |
101
+ | **Project Admin** | Scoped to one project. Can manage device tags, view project admin settings, list users. v2 API tools (agents, regions, license data) return 403. |
102
+ | **Project User** | Scoped to one project. Read/test operations only. Cannot manage tags or access admin settings; deleting reports requires the project's `allowUsersDeleteTests` setting. |
99
103
 
100
- When a Cloud Admin tool is called with a project-level key, the MCP returns a plain-language error explaining what happened — and, if a Cloud Admin profile is configured, a ready-to-use `switch_environment(...)` command.
104
+ > **The key format does not tell you the access level.** Older keys are long `eyJ…` tokens; since Digital.ai 24.11 newly generated keys use a shorter `aut_1_…` token, and that applies to Cloud Admins too. The MCP server therefore detects your access level from the API (`GET /api/v1/users/my-account-info` → `Admin` / `ProjectAdmin` / `User`) and never from the key's prefix. `get_server_info` and `list_environments` show what was detected for each profile.
105
+
106
+ When a Cloud Admin tool is called with a project-level role, the MCP returns a plain-language error explaining what happened — and, if a Cloud Admin profile is known, a ready-to-use `switch_environment(...)` command.
101
107
 
102
108
  ### Finding your key
103
109
 
@@ -112,16 +118,16 @@ The key shown is tied to the **project currently selected in the portal**. If yo
112
118
  Configure named profiles in `.env` to switch contexts at runtime without editing files:
113
119
 
114
120
  ```
115
- # Default connection (typically Cloud Admin credentials)
121
+ # Default connection (typically Cloud Admin credentials — either key format works)
116
122
  DIGITAL_AI_BASE_URL=https://your-tenant.experitest.com
117
- DIGITAL_AI_ACCESS_KEY=eyJ...your-cloud-admin-key...
123
+ DIGITAL_AI_ACCESS_KEY=...your-cloud-admin-key...
118
124
 
119
125
  # Project-scoped profiles
120
126
  DAI_PROFILE_QA_URL=https://your-tenant.experitest.com
121
- DAI_PROFILE_QA_KEY=aut_1_...your-qa-key...
127
+ DAI_PROFILE_QA_KEY=...your-qa-key...
122
128
 
123
129
  DAI_PROFILE_STAGING_URL=https://your-tenant.experitest.com
124
- DAI_PROFILE_STAGING_KEY=aut_1_...your-staging-key...
130
+ DAI_PROFILE_STAGING_KEY=...your-staging-key...
125
131
  ```
126
132
 
127
133
  Then ask your AI:
@@ -225,9 +231,78 @@ docker build -t digital-ai-testing-mcp:latest .
225
231
  | `MCP_DEPLOYMENT_MODE` | Optional | `docker` | `docker` (default) or `local` — set to `local` when running via the npm package ([Option A](#option-a--install-via-npm-recommended)) so tool descriptions and upload/download behavior reflect that the server's filesystem is your own machine, not a container's |
226
232
  | `REQUEST_TIMEOUT_MS` | Optional | `30000` | API request timeout in milliseconds |
227
233
  | `UPLOAD_TIMEOUT_MS` | Optional | `120000` | File upload timeout in milliseconds |
234
+ | `MCP_TOOLSETS` | Optional | all | Comma-separated toolsets to load in full: `devices`, `apps`, `reporting`, `performance`, `inspection`, `authoring`, `browsers`, `admin` (or `all`, or `core` for only the always-loaded core). See [Toolsets](#toolsets-reducing-context-size) |
235
+ | `MCP_DEBUG_MODE` | Optional | `false` | `true` records remediation notes and a tool-call event log to help improve this MCP. See [Debug mode](#debug-mode-remediation-notes) |
236
+ | `MCP_REMEDIATION_DIR` | Optional | — | Debug mode only: folder to write remediation files to (overrides project detection) |
237
+ | `AXE_DEVTOOLS_API_KEY` | Optional | — | Deque Axe DevTools Mobile key used by `includeAxeScan` boilerplate |
238
+ | `TEST_ORCHESTRATOR_JAR_URL` / `TEST_ORCHESTRATOR_JAR_SHA256` | Optional | pinned | Override the Test Orchestrator agent download location; both are required |
239
+ | `ACCESS_PROBE_TIMEOUT_MS` | Optional | `8000` | Timeout for the access-level detection call |
228
240
 
229
241
  Additional `DAI_PROFILE_{NAME}_URL` / `DAI_PROFILE_{NAME}_KEY` pairs configure named profiles for multi-project or multi-environment use. See [Access Keys](#access-keys) and `.env.example` for examples.
230
242
 
243
+ ### Toolsets (reducing context size)
244
+
245
+ All 202 tool definitions add up to roughly 68K tokens of context. If your AI client is short on context — or runs several MCP servers — set `MCP_TOOLSETS` to the areas you use, e.g. `MCP_TOOLSETS=reporting` (about 25K tokens) or `MCP_TOOLSETS=inspection,authoring` (about 32K).
246
+
247
+ Nothing becomes unavailable. Tools outside the chosen toolsets are still listed, with a one-line description. When the AI calls one, the server loads that whole toolset and replies with the tool's full description and parameter guidance, then the AI calls it again — so the detailed usage guidance is always seen before the tool runs. The AI can also load toolsets up front with `enable_toolset`. Leave `MCP_TOOLSETS` unset (the default) to load everything in full, exactly as before.
248
+
249
+ | Toolset | Covers |
250
+ |---|---|
251
+ | `devices` | Devices, device groups, reservations, health, agents, regions |
252
+ | `apps` | Application repository, file repository, provisioning profiles |
253
+ | `reporting` | Test reports, analytics, root-cause analysis, test views, coverage |
254
+ | `performance` | Performance transactions and comparisons, NV servers |
255
+ | `inspection` | Live device and browser inspection, Android Auto / CarPlay, remote debug |
256
+ | `authoring` | Test boilerplate, Espresso / XCUITest / Maestro test runs |
257
+ | `browsers` | Browser listing and Selenium sessions |
258
+ | `admin` | Users, projects, backups, usage reports, POC and project workflows |
259
+
260
+ Always loaded in full: environment and connection tools (`get_server_info`, `switch_environment`, …), `enable_toolset`, and `validate_test_script`.
261
+
262
+ ### Debug mode (remediation notes)
263
+
264
+ Set `MCP_DEBUG_MODE=true` while evaluating or developing the MCP to collect feedback for improving it. With the default `false`, nothing changes.
265
+
266
+ When it is on:
267
+
268
+ - **The AI records notes.** The server asks the AI client to call `record_remediation_note` (a tool that exists only in debug mode) whenever it:
269
+ - hits an error;
270
+ - wastes calls because guidance was unclear;
271
+ - is corrected or stopped by you;
272
+ - gives up;
273
+ - finds a better path than the one recommended;
274
+ - spots a way to make things cleaner or cheaper.
275
+
276
+ It records each note once the situation is resolved or abandoned, not in the middle of your task.
277
+ - **The server logs what it sees.** Every tool call is logged with its duration, response size and outcome (ok, error or guard), with flags for retries and repeated calls. This is objective data the AI can't under-report.
278
+ - **Errors get a reminder.** The first time a tool returns an error or a guard, the response ends with a one-line reminder to record a note.
279
+
280
+ Where the files go: a `remediation/` folder in the root of **the project the AI is working in**, so each note sits next to the app and tests it's about.
281
+
282
+ | Install | Notes | Event log |
283
+ |---|---|---|
284
+ | npm (`MCP_DEPLOYMENT_MODE=local`) | The server writes `<project>/remediation/<yyyymmddhhmmss>-<id>.md` | The server writes `<project>/remediation/<yyyymmddhhmmss>-<id>.events.jsonl` |
285
+ | Docker | Returned to the AI, which saves them to `remediation/` in its project if it can write files (Claude Code can; Claude Desktop needs a filesystem tool) | Kept in server memory; a summary is included in every note |
286
+
287
+ With the npm install, the server finds the project in this order, and records the step it used as `location:` in each file:
288
+ 1. `MCP_REMEDIATION_DIR`, if set (files go directly into that folder);
289
+ 2. the workspace folder the AI client reports;
290
+ 3. the folder the server was started in, if it looks like a project;
291
+ 4. `~/remediation`.
292
+
293
+ The `remediation/` folder contains its own `.gitignore` (`*`), so it is never committed to the project it lives in. There is one file per session; the `<id>` suffix keeps parallel sessions apart, and each file records the machine and user, so files collected from several machines can be merged. Keys, tokens, emails and signed-URL parameters are redacted before anything is written or returned.
294
+
295
+ Debug mode costs extra tokens (the instructions and the notes). Leave it off for normal use, and don't use debug sessions as token baselines.
296
+
297
+ To process a batch, collect the `remediation/` folders you want to analyse, from one machine or several, and run `npm run remediation:digest -- --dir <folder> --dir <folder>` in a clone of this repository (the npm package doesn't include it). With no `--dir`, it reads `$MCP_REMEDIATION_DIR`, or else `./remediation` and `~/remediation`. It condenses every unprocessed file into one summary, so the development agent reads a few KB instead of every raw note:
298
+ - notes by category, and issues that recur across sessions;
299
+ - per-tool errors, guards, retries, response sizes and latency;
300
+ - every note in a table.
301
+
302
+ Useful options:
303
+ - `--out <file>` writes the summary to a file; `--since YYYY-MM-DD` limits it to recent sessions; `--json` gives machine-readable output.
304
+ - `--archive <session…>` or `--archive-all` moves handled files into `processed/<date>/` inside the folder they came from. Nothing is deleted.
305
+
231
306
  ---
232
307
 
233
308
  ## Connecting AI Clients
@@ -418,7 +493,7 @@ Once connected, talk to the server in plain language — no tool names needed:
418
493
  - *"Find an available Android phone, then generate a Java JUnit5 test boilerplate"*
419
494
  - *"Show the pass/fail breakdown by OS — are Android and iOS failing at different rates?"*
420
495
  - *"Did the latest release introduce a CPU regression? Compare version 10553 vs 10554 on Android."*
421
- - *"Set up a new POC for Acme Corp — 6 devices in US2, 3 iOS and 3 Android, through August 31st"*
496
+ - *"Set up a new POC for Acme Corp — 6 devices in US2, 3 iOS and 3 Android, for the next 4 weeks"*
422
497
 
423
498
  **[→ docs/examples.md](docs/examples.md)** has 200+ prompts organized by scenario: farm health, device groups, regression sign-off, test analytics, performance, usage reports, interactive inspection, POC lifecycle, and more.
424
499
 
@@ -426,7 +501,7 @@ Once connected, talk to the server in plain language — no tool names needed:
426
501
 
427
502
  ## Capabilities
428
503
 
429
- 191 tools across 26 capability domains. The complete per-tool reference — descriptions, filters, auth requirements, and usage notes — lives in **[docs/tools.md](docs/tools.md)**.
504
+ 202 tools across 27 capability domains. The complete per-tool reference — descriptions, filters, auth requirements, and usage notes — lives in **[docs/tools.md](docs/tools.md)**.
430
505
 
431
506
  | Domain | Tools | Highlights |
432
507
  |---|---|---|
@@ -442,21 +517,22 @@ Once connected, talk to the server in plain language — no tool names needed:
442
517
  | [Backup](docs/tools.md#backup) | 1 | Trigger a system backup |
443
518
  | [Health & Diagnostics](docs/tools.md#health--diagnostics) | 11 | Farm health, readiness checks, license utilization |
444
519
  | [Coverage Analytics](docs/tools.md#coverage-analytics) | 2 | Tested-vs-available device gap analysis |
445
- | [Reporting](docs/tools.md#reporting) | 20 | Search/filter test reports, failure summarization, stability, trends, logs, attachment download command, cleanup |
520
+ | [Reporting](docs/tools.md#reporting) | 23 | Search/filter test reports, root-cause analysis, failure summarization, stability, trends, logs, video download (byte ranges), share links, attachment download command, cleanup |
521
+ | [Test Runs](docs/tools.md#test-runs-espresso--xcuitest--maestro) | 5 | Run Espresso, XCUITest and Maestro suites on the platform; status, cancel, local command; generate Maestro flows |
446
522
  | [Test Views](docs/tools.md#test-views) | 7 | Dashboard view groups |
447
- | [Transactions & Performance](docs/tools.md#transactions--performance) | 4 | CPU/memory/battery/Speed Index analytics *(Cloud Admin)* |
523
+ | [Transactions & Performance](docs/tools.md#transactions--performance) | 4 | CPU/memory/battery/Speed Index analytics |
448
524
  | [Agents](docs/tools.md#agents) | 2 | Host machine status *(Cloud Admin)* |
449
525
  | [Regions](docs/tools.md#regions) | 2 | Region status and infrastructure topology *(Cloud Admin)* |
450
526
  | [NV Servers](docs/tools.md#nv-servers) | 2 | Network Virtualization servers *(Cloud Admin)* |
451
527
  | [Environment Management](docs/tools.md#environment-management) | 2 | Named connection profiles, runtime switching |
452
528
  | [Workflows](docs/tools.md#workflows) | 6 | POC and project lifecycle orchestration |
453
- | [Boilerplate Generation](docs/tools.md#boilerplate-generation) | 3 | Ready-to-run Appium and Selenium test scripts in 4 languages; validate scripts before delivery |
529
+ | [Boilerplate Generation](docs/tools.md#boilerplate-generation) | 4 | Ready-to-run Appium and Selenium test scripts in 4 languages, Test Orchestrator agent install; validate scripts before delivery |
454
530
  | [Remote Debug](docs/tools.md#remote-debug) | 1 | Connect a cloud device as a local ADB device |
455
- | [Mobile Inspection Sessions](docs/tools.md#mobile-inspection-sessions) | 22 | AI-driven live device interaction — screenshots, element discovery, full gesture set, keys, app/device control |
531
+ | [Mobile Inspection Sessions](docs/tools.md#mobile-inspection-sessions) | 24 | AI-driven live device interaction — screenshots, element discovery, full gesture set, keys, app/device control, Android Auto / CarPlay projection, mock biometric authentication |
456
532
  | [Web Inspection Sessions](docs/tools.md#web-inspection-sessions) | 8 | AI-driven live browser interaction — navigate, Shadow DOM element discovery, CSS selector verification, browser-neutral Selenium script generation |
457
- | [Performance Comparison](docs/tools.md#performance-comparison) | 4 | Two-set Speed Index comparison with confound detection, MAD outlier exclusion, and fresh-sample generation *(Cloud Admin)* |
458
- | [Usage Reports](docs/tools.md#usage-reports) | 3 | Platform usage-report CSV export and in-memory aggregation — device reservations, users/devices/browser usage, license usage *(Cloud Admin)* |
459
- | [Resources & Prompts](docs/tools.md#resources--prompts) | — | 2 ambient resources, 6 guided prompts |
533
+ | [Performance Comparison](docs/tools.md#performance-comparison) | 4 | Two-set Speed Index comparison with confound detection, MAD outlier exclusion, and fresh-sample generation |
534
+ | [Usage Reports](docs/tools.md#usage-reports) | 3 | Platform usage-report CSV export and in-memory aggregation — device reservations, users/devices/browser usage, license usage; multi-section CSVs with a User Tag column, about 2 years of retention; large unfiltered exports run only after you explicitly agree (`confirmLargeExport`) *(Cloud Admin)* |
535
+ | [Resources & Prompts](docs/tools.md#resources--prompts) | — | 2 ambient resources, 7 guided prompts |
460
536
 
461
537
  ---
462
538
 
@@ -479,7 +555,7 @@ Collects all parameters upfront, presents a confirmation summary to the operator
479
555
  7. Remove devices from the Default group — done only after the project exists, so a project-creation failure never strands devices
480
556
  8. Locate the Default project
481
557
  9. Create users, assign to the POC project, remove from Default, and tag each account with the POC tag. Cloud Admin access is never granted through this workflow.
482
- 10. Assign ExperiBank (or a specified app) to the POC project — falls back to the latest available version if the exact version is not found
558
+ 10. Assign ExperiBank (or the app identified by `androidPackageName` / `iosBundleId`) to the POC project — if a binary isn't in the repository, the summary reports it as a manual upload and the workflow continues
483
559
 
484
560
  If any step fails, the workflow stops, reports which steps completed with all created resource IDs, and offers resume (re-invoke — existing resources are detected and reused) or unwind (`delete_poc`) paths.
485
561
 
@@ -490,7 +566,7 @@ If any step fails, the workflow stops, reports which steps completed with all cr
490
566
  | `customerName` | e.g. `"Acme Corp"` → derives project `"Acme Corp POC"` and tag `"acmecorppoc"` |
491
567
  | `region` | Testing region, e.g. `"US2"`, `"EU"`, `"SG"` |
492
568
  | `salesforceUrl` | Salesforce Opportunity URL — recorded in project notes |
493
- | `endDate` | Accepts ISO (`"2026-08-31"`), relative offsets (`"+14d"`, `"+2w"`), or natural language (`"in 2 weeks"`) |
569
+ | `endDate` | Accepts ISO (`"YYYY-MM-DD"`), relative offsets (`"+14d"`, `"+2w"`), or natural language (`"in 2 weeks"`) |
494
570
  | `users` | Array of `{email, firstName, lastName, role}` — role must be `"User"` or `"ProjectAdmin"` |
495
571
 
496
572
  **Optional parameters:**
@@ -502,7 +578,8 @@ If any step fails, the workflow stops, reports which steps completed with all cr
502
578
  | `androidCount` | `floor(deviceCount / 2)` |
503
579
  | `automationType` | `"appium-server"` (alternative: `"appium-grid"`) |
504
580
  | `appName` | `"ExperiBank"` |
505
- | `appVersion` | `"1.0"` |
581
+ | `androidPackageName` | `"com.experitest.ExperiBank"` |
582
+ | `iosBundleId` | `"com.experitest.ExperiBank"` |
506
583
 
507
584
  Steps 1 and 6 check for an existing group/project by name before creating — safe to re-run after a partial failure.
508
585
 
@@ -544,7 +621,7 @@ At the start, the workflow asks whether to create a **simple** project record on
544
621
 
545
622
  Full setup steps:
546
623
 
547
- 1. Create (or reuse) the project with the specified name and automation type; record the memo in project notes
624
+ 1. Create (or reuse) the project with the specified name and automation type; record `projectMemo` in project notes
548
625
  2. Create (or reuse) a device group and link it to the project
549
626
  3. Pre-flight check that the target app exists in the repository
550
627
  4. Select available devices matching the target OS and region — presented to the operator for confirmation
@@ -571,8 +648,10 @@ Full setup steps:
571
648
  | `automationType` | `"appium-server"` | `"appium-server"` or `"appium-grid"` |
572
649
  | `isolateDevices` | `false` | Remove devices from all other groups before assignment |
573
650
  | `users` | `[]` | Array of `{email, firstName, lastName, role}` |
574
- | `appName` | _(none)_ | App to assign to the project |
575
- | `memo` | _(none)_ | Notes to record on the project |
651
+ | `appName` | `"ExperiBank"` | App to assign to the project |
652
+ | `androidPackageName` | `"com.experitest.ExperiBank"` | Android package used to find the app |
653
+ | `iosBundleId` | `"com.experitest.ExperiBank"` | iOS bundle ID used to find the app |
654
+ | `projectMemo` | _(none)_ | Notes to record on the project |
576
655
 
577
656
  ---
578
657
 
@@ -620,6 +699,20 @@ The server supports two test-creation modes for both mobile and web, and the age
620
699
 
621
700
  Generates a complete, pre-configured Appium test script. The Digital.ai server URL and access key are pre-filled from the active connection profile — switch profiles with `switch_environment` first to generate scripts carrying a project-scoped key instead of your admin key.
622
701
 
702
+ #### Test Orchestrator (default for Java on Appium Server)
703
+
704
+ For **Java (JUnit 5 / TestNG) on an Appium Server project**, the generated project is wired for the **Digital.ai Test Orchestrator** (the Smart Agent): failed tests are retried automatically, each attempt is reported separately, the real pass/fail is synced to Reporter, and runs are grouped by Build ID — with no change to the test code. Appium Grid, Python and NodeJS projects get the classic boilerplate. Pass `orchestration: "off"` to get the classic boilerplate for Java too.
705
+
706
+ | Step | What happens |
707
+ |---|---|
708
+ | 1. `get_test_boilerplate` | Adds conditional agent wiring to `build.gradle` / `pom.xml`, an `orchestrator/config.template.yml` (retries default to 2; 0 for placeholder or performance tests), plus `orchestrator/.gitignore` and `lib/.gitignore` so the rendered key and the agent JAR can never be committed. No root `.gitignore` is written (it would overwrite yours). |
709
+ | 2. `install_test_orchestrator_agent(projectDir)` | Downloads the agent on demand, verifies its SHA-256 and places it at `lib/smart-agent.jar`. Under Docker it returns a download command (bash and PowerShell) that does the same and deletes the file on a checksum mismatch. If the download fails, the response gives the URL, the checksum and the target path for a manual install. |
710
+ | 3. Set `DIGITAL_AI_ACCESS_KEY` and run `gradle test` / `mvn test` | The agent attaches only when the JAR is present **and** the key is set; the key is rendered into the git-ignored `orchestrator/rendered/` at build time, never into a tracked file, and Java 17 is required only in that case. Otherwise the project builds and runs exactly like the classic boilerplate (Gradle prints why the agent was not attached). Set `BUILD_ID` to group a CI build's results. |
711
+
712
+ Requirements: a **Java 17 or 21** JDK with a current truststore, JUnit 5 or TestNG 7+, Appium java-client 8–10. See [Limitations](docs/limitations.md#17-test-orchestrator-known-issues) for known issues.
713
+
714
+ The agent JAR is **not** shipped with this server. By default it is downloaded from the [Digital.ai Test Orchestrator sample repository](https://github.com/raheekhandigitalai/Digital.ai-Testing-Test-Orchestrator), pinned to a fixed commit and verified by SHA-256 (see `resources/test-orchestrator/agent.json`). To use a different location — for example an internal mirror — set `TEST_ORCHESTRATOR_JAR_URL` **and** `TEST_ORCHESTRATOR_JAR_SHA256` (both required; a URL without a checksum is refused).
715
+
623
716
  | Parameter | Values | Default |
624
717
  |---|---|---|
625
718
  | `platform` | `android` \| `ios` | _(required)_ |
@@ -635,23 +728,25 @@ Generates a complete, pre-configured Appium test script. The Digital.ai server U
635
728
  | `includePerformanceTransactions` | `true` \| `false` | `false` |
636
729
  | `includeAxeScan` | `true` \| `false` | `false` |
637
730
  | `confirmSelectorsVerified` | `true` \| `false` | _(optional)_ |
731
+ | `orchestration` | `auto` \| `on` \| `off` | `auto` |
732
+ | `orchestrationMaxRetries` | `0`–`5` | `2` (`0` for placeholder or performance tests) |
733
+ | `automotiveProjection` | Android Auto / CarPlay resolution | _(optional)_ |
638
734
  | `outputFormat` | `json` \| `human` | `json` |
639
735
 
640
- > **Inspection gate (v42):** when you target a real app (`appId`/`packageName`/`bundleIdentifier`), this tool returns **no code** and a structured `{ status: "blocked", reason: "no_verified_selectors" }` error UNLESS a live inspection session exists in the MCP process, or you set `confirmSelectorsVerified: true` (escape hatch for selectors already captured via rdb/UIAutomator, `open_mobile_studio`, or authoritative source). The built-in ExperiBank demo (no app identifiers) is never gated. This makes it structurally impossible to receive a placeholder scaffold and pass it off as a finished test.
736
+ > **Inspection gate:** when you target a real app (`appId`/`packageName`/`bundleIdentifier`), this tool returns **no code** and a structured `{ status: "blocked", reason: "no_verified_selectors" }` error UNLESS a live inspection session exists in the MCP process, or you set `confirmSelectorsVerified: true` (escape hatch for selectors already captured via rdb/UIAutomator, `open_mobile_studio`, or authoritative source). The built-in ExperiBank demo (no app identifiers) is never gated. This makes it structurally impossible to receive a placeholder scaffold and pass it off as a finished test.
641
737
 
642
738
  **Recommended workflow** for building a new test:
643
739
 
644
740
  1. `get_application_info` — confirm package name, launch activity, and app ID
645
- 2. `get_test_boilerplate` — generate starter test with capabilities pre-filled (this step)
646
- 3. `open_mobile_studio` or `get_automation_properties` — inspect live element IDs (no ADB required)
647
- 4. `release_orphaned_sessions(maxAgeHours=4, dryRun=true)` — pre-flight device check
648
- 5. `find_available_device` — select a healthy device and read its `region`
649
- 6. Write/run the test
650
- 7. `release_device` — explicit cleanup
741
+ 2. `find_available_device` — select a healthy device and read its `region`
742
+ 3. `start_inspection_session` (or `open_mobile_studio`) — capture real element IDs
743
+ 4. `get_test_boilerplate(appId, region, …)` — generates code while the inspection session is live, or with `confirmSelectorsVerified: true`
744
+ 5. `validate_test_script` — before saving or presenting the test
745
+ 6. Run the test, then `release_device` — explicit cleanup
651
746
 
652
747
  **Performance transactions:** Pass `includePerformanceTransactions: true` to bracket the test body with `startPerformanceTransaction` / `endPerformanceTransaction` calls. The start arg is the NV network profile (defaults to `"Monitor"` — observe without throttling); the end arg is the transaction name. The platform records CPU, memory, battery, and Speed Index metrics; results appear in the reporter Transactions tab (~1 min after `endPerformanceTransaction`) and are queryable via `list_transactions`. Pre-requisite: an NV server must be ONLINE and tunnel-connected in the device region (`list_nv_servers`). Note: a throttling profile activates NV shaping immediately; `"Monitor"` is pass-through and carries no ANR risk.
653
748
 
654
- **Accessibility scanning:** Pass `includeAxeScan: true` to inject a Deque Axe DevTools Mobile accessibility scan (`mobile: axeScan` executeScript call). Sets the required `appium:automationName` capability automatically (`AxeUiAutomator2` for Android, `AxeXCUITest` for iOS). Requires `AXE_DEVTOOLS_API_KEY` in the MCP environment.
749
+ **Accessibility scanning:** Pass `includeAxeScan: true` to inject a Deque Axe DevTools Mobile accessibility scan (`mobile: axeScan` executeScript call). Sets the required `appium:automationName` capability automatically (`AxeUiAutomator2` for Android, `AxeXCUITest` for iOS). Also pins `appiumVersion` to 2.16.2 (the Axe driver is unavailable on Appium Server 3.3.0+; don't raise it). The key comes from `AXE_DEVTOOLS_API_KEY`; if that's unset, a placeholder is written.
655
750
 
656
751
  Both flags can be combined — the Axe scan runs inside the performance transaction boundary.
657
752
 
@@ -661,7 +756,7 @@ Both flags can be combined — the Axe scan runs inside the performance transact
661
756
 
662
757
  ### `validate_test_script`
663
758
 
664
- Delivery backstop (v43) for the case the `get_test_boilerplate` gate cannot catch — a test written by hand with guessed selectors. Pass the full `scriptContent` before presenting or saving any test; it scans for unreplaced `<…>` placeholder selectors, the deliberate scaffold fail-guard, placeholder/fabricated credentials, and resource IDs from a known prior fabrication incident. Returns `isError` with a `fail` verdict when any high-severity pattern is found, so a non-functional test cannot be delivered as finished. A `pass` is necessary but not sufficient — it confirms obvious placeholders are gone, not that selectors are real; a live inspection remains the authoritative source.
759
+ Delivery backstop for the case the `get_test_boilerplate` gate cannot catch — a test written by hand with guessed selectors. Pass the full `scriptContent` before presenting or saving any test; it scans for unreplaced `<…>` placeholder selectors, the deliberate scaffold fail-guard, placeholder/fabricated credentials, and known fabricated resource IDs. Returns `isError` with a `fail` verdict when any high-severity pattern is found, so a non-functional test cannot be delivered as finished. A `pass` is necessary but not sufficient — it confirms obvious placeholders are gone, not that selectors are real; a live inspection remains the authoritative source.
665
760
 
666
761
  | Parameter | Values | Default |
667
762
  |---|---|---|
@@ -676,7 +771,7 @@ Delivery backstop (v43) for the case the `get_test_boilerplate` gate cannot catc
676
771
  | Language | Files |
677
772
  |---|---|
678
773
  | `java-junit5` / `java-testng` | `AndroidNative.java` or `iOSNative.java` + `build.gradle` + `pom.xml` |
679
- | `nodejs` | `wdio.conf.js` + test file + setup shell steps |
774
+ | `nodejs` | `wdio.conf.js` + test file + setup shell steps (WebdriverIO 7.40 for Appium Grid projects, WebdriverIO 9 for Appium Server) |
680
775
  | `python` | Test file + `requirements.txt` |
681
776
 
682
777
  All boilerplate defaults to the ExperiBank demo app as the starting point. ExperiBank is available on most farm devices and can verify connectivity before switching to your own app.
@@ -689,15 +784,15 @@ Generates a complete, pre-configured Selenium test script for browser testing. T
689
784
  |---|---|---|
690
785
  | `language` | `java-junit5` \| `java-testng` \| `nodejs` \| `python` | _(required)_ |
691
786
  | `url` | URL of the page being tested | _(optional)_ |
692
- | `testName` | Any string | `"My First Web Test"` |
693
- | `targetBrowser` | `chrome` \| `firefox` \| `edge` \| `safari` | _(optional)_ |
787
+ | `testName` | Any string | `"WebTest"` |
788
+ | `targetBrowser` | `chrome` \| `firefox` \| `MicrosoftEdge` \| `safari` \| `opera` (names as `list_available_browsers` returns them; `edge` is accepted for `MicrosoftEdge`) | _(optional)_ |
694
789
  | `shadowDomSupport` | `auto` \| `always` \| `never` | `auto` |
695
790
  | `confirmSelectorsVerified` | `true` \| `false` | _(optional)_ |
696
791
  | `outputFormat` | `json` \| `human` | `json` |
697
792
 
698
793
  > **Inspection gate:** when you target a real URL (`url` provided), this tool returns **no code** and a structured `{ status: "blocked", reason: "no_verified_selectors" }` error UNLESS a live browser inspection session exists in the MCP process, or you set `confirmSelectorsVerified: true` (escape hatch for selectors captured via a prior `start_browser_inspection_session` session). This mirrors the mobile boilerplate gate.
699
794
 
700
- **Browser-neutral by default:** when `targetBrowser` is omitted, the generated script reads the browser name at runtime from an environment variable (`BROWSER`). This means the same script runs on Chrome, Firefox, Edge, and Safari without modification — just change the env var or CI matrix. Specify `targetBrowser` only when you need browser-specific options (e.g. Safari-only session setup).
795
+ **Browser-neutral by default:** when `targetBrowser` is omitted, the generated script reads the browser name at runtime from an environment variable (`BROWSER`). This means the same script runs on Chrome, Firefox, Microsoft Edge, Safari and Opera without modification — just change the env var or CI matrix. Specify `targetBrowser` only when you need browser-specific options (e.g. Safari-only session setup).
701
796
 
702
797
  **Shadow DOM support:** when `shadowDomSupport` is `auto` (the default) or `always`, the generated script includes a `shadowQuery(host, cssSelector)` helper that traverses into shadow roots using `executeScript`. Use this for apps built with Web Components, Lit, Polymer, or other shadow-DOM frameworks. `get_page_dom` detects shadow DOM automatically during inspection and reports `hasShadowDom: true` when present.
703
798
 
@@ -726,7 +821,7 @@ The headline metric is reported three ways every time — **trimmed mean, median
726
821
  | `detect_performance_outliers` | Robust median/MAD outlier flagging on a single set; returns the kept set and recommended exclusions/re-runs. |
727
822
  | `performance_transaction_control` | Phase 2: generate fresh samples inside an inspection session — `start` (with an NV profile) → run the verified flow → `end` (names the record). Records appear in the reporter ~1 min later. |
728
823
 
729
- > **Why the confound check matters:** comparing transactions 1894 (Speed Index 1015) and 1895 (1000) looks like a 1.5% version delta — but they ran on *different device models, different OS versions, and different projects*, and 1894 has no CPU/memory telemetry. `assess_comparison_confounds` returns **confounded** and names all three, so the delta is never misread as a version regression.
824
+ > **Why the confound check matters:** comparing two transactions with Speed Index 1015 vs 1000 looks like a 1.5% version delta — but if they ran on *different device models, different OS versions, and different projects*, and one has no CPU/memory telemetry, `assess_comparison_confounds` returns **confounded** and names each cause, so the delta is never misread as a version regression.
730
825
 
731
826
  The **`performance_comparison_report` prompt** orchestrates the full workflow end to end: define the axis → scrub confounds → negotiate sample size → **require explicit plan confirmation with a time estimate** → run the series with outlier-driven re-runs → report the delta with root-cause reasoning. True host/background interference is not directly observable; the report approximates device quietness (idle, healthy status, no concurrent reservation) and states that limit explicitly.
732
827
 
@@ -771,15 +866,17 @@ Use filters to narrow results rather than raising `maxResults`. When results are
771
866
  | Tool | Available Filters | Available `sortBy` values |
772
867
  |---|---|---|
773
868
  | `list_users` | firstName, lastName, email, authenticationType, isCloudAdmin, tag | firstName, lastName, email, userName, authenticationType |
774
- | `list_devices` | query (@-syntax), region, model | deviceName, deviceOs, osVersion, manufacturer, displayStatus, agentName, region |
869
+ | `list_devices` | query (@-syntax), os, region, model | deviceName, deviceOs, osVersion, manufacturer, displayStatus, agentName, region |
775
870
  | `get_devices_in_group` | osType, status, category, excludeTags, requireTags | _(none)_ |
776
871
  | `list_reservations` | username, project, deviceUid | reservationStart, reservationEnd, username, project, deviceUid |
777
872
  | `list_projects` | name (partial match) | name, id |
778
- | `list_project_users` | username (partial), role (exact) | firstName, lastName, email, userName, role |
779
- | `list_applications` | nameContains, osType, packageName, bundleIdentifier, fileType, isForSimulator | applicationName, version, createdAt (default), osType |
873
+ | `list_project_users` | username (partial), role (exact) | username, role |
874
+ | `list_applications` | nameContains, osType, packageName, bundleIdentifier, uniqueName, fileType, isForSimulator | applicationName, releaseVersion, buildVersion, createdAt (default), osType, fileType |
780
875
  | `list_test_reports` | status, name, has_attachment, success; startDate/endDate (date range) | start_time |
781
876
 
782
- All list tools accept `sortOrder: "asc" | "desc"` (default: `"asc"`).
877
+ All list tools accept `sortOrder: "asc" | "desc"` (default `"asc"`, except `list_applications`, which defaults to `"desc"`, newest first).
878
+
879
+ `list_test_reports` sorting is performed by the platform for every access level. If the platform ever refuses it for a credential, the server transparently falls back to a client-side scan (up to 5,000 records) so "newest first" results stay correct, just slower on large report sets.
783
880
 
784
881
  ### Test Reporting Schema
785
882
 
@@ -817,8 +914,8 @@ To diagnose a failure, list with `list_test_reports`, then call `get_test_report
817
914
 
818
915
  | Identifier | Source | How to retrieve |
819
916
  |---|---|---|
820
- | `test_id` | List results | `get_test_report(testId: N)` |
821
- | `uuid` | List results | No direct endpoint — look up via `test_id` |
917
+ | `uuid` | List results, report URLs (`/reporter/video-report/<uuid>`) | `get_test_report(uuid: "...")` or `get_test_report(reportUrl: "...")` — preferred, globally unique |
918
+ | `test_id` | List results | `get_test_report(testId: N)` — unique only within a project; the same number can be a different test in another project |
822
919
  | `report_api_id` | Returned by session-start tools | `get_test_by_report_id(reportApiId: "...")` — wait ~60 s after session close |
823
920
 
824
921
  **Examples:**
@@ -857,11 +954,15 @@ list_test_reports
857
954
  get_test_report(testId: 69)
858
955
  → cause, errorCategory, errorClassification, errorDetail (stack trace)
859
956
 
957
+ # 2b. Platform root-cause analysis (read-only)
958
+ get_root_cause_analysis(uuid: "…") # or reportUrl / testId
959
+
860
960
  # 3. Full Appium log as text, when the structured fields aren't enough
861
961
  get_test_log(uuid: "…", logType: "appium")
862
962
 
863
- # 4. Pull the binary artifacts (session video .mp4, full ZIP) to YOUR machine
864
- # — only needed if the server runs in Docker/remote (not the npm install)
963
+ # 4. Pull the binary artifacts (session video .mp4, full ZIP) to YOUR machine.
964
+ # With the npm install, call download_test_video / download_test_attachments
965
+ # directly; under Docker/remote, generate a local command instead:
865
966
  get_test_attachments_download_command(uuid: "…", localPlatform: "macos")
866
967
  → a curl command you run locally; the file never transits the container
867
968
  ```
@@ -880,6 +981,7 @@ For those deployments, every file-transfer tool has a **command-generator siblin
880
981
  | Upload repo file | `upload_repository_file` | `get_repository_upload_command` |
881
982
  | Upload profile | `upload_provisioning_profile` | `get_provisioning_profile_upload_command` |
882
983
  | Download test attachments (.mp4, logs) | `download_test_attachments` | `get_test_attachments_download_command` |
984
+ | Download test video (.mp4, byte ranges) | `download_test_video` | `get_test_attachments_download_command` (`artifact: "video"`) |
883
985
  | Download repo file | `download_repository_file` | `get_repository_file_download_command` |
884
986
  | Download profile | `download_provisioning_profile` | `get_provisioning_profile_download_command` |
885
987
  | Download app language files | `extract_app_language_files` | `get_app_language_files_download_command` |
@@ -899,7 +1001,7 @@ The first call describes exactly what will be deleted. The second call — with
899
1001
 
900
1002
  Two further guards run before any request leaves the server:
901
1003
 
902
- - **Auth pre-flight:** report-deletion tools (`delete_test_reports*`, `cleanup_inspection_sessions`) check the active key type first and return a clear "switch to a Cloud Admin profile" message instead of an opaque CSRF 401 when called with a project-level key.
1004
+ - **Auth pre-flight:** report-deletion tools (`delete_test_reports*`, `cleanup_inspection_sessions`) check the API-detected access level first. A project-level role is refused up front only when its project's `allowUsersDeleteTests` setting is false, with a message telling you to have a Cloud Admin enable it (or switch to a Cloud Admin profile) instead of an opaque 403.
903
1005
  - **Upload path validation:** tools that read local files for upload refuse relative paths, path traversal, and credential-file names (`.env*`, SSH private keys) — a misdirected request cannot publish secrets to the cloud repository.
904
1006
 
905
1007
  ---
@@ -933,9 +1035,12 @@ npm run test:utils # utility functions
933
1035
  npm run test:transactions # performance transactions
934
1036
  npm run test:analytics # analytics tools
935
1037
  npm run test:infrastructure # agents / regions / NV servers / sessions
1038
+ npm run test:live # live API-behavior probes (requires .env; not part of npm test)
1039
+ npm run test:fidelity # agent tool-choice evaluation (builds first)
1040
+ npm run remediation:digest # summarize debug-mode remediation notes
936
1041
  ```
937
1042
 
938
- Tests can also be run from VS Code: `Ctrl+Shift+P` → "Tasks: Run Task".
1043
+ See `package.json` for the full list of scripts.
939
1044
 
940
1045
  ---
941
1046
 
@@ -946,7 +1051,7 @@ Tests can also be run from VS Code: `Ctrl+Shift+P` → "Tasks: Run Task".
946
1051
  | Symptom | Cause | Fix |
947
1052
  |---|---|---|
948
1053
  | `403 Forbidden` on admin tools (agents, regions, license) | Active profile is a project-level key — v2 endpoints require Cloud Admin access | Ask Agent: *"switch to my Cloud Admin profile"* — it will call `list_environments` and pick the right one |
949
- | Report delete tools return "Cloud Admin access required" | Reporter mutation endpoints are CSRF-blocked for project-level keys (Project Admin and Project User) | Switch to a Cloud Admin profile, re-run, switch back |
1054
+ | Report delete tools refuse with "Reporter deletes are disabled for project …" | The project's `allowUsersDeleteTests` setting is false, so project-level roles can't delete reports | Ask a Cloud Admin to enable it for the project, or switch to a Cloud Admin profile |
950
1055
  | `install_application` returns 400 on a device you can see | Device is reserved via an rdb (remote debug) session | Install **first**, then run `get_remote_debug_command` — not the other way around |
951
1056
  | `install_application` returns 400 (no rdb involved) | App not assigned to a project containing the target device | Call `assign_app_to_project` first |
952
1057
  | Repeatable `NoSuchElementException` while sibling tests pass | Device health, not test code — device stuck in a wrong state or offline-but-pooled | Run `get_device_health_summary`; scope the deviceQuery with `@region='<healthy-region>'` |
@@ -963,7 +1068,7 @@ On Windows this usually means the `--env-file` path was stored with backslashes
963
1068
  Run this sequence:
964
1069
 
965
1070
  ```
966
- 1. get_server_info — confirm tool count (expect 191) and active profile
1071
+ 1. get_server_info — confirm tool count (expect 202; 203 with MCP_DEBUG_MODE=true) and active profile
967
1072
  2. check_workflow_readiness — which dependency tools are present or missing
968
1073
  3. check_connectivity — confirm the backend API is reachable
969
1074
  ```
@@ -973,7 +1078,7 @@ Run this sequence:
973
1078
  ```json
974
1079
  {
975
1080
  "allWorkflowsReady": true,
976
- "registeredToolCount": 191,
1081
+ "registeredToolCount": 202,
977
1082
  "workflows": {
978
1083
  "create_poc": { "ready": true, "missingRead": [], "missingWrite": [] },
979
1084
  "setup_project": { "ready": true, "missingRead": [], "missingWrite": [] }
@@ -999,12 +1104,12 @@ The server also logs a readiness check at startup (visible in your terminal, or
999
1104
 
1000
1105
  The four most commonly encountered:
1001
1106
 
1002
- 1. **No API to trigger Appium test execution.** Tests launch from Appium clients (IDE, CI scripts). The server manages devices, apps, reservations, and results — it cannot start an Appium session itself. (Interactive [mobile inspection sessions](docs/tools.md#mobile-inspection-sessions) and [browser inspection sessions](docs/tools.md#web-inspection-sessions) are the exception: live WebDriver sessions for element discovery on Android, iOS, and desktop browsers.)
1107
+ 1. **No API to trigger Appium or Selenium test execution.** Those tests launch from Appium clients (IDE, CI scripts); the server manages devices, apps, reservations, and results. Espresso, XCUITest and Maestro suites are different: the platform runs them itself via `execute_test_run` / `get_test_run_status` / `cancel_test_run` / `get_test_run_command` (Maestro is Android-only). Espresso and Maestro runs are verified; XCUITest uses the same API but is not yet verified end to end. Interactive [mobile inspection sessions](docs/tools.md#mobile-inspection-sessions) and [browser inspection sessions](docs/tools.md#web-inspection-sessions) are also live WebDriver sessions for element discovery on Android, iOS, and desktop browsers.
1003
1108
  2. **Inspection sessions support Android and iOS** on both the legacy Appium Grid (JWP) and Appium Server (W3C/OSS) projects. iOS caveats: no clear-app-data, no clipboard on Grid devices, and back navigation uses the nav-bar button (iOS has no Back button).
1004
1109
  3. **No user disable/lock endpoint.** Removing access for a user provisioned solely for a POC means deleting the account — which is why `close_poc` requires per-user confirmation.
1005
- 4. **Reporter API restrictions for project-level keys** — server-side sort and report deletion require Cloud Admin access. Tools compensate automatically (client-side sorting, clear pre-flight errors), at the cost of slower full-scan queries under project-level keys.
1110
+ 4. **Deleting test reports with a project-level key depends on a per-project setting** (`allowUsersDeleteTests`). Cloud Admin can always delete. When the setting is off, the tools refuse up front and explain how to get it enabled.
1006
1111
 
1007
- See [docs/limitations.md](docs/limitations.md) for the full list of 15.
1112
+ See [docs/limitations.md](docs/limitations.md) for the full list.
1008
1113
 
1009
1114
  ---
1010
1115
 
@@ -0,0 +1,46 @@
1
+ import type { MyAccountInfo } from '../types/digital-ai.js';
2
+ import { type AccessInfo } from '../utils/access-level.js';
3
+ /** Resolve (and cache) the access level for an explicit url + key. */
4
+ export declare function resolveAccessInfo(url: string, key: string): Promise<AccessInfo>;
5
+ /**
6
+ * Seed the cache for the ACTIVE credential from a my-account-info body the caller already fetched through
7
+ * the main client. Only a recognised role is primed; an unrecognised one is left for the resolver's probe.
8
+ */
9
+ export declare function primeAccessInfoFromAccount(me: MyAccountInfo): void;
10
+ /** Access level of the ACTIVE profile (honours switch_environment). */
11
+ export declare function getAccessInfo(): Promise<AccessInfo>;
12
+ /**
13
+ * Cache-only view of the active profile's access level — NEVER touches the network. For callers that must
14
+ * stay offline (command generators the user reaches for precisely when the server is unreachable).
15
+ * Returns undefined when nothing is known yet.
16
+ */
17
+ export declare function peekActiveAccessInfo(): AccessInfo | undefined;
18
+ /** Access level of a NAMED profile, using that profile's own credentials. */
19
+ export declare function resolveProfileAccess(profileName: string): Promise<AccessInfo>;
20
+ /**
21
+ * Gate for reporter DELETES. Cloud Admin (and an undeterminable level) always proceeds.
22
+ *
23
+ * Project-level roles are refused up front ONLY when the project the delete will actually hit has a
24
+ * readable `allowUsersDeleteTests === false` — verified live: with it false, Cloud Admin deleted fine
25
+ * while ProjectAdmin and User both got 403 "You have no permission to delete tests". A ProjectAdmin key
26
+ * can read its own project's flag via GET /reporter/api/projects; a User key sees an empty list.
27
+ *
28
+ * Which project is "the one the delete will hit":
29
+ * - `projectName` given → exactly that project. If it is not in the visible list we do NOT substitute
30
+ * another one; the call proceeds and the platform decides.
31
+ * - no `projectName` → the credential's own project (reporter calls default to that scope; a numeric
32
+ * projectId is ignored by the reporter delete endpoint, so it cannot redirect the delete elsewhere).
33
+ * If that is not visible and exactly one project is, that one.
34
+ * When the flag is true or unreadable the call proceeds (whether a project role succeeds once the flag is
35
+ * true is NOT verified). Returns the refusal text, or null to proceed.
36
+ */
37
+ export declare function checkDeleteAllowed(projectName?: string): Promise<string | null>;
38
+ /**
39
+ * True unless the platform has already refused server-side sort for the ACTIVE credential. Sort is
40
+ * attempted for every role and falls back automatically (see listTests) — role is deliberately NOT
41
+ * consulted, because which roles may sort is a platform behavior that has changed before.
42
+ */
43
+ export declare function serverSortAvailable(): boolean;
44
+ /** Record that the platform refused sort for the ACTIVE credential. */
45
+ export declare function markActiveServerSortRefused(): void;
46
+ //# sourceMappingURL=access-level.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"access-level.d.ts","sourceRoot":"","sources":["../../src/api/access-level.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAWL,KAAK,UAAU,EAEhB,MAAM,0BAA0B,CAAC;AAqBlC,sEAAsE;AACtE,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAWrF;AAkCD;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,EAAE,EAAE,aAAa,GAAG,IAAI,CASlE;AAED,uEAAuE;AACvE,wBAAsB,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC,CAOzD;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,IAAI,UAAU,GAAG,SAAS,CAI7D;AAED,6EAA6E;AAC7E,wBAAsB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAInF;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,kBAAkB,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CA2BrF;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AAED,uEAAuE;AACvE,wBAAgB,2BAA2B,IAAI,IAAI,CAElD"}