digital-ai-testing-mcp 1.2.7 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +161 -56
- package/dist/api/access-level.d.ts +46 -0
- package/dist/api/access-level.d.ts.map +1 -0
- package/dist/api/access-level.js +183 -0
- package/dist/api/access-level.js.map +1 -0
- package/dist/api/agents.js +1 -1
- package/dist/api/agents.js.map +1 -1
- package/dist/api/browser-inspection.d.ts.map +1 -1
- package/dist/api/browser-inspection.js +18 -17
- package/dist/api/browser-inspection.js.map +1 -1
- package/dist/api/client.d.ts +14 -2
- package/dist/api/client.d.ts.map +1 -1
- package/dist/api/client.js +60 -43
- package/dist/api/client.js.map +1 -1
- package/dist/api/device-groups.js +1 -1
- package/dist/api/device-groups.js.map +1 -1
- package/dist/api/license.js +1 -1
- package/dist/api/license.js.map +1 -1
- package/dist/api/nv-servers.js +1 -1
- package/dist/api/nv-servers.js.map +1 -1
- package/dist/api/projects.d.ts +16 -3
- package/dist/api/projects.d.ts.map +1 -1
- package/dist/api/projects.js +60 -18
- package/dist/api/projects.js.map +1 -1
- package/dist/api/regions.js +1 -1
- package/dist/api/regions.js.map +1 -1
- package/dist/api/reporter-projects.d.ts.map +1 -1
- package/dist/api/reporter-projects.js +2 -1
- package/dist/api/reporter-projects.js.map +1 -1
- package/dist/api/reporting.d.ts +91 -0
- package/dist/api/reporting.d.ts.map +1 -1
- package/dist/api/reporting.js +220 -22
- package/dist/api/reporting.js.map +1 -1
- package/dist/api/sessions.js +1 -1
- package/dist/api/sessions.js.map +1 -1
- package/dist/api/test-orchestrator.d.ts +59 -0
- package/dist/api/test-orchestrator.d.ts.map +1 -0
- package/dist/api/test-orchestrator.js +126 -0
- package/dist/api/test-orchestrator.js.map +1 -0
- package/dist/api/test-runs.d.ts +80 -0
- package/dist/api/test-runs.d.ts.map +1 -0
- package/dist/api/test-runs.js +209 -0
- package/dist/api/test-runs.js.map +1 -0
- package/dist/api/test-views.d.ts.map +1 -1
- package/dist/api/test-views.js +5 -1
- package/dist/api/test-views.js.map +1 -1
- package/dist/api/transactions.d.ts +6 -0
- package/dist/api/transactions.d.ts.map +1 -1
- package/dist/api/transactions.js +19 -3
- package/dist/api/transactions.js.map +1 -1
- package/dist/api/usage-reports.d.ts.map +1 -1
- package/dist/api/usage-reports.js +3 -3
- package/dist/api/usage-reports.js.map +1 -1
- package/dist/api/webdriver.d.ts +39 -1
- package/dist/api/webdriver.d.ts.map +1 -1
- package/dist/api/webdriver.js +177 -71
- package/dist/api/webdriver.js.map +1 -1
- package/dist/index.js +82 -33
- package/dist/index.js.map +1 -1
- package/dist/tools/agent-tools.js +2 -2
- package/dist/tools/agent-tools.js.map +1 -1
- package/dist/tools/application-tools.d.ts.map +1 -1
- package/dist/tools/application-tools.js +17 -10
- package/dist/tools/application-tools.js.map +1 -1
- package/dist/tools/boilerplate-tools.d.ts +3 -0
- package/dist/tools/boilerplate-tools.d.ts.map +1 -1
- package/dist/tools/boilerplate-tools.js +302 -21
- package/dist/tools/boilerplate-tools.js.map +1 -1
- package/dist/tools/browser-tools.d.ts.map +1 -1
- package/dist/tools/browser-tools.js +8 -5
- package/dist/tools/browser-tools.js.map +1 -1
- package/dist/tools/debug-tools.d.ts.map +1 -1
- package/dist/tools/debug-tools.js +6 -3
- package/dist/tools/debug-tools.js.map +1 -1
- package/dist/tools/device-group-tools.js +5 -5
- package/dist/tools/device-group-tools.js.map +1 -1
- package/dist/tools/device-tools.d.ts.map +1 -1
- package/dist/tools/device-tools.js +5 -4
- package/dist/tools/device-tools.js.map +1 -1
- package/dist/tools/health-tools.d.ts.map +1 -1
- package/dist/tools/health-tools.js +6 -5
- package/dist/tools/health-tools.js.map +1 -1
- package/dist/tools/inspection-tools.d.ts.map +1 -1
- package/dist/tools/inspection-tools.js +122 -22
- package/dist/tools/inspection-tools.js.map +1 -1
- package/dist/tools/meta-tools.d.ts +4 -1
- package/dist/tools/meta-tools.d.ts.map +1 -1
- package/dist/tools/meta-tools.js +87 -58
- package/dist/tools/meta-tools.js.map +1 -1
- package/dist/tools/performance-tools.d.ts.map +1 -1
- package/dist/tools/performance-tools.js +2 -8
- package/dist/tools/performance-tools.js.map +1 -1
- package/dist/tools/project-tools.d.ts.map +1 -1
- package/dist/tools/project-tools.js +38 -19
- package/dist/tools/project-tools.js.map +1 -1
- package/dist/tools/provisioning-profile-tools.d.ts.map +1 -1
- package/dist/tools/provisioning-profile-tools.js +5 -4
- package/dist/tools/provisioning-profile-tools.js.map +1 -1
- package/dist/tools/remediation-tools.d.ts +7 -0
- package/dist/tools/remediation-tools.d.ts.map +1 -0
- package/dist/tools/remediation-tools.js +50 -0
- package/dist/tools/remediation-tools.js.map +1 -0
- package/dist/tools/reporting-tools.d.ts.map +1 -1
- package/dist/tools/reporting-tools.js +393 -147
- package/dist/tools/reporting-tools.js.map +1 -1
- package/dist/tools/repository-tools.d.ts.map +1 -1
- package/dist/tools/repository-tools.js +5 -4
- package/dist/tools/repository-tools.js.map +1 -1
- package/dist/tools/reservation-tools.js +2 -2
- package/dist/tools/reservation-tools.js.map +1 -1
- package/dist/tools/resources.js +2 -2
- package/dist/tools/resources.js.map +1 -1
- package/dist/tools/test-run-tools.d.ts +3 -0
- package/dist/tools/test-run-tools.d.ts.map +1 -0
- package/dist/tools/test-run-tools.js +284 -0
- package/dist/tools/test-run-tools.js.map +1 -0
- package/dist/tools/test-view-tools.d.ts.map +1 -1
- package/dist/tools/test-view-tools.js +11 -11
- package/dist/tools/test-view-tools.js.map +1 -1
- package/dist/tools/toolset-tools.d.ts +8 -0
- package/dist/tools/toolset-tools.d.ts.map +1 -0
- package/dist/tools/toolset-tools.js +40 -0
- package/dist/tools/toolset-tools.js.map +1 -0
- package/dist/tools/transaction-tools.d.ts.map +1 -1
- package/dist/tools/transaction-tools.js +17 -7
- package/dist/tools/transaction-tools.js.map +1 -1
- package/dist/tools/usage-report-tools.d.ts.map +1 -1
- package/dist/tools/usage-report-tools.js +43 -15
- package/dist/tools/usage-report-tools.js.map +1 -1
- package/dist/tools/user-tools.js +2 -2
- package/dist/tools/user-tools.js.map +1 -1
- package/dist/tools/web-inspection-tools.d.ts.map +1 -1
- package/dist/tools/web-inspection-tools.js +45 -33
- package/dist/tools/web-inspection-tools.js.map +1 -1
- package/dist/tools/workflow-tools.js +1 -1
- package/dist/types/digital-ai.d.ts +19 -1
- package/dist/types/digital-ai.d.ts.map +1 -1
- package/dist/utils/access-level.d.ts +39 -0
- package/dist/utils/access-level.d.ts.map +1 -0
- package/dist/utils/access-level.js +111 -0
- package/dist/utils/access-level.js.map +1 -0
- package/dist/utils/browser-name.d.ts +2 -0
- package/dist/utils/browser-name.d.ts.map +1 -0
- package/dist/utils/browser-name.js +19 -0
- package/dist/utils/browser-name.js.map +1 -0
- package/dist/utils/command-payload.d.ts +19 -0
- package/dist/utils/command-payload.d.ts.map +1 -0
- package/dist/utils/command-payload.js +13 -0
- package/dist/utils/command-payload.js.map +1 -0
- package/dist/utils/csv.d.ts +6 -0
- package/dist/utils/csv.d.ts.map +1 -1
- package/dist/utils/csv.js +28 -2
- package/dist/utils/csv.js.map +1 -1
- package/dist/utils/download-command.d.ts.map +1 -1
- package/dist/utils/download-command.js +5 -4
- package/dist/utils/download-command.js.map +1 -1
- package/dist/utils/locality.d.ts +7 -0
- package/dist/utils/locality.d.ts.map +1 -1
- package/dist/utils/locality.js +19 -0
- package/dist/utils/locality.js.map +1 -1
- package/dist/utils/maestro-flow.d.ts +38 -0
- package/dist/utils/maestro-flow.d.ts.map +1 -0
- package/dist/utils/maestro-flow.js +114 -0
- package/dist/utils/maestro-flow.js.map +1 -0
- package/dist/utils/metric-semantics.d.ts +22 -0
- package/dist/utils/metric-semantics.d.ts.map +1 -0
- package/dist/utils/metric-semantics.js +24 -0
- package/dist/utils/metric-semantics.js.map +1 -0
- package/dist/utils/pagination.d.ts +11 -0
- package/dist/utils/pagination.d.ts.map +1 -1
- package/dist/utils/pagination.js +16 -0
- package/dist/utils/pagination.js.map +1 -1
- package/dist/utils/path-guard.js +1 -1
- package/dist/utils/path-guard.js.map +1 -1
- package/dist/utils/profile-loader.d.ts +4 -2
- package/dist/utils/profile-loader.d.ts.map +1 -1
- package/dist/utils/profile-loader.js +5 -4
- package/dist/utils/profile-loader.js.map +1 -1
- package/dist/utils/remediation.d.ts +113 -0
- package/dist/utils/remediation.d.ts.map +1 -0
- package/dist/utils/remediation.js +392 -0
- package/dist/utils/remediation.js.map +1 -0
- package/dist/utils/report-ref.d.ts +23 -0
- package/dist/utils/report-ref.d.ts.map +1 -0
- package/dist/utils/report-ref.js +31 -0
- package/dist/utils/report-ref.js.map +1 -0
- package/dist/utils/response-formatter.d.ts +2 -1
- package/dist/utils/response-formatter.d.ts.map +1 -1
- package/dist/utils/response-formatter.js +32 -21
- package/dist/utils/response-formatter.js.map +1 -1
- package/dist/utils/test-orchestrator.d.ts +102 -0
- package/dist/utils/test-orchestrator.d.ts.map +1 -0
- package/dist/utils/test-orchestrator.js +236 -0
- package/dist/utils/test-orchestrator.js.map +1 -0
- package/dist/utils/test-status.d.ts +48 -0
- package/dist/utils/test-status.d.ts.map +1 -0
- package/dist/utils/test-status.js +92 -0
- package/dist/utils/test-status.js.map +1 -0
- package/dist/utils/tool-catalog.d.ts +6 -0
- package/dist/utils/tool-catalog.d.ts.map +1 -0
- package/dist/utils/tool-catalog.js +15 -0
- package/dist/utils/tool-catalog.js.map +1 -0
- package/dist/utils/toolsets.d.ts +79 -0
- package/dist/utils/toolsets.d.ts.map +1 -0
- package/dist/utils/toolsets.js +156 -0
- package/dist/utils/toolsets.js.map +1 -0
- package/dist/utils/upload-command.d.ts.map +1 -1
- package/dist/utils/upload-command.js +5 -4
- package/dist/utils/upload-command.js.map +1 -1
- package/dist/utils/usage-report-guard.d.ts +7 -0
- package/dist/utils/usage-report-guard.d.ts.map +1 -1
- package/dist/utils/usage-report-guard.js +18 -1
- package/dist/utils/usage-report-guard.js.map +1 -1
- package/dist/utils/usage-report-summary.d.ts +12 -0
- package/dist/utils/usage-report-summary.d.ts.map +1 -1
- package/dist/utils/usage-report-summary.js +25 -6
- package/dist/utils/usage-report-summary.js.map +1 -1
- package/package.json +40 -4
- package/resources/boilerplate/Android-Native/Java JUnit5/gradle +6 -1
- package/resources/boilerplate/Android-Native/Java JUnit5/gradle-oss +6 -1
- package/resources/boilerplate/Android-Native/java TestNG/AndroidNative-oss.java +2 -2
- package/resources/boilerplate/Android-Native/java TestNG/AndroidNative.java +2 -2
- package/resources/boilerplate/Android-Native/java TestNG/gradle +1 -1
- package/resources/boilerplate/Android-Native/java TestNG/gradle-oss +1 -1
- package/resources/boilerplate/iOS-Native/java-JUnit5/gradle +6 -1
- package/resources/boilerplate/iOS-Native/java-JUnit5/gradle-oss +6 -1
- package/resources/boilerplate/iOS-Native/java-TestNG/gradle +1 -1
- package/resources/boilerplate/iOS-Native/java-TestNG/gradle-oss +1 -1
- package/resources/boilerplate/iOS-Native/java-TestNG/iOSNative-oss.java +2 -2
- package/resources/boilerplate/iOS-Native/java-TestNG/iOSNative.java +2 -2
- package/resources/test-orchestrator/agent.json +19 -0
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
# Digital.ai Testing — MCP Server
|
|
8
8
|
|
|
9
|
-
An MCP (Model Context Protocol) server that connects AI assistants to a Digital.ai Testing device farm. The server exposes **
|
|
9
|
+
An MCP (Model Context Protocol) server that connects AI assistants to a Digital.ai Testing device farm. The server exposes **202 tools**, **2 resources**, and **7 prompts** covering 27 capability areas: device management, test execution (including Espresso, XCUITest and Maestro test runs), app lifecycle, reporting and root-cause analysis, analytics, performance, project administration, interactive mobile inspection, interactive browser inspection, and more.
|
|
10
10
|
|
|
11
11
|
---
|
|
12
12
|
|
|
@@ -53,6 +53,8 @@ Then ask: *"Show me the overall health of the device farm."*
|
|
|
53
53
|
- [Access Keys](#access-keys)
|
|
54
54
|
- [Installation](#installation)
|
|
55
55
|
- [Configuration](#configuration)
|
|
56
|
+
- [Toolsets](#toolsets-reducing-context-size)
|
|
57
|
+
- [Debug mode](#debug-mode-remediation-notes)
|
|
56
58
|
- [Connecting AI Clients](#connecting-ai-clients)
|
|
57
59
|
- [Claude Desktop](#claude-desktop)
|
|
58
60
|
- [Claude Code (VS Code)](#claude-code-vs-code)
|
|
@@ -66,11 +68,13 @@ Then ask: *"Show me the overall health of the device farm."*
|
|
|
66
68
|
- [Project Lifecycle](#project-lifecycle)
|
|
67
69
|
- [Boilerplate Generation](#boilerplate-generation)
|
|
68
70
|
- [Recommended agent guardrails](docs/recommended-agent-guardrails.md)
|
|
71
|
+
- [Performance Comparison Reports](#performance-comparison-reports)
|
|
69
72
|
- [Reference](#reference)
|
|
70
73
|
- [Response Format](#response-format)
|
|
71
74
|
- [Understanding maxResults](#understanding-maxresults)
|
|
72
75
|
- [List Filters & Sorting](#list-filters--sorting)
|
|
73
76
|
- [Test Reporting Schema](#test-reporting-schema)
|
|
77
|
+
- [Local File Transfer](#local-file-transfer-docker--remote-deployments-only)
|
|
74
78
|
- [Safety Guards](#safety-guards)
|
|
75
79
|
- [Development](#development)
|
|
76
80
|
- [Troubleshooting](#troubleshooting)
|
|
@@ -91,13 +95,15 @@ Then ask: *"Show me the overall health of the device farm."*
|
|
|
91
95
|
|
|
92
96
|
Your Digital.ai access key determines what the MCP server can do on your behalf. There are three access levels.
|
|
93
97
|
|
|
94
|
-
| Access level |
|
|
95
|
-
|
|
96
|
-
| **Cloud Admin** |
|
|
97
|
-
| **Project Admin** |
|
|
98
|
-
| **Project User** |
|
|
98
|
+
| Access level | Access |
|
|
99
|
+
|---|---|
|
|
100
|
+
| **Cloud Admin** | All tools: device management, user provisioning, project administration, infrastructure, performance data |
|
|
101
|
+
| **Project Admin** | Scoped to one project. Can manage device tags, view project admin settings, list users. v2 API tools (agents, regions, license data) return 403. |
|
|
102
|
+
| **Project User** | Scoped to one project. Read/test operations only. Cannot manage tags or access admin settings; deleting reports requires the project's `allowUsersDeleteTests` setting. |
|
|
99
103
|
|
|
100
|
-
|
|
104
|
+
> **The key format does not tell you the access level.** Older keys are long `eyJ…` tokens; since Digital.ai 24.11 newly generated keys use a shorter `aut_1_…` token, and that applies to Cloud Admins too. The MCP server therefore detects your access level from the API (`GET /api/v1/users/my-account-info` → `Admin` / `ProjectAdmin` / `User`) and never from the key's prefix. `get_server_info` and `list_environments` show what was detected for each profile.
|
|
105
|
+
|
|
106
|
+
When a Cloud Admin tool is called with a project-level role, the MCP returns a plain-language error explaining what happened — and, if a Cloud Admin profile is known, a ready-to-use `switch_environment(...)` command.
|
|
101
107
|
|
|
102
108
|
### Finding your key
|
|
103
109
|
|
|
@@ -112,16 +118,16 @@ The key shown is tied to the **project currently selected in the portal**. If yo
|
|
|
112
118
|
Configure named profiles in `.env` to switch contexts at runtime without editing files:
|
|
113
119
|
|
|
114
120
|
```
|
|
115
|
-
# Default connection (typically Cloud Admin credentials)
|
|
121
|
+
# Default connection (typically Cloud Admin credentials — either key format works)
|
|
116
122
|
DIGITAL_AI_BASE_URL=https://your-tenant.experitest.com
|
|
117
|
-
DIGITAL_AI_ACCESS_KEY
|
|
123
|
+
DIGITAL_AI_ACCESS_KEY=...your-cloud-admin-key...
|
|
118
124
|
|
|
119
125
|
# Project-scoped profiles
|
|
120
126
|
DAI_PROFILE_QA_URL=https://your-tenant.experitest.com
|
|
121
|
-
DAI_PROFILE_QA_KEY
|
|
127
|
+
DAI_PROFILE_QA_KEY=...your-qa-key...
|
|
122
128
|
|
|
123
129
|
DAI_PROFILE_STAGING_URL=https://your-tenant.experitest.com
|
|
124
|
-
DAI_PROFILE_STAGING_KEY
|
|
130
|
+
DAI_PROFILE_STAGING_KEY=...your-staging-key...
|
|
125
131
|
```
|
|
126
132
|
|
|
127
133
|
Then ask your AI:
|
|
@@ -225,9 +231,78 @@ docker build -t digital-ai-testing-mcp:latest .
|
|
|
225
231
|
| `MCP_DEPLOYMENT_MODE` | Optional | `docker` | `docker` (default) or `local` — set to `local` when running via the npm package ([Option A](#option-a--install-via-npm-recommended)) so tool descriptions and upload/download behavior reflect that the server's filesystem is your own machine, not a container's |
|
|
226
232
|
| `REQUEST_TIMEOUT_MS` | Optional | `30000` | API request timeout in milliseconds |
|
|
227
233
|
| `UPLOAD_TIMEOUT_MS` | Optional | `120000` | File upload timeout in milliseconds |
|
|
234
|
+
| `MCP_TOOLSETS` | Optional | all | Comma-separated toolsets to load in full: `devices`, `apps`, `reporting`, `performance`, `inspection`, `authoring`, `browsers`, `admin` (or `all`, or `core` for only the always-loaded core). See [Toolsets](#toolsets-reducing-context-size) |
|
|
235
|
+
| `MCP_DEBUG_MODE` | Optional | `false` | `true` records remediation notes and a tool-call event log to help improve this MCP. See [Debug mode](#debug-mode-remediation-notes) |
|
|
236
|
+
| `MCP_REMEDIATION_DIR` | Optional | — | Debug mode only: folder to write remediation files to (overrides project detection) |
|
|
237
|
+
| `AXE_DEVTOOLS_API_KEY` | Optional | — | Deque Axe DevTools Mobile key used by `includeAxeScan` boilerplate |
|
|
238
|
+
| `TEST_ORCHESTRATOR_JAR_URL` / `TEST_ORCHESTRATOR_JAR_SHA256` | Optional | pinned | Override the Test Orchestrator agent download location; both are required |
|
|
239
|
+
| `ACCESS_PROBE_TIMEOUT_MS` | Optional | `8000` | Timeout for the access-level detection call |
|
|
228
240
|
|
|
229
241
|
Additional `DAI_PROFILE_{NAME}_URL` / `DAI_PROFILE_{NAME}_KEY` pairs configure named profiles for multi-project or multi-environment use. See [Access Keys](#access-keys) and `.env.example` for examples.
|
|
230
242
|
|
|
243
|
+
### Toolsets (reducing context size)
|
|
244
|
+
|
|
245
|
+
All 202 tool definitions add up to roughly 68K tokens of context. If your AI client is short on context — or runs several MCP servers — set `MCP_TOOLSETS` to the areas you use, e.g. `MCP_TOOLSETS=reporting` (about 25K tokens) or `MCP_TOOLSETS=inspection,authoring` (about 32K).
|
|
246
|
+
|
|
247
|
+
Nothing becomes unavailable. Tools outside the chosen toolsets are still listed, with a one-line description. When the AI calls one, the server loads that whole toolset and replies with the tool's full description and parameter guidance, then the AI calls it again — so the detailed usage guidance is always seen before the tool runs. The AI can also load toolsets up front with `enable_toolset`. Leave `MCP_TOOLSETS` unset (the default) to load everything in full, exactly as before.
|
|
248
|
+
|
|
249
|
+
| Toolset | Covers |
|
|
250
|
+
|---|---|
|
|
251
|
+
| `devices` | Devices, device groups, reservations, health, agents, regions |
|
|
252
|
+
| `apps` | Application repository, file repository, provisioning profiles |
|
|
253
|
+
| `reporting` | Test reports, analytics, root-cause analysis, test views, coverage |
|
|
254
|
+
| `performance` | Performance transactions and comparisons, NV servers |
|
|
255
|
+
| `inspection` | Live device and browser inspection, Android Auto / CarPlay, remote debug |
|
|
256
|
+
| `authoring` | Test boilerplate, Espresso / XCUITest / Maestro test runs |
|
|
257
|
+
| `browsers` | Browser listing and Selenium sessions |
|
|
258
|
+
| `admin` | Users, projects, backups, usage reports, POC and project workflows |
|
|
259
|
+
|
|
260
|
+
Always loaded in full: environment and connection tools (`get_server_info`, `switch_environment`, …), `enable_toolset`, and `validate_test_script`.
|
|
261
|
+
|
|
262
|
+
### Debug mode (remediation notes)
|
|
263
|
+
|
|
264
|
+
Set `MCP_DEBUG_MODE=true` while evaluating or developing the MCP to collect feedback for improving it. With the default `false`, nothing changes.
|
|
265
|
+
|
|
266
|
+
When it is on:
|
|
267
|
+
|
|
268
|
+
- **The AI records notes.** The server asks the AI client to call `record_remediation_note` (a tool that exists only in debug mode) whenever it:
|
|
269
|
+
- hits an error;
|
|
270
|
+
- wastes calls because guidance was unclear;
|
|
271
|
+
- is corrected or stopped by you;
|
|
272
|
+
- gives up;
|
|
273
|
+
- finds a better path than the one recommended;
|
|
274
|
+
- spots a way to make things cleaner or cheaper.
|
|
275
|
+
|
|
276
|
+
It records each note once the situation is resolved or abandoned, not in the middle of your task.
|
|
277
|
+
- **The server logs what it sees.** Every tool call is logged with its duration, response size and outcome (ok, error or guard), with flags for retries and repeated calls. This is objective data the AI can't under-report.
|
|
278
|
+
- **Errors get a reminder.** The first time a tool returns an error or a guard, the response ends with a one-line reminder to record a note.
|
|
279
|
+
|
|
280
|
+
Where the files go: a `remediation/` folder in the root of **the project the AI is working in**, so each note sits next to the app and tests it's about.
|
|
281
|
+
|
|
282
|
+
| Install | Notes | Event log |
|
|
283
|
+
|---|---|---|
|
|
284
|
+
| npm (`MCP_DEPLOYMENT_MODE=local`) | The server writes `<project>/remediation/<yyyymmddhhmmss>-<id>.md` | The server writes `<project>/remediation/<yyyymmddhhmmss>-<id>.events.jsonl` |
|
|
285
|
+
| Docker | Returned to the AI, which saves them to `remediation/` in its project if it can write files (Claude Code can; Claude Desktop needs a filesystem tool) | Kept in server memory; a summary is included in every note |
|
|
286
|
+
|
|
287
|
+
With the npm install, the server finds the project in this order, and records the step it used as `location:` in each file:
|
|
288
|
+
1. `MCP_REMEDIATION_DIR`, if set (files go directly into that folder);
|
|
289
|
+
2. the workspace folder the AI client reports;
|
|
290
|
+
3. the folder the server was started in, if it looks like a project;
|
|
291
|
+
4. `~/remediation`.
|
|
292
|
+
|
|
293
|
+
The `remediation/` folder contains its own `.gitignore` (`*`), so it is never committed to the project it lives in. There is one file per session; the `<id>` suffix keeps parallel sessions apart, and each file records the machine and user, so files collected from several machines can be merged. Keys, tokens, emails and signed-URL parameters are redacted before anything is written or returned.
|
|
294
|
+
|
|
295
|
+
Debug mode costs extra tokens (the instructions and the notes). Leave it off for normal use, and don't use debug sessions as token baselines.
|
|
296
|
+
|
|
297
|
+
To process a batch, collect the `remediation/` folders you want to analyse, from one machine or several, and run `npm run remediation:digest -- --dir <folder> --dir <folder>` in a clone of this repository (the npm package doesn't include it). With no `--dir`, it reads `$MCP_REMEDIATION_DIR`, or else `./remediation` and `~/remediation`. It condenses every unprocessed file into one summary, so the development agent reads a few KB instead of every raw note:
|
|
298
|
+
- notes by category, and issues that recur across sessions;
|
|
299
|
+
- per-tool errors, guards, retries, response sizes and latency;
|
|
300
|
+
- every note in a table.
|
|
301
|
+
|
|
302
|
+
Useful options:
|
|
303
|
+
- `--out <file>` writes the summary to a file; `--since YYYY-MM-DD` limits it to recent sessions; `--json` gives machine-readable output.
|
|
304
|
+
- `--archive <session…>` or `--archive-all` moves handled files into `processed/<date>/` inside the folder they came from. Nothing is deleted.
|
|
305
|
+
|
|
231
306
|
---
|
|
232
307
|
|
|
233
308
|
## Connecting AI Clients
|
|
@@ -418,7 +493,7 @@ Once connected, talk to the server in plain language — no tool names needed:
|
|
|
418
493
|
- *"Find an available Android phone, then generate a Java JUnit5 test boilerplate"*
|
|
419
494
|
- *"Show the pass/fail breakdown by OS — are Android and iOS failing at different rates?"*
|
|
420
495
|
- *"Did the latest release introduce a CPU regression? Compare version 10553 vs 10554 on Android."*
|
|
421
|
-
- *"Set up a new POC for Acme Corp — 6 devices in US2, 3 iOS and 3 Android,
|
|
496
|
+
- *"Set up a new POC for Acme Corp — 6 devices in US2, 3 iOS and 3 Android, for the next 4 weeks"*
|
|
422
497
|
|
|
423
498
|
**[→ docs/examples.md](docs/examples.md)** has 200+ prompts organized by scenario: farm health, device groups, regression sign-off, test analytics, performance, usage reports, interactive inspection, POC lifecycle, and more.
|
|
424
499
|
|
|
@@ -426,7 +501,7 @@ Once connected, talk to the server in plain language — no tool names needed:
|
|
|
426
501
|
|
|
427
502
|
## Capabilities
|
|
428
503
|
|
|
429
|
-
|
|
504
|
+
202 tools across 27 capability domains. The complete per-tool reference — descriptions, filters, auth requirements, and usage notes — lives in **[docs/tools.md](docs/tools.md)**.
|
|
430
505
|
|
|
431
506
|
| Domain | Tools | Highlights |
|
|
432
507
|
|---|---|---|
|
|
@@ -442,21 +517,22 @@ Once connected, talk to the server in plain language — no tool names needed:
|
|
|
442
517
|
| [Backup](docs/tools.md#backup) | 1 | Trigger a system backup |
|
|
443
518
|
| [Health & Diagnostics](docs/tools.md#health--diagnostics) | 11 | Farm health, readiness checks, license utilization |
|
|
444
519
|
| [Coverage Analytics](docs/tools.md#coverage-analytics) | 2 | Tested-vs-available device gap analysis |
|
|
445
|
-
| [Reporting](docs/tools.md#reporting) |
|
|
520
|
+
| [Reporting](docs/tools.md#reporting) | 23 | Search/filter test reports, root-cause analysis, failure summarization, stability, trends, logs, video download (byte ranges), share links, attachment download command, cleanup |
|
|
521
|
+
| [Test Runs](docs/tools.md#test-runs-espresso--xcuitest--maestro) | 5 | Run Espresso, XCUITest and Maestro suites on the platform; status, cancel, local command; generate Maestro flows |
|
|
446
522
|
| [Test Views](docs/tools.md#test-views) | 7 | Dashboard view groups |
|
|
447
|
-
| [Transactions & Performance](docs/tools.md#transactions--performance) | 4 | CPU/memory/battery/Speed Index analytics
|
|
523
|
+
| [Transactions & Performance](docs/tools.md#transactions--performance) | 4 | CPU/memory/battery/Speed Index analytics |
|
|
448
524
|
| [Agents](docs/tools.md#agents) | 2 | Host machine status *(Cloud Admin)* |
|
|
449
525
|
| [Regions](docs/tools.md#regions) | 2 | Region status and infrastructure topology *(Cloud Admin)* |
|
|
450
526
|
| [NV Servers](docs/tools.md#nv-servers) | 2 | Network Virtualization servers *(Cloud Admin)* |
|
|
451
527
|
| [Environment Management](docs/tools.md#environment-management) | 2 | Named connection profiles, runtime switching |
|
|
452
528
|
| [Workflows](docs/tools.md#workflows) | 6 | POC and project lifecycle orchestration |
|
|
453
|
-
| [Boilerplate Generation](docs/tools.md#boilerplate-generation) |
|
|
529
|
+
| [Boilerplate Generation](docs/tools.md#boilerplate-generation) | 4 | Ready-to-run Appium and Selenium test scripts in 4 languages, Test Orchestrator agent install; validate scripts before delivery |
|
|
454
530
|
| [Remote Debug](docs/tools.md#remote-debug) | 1 | Connect a cloud device as a local ADB device |
|
|
455
|
-
| [Mobile Inspection Sessions](docs/tools.md#mobile-inspection-sessions) |
|
|
531
|
+
| [Mobile Inspection Sessions](docs/tools.md#mobile-inspection-sessions) | 24 | AI-driven live device interaction — screenshots, element discovery, full gesture set, keys, app/device control, Android Auto / CarPlay projection, mock biometric authentication |
|
|
456
532
|
| [Web Inspection Sessions](docs/tools.md#web-inspection-sessions) | 8 | AI-driven live browser interaction — navigate, Shadow DOM element discovery, CSS selector verification, browser-neutral Selenium script generation |
|
|
457
|
-
| [Performance Comparison](docs/tools.md#performance-comparison) | 4 | Two-set Speed Index comparison with confound detection, MAD outlier exclusion, and fresh-sample generation
|
|
458
|
-
| [Usage Reports](docs/tools.md#usage-reports) | 3 | Platform usage-report CSV export and in-memory aggregation — device reservations, users/devices/browser usage, license usage *(Cloud Admin)* |
|
|
459
|
-
| [Resources & Prompts](docs/tools.md#resources--prompts) | — | 2 ambient resources,
|
|
533
|
+
| [Performance Comparison](docs/tools.md#performance-comparison) | 4 | Two-set Speed Index comparison with confound detection, MAD outlier exclusion, and fresh-sample generation |
|
|
534
|
+
| [Usage Reports](docs/tools.md#usage-reports) | 3 | Platform usage-report CSV export and in-memory aggregation — device reservations, users/devices/browser usage, license usage; multi-section CSVs with a User Tag column, about 2 years of retention; large unfiltered exports run only after you explicitly agree (`confirmLargeExport`) *(Cloud Admin)* |
|
|
535
|
+
| [Resources & Prompts](docs/tools.md#resources--prompts) | — | 2 ambient resources, 7 guided prompts |
|
|
460
536
|
|
|
461
537
|
---
|
|
462
538
|
|
|
@@ -479,7 +555,7 @@ Collects all parameters upfront, presents a confirmation summary to the operator
|
|
|
479
555
|
7. Remove devices from the Default group — done only after the project exists, so a project-creation failure never strands devices
|
|
480
556
|
8. Locate the Default project
|
|
481
557
|
9. Create users, assign to the POC project, remove from Default, and tag each account with the POC tag. Cloud Admin access is never granted through this workflow.
|
|
482
|
-
10. Assign ExperiBank (or
|
|
558
|
+
10. Assign ExperiBank (or the app identified by `androidPackageName` / `iosBundleId`) to the POC project — if a binary isn't in the repository, the summary reports it as a manual upload and the workflow continues
|
|
483
559
|
|
|
484
560
|
If any step fails, the workflow stops, reports which steps completed with all created resource IDs, and offers resume (re-invoke — existing resources are detected and reused) or unwind (`delete_poc`) paths.
|
|
485
561
|
|
|
@@ -490,7 +566,7 @@ If any step fails, the workflow stops, reports which steps completed with all cr
|
|
|
490
566
|
| `customerName` | e.g. `"Acme Corp"` → derives project `"Acme Corp POC"` and tag `"acmecorppoc"` |
|
|
491
567
|
| `region` | Testing region, e.g. `"US2"`, `"EU"`, `"SG"` |
|
|
492
568
|
| `salesforceUrl` | Salesforce Opportunity URL — recorded in project notes |
|
|
493
|
-
| `endDate` | Accepts ISO (`"
|
|
569
|
+
| `endDate` | Accepts ISO (`"YYYY-MM-DD"`), relative offsets (`"+14d"`, `"+2w"`), or natural language (`"in 2 weeks"`) |
|
|
494
570
|
| `users` | Array of `{email, firstName, lastName, role}` — role must be `"User"` or `"ProjectAdmin"` |
|
|
495
571
|
|
|
496
572
|
**Optional parameters:**
|
|
@@ -502,7 +578,8 @@ If any step fails, the workflow stops, reports which steps completed with all cr
|
|
|
502
578
|
| `androidCount` | `floor(deviceCount / 2)` |
|
|
503
579
|
| `automationType` | `"appium-server"` (alternative: `"appium-grid"`) |
|
|
504
580
|
| `appName` | `"ExperiBank"` |
|
|
505
|
-
| `
|
|
581
|
+
| `androidPackageName` | `"com.experitest.ExperiBank"` |
|
|
582
|
+
| `iosBundleId` | `"com.experitest.ExperiBank"` |
|
|
506
583
|
|
|
507
584
|
Steps 1 and 6 check for an existing group/project by name before creating — safe to re-run after a partial failure.
|
|
508
585
|
|
|
@@ -544,7 +621,7 @@ At the start, the workflow asks whether to create a **simple** project record on
|
|
|
544
621
|
|
|
545
622
|
Full setup steps:
|
|
546
623
|
|
|
547
|
-
1. Create (or reuse) the project with the specified name and automation type; record
|
|
624
|
+
1. Create (or reuse) the project with the specified name and automation type; record `projectMemo` in project notes
|
|
548
625
|
2. Create (or reuse) a device group and link it to the project
|
|
549
626
|
3. Pre-flight check that the target app exists in the repository
|
|
550
627
|
4. Select available devices matching the target OS and region — presented to the operator for confirmation
|
|
@@ -571,8 +648,10 @@ Full setup steps:
|
|
|
571
648
|
| `automationType` | `"appium-server"` | `"appium-server"` or `"appium-grid"` |
|
|
572
649
|
| `isolateDevices` | `false` | Remove devices from all other groups before assignment |
|
|
573
650
|
| `users` | `[]` | Array of `{email, firstName, lastName, role}` |
|
|
574
|
-
| `appName` |
|
|
575
|
-
| `
|
|
651
|
+
| `appName` | `"ExperiBank"` | App to assign to the project |
|
|
652
|
+
| `androidPackageName` | `"com.experitest.ExperiBank"` | Android package used to find the app |
|
|
653
|
+
| `iosBundleId` | `"com.experitest.ExperiBank"` | iOS bundle ID used to find the app |
|
|
654
|
+
| `projectMemo` | _(none)_ | Notes to record on the project |
|
|
576
655
|
|
|
577
656
|
---
|
|
578
657
|
|
|
@@ -620,6 +699,20 @@ The server supports two test-creation modes for both mobile and web, and the age
|
|
|
620
699
|
|
|
621
700
|
Generates a complete, pre-configured Appium test script. The Digital.ai server URL and access key are pre-filled from the active connection profile — switch profiles with `switch_environment` first to generate scripts carrying a project-scoped key instead of your admin key.
|
|
622
701
|
|
|
702
|
+
#### Test Orchestrator (default for Java on Appium Server)
|
|
703
|
+
|
|
704
|
+
For **Java (JUnit 5 / TestNG) on an Appium Server project**, the generated project is wired for the **Digital.ai Test Orchestrator** (the Smart Agent): failed tests are retried automatically, each attempt is reported separately, the real pass/fail is synced to Reporter, and runs are grouped by Build ID — with no change to the test code. Appium Grid, Python and NodeJS projects get the classic boilerplate. Pass `orchestration: "off"` to get the classic boilerplate for Java too.
|
|
705
|
+
|
|
706
|
+
| Step | What happens |
|
|
707
|
+
|---|---|
|
|
708
|
+
| 1. `get_test_boilerplate` | Adds conditional agent wiring to `build.gradle` / `pom.xml`, an `orchestrator/config.template.yml` (retries default to 2; 0 for placeholder or performance tests), plus `orchestrator/.gitignore` and `lib/.gitignore` so the rendered key and the agent JAR can never be committed. No root `.gitignore` is written (it would overwrite yours). |
|
|
709
|
+
| 2. `install_test_orchestrator_agent(projectDir)` | Downloads the agent on demand, verifies its SHA-256 and places it at `lib/smart-agent.jar`. Under Docker it returns a download command (bash and PowerShell) that does the same and deletes the file on a checksum mismatch. If the download fails, the response gives the URL, the checksum and the target path for a manual install. |
|
|
710
|
+
| 3. Set `DIGITAL_AI_ACCESS_KEY` and run `gradle test` / `mvn test` | The agent attaches only when the JAR is present **and** the key is set; the key is rendered into the git-ignored `orchestrator/rendered/` at build time, never into a tracked file, and Java 17 is required only in that case. Otherwise the project builds and runs exactly like the classic boilerplate (Gradle prints why the agent was not attached). Set `BUILD_ID` to group a CI build's results. |
|
|
711
|
+
|
|
712
|
+
Requirements: a **Java 17 or 21** JDK with a current truststore, JUnit 5 or TestNG 7+, Appium java-client 8–10. See [Limitations](docs/limitations.md#17-test-orchestrator-known-issues) for known issues.
|
|
713
|
+
|
|
714
|
+
The agent JAR is **not** shipped with this server. By default it is downloaded from the [Digital.ai Test Orchestrator sample repository](https://github.com/raheekhandigitalai/Digital.ai-Testing-Test-Orchestrator), pinned to a fixed commit and verified by SHA-256 (see `resources/test-orchestrator/agent.json`). To use a different location — for example an internal mirror — set `TEST_ORCHESTRATOR_JAR_URL` **and** `TEST_ORCHESTRATOR_JAR_SHA256` (both required; a URL without a checksum is refused).
|
|
715
|
+
|
|
623
716
|
| Parameter | Values | Default |
|
|
624
717
|
|---|---|---|
|
|
625
718
|
| `platform` | `android` \| `ios` | _(required)_ |
|
|
@@ -635,23 +728,25 @@ Generates a complete, pre-configured Appium test script. The Digital.ai server U
|
|
|
635
728
|
| `includePerformanceTransactions` | `true` \| `false` | `false` |
|
|
636
729
|
| `includeAxeScan` | `true` \| `false` | `false` |
|
|
637
730
|
| `confirmSelectorsVerified` | `true` \| `false` | _(optional)_ |
|
|
731
|
+
| `orchestration` | `auto` \| `on` \| `off` | `auto` |
|
|
732
|
+
| `orchestrationMaxRetries` | `0`–`5` | `2` (`0` for placeholder or performance tests) |
|
|
733
|
+
| `automotiveProjection` | Android Auto / CarPlay resolution | _(optional)_ |
|
|
638
734
|
| `outputFormat` | `json` \| `human` | `json` |
|
|
639
735
|
|
|
640
|
-
> **Inspection gate
|
|
736
|
+
> **Inspection gate:** when you target a real app (`appId`/`packageName`/`bundleIdentifier`), this tool returns **no code** and a structured `{ status: "blocked", reason: "no_verified_selectors" }` error UNLESS a live inspection session exists in the MCP process, or you set `confirmSelectorsVerified: true` (escape hatch for selectors already captured via rdb/UIAutomator, `open_mobile_studio`, or authoritative source). The built-in ExperiBank demo (no app identifiers) is never gated. This makes it structurally impossible to receive a placeholder scaffold and pass it off as a finished test.
|
|
641
737
|
|
|
642
738
|
**Recommended workflow** for building a new test:
|
|
643
739
|
|
|
644
740
|
1. `get_application_info` — confirm package name, launch activity, and app ID
|
|
645
|
-
2. `
|
|
646
|
-
3. `
|
|
647
|
-
4. `
|
|
648
|
-
5. `
|
|
649
|
-
6.
|
|
650
|
-
7. `release_device` — explicit cleanup
|
|
741
|
+
2. `find_available_device` — select a healthy device and read its `region`
|
|
742
|
+
3. `start_inspection_session` (or `open_mobile_studio`) — capture real element IDs
|
|
743
|
+
4. `get_test_boilerplate(appId, region, …)` — generates code while the inspection session is live, or with `confirmSelectorsVerified: true`
|
|
744
|
+
5. `validate_test_script` — before saving or presenting the test
|
|
745
|
+
6. Run the test, then `release_device` — explicit cleanup
|
|
651
746
|
|
|
652
747
|
**Performance transactions:** Pass `includePerformanceTransactions: true` to bracket the test body with `startPerformanceTransaction` / `endPerformanceTransaction` calls. The start arg is the NV network profile (defaults to `"Monitor"` — observe without throttling); the end arg is the transaction name. The platform records CPU, memory, battery, and Speed Index metrics; results appear in the reporter Transactions tab (~1 min after `endPerformanceTransaction`) and are queryable via `list_transactions`. Pre-requisite: an NV server must be ONLINE and tunnel-connected in the device region (`list_nv_servers`). Note: a throttling profile activates NV shaping immediately; `"Monitor"` is pass-through and carries no ANR risk.
|
|
653
748
|
|
|
654
|
-
**Accessibility scanning:** Pass `includeAxeScan: true` to inject a Deque Axe DevTools Mobile accessibility scan (`mobile: axeScan` executeScript call). Sets the required `appium:automationName` capability automatically (`AxeUiAutomator2` for Android, `AxeXCUITest` for iOS).
|
|
749
|
+
**Accessibility scanning:** Pass `includeAxeScan: true` to inject a Deque Axe DevTools Mobile accessibility scan (`mobile: axeScan` executeScript call). Sets the required `appium:automationName` capability automatically (`AxeUiAutomator2` for Android, `AxeXCUITest` for iOS). Also pins `appiumVersion` to 2.16.2 (the Axe driver is unavailable on Appium Server 3.3.0+; don't raise it). The key comes from `AXE_DEVTOOLS_API_KEY`; if that's unset, a placeholder is written.
|
|
655
750
|
|
|
656
751
|
Both flags can be combined — the Axe scan runs inside the performance transaction boundary.
|
|
657
752
|
|
|
@@ -661,7 +756,7 @@ Both flags can be combined — the Axe scan runs inside the performance transact
|
|
|
661
756
|
|
|
662
757
|
### `validate_test_script`
|
|
663
758
|
|
|
664
|
-
Delivery backstop
|
|
759
|
+
Delivery backstop for the case the `get_test_boilerplate` gate cannot catch — a test written by hand with guessed selectors. Pass the full `scriptContent` before presenting or saving any test; it scans for unreplaced `<…>` placeholder selectors, the deliberate scaffold fail-guard, placeholder/fabricated credentials, and known fabricated resource IDs. Returns `isError` with a `fail` verdict when any high-severity pattern is found, so a non-functional test cannot be delivered as finished. A `pass` is necessary but not sufficient — it confirms obvious placeholders are gone, not that selectors are real; a live inspection remains the authoritative source.
|
|
665
760
|
|
|
666
761
|
| Parameter | Values | Default |
|
|
667
762
|
|---|---|---|
|
|
@@ -676,7 +771,7 @@ Delivery backstop (v43) for the case the `get_test_boilerplate` gate cannot catc
|
|
|
676
771
|
| Language | Files |
|
|
677
772
|
|---|---|
|
|
678
773
|
| `java-junit5` / `java-testng` | `AndroidNative.java` or `iOSNative.java` + `build.gradle` + `pom.xml` |
|
|
679
|
-
| `nodejs` | `wdio.conf.js` + test file + setup shell steps |
|
|
774
|
+
| `nodejs` | `wdio.conf.js` + test file + setup shell steps (WebdriverIO 7.40 for Appium Grid projects, WebdriverIO 9 for Appium Server) |
|
|
680
775
|
| `python` | Test file + `requirements.txt` |
|
|
681
776
|
|
|
682
777
|
All boilerplate defaults to the ExperiBank demo app as the starting point. ExperiBank is available on most farm devices and can verify connectivity before switching to your own app.
|
|
@@ -689,15 +784,15 @@ Generates a complete, pre-configured Selenium test script for browser testing. T
|
|
|
689
784
|
|---|---|---|
|
|
690
785
|
| `language` | `java-junit5` \| `java-testng` \| `nodejs` \| `python` | _(required)_ |
|
|
691
786
|
| `url` | URL of the page being tested | _(optional)_ |
|
|
692
|
-
| `testName` | Any string | `"
|
|
693
|
-
| `targetBrowser` | `chrome` \| `firefox` \| `
|
|
787
|
+
| `testName` | Any string | `"WebTest"` |
|
|
788
|
+
| `targetBrowser` | `chrome` \| `firefox` \| `MicrosoftEdge` \| `safari` \| `opera` (names as `list_available_browsers` returns them; `edge` is accepted for `MicrosoftEdge`) | _(optional)_ |
|
|
694
789
|
| `shadowDomSupport` | `auto` \| `always` \| `never` | `auto` |
|
|
695
790
|
| `confirmSelectorsVerified` | `true` \| `false` | _(optional)_ |
|
|
696
791
|
| `outputFormat` | `json` \| `human` | `json` |
|
|
697
792
|
|
|
698
793
|
> **Inspection gate:** when you target a real URL (`url` provided), this tool returns **no code** and a structured `{ status: "blocked", reason: "no_verified_selectors" }` error UNLESS a live browser inspection session exists in the MCP process, or you set `confirmSelectorsVerified: true` (escape hatch for selectors captured via a prior `start_browser_inspection_session` session). This mirrors the mobile boilerplate gate.
|
|
699
794
|
|
|
700
|
-
**Browser-neutral by default:** when `targetBrowser` is omitted, the generated script reads the browser name at runtime from an environment variable (`BROWSER`). This means the same script runs on Chrome, Firefox, Edge, and
|
|
795
|
+
**Browser-neutral by default:** when `targetBrowser` is omitted, the generated script reads the browser name at runtime from an environment variable (`BROWSER`). This means the same script runs on Chrome, Firefox, Microsoft Edge, Safari and Opera without modification — just change the env var or CI matrix. Specify `targetBrowser` only when you need browser-specific options (e.g. Safari-only session setup).
|
|
701
796
|
|
|
702
797
|
**Shadow DOM support:** when `shadowDomSupport` is `auto` (the default) or `always`, the generated script includes a `shadowQuery(host, cssSelector)` helper that traverses into shadow roots using `executeScript`. Use this for apps built with Web Components, Lit, Polymer, or other shadow-DOM frameworks. `get_page_dom` detects shadow DOM automatically during inspection and reports `hasShadowDom: true` when present.
|
|
703
798
|
|
|
@@ -726,7 +821,7 @@ The headline metric is reported three ways every time — **trimmed mean, median
|
|
|
726
821
|
| `detect_performance_outliers` | Robust median/MAD outlier flagging on a single set; returns the kept set and recommended exclusions/re-runs. |
|
|
727
822
|
| `performance_transaction_control` | Phase 2: generate fresh samples inside an inspection session — `start` (with an NV profile) → run the verified flow → `end` (names the record). Records appear in the reporter ~1 min later. |
|
|
728
823
|
|
|
729
|
-
> **Why the confound check matters:** comparing transactions
|
|
824
|
+
> **Why the confound check matters:** comparing two transactions with Speed Index 1015 vs 1000 looks like a 1.5% version delta — but if they ran on *different device models, different OS versions, and different projects*, and one has no CPU/memory telemetry, `assess_comparison_confounds` returns **confounded** and names each cause, so the delta is never misread as a version regression.
|
|
730
825
|
|
|
731
826
|
The **`performance_comparison_report` prompt** orchestrates the full workflow end to end: define the axis → scrub confounds → negotiate sample size → **require explicit plan confirmation with a time estimate** → run the series with outlier-driven re-runs → report the delta with root-cause reasoning. True host/background interference is not directly observable; the report approximates device quietness (idle, healthy status, no concurrent reservation) and states that limit explicitly.
|
|
732
827
|
|
|
@@ -771,15 +866,17 @@ Use filters to narrow results rather than raising `maxResults`. When results are
|
|
|
771
866
|
| Tool | Available Filters | Available `sortBy` values |
|
|
772
867
|
|---|---|---|
|
|
773
868
|
| `list_users` | firstName, lastName, email, authenticationType, isCloudAdmin, tag | firstName, lastName, email, userName, authenticationType |
|
|
774
|
-
| `list_devices` | query (@-syntax), region, model | deviceName, deviceOs, osVersion, manufacturer, displayStatus, agentName, region |
|
|
869
|
+
| `list_devices` | query (@-syntax), os, region, model | deviceName, deviceOs, osVersion, manufacturer, displayStatus, agentName, region |
|
|
775
870
|
| `get_devices_in_group` | osType, status, category, excludeTags, requireTags | _(none)_ |
|
|
776
871
|
| `list_reservations` | username, project, deviceUid | reservationStart, reservationEnd, username, project, deviceUid |
|
|
777
872
|
| `list_projects` | name (partial match) | name, id |
|
|
778
|
-
| `list_project_users` | username (partial), role (exact) |
|
|
779
|
-
| `list_applications` | nameContains, osType, packageName, bundleIdentifier, fileType, isForSimulator | applicationName,
|
|
873
|
+
| `list_project_users` | username (partial), role (exact) | username, role |
|
|
874
|
+
| `list_applications` | nameContains, osType, packageName, bundleIdentifier, uniqueName, fileType, isForSimulator | applicationName, releaseVersion, buildVersion, createdAt (default), osType, fileType |
|
|
780
875
|
| `list_test_reports` | status, name, has_attachment, success; startDate/endDate (date range) | start_time |
|
|
781
876
|
|
|
782
|
-
All list tools accept `sortOrder: "asc" | "desc"` (default
|
|
877
|
+
All list tools accept `sortOrder: "asc" | "desc"` (default `"asc"`, except `list_applications`, which defaults to `"desc"`, newest first).
|
|
878
|
+
|
|
879
|
+
`list_test_reports` sorting is performed by the platform for every access level. If the platform ever refuses it for a credential, the server transparently falls back to a client-side scan (up to 5,000 records) so "newest first" results stay correct, just slower on large report sets.
|
|
783
880
|
|
|
784
881
|
### Test Reporting Schema
|
|
785
882
|
|
|
@@ -817,8 +914,8 @@ To diagnose a failure, list with `list_test_reports`, then call `get_test_report
|
|
|
817
914
|
|
|
818
915
|
| Identifier | Source | How to retrieve |
|
|
819
916
|
|---|---|---|
|
|
820
|
-
| `
|
|
821
|
-
| `
|
|
917
|
+
| `uuid` | List results, report URLs (`/reporter/video-report/<uuid>`) | `get_test_report(uuid: "...")` or `get_test_report(reportUrl: "...")` — preferred, globally unique |
|
|
918
|
+
| `test_id` | List results | `get_test_report(testId: N)` — unique only within a project; the same number can be a different test in another project |
|
|
822
919
|
| `report_api_id` | Returned by session-start tools | `get_test_by_report_id(reportApiId: "...")` — wait ~60 s after session close |
|
|
823
920
|
|
|
824
921
|
**Examples:**
|
|
@@ -857,11 +954,15 @@ list_test_reports
|
|
|
857
954
|
get_test_report(testId: 69)
|
|
858
955
|
→ cause, errorCategory, errorClassification, errorDetail (stack trace)
|
|
859
956
|
|
|
957
|
+
# 2b. Platform root-cause analysis (read-only)
|
|
958
|
+
get_root_cause_analysis(uuid: "…") # or reportUrl / testId
|
|
959
|
+
|
|
860
960
|
# 3. Full Appium log as text, when the structured fields aren't enough
|
|
861
961
|
get_test_log(uuid: "…", logType: "appium")
|
|
862
962
|
|
|
863
|
-
# 4. Pull the binary artifacts (session video .mp4, full ZIP) to YOUR machine
|
|
864
|
-
#
|
|
963
|
+
# 4. Pull the binary artifacts (session video .mp4, full ZIP) to YOUR machine.
|
|
964
|
+
# With the npm install, call download_test_video / download_test_attachments
|
|
965
|
+
# directly; under Docker/remote, generate a local command instead:
|
|
865
966
|
get_test_attachments_download_command(uuid: "…", localPlatform: "macos")
|
|
866
967
|
→ a curl command you run locally; the file never transits the container
|
|
867
968
|
```
|
|
@@ -880,6 +981,7 @@ For those deployments, every file-transfer tool has a **command-generator siblin
|
|
|
880
981
|
| Upload repo file | `upload_repository_file` | `get_repository_upload_command` |
|
|
881
982
|
| Upload profile | `upload_provisioning_profile` | `get_provisioning_profile_upload_command` |
|
|
882
983
|
| Download test attachments (.mp4, logs) | `download_test_attachments` | `get_test_attachments_download_command` |
|
|
984
|
+
| Download test video (.mp4, byte ranges) | `download_test_video` | `get_test_attachments_download_command` (`artifact: "video"`) |
|
|
883
985
|
| Download repo file | `download_repository_file` | `get_repository_file_download_command` |
|
|
884
986
|
| Download profile | `download_provisioning_profile` | `get_provisioning_profile_download_command` |
|
|
885
987
|
| Download app language files | `extract_app_language_files` | `get_app_language_files_download_command` |
|
|
@@ -899,7 +1001,7 @@ The first call describes exactly what will be deleted. The second call — with
|
|
|
899
1001
|
|
|
900
1002
|
Two further guards run before any request leaves the server:
|
|
901
1003
|
|
|
902
|
-
- **Auth pre-flight:** report-deletion tools (`delete_test_reports*`, `cleanup_inspection_sessions`) check the
|
|
1004
|
+
- **Auth pre-flight:** report-deletion tools (`delete_test_reports*`, `cleanup_inspection_sessions`) check the API-detected access level first. A project-level role is refused up front only when its project's `allowUsersDeleteTests` setting is false, with a message telling you to have a Cloud Admin enable it (or switch to a Cloud Admin profile) instead of an opaque 403.
|
|
903
1005
|
- **Upload path validation:** tools that read local files for upload refuse relative paths, path traversal, and credential-file names (`.env*`, SSH private keys) — a misdirected request cannot publish secrets to the cloud repository.
|
|
904
1006
|
|
|
905
1007
|
---
|
|
@@ -933,9 +1035,12 @@ npm run test:utils # utility functions
|
|
|
933
1035
|
npm run test:transactions # performance transactions
|
|
934
1036
|
npm run test:analytics # analytics tools
|
|
935
1037
|
npm run test:infrastructure # agents / regions / NV servers / sessions
|
|
1038
|
+
npm run test:live # live API-behavior probes (requires .env; not part of npm test)
|
|
1039
|
+
npm run test:fidelity # agent tool-choice evaluation (builds first)
|
|
1040
|
+
npm run remediation:digest # summarize debug-mode remediation notes
|
|
936
1041
|
```
|
|
937
1042
|
|
|
938
|
-
|
|
1043
|
+
See `package.json` for the full list of scripts.
|
|
939
1044
|
|
|
940
1045
|
---
|
|
941
1046
|
|
|
@@ -946,7 +1051,7 @@ Tests can also be run from VS Code: `Ctrl+Shift+P` → "Tasks: Run Task".
|
|
|
946
1051
|
| Symptom | Cause | Fix |
|
|
947
1052
|
|---|---|---|
|
|
948
1053
|
| `403 Forbidden` on admin tools (agents, regions, license) | Active profile is a project-level key — v2 endpoints require Cloud Admin access | Ask Agent: *"switch to my Cloud Admin profile"* — it will call `list_environments` and pick the right one |
|
|
949
|
-
| Report delete tools
|
|
1054
|
+
| Report delete tools refuse with "Reporter deletes are disabled for project …" | The project's `allowUsersDeleteTests` setting is false, so project-level roles can't delete reports | Ask a Cloud Admin to enable it for the project, or switch to a Cloud Admin profile |
|
|
950
1055
|
| `install_application` returns 400 on a device you can see | Device is reserved via an rdb (remote debug) session | Install **first**, then run `get_remote_debug_command` — not the other way around |
|
|
951
1056
|
| `install_application` returns 400 (no rdb involved) | App not assigned to a project containing the target device | Call `assign_app_to_project` first |
|
|
952
1057
|
| Repeatable `NoSuchElementException` while sibling tests pass | Device health, not test code — device stuck in a wrong state or offline-but-pooled | Run `get_device_health_summary`; scope the deviceQuery with `@region='<healthy-region>'` |
|
|
@@ -963,7 +1068,7 @@ On Windows this usually means the `--env-file` path was stored with backslashes
|
|
|
963
1068
|
Run this sequence:
|
|
964
1069
|
|
|
965
1070
|
```
|
|
966
|
-
1. get_server_info — confirm tool count (expect
|
|
1071
|
+
1. get_server_info — confirm tool count (expect 202; 203 with MCP_DEBUG_MODE=true) and active profile
|
|
967
1072
|
2. check_workflow_readiness — which dependency tools are present or missing
|
|
968
1073
|
3. check_connectivity — confirm the backend API is reachable
|
|
969
1074
|
```
|
|
@@ -973,7 +1078,7 @@ Run this sequence:
|
|
|
973
1078
|
```json
|
|
974
1079
|
{
|
|
975
1080
|
"allWorkflowsReady": true,
|
|
976
|
-
"registeredToolCount":
|
|
1081
|
+
"registeredToolCount": 202,
|
|
977
1082
|
"workflows": {
|
|
978
1083
|
"create_poc": { "ready": true, "missingRead": [], "missingWrite": [] },
|
|
979
1084
|
"setup_project": { "ready": true, "missingRead": [], "missingWrite": [] }
|
|
@@ -999,12 +1104,12 @@ The server also logs a readiness check at startup (visible in your terminal, or
|
|
|
999
1104
|
|
|
1000
1105
|
The four most commonly encountered:
|
|
1001
1106
|
|
|
1002
|
-
1. **No API to trigger Appium test execution.**
|
|
1107
|
+
1. **No API to trigger Appium or Selenium test execution.** Those tests launch from Appium clients (IDE, CI scripts); the server manages devices, apps, reservations, and results. Espresso, XCUITest and Maestro suites are different: the platform runs them itself via `execute_test_run` / `get_test_run_status` / `cancel_test_run` / `get_test_run_command` (Maestro is Android-only). Espresso and Maestro runs are verified; XCUITest uses the same API but is not yet verified end to end. Interactive [mobile inspection sessions](docs/tools.md#mobile-inspection-sessions) and [browser inspection sessions](docs/tools.md#web-inspection-sessions) are also live WebDriver sessions for element discovery on Android, iOS, and desktop browsers.
|
|
1003
1108
|
2. **Inspection sessions support Android and iOS** on both the legacy Appium Grid (JWP) and Appium Server (W3C/OSS) projects. iOS caveats: no clear-app-data, no clipboard on Grid devices, and back navigation uses the nav-bar button (iOS has no Back button).
|
|
1004
1109
|
3. **No user disable/lock endpoint.** Removing access for a user provisioned solely for a POC means deleting the account — which is why `close_poc` requires per-user confirmation.
|
|
1005
|
-
4. **
|
|
1110
|
+
4. **Deleting test reports with a project-level key depends on a per-project setting** (`allowUsersDeleteTests`). Cloud Admin can always delete. When the setting is off, the tools refuse up front and explain how to get it enabled.
|
|
1006
1111
|
|
|
1007
|
-
See [docs/limitations.md](docs/limitations.md) for the full list
|
|
1112
|
+
See [docs/limitations.md](docs/limitations.md) for the full list.
|
|
1008
1113
|
|
|
1009
1114
|
---
|
|
1010
1115
|
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { MyAccountInfo } from '../types/digital-ai.js';
|
|
2
|
+
import { type AccessInfo } from '../utils/access-level.js';
|
|
3
|
+
/** Resolve (and cache) the access level for an explicit url + key. */
|
|
4
|
+
export declare function resolveAccessInfo(url: string, key: string): Promise<AccessInfo>;
|
|
5
|
+
/**
|
|
6
|
+
* Seed the cache for the ACTIVE credential from a my-account-info body the caller already fetched through
|
|
7
|
+
* the main client. Only a recognised role is primed; an unrecognised one is left for the resolver's probe.
|
|
8
|
+
*/
|
|
9
|
+
export declare function primeAccessInfoFromAccount(me: MyAccountInfo): void;
|
|
10
|
+
/** Access level of the ACTIVE profile (honours switch_environment). */
|
|
11
|
+
export declare function getAccessInfo(): Promise<AccessInfo>;
|
|
12
|
+
/**
|
|
13
|
+
* Cache-only view of the active profile's access level — NEVER touches the network. For callers that must
|
|
14
|
+
* stay offline (command generators the user reaches for precisely when the server is unreachable).
|
|
15
|
+
* Returns undefined when nothing is known yet.
|
|
16
|
+
*/
|
|
17
|
+
export declare function peekActiveAccessInfo(): AccessInfo | undefined;
|
|
18
|
+
/** Access level of a NAMED profile, using that profile's own credentials. */
|
|
19
|
+
export declare function resolveProfileAccess(profileName: string): Promise<AccessInfo>;
|
|
20
|
+
/**
|
|
21
|
+
* Gate for reporter DELETES. Cloud Admin (and an undeterminable level) always proceeds.
|
|
22
|
+
*
|
|
23
|
+
* Project-level roles are refused up front ONLY when the project the delete will actually hit has a
|
|
24
|
+
* readable `allowUsersDeleteTests === false` — verified live: with it false, Cloud Admin deleted fine
|
|
25
|
+
* while ProjectAdmin and User both got 403 "You have no permission to delete tests". A ProjectAdmin key
|
|
26
|
+
* can read its own project's flag via GET /reporter/api/projects; a User key sees an empty list.
|
|
27
|
+
*
|
|
28
|
+
* Which project is "the one the delete will hit":
|
|
29
|
+
* - `projectName` given → exactly that project. If it is not in the visible list we do NOT substitute
|
|
30
|
+
* another one; the call proceeds and the platform decides.
|
|
31
|
+
* - no `projectName` → the credential's own project (reporter calls default to that scope; a numeric
|
|
32
|
+
* projectId is ignored by the reporter delete endpoint, so it cannot redirect the delete elsewhere).
|
|
33
|
+
* If that is not visible and exactly one project is, that one.
|
|
34
|
+
* When the flag is true or unreadable the call proceeds (whether a project role succeeds once the flag is
|
|
35
|
+
* true is NOT verified). Returns the refusal text, or null to proceed.
|
|
36
|
+
*/
|
|
37
|
+
export declare function checkDeleteAllowed(projectName?: string): Promise<string | null>;
|
|
38
|
+
/**
|
|
39
|
+
* True unless the platform has already refused server-side sort for the ACTIVE credential. Sort is
|
|
40
|
+
* attempted for every role and falls back automatically (see listTests) — role is deliberately NOT
|
|
41
|
+
* consulted, because which roles may sort is a platform behavior that has changed before.
|
|
42
|
+
*/
|
|
43
|
+
export declare function serverSortAvailable(): boolean;
|
|
44
|
+
/** Record that the platform refused sort for the ACTIVE credential. */
|
|
45
|
+
export declare function markActiveServerSortRefused(): void;
|
|
46
|
+
//# sourceMappingURL=access-level.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"access-level.d.ts","sourceRoot":"","sources":["../../src/api/access-level.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAWL,KAAK,UAAU,EAEhB,MAAM,0BAA0B,CAAC;AAqBlC,sEAAsE;AACtE,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAWrF;AAkCD;;;GAGG;AACH,wBAAgB,0BAA0B,CAAC,EAAE,EAAE,aAAa,GAAG,IAAI,CASlE;AAED,uEAAuE;AACvE,wBAAsB,aAAa,IAAI,OAAO,CAAC,UAAU,CAAC,CAOzD;AAED;;;;GAIG;AACH,wBAAgB,oBAAoB,IAAI,UAAU,GAAG,SAAS,CAI7D;AAED,6EAA6E;AAC7E,wBAAsB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAInF;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,kBAAkB,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CA2BrF;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,IAAI,OAAO,CAE7C;AAED,uEAAuE;AACvE,wBAAgB,2BAA2B,IAAI,IAAI,CAElD"}
|