@alex_neo/playwright-azure-reporter 1.22.0 → 1.22.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -1
- package/dist/playwright-azure-reporter.d.ts +16 -1
- package/dist/playwright-azure-reporter.js +86 -14
- package/dist/playwright-azure-reporter.js.map +1 -1
- package/dist/test-point-diagnostics.d.ts +43 -0
- package/dist/test-point-diagnostics.js +76 -0
- package/dist/test-point-diagnostics.js.map +1 -0
- package/dist/utils/index.d.ts +5 -0
- package/dist/utils/index.js +16 -0
- package/dist/utils/index.js.map +1 -1
- package/package.json +5 -5
- package/CHANGELOG.md +0 -589
package/README.md
CHANGED
|
@@ -27,6 +27,7 @@ A Playwright reporter that publishes test results to Azure DevOps Test Plans.
|
|
|
27
27
|
- [Auto-Mark as Automated](#auto-mark-as-automated)
|
|
28
28
|
- [CI/CD Integration](#cicd-integration)
|
|
29
29
|
- [Playwright Compatibility](#playwright-compatibility)
|
|
30
|
+
- [Troubleshooting](#troubleshooting)
|
|
30
31
|
- [Releases](#releases)
|
|
31
32
|
|
|
32
33
|
## Quick Start
|
|
@@ -62,6 +63,8 @@ export default defineConfig({
|
|
|
62
63
|
|
|
63
64
|
Each test must reference an existing Azure DevOps test case ID. See [Defining Test Case IDs](#defining-test-case-ids) for all supported formats.
|
|
64
65
|
|
|
66
|
+
`planId` also accepts a numeric string, so `planId: process.env.AZURE_PLAN_ID` works without `Number()`. Something not published? See [Troubleshooting](docs/troubleshooting.md).
|
|
67
|
+
|
|
65
68
|
## Defining Test Case IDs
|
|
66
69
|
|
|
67
70
|
You must link each test to an existing test case in Azure DevOps before running tests. There are three ways to do this.
|
|
@@ -170,6 +173,25 @@ The reporter supports three authentication methods:
|
|
|
170
173
|
}
|
|
171
174
|
```
|
|
172
175
|
|
|
176
|
+
#### `System.AccessToken` in Azure Pipelines
|
|
177
|
+
|
|
178
|
+
The job access token of a pipeline works with `authType: 'accessToken'`: the reporter sends it as a Bearer token. Map it into the environment explicitly, pipelines do not pass it to scripts by default:
|
|
179
|
+
|
|
180
|
+
```yaml
|
|
181
|
+
- script: npx playwright test
|
|
182
|
+
env:
|
|
183
|
+
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
```typescript
|
|
187
|
+
{
|
|
188
|
+
authType: 'accessToken',
|
|
189
|
+
token: process.env.SYSTEM_ACCESSTOKEN,
|
|
190
|
+
}
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
The token acts as the pipeline's build service identity, so that identity needs the permissions a PAT would need (see [Required Token Scopes](#required-token-scopes)): create test runs and publish results, read the test plan, and, for `autoMarkTestCasesAsAutomated`, edit work items. If publishing fails with 401 or 403, check these permissions for the build service account of the project.
|
|
194
|
+
|
|
173
195
|
**When to use each:**
|
|
174
196
|
|
|
175
197
|
- **PAT**: Personal Access Token from Azure DevOps. Suitable for most use cases.
|
|
@@ -222,6 +244,10 @@ To publish results to an existing test run instead of creating a new one:
|
|
|
222
244
|
|
|
223
245
|
> **Note:** When using `isExistingTestRun`, the test run is not completed automatically. You must complete it manually.
|
|
224
246
|
|
|
247
|
+
#### Sharded Runs
|
|
248
|
+
|
|
249
|
+
Every shard of `--shard=1/4` ... `--shard=4/4` is a separate Playwright process, so a reporter loaded in each shard creates one test run per shard. To publish a sharded run as a single test run, let the shards write Playwright blob reports and run this reporter once with `npx playwright merge-reports`. No test is executed again, the recorded results of all shards are replayed to the reporter. See [Sharded Runs](docs/sharded-runs.md) for the configuration and examples for Azure Pipelines and GitHub Actions.
|
|
250
|
+
|
|
225
251
|
### Retry Results
|
|
226
252
|
|
|
227
253
|
Control how retry attempts are published with `publishRetryResults`:
|
|
@@ -413,7 +439,7 @@ import { TestPoint } from 'azure-devops-node-api/interfaces/TestInterfaces';
|
|
|
413
439
|
case 'webkit':
|
|
414
440
|
return testPoints.filter((tp) => tp.configuration.id === '5');
|
|
415
441
|
default:
|
|
416
|
-
|
|
442
|
+
return []; // no point: the result of this test is not published
|
|
417
443
|
}
|
|
418
444
|
},
|
|
419
445
|
testRunConfig: {
|
|
@@ -426,6 +452,8 @@ import { TestPoint } from 'azure-devops-node-api/interfaces/TestInterfaces';
|
|
|
426
452
|
|
|
427
453
|
> **Important:** Define `testRunConfig.configurationIds` and/or `testPointMapper`, otherwise results are published for all configurations.
|
|
428
454
|
|
|
455
|
+
> **Note:** If `testPointMapper` throws for a test, the reporter logs the error and skips that test only. The other results are still published.
|
|
456
|
+
|
|
429
457
|
#### `rootSuiteId`
|
|
430
458
|
|
|
431
459
|
Restricts test point resolution to a specific suite. Useful when different suites (e.g. `smoke`, `integration`) contain the same test case IDs. Can also be set via `AZURE_PW_ROOT_SUITE_ID` env variable. Reporter options take precedence.
|
|
@@ -674,6 +702,10 @@ When the Playwright version cannot be detected, the check is skipped in both mod
|
|
|
674
702
|
|
|
675
703
|
<!-- The table above is generated from src/compat.ts: run `yarn build && yarn feature-matrix` and paste the output when the matrix changes. A test fails when README and the matrix differ. -->
|
|
676
704
|
|
|
705
|
+
## Troubleshooting
|
|
706
|
+
|
|
707
|
+
The most common problems, with what the log says and how to fix them, are collected in [Troubleshooting](docs/troubleshooting.md): no test points found, reporting disabled or 401, "User identity not found", attachments not uploaded, results missing from the pipeline Tests tab.
|
|
708
|
+
|
|
677
709
|
## Releases
|
|
678
710
|
|
|
679
711
|
For detailed information about stable releases, beta releases, and version management, see [RELEASES.md](RELEASES.md).
|
|
@@ -40,7 +40,8 @@ export interface AzureReporterOptions {
|
|
|
40
40
|
token?: string;
|
|
41
41
|
authType?: 'pat' | 'accessToken' | 'managedIdentity' | undefined;
|
|
42
42
|
applicationIdURI?: string | undefined;
|
|
43
|
-
planId:
|
|
43
|
+
/** Test plan id. A numeric string is accepted too, so `planId: process.env.AZURE_PLAN_ID` works as is. */
|
|
44
|
+
planId: number | string;
|
|
44
45
|
orgUrl: string;
|
|
45
46
|
projectName: string;
|
|
46
47
|
publishTestResultsMode?: TPublishTestResults;
|
|
@@ -174,6 +175,8 @@ declare class AzureDevOpsReporter implements Reporter {
|
|
|
174
175
|
private _workItemStepsCache;
|
|
175
176
|
private _testStepsAlignmentWarned;
|
|
176
177
|
private _testCasesWithoutSteps;
|
|
178
|
+
private _testPointMapperErrors;
|
|
179
|
+
private _diagnosedMissingPoints;
|
|
177
180
|
private _unmatched;
|
|
178
181
|
constructor(options: AzureReporterOptions);
|
|
179
182
|
private _initPublishTestSteps;
|
|
@@ -249,6 +252,18 @@ declare class AzureDevOpsReporter implements Reporter {
|
|
|
249
252
|
* @returns ValueEntity representation of the test case result
|
|
250
253
|
*/
|
|
251
254
|
private _convertTestCaseResultToValueEntity;
|
|
255
|
+
/**
|
|
256
|
+
* Calls `testPointMapper` for one test. An exception thrown by the mapper only skips that test: before, it
|
|
257
|
+
* reached the catch of the points lookup and disabled reporting for the whole run.
|
|
258
|
+
*/
|
|
259
|
+
private _mapTestPoints;
|
|
260
|
+
/**
|
|
261
|
+
* Why a test got no test point. Runs only on that failure path: one extra points query without the
|
|
262
|
+
* configuration filter for the test, at most for `MAX_DIAGNOSED_TESTS` tests per run.
|
|
263
|
+
* Not used in testRunADO mode, where points are matched against the results of the existing run.
|
|
264
|
+
*/
|
|
265
|
+
private _explainMissingTestPoints;
|
|
266
|
+
private _missingTestPointsWarning;
|
|
252
267
|
private _getTestPointsOfTestResults;
|
|
253
268
|
private _addReportingOverride;
|
|
254
269
|
private _getPointsList;
|
|
@@ -48,6 +48,7 @@ const compat_1 = require("./compat");
|
|
|
48
48
|
const global_errors_1 = require("./global-errors");
|
|
49
49
|
const logger_1 = __importDefault(require("./logger"));
|
|
50
50
|
const run_context_1 = require("./run-context");
|
|
51
|
+
const test_point_diagnostics_1 = require("./test-point-diagnostics");
|
|
51
52
|
const test_steps_1 = require("./test-steps");
|
|
52
53
|
const utils_1 = require("./utils");
|
|
53
54
|
// https://learn.microsoft.com/en-us/azure/devops/report/analytics/entity-reference-test-plans?view=azure-devops#testoutcome-enumerated-type-members
|
|
@@ -61,6 +62,8 @@ var EAzureTestStatuses;
|
|
|
61
62
|
EAzureTestStatuses["timedOut"] = "Timeout";
|
|
62
63
|
EAzureTestStatuses["interrupted"] = "Aborted";
|
|
63
64
|
})(EAzureTestStatuses || (EAzureTestStatuses = {}));
|
|
65
|
+
/** Tests per run that get an explanation of their missing test points (one extra request each). */
|
|
66
|
+
const MAX_DIAGNOSED_TESTS = 50;
|
|
64
67
|
// eslint-disable-next-line
|
|
65
68
|
const attachmentTypesArray = ['screenshot', 'video', 'trace', 'error-context'];
|
|
66
69
|
function toError(error) {
|
|
@@ -150,6 +153,8 @@ class AzureDevOpsReporter {
|
|
|
150
153
|
_workItemStepsCache = new Map();
|
|
151
154
|
_testStepsAlignmentWarned = false;
|
|
152
155
|
_testCasesWithoutSteps = new Set();
|
|
156
|
+
_testPointMapperErrors = new WeakMap();
|
|
157
|
+
_diagnosedMissingPoints = 0;
|
|
153
158
|
_unmatched = {
|
|
154
159
|
noTestPoints: [],
|
|
155
160
|
};
|
|
@@ -331,9 +336,9 @@ class AzureDevOpsReporter {
|
|
|
331
336
|
}
|
|
332
337
|
this._orgUrl = options.orgUrl;
|
|
333
338
|
this._projectName = options.projectName;
|
|
334
|
-
const parsedPlanId =
|
|
335
|
-
if (
|
|
336
|
-
this._logger?.warn(`'planId' value "${options.planId}" is not a
|
|
339
|
+
const parsedPlanId = (0, utils_1.parsePlanId)(options.planId);
|
|
340
|
+
if (parsedPlanId === undefined) {
|
|
341
|
+
this._logger?.warn(`'planId' value "${options.planId}" is not a positive integer. Reporting is disabled.`);
|
|
337
342
|
this._setIsDisable(true);
|
|
338
343
|
return;
|
|
339
344
|
}
|
|
@@ -342,9 +347,8 @@ class AzureDevOpsReporter {
|
|
|
342
347
|
this._authType = options.authType || 'pat';
|
|
343
348
|
this._applicationIdURI = options.applicationIdURI;
|
|
344
349
|
this._environment = options?.environment || undefined;
|
|
345
|
-
this.
|
|
346
|
-
|
|
347
|
-
`${this._environment ? `[${this._environment}]:` : ''}Test plan ${this._planId}`;
|
|
350
|
+
const environmentPrefix = this._environment ? `[${this._environment}]: ` : '';
|
|
351
|
+
this._testRunTitle = `${environmentPrefix}${options?.testRunTitle || 'Playwright Test Run'}`;
|
|
348
352
|
this._uploadAttachments = options?.uploadAttachments || false;
|
|
349
353
|
// Authentication handler and connection will be initialized in onBegin() for managedIdentity support
|
|
350
354
|
this._testRunConfig = options?.testRunConfig || undefined;
|
|
@@ -1239,13 +1243,79 @@ class AzureDevOpsReporter {
|
|
|
1239
1243
|
url: testCaseResult.url,
|
|
1240
1244
|
};
|
|
1241
1245
|
}
|
|
1246
|
+
/**
|
|
1247
|
+
* Calls `testPointMapper` for one test. An exception thrown by the mapper only skips that test: before, it
|
|
1248
|
+
* reached the catch of the points lookup and disabled reporting for the whole run.
|
|
1249
|
+
*/
|
|
1250
|
+
async _mapTestPoints(testsResult, testPoints) {
|
|
1251
|
+
try {
|
|
1252
|
+
return await this._testPointMapper(testsResult.testCase, testPoints);
|
|
1253
|
+
}
|
|
1254
|
+
catch (error) {
|
|
1255
|
+
const message = toError(error).message;
|
|
1256
|
+
this._testPointMapperErrors.set(testsResult, message);
|
|
1257
|
+
this._logger?.warn(`testPointMapper threw for test "${testsResult.testCase.title}": ${message}. The result is not published.`);
|
|
1258
|
+
return [];
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
/**
|
|
1262
|
+
* Why a test got no test point. Runs only on that failure path: one extra points query without the
|
|
1263
|
+
* configuration filter for the test, at most for `MAX_DIAGNOSED_TESTS` tests per run.
|
|
1264
|
+
* Not used in testRunADO mode, where points are matched against the results of the existing run.
|
|
1265
|
+
*/
|
|
1266
|
+
async _explainMissingTestPoints(testsResult) {
|
|
1267
|
+
if (this._publishTestResultsMode === 'testRunADO')
|
|
1268
|
+
return undefined;
|
|
1269
|
+
if (this._diagnosedMissingPoints >= MAX_DIAGNOSED_TESTS)
|
|
1270
|
+
return undefined;
|
|
1271
|
+
this._diagnosedMissingPoints++;
|
|
1272
|
+
if (this._diagnosedMissingPoints === MAX_DIAGNOSED_TESTS) {
|
|
1273
|
+
this._logger?.info(`Explained missing test points for ${MAX_DIAGNOSED_TESTS} tests, the rest only get the summary line.`);
|
|
1274
|
+
}
|
|
1275
|
+
try {
|
|
1276
|
+
const { testCaseIds } = testsResult.testCase;
|
|
1277
|
+
const mapperError = this._testPointMapperErrors.get(testsResult);
|
|
1278
|
+
let points = [];
|
|
1279
|
+
if (mapperError === undefined) {
|
|
1280
|
+
if (!this._testApi)
|
|
1281
|
+
this._testApi = await this._connection.getTestApi();
|
|
1282
|
+
const response = await this._testApi.getPointsByQuery({ pointsFilter: { testcaseIds: testCaseIds.map((id) => parseInt(id, 10)) } }, this._projectName);
|
|
1283
|
+
points = response?.points ?? [];
|
|
1284
|
+
}
|
|
1285
|
+
return (0, test_point_diagnostics_1.diagnoseMissingTestPoints)({
|
|
1286
|
+
testCaseIds,
|
|
1287
|
+
planId: this._planId,
|
|
1288
|
+
configurationIds: this._testRunConfig?.configurationIds,
|
|
1289
|
+
rootSuite: this._rootSuiteId
|
|
1290
|
+
? {
|
|
1291
|
+
id: this._rootSuiteId,
|
|
1292
|
+
pointIds: new Set(this._expectedTestPointsByRootSuite.map((point) => String(point.id))),
|
|
1293
|
+
}
|
|
1294
|
+
: undefined,
|
|
1295
|
+
points,
|
|
1296
|
+
mapperError,
|
|
1297
|
+
}).map((diagnosis) => diagnosis.message);
|
|
1298
|
+
}
|
|
1299
|
+
catch (error) {
|
|
1300
|
+
this._logger?.debug(`Could not explain the missing test points: ${toError(error).message}`);
|
|
1301
|
+
return undefined;
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
_missingTestPointsWarning(testCase, reasons) {
|
|
1305
|
+
const configurations = this._testRunConfig?.configurationIds;
|
|
1306
|
+
const head = `No test points found for test case [${testCase.testCaseIds}] associated with test plan ${this._planId} ${configurations ? `for configurations [${configurations.join(', ')}]` : ''}`;
|
|
1307
|
+
if (reasons?.length) {
|
|
1308
|
+
return `${head}\nTest "${testCase.title}":\n${reasons.map((reason) => ` - ${reason}`).join('\n')}`;
|
|
1309
|
+
}
|
|
1310
|
+
return `${head}\nCheck, maybe testPlanId or assigned configurations per test case, what you specified, is incorrect.\n Also check testPointMapper function.`;
|
|
1311
|
+
}
|
|
1242
1312
|
async _getTestPointsOfTestResults(testsResults, existingTestCaseResults) {
|
|
1243
1313
|
const result = new Map();
|
|
1244
1314
|
if (this._rootSuiteId) {
|
|
1245
1315
|
for (const testsResult of testsResults) {
|
|
1246
1316
|
const currentTestPoints = this._filterTestPlanTestPoints(this._convertTestPlanTestPointsToTestPoints(this._expectedTestPointsByRootSuite), testsResult);
|
|
1247
1317
|
if (currentTestPoints && currentTestPoints.length > 0) {
|
|
1248
|
-
const mappedTestPoints = await this.
|
|
1318
|
+
const mappedTestPoints = await this._mapTestPoints(testsResult, currentTestPoints);
|
|
1249
1319
|
if (mappedTestPoints && mappedTestPoints.length > 0) {
|
|
1250
1320
|
result.set(testsResult, mappedTestPoints);
|
|
1251
1321
|
}
|
|
@@ -1284,7 +1354,7 @@ class AzureDevOpsReporter {
|
|
|
1284
1354
|
currentTestPoints = this._filterTestTestPointsByRunResults(existingTestCaseResults, pointsQueryResult.points || [], testsResult);
|
|
1285
1355
|
}
|
|
1286
1356
|
if (currentTestPoints && currentTestPoints.length > 0) {
|
|
1287
|
-
const mappedTestPoints = await this.
|
|
1357
|
+
const mappedTestPoints = await this._mapTestPoints(testsResult, currentTestPoints);
|
|
1288
1358
|
if (mappedTestPoints && mappedTestPoints.length > 0) {
|
|
1289
1359
|
result.set(testsResult, mappedTestPoints);
|
|
1290
1360
|
}
|
|
@@ -1933,6 +2003,7 @@ class AzureDevOpsReporter {
|
|
|
1933
2003
|
const mappedTestPoints = (await this._getTestPointsOfTestResults([toBePublished], [])).get(toBePublished);
|
|
1934
2004
|
this._logger?.debug(`[publishCaseResult] Mapped test points: ${JSON.stringify(mappedTestPoints, null, 2)}`);
|
|
1935
2005
|
if (!mappedTestPoints || mappedTestPoints.length == 0) {
|
|
2006
|
+
const reasons = await this._explainMissingTestPoints(toBePublished);
|
|
1936
2007
|
// Track for summary if enabled
|
|
1937
2008
|
if (this._testCaseSummaryEnabled) {
|
|
1938
2009
|
this._unmatched.noTestPoints.push({
|
|
@@ -1940,13 +2011,12 @@ class AzureDevOpsReporter {
|
|
|
1940
2011
|
file: test.location?.file || 'unknown',
|
|
1941
2012
|
line: test.location?.line || 0,
|
|
1942
2013
|
testCaseIds: caseIds,
|
|
2014
|
+
reasons,
|
|
1943
2015
|
});
|
|
1944
2016
|
}
|
|
1945
2017
|
// Ensure we don't hang in onEnd wait loop
|
|
1946
2018
|
this._removePublished(testCase.testAlias);
|
|
1947
|
-
this._logger?.warn(
|
|
1948
|
-
? `for configurations [${this._testRunConfig?.configurationIds?.join(', ')}]`
|
|
1949
|
-
: ''}\nCheck, maybe testPlanId or assigned configurations per test case, what you specified, is incorrect.\n Also check testPointMapper function.`);
|
|
2019
|
+
this._logger?.warn(this._missingTestPointsWarning(testCase, reasons));
|
|
1950
2020
|
return;
|
|
1951
2021
|
}
|
|
1952
2022
|
this._publishedResultsCount.points += mappedTestPoints?.length || 0;
|
|
@@ -2080,6 +2150,7 @@ class AzureDevOpsReporter {
|
|
|
2080
2150
|
}
|
|
2081
2151
|
}
|
|
2082
2152
|
else {
|
|
2153
|
+
const reasons = await this._explainMissingTestPoints(key);
|
|
2083
2154
|
// Track for summary if enabled
|
|
2084
2155
|
if (this._testCaseSummaryEnabled) {
|
|
2085
2156
|
this._unmatched.noTestPoints.push({
|
|
@@ -2087,11 +2158,10 @@ class AzureDevOpsReporter {
|
|
|
2087
2158
|
file: testCase.location?.file || 'unknown',
|
|
2088
2159
|
line: testCase.location?.line || 0,
|
|
2089
2160
|
testCaseIds: testCase.testCaseIds,
|
|
2161
|
+
reasons,
|
|
2090
2162
|
});
|
|
2091
2163
|
}
|
|
2092
|
-
this._logger?.warn(
|
|
2093
|
-
? `for configurations [${this._testRunConfig?.configurationIds?.join(', ')}]`
|
|
2094
|
-
: ''}\nCheck, maybe testPlanId or assigned configurations per test case, what you specified, is incorrect.\n Also check testPointMapper function.`);
|
|
2164
|
+
this._logger?.warn(this._missingTestPointsWarning(testCase, reasons));
|
|
2095
2165
|
}
|
|
2096
2166
|
}
|
|
2097
2167
|
if (testCaseResults.length === 0) {
|
|
@@ -2214,6 +2284,8 @@ class AzureDevOpsReporter {
|
|
|
2214
2284
|
summary += `- **${test.title}**\n`;
|
|
2215
2285
|
summary += ` - File: ${test.file}:${test.line}\n`;
|
|
2216
2286
|
summary += ` - Test Case IDs: [${test.testCaseIds.join(', ')}]\n`;
|
|
2287
|
+
for (const reason of test.reasons ?? [])
|
|
2288
|
+
summary += ` - Reason: ${reason}\n`;
|
|
2217
2289
|
});
|
|
2218
2290
|
summary += `\n`;
|
|
2219
2291
|
summary += `## Recommendations\n\n`;
|