@alex_neo/playwright-azure-reporter 1.14.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/publish.yml +2 -2
- package/CHANGELOG.md +9 -68
- package/README.md +71 -0
- package/dist/playwright-azure-reporter.d.ts +16 -0
- package/dist/playwright-azure-reporter.js +280 -18
- package/dist/playwright-azure-reporter.js.map +1 -1
- package/package.json +1 -1
- package/tests/reporter/assets/azure-reporter/azureAreas.js +131 -76
- package/tests/reporter/assets/azure-reporter/azureTestPlanOptionsResponse.json +195 -0
- package/tests/reporter/playwright.config.ts +4 -1
- package/tests/reporter/reporter-publish-testResult.spec.ts +22 -1
- package/tests/reporter/reporter-testCaseSummary.spec.ts +833 -0
|
@@ -132,8 +132,8 @@ jobs:
|
|
|
132
132
|
echo "New version: $NEW_VERSION"
|
|
133
133
|
echo "new-version=$NEW_VERSION" >> $GITHUB_OUTPUT
|
|
134
134
|
|
|
135
|
-
# Update package.json version
|
|
136
|
-
npm version "$NEW_VERSION" --no-git-tag-version
|
|
135
|
+
# Update package.json version (skip scripts to avoid duplicate changelog generation)
|
|
136
|
+
npm version "$NEW_VERSION" --no-git-tag-version --ignore-scripts
|
|
137
137
|
|
|
138
138
|
- name: Update changelog
|
|
139
139
|
if: github.event_name == 'workflow_dispatch'
|
package/CHANGELOG.md
CHANGED
|
@@ -1,33 +1,21 @@
|
|
|
1
|
-
# [1.
|
|
1
|
+
# [1.15.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.14.0...v1.15.0) (2025-09-16)
|
|
2
2
|
|
|
3
3
|
|
|
4
|
-
###
|
|
4
|
+
### Bug Fixes
|
|
5
5
|
|
|
6
|
-
* **
|
|
7
|
-
* **
|
|
6
|
+
* **attachments:** implement retry mechanism for file access during upload ([2bd80e5](https://github.com/alexneo2003/playwright-azure-reporter/commit/2bd80e5d233504f2a643e629692533aaae253a53)), closes [#120](https://github.com/alexneo2003/playwright-azure-reporter/issues/120)
|
|
7
|
+
* **changelog:** prevent duplicate entries in publish workflow ([d35cc9c](https://github.com/alexneo2003/playwright-azure-reporter/commit/d35cc9c4cab6a2dce4d9a0880b24bb74964ab7e8))
|
|
8
8
|
|
|
9
9
|
|
|
10
|
-
###
|
|
10
|
+
### Features
|
|
11
11
|
|
|
12
|
-
* **
|
|
12
|
+
* **summary:** add testCaseSummary feature with publishToRun support and configuration name resolution ([37c9e43](https://github.com/alexneo2003/playwright-azure-reporter/commit/37c9e4323353d1ddf1bb12267f95acd0a83838b9))
|
|
13
|
+
* **testCaseSummary:** optimize attachment uploads and replace dynamic imports ([a7e0d81](https://github.com/alexneo2003/playwright-azure-reporter/commit/a7e0d81acbea9eb5ce530d800f75dcd94f9f80ac))
|
|
13
14
|
|
|
14
|
-
Features:
|
|
15
|
-
- New `authType` option in AzureReporterOptions interface
|
|
16
|
-
- Support for both azdev.getPersonalAccessTokenHandler and azdev.getHandlerFromToken
|
|
17
|
-
- Backward compatibility with existing PAT-based configurations
|
|
18
|
-
- Graceful handling of invalid authType values
|
|
19
15
|
|
|
20
|
-
|
|
21
|
-
- 4 integration tests for authType configuration scenarios
|
|
22
|
-
- 3 unit tests with Azure DevOps API method mocking
|
|
23
|
-
- 5 authentication handler creation tests
|
|
24
|
-
- Complete coverage of default behavior and edge cases
|
|
16
|
+
### BREAKING CHANGES
|
|
25
17
|
|
|
26
|
-
|
|
27
|
-
- Updated README.md with authentication section and token scope requirements
|
|
28
|
-
- Created comprehensive authType-examples.md with usage examples
|
|
29
|
-
- Added configuration guidance for both PAT and OAuth scenarios
|
|
30
|
-
- Included troubleshooting section for common authentication issues
|
|
18
|
+
* **summary:** none (feature is opt-in via testCaseSummary.enabled)
|
|
31
19
|
|
|
32
20
|
|
|
33
21
|
|
|
@@ -73,39 +61,10 @@ Documentation:
|
|
|
73
61
|
|
|
74
62
|
|
|
75
63
|
|
|
76
|
-
## [1.13.3](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.13.2-beta.1...v1.13.3) (2025-09-02)
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
### Bug Fixes
|
|
80
|
-
|
|
81
|
-
* add missing conventional-changelog and conventional-github-releaser dependencies ([93bfa1c](https://github.com/alexneo2003/playwright-azure-reporter/commit/93bfa1c45eaff6205b5bf0333cca246976159028))
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
64
|
## [1.13.2-beta.1](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.13.2-beta.0...v1.13.2-beta.1) (2025-08-31)
|
|
86
65
|
|
|
87
66
|
|
|
88
67
|
|
|
89
|
-
## [1.13.2-beta.1](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.13.2-beta.0...v1.13.2-beta.1) (2025-08-31)
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
## [1.13.2-beta.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.13.1...v1.13.2-beta.0) (2025-08-31)
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
### Bug Fixes
|
|
97
|
-
|
|
98
|
-
* add missing conventional-changelog dependencies for beta release ([caa4a7b](https://github.com/alexneo2003/playwright-azure-reporter/commit/caa4a7bce86f31684bef69c8610f7b52ab8a272f))
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
### Features
|
|
102
|
-
|
|
103
|
-
* add automatic beta releases for PR merges ([39191e8](https://github.com/alexneo2003/playwright-azure-reporter/commit/39191e8ac0ea20e1f133e2c173146993b93e4dd2))
|
|
104
|
-
* add PR beta release functionality with manual confirmation ([4c61d21](https://github.com/alexneo2003/playwright-azure-reporter/commit/4c61d21db88a4954cce5131fd0349818925435df))
|
|
105
|
-
* add repository owner permission check for PR beta releases ([1242bb3](https://github.com/alexneo2003/playwright-azure-reporter/commit/1242bb322783f40e80f89bd75f58c1ede2ee6def))
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
68
|
## [1.13.2-beta.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.13.1...v1.13.2-beta.0) (2025-08-31)
|
|
110
69
|
|
|
111
70
|
|
|
@@ -406,24 +365,6 @@ Documentation:
|
|
|
406
365
|
|
|
407
366
|
|
|
408
367
|
|
|
409
|
-
## [1.4.1-beta.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.4.0-beta.0...v1.4.1-beta.0) (2022-12-30)
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
### Bug Fixes
|
|
413
|
-
|
|
414
|
-
* fixing some throwe error ([733e759](https://github.com/alexneo2003/playwright-azure-reporter/commit/733e759f82bf7dc17176af7f3ef04ce43593ceeb))
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
# [1.4.0-beta.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.3.1-beta.0...v1.4.0-beta.0) (2022-12-29)
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
### Features
|
|
422
|
-
|
|
423
|
-
* publishTestResultsMode config to set mode when publish test results ([4e25019](https://github.com/alexneo2003/playwright-azure-reporter/commit/4e2501926beb4680c4ef770273847fae39c57b71))
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
368
|
## [1.3.1-beta.0](https://github.com/alexneo2003/playwright-azure-reporter/compare/v1.3.0-beta.0...v1.3.1-beta.0) (2022-12-22)
|
|
428
369
|
|
|
429
370
|
|
package/README.md
CHANGED
|
@@ -372,6 +372,77 @@ Reporter options (\* - required):
|
|
|
372
372
|
>
|
|
373
373
|
> And when you run tests with the `Smoke Tests` project without specifying the `rootSuiteId`, the test results will be published under the root suite `Automation Tests` for test cases in the `Smoke Tests` suite and `Integration Tests` suite. It will look like you have redundant results inside the test run for the same test cases in different suites. To avoid this, you can specify the `rootSuiteId: 5` to publish test results only under the `Smoke Tests` suite.
|
|
374
374
|
|
|
375
|
+
- `testCaseSummary` [object] - Configuration for generating a summary report of test cases that don't match the test plan. Default: `undefined`.
|
|
376
|
+
- `enabled` [boolean] - Enable test case summary generation. Default: `false`.
|
|
377
|
+
- `outputPath` [string] - File path where the summary report will be written. If not specified, no file will be created. Default: `undefined`.
|
|
378
|
+
- `consoleOutput` [boolean] - Whether to output the summary to console. Default: `true`.
|
|
379
|
+
- `publishToRun` [boolean] - When `true`, the generated summary (Markdown) is uploaded as an attachment to the Azure DevOps test run (if a run ID exists). Default: `false`. Works together with `outputPath` (file still written if specified) and `consoleOutput` (console printing still occurs unless you explicitly set `consoleOutput: false`).
|
|
380
|
+
|
|
381
|
+
**Example:**
|
|
382
|
+
|
|
383
|
+
```typescript
|
|
384
|
+
testCaseSummary: {
|
|
385
|
+
enabled: true,
|
|
386
|
+
outputPath: './test-case-summary.md',
|
|
387
|
+
consoleOutput: true,
|
|
388
|
+
publishToRun: true
|
|
389
|
+
}
|
|
390
|
+
```
|
|
391
|
+
|
|
392
|
+
When enabled, the reporter will track test cases that have test case IDs but no matching test points in the Azure DevOps test plan. This helps identify:
|
|
393
|
+
- Test cases that don't exist in the specified test plan
|
|
394
|
+
- Test cases that are not assigned to the correct configurations
|
|
395
|
+
- Test cases that are not included in the test plan suite structure
|
|
396
|
+
|
|
397
|
+
The summary includes recommendations for resolving issues, including specific configuration IDs and names when available.
|
|
398
|
+
|
|
399
|
+
**Behavior matrix:**
|
|
400
|
+
- Only `consoleOutput: true`: Printed to console only
|
|
401
|
+
- `outputPath` + `consoleOutput: true`: Printed and written to file
|
|
402
|
+
- `publishToRun: true` + (optional `outputPath`): Uploaded as run attachment; also printed and/or written depending on the other flags
|
|
403
|
+
- Set any channel off explicitly by setting its flag to `false` (e.g. `consoleOutput: false` to suppress console)
|
|
404
|
+
|
|
405
|
+
**Example summary (actual generated format):**
|
|
406
|
+
|
|
407
|
+
```markdown
|
|
408
|
+
# Test Case Summary
|
|
409
|
+
|
|
410
|
+
⚠️ Found 1 test(s) with test case IDs that don't match the test plan:
|
|
411
|
+
|
|
412
|
+
## Tests with No Matching Test Points (1)
|
|
413
|
+
|
|
414
|
+
These tests have valid test case IDs but no matching test points in the test plan:
|
|
415
|
+
|
|
416
|
+
- **[777] Test with file output**
|
|
417
|
+
- File: /path/to/project/test-results/.../a.spec.js:3
|
|
418
|
+
- Test Case IDs: [777]
|
|
419
|
+
|
|
420
|
+
## Recommendations
|
|
421
|
+
|
|
422
|
+
- Verify test case IDs exist in Azure DevOps test plan 4
|
|
423
|
+
- Check that test cases are assigned to configurations: [10, 20] (Firefox on Ubuntu, Safari on macOS)
|
|
424
|
+
- Ensure test cases are included in the test plan suite structure
|
|
425
|
+
- Add missing test cases to the test plan or assign them to the correct configurations
|
|
426
|
+
```
|
|
427
|
+
|
|
428
|
+
When `outputPath` is set you'll see a log line:
|
|
429
|
+
|
|
430
|
+
```text
|
|
431
|
+
Test case summary written to: ./test-case-summary.md
|
|
432
|
+
```
|
|
433
|
+
|
|
434
|
+
And if `publishToRun: true` a Markdown attachment with the same content is uploaded to the run.
|
|
435
|
+
|
|
436
|
+
When all tests with IDs are matched you'll instead see:
|
|
437
|
+
|
|
438
|
+
```text
|
|
439
|
+
Test case summary: All tests with test case IDs found matching test points in the test plan.
|
|
440
|
+
```
|
|
441
|
+
|
|
442
|
+
> Note: Key summary lines are force-logged so they appear even when general reporter logging is disabled; this ensures visibility in CI logs.
|
|
443
|
+
|
|
444
|
+
> **Important:** When `publishToRun: true` is enabled, the test case summary file will only be uploaded as an attachment to the Azure DevOps test run if there are unmatched test points (i.e., `this._unmatched.noTestPoints.length > 0`). If all tests with test case IDs match the test plan, no attachment will be uploaded to avoid unnecessary file uploads, though the summary will still be displayed in console and/or written to file if those options are enabled.
|
|
445
|
+
|
|
375
446
|
## Usefulness
|
|
376
447
|
|
|
377
448
|
- **AZURE_PW_TEST_RUN_ID** - Id of current test run. It will be set in environment variables after test run created. Can be accessed by `process.env.AZURE_PW_TEST_RUN_ID`. Pay attention what `publishTestResultsMode` configuration you use. If you use `testResult` mode - this variable will be set when test run created, at the start of tests execution, if you use `testRun` mode - this variable will be set when test run completed, at the end of tests execution.
|
|
@@ -27,6 +27,12 @@ export interface AzureReporterOptions {
|
|
|
27
27
|
testCaseIdZone?: TTestCaseIdZone;
|
|
28
28
|
rootSuiteId?: number;
|
|
29
29
|
uploadLogs?: boolean;
|
|
30
|
+
testCaseSummary?: {
|
|
31
|
+
enabled?: boolean;
|
|
32
|
+
outputPath?: string;
|
|
33
|
+
consoleOutput?: boolean;
|
|
34
|
+
publishToRun?: boolean;
|
|
35
|
+
};
|
|
30
36
|
}
|
|
31
37
|
declare class AzureDevOpsReporter implements Reporter {
|
|
32
38
|
private _logger;
|
|
@@ -74,6 +80,11 @@ declare class AzureDevOpsReporter implements Reporter {
|
|
|
74
80
|
private _rejectTestPointsByRootSuite;
|
|
75
81
|
private _uploadLogs;
|
|
76
82
|
private _configurationNames;
|
|
83
|
+
private _testCaseSummaryEnabled;
|
|
84
|
+
private _testCaseSummaryOutputPath?;
|
|
85
|
+
private _testCaseSummaryConsoleOutput;
|
|
86
|
+
private _testCaseSummaryPublishToRun;
|
|
87
|
+
private _unmatched;
|
|
77
88
|
constructor(options: AzureReporterOptions);
|
|
78
89
|
_validateOptions(options: AzureReporterOptions): void;
|
|
79
90
|
private _getAuthenticationHandler;
|
|
@@ -82,6 +93,7 @@ declare class AzureDevOpsReporter implements Reporter {
|
|
|
82
93
|
onBegin(): Promise<void>;
|
|
83
94
|
onTestEnd(test: TestCase, testResult: TestResult): Promise<void>;
|
|
84
95
|
onEnd(): Promise<void>;
|
|
96
|
+
private _performOnEnd;
|
|
85
97
|
printsToStdio(): boolean;
|
|
86
98
|
private _anonymizeString;
|
|
87
99
|
private _anonymizeObject;
|
|
@@ -102,6 +114,7 @@ declare class AzureDevOpsReporter implements Reporter {
|
|
|
102
114
|
private _getPointsList;
|
|
103
115
|
private _recursivelyGetPointsList;
|
|
104
116
|
private _retrieveTestPoints;
|
|
117
|
+
private _waitForFileAccess;
|
|
105
118
|
private _uploadAttachmentsFunc;
|
|
106
119
|
private _uploadLogsAttachmentsFunc;
|
|
107
120
|
private _mapToAzureState;
|
|
@@ -109,6 +122,9 @@ declare class AzureDevOpsReporter implements Reporter {
|
|
|
109
122
|
private _prepareLogAttachments;
|
|
110
123
|
private _publishCaseResult;
|
|
111
124
|
private _publishTestRunResults;
|
|
125
|
+
private _generateTestCaseSummary;
|
|
126
|
+
private _outputTestCaseSummary;
|
|
112
127
|
private _setAzurePWTestRunId;
|
|
128
|
+
private _publishTestCaseSummaryAttachment;
|
|
113
129
|
}
|
|
114
130
|
export default AzureDevOpsReporter;
|