@lokalise/playwright-reporters 1.9.2 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/datadog/index.d.ts +1 -1
- package/dist/datadog/report.types.d.ts +14 -0
- package/dist/datadog/reporter.d.ts +21 -0
- package/dist/datadog/reporter.js +209 -2
- package/dist/datadog/transform.js +14 -0
- package/package.json +1 -1
package/README.md
CHANGED
package/dist/datadog/index.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
export { default as DatadogReporter, defineDatadogReporterConfig } from './reporter';
|
|
2
|
-
export type { DatadogReporterOptions, DatadogSite, MetricTags, TestRunMetrics, TestMetrics, } from './report.types';
|
|
2
|
+
export type { DatadogReporterOptions, DatadogSite, MetricTags, TestRunMetrics, TestMetrics, TerminationReason, } from './report.types';
|
|
@@ -92,6 +92,10 @@ export interface MetricSeries {
|
|
|
92
92
|
export interface DatadogMetricsPayload {
|
|
93
93
|
series: MetricSeries[];
|
|
94
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* Termination reasons for early exit
|
|
97
|
+
*/
|
|
98
|
+
export type TerminationReason = 'failure_rate_exceeded' | 'user_interrupted' | 'timeout' | 'unknown';
|
|
95
99
|
/**
|
|
96
100
|
* Aggregated test run data for metrics generation
|
|
97
101
|
*/
|
|
@@ -130,6 +134,12 @@ export interface TestRunMetrics {
|
|
|
130
134
|
workerCount: number;
|
|
131
135
|
/** Average test duration in milliseconds */
|
|
132
136
|
avgTestDurationMs: number;
|
|
137
|
+
/** Whether the test run was terminated early */
|
|
138
|
+
terminatedEarly?: boolean;
|
|
139
|
+
/** Reason for early termination */
|
|
140
|
+
terminationReason?: TerminationReason;
|
|
141
|
+
/** Number of tests that actually completed (for partial runs) */
|
|
142
|
+
testsCompleted?: number;
|
|
133
143
|
}
|
|
134
144
|
/**
|
|
135
145
|
* Individual test metrics
|
|
@@ -176,4 +186,8 @@ export interface MetricTags {
|
|
|
176
186
|
jobName?: string;
|
|
177
187
|
/** CI build number */
|
|
178
188
|
buildNumber?: string;
|
|
189
|
+
/** Whether the test run was terminated early */
|
|
190
|
+
terminatedEarly?: boolean;
|
|
191
|
+
/** Reason for early termination */
|
|
192
|
+
terminationReason?: TerminationReason;
|
|
179
193
|
}
|
|
@@ -23,7 +23,28 @@ export default class DatadogReporter implements Reporter {
|
|
|
23
23
|
private readonly testMetrics;
|
|
24
24
|
private projectNames;
|
|
25
25
|
private readonly currentTestNavigationMs;
|
|
26
|
+
private testsCompleted;
|
|
27
|
+
private metricsSubmitted;
|
|
28
|
+
private terminatedEarly;
|
|
29
|
+
private terminationReason;
|
|
26
30
|
constructor(options: DatadogReporterOptions);
|
|
31
|
+
/**
|
|
32
|
+
* Register handlers for process signals to capture metrics on early termination
|
|
33
|
+
*/
|
|
34
|
+
private registerSignalHandlers;
|
|
35
|
+
/**
|
|
36
|
+
* Detect the reason for early termination based on signal and current state
|
|
37
|
+
*/
|
|
38
|
+
private detectTerminationReason;
|
|
39
|
+
/**
|
|
40
|
+
* Submit partial metrics synchronously (for use in signal handlers)
|
|
41
|
+
* Uses a fire-and-forget HTTP request since we can't await in exit handlers
|
|
42
|
+
*/
|
|
43
|
+
private submitPartialMetricsSync;
|
|
44
|
+
/**
|
|
45
|
+
* Send metrics using a detached child process (works in exit handlers)
|
|
46
|
+
*/
|
|
47
|
+
private sendMetricsViaChildProcess;
|
|
27
48
|
onBegin(config: FullConfig, suite: Suite): Promise<void>;
|
|
28
49
|
onStepEnd(test: TestCase, result: TestResult, step: TestStep): void;
|
|
29
50
|
onTestEnd(test: TestCase, result: TestResult): void;
|
package/dist/datadog/reporter.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.defineDatadogReporterConfig = void 0;
|
|
4
4
|
/* eslint-disable max-statements */
|
|
5
|
+
const child_process_1 = require("child_process");
|
|
5
6
|
const faker_1 = require("@faker-js/faker");
|
|
6
7
|
const datadog_1 = require("./datadog");
|
|
7
8
|
const transform_1 = require("./transform");
|
|
@@ -38,6 +39,11 @@ class DatadogReporter {
|
|
|
38
39
|
testMetrics;
|
|
39
40
|
projectNames;
|
|
40
41
|
currentTestNavigationMs;
|
|
42
|
+
// Early termination tracking
|
|
43
|
+
testsCompleted;
|
|
44
|
+
metricsSubmitted;
|
|
45
|
+
terminatedEarly;
|
|
46
|
+
terminationReason;
|
|
41
47
|
constructor(options) {
|
|
42
48
|
this.options = options;
|
|
43
49
|
this.testRunId = faker_1.faker.string.uuid();
|
|
@@ -63,6 +69,186 @@ class DatadogReporter {
|
|
|
63
69
|
this.testMetrics = [];
|
|
64
70
|
this.projectNames = [];
|
|
65
71
|
this.currentTestNavigationMs = new Map();
|
|
72
|
+
// Early termination tracking
|
|
73
|
+
this.testsCompleted = 0;
|
|
74
|
+
this.metricsSubmitted = false;
|
|
75
|
+
this.terminatedEarly = false;
|
|
76
|
+
this.terminationReason = undefined;
|
|
77
|
+
// Register signal handlers for early termination
|
|
78
|
+
this.registerSignalHandlers();
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Register handlers for process signals to capture metrics on early termination
|
|
82
|
+
*/
|
|
83
|
+
registerSignalHandlers() {
|
|
84
|
+
const handleEarlyTermination = (signal) => {
|
|
85
|
+
if (this.metricsSubmitted) {
|
|
86
|
+
return; // Already submitted, nothing to do
|
|
87
|
+
}
|
|
88
|
+
this.terminatedEarly = true;
|
|
89
|
+
this.terminationReason = this.detectTerminationReason(signal);
|
|
90
|
+
if (this.options.debug) {
|
|
91
|
+
// eslint-disable-next-line no-console
|
|
92
|
+
console.log(`[Datadog Reporter] Early termination detected (${signal}), submitting partial metrics...`);
|
|
93
|
+
}
|
|
94
|
+
// Submit partial metrics synchronously before exit
|
|
95
|
+
this.submitPartialMetricsSync();
|
|
96
|
+
};
|
|
97
|
+
// Handle SIGINT (Ctrl+C)
|
|
98
|
+
process.on('SIGINT', () => handleEarlyTermination('SIGINT'));
|
|
99
|
+
// Handle SIGTERM (kill command)
|
|
100
|
+
process.on('SIGTERM', () => handleEarlyTermination('SIGTERM'));
|
|
101
|
+
// Handle beforeExit (process about to exit naturally)
|
|
102
|
+
process.on('beforeExit', () => {
|
|
103
|
+
if (!this.metricsSubmitted && this.testsCompleted > 0) {
|
|
104
|
+
this.terminatedEarly = true;
|
|
105
|
+
this.terminationReason = this.terminationReason ?? 'unknown';
|
|
106
|
+
handleEarlyTermination('beforeExit');
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
// Handle uncaught exceptions
|
|
110
|
+
process.on('uncaughtException', () => {
|
|
111
|
+
if (!this.metricsSubmitted && this.testsCompleted > 0) {
|
|
112
|
+
this.terminatedEarly = true;
|
|
113
|
+
this.terminationReason = 'unknown';
|
|
114
|
+
handleEarlyTermination('uncaughtException');
|
|
115
|
+
}
|
|
116
|
+
});
|
|
117
|
+
// Handle exit event (last chance to do sync operations)
|
|
118
|
+
process.on('exit', () => {
|
|
119
|
+
if (!this.metricsSubmitted && this.testsCompleted > 0) {
|
|
120
|
+
this.terminatedEarly = true;
|
|
121
|
+
this.terminationReason = this.terminationReason ?? 'unknown';
|
|
122
|
+
// Note: We can only do sync operations in 'exit' handler
|
|
123
|
+
// The submitPartialMetricsSync method handles this
|
|
124
|
+
this.submitPartialMetricsSync();
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Detect the reason for early termination based on signal and current state
|
|
130
|
+
*/
|
|
131
|
+
detectTerminationReason(signal) {
|
|
132
|
+
// Check if failure rate exceeded by looking at current stats
|
|
133
|
+
const completedNonSkipped = this.testsCompleted - this.skippedTests;
|
|
134
|
+
if (completedNonSkipped > 0) {
|
|
135
|
+
const currentFailureRate = (this.failedTests / completedNonSkipped) * 100;
|
|
136
|
+
// If failure rate is high and we haven't completed all tests, likely terminated by failure rate monitor
|
|
137
|
+
if (currentFailureRate > 5 && this.testsCompleted < this.totalTests) {
|
|
138
|
+
return 'failure_rate_exceeded';
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
if (signal === 'SIGINT') {
|
|
142
|
+
return 'user_interrupted';
|
|
143
|
+
}
|
|
144
|
+
return 'unknown';
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Submit partial metrics synchronously (for use in signal handlers)
|
|
148
|
+
* Uses a fire-and-forget HTTP request since we can't await in exit handlers
|
|
149
|
+
*/
|
|
150
|
+
submitPartialMetricsSync() {
|
|
151
|
+
if (this.metricsSubmitted || this.options.dryRun) {
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
try {
|
|
155
|
+
this.runEndTime = Date.now();
|
|
156
|
+
// Calculate metrics from what we have
|
|
157
|
+
const completedNonSkipped = this.testsCompleted - this.skippedTests;
|
|
158
|
+
const avgTestDurationMs = this.testDurations.length > 0
|
|
159
|
+
? Math.round(this.testDurations.reduce((a, b) => a + b, 0) / this.testDurations.length)
|
|
160
|
+
: 0;
|
|
161
|
+
const runMetrics = {
|
|
162
|
+
testRunId: this.testRunId,
|
|
163
|
+
timestamp: Math.floor(this.runStartTime / 1000),
|
|
164
|
+
totalTests: this.totalTests,
|
|
165
|
+
passedTests: this.passedTests,
|
|
166
|
+
failedTests: this.failedTests,
|
|
167
|
+
flakyTests: this.flakyTests,
|
|
168
|
+
skippedTests: this.skippedTests,
|
|
169
|
+
timedOutTests: this.timedOutTests,
|
|
170
|
+
durationMs: this.runEndTime - this.runStartTime,
|
|
171
|
+
browserCreationMs: this.browserCreationMs,
|
|
172
|
+
retryTimeMs: this.retryTimeMs,
|
|
173
|
+
navigationMs: this.navigationMs,
|
|
174
|
+
allPassed: false, // Early termination means not all passed
|
|
175
|
+
passRate: (0, transform_1.calculatePassRate)(this.passedTests, completedNonSkipped),
|
|
176
|
+
flakyRate: (0, transform_1.calculateFlakyRate)(this.flakyTests, completedNonSkipped),
|
|
177
|
+
workerCount: this.workerCount,
|
|
178
|
+
avgTestDurationMs,
|
|
179
|
+
terminatedEarly: this.terminatedEarly,
|
|
180
|
+
terminationReason: this.terminationReason,
|
|
181
|
+
testsCompleted: this.testsCompleted,
|
|
182
|
+
};
|
|
183
|
+
const baseTags = {
|
|
184
|
+
branch: this.options.ci?.branchName,
|
|
185
|
+
project: this.projectNames.length > 0 ? this.projectNames : undefined,
|
|
186
|
+
prenvId: this.options.prenvId,
|
|
187
|
+
triggeredBy: this.options.ci?.triggeredBy,
|
|
188
|
+
env: this.options.env,
|
|
189
|
+
service: this.options.service ?? 'playwright-tests',
|
|
190
|
+
product: this.options.product,
|
|
191
|
+
jobName: this.options.ci?.jobName,
|
|
192
|
+
buildNumber: this.options.ci?.buildNumber,
|
|
193
|
+
terminatedEarly: this.terminatedEarly,
|
|
194
|
+
terminationReason: this.terminationReason,
|
|
195
|
+
};
|
|
196
|
+
const payload = (0, transform_1.buildMetricsPayload)(runMetrics, this.testMetrics, baseTags);
|
|
197
|
+
// eslint-disable-next-line no-console
|
|
198
|
+
console.log(`\n[Datadog Reporter] Early Termination - Submitting partial metrics:`);
|
|
199
|
+
// eslint-disable-next-line no-console
|
|
200
|
+
console.log(` Tests Completed: ${this.testsCompleted}/${this.totalTests}`);
|
|
201
|
+
// eslint-disable-next-line no-console
|
|
202
|
+
console.log(` Passed: ${this.passedTests}, Failed: ${this.failedTests}`);
|
|
203
|
+
// eslint-disable-next-line no-console
|
|
204
|
+
console.log(` Termination Reason: ${this.terminationReason}`);
|
|
205
|
+
// Use synchronous HTTP request via child_process for exit handlers
|
|
206
|
+
// This is a workaround since we can't use async in exit handlers
|
|
207
|
+
this.sendMetricsViaChildProcess(payload);
|
|
208
|
+
this.metricsSubmitted = true;
|
|
209
|
+
}
|
|
210
|
+
catch (error) {
|
|
211
|
+
// eslint-disable-next-line no-console
|
|
212
|
+
console.error('[Datadog Reporter] Error submitting partial metrics:', error);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Send metrics using a detached child process (works in exit handlers)
|
|
217
|
+
*/
|
|
218
|
+
sendMetricsViaChildProcess(payload) {
|
|
219
|
+
try {
|
|
220
|
+
const site = this.options.site ?? 'datadoghq.eu';
|
|
221
|
+
const endpoints = {
|
|
222
|
+
'datadoghq.com': 'https://api.datadoghq.com',
|
|
223
|
+
'datadoghq.eu': 'https://api.datadoghq.eu',
|
|
224
|
+
'us3.datadoghq.com': 'https://api.us3.datadoghq.com',
|
|
225
|
+
'us5.datadoghq.com': 'https://api.us5.datadoghq.com',
|
|
226
|
+
'ap1.datadoghq.com': 'https://api.ap1.datadoghq.com',
|
|
227
|
+
'ddog-gov.com': 'https://api.ddog-gov.com',
|
|
228
|
+
};
|
|
229
|
+
const endpoint = endpoints[site] || endpoints['datadoghq.eu'];
|
|
230
|
+
// Use curl to send metrics (available on most systems)
|
|
231
|
+
const curlArgs = [
|
|
232
|
+
'-X', 'POST',
|
|
233
|
+
`${endpoint}/api/v2/series`,
|
|
234
|
+
'-H', 'Content-Type: application/json',
|
|
235
|
+
'-H', `DD-API-KEY: ${this.options.apiKey}`,
|
|
236
|
+
'-d', JSON.stringify(payload),
|
|
237
|
+
'--silent',
|
|
238
|
+
'--show-error',
|
|
239
|
+
];
|
|
240
|
+
const child = (0, child_process_1.spawn)('curl', curlArgs, {
|
|
241
|
+
detached: true,
|
|
242
|
+
stdio: 'ignore',
|
|
243
|
+
});
|
|
244
|
+
child.unref();
|
|
245
|
+
// eslint-disable-next-line no-console
|
|
246
|
+
console.log('[Datadog Reporter] Metrics submission initiated via curl');
|
|
247
|
+
}
|
|
248
|
+
catch (error) {
|
|
249
|
+
// eslint-disable-next-line no-console
|
|
250
|
+
console.error('[Datadog Reporter] Failed to spawn curl process:', error);
|
|
251
|
+
}
|
|
66
252
|
}
|
|
67
253
|
async onBegin(config, suite) {
|
|
68
254
|
try {
|
|
@@ -119,6 +305,10 @@ class DatadogReporter {
|
|
|
119
305
|
const projectName = test.parent?.project()?.name;
|
|
120
306
|
const testKey = `${test.id}-${currentRetryCount}`;
|
|
121
307
|
const testNavigationMs = this.currentTestNavigationMs.get(testKey) || 0;
|
|
308
|
+
// Increment completed tests counter (only on final attempt for each test)
|
|
309
|
+
if (currentRetryCount === 0 || currentRetryCount === maxRetriesCount || status === 'passed') {
|
|
310
|
+
this.testsCompleted++;
|
|
311
|
+
}
|
|
122
312
|
// Track test duration for average calculation (only on final attempt)
|
|
123
313
|
if (currentRetryCount === 0 || currentRetryCount === maxRetriesCount) {
|
|
124
314
|
this.testDurations.push(duration);
|
|
@@ -201,12 +391,23 @@ class DatadogReporter {
|
|
|
201
391
|
}
|
|
202
392
|
}
|
|
203
393
|
async onEnd(result) {
|
|
394
|
+
// Skip if metrics were already submitted via signal handler
|
|
395
|
+
if (this.metricsSubmitted) {
|
|
396
|
+
if (this.options.debug) {
|
|
397
|
+
// eslint-disable-next-line no-console
|
|
398
|
+
console.log('[Datadog Reporter] Metrics already submitted, skipping onEnd');
|
|
399
|
+
}
|
|
400
|
+
return;
|
|
401
|
+
}
|
|
204
402
|
try {
|
|
205
403
|
this.runEndTime = Date.now();
|
|
206
404
|
// Calculate average test duration
|
|
207
405
|
const avgTestDurationMs = this.testDurations.length > 0
|
|
208
406
|
? Math.round(this.testDurations.reduce((a, b) => a + b, 0) / this.testDurations.length)
|
|
209
407
|
: 0;
|
|
408
|
+
// Determine if this was a normal completion or interrupted
|
|
409
|
+
const wasInterrupted = result.status === 'interrupted';
|
|
410
|
+
const completedNonSkipped = this.totalTests - this.skippedTests;
|
|
210
411
|
// Build run metrics
|
|
211
412
|
const runMetrics = {
|
|
212
413
|
testRunId: this.testRunId,
|
|
@@ -222,10 +423,13 @@ class DatadogReporter {
|
|
|
222
423
|
retryTimeMs: this.retryTimeMs,
|
|
223
424
|
navigationMs: this.navigationMs,
|
|
224
425
|
allPassed: this.failedTests === 0 && result.status === 'passed',
|
|
225
|
-
passRate: (0, transform_1.calculatePassRate)(this.passedTests,
|
|
226
|
-
flakyRate: (0, transform_1.calculateFlakyRate)(this.flakyTests,
|
|
426
|
+
passRate: (0, transform_1.calculatePassRate)(this.passedTests, completedNonSkipped),
|
|
427
|
+
flakyRate: (0, transform_1.calculateFlakyRate)(this.flakyTests, completedNonSkipped),
|
|
227
428
|
workerCount: this.workerCount,
|
|
228
429
|
avgTestDurationMs,
|
|
430
|
+
terminatedEarly: wasInterrupted || this.terminatedEarly,
|
|
431
|
+
terminationReason: wasInterrupted ? 'user_interrupted' : this.terminationReason,
|
|
432
|
+
testsCompleted: this.testsCompleted,
|
|
229
433
|
};
|
|
230
434
|
// Build base tags
|
|
231
435
|
const baseTags = {
|
|
@@ -238,6 +442,8 @@ class DatadogReporter {
|
|
|
238
442
|
product: this.options.product,
|
|
239
443
|
jobName: this.options.ci?.jobName,
|
|
240
444
|
buildNumber: this.options.ci?.buildNumber,
|
|
445
|
+
terminatedEarly: runMetrics.terminatedEarly,
|
|
446
|
+
terminationReason: runMetrics.terminationReason,
|
|
241
447
|
};
|
|
242
448
|
// Build metrics payload
|
|
243
449
|
const payload = (0, transform_1.buildMetricsPayload)(runMetrics, this.testMetrics, baseTags);
|
|
@@ -279,6 +485,7 @@ class DatadogReporter {
|
|
|
279
485
|
await this.datadog.submitMetrics(payload);
|
|
280
486
|
await this.datadog.dispose();
|
|
281
487
|
}
|
|
488
|
+
this.metricsSubmitted = true;
|
|
282
489
|
if (this.options.debug) {
|
|
283
490
|
// eslint-disable-next-line no-console
|
|
284
491
|
console.log('[Datadog Reporter] Finished');
|
|
@@ -39,6 +39,12 @@ const formatTags = (tags) => {
|
|
|
39
39
|
if (tags.buildNumber) {
|
|
40
40
|
formattedTags.push(`build_number:${tags.buildNumber}`);
|
|
41
41
|
}
|
|
42
|
+
if (tags.terminatedEarly !== undefined) {
|
|
43
|
+
formattedTags.push(`terminated_early:${tags.terminatedEarly}`);
|
|
44
|
+
}
|
|
45
|
+
if (tags.terminationReason) {
|
|
46
|
+
formattedTags.push(`termination_reason:${tags.terminationReason}`);
|
|
47
|
+
}
|
|
42
48
|
return formattedTags;
|
|
43
49
|
};
|
|
44
50
|
exports.formatTags = formatTags;
|
|
@@ -116,6 +122,14 @@ const transformTestRunMetrics = (runMetrics, baseTags) => {
|
|
|
116
122
|
if (runMetrics.timedOutTests > 0) {
|
|
117
123
|
series.push(createCount('test_run.timeout_count', runMetrics.timedOutTests, timestamp, tags));
|
|
118
124
|
}
|
|
125
|
+
// Tests completed (useful for partial/terminated runs)
|
|
126
|
+
if (runMetrics.testsCompleted !== undefined) {
|
|
127
|
+
series.push(createGauge('test_run.tests_completed', runMetrics.testsCompleted, timestamp, tags));
|
|
128
|
+
}
|
|
129
|
+
// Early termination indicator (1 if terminated early, 0 otherwise)
|
|
130
|
+
if (runMetrics.terminatedEarly !== undefined) {
|
|
131
|
+
series.push(createGauge('test_run.terminated_early', runMetrics.terminatedEarly ? 1 : 0, timestamp, tags));
|
|
132
|
+
}
|
|
119
133
|
return series;
|
|
120
134
|
};
|
|
121
135
|
exports.transformTestRunMetrics = transformTestRunMetrics;
|