@lokalise/playwright-reporters 1.9.2 → 1.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -103,3 +103,4 @@ When performing a release, make sure to follow our conventional commit approach,
103
103
 
104
104
  This project is APACHE, VERSION 2.0 licensed, see LICENSE.md for details.
105
105
 
106
+
@@ -1,2 +1,2 @@
1
1
  export { default as DatadogReporter, defineDatadogReporterConfig } from './reporter';
2
- export type { DatadogReporterOptions, DatadogSite, MetricTags, TestRunMetrics, TestMetrics, } from './report.types';
2
+ export type { DatadogReporterOptions, DatadogSite, MetricTags, TestRunMetrics, TestMetrics, TerminationReason, } from './report.types';
@@ -92,6 +92,10 @@ export interface MetricSeries {
92
92
  export interface DatadogMetricsPayload {
93
93
  series: MetricSeries[];
94
94
  }
95
+ /**
96
+ * Termination reasons for early exit
97
+ */
98
+ export type TerminationReason = 'failure_rate_exceeded' | 'user_interrupted' | 'timeout' | 'unknown';
95
99
  /**
96
100
  * Aggregated test run data for metrics generation
97
101
  */
@@ -130,6 +134,12 @@ export interface TestRunMetrics {
130
134
  workerCount: number;
131
135
  /** Average test duration in milliseconds */
132
136
  avgTestDurationMs: number;
137
+ /** Whether the test run was terminated early */
138
+ terminatedEarly?: boolean;
139
+ /** Reason for early termination */
140
+ terminationReason?: TerminationReason;
141
+ /** Number of tests that actually completed (for partial runs) */
142
+ testsCompleted?: number;
133
143
  }
134
144
  /**
135
145
  * Individual test metrics
@@ -176,4 +186,8 @@ export interface MetricTags {
176
186
  jobName?: string;
177
187
  /** CI build number */
178
188
  buildNumber?: string;
189
+ /** Whether the test run was terminated early */
190
+ terminatedEarly?: boolean;
191
+ /** Reason for early termination */
192
+ terminationReason?: TerminationReason;
179
193
  }
@@ -23,7 +23,28 @@ export default class DatadogReporter implements Reporter {
23
23
  private readonly testMetrics;
24
24
  private projectNames;
25
25
  private readonly currentTestNavigationMs;
26
+ private testsCompleted;
27
+ private metricsSubmitted;
28
+ private terminatedEarly;
29
+ private terminationReason;
26
30
  constructor(options: DatadogReporterOptions);
31
+ /**
32
+ * Register handlers for process signals to capture metrics on early termination
33
+ */
34
+ private registerSignalHandlers;
35
+ /**
36
+ * Detect the reason for early termination based on signal and current state
37
+ */
38
+ private detectTerminationReason;
39
+ /**
40
+ * Submit partial metrics synchronously (for use in signal handlers)
41
+ * Uses a fire-and-forget HTTP request since we can't await in exit handlers
42
+ */
43
+ private submitPartialMetricsSync;
44
+ /**
45
+ * Send metrics using a detached child process (works in exit handlers)
46
+ */
47
+ private sendMetricsViaChildProcess;
27
48
  onBegin(config: FullConfig, suite: Suite): Promise<void>;
28
49
  onStepEnd(test: TestCase, result: TestResult, step: TestStep): void;
29
50
  onTestEnd(test: TestCase, result: TestResult): void;
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.defineDatadogReporterConfig = void 0;
4
4
  /* eslint-disable max-statements */
5
+ const child_process_1 = require("child_process");
5
6
  const faker_1 = require("@faker-js/faker");
6
7
  const datadog_1 = require("./datadog");
7
8
  const transform_1 = require("./transform");
@@ -38,6 +39,11 @@ class DatadogReporter {
38
39
  testMetrics;
39
40
  projectNames;
40
41
  currentTestNavigationMs;
42
+ // Early termination tracking
43
+ testsCompleted;
44
+ metricsSubmitted;
45
+ terminatedEarly;
46
+ terminationReason;
41
47
  constructor(options) {
42
48
  this.options = options;
43
49
  this.testRunId = faker_1.faker.string.uuid();
@@ -63,6 +69,186 @@ class DatadogReporter {
63
69
  this.testMetrics = [];
64
70
  this.projectNames = [];
65
71
  this.currentTestNavigationMs = new Map();
72
+ // Early termination tracking
73
+ this.testsCompleted = 0;
74
+ this.metricsSubmitted = false;
75
+ this.terminatedEarly = false;
76
+ this.terminationReason = undefined;
77
+ // Register signal handlers for early termination
78
+ this.registerSignalHandlers();
79
+ }
80
+ /**
81
+ * Register handlers for process signals to capture metrics on early termination
82
+ */
83
+ registerSignalHandlers() {
84
+ const handleEarlyTermination = (signal) => {
85
+ if (this.metricsSubmitted) {
86
+ return; // Already submitted, nothing to do
87
+ }
88
+ this.terminatedEarly = true;
89
+ this.terminationReason = this.detectTerminationReason(signal);
90
+ if (this.options.debug) {
91
+ // eslint-disable-next-line no-console
92
+ console.log(`[Datadog Reporter] Early termination detected (${signal}), submitting partial metrics...`);
93
+ }
94
+ // Submit partial metrics synchronously before exit
95
+ this.submitPartialMetricsSync();
96
+ };
97
+ // Handle SIGINT (Ctrl+C)
98
+ process.on('SIGINT', () => handleEarlyTermination('SIGINT'));
99
+ // Handle SIGTERM (kill command)
100
+ process.on('SIGTERM', () => handleEarlyTermination('SIGTERM'));
101
+ // Handle beforeExit (process about to exit naturally)
102
+ process.on('beforeExit', () => {
103
+ if (!this.metricsSubmitted && this.testsCompleted > 0) {
104
+ this.terminatedEarly = true;
105
+ this.terminationReason = this.terminationReason ?? 'unknown';
106
+ handleEarlyTermination('beforeExit');
107
+ }
108
+ });
109
+ // Handle uncaught exceptions
110
+ process.on('uncaughtException', () => {
111
+ if (!this.metricsSubmitted && this.testsCompleted > 0) {
112
+ this.terminatedEarly = true;
113
+ this.terminationReason = 'unknown';
114
+ handleEarlyTermination('uncaughtException');
115
+ }
116
+ });
117
+ // Handle exit event (last chance to do sync operations)
118
+ process.on('exit', () => {
119
+ if (!this.metricsSubmitted && this.testsCompleted > 0) {
120
+ this.terminatedEarly = true;
121
+ this.terminationReason = this.terminationReason ?? 'unknown';
122
+ // Note: We can only do sync operations in 'exit' handler
123
+ // The submitPartialMetricsSync method handles this
124
+ this.submitPartialMetricsSync();
125
+ }
126
+ });
127
+ }
128
+ /**
129
+ * Detect the reason for early termination based on signal and current state
130
+ */
131
+ detectTerminationReason(signal) {
132
+ // Check if failure rate exceeded by looking at current stats
133
+ const completedNonSkipped = this.testsCompleted - this.skippedTests;
134
+ if (completedNonSkipped > 0) {
135
+ const currentFailureRate = (this.failedTests / completedNonSkipped) * 100;
136
+ // If failure rate is high and we haven't completed all tests, likely terminated by failure rate monitor
137
+ if (currentFailureRate > 5 && this.testsCompleted < this.totalTests) {
138
+ return 'failure_rate_exceeded';
139
+ }
140
+ }
141
+ if (signal === 'SIGINT') {
142
+ return 'user_interrupted';
143
+ }
144
+ return 'unknown';
145
+ }
146
+ /**
147
+ * Submit partial metrics synchronously (for use in signal handlers)
148
+ * Uses a fire-and-forget HTTP request since we can't await in exit handlers
149
+ */
150
+ submitPartialMetricsSync() {
151
+ if (this.metricsSubmitted || this.options.dryRun) {
152
+ return;
153
+ }
154
+ try {
155
+ this.runEndTime = Date.now();
156
+ // Calculate metrics from what we have
157
+ const completedNonSkipped = this.testsCompleted - this.skippedTests;
158
+ const avgTestDurationMs = this.testDurations.length > 0
159
+ ? Math.round(this.testDurations.reduce((a, b) => a + b, 0) / this.testDurations.length)
160
+ : 0;
161
+ const runMetrics = {
162
+ testRunId: this.testRunId,
163
+ timestamp: Math.floor(this.runStartTime / 1000),
164
+ totalTests: this.totalTests,
165
+ passedTests: this.passedTests,
166
+ failedTests: this.failedTests,
167
+ flakyTests: this.flakyTests,
168
+ skippedTests: this.skippedTests,
169
+ timedOutTests: this.timedOutTests,
170
+ durationMs: this.runEndTime - this.runStartTime,
171
+ browserCreationMs: this.browserCreationMs,
172
+ retryTimeMs: this.retryTimeMs,
173
+ navigationMs: this.navigationMs,
174
+ allPassed: false, // Early termination means not all passed
175
+ passRate: (0, transform_1.calculatePassRate)(this.passedTests, completedNonSkipped),
176
+ flakyRate: (0, transform_1.calculateFlakyRate)(this.flakyTests, completedNonSkipped),
177
+ workerCount: this.workerCount,
178
+ avgTestDurationMs,
179
+ terminatedEarly: this.terminatedEarly,
180
+ terminationReason: this.terminationReason,
181
+ testsCompleted: this.testsCompleted,
182
+ };
183
+ const baseTags = {
184
+ branch: this.options.ci?.branchName,
185
+ project: this.projectNames.length > 0 ? this.projectNames : undefined,
186
+ prenvId: this.options.prenvId,
187
+ triggeredBy: this.options.ci?.triggeredBy,
188
+ env: this.options.env,
189
+ service: this.options.service ?? 'playwright-tests',
190
+ product: this.options.product,
191
+ jobName: this.options.ci?.jobName,
192
+ buildNumber: this.options.ci?.buildNumber,
193
+ terminatedEarly: this.terminatedEarly,
194
+ terminationReason: this.terminationReason,
195
+ };
196
+ const payload = (0, transform_1.buildMetricsPayload)(runMetrics, this.testMetrics, baseTags);
197
+ // eslint-disable-next-line no-console
198
+ console.log(`\n[Datadog Reporter] Early Termination - Submitting partial metrics:`);
199
+ // eslint-disable-next-line no-console
200
+ console.log(` Tests Completed: ${this.testsCompleted}/${this.totalTests}`);
201
+ // eslint-disable-next-line no-console
202
+ console.log(` Passed: ${this.passedTests}, Failed: ${this.failedTests}`);
203
+ // eslint-disable-next-line no-console
204
+ console.log(` Termination Reason: ${this.terminationReason}`);
205
+ // Use synchronous HTTP request via child_process for exit handlers
206
+ // This is a workaround since we can't use async in exit handlers
207
+ this.sendMetricsViaChildProcess(payload);
208
+ this.metricsSubmitted = true;
209
+ }
210
+ catch (error) {
211
+ // eslint-disable-next-line no-console
212
+ console.error('[Datadog Reporter] Error submitting partial metrics:', error);
213
+ }
214
+ }
215
+ /**
216
+ * Send metrics using a detached child process (works in exit handlers)
217
+ */
218
+ sendMetricsViaChildProcess(payload) {
219
+ try {
220
+ const site = this.options.site ?? 'datadoghq.eu';
221
+ const endpoints = {
222
+ 'datadoghq.com': 'https://api.datadoghq.com',
223
+ 'datadoghq.eu': 'https://api.datadoghq.eu',
224
+ 'us3.datadoghq.com': 'https://api.us3.datadoghq.com',
225
+ 'us5.datadoghq.com': 'https://api.us5.datadoghq.com',
226
+ 'ap1.datadoghq.com': 'https://api.ap1.datadoghq.com',
227
+ 'ddog-gov.com': 'https://api.ddog-gov.com',
228
+ };
229
+ const endpoint = endpoints[site] || endpoints['datadoghq.eu'];
230
+ // Use curl to send metrics (available on most systems)
231
+ const curlArgs = [
232
+ '-X', 'POST',
233
+ `${endpoint}/api/v2/series`,
234
+ '-H', 'Content-Type: application/json',
235
+ '-H', `DD-API-KEY: ${this.options.apiKey}`,
236
+ '-d', JSON.stringify(payload),
237
+ '--silent',
238
+ '--show-error',
239
+ ];
240
+ const child = (0, child_process_1.spawn)('curl', curlArgs, {
241
+ detached: true,
242
+ stdio: 'ignore',
243
+ });
244
+ child.unref();
245
+ // eslint-disable-next-line no-console
246
+ console.log('[Datadog Reporter] Metrics submission initiated via curl');
247
+ }
248
+ catch (error) {
249
+ // eslint-disable-next-line no-console
250
+ console.error('[Datadog Reporter] Failed to spawn curl process:', error);
251
+ }
66
252
  }
67
253
  async onBegin(config, suite) {
68
254
  try {
@@ -119,6 +305,10 @@ class DatadogReporter {
119
305
  const projectName = test.parent?.project()?.name;
120
306
  const testKey = `${test.id}-${currentRetryCount}`;
121
307
  const testNavigationMs = this.currentTestNavigationMs.get(testKey) || 0;
308
+ // Increment completed tests counter (only on final attempt for each test)
309
+ if (currentRetryCount === 0 || currentRetryCount === maxRetriesCount || status === 'passed') {
310
+ this.testsCompleted++;
311
+ }
122
312
  // Track test duration for average calculation (only on final attempt)
123
313
  if (currentRetryCount === 0 || currentRetryCount === maxRetriesCount) {
124
314
  this.testDurations.push(duration);
@@ -201,12 +391,23 @@ class DatadogReporter {
201
391
  }
202
392
  }
203
393
  async onEnd(result) {
394
+ // Skip if metrics were already submitted via signal handler
395
+ if (this.metricsSubmitted) {
396
+ if (this.options.debug) {
397
+ // eslint-disable-next-line no-console
398
+ console.log('[Datadog Reporter] Metrics already submitted, skipping onEnd');
399
+ }
400
+ return;
401
+ }
204
402
  try {
205
403
  this.runEndTime = Date.now();
206
404
  // Calculate average test duration
207
405
  const avgTestDurationMs = this.testDurations.length > 0
208
406
  ? Math.round(this.testDurations.reduce((a, b) => a + b, 0) / this.testDurations.length)
209
407
  : 0;
408
+ // Determine if this was a normal completion or interrupted
409
+ const wasInterrupted = result.status === 'interrupted';
410
+ const completedNonSkipped = this.totalTests - this.skippedTests;
210
411
  // Build run metrics
211
412
  const runMetrics = {
212
413
  testRunId: this.testRunId,
@@ -222,10 +423,13 @@ class DatadogReporter {
222
423
  retryTimeMs: this.retryTimeMs,
223
424
  navigationMs: this.navigationMs,
224
425
  allPassed: this.failedTests === 0 && result.status === 'passed',
225
- passRate: (0, transform_1.calculatePassRate)(this.passedTests, this.totalTests - this.skippedTests),
226
- flakyRate: (0, transform_1.calculateFlakyRate)(this.flakyTests, this.totalTests - this.skippedTests),
426
+ passRate: (0, transform_1.calculatePassRate)(this.passedTests, completedNonSkipped),
427
+ flakyRate: (0, transform_1.calculateFlakyRate)(this.flakyTests, completedNonSkipped),
227
428
  workerCount: this.workerCount,
228
429
  avgTestDurationMs,
430
+ terminatedEarly: wasInterrupted || this.terminatedEarly,
431
+ terminationReason: wasInterrupted ? 'user_interrupted' : this.terminationReason,
432
+ testsCompleted: this.testsCompleted,
229
433
  };
230
434
  // Build base tags
231
435
  const baseTags = {
@@ -238,6 +442,8 @@ class DatadogReporter {
238
442
  product: this.options.product,
239
443
  jobName: this.options.ci?.jobName,
240
444
  buildNumber: this.options.ci?.buildNumber,
445
+ terminatedEarly: runMetrics.terminatedEarly,
446
+ terminationReason: runMetrics.terminationReason,
241
447
  };
242
448
  // Build metrics payload
243
449
  const payload = (0, transform_1.buildMetricsPayload)(runMetrics, this.testMetrics, baseTags);
@@ -279,6 +485,7 @@ class DatadogReporter {
279
485
  await this.datadog.submitMetrics(payload);
280
486
  await this.datadog.dispose();
281
487
  }
488
+ this.metricsSubmitted = true;
282
489
  if (this.options.debug) {
283
490
  // eslint-disable-next-line no-console
284
491
  console.log('[Datadog Reporter] Finished');
@@ -39,6 +39,12 @@ const formatTags = (tags) => {
39
39
  if (tags.buildNumber) {
40
40
  formattedTags.push(`build_number:${tags.buildNumber}`);
41
41
  }
42
+ if (tags.terminatedEarly !== undefined) {
43
+ formattedTags.push(`terminated_early:${tags.terminatedEarly}`);
44
+ }
45
+ if (tags.terminationReason) {
46
+ formattedTags.push(`termination_reason:${tags.terminationReason}`);
47
+ }
42
48
  return formattedTags;
43
49
  };
44
50
  exports.formatTags = formatTags;
@@ -116,6 +122,14 @@ const transformTestRunMetrics = (runMetrics, baseTags) => {
116
122
  if (runMetrics.timedOutTests > 0) {
117
123
  series.push(createCount('test_run.timeout_count', runMetrics.timedOutTests, timestamp, tags));
118
124
  }
125
+ // Tests completed (useful for partial/terminated runs)
126
+ if (runMetrics.testsCompleted !== undefined) {
127
+ series.push(createGauge('test_run.tests_completed', runMetrics.testsCompleted, timestamp, tags));
128
+ }
129
+ // Early termination indicator (1 if terminated early, 0 otherwise)
130
+ if (runMetrics.terminatedEarly !== undefined) {
131
+ series.push(createGauge('test_run.terminated_early', runMetrics.terminatedEarly ? 1 : 0, timestamp, tags));
132
+ }
119
133
  return series;
120
134
  };
121
135
  exports.transformTestRunMetrics = transformTestRunMetrics;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lokalise/playwright-reporters",
3
- "version": "1.9.2",
3
+ "version": "1.10.0",
4
4
  "scripts": {
5
5
  "lint:eslint": "eslint --cache . --ext .js,.cjs,.ts",
6
6
  "lint:ts": "tsc --noEmit",