@clipboard-health/playwright-reporter-llm 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +111 -0
- package/package.json +29 -0
- package/src/index.d.ts +2 -0
- package/src/index.js +7 -0
- package/src/index.js.map +1 -0
- package/src/lib/reporter.d.ts +14 -0
- package/src/lib/reporter.js +245 -0
- package/src/lib/reporter.js.map +1 -0
- package/src/lib/types.d.ts +73 -0
- package/src/lib/types.js +3 -0
- package/src/lib/types.js.map +1 -0
package/README.md
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# @clipboard-health/playwright-reporter-llm <!-- omit from toc -->
|
|
2
|
+
|
|
3
|
+
Playwright reporter that outputs structured JSON for LLM agents. Minimal console output, flat schema, easy to filter to failures.
|
|
4
|
+
|
|
5
|
+
## Table of contents <!-- omit from toc -->
|
|
6
|
+
|
|
7
|
+
- [Install](#install)
|
|
8
|
+
- [Usage](#usage)
|
|
9
|
+
- [Local development commands](#local-development-commands)
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npm install @clipboard-health/playwright-reporter-llm
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
```typescript
|
|
18
|
+
// playwright.config.ts
|
|
19
|
+
import { defineConfig } from "@playwright/test";
|
|
20
|
+
|
|
21
|
+
export default defineConfig({
|
|
22
|
+
reporter: [
|
|
23
|
+
["@clipboard-health/playwright-reporter-llm", { outputFile: "test-results/llm-report.json" }],
|
|
24
|
+
],
|
|
25
|
+
});
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Usage
|
|
29
|
+
|
|
30
|
+
### Options
|
|
31
|
+
|
|
32
|
+
| Option | Default | Description |
|
|
33
|
+
| ------------ | ------------------------------ | ------------------- |
|
|
34
|
+
| `outputFile` | `test-results/llm-report.json` | Path to JSON output |
|
|
35
|
+
|
|
36
|
+
### Console output
|
|
37
|
+
|
|
38
|
+
```text
|
|
39
|
+
.......F..........F.S....
|
|
40
|
+
26 tests | 23 passed | 2 failed | 1 skipped (4.2s)
|
|
41
|
+
Report: test-results/llm-report.json
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
### Report schema
|
|
45
|
+
|
|
46
|
+
```json
|
|
47
|
+
{
|
|
48
|
+
"schemaVersion": 1,
|
|
49
|
+
"timestamp": "2026-02-25T19:00:00.000Z",
|
|
50
|
+
"durationMs": 4200,
|
|
51
|
+
"summary": {
|
|
52
|
+
"total": 26,
|
|
53
|
+
"passed": 23,
|
|
54
|
+
"failed": 2,
|
|
55
|
+
"flaky": 0,
|
|
56
|
+
"skipped": 1,
|
|
57
|
+
"timedOut": 0,
|
|
58
|
+
"interrupted": 0
|
|
59
|
+
},
|
|
60
|
+
"environment": {
|
|
61
|
+
"playwrightVersion": "1.58.2",
|
|
62
|
+
"nodeVersion": "v24.13.1",
|
|
63
|
+
"os": "darwin",
|
|
64
|
+
"workers": 4,
|
|
65
|
+
"retries": 1,
|
|
66
|
+
"projects": ["chromium"]
|
|
67
|
+
},
|
|
68
|
+
"tests": [
|
|
69
|
+
{
|
|
70
|
+
"id": "abc123",
|
|
71
|
+
"title": "Suite > should work",
|
|
72
|
+
"status": "failed",
|
|
73
|
+
"flaky": false,
|
|
74
|
+
"durationMs": 150,
|
|
75
|
+
"location": { "file": "tests/example.spec.ts", "line": 10, "column": 5 },
|
|
76
|
+
"project": "chromium",
|
|
77
|
+
"tags": [],
|
|
78
|
+
"annotations": [],
|
|
79
|
+
"retries": 1,
|
|
80
|
+
"errors": [
|
|
81
|
+
{
|
|
82
|
+
"message": "Expected: 1, Received: 2",
|
|
83
|
+
"stack": "...",
|
|
84
|
+
"diff": { "expected": "1", "actual": "2" },
|
|
85
|
+
"location": { "file": "tests/example.spec.ts", "line": 12, "column": 5 }
|
|
86
|
+
}
|
|
87
|
+
],
|
|
88
|
+
"attachments": [
|
|
89
|
+
{ "name": "screenshot", "contentType": "image/png", "path": "screenshot.png" }
|
|
90
|
+
],
|
|
91
|
+
"stdout": "",
|
|
92
|
+
"stderr": ""
|
|
93
|
+
}
|
|
94
|
+
],
|
|
95
|
+
"globalErrors": []
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Key fields for agents:
|
|
100
|
+
|
|
101
|
+
- **`summary`** -- quick pass/fail counts
|
|
102
|
+
- **`tests[].errors[].message`** -- ANSI-stripped, clean error text
|
|
103
|
+
- **`tests[].errors[].diff`** -- extracted expected/actual from assertion errors
|
|
104
|
+
- **`tests[].errors[].location`** -- exact file and line of failure
|
|
105
|
+
- **`tests[].flaky`** -- true if test passed after retry
|
|
106
|
+
- **`tests[].attachments[].path`** -- relative to Playwright outputDir
|
|
107
|
+
- **`stdout`/`stderr`** -- capped at 4KB with `[truncated]` marker
|
|
108
|
+
|
|
109
|
+
## Local development commands
|
|
110
|
+
|
|
111
|
+
See [`package.json`](./package.json) `scripts` for a list of commands.
|
package/package.json
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@clipboard-health/playwright-reporter-llm",
|
|
3
|
+
"description": "Playwright reporter that outputs structured JSON for LLM agents. Minimal console output, flat schema, easy to filter to failures.",
|
|
4
|
+
"version": "1.0.0",
|
|
5
|
+
"bugs": "https://github.com/ClipboardHealth/core-utils/issues",
|
|
6
|
+
"dependencies": {
|
|
7
|
+
"tslib": "2.8.1"
|
|
8
|
+
},
|
|
9
|
+
"devDependencies": {
|
|
10
|
+
"@playwright/test": "1.58.2"
|
|
11
|
+
},
|
|
12
|
+
"keywords": [],
|
|
13
|
+
"license": "MIT",
|
|
14
|
+
"main": "./src/index.js",
|
|
15
|
+
"peerDependencies": {
|
|
16
|
+
"@playwright/test": ">=1.50.0"
|
|
17
|
+
},
|
|
18
|
+
"publishConfig": {
|
|
19
|
+
"access": "public"
|
|
20
|
+
},
|
|
21
|
+
"repository": {
|
|
22
|
+
"directory": "packages/playwright-reporter-llm",
|
|
23
|
+
"type": "git",
|
|
24
|
+
"url": "git+https://github.com/ClipboardHealth/core-utils.git"
|
|
25
|
+
},
|
|
26
|
+
"type": "commonjs",
|
|
27
|
+
"typings": "./src/index.d.ts",
|
|
28
|
+
"types": "./src/index.d.ts"
|
|
29
|
+
}
|
package/src/index.d.ts
ADDED
package/src/index.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.default = void 0;
|
|
4
|
+
const tslib_1 = require("tslib");
|
|
5
|
+
var reporter_1 = require("./lib/reporter");
|
|
6
|
+
Object.defineProperty(exports, "default", { enumerable: true, get: function () { return tslib_1.__importDefault(reporter_1).default; } });
|
|
7
|
+
//# sourceMappingURL=index.js.map
|
package/src/index.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../packages/playwright-reporter-llm/src/index.ts"],"names":[],"mappings":";;;;AAAA,2CAAyC;AAAhC,4HAAA,OAAO,OAAA"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError as PlaywrightTestError, TestResult } from "@playwright/test/reporter";
|
|
2
|
+
import type { LlmReporterOptions } from "./types";
|
|
3
|
+
export default class LlmReporter implements Reporter {
|
|
4
|
+
private readonly outputFile;
|
|
5
|
+
private config;
|
|
6
|
+
private readonly entriesById;
|
|
7
|
+
private readonly globalErrors;
|
|
8
|
+
private startTimeMs;
|
|
9
|
+
constructor(options?: LlmReporterOptions);
|
|
10
|
+
onBegin(config: FullConfig, _suite: Suite): void;
|
|
11
|
+
onTestEnd(test: TestCase, result: TestResult): void;
|
|
12
|
+
onError(error: PlaywrightTestError): void;
|
|
13
|
+
onEnd(_result: FullResult): void;
|
|
14
|
+
}
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const tslib_1 = require("tslib");
|
|
4
|
+
const node_crypto_1 = require("node:crypto");
|
|
5
|
+
const node_fs_1 = require("node:fs");
|
|
6
|
+
const node_path_1 = tslib_1.__importDefault(require("node:path"));
|
|
7
|
+
const STDOUT_CAP = 4096;
|
|
8
|
+
const TRUNCATION_MARKER = "[truncated]";
|
|
9
|
+
function stripAnsi(text) {
|
|
10
|
+
// eslint-disable-next-line no-control-regex
|
|
11
|
+
return text.replaceAll(/\u001B\[[0-9;]*m/g, "");
|
|
12
|
+
}
|
|
13
|
+
function capOutput(text) {
|
|
14
|
+
if (text.length <= STDOUT_CAP) {
|
|
15
|
+
return text;
|
|
16
|
+
}
|
|
17
|
+
return `${text.slice(0, STDOUT_CAP - TRUNCATION_MARKER.length)}${TRUNCATION_MARKER}`;
|
|
18
|
+
}
|
|
19
|
+
function buildFullTitle(test) {
|
|
20
|
+
const parts = [];
|
|
21
|
+
let current = test.parent;
|
|
22
|
+
while (current) {
|
|
23
|
+
if (current.title && current.type === "describe") {
|
|
24
|
+
parts.unshift(current.title);
|
|
25
|
+
}
|
|
26
|
+
current = current.parent;
|
|
27
|
+
}
|
|
28
|
+
parts.push(test.title);
|
|
29
|
+
return parts.join(" > ");
|
|
30
|
+
}
|
|
31
|
+
function filterStackToUserCode(stack) {
|
|
32
|
+
return stack
|
|
33
|
+
.split("\n")
|
|
34
|
+
.filter((line) => {
|
|
35
|
+
if (!line.includes(" at ")) {
|
|
36
|
+
return true;
|
|
37
|
+
}
|
|
38
|
+
return !line.includes("node_modules") && !line.includes("node:internal");
|
|
39
|
+
})
|
|
40
|
+
.join("\n");
|
|
41
|
+
}
|
|
42
|
+
function extractDiff(message) {
|
|
43
|
+
const lines = message.split("\n");
|
|
44
|
+
let expectedValue;
|
|
45
|
+
let actualValue;
|
|
46
|
+
for (const line of lines) {
|
|
47
|
+
const expectedMatch = /^Expected\b[^:]*:[ \t]*(.+)/i.exec(line);
|
|
48
|
+
if (expectedMatch?.[1]) {
|
|
49
|
+
expectedValue = stripAnsi(expectedMatch[1]).trim();
|
|
50
|
+
}
|
|
51
|
+
const receivedMatch = /^Received\b[^:]*:[ \t]*(.+)/i.exec(line);
|
|
52
|
+
if (receivedMatch?.[1]) {
|
|
53
|
+
actualValue = stripAnsi(receivedMatch[1]).trim();
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
if (expectedValue && actualValue) {
|
|
57
|
+
return { expected: expectedValue, actual: actualValue };
|
|
58
|
+
}
|
|
59
|
+
return undefined;
|
|
60
|
+
}
|
|
61
|
+
function buildTestError(error) {
|
|
62
|
+
const testError = {
|
|
63
|
+
message: stripAnsi(error.message ?? "Unknown error"),
|
|
64
|
+
};
|
|
65
|
+
if (error.stack) {
|
|
66
|
+
testError.stack = filterStackToUserCode(stripAnsi(error.stack));
|
|
67
|
+
}
|
|
68
|
+
if (error.snippet) {
|
|
69
|
+
testError.snippet = stripAnsi(error.snippet);
|
|
70
|
+
}
|
|
71
|
+
const diff = extractDiff(error.message ?? "");
|
|
72
|
+
if (diff) {
|
|
73
|
+
testError.diff = diff;
|
|
74
|
+
}
|
|
75
|
+
if (error.location) {
|
|
76
|
+
const { file, line, column } = error.location;
|
|
77
|
+
testError.location = { file, line, column };
|
|
78
|
+
}
|
|
79
|
+
return testError;
|
|
80
|
+
}
|
|
81
|
+
function statusIndicator(status) {
|
|
82
|
+
switch (status) {
|
|
83
|
+
case "passed": {
|
|
84
|
+
return ".";
|
|
85
|
+
}
|
|
86
|
+
case "failed": {
|
|
87
|
+
return "F";
|
|
88
|
+
}
|
|
89
|
+
case "timedOut": {
|
|
90
|
+
return "T";
|
|
91
|
+
}
|
|
92
|
+
case "interrupted": {
|
|
93
|
+
return "I";
|
|
94
|
+
}
|
|
95
|
+
case "skipped": {
|
|
96
|
+
return "S";
|
|
97
|
+
}
|
|
98
|
+
default: {
|
|
99
|
+
return "?";
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
function collectStdio(result, channel) {
|
|
104
|
+
const text = result[channel]
|
|
105
|
+
.map((chunk) => (typeof chunk === "string" ? chunk : chunk.toString("utf8")))
|
|
106
|
+
.join("");
|
|
107
|
+
return capOutput(stripAnsi(text));
|
|
108
|
+
}
|
|
109
|
+
class LlmReporter {
|
|
110
|
+
outputFile;
|
|
111
|
+
config;
|
|
112
|
+
entriesById = new Map();
|
|
113
|
+
globalErrors = [];
|
|
114
|
+
startTimeMs = 0;
|
|
115
|
+
constructor(options = {}) {
|
|
116
|
+
this.outputFile = options.outputFile ?? "test-results/llm-report.json";
|
|
117
|
+
}
|
|
118
|
+
onBegin(config, _suite) {
|
|
119
|
+
this.config = config;
|
|
120
|
+
this.startTimeMs = Date.now();
|
|
121
|
+
this.entriesById.clear();
|
|
122
|
+
this.globalErrors.length = 0;
|
|
123
|
+
(0, node_fs_1.rmSync)(this.outputFile, { force: true });
|
|
124
|
+
}
|
|
125
|
+
onTestEnd(test, result) {
|
|
126
|
+
const config = this.config;
|
|
127
|
+
if (!config) {
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
const file = node_path_1.default.relative(config.rootDir, test.location.file);
|
|
131
|
+
const fullTitle = buildFullTitle(test);
|
|
132
|
+
const project = test.parent.project()?.name ?? "";
|
|
133
|
+
const status = result.status;
|
|
134
|
+
const retries = test.results.length - 1;
|
|
135
|
+
const isFlaky = status === "passed" && retries > 0;
|
|
136
|
+
process.stdout.write(statusIndicator(status));
|
|
137
|
+
const outputDirectory = test.parent.project()?.outputDir ?? config.rootDir;
|
|
138
|
+
const attachments = result.attachments.map((a) => ({
|
|
139
|
+
name: a.name,
|
|
140
|
+
contentType: a.contentType,
|
|
141
|
+
...(a.path && { path: node_path_1.default.relative(outputDirectory, a.path) }),
|
|
142
|
+
}));
|
|
143
|
+
const entry = {
|
|
144
|
+
id: test.id,
|
|
145
|
+
title: fullTitle,
|
|
146
|
+
status,
|
|
147
|
+
flaky: isFlaky,
|
|
148
|
+
durationMs: result.duration,
|
|
149
|
+
location: { file, line: test.location.line, column: test.location.column },
|
|
150
|
+
project,
|
|
151
|
+
tags: test.tags,
|
|
152
|
+
annotations: test.annotations,
|
|
153
|
+
retries,
|
|
154
|
+
errors: result.errors.map(buildTestError),
|
|
155
|
+
attachments,
|
|
156
|
+
stdout: collectStdio(result, "stdout"),
|
|
157
|
+
stderr: collectStdio(result, "stderr"),
|
|
158
|
+
};
|
|
159
|
+
this.entriesById.set(test.id, entry);
|
|
160
|
+
}
|
|
161
|
+
onError(error) {
|
|
162
|
+
const globalError = {
|
|
163
|
+
message: stripAnsi(error.message ?? "Unknown error"),
|
|
164
|
+
};
|
|
165
|
+
if (error.stack) {
|
|
166
|
+
globalError.stack = filterStackToUserCode(stripAnsi(error.stack));
|
|
167
|
+
}
|
|
168
|
+
this.globalErrors.push(globalError);
|
|
169
|
+
}
|
|
170
|
+
onEnd(_result) {
|
|
171
|
+
const durationMs = Date.now() - this.startTimeMs;
|
|
172
|
+
const config = this.config;
|
|
173
|
+
if (!config) {
|
|
174
|
+
// eslint-disable-next-line no-console
|
|
175
|
+
console.error("LlmReporter: onEnd called without onBegin — skipping report.");
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
const entries = [...this.entriesById.values()];
|
|
179
|
+
const summary = {
|
|
180
|
+
total: entries.length,
|
|
181
|
+
passed: 0,
|
|
182
|
+
failed: 0,
|
|
183
|
+
flaky: 0,
|
|
184
|
+
skipped: 0,
|
|
185
|
+
timedOut: 0,
|
|
186
|
+
interrupted: 0,
|
|
187
|
+
};
|
|
188
|
+
for (const entry of entries) {
|
|
189
|
+
if (entry.flaky) {
|
|
190
|
+
summary.flaky++;
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
summary[entry.status]++;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
const report = {
|
|
197
|
+
schemaVersion: 1,
|
|
198
|
+
timestamp: new Date(this.startTimeMs).toISOString(),
|
|
199
|
+
durationMs,
|
|
200
|
+
summary,
|
|
201
|
+
environment: {
|
|
202
|
+
playwrightVersion: config.version,
|
|
203
|
+
nodeVersion: process.version,
|
|
204
|
+
os: process.platform,
|
|
205
|
+
workers: config.workers,
|
|
206
|
+
retries: Math.max(0, ...config.projects.map((p) => p.retries)),
|
|
207
|
+
projects: config.projects.map((p) => p.name),
|
|
208
|
+
},
|
|
209
|
+
tests: entries,
|
|
210
|
+
globalErrors: this.globalErrors,
|
|
211
|
+
};
|
|
212
|
+
const directory = node_path_1.default.dirname(this.outputFile);
|
|
213
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename
|
|
214
|
+
(0, node_fs_1.mkdirSync)(directory, { recursive: true });
|
|
215
|
+
const temporaryFile = node_path_1.default.join(directory, `.llm-report-${(0, node_crypto_1.randomUUID)()}.tmp`);
|
|
216
|
+
try {
|
|
217
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename
|
|
218
|
+
(0, node_fs_1.writeFileSync)(temporaryFile, JSON.stringify(report, undefined, 2));
|
|
219
|
+
// eslint-disable-next-line security/detect-non-literal-fs-filename
|
|
220
|
+
(0, node_fs_1.renameSync)(temporaryFile, this.outputFile);
|
|
221
|
+
}
|
|
222
|
+
catch (error) {
|
|
223
|
+
// eslint-disable-next-line no-console
|
|
224
|
+
console.error(`LlmReporter: Failed to write report to ${this.outputFile}:`, error);
|
|
225
|
+
}
|
|
226
|
+
finally {
|
|
227
|
+
(0, node_fs_1.rmSync)(temporaryFile, { force: true });
|
|
228
|
+
}
|
|
229
|
+
const durationSeconds = (durationMs / 1000).toFixed(1);
|
|
230
|
+
const parts = [
|
|
231
|
+
`${summary.total} tests`,
|
|
232
|
+
`${summary.passed} passed`,
|
|
233
|
+
`${summary.failed} failed`,
|
|
234
|
+
`${summary.skipped} skipped`,
|
|
235
|
+
...(summary.flaky > 0 ? [`${summary.flaky} flaky`] : []),
|
|
236
|
+
...(summary.timedOut > 0 ? [`${summary.timedOut} timedOut`] : []),
|
|
237
|
+
...(summary.interrupted > 0 ? [`${summary.interrupted} interrupted`] : []),
|
|
238
|
+
];
|
|
239
|
+
const consoleSummary = `\n${parts.join(" | ")} (${durationSeconds}s)\n`;
|
|
240
|
+
process.stdout.write(consoleSummary);
|
|
241
|
+
process.stdout.write(`Report: ${this.outputFile}\n`);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
exports.default = LlmReporter;
|
|
245
|
+
//# sourceMappingURL=reporter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../../../../packages/playwright-reporter-llm/src/lib/reporter.ts"],"names":[],"mappings":";;;AAAA,6CAAyC;AACzC,qCAAuE;AACvE,kEAA6B;AAuB7B,MAAM,UAAU,GAAG,IAAI,CAAC;AACxB,MAAM,iBAAiB,GAAG,aAAa,CAAC;AAExC,SAAS,SAAS,CAAC,IAAY;IAC7B,4CAA4C;IAC5C,OAAO,IAAI,CAAC,UAAU,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,iBAAiB,EAAE,CAAC;AACvF,CAAC;AAED,SAAS,cAAc,CAAC,IAAc;IACpC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,OAAO,GAAsB,IAAI,CAAC,MAAM,CAAC;IAC7C,OAAO,OAAO,EAAE,CAAC;QACf,IAAI,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACjD,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;IAC3B,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAa;IAC1C,OAAO,KAAK;SACT,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACf,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;IAC3E,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAClC,IAAI,aAAiC,CAAC;IACtC,IAAI,WAA+B,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,aAAa,GAAG,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,aAAa,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACrD,CAAC;QACD,MAAM,aAAa,GAAG,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,WAAW,GAAG,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAED,IAAI,aAAa,IAAI,WAAW,EAAE,CAAC;QACjC,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC1D,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,cAAc,CAAC,KAA0B;IAChD,MAAM,SAAS,GAAc;QAC3B,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,IAAI,eAAe,CAAC;KACrD,CAAC;IAEF,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,SAAS,CAAC,KAAK,GAAG,qBAAqB,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IAClE,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClB,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAC9C,IAAI,IAAI,EAAE,CAAC;QACT,SAAS,CAAC,IAAI,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACnB,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC9C,SAAS,CAAC,QAAQ,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAC9C,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,MAAkB;IACzC,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,OAAO,GAAG,CAAC;QACb,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,OAAO,GAAG,CAAC;QACb,CAAC;QACD,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,OAAO,GAAG,CAAC;QACb,CAAC;QACD,KAAK,aAAa,CAAC,CAAC,CAAC;YACnB,OAAO,GAAG,CAAC;QACb,CAAC;QACD,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,OAAO,GAAG,CAAC;QACb,CAAC;QACD,OAAO,CAAC,CAAC,CAAC;YACR,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,MAAkB,EAAE,OAA4B;IACpE,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC;SACzB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;SAC5E,IAAI,CAAC,EAAE,CAAC,CAAC;IACZ,OAAO,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,MAAqB,WAAW;IACb,UAAU,CAAS;IAC5B,MAAM,CAAyB;IACtB,WAAW,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC9C,YAAY,GAAkB,EAAE,CAAC;IAC1C,WAAW,GAAG,CAAC,CAAC;IAExB,YAAmB,UAA8B,EAAE;QACjD,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,8BAA8B,CAAC;IACzE,CAAC;IAEM,OAAO,CAAC,MAAkB,EAAE,MAAa;QAC9C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9B,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7B,IAAA,gBAAM,EAAC,IAAI,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,CAAC;IAEM,SAAS,CAAC,IAAc,EAAE,MAAkB;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,mBAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC/D,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC;QAClD,MAAM,MAAM,GAAe,MAAM,CAAC,MAAM,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,MAAM,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,CAAC;QAEnD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;QAE9C,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC;QAC3E,MAAM,WAAW,GAAqB,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACnE,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,IAAI,EAAE,mBAAI,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;SAChE,CAAC,CAAC,CAAC;QAEJ,MAAM,KAAK,GAAiB;YAC1B,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,KAAK,EAAE,SAAS;YAChB,MAAM;YACN,KAAK,EAAE,OAAO;YACd,UAAU,EAAE,MAAM,CAAC,QAAQ;YAC3B,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE;YAC1E,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO;YACP,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC;YACzC,WAAW;YACX,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC;YACtC,MAAM,EAAE,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC;SACvC,CAAC;QAEF,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;IACvC,CAAC;IAEM,OAAO,CAAC,KAA0B;QACvC,MAAM,WAAW,GAAgB;YAC/B,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,IAAI,eAAe,CAAC;SACrD,CAAC;QACF,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,WAAW,CAAC,KAAK,GAAG,qBAAqB,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QACpE,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACtC,CAAC;IAEM,KAAK,CAAC,OAAmB;QAC9B,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC;QACjD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,sCAAsC;YACtC,OAAO,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;YAC9E,OAAO;QACT,CAAC;QAED,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAgB;YAC3B,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,MAAM,EAAE,CAAC;YACT,MAAM,EAAE,CAAC;YACT,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,CAAC;YACV,QAAQ,EAAE,CAAC;YACX,WAAW,EAAE,CAAC;SACf,CAAC;QAEF,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAkB;YAC5B,aAAa,EAAE,CAAC;YAChB,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE;YACnD,UAAU;YACV,OAAO;YACP,WAAW,EAAE;gBACX,iBAAiB,EAAE,MAAM,CAAC,OAAO;gBACjC,WAAW,EAAE,OAAO,CAAC,OAAO;gBAC5B,EAAE,EAAE,OAAO,CAAC,QAAQ;gBACpB,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBAC9D,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;aAC7C;YACD,KAAK,EAAE,OAAO;YACd,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAC;QAEF,MAAM,SAAS,GAAG,mBAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChD,mEAAmE;QACnE,IAAA,mBAAS,EAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,MAAM,aAAa,GAAG,mBAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,IAAA,wBAAU,GAAE,MAAM,CAAC,CAAC;QAC9E,IAAI,CAAC;YACH,mEAAmE;YACnE,IAAA,uBAAa,EAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;YACnE,mEAAmE;YACnE,IAAA,oBAAU,EAAC,aAAa,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,sCAAsC;YACtC,OAAO,CAAC,KAAK,CAAC,0CAA0C,IAAI,CAAC,UAAU,GAAG,EAAE,KAAK,CAAC,CAAC;QACrF,CAAC;gBAAS,CAAC;YACT,IAAA,gBAAM,EAAC,aAAa,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzC,CAAC;QAED,MAAM,eAAe,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG;YACZ,GAAG,OAAO,CAAC,KAAK,QAAQ;YACxB,GAAG,OAAO,CAAC,MAAM,SAAS;YAC1B,GAAG,OAAO,CAAC,MAAM,SAAS;YAC1B,GAAG,OAAO,CAAC,OAAO,UAAU;YAC5B,GAAG,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,QAAQ,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,GAAG,CAAC,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,cAAc,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC3E,CAAC;QACF,MAAM,cAAc,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,eAAe,MAAM,CAAC;QACxE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QACrC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;IACvD,CAAC;CACF;AAlJD,8BAkJC"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
export interface TestSummary {
|
|
2
|
+
total: number;
|
|
3
|
+
passed: number;
|
|
4
|
+
failed: number;
|
|
5
|
+
flaky: number;
|
|
6
|
+
skipped: number;
|
|
7
|
+
timedOut: number;
|
|
8
|
+
interrupted: number;
|
|
9
|
+
}
|
|
10
|
+
export interface TestEnvironment {
|
|
11
|
+
playwrightVersion: string;
|
|
12
|
+
nodeVersion: string;
|
|
13
|
+
os: string;
|
|
14
|
+
workers: number;
|
|
15
|
+
retries: number;
|
|
16
|
+
projects: string[];
|
|
17
|
+
}
|
|
18
|
+
export interface GlobalError {
|
|
19
|
+
message: string;
|
|
20
|
+
stack?: string;
|
|
21
|
+
}
|
|
22
|
+
export type TestStatus = "passed" | "failed" | "timedOut" | "skipped" | "interrupted";
|
|
23
|
+
export interface TestLocation {
|
|
24
|
+
file: string;
|
|
25
|
+
line: number;
|
|
26
|
+
column: number;
|
|
27
|
+
}
|
|
28
|
+
export interface TestError {
|
|
29
|
+
message: string;
|
|
30
|
+
stack?: string;
|
|
31
|
+
snippet?: string;
|
|
32
|
+
diff?: {
|
|
33
|
+
expected: string;
|
|
34
|
+
actual: string;
|
|
35
|
+
};
|
|
36
|
+
location?: TestLocation;
|
|
37
|
+
}
|
|
38
|
+
export interface TestAttachment {
|
|
39
|
+
name: string;
|
|
40
|
+
contentType: string;
|
|
41
|
+
path?: string;
|
|
42
|
+
}
|
|
43
|
+
export interface LlmTestEntry {
|
|
44
|
+
id: string;
|
|
45
|
+
title: string;
|
|
46
|
+
status: TestStatus;
|
|
47
|
+
flaky: boolean;
|
|
48
|
+
durationMs: number;
|
|
49
|
+
location: TestLocation;
|
|
50
|
+
project: string;
|
|
51
|
+
tags: string[];
|
|
52
|
+
annotations: Array<{
|
|
53
|
+
type: string;
|
|
54
|
+
description?: string;
|
|
55
|
+
}>;
|
|
56
|
+
retries: number;
|
|
57
|
+
errors: TestError[];
|
|
58
|
+
attachments: TestAttachment[];
|
|
59
|
+
stdout: string;
|
|
60
|
+
stderr: string;
|
|
61
|
+
}
|
|
62
|
+
export interface LlmReporterOptions {
|
|
63
|
+
outputFile?: string;
|
|
64
|
+
}
|
|
65
|
+
export interface LlmTestReport {
|
|
66
|
+
schemaVersion: 1;
|
|
67
|
+
timestamp: string;
|
|
68
|
+
durationMs: number;
|
|
69
|
+
summary: TestSummary;
|
|
70
|
+
environment: TestEnvironment;
|
|
71
|
+
tests: LlmTestEntry[];
|
|
72
|
+
globalErrors: GlobalError[];
|
|
73
|
+
}
|
package/src/lib/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../../packages/playwright-reporter-llm/src/lib/types.ts"],"names":[],"mappings":""}
|