flakewatch-core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/config.test.d.ts +2 -0
- package/dist/__tests__/config.test.d.ts.map +1 -0
- package/dist/__tests__/config.test.js +65 -0
- package/dist/__tests__/config.test.js.map +1 -0
- package/dist/__tests__/context.test.d.ts +2 -0
- package/dist/__tests__/context.test.d.ts.map +1 -0
- package/dist/__tests__/context.test.js +76 -0
- package/dist/__tests__/context.test.js.map +1 -0
- package/dist/__tests__/grouping.test.d.ts +2 -0
- package/dist/__tests__/grouping.test.d.ts.map +1 -0
- package/dist/__tests__/grouping.test.js +124 -0
- package/dist/__tests__/grouping.test.js.map +1 -0
- package/dist/__tests__/report.test.d.ts +2 -0
- package/dist/__tests__/report.test.d.ts.map +1 -0
- package/dist/__tests__/report.test.js +102 -0
- package/dist/__tests__/report.test.js.map +1 -0
- package/dist/config.d.ts +41 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +91 -0
- package/dist/config.js.map +1 -0
- package/dist/context.d.ts +60 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +175 -0
- package/dist/context.js.map +1 -0
- package/dist/grouping.d.ts +23 -0
- package/dist/grouping.d.ts.map +1 -0
- package/dist/grouping.js +88 -0
- package/dist/grouping.js.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/llm.d.ts +12 -0
- package/dist/llm.d.ts.map +1 -0
- package/dist/llm.js +188 -0
- package/dist/llm.js.map +1 -0
- package/dist/playwright-json.d.ts +46 -0
- package/dist/playwright-json.d.ts.map +1 -0
- package/dist/playwright-json.js +46 -0
- package/dist/playwright-json.js.map +1 -0
- package/dist/report.d.ts +7 -0
- package/dist/report.d.ts.map +1 -0
- package/dist/report.js +121 -0
- package/dist/report.js.map +1 -0
- package/dist/triage.d.ts +25 -0
- package/dist/triage.d.ts.map +1 -0
- package/dist/triage.js +91 -0
- package/dist/triage.js.map +1 -0
- package/dist/verdicts.d.ts +56 -0
- package/dist/verdicts.d.ts.map +1 -0
- package/dist/verdicts.js +2 -0
- package/dist/verdicts.js.map +1 -0
- package/package.json +35 -0
package/dist/context.js
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { resolve, dirname, relative } from 'node:path';
|
|
3
|
+
/**
|
|
4
|
+
* Package a test failure with all relevant context for LLM analysis.
|
|
5
|
+
*/
|
|
6
|
+
export async function packageFailureContext(failure, config) {
|
|
7
|
+
const testSource = await readTestSource(failure.testFile, config.maxTestSourceTokens);
|
|
8
|
+
const pageObjectSources = await extractPageObjectImports(failure.testFile, testSource, config.maxTestSourceTokens);
|
|
9
|
+
const intentAnnotation = extractIntentAnnotation(testSource);
|
|
10
|
+
const failingLine = failure.error.stack
|
|
11
|
+
? parseFailingLine(failure.error.stack, failure.testFile)
|
|
12
|
+
: undefined;
|
|
13
|
+
const context = {
|
|
14
|
+
testId: failure.testId,
|
|
15
|
+
testTitle: failure.testTitle,
|
|
16
|
+
testFile: failure.testFile,
|
|
17
|
+
testSource,
|
|
18
|
+
pageObjectSources,
|
|
19
|
+
intentAnnotation,
|
|
20
|
+
errorMessage: failure.error.message,
|
|
21
|
+
errorStack: failure.error.stack,
|
|
22
|
+
failingLine,
|
|
23
|
+
screenshotPath: failure.screenshotPath,
|
|
24
|
+
};
|
|
25
|
+
// Optional context based on config
|
|
26
|
+
if (config.includeTrace && failure.tracePath) {
|
|
27
|
+
context.traceActions = await parseTraceFile(failure.tracePath);
|
|
28
|
+
}
|
|
29
|
+
return context;
|
|
30
|
+
}
|
|
31
|
+
async function readTestSource(testFile, maxTokens) {
|
|
32
|
+
try {
|
|
33
|
+
const content = await readFile(resolve(testFile), 'utf-8');
|
|
34
|
+
// Rough token estimate: ~4 chars per token
|
|
35
|
+
const maxChars = maxTokens * 4;
|
|
36
|
+
if (content.length > maxChars) {
|
|
37
|
+
return content.slice(0, maxChars) + '\n// ... (truncated)';
|
|
38
|
+
}
|
|
39
|
+
return content;
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return '// Failed to read test source';
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Extract page object imports from the test source and read their contents.
|
|
47
|
+
*/
|
|
48
|
+
async function extractPageObjectImports(testFile, testSource, maxTokens) {
|
|
49
|
+
const sources = new Map();
|
|
50
|
+
const importRegex = /import\s+.*from\s+['"](\.[^'"]+)['"]/g;
|
|
51
|
+
const testDir = dirname(resolve(testFile));
|
|
52
|
+
let totalChars = 0;
|
|
53
|
+
const maxChars = maxTokens * 4;
|
|
54
|
+
for (const match of testSource.matchAll(importRegex)) {
|
|
55
|
+
const importPath = match[1];
|
|
56
|
+
// Try common extensions
|
|
57
|
+
for (const ext of ['', '.ts', '.js', '.tsx', '.jsx']) {
|
|
58
|
+
const fullPath = resolve(testDir, importPath + ext);
|
|
59
|
+
try {
|
|
60
|
+
const content = await readFile(fullPath, 'utf-8');
|
|
61
|
+
if (totalChars + content.length > maxChars)
|
|
62
|
+
break;
|
|
63
|
+
sources.set(relative(process.cwd(), fullPath), content);
|
|
64
|
+
totalChars += content.length;
|
|
65
|
+
break;
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
// Try next extension
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return sources;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Extract @intent annotation from test source.
|
|
76
|
+
*/
|
|
77
|
+
function extractIntentAnnotation(source) {
|
|
78
|
+
const match = source.match(/\/\/\s*@intent\s+(.+)/);
|
|
79
|
+
return match?.[1]?.trim();
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Parse the stack trace to find the failing line in the test file.
|
|
83
|
+
*/
|
|
84
|
+
function parseFailingLine(stack, testFile) {
|
|
85
|
+
const escapedFile = testFile.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
86
|
+
const regex = new RegExp(`${escapedFile}:(\\d+)(?::(\\d+))?`);
|
|
87
|
+
const match = stack.match(regex);
|
|
88
|
+
if (!match)
|
|
89
|
+
return undefined;
|
|
90
|
+
return {
|
|
91
|
+
file: testFile,
|
|
92
|
+
line: parseInt(match[1], 10),
|
|
93
|
+
column: match[2] ? parseInt(match[2], 10) : undefined,
|
|
94
|
+
code: '', // Will be filled by reading the file
|
|
95
|
+
context: '',
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
async function parseTraceFile(tracePath) {
|
|
99
|
+
// Playwright traces are zip files — for now, return empty.
|
|
100
|
+
// Full implementation will use playwright's trace viewer API.
|
|
101
|
+
return [];
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Format failure context as a prompt for LLM analysis.
|
|
105
|
+
*/
|
|
106
|
+
export function formatContextForLLM(context) {
|
|
107
|
+
const parts = [];
|
|
108
|
+
parts.push(`## Failed Test: ${context.testTitle}`);
|
|
109
|
+
parts.push(`**File:** ${context.testFile}`);
|
|
110
|
+
if (context.intentAnnotation) {
|
|
111
|
+
parts.push(`**Declared Intent:** ${context.intentAnnotation}`);
|
|
112
|
+
}
|
|
113
|
+
parts.push('');
|
|
114
|
+
parts.push('### Error');
|
|
115
|
+
parts.push('```');
|
|
116
|
+
parts.push(context.errorMessage);
|
|
117
|
+
if (context.errorStack) {
|
|
118
|
+
parts.push('');
|
|
119
|
+
parts.push(context.errorStack);
|
|
120
|
+
}
|
|
121
|
+
parts.push('```');
|
|
122
|
+
if (context.failingLine) {
|
|
123
|
+
parts.push('');
|
|
124
|
+
parts.push(`**Failing line:** ${context.failingLine.file}:${context.failingLine.line}`);
|
|
125
|
+
if (context.failingLine.code) {
|
|
126
|
+
parts.push('```');
|
|
127
|
+
parts.push(context.failingLine.code);
|
|
128
|
+
parts.push('```');
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
parts.push('');
|
|
132
|
+
parts.push('### Test Source');
|
|
133
|
+
parts.push('```typescript');
|
|
134
|
+
parts.push(context.testSource);
|
|
135
|
+
parts.push('```');
|
|
136
|
+
if (context.pageObjectSources.size > 0) {
|
|
137
|
+
parts.push('');
|
|
138
|
+
parts.push('### Imported Page Objects');
|
|
139
|
+
for (const [path, source] of context.pageObjectSources) {
|
|
140
|
+
parts.push(`\n**${path}:**`);
|
|
141
|
+
parts.push('```typescript');
|
|
142
|
+
parts.push(source);
|
|
143
|
+
parts.push('```');
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
if (context.screenshotPath) {
|
|
147
|
+
parts.push('');
|
|
148
|
+
parts.push(`### Failure Screenshot`);
|
|
149
|
+
parts.push(`Screenshot available at: ${context.screenshotPath}`);
|
|
150
|
+
}
|
|
151
|
+
if (context.traceActions && context.traceActions.length > 0) {
|
|
152
|
+
parts.push('');
|
|
153
|
+
parts.push('### Trace Actions');
|
|
154
|
+
for (const action of context.traceActions) {
|
|
155
|
+
const status = action.error ? `FAILED: ${action.error}` : 'OK';
|
|
156
|
+
parts.push(`- ${action.action}${action.selector ? ` (${action.selector})` : ''} [${action.duration}ms] ${status}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (context.consoleMessages && context.consoleMessages.length > 0) {
|
|
160
|
+
parts.push('');
|
|
161
|
+
parts.push('### Browser Console');
|
|
162
|
+
for (const msg of context.consoleMessages) {
|
|
163
|
+
parts.push(`- ${msg}`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
if (context.networkRequests && context.networkRequests.length > 0) {
|
|
167
|
+
parts.push('');
|
|
168
|
+
parts.push('### Recent Network Requests');
|
|
169
|
+
for (const req of context.networkRequests) {
|
|
170
|
+
parts.push(`- ${req.method} ${req.url} → ${req.status ?? 'pending'} (${req.duration ?? '?'}ms)`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return parts.join('\n');
|
|
174
|
+
}
|
|
175
|
+
//# sourceMappingURL=context.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context.js","sourceRoot":"","sources":["../src/context.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAgEvD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,OAAoB,EACpB,MAAqB;IAErB,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACtF,MAAM,iBAAiB,GAAG,MAAM,wBAAwB,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,CAAC,mBAAmB,CAAC,CAAC;IACnH,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,UAAU,CAAC,CAAC;IAC7D,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK;QACrC,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC;QACzD,CAAC,CAAC,SAAS,CAAC;IAEd,MAAM,OAAO,GAAmB;QAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,UAAU;QACV,iBAAiB;QACjB,gBAAgB;QAChB,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,OAAO;QACnC,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK;QAC/B,WAAW;QACX,cAAc,EAAE,OAAO,CAAC,cAAc;KACvC,CAAC;IAEF,mCAAmC;IACnC,IAAI,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QAC7C,OAAO,CAAC,YAAY,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACjE,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,QAAgB,EAChB,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;QAC3D,2CAA2C;QAC3C,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC;QAC/B,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC9B,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,GAAG,sBAAsB,CAAC;QAC7D,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,+BAA+B,CAAC;IACzC,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,wBAAwB,CACrC,QAAgB,EAChB,UAAkB,EAClB,SAAiB;IAEjB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,MAAM,WAAW,GAAG,uCAAuC,CAAC;IAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3C,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC;IAE/B,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QACrD,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5B,wBAAwB;QACxB,KAAK,MAAM,GAAG,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,EAAE,UAAU,GAAG,GAAG,CAAC,CAAC;YACpD,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAClD,IAAI,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,QAAQ;oBAAE,MAAM;gBAClD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;gBACxD,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;gBAC7B,MAAM;YACR,CAAC;YAAC,MAAM,CAAC;gBACP,qBAAqB;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAS,uBAAuB,CAAC,MAAc;IAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACpD,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CACvB,KAAa,EACb,QAAgB;IAEhB,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,GAAG,WAAW,qBAAqB,CAAC,CAAC;IAC9D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACjC,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAE7B,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QAC5B,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS;QACrD,IAAI,EAAE,EAAE,EAAE,qCAAqC;QAC/C,OAAO,EAAE,EAAE;KACZ,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,SAAiB;IAC7C,2DAA2D;IAC3D,8DAA8D;IAC9D,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAuB;IACzD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,mBAAmB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACnD,KAAK,CAAC,IAAI,CAAC,aAAa,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IAE5C,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,wBAAwB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACxB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;IACjC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAElB,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,qBAAqB,OAAO,CAAC,WAAW,CAAC,IAAI,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;QACxF,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC9B,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC5B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC/B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAElB,IAAI,OAAO,CAAC,iBAAiB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACxC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;YACvD,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACnB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,4BAA4B,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAChC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YAC1C,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;YAC/D,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,QAAQ,OAAO,MAAM,EAAE,CAAC,CAAC;QACrH,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,eAAe,IAAI,OAAO,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,eAAe,IAAI,OAAO,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAC1C,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,GAAG,MAAM,GAAG,CAAC,MAAM,IAAI,SAAS,KAAK,GAAG,CAAC,QAAQ,IAAI,GAAG,KAAK,CAAC,CAAC;QACnG,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { TestFailure } from './context.js';
|
|
2
|
+
export interface FailureGroup {
|
|
3
|
+
id: string;
|
|
4
|
+
signature: string;
|
|
5
|
+
failures: TestFailure[];
|
|
6
|
+
sample: TestFailure;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Group failures by normalized error signature.
|
|
10
|
+
* Pure string matching — no LLM involved.
|
|
11
|
+
*/
|
|
12
|
+
export declare function groupFailures(failures: TestFailure[]): FailureGroup[];
|
|
13
|
+
/**
|
|
14
|
+
* Select which groups to investigate, respecting limits and circuit breaker.
|
|
15
|
+
*/
|
|
16
|
+
export declare function selectGroupsForInvestigation(groups: FailureGroup[], totalTests: number, config: {
|
|
17
|
+
maxTotal: number;
|
|
18
|
+
massFailureThreshold: number;
|
|
19
|
+
}): {
|
|
20
|
+
selected: FailureGroup[];
|
|
21
|
+
circuitBroken: boolean;
|
|
22
|
+
};
|
|
23
|
+
//# sourceMappingURL=grouping.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"grouping.d.ts","sourceRoot":"","sources":["../src/grouping.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,EAAE,WAAW,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,YAAY,EAAE,CA0BrE;AAqDD;;GAEG;AACH,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,YAAY,EAAE,EACtB,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,oBAAoB,EAAE,MAAM,CAAA;CAAE,GACzD;IAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IAAC,aAAa,EAAE,OAAO,CAAA;CAAE,CAiBtD"}
|
package/dist/grouping.js
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Group failures by normalized error signature.
|
|
3
|
+
* Pure string matching — no LLM involved.
|
|
4
|
+
*/
|
|
5
|
+
export function groupFailures(failures) {
|
|
6
|
+
const groups = new Map();
|
|
7
|
+
for (const failure of failures) {
|
|
8
|
+
const signature = normalizeErrorSignature(failure.error.message, failure.error.stack);
|
|
9
|
+
const existing = groups.get(signature);
|
|
10
|
+
if (existing) {
|
|
11
|
+
existing.push(failure);
|
|
12
|
+
}
|
|
13
|
+
else {
|
|
14
|
+
groups.set(signature, [failure]);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
return Array.from(groups.entries()).map(([signature, groupFailures], i) => {
|
|
18
|
+
// Pick the simplest test as the sample (shortest source path as heuristic)
|
|
19
|
+
const sample = groupFailures.sort((a, b) => a.testFile.length - b.testFile.length)[0];
|
|
20
|
+
return {
|
|
21
|
+
id: `group-${i}`,
|
|
22
|
+
signature,
|
|
23
|
+
failures: groupFailures,
|
|
24
|
+
sample,
|
|
25
|
+
};
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Normalize an error message into a stable signature for grouping.
|
|
30
|
+
* Strips dynamic values (line numbers, selectors with IDs, timestamps, etc.)
|
|
31
|
+
*/
|
|
32
|
+
function normalizeErrorSignature(message, stack) {
|
|
33
|
+
let sig = message;
|
|
34
|
+
// Strip ANSI escape codes
|
|
35
|
+
sig = sig.replace(/\x1b\[[0-9;]*m/g, '');
|
|
36
|
+
// Strip specific DOM selectors with dynamic IDs
|
|
37
|
+
sig = sig.replace(/#[a-zA-Z0-9_-]+/g, '#<id>');
|
|
38
|
+
// Strip quoted strings that look like dynamic values
|
|
39
|
+
sig = sig.replace(/"[^"]{40,}"/g, '"<dynamic>"');
|
|
40
|
+
// Strip numbers that look like line/col references
|
|
41
|
+
sig = sig.replace(/:\d+:\d+/g, ':<line>:<col>');
|
|
42
|
+
// Strip timestamps
|
|
43
|
+
sig = sig.replace(/\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}[^\s]*/g, '<timestamp>');
|
|
44
|
+
// Strip timeout values
|
|
45
|
+
sig = sig.replace(/\d+ms/g, '<duration>');
|
|
46
|
+
// Use the first meaningful stack frame to distinguish errors at different call sites
|
|
47
|
+
if (stack) {
|
|
48
|
+
const firstFrame = extractFirstUserFrame(stack);
|
|
49
|
+
if (firstFrame) {
|
|
50
|
+
sig += ` @ ${firstFrame}`;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return sig.trim();
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Extract the first stack frame that's in user code (not node_modules).
|
|
57
|
+
* Strip line numbers for a stable signature.
|
|
58
|
+
*/
|
|
59
|
+
function extractFirstUserFrame(stack) {
|
|
60
|
+
const lines = stack.split('\n');
|
|
61
|
+
for (const line of lines) {
|
|
62
|
+
const match = line.match(/at\s+.*?\(?((?!node_modules)[^\s)]+):\d+:\d+\)?/);
|
|
63
|
+
if (match) {
|
|
64
|
+
return match[1];
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return undefined;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Select which groups to investigate, respecting limits and circuit breaker.
|
|
71
|
+
*/
|
|
72
|
+
export function selectGroupsForInvestigation(groups, totalTests, config) {
|
|
73
|
+
const totalFailures = groups.reduce((sum, g) => sum + g.failures.length, 0);
|
|
74
|
+
const failureRate = totalFailures / totalTests;
|
|
75
|
+
// Circuit breaker: if too many failures, only investigate top groups
|
|
76
|
+
if (failureRate > config.massFailureThreshold) {
|
|
77
|
+
const topGroups = groups
|
|
78
|
+
.sort((a, b) => b.failures.length - a.failures.length)
|
|
79
|
+
.slice(0, 3);
|
|
80
|
+
return { selected: topGroups, circuitBroken: true };
|
|
81
|
+
}
|
|
82
|
+
// Otherwise, investigate up to maxTotal groups
|
|
83
|
+
return {
|
|
84
|
+
selected: groups.slice(0, config.maxTotal),
|
|
85
|
+
circuitBroken: false,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=grouping.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"grouping.js","sourceRoot":"","sources":["../src/grouping.ts"],"names":[],"mappings":"AASA;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,QAAuB;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAEhD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,uBAAuB,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtF,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACvC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,aAAa,CAAC,EAAE,CAAC,EAAE,EAAE;QACxE,2EAA2E;QAC3E,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAC/B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAChD,CAAC,CAAC,CAAC,CAAC;QAEL,OAAO;YACL,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,SAAS;YACT,QAAQ,EAAE,aAAa;YACvB,MAAM;SACP,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,SAAS,uBAAuB,CAAC,OAAe,EAAE,KAAc;IAC9D,IAAI,GAAG,GAAG,OAAO,CAAC;IAElB,0BAA0B;IAC1B,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;IAEzC,gDAAgD;IAChD,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IAE/C,qDAAqD;IACrD,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;IAEjD,mDAAmD;IACnD,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,eAAe,CAAC,CAAC;IAEhD,mBAAmB;IACnB,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,+CAA+C,EAAE,aAAa,CAAC,CAAC;IAElF,uBAAuB;IACvB,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAE1C,qFAAqF;IACrF,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,UAAU,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAC;QAChD,IAAI,UAAU,EAAE,CAAC;YACf,GAAG,IAAI,MAAM,UAAU,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;AACpB,CAAC;AAED;;;GAGG;AACH,SAAS,qBAAqB,CAAC,KAAa;IAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;QAC5E,IAAI,KAAK,EAAE,CAAC;YACV,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,4BAA4B,CAC1C,MAAsB,EACtB,UAAkB,EAClB,MAA0D;IAE1D,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,WAAW,GAAG,aAAa,GAAG,UAAU,CAAC;IAE/C,qEAAqE;IACrE,IAAI,WAAW,GAAG,MAAM,CAAC,oBAAoB,EAAE,CAAC;QAC9C,MAAM,SAAS,GAAG,MAAM;aACrB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;aACrD,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACf,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IACtD,CAAC;IAED,+CAA+C;IAC/C,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC;QAC1C,aAAa,EAAE,KAAK;KACrB,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export { loadConfig, validateConfig } from './config.js';
|
|
2
|
+
export type { SmartRetryConfig, LLMConfig, BrowserConfig, InvestigationConfig, InvestigationMode, AuthConfig, ContextConfig, OutputConfig } from './config.js';
|
|
3
|
+
export { packageFailureContext, formatContextForLLM } from './context.js';
|
|
4
|
+
export type { TestFailure, FailureContext, FailingLine, TraceAction, NetworkRequest } from './context.js';
|
|
5
|
+
export { groupFailures, selectGroupsForInvestigation } from './grouping.js';
|
|
6
|
+
export type { FailureGroup } from './grouping.js';
|
|
7
|
+
export { createLLMProvider } from './llm.js';
|
|
8
|
+
export type { LLMProvider, LLMAnalysisResult } from './llm.js';
|
|
9
|
+
export type { Verdict, VerdictType, VerdictEvidence, ProposedFix, ProposedComment, InvestigationResult, TriageReport, FailureGroupSummary } from './verdicts.js';
|
|
10
|
+
export { formatMarkdownReport, formatGitHubComment, formatJsonReport, writeReport } from './report.js';
|
|
11
|
+
export { parsePlaywrightJsonReport } from './playwright-json.js';
|
|
12
|
+
export { triageFailures } from './triage.js';
|
|
13
|
+
export type { TriageOptions, BrowserInvestigatorFn } from './triage.js';
|
|
14
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACzD,YAAY,EAAE,gBAAgB,EAAE,SAAS,EAAE,aAAa,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE/J,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAC1E,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,WAAW,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE1G,OAAO,EAAE,aAAa,EAAE,4BAA4B,EAAE,MAAM,eAAe,CAAC;AAC5E,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,OAAO,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAC7C,YAAY,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAE/D,YAAY,EAAE,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,WAAW,EAAE,eAAe,EAAE,mBAAmB,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAEjK,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAEvG,OAAO,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,YAAY,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { loadConfig, validateConfig } from './config.js';
|
|
2
|
+
export { packageFailureContext, formatContextForLLM } from './context.js';
|
|
3
|
+
export { groupFailures, selectGroupsForInvestigation } from './grouping.js';
|
|
4
|
+
export { createLLMProvider } from './llm.js';
|
|
5
|
+
export { formatMarkdownReport, formatGitHubComment, formatJsonReport, writeReport } from './report.js';
|
|
6
|
+
export { parsePlaywrightJsonReport } from './playwright-json.js';
|
|
7
|
+
export { triageFailures } from './triage.js';
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAGzD,OAAO,EAAE,qBAAqB,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAG1E,OAAO,EAAE,aAAa,EAAE,4BAA4B,EAAE,MAAM,eAAe,CAAC;AAG5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAC;AAK7C,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAEvG,OAAO,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/llm.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { LLMConfig } from './config.js';
|
|
2
|
+
import type { Verdict } from './verdicts.js';
|
|
3
|
+
import type { FailureContext } from './context.js';
|
|
4
|
+
export interface LLMProvider {
|
|
5
|
+
analyze(context: FailureContext, screenshotBase64?: string): Promise<LLMAnalysisResult>;
|
|
6
|
+
}
|
|
7
|
+
export interface LLMAnalysisResult {
|
|
8
|
+
verdict: Verdict;
|
|
9
|
+
tokensUsed: number;
|
|
10
|
+
}
|
|
11
|
+
export declare function createLLMProvider(config: LLMConfig): LLMProvider;
|
|
12
|
+
//# sourceMappingURL=llm.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../src/llm.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,KAAK,EAAE,OAAO,EAAe,MAAM,eAAe,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAGnD,MAAM,WAAW,WAAW;IAC1B,OAAO,CAAC,OAAO,EAAE,cAAc,EAAE,gBAAgB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CACzF;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AA8CD,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW,CAKhE"}
|
package/dist/llm.js
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import Anthropic from '@anthropic-ai/sdk';
|
|
2
|
+
import { formatContextForLLM } from './context.js';
|
|
3
|
+
const SYSTEM_PROMPT = `You are an expert test failure analyst. Given a failing Playwright test with its source code, error message, stack trace, and optionally a failure screenshot, your job is to determine the root cause.
|
|
4
|
+
|
|
5
|
+
Analyze the failure and produce a verdict:
|
|
6
|
+
|
|
7
|
+
- **likely_bug**: The application appears to have a real defect. The test's intent seems correct, but the app isn't behaving as expected.
|
|
8
|
+
- **stale_test**: The application works correctly, but the test is outdated. Common causes: renamed selectors, changed UI structure, updated text content, removed elements.
|
|
9
|
+
- **ambiguous_test**: You can't confidently determine what the test is trying to verify. The test code is unclear, poorly named, or uses magic values without explanation.
|
|
10
|
+
- **data_issue**: The test expects specific data that isn't present. The app and test are both correct, but the test environment/data is wrong.
|
|
11
|
+
- **flaky**: The error looks intermittent (timeouts, race conditions, animation timing).
|
|
12
|
+
|
|
13
|
+
For each verdict, provide:
|
|
14
|
+
1. A confidence score (0.0 to 1.0)
|
|
15
|
+
2. A clear summary explaining your reasoning
|
|
16
|
+
3. Detailed evidence supporting the verdict
|
|
17
|
+
|
|
18
|
+
For **stale_test** verdicts, also propose a code fix:
|
|
19
|
+
- Include the original code and the fixed code
|
|
20
|
+
- Explain what changed and why
|
|
21
|
+
|
|
22
|
+
For **ambiguous_test** verdicts, propose clarifying @intent comments to add to the test.
|
|
23
|
+
|
|
24
|
+
Respond with valid JSON matching this schema:
|
|
25
|
+
{
|
|
26
|
+
"type": "likely_bug" | "stale_test" | "ambiguous_test" | "data_issue" | "flaky",
|
|
27
|
+
"confidence": number,
|
|
28
|
+
"summary": string,
|
|
29
|
+
"evidence": {
|
|
30
|
+
"reasoning": string,
|
|
31
|
+
"networkErrors": [{ "url": string, "status": number, "method": string }] | undefined,
|
|
32
|
+
"consoleErrors": [string] | undefined
|
|
33
|
+
},
|
|
34
|
+
"proposedFix": {
|
|
35
|
+
"filePath": string,
|
|
36
|
+
"originalCode": string,
|
|
37
|
+
"fixedCode": string,
|
|
38
|
+
"explanation": string
|
|
39
|
+
} | undefined,
|
|
40
|
+
"proposedComments": [{
|
|
41
|
+
"filePath": string,
|
|
42
|
+
"line": number,
|
|
43
|
+
"comment": string
|
|
44
|
+
}] | undefined
|
|
45
|
+
}`;
|
|
46
|
+
export function createLLMProvider(config) {
|
|
47
|
+
if (config.provider === 'anthropic') {
|
|
48
|
+
return new AnthropicProvider(config);
|
|
49
|
+
}
|
|
50
|
+
throw new Error(`Unsupported LLM provider: ${config.provider}`);
|
|
51
|
+
}
|
|
52
|
+
async function callWithRetry(client, params, maxRetries = 3) {
|
|
53
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
54
|
+
try {
|
|
55
|
+
return await client.messages.create(params);
|
|
56
|
+
}
|
|
57
|
+
catch (e) {
|
|
58
|
+
const isRateLimit = e instanceof Anthropic.RateLimitError ||
|
|
59
|
+
(e instanceof Error && e.message.includes('rate_limit'));
|
|
60
|
+
if (isRateLimit && attempt < maxRetries) {
|
|
61
|
+
const waitMs = Math.min(30_000, (attempt + 1) * 15_000);
|
|
62
|
+
console.log(` Rate limited, waiting ${waitMs / 1000}s...`);
|
|
63
|
+
await new Promise((r) => setTimeout(r, waitMs));
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
throw e;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
throw new Error('Unreachable');
|
|
70
|
+
}
|
|
71
|
+
class AnthropicProvider {
|
|
72
|
+
client;
|
|
73
|
+
model;
|
|
74
|
+
constructor(config) {
|
|
75
|
+
this.client = new Anthropic({ apiKey: config.apiKey });
|
|
76
|
+
this.model = config.model;
|
|
77
|
+
}
|
|
78
|
+
async analyze(context, screenshotBase64) {
|
|
79
|
+
const userContent = [];
|
|
80
|
+
// Add the failure context as text
|
|
81
|
+
userContent.push({
|
|
82
|
+
type: 'text',
|
|
83
|
+
text: formatContextForLLM(context),
|
|
84
|
+
});
|
|
85
|
+
// Add screenshot if available
|
|
86
|
+
if (screenshotBase64) {
|
|
87
|
+
userContent.push({
|
|
88
|
+
type: 'image',
|
|
89
|
+
source: {
|
|
90
|
+
type: 'base64',
|
|
91
|
+
media_type: 'image/png',
|
|
92
|
+
data: screenshotBase64,
|
|
93
|
+
},
|
|
94
|
+
});
|
|
95
|
+
userContent.push({
|
|
96
|
+
type: 'text',
|
|
97
|
+
text: 'Above is the screenshot captured at the moment of failure. Use it to understand what the page actually looked like when the test failed.',
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
userContent.push({
|
|
101
|
+
type: 'text',
|
|
102
|
+
text: 'Analyze this test failure and respond with the JSON verdict.',
|
|
103
|
+
});
|
|
104
|
+
const response = await callWithRetry(this.client, {
|
|
105
|
+
model: this.model,
|
|
106
|
+
max_tokens: 2048,
|
|
107
|
+
system: SYSTEM_PROMPT,
|
|
108
|
+
messages: [{ role: 'user', content: userContent }],
|
|
109
|
+
});
|
|
110
|
+
const textBlock = response.content.find((block) => block.type === 'text');
|
|
111
|
+
if (!textBlock) {
|
|
112
|
+
throw new Error('No text response from LLM');
|
|
113
|
+
}
|
|
114
|
+
const verdict = parseVerdictResponse(textBlock.text);
|
|
115
|
+
const tokensUsed = (response.usage.input_tokens ?? 0) + (response.usage.output_tokens ?? 0);
|
|
116
|
+
return { verdict, tokensUsed };
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
function parseVerdictResponse(text) {
|
|
120
|
+
// Extract JSON from the response (may be wrapped in markdown code block)
|
|
121
|
+
const jsonMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/) ?? [
|
|
122
|
+
null,
|
|
123
|
+
text,
|
|
124
|
+
];
|
|
125
|
+
const jsonStr = sanitizeJsonString(jsonMatch[1].trim());
|
|
126
|
+
try {
|
|
127
|
+
const parsed = JSON.parse(jsonStr);
|
|
128
|
+
return validateVerdict(parsed);
|
|
129
|
+
}
|
|
130
|
+
catch (e) {
|
|
131
|
+
// Try to parse the entire text as JSON
|
|
132
|
+
try {
|
|
133
|
+
const parsed = JSON.parse(sanitizeJsonString(text.trim()));
|
|
134
|
+
return validateVerdict(parsed);
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
throw new Error(`Failed to parse LLM verdict response: ${e.message}\nResponse: ${text.slice(0, 500)}`);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Fix common issues in LLM-generated JSON:
|
|
143
|
+
* - Replace bare `undefined` with `null`
|
|
144
|
+
* - Strip trailing commas before } or ]
|
|
145
|
+
*/
|
|
146
|
+
function sanitizeJsonString(str) {
|
|
147
|
+
return str
|
|
148
|
+
.replace(/:\s*undefined\b/g, ': null')
|
|
149
|
+
.replace(/,\s*([}\]])/g, '$1');
|
|
150
|
+
}
|
|
151
|
+
const VALID_VERDICTS = new Set([
|
|
152
|
+
'likely_bug',
|
|
153
|
+
'stale_test',
|
|
154
|
+
'ambiguous_test',
|
|
155
|
+
'data_issue',
|
|
156
|
+
'flaky',
|
|
157
|
+
]);
|
|
158
|
+
function validateVerdict(parsed) {
|
|
159
|
+
if (!parsed || typeof parsed !== 'object') {
|
|
160
|
+
throw new Error('Verdict must be an object');
|
|
161
|
+
}
|
|
162
|
+
const obj = parsed;
|
|
163
|
+
if (!VALID_VERDICTS.has(obj['type'])) {
|
|
164
|
+
throw new Error(`Invalid verdict type: ${obj['type']}`);
|
|
165
|
+
}
|
|
166
|
+
const confidence = Number(obj['confidence']);
|
|
167
|
+
if (isNaN(confidence) || confidence < 0 || confidence > 1) {
|
|
168
|
+
throw new Error(`Invalid confidence: ${obj['confidence']}`);
|
|
169
|
+
}
|
|
170
|
+
if (typeof obj['summary'] !== 'string') {
|
|
171
|
+
throw new Error('Verdict must include a summary string');
|
|
172
|
+
}
|
|
173
|
+
return {
|
|
174
|
+
type: obj['type'],
|
|
175
|
+
confidence,
|
|
176
|
+
summary: obj['summary'],
|
|
177
|
+
evidence: {
|
|
178
|
+
reasoning: typeof obj['evidence']?.['reasoning'] === 'string'
|
|
179
|
+
? obj['evidence']['reasoning']
|
|
180
|
+
: obj['summary'],
|
|
181
|
+
networkErrors: obj['evidence']?.['networkErrors'],
|
|
182
|
+
consoleErrors: obj['evidence']?.['consoleErrors'],
|
|
183
|
+
},
|
|
184
|
+
proposedFix: obj['proposedFix'],
|
|
185
|
+
proposedComments: obj['proposedComments'],
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
//# sourceMappingURL=llm.js.map
|
package/dist/llm.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../src/llm.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAI1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAWnD,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA0CpB,CAAC;AAEH,MAAM,UAAU,iBAAiB,CAAC,MAAiB;IACjD,IAAI,MAAM,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;QACpC,OAAO,IAAI,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACvC,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,MAAiB,EACjB,MAA0D,EAC1D,UAAU,GAAG,CAAC;IAEd,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,OAAO,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,CAAU,EAAE,CAAC;YACpB,MAAM,WAAW,GACf,CAAC,YAAY,SAAS,CAAC,cAAc;gBACrC,CAAC,CAAC,YAAY,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;YAC3D,IAAI,WAAW,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;gBACxC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;gBACxD,OAAO,CAAC,GAAG,CAAC,2BAA2B,MAAM,GAAG,IAAI,MAAM,CAAC,CAAC;gBAC5D,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;gBAChD,SAAS;YACX,CAAC;YACD,MAAM,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,iBAAiB;IACb,MAAM,CAAY;IAClB,KAAK,CAAS;IAEtB,YAAY,MAAiB;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO,CACX,OAAuB,EACvB,gBAAyB;QAEzB,MAAM,WAAW,GAA2C,EAAE,CAAC;QAE/D,kCAAkC;QAClC,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,mBAAmB,CAAC,OAAO,CAAC;SACnC,CAAC,CAAC;QAEH,8BAA8B;QAC9B,IAAI,gBAAgB,EAAE,CAAC;YACrB,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE,WAAW;oBACvB,IAAI,EAAE,gBAAgB;iBACvB;aACF,CAAC,CAAC;YACH,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,MAAM;gBACZ,IAAI,EAAE,0IAA0I;aACjJ,CAAC,CAAC;QACL,CAAC;QAED,WAAW,CAAC,IAAI,CAAC;YACf,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,8DAA8D;SACrE,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE;YAChD,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,UAAU,EAAE,IAAI;YAChB,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;SACnD,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CACrC,CAAC,KAAK,EAAyC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CACxE,CAAC;QAEF,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,OAAO,GAAG,oBAAoB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,UAAU,GACd,CAAC,QAAQ,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC;QAE3E,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;IACjC,CAAC;CACF;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,yEAAyE;IACzE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,uCAAuC,CAAC,IAAI;QACvE,IAAI;QACJ,IAAI;KACL,CAAC;IACF,MAAM,OAAO,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAExD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACnC,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,uCAAuC;QACvC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC3D,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CACb,yCAA0C,CAAW,CAAC,OAAO,eAAe,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CACjG,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,GAAW;IACrC,OAAO,GAAG;SACP,OAAO,CAAC,kBAAkB,EAAE,QAAQ,CAAC;SACrC,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,cAAc,GAAqB,IAAI,GAAG,CAAC;IAC/C,YAAY;IACZ,YAAY;IACZ,gBAAgB;IAChB,YAAY;IACZ,OAAO;CACR,CAAC,CAAC;AAEH,SAAS,eAAe,CAAC,MAAe;IACtC,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CAAC,2BAA2B,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,GAAG,GAAG,MAAiC,CAAC;IAE9C,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAgB,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC;IAC7C,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,uBAAuB,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,IAAI,EAAE,GAAG,CAAC,MAAM,CAAgB;QAChC,UAAU;QACV,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC;QACvB,QAAQ,EAAE;YACR,SAAS,EACP,OAAQ,GAAG,CAAC,UAAU,CAA6B,EAAE,CAAC,WAAW,CAAC,KAAK,QAAQ;gBAC7E,CAAC,CAAE,GAAG,CAAC,UAAU,CAA6B,CAAC,WAAW,CAAW;gBACrE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC;YACpB,aAAa,EAAG,GAAG,CAAC,UAAU,CAA6B,EAAE,CAAC,eAAe,CAAyC;YACtH,aAAa,EAAG,GAAG,CAAC,UAAU,CAA6B,EAAE,CAAC,eAAe,CAAyC;SACvH;QACD,WAAW,EAAE,GAAG,CAAC,aAAa,CAA2B;QACzD,gBAAgB,EAAE,GAAG,CAAC,kBAAkB,CAAgC;KACzE,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { TestFailure } from './context.js';
|
|
2
|
+
interface PlaywrightJsonReport {
|
|
3
|
+
config: {
|
|
4
|
+
rootDir: string;
|
|
5
|
+
};
|
|
6
|
+
suites: PlaywrightSuite[];
|
|
7
|
+
}
|
|
8
|
+
interface PlaywrightSuite {
|
|
9
|
+
title: string;
|
|
10
|
+
file: string;
|
|
11
|
+
suites?: PlaywrightSuite[];
|
|
12
|
+
specs?: PlaywrightSpec[];
|
|
13
|
+
}
|
|
14
|
+
interface PlaywrightSpec {
|
|
15
|
+
title: string;
|
|
16
|
+
file: string;
|
|
17
|
+
line: number;
|
|
18
|
+
tests: PlaywrightTest[];
|
|
19
|
+
}
|
|
20
|
+
interface PlaywrightTest {
|
|
21
|
+
projectName: string;
|
|
22
|
+
results: PlaywrightResult[];
|
|
23
|
+
}
|
|
24
|
+
interface PlaywrightResult {
|
|
25
|
+
status: string;
|
|
26
|
+
duration: number;
|
|
27
|
+
retry: number;
|
|
28
|
+
error?: {
|
|
29
|
+
message: string;
|
|
30
|
+
stack?: string;
|
|
31
|
+
};
|
|
32
|
+
attachments: Array<{
|
|
33
|
+
name: string;
|
|
34
|
+
contentType: string;
|
|
35
|
+
path?: string;
|
|
36
|
+
}>;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Parse Playwright's JSON reporter output into TestFailure objects.
|
|
40
|
+
*/
|
|
41
|
+
export declare function parsePlaywrightJsonReport(report: PlaywrightJsonReport): {
|
|
42
|
+
failures: TestFailure[];
|
|
43
|
+
totalTests: number;
|
|
44
|
+
};
|
|
45
|
+
export {};
|
|
46
|
+
//# sourceMappingURL=playwright-json.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"playwright-json.d.ts","sourceRoot":"","sources":["../src/playwright-json.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,UAAU,oBAAoB;IAC5B,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5B,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B;AAED,UAAU,eAAe;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,eAAe,EAAE,CAAC;IAC3B,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC;CAC1B;AAED,UAAU,cAAc;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,cAAc,EAAE,CAAC;CACzB;AAED,UAAU,cAAc;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,gBAAgB,EAAE,CAAC;CAC7B;AAED,UAAU,gBAAgB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC5C,WAAW,EAAE,KAAK,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,IAAI,CAAC,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,oBAAoB,GAC3B;IAAE,QAAQ,EAAE,WAAW,EAAE,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAiDjD"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
/**
|
|
3
|
+
* Parse Playwright's JSON reporter output into TestFailure objects.
|
|
4
|
+
*/
|
|
5
|
+
export function parsePlaywrightJsonReport(report) {
|
|
6
|
+
const failures = [];
|
|
7
|
+
let totalTests = 0;
|
|
8
|
+
const rootDir = report.config.rootDir;
|
|
9
|
+
function walkSuites(suites) {
|
|
10
|
+
for (const suite of suites) {
|
|
11
|
+
if (suite.suites)
|
|
12
|
+
walkSuites(suite.suites);
|
|
13
|
+
if (!suite.specs)
|
|
14
|
+
continue;
|
|
15
|
+
for (const spec of suite.specs) {
|
|
16
|
+
for (const test of spec.tests) {
|
|
17
|
+
totalTests++;
|
|
18
|
+
const result = test.results[test.results.length - 1];
|
|
19
|
+
if (!result || (result.status !== 'failed' && result.status !== 'timedOut')) {
|
|
20
|
+
continue;
|
|
21
|
+
}
|
|
22
|
+
const testFile = join(rootDir, spec.file);
|
|
23
|
+
const screenshot = result.attachments.find((a) => a.contentType === 'image/png' && a.path);
|
|
24
|
+
const trace = result.attachments.find((a) => a.name === 'trace' && a.path);
|
|
25
|
+
failures.push({
|
|
26
|
+
testId: `${spec.file}:${spec.line}`,
|
|
27
|
+
testTitle: spec.title,
|
|
28
|
+
testFile,
|
|
29
|
+
line: spec.line,
|
|
30
|
+
error: {
|
|
31
|
+
message: result.error?.message ?? 'Unknown error',
|
|
32
|
+
stack: result.error?.stack,
|
|
33
|
+
},
|
|
34
|
+
duration: result.duration,
|
|
35
|
+
retry: result.retry,
|
|
36
|
+
screenshotPath: screenshot?.path,
|
|
37
|
+
tracePath: trace?.path,
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
walkSuites(report.suites);
|
|
44
|
+
return { failures, totalTests };
|
|
45
|
+
}
|
|
46
|
+
//# sourceMappingURL=playwright-json.js.map
|