@veda-hq/pariksha 0.5.17 → 0.5.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +209 -2
- package/dist/cli.js +154 -1
- package/dist/duplicate-tests.js +367 -0
- package/dist/flaky-tests.js +252 -0
- package/dist/hook-preamble.js +343 -0
- package/dist/masked-exit-codes.js +762 -0
- package/dist/shell-command.js +425 -0
- package/dist/shell-gates.js +283 -0
- package/dist/shell-pitfalls.js +408 -0
- package/dist/shell-sources.js +165 -0
- package/dist/shell-text.js +417 -22
- package/dist/vitest-json-report.js +129 -0
- package/package.json +2 -2
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
// duplicate-tests: test cases that look copied, by a weighted similarity of
|
|
2
|
+
// their assertion matchers, their names and their describe paths.
|
|
3
|
+
//
|
|
4
|
+
// Read with the TypeScript compiler API (the optional peer), walking the nodes
|
|
5
|
+
// a Babel CallExpression visitor would reach: an optional-chain call is not
|
|
6
|
+
// one, and parentheses are not nodes.
|
|
7
|
+
import { basename } from 'node:path';
|
|
8
|
+
export const DEFAULT_CONFIG = {
|
|
9
|
+
include: /\.(test|spec)\.(ts|tsx|js|jsx)$/,
|
|
10
|
+
threshold: 0.85,
|
|
11
|
+
commonNames: ['should be defined', 'should exist', 'should return', 'should throw', 'should create', 'should update', 'should delete', 'should get', 'should set'],
|
|
12
|
+
oppositePairs: [
|
|
13
|
+
['success', 'failure'],
|
|
14
|
+
['success', 'error'],
|
|
15
|
+
['pass', 'fail'],
|
|
16
|
+
['valid', 'invalid'],
|
|
17
|
+
['correct', 'incorrect'],
|
|
18
|
+
['positive', 'negative'],
|
|
19
|
+
['zero', 'negative'],
|
|
20
|
+
['true', 'false'],
|
|
21
|
+
['enabled', 'disabled'],
|
|
22
|
+
['before', 'after'],
|
|
23
|
+
['start', 'end'],
|
|
24
|
+
['to', 'from'],
|
|
25
|
+
['forward', 'backward'],
|
|
26
|
+
['add', 'remove'],
|
|
27
|
+
['create', 'delete'],
|
|
28
|
+
],
|
|
29
|
+
};
|
|
30
|
+
const isRecord = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
31
|
+
const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === 'string');
|
|
32
|
+
/** The config a product passes; each field replaces its default, and an unknown field is refused. */
|
|
33
|
+
export const parseConfig = (text) => {
|
|
34
|
+
if (text === undefined)
|
|
35
|
+
return DEFAULT_CONFIG;
|
|
36
|
+
const raw = JSON.parse(text);
|
|
37
|
+
if (!isRecord(raw))
|
|
38
|
+
throw new Error('the config is not a JSON object');
|
|
39
|
+
const known = ['include', 'threshold', 'commonNames', 'oppositePairs'];
|
|
40
|
+
const unknown = Object.keys(raw).filter((k) => !known.includes(k));
|
|
41
|
+
if (unknown.length > 0)
|
|
42
|
+
throw new Error(`the config has unknown field(s): ${unknown.join(', ')}`);
|
|
43
|
+
const { include, threshold, commonNames, oppositePairs } = raw;
|
|
44
|
+
if (include !== undefined && typeof include !== 'string')
|
|
45
|
+
throw new Error('include is not a regex string');
|
|
46
|
+
if (threshold !== undefined && (typeof threshold !== 'number' || !Number.isFinite(threshold)))
|
|
47
|
+
throw new Error('threshold is not a number');
|
|
48
|
+
if (commonNames !== undefined && !isStrings(commonNames))
|
|
49
|
+
throw new Error('commonNames is not a list of strings');
|
|
50
|
+
if (oppositePairs !== undefined && !(Array.isArray(oppositePairs) && oppositePairs.every((p) => isStrings(p) && p.length === 2))) {
|
|
51
|
+
throw new Error('oppositePairs is not a list of [word, word] pairs');
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
include: include === undefined ? DEFAULT_CONFIG.include : new RegExp(include),
|
|
55
|
+
threshold: threshold ?? DEFAULT_CONFIG.threshold,
|
|
56
|
+
commonNames: commonNames ?? DEFAULT_CONFIG.commonNames,
|
|
57
|
+
oppositePairs: oppositePairs ?? DEFAULT_CONFIG.oppositePairs,
|
|
58
|
+
};
|
|
59
|
+
};
|
|
60
|
+
/** Through parentheses, which Babel does not keep as nodes. */
|
|
61
|
+
const unwrap = (ts, node) => {
|
|
62
|
+
let n = node;
|
|
63
|
+
while (ts.isParenthesizedExpression(n))
|
|
64
|
+
n = n.expression;
|
|
65
|
+
return n;
|
|
66
|
+
};
|
|
67
|
+
const inChain = (ts, node) => (node.flags & ts.NodeFlags.OptionalChain) !== 0;
|
|
68
|
+
/** A call Babel reads as a CallExpression: not one inside an optional chain. */
|
|
69
|
+
const isPlainCall = (ts, node) => ts.isCallExpression(node) && !inChain(ts, node);
|
|
70
|
+
const calleeName = (ts, call) => {
|
|
71
|
+
const callee = unwrap(ts, call.expression);
|
|
72
|
+
return ts.isIdentifier(callee) ? callee.text : undefined;
|
|
73
|
+
};
|
|
74
|
+
/**
|
|
75
|
+
* The matcher a call names, as Babel's MemberExpression with an Identifier
|
|
76
|
+
* property: `a.toBe()`, and `a[toBe]()` too; never `a?.toBe()` or `a['toBe']()`.
|
|
77
|
+
*/
|
|
78
|
+
const memberProperty = (ts, call) => {
|
|
79
|
+
const callee = unwrap(ts, call.expression);
|
|
80
|
+
if (inChain(ts, callee))
|
|
81
|
+
return undefined;
|
|
82
|
+
if (ts.isPropertyAccessExpression(callee))
|
|
83
|
+
return ts.isIdentifier(callee.name) ? callee.name.text : undefined;
|
|
84
|
+
if (ts.isElementAccessExpression(callee)) {
|
|
85
|
+
const arg = unwrap(ts, callee.argumentExpression);
|
|
86
|
+
return ts.isIdentifier(arg) ? arg.text : undefined;
|
|
87
|
+
}
|
|
88
|
+
return undefined;
|
|
89
|
+
};
|
|
90
|
+
const sourceText = (sf, node) => sf.text.slice(node.getStart(sf), node.end);
|
|
91
|
+
/**
|
|
92
|
+
* The name a describe/test/it call gives its block. A string literal (or a
|
|
93
|
+
* template without expressions) yields its value; `describe(X.name)` yields
|
|
94
|
+
* `X`, the unit under test; any other argument its source text.
|
|
95
|
+
*/
|
|
96
|
+
export const extractTestName = (ts, sf, call) => {
|
|
97
|
+
const [arg] = call.arguments;
|
|
98
|
+
if (arg === undefined)
|
|
99
|
+
return 'unnamed test';
|
|
100
|
+
const first = unwrap(ts, arg);
|
|
101
|
+
if (ts.isStringLiteral(first) || ts.isNoSubstitutionTemplateLiteral(first))
|
|
102
|
+
return first.text;
|
|
103
|
+
const named = ts.isPropertyAccessExpression(first) && !inChain(ts, first) && ts.isIdentifier(first.name) && first.name.text === 'name' ? unwrap(ts, first.expression) : first;
|
|
104
|
+
return sourceText(sf, named);
|
|
105
|
+
};
|
|
106
|
+
/**
|
|
107
|
+
* TS for a `.ts` file, TSX for the rest. Babel had JSX on everywhere but reads
|
|
108
|
+
* `<T>(x) => …` in a `.ts` file as a generic arrow; TypeScript under TSX reads
|
|
109
|
+
* it as an element and loses the rest of the file.
|
|
110
|
+
*/
|
|
111
|
+
const scriptKind = (ts, file) => (/\.[cm]?ts$/.test(file) ? ts.ScriptKind.TS : ts.ScriptKind.TSX);
|
|
112
|
+
/** The syntax errors the parse recorded; `parseDiagnostics` is the source file's own list, read without a program. */
|
|
113
|
+
const syntaxErrors = (sf) => sf.parseDiagnostics ?? [];
|
|
114
|
+
/**
|
|
115
|
+
* Syntax TypeScript reads and the Babel parse this replaces (plugins
|
|
116
|
+
* typescript, jsx and decorators-legacy) threw on, so the file gave no
|
|
117
|
+
* signatures: a `<T>value` assertion in a `.ts` file, which JSX reads as an
|
|
118
|
+
* unterminated element, and an `accessor` field, which needs a plugin it did
|
|
119
|
+
* not have.
|
|
120
|
+
*/
|
|
121
|
+
const babelRefusal = (ts, kind, node) => {
|
|
122
|
+
if (kind === ts.ScriptKind.TS && ts.isTypeAssertionExpression(node))
|
|
123
|
+
return 'a <T>value type assertion, which a JSX parse reads as an unterminated element';
|
|
124
|
+
if (ts.isPropertyDeclaration(node) && node.modifiers?.some((m) => m.kind === ts.SyntaxKind.AccessorKeyword) === true)
|
|
125
|
+
return 'an accessor field, which needs the decoratorAutoAccessors parser plugin';
|
|
126
|
+
if (ts.isClassLike(node)) {
|
|
127
|
+
const modifiers = node.modifiers ?? [];
|
|
128
|
+
const exported = modifiers.findIndex((m) => m.kind === ts.SyntaxKind.ExportKeyword);
|
|
129
|
+
// `export default @dec class` is a decorated class expression, which they do take.
|
|
130
|
+
const isDefault = modifiers.some((m) => m.kind === ts.SyntaxKind.DefaultKeyword);
|
|
131
|
+
if (exported >= 0 && !isDefault && modifiers.slice(exported).some((m) => ts.isDecorator(m)))
|
|
132
|
+
return 'a decorator after export, which legacy decorators do not take';
|
|
133
|
+
}
|
|
134
|
+
return undefined;
|
|
135
|
+
};
|
|
136
|
+
/** Why the file gives no signatures: a syntax error, or what Babel refused; undefined when it reads. */
|
|
137
|
+
const parseError = (ts, sf, refused) => {
|
|
138
|
+
const [first] = syntaxErrors(sf);
|
|
139
|
+
const at = (pos) => {
|
|
140
|
+
const { line, character } = sf.getLineAndCharacterOfPosition(pos);
|
|
141
|
+
return `(${String(line + 1)}:${String(character)})`;
|
|
142
|
+
};
|
|
143
|
+
if (first !== undefined)
|
|
144
|
+
return `${ts.flattenDiagnosticMessageText(first.messageText, '\n')} ${at(first.start)}`;
|
|
145
|
+
if (refused !== undefined)
|
|
146
|
+
return `${refused.why} ${at(refused.node.getStart(sf))}`;
|
|
147
|
+
return undefined;
|
|
148
|
+
};
|
|
149
|
+
/** Every test with an assertion in one file, in the order a depth-first walk meets them. */
|
|
150
|
+
export const extractTestSignatures = (ts, file, content) => {
|
|
151
|
+
const parse = (k) => ts.createSourceFile(file, content, ts.ScriptTarget.Latest, true, k);
|
|
152
|
+
let kind = scriptKind(ts, file);
|
|
153
|
+
let sf = parse(kind);
|
|
154
|
+
if (kind === ts.ScriptKind.TS && syntaxErrors(sf).length > 0) {
|
|
155
|
+
// JSX in a `.ts` file, which Babel read: take the TSX parse when it is clean.
|
|
156
|
+
const tsx = parse(ts.ScriptKind.TSX);
|
|
157
|
+
if (syntaxErrors(tsx).length === 0) {
|
|
158
|
+
kind = ts.ScriptKind.TSX;
|
|
159
|
+
sf = tsx;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
const signatures = [];
|
|
163
|
+
const describeStack = [];
|
|
164
|
+
let refused;
|
|
165
|
+
const assertions = (call) => {
|
|
166
|
+
const pattern = [];
|
|
167
|
+
const visit = (node) => {
|
|
168
|
+
if (isPlainCall(ts, node)) {
|
|
169
|
+
if (calleeName(ts, node) === 'expect')
|
|
170
|
+
pattern.push('expect()');
|
|
171
|
+
const matcher = memberProperty(ts, node);
|
|
172
|
+
if (matcher?.startsWith('to') === true)
|
|
173
|
+
pattern.push(matcher);
|
|
174
|
+
}
|
|
175
|
+
ts.forEachChild(node, visit);
|
|
176
|
+
};
|
|
177
|
+
ts.forEachChild(call, visit);
|
|
178
|
+
return pattern;
|
|
179
|
+
};
|
|
180
|
+
const walk = (node) => {
|
|
181
|
+
if (refused === undefined) {
|
|
182
|
+
const why = babelRefusal(ts, kind, node);
|
|
183
|
+
if (why !== undefined)
|
|
184
|
+
refused = { node, why };
|
|
185
|
+
}
|
|
186
|
+
const call = isPlainCall(ts, node) ? node : undefined;
|
|
187
|
+
const name = call === undefined ? undefined : calleeName(ts, call);
|
|
188
|
+
if (call !== undefined && name === 'describe')
|
|
189
|
+
describeStack.push(extractTestName(ts, sf, call));
|
|
190
|
+
if (call !== undefined && (name === 'test' || name === 'it')) {
|
|
191
|
+
const assertionPattern = assertions(call);
|
|
192
|
+
if (assertionPattern.length > 0) {
|
|
193
|
+
signatures.push({
|
|
194
|
+
file,
|
|
195
|
+
testName: extractTestName(ts, sf, call),
|
|
196
|
+
describePath: [...describeStack],
|
|
197
|
+
assertionPattern,
|
|
198
|
+
line: sf.getLineAndCharacterOfPosition(call.getStart(sf)).line + 1,
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
ts.forEachChild(node, walk);
|
|
203
|
+
if (name === 'describe')
|
|
204
|
+
describeStack.pop();
|
|
205
|
+
};
|
|
206
|
+
walk(sf);
|
|
207
|
+
const error = parseError(ts, sf, refused);
|
|
208
|
+
return error === undefined ? { signatures } : { signatures: [], parseError: error };
|
|
209
|
+
};
|
|
210
|
+
const levenshtein = (a, b) => {
|
|
211
|
+
let prev = Array.from({ length: a.length + 1 }, (_, j) => j);
|
|
212
|
+
for (let i = 1; i <= b.length; i++) {
|
|
213
|
+
const row = [i];
|
|
214
|
+
for (let j = 1; j <= a.length; j++) {
|
|
215
|
+
row[j] = b.charAt(i - 1) === a.charAt(j - 1) ? prev[j - 1] : Math.min(prev[j - 1] + 1, row[j - 1] + 1, prev[j] + 1);
|
|
216
|
+
}
|
|
217
|
+
prev = row;
|
|
218
|
+
}
|
|
219
|
+
return prev[a.length];
|
|
220
|
+
};
|
|
221
|
+
/** 1 minus the edit distance over the longer length, case-insensitive; 1 for two empty strings. */
|
|
222
|
+
export const stringSimilarity = (a, b) => {
|
|
223
|
+
const max = Math.max(a.length, b.length);
|
|
224
|
+
return max > 0 ? 1 - levenshtein(a.toLowerCase(), b.toLowerCase()) / max : 1;
|
|
225
|
+
};
|
|
226
|
+
/** Jaccard similarity of two matcher lists, as sets; 0 when both are empty. */
|
|
227
|
+
export const patternSimilarity = (a, b) => {
|
|
228
|
+
const setA = new Set(a);
|
|
229
|
+
const setB = new Set(b);
|
|
230
|
+
const union = new Set([...setA, ...setB]);
|
|
231
|
+
return union.size > 0 ? [...setA].filter((x) => setB.has(x)).length / union.size : 0;
|
|
232
|
+
};
|
|
233
|
+
/**
|
|
234
|
+
* Two tests in no describe block: full credit in one file; across files the
|
|
235
|
+
* file stems (the basename up to its first dot) are the only names they have.
|
|
236
|
+
*/
|
|
237
|
+
const neitherDescribed = (file1, file2) => {
|
|
238
|
+
if (file1 === file2)
|
|
239
|
+
return 1;
|
|
240
|
+
const stem = (f) => basename(f).split('.')[0];
|
|
241
|
+
return stringSimilarity(stem(file1), stem(file2));
|
|
242
|
+
};
|
|
243
|
+
/** 50% matchers, 20% name, 30% describe path. */
|
|
244
|
+
export const compositeSimilarity = (a, b) => {
|
|
245
|
+
const path1 = a.describePath.join(' > ');
|
|
246
|
+
const path2 = b.describePath.join(' > ');
|
|
247
|
+
const describeSim = path1.length > 0 && path2.length > 0 ? stringSimilarity(path1, path2) : path1.length === 0 && path2.length === 0 ? neitherDescribed(a.file, b.file) : 0;
|
|
248
|
+
return patternSimilarity(a.assertionPattern, b.assertionPattern) * 0.5 + stringSimilarity(a.testName, b.testName) * 0.2 + describeSim * 0.3;
|
|
249
|
+
};
|
|
250
|
+
const MIN_ASSERTIONS = 3;
|
|
251
|
+
const HTTP_INDICATORS = /\b(GET|POST|PUT|DELETE|PATCH|200|201|400|401|403|404|500|502)\b/gi;
|
|
252
|
+
// Case-sensitive on purpose: lowercase "and"/"or" are prose, uppercase AND/OR are domain tokens.
|
|
253
|
+
const COMBINATOR_INDICATORS = /\b(AND|OR)\b/g;
|
|
254
|
+
const isCommon = (config, name) => {
|
|
255
|
+
const normalized = name.toLowerCase().trim();
|
|
256
|
+
return config.commonNames.some((p) => normalized === p || normalized.startsWith(`${p} `));
|
|
257
|
+
};
|
|
258
|
+
/** Two names for opposite or complementary cases: an opposite pair, other HTTP tokens, other AND/OR tokens, another endpoint. */
|
|
259
|
+
export const areOppositeScenarios = (config, name1, name2) => {
|
|
260
|
+
const lower1 = name1.toLowerCase();
|
|
261
|
+
const lower2 = name2.toLowerCase();
|
|
262
|
+
for (const [w1, w2] of config.oppositePairs) {
|
|
263
|
+
if ((lower1.includes(w1) && lower2.includes(w2)) || (lower1.includes(w2) && lower2.includes(w1)))
|
|
264
|
+
return true;
|
|
265
|
+
}
|
|
266
|
+
const http1 = Array.from(name1.matchAll(HTTP_INDICATORS), (m) => m[0]);
|
|
267
|
+
const http2 = Array.from(name2.matchAll(HTTP_INDICATORS), (m) => m[0]);
|
|
268
|
+
if (http1.length > 0 && http2.length > 0 && http1.some((m) => !http2.includes(m)))
|
|
269
|
+
return true;
|
|
270
|
+
const comb1 = Array.from(name1.matchAll(COMBINATOR_INDICATORS), (m) => m[0]);
|
|
271
|
+
const comb2 = Array.from(name2.matchAll(COMBINATOR_INDICATORS), (m) => m[0]);
|
|
272
|
+
if (comb1.length > 0 && comb2.length > 0 && (comb1.some((c) => !comb2.includes(c)) || comb2.some((c) => !comb1.includes(c))))
|
|
273
|
+
return true;
|
|
274
|
+
const end1 = name1.match(/\/\w+/g) ?? [];
|
|
275
|
+
const end2 = name2.match(/\/\w+/g) ?? [];
|
|
276
|
+
return end1.length > 0 && end2.length > 0 && end1[0] !== end2[0];
|
|
277
|
+
};
|
|
278
|
+
const FILLER = new Set(['should', 'when', 'with', 'the', 'a', 'an', 'it', 'is', 'are', 'be', 'to', 'for', 'of', 'in', 'on', 'and', 'or', 'not', 'if', 'that', 'this', 'can', 'does']);
|
|
279
|
+
const words = (name) => name
|
|
280
|
+
.toLowerCase()
|
|
281
|
+
.replace(/[^a-z0-9 ]/g, ' ')
|
|
282
|
+
.split(/\s+/)
|
|
283
|
+
.filter((w) => w.length > 1 && !FILLER.has(w));
|
|
284
|
+
/**
|
|
285
|
+
* Two tests in one file that are meant to differ: opposite names, the same
|
|
286
|
+
* name (so different describe blocks), a word only one has, or the same words
|
|
287
|
+
* in another order (a mirrored case).
|
|
288
|
+
*/
|
|
289
|
+
export const areSameFileComplementary = (config, file1, file2, name1, name2) => {
|
|
290
|
+
if (file1 !== file2)
|
|
291
|
+
return false;
|
|
292
|
+
if (areOppositeScenarios(config, name1, name2))
|
|
293
|
+
return true;
|
|
294
|
+
if (name1 === name2)
|
|
295
|
+
return true;
|
|
296
|
+
const words1 = new Set(words(name1));
|
|
297
|
+
const words2 = new Set(words(name2));
|
|
298
|
+
if ([...words1].some((w) => !words2.has(w)) || [...words2].some((w) => !words1.has(w)))
|
|
299
|
+
return true;
|
|
300
|
+
const seq1 = words(name1);
|
|
301
|
+
const seq2 = words(name2);
|
|
302
|
+
return [...seq1].sort().join(' ') === [...seq2].sort().join(' ') && seq1.join(' ') !== seq2.join(' ');
|
|
303
|
+
};
|
|
304
|
+
/** Every pair at or above the threshold, in signature order. */
|
|
305
|
+
export const findDuplicates = (config, signatures) => {
|
|
306
|
+
const matches = [];
|
|
307
|
+
const processed = new Set();
|
|
308
|
+
for (let i = 0; i < signatures.length; i++) {
|
|
309
|
+
for (let j = i + 1; j < signatures.length; j++) {
|
|
310
|
+
const a = signatures[i];
|
|
311
|
+
const b = signatures[j];
|
|
312
|
+
if (a.assertionPattern.length < MIN_ASSERTIONS || b.assertionPattern.length < MIN_ASSERTIONS)
|
|
313
|
+
continue;
|
|
314
|
+
if (isCommon(config, a.testName) || isCommon(config, b.testName))
|
|
315
|
+
continue;
|
|
316
|
+
if (areOppositeScenarios(config, a.testName, b.testName))
|
|
317
|
+
continue;
|
|
318
|
+
if (areSameFileComplementary(config, a.file, b.file, a.testName, b.testName))
|
|
319
|
+
continue;
|
|
320
|
+
const key = `${a.file}:${a.testName}-${b.file}:${b.testName}`;
|
|
321
|
+
if (processed.has(key))
|
|
322
|
+
continue;
|
|
323
|
+
const similarity = compositeSimilarity(a, b);
|
|
324
|
+
// Different files test different modules: a higher bar.
|
|
325
|
+
const bar = a.file !== b.file ? Math.max(config.threshold, 0.95) : config.threshold;
|
|
326
|
+
if (similarity >= bar) {
|
|
327
|
+
matches.push({ file1: a.file, test1: a.testName, line1: a.line, file2: b.file, test2: b.testName, line2: b.line, similarity });
|
|
328
|
+
processed.add(key);
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return matches;
|
|
333
|
+
};
|
|
334
|
+
/** The report, line by line, and the exit code; reading failures warn on stderr and the file is skipped. */
|
|
335
|
+
export const runDuplicateTests = (ts, config, mode, files) => {
|
|
336
|
+
const out = [`🔍 Searching for duplicate tests (Threshold: ${String(config.threshold)})...`, ` Mode: ${mode}`];
|
|
337
|
+
const err = [];
|
|
338
|
+
if (files.length === 0) {
|
|
339
|
+
out.push('No files found to analyze.');
|
|
340
|
+
return { code: 0, out, err };
|
|
341
|
+
}
|
|
342
|
+
out.push(` Analyzing ${String(files.length)} files...`);
|
|
343
|
+
const signatures = [];
|
|
344
|
+
for (const f of files) {
|
|
345
|
+
try {
|
|
346
|
+
const read = extractTestSignatures(ts, f.file, f.read());
|
|
347
|
+
if (read.parseError !== undefined)
|
|
348
|
+
err.push(`❌ Parse error: ${read.parseError}`);
|
|
349
|
+
signatures.push(...read.signatures);
|
|
350
|
+
}
|
|
351
|
+
catch (e) {
|
|
352
|
+
err.push(`⚠️ Failed to parse ${f.path}: ${e.message}`);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
out.push(` Extracted ${String(signatures.length)} test signatures.`);
|
|
356
|
+
const duplicates = findDuplicates(config, signatures);
|
|
357
|
+
if (duplicates.length === 0) {
|
|
358
|
+
out.push('', '✅ No duplicates found.');
|
|
359
|
+
return { code: 0, out, err };
|
|
360
|
+
}
|
|
361
|
+
out.push('', '❌ Found potential duplicate tests:');
|
|
362
|
+
for (const d of [...duplicates].sort((x, y) => y.similarity - x.similarity)) {
|
|
363
|
+
out.push('', ` ${(d.similarity * 100).toFixed(0)}% Match:`, ` A: ${d.file1}:${String(d.line1)} "${d.test1}"`, ` B: ${d.file2}:${String(d.line2)} "${d.test2}"`);
|
|
364
|
+
}
|
|
365
|
+
out.push('', `Total potential duplicates: ${String(duplicates.length)}`);
|
|
366
|
+
return { code: 1, out, err };
|
|
367
|
+
};
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
// flaky-tests: run the listed test files several times with Vitest and
|
|
2
|
+
// refuse a test that passes in some runs and fails in others.
|
|
3
|
+
//
|
|
4
|
+
// Verdicts (evaluateRuns):
|
|
5
|
+
// - a test passes in some runs and fails in others → flaky, exit 1
|
|
6
|
+
// - every run fails → not flaky, still exit 1
|
|
7
|
+
// - a run fails without collecting a test case → runner failure, exit 1
|
|
8
|
+
// - every run passes → exit 0
|
|
9
|
+
// A file that fails every time is not flaky, but it is not a pass either.
|
|
10
|
+
import { spawnSync } from 'node:child_process';
|
|
11
|
+
import { relative, resolve } from 'node:path';
|
|
12
|
+
import { readVitestJsonReports } from "./vitest-json-report.js";
|
|
13
|
+
/** `npx vitest …`, its output captured. */
|
|
14
|
+
export const npxVitest = (args, cwd) => spawnSync('npx', ['vitest', ...args], { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 10 * 1024 * 1024 });
|
|
15
|
+
export const DEFAULT_CONFIG = { include: /\.(test|spec)\.(ts|tsx|js|jsx)$/, packageRoots: [], exclude: [] };
|
|
16
|
+
const isRecord = (v) => typeof v === 'object' && v !== null && !Array.isArray(v);
|
|
17
|
+
const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === 'string');
|
|
18
|
+
/** The config a product passes; each field replaces its default, and an unknown field is refused. */
|
|
19
|
+
export const parseConfig = (text) => {
|
|
20
|
+
if (text === undefined)
|
|
21
|
+
return DEFAULT_CONFIG;
|
|
22
|
+
const raw = JSON.parse(text);
|
|
23
|
+
if (!isRecord(raw))
|
|
24
|
+
throw new Error('the config is not a JSON object');
|
|
25
|
+
const unknown = Object.keys(raw).filter((k) => !['include', 'packageRoots', 'exclude'].includes(k));
|
|
26
|
+
if (unknown.length > 0)
|
|
27
|
+
throw new Error(`the config has unknown field(s): ${unknown.join(', ')}`);
|
|
28
|
+
const { include, packageRoots, exclude } = raw;
|
|
29
|
+
if (include !== undefined && typeof include !== 'string')
|
|
30
|
+
throw new Error('include is not a regex string');
|
|
31
|
+
if (packageRoots !== undefined && !isStrings(packageRoots))
|
|
32
|
+
throw new Error('packageRoots is not a list of strings');
|
|
33
|
+
if (exclude !== undefined && !isStrings(exclude))
|
|
34
|
+
throw new Error('exclude is not a list of strings');
|
|
35
|
+
return {
|
|
36
|
+
include: include === undefined ? DEFAULT_CONFIG.include : new RegExp(include),
|
|
37
|
+
packageRoots: packageRoots ?? DEFAULT_CONFIG.packageRoots,
|
|
38
|
+
exclude: exclude ?? DEFAULT_CONFIG.exclude,
|
|
39
|
+
};
|
|
40
|
+
};
|
|
41
|
+
/** A run count: an integer from 1 to 1000, or null. */
|
|
42
|
+
export const parseRuns = (text) => {
|
|
43
|
+
if (!/^[+-]?\d+$/.test(text))
|
|
44
|
+
return null;
|
|
45
|
+
const value = Number(text);
|
|
46
|
+
return Number.isSafeInteger(value) && value >= 1 && value <= 1000 ? value : null;
|
|
47
|
+
};
|
|
48
|
+
export const DEFAULT_RUNS = 5;
|
|
49
|
+
/** Keep the tail of the runner output so a huge stack trace stays readable. */
|
|
50
|
+
const MAX_OUTPUT_CHARS = 6000;
|
|
51
|
+
/** A JSON line: the reporter's payload, or a test's own logged object. */
|
|
52
|
+
const isReporterPayload = (line) => line.trim().startsWith('{');
|
|
53
|
+
/** The output that is not the JSON payload: what a human needs when a suite fails to load. */
|
|
54
|
+
export const runnerNoise = (stdout, stderr) => {
|
|
55
|
+
const stdoutNoise = stdout
|
|
56
|
+
.split('\n')
|
|
57
|
+
.filter((l) => l.trim().length > 0 && !isReporterPayload(l))
|
|
58
|
+
.join('\n');
|
|
59
|
+
return [stderr.trim(), stdoutNoise.trim()].filter(Boolean).join('\n');
|
|
60
|
+
};
|
|
61
|
+
/** A root's run that exited non-zero delivering no test case: an unloadable file or a crashed runner. */
|
|
62
|
+
export const isEmptyFailure = (status, collected) => status !== 0 && collected === 0;
|
|
63
|
+
const tail = (text) => (text.length > MAX_OUTPUT_CHARS ? `…(truncated)…\n${text.slice(-MAX_OUTPUT_CHARS)}` : text);
|
|
64
|
+
/** The verdict on the collected per-test stats and per-run outcomes; no I/O. */
|
|
65
|
+
export const evaluateRuns = (results, runOutcomes) => {
|
|
66
|
+
const lines = [];
|
|
67
|
+
const allTests = Array.from(results.values());
|
|
68
|
+
const flakyTests = allTests.filter((r) => r.isFlaky);
|
|
69
|
+
const consistentFailures = allTests.filter((r) => r.failures === r.runs && r.runs > 0);
|
|
70
|
+
const failedRuns = runOutcomes.filter((r) => !r.passed);
|
|
71
|
+
// Judged per root and per run, never in aggregate: a run that collected
|
|
72
|
+
// tests must not launder one that could not.
|
|
73
|
+
const emptyFailedRuns = failedRuns.filter((r) => r.emptyFailure);
|
|
74
|
+
const allRunsFailed = runOutcomes.length > 0 && failedRuns.length === runOutcomes.length;
|
|
75
|
+
const lastFailureOutput = failedRuns
|
|
76
|
+
.map((r) => r.output)
|
|
77
|
+
.filter(Boolean)
|
|
78
|
+
.at(-1);
|
|
79
|
+
const flakyLines = [];
|
|
80
|
+
for (const t of flakyTests) {
|
|
81
|
+
flakyLines.push('', ` ${t.file}`, ` Rate: ${(t.flakinessRate * 100).toFixed(0)}% (${String(t.failures)}/${String(t.runs)} failed)`, ` Test: ${t.testName}`);
|
|
82
|
+
const [first] = t.failureMessages;
|
|
83
|
+
if (first !== undefined)
|
|
84
|
+
flakyLines.push(` Error: ${first.substring(0, 200)}...`);
|
|
85
|
+
}
|
|
86
|
+
// A run that did not execute the file takes the verdict; flaky tests seen in the other runs are still listed.
|
|
87
|
+
if (emptyFailedRuns.length > 0) {
|
|
88
|
+
lines.push('', '❌ Test execution failed: the runner exited non-zero without collecting a single test case.', ` ${String(emptyFailedRuns.length)}/${String(runOutcomes.length)} run(s) failed that way. The file under test is not flaky — it did not run at all.`, '', ' Runner output (last failing run):', '', tail(lastFailureOutput ?? ' (the runner produced no output)'));
|
|
89
|
+
if (flakyTests.length > 0)
|
|
90
|
+
lines.push('', '⚠️ The runs that did execute also found flaky tests:', ...flakyLines);
|
|
91
|
+
return { verdict: 'runner-failure', exitCode: 1, lines };
|
|
92
|
+
}
|
|
93
|
+
if (flakyTests.length > 0) {
|
|
94
|
+
lines.push('', '⚠️ Found flaky tests:', ...flakyLines);
|
|
95
|
+
return { verdict: 'flaky', exitCode: 1, lines };
|
|
96
|
+
}
|
|
97
|
+
if (allRunsFailed || consistentFailures.length > 0) {
|
|
98
|
+
lines.push('', '❌ No flaky tests found, but the tests failed in EVERY run.', ' A deterministic failure is not flakiness — it is a broken test or environment.', '');
|
|
99
|
+
for (const t of consistentFailures) {
|
|
100
|
+
lines.push(` ❌ ${t.file}`, ` Test: ${t.testName}`);
|
|
101
|
+
const [first] = t.failureMessages;
|
|
102
|
+
if (first !== undefined)
|
|
103
|
+
lines.push(` Error: ${first.split('\n')[0].substring(0, 150)}...`);
|
|
104
|
+
lines.push('');
|
|
105
|
+
}
|
|
106
|
+
if (lastFailureOutput)
|
|
107
|
+
lines.push(' Runner output (last failing run):', '', tail(lastFailureOutput));
|
|
108
|
+
return { verdict: 'consistent-failure', exitCode: 1, lines };
|
|
109
|
+
}
|
|
110
|
+
lines.push('', `✅ No flaky tests found across ${String(results.size)} test cases.`);
|
|
111
|
+
return { verdict: 'clean', exitCode: 0, lines };
|
|
112
|
+
};
|
|
113
|
+
/**
|
|
114
|
+
* Fold one run's JSON reporter output into `results`. An unloadable file is a
|
|
115
|
+
* suite with a non-empty `message` and no assertion results, and the reporter
|
|
116
|
+
* prints nothing to stderr, so the message alone is its cause.
|
|
117
|
+
*/
|
|
118
|
+
export const parseTestResults = (output, results) => {
|
|
119
|
+
const suiteErrors = [];
|
|
120
|
+
let collected = 0;
|
|
121
|
+
const { reports, errors } = readVitestJsonReports(output);
|
|
122
|
+
for (const report of reports) {
|
|
123
|
+
for (const testResult of report.testResults) {
|
|
124
|
+
const file = testResult.name;
|
|
125
|
+
if (testResult.message)
|
|
126
|
+
suiteErrors.push(`${file}\n ${testResult.message}`);
|
|
127
|
+
for (const test of testResult.assertionResults) {
|
|
128
|
+
const testName = test.title || test.fullName || 'unknown';
|
|
129
|
+
const key = `${file}::${testName}`;
|
|
130
|
+
const stats = results.get(key) ?? { testName, file, runs: 0, passes: 0, failures: 0, isFlaky: false, flakinessRate: 0, failureMessages: [] };
|
|
131
|
+
results.set(key, stats);
|
|
132
|
+
stats.runs++;
|
|
133
|
+
collected++;
|
|
134
|
+
if (test.status === 'passed')
|
|
135
|
+
stats.passes++;
|
|
136
|
+
else if (test.status === 'failed') {
|
|
137
|
+
stats.failures++;
|
|
138
|
+
const [first] = test.failureMessages;
|
|
139
|
+
if (first !== undefined)
|
|
140
|
+
stats.failureMessages.push(first);
|
|
141
|
+
}
|
|
142
|
+
// 'skipped', 'todo' and 'pending' count as collected, not as outcomes.
|
|
143
|
+
if (stats.passes > 0 && stats.failures > 0) {
|
|
144
|
+
stats.isFlaky = true;
|
|
145
|
+
stats.flakinessRate = stats.failures / stats.runs;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
return { suiteErrors, collected, reports: reports.length, failedReports: reports.filter((r) => !r.success).length, reportErrors: errors };
|
|
151
|
+
};
|
|
152
|
+
/**
|
|
153
|
+
* Did this root's run leave a report the detector can trust? A clean exit
|
|
154
|
+
* with no report, a report that broke the contract, or a clean exit over a
|
|
155
|
+
* report saying `success: false` is a harness failure.
|
|
156
|
+
*/
|
|
157
|
+
export const harnessFailure = (status, parsed) => {
|
|
158
|
+
if (parsed.reportErrors.length > 0)
|
|
159
|
+
return parsed.reportErrors.join('\n');
|
|
160
|
+
if (status === 0 && parsed.reports === 0)
|
|
161
|
+
return 'vitest exited 0 but printed no JSON report';
|
|
162
|
+
if (status === 0 && parsed.failedReports > 0)
|
|
163
|
+
return 'vitest exited 0 but its JSON report says success: false';
|
|
164
|
+
return null;
|
|
165
|
+
};
|
|
166
|
+
export const readRootRun = (root, status, stdout, results) => {
|
|
167
|
+
// Judged on a scratch map first: an untrusted root's outcomes must not reach the shared results.
|
|
168
|
+
const parsed = parseTestResults(stdout, new Map());
|
|
169
|
+
const harness = harnessFailure(status, parsed);
|
|
170
|
+
if (harness === null)
|
|
171
|
+
parseTestResults(stdout, results);
|
|
172
|
+
return {
|
|
173
|
+
parsed,
|
|
174
|
+
trusted: harness === null,
|
|
175
|
+
emptyFailure: harness !== null || isEmptyFailure(status, parsed.collected),
|
|
176
|
+
errors: harness === null ? '' : `[${root}] untrusted test report: ${harness}\n`,
|
|
177
|
+
};
|
|
178
|
+
};
|
|
179
|
+
/**
|
|
180
|
+
* The files grouped by package root, each relative to its root; '.' (the
|
|
181
|
+
* repository root) first, for files under none. A root holds a file at
|
|
182
|
+
* `<root>/…`, or the path equal to it.
|
|
183
|
+
*/
|
|
184
|
+
export const groupByRoot = (config, files) => {
|
|
185
|
+
const byRoot = new Map([['.', []]]);
|
|
186
|
+
for (const file of files) {
|
|
187
|
+
const root = config.packageRoots.find((r) => file.startsWith(`${r}/`) || file === r) ?? '.';
|
|
188
|
+
const list = byRoot.get(root) ?? [];
|
|
189
|
+
byRoot.set(root, list);
|
|
190
|
+
list.push(file);
|
|
191
|
+
}
|
|
192
|
+
return byRoot;
|
|
193
|
+
};
|
|
194
|
+
/**
|
|
195
|
+
* The detector: `runs` passes over the listed files (paths relative to
|
|
196
|
+
* `projectRoot`), each package root's files in one Vitest call from that root.
|
|
197
|
+
* Lines go to `out` as the original printed them.
|
|
198
|
+
*/
|
|
199
|
+
export const runFlakyTests = (config, runs, mode, projectRoot, files, runner, out) => {
|
|
200
|
+
out(`🔄 Detecting flaky tests (${String(runs)} runs)...`);
|
|
201
|
+
out(` Mode: ${mode}`);
|
|
202
|
+
if (files.length === 0) {
|
|
203
|
+
out('No files found to analyze.');
|
|
204
|
+
return 0;
|
|
205
|
+
}
|
|
206
|
+
out(` Running on ${String(files.length)} files...`);
|
|
207
|
+
const results = new Map();
|
|
208
|
+
const byRoot = groupByRoot(config, files);
|
|
209
|
+
const runOutcomes = [];
|
|
210
|
+
for (let i = 1; i <= runs; i++) {
|
|
211
|
+
let runSuccess = true;
|
|
212
|
+
let runEmptyFailure = false;
|
|
213
|
+
let groupErrors = '';
|
|
214
|
+
for (const [root, rootFiles] of byRoot) {
|
|
215
|
+
if (rootFiles.length === 0)
|
|
216
|
+
continue;
|
|
217
|
+
const args = ['run', '--reporter=json', '--passWithNoTests', ...config.exclude.flatMap((g) => ['--exclude', g]), ...rootFiles.map((f) => relative(root, f))];
|
|
218
|
+
try {
|
|
219
|
+
const result = runner(args, resolve(projectRoot, root));
|
|
220
|
+
// A runner that never started has no output to judge.
|
|
221
|
+
if (result.error)
|
|
222
|
+
throw result.error;
|
|
223
|
+
// A runner killed before it wrote anything leaves null streams.
|
|
224
|
+
const stdout = result.stdout ?? '';
|
|
225
|
+
const stderr = result.stderr ?? '';
|
|
226
|
+
const rootRun = readRootRun(root, result.status, stdout, results);
|
|
227
|
+
runEmptyFailure ||= rootRun.emptyFailure;
|
|
228
|
+
runSuccess &&= rootRun.trusted;
|
|
229
|
+
groupErrors += rootRun.errors;
|
|
230
|
+
if (result.status !== 0) {
|
|
231
|
+
runSuccess = false;
|
|
232
|
+
const noise = runnerNoise(stdout, stderr);
|
|
233
|
+
groupErrors += `[${root}] vitest exited with status ${String(result.status)}\n`;
|
|
234
|
+
for (const suiteError of rootRun.parsed.suiteErrors)
|
|
235
|
+
groupErrors += `${suiteError}\n`;
|
|
236
|
+
if (noise)
|
|
237
|
+
groupErrors += `${noise}\n`;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
catch (e) {
|
|
241
|
+
runSuccess = false;
|
|
242
|
+
runEmptyFailure = true;
|
|
243
|
+
groupErrors += `[${root}] Spawn Error: ${e.message}\n`;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
out(` Run ${String(i)}/${String(runs)}: ${runSuccess ? '✅ Passed' : '❌ Failed'}`);
|
|
247
|
+
runOutcomes.push({ passed: runSuccess, emptyFailure: runEmptyFailure, output: groupErrors });
|
|
248
|
+
}
|
|
249
|
+
const evaluation = evaluateRuns(results, runOutcomes);
|
|
250
|
+
evaluation.lines.forEach(out);
|
|
251
|
+
return evaluation.exitCode;
|
|
252
|
+
};
|