@specter-automation/test 1.0.151 → 1.0.153
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -2,6 +2,3 @@ export declare function setActiveWorld(world: any): void;
|
|
|
2
2
|
export declare function getActiveWorld(): any;
|
|
3
3
|
export declare function setAiDiagnosticsFormatter(fn: (diag: any) => string): void;
|
|
4
4
|
export declare function captureSpecterFailure(world: any, testInfo: any, source?: string): Promise<void>;
|
|
5
|
-
export declare function extractFunctionParams(fn: any): string;
|
|
6
|
-
export declare function wrapTestFunction(fn: any): any;
|
|
7
|
-
export declare function wrapTestInstance<T extends (...args: any[]) => any>(targetTest: T): T;
|
|
@@ -4,9 +4,6 @@ exports.setActiveWorld = setActiveWorld;
|
|
|
4
4
|
exports.getActiveWorld = getActiveWorld;
|
|
5
5
|
exports.setAiDiagnosticsFormatter = setAiDiagnosticsFormatter;
|
|
6
6
|
exports.captureSpecterFailure = captureSpecterFailure;
|
|
7
|
-
exports.extractFunctionParams = extractFunctionParams;
|
|
8
|
-
exports.wrapTestFunction = wrapTestFunction;
|
|
9
|
-
exports.wrapTestInstance = wrapTestInstance;
|
|
10
7
|
let activeWorld = null;
|
|
11
8
|
let diagnosticsFormatter = null;
|
|
12
9
|
function setActiveWorld(world) {
|
|
@@ -60,69 +57,3 @@ async function captureSpecterFailure(world, testInfo, source = 'test') {
|
|
|
60
57
|
console.error(`[Specter] Failed to capture AI diagnostics on test failure (${source}):`, e);
|
|
61
58
|
}
|
|
62
59
|
}
|
|
63
|
-
function extractFunctionParams(fn) {
|
|
64
|
-
const str = fn.toString();
|
|
65
|
-
const match = str.match(/^(?:async\s*)?(?:function\s*)?\s*\(([^)]*)\)/);
|
|
66
|
-
if (match)
|
|
67
|
-
return match[1];
|
|
68
|
-
const arrowMatch = str.match(/^(?:async\s*)?([a-zA-Z0-9_{}\s,]+)=>/);
|
|
69
|
-
if (arrowMatch)
|
|
70
|
-
return arrowMatch[1];
|
|
71
|
-
return "fixtures, testInfo";
|
|
72
|
-
}
|
|
73
|
-
function wrapTestFunction(fn) {
|
|
74
|
-
if (typeof fn !== "function")
|
|
75
|
-
return fn;
|
|
76
|
-
const params = extractFunctionParams(fn);
|
|
77
|
-
try {
|
|
78
|
-
const wrapperFactory = new Function("fn", "captureSpecterFailure", "getActiveWorld", `
|
|
79
|
-
return async function(${params}) {
|
|
80
|
-
try {
|
|
81
|
-
return await fn.apply(this, arguments);
|
|
82
|
-
} catch (err) {
|
|
83
|
-
const testInfo = (arguments.length > 1 && arguments[1]) ? arguments[1] : undefined;
|
|
84
|
-
const world = (testInfo && testInfo._specterWorld) || (arguments[0] && arguments[0].world) || (arguments[0] && arguments[0].server && arguments[0].server.world) || getActiveWorld();
|
|
85
|
-
if (world && testInfo) {
|
|
86
|
-
try {
|
|
87
|
-
await captureSpecterFailure(world, testInfo, 'test-body');
|
|
88
|
-
} catch (e) {}
|
|
89
|
-
}
|
|
90
|
-
throw err;
|
|
91
|
-
}
|
|
92
|
-
};
|
|
93
|
-
`);
|
|
94
|
-
return wrapperFactory(fn, captureSpecterFailure, getActiveWorld);
|
|
95
|
-
}
|
|
96
|
-
catch (e) {
|
|
97
|
-
return fn;
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
function wrapTestInstance(targetTest) {
|
|
101
|
-
const wrapCall = (originalFn) => {
|
|
102
|
-
return function (title, ...rest) {
|
|
103
|
-
if (rest.length > 0 && typeof rest[rest.length - 1] === "function") {
|
|
104
|
-
rest[rest.length - 1] = wrapTestFunction(rest[rest.length - 1]);
|
|
105
|
-
}
|
|
106
|
-
return originalFn.apply(this, [title, ...rest]);
|
|
107
|
-
};
|
|
108
|
-
};
|
|
109
|
-
const wrapped = wrapCall(targetTest);
|
|
110
|
-
Object.setPrototypeOf(wrapped, targetTest);
|
|
111
|
-
for (const key of Object.getOwnPropertyNames(targetTest)) {
|
|
112
|
-
const desc = Object.getOwnPropertyDescriptor(targetTest, key);
|
|
113
|
-
if (desc)
|
|
114
|
-
Object.defineProperty(wrapped, key, desc);
|
|
115
|
-
}
|
|
116
|
-
for (const modifier of ["only", "skip", "fixme", "fail"]) {
|
|
117
|
-
if (typeof targetTest[modifier] === "function") {
|
|
118
|
-
wrapped[modifier] = wrapCall(targetTest[modifier]);
|
|
119
|
-
Object.setPrototypeOf(wrapped[modifier], targetTest[modifier]);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
const origExtend = targetTest.extend.bind(targetTest);
|
|
123
|
-
wrapped.extend = function (...args) {
|
|
124
|
-
const extended = origExtend(...args);
|
|
125
|
-
return wrapTestInstance(extended);
|
|
126
|
-
};
|
|
127
|
-
return wrapped;
|
|
128
|
-
}
|
|
@@ -10,6 +10,8 @@ export default class SpecterReporter implements Reporter {
|
|
|
10
10
|
private runEnded;
|
|
11
11
|
private testStepCounters;
|
|
12
12
|
private stepIdMap;
|
|
13
|
+
/** Specter expect steps whose wrapper step has started */
|
|
14
|
+
private wrappedExpectSteps;
|
|
13
15
|
/** Playwright's outer `Expect "<matcher>"` steps for Specter matchers. They are shown under their wrapper step's name. */
|
|
14
16
|
private specterExpectSteps;
|
|
15
17
|
/** Image paths already sent to the runner, per test, so step and test attachments aren't sent twice. */
|
|
@@ -49,8 +51,15 @@ export default class SpecterReporter implements Reporter {
|
|
|
49
51
|
onBegin(config: FullConfig, suite: Suite): void;
|
|
50
52
|
onTestBegin(test: TestCase): void;
|
|
51
53
|
private isNestedExpect;
|
|
52
|
-
/** The step opened by instrumentMatchers directly under a Specter expect step. */
|
|
54
|
+
/** The step opened by instrumentMatchers directly under a Specter expect step (the first one there). */
|
|
55
|
+
private specterWrapperSteps;
|
|
53
56
|
private isSpecterWrapper;
|
|
57
|
+
/**
|
|
58
|
+
* Any other step directly under a Specter expect step, opened after the matcher's own step ended: the failure
|
|
59
|
+
* screenshot a failing matcher takes, for one. Not shown as rows (and never taken for the wrapper, which would
|
|
60
|
+
* rename the expect step and leave rows that never end); their images still go to the expect step's row.
|
|
61
|
+
*/
|
|
62
|
+
private isExtraStepInMatcher;
|
|
54
63
|
private readPerfCapture;
|
|
55
64
|
private readMatcherReport;
|
|
56
65
|
private getParentStepName;
|
|
@@ -141,6 +141,8 @@ class SpecterReporter {
|
|
|
141
141
|
runEnded = false;
|
|
142
142
|
testStepCounters = new Map();
|
|
143
143
|
stepIdMap = new WeakMap();
|
|
144
|
+
/** Specter expect steps whose wrapper step has started */
|
|
145
|
+
wrappedExpectSteps = new WeakSet();
|
|
144
146
|
/** Playwright's outer `Expect "<matcher>"` steps for Specter matchers. They are shown under their wrapper step's name. */
|
|
145
147
|
specterExpectSteps = new WeakSet();
|
|
146
148
|
/** Image paths already sent to the runner, per test, so step and test attachments aren't sent twice. */
|
|
@@ -322,9 +324,19 @@ class SpecterReporter {
|
|
|
322
324
|
}
|
|
323
325
|
return false;
|
|
324
326
|
}
|
|
325
|
-
/** The step opened by instrumentMatchers directly under a Specter expect step. */
|
|
327
|
+
/** The step opened by instrumentMatchers directly under a Specter expect step (the first one there). */
|
|
328
|
+
specterWrapperSteps = new WeakSet();
|
|
326
329
|
isSpecterWrapper(step) {
|
|
327
|
-
return
|
|
330
|
+
return this.specterWrapperSteps.has(step);
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Any other step directly under a Specter expect step, opened after the matcher's own step ended: the failure
|
|
334
|
+
* screenshot a failing matcher takes, for one. Not shown as rows (and never taken for the wrapper, which would
|
|
335
|
+
* rename the expect step and leave rows that never end); their images still go to the expect step's row.
|
|
336
|
+
*/
|
|
337
|
+
isExtraStepInMatcher(step) {
|
|
338
|
+
return step.category === 'test.step' && !!step.parent && this.specterExpectSteps.has(step.parent)
|
|
339
|
+
&& !this.specterWrapperSteps.has(step);
|
|
328
340
|
}
|
|
329
341
|
readPerfCapture(step) {
|
|
330
342
|
const attachment = step.attachments?.find(a => a.name === specter_perf_1.SPECTER_PERF_ATTACHMENT && a.body);
|
|
@@ -376,6 +388,14 @@ class SpecterReporter {
|
|
|
376
388
|
this.specterExpectSteps.add(step);
|
|
377
389
|
return;
|
|
378
390
|
}
|
|
391
|
+
// The first test.step under a Specter expect step is the matcher's wrapper; later ones are extras
|
|
392
|
+
if (step.category === 'test.step' && step.parent && this.specterExpectSteps.has(step.parent)) {
|
|
393
|
+
if (this.wrappedExpectSteps.has(step.parent)) {
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
this.wrappedExpectSteps.add(step.parent);
|
|
397
|
+
this.specterWrapperSteps.add(step);
|
|
398
|
+
}
|
|
379
399
|
const isCustomWrapper = this.isSpecterWrapper(step);
|
|
380
400
|
// 2. Ignore other expect wrapper steps (e.g. from helpers that title steps "expect(...)")
|
|
381
401
|
if (step.category === 'test.step' && step.title.startsWith('expect(') && !isCustomWrapper) {
|
|
@@ -429,8 +449,8 @@ class SpecterReporter {
|
|
|
429
449
|
}
|
|
430
450
|
return;
|
|
431
451
|
}
|
|
432
|
-
// 2. Ignore inner expect steps nested under matchers
|
|
433
|
-
if (this.isNestedExpect(step)) {
|
|
452
|
+
// 2. Ignore inner expect steps nested under matchers, and other steps a matcher opened after its own
|
|
453
|
+
if (this.isNestedExpect(step) || this.isExtraStepInMatcher(step)) {
|
|
434
454
|
return;
|
|
435
455
|
}
|
|
436
456
|
const isSpecterExpect = this.specterExpectSteps.has(step);
|
|
@@ -40,6 +40,5 @@ type CreateWorldOptions = {
|
|
|
40
40
|
autoJoinHost?: boolean;
|
|
41
41
|
launchClient?: boolean;
|
|
42
42
|
};
|
|
43
|
-
declare const
|
|
44
|
-
export declare const test: typeof extendedTest;
|
|
43
|
+
export declare const test: import("playwright/test").TestType<import("playwright/test").PlaywrightTestArgs & import("playwright/test").PlaywrightTestOptions & SpecterTestFixtures & SpecterTestOptions, import("playwright/test").PlaywrightWorkerArgs & import("playwright/test").PlaywrightWorkerOptions & SpecterWorkerFixtures>;
|
|
45
44
|
export {};
|
|
@@ -218,10 +218,25 @@ const extendedTest = test_1.test.extend({
|
|
|
218
218
|
}
|
|
219
219
|
},
|
|
220
220
|
autoScreenshot: [async ({ world }, use, testInfo) => {
|
|
221
|
+
// Capture as soon as the test function ends, before afterEach hooks and fixture teardown change the scene.
|
|
222
|
+
// Playwright runs these callbacks between the two (its own screenshot-on-failure uses them). Wrapping test()
|
|
223
|
+
// instead would make Playwright record the wrapper as every test's location (file and line).
|
|
224
|
+
const onTestFunctionEnd = async () => {
|
|
225
|
+
if (testInfo.status !== 'passed' && testInfo.status !== 'skipped') {
|
|
226
|
+
await (0, failure_capture_1.captureSpecterFailure)(world, testInfo, 'test-end');
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
const callbacks = testInfo._onDidFinishTestFunctionCallbacks;
|
|
230
|
+
if (callbacks instanceof Set) {
|
|
231
|
+
callbacks.add(onTestFunctionEnd);
|
|
232
|
+
}
|
|
221
233
|
try {
|
|
222
234
|
await use();
|
|
223
235
|
}
|
|
224
236
|
finally {
|
|
237
|
+
if (callbacks instanceof Set) {
|
|
238
|
+
callbacks.delete(onTestFunctionEnd);
|
|
239
|
+
}
|
|
225
240
|
// FIX: Catch 'failed', 'timedOut', and 'interrupted' statuses if not already captured earlier
|
|
226
241
|
if (testInfo.status !== 'passed' && testInfo.status !== 'skipped') {
|
|
227
242
|
if (!testInfo._specterFailureCaptured) {
|
|
@@ -232,4 +247,4 @@ const extendedTest = test_1.test.extend({
|
|
|
232
247
|
}, { auto: true }]
|
|
233
248
|
});
|
|
234
249
|
(0, failure_capture_1.setAiDiagnosticsFormatter)(gas_matchers_1.formatAiDiagnostics);
|
|
235
|
-
exports.test =
|
|
250
|
+
exports.test = extendedTest;
|