@specter-automation/test 1.0.130 → 1.0.132
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/locators/actor-locator.d.ts +20 -0
- package/dist/core/locators/actor-locator.js +44 -0
- package/dist/core/locators/widget-locator.d.ts +7 -0
- package/dist/core/locators/widget-locator.js +11 -0
- package/dist/core/specter-client.d.ts +36 -1
- package/dist/core/specter-client.js +69 -0
- package/dist/core/specter-matcher-report.d.ts +3 -2
- package/dist/core/specter-matcher-report.js +5 -0
- package/dist/core/specter-perf.d.ts +95 -0
- package/dist/core/specter-perf.js +49 -0
- package/dist/core/specter-reporter.d.ts +1 -0
- package/dist/core/specter-reporter.js +18 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +6 -1
- package/dist/matchers/instrument.js +13 -2
- package/dist/matchers/performance-matchers.d.ts +101 -4
- package/dist/matchers/performance-matchers.js +121 -0
- package/package.json +1 -1
|
@@ -66,6 +66,26 @@ export declare class ActorLocator<T = any> extends SpecterLocator<T> {
|
|
|
66
66
|
duration?: number;
|
|
67
67
|
color?: string;
|
|
68
68
|
}): Promise<void>;
|
|
69
|
+
/**
|
|
70
|
+
* Clicks the actor.
|
|
71
|
+
*
|
|
72
|
+
* @param options.strategy
|
|
73
|
+
* - 'function' (default): runs its click events (OnClicked, the blueprint's ActorOnClicked) when it has any.
|
|
74
|
+
* When nothing listens for clicks on it, it is clicked through the game view instead.
|
|
75
|
+
* - 'viewport': clicks where it is on screen. The click goes straight to the game's view, so the game reads it like
|
|
76
|
+
* a player's (GetMousePosition, the button press, Enhanced Input, click events) without moving the real mouse
|
|
77
|
+
* or needing focus. Fails when something else is in front of it.
|
|
78
|
+
* - 'broadcast': runs its click events whether or not anything listens.
|
|
79
|
+
* @param options.button 'left' (default), 'right' or 'middle'
|
|
80
|
+
* @param options.timeout How long to keep trying, e.g. while it walks on screen (default: 5000ms)
|
|
81
|
+
* @param options.context Optional text to tell identical steps apart in the report
|
|
82
|
+
*/
|
|
83
|
+
click(options?: {
|
|
84
|
+
strategy?: 'function' | 'viewport' | 'broadcast';
|
|
85
|
+
button?: 'left' | 'right' | 'middle';
|
|
86
|
+
timeout?: number;
|
|
87
|
+
context?: string;
|
|
88
|
+
}): Promise<void>;
|
|
69
89
|
/**
|
|
70
90
|
* Explicitly destroys this actor in the Unreal Engine world.
|
|
71
91
|
*/
|
|
@@ -138,6 +138,50 @@ class ActorLocator extends specter_locator_1.SpecterLocator {
|
|
|
138
138
|
async updateStatsBillboard(options) {
|
|
139
139
|
return this.showStatsBillboard(options);
|
|
140
140
|
}
|
|
141
|
+
/**
|
|
142
|
+
* Clicks the actor.
|
|
143
|
+
*
|
|
144
|
+
* @param options.strategy
|
|
145
|
+
* - 'function' (default): runs its click events (OnClicked, the blueprint's ActorOnClicked) when it has any.
|
|
146
|
+
* When nothing listens for clicks on it, it is clicked through the game view instead.
|
|
147
|
+
* - 'viewport': clicks where it is on screen. The click goes straight to the game's view, so the game reads it like
|
|
148
|
+
* a player's (GetMousePosition, the button press, Enhanced Input, click events) without moving the real mouse
|
|
149
|
+
* or needing focus. Fails when something else is in front of it.
|
|
150
|
+
* - 'broadcast': runs its click events whether or not anything listens.
|
|
151
|
+
* @param options.button 'left' (default), 'right' or 'middle'
|
|
152
|
+
* @param options.timeout How long to keep trying, e.g. while it walks on screen (default: 5000ms)
|
|
153
|
+
* @param options.context Optional text to tell identical steps apart in the report
|
|
154
|
+
*/
|
|
155
|
+
async click(options) {
|
|
156
|
+
const baseName = `Click Actor @ ${this.toString()}`;
|
|
157
|
+
const stepName = options?.context ? `${baseName} [${options.context}]` : baseName;
|
|
158
|
+
return await test_1.default.step(stepName, async () => {
|
|
159
|
+
const strategy = options?.strategy || 'function';
|
|
160
|
+
const button = options?.button || 'left';
|
|
161
|
+
const timeout = options?.timeout ?? 5000;
|
|
162
|
+
const startTime = Date.now();
|
|
163
|
+
let lastError = null;
|
|
164
|
+
while (Date.now() - startTime < timeout) {
|
|
165
|
+
try {
|
|
166
|
+
const response = await this.evaluate('ClickActor', { strategy, button }, undefined, true);
|
|
167
|
+
if (response?.status === 'success') {
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
lastError = new Error(response?.error || 'ClickActor failed');
|
|
171
|
+
}
|
|
172
|
+
catch (err) {
|
|
173
|
+
lastError = err;
|
|
174
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
175
|
+
if (msg.toLowerCase().includes('strict mode')) {
|
|
176
|
+
throw err;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
await new Promise(resolve => setTimeout(resolve, 100));
|
|
180
|
+
}
|
|
181
|
+
const details = lastError ? `\nUnderlying Error: ${lastError.message}` : '';
|
|
182
|
+
throw new Error(`[Specter Timeout] Timed out after ${timeout}ms attempting to click actor: ${this.toString()}.${details}`);
|
|
183
|
+
});
|
|
184
|
+
}
|
|
141
185
|
/**
|
|
142
186
|
* Explicitly destroys this actor in the Unreal Engine world.
|
|
143
187
|
*/
|
|
@@ -78,6 +78,13 @@ export declare class WidgetLocator<T = any> extends SpecterLocator<T> {
|
|
|
78
78
|
getByText(text: string | RegExp, options?: {
|
|
79
79
|
exact?: boolean;
|
|
80
80
|
}): WidgetLocator;
|
|
81
|
+
/**
|
|
82
|
+
* Finds UI inside this one by a name that stays the same when its words change: a UMG widget's name in the
|
|
83
|
+
* designer, the test id a hand-drawn HUD reports with FSpecterHUD, or a Slate widget's tag or Automation Driver id.
|
|
84
|
+
*/
|
|
85
|
+
getByTestId(testId: string, options?: {
|
|
86
|
+
exact?: boolean;
|
|
87
|
+
}): WidgetLocator;
|
|
81
88
|
/**
|
|
82
89
|
* Alias for getChild / navigating down the widget hierarchy.
|
|
83
90
|
*/
|
|
@@ -181,6 +181,17 @@ class WidgetLocator extends specter_locator_1.SpecterLocator {
|
|
|
181
181
|
];
|
|
182
182
|
return new WidgetLocator(this.transport, newPipeline);
|
|
183
183
|
}
|
|
184
|
+
/**
|
|
185
|
+
* Finds UI inside this one by a name that stays the same when its words change: a UMG widget's name in the
|
|
186
|
+
* designer, the test id a hand-drawn HUD reports with FSpecterHUD, or a Slate widget's tag or Automation Driver id.
|
|
187
|
+
*/
|
|
188
|
+
getByTestId(testId, options) {
|
|
189
|
+
const newPipeline = [
|
|
190
|
+
...this.pipeline,
|
|
191
|
+
{ action: 'getWidgetByTestId', value: testId, exact: options?.exact ?? true }
|
|
192
|
+
];
|
|
193
|
+
return new WidgetLocator(this.transport, newPipeline);
|
|
194
|
+
}
|
|
184
195
|
/**
|
|
185
196
|
* Alias for getChild / navigating down the widget hierarchy.
|
|
186
197
|
*/
|
|
@@ -5,6 +5,7 @@ import { SpecterKeyboard } from './specter-keyboard';
|
|
|
5
5
|
import { UnrealKey } from './specter-keys';
|
|
6
6
|
import { SpecterInputSequence } from './specter-input-sequence';
|
|
7
7
|
import { SpecterNetwork } from './specter-network';
|
|
8
|
+
import { type MeasurePerformanceOptions, type PerformanceCapture } from './specter-perf';
|
|
8
9
|
export interface SpecterRegistry {
|
|
9
10
|
}
|
|
10
11
|
export interface SpecterFunctionRegistry {
|
|
@@ -196,7 +197,14 @@ export interface SpawnOptions {
|
|
|
196
197
|
name?: string;
|
|
197
198
|
}
|
|
198
199
|
export interface ClickOptions {
|
|
199
|
-
|
|
200
|
+
/**
|
|
201
|
+
* - 'function' (default): runs the click handler directly; falls back to a click when there is none.
|
|
202
|
+
* - 'hardware': a real mouse click through Slate. Moves the real cursor.
|
|
203
|
+
* - 'viewport': a click handed straight to the game's view, as the game reads a player's click. The real cursor
|
|
204
|
+
* doesn't move and the game doesn't need focus. For things the game draws or picks itself: HUD elements.
|
|
205
|
+
* - 'broadcast': only the click handler; fails when there is none.
|
|
206
|
+
*/
|
|
207
|
+
strategy?: 'hardware' | 'broadcast' | 'function' | 'viewport';
|
|
200
208
|
timeout?: number;
|
|
201
209
|
tolerance?: number;
|
|
202
210
|
waitForStable?: boolean;
|
|
@@ -381,6 +389,13 @@ export declare class SpecterWorld {
|
|
|
381
389
|
getByText(text: string | RegExp, options?: {
|
|
382
390
|
exact?: boolean;
|
|
383
391
|
}): WidgetLocator;
|
|
392
|
+
/**
|
|
393
|
+
* Finds UI by a name that stays the same when its words change: a UMG widget's name in the designer, the test
|
|
394
|
+
* id a hand-drawn HUD reports with FSpecterHUD, or a Slate widget's tag or Automation Driver id.
|
|
395
|
+
*/
|
|
396
|
+
getByTestId(testId: string, options?: {
|
|
397
|
+
exact?: boolean;
|
|
398
|
+
}): WidgetLocator;
|
|
384
399
|
getByPlayerId(playerId: number): ActorLocator;
|
|
385
400
|
subsystem(subsystemName: string): SpecterLocator;
|
|
386
401
|
/**
|
|
@@ -534,6 +549,26 @@ export declare class SpecterWorld {
|
|
|
534
549
|
frameTimeMs: number;
|
|
535
550
|
memoryMb: number;
|
|
536
551
|
}>;
|
|
552
|
+
/**
|
|
553
|
+
* Measures how smoothly the game runs while `action` plays out: every frame from start to end, not a single reading.
|
|
554
|
+
* Returns frame-time percentiles, FPS, game/render/GPU thread times, draw calls, memory, and each hitch with what
|
|
555
|
+
* happened in the game around it (garbage collection, level streaming, timeline events).
|
|
556
|
+
*
|
|
557
|
+
* ```ts
|
|
558
|
+
* const perf = await world.measurePerformance(async () => {
|
|
559
|
+
* await hero.ability('Q').activate();
|
|
560
|
+
* await world.waitForTimeout(3000);
|
|
561
|
+
* }, { name: 'Ability barrage' });
|
|
562
|
+
*
|
|
563
|
+
* await expect(perf).toHaveFrameTimeP95Below(20);
|
|
564
|
+
* await expect(perf).toHaveNoHitchesOver(50);
|
|
565
|
+
* await expect(perf).toHavePeakMemoryBelow(4096);
|
|
566
|
+
* ```
|
|
567
|
+
*
|
|
568
|
+
* In the Specter runner, the "Measure performance" step shows a frame-time graph, the numbers, and the hitches.
|
|
569
|
+
* Measure in a packaged or standalone build for numbers that match what players get; the editor adds its own cost.
|
|
570
|
+
*/
|
|
571
|
+
measurePerformance(action: () => Promise<unknown> | unknown, options?: MeasurePerformanceOptions): Promise<PerformanceCapture>;
|
|
537
572
|
/**
|
|
538
573
|
* Checks if a navigation path exists between start and end vectors on the NavMesh.
|
|
539
574
|
*/
|
|
@@ -10,6 +10,7 @@ const locators_1 = require("./locators");
|
|
|
10
10
|
const test_1 = __importDefault(require("@playwright/test"));
|
|
11
11
|
const specter_keyboard_1 = require("./specter-keyboard");
|
|
12
12
|
const specter_network_1 = require("./specter-network");
|
|
13
|
+
const specter_perf_1 = require("./specter-perf");
|
|
13
14
|
var SpecterEventType;
|
|
14
15
|
(function (SpecterEventType) {
|
|
15
16
|
SpecterEventType["StandardVariable"] = "StandardVariable";
|
|
@@ -732,6 +733,13 @@ class SpecterWorld {
|
|
|
732
733
|
const exactMatch = options?.exact ?? (typeof text === 'string' ? true : false);
|
|
733
734
|
return new locators_1.WidgetLocator(this.transport, [{ action: 'getWidgetByText', value: textStr, exact: exactMatch }]);
|
|
734
735
|
}
|
|
736
|
+
/**
|
|
737
|
+
* Finds UI by a name that stays the same when its words change: a UMG widget's name in the designer, the test
|
|
738
|
+
* id a hand-drawn HUD reports with FSpecterHUD, or a Slate widget's tag or Automation Driver id.
|
|
739
|
+
*/
|
|
740
|
+
getByTestId(testId, options) {
|
|
741
|
+
return new locators_1.WidgetLocator(this.transport, [{ action: 'getWidgetByTestId', value: testId, exact: options?.exact ?? true }]);
|
|
742
|
+
}
|
|
735
743
|
getByPlayerId(playerId) {
|
|
736
744
|
return new locators_1.ActorLocator(this.transport, [{ action: 'getByPlayerId', value: playerId.toString() }]);
|
|
737
745
|
}
|
|
@@ -1160,6 +1168,67 @@ class SpecterWorld {
|
|
|
1160
1168
|
const response = await this.evaluate("GetPerformanceMetrics", {});
|
|
1161
1169
|
return response.data ?? { fps: 60, frameTimeMs: 16.66, memoryMb: 0 };
|
|
1162
1170
|
}
|
|
1171
|
+
/**
|
|
1172
|
+
* Measures how smoothly the game runs while `action` plays out: every frame from start to end, not a single reading.
|
|
1173
|
+
* Returns frame-time percentiles, FPS, game/render/GPU thread times, draw calls, memory, and each hitch with what
|
|
1174
|
+
* happened in the game around it (garbage collection, level streaming, timeline events).
|
|
1175
|
+
*
|
|
1176
|
+
* ```ts
|
|
1177
|
+
* const perf = await world.measurePerformance(async () => {
|
|
1178
|
+
* await hero.ability('Q').activate();
|
|
1179
|
+
* await world.waitForTimeout(3000);
|
|
1180
|
+
* }, { name: 'Ability barrage' });
|
|
1181
|
+
*
|
|
1182
|
+
* await expect(perf).toHaveFrameTimeP95Below(20);
|
|
1183
|
+
* await expect(perf).toHaveNoHitchesOver(50);
|
|
1184
|
+
* await expect(perf).toHavePeakMemoryBelow(4096);
|
|
1185
|
+
* ```
|
|
1186
|
+
*
|
|
1187
|
+
* In the Specter runner, the "Measure performance" step shows a frame-time graph, the numbers, and the hitches.
|
|
1188
|
+
* Measure in a packaged or standalone build for numbers that match what players get; the editor adds its own cost.
|
|
1189
|
+
*/
|
|
1190
|
+
async measurePerformance(action, options = {}) {
|
|
1191
|
+
const name = options.name ?? 'performance';
|
|
1192
|
+
const hitchMs = options.hitchMs ?? 50;
|
|
1193
|
+
const portTag = this.transport.friendlyPortTag;
|
|
1194
|
+
const stepTitle = `${portTag}Measure performance: ${name}`;
|
|
1195
|
+
return await test_1.default.step(stepTitle, async () => {
|
|
1196
|
+
await this.transport.sendCommand({ command: 'StartPerfCapture', payload: { name, hitchMs } });
|
|
1197
|
+
let failed = false;
|
|
1198
|
+
let failure;
|
|
1199
|
+
try {
|
|
1200
|
+
await action();
|
|
1201
|
+
}
|
|
1202
|
+
catch (error) {
|
|
1203
|
+
failed = true;
|
|
1204
|
+
failure = error;
|
|
1205
|
+
}
|
|
1206
|
+
let capture;
|
|
1207
|
+
try {
|
|
1208
|
+
const response = await this.transport.sendCommand({
|
|
1209
|
+
command: 'StopPerfCapture',
|
|
1210
|
+
payload: { stepName: stepTitle, client: portTag.replace(/^\[|\]\s*$/g, '') }
|
|
1211
|
+
});
|
|
1212
|
+
capture = response.data;
|
|
1213
|
+
}
|
|
1214
|
+
catch (error) {
|
|
1215
|
+
// The action's own failure matters more than not getting its numbers
|
|
1216
|
+
if (!failed)
|
|
1217
|
+
throw error;
|
|
1218
|
+
}
|
|
1219
|
+
if (capture) {
|
|
1220
|
+
// Hands the capture to the runner via the SpecterReporter (attachments.push adds no "Attach" step)
|
|
1221
|
+
test_1.default.info().attachments.push({
|
|
1222
|
+
name: specter_perf_1.SPECTER_PERF_ATTACHMENT,
|
|
1223
|
+
contentType: 'application/json',
|
|
1224
|
+
body: Buffer.from(JSON.stringify(capture), 'utf-8')
|
|
1225
|
+
});
|
|
1226
|
+
}
|
|
1227
|
+
if (failed)
|
|
1228
|
+
throw failure;
|
|
1229
|
+
return capture;
|
|
1230
|
+
});
|
|
1231
|
+
}
|
|
1163
1232
|
/**
|
|
1164
1233
|
* Checks if a navigation path exists between start and end vectors on the NavMesh.
|
|
1165
1234
|
*/
|
|
@@ -9,6 +9,7 @@ import type { LocatorMatchers } from '../matchers/locator-matchers';
|
|
|
9
9
|
import type { WidgetMatchers } from '../matchers/widget-matchers';
|
|
10
10
|
import type { MaterialMatchers } from '../matchers/material-matchers';
|
|
11
11
|
import type { AiMatchers } from '../matchers/ai-matchers';
|
|
12
|
+
import type { PerformanceMatchers } from '../matchers/performance-matchers';
|
|
12
13
|
/** Attachment name used to carry a matcher result from the worker to the reporter. Leading "_" hides it in Playwright's reports. */
|
|
13
14
|
export declare const SPECTER_MATCHER_ATTACHMENT = "_specter-matcher";
|
|
14
15
|
/** Structured result of one Specter matcher call. */
|
|
@@ -25,12 +26,12 @@ export interface SpecterMatcherReport {
|
|
|
25
26
|
expected?: unknown;
|
|
26
27
|
actual?: unknown;
|
|
27
28
|
}
|
|
28
|
-
export type SpecterMatcherName = keyof GasMatchers | keyof ActorMatchers | keyof LocatorMatchers | keyof WidgetMatchers | keyof MaterialMatchers | keyof AiMatchers;
|
|
29
|
+
export type SpecterMatcherName = keyof GasMatchers | keyof ActorMatchers | keyof LocatorMatchers | keyof WidgetMatchers | keyof MaterialMatchers | keyof AiMatchers | keyof PerformanceMatchers;
|
|
29
30
|
/**
|
|
30
31
|
* Every Specter matcher. The compile-time check below fails the build if a
|
|
31
32
|
* matcher is added to one of the matcher interfaces but not to this list.
|
|
32
33
|
*/
|
|
33
|
-
export declare const SPECTER_MATCHER_NAMES: readonly ["toHaveAttributeValue", "toHaveAttributeChange", "toHaveGameplayTag", "toHaveGameplayEffect", "toHaveAppliedGameplayEffect", "toHaveGameplayAbility", "toHaveAbilityCooldown", "toHaveGameplayEffectStackCount", "toHaveGameplayEffectLevel", "toHaveAttributeBaseValue", "toBeExecutingAbility", "toHaveActivatedAbility", "toHaveGameplayTagCount", "toHaveAnyGameplayTags", "toHaveAllGameplayTags", "toHaveMovedFurtherThan", "toHaveLocation", "toHaveInputMappingContext", "toHaveInputActionValue", "toHavePlayerScore", "toBeNiagaraActive", "toHaveNiagaraAsset", "toBeAudioPlaying", "toHaveSoundAsset", "toBePhysicsSimulating", "toBePhysicsAsleep", "toHaveVelocity", "toHaveCameraFOV", "toBeCameraShaking", "toHaveCameraViewTarget", "toBeAINavigating", "toHaveTeamId", "toBeHostileTowards", "toBePossessed", "toBePlayerControlled", "toBeLocallyControlled", "toHaveController", "toBeFalling", "toBeSwimming", "toBeFlying", "toBeCrouching", "toBeMovingOnGround", "toHaveMovementMode", "toBeVisible", "toHavePropertyValue", "toBeInViewport", "toBeDestroyed", "toHaveCount", "toHaveText", "toHaveValue", "toContainText", "toBeEnabled", "toHaveMaterialScalarParameter", "toHaveMaterialVectorParameter", "toHaveMaterial", "toBeDynamicMaterial", "toHaveMaterialTextureParameter", "toHaveBlackboardValue", "toHaveAIState", "toBeMovingTowards", "toBeMovingAwayFrom", "toBeWithinDistance", "toBeFollowing", "toHaveLineOfSightTo", "toBeRunningBehaviorTree"];
|
|
34
|
+
export declare const SPECTER_MATCHER_NAMES: readonly ["toHaveAttributeValue", "toHaveAttributeChange", "toHaveGameplayTag", "toHaveGameplayEffect", "toHaveAppliedGameplayEffect", "toHaveGameplayAbility", "toHaveAbilityCooldown", "toHaveGameplayEffectStackCount", "toHaveGameplayEffectLevel", "toHaveAttributeBaseValue", "toBeExecutingAbility", "toHaveActivatedAbility", "toHaveGameplayTagCount", "toHaveAnyGameplayTags", "toHaveAllGameplayTags", "toHaveMovedFurtherThan", "toHaveLocation", "toHaveInputMappingContext", "toHaveInputActionValue", "toHavePlayerScore", "toBeNiagaraActive", "toHaveNiagaraAsset", "toBeAudioPlaying", "toHaveSoundAsset", "toBePhysicsSimulating", "toBePhysicsAsleep", "toHaveVelocity", "toHaveCameraFOV", "toBeCameraShaking", "toHaveCameraViewTarget", "toBeAINavigating", "toHaveTeamId", "toBeHostileTowards", "toBePossessed", "toBePlayerControlled", "toBeLocallyControlled", "toHaveController", "toBeFalling", "toBeSwimming", "toBeFlying", "toBeCrouching", "toBeMovingOnGround", "toHaveMovementMode", "toBeVisible", "toHavePropertyValue", "toBeInViewport", "toBeDestroyed", "toHaveCount", "toHaveText", "toHaveValue", "toContainText", "toBeEnabled", "toHaveMaterialScalarParameter", "toHaveMaterialVectorParameter", "toHaveMaterial", "toBeDynamicMaterial", "toHaveMaterialTextureParameter", "toHaveBlackboardValue", "toHaveAIState", "toBeMovingTowards", "toBeMovingAwayFrom", "toBeWithinDistance", "toBeFollowing", "toHaveLineOfSightTo", "toBeRunningBehaviorTree", "toHaveFPS", "toHaveMinFPS", "toHaveFrameTimeBelow", "toHaveMemoryBelow", "toMaintainPerformanceBudget", "toHaveFrameTimeP95Below", "toHaveFrameTimeP99Below", "toHaveAverageFrameTimeBelow", "toHaveAverageFPSAbove", "toHaveOnePercentLowFPSAbove", "toHaveNoHitchesOver", "toHavePeakMemoryBelow", "toHaveMemoryGrowthBelow", "toHaveDrawCallsBelow"];
|
|
34
35
|
export declare function isSpecterMatcher(name: string): boolean;
|
|
35
36
|
/**
|
|
36
37
|
* Parses Playwright's expect step title, e.g. `Expect "soft not toHaveGameplayTag"`.
|
|
@@ -33,6 +33,11 @@ exports.SPECTER_MATCHER_NAMES = [
|
|
|
33
33
|
// ai-matchers
|
|
34
34
|
'toHaveBlackboardValue', 'toHaveAIState', 'toBeMovingTowards', 'toBeMovingAwayFrom',
|
|
35
35
|
'toBeWithinDistance', 'toBeFollowing', 'toHaveLineOfSightTo', 'toBeRunningBehaviorTree',
|
|
36
|
+
// performance-matchers
|
|
37
|
+
'toHaveFPS', 'toHaveMinFPS', 'toHaveFrameTimeBelow', 'toHaveMemoryBelow', 'toMaintainPerformanceBudget',
|
|
38
|
+
'toHaveFrameTimeP95Below', 'toHaveFrameTimeP99Below', 'toHaveAverageFrameTimeBelow', 'toHaveAverageFPSAbove',
|
|
39
|
+
'toHaveOnePercentLowFPSAbove', 'toHaveNoHitchesOver', 'toHavePeakMemoryBelow', 'toHaveMemoryGrowthBelow',
|
|
40
|
+
'toHaveDrawCallsBelow',
|
|
36
41
|
];
|
|
37
42
|
// If this line errors, add the names shown in the error to SPECTER_MATCHER_NAMES.
|
|
38
43
|
const _allMatchersListed = true;
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Performance captures: what world.measurePerformance() returns, and what the performance matchers check.
|
|
3
|
+
* Keep this file free of runtime imports so the reporter can load it.
|
|
4
|
+
*/
|
|
5
|
+
/** Attachment carrying a capture from the worker to the SpecterReporter, which hands it to the runner. */
|
|
6
|
+
export declare const SPECTER_PERF_ATTACHMENT = "_specter-perf";
|
|
7
|
+
export interface PerformanceStat {
|
|
8
|
+
avg: number;
|
|
9
|
+
p95: number;
|
|
10
|
+
}
|
|
11
|
+
/** A frame that took longer than the capture's hitch threshold, and what happened in the game around it. */
|
|
12
|
+
export interface PerformanceHitch {
|
|
13
|
+
/** Seconds into the capture (real time). */
|
|
14
|
+
timeSeconds: number;
|
|
15
|
+
/** Seconds since the test started, in game time (the runner timeline's time); -1 if unknown. */
|
|
16
|
+
testSeconds: number;
|
|
17
|
+
frameMs: number;
|
|
18
|
+
gameMs: number;
|
|
19
|
+
renderMs: number;
|
|
20
|
+
gpuMs: number;
|
|
21
|
+
/** Whichever took longest that frame: "Game thread", "Render thread" or "GPU". A hint for where to look. */
|
|
22
|
+
bound: string;
|
|
23
|
+
/** Garbage collection, level streaming and timeline events from that frame and the one before it. */
|
|
24
|
+
events: string[];
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Everything measured between the start and end of a `world.measurePerformance()` call.
|
|
28
|
+
* Frame times are in milliseconds, memory in megabytes (used physical memory of the game process).
|
|
29
|
+
*/
|
|
30
|
+
export interface PerformanceCapture {
|
|
31
|
+
name: string;
|
|
32
|
+
testName: string;
|
|
33
|
+
stepName: string;
|
|
34
|
+
client: string;
|
|
35
|
+
durationSeconds: number;
|
|
36
|
+
frameCount: number;
|
|
37
|
+
/** Seconds since the test started, in game time, when the capture began; -1 if unknown. */
|
|
38
|
+
testStartSeconds: number;
|
|
39
|
+
frameTimeMs: {
|
|
40
|
+
avg: number;
|
|
41
|
+
p50: number;
|
|
42
|
+
p90: number;
|
|
43
|
+
p95: number;
|
|
44
|
+
p99: number;
|
|
45
|
+
max: number;
|
|
46
|
+
};
|
|
47
|
+
/** `onePercentLow` is the frame rate of the slowest 1% of frames (1000 / p99). */
|
|
48
|
+
fps: {
|
|
49
|
+
avg: number;
|
|
50
|
+
onePercentLow: number;
|
|
51
|
+
min: number;
|
|
52
|
+
};
|
|
53
|
+
gameThreadMs: PerformanceStat;
|
|
54
|
+
renderThreadMs: PerformanceStat;
|
|
55
|
+
gpuMs: PerformanceStat;
|
|
56
|
+
drawCalls: {
|
|
57
|
+
avg: number;
|
|
58
|
+
max: number;
|
|
59
|
+
};
|
|
60
|
+
memoryMb: {
|
|
61
|
+
start: number;
|
|
62
|
+
peak: number;
|
|
63
|
+
end: number;
|
|
64
|
+
growth: number;
|
|
65
|
+
};
|
|
66
|
+
hitchThresholdMs: number;
|
|
67
|
+
/** Every frame over the threshold (`hitches` lists the slowest 50). */
|
|
68
|
+
hitchCount: number;
|
|
69
|
+
/** Slowest first. */
|
|
70
|
+
hitches: PerformanceHitch[];
|
|
71
|
+
/** Graph points (every frame, or the slowest frame of each slice for long captures), as columns. */
|
|
72
|
+
frames: {
|
|
73
|
+
t: number[];
|
|
74
|
+
frameMs: number[];
|
|
75
|
+
gameMs: number[];
|
|
76
|
+
renderMs: number[];
|
|
77
|
+
gpuMs: number[];
|
|
78
|
+
drawCalls: number[];
|
|
79
|
+
memoryMb: number[];
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
export interface MeasurePerformanceOptions {
|
|
83
|
+
/** Shown in the runner and in failure messages. Default: "performance". */
|
|
84
|
+
name?: string;
|
|
85
|
+
/** Frames slower than this many milliseconds count as hitches. Default: 50. */
|
|
86
|
+
hitchMs?: number;
|
|
87
|
+
}
|
|
88
|
+
export declare function isPerformanceCapture(value: unknown): value is PerformanceCapture;
|
|
89
|
+
/** "112 ms at 4.20 s in the test (Game thread): Garbage collection (38.0 ms); GE Applied [Stun]" */
|
|
90
|
+
export declare function describeHitch(hitch: PerformanceHitch): string;
|
|
91
|
+
/** The slowest few hitches, one per line, for failure messages. */
|
|
92
|
+
export declare function describeHitches(capture: PerformanceCapture, limit?: number): string;
|
|
93
|
+
/** "avg 8.2 ms · p95 11.4 ms · 121 fps · 2 hitches over 50 ms" */
|
|
94
|
+
export declare function summarizeCapture(capture: PerformanceCapture): string;
|
|
95
|
+
export declare function formatMs(value: number): string;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Performance captures: what world.measurePerformance() returns, and what the performance matchers check.
|
|
4
|
+
* Keep this file free of runtime imports so the reporter can load it.
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.SPECTER_PERF_ATTACHMENT = void 0;
|
|
8
|
+
exports.isPerformanceCapture = isPerformanceCapture;
|
|
9
|
+
exports.describeHitch = describeHitch;
|
|
10
|
+
exports.describeHitches = describeHitches;
|
|
11
|
+
exports.summarizeCapture = summarizeCapture;
|
|
12
|
+
exports.formatMs = formatMs;
|
|
13
|
+
/** Attachment carrying a capture from the worker to the SpecterReporter, which hands it to the runner. */
|
|
14
|
+
exports.SPECTER_PERF_ATTACHMENT = '_specter-perf';
|
|
15
|
+
function isPerformanceCapture(value) {
|
|
16
|
+
return !!value && typeof value === 'object'
|
|
17
|
+
&& typeof value.frameCount === 'number'
|
|
18
|
+
&& typeof value.frameTimeMs === 'object';
|
|
19
|
+
}
|
|
20
|
+
/** "112 ms at 4.20 s in the test (Game thread): Garbage collection (38.0 ms); GE Applied [Stun]" */
|
|
21
|
+
function describeHitch(hitch) {
|
|
22
|
+
const when = hitch.testSeconds >= 0
|
|
23
|
+
? `at ${hitch.testSeconds.toFixed(2)} s in the test`
|
|
24
|
+
: `${hitch.timeSeconds.toFixed(2)} s into the capture`;
|
|
25
|
+
const events = hitch.events.length > 0 ? `: ${hitch.events.join('; ')}` : '';
|
|
26
|
+
return `${formatMs(hitch.frameMs)} ${when} (${hitch.bound})${events}`;
|
|
27
|
+
}
|
|
28
|
+
/** The slowest few hitches, one per line, for failure messages. */
|
|
29
|
+
function describeHitches(capture, limit = 5) {
|
|
30
|
+
if (capture.hitchCount === 0) {
|
|
31
|
+
return `No frame took longer than ${formatMs(capture.hitchThresholdMs)}.`;
|
|
32
|
+
}
|
|
33
|
+
const lines = capture.hitches.slice(0, limit).map(h => ` - ${describeHitch(h)}`);
|
|
34
|
+
const more = capture.hitchCount - Math.min(capture.hitches.length, limit);
|
|
35
|
+
if (more > 0) {
|
|
36
|
+
lines.push(` - and ${more} more`);
|
|
37
|
+
}
|
|
38
|
+
return `${capture.hitchCount} frame${capture.hitchCount === 1 ? '' : 's'} over ${formatMs(capture.hitchThresholdMs)}, slowest first:\n${lines.join('\n')}`;
|
|
39
|
+
}
|
|
40
|
+
/** "avg 8.2 ms · p95 11.4 ms · 121 fps · 2 hitches over 50 ms" */
|
|
41
|
+
function summarizeCapture(capture) {
|
|
42
|
+
const hitches = capture.hitchCount === 0
|
|
43
|
+
? `no hitches over ${formatMs(capture.hitchThresholdMs)}`
|
|
44
|
+
: `${capture.hitchCount} hitch${capture.hitchCount === 1 ? '' : 'es'} over ${formatMs(capture.hitchThresholdMs)}`;
|
|
45
|
+
return `avg ${formatMs(capture.frameTimeMs.avg)} · p95 ${formatMs(capture.frameTimeMs.p95)} · ${Math.round(capture.fps.avg)} fps · ${hitches}`;
|
|
46
|
+
}
|
|
47
|
+
function formatMs(value) {
|
|
48
|
+
return value >= 100 ? `${value.toFixed(0)} ms` : `${value.toFixed(1)} ms`;
|
|
49
|
+
}
|
|
@@ -37,6 +37,7 @@ export default class SpecterReporter implements Reporter {
|
|
|
37
37
|
private isNestedExpect;
|
|
38
38
|
/** The step opened by instrumentMatchers directly under a Specter expect step. */
|
|
39
39
|
private isSpecterWrapper;
|
|
40
|
+
private readPerfCapture;
|
|
40
41
|
private readMatcherReport;
|
|
41
42
|
private getParentStepName;
|
|
42
43
|
onStepBegin(test: TestCase, result: TestResult, step: TestStep): void;
|
|
@@ -43,6 +43,7 @@ const os = __importStar(require("os"));
|
|
|
43
43
|
const path = __importStar(require("path"));
|
|
44
44
|
const specter_matcher_report_1 = require("./specter-matcher-report");
|
|
45
45
|
const specter_zip_1 = require("./specter-zip");
|
|
46
|
+
const specter_perf_1 = require("./specter-perf");
|
|
46
47
|
function pad(value, length = 2) {
|
|
47
48
|
return String(value).padStart(length, '0');
|
|
48
49
|
}
|
|
@@ -266,6 +267,17 @@ class SpecterReporter {
|
|
|
266
267
|
isSpecterWrapper(step) {
|
|
267
268
|
return step.category === 'test.step' && !!step.parent && this.specterExpectSteps.has(step.parent);
|
|
268
269
|
}
|
|
270
|
+
readPerfCapture(step) {
|
|
271
|
+
const attachment = step.attachments?.find(a => a.name === specter_perf_1.SPECTER_PERF_ATTACHMENT && a.body);
|
|
272
|
+
if (!attachment?.body)
|
|
273
|
+
return undefined;
|
|
274
|
+
try {
|
|
275
|
+
return JSON.parse(attachment.body.toString('utf-8'));
|
|
276
|
+
}
|
|
277
|
+
catch {
|
|
278
|
+
return undefined;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
269
281
|
readMatcherReport(step) {
|
|
270
282
|
const attachment = step.attachments?.find(a => a.name === specter_matcher_report_1.SPECTER_MATCHER_ATTACHMENT && a.body);
|
|
271
283
|
if (!attachment?.body)
|
|
@@ -395,7 +407,12 @@ class SpecterReporter {
|
|
|
395
407
|
duration: duration,
|
|
396
408
|
errorMessage: step.error ? this.formatError(step.error, result) : ''
|
|
397
409
|
};
|
|
398
|
-
// 4.
|
|
410
|
+
// 4. Frame timing measured in this step (world.measurePerformance), for the runner's perf panel
|
|
411
|
+
const perf = this.readPerfCapture(step);
|
|
412
|
+
if (perf) {
|
|
413
|
+
payload.perf = perf;
|
|
414
|
+
}
|
|
415
|
+
// 5. Structured result for the runner's results grid (Specter matchers only)
|
|
399
416
|
const report = isSpecterExpect ? this.readMatcherReport(step) : undefined;
|
|
400
417
|
if (report) {
|
|
401
418
|
payload.matcherResult = {
|
package/dist/index.d.ts
CHANGED
|
@@ -6,6 +6,8 @@ export { default as SpecterReporter } from './core/specter-reporter';
|
|
|
6
6
|
export { expectAll } from './core/expect-all';
|
|
7
7
|
export { SPECTER_MATCHER_ATTACHMENT } from './core/specter-matcher-report';
|
|
8
8
|
export type { SpecterMatcherReport } from './core/specter-matcher-report';
|
|
9
|
+
export { SPECTER_PERF_ATTACHMENT, describeHitch, describeHitches, summarizeCapture } from './core/specter-perf';
|
|
10
|
+
export type { PerformanceCapture, PerformanceHitch, PerformanceStat, MeasurePerformanceOptions } from './core/specter-perf';
|
|
9
11
|
export type { UnrealKey } from './core/specter-keys';
|
|
10
12
|
export type { SpecterActorEvents, SpecterWidgetEvents, SpecterWorldEvents, CustomActorEvents, CustomWidgetEvents, CustomWorldEvents, ActorEventMap, WidgetEventMap, WorldEventMap, EventPredicate, WaitForEventOptions } from './core/events';
|
|
11
13
|
export { SpecterInputSequence } from './core/specter-input-sequence';
|
package/dist/index.js
CHANGED
|
@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
3
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.visualExpect = exports.performanceExpect = exports.aiExpect = exports.materialExpect = exports.runConditionalInputLoop = exports.evaluateCondition = exports.SpecterNetwork = exports.SpecterInputSequence = exports.SPECTER_MATCHER_ATTACHMENT = exports.expectAll = exports.SpecterReporter = exports.AiControllerLocator = exports.ActorLocator = exports.WidgetLocator = exports.SpecterLocator = exports.VectorUtils = exports.SpecterWorld = exports.SpecterTransport = exports.expect = exports.test = void 0;
|
|
6
|
+
exports.visualExpect = exports.performanceExpect = exports.aiExpect = exports.materialExpect = exports.runConditionalInputLoop = exports.evaluateCondition = exports.SpecterNetwork = exports.SpecterInputSequence = exports.summarizeCapture = exports.describeHitches = exports.describeHitch = exports.SPECTER_PERF_ATTACHMENT = exports.SPECTER_MATCHER_ATTACHMENT = exports.expectAll = exports.SpecterReporter = exports.AiControllerLocator = exports.ActorLocator = exports.WidgetLocator = exports.SpecterLocator = exports.VectorUtils = exports.SpecterWorld = exports.SpecterTransport = exports.expect = exports.test = void 0;
|
|
7
7
|
var test_wrapper_1 = require("./core/test-wrapper");
|
|
8
8
|
Object.defineProperty(exports, "test", { enumerable: true, get: function () { return test_wrapper_1.test; } });
|
|
9
9
|
Object.defineProperty(exports, "expect", { enumerable: true, get: function () { return test_wrapper_1.expect; } });
|
|
@@ -22,6 +22,11 @@ var expect_all_1 = require("./core/expect-all");
|
|
|
22
22
|
Object.defineProperty(exports, "expectAll", { enumerable: true, get: function () { return expect_all_1.expectAll; } });
|
|
23
23
|
var specter_matcher_report_1 = require("./core/specter-matcher-report");
|
|
24
24
|
Object.defineProperty(exports, "SPECTER_MATCHER_ATTACHMENT", { enumerable: true, get: function () { return specter_matcher_report_1.SPECTER_MATCHER_ATTACHMENT; } });
|
|
25
|
+
var specter_perf_1 = require("./core/specter-perf");
|
|
26
|
+
Object.defineProperty(exports, "SPECTER_PERF_ATTACHMENT", { enumerable: true, get: function () { return specter_perf_1.SPECTER_PERF_ATTACHMENT; } });
|
|
27
|
+
Object.defineProperty(exports, "describeHitch", { enumerable: true, get: function () { return specter_perf_1.describeHitch; } });
|
|
28
|
+
Object.defineProperty(exports, "describeHitches", { enumerable: true, get: function () { return specter_perf_1.describeHitches; } });
|
|
29
|
+
Object.defineProperty(exports, "summarizeCapture", { enumerable: true, get: function () { return specter_perf_1.summarizeCapture; } });
|
|
25
30
|
var specter_input_sequence_1 = require("./core/specter-input-sequence");
|
|
26
31
|
Object.defineProperty(exports, "SpecterInputSequence", { enumerable: true, get: function () { return specter_input_sequence_1.SpecterInputSequence; } });
|
|
27
32
|
var specter_network_1 = require("./core/specter-network");
|
|
@@ -6,6 +6,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.instrumentMatchers = instrumentMatchers;
|
|
7
7
|
const test_1 = __importDefault(require("@playwright/test"));
|
|
8
8
|
const specter_matcher_report_1 = require("../core/specter-matcher-report");
|
|
9
|
+
const specter_perf_1 = require("../core/specter-perf");
|
|
9
10
|
const MAX_ARG_LENGTH = 60;
|
|
10
11
|
const MAX_ARGS_LENGTH = 140;
|
|
11
12
|
const MAX_REPORT_VALUE_LENGTH = 4000;
|
|
@@ -68,12 +69,16 @@ function isLocatorLike(value) {
|
|
|
68
69
|
function isWorldLike(value) {
|
|
69
70
|
return !!value && typeof value === 'object' && typeof value.getPerformanceMetrics === 'function';
|
|
70
71
|
}
|
|
72
|
+
/** A capture from world.measurePerformance(): shown by name rather than as a wall of numbers */
|
|
73
|
+
function captureName(value) {
|
|
74
|
+
return (0, specter_perf_1.isPerformanceCapture)(value) ? `measurePerformance('${truncate(value.name, MAX_ARG_LENGTH)}')` : undefined;
|
|
75
|
+
}
|
|
71
76
|
function describe(receiver) {
|
|
72
77
|
if (isLocatorLike(receiver))
|
|
73
78
|
return receiver.toString();
|
|
74
79
|
if (isWorldLike(receiver))
|
|
75
80
|
return 'world';
|
|
76
|
-
return formatArg(receiver);
|
|
81
|
+
return captureName(receiver) ?? formatArg(receiver);
|
|
77
82
|
}
|
|
78
83
|
function selectorOf(receiver) {
|
|
79
84
|
if (isLocatorLike(receiver)) {
|
|
@@ -81,7 +86,7 @@ function selectorOf(receiver) {
|
|
|
81
86
|
}
|
|
82
87
|
if (isWorldLike(receiver))
|
|
83
88
|
return 'world';
|
|
84
|
-
return formatArg(receiver);
|
|
89
|
+
return captureName(receiver) ?? formatArg(receiver);
|
|
85
90
|
}
|
|
86
91
|
function clientTagOf(receiver) {
|
|
87
92
|
try {
|
|
@@ -89,6 +94,8 @@ function clientTagOf(receiver) {
|
|
|
89
94
|
return receiver.portTag;
|
|
90
95
|
if (isWorldLike(receiver) && typeof receiver.transport?.friendlyPortTag === 'string')
|
|
91
96
|
return receiver.transport.friendlyPortTag;
|
|
97
|
+
if ((0, specter_perf_1.isPerformanceCapture)(receiver) && receiver.client)
|
|
98
|
+
return `[${receiver.client}] `;
|
|
92
99
|
return '';
|
|
93
100
|
}
|
|
94
101
|
catch {
|
|
@@ -118,6 +125,8 @@ function formatArg(value) {
|
|
|
118
125
|
return '() => …';
|
|
119
126
|
if (isLocatorLike(value))
|
|
120
127
|
return value.toString();
|
|
128
|
+
if ((0, specter_perf_1.isPerformanceCapture)(value))
|
|
129
|
+
return captureName(value);
|
|
121
130
|
return truncate(safeStringify(toReportValue(value)), MAX_ARG_LENGTH);
|
|
122
131
|
}
|
|
123
132
|
/** Converts values into something JSON-safe and readable: locators become their description, functions a placeholder. */
|
|
@@ -132,6 +141,8 @@ function toReportValue(value, seen = new WeakSet()) {
|
|
|
132
141
|
return value;
|
|
133
142
|
if (isLocatorLike(value))
|
|
134
143
|
return value.toString();
|
|
144
|
+
if ((0, specter_perf_1.isPerformanceCapture)(value))
|
|
145
|
+
return captureName(value);
|
|
135
146
|
if (seen.has(value))
|
|
136
147
|
return '[Circular]';
|
|
137
148
|
seen.add(value);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { SpecterWorld } from '../core/specter-client';
|
|
2
2
|
import { SpecterLocator } from '../core/locators';
|
|
3
|
+
import { type PerformanceCapture } from '../core/specter-perf';
|
|
3
4
|
export interface PerformanceBudgetOptions {
|
|
4
5
|
/** Minimum FPS allowed across all samples (no frame drops below this threshold). */
|
|
5
6
|
minFps?: number;
|
|
@@ -30,7 +31,32 @@ declare global {
|
|
|
30
31
|
toHaveMemoryBelow(this: Matchers<R, SpecterWorld | SpecterLocator>, maxMemoryMb: number, options?: {
|
|
31
32
|
timeout?: number;
|
|
32
33
|
}): Promise<R>;
|
|
33
|
-
toMaintainPerformanceBudget(this: Matchers<R, SpecterWorld | SpecterLocator>, budget: PerformanceBudgetOptions): Promise<R>;
|
|
34
|
+
toMaintainPerformanceBudget(this: Matchers<R, SpecterWorld | SpecterLocator | PerformanceCapture>, budget: PerformanceBudgetOptions): Promise<R>;
|
|
35
|
+
/** 95% of the capture's frames took at most this many milliseconds. */
|
|
36
|
+
toHaveFrameTimeP95Below(this: Matchers<R, PerformanceCapture>, maxMs: number): Promise<R>;
|
|
37
|
+
/** 99% of the capture's frames took at most this many milliseconds. */
|
|
38
|
+
toHaveFrameTimeP99Below(this: Matchers<R, PerformanceCapture>, maxMs: number): Promise<R>;
|
|
39
|
+
/** The capture's average frame time is at most this many milliseconds. */
|
|
40
|
+
toHaveAverageFrameTimeBelow(this: Matchers<R, PerformanceCapture>, maxMs: number): Promise<R>;
|
|
41
|
+
/** The capture averaged at least this frame rate. */
|
|
42
|
+
toHaveAverageFPSAbove(this: Matchers<R, PerformanceCapture>, minFps: number): Promise<R>;
|
|
43
|
+
/** The slowest 1% of the capture's frames still ran at least this frame rate. */
|
|
44
|
+
toHaveOnePercentLowFPSAbove(this: Matchers<R, PerformanceCapture>, minFps: number): Promise<R>;
|
|
45
|
+
/**
|
|
46
|
+
* No frame of the capture took longer than this many milliseconds (or at most `allowed` did).
|
|
47
|
+
* The failure lists the slowest hitches with what happened in the game around each one.
|
|
48
|
+
*/
|
|
49
|
+
toHaveNoHitchesOver(this: Matchers<R, PerformanceCapture>, maxMs: number, options?: {
|
|
50
|
+
allowed?: number;
|
|
51
|
+
}): Promise<R>;
|
|
52
|
+
/** Used memory never went above this many megabytes during the capture. */
|
|
53
|
+
toHavePeakMemoryBelow(this: Matchers<R, PerformanceCapture>, maxMb: number): Promise<R>;
|
|
54
|
+
/** Used memory grew by at most this many megabytes from the start of the capture to its end. */
|
|
55
|
+
toHaveMemoryGrowthBelow(this: Matchers<R, PerformanceCapture>, maxMb: number): Promise<R>;
|
|
56
|
+
/** Draw calls a frame stayed at or under this count, on average (default) or at most. */
|
|
57
|
+
toHaveDrawCallsBelow(this: Matchers<R, PerformanceCapture>, maxDrawCalls: number, options?: {
|
|
58
|
+
stat?: 'avg' | 'max';
|
|
59
|
+
}): Promise<R>;
|
|
34
60
|
}
|
|
35
61
|
}
|
|
36
62
|
}
|
|
@@ -49,6 +75,19 @@ export interface PerformanceMatchers {
|
|
|
49
75
|
timeout?: number;
|
|
50
76
|
}): Promise<void>;
|
|
51
77
|
toMaintainPerformanceBudget(budget: PerformanceBudgetOptions): Promise<void>;
|
|
78
|
+
toHaveFrameTimeP95Below(maxMs: number): Promise<void>;
|
|
79
|
+
toHaveFrameTimeP99Below(maxMs: number): Promise<void>;
|
|
80
|
+
toHaveAverageFrameTimeBelow(maxMs: number): Promise<void>;
|
|
81
|
+
toHaveAverageFPSAbove(minFps: number): Promise<void>;
|
|
82
|
+
toHaveOnePercentLowFPSAbove(minFps: number): Promise<void>;
|
|
83
|
+
toHaveNoHitchesOver(maxMs: number, options?: {
|
|
84
|
+
allowed?: number;
|
|
85
|
+
}): Promise<void>;
|
|
86
|
+
toHavePeakMemoryBelow(maxMb: number): Promise<void>;
|
|
87
|
+
toHaveMemoryGrowthBelow(maxMb: number): Promise<void>;
|
|
88
|
+
toHaveDrawCallsBelow(maxDrawCalls: number, options?: {
|
|
89
|
+
stat?: 'avg' | 'max';
|
|
90
|
+
}): Promise<void>;
|
|
52
91
|
}
|
|
53
92
|
export declare const performanceExpect: import("playwright/test").Expect<{
|
|
54
93
|
toHaveFPS(this: import("playwright/test").ExpectMatcherState, receiver: any, expectedFps: number, options?: {
|
|
@@ -105,14 +144,72 @@ export declare const performanceExpect: import("playwright/test").Expect<{
|
|
|
105
144
|
expected: number;
|
|
106
145
|
}>;
|
|
107
146
|
toMaintainPerformanceBudget(this: import("playwright/test").ExpectMatcherState, receiver: any, budget: PerformanceBudgetOptions): Promise<{
|
|
147
|
+
pass: boolean;
|
|
148
|
+
message: () => string;
|
|
149
|
+
actual: string;
|
|
150
|
+
expected: PerformanceBudgetOptions;
|
|
151
|
+
} | {
|
|
108
152
|
pass: false;
|
|
109
153
|
message: () => string;
|
|
110
154
|
actual: number;
|
|
111
155
|
expected: number;
|
|
112
|
-
}
|
|
156
|
+
}>;
|
|
157
|
+
toHaveFrameTimeP95Below(this: import("playwright/test").ExpectMatcherState, receiver: any, maxMs: number): Promise<{
|
|
113
158
|
pass: boolean;
|
|
114
159
|
message: () => string;
|
|
115
|
-
actual:
|
|
116
|
-
expected:
|
|
160
|
+
actual: number;
|
|
161
|
+
expected: number;
|
|
162
|
+
}>;
|
|
163
|
+
toHaveFrameTimeP99Below(this: import("playwright/test").ExpectMatcherState, receiver: any, maxMs: number): Promise<{
|
|
164
|
+
pass: boolean;
|
|
165
|
+
message: () => string;
|
|
166
|
+
actual: number;
|
|
167
|
+
expected: number;
|
|
168
|
+
}>;
|
|
169
|
+
toHaveAverageFrameTimeBelow(this: import("playwright/test").ExpectMatcherState, receiver: any, maxMs: number): Promise<{
|
|
170
|
+
pass: boolean;
|
|
171
|
+
message: () => string;
|
|
172
|
+
actual: number;
|
|
173
|
+
expected: number;
|
|
174
|
+
}>;
|
|
175
|
+
toHaveAverageFPSAbove(this: import("playwright/test").ExpectMatcherState, receiver: any, minFps: number): Promise<{
|
|
176
|
+
pass: boolean;
|
|
177
|
+
message: () => string;
|
|
178
|
+
actual: number;
|
|
179
|
+
expected: number;
|
|
180
|
+
}>;
|
|
181
|
+
toHaveOnePercentLowFPSAbove(this: import("playwright/test").ExpectMatcherState, receiver: any, minFps: number): Promise<{
|
|
182
|
+
pass: boolean;
|
|
183
|
+
message: () => string;
|
|
184
|
+
actual: number;
|
|
185
|
+
expected: number;
|
|
186
|
+
}>;
|
|
187
|
+
toHaveNoHitchesOver(this: import("playwright/test").ExpectMatcherState, receiver: any, maxMs: number, options?: {
|
|
188
|
+
allowed?: number;
|
|
189
|
+
}): Promise<{
|
|
190
|
+
pass: boolean;
|
|
191
|
+
message: () => string;
|
|
192
|
+
actual: number;
|
|
193
|
+
expected: number;
|
|
194
|
+
}>;
|
|
195
|
+
toHavePeakMemoryBelow(this: import("playwright/test").ExpectMatcherState, receiver: any, maxMb: number): Promise<{
|
|
196
|
+
pass: boolean;
|
|
197
|
+
message: () => string;
|
|
198
|
+
actual: number;
|
|
199
|
+
expected: number;
|
|
200
|
+
}>;
|
|
201
|
+
toHaveMemoryGrowthBelow(this: import("playwright/test").ExpectMatcherState, receiver: any, maxMb: number): Promise<{
|
|
202
|
+
pass: boolean;
|
|
203
|
+
message: () => string;
|
|
204
|
+
actual: number;
|
|
205
|
+
expected: number;
|
|
206
|
+
}>;
|
|
207
|
+
toHaveDrawCallsBelow(this: import("playwright/test").ExpectMatcherState, receiver: any, maxDrawCalls: number, options?: {
|
|
208
|
+
stat?: "avg" | "max";
|
|
209
|
+
}): Promise<{
|
|
210
|
+
pass: boolean;
|
|
211
|
+
message: () => string;
|
|
212
|
+
actual: number;
|
|
213
|
+
expected: number;
|
|
117
214
|
}>;
|
|
118
215
|
}>;
|
|
@@ -3,6 +3,48 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.performanceExpect = void 0;
|
|
4
4
|
const test_1 = require("@playwright/test");
|
|
5
5
|
const instrument_1 = require("./instrument");
|
|
6
|
+
const specter_perf_1 = require("../core/specter-perf");
|
|
7
|
+
function requireCapture(receiver, matcher) {
|
|
8
|
+
if (!(0, specter_perf_1.isPerformanceCapture)(receiver)) {
|
|
9
|
+
throw new Error(`${matcher} checks a capture from world.measurePerformance(), e.g.\n` +
|
|
10
|
+
` const perf = await world.measurePerformance(async () => { ... });\n` +
|
|
11
|
+
` expect(perf).${matcher}(...);`);
|
|
12
|
+
}
|
|
13
|
+
return receiver;
|
|
14
|
+
}
|
|
15
|
+
/** Result of a threshold check on a capture, with the capture's summary (and hitches) in the failure message. */
|
|
16
|
+
function captureCheck(isNot, capture, what, pass, expectedText, actualText, actual, expected, withHitches = false) {
|
|
17
|
+
return {
|
|
18
|
+
pass,
|
|
19
|
+
message: () => {
|
|
20
|
+
const lines = [
|
|
21
|
+
`Expected ${what} of "${capture.name}" to${isNot ? ' not' : ''} be ${expectedText}, but it was ${actualText}.`,
|
|
22
|
+
`${(0, specter_perf_1.summarizeCapture)(capture)} (${capture.frameCount} frames in ${capture.durationSeconds.toFixed(2)} s)`
|
|
23
|
+
];
|
|
24
|
+
if (withHitches || capture.hitchCount > 0) {
|
|
25
|
+
lines.push((0, specter_perf_1.describeHitches)(capture));
|
|
26
|
+
}
|
|
27
|
+
return lines.join('\n');
|
|
28
|
+
},
|
|
29
|
+
actual,
|
|
30
|
+
expected
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
/** Frames over a threshold. Exact when the capture has every frame or listed every hitch over it; otherwise a lower bound. */
|
|
34
|
+
function countFramesOver(capture, maxMs) {
|
|
35
|
+
if (maxMs === capture.hitchThresholdMs) {
|
|
36
|
+
return { count: capture.hitchCount, atLeast: false };
|
|
37
|
+
}
|
|
38
|
+
if (capture.frames.frameMs.length === capture.frameCount) {
|
|
39
|
+
return { count: capture.frames.frameMs.filter(ms => ms > maxMs).length, atLeast: false };
|
|
40
|
+
}
|
|
41
|
+
if (maxMs > capture.hitchThresholdMs) {
|
|
42
|
+
const listed = capture.hitches.filter(h => h.frameMs > maxMs).length;
|
|
43
|
+
const allListed = capture.hitches.length >= capture.hitchCount || listed < capture.hitches.length;
|
|
44
|
+
return { count: listed, atLeast: !allListed };
|
|
45
|
+
}
|
|
46
|
+
return undefined;
|
|
47
|
+
}
|
|
6
48
|
async function fetchMetrics(receiver) {
|
|
7
49
|
if (receiver && typeof receiver.getPerformanceMetrics === 'function') {
|
|
8
50
|
return await receiver.getPerformanceMetrics();
|
|
@@ -123,6 +165,32 @@ exports.performanceExpect = test_1.expect.extend((0, instrument_1.instrumentMatc
|
|
|
123
165
|
},
|
|
124
166
|
async toMaintainPerformanceBudget(receiver, budget) {
|
|
125
167
|
const isNot = this.isNot;
|
|
168
|
+
// A capture from world.measurePerformance(): check the budget against every frame it recorded
|
|
169
|
+
if ((0, specter_perf_1.isPerformanceCapture)(receiver)) {
|
|
170
|
+
const capture = receiver;
|
|
171
|
+
const violations = [];
|
|
172
|
+
if (budget.minFps !== undefined && capture.fps.min < budget.minFps) {
|
|
173
|
+
violations.push(`- Min FPS: ${capture.fps.min.toFixed(1)} (Budget floor: >= ${budget.minFps}) [FAILED]`);
|
|
174
|
+
}
|
|
175
|
+
if (budget.avgFps !== undefined && capture.fps.avg < budget.avgFps) {
|
|
176
|
+
violations.push(`- Avg FPS: ${capture.fps.avg.toFixed(1)} (Budget average: >= ${budget.avgFps}) [FAILED]`);
|
|
177
|
+
}
|
|
178
|
+
if (budget.maxFrameTimeMs !== undefined && capture.frameTimeMs.max > budget.maxFrameTimeMs) {
|
|
179
|
+
violations.push(`- Peak Frame Time: ${capture.frameTimeMs.max.toFixed(2)}ms (Budget ceiling: <= ${budget.maxFrameTimeMs}ms) [FAILED]`);
|
|
180
|
+
}
|
|
181
|
+
if (budget.maxMemoryMb !== undefined && capture.memoryMb.peak > budget.maxMemoryMb) {
|
|
182
|
+
violations.push(`- Peak Memory: ${capture.memoryMb.peak.toFixed(1)}MB (Budget ceiling: <= ${budget.maxMemoryMb}MB) [FAILED]`);
|
|
183
|
+
}
|
|
184
|
+
const summary = `${(0, specter_perf_1.summarizeCapture)(capture)} (${capture.frameCount} frames in ${capture.durationSeconds.toFixed(2)} s)`;
|
|
185
|
+
return {
|
|
186
|
+
pass: violations.length === 0,
|
|
187
|
+
message: () => violations.length === 0
|
|
188
|
+
? `Expected "${capture.name}" to break the performance budget, but it held.\n${summary}`
|
|
189
|
+
: `Performance budget violation in "${capture.name}":\n${violations.join('\n')}\n${summary}\n${(0, specter_perf_1.describeHitches)(capture)}`,
|
|
190
|
+
actual: summary,
|
|
191
|
+
expected: budget
|
|
192
|
+
};
|
|
193
|
+
}
|
|
126
194
|
const duration = budget.sampleDurationMs ?? 1000;
|
|
127
195
|
const interval = budget.sampleIntervalMs ?? 100;
|
|
128
196
|
const startTime = Date.now();
|
|
@@ -174,5 +242,58 @@ exports.performanceExpect = test_1.expect.extend((0, instrument_1.instrumentMatc
|
|
|
174
242
|
actual: summary,
|
|
175
243
|
expected: budget
|
|
176
244
|
};
|
|
245
|
+
},
|
|
246
|
+
async toHaveFrameTimeP95Below(receiver, maxMs) {
|
|
247
|
+
const capture = requireCapture(receiver, 'toHaveFrameTimeP95Below');
|
|
248
|
+
const actual = capture.frameTimeMs.p95;
|
|
249
|
+
return captureCheck(this.isNot, capture, '95th percentile frame time', actual <= maxMs, `<= ${(0, specter_perf_1.formatMs)(maxMs)}`, (0, specter_perf_1.formatMs)(actual), actual, maxMs);
|
|
250
|
+
},
|
|
251
|
+
async toHaveFrameTimeP99Below(receiver, maxMs) {
|
|
252
|
+
const capture = requireCapture(receiver, 'toHaveFrameTimeP99Below');
|
|
253
|
+
const actual = capture.frameTimeMs.p99;
|
|
254
|
+
return captureCheck(this.isNot, capture, '99th percentile frame time', actual <= maxMs, `<= ${(0, specter_perf_1.formatMs)(maxMs)}`, (0, specter_perf_1.formatMs)(actual), actual, maxMs);
|
|
255
|
+
},
|
|
256
|
+
async toHaveAverageFrameTimeBelow(receiver, maxMs) {
|
|
257
|
+
const capture = requireCapture(receiver, 'toHaveAverageFrameTimeBelow');
|
|
258
|
+
const actual = capture.frameTimeMs.avg;
|
|
259
|
+
return captureCheck(this.isNot, capture, 'average frame time', actual <= maxMs, `<= ${(0, specter_perf_1.formatMs)(maxMs)}`, (0, specter_perf_1.formatMs)(actual), actual, maxMs);
|
|
260
|
+
},
|
|
261
|
+
async toHaveAverageFPSAbove(receiver, minFps) {
|
|
262
|
+
const capture = requireCapture(receiver, 'toHaveAverageFPSAbove');
|
|
263
|
+
const actual = capture.fps.avg;
|
|
264
|
+
return captureCheck(this.isNot, capture, 'average FPS', actual >= minFps, `>= ${minFps}`, `${actual.toFixed(1)} fps`, actual, minFps);
|
|
265
|
+
},
|
|
266
|
+
async toHaveOnePercentLowFPSAbove(receiver, minFps) {
|
|
267
|
+
const capture = requireCapture(receiver, 'toHaveOnePercentLowFPSAbove');
|
|
268
|
+
const actual = capture.fps.onePercentLow;
|
|
269
|
+
return captureCheck(this.isNot, capture, '1% low FPS', actual >= minFps, `>= ${minFps}`, `${actual.toFixed(1)} fps`, actual, minFps, true);
|
|
270
|
+
},
|
|
271
|
+
async toHaveNoHitchesOver(receiver, maxMs, options) {
|
|
272
|
+
const capture = requireCapture(receiver, 'toHaveNoHitchesOver');
|
|
273
|
+
const allowed = options?.allowed ?? 0;
|
|
274
|
+
const over = countFramesOver(capture, maxMs);
|
|
275
|
+
if (!over) {
|
|
276
|
+
throw new Error(`"${capture.name}" only kept the frames over ${(0, specter_perf_1.formatMs)(capture.hitchThresholdMs)}, so it can't count frames over ${(0, specter_perf_1.formatMs)(maxMs)}. ` +
|
|
277
|
+
`Measure with { hitchMs: ${maxMs} } to check that.`);
|
|
278
|
+
}
|
|
279
|
+
const countText = `${over.atLeast ? 'at least ' : ''}${over.count} frame${over.count === 1 ? '' : 's'} over ${(0, specter_perf_1.formatMs)(maxMs)}`;
|
|
280
|
+
const expectedText = allowed === 0 ? `free of frames over ${(0, specter_perf_1.formatMs)(maxMs)}` : `at most ${allowed} frame${allowed === 1 ? '' : 's'} over ${(0, specter_perf_1.formatMs)(maxMs)}`;
|
|
281
|
+
return captureCheck(this.isNot, capture, 'the frame times', over.count <= allowed, expectedText, countText, over.count, allowed, true);
|
|
282
|
+
},
|
|
283
|
+
async toHavePeakMemoryBelow(receiver, maxMb) {
|
|
284
|
+
const capture = requireCapture(receiver, 'toHavePeakMemoryBelow');
|
|
285
|
+
const actual = capture.memoryMb.peak;
|
|
286
|
+
return captureCheck(this.isNot, capture, 'peak memory', actual <= maxMb, `<= ${maxMb} MB`, `${actual.toFixed(0)} MB`, actual, maxMb);
|
|
287
|
+
},
|
|
288
|
+
async toHaveMemoryGrowthBelow(receiver, maxMb) {
|
|
289
|
+
const capture = requireCapture(receiver, 'toHaveMemoryGrowthBelow');
|
|
290
|
+
const actual = capture.memoryMb.growth;
|
|
291
|
+
return captureCheck(this.isNot, capture, 'memory growth', actual <= maxMb, `<= ${maxMb} MB`, `${actual.toFixed(0)} MB (${capture.memoryMb.start.toFixed(0)} MB -> ${capture.memoryMb.end.toFixed(0)} MB)`, actual, maxMb);
|
|
292
|
+
},
|
|
293
|
+
async toHaveDrawCallsBelow(receiver, maxDrawCalls, options) {
|
|
294
|
+
const capture = requireCapture(receiver, 'toHaveDrawCallsBelow');
|
|
295
|
+
const stat = options?.stat ?? 'avg';
|
|
296
|
+
const actual = stat === 'max' ? capture.drawCalls.max : capture.drawCalls.avg;
|
|
297
|
+
return captureCheck(this.isNot, capture, stat === 'max' ? 'most draw calls in a frame' : 'average draw calls a frame', actual <= maxDrawCalls, `<= ${maxDrawCalls}`, `${Math.round(actual)}`, actual, maxDrawCalls);
|
|
177
298
|
}
|
|
178
299
|
}));
|