game-harness 1.0.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -0
- package/CHANGELOG.md +25 -2
- package/MIGRATION.md +144 -0
- package/README.md +39 -10
- package/dist/cjs/bin/visual-battery.js +19 -1
- package/dist/cjs/chromium-launch.js +18 -3
- package/dist/cjs/release-ladder.js +1 -1
- package/dist/cjs/visual-battery.js +70 -6
- package/dist/esm/bin/visual-battery.js +19 -1
- package/dist/esm/chromium-launch.js +17 -3
- package/dist/esm/release-ladder.js +1 -1
- package/dist/esm/visual-battery.js +71 -7
- package/dist/types/chromium-launch.d.cts +17 -3
- package/dist/types/chromium-launch.d.ts +17 -3
- package/dist/types/index.d.cts +1 -1
- package/dist/types/index.d.ts +1 -1
- package/dist/types/release-ladder.d.cts +1 -1
- package/dist/types/release-ladder.d.ts +1 -1
- package/dist/types/visual-battery.d.cts +8 -5
- package/dist/types/visual-battery.d.ts +8 -5
- package/docs/architecture.md +8 -4
- package/docs/decisions.md +165 -0
- package/docs/getting-started.md +1 -1
- package/docs/guides/chromium-and-silent-qa.md +9 -0
- package/docs/guides/visual-battery.md +23 -2
- package/docs/introduction.md +1 -1
- package/docs/reference/troubleshooting.md +3 -2
- package/docs/sourcey.config.ts +1 -1
- package/llms.txt +1 -0
- package/maestro/smoke.template.yaml +1 -1
- package/package.json +13 -7
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { execFileSync } from 'node:child_process';
|
|
2
|
-
import { existsSync, readdirSync, realpathSync, statSync } from 'node:fs';
|
|
2
|
+
import { existsSync, readFileSync, readdirSync, realpathSync, statSync } from 'node:fs';
|
|
3
3
|
import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path';
|
|
4
4
|
import spawn from 'cross-spawn';
|
|
5
|
+
import { PNG } from 'pngjs';
|
|
6
|
+
import which from 'which';
|
|
5
7
|
/**
|
|
6
8
|
* Thrown by every `runVisualBattery()` failure path — a missing harness dir,
|
|
7
9
|
* no discovered `.browser.test.ts(x)` files, a misplaced `__screenshots__`
|
|
@@ -51,9 +53,9 @@ function canonicalizePath(target) {
|
|
|
51
53
|
*
|
|
52
54
|
* Runs every `.browser.test.tsx` harness file under `harnessDir`, then
|
|
53
55
|
* diffs the resulting `__screenshots__/*.png` baselines against what's
|
|
54
|
-
* committed in git.
|
|
55
|
-
*
|
|
56
|
-
*
|
|
56
|
+
* committed in git. Modified PNGs tolerate RGBA channel deltas up to 2 by
|
|
57
|
+
* default; any pixel beyond that precise tolerance is drift. Noise-only
|
|
58
|
+
* renders are restored to committed bytes. New/deleted baselines always drift.
|
|
57
59
|
*
|
|
58
60
|
* - Update mode (`ci: false`, the default): runs the harnesses, lets new
|
|
59
61
|
* baselines land on disk, reports what changed so a human can review
|
|
@@ -68,11 +70,19 @@ function canonicalizePath(target) {
|
|
|
68
70
|
* exits the process.
|
|
69
71
|
*/
|
|
70
72
|
export function runVisualBattery(harnessDir, options = {}) {
|
|
71
|
-
const { ci = false, cwd = process.cwd(), testCommand = 'pnpm test:browser', baselineProfile, isolatedHarnessFiles = [], log = defaultLog, error = defaultError, } = options;
|
|
73
|
+
const { ci = false, cwd = process.cwd(), testCommand = 'pnpm test:browser', baselineProfile, isolatedHarnessFiles = [], log = defaultLog, error = defaultError, maxChannelDelta = 2, maxDifferentPixelRatio = 0, } = options;
|
|
72
74
|
const die = (msg) => {
|
|
73
75
|
error(msg);
|
|
74
76
|
throw new VisualBatteryError(msg);
|
|
75
77
|
};
|
|
78
|
+
if (!Number.isInteger(maxChannelDelta) || maxChannelDelta < 0 || maxChannelDelta > 255) {
|
|
79
|
+
die('maxChannelDelta must be an integer between 0 and 255');
|
|
80
|
+
}
|
|
81
|
+
if (!Number.isFinite(maxDifferentPixelRatio) ||
|
|
82
|
+
maxDifferentPixelRatio < 0 ||
|
|
83
|
+
maxDifferentPixelRatio > 1) {
|
|
84
|
+
die('maxDifferentPixelRatio must be between 0 and 1');
|
|
85
|
+
}
|
|
76
86
|
const resolvedCwd = resolve(cwd);
|
|
77
87
|
const canonicalCwd = (() => {
|
|
78
88
|
try {
|
|
@@ -133,10 +143,20 @@ export function runVisualBattery(harnessDir, options = {}) {
|
|
|
133
143
|
log(`running ${harnessFiles.length} harness file(s):`);
|
|
134
144
|
for (const f of harnessFiles)
|
|
135
145
|
log(` - ${f}`);
|
|
146
|
+
// Resolve the consumer's Git installation once, including PATHEXT on Windows.
|
|
147
|
+
// All Git subprocesses use the resulting absolute executable path.
|
|
148
|
+
const gitExecutable = (() => {
|
|
149
|
+
try {
|
|
150
|
+
return resolve(which.sync('git'));
|
|
151
|
+
}
|
|
152
|
+
catch (err) {
|
|
153
|
+
return die(`git executable not found: ${err}`);
|
|
154
|
+
}
|
|
155
|
+
})();
|
|
136
156
|
if (ci) {
|
|
137
157
|
let beforeStatus = '';
|
|
138
158
|
try {
|
|
139
|
-
beforeStatus = execFileSync(
|
|
159
|
+
beforeStatus = execFileSync(gitExecutable, ['status', '--porcelain', '--', relativeBaselinesDir], {
|
|
140
160
|
cwd,
|
|
141
161
|
encoding: 'utf-8',
|
|
142
162
|
});
|
|
@@ -206,7 +226,7 @@ export function runVisualBattery(harnessDir, options = {}) {
|
|
|
206
226
|
log(`baseline screenshots produced: ${baselineCount}`);
|
|
207
227
|
let afterStatus = '';
|
|
208
228
|
try {
|
|
209
|
-
afterStatus = execFileSync(
|
|
229
|
+
afterStatus = execFileSync(gitExecutable, ['status', '--porcelain', '-z', '--untracked-files=all', '--', relativeBaselinesDir], {
|
|
210
230
|
cwd,
|
|
211
231
|
encoding: 'utf-8',
|
|
212
232
|
});
|
|
@@ -214,6 +234,50 @@ export function runVisualBattery(harnessDir, options = {}) {
|
|
|
214
234
|
catch (err) {
|
|
215
235
|
die(`git status failed after harness run: ${err}`);
|
|
216
236
|
}
|
|
237
|
+
const drift = [];
|
|
238
|
+
const entries = afterStatus.split('\0').filter(Boolean);
|
|
239
|
+
for (let index = 0; index < entries.length; index += 1) {
|
|
240
|
+
const entry = entries[index];
|
|
241
|
+
const status = entry.slice(0, 2);
|
|
242
|
+
const path = entry.slice(3);
|
|
243
|
+
// Renames/copies have a second NUL-delimited path; neither is noise.
|
|
244
|
+
if (/[RC]/.test(status))
|
|
245
|
+
index += 1;
|
|
246
|
+
if (status !== ' M' || !path.endsWith('.png')) {
|
|
247
|
+
drift.push(entry);
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
try {
|
|
251
|
+
const committed = PNG.sync.read(execFileSync(gitExecutable, ['show', `HEAD:./${path}`], { cwd }));
|
|
252
|
+
const current = PNG.sync.read(readFileSync(resolve(cwd, path)));
|
|
253
|
+
if (committed.width !== current.width || committed.height !== current.height) {
|
|
254
|
+
drift.push(entry);
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
let changed = 0;
|
|
258
|
+
let beyond = 0;
|
|
259
|
+
for (let offset = 0; offset < current.data.length; offset += 4) {
|
|
260
|
+
const delta = Math.max(Math.abs(current.data[offset] - committed.data[offset]), Math.abs(current.data[offset + 1] - committed.data[offset + 1]), Math.abs(current.data[offset + 2] - committed.data[offset + 2]), Math.abs(current.data[offset + 3] - committed.data[offset + 3]));
|
|
261
|
+
if (delta > 0)
|
|
262
|
+
changed += 1;
|
|
263
|
+
if (delta > maxChannelDelta)
|
|
264
|
+
beyond += 1;
|
|
265
|
+
}
|
|
266
|
+
if (beyond / (current.width * current.height) > maxDifferentPixelRatio) {
|
|
267
|
+
drift.push(entry);
|
|
268
|
+
continue;
|
|
269
|
+
}
|
|
270
|
+
execFileSync(gitExecutable, ['checkout', '--', path], { cwd });
|
|
271
|
+
log(beyond === 0
|
|
272
|
+
? `${path}: rasterization noise (${changed} px within ±${maxChannelDelta})`
|
|
273
|
+
: `${path}: accepted pixel tolerance (${beyond} px beyond ±${maxChannelDelta})`);
|
|
274
|
+
}
|
|
275
|
+
catch {
|
|
276
|
+
// Missing/unreadable committed PNGs or failed restoration must fail closed.
|
|
277
|
+
drift.push(entry);
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
afterStatus = drift.join('\n');
|
|
217
281
|
if (afterStatus.trim().length === 0) {
|
|
218
282
|
log('baselines clean — no visual drift detected.');
|
|
219
283
|
return;
|
|
@@ -15,13 +15,27 @@ export interface ChromiumLaunchProfileOptions {
|
|
|
15
15
|
env?: Readonly<ChromiumEnvironment>;
|
|
16
16
|
}
|
|
17
17
|
export interface ChromiumLaunchProfile {
|
|
18
|
-
/**
|
|
18
|
+
/**
|
|
19
|
+
* De-duplicated Chromium launch arguments: the `gpuMode` renderer flags, the
|
|
20
|
+
* {@link CHROMIUM_ANTI_THROTTLING_ARGS}, any caller `args`, and always
|
|
21
|
+
* `--mute-audio` last.
|
|
22
|
+
*/
|
|
19
23
|
args: string[];
|
|
20
24
|
/** Present when `env` overrides were supplied or `gpuMode` is `linux-hardware-vulkan` (which also sets `EGL_PLATFORM`); merged on top of `process.env`. */
|
|
21
25
|
env?: ChromiumEnvironment;
|
|
22
26
|
}
|
|
23
27
|
/**
|
|
24
|
-
*
|
|
25
|
-
*
|
|
28
|
+
* Keeps timers and rendering on schedule in pages Chromium considers
|
|
29
|
+
* backgrounded. Several headed windows commonly run at once during a test
|
|
30
|
+
* suite; without these switches a background tab's timers are coalesced to
|
|
31
|
+
* one wake-up per second, and an occluded window or backgrounded renderer is
|
|
32
|
+
* deprioritised, so suites time out for reasons unrelated to the game.
|
|
33
|
+
* Page Visibility still reports `hidden`, so visibility handling stays
|
|
34
|
+
* testable.
|
|
35
|
+
*/
|
|
36
|
+
export declare const CHROMIUM_ANTI_THROTTLING_ARGS: readonly string[];
|
|
37
|
+
/**
|
|
38
|
+
* Builds the shared Chromium renderer, scheduling and silence profile without
|
|
39
|
+
* deciding whether the browser is headed. Callers own that explicit choice.
|
|
26
40
|
*/
|
|
27
41
|
export declare function createChromiumLaunchProfile(options?: ChromiumLaunchProfileOptions): ChromiumLaunchProfile;
|
|
@@ -15,13 +15,27 @@ export interface ChromiumLaunchProfileOptions {
|
|
|
15
15
|
env?: Readonly<ChromiumEnvironment>;
|
|
16
16
|
}
|
|
17
17
|
export interface ChromiumLaunchProfile {
|
|
18
|
-
/**
|
|
18
|
+
/**
|
|
19
|
+
* De-duplicated Chromium launch arguments: the `gpuMode` renderer flags, the
|
|
20
|
+
* {@link CHROMIUM_ANTI_THROTTLING_ARGS}, any caller `args`, and always
|
|
21
|
+
* `--mute-audio` last.
|
|
22
|
+
*/
|
|
19
23
|
args: string[];
|
|
20
24
|
/** Present when `env` overrides were supplied or `gpuMode` is `linux-hardware-vulkan` (which also sets `EGL_PLATFORM`); merged on top of `process.env`. */
|
|
21
25
|
env?: ChromiumEnvironment;
|
|
22
26
|
}
|
|
23
27
|
/**
|
|
24
|
-
*
|
|
25
|
-
*
|
|
28
|
+
* Keeps timers and rendering on schedule in pages Chromium considers
|
|
29
|
+
* backgrounded. Several headed windows commonly run at once during a test
|
|
30
|
+
* suite; without these switches a background tab's timers are coalesced to
|
|
31
|
+
* one wake-up per second, and an occluded window or backgrounded renderer is
|
|
32
|
+
* deprioritised, so suites time out for reasons unrelated to the game.
|
|
33
|
+
* Page Visibility still reports `hidden`, so visibility handling stays
|
|
34
|
+
* testable.
|
|
35
|
+
*/
|
|
36
|
+
export declare const CHROMIUM_ANTI_THROTTLING_ARGS: readonly string[];
|
|
37
|
+
/**
|
|
38
|
+
* Builds the shared Chromium renderer, scheduling and silence profile without
|
|
39
|
+
* deciding whether the browser is headed. Callers own that explicit choice.
|
|
26
40
|
*/
|
|
27
41
|
export declare function createChromiumLaunchProfile(options?: ChromiumLaunchProfileOptions): ChromiumLaunchProfile;
|
package/dist/types/index.d.cts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { type LighthouseAssertionsOverrides, type LighthouseCiConfig, lighthouseAssertions, } from './lighthouse.js';
|
|
2
2
|
export { type ReleaseLadderResult, type ReleaseLadderStep, type VerifyReleaseLadderOptions, verifyReleaseLadder, } from './release-ladder.js';
|
|
3
|
-
export { runVisualBattery, VisualBatteryError, type VisualBatteryOptions, } from './visual-battery.js';
|
|
3
|
+
export { runVisualBattery, type VisualBatteryCommand, VisualBatteryError, type VisualBatteryOptions, } from './visual-battery.js';
|
package/dist/types/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { type LighthouseAssertionsOverrides, type LighthouseCiConfig, lighthouseAssertions, } from './lighthouse.js';
|
|
2
2
|
export { type ReleaseLadderResult, type ReleaseLadderStep, type VerifyReleaseLadderOptions, verifyReleaseLadder, } from './release-ladder.js';
|
|
3
|
-
export { runVisualBattery, VisualBatteryError, type VisualBatteryOptions, } from './visual-battery.js';
|
|
3
|
+
export { runVisualBattery, type VisualBatteryCommand, VisualBatteryError, type VisualBatteryOptions, } from './visual-battery.js';
|
|
@@ -26,7 +26,7 @@ export interface VerifyReleaseLadderOptions {
|
|
|
26
26
|
* screenshots → native sync, or whatever a given repo's ladder is), run in
|
|
27
27
|
* sequence and stopped at the first failure with a labeled summary.
|
|
28
28
|
*
|
|
29
|
-
* Generalizes the
|
|
29
|
+
* Generalizes the common pattern of many discrete
|
|
30
30
|
* `node scripts/verify-X.mjs` files composed via a shell `&&` chain into a
|
|
31
31
|
* single reusable primitive: each step is a plain function (sync or async),
|
|
32
32
|
* so a repo can inline its logic or delegate to existing scripts via
|
|
@@ -26,7 +26,7 @@ export interface VerifyReleaseLadderOptions {
|
|
|
26
26
|
* screenshots → native sync, or whatever a given repo's ladder is), run in
|
|
27
27
|
* sequence and stopped at the first failure with a labeled summary.
|
|
28
28
|
*
|
|
29
|
-
* Generalizes the
|
|
29
|
+
* Generalizes the common pattern of many discrete
|
|
30
30
|
* `node scripts/verify-X.mjs` files composed via a shell `&&` chain into a
|
|
31
31
|
* single reusable primitive: each step is a plain function (sync or async),
|
|
32
32
|
* so a repo can inline its logic or delegate to existing scripts via
|
|
@@ -5,6 +5,10 @@ export interface VisualBatteryCommand {
|
|
|
5
5
|
args?: readonly string[];
|
|
6
6
|
}
|
|
7
7
|
export interface VisualBatteryOptions {
|
|
8
|
+
/** Maximum absolute RGBA channel delta per pixel (0–255 integer). Defaults to 2. */
|
|
9
|
+
maxChannelDelta?: number;
|
|
10
|
+
/** Allowed fraction of pixels beyond the channel tolerance (0–1). Defaults to 0. */
|
|
11
|
+
maxDifferentPixelRatio?: number;
|
|
8
12
|
/** CI mode: refuses to run with a dirty baseline dir, fails on drift instead of updating. */
|
|
9
13
|
ci?: boolean;
|
|
10
14
|
/** Repo root the git commands run relative to. Defaults to `process.cwd()`. */
|
|
@@ -21,8 +25,7 @@ export interface VisualBatteryOptions {
|
|
|
21
25
|
/**
|
|
22
26
|
* Optional platform/profile directory below the baseline root (for example
|
|
23
27
|
* `linux`). The profile is also exposed to Vite browser tests as
|
|
24
|
-
* `VITE_VISUAL_BASELINE_PROFILE`,
|
|
25
|
-
* strict on renderers that cannot produce identical PNGs.
|
|
28
|
+
* `VITE_VISUAL_BASELINE_PROFILE`, keeping distinct renderer baselines separate.
|
|
26
29
|
*/
|
|
27
30
|
baselineProfile?: string;
|
|
28
31
|
/**
|
|
@@ -50,9 +53,9 @@ export declare class VisualBatteryError extends Error {
|
|
|
50
53
|
*
|
|
51
54
|
* Runs every `.browser.test.tsx` harness file under `harnessDir`, then
|
|
52
55
|
* diffs the resulting `__screenshots__/*.png` baselines against what's
|
|
53
|
-
* committed in git.
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
+
* committed in git. Modified PNGs tolerate RGBA channel deltas up to 2 by
|
|
57
|
+
* default; any pixel beyond that precise tolerance is drift. Noise-only
|
|
58
|
+
* renders are restored to committed bytes. New/deleted baselines always drift.
|
|
56
59
|
*
|
|
57
60
|
* - Update mode (`ci: false`, the default): runs the harnesses, lets new
|
|
58
61
|
* baselines land on disk, reports what changed so a human can review
|
|
@@ -5,6 +5,10 @@ export interface VisualBatteryCommand {
|
|
|
5
5
|
args?: readonly string[];
|
|
6
6
|
}
|
|
7
7
|
export interface VisualBatteryOptions {
|
|
8
|
+
/** Maximum absolute RGBA channel delta per pixel (0–255 integer). Defaults to 2. */
|
|
9
|
+
maxChannelDelta?: number;
|
|
10
|
+
/** Allowed fraction of pixels beyond the channel tolerance (0–1). Defaults to 0. */
|
|
11
|
+
maxDifferentPixelRatio?: number;
|
|
8
12
|
/** CI mode: refuses to run with a dirty baseline dir, fails on drift instead of updating. */
|
|
9
13
|
ci?: boolean;
|
|
10
14
|
/** Repo root the git commands run relative to. Defaults to `process.cwd()`. */
|
|
@@ -21,8 +25,7 @@ export interface VisualBatteryOptions {
|
|
|
21
25
|
/**
|
|
22
26
|
* Optional platform/profile directory below the baseline root (for example
|
|
23
27
|
* `linux`). The profile is also exposed to Vite browser tests as
|
|
24
|
-
* `VITE_VISUAL_BASELINE_PROFILE`,
|
|
25
|
-
* strict on renderers that cannot produce identical PNGs.
|
|
28
|
+
* `VITE_VISUAL_BASELINE_PROFILE`, keeping distinct renderer baselines separate.
|
|
26
29
|
*/
|
|
27
30
|
baselineProfile?: string;
|
|
28
31
|
/**
|
|
@@ -50,9 +53,9 @@ export declare class VisualBatteryError extends Error {
|
|
|
50
53
|
*
|
|
51
54
|
* Runs every `.browser.test.tsx` harness file under `harnessDir`, then
|
|
52
55
|
* diffs the resulting `__screenshots__/*.png` baselines against what's
|
|
53
|
-
* committed in git.
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
+
* committed in git. Modified PNGs tolerate RGBA channel deltas up to 2 by
|
|
57
|
+
* default; any pixel beyond that precise tolerance is drift. Noise-only
|
|
58
|
+
* renders are restored to committed bytes. New/deleted baselines always drift.
|
|
56
59
|
*
|
|
57
60
|
* - Update mode (`ci: false`, the default): runs the harnesses, lets new
|
|
58
61
|
* baselines land on disk, reports what changed so a human can review
|
package/docs/architecture.md
CHANGED
|
@@ -29,8 +29,8 @@ either framework.
|
|
|
29
29
|
2. When a server is configured, its readiness URL must be unreachable first.
|
|
30
30
|
The command is spawned directly with an argument array and must bind with
|
|
31
31
|
strict-port behavior.
|
|
32
|
-
3. A fresh browser launches with the selected renderer profile
|
|
33
|
-
final `--mute-audio` argument.
|
|
32
|
+
3. A fresh browser launches with the selected renderer profile, the
|
|
33
|
+
anti-throttling switches, and exactly one final `--mute-audio` argument.
|
|
34
34
|
4. Local-storage sentinels are installed before application code.
|
|
35
35
|
5. `openSilentGame()` adds the runtime-only mute query and waits for the
|
|
36
36
|
application readiness marker.
|
|
@@ -52,8 +52,12 @@ runs selected harnesses in direct child processes, requires at least one PNG,
|
|
|
52
52
|
and asks Git for scoped status with argument arrays rather than shell commands.
|
|
53
53
|
|
|
54
54
|
Update mode reports reviewed changes. CI mode also refuses a dirty starting
|
|
55
|
-
state and fails on any resulting drift.
|
|
56
|
-
|
|
55
|
+
state and fails on any resulting drift. Modified PNGs are decoded with pngjs;
|
|
56
|
+
dimensions must match, and every pixel whose maximum RGBA channel delta exceeds
|
|
57
|
+
`maxChannelDelta` (default 2) counts toward `maxDifferentPixelRatio` (default 0).
|
|
58
|
+
Accepted renders are restored to committed bytes. New/deleted or unreadable
|
|
59
|
+
baselines remain drift. Renderer-specific output belongs in a separate profile
|
|
60
|
+
when the difference exceeds rasterization noise.
|
|
57
61
|
|
|
58
62
|
## Configuration philosophy
|
|
59
63
|
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Decisions
|
|
3
|
+
description: Why the package is shaped the way it is, and how a predecessor harness converged into it.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
Each entry records a decision, the reason for it, and what it means for a
|
|
7
|
+
consumer. Newest first.
|
|
8
|
+
|
|
9
|
+
## Pixel tolerance for screenshot baselines
|
|
10
|
+
|
|
11
|
+
Byte equality is not stable across renders, even on the same machine. Decode
|
|
12
|
+
modified PNGs with pngjs and compare the maximum absolute RGBA channel delta
|
|
13
|
+
per pixel. `maxChannelDelta` defaults to 2 (out of 255), and
|
|
14
|
+
`maxDifferentPixelRatio` defaults to 0: any pixel beyond that tolerance is
|
|
15
|
+
drift. Dimension changes, new/deleted files, and unreadable PNGs remain drift.
|
|
16
|
+
Noise-only files are restored to committed bytes, keeping the Git diff clean.
|
|
17
|
+
Both thresholds are configurable; raising the ratio explicitly permits real
|
|
18
|
+
changes and should be a deliberate consumer decision.
|
|
19
|
+
|
|
20
|
+
## Converging a predecessor `test-harness` package
|
|
21
|
+
|
|
22
|
+
This entry compares the supported migration paths export by export so that
|
|
23
|
+
consumers of an earlier `test-harness` package can move to Game Harness. The step-by-step
|
|
24
|
+
consumer mapping is in [MIGRATION.md](https://github.com/jbcom/game-harness/blob/main/MIGRATION.md).
|
|
25
|
+
|
|
26
|
+
**Decision:** Game Harness is the one canonical harness. Where both packages
|
|
27
|
+
implement the same thing, Game Harness keeps its own public shape; anything the
|
|
28
|
+
predecessor did that Game Harness did not is added here, with tests.
|
|
29
|
+
|
|
30
|
+
### Method
|
|
31
|
+
|
|
32
|
+
1. Diffed every source file of the predecessor's last release against this
|
|
33
|
+
repository's `main`.
|
|
34
|
+
2. Ran the predecessor's own unit suite against Game Harness's source. Every
|
|
35
|
+
behavioural test passed (97 of 107). The ten failures were all in the
|
|
36
|
+
visual battery and mocked `execSync`, an implementation detail Game Harness
|
|
37
|
+
replaced with shell-free `execFileSync`; the behaviour those tests describe
|
|
38
|
+
is covered by Game Harness's own suite. The predecessor's package-contract
|
|
39
|
+
test checks its own package name and is not applicable.
|
|
40
|
+
|
|
41
|
+
### Entry points
|
|
42
|
+
|
|
43
|
+
| Subpath | Predecessor | Game Harness | Notes |
|
|
44
|
+
| --------------------- | ----------- | ------------ | ----------------------------------------------------------- |
|
|
45
|
+
| `.` (root) | yes | yes | Same re-exports: Lighthouse, release ladder, visual battery |
|
|
46
|
+
| `/vitest` | yes | yes | Same function; Game Harness validates its inputs |
|
|
47
|
+
| `/playwright` | yes | yes | Same functions; Game Harness validates and normalizes |
|
|
48
|
+
| `/silent-qa` | yes | yes | Game Harness adds `activateSilentQaAsync` |
|
|
49
|
+
| `/production-runtime` | yes | yes | Same functions; Game Harness validates URLs and durations |
|
|
50
|
+
| `/chromium` | yes | yes | Game Harness adds `CHROMIUM_ANTI_THROTTLING_ARGS` |
|
|
51
|
+
| `/lighthouse` | yes | yes | Same function; result type always carries `staticDistDir` |
|
|
52
|
+
| `/release-ladder` | yes | yes | Identical behaviour |
|
|
53
|
+
| `/visual-battery` | yes | yes | Game Harness adds `VisualBatteryCommand` and path checks |
|
|
54
|
+
| `/package.json` | yes | yes | |
|
|
55
|
+
|
|
56
|
+
Both packages ship ESM and CommonJS builds. Game Harness additionally ships
|
|
57
|
+
separate `.d.cts` declarations for `require` consumers.
|
|
58
|
+
|
|
59
|
+
### Exports
|
|
60
|
+
|
|
61
|
+
Every named export of the predecessor exists in Game Harness under the same
|
|
62
|
+
name and subpath. Game Harness adds three:
|
|
63
|
+
|
|
64
|
+
| Export | Subpath | Why |
|
|
65
|
+
| ------------------------------- | ----------------------- | -------------------------------------------------------------------- |
|
|
66
|
+
| `activateSilentQaAsync` | `/silent-qa` | Publishes the readiness marker only after an async mute resolves |
|
|
67
|
+
| `VisualBatteryCommand` | `/visual-battery`, root | Runs the browser suite without a shell |
|
|
68
|
+
| `CHROMIUM_ANTI_THROTTLING_ARGS` | `/chromium` | The scheduling switches every launch profile now carries (see below) |
|
|
69
|
+
|
|
70
|
+
### Bins
|
|
71
|
+
|
|
72
|
+
| Predecessor | Game Harness |
|
|
73
|
+
| ----------------------------- | ----------------------------------------------------------- |
|
|
74
|
+
| `test-harness-visual-battery` | `game-harness-visual-battery`, and the old name as an alias |
|
|
75
|
+
|
|
76
|
+
Game Harness's CLI also rejects unknown flags and more than one positional
|
|
77
|
+
directory (exit code 2) instead of silently ignoring them.
|
|
78
|
+
|
|
79
|
+
### Peers
|
|
80
|
+
|
|
81
|
+
| Peer | Predecessor | Game Harness |
|
|
82
|
+
| ---------------------------- | ------------- | ------------- |
|
|
83
|
+
| `@playwright/test` | `>=1.62.1 <2` | `>=1.62.1 <2` |
|
|
84
|
+
| `vitest` | `>=4.1.10 <5` | `>=4.1.10 <6` |
|
|
85
|
+
| `@vitest/browser-playwright` | `>=4.1.10 <5` | `>=4.1.10 <6` |
|
|
86
|
+
| Node (`engines`) | `>=24.19.0` | `>=22` |
|
|
87
|
+
|
|
88
|
+
All framework peers stay optional; the root never loads them. CI proves the
|
|
89
|
+
packed tarball against Vitest 4.1.10 and 5.0.3.
|
|
90
|
+
|
|
91
|
+
### Where the shapes differ
|
|
92
|
+
|
|
93
|
+
Game Harness's shape wins in each case. The consumer-facing consequence is
|
|
94
|
+
listed in MIGRATION.md.
|
|
95
|
+
|
|
96
|
+
- **Visual battery `testCommand`.** The predecessor passed a string to a
|
|
97
|
+
shell, so shell syntax (`FOO=1 pnpm …`, `&&`, quoting) worked by accident.
|
|
98
|
+
Game Harness never starts a shell: a string is split on whitespace, or a
|
|
99
|
+
`{ command, args }` object is executed directly. Kept because a shell turns
|
|
100
|
+
a configuration value into a command-injection surface and behaves
|
|
101
|
+
differently on Windows.
|
|
102
|
+
- **Strict input validation.** Game Harness throws a `TypeError` for an
|
|
103
|
+
invalid `PLAYWRIGHT_PORT`/`PW_PORT` (the predecessor silently fell back to
|
|
104
|
+
the derived port), an empty or unknown `deviceTiers` list, a non-positive
|
|
105
|
+
`ciTimeoutMultiplier`, a `basePath` with a query, fragment or traversal, an
|
|
106
|
+
empty Vitest project name, `instances` or `include`, and empty or invalid
|
|
107
|
+
Lighthouse overrides. Kept because a misconfigured gate that silently
|
|
108
|
+
"passes" is not release evidence.
|
|
109
|
+
- **Async mute callbacks.** The predecessor's `activateSilentQa` accepted a
|
|
110
|
+
promise-returning callback and published the readiness marker before the
|
|
111
|
+
mute finished. Game Harness throws and points to `activateSilentQaAsync`.
|
|
112
|
+
- **`LighthouseCiConfig.ci.collect.staticDistDir`** is required in the result
|
|
113
|
+
type. Every preset already sets it, so only code that builds the type by
|
|
114
|
+
hand without it is affected.
|
|
115
|
+
- **Visual battery paths.** Game Harness requires the harness and baseline
|
|
116
|
+
directories to stay inside `cwd`, sorts harness files for a deterministic
|
|
117
|
+
run order, and fails when a run produces no PNG baselines.
|
|
118
|
+
|
|
119
|
+
### Deliberately not carried over
|
|
120
|
+
|
|
121
|
+
- **The predecessor's repository tooling:** its Gitea release workflow,
|
|
122
|
+
release-label script, anonymous-registry environment helper and the test
|
|
123
|
+
that pinned that workflow. Game Harness publishes from GitHub with npm
|
|
124
|
+
trusted publishing and provenance, so none of it applies.
|
|
125
|
+
- **Shell execution for `testCommand`**, for the reason above. A consumer
|
|
126
|
+
that needs environment variables or chained commands puts them in a
|
|
127
|
+
package script and passes that script's name.
|
|
128
|
+
|
|
129
|
+
## Every launch profile disables background throttling
|
|
130
|
+
|
|
131
|
+
**Decision:** `createChromiumLaunchProfile()` always adds
|
|
132
|
+
`--disable-background-timer-throttling`, `--disable-renderer-backgrounding`
|
|
133
|
+
and `--disable-backgrounding-occluded-windows`, exported as
|
|
134
|
+
`CHROMIUM_ANTI_THROTTLING_ARGS`. Every Playwright project, Vitest Browser
|
|
135
|
+
instance and production-runtime launch inherits them.
|
|
136
|
+
|
|
137
|
+
**Why:** a browser suite commonly runs several headed windows at once. A tab
|
|
138
|
+
Chromium considers backgrounded has its timers coalesced to one wake-up per
|
|
139
|
+
second, and an occluded window or backgrounded renderer is deprioritised, so
|
|
140
|
+
suites time out for reasons that have nothing to do with the game.
|
|
141
|
+
|
|
142
|
+
Playwright adds the same switches to its own default arguments today, but the
|
|
143
|
+
profile is the package's contract with every launcher it serves: a custom
|
|
144
|
+
script, a launcher that does not use Playwright, or a caller that passes
|
|
145
|
+
`ignoreDefaultArgs`. Making the switches explicit keeps the guarantee from
|
|
146
|
+
depending on another tool's defaults. They do not change Page Visibility: a
|
|
147
|
+
backgrounded page still reports `hidden`, so a game's pause-on-hide logic
|
|
148
|
+
remains testable.
|
|
149
|
+
|
|
150
|
+
**Evidence:** `pnpm test:chromium` launches headed Chromium over the raw
|
|
151
|
+
DevTools protocol (Playwright's own switches and focus emulation would mask
|
|
152
|
+
the effect), opens a 50 ms timer in one tab and a second tab in front of it,
|
|
153
|
+
and samples the background tab for three seconds. With the profile the timer
|
|
154
|
+
keeps its schedule; a control case without the switches shows the one-second
|
|
155
|
+
throttling, which proves the scenario really backgrounds the tab.
|
|
156
|
+
|
|
157
|
+
## Vitest 4 and 5
|
|
158
|
+
|
|
159
|
+
**Decision:** the `vitest` and `@vitest/browser-playwright` peers accept
|
|
160
|
+
`>=4.1.10 <6`. Development runs on Vitest 5, and the packed-consumer smoke
|
|
161
|
+
runs once against 4.1.10 (the floor) and once against 5.0.3.
|
|
162
|
+
|
|
163
|
+
**Why:** consumers are moving to Vitest 5, and the config fragment
|
|
164
|
+
`defineBrowserTestConfig()` emits is valid for both majors without a code
|
|
165
|
+
change.
|
package/docs/getting-started.md
CHANGED
|
@@ -31,7 +31,7 @@ supported. CI pins the primary gate to the version in `.nvmrc` (currently
|
|
|
31
31
|
24.19.0, the latest Node 24 LTS patch) and additionally runs the full test
|
|
32
32
|
and build suite against Node 22 on Linux to prove the floor of that range,
|
|
33
33
|
alongside macOS and Windows portability on the pinned version. The current
|
|
34
|
-
conformance matrix is Playwright 1.62.1 and Vitest Browser 4.1.10.
|
|
34
|
+
conformance matrix is Playwright 1.62.1 and Vitest Browser 4.1.10 and 5.0.3.
|
|
35
35
|
Package-boundary consumers run with a credential-free home directory and npm
|
|
36
36
|
configuration, install only the peer family needed by each entry point, and
|
|
37
37
|
exercise ESM, CommonJS, the CLI, silent runtime markers, and Chromium launch
|
|
@@ -24,6 +24,15 @@ exposing `/dev/dri/renderD128`) and always de-duplicates and appends
|
|
|
24
24
|
`--mute-audio` last, so a caller-supplied arg list can never accidentally
|
|
25
25
|
drop the silence guard.
|
|
26
26
|
|
|
27
|
+
Every profile also carries `CHROMIUM_ANTI_THROTTLING_ARGS`:
|
|
28
|
+
`--disable-background-timer-throttling`, `--disable-renderer-backgrounding`
|
|
29
|
+
and `--disable-backgrounding-occluded-windows`. When several headed windows
|
|
30
|
+
run at once, Chromium would otherwise coalesce a background tab's timers to
|
|
31
|
+
one wake-up per second and deprioritise occluded windows, and suites time out
|
|
32
|
+
for reasons unrelated to the game. Page Visibility is unaffected: a
|
|
33
|
+
backgrounded page still reports `hidden`, so pause-on-hide logic stays
|
|
34
|
+
testable.
|
|
35
|
+
|
|
27
36
|
Every browser launched by `definePlaywrightConfig()` or
|
|
28
37
|
`defineBrowserTestConfig()` receives Chromium's `--mute-audio` argument as a
|
|
29
38
|
defense-in-depth guard, including projects with custom launch options. The
|
|
@@ -3,6 +3,27 @@ title: Visual battery
|
|
|
3
3
|
description: Deterministic, fail-closed screenshot baseline orchestration.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
|
+
Modified PNGs are decoded and compared against `HEAD` pixel by pixel.
|
|
7
|
+
`maxChannelDelta` defaults to 2 (integer 0–255): a pixel differs only when
|
|
8
|
+
its maximum absolute RGBA channel delta exceeds this tolerance.
|
|
9
|
+
`maxDifferentPixelRatio` defaults to 0 (range 0–1): any pixel beyond the
|
|
10
|
+
channel tolerance is drift. Dimension changes, new/deleted baselines, and
|
|
11
|
+
unreadable PNGs always count as drift. Accepted renders are restored to
|
|
12
|
+
committed bytes; noise-only files log `rasterization noise (N px within ±2)`.
|
|
13
|
+
The CI dirty-start check still rejects all uncommitted baseline changes.
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
runVisualBattery('tests/harness', {
|
|
17
|
+
ci: true,
|
|
18
|
+
maxChannelDelta: 2,
|
|
19
|
+
maxDifferentPixelRatio: 0,
|
|
20
|
+
});
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
CLI equivalents: `--max-channel-delta 2 --max-different-pixel-ratio 0`.
|
|
24
|
+
Use a channel delta of 0 for exact decoded pixels. Raising the ratio explicitly
|
|
25
|
+
permits some pixels beyond the tolerance; do so deliberately.
|
|
26
|
+
|
|
6
27
|
Run the default harness directory in update mode, or enforce committed
|
|
7
28
|
baselines in CI:
|
|
8
29
|
|
|
@@ -24,8 +45,8 @@ The battery rejects any second `__screenshots__` directory nested elsewhere
|
|
|
24
45
|
under the harness tree; otherwise an apparently green run could leave an
|
|
25
46
|
important screenshot outside the Git diff gate.
|
|
26
47
|
|
|
27
|
-
When
|
|
28
|
-
|
|
48
|
+
When renderers produce genuinely different pixels, keep separate profiles.
|
|
49
|
+
Pass `baselineProfile: 'linux'`; the
|
|
29
50
|
battery compares `__screenshots__/linux/` and exposes the same value to Vite
|
|
30
51
|
as `VITE_VISUAL_BASELINE_PROFILE`. Screenshot helpers should include that
|
|
31
52
|
optional directory in their path:
|
package/docs/introduction.md
CHANGED
|
@@ -4,7 +4,7 @@ description: Release-grade browser QA primitives for TypeScript games.
|
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
Game Harness turns a successful build into evidence: silent browser sessions,
|
|
7
|
-
deterministic device tiers,
|
|
7
|
+
deterministic device tiers, pixel-tolerant screenshot gates, production-runtime proof,
|
|
8
8
|
and Lighthouse gates.
|
|
9
9
|
|
|
10
10
|
It is intentionally a focused library rather than a test framework. Your game
|
|
@@ -23,8 +23,9 @@ description: Common failure modes and their causes.
|
|
|
23
23
|
## Releases and support
|
|
24
24
|
|
|
25
25
|
Conventional commits on `main` are collected into a release pull request by
|
|
26
|
-
release-please. Merging that pull request creates the GitHub release
|
|
27
|
-
|
|
26
|
+
release-please. Merging that pull request creates the GitHub release; its
|
|
27
|
+
published-release event starts `cd.yml`, which verifies and publishes the exact
|
|
28
|
+
tag to npm through OIDC with provenance.
|
|
28
29
|
Changes are recorded in the [CHANGELOG](https://github.com/jbcom/game-harness/blob/main/CHANGELOG.md).
|
|
29
30
|
|
|
30
31
|
Report defects and feature requests through the
|
package/docs/sourcey.config.ts
CHANGED
package/llms.txt
CHANGED
|
@@ -23,6 +23,7 @@ agent-oriented quick reference before writing code against this package.
|
|
|
23
23
|
- [Visual battery guide](https://jonbogaty.com/game-harness/guides/visual-battery/): Deterministic screenshot baseline orchestration.
|
|
24
24
|
- [Lighthouse presets & release ladder guide](https://jonbogaty.com/game-harness/guides/lighthouse-and-release-ladder/): CI Lighthouse assertions and the release-step orchestrator.
|
|
25
25
|
- [Architecture reference](https://jonbogaty.com/game-harness/reference/architecture/): Module boundaries, runtime evidence flow, build/distribution guarantees.
|
|
26
|
+
- [Decisions reference](https://jonbogaty.com/game-harness/decisions/): Migration and compatibility decisions and their rationale.
|
|
26
27
|
- [Troubleshooting reference](https://jonbogaty.com/game-harness/reference/troubleshooting/): Common failure modes and releases/support.
|
|
27
28
|
|
|
28
29
|
## Optional
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# render its first screen" flow; add game-specific flows alongside it as
|
|
4
4
|
# separate .maestro/*.yaml files (they're app-specific user journeys, not
|
|
5
5
|
# shareable config — see game-harness's package notes).
|
|
6
|
-
appId: __APP_ID__ # e.g. com.
|
|
6
|
+
appId: __APP_ID__ # e.g. com.example.game — must match capacitor.config.ts's appId
|
|
7
7
|
---
|
|
8
8
|
- launchApp
|
|
9
9
|
- assertVisible:
|