@roku-sdk/device-test-client 0.0.0-stage → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BREAKING_CHANGES.md +17 -0
- package/LICENSE +21 -0
- package/README.md +144 -2
- package/docs/baselines.md +221 -0
- package/docs/configuration.md +258 -0
- package/docs/dom-assertions.md +264 -0
- package/docs/errors.md +544 -0
- package/docs/examples/deep-link-cold-start.md +85 -0
- package/docs/examples/focus-navigation.md +81 -0
- package/docs/examples/log-driven-assertion.md +99 -0
- package/docs/examples/precondition-skips.md +150 -0
- package/docs/examples/screenshot-regression.md +103 -0
- package/docs/examples/smoke-test.md +69 -0
- package/docs/getting-started.md +195 -0
- package/docs/jest-integration.md +331 -0
- package/docs/log-assertions.md +349 -0
- package/docs/preconditions.md +317 -0
- package/docs/rkclient.md +657 -0
- package/docs/screenshot-assertions.md +316 -0
- package/package.json +81 -3
- package/src/assertions/assertions.ts +661 -0
- package/src/assertions/baseline-env-guard.ts +43 -0
- package/src/assertions/baselines.ts +178 -0
- package/src/assertions/screenshot-artifacts.ts +39 -0
- package/src/assertions/screenshot-diff.ts +447 -0
- package/src/client/build-cache.ts +292 -0
- package/src/client/client.ts +1964 -0
- package/src/client/deploy.ts +407 -0
- package/src/client/ecp-queue.ts +112 -0
- package/src/client/ecp-retry.ts +94 -0
- package/src/client/factory.ts +170 -0
- package/src/client/health.ts +345 -0
- package/src/client/parsers.ts +162 -0
- package/src/client/server-url.ts +137 -0
- package/src/constants.ts +520 -0
- package/src/device/device-class.ts +57 -0
- package/src/device/device-env-sync.ts +58 -0
- package/src/device/device-resolver-sync.ts +43 -0
- package/src/device/device-resolver.ts +141 -0
- package/src/device/lease-provisioning.ts +54 -0
- package/src/errors.ts +774 -0
- package/src/index.ts +181 -0
- package/src/internal/constants.ts +3 -0
- package/src/jest/artifacts.ts +87 -0
- package/src/jest/device-test-environment.ts +420 -0
- package/src/jest/failure-artifacts.ts +169 -0
- package/src/jest/global-setup.ts +137 -0
- package/src/jest/global-teardown.ts +132 -0
- package/src/jest/install-hooks.ts +490 -0
- package/src/jest/jest-setup.ts +53 -0
- package/src/jest/preconditions.ts +239 -0
- package/src/jest/telemetry.ts +331 -0
- package/src/logs/device-log-sink.ts +69 -0
- package/src/logs/log-buffer.ts +278 -0
- package/src/logs/log-session.ts +436 -0
- package/src/logs/log-stream-client.ts +296 -0
- package/src/paths.ts +65 -0
- package/src/types.ts +472 -0
- package/support/constants.ts +17 -0
- package/support/deploy.ts +27 -0
- package/support/paths.ts +13 -0
package/src/index.ts
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
// Copyright (c) 2026 Roku, Inc.
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
/**
|
|
4
|
+
* `@roku-sdk/device-test-client` — public surface.
|
|
5
|
+
*
|
|
6
|
+
* SemVer major 2.0.0 trimmed the barrel to the symbols real
|
|
7
|
+
* consumers actually import. Log primitives, failure-bundle writers,
|
|
8
|
+
* XML parsers, low-level baseline/screenshot-artifact setters, async
|
|
9
|
+
* device resolvers, and sync device-env helpers are no longer part
|
|
10
|
+
* of the documented API.
|
|
11
|
+
*
|
|
12
|
+
* **Harness support entry points.** The `package.json` `exports` map
|
|
13
|
+
* has no `./src/*` wildcard. Symbols kept off this barrel for harness
|
|
14
|
+
* code are on documented `./support/<name>` entry points:
|
|
15
|
+
*
|
|
16
|
+
* - `@roku-sdk/device-test-client/support/deploy` — `resolveAppPath`
|
|
17
|
+
* and `invalidateBuildCacheForApp` for a deploy guard that wipes
|
|
18
|
+
* and rebuilds an app between retries.
|
|
19
|
+
* - `@roku-sdk/device-test-client/support/constants` — the subset of
|
|
20
|
+
* `src/constants.ts` (currently `ARTIFACT_FILE`) that `*.config.ts`
|
|
21
|
+
* files load at parse time. This barrel is unreachable from
|
|
22
|
+
* CommonJS-mode config loaders because it transitively pulls in
|
|
23
|
+
* `@roku-sdk/tooling` ESM (`import.meta.url`); this entry point
|
|
24
|
+
* bypasses that chain.
|
|
25
|
+
* - `@roku-sdk/device-test-client/support/paths` — load-safe
|
|
26
|
+
* `test-results/` path helpers used by multi-device test setup.
|
|
27
|
+
*
|
|
28
|
+
* The former `./internal/<name>` subpaths were removed (BREAKING_CHANGES.md).
|
|
29
|
+
* Prefer extending an existing `support/<name>` entry point over
|
|
30
|
+
* minting a parallel one, and never reintroduce the wildcard: every
|
|
31
|
+
* `./src/<x>` deep import bound the module topology into the
|
|
32
|
+
* published API surface.
|
|
33
|
+
*
|
|
34
|
+
* Constants surface (M14): `index.ts` no longer re-exports every
|
|
35
|
+
* value in `constants.ts`. The curated set below covers the env
|
|
36
|
+
* vars, file/global keys, default budgets, and bundle-layout
|
|
37
|
+
* names that consumers legitimately read; pure-implementation
|
|
38
|
+
* magic numbers (ECP queue gaps, log-buffer caps, pixelmatch YIQ
|
|
39
|
+
* cutoff, error-formatter character caps, readiness-gate poll
|
|
40
|
+
* mechanics, SSE accumulator limits) are intentionally NOT
|
|
41
|
+
* re-exported.
|
|
42
|
+
*
|
|
43
|
+
* Errors and types (`errors.ts`, `types.ts`) re-export wholesale —
|
|
44
|
+
* the entire error catalogue must stay reachable for `instanceof`
|
|
45
|
+
* checks, and the type surface is read-only at runtime.
|
|
46
|
+
*
|
|
47
|
+
* The trimmed surface is snapshot-guarded by
|
|
48
|
+
* `__tests__/public-api-surface.test.ts`; any add/remove/typeof flip
|
|
49
|
+
* requires an explicit snapshot update + reviewer sign-off.
|
|
50
|
+
*
|
|
51
|
+
* See `README.md` (Versioning) for the public API contract and
|
|
52
|
+
* `docs/getting-started.md` for setup migration guidance.
|
|
53
|
+
*/
|
|
54
|
+
|
|
55
|
+
export * from "./types";
|
|
56
|
+
export * from "./errors";
|
|
57
|
+
|
|
58
|
+
// Curated public constants. KEEP in this list any constant a real
|
|
59
|
+
// consumer needs to align to (env-var names, file/global keys,
|
|
60
|
+
// default budgets that overlap with public option types, server
|
|
61
|
+
// URL / port, bundle layout names CI pipelines parse). DO NOT add
|
|
62
|
+
// pure-implementation magic numbers (queue mechanics, formatter
|
|
63
|
+
// caps, parser limits) — those stay file-internal. If a partner
|
|
64
|
+
// truly needs one, promote it through this barrel via a snapshot
|
|
65
|
+
// update (the `./src/*` deep-import escape hatch was removed in
|
|
66
|
+
// to keep the published exports map auditable).
|
|
67
|
+
export {
|
|
68
|
+
// Env vars + file/global keys.
|
|
69
|
+
ARTIFACT_FILE,
|
|
70
|
+
DEVICE_ENV_GLOBAL_KEY,
|
|
71
|
+
ENV_VAR,
|
|
72
|
+
// Server URL / port.
|
|
73
|
+
DEFAULT_RK_SERVER_URL,
|
|
74
|
+
ROKU_LOG_PORT,
|
|
75
|
+
// Default budgets (overlap with `RkClientOptions` / `WaitOptions`
|
|
76
|
+
// / `BuildAndDeployOptions` / `CaptureStableOptions`).
|
|
77
|
+
BUILD_SSE_IDLE_TIMEOUT_DEFAULT_MS,
|
|
78
|
+
DEFAULT_KEY_DELAY,
|
|
79
|
+
DEFAULT_REQUEST_TIMEOUT_MS,
|
|
80
|
+
DEVICE_ONLINE_DEFAULT_TIMEOUT_MS,
|
|
81
|
+
DEVICE_ONLINE_POLL_INTERVAL_MS,
|
|
82
|
+
DEVICE_RECOVERY_DEFAULT_RETRIES,
|
|
83
|
+
INSTALLER_PORT_PROBE_DEFAULT_TIMEOUT_MS,
|
|
84
|
+
INSTALLER_PORT_PROBE_INTERVAL_MS,
|
|
85
|
+
READINESS_TIMEOUT_MS,
|
|
86
|
+
SCREENSHOT_STABLE_DEFAULT_INTERVAL_MS,
|
|
87
|
+
WAIT_FOR_LOG_DEFAULT_TIMEOUT_MS,
|
|
88
|
+
WAIT_FOR_NODE_DEFAULT_INTERVAL_MS,
|
|
89
|
+
// Public threshold + alignment cap (overlap with `expectScreenshot` opts).
|
|
90
|
+
ALIGN_TOLERANCE_MAX_PX,
|
|
91
|
+
DEFAULT_SCREENSHOT_THRESHOLD,
|
|
92
|
+
// Validation sets consumers can read.
|
|
93
|
+
SUPPORTED_VIDEO_MODES,
|
|
94
|
+
SYSTEM_DIALOG_TAGS,
|
|
95
|
+
// Bundle / artifact layout — CI pipelines key off these names.
|
|
96
|
+
BASELINE_RESOLUTION_DIR,
|
|
97
|
+
FAILURE_ARTIFACT_FILE,
|
|
98
|
+
FAILURE_RECENT_LOG_LINES,
|
|
99
|
+
FAILURES_DIR,
|
|
100
|
+
SCREENSHOT_DIFF_DIR
|
|
101
|
+
} from "./constants";
|
|
102
|
+
|
|
103
|
+
export { RkClient } from "./client/client";
|
|
104
|
+
export { createDefaultClient } from "./client/factory";
|
|
105
|
+
// Exported so tests asserting the home-vs-channel contract check it the
|
|
106
|
+
// same way crash-detection does, instead of pinning field values that
|
|
107
|
+
// differ per firmware.
|
|
108
|
+
export { isHomeScreenActiveApp } from "./client/parsers";
|
|
109
|
+
export {
|
|
110
|
+
getFatalReason,
|
|
111
|
+
getUnhealthyReason,
|
|
112
|
+
isDeviceFatal,
|
|
113
|
+
isDeviceHealthy,
|
|
114
|
+
markDeviceFatal,
|
|
115
|
+
markDeviceUnhealthy,
|
|
116
|
+
resetDeviceHealth
|
|
117
|
+
} from "./client/health";
|
|
118
|
+
export { createGlobalSetup } from "./jest/global-setup";
|
|
119
|
+
export type { GlobalSetupOptions } from "./jest/global-setup";
|
|
120
|
+
export { createGlobalTeardown } from "./jest/global-teardown";
|
|
121
|
+
export type { CreateGlobalTeardownOptions } from "./jest/global-teardown";
|
|
122
|
+
export { installDeviceTestHooks } from "./jest/install-hooks";
|
|
123
|
+
export type { DeviceRecoveryOptions, InstallDeviceTestHooksOptions } from "./jest/install-hooks";
|
|
124
|
+
export { describeOnDevice, itOnDevice, requirePrecondition } from "./jest/preconditions";
|
|
125
|
+
export type { OnDeviceDescribeOptions, OnDeviceTestOptions } from "./jest/preconditions";
|
|
126
|
+
// Device-lease telemetry helpers for the parallel-device rollout.
|
|
127
|
+
// Curated for partners writing their own `globalSetup` /
|
|
128
|
+
// `globalTeardown`. Each export earns its keep:
|
|
129
|
+
//
|
|
130
|
+
// - `resolveTelemetry`: bootstrap entry point — turns a
|
|
131
|
+
// known device id + env into the fully-resolved triple.
|
|
132
|
+
// - `emitDeviceLeaseLog`: the side-effecting "emit to stdout"
|
|
133
|
+
// that the dashboard log-aggregator consumes.
|
|
134
|
+
// - `writeJUnitTelemetryPropertiesFile`: the side-effecting
|
|
135
|
+
// "write the CJS file `jest-junit` requires".
|
|
136
|
+
// - `formatDeviceLeaseLog`: pure formatter, exposed so partners
|
|
137
|
+
// piping through structured loggers (e.g. pino) can re-emit
|
|
138
|
+
// the canonical line without duplicating the wire contract.
|
|
139
|
+
// - `buildAcquireEvent` / `buildReleaseEvent` /
|
|
140
|
+
// `buildJUnitTelemetryProperties`: pure builders so partners
|
|
141
|
+
// can construct events without memorising the field shape.
|
|
142
|
+
// - `DEVICE_LEASE_LOG_PREFIX`: surfaced for log-parser code.
|
|
143
|
+
// - `DEFAULT_PROVIDER_KIND` / `DEFAULT_SHARD_NAME`: the
|
|
144
|
+
// documented fallback values when env vars are unset; surfaced
|
|
145
|
+
// so partner tests can assert against them by name.
|
|
146
|
+
//
|
|
147
|
+
// Anything NOT in this list (e.g. internal type guards) is file-
|
|
148
|
+
// internal. The `./src/*` deep-import escape hatch was removed in
|
|
149
|
+
// promote the symbol through this barrel (with a
|
|
150
|
+
// public-API snapshot update) if a partner needs it.
|
|
151
|
+
export {
|
|
152
|
+
buildAcquireEvent,
|
|
153
|
+
buildJUnitTelemetryProperties,
|
|
154
|
+
buildReleaseEvent,
|
|
155
|
+
DEFAULT_PROVIDER_KIND,
|
|
156
|
+
DEFAULT_SHARD_NAME,
|
|
157
|
+
DEVICE_LEASE_LOG_PREFIX,
|
|
158
|
+
emitDeviceLeaseLog,
|
|
159
|
+
formatDeviceLeaseLog,
|
|
160
|
+
resolveTelemetry,
|
|
161
|
+
writeJUnitTelemetryPropertiesFile
|
|
162
|
+
} from "./jest/telemetry";
|
|
163
|
+
export type {
|
|
164
|
+
DeviceLeaseAcquireEvent,
|
|
165
|
+
DeviceLeaseEvent,
|
|
166
|
+
DeviceLeaseReleaseEvent,
|
|
167
|
+
DeviceLeaseTelemetry,
|
|
168
|
+
JUnitTelemetryProperties
|
|
169
|
+
} from "./jest/telemetry";
|
|
170
|
+
export { getDeviceClass } from "./device/device-class";
|
|
171
|
+
export type { DeviceClass } from "./device/device-class";
|
|
172
|
+
export {
|
|
173
|
+
expectFocused,
|
|
174
|
+
expectScreenshot,
|
|
175
|
+
expectTree,
|
|
176
|
+
findAllNodes,
|
|
177
|
+
findFocusedNode,
|
|
178
|
+
findNode,
|
|
179
|
+
matchesPredicate
|
|
180
|
+
} from "./assertions/assertions";
|
|
181
|
+
export type { ScreenshotAssertion, TreeAssertion } from "./assertions/assertions";
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// Copyright (c) 2026 Roku, Inc.
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
/**
|
|
4
|
+
* Artifact helpers shared between `globalSetup` and install hooks.
|
|
5
|
+
* Extracted so orchestration files focus on lifecycle wiring;
|
|
6
|
+
* `jest-setup.ts` re-exports every symbol for API compatibility.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { mkdirSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
|
|
12
|
+
import { ARTIFACT_FILE, SUPPORTED_VIDEO_MODES } from "../constants";
|
|
13
|
+
import type { LeaseProvisioning } from "../device/lease-provisioning";
|
|
14
|
+
import type { DeviceInfo } from "../types";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Removes everything inside `testResultsDir` except a `.gitkeep` marker, then
|
|
18
|
+
* ensures the directory itself exists. Safe to call when the directory is
|
|
19
|
+
* missing.
|
|
20
|
+
*/
|
|
21
|
+
export function cleanArtifacts(testResultsDir: string): void {
|
|
22
|
+
mkdirSync(testResultsDir, { recursive: true });
|
|
23
|
+
for (const entry of readdirSync(testResultsDir, { withFileTypes: true })) {
|
|
24
|
+
if (entry.name === ".gitkeep") continue;
|
|
25
|
+
rmSync(join(testResultsDir, entry.name), { recursive: true, force: true });
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function assertSupportedVideoMode(videoMode: string): void {
|
|
30
|
+
if (!SUPPORTED_VIDEO_MODES.has(videoMode)) {
|
|
31
|
+
throw new Error(
|
|
32
|
+
`Unrecognized device videoMode "${videoMode}". ` + `Supported: ${[...SUPPORTED_VIDEO_MODES].join(", ")}.`
|
|
33
|
+
);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* On-disk shape of `device-env.json`: the ECP `DeviceInfo` plus whatever
|
|
39
|
+
* provisioning provenance the lease carried.
|
|
40
|
+
*
|
|
41
|
+
* A superset rather than extra `DeviceInfo` fields, because `DeviceInfo` is
|
|
42
|
+
* the return type of `queryDeviceInfo()` and must keep meaning "what ECP
|
|
43
|
+
* reported". Readers of the artifact stay valid either way: the snapshot
|
|
44
|
+
* validator in `client/factory.ts` checks required fields only, and
|
|
45
|
+
* `readCachedDeviceInfoSync` passes extras through untouched.
|
|
46
|
+
*/
|
|
47
|
+
export type DeviceEnvSnapshot = DeviceInfo & LeaseProvisioning;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Atomic write of `device-env.json` (write-temp + rename) so
|
|
51
|
+
* concurrent readers never see a torn JSON payload.
|
|
52
|
+
*
|
|
53
|
+
* `provisioning` keys with an `undefined` value are dropped by
|
|
54
|
+
* `JSON.stringify`, so an unknown field is absent from the artifact rather
|
|
55
|
+
* than present-and-null — see `lease-provisioning.ts` for why absence is
|
|
56
|
+
* the correct encoding of "unknown".
|
|
57
|
+
*/
|
|
58
|
+
export function writeDeviceEnvSnapshot(
|
|
59
|
+
testResultsDir: string,
|
|
60
|
+
info: DeviceInfo,
|
|
61
|
+
provisioning: LeaseProvisioning = {}
|
|
62
|
+
): void {
|
|
63
|
+
const file = join(testResultsDir, ARTIFACT_FILE.DEVICE_ENV);
|
|
64
|
+
const snapshot: DeviceEnvSnapshot = { ...info, ...provisioning };
|
|
65
|
+
// Per-pid+timestamp tmp name so concurrent globalSetups don't
|
|
66
|
+
// clobber each other's in-flight tmp files.
|
|
67
|
+
const tmp = `${file}.${process.pid}.${Date.now()}.tmp`;
|
|
68
|
+
writeFileSync(tmp, `${JSON.stringify(snapshot, null, 2)}\n`, "utf8");
|
|
69
|
+
renameSync(tmp, file);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function logDeviceDiagnostic(info: DeviceInfo, provisioning: LeaseProvisioning = {}): void {
|
|
73
|
+
// The rootfs is appended to the SAME line rather than logged separately
|
|
74
|
+
// so a `grep "\[device-test-client\] device:"` over a CI log still yields
|
|
75
|
+
// one self-contained record per job.
|
|
76
|
+
const rootfs = provisioning.rootfsArtifactoryPath;
|
|
77
|
+
const rootfsSuffix = rootfs === undefined ? "" : `, rootfs ${rootfs}`;
|
|
78
|
+
console.log(
|
|
79
|
+
`[device-test-client] device: ${info.modelName} (${info.modelNumber}), ` +
|
|
80
|
+
`firmware ${info.firmwareVersion}, videoMode ${info.videoMode}, locale ${info.locale}${rootfsSuffix}`
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Trim a single trailing `/` — shared helper for `rk server` URL composition. */
|
|
85
|
+
export function trimTrailingSlash(s: string): string {
|
|
86
|
+
return s.endsWith("/") ? s.slice(0, -1) : s;
|
|
87
|
+
}
|
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
// Copyright (c) 2026 Roku, Inc.
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
/**
|
|
4
|
+
* Custom Jest environment that hooks `handleTestEvent` to capture
|
|
5
|
+
* device-side artifacts on failure and writes them to
|
|
6
|
+
* `<testResultsDir>/failures/<sanitized>/`. Used over `afterEach`
|
|
7
|
+
* because circus emits structured `test_fn_failure`/`hook_failure`
|
|
8
|
+
* events with the actual error.
|
|
9
|
+
*
|
|
10
|
+
* Capture runs EAGERLY on each failure event (not at teardown) so
|
|
11
|
+
* screenshots/DOM/logs reflect the device state at the moment of
|
|
12
|
+
* failure rather than after the next test's `beforeAll` deploys
|
|
13
|
+
* over it. Disk writes are still deferred to `teardown` (off the
|
|
14
|
+
* hot path).
|
|
15
|
+
*
|
|
16
|
+
* Globals set by the consumer's `setup.ts`:
|
|
17
|
+
* - `DEVICE_ENV_GLOBAL_KEY.CLIENT` — RkClient
|
|
18
|
+
* - `DEVICE_ENV_GLOBAL_KEY.TEST_RESULTS_DIR` — abs path
|
|
19
|
+
* - `DEVICE_ENV_GLOBAL_KEY.CAPTURE_ENABLED` — opt-out flag
|
|
20
|
+
*
|
|
21
|
+
* All optional. Capture short-circuits when `CAPTURE_ENABLED ===
|
|
22
|
+
* false`, or when both `CAPTURE_ENABLED` and `TEST_RESULTS_DIR`
|
|
23
|
+
* are undefined (vanilla consumer that picked this env via
|
|
24
|
+
* misconfigured preset).
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { join } from "node:path";
|
|
28
|
+
|
|
29
|
+
import { DEVICE_ENV_GLOBAL_KEY, FAILURE_RECENT_LOG_LINES } from "../constants";
|
|
30
|
+
import { TEST_RESULTS_DIR_NAME } from "../paths";
|
|
31
|
+
import type { LogLine } from "../types";
|
|
32
|
+
import { ensureFailuresDir, sanitizeTestPath, writeFailureBundle } from "./failure-artifacts";
|
|
33
|
+
|
|
34
|
+
// Runtime `require()` of `jest-environment-node` keeps the
|
|
35
|
+
// dependency out of our TS type graph and picks whichever Jest
|
|
36
|
+
// version is hosting the worker (29 or 30).
|
|
37
|
+
|
|
38
|
+
interface NodeEnvironmentConstructor {
|
|
39
|
+
new (config: unknown, context?: unknown): NodeEnvironmentLike;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Base class shape; `any` to avoid coupling to Jest 29 vs 30. */
|
|
43
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
44
|
+
type NodeEnvironmentLike = any;
|
|
45
|
+
|
|
46
|
+
/* eslint-disable @typescript-eslint/no-require-imports */
|
|
47
|
+
const NodeEnvModule = require("jest-environment-node");
|
|
48
|
+
/* eslint-enable @typescript-eslint/no-require-imports */
|
|
49
|
+
|
|
50
|
+
// Both Jest 29 and 30 export the class as `TestEnvironment`.
|
|
51
|
+
const NodeEnvironment: NodeEnvironmentConstructor = NodeEnvModule.TestEnvironment;
|
|
52
|
+
|
|
53
|
+
// Local circus-event shapes (stable across Jest 29/30) so we don't
|
|
54
|
+
// need `@jest/types` as a dep.
|
|
55
|
+
|
|
56
|
+
interface CircusNamedBlock {
|
|
57
|
+
name: string;
|
|
58
|
+
parent?: CircusNamedBlock;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
interface CircusTestEntry extends CircusNamedBlock {
|
|
62
|
+
errors?: unknown[];
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
interface CircusHook extends CircusNamedBlock {
|
|
66
|
+
type: string;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
interface TestFnFailureEvent {
|
|
70
|
+
name: "test_fn_failure";
|
|
71
|
+
test: CircusTestEntry;
|
|
72
|
+
error?: unknown;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
interface HookFailureEvent {
|
|
76
|
+
name: "hook_failure";
|
|
77
|
+
hook: CircusHook;
|
|
78
|
+
test?: CircusTestEntry;
|
|
79
|
+
error?: unknown;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
interface TeardownEvent {
|
|
83
|
+
name: "teardown";
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
type KnownEvent = TestFnFailureEvent | HookFailureEvent | TeardownEvent;
|
|
87
|
+
|
|
88
|
+
// Structural client shape — keeps the environment decoupled from
|
|
89
|
+
// the full client module graph so unit tests can inject fakes.
|
|
90
|
+
export interface FailureCaptureClient {
|
|
91
|
+
captureScreenshot(): Promise<Buffer>;
|
|
92
|
+
queryAppUi(): Promise<unknown>;
|
|
93
|
+
recentLogLines(n: number): ReadonlyArray<{ seq: number; line: LogLine }>;
|
|
94
|
+
/** Optional; older clients/fakes may omit. Probed before calling. */
|
|
95
|
+
getLogStreamHealth?(): Promise<Record<string, unknown>>;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
interface PendingFailure {
|
|
99
|
+
/** describe ancestors + it name (root excluded). */
|
|
100
|
+
testPath: string[];
|
|
101
|
+
error: unknown;
|
|
102
|
+
/** Device-side artifacts captured at failure event time (best-effort). */
|
|
103
|
+
screenshot: Buffer | null;
|
|
104
|
+
domTree: unknown;
|
|
105
|
+
recentLogs: ReadonlyArray<{ seq: number; line: LogLine }> | null;
|
|
106
|
+
streamHealth: Record<string, unknown> | null;
|
|
107
|
+
/** Capture-time warnings; merged with write-time on flush. */
|
|
108
|
+
captureWarnings: Array<{ artifact: string; cause: Error }>;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Subclass of `jest-environment-node`'s `TestEnvironment`. Implements
|
|
113
|
+
* `handleTestEvent` to buffer failures and flush them on `teardown`.
|
|
114
|
+
*/
|
|
115
|
+
export default class DeviceTestEnvironment extends NodeEnvironment {
|
|
116
|
+
/** Failure buffer — flushed once on the single `teardown` event. */
|
|
117
|
+
private pendingFailures: PendingFailure[] = [];
|
|
118
|
+
|
|
119
|
+
async handleTestEvent(event: unknown, _state: unknown): Promise<void> {
|
|
120
|
+
const typed = event as KnownEvent;
|
|
121
|
+
if (typed.name === "test_fn_failure") {
|
|
122
|
+
if (!this.shouldCapture()) return;
|
|
123
|
+
await this.recordFailure(buildTestPath(typed.test), pickError(typed.test.errors, typed.error));
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
if (typed.name === "hook_failure") {
|
|
127
|
+
if (!this.shouldCapture()) return;
|
|
128
|
+
const testPath = typed.test ? buildTestPath(typed.test) : buildHookPath(typed.hook);
|
|
129
|
+
await this.recordFailure(testPath, typed.error ?? new Error(`${typed.hook.type} failed`));
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
if (typed.name === "teardown") {
|
|
133
|
+
await this.flushPendingFailures();
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Capture gate. `CAPTURE_ENABLED === false` → off; both
|
|
139
|
+
* `CAPTURE_ENABLED` and `TEST_RESULTS_DIR` undefined → off
|
|
140
|
+
* (vanilla consumer didn't opt in). Defensive overlap with the
|
|
141
|
+
* flush guard in case globals mutate between events.
|
|
142
|
+
*/
|
|
143
|
+
private shouldCapture(): boolean {
|
|
144
|
+
const envGlobal = this.global as Record<string, unknown>;
|
|
145
|
+
const captureEnabled = envGlobal[DEVICE_ENV_GLOBAL_KEY.CAPTURE_ENABLED];
|
|
146
|
+
if (captureEnabled === false) return false;
|
|
147
|
+
if (captureEnabled === undefined && envGlobal[DEVICE_ENV_GLOBAL_KEY.TEST_RESULTS_DIR] === undefined) {
|
|
148
|
+
return false;
|
|
149
|
+
}
|
|
150
|
+
return true;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Capture device-side artifacts NOW (eager). Runs inside Jest's
|
|
155
|
+
* awaited `handleTestEvent`, blocking the next `beforeEach` until
|
|
156
|
+
* the snapshot is in hand. Each call wrapped in `captureArtifact`
|
|
157
|
+
* so one throw doesn't abort the others.
|
|
158
|
+
*/
|
|
159
|
+
private async recordFailure(testPath: string[], error: unknown): Promise<void> {
|
|
160
|
+
const envGlobal = this.global as Record<string, unknown>;
|
|
161
|
+
const client = envGlobal[DEVICE_ENV_GLOBAL_KEY.CLIENT] as FailureCaptureClient | undefined;
|
|
162
|
+
|
|
163
|
+
const captureWarnings: Array<{ artifact: string; cause: Error }> = [];
|
|
164
|
+
let screenshot: Buffer | null = null;
|
|
165
|
+
let domTree: unknown = null;
|
|
166
|
+
let recentLogs: ReadonlyArray<{ seq: number; line: LogLine }> | null = null;
|
|
167
|
+
let streamHealth: Record<string, unknown> | null = null;
|
|
168
|
+
|
|
169
|
+
if (client) {
|
|
170
|
+
screenshot = await captureArtifact("screenshot", () => client.captureScreenshot(), captureWarnings);
|
|
171
|
+
domTree = await captureArtifact("dom-tree", () => client.queryAppUi(), captureWarnings);
|
|
172
|
+
recentLogs =
|
|
173
|
+
(await captureArtifact(
|
|
174
|
+
"recent-logs",
|
|
175
|
+
async () => client.recentLogLines(FAILURE_RECENT_LOG_LINES),
|
|
176
|
+
captureWarnings
|
|
177
|
+
)) ?? [];
|
|
178
|
+
if (typeof client.getLogStreamHealth === "function") {
|
|
179
|
+
streamHealth = await captureArtifact(
|
|
180
|
+
"stream-health",
|
|
181
|
+
async () => {
|
|
182
|
+
// Re-narrow locally; the closure doesn't see
|
|
183
|
+
// the typeof guard above.
|
|
184
|
+
const fn = client.getLogStreamHealth!;
|
|
185
|
+
return await fn.call(client);
|
|
186
|
+
},
|
|
187
|
+
captureWarnings
|
|
188
|
+
);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Surface the one-line stream-health summary so CI stdout
|
|
193
|
+
// shows gap context inline with the failure without
|
|
194
|
+
// requiring the reviewer to fetch the on-disk bundle.
|
|
195
|
+
// Routed through `console.warn` (not error.message/.stack
|
|
196
|
+
// mutation) because (a) Jest's reporter pipes our console
|
|
197
|
+
// calls right next to the failure, so visibility is
|
|
198
|
+
// preserved, and (b) mutating foreign Error objects
|
|
199
|
+
// throws on frozen errors and breaks custom matchers that
|
|
200
|
+
// pin on `expect(err.message).toMatch(...)`. The bundle
|
|
201
|
+
// already persists `streamHealth` separately for offline
|
|
202
|
+
// analysis, so this is purely an inline-visibility hook.
|
|
203
|
+
emitStreamHealthSummary(testPath, streamHealth);
|
|
204
|
+
|
|
205
|
+
this.pendingFailures.push({
|
|
206
|
+
testPath,
|
|
207
|
+
error,
|
|
208
|
+
screenshot,
|
|
209
|
+
domTree,
|
|
210
|
+
recentLogs,
|
|
211
|
+
streamHealth,
|
|
212
|
+
captureWarnings
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* Drain `pendingFailures` and write one bundle per failure. All
|
|
218
|
+
* work is best-effort and must not propagate out of `teardown`.
|
|
219
|
+
* Emits at most ONE warning per bundle covering every capture +
|
|
220
|
+
* write failure for that bundle.
|
|
221
|
+
*/
|
|
222
|
+
private async flushPendingFailures(): Promise<void> {
|
|
223
|
+
if (this.pendingFailures.length === 0) return;
|
|
224
|
+
const envGlobal = this.global as Record<string, unknown>;
|
|
225
|
+
|
|
226
|
+
// Defensive re-check: a late `CAPTURE_ENABLED = false`
|
|
227
|
+
// write still suppresses on-disk output.
|
|
228
|
+
if (!this.shouldCapture()) {
|
|
229
|
+
this.pendingFailures = [];
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const resultsDir = resolveTestResultsDir(envGlobal);
|
|
234
|
+
let failuresDir: string;
|
|
235
|
+
try {
|
|
236
|
+
failuresDir = ensureFailuresDir(resultsDir);
|
|
237
|
+
} catch (err) {
|
|
238
|
+
console.warn(
|
|
239
|
+
`[device-test-client] could not create failures dir at "${resultsDir}": ` +
|
|
240
|
+
`${(err as Error).message}. Failure diagnostics will be skipped.`,
|
|
241
|
+
err
|
|
242
|
+
);
|
|
243
|
+
this.pendingFailures = [];
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
for (const failure of this.pendingFailures) {
|
|
248
|
+
const sanitized = sanitizeTestPath(failure.testPath);
|
|
249
|
+
const bundleDir = join(failuresDir, sanitized);
|
|
250
|
+
|
|
251
|
+
// Capture-time + write-time warnings → ONE warn per bundle.
|
|
252
|
+
const warnings: Array<{ artifact: string; cause: Error }> = [...failure.captureWarnings];
|
|
253
|
+
|
|
254
|
+
const result = writeFailureBundle(bundleDir, {
|
|
255
|
+
screenshot: failure.screenshot,
|
|
256
|
+
domTree: failure.domTree,
|
|
257
|
+
recentLogs: failure.recentLogs,
|
|
258
|
+
streamHealth: failure.streamHealth,
|
|
259
|
+
error: failure.error
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
for (const f of result.failed) {
|
|
263
|
+
warnings.push(f);
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
if (warnings.length > 0) {
|
|
267
|
+
emitBundleWarning(sanitized, warnings);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// One-line pointer so reviewers find the bundle without
|
|
271
|
+
// knowing the `test-results/failures/...` convention.
|
|
272
|
+
// Skipped when nothing landed (mkdir failure).
|
|
273
|
+
if (result.written.length > 0) {
|
|
274
|
+
console.log(
|
|
275
|
+
`[device-test-client] failure bundle written: ${bundleDir} ` + `(${result.written.join(", ")})`
|
|
276
|
+
);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
this.pendingFailures = [];
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* Await `fn`; on throw, push `{artifact, cause}` and return null so
|
|
286
|
+
* the bundle writer still gets a partial input. Caller emits one
|
|
287
|
+
* consolidated warning per bundle.
|
|
288
|
+
*/
|
|
289
|
+
async function captureArtifact<T>(
|
|
290
|
+
artifact: string,
|
|
291
|
+
fn: () => Promise<T>,
|
|
292
|
+
warnings: Array<{ artifact: string; cause: Error }>
|
|
293
|
+
): Promise<T | null> {
|
|
294
|
+
try {
|
|
295
|
+
return await fn();
|
|
296
|
+
} catch (err) {
|
|
297
|
+
warnings.push({ artifact, cause: toError(err) });
|
|
298
|
+
return null;
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/**
|
|
303
|
+
* One warn per bundle. Pass each underlying `Error` separately so
|
|
304
|
+
* Node's console printer renders the full stack.
|
|
305
|
+
*/
|
|
306
|
+
function emitBundleWarning(sanitized: string, warnings: ReadonlyArray<{ artifact: string; cause: Error }>): void {
|
|
307
|
+
const summary = warnings.map(w => `${w.artifact}=${w.cause.message}`).join("; ");
|
|
308
|
+
console.warn(`[device-test-client] bundle "${sanitized}" had warnings: ${summary}`, ...warnings.map(w => w.cause));
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function toError(err: unknown): Error {
|
|
312
|
+
return err instanceof Error ? err : new Error(String(err));
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
/**
|
|
316
|
+
* Emit a one-line stream-health summary via `console.warn` so
|
|
317
|
+
* Jest's reporter renders it inline with the failure. Replaces an
|
|
318
|
+
* earlier in-place mutation of `error.message`/`error.stack`,
|
|
319
|
+
* which threw on frozen errors and broke custom matchers that
|
|
320
|
+
* pinned on `expect(err.message).toMatch(...)`.
|
|
321
|
+
*
|
|
322
|
+
* Skipped on trivial snapshots so healthy runs don't gain noise.
|
|
323
|
+
* `testPath` is included in the prefix so a multi-failure run
|
|
324
|
+
* still attributes each summary to the right test.
|
|
325
|
+
*/
|
|
326
|
+
function emitStreamHealthSummary(testPath: string[], health: Record<string, unknown> | null): void {
|
|
327
|
+
if (!health) return;
|
|
328
|
+
const summary = formatStreamHealthSummary(health);
|
|
329
|
+
if (!summary) return;
|
|
330
|
+
const label = testPath.length > 0 ? testPath.join(" › ") : "<unknown test>";
|
|
331
|
+
console.warn(`[device-test-client] [log-stream-health] ${label}: ${summary}`);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* One-line `LogStreamHealth` summary; `""` for trivially-clean
|
|
336
|
+
* snapshots so healthy runs stay terse. Reads fields defensively
|
|
337
|
+
* to avoid an env→types import edge.
|
|
338
|
+
*/
|
|
339
|
+
function formatStreamHealthSummary(health: Record<string, unknown>): string {
|
|
340
|
+
const gapCount = numberFrom(health.gapCount, 0);
|
|
341
|
+
const totalGapSeqSpan = numberFrom(health.totalGapSeqSpan, 0);
|
|
342
|
+
const openGap = health.openGap === true;
|
|
343
|
+
const server = isObject(health.serverMetrics) ? health.serverMetrics : null;
|
|
344
|
+
const reconnectCount = server ? numberFrom(server.reconnectCount, 0) : 0;
|
|
345
|
+
const totalDowntime = server ? numberFrom(server.totalDowntime, 0) : 0;
|
|
346
|
+
const trivial = gapCount === 0 && reconnectCount === 0 && !openGap;
|
|
347
|
+
if (trivial) return "";
|
|
348
|
+
const parts: string[] = [];
|
|
349
|
+
parts.push(`client gaps=${gapCount} (span=${totalGapSeqSpan} seqs)`);
|
|
350
|
+
if (openGap) parts.push(`open-gap=true`);
|
|
351
|
+
if (server) {
|
|
352
|
+
parts.push(`server reconnects=${reconnectCount} downtime=${Math.round(totalDowntime)}ms`);
|
|
353
|
+
} else if (health.serverMetrics === null) {
|
|
354
|
+
parts.push(`server metrics unavailable (rk server unreachable)`);
|
|
355
|
+
}
|
|
356
|
+
return parts.join("; ");
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
function numberFrom(v: unknown, fallback: number): number {
|
|
360
|
+
return typeof v === "number" && Number.isFinite(v) ? v : fallback;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function isObject(v: unknown): v is Record<string, unknown> {
|
|
364
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/** Build `[...describeAncestors, testName]`; drops ROOT_DESCRIBE_BLOCK. */
|
|
368
|
+
function buildTestPath(test: CircusTestEntry): string[] {
|
|
369
|
+
const parts: string[] = [];
|
|
370
|
+
const ancestors: string[] = [];
|
|
371
|
+
let cursor: CircusNamedBlock | undefined = test.parent;
|
|
372
|
+
while (cursor) {
|
|
373
|
+
ancestors.unshift(cursor.name);
|
|
374
|
+
cursor = cursor.parent;
|
|
375
|
+
}
|
|
376
|
+
for (const name of ancestors) {
|
|
377
|
+
if (name === "ROOT_DESCRIBE_BLOCK") continue;
|
|
378
|
+
parts.push(name);
|
|
379
|
+
}
|
|
380
|
+
parts.push(test.name);
|
|
381
|
+
return parts;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** Path for a hook failure with no test; leaf = `hook-<type>`. */
|
|
385
|
+
function buildHookPath(hook: CircusHook): string[] {
|
|
386
|
+
const parts: string[] = [];
|
|
387
|
+
const ancestors: string[] = [];
|
|
388
|
+
let cursor: CircusNamedBlock | undefined = hook.parent;
|
|
389
|
+
while (cursor) {
|
|
390
|
+
ancestors.unshift(cursor.name);
|
|
391
|
+
cursor = cursor.parent;
|
|
392
|
+
}
|
|
393
|
+
for (const name of ancestors) {
|
|
394
|
+
if (name === "ROOT_DESCRIBE_BLOCK") continue;
|
|
395
|
+
parts.push(name);
|
|
396
|
+
}
|
|
397
|
+
parts.push(`hook-${hook.type}`);
|
|
398
|
+
return parts;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* Pick the most actionable error from a circus entry. Last entry
|
|
403
|
+
* wins (follow-on errors are less interesting than the original).
|
|
404
|
+
*/
|
|
405
|
+
function pickError(errors: unknown[] | undefined, fallback: unknown): unknown {
|
|
406
|
+
if (errors && errors.length > 0) {
|
|
407
|
+
const last = errors[errors.length - 1];
|
|
408
|
+
if (Array.isArray(last) && last.length > 0) return last[0];
|
|
409
|
+
return last;
|
|
410
|
+
}
|
|
411
|
+
return fallback ?? new Error("unknown test failure");
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
function resolveTestResultsDir(envGlobal: Record<string, unknown>): string {
|
|
415
|
+
const fromGlobal = envGlobal[DEVICE_ENV_GLOBAL_KEY.TEST_RESULTS_DIR];
|
|
416
|
+
if (typeof fromGlobal === "string" && fromGlobal.length > 0) {
|
|
417
|
+
return fromGlobal;
|
|
418
|
+
}
|
|
419
|
+
return join(process.cwd(), TEST_RESULTS_DIR_NAME);
|
|
420
|
+
}
|