@delacour/warden-e2e 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{chunk-qtkmfjbq.mjs → chunk-7ns30pbp.mjs} +5482 -4235
- package/index.mjs +119 -70
- package/package.json +2 -2
- package/preload.mjs +215 -939
- package/types/core/src/providers/android.d.ts +8 -0
- package/types/core/src/store.d.ts +15 -1
- package/types/core/src/types.d.ts +20 -0
- package/types/e2e/src/app.d.ts +1 -1
- package/types/e2e/src/device.d.ts +9 -1
- package/types/e2e/src/devtools/native-log.d.ts +50 -0
- package/types/e2e/src/driver/android/uiad-backend.d.ts +1 -1
- package/types/e2e/src/driver/driver.types.d.ts +1 -1
- package/types/e2e/src/driver/ios/hid-recovery.d.ts +36 -0
- package/types/e2e/src/driver/ios/simctl.d.ts +15 -0
- package/types/e2e/src/driver/observation-cache.d.ts +1 -1
- package/types/e2e/src/env.d.ts +26 -9
- package/types/e2e/src/errors.d.ts +1 -1
- package/types/e2e/src/fixture-session.d.ts +5 -1
- package/types/e2e/src/index.d.ts +3 -0
- package/types/e2e/src/perf/budgets.d.ts +33 -0
- package/types/e2e/src/perf/device-perf.d.ts +45 -0
- package/types/e2e/src/perf/perf.types.d.ts +59 -0
- package/types/e2e/src/preload.d.ts +2 -2
- package/types/e2e/src/runner/artifacts.d.ts +5 -5
- package/types/e2e/src/runner/failure.d.ts +12 -0
- package/types/e2e/src/runner/fixtures.types.d.ts +1 -1
- package/types/e2e/src/runner/scope.d.ts +5 -3
- package/types/e2e/src/runner/test.d.ts +7 -0
- package/types/e2e/src/screen/actions.d.ts +1 -1
- package/types/e2e/src/session.d.ts +19 -0
|
@@ -115,6 +115,14 @@ export declare const GOLDEN_PORT_LABEL = "emulator golden ";
|
|
|
115
115
|
* and unverified on a device; a successful load logs `Successfully loaded snapshot …`, which does not match.
|
|
116
116
|
*/
|
|
117
117
|
export declare function snapshotLoadFailed(log: string): boolean;
|
|
118
|
+
/**
|
|
119
|
+
* Global settings applied after every boot. On a fresh playstore image the first `adb install` can pop a Play
|
|
120
|
+
* Protect consent dialog (a GMS activity the e2e alert detector cannot classify), and every adb install otherwise
|
|
121
|
+
* waits on the GMS verifier (up to the 10 s verifier_timeout). Warden's emulators run `-read-only -no-snapshot-save`,
|
|
122
|
+
* so /data changes are a per-boot overlay and must be re-applied on each boot.
|
|
123
|
+
*/
|
|
124
|
+
export declare const BOOT_SETTINGS: ReadonlyArray<readonly [key: string, value: string]>;
|
|
125
|
+
export declare function bootSettingsCommand(): string;
|
|
118
126
|
/**
|
|
119
127
|
* Android emulators on `adb` + `emulator`. Warden never boots an existing emulator — it launches a
|
|
120
128
|
* new headless `-read-only` instance of an AVD (so one AVD backs many emulators) on a leased even
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Database } from "bun:sqlite";
|
|
2
2
|
import { type PidAlive } from "./liveness";
|
|
3
|
-
import type { Lease, Owner, Platform, Resource } from "./types";
|
|
3
|
+
import type { Lease, Owner, Platform, QueueEntry, Resource } from "./types";
|
|
4
4
|
/** `$WARDEN_HOME`, else `~/.warden`. */
|
|
5
5
|
export declare function wardenHome(env?: Record<string, string | undefined>): string;
|
|
6
6
|
export declare function defaultDbPath(env?: Record<string, string | undefined>): string;
|
|
@@ -9,6 +9,8 @@ export declare function defaultDbPath(env?: Record<string, string | undefined>):
|
|
|
9
9
|
* edit an entry that has shipped.
|
|
10
10
|
*/
|
|
11
11
|
export declare const MIGRATIONS: readonly string[];
|
|
12
|
+
/** A queued claim that has not refreshed within this long is dead (its waiter polls every few seconds). */
|
|
13
|
+
export declare const QUEUE_TTL_MS = 60000;
|
|
12
14
|
export declare function ownerKey(owner: Owner): string;
|
|
13
15
|
export type NewLease = {
|
|
14
16
|
resource: Resource;
|
|
@@ -17,6 +19,7 @@ export type NewLease = {
|
|
|
17
19
|
label?: string;
|
|
18
20
|
pid?: number;
|
|
19
21
|
bootedByOwner?: boolean;
|
|
22
|
+
shared?: boolean;
|
|
20
23
|
};
|
|
21
24
|
/** A device warden created (sim) or launched (emulator). */
|
|
22
25
|
export type DeviceRecord = {
|
|
@@ -45,6 +48,8 @@ export declare class Store {
|
|
|
45
48
|
/** Run `fn` under `BEGIN IMMEDIATE` (write lock taken up front). Nested calls join the outer txn. */
|
|
46
49
|
transaction<T>(fn: () => T): T;
|
|
47
50
|
insertLease(input: NewLease, now: number): Lease;
|
|
51
|
+
/** Mark leases shared (`Lease.shared`): `warden dev` on the device leases it just claimed. */
|
|
52
|
+
markShared(ids: readonly string[]): void;
|
|
48
53
|
/** Record that this lease's owner booted its device (claim boot/create, hook saw it shut down). */
|
|
49
54
|
markBootedByOwner(id: string): void;
|
|
50
55
|
/** Record the process a lease now lives with (an emulator launched after its port was leased). */
|
|
@@ -65,6 +70,15 @@ export declare class Store {
|
|
|
65
70
|
heartbeat(ids: readonly string[], now: number, periodMs?: number): number;
|
|
66
71
|
/** Delete leases that are not alive (`isLeaseAlive`: pid dead AND heartbeat expired). Returns reclaimed ids. */
|
|
67
72
|
reclaimStale(now: number, pidAlive: PidAlive): string[];
|
|
73
|
+
/** Insert a queue row; an existing id keeps its `enqueuedAt` (queue age) and updates the rest. */
|
|
74
|
+
enqueue(entry: QueueEntry): QueueEntry;
|
|
75
|
+
/** Stamp `heartbeat_at`; false when the row is gone (reclaimed or dequeued). */
|
|
76
|
+
refreshQueue(id: string, now: number): boolean;
|
|
77
|
+
dequeue(id: string): boolean;
|
|
78
|
+
/** Raw insertion order (`enqueued_at`, id); use `queueOrder` for admission order. */
|
|
79
|
+
listQueue(): QueueEntry[];
|
|
80
|
+
/** Delete queue rows whose waiter is gone: pid dead, or heartbeat older than `QUEUE_TTL_MS`. */
|
|
81
|
+
reclaimStaleQueue(now: number, pidAlive: PidAlive): string[];
|
|
68
82
|
recordDevice(device: Omit<DeviceRecord, "createdAt" | "lastUsedAt">, now: number): void;
|
|
69
83
|
listDevices(platform?: Platform): DeviceRecord[];
|
|
70
84
|
touchDevice(platform: Platform, id: string, now: number): void;
|
|
@@ -50,6 +50,11 @@ export type Lease = {
|
|
|
50
50
|
* record their own `pid`). With a pid, a dead pid makes the lease stale after `LOST_HEARTBEATS` periods.
|
|
51
51
|
*/
|
|
52
52
|
heartbeatMs?: number;
|
|
53
|
+
/**
|
|
54
|
+
* The holding process (`warden dev`) lets its owner's other processes drive the device beside it: `warden ui`
|
|
55
|
+
* attaches to it instead of refusing with `device_busy`, and never releases it. Other owners still see it as foreign.
|
|
56
|
+
*/
|
|
57
|
+
shared?: boolean;
|
|
53
58
|
};
|
|
54
59
|
export type DeviceState = "booted" | "shutdown" | "booting";
|
|
55
60
|
/** A device as seen by a provider, enriched with warden's own bookkeeping. */
|
|
@@ -104,3 +109,18 @@ export declare function describeOwner(owner: Owner): string;
|
|
|
104
109
|
/** Owner location for messages: `repo/worktree` when known. */
|
|
105
110
|
export declare function ownerLocation(owner: Owner): string | undefined;
|
|
106
111
|
export declare function sameOwner(a: Owner, b: Owner): boolean;
|
|
112
|
+
/** A claim waiting for host capacity to boot devices (`boot_queue` row). */
|
|
113
|
+
export type QueueEntry = {
|
|
114
|
+
id: string;
|
|
115
|
+
platform: Platform;
|
|
116
|
+
profile: string;
|
|
117
|
+
/** capacity units the boots need */
|
|
118
|
+
units: number;
|
|
119
|
+
/** numeric base priority (see `PRIORITY` in queue.ts); aging adds to it */
|
|
120
|
+
priority: number;
|
|
121
|
+
enqueuedAt: number;
|
|
122
|
+
heartbeatAt: number;
|
|
123
|
+
owner: Owner;
|
|
124
|
+
label?: string;
|
|
125
|
+
pid?: number;
|
|
126
|
+
};
|
package/types/e2e/src/app.d.ts
CHANGED
|
@@ -6,7 +6,7 @@ import { type Locator } from "./screen/locator";
|
|
|
6
6
|
export type OpenWaitUntil = {
|
|
7
7
|
/** a locator made by `screen` that is visible once the app's first screen is up */
|
|
8
8
|
until: Locator;
|
|
9
|
-
/** how long `until` gets to show; default the assertion timeout (`
|
|
9
|
+
/** how long `until` gets to show; default the assertion timeout (`WARDEN_E2E_ASSERTION_TIMEOUT`) */
|
|
10
10
|
timeout?: number;
|
|
11
11
|
};
|
|
12
12
|
export type OpenOptions = {
|
|
@@ -3,6 +3,7 @@ import type { Driver, Orientation } from "./driver/driver.types";
|
|
|
3
3
|
import { type SettleDetail } from "./driver/settle";
|
|
4
4
|
import type { E2eEnv } from "./env";
|
|
5
5
|
import { type Clock, type FixtureSession } from "./fixture-session";
|
|
6
|
+
import { type DevicePerf, type PerfExec } from "./perf/device-perf";
|
|
6
7
|
import type { Locator } from "./screen/locator";
|
|
7
8
|
export type AlertAction = "accept" | "dismiss" | {
|
|
8
9
|
button: string;
|
|
@@ -72,11 +73,18 @@ export type Device = {
|
|
|
72
73
|
* `WARDEN_METRO_URL` (a release build). The first access starts attaching, so console capture begins there.
|
|
73
74
|
*/
|
|
74
75
|
readonly devtools: DeviceDevtools | undefined;
|
|
76
|
+
/**
|
|
77
|
+
* Perf profiling of the app (`warden perf` over the CLI that started the run): `start`, `mark`, then `stop` for the
|
|
78
|
+
* typed report. The report is copied under the attempt's `perf/`; budgets from `warden.config.ts` fail `stop`.
|
|
79
|
+
*/
|
|
80
|
+
readonly perf: DevicePerf;
|
|
75
81
|
};
|
|
76
82
|
export type DeviceDeps = {
|
|
77
83
|
session: FixtureSession;
|
|
78
84
|
driver: Driver;
|
|
79
85
|
env: E2eEnv;
|
|
80
86
|
clock?: Clock;
|
|
87
|
+
/** runs `warden …` for `device.perf`; default spawns `WARDEN_BIN` */
|
|
88
|
+
perfExec?: PerfExec;
|
|
81
89
|
};
|
|
82
|
-
export declare function createDevice({ session, driver, env, clock }: DeviceDeps): Device;
|
|
90
|
+
export declare function createDevice({ session, driver, env, clock, perfExec }: DeviceDeps): Device;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { Exec } from "../../../core/src/exec";
|
|
2
|
+
export declare const NATIVE_LOG = "native.log";
|
|
3
|
+
/** The app a failing attempt ran: `native.log` and crash reports are read for it only. */
|
|
4
|
+
export type NativeTarget = {
|
|
5
|
+
platform: "ios";
|
|
6
|
+
udid: string;
|
|
7
|
+
bundleId: string;
|
|
8
|
+
} | {
|
|
9
|
+
platform: "android";
|
|
10
|
+
serial: string;
|
|
11
|
+
pkg: string;
|
|
12
|
+
};
|
|
13
|
+
export type CrashReport = {
|
|
14
|
+
name: string;
|
|
15
|
+
mtimeMs: number;
|
|
16
|
+
};
|
|
17
|
+
/** Filesystem seam: where crash reports are read from and `native.log` goes. */
|
|
18
|
+
export type NativeFs = {
|
|
19
|
+
listReports(dir: string): CrashReport[];
|
|
20
|
+
copy(from: string, to: string): void;
|
|
21
|
+
write(path: string, text: string): void;
|
|
22
|
+
};
|
|
23
|
+
export type NativeLogDeps = {
|
|
24
|
+
exec: Exec;
|
|
25
|
+
/** path of the `adb` binary */
|
|
26
|
+
adb: string;
|
|
27
|
+
fs: NativeFs;
|
|
28
|
+
/** `~/Library/Logs/DiagnosticReports` */
|
|
29
|
+
reportsDir: string;
|
|
30
|
+
};
|
|
31
|
+
export type NativeLogWindow = {
|
|
32
|
+
/** the attempt's directory; files are written into it */
|
|
33
|
+
dir: string;
|
|
34
|
+
/** when the attempt started (ms since epoch) */
|
|
35
|
+
sinceMs: number;
|
|
36
|
+
nowMs: number;
|
|
37
|
+
timeoutMs?: number;
|
|
38
|
+
};
|
|
39
|
+
export type NativeCapture = {
|
|
40
|
+
files: string[];
|
|
41
|
+
problems: string[];
|
|
42
|
+
};
|
|
43
|
+
export declare function realNativeFs(): NativeFs;
|
|
44
|
+
export declare function realNativeLogDeps(exec: Exec, adb: string): NativeLogDeps;
|
|
45
|
+
/**
|
|
46
|
+
* `native.log` (and iOS `.ips` crash reports) for a failed attempt, read after the fact: iOS
|
|
47
|
+
* `simctl spawn log show` for the app's process over the attempt's window, Android `logcat -d` for its pid
|
|
48
|
+
* plus the crash buffer. Reading afterwards needs no stream to start, stop or leak. Never throws.
|
|
49
|
+
*/
|
|
50
|
+
export declare function captureNativeLog(target: NativeTarget, deps: NativeLogDeps, win: NativeLogWindow): Promise<NativeCapture>;
|
|
@@ -5,7 +5,7 @@ import type { Clock } from "./android-device";
|
|
|
5
5
|
import { type UiadClient } from "./uiad-client";
|
|
6
6
|
import { launchUiad, type PushDex } from "./uiad-device";
|
|
7
7
|
/**
|
|
8
|
-
* The driver's hold on `warden-uiad`: started when the first call that can use it arrives (a
|
|
8
|
+
* The driver's hold on `warden-uiad`: started when the first call that can use it arrives (a spec that only opens
|
|
9
9
|
* links never pays for it), reused for every call after, and given up on, for this driver, when it cannot be made to
|
|
10
10
|
* work. "Given up" means the driver goes on with `uiautomator dump` and `input`: the server is an accelerator, never a
|
|
11
11
|
* requirement. The one failure that is not answered with a fallback is a busy UiAutomation slot, because the dump needs
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { AsyncResult } from "../../../types/src/result";
|
|
2
2
|
/**
|
|
3
3
|
* The device driver contract every layer of warden-e2e builds on: the spec API (`screen`, `device`,
|
|
4
|
-
* `app`), `warden ui`,
|
|
4
|
+
* `app`), `warden ui`, specs and the devtools facet all talk to a `Driver`. Implementations:
|
|
5
5
|
* `ios/ios-driver.ts` (warden-simd over NDJSON + `xcrun simctl`) and `android/android-driver.ts`
|
|
6
6
|
* (persistent `adb shell` + uiautomator). `driver.testing.ts` has the scripted FakeDriver for tests.
|
|
7
7
|
*
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { Exec } from "../../../../core/src/exec";
|
|
2
|
+
import { type AsyncResult } from "../../../../types/src/result";
|
|
3
|
+
import { connectSimd } from "./simd-client";
|
|
4
|
+
export type HidRecoveryDeps = {
|
|
5
|
+
exec: Exec;
|
|
6
|
+
log?: (line: string) => void;
|
|
7
|
+
/** attach to the resident helper's socket (default {@link connectSimd}) */
|
|
8
|
+
connect?: typeof connectSimd;
|
|
9
|
+
/** whether the socket file is still there (default: `existsSync`) */
|
|
10
|
+
socketExists?: (path: string) => boolean;
|
|
11
|
+
sleep?: (ms: number) => Promise<void>;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Stops the resident `warden-simd` daemon on `socketPath` (the warm one `warden e2e` started): it is bound to the
|
|
15
|
+
* boot that just ended, and the reopened driver would otherwise attach to it and hit `E_HID` again. Asks it to
|
|
16
|
+
* `shutdown` and waits for the socket to go; with no daemon there (a stdio helper dies with its driver) nothing
|
|
17
|
+
* is done. The next driver open spawns a helper on the new boot.
|
|
18
|
+
*/
|
|
19
|
+
export declare function stopResidentHelper(deps: HidRecoveryDeps, socketPath: string): AsyncResult<"stopped" | "none">;
|
|
20
|
+
/**
|
|
21
|
+
* The error of a tap/swipe/type on a sim whose HID connection is gone (wrapped as `ACTION_FAILED` or `DRIVER`), or
|
|
22
|
+
* of a helper that finds the sim shut down: quitting Device Hub normally shuts down every booted sim (DHW-0 V5), and
|
|
23
|
+
* the next call then fails `E_NOT_BOOTED`.
|
|
24
|
+
*/
|
|
25
|
+
export declare function isHidLost(error: unknown): boolean;
|
|
26
|
+
/**
|
|
27
|
+
* Shuts the sim down and boots it again: `bootstatus -b` boots a shut-down sim itself and waits for it, and opens
|
|
28
|
+
* no Simulator.app window (the same windowless path warden claims use). The caller reconnects the helper and
|
|
29
|
+
* relaunches the app.
|
|
30
|
+
*/
|
|
31
|
+
export declare function rebootSim(deps: HidRecoveryDeps, udid: string): AsyncResult<void>;
|
|
32
|
+
/**
|
|
33
|
+
* Runs `WARDEN_AFTER_REBOOT` (`{udid}` replaced) once the sim is back — `warden` uses it to reopen the sim's Device
|
|
34
|
+
* Hub window and tile again. Best effort: a failure is logged, never raised, and never fails the test.
|
|
35
|
+
*/
|
|
36
|
+
export declare function runAfterReboot(deps: HidRecoveryDeps, argv: readonly string[] | undefined, udid: string): Promise<void>;
|
|
@@ -43,6 +43,17 @@ export declare const statusBarClearArgv: (udid: string) => string[];
|
|
|
43
43
|
* (never interpolated into the script): `printf %s "$1" | xcrun simctl pbcopy "$2"`.
|
|
44
44
|
*/
|
|
45
45
|
export declare function pbcopyArgv(udid: string, text: string): string[];
|
|
46
|
+
/**
|
|
47
|
+
* The same text through `devicectl device pasteboard copy`, the CoreDevice side of the simulator pasteboard.
|
|
48
|
+
* On Xcode 27.1 (iOS 26.5 runtime) `simctl pbcopy` exits 0 without changing the pasteboard, which then still
|
|
49
|
+
* holds whatever CoreDevice's `dtpasteboardd` last mirrored from the Mac; `devicectl` writes it in ~350 ms.
|
|
50
|
+
*/
|
|
51
|
+
export declare function devicectlPbcopyArgv(udid: string, text: string): string[];
|
|
52
|
+
/** The channels {@link createSimctl}'s `pbcopy` tries, in order; each write is confirmed with `simctl pbpaste`. */
|
|
53
|
+
export declare const PBCOPY_CHANNELS: readonly {
|
|
54
|
+
name: string;
|
|
55
|
+
argv: (udid: string, text: string) => string[];
|
|
56
|
+
}[];
|
|
46
57
|
/** `launchctl list` rows are `PID STATUS LABEL`; PID is `-` for loaded-but-not-running jobs. */
|
|
47
58
|
export declare function parseRunning(launchctlOutput: string, bundleId: string): boolean;
|
|
48
59
|
/** Map a failed simctl run to a `DriverError`. `appCommand` marks launch-like calls where "not installed" means the app, not the device. */
|
|
@@ -72,6 +83,10 @@ export type Simctl = {
|
|
|
72
83
|
clearData(bundleId: string, opts?: {
|
|
73
84
|
keychain?: boolean;
|
|
74
85
|
}): DriverResult<void>;
|
|
86
|
+
/**
|
|
87
|
+
* Puts `text` on the simulator pasteboard and confirms it with `pbpaste` before answering, trying every
|
|
88
|
+
* {@link PBCOPY_CHANNELS} entry: `E_IO` when none took, so a paste never lands other text.
|
|
89
|
+
*/
|
|
75
90
|
pbcopy(text: string): DriverResult<void>;
|
|
76
91
|
pbpaste(): DriverResult<string>;
|
|
77
92
|
appearance(set?: "light" | "dark"): DriverResult<"light" | "dark">;
|
|
@@ -4,7 +4,7 @@ import type { AlertInfo, Driver, DriverResult, Platform, Snapshot } from "./driv
|
|
|
4
4
|
* does not walk the screen again (`isVisible()` then `text()` was two snapshots; on Android each one is a ~1.9 s
|
|
5
5
|
* `uiautomator dump`).
|
|
6
6
|
*
|
|
7
|
-
* It is a `Driver` wrapper, because every layer that can change the screen (`screen`, `app`, `device`,
|
|
7
|
+
* It is a `Driver` wrapper, because every layer that can change the screen (`screen`, `app`, `device`, specs) shares
|
|
8
8
|
* the one driver: whatever calls it, an input moves the *generation* on and the kept snapshot dies with the old one.
|
|
9
9
|
* A snapshot is also dropped once it is older than the platform's max age, input or not (the app can change by itself).
|
|
10
10
|
* Which reads may reuse one is the caller's call (`observe({ reuse })`): only the first look of a wait does, so a
|
package/types/e2e/src/env.d.ts
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import type { Result } from "../../types/src/result";
|
|
2
|
-
/**
|
|
2
|
+
/**
|
|
3
|
+
* The env names `warden e2e` writes and the spec process reads (`apps/cli` imports these). `WARDEN_HEADLESS` and
|
|
4
|
+
* `WARDEN_WORKER` are also set for the spec process but are for the spec's own tools (`process.env`): nothing here
|
|
5
|
+
* reads them, so a bad value cannot fail spec-env validation.
|
|
6
|
+
*/
|
|
3
7
|
export declare const ENV_NAMES: {
|
|
4
8
|
readonly udid: "WARDEN_UDID";
|
|
5
9
|
readonly deviceName: "WARDEN_DEVICE_NAME";
|
|
@@ -12,15 +16,24 @@ export declare const ENV_NAMES: {
|
|
|
12
16
|
readonly timeout: "WARDEN_E2E_TIMEOUT";
|
|
13
17
|
readonly actionTimeout: "WARDEN_E2E_ACTION_TIMEOUT";
|
|
14
18
|
readonly assertionTimeout: "WARDEN_E2E_ASSERTION_TIMEOUT";
|
|
15
|
-
readonly
|
|
19
|
+
readonly specsDir: "WARDEN_E2E_SPECS_DIR";
|
|
16
20
|
readonly helperDir: "WARDEN_HELPER_DIR";
|
|
17
21
|
readonly simd: "WARDEN_SIMD";
|
|
18
22
|
readonly simdSocket: "WARDEN_SIMD_SOCKET";
|
|
19
|
-
readonly headless: "WARDEN_HEADLESS";
|
|
20
23
|
readonly logTimestamps: "WARDEN_LOG_TIMESTAMPS";
|
|
21
|
-
|
|
24
|
+
/**
|
|
25
|
+
* JSON argv (`{udid}` is replaced) the session runs after it rebooted the sim following a lost HID connection;
|
|
26
|
+
* `warden` sets it for non-headless runs so the sim's viewer window comes back (packages/e2e cannot import apps/cli)
|
|
27
|
+
*/
|
|
28
|
+
readonly afterReboot: "WARDEN_AFTER_REBOOT";
|
|
22
29
|
/** the `warden` that started this process: the preload refuses to run when it is not this package's version */
|
|
23
30
|
readonly version: "WARDEN_VERSION";
|
|
31
|
+
/** JSON argv that re-runs the `warden` that started this process (`device.perf` shells out to it) */
|
|
32
|
+
readonly bin: "WARDEN_BIN";
|
|
33
|
+
/** `warden e2e --perf`: the capture flags (`all`, or a comma list like `react,js`) every attempt records */
|
|
34
|
+
readonly perf: "WARDEN_PERF";
|
|
35
|
+
/** JSON of the suite's `perf.budgets`; a breach fails the test with `PERF_BUDGET` */
|
|
36
|
+
readonly perfBudgets: "WARDEN_PERF_BUDGETS";
|
|
24
37
|
};
|
|
25
38
|
export declare const DEFAULT_TIMEOUT_MS = 120000;
|
|
26
39
|
export declare const DEFAULT_ACTION_TIMEOUT_MS = 30000;
|
|
@@ -39,18 +52,22 @@ export type E2eEnv = {
|
|
|
39
52
|
timeoutMs: number;
|
|
40
53
|
actionTimeoutMs: number;
|
|
41
54
|
assertionTimeoutMs: number;
|
|
42
|
-
|
|
55
|
+
specsDir?: string;
|
|
43
56
|
helperDir?: string;
|
|
44
57
|
simd?: string;
|
|
45
58
|
simdSocket?: string;
|
|
46
|
-
/** nothing in this package reads it: kept for specs and helpers that honour a headless run (`guides/headless`) */
|
|
47
|
-
headless: boolean;
|
|
48
59
|
/** warden already stamps the log lines: the spec reporter prints no clock of its own */
|
|
49
60
|
logTimestamps: boolean;
|
|
50
|
-
/**
|
|
51
|
-
|
|
61
|
+
/** `WARDEN_AFTER_REBOOT`: argv run after a HID-loss reboot, `{udid}` not yet replaced (see `runAfterReboot`) */
|
|
62
|
+
afterReboot?: string[];
|
|
52
63
|
/** `WARDEN_VERSION`: the warden that started this run */
|
|
53
64
|
wardenVersion?: string;
|
|
65
|
+
/** `WARDEN_BIN`: argv of the `warden` that started this run */
|
|
66
|
+
wardenBin?: string[];
|
|
67
|
+
/** `WARDEN_PERF`: capture names every attempt records (`all`, `react`, `js`…) */
|
|
68
|
+
perf?: string[];
|
|
69
|
+
/** `WARDEN_PERF_BUDGETS`: raw JSON of the suite's budgets */
|
|
70
|
+
perfBudgets?: string;
|
|
54
71
|
};
|
|
55
72
|
/**
|
|
56
73
|
* Reads the spec env contract. `undefined` data = not running under warden (no `WARDEN_UDID`); an
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ErrorResult } from "../../types/src/result";
|
|
2
|
-
export type E2eErrorCode = "LOCATOR_NOT_FOUND" | "LOCATOR_AMBIGUOUS" | "INVALID_LOCATOR" | "ACTION_FAILED" | "TIMEOUT" | "NOT_ACTIONABLE" | "INVALID_ARGUMENT" | "SESSION" | "DRIVER" | "ASSERTION";
|
|
2
|
+
export type E2eErrorCode = "LOCATOR_NOT_FOUND" | "LOCATOR_AMBIGUOUS" | "INVALID_LOCATOR" | "ACTION_FAILED" | "TIMEOUT" | "NOT_ACTIONABLE" | "INVALID_ARGUMENT" | "SESSION" | "DRIVER" | "ASSERTION" | "PERF_BUDGET";
|
|
3
3
|
/** Plain shape of a spec error: what `e2eErr()` accepts and what a `Result` carries. */
|
|
4
4
|
export type E2eErrorShape = {
|
|
5
5
|
code: E2eErrorCode;
|
|
@@ -11,8 +11,12 @@ export type FixtureSession = {
|
|
|
11
11
|
readonly out: string;
|
|
12
12
|
/** the running attempt's deadline / cancel signal; every driver call honours it */
|
|
13
13
|
readonly signal?: AbortSignal;
|
|
14
|
-
/** `<out>/<
|
|
14
|
+
/** `<out>/<specId>/<title-slug>-<hash6>` of the running attempt */
|
|
15
15
|
readonly attemptDir?: string;
|
|
16
|
+
/** the running attempt's recorded files (`device.perf` adds its `perf/` ones) */
|
|
17
|
+
readonly attempt?: {
|
|
18
|
+
readonly files: Set<string>;
|
|
19
|
+
};
|
|
16
20
|
/** the running attempt's `trace.jsonl` */
|
|
17
21
|
readonly trace?: {
|
|
18
22
|
write(event: TraceEvent): void;
|
package/types/e2e/src/index.d.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
export type { OpenOptions, OpenWaitUntil } from "./app";
|
|
2
2
|
export { E2eError } from "./errors";
|
|
3
3
|
export { expect } from "./expect/expect";
|
|
4
|
+
export type { PerfBreach, PerfBudgets } from "./perf/budgets";
|
|
5
|
+
export type { DevicePerf } from "./perf/device-perf";
|
|
6
|
+
export type { PerfMetrics, PerfReport, PerfStartOptions } from "./perf/perf.types";
|
|
4
7
|
export type { App, Device, Screen, SystemAlerts, TestFixtures } from "./runner/fixtures.types";
|
|
5
8
|
export { afterAll, afterEach, beforeAll, beforeEach, describe, test } from "./runner/test";
|
|
6
9
|
export type { Locator, LocatorReading, PinchOptions, RotateOptions, TapOptions, TapUntil } from "./screen/locator";
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { PerfReport } from "./perf.types";
|
|
2
|
+
/** Ceilings a perf report must stay under; every field is optional and lower-is-better. */
|
|
3
|
+
export type PerfBudgets = {
|
|
4
|
+
/** cold start, ms */
|
|
5
|
+
startupColdMs?: number;
|
|
6
|
+
/** share of janky frames, percent */
|
|
7
|
+
jankPct?: number;
|
|
8
|
+
/** the slowest React commit, ms */
|
|
9
|
+
maxCommitMs?: number;
|
|
10
|
+
/** 95th percentile app CPU, percent */
|
|
11
|
+
p95CpuPct?: number;
|
|
12
|
+
/** peak memory (PSS on Android, RSS on iOS), MB */
|
|
13
|
+
peakMemoryMb?: number;
|
|
14
|
+
};
|
|
15
|
+
export declare const BUDGET_KEYS: readonly ["startupColdMs", "jankPct", "maxCommitMs", "p95CpuPct", "peakMemoryMb"];
|
|
16
|
+
export type BudgetKey = (typeof BUDGET_KEYS)[number];
|
|
17
|
+
/** One metric over its ceiling. */
|
|
18
|
+
export type PerfBreach = {
|
|
19
|
+
metric: BudgetKey;
|
|
20
|
+
value: number;
|
|
21
|
+
budget: number;
|
|
22
|
+
};
|
|
23
|
+
/** What the report measured for `metric`, or `undefined` when its capture did not run. */
|
|
24
|
+
export declare function measured(report: Pick<PerfReport, "metrics">, metric: BudgetKey): number | undefined;
|
|
25
|
+
/**
|
|
26
|
+
* Budgets set in `overrides` replace the same key of `base` (config); a metric the report has no number for (its
|
|
27
|
+
* capture did not run) cannot breach.
|
|
28
|
+
*/
|
|
29
|
+
export declare function checkBudgets(report: Pick<PerfReport, "metrics">, base: PerfBudgets | undefined, overrides?: PerfBudgets): PerfBreach[];
|
|
30
|
+
/** The failure text: each breach, then where the full report is. */
|
|
31
|
+
export declare function describeBreaches(breaches: readonly PerfBreach[], reportPath: string | undefined): string;
|
|
32
|
+
/** Parses the `WARDEN_PERF_BUDGETS` JSON warden writes; unknown keys and non-numbers are dropped. */
|
|
33
|
+
export declare function parseBudgets(raw: string | undefined): PerfBudgets;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { type PerfBudgets } from "./budgets";
|
|
2
|
+
import type { PerfReport, PerfStartOptions } from "./perf.types";
|
|
3
|
+
/** One `warden` invocation: argv (without the warden command itself) in, exit code and streams out. */
|
|
4
|
+
export type PerfExec = (argv: readonly string[]) => Promise<{
|
|
5
|
+
code: number;
|
|
6
|
+
stdout: string;
|
|
7
|
+
stderr: string;
|
|
8
|
+
}>;
|
|
9
|
+
/** Subdirectory of the attempt dir a perf session's report is copied to. */
|
|
10
|
+
export declare const PERF_DIR = "perf";
|
|
11
|
+
export type DevicePerf = {
|
|
12
|
+
/** opens a perf session on this device; one per device at a time */
|
|
13
|
+
start(options?: PerfStartOptions): Promise<{
|
|
14
|
+
sessionId: string;
|
|
15
|
+
}>;
|
|
16
|
+
/** drops a label on the session timeline; commits, hangs and jank are tagged with the last mark before them */
|
|
17
|
+
mark(label: string): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Ends the session and returns its report (`reportPath` is the copy under the attempt's `perf/`). Throws
|
|
20
|
+
* `PERF_BUDGET` when a metric is over the suite's budget (`warden.config.ts` `perf.budgets`) or `budgets`, which
|
|
21
|
+
* replaces the suite's per key.
|
|
22
|
+
*/
|
|
23
|
+
stop(options?: {
|
|
24
|
+
budgets?: PerfBudgets;
|
|
25
|
+
}): Promise<PerfReport>;
|
|
26
|
+
/** a session this device opened and has not stopped */
|
|
27
|
+
readonly active: boolean;
|
|
28
|
+
};
|
|
29
|
+
export type DevicePerfDeps = {
|
|
30
|
+
exec: PerfExec;
|
|
31
|
+
udid: string;
|
|
32
|
+
/** budgets of the suite (`WARDEN_PERF_BUDGETS`) */
|
|
33
|
+
budgets: PerfBudgets;
|
|
34
|
+
/** the running attempt's dir; absent outside an attempt (the report stays in the session dir) */
|
|
35
|
+
attemptDir: () => string | undefined;
|
|
36
|
+
/** records a file written into the attempt dir (relative path) so `results.jsonl` lists it */
|
|
37
|
+
addFile?: (relative: string) => void;
|
|
38
|
+
/** `warden` is not reachable (no `WARDEN_BIN`) */
|
|
39
|
+
unavailable?: string;
|
|
40
|
+
};
|
|
41
|
+
export declare function createDevicePerf(deps: DevicePerfDeps): DevicePerf;
|
|
42
|
+
/** `--perf=react,js` (or `all`) as the options `perf.start` takes; an unknown name is ignored, none named = warden's default set. */
|
|
43
|
+
export declare function perfStartFor(names: readonly string[]): PerfStartOptions;
|
|
44
|
+
/** Runs the `warden` that started the spec process (`WARDEN_BIN`). */
|
|
45
|
+
export declare function spawnWarden(bin: readonly string[] | undefined): PerfExec;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/** The capture switches `device.perf.start` takes (the `warden perf start` flags of the same name). */
|
|
2
|
+
export type PerfStartOptions = {
|
|
3
|
+
js?: boolean;
|
|
4
|
+
react?: boolean;
|
|
5
|
+
native?: boolean | "perfetto" | "simpleperf";
|
|
6
|
+
metrics?: boolean;
|
|
7
|
+
frames?: boolean;
|
|
8
|
+
logs?: boolean;
|
|
9
|
+
all?: boolean;
|
|
10
|
+
/** stop the app, arm the captures, launch it: startup and early JS/React are recorded */
|
|
11
|
+
relaunch?: boolean;
|
|
12
|
+
/** like `relaunch`, but wipe the app's data first */
|
|
13
|
+
reset?: boolean;
|
|
14
|
+
/** hard cap on the session; warden's default is 30 s, the maximum 5 min */
|
|
15
|
+
maxDurationMs?: number;
|
|
16
|
+
};
|
|
17
|
+
/** The numbers of a report, each lower-is-better (the `metrics` of `report.json`). */
|
|
18
|
+
export type PerfMetrics = {
|
|
19
|
+
startupColdMs?: number;
|
|
20
|
+
startupWarmMs?: number;
|
|
21
|
+
jankPct?: number;
|
|
22
|
+
fps?: number;
|
|
23
|
+
cpuP95Pct?: number;
|
|
24
|
+
peakPssKb?: number;
|
|
25
|
+
peakRssKb?: number;
|
|
26
|
+
/** the slowest React commit */
|
|
27
|
+
topCommitMs?: number;
|
|
28
|
+
hangCount?: number;
|
|
29
|
+
hangTotalMs?: number;
|
|
30
|
+
topFunctions: Array<{
|
|
31
|
+
name: string;
|
|
32
|
+
selfPct: number;
|
|
33
|
+
}>;
|
|
34
|
+
};
|
|
35
|
+
/** `report.json` of a perf session, as far as a spec reads it (the full document is at `reportPath`). */
|
|
36
|
+
export type PerfReport = {
|
|
37
|
+
version: 1;
|
|
38
|
+
sessionId: string;
|
|
39
|
+
app?: string;
|
|
40
|
+
durationMs: number;
|
|
41
|
+
stopReason?: string;
|
|
42
|
+
banners: string[];
|
|
43
|
+
summary: string[];
|
|
44
|
+
findings: Array<{
|
|
45
|
+
kind: string;
|
|
46
|
+
severity: string;
|
|
47
|
+
costMs: number;
|
|
48
|
+
evidence: string;
|
|
49
|
+
fix: string;
|
|
50
|
+
}>;
|
|
51
|
+
metrics: PerfMetrics;
|
|
52
|
+
marks: Array<{
|
|
53
|
+
label: string;
|
|
54
|
+
atMs: number;
|
|
55
|
+
}>;
|
|
56
|
+
next: string[];
|
|
57
|
+
/** where `report.json` is (inside the attempt's `perf/` dir when the spec runs under `warden e2e`) */
|
|
58
|
+
reportPath?: string;
|
|
59
|
+
};
|
|
@@ -14,8 +14,8 @@ export type PreloadDeps = {
|
|
|
14
14
|
exit: (code: number) => void;
|
|
15
15
|
};
|
|
16
16
|
/**
|
|
17
|
-
* The protocol handshake: `warden e2e` and this package version in lock-step. A mismatch
|
|
18
|
-
*
|
|
17
|
+
* The protocol handshake: `warden e2e` and this package version in lock-step. A mismatch refuses the run (exit 2)
|
|
18
|
+
* with the install fix spelled out. Nothing to compare without `WARDEN_VERSION`
|
|
19
19
|
* (a by-hand run) or from the workspace sources (not a published build).
|
|
20
20
|
*/
|
|
21
21
|
export declare function versionMismatch(wardenVersion: string | undefined, e2eVersion: string): string | undefined;
|
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
export declare function slug(text: string, fallback?: string): string;
|
|
3
3
|
/** First 6 hex chars of a sha1 over the parts: keeps equal slugs from colliding. */
|
|
4
4
|
export declare function hash6(...parts: string[]): string;
|
|
5
|
-
/** `<out>/<
|
|
6
|
-
export declare function attemptDir(out: string,
|
|
5
|
+
/** `<out>/<specId>/<title-slug>-<hash6>/` for one test attempt. */
|
|
6
|
+
export declare function attemptDir(out: string, specId: string, title: readonly string[], hash: string): string;
|
|
7
7
|
/** `<dir>/screenshots/NNN-name.png` (3-digit, 1-based order). */
|
|
8
8
|
export declare function screenshotPath(dir: string, n: number, name: string): string;
|
|
9
9
|
/**
|
|
10
|
-
* Stable per-file id: the path under `
|
|
11
|
-
* `auth/sign-in`), segments slugged. A file outside `
|
|
10
|
+
* Stable per-file id: the path under `specsDir` without extension (`auth/sign-in.e2e.ts` →
|
|
11
|
+
* `auth/sign-in`), segments slugged. A file outside `specsDir` (or no `specsDir`) uses its basename.
|
|
12
12
|
*/
|
|
13
|
-
export declare function
|
|
13
|
+
export declare function specIdOf(file: string, specsDir?: string): string;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { DevtoolsFacet } from "../devtools/devtools.types";
|
|
2
|
+
import { type NativeLogDeps, type NativeTarget } from "../devtools/native-log";
|
|
2
3
|
import type { Driver } from "../driver/driver.types";
|
|
4
|
+
import type { E2eEnv } from "../env";
|
|
3
5
|
export declare const FAILURE_PNG = "failure.png";
|
|
4
6
|
export declare const SCREEN_AT_FAILURE = "screen-at-failure.txt";
|
|
5
7
|
export declare const CONSOLE_JSONL = "console.jsonl";
|
|
@@ -26,3 +28,13 @@ export declare function captureFailure(driver: Driver, dir: string, timeoutMs?:
|
|
|
26
28
|
* {@link captureFailure}: never throws, gives up after `timeoutMs` (the network drain talks to the page).
|
|
27
29
|
*/
|
|
28
30
|
export declare function captureDevtoolsArtifacts(facet: DevtoolsFacet, dir: string, timeoutMs?: number, budget?: CaptureBudget): Promise<FailureCapture>;
|
|
31
|
+
/** The app a failed attempt ran, from the run's env. */
|
|
32
|
+
export declare function nativeTarget(env: Pick<E2eEnv, "platform" | "udid" | "bundleId">): NativeTarget;
|
|
33
|
+
/**
|
|
34
|
+
* `native.log` (+ iOS `.ips` crash reports) of the failed attempt, see {@link captureNativeLog}. Best effort
|
|
35
|
+
* like {@link captureFailure}: never throws, limited by `timeoutMs` and what `budget` has left.
|
|
36
|
+
*/
|
|
37
|
+
export declare function captureNativeArtifacts(target: NativeTarget, deps: NativeLogDeps, win: {
|
|
38
|
+
dir: string;
|
|
39
|
+
sinceMs: number;
|
|
40
|
+
}, timeoutMs?: number, budget?: CaptureBudget): Promise<FailureCapture>;
|
|
@@ -46,7 +46,7 @@ export type FixtureFactory = (ctx: FixtureContext) => BaseFixtures | Promise<Bas
|
|
|
46
46
|
/** The running attempt: where steps, screenshots and failure files go. Read it through the session. */
|
|
47
47
|
export type AttemptContext = {
|
|
48
48
|
readonly signal: AbortSignal;
|
|
49
|
-
/** `<out>/<
|
|
49
|
+
/** `<out>/<specId>/<title-slug>-<hash6>/` */
|
|
50
50
|
readonly dir: string;
|
|
51
51
|
/** spec name tagging step lines */
|
|
52
52
|
readonly short: string;
|
|
@@ -21,8 +21,10 @@ export type ResolvedUse = {
|
|
|
21
21
|
launch: LaunchMode;
|
|
22
22
|
systemAlerts: SystemAlerts;
|
|
23
23
|
};
|
|
24
|
-
/**
|
|
24
|
+
/** Scope hooks are stored against the base fixtures; `test.extend(...).beforeEach` hooks are narrowed back to theirs by the runner (bun's own hooks take no arguments). */
|
|
25
25
|
export type Hook = (fixtures: BaseFixtures) => void | Promise<void>;
|
|
26
|
+
/** Marks `hook` as belonging to tests built from `fixtures` (entries compared by identity, in definition order). */
|
|
27
|
+
export declare function ownHook(hook: Hook, fixtures: readonly object[]): Hook;
|
|
26
28
|
export type Scope = {
|
|
27
29
|
/** describe titles from the root down to this scope */
|
|
28
30
|
readonly title: readonly string[];
|
|
@@ -45,8 +47,8 @@ export declare function createScope(title: string, serial: boolean): Scope;
|
|
|
45
47
|
export declare function runInScope(scope: Scope, body: () => void): void;
|
|
46
48
|
/** Root first, `scope` last. */
|
|
47
49
|
export declare function scopeChain(scope: Scope): Scope[];
|
|
48
|
-
/** `beforeEach`: outermost first. `afterEach`: innermost first. */
|
|
49
|
-
export declare function hooksFor(scope: Scope, kind: "beforeEach" | "afterEach"): Hook[];
|
|
50
|
+
/** `beforeEach`: outermost first. `afterEach`: innermost first. Hooks owned by another fixture set are left out. */
|
|
51
|
+
export declare function hooksFor(scope: Scope, kind: "beforeEach" | "afterEach", fixtures?: readonly object[]): Hook[];
|
|
50
52
|
/** Session defaults (env contract) under the nearest `test.use`, under the test's own `timeout`. */
|
|
51
53
|
export declare function resolveUse(scope: Scope, defaults: Pick<ResolvedUse, "timeoutMs" | "actionTimeoutMs" | "assertionTimeoutMs">, testTimeout?: number): ResolvedUse;
|
|
52
54
|
/** Timeouts known at collection time: the env when set, else the contract defaults. */
|
|
@@ -31,6 +31,13 @@ export type TestApi<F extends object> = TestCallable<F> & {
|
|
|
31
31
|
skipIf(condition: boolean): TestCallable<F>;
|
|
32
32
|
/** applies to every test collected in the current scope (file or `describe`) */
|
|
33
33
|
use(options: UseOptions): void;
|
|
34
|
+
/**
|
|
35
|
+
* Like the top-level `beforeEach`, typed with this `test`'s fixtures (the ones `extend` added too). It runs only
|
|
36
|
+
* for tests made with this `test` or one extended from it, not for the base `test`'s.
|
|
37
|
+
*/
|
|
38
|
+
beforeEach(hook: (fixtures: F) => void | Promise<void>): void;
|
|
39
|
+
/** `beforeEach`'s counterpart: runs after each such test, even when it failed. */
|
|
40
|
+
afterEach(hook: (fixtures: F) => void | Promise<void>): void;
|
|
34
41
|
/** a `test` whose bodies also receive `Extra` */
|
|
35
42
|
extend<Extra extends object>(defs: FixtureDefs<F, Extra>): TestApi<F & Extra>;
|
|
36
43
|
};
|
|
@@ -21,7 +21,7 @@ export declare function focusPoint(frame: Rect): Point;
|
|
|
21
21
|
export declare function swipeVector(direction: SwipeDirection, rect: Rect, momentum?: Momentum): SwipeVector;
|
|
22
22
|
/**
|
|
23
23
|
* The one swipe geometry: the finger path inside `rect`, centred, covering `fraction` (0-1) of it, that scrolls the
|
|
24
|
-
* content `direction` (`down` reveals what is below, so the finger travels up). Spec swipes,
|
|
24
|
+
* content `direction` (`down` reveals what is below, so the finger travels up). Spec swipes, spec `swipe`/`scroll` steps
|
|
25
25
|
* and `warden ui scroll` use it as is; `warden ui gesture swipe <dir>` names the finger's direction instead and flips it
|
|
26
26
|
* (`fingerPath` in the CLI).
|
|
27
27
|
*/
|