@quiesco/core 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/bundle-check.d.ts +16 -0
  2. package/dist/bundle-check.d.ts.map +1 -0
  3. package/dist/bundle-check.js +84 -0
  4. package/dist/bundle-check.js.map +1 -0
  5. package/dist/config.d.ts +25 -0
  6. package/dist/config.d.ts.map +1 -1
  7. package/dist/config.js.map +1 -1
  8. package/dist/devices/android.d.ts.map +1 -1
  9. package/dist/devices/android.js +1 -0
  10. package/dist/devices/android.js.map +1 -1
  11. package/dist/devices/ios-runner.d.ts +24 -0
  12. package/dist/devices/ios-runner.d.ts.map +1 -1
  13. package/dist/devices/ios-runner.js +27 -0
  14. package/dist/devices/ios-runner.js.map +1 -1
  15. package/dist/devices/ios.d.ts +7 -1
  16. package/dist/devices/ios.d.ts.map +1 -1
  17. package/dist/devices/ios.js +22 -2
  18. package/dist/devices/ios.js.map +1 -1
  19. package/dist/driver.d.ts +22 -2
  20. package/dist/driver.d.ts.map +1 -1
  21. package/dist/driver.js +35 -1
  22. package/dist/driver.js.map +1 -1
  23. package/dist/index.d.ts +1 -1
  24. package/dist/index.d.ts.map +1 -1
  25. package/dist/lifecycle.d.ts +13 -0
  26. package/dist/lifecycle.d.ts.map +1 -1
  27. package/dist/lifecycle.js +34 -3
  28. package/dist/lifecycle.js.map +1 -1
  29. package/dist/repeated-failures.d.ts +43 -0
  30. package/dist/repeated-failures.d.ts.map +1 -0
  31. package/dist/repeated-failures.js +111 -0
  32. package/dist/repeated-failures.js.map +1 -0
  33. package/dist/session.d.ts +11 -1
  34. package/dist/session.d.ts.map +1 -1
  35. package/dist/session.js +47 -3
  36. package/dist/session.js.map +1 -1
  37. package/ios-runner/Runner/SystemUIServer.swift +17 -0
  38. package/package.json +2 -2
  39. package/src/bundle-check.ts +83 -0
  40. package/src/config.ts +28 -1
  41. package/src/devices/android.ts +1 -0
  42. package/src/devices/ios-runner.ts +33 -0
  43. package/src/devices/ios.ts +23 -3
  44. package/src/driver.ts +50 -2
  45. package/src/index.ts +1 -1
  46. package/src/lifecycle.ts +38 -3
  47. package/src/repeated-failures.ts +114 -0
  48. package/src/session.ts +47 -4
@@ -0,0 +1,83 @@
1
+ /**
2
+ * Stale JS bundles (issue #3): when the app's bundle does not follow source changes (Metro started with
3
+ * `CI` set, or a one-off `expo export`), edits made after it was built never reach the app, and nothing
4
+ * said so. The agent reports where its bundle comes from (`hello.bundle`); this compares that with the
5
+ * project's source files.
6
+ */
7
+ import fs from 'node:fs';
8
+ import path from 'node:path';
9
+
10
+ import type { BundleInfo } from '@quiesco/protocol';
11
+
12
+ const SOURCE = /\.(?:[cm]?[jt]sx?)$/;
13
+ /** Not app source: dependencies, generated output, native projects, test artifacts. */
14
+ const SKIP_DIRS = new Set(['node_modules', '.git', '.expo', 'ios', 'android', 'dist', 'web-build', 'build', 'artifacts', 'coverage', 'reports']);
15
+ /**
16
+ * Test code is not in the app's bundle, so editing a step file needs no rebuild: the usual test
17
+ * directories, test files, and the tool configs at the project root (quiesco.config.mts, cucumber.mjs, …).
18
+ */
19
+ const TEST_DIRS = new Set(['features', 'e2e', '__tests__', '__mocks__', 'test', 'tests', 'step_definitions']);
20
+ const TEST_FILE = /\.(?:test|spec|steps?)\.[cm]?[jt]sx?$/;
21
+ const ROOT_TOOL_FILE = /^(?:[\w-]+\.config|cucumber|\.?[\w-]*rc)\.[cm]?[jt]s$/;
22
+ /** Bounds the walk in a large project; the newest of these is enough to tell. */
23
+ const MAX_FILES = 5000;
24
+
25
+ /** The most recently changed source file under `root`, or null. */
26
+ export function newestSource(root: string): { file: string; mtimeMs: number } | null {
27
+ let newest: { file: string; mtimeMs: number } | null = null;
28
+ let seen = 0;
29
+ const walk = (dir: string): void => {
30
+ let entries: fs.Dirent[];
31
+ try {
32
+ entries = fs.readdirSync(dir, { withFileTypes: true });
33
+ } catch {
34
+ return;
35
+ }
36
+ for (const e of entries) {
37
+ if (seen >= MAX_FILES) return;
38
+ const p = path.join(dir, e.name);
39
+ if (e.isDirectory()) {
40
+ if (!SKIP_DIRS.has(e.name) && !TEST_DIRS.has(e.name) && !e.name.startsWith('.')) walk(p);
41
+ } else if (e.isFile() && SOURCE.test(e.name) && !e.name.endsWith('.d.ts') && !TEST_FILE.test(e.name) && !(dir === root && ROOT_TOOL_FILE.test(e.name))) {
42
+ seen++;
43
+ const mtimeMs = fs.statSync(p, { throwIfNoEntry: false })?.mtimeMs;
44
+ if (mtimeMs !== undefined && (newest === null || mtimeMs > newest.mtimeMs)) newest = { file: p, mtimeMs };
45
+ }
46
+ }
47
+ };
48
+ walk(root);
49
+ return newest;
50
+ }
51
+
52
+ const time = (ms: number): string => new Date(ms).toTimeString().slice(0, 8);
53
+
54
+ /** `bundle built 10:02:11, Metro watching files` for the connection log line. */
55
+ export function describeBundle(bundle: BundleInfo | undefined): string {
56
+ if (bundle === undefined) return 'bundle unknown (no withQuiesco() in metro.config.js)';
57
+ if (bundle.watching) return `Metro started ${time(bundle.builtAt)}, watching files`;
58
+ return `bundle built ${time(bundle.builtAt)}, ${bundle.reason === 'CI' ? 'Metro not watching files (CI is set)' : 'a one-off export'}`;
59
+ }
60
+
61
+ /**
62
+ * A warning when the bundle cannot have the project's latest changes, else null.
63
+ * - Metro started with `CI` set: always (unless this host also runs in CI, where that is intended), since
64
+ * later edits will not arrive either.
65
+ * - Any bundle that does not follow changes: when a source file is newer than the bundle.
66
+ */
67
+ export function staleBundleWarning(bundle: BundleInfo | undefined, projectRoot: string, hostEnv: NodeJS.ProcessEnv = process.env): string | null {
68
+ if (bundle === undefined || bundle.watching) return null;
69
+ const fix =
70
+ bundle.reason === 'CI'
71
+ ? 'Start Metro without CI (EXPO_PUBLIC_QUIESCO=1 npx expo start), or restart it after each change'
72
+ : 'Build it again (e.g. EXPO_PUBLIC_QUIESCO=1 npx expo export --platform web)';
73
+ const newest = newestSource(projectRoot);
74
+ // A second of slack: the export reads files written just before it started.
75
+ if (newest !== null && newest.mtimeMs > bundle.builtAt + 1000) {
76
+ return `The app runs a JS bundle built at ${time(bundle.builtAt)}, but ${path.relative(projectRoot, newest.file)} changed at ${time(newest.mtimeMs)}: the app does not have that change (${bundle.reason === 'CI' ? 'Metro was started with CI set, so it does not watch files' : 'the bundle is a one-off export'}). ${fix}.`;
77
+ }
78
+ const hostInCi = hostEnv['CI'] !== undefined && hostEnv['CI'] !== '' && hostEnv['CI'] !== '0' && hostEnv['CI'].toLowerCase() !== 'false';
79
+ if (bundle.reason === 'CI' && !hostInCi) {
80
+ return `Metro was started with CI set, so it does not watch files: changes you make now will not reach the app. ${fix}.`;
81
+ }
82
+ return null;
83
+ }
package/src/config.ts CHANGED
@@ -159,7 +159,34 @@ export interface QuiescoConfig {
159
159
  */
160
160
  gestureSlop?: Partial<GestureSlop>;
161
161
  };
162
- reset?: { default?: 'soft' | 'hard' };
162
+ reset?: {
163
+ default?: 'soft' | 'hard';
164
+ /**
165
+ * An element that must be visible after every reset, e.g. `{ id: 'home' }`: the app is back in its
166
+ * initial state. When it is not, the reset fails with a clear message instead of the scenario's first
167
+ * step. Expo Router apps already have the route checked (issue #1); this is for any app.
168
+ */
169
+ expectVisible?: MatcherInput;
170
+ };
171
+ /**
172
+ * The Metro dev server the app loads its JS from (issue #5): `quiesco test` starts one with
173
+ * `EXPO_PUBLIC_QUIESCO=1` when the app needs it (`devClient.metroUrl`, or `apps.web.url`) and nothing
174
+ * answers there, waits until it is ready, and stops it afterwards. A running one is reused.
175
+ */
176
+ devServer?: {
177
+ /** Start Metro when it is not running. Default true. */
178
+ autoStart?: boolean;
179
+ /** The command, with `{port}` replaced. Default `npx expo start --port {port}`. */
180
+ command?: string[];
181
+ /** How long to wait for it to be ready. Default 120 s. */
182
+ timeoutMs?: number;
183
+ };
184
+ /**
185
+ * Stop after this many scenarios in a row failed the same way (same error at the same step, issue #9): the
186
+ * app is most likely stuck in a state every scenario inherits. The rest are skipped. Default 3; `false`
187
+ * runs every scenario.
188
+ */
189
+ bailOnRepeatedFailure?: number | false;
163
190
  server?: {
164
191
  /** Port of the Quiesco WebSocket server. Default 7357. */
165
192
  port?: number;
@@ -929,6 +929,7 @@ export class AndroidProvider implements DeviceProvider {
929
929
  async injectInput(input: HostInput): Promise<void> {
930
930
  switch (input.kind) {
931
931
  case 'back':
932
+ case 'goBack':
932
933
  await this.shell('input keyevent KEYCODE_BACK');
933
934
  return;
934
935
  case 'navigate':
@@ -26,6 +26,29 @@ export interface RunnerScreen {
26
26
  others: { label: string; id: string; frame: number[] }[];
27
27
  }
28
28
 
29
+ /** The app's navigation bars and their buttons (Runner/SystemUIServer.swift `/navigationBar`). */
30
+ export interface RunnerNavigationBar {
31
+ bars: { frame: number[]; buttons: { label: string; id: string; frame: number[]; hittable: boolean }[] }[];
32
+ }
33
+
34
+ /**
35
+ * The back button of the app's top navigation bar, or null (issue #4): UIKit names it `BackButton`; else
36
+ * the first hittable button in the left third of a hittable bar (a custom `headerLeft`). With nested
37
+ * navigators only the visible bars count; the lowest one on screen is the innermost stack.
38
+ */
39
+ export function backButton(nav: RunnerNavigationBar): { frame: number[] } | null {
40
+ const bars = [...nav.bars].sort((a, b) => (b.frame[1] ?? 0) - (a.frame[1] ?? 0));
41
+ for (const bar of bars) {
42
+ const named = bar.buttons.find((b) => b.hittable && b.id === 'BackButton');
43
+ if (named !== undefined) return named;
44
+ }
45
+ for (const bar of bars) {
46
+ const left = bar.buttons.find((b) => b.hittable && (b.frame[0] ?? 0) < (bar.frame[0] ?? 0) + (bar.frame[2] ?? 0) / 3);
47
+ if (left !== undefined) return left;
48
+ }
49
+ return null;
50
+ }
51
+
29
52
  /** Synthetic ids of a permission alert's buttons (they have no identifiers and follow the device language). */
30
53
  export const PERMISSION_ALLOW = 'quiesco.permission.allow';
31
54
  export const PERMISSION_DENY = 'quiesco.permission.deny';
@@ -131,6 +154,16 @@ export class IosSystemRunner {
131
154
  await this.call(() => this.request('POST', `/tap?x=${x.toFixed(1)}&y=${y.toFixed(1)}`));
132
155
  }
133
156
 
157
+ /** The navigation bars of the app `bundleId` (issue #4). */
158
+ async navigationBar(bundleId: string): Promise<RunnerNavigationBar> {
159
+ return this.call(() => this.request<RunnerNavigationBar>('GET', `/navigationBar?bundleId=${encodeURIComponent(bundleId)}`));
160
+ }
161
+
162
+ /** The interactive-pop gesture: a drag from the left edge of the screen (issue #4). */
163
+ async edgeSwipeBack(): Promise<void> {
164
+ await this.call(() => this.request('POST', '/edgeSwipeBack'));
165
+ }
166
+
134
167
  async openNotificationCenter(): Promise<void> {
135
168
  await this.call(() => this.request('POST', '/notificationCenter'));
136
169
  }
@@ -34,7 +34,7 @@ import type { Exec } from './exec.js';
34
34
  import { execCommand } from './exec.js';
35
35
  import { LogBuffer, exited, spawnProcess } from './streams.js';
36
36
  import type { ChildLike, SpawnProcess } from './streams.js';
37
- import { IosSystemRunner, PERMISSION_ALLOW, PERMISSION_DENY, dismissButton, screenNodes } from './ios-runner.js';
37
+ import { IosSystemRunner, PERMISSION_ALLOW, PERMISSION_DENY, backButton, dismissButton, screenNodes } from './ios-runner.js';
38
38
  import type { Permission } from './permissions.js';
39
39
  import { SERIAL_WORKER } from './provider.js';
40
40
  import type { ColorScheme, DeviceProvider, InAppControls, LaunchParams, LaunchUrlHandling, NotificationSpec, StateChange, StatusBarOptions, SystemNode, WorkerInfo } from './provider.js';
@@ -138,7 +138,7 @@ export interface IosProviderOptions {
138
138
  }
139
139
 
140
140
  /** What IosProvider uses of IosSystemRunner. */
141
- export type SystemRunnerLike = Pick<IosSystemRunner, 'running' | 'screen' | 'tap' | 'openNotificationCenter' | 'stop'>;
141
+ export type SystemRunnerLike = Pick<IosSystemRunner, 'running' | 'screen' | 'tap' | 'openNotificationCenter' | 'stop'> & Partial<Pick<IosSystemRunner, 'navigationBar' | 'edgeSwipeBack'>>;
142
142
 
143
143
  /** `27.0` / `iOS 27.0` / `27` → does the runtime identifier match? */
144
144
  export function runtimeMatches(runtime: string, version: string): boolean {
@@ -687,6 +687,23 @@ export class IosProvider implements DeviceProvider {
687
687
  await this.runner.tap((node.bounds.x1 + node.bounds.x2) / 2, (node.bounds.y1 + node.bounds.y2) / 2);
688
688
  }
689
689
 
690
+ /**
691
+ * `app.goBack()` on iOS (issue #4), as a user would: tap the navigation bar's back button when the app
692
+ * shows one, else the edge swipe from the left (interactive pop). Through the XCUITest runner, which sees
693
+ * the app's native navigation bar (the agent cannot: it is not a React element).
694
+ */
695
+ async goBack(): Promise<void> {
696
+ const runner = this.runner;
697
+ if (runner.navigationBar === undefined || runner.edgeSwipeBack === undefined) throw new Error('the iOS system UI runner cannot go back');
698
+ const back = backButton(await runner.navigationBar(this.bundleId));
699
+ if (back !== null) {
700
+ const [x = 0, y = 0, w = 0, h = 0] = back.frame;
701
+ await runner.tap(x + w / 2, y + h / 2);
702
+ return;
703
+ }
704
+ await runner.edgeSwipeBack();
705
+ }
706
+
690
707
  /**
691
708
  * Tap the notification titled `title`: its banner while it is shown, else in Notification Center
692
709
  * (swiped down from the top edge). Banners read "<time>, <title>, <body>".
@@ -833,8 +850,11 @@ export class IosProvider implements DeviceProvider {
833
850
  throw new QuiescoError('ACTION_NOT_SUPPORTED', 'pressBack is not available on iOS: there is no system back button', {
834
851
  action: 'pressBack',
835
852
  platform: 'ios',
836
- hint: 'tap the app\'s own back control instead (e.g. the navigation bar back button), or tag the scenario @android',
853
+ hint: 'use app.goBack(): the navigation bar back button, else the edge swipe',
837
854
  });
855
+ case 'goBack':
856
+ await this.goBack();
857
+ return;
838
858
  case 'navigate':
839
859
  throw new QuiescoError(
840
860
  'ACTION_NOT_SUPPORTED',
package/src/driver.ts CHANGED
@@ -9,10 +9,14 @@ import type {
9
9
  AgentEvents,
10
10
  AgentMethods,
11
11
  AssertionSpec,
12
+ BackdoorParams,
13
+ BackdoorResult,
14
+ Backdoors,
12
15
  ControlValue,
13
16
  Direction,
14
17
  Edge,
15
18
  ElementSummary,
19
+ JsonCompatible,
16
20
  JsonValue,
17
21
  KeyName,
18
22
  ParamsOf,
@@ -209,6 +213,23 @@ export class Driver {
209
213
  async pressBack(opts?: ActionOptions): Promise<void> {
210
214
  await this.action({ type: 'pressBack' }, opts);
211
215
  }
216
+ /**
217
+ * Go back as a user would (issue #4): Android back, browser back on the web, and on iOS the navigation
218
+ * bar's back button, else the edge swipe. In Expo Router apps it fails when the route did not change.
219
+ */
220
+ async goBack(opts?: ActionOptions): Promise<void> {
221
+ const before = await this.currentRoute();
222
+ await this.action({ type: 'goBack' }, opts);
223
+ if (before === null || opts?.sync === false) return;
224
+ const after = await this.currentRoute();
225
+ if (after === before) {
226
+ throw new QuiescoError('ASSERTION_FAILED', `goBack did not leave ${before}: there is no screen to go back to here, or the back button / swipe was not handled (e.g. gestureEnabled: false without a back button)`, {
227
+ expected: `a route other than ${before}`,
228
+ actual: after,
229
+ elements: [],
230
+ });
231
+ }
232
+ }
212
233
  async openUrl(url: string, opts?: ActionOptions): Promise<void> {
213
234
  await this.action({ type: 'openUrl', url }, opts);
214
235
  }
@@ -410,12 +431,22 @@ export class Driver {
410
431
  return (await this.call('getIdleState', {})).trackers;
411
432
  }
412
433
 
413
- async backdoor<T extends JsonValue = JsonValue>(name: string, ...args: JsonValue[]): Promise<T> {
434
+ /**
435
+ * Call a backdoor the app registered (`quiesco.backdoor`). With a `Backdoors` declaration (issue #6) the
436
+ * name, the arguments and the result are checked against it; without one, the arguments only need to be
437
+ * JSON-compatible (interfaces included) and the result is `T`.
438
+ */
439
+ async backdoor<K extends keyof Backdoors>(name: K, ...args: BackdoorParams<K>): Promise<BackdoorResult<K, never>>;
440
+ async backdoor<T = JsonValue, const A extends readonly unknown[] = readonly unknown[]>(
441
+ name: [keyof Backdoors] extends [never] ? string : never,
442
+ ...args: A & { readonly [I in keyof A]: JsonCompatible<A[I]> }
443
+ ): Promise<T>;
444
+ async backdoor(name: string, ...args: readonly unknown[]): Promise<unknown> {
414
445
  const bad = findNonJson(args, '$args');
415
446
  if (bad !== null) {
416
447
  throw new QuiescoError('NOT_SERIALIZABLE', `Backdoor "${name}" argument at ${bad.path} is not JSON (${bad.reason})`, { where: 'args', path: bad.path });
417
448
  }
418
- return (await this.call('backdoor', { name, args })).value as T;
449
+ return (await this.call('backdoor', { name, args: args as JsonValue[] })).value;
419
450
  }
420
451
 
421
452
  /** Soft reset (storage, onReset hooks, remount, wait for idle). Returns its duration in ms. */
@@ -423,6 +454,23 @@ export class Driver {
423
454
  return (await this.call('reset', {})).durationMs;
424
455
  }
425
456
 
457
+ /** The condensed element list for failure evidence: elements with a testID, label or text (issue #2). */
458
+ async listElements(): Promise<ResultOf<AgentMethods['listElements']>> {
459
+ return this.call('listElements', {});
460
+ }
461
+
462
+ /**
463
+ * The app's focused route (`/students/new`), or null when it has none (no Expo Router), when the agent is
464
+ * older than this host, or when it does not answer. For failure messages: never throws.
465
+ */
466
+ async currentRoute(): Promise<string | null> {
467
+ try {
468
+ return (await this.call('route', {})).route;
469
+ } catch {
470
+ return null;
471
+ }
472
+ }
473
+
426
474
  /** Take an app error reported between calls (the runner may fail the scenario with it). */
427
475
  takeAppError(): QuiescoError | null {
428
476
  const e = this.appErrors.shift();
package/src/index.ts CHANGED
@@ -1,5 +1,5 @@
1
1
  export { PROTOCOL_VERSION } from '@quiesco/protocol';
2
- export type { ActionMode, ElementSummary, JsonValue, Platform, Rect, TrackerState, TreeNode } from '@quiesco/protocol';
2
+ export type { ActionMode, Backdoors, ElementSummary, JsonCompatible, JsonValue, Platform, Rect, TrackerState, TreeNode } from '@quiesco/protocol';
3
3
 
4
4
  export { defineConfig, loadConfig, resolveConfig, resolvePlatform, serverPort, toAgentConfig, ignoredAppErrors, DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_IGNORED_APP_ERRORS } from './config.js';
5
5
  export type { AndroidAppConfig, AndroidDeviceConfig, ConfigContext, QuiescoConfigInput, IosAppConfig, IosDeviceConfig, QuiescoConfig, WebAppConfig, WebDeviceConfig, WebViewport } from './config.js';
package/src/lifecycle.ts CHANGED
@@ -10,6 +10,9 @@ import type { QuiescoConfig } from './config.js';
10
10
  import type { Platform } from '@quiesco/protocol';
11
11
  import { SERIAL_WORKER } from './devices/provider.js';
12
12
  import type { WorkerInfo } from './devices/provider.js';
13
+ import { consoleLogger } from './logger.js';
14
+ import type { Logger } from './logger.js';
15
+ import { DEFAULT_REPEATED_FAILURES, RepeatedFailures } from './repeated-failures.js';
13
16
  import { createSession } from './session.js';
14
17
  import type { Session, SessionOptions } from './session.js';
15
18
  import { flattenTags, runsOnPlatform, tagNames } from './tags.js';
@@ -28,11 +31,15 @@ export interface LifecycleOptions {
28
31
  * gets it, and the session takes its own device, port and artifacts directory. Default: the only worker.
29
32
  */
30
33
  worker?: WorkerInfo;
34
+ /** Where the repeated-failure hint goes. Default: the console. */
35
+ logger?: Logger;
31
36
  }
32
37
 
33
38
  export interface ScenarioResult {
34
39
  tags: readonly string[];
35
40
  failed: boolean;
41
+ /** The failure (an Error, or the runner's message), to tell repeated identical failures apart (issue #9). */
42
+ error?: unknown;
36
43
  }
37
44
 
38
45
  export interface AfterResult {
@@ -53,6 +60,7 @@ export class Lifecycle {
53
60
  private platform: Platform | null = null;
54
61
  /** Whether the current test started (not skipped): only then does it have artifacts to end. */
55
62
  private started = false;
63
+ private repeated = new RepeatedFailures();
56
64
 
57
65
  constructor(private readonly options: LifecycleOptions = {}) {}
58
66
 
@@ -73,9 +81,18 @@ export class Lifecycle {
73
81
  this.options.config ??
74
82
  (await loadConfig({ ...(env['QUIESCO_CONFIG'] !== undefined ? { file: env['QUIESCO_CONFIG'] } : {}), worker: { workerIndex: worker.index, workerCount: worker.count } })).config;
75
83
  this.platform = this.options.platform ?? resolvePlatform(undefined, env);
84
+ this.repeated = new RepeatedFailures(this.config.bailOnRepeatedFailure ?? DEFAULT_REPEATED_FAILURES);
76
85
  this.session = await (this.options.createSession ?? createSession)({ config: this.config, platform: this.platform, worker });
77
86
  }
78
87
 
88
+ /**
89
+ * Why the remaining tests are skipped (`bailOnRepeatedFailure`, issue #9), or null while they run.
90
+ * Runners that decide on skipping before `before` (Jest) check this too.
91
+ */
92
+ get bailedOut(): string | null {
93
+ return this.repeated.bailedOut;
94
+ }
95
+
79
96
  /** Whether a test with these tags (grouped outermost level first) runs on this run's platform. */
80
97
  runsHere(tags: ScenarioTags): boolean {
81
98
  if (this.config === null || this.platform === null) throw new Error('Quiesco beforeAll did not complete');
@@ -88,6 +105,10 @@ export class Lifecycle {
88
105
  */
89
106
  async before(tags: ScenarioTags, name = 'scenario'): Promise<'skipped' | undefined> {
90
107
  if (!this.runsHere(tags)) return 'skipped';
108
+ if (this.repeated.bailedOut !== null) {
109
+ this.repeated.countSkipped();
110
+ return 'skipped';
111
+ }
91
112
  // Artifacts first: the reset is part of the test's video and timeline.
92
113
  await this.current.startScenario({ name });
93
114
  this.started = true;
@@ -111,14 +132,28 @@ export class Lifecycle {
111
132
  }
112
133
  }
113
134
  if (!started) {
114
- // Skipped by platform (or the before hook failed before the artifacts started): evidence only.
115
- return result.failed ? await this.session.afterScenario({ failed: true }) : { attachments: [], files: [] };
135
+ // Skipped (platform, bail), or the before hook failed before the artifacts started: evidence only.
136
+ if (!result.failed) return { attachments: [], files: [] };
137
+ const evidence = await this.session.afterScenario({ failed: true });
138
+ await this.recordOutcome(result);
139
+ return evidence;
116
140
  }
117
- return this.session.afterScenario({ failed: result.failed });
141
+ const evidence = await this.session.afterScenario({ failed: result.failed });
142
+ await this.recordOutcome(result);
143
+ return evidence;
144
+ }
145
+
146
+ /** Repeated identical failures: the hint once per streak, and the bail (issue #9). */
147
+ private async recordOutcome(result: ScenarioResult): Promise<void> {
148
+ const session = this.session;
149
+ const hint = await this.repeated.record(result, async () => (session === null ? null : session.driver.currentRoute()));
150
+ if (hint !== null) (this.options.logger ?? consoleLogger).warn(hint.message);
118
151
  }
119
152
 
120
153
  /** Close the connection and the device (per config). */
121
154
  async afterAll(): Promise<void> {
155
+ const skipped = this.repeated.skippedCount;
156
+ if (skipped > 0) (this.options.logger ?? consoleLogger).warn(`${skipped} scenario${skipped === 1 ? ' was' : 's were'} skipped after repeated identical failures (bailOnRepeatedFailure)`);
122
157
  const s = this.session;
123
158
  this.session = null;
124
159
  await s?.dispose();
@@ -0,0 +1,114 @@
1
+ /**
2
+ * Repeated identical failures (issue #9): when the app gets into a state that every following scenario
3
+ * inherits, each one fails the same way. After `threshold` such scenarios in a row, say so once; with
4
+ * `bailOnRepeatedFailure` on (the default), skip the rest instead of waiting for each one's timeouts.
5
+ */
6
+
7
+ /** Default for `bailOnRepeatedFailure`, and the streak length that prints the hint when bailing is off. */
8
+ export const DEFAULT_REPEATED_FAILURES = 3;
9
+
10
+ /**
11
+ * What makes two failures "the same": the error's name and the first line of its message, with numbers
12
+ * (durations, counts, ports) replaced so that "after 10012ms" and "after 10009ms" match.
13
+ */
14
+ export function failureSignature(error: unknown): string {
15
+ const { name, line } = describeFailure(error);
16
+ return `${name}: ${line.replace(/\d+/g, 'N')}`;
17
+ }
18
+
19
+ /** The error's name (QuiescoError: its errorName) and the first non-empty line of its message. */
20
+ function describeFailure(error: unknown): { name: string; line: string } {
21
+ // jest-circus keeps some failures as [error, asyncError]
22
+ if (Array.isArray(error)) return describeFailure(error[0]);
23
+ let name = 'Error';
24
+ let message: string;
25
+ if (error instanceof Error) {
26
+ const errorName = (error as { errorName?: unknown }).errorName;
27
+ name = errorName !== undefined ? String(errorName) : error.name;
28
+ message = error.message;
29
+ } else if (typeof error === 'string') {
30
+ message = error;
31
+ } else if (error !== null && typeof error === 'object' && typeof (error as { message?: unknown }).message === 'string') {
32
+ // cucumber-js result.exception { type, message }, Vitest's serialized errors { name, message }
33
+ const o = error as { message: string; type?: unknown; name?: unknown };
34
+ if (typeof o.type === 'string') name = o.type;
35
+ else if (typeof o.name === 'string') name = o.name;
36
+ message = o.message;
37
+ } else {
38
+ message = String(error);
39
+ }
40
+ return { name, line: (message.split('\n').find((l) => l.trim() !== '') ?? '').trim() };
41
+ }
42
+
43
+ export interface StreakHint {
44
+ /** The message to print (once per streak). */
45
+ message: string;
46
+ /** Whether the remaining scenarios are skipped from now on. */
47
+ bail: boolean;
48
+ }
49
+
50
+ export class RepeatedFailures {
51
+ private signature: string | null = null;
52
+ private firstLine = '';
53
+ private count = 0;
54
+ private hinted = false;
55
+ private bailReason: string | null = null;
56
+ private skipped = 0;
57
+
58
+ /** `bail`: the streak length that stops the run, or false (hint only, at DEFAULT_REPEATED_FAILURES). */
59
+ constructor(private readonly bail: number | false = DEFAULT_REPEATED_FAILURES) {
60
+ if (bail !== false && (!Number.isInteger(bail) || bail < 1)) throw new TypeError(`bailOnRepeatedFailure must be a whole number ≥ 1 or false (got ${String(bail)})`);
61
+ }
62
+
63
+ /** Why the remaining scenarios are skipped, or null while they run. */
64
+ get bailedOut(): string | null {
65
+ return this.bailReason;
66
+ }
67
+
68
+ /** A scenario skipped because of the bail. */
69
+ countSkipped(): void {
70
+ this.skipped++;
71
+ }
72
+
73
+ get skippedCount(): number {
74
+ return this.skipped;
75
+ }
76
+
77
+ /**
78
+ * A finished scenario. Passing ones end the streak, failures extend it or start a new one. Returns the hint
79
+ * when this one completes a streak (`route`: where the app is, for the message).
80
+ */
81
+ async record(result: { failed: boolean; error?: unknown }, route: () => Promise<string | null> = async () => null): Promise<StreakHint | null> {
82
+ if (!result.failed) {
83
+ this.signature = null;
84
+ this.count = 0;
85
+ this.hinted = false;
86
+ return null;
87
+ }
88
+ const signature = failureSignature(result.error);
89
+ if (signature === this.signature) {
90
+ this.count++;
91
+ } else {
92
+ this.signature = signature;
93
+ this.count = 1;
94
+ this.hinted = false;
95
+ }
96
+ const { name, line } = describeFailure(result.error);
97
+ this.firstLine = `${name}: ${line}`;
98
+ const threshold = this.bail === false ? DEFAULT_REPEATED_FAILURES : this.bail;
99
+ if (this.hinted || this.count < threshold) return null;
100
+ this.hinted = true;
101
+ const where = await route().catch(() => null);
102
+ const lines = [
103
+ `the last ${this.count} scenarios failed the same way: ${this.firstLine}`,
104
+ `The app may not be returning to its initial state after the reset${where !== null ? ` (current route: ${where})` : ''}.`,
105
+ ];
106
+ if (this.bail !== false) {
107
+ this.bailReason = `skipped: the ${this.count} scenarios before it failed the same way (bailOnRepeatedFailure: ${this.bail})`;
108
+ lines.push(`Skipping the remaining scenarios (bailOnRepeatedFailure: ${this.bail} in quiesco.config; false runs them all).`);
109
+ } else {
110
+ lines.push('Set reset.expectVisible in quiesco.config to have the reset check the initial state.');
111
+ }
112
+ return { message: lines.join('\n'), bail: this.bail !== false };
113
+ }
114
+ }
package/src/session.ts CHANGED
@@ -5,7 +5,7 @@
5
5
  */
6
6
  import { randomUUID } from 'node:crypto';
7
7
 
8
- import type { AgentConfig, Platform, TrackerState, TreeNode } from '@quiesco/protocol';
8
+ import type { AgentConfig, AgentMethods, Platform, ResultOf, TrackerState, TreeNode } from '@quiesco/protocol';
9
9
 
10
10
  import type { QuiescoConfig } from './config.js';
11
11
  import { DEFAULT_CONNECT_TIMEOUT_MS, ignoredAppErrors, serverPort, toAgentConfig } from './config.js';
@@ -16,6 +16,7 @@ import type { DeviceProvider, LaunchParams, LaunchUrlHandling, WorkerInfo } from
16
16
  import { DeviceApi } from './device.js';
17
17
  import type { AppRunState, DeviceHost } from './device.js';
18
18
  import { Artifacts } from './artifacts.js';
19
+ import { describeBundle, staleBundleWarning } from './bundle-check.js';
19
20
  import type { Attachment, ScenarioArtifacts } from './artifacts.js';
20
21
  import { Driver } from './driver.js';
21
22
  import { failureRects, highlight } from './screenshots.js';
@@ -62,6 +63,8 @@ export interface SessionOptions {
62
63
  export interface Evidence {
63
64
  screenshot?: Buffer;
64
65
  tree?: TreeNode | null;
66
+ /** The condensed element list and the route (issue #2). */
67
+ elements?: ResultOf<AgentMethods['listElements']>;
65
68
  idle?: TrackerState[];
66
69
  /** Why a piece could not be collected. */
67
70
  errors: string[];
@@ -214,10 +217,26 @@ export class Session {
214
217
  this.appState = 'terminated';
215
218
  await this.launch();
216
219
  this.appState = 'running';
217
- return;
220
+ } else {
221
+ if (this.appState === 'background') await this.device.bringToForeground();
222
+ await this.driver.reset();
223
+ }
224
+ await this.expectInitialState();
225
+ }
226
+
227
+ /** `reset.expectVisible`: the reset must leave that element visible (issue #9). */
228
+ private async expectInitialState(): Promise<void> {
229
+ const target = this.config.reset?.expectVisible;
230
+ if (target === undefined) return;
231
+ try {
232
+ await this.driver.expectVisible(target);
233
+ } catch (e) {
234
+ // A lost connection is not about the app's state: beforeScenario starts the app again for that.
235
+ if (!(e instanceof QuiescoError) || RESTART_ON.has(e.errorName)) throw e;
236
+ const route = await this.driver.currentRoute();
237
+ const message = `the app is not back in its initial state${route !== null ? ` (current route: ${route})` : ''}: ${e.message} (reset.expectVisible in quiesco.config)`;
238
+ throw new QuiescoError('RESET_FAILED', `Reset failed at expectVisible: ${message}`, { step: 'expectVisible', message });
218
239
  }
219
- if (this.appState === 'background') await this.device.bringToForeground();
220
- await this.driver.reset();
221
240
  }
222
241
 
223
242
  /**
@@ -238,6 +257,7 @@ export class Session {
238
257
  if (result.failed) {
239
258
  const ev = await this.collectEvidence();
240
259
  if (ev.screenshot !== undefined) evidence.push({ data: ev.screenshot, mediaType: 'image/png', fileName: 'failure.png' });
260
+ if (ev.elements !== undefined) evidence.push({ data: JSON.stringify(ev.elements, null, 2), mediaType: 'application/json', fileName: 'elements.json' });
241
261
  if (ev.tree !== undefined) evidence.push({ data: JSON.stringify({ elementTree: ev.tree }, null, 2), mediaType: 'application/json', fileName: 'element-tree.json' });
242
262
  if (ev.idle !== undefined) evidence.push({ data: JSON.stringify({ idleState: ev.idle }, null, 2), mediaType: 'application/json', fileName: 'idle-state.json' });
243
263
  if (ev.errors.length > 0) evidence.push({ data: `Quiesco could not collect: ${ev.errors.join('; ')}`, mediaType: 'text/plain', fileName: 'evidence-errors.txt' });
@@ -257,6 +277,13 @@ export class Session {
257
277
  (t) => (ev.tree = t),
258
278
  (e: unknown) => ev.errors.push(`element tree: ${(e as Error).message}`),
259
279
  ),
280
+ this.driver.listElements().then(
281
+ (l) => (ev.elements = l),
282
+ // An agent older than this host does not have it: not worth an error line.
283
+ (e: unknown) => {
284
+ if (!(e instanceof QuiescoError && e.code === -32601)) ev.errors.push(`element list: ${(e as Error).message}`);
285
+ },
286
+ ),
260
287
  this.driver.getIdleState().then(
261
288
  (s) => (ev.idle = s),
262
289
  (e: unknown) => ev.errors.push(`idle state: ${(e as Error).message}`),
@@ -293,6 +320,21 @@ export class Session {
293
320
  await this.provider.shutdown().catch((e: unknown) => this.logger.warn(`device shutdown failed: ${(e as Error).message}`));
294
321
  }
295
322
 
323
+ /** Where the project's sources are, for the stale-bundle check: the test runner runs in the project root. */
324
+ private readonly projectRoot = process.cwd();
325
+ /** Bundles already checked (by build time): a reconnect of the same bundle is not checked again. */
326
+ private readonly checkedBundles = new Set<number>();
327
+
328
+ /** Issue #3: say when the app's JS bundle cannot have the project's latest changes. Once per bundle. */
329
+ private checkBundle(conn: AgentConnection): void {
330
+ const bundle = conn.info.bundle;
331
+ this.logger.debug(`app JS: ${describeBundle(bundle)}`);
332
+ if (bundle === undefined || this.checkedBundles.has(bundle.builtAt)) return;
333
+ this.checkedBundles.add(bundle.builtAt);
334
+ const warning = staleBundleWarning(bundle, this.projectRoot);
335
+ if (warning !== null) this.logger.warn(warning);
336
+ }
337
+
296
338
  /**
297
339
  * The open connection; after a disconnect (web page reload, app restart) waits for the app to come
298
340
  * back within the connect timeout.
@@ -314,6 +356,7 @@ export class Session {
314
356
  private adopt(conn: AgentConnection): void {
315
357
  if (this.connection === conn) return;
316
358
  this.connection = conn;
359
+ this.checkBundle(conn);
317
360
  conn.on('log', (l) => this.logger[l.level](`[app] ${l.message}`));
318
361
  this.driver.attach(conn);
319
362
  }