@e2e-dev/mobile 0.12.5 → 0.12.6-nightly-20261101030202
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.build.json +1 -1
- package/dist/device.js +1 -1
- package/dist/engine.js +4 -4
- package/dist/support.d.ts +14 -4
- package/dist/support.js +30 -8
- package/dist/surface.d.ts +14 -9
- package/dist/surface.js +66 -63
- package/package.json +5 -5
package/dist/.build.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":"0.12.
|
|
1
|
+
{"version":"0.12.6-nightly-20261101030202"}
|
package/dist/device.js
CHANGED
|
@@ -114,7 +114,7 @@ export function createDeviceFixture(surface, context) {
|
|
|
114
114
|
await surface.screenCommand('device.alert', (client) => client.command.alert({ ...surface.selection(), action }), context.signal);
|
|
115
115
|
},
|
|
116
116
|
async dismissKeyboard() {
|
|
117
|
-
await surface.dismissKeyboard(context.signal);
|
|
117
|
+
await surface.dismissKeyboard(context.signal, surface.inputBudget('device.dismissKeyboard', context.timeouts.action));
|
|
118
118
|
},
|
|
119
119
|
async clipboard() {
|
|
120
120
|
const result = await surface.command('device.clipboard', (client) => client.command.clipboard({ ...surface.selection(), action: 'read' }), context.signal);
|
package/dist/engine.js
CHANGED
|
@@ -27,10 +27,10 @@ export function buildEngine(surface) {
|
|
|
27
27
|
startAttempt: (context) => surface.startAttempt(context),
|
|
28
28
|
endAttempt: (context) => surface.endAttempt(context),
|
|
29
29
|
dispose: (context) => surface.dispose(context),
|
|
30
|
-
observe: (operation, options) => surface.observe(operation, options),
|
|
31
|
-
locate: (expression, operation) => surface.locate(expression, operation),
|
|
32
30
|
// The harness signal does not abort at an operation's budget, so each
|
|
33
|
-
//
|
|
31
|
+
// operation enforces its own; see InputBudget in support.ts.
|
|
32
|
+
observe: (operation, options) => surface.observe(operation, options, surface.inputBudget('observe', operation.timeoutMs)),
|
|
33
|
+
locate: (expression, operation) => surface.locate(expression, operation, surface.inputBudget('locate', operation.timeoutMs)),
|
|
34
34
|
perform: (ref, action, operation) => surface.perform(ref, action, operation, surface.inputBudget(action.kind, operation.timeoutMs)),
|
|
35
35
|
actions: DEVICE_ACTIONS,
|
|
36
36
|
performAt: (point, action, operation) => surface.performAt(point, action, operation, surface.inputBudget(action.kind, operation.timeoutMs)),
|
|
@@ -50,7 +50,7 @@ export function buildEngine(surface) {
|
|
|
50
50
|
reset: (operation) => surface.reset(operation),
|
|
51
51
|
},
|
|
52
52
|
artifacts: {
|
|
53
|
-
screenshot: (label, operation) => surface.screenshot(label, operation),
|
|
53
|
+
screenshot: (label, operation) => surface.screenshot(label, operation, surface.inputBudget('screenshot', operation.timeoutMs)),
|
|
54
54
|
startVideo: (operation) => surface.startVideo(operation),
|
|
55
55
|
stopVideo: (operation) => surface.stopVideo(operation),
|
|
56
56
|
},
|
package/dist/support.d.ts
CHANGED
|
@@ -13,10 +13,9 @@ export interface Rect {
|
|
|
13
13
|
}
|
|
14
14
|
export declare function cancelled(text: string): EngineError;
|
|
15
15
|
/**
|
|
16
|
-
* One
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* last snapshot and a target settling before a tap is relocated in full.
|
|
16
|
+
* One operation's budget (its `timeoutMs`), which the harness signal does not
|
|
17
|
+
* enforce. Every device command the operation sends runs under it: the
|
|
18
|
+
* inputs through `send`, the snapshots and screenshots through `read`.
|
|
20
19
|
*/
|
|
21
20
|
export declare class InputBudget {
|
|
22
21
|
private readonly label;
|
|
@@ -38,6 +37,17 @@ export declare class InputBudget {
|
|
|
38
37
|
* landed, so it is `ACTION_MAY_HAVE_COMMITTED` and the runner never repeats it.
|
|
39
38
|
*/
|
|
40
39
|
send<T>(dispatch: () => Promise<T>, holdMs?: number): Promise<T>;
|
|
40
|
+
/**
|
|
41
|
+
* Runs one device read with what is left of the budget. With none left it
|
|
42
|
+
* reads nothing. A read that does not answer in time is `OPERATION_TIMEOUT`,
|
|
43
|
+
* or `ACTION_MAY_HAVE_COMMITTED` once an input of this operation went out,
|
|
44
|
+
* so the runner never repeats that input.
|
|
45
|
+
*/
|
|
46
|
+
read<T>(dispatch: () => Promise<T>): Promise<T>;
|
|
47
|
+
/** Milliseconds left of the budget; zero or less once it is spent. */
|
|
48
|
+
remainingMs(): number;
|
|
49
|
+
/** The failure of a command the spent budget did not send. */
|
|
50
|
+
private spent;
|
|
41
51
|
private maybeCommitted;
|
|
42
52
|
}
|
|
43
53
|
export declare function invalidState(text: string): EngineError;
|
package/dist/support.js
CHANGED
|
@@ -9,10 +9,9 @@ export function cancelled(text) {
|
|
|
9
9
|
*/
|
|
10
10
|
const MIN_ANSWER_MS = 250;
|
|
11
11
|
/**
|
|
12
|
-
* One
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* last snapshot and a target settling before a tap is relocated in full.
|
|
12
|
+
* One operation's budget (its `timeoutMs`), which the harness signal does not
|
|
13
|
+
* enforce. Every device command the operation sends runs under it: the
|
|
14
|
+
* inputs through `send`, the snapshots and screenshots through `read`.
|
|
16
15
|
*/
|
|
17
16
|
export class InputBudget {
|
|
18
17
|
label;
|
|
@@ -38,10 +37,9 @@ export class InputBudget {
|
|
|
38
37
|
* landed, so it is `ACTION_MAY_HAVE_COMMITTED` and the runner never repeats it.
|
|
39
38
|
*/
|
|
40
39
|
send(dispatch, holdMs = 0) {
|
|
41
|
-
const remaining = this.
|
|
42
|
-
if (remaining <= 0)
|
|
43
|
-
return Promise.reject(this.
|
|
44
|
-
}
|
|
40
|
+
const remaining = this.remainingMs();
|
|
41
|
+
if (remaining <= 0)
|
|
42
|
+
return Promise.reject(this.spent());
|
|
45
43
|
if (holdMs >= remaining) {
|
|
46
44
|
return Promise.reject(new EngineError('NOT_ACTIONABLE', `${this.label} holds for ${holdMs} ms, but only ${Math.floor(remaining)} ms of its timeout are left; raise the timeout or shorten the hold`, { retryable: false }));
|
|
47
45
|
}
|
|
@@ -49,6 +47,30 @@ export class InputBudget {
|
|
|
49
47
|
this.sent = true;
|
|
50
48
|
return withTimeout(pending, Math.max(remaining, MIN_ANSWER_MS), () => this.maybeCommitted());
|
|
51
49
|
}
|
|
50
|
+
/**
|
|
51
|
+
* Runs one device read with what is left of the budget. With none left it
|
|
52
|
+
* reads nothing. A read that does not answer in time is `OPERATION_TIMEOUT`,
|
|
53
|
+
* or `ACTION_MAY_HAVE_COMMITTED` once an input of this operation went out,
|
|
54
|
+
* so the runner never repeats that input.
|
|
55
|
+
*/
|
|
56
|
+
read(dispatch) {
|
|
57
|
+
const remaining = this.remainingMs();
|
|
58
|
+
if (remaining <= 0)
|
|
59
|
+
return Promise.reject(this.spent());
|
|
60
|
+
return withTimeout(this.track(dispatch()), remaining, () => this.sent
|
|
61
|
+
? this.maybeCommitted()
|
|
62
|
+
: new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before the device answered`, { retryable: false }));
|
|
63
|
+
}
|
|
64
|
+
/** Milliseconds left of the budget; zero or less once it is spent. */
|
|
65
|
+
remainingMs() {
|
|
66
|
+
return this.endsAt - Date.now();
|
|
67
|
+
}
|
|
68
|
+
/** The failure of a command the spent budget did not send. */
|
|
69
|
+
spent() {
|
|
70
|
+
return this.sent
|
|
71
|
+
? this.maybeCommitted()
|
|
72
|
+
: new EngineError('OPERATION_TIMEOUT', `${this.label} timed out before it was sent`, { retryable: false });
|
|
73
|
+
}
|
|
52
74
|
maybeCommitted() {
|
|
53
75
|
return new EngineError('ACTION_MAY_HAVE_COMMITTED', `${this.label} timed out before the device answered; its input may have been dispatched`, {
|
|
54
76
|
retryable: false,
|
package/dist/surface.d.ts
CHANGED
|
@@ -155,7 +155,7 @@ export declare class AgentDeviceSurface {
|
|
|
155
155
|
* not yet arrived.
|
|
156
156
|
*/
|
|
157
157
|
screenCommand<T>(label: string, run: (client: AgentDeviceClient) => Promise<T>, signal: AbortSignal): Promise<T>;
|
|
158
|
-
/** The budget one
|
|
158
|
+
/** The budget one operation runs under; its device commands stay in flight until they settle. */
|
|
159
159
|
inputBudget(label: string, timeoutMs: number): InputBudget;
|
|
160
160
|
/** Registers one device command as in flight until it settles. */
|
|
161
161
|
private track;
|
|
@@ -272,7 +272,8 @@ export declare class AgentDeviceSurface {
|
|
|
272
272
|
* One capture, taken again once when the iOS runner acquired the tree but
|
|
273
273
|
* failed its own presentation check on it (a viewport it could not read, a
|
|
274
274
|
* malformed graph). The check is per capture and a capture is a read, so
|
|
275
|
-
* the retry is safe; a second failure is the runner's and propagates.
|
|
275
|
+
* the retry is safe; a second failure is the runner's and propagates. Each
|
|
276
|
+
* capture runs under `budget`, so one the budget abandoned is not retried.
|
|
276
277
|
*/
|
|
277
278
|
private capture;
|
|
278
279
|
/**
|
|
@@ -285,8 +286,11 @@ export declare class AgentDeviceSurface {
|
|
|
285
286
|
* Snapshot for a locate; undefined for a screen agent-device could not read
|
|
286
287
|
* yet (an app still on its splash, or one iOS does not report running yet
|
|
287
288
|
* after a launch). The harness polls a locate until its deadline, so that
|
|
288
|
-
* screen matches nothing and the poll waits for it to render.
|
|
289
|
-
*
|
|
289
|
+
* screen matches nothing and the poll waits for it to render. When what is
|
|
290
|
+
* left of the budget cannot fit another capture as long as this one, the
|
|
291
|
+
* locate fails with the reason instead: the budget bounds every capture, so
|
|
292
|
+
* an unreadable one never outlasts it, and the wait that ends on an
|
|
293
|
+
* unreadable screen says so.
|
|
290
294
|
*/
|
|
291
295
|
private locateSnapshot;
|
|
292
296
|
/**
|
|
@@ -314,9 +318,9 @@ export declare class AgentDeviceSurface {
|
|
|
314
318
|
private viewportFor;
|
|
315
319
|
/** The device's logical screen size as its screenshot reports it; undefined when it reports none. */
|
|
316
320
|
private probeViewport;
|
|
317
|
-
observe(operation: OperationContext, options
|
|
321
|
+
observe(operation: OperationContext, options: EngineObserveOptions | undefined, budget: InputBudget): Promise<EngineSnapshot>;
|
|
318
322
|
/** Retains only action bindings for matches, independently of the observation generation. */
|
|
319
|
-
locate(expression: LocatorExpression, operation: OperationContext): Promise<readonly SemanticNode[]>;
|
|
323
|
+
locate(expression: LocatorExpression, operation: OperationContext, budget: InputBudget): Promise<readonly SemanticNode[]>;
|
|
320
324
|
/**
|
|
321
325
|
* Hands the harness the capture a locate matched against, for the trace:
|
|
322
326
|
* the screen the step acted on, at no cost of another snapshot. A screen
|
|
@@ -402,7 +406,7 @@ export declare class AgentDeviceSurface {
|
|
|
402
406
|
* makes that guess, the way a user does and Maestro's `hideKeyboard` does,
|
|
403
407
|
* in {@link dismissKeyboardByGesture}.
|
|
404
408
|
*/
|
|
405
|
-
dismissKeyboard(signal: AbortSignal, input
|
|
409
|
+
dismissKeyboard(signal: AbortSignal, input: InputBudget): Promise<void>;
|
|
406
410
|
/**
|
|
407
411
|
* A keyboard with no dismiss key goes the way a user sends it away: a short
|
|
408
412
|
* drag on the screen, which a scroll view turns into a dismissal, each
|
|
@@ -458,7 +462,7 @@ export declare class AgentDeviceSurface {
|
|
|
458
462
|
* bounds cannot be masked, and an image that cannot be redacted is not
|
|
459
463
|
* written at all.
|
|
460
464
|
*/
|
|
461
|
-
screenshot(label: string | undefined, operation: OperationContext): Promise<string>;
|
|
465
|
+
screenshot(label: string | undefined, operation: OperationContext, budget: InputBudget): Promise<string>;
|
|
462
466
|
/** Raw device pixels; cleanup follows the capture even when its caller abandons it. */
|
|
463
467
|
private rawScreenshot;
|
|
464
468
|
/**
|
|
@@ -476,7 +480,8 @@ export declare class AgentDeviceSurface {
|
|
|
476
480
|
private maskedScreenshot;
|
|
477
481
|
/**
|
|
478
482
|
* Viewport pixels for an observation. Best-effort: a screenshot that cannot
|
|
479
|
-
* be produced or redacted costs the observation its image, not the step
|
|
483
|
+
* be produced or redacted costs the observation its image, not the step,
|
|
484
|
+
* unless the operation's budget ran out.
|
|
480
485
|
*/
|
|
481
486
|
private capturePixels;
|
|
482
487
|
}
|
package/dist/surface.js
CHANGED
|
@@ -125,10 +125,6 @@ function depthBelow(entry, ancestor) {
|
|
|
125
125
|
}
|
|
126
126
|
return hops;
|
|
127
127
|
}
|
|
128
|
-
/** Sends an input under its operation's budget, or directly from a fixture call, which has none. */
|
|
129
|
-
function sendInput(input, dispatch) {
|
|
130
|
-
return input === undefined ? dispatch() : input.send(dispatch);
|
|
131
|
-
}
|
|
132
128
|
function sleep(ms, signal) {
|
|
133
129
|
return new Promise((resolve) => {
|
|
134
130
|
const timer = setTimeout(done, ms);
|
|
@@ -320,7 +316,7 @@ export class AgentDeviceSurface {
|
|
|
320
316
|
this.markAction(before);
|
|
321
317
|
return result;
|
|
322
318
|
}
|
|
323
|
-
/** The budget one
|
|
319
|
+
/** The budget one operation runs under; its device commands stay in flight until they settle. */
|
|
324
320
|
inputBudget(label, timeoutMs) {
|
|
325
321
|
return new InputBudget(label, timeoutMs, (pending) => this.track(pending));
|
|
326
322
|
}
|
|
@@ -703,14 +699,21 @@ export class AgentDeviceSurface {
|
|
|
703
699
|
this.installedApp = installed.app;
|
|
704
700
|
return installed;
|
|
705
701
|
}
|
|
706
|
-
async snapshot(signal, interactiveOnly) {
|
|
702
|
+
async snapshot(signal, interactiveOnly, budget) {
|
|
707
703
|
let last = {};
|
|
704
|
+
let captureMs = 0;
|
|
708
705
|
for (const backoffMs of SPARSE_RETRY_BACKOFF_MS) {
|
|
709
|
-
if (backoffMs > 0)
|
|
706
|
+
if (backoffMs > 0) {
|
|
707
|
+
// A sparse capture is still the screen: a retry the budget cannot fit returns it instead of timing out.
|
|
708
|
+
if (budget !== undefined && budget.remainingMs() <= backoffMs + captureMs)
|
|
709
|
+
return last;
|
|
710
710
|
await sleep(backoffMs, signal);
|
|
711
|
+
}
|
|
711
712
|
if (signal.aborted)
|
|
712
713
|
throw new EngineError('CANCELLED', 'snapshot cancelled', { retryable: false });
|
|
713
|
-
|
|
714
|
+
const startedAt = Date.now();
|
|
715
|
+
last = await this.capture(signal, interactiveOnly, budget);
|
|
716
|
+
captureMs = Date.now() - startedAt;
|
|
714
717
|
this.launching = false;
|
|
715
718
|
if (last.appBundleId !== undefined || last.appName !== undefined) {
|
|
716
719
|
this.appIdentity = last.appBundleId ?? last.appName;
|
|
@@ -724,10 +727,11 @@ export class AgentDeviceSurface {
|
|
|
724
727
|
* One capture, taken again once when the iOS runner acquired the tree but
|
|
725
728
|
* failed its own presentation check on it (a viewport it could not read, a
|
|
726
729
|
* malformed graph). The check is per capture and a capture is a read, so
|
|
727
|
-
* the retry is safe; a second failure is the runner's and propagates.
|
|
730
|
+
* the retry is safe; a second failure is the runner's and propagates. Each
|
|
731
|
+
* capture runs under `budget`, so one the budget abandoned is not retried.
|
|
728
732
|
*/
|
|
729
|
-
async capture(signal, interactiveOnly) {
|
|
730
|
-
const take = () => this.command('snapshot', (client) => client.capture.snapshot({ interactiveOnly }), signal);
|
|
733
|
+
async capture(signal, interactiveOnly, budget) {
|
|
734
|
+
const take = () => budget.read(() => this.command('snapshot', (client) => client.capture.snapshot({ interactiveOnly }), signal));
|
|
731
735
|
try {
|
|
732
736
|
return await take();
|
|
733
737
|
}
|
|
@@ -742,9 +746,9 @@ export class AgentDeviceSurface {
|
|
|
742
746
|
* anything is open is an empty screen rather than a fault: the model's next
|
|
743
747
|
* move is the open tool, and failing the step would take that move away.
|
|
744
748
|
*/
|
|
745
|
-
async snapshotOrEmpty(operation, interactiveOnly) {
|
|
749
|
+
async snapshotOrEmpty(operation, interactiveOnly, budget) {
|
|
746
750
|
try {
|
|
747
|
-
return await this.snapshot(operation.signal, interactiveOnly);
|
|
751
|
+
return await this.snapshot(operation.signal, interactiveOnly, budget);
|
|
748
752
|
}
|
|
749
753
|
catch (cause) {
|
|
750
754
|
if (!this.managesApp && cause instanceof EngineError && cause.code === 'INVALID_STATE')
|
|
@@ -756,17 +760,20 @@ export class AgentDeviceSurface {
|
|
|
756
760
|
* Snapshot for a locate; undefined for a screen agent-device could not read
|
|
757
761
|
* yet (an app still on its splash, or one iOS does not report running yet
|
|
758
762
|
* after a launch). The harness polls a locate until its deadline, so that
|
|
759
|
-
* screen matches nothing and the poll waits for it to render.
|
|
760
|
-
*
|
|
763
|
+
* screen matches nothing and the poll waits for it to render. When what is
|
|
764
|
+
* left of the budget cannot fit another capture as long as this one, the
|
|
765
|
+
* locate fails with the reason instead: the budget bounds every capture, so
|
|
766
|
+
* an unreadable one never outlasts it, and the wait that ends on an
|
|
767
|
+
* unreadable screen says so.
|
|
761
768
|
*/
|
|
762
|
-
async locateSnapshot(operation) {
|
|
769
|
+
async locateSnapshot(operation, budget) {
|
|
763
770
|
const startedMs = Date.now();
|
|
764
771
|
try {
|
|
765
|
-
return await this.snapshotOrEmpty(operation, false);
|
|
772
|
+
return await this.snapshotOrEmpty(operation, false, budget);
|
|
766
773
|
}
|
|
767
774
|
catch (cause) {
|
|
768
775
|
const arriving = isUnreadableCapture(cause) || (this.launching && isAppNotRunning(cause));
|
|
769
|
-
if (operation.signal.aborted || !arriving || Date.now() - startedMs
|
|
776
|
+
if (operation.signal.aborted || !arriving || budget.remainingMs() < Date.now() - startedMs)
|
|
770
777
|
throw cause;
|
|
771
778
|
return undefined;
|
|
772
779
|
}
|
|
@@ -810,21 +817,21 @@ export class AgentDeviceSurface {
|
|
|
810
817
|
* measures every rect and point against the viewport, so an invented one
|
|
811
818
|
* would misplace every tap.
|
|
812
819
|
*/
|
|
813
|
-
async viewportFor(projected, signal) {
|
|
820
|
+
async viewportFor(projected, signal, budget) {
|
|
814
821
|
const read = projected.viewport;
|
|
815
822
|
const known = this.knownViewport;
|
|
816
823
|
if (known !== undefined) {
|
|
817
824
|
if (read === undefined)
|
|
818
825
|
return known;
|
|
819
826
|
if (!sameScreen(read, known)) {
|
|
820
|
-
const probed = await this.probeViewport(signal);
|
|
827
|
+
const probed = await this.probeViewport(signal, budget);
|
|
821
828
|
if (probed !== undefined && !sameScreen(read, probed))
|
|
822
829
|
return known;
|
|
823
830
|
}
|
|
824
831
|
this.knownViewport = read;
|
|
825
832
|
return read;
|
|
826
833
|
}
|
|
827
|
-
const probed = await this.probeViewport(signal);
|
|
834
|
+
const probed = await this.probeViewport(signal, budget);
|
|
828
835
|
if (probed === undefined) {
|
|
829
836
|
throw new EngineError('ENGINE_FAILURE', 'the device reported no screen geometry: the snapshot has no bounds and the screenshot no logical size, so the viewport is unknown', { retryable: false });
|
|
830
837
|
}
|
|
@@ -832,17 +839,17 @@ export class AgentDeviceSurface {
|
|
|
832
839
|
return probed;
|
|
833
840
|
}
|
|
834
841
|
/** The device's logical screen size as its screenshot reports it; undefined when it reports none. */
|
|
835
|
-
async probeViewport(signal) {
|
|
836
|
-
const result = await this.captureScreenshot(signal, async (shot) => shot);
|
|
842
|
+
async probeViewport(signal, budget) {
|
|
843
|
+
const result = await this.captureScreenshot(signal, async (shot) => shot, {}, budget);
|
|
837
844
|
return logicalScreenSize(result);
|
|
838
845
|
}
|
|
839
|
-
async observe(operation, options) {
|
|
840
|
-
const raw = await this.snapshotOrEmpty(operation, this.options.snapshot === 'interactive');
|
|
846
|
+
async observe(operation, options, budget) {
|
|
847
|
+
const raw = await this.snapshotOrEmpty(operation, this.options.snapshot === 'interactive', budget);
|
|
841
848
|
const projected = this.project(raw);
|
|
842
849
|
this.generation = new Map(projected.index.map((entry) => [entry.id, this.bind(entry, projected)]));
|
|
843
|
-
const viewport = await this.viewportFor(projected, operation.signal);
|
|
850
|
+
const viewport = await this.viewportFor(projected, operation.signal, budget);
|
|
844
851
|
const location = screenLocation(raw.appBundleId ?? raw.appName ?? this.appIdentity, screenTitle(projected));
|
|
845
|
-
const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }) : undefined;
|
|
852
|
+
const capture = options?.pixels === true ? await this.capturePixels(operation, projected, viewport, { normalizeStatusBar: options.comparable === true }, budget) : undefined;
|
|
846
853
|
const keyboardVisible = measuredKeyboard(raw);
|
|
847
854
|
return {
|
|
848
855
|
root: screenRoot(projected.roots, viewport),
|
|
@@ -854,8 +861,8 @@ export class AgentDeviceSurface {
|
|
|
854
861
|
};
|
|
855
862
|
}
|
|
856
863
|
/** Retains only action bindings for matches, independently of the observation generation. */
|
|
857
|
-
async locate(expression, operation) {
|
|
858
|
-
const raw = await this.locateSnapshot(operation);
|
|
864
|
+
async locate(expression, operation, budget) {
|
|
865
|
+
const raw = await this.locateSnapshot(operation, budget);
|
|
859
866
|
if (raw === undefined)
|
|
860
867
|
return [];
|
|
861
868
|
const projected = this.project(raw);
|
|
@@ -1003,9 +1010,9 @@ export class AgentDeviceSurface {
|
|
|
1003
1010
|
return true;
|
|
1004
1011
|
}
|
|
1005
1012
|
/** Refreshes a control after a transition wait, keeping the binding when no wait was needed. */
|
|
1006
|
-
async settled(entry, operation) {
|
|
1013
|
+
async settled(entry, operation, budget) {
|
|
1007
1014
|
if (await this.waitForTransition(entry, operation))
|
|
1008
|
-
return this.relocated(entry, operation);
|
|
1015
|
+
return this.relocated(entry, operation, budget);
|
|
1009
1016
|
return entry;
|
|
1010
1017
|
}
|
|
1011
1018
|
/**
|
|
@@ -1013,8 +1020,8 @@ export class AgentDeviceSurface {
|
|
|
1013
1020
|
* is a retryable NODE_STALE, so the locator resolves again; input mutations
|
|
1014
1021
|
* also refuse a truncated snapshot or changed security facts.
|
|
1015
1022
|
*/
|
|
1016
|
-
async relocated(entry, operation, input = false) {
|
|
1017
|
-
const raw = await this.snapshotOrEmpty(operation, false);
|
|
1023
|
+
async relocated(entry, operation, budget, input = false) {
|
|
1024
|
+
const raw = await this.snapshotOrEmpty(operation, false, budget);
|
|
1018
1025
|
const projected = this.project(raw);
|
|
1019
1026
|
const found = this.refind(entry, projected.index);
|
|
1020
1027
|
if (input && (isTruncated(raw) || found === undefined ||
|
|
@@ -1079,12 +1086,12 @@ export class AgentDeviceSurface {
|
|
|
1079
1086
|
if (filling) {
|
|
1080
1087
|
if (deterministic)
|
|
1081
1088
|
await this.waitForTransition(entry, operation);
|
|
1082
|
-
target = await this.relocated(entry, operation, true);
|
|
1089
|
+
target = await this.relocated(entry, operation, input, true);
|
|
1083
1090
|
if (!target.takesText)
|
|
1084
1091
|
throw refuseFill(target);
|
|
1085
1092
|
}
|
|
1086
1093
|
else if (deterministic) {
|
|
1087
|
-
target = await this.settled(entry, operation);
|
|
1094
|
+
target = await this.settled(entry, operation, input);
|
|
1088
1095
|
}
|
|
1089
1096
|
switch (action.kind) {
|
|
1090
1097
|
case 'tap':
|
|
@@ -1245,7 +1252,7 @@ export class AgentDeviceSurface {
|
|
|
1245
1252
|
async dismissKeyboard(signal, input) {
|
|
1246
1253
|
const before = this.latestScreen;
|
|
1247
1254
|
try {
|
|
1248
|
-
await this.command('keyboard.dismiss', (client) =>
|
|
1255
|
+
await this.command('keyboard.dismiss', (client) => input.send(() => client.command.keyboard({ action: 'dismiss' })), signal);
|
|
1249
1256
|
}
|
|
1250
1257
|
catch (cause) {
|
|
1251
1258
|
if (!(cause instanceof EngineError && cause.code === 'UNSUPPORTED_CAPABILITY'))
|
|
@@ -1277,12 +1284,12 @@ export class AgentDeviceSurface {
|
|
|
1277
1284
|
* keyboard.
|
|
1278
1285
|
*/
|
|
1279
1286
|
async dismissKeyboardByGesture(signal, input) {
|
|
1280
|
-
let raw = await this.dismissKeyboardTip(await this.snapshot(signal, false), signal, input);
|
|
1287
|
+
let raw = await this.dismissKeyboardTip(await this.snapshot(signal, false, input), signal, input);
|
|
1281
1288
|
if (!keyboardShowing(raw))
|
|
1282
1289
|
return;
|
|
1283
1290
|
for (const axis of ['horizontal', 'vertical']) {
|
|
1284
1291
|
const projected = projectSnapshot(raw.nodes ?? [], { mintId: () => 'keyboard', viewport: raw.viewport });
|
|
1285
|
-
const viewport = await this.viewportFor(projected, signal);
|
|
1292
|
+
const viewport = await this.viewportFor(projected, signal, input);
|
|
1286
1293
|
const from = keyboardDragStart(projected, viewport, keyboardTop(raw, viewport));
|
|
1287
1294
|
if (from === undefined) {
|
|
1288
1295
|
throw new EngineError('UNSUPPORTED_CAPABILITY', `keyboard dismissal failed: the keyboard has no dismiss key, and every point above it (in a scroll view, when the screen has one) is on a control, so a drag would press one. ${KEYBOARD_ALTERNATIVES}`, { retryable: false });
|
|
@@ -1290,8 +1297,8 @@ export class AgentDeviceSurface {
|
|
|
1290
1297
|
const to = axis === 'horizontal'
|
|
1291
1298
|
? { x: from.x - viewport.width * KEYBOARD_DRAG_RATIO, y: from.y }
|
|
1292
1299
|
: { x: from.x, y: from.y - viewport.height * KEYBOARD_DRAG_RATIO };
|
|
1293
|
-
await this.command('keyboard.dismiss', (client) =>
|
|
1294
|
-
raw = await this.snapshot(signal, false);
|
|
1300
|
+
await this.command('keyboard.dismiss', (client) => input.send(() => client.interactions.swipe({ from, to })), signal);
|
|
1301
|
+
raw = await this.snapshot(signal, false, input);
|
|
1295
1302
|
if (!keyboardShowing(raw))
|
|
1296
1303
|
return;
|
|
1297
1304
|
}
|
|
@@ -1312,8 +1319,8 @@ export class AgentDeviceSurface {
|
|
|
1312
1319
|
const button = nodes.find((node) => node.label === 'Continue' && node.rect !== undefined && node.rect.y > tipTop);
|
|
1313
1320
|
if (button?.rect === undefined)
|
|
1314
1321
|
return raw;
|
|
1315
|
-
await this.command('keyboard.dismiss', (client) =>
|
|
1316
|
-
return await this.snapshot(signal, false);
|
|
1322
|
+
await this.command('keyboard.dismiss', (client) => input.send(() => client.interactions.press(centreOf(button.rect))), signal);
|
|
1323
|
+
return await this.snapshot(signal, false, input);
|
|
1317
1324
|
}
|
|
1318
1325
|
async back(operation, input) {
|
|
1319
1326
|
const settle = operation.origin === 'test' ? {} : this.settleOptions;
|
|
@@ -1363,11 +1370,11 @@ export class AgentDeviceSurface {
|
|
|
1363
1370
|
* bounds cannot be masked, and an image that cannot be redacted is not
|
|
1364
1371
|
* written at all.
|
|
1365
1372
|
*/
|
|
1366
|
-
async screenshot(label, operation) {
|
|
1373
|
+
async screenshot(label, operation, budget) {
|
|
1367
1374
|
const attempt = this.attempt;
|
|
1368
1375
|
if (attempt === undefined)
|
|
1369
1376
|
throw invalidState('screenshot outside an attempt');
|
|
1370
|
-
const masked = await this.maskedScreenshot(operation
|
|
1377
|
+
const masked = await this.maskedScreenshot(operation, budget);
|
|
1371
1378
|
attempt.screenshots += 1;
|
|
1372
1379
|
const name = `${String(attempt.screenshots).padStart(3, '0')}-${sanitizeFilename(label ?? 'screenshot')}.png`;
|
|
1373
1380
|
const relative = path.join('screenshots', name);
|
|
@@ -1376,16 +1383,16 @@ export class AgentDeviceSurface {
|
|
|
1376
1383
|
return relative;
|
|
1377
1384
|
}
|
|
1378
1385
|
/** Raw device pixels; cleanup follows the capture even when its caller abandons it. */
|
|
1379
|
-
rawScreenshot(signal, capture
|
|
1380
|
-
return this.captureScreenshot(signal, async (shot, file) => new Uint8Array(readFileSync(shot.path ?? file)), capture);
|
|
1386
|
+
rawScreenshot(signal, capture, budget) {
|
|
1387
|
+
return this.captureScreenshot(signal, async (shot, file) => new Uint8Array(readFileSync(shot.path ?? file)), capture, budget);
|
|
1381
1388
|
}
|
|
1382
1389
|
/**
|
|
1383
1390
|
* One screenshot into a temp directory that is removed once `read` has
|
|
1384
1391
|
* taken what it needs from the response or the file, even when the caller
|
|
1385
1392
|
* abandons the capture.
|
|
1386
1393
|
*/
|
|
1387
|
-
captureScreenshot(signal, read, capture
|
|
1388
|
-
return this.command('screenshot', async (client) => {
|
|
1394
|
+
captureScreenshot(signal, read, capture, budget) {
|
|
1395
|
+
return budget.read(() => this.command('screenshot', async (client) => {
|
|
1389
1396
|
const directory = mkdtempSync(path.join(tmpdir(), 'e2e-agent-device-'));
|
|
1390
1397
|
const file = path.join(directory, 'screenshot.png');
|
|
1391
1398
|
try {
|
|
@@ -1394,7 +1401,7 @@ export class AgentDeviceSurface {
|
|
|
1394
1401
|
finally {
|
|
1395
1402
|
rmSync(directory, { recursive: true, force: true });
|
|
1396
1403
|
}
|
|
1397
|
-
}, signal);
|
|
1404
|
+
}, signal));
|
|
1398
1405
|
}
|
|
1399
1406
|
/**
|
|
1400
1407
|
* Screenshot with every secure node on the current screen painted over.
|
|
@@ -1402,16 +1409,9 @@ export class AgentDeviceSurface {
|
|
|
1402
1409
|
* throws when a secure field cannot be covered, because an image that may
|
|
1403
1410
|
* hold a credential must not leave the engine.
|
|
1404
1411
|
*/
|
|
1405
|
-
async maskedScreenshot(
|
|
1406
|
-
const
|
|
1407
|
-
|
|
1408
|
-
timeoutMs: 30_000,
|
|
1409
|
-
runId: '',
|
|
1410
|
-
attemptId: '',
|
|
1411
|
-
origin: 'test',
|
|
1412
|
-
};
|
|
1413
|
-
const projected = this.project(await this.snapshotOrEmpty(operation, false));
|
|
1414
|
-
const data = await this.rawScreenshot(signal);
|
|
1412
|
+
async maskedScreenshot(operation, budget) {
|
|
1413
|
+
const projected = this.project(await this.snapshotOrEmpty(operation, false, budget));
|
|
1414
|
+
const data = await this.rawScreenshot(operation.signal, {}, budget);
|
|
1415
1415
|
const masked = redactSecure(data, projected, this.knownViewport);
|
|
1416
1416
|
if (masked === undefined) {
|
|
1417
1417
|
throw new EngineError('ENGINE_FAILURE', 'a secure field on screen could not be masked; screenshot withheld', {
|
|
@@ -1422,14 +1422,17 @@ export class AgentDeviceSurface {
|
|
|
1422
1422
|
}
|
|
1423
1423
|
/**
|
|
1424
1424
|
* Viewport pixels for an observation. Best-effort: a screenshot that cannot
|
|
1425
|
-
* be produced or redacted costs the observation its image, not the step
|
|
1425
|
+
* be produced or redacted costs the observation its image, not the step,
|
|
1426
|
+
* unless the operation's budget ran out.
|
|
1426
1427
|
*/
|
|
1427
|
-
async capturePixels(operation, projected, viewport, capture) {
|
|
1428
|
+
async capturePixels(operation, projected, viewport, capture, budget) {
|
|
1428
1429
|
let raw;
|
|
1429
1430
|
try {
|
|
1430
|
-
raw = await this.rawScreenshot(operation.signal, capture);
|
|
1431
|
+
raw = await this.rawScreenshot(operation.signal, capture, budget);
|
|
1431
1432
|
}
|
|
1432
|
-
catch {
|
|
1433
|
+
catch (cause) {
|
|
1434
|
+
if (cause instanceof EngineError && cause.code === 'OPERATION_TIMEOUT')
|
|
1435
|
+
throw cause;
|
|
1433
1436
|
return undefined;
|
|
1434
1437
|
}
|
|
1435
1438
|
const redacted = redactSecure(raw, projected, viewport);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@e2e-dev/mobile",
|
|
3
|
-
"version": "0.12.
|
|
3
|
+
"version": "0.12.6-nightly-20261101030202",
|
|
4
4
|
"description": "Mobile engine (iOS and Android) for e2e agentic end-to-end testing, built on agent-device",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"e2e",
|
|
@@ -61,7 +61,7 @@
|
|
|
61
61
|
},
|
|
62
62
|
"peerDependencies": {
|
|
63
63
|
"ai": "^7.0.0",
|
|
64
|
-
"e2e": "
|
|
64
|
+
"e2e": "0.22.1-nightly-20261101030202"
|
|
65
65
|
},
|
|
66
66
|
"peerDependenciesMeta": {
|
|
67
67
|
"ai": {
|
|
@@ -69,10 +69,10 @@
|
|
|
69
69
|
}
|
|
70
70
|
},
|
|
71
71
|
"devDependencies": {
|
|
72
|
-
"@types/node": "26.6.
|
|
72
|
+
"@types/node": "26.6.6",
|
|
73
73
|
"@types/pngjs": "6.0.5",
|
|
74
|
-
"ai": "7.0.
|
|
75
|
-
"e2e": "0.22.
|
|
74
|
+
"ai": "7.0.159",
|
|
75
|
+
"e2e": "0.22.1-nightly-20261101030202",
|
|
76
76
|
"typescript": "7.0.2",
|
|
77
77
|
"vitest": "5.0.5"
|
|
78
78
|
},
|