mobile-debug-mcp 0.26.5 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +586 -192
- package/dist/server/common.js +172 -2
- package/dist/server-core.js +1 -1
- package/docs/CHANGELOG.md +6 -0
- package/docs/ROADMAP.md +109 -11
- package/docs/rfcs/010-verification-stabilization-and-temporal-convergence.md +265 -0
- package/docs/rfcs/011-recovery-and-replanning-for-failed-or-ambiguous-interaction-flows.md +321 -0
- package/docs/rfcs/011.1-recovery-contract-types-and-runtime-wiring-spec.md +253 -0
- package/docs/rfcs/012-action-trace-and-xecution-observability.md +242 -0
- package/docs/specs/mcp-tooling-spec-v1.md +26 -0
- package/docs/tools/interact.md +54 -0
- package/package.json +1 -1
- package/src/interact/index.ts +657 -194
- package/src/server/common.ts +236 -3
- package/src/server-core.ts +1 -1
- package/src/types.ts +59 -0
- package/test/device/manual/observe/rfc012_trace.manual.ts +51 -0
- package/test/unit/interact/adjust_control.test.ts +77 -1
- package/test/unit/interact/expect_tools.test.ts +57 -25
- package/test/unit/interact/verification_stabilization.test.ts +94 -0
- package/test/unit/server/common.test.ts +60 -1
package/src/server/common.ts
CHANGED
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
import type {
|
|
2
|
+
ActionTrace,
|
|
2
3
|
ActionExecutionResult,
|
|
3
4
|
ActionFailureCode,
|
|
4
|
-
ActionTargetResolved
|
|
5
|
+
ActionTargetResolved,
|
|
6
|
+
FailureClass,
|
|
7
|
+
RecoveryState,
|
|
8
|
+
TraceResult,
|
|
9
|
+
TraceStage,
|
|
10
|
+
TraceStep
|
|
5
11
|
} from '../types.js'
|
|
6
12
|
import { ToolsObserve } from '../observe/index.js'
|
|
7
13
|
|
|
14
|
+
export const DEFAULT_MAX_RECOVERY_ATTEMPTS = 3
|
|
15
|
+
export const DEFAULT_MAX_RETRY_DEPTH = 3
|
|
16
|
+
|
|
8
17
|
export function wrapResponse<T>(data: T) {
|
|
9
18
|
return {
|
|
10
19
|
content: [{
|
|
@@ -87,6 +96,172 @@ export async function captureActionFingerprint(platform?: 'android' | 'ios', dev
|
|
|
87
96
|
}
|
|
88
97
|
}
|
|
89
98
|
|
|
99
|
+
export function createTraceStep({
|
|
100
|
+
stage,
|
|
101
|
+
timestamp,
|
|
102
|
+
result,
|
|
103
|
+
attemptIndex,
|
|
104
|
+
cycleId,
|
|
105
|
+
metadata
|
|
106
|
+
}: {
|
|
107
|
+
stage: TraceStage
|
|
108
|
+
timestamp: number
|
|
109
|
+
result: TraceResult
|
|
110
|
+
attemptIndex: number
|
|
111
|
+
cycleId?: number
|
|
112
|
+
metadata?: Record<string, unknown>
|
|
113
|
+
}): TraceStep {
|
|
114
|
+
return {
|
|
115
|
+
stage,
|
|
116
|
+
timestamp,
|
|
117
|
+
result,
|
|
118
|
+
attempt_index: attemptIndex,
|
|
119
|
+
...(cycleId !== undefined ? { cycle_id: cycleId } : {}),
|
|
120
|
+
...(metadata ? { metadata } : {})
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export function buildActionTrace({
|
|
125
|
+
actionId,
|
|
126
|
+
actionType,
|
|
127
|
+
sourceModule,
|
|
128
|
+
selector,
|
|
129
|
+
resolved,
|
|
130
|
+
success,
|
|
131
|
+
failure,
|
|
132
|
+
details,
|
|
133
|
+
recovery,
|
|
134
|
+
attempts = 1,
|
|
135
|
+
steps
|
|
136
|
+
}: {
|
|
137
|
+
actionId: string
|
|
138
|
+
actionType: string
|
|
139
|
+
sourceModule: 'server' | 'interact'
|
|
140
|
+
selector: Record<string, unknown> | null
|
|
141
|
+
resolved?: Partial<ActionTargetResolved> | null
|
|
142
|
+
success: boolean
|
|
143
|
+
failure?: { failureCode: ActionFailureCode; retryable: boolean }
|
|
144
|
+
details?: Record<string, unknown>
|
|
145
|
+
recovery?: RecoveryState
|
|
146
|
+
attempts?: number
|
|
147
|
+
steps?: TraceStep[]
|
|
148
|
+
}): ActionTrace {
|
|
149
|
+
if (steps && steps.length > 0) {
|
|
150
|
+
return {
|
|
151
|
+
action_id: actionId,
|
|
152
|
+
steps,
|
|
153
|
+
final_outcome: success ? 'success' : 'failure',
|
|
154
|
+
attempts: Math.max(1, Math.floor(attempts || steps.length))
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const start = Date.now()
|
|
159
|
+
const builtSteps: TraceStep[] = []
|
|
160
|
+
let attemptIndex = 0
|
|
161
|
+
const totalAttempts = Math.max(1, Math.floor(attempts || 1))
|
|
162
|
+
|
|
163
|
+
if (selector || resolved) {
|
|
164
|
+
const stageResult: TraceResult = resolved ? 'success' : 'failure'
|
|
165
|
+
builtSteps.push(createTraceStep({
|
|
166
|
+
stage: 'resolve',
|
|
167
|
+
timestamp: start,
|
|
168
|
+
result: stageResult,
|
|
169
|
+
attemptIndex: attemptIndex++,
|
|
170
|
+
metadata: {
|
|
171
|
+
action_type: actionType,
|
|
172
|
+
source_module: sourceModule,
|
|
173
|
+
selector: selector ?? null,
|
|
174
|
+
resolved: resolved ? normalizeResolvedTarget(resolved) : null
|
|
175
|
+
}
|
|
176
|
+
}))
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
builtSteps.push(createTraceStep({
|
|
180
|
+
stage: 'execute',
|
|
181
|
+
timestamp: Date.now(),
|
|
182
|
+
result: success ? 'success' : 'failure',
|
|
183
|
+
attemptIndex: attemptIndex++,
|
|
184
|
+
metadata: {
|
|
185
|
+
action_type: actionType,
|
|
186
|
+
source_module: sourceModule,
|
|
187
|
+
...(failure ? { failure_code: failure.failureCode, retryable: failure.retryable } : {})
|
|
188
|
+
}
|
|
189
|
+
}))
|
|
190
|
+
|
|
191
|
+
const hasStabilizeDetails = Boolean(details && (
|
|
192
|
+
Object.prototype.hasOwnProperty.call(details, 'stabilization_attempts') ||
|
|
193
|
+
Object.prototype.hasOwnProperty.call(details, 'stable_observation_count') ||
|
|
194
|
+
Object.prototype.hasOwnProperty.call(details, 'snapshot_freshness_ms')
|
|
195
|
+
))
|
|
196
|
+
const hasVerifyDetails = Boolean(details && (
|
|
197
|
+
Object.prototype.hasOwnProperty.call(details, 'within_tolerance') ||
|
|
198
|
+
Object.prototype.hasOwnProperty.call(details, 'converged') ||
|
|
199
|
+
Object.prototype.hasOwnProperty.call(details, 'observed_state')
|
|
200
|
+
))
|
|
201
|
+
|
|
202
|
+
if (hasStabilizeDetails) {
|
|
203
|
+
builtSteps.push(createTraceStep({
|
|
204
|
+
stage: 'stabilize',
|
|
205
|
+
timestamp: Date.now(),
|
|
206
|
+
result: success ? 'success' : 'failure',
|
|
207
|
+
attemptIndex: attemptIndex++,
|
|
208
|
+
metadata: {
|
|
209
|
+
stabilization_attempts: (details?.stabilization_attempts as number | undefined) ?? null,
|
|
210
|
+
stable_observation_count: (details?.stable_observation_count as number | undefined) ?? null,
|
|
211
|
+
snapshot_freshness_ms: (details?.snapshot_freshness_ms as number | undefined) ?? null
|
|
212
|
+
}
|
|
213
|
+
}))
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
if (hasVerifyDetails) {
|
|
217
|
+
builtSteps.push(createTraceStep({
|
|
218
|
+
stage: 'verify',
|
|
219
|
+
timestamp: Date.now(),
|
|
220
|
+
result: success ? 'success' : 'failure',
|
|
221
|
+
attemptIndex: attemptIndex++,
|
|
222
|
+
metadata: {
|
|
223
|
+
within_tolerance: details?.within_tolerance ?? null,
|
|
224
|
+
converged: details?.converged ?? null,
|
|
225
|
+
actual_state: details?.actual_state ?? null,
|
|
226
|
+
reason: details?.reason ?? null
|
|
227
|
+
}
|
|
228
|
+
}))
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
if (failure) {
|
|
232
|
+
builtSteps.push(createTraceStep({
|
|
233
|
+
stage: 'recover',
|
|
234
|
+
timestamp: Date.now(),
|
|
235
|
+
result: failure.retryable ? 'retry' : 'failure',
|
|
236
|
+
attemptIndex: attemptIndex++,
|
|
237
|
+
metadata: {
|
|
238
|
+
failure_class: recovery?.failure_class ?? mapFailureCodeToFailureClass(failure.failureCode),
|
|
239
|
+
runtime_code: failure.failureCode,
|
|
240
|
+
retry_allowed: failure.retryable,
|
|
241
|
+
recovery_attempts: recovery?.recovery_attempts ?? 0,
|
|
242
|
+
retry_depth: recovery?.retry_depth ?? 0
|
|
243
|
+
}
|
|
244
|
+
}))
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
if (!builtSteps.length) {
|
|
248
|
+
builtSteps.push(createTraceStep({
|
|
249
|
+
stage: 'execute',
|
|
250
|
+
timestamp: start,
|
|
251
|
+
result: success ? 'success' : 'failure',
|
|
252
|
+
attemptIndex: 0,
|
|
253
|
+
metadata: { action_type: actionType, source_module: sourceModule }
|
|
254
|
+
}))
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
return {
|
|
258
|
+
action_id: actionId,
|
|
259
|
+
steps: builtSteps,
|
|
260
|
+
final_outcome: success ? 'success' : 'failure',
|
|
261
|
+
attempts: totalAttempts
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
90
265
|
export function normalizeResolvedTarget(value: Partial<ActionTargetResolved> | null = null): ActionTargetResolved | null {
|
|
91
266
|
if (!value) return null
|
|
92
267
|
return {
|
|
@@ -103,6 +278,7 @@ export function normalizeResolvedTarget(value: Partial<ActionTargetResolved> | n
|
|
|
103
278
|
|
|
104
279
|
export function inferGenericFailure(message: string | undefined): { failureCode: ActionFailureCode; retryable: boolean } {
|
|
105
280
|
if (message && /timeout/i.test(message)) return { failureCode: 'TIMEOUT', retryable: true }
|
|
281
|
+
if (message && /semantic mismatch/i.test(message)) return { failureCode: 'SEMANTIC_MISMATCH', retryable: false }
|
|
106
282
|
return { failureCode: 'UNKNOWN', retryable: false }
|
|
107
283
|
}
|
|
108
284
|
|
|
@@ -129,6 +305,42 @@ export function determineActionLifecycleState({
|
|
|
129
305
|
return ACTION_LIFECYCLE_STATE_BY_OUTCOME.success
|
|
130
306
|
}
|
|
131
307
|
|
|
308
|
+
function mapFailureCodeToFailureClass(code: ActionFailureCode): FailureClass {
|
|
309
|
+
switch (code) {
|
|
310
|
+
case 'ELEMENT_NOT_FOUND':
|
|
311
|
+
case 'AMBIGUOUS_TARGET':
|
|
312
|
+
case 'STALE_REFERENCE':
|
|
313
|
+
return 'TargetResolutionFailure'
|
|
314
|
+
case 'ELEMENT_NOT_INTERACTABLE':
|
|
315
|
+
return 'ExecutionFailure'
|
|
316
|
+
case 'TIMEOUT':
|
|
317
|
+
case 'ACTION_REJECTED':
|
|
318
|
+
case 'NAVIGATION_NO_CHANGE':
|
|
319
|
+
case 'UNKNOWN':
|
|
320
|
+
return 'ExecutionFailure'
|
|
321
|
+
case 'VERIFICATION_FAILED':
|
|
322
|
+
case 'EXPECT_STATE_MISMATCH':
|
|
323
|
+
return 'VerificationFailure'
|
|
324
|
+
case 'CONTROL_CONVERGENCE_FAILED':
|
|
325
|
+
return 'ControlConvergenceFailure'
|
|
326
|
+
case 'SEMANTIC_MISMATCH':
|
|
327
|
+
return 'SemanticMismatchFailure'
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
function buildRecoveryState(failureCode: ActionFailureCode, retryable: boolean): RecoveryState {
|
|
332
|
+
return {
|
|
333
|
+
failure_class: mapFailureCodeToFailureClass(failureCode),
|
|
334
|
+
runtime_code: failureCode,
|
|
335
|
+
recovery_attempts: 0,
|
|
336
|
+
max_recovery_attempts: DEFAULT_MAX_RECOVERY_ATTEMPTS,
|
|
337
|
+
retry_depth: 0,
|
|
338
|
+
max_retry_depth: DEFAULT_MAX_RETRY_DEPTH,
|
|
339
|
+
is_terminal: false,
|
|
340
|
+
retry_allowed: retryable
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
132
344
|
export function buildActionExecutionResult({
|
|
133
345
|
actionType,
|
|
134
346
|
device,
|
|
@@ -139,7 +351,8 @@ export function buildActionExecutionResult({
|
|
|
139
351
|
uiFingerprintAfter,
|
|
140
352
|
failure,
|
|
141
353
|
details,
|
|
142
|
-
sourceModule
|
|
354
|
+
sourceModule,
|
|
355
|
+
traceSteps
|
|
143
356
|
}: {
|
|
144
357
|
actionType: string
|
|
145
358
|
device?: ActionExecutionResult['device']
|
|
@@ -151,11 +364,17 @@ export function buildActionExecutionResult({
|
|
|
151
364
|
failure?: { failureCode: ActionFailureCode; retryable: boolean }
|
|
152
365
|
details?: Record<string, unknown>
|
|
153
366
|
sourceModule: 'server' | 'interact'
|
|
367
|
+
traceSteps?: TraceStep[]
|
|
154
368
|
}): ActionExecutionResult {
|
|
155
369
|
const timestampMs = Date.now()
|
|
156
370
|
const timestamp = new Date(timestampMs).toISOString()
|
|
371
|
+
const actionId = nextActionId(actionType, timestampMs)
|
|
372
|
+
const recoveryState = failure ? buildRecoveryState(failure.failureCode, failure.retryable) : undefined
|
|
373
|
+
const attempts = typeof details?.attempts === 'number' && Number.isFinite(details.attempts)
|
|
374
|
+
? Math.max(1, Math.floor(details.attempts))
|
|
375
|
+
: 1
|
|
157
376
|
return {
|
|
158
|
-
action_id:
|
|
377
|
+
action_id: actionId,
|
|
159
378
|
timestamp,
|
|
160
379
|
action_type: actionType,
|
|
161
380
|
lifecycle_state: determineActionLifecycleState({ success, failure }),
|
|
@@ -167,6 +386,20 @@ export function buildActionExecutionResult({
|
|
|
167
386
|
},
|
|
168
387
|
success,
|
|
169
388
|
...(failure ? { failure_code: failure.failureCode, retryable: failure.retryable } : {}),
|
|
389
|
+
...(recoveryState ? { recovery: recoveryState } : {}),
|
|
390
|
+
trace: buildActionTrace({
|
|
391
|
+
actionId,
|
|
392
|
+
actionType,
|
|
393
|
+
sourceModule,
|
|
394
|
+
selector,
|
|
395
|
+
resolved,
|
|
396
|
+
success,
|
|
397
|
+
failure,
|
|
398
|
+
details,
|
|
399
|
+
recovery: recoveryState,
|
|
400
|
+
attempts,
|
|
401
|
+
steps: traceSteps
|
|
402
|
+
}),
|
|
170
403
|
ui_fingerprint_before: uiFingerprintBefore,
|
|
171
404
|
ui_fingerprint_after: uiFingerprintAfter,
|
|
172
405
|
...(details ? { details } : {})
|
package/src/server-core.ts
CHANGED
package/src/types.ts
CHANGED
|
@@ -113,6 +113,39 @@ export interface UIElementSemanticMetadata {
|
|
|
113
113
|
state_shape?: 'continuous' | 'discrete' | 'semantic' | null;
|
|
114
114
|
}
|
|
115
115
|
|
|
116
|
+
export type TraceStage = 'resolve' | 'execute' | 'verify' | 'stabilize' | 'recover';
|
|
117
|
+
export type TraceResult = 'success' | 'failure' | 'retry';
|
|
118
|
+
|
|
119
|
+
export interface TraceStep {
|
|
120
|
+
stage: TraceStage;
|
|
121
|
+
timestamp: number;
|
|
122
|
+
result: TraceResult;
|
|
123
|
+
attempt_index: number;
|
|
124
|
+
cycle_id?: number;
|
|
125
|
+
metadata?: Record<string, unknown>;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export interface ActionTrace {
|
|
129
|
+
action_id: string;
|
|
130
|
+
steps: TraceStep[];
|
|
131
|
+
final_outcome: 'success' | 'failure';
|
|
132
|
+
attempts: number;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export type FailureClass =
|
|
136
|
+
| 'TargetResolutionFailure'
|
|
137
|
+
| 'ExecutionFailure'
|
|
138
|
+
| 'VerificationFailure'
|
|
139
|
+
| 'ControlConvergenceFailure'
|
|
140
|
+
| 'SemanticMismatchFailure';
|
|
141
|
+
|
|
142
|
+
export type RecoveryStrategy =
|
|
143
|
+
| 're_resolve'
|
|
144
|
+
| 'alternate_candidate'
|
|
145
|
+
| 'state_refresh'
|
|
146
|
+
| 'retry_adjustment'
|
|
147
|
+
| 'step_back';
|
|
148
|
+
|
|
116
149
|
export interface LoadingState {
|
|
117
150
|
active: boolean;
|
|
118
151
|
signal: string;
|
|
@@ -240,8 +273,25 @@ export type ActionFailureCode =
|
|
|
240
273
|
| 'NAVIGATION_NO_CHANGE'
|
|
241
274
|
| 'AMBIGUOUS_TARGET'
|
|
242
275
|
| 'STALE_REFERENCE'
|
|
276
|
+
| 'ACTION_REJECTED'
|
|
277
|
+
| 'VERIFICATION_FAILED'
|
|
278
|
+
| 'EXPECT_STATE_MISMATCH'
|
|
279
|
+
| 'CONTROL_CONVERGENCE_FAILED'
|
|
280
|
+
| 'SEMANTIC_MISMATCH'
|
|
243
281
|
| 'UNKNOWN'
|
|
244
282
|
|
|
283
|
+
export interface RecoveryState {
|
|
284
|
+
failure_class: FailureClass;
|
|
285
|
+
runtime_code: ActionFailureCode;
|
|
286
|
+
recovery_strategy?: RecoveryStrategy;
|
|
287
|
+
recovery_attempts: number;
|
|
288
|
+
max_recovery_attempts: number;
|
|
289
|
+
retry_depth: number;
|
|
290
|
+
max_retry_depth: number;
|
|
291
|
+
is_terminal: boolean;
|
|
292
|
+
retry_allowed?: boolean;
|
|
293
|
+
}
|
|
294
|
+
|
|
245
295
|
export interface ActionTargetResolved {
|
|
246
296
|
elementId: string | null;
|
|
247
297
|
text: string | null;
|
|
@@ -345,6 +395,8 @@ export interface ActionExecutionResult {
|
|
|
345
395
|
success: boolean;
|
|
346
396
|
failure_code?: ActionFailureCode;
|
|
347
397
|
retryable?: boolean;
|
|
398
|
+
recovery?: RecoveryState;
|
|
399
|
+
trace: ActionTrace;
|
|
348
400
|
ui_fingerprint_before?: string | null;
|
|
349
401
|
ui_fingerprint_after?: string | null;
|
|
350
402
|
details?: Record<string, unknown>;
|
|
@@ -368,6 +420,7 @@ export interface ExpectScreenResponse {
|
|
|
368
420
|
matched: boolean;
|
|
369
421
|
reason: string;
|
|
370
422
|
};
|
|
423
|
+
trace: ActionTrace;
|
|
371
424
|
}
|
|
372
425
|
|
|
373
426
|
export interface ExpectElementVisibleResponse {
|
|
@@ -391,6 +444,7 @@ export interface ExpectElementVisibleResponse {
|
|
|
391
444
|
reason?: string;
|
|
392
445
|
failure_code?: 'TIMEOUT' | 'ELEMENT_NOT_FOUND' | 'UNKNOWN';
|
|
393
446
|
retryable?: boolean;
|
|
447
|
+
trace: ActionTrace;
|
|
394
448
|
}
|
|
395
449
|
|
|
396
450
|
export interface ExpectStateResponse {
|
|
@@ -415,6 +469,11 @@ export interface ExpectStateResponse {
|
|
|
415
469
|
reason?: string;
|
|
416
470
|
failure_code?: 'ELEMENT_NOT_FOUND' | 'UNKNOWN';
|
|
417
471
|
retryable?: boolean;
|
|
472
|
+
stabilization_attempts?: number;
|
|
473
|
+
stabilization_window_ms?: number;
|
|
474
|
+
stable_observation_count?: number;
|
|
475
|
+
snapshot_freshness_ms?: number;
|
|
476
|
+
trace: ActionTrace;
|
|
418
477
|
}
|
|
419
478
|
|
|
420
479
|
export interface AdjustControlResponse extends ActionExecutionResult {
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import assert from 'assert'
|
|
2
|
+
import { ToolsInteract } from '../../../../src/interact/index.js'
|
|
3
|
+
import { ToolsObserve } from '../../../../src/observe/index.js'
|
|
4
|
+
|
|
5
|
+
async function verifyTrace(platform: 'android' | 'ios', deviceId?: string) {
|
|
6
|
+
const fingerprint = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as any
|
|
7
|
+
assert.ok(fingerprint, `${platform}: missing fingerprint response`)
|
|
8
|
+
assert.ok(fingerprint.fingerprint || fingerprint.activity, `${platform}: missing fingerprint or activity`)
|
|
9
|
+
|
|
10
|
+
const expected = fingerprint.fingerprint
|
|
11
|
+
? { fingerprint: fingerprint.fingerprint }
|
|
12
|
+
: { screen: fingerprint.activity }
|
|
13
|
+
|
|
14
|
+
const response = await ToolsInteract.expectScreenHandler({
|
|
15
|
+
platform,
|
|
16
|
+
deviceId,
|
|
17
|
+
...expected
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
assert.strictEqual(response.success, true, `${platform}: expect_screen did not succeed`)
|
|
21
|
+
assert.ok(response.trace, `${platform}: trace missing`)
|
|
22
|
+
assert.strictEqual(response.trace.final_outcome, 'success', `${platform}: trace outcome mismatch`)
|
|
23
|
+
assert.ok(Array.isArray(response.trace.steps) && response.trace.steps.length > 0, `${platform}: trace steps missing`)
|
|
24
|
+
assert.strictEqual(response.trace.steps[0].stage, 'verify', `${platform}: expected verify stage`)
|
|
25
|
+
|
|
26
|
+
console.log(`${platform}: RFC 012 trace verified`)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async function main() {
|
|
30
|
+
const args = process.argv.slice(2)
|
|
31
|
+
const platform = args[0] as 'android' | 'ios' | undefined
|
|
32
|
+
const deviceId = args[1]
|
|
33
|
+
|
|
34
|
+
if (platform === 'android') {
|
|
35
|
+
await verifyTrace('android', deviceId)
|
|
36
|
+
return
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (platform === 'ios') {
|
|
40
|
+
await verifyTrace('ios', deviceId)
|
|
41
|
+
return
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
await verifyTrace('android')
|
|
45
|
+
await verifyTrace('ios')
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
main().catch((error) => {
|
|
49
|
+
console.error(error)
|
|
50
|
+
process.exit(1)
|
|
51
|
+
})
|
|
@@ -346,9 +346,85 @@ async function run() {
|
|
|
346
346
|
assert.strictEqual(cachedResolveAdjust.success, true)
|
|
347
347
|
assert.strictEqual(cachedResolveAdjust.converged, true)
|
|
348
348
|
assert.strictEqual(cachedResolveAdjust.within_tolerance, true)
|
|
349
|
-
assert.
|
|
349
|
+
assert.ok(cachedResolveAdjust.attempts >= 3)
|
|
350
350
|
assert.strictEqual(treeFetches, 1, 'second attempt should reuse the resolved element instead of refetching the UI tree')
|
|
351
351
|
|
|
352
|
+
const probeTapStart = tapCalls.length
|
|
353
|
+
const probeSwipeStart = swipeCalls.length
|
|
354
|
+
let probeVerificationCount = 0
|
|
355
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => ({
|
|
356
|
+
device: { platform: 'android', id: 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
357
|
+
screen: '',
|
|
358
|
+
resolution: { width: 1080, height: 2400 },
|
|
359
|
+
elements: [
|
|
360
|
+
{
|
|
361
|
+
text: 'Duration',
|
|
362
|
+
type: 'android.widget.SeekBar',
|
|
363
|
+
contentDescription: null,
|
|
364
|
+
clickable: true,
|
|
365
|
+
enabled: true,
|
|
366
|
+
visible: true,
|
|
367
|
+
bounds: [0, 0, 200, 40],
|
|
368
|
+
resourceId: 'seek_duration',
|
|
369
|
+
state: {
|
|
370
|
+
value: 10,
|
|
371
|
+
raw_value: 10,
|
|
372
|
+
value_range: { min: 0, max: 20 }
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
]
|
|
376
|
+
})
|
|
377
|
+
|
|
378
|
+
;(ToolsInteract as any).tapHandler = async ({ platform, x, y, deviceId }: any) => {
|
|
379
|
+
tapCalls.push({ platform, x, y, deviceId })
|
|
380
|
+
return {
|
|
381
|
+
device: { platform: platform || 'android', id: deviceId || 'mock-device', osVersion: '14', model: 'Pixel', simulator: true },
|
|
382
|
+
success: true,
|
|
383
|
+
x,
|
|
384
|
+
y
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
;(ToolsInteract as any).expectStateHandler = async () => {
|
|
389
|
+
probeVerificationCount++
|
|
390
|
+
const value = probeVerificationCount === 1 ? 11 : 12
|
|
391
|
+
return {
|
|
392
|
+
success: true,
|
|
393
|
+
selector: { text: 'Duration' },
|
|
394
|
+
element_id: wait.element.elementId,
|
|
395
|
+
expected_state: { property: 'value', expected: 12 },
|
|
396
|
+
element: {
|
|
397
|
+
elementId: wait.element.elementId,
|
|
398
|
+
text: 'Duration',
|
|
399
|
+
resource_id: 'seek_duration',
|
|
400
|
+
accessibility_id: null,
|
|
401
|
+
class: 'android.widget.SeekBar',
|
|
402
|
+
bounds: [0, 0, 200, 40],
|
|
403
|
+
index: 0,
|
|
404
|
+
state: { value, raw_value: value, value_range: { min: 0, max: 20 } }
|
|
405
|
+
},
|
|
406
|
+
observed_state: { property: 'value', value, raw_value: value },
|
|
407
|
+
reason: value === 12 ? 'value matches expected value' : 'value still below target'
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
const probeAdjust = await ToolsInteract.adjustControlHandler({
|
|
412
|
+
element_id: wait.element.elementId,
|
|
413
|
+
property: 'value',
|
|
414
|
+
targetValue: 12,
|
|
415
|
+
tolerance: 0.5,
|
|
416
|
+
maxAttempts: 3,
|
|
417
|
+
platform: 'android'
|
|
418
|
+
})
|
|
419
|
+
|
|
420
|
+
assert.strictEqual(probeAdjust.success, true)
|
|
421
|
+
assert.strictEqual(probeAdjust.converged, true)
|
|
422
|
+
assert.strictEqual(probeAdjust.within_tolerance, true)
|
|
423
|
+
assert.strictEqual(probeAdjust.adjustment_mode, 'coordinate')
|
|
424
|
+
assert.strictEqual(probeAdjust.attempts, 2)
|
|
425
|
+
assert.strictEqual(tapCalls.length, probeTapStart + 2)
|
|
426
|
+
assert.strictEqual(swipeCalls.length, probeSwipeStart)
|
|
427
|
+
|
|
352
428
|
console.log('adjust_control unit tests passed')
|
|
353
429
|
} finally {
|
|
354
430
|
;(Observe as any).ToolsObserve.getUITreeHandler = originalGetUITreeHandler
|
|
@@ -6,22 +6,23 @@ async function run() {
|
|
|
6
6
|
console.log('Starting expect_* unit tests...')
|
|
7
7
|
const originalGetScreenFingerprintHandler = (Observe as any).ToolsObserve.getScreenFingerprintHandler
|
|
8
8
|
const originalGetCurrentScreenHandler = (Observe as any).ToolsObserve.getCurrentScreenHandler
|
|
9
|
+
const originalGetUITreeHandler = (Observe as any).ToolsObserve.getUITreeHandler
|
|
9
10
|
const originalWaitForUIHandler = (ToolsInteract as any).waitForUIHandler
|
|
10
11
|
|
|
11
12
|
try {
|
|
12
13
|
;(Observe as any).ToolsObserve.getScreenFingerprintHandler = async () => ({ fingerprint: 'fp_home', activity: 'com.example.HomeActivity' })
|
|
13
14
|
let expectScreen = await ToolsInteract.expectScreenHandler({ platform: 'android', fingerprint: 'fp_home' })
|
|
14
|
-
assert.
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
reason: 'observed fingerprint matches expected fingerprint fp_home'
|
|
23
|
-
}
|
|
15
|
+
assert.strictEqual(expectScreen.success, true)
|
|
16
|
+
assert.deepStrictEqual(expectScreen.observed_screen, { fingerprint: 'fp_home', screen: 'com.example.HomeActivity' })
|
|
17
|
+
assert.deepStrictEqual(expectScreen.expected_screen, { fingerprint: 'fp_home', screen: null })
|
|
18
|
+
assert.strictEqual(expectScreen.confidence, 1)
|
|
19
|
+
assert.deepStrictEqual(expectScreen.comparison, {
|
|
20
|
+
basis: 'fingerprint',
|
|
21
|
+
matched: true,
|
|
22
|
+
reason: 'observed fingerprint matches expected fingerprint fp_home'
|
|
24
23
|
})
|
|
24
|
+
assert.strictEqual(expectScreen.trace.final_outcome, 'success')
|
|
25
|
+
assert.strictEqual(expectScreen.trace.steps[0].stage, 'verify')
|
|
25
26
|
|
|
26
27
|
;(Observe as any).ToolsObserve.getCurrentScreenHandler = async () => ({
|
|
27
28
|
activity: 'com.example.HomeActivity',
|
|
@@ -58,6 +59,8 @@ async function run() {
|
|
|
58
59
|
assert.strictEqual(expectElementVisible.element?.resource_id, 'rid_ready')
|
|
59
60
|
assert.strictEqual(expectElementVisible.expected_condition, 'visible')
|
|
60
61
|
assert.strictEqual(expectElementVisible.reason, 'selector is visible')
|
|
62
|
+
assert.strictEqual(expectElementVisible.trace.final_outcome, 'success')
|
|
63
|
+
assert.strictEqual(expectElementVisible.trace.steps[0].stage, 'verify')
|
|
61
64
|
|
|
62
65
|
;(ToolsInteract as any).waitForUIHandler = async () => ({
|
|
63
66
|
status: 'timeout',
|
|
@@ -73,27 +76,56 @@ async function run() {
|
|
|
73
76
|
selector: { text: 'Missing' },
|
|
74
77
|
platform: 'android'
|
|
75
78
|
})
|
|
76
|
-
assert.
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
79
|
+
assert.strictEqual(timeoutResult.success, false)
|
|
80
|
+
assert.deepStrictEqual(timeoutResult.selector, { text: 'Missing' })
|
|
81
|
+
assert.strictEqual(timeoutResult.element_id, null)
|
|
82
|
+
assert.strictEqual(timeoutResult.expected_condition, 'visible')
|
|
83
|
+
assert.deepStrictEqual(timeoutResult.observed, {
|
|
84
|
+
status: 'timeout',
|
|
85
|
+
matched_count: 0,
|
|
86
|
+
condition_satisfied: false,
|
|
87
|
+
selected_index: null,
|
|
88
|
+
last_matched_element: null
|
|
89
|
+
})
|
|
90
|
+
assert.strictEqual(timeoutResult.reason, 'Condition visible not satisfied within timeout; observed 0 match(es)')
|
|
91
|
+
assert.strictEqual(timeoutResult.failure_code, 'TIMEOUT')
|
|
92
|
+
assert.strictEqual(timeoutResult.retryable, true)
|
|
93
|
+
assert.strictEqual(timeoutResult.trace.final_outcome, 'failure')
|
|
94
|
+
|
|
95
|
+
let uiTreeCallCount = 0
|
|
96
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = async () => {
|
|
97
|
+
uiTreeCallCount++
|
|
98
|
+
return {
|
|
99
|
+
device: { platform: 'android', id: 'emulator-5554' },
|
|
100
|
+
captured_at_ms: Date.now(),
|
|
101
|
+
elements: [{
|
|
102
|
+
elementId: 'el_state',
|
|
103
|
+
text: 'Stateful',
|
|
104
|
+
resource_id: 'rid_state',
|
|
105
|
+
accessibility_id: null,
|
|
106
|
+
class: 'TextView',
|
|
107
|
+
bounds: [0, 0, 10, 10],
|
|
108
|
+
index: 0,
|
|
109
|
+
state: { enabled: true, value: 7 }
|
|
110
|
+
}]
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const expectState = await ToolsInteract.expectStateHandler({
|
|
114
|
+
selector: { text: 'Stateful' },
|
|
115
|
+
property: 'value',
|
|
116
|
+
expected: 7,
|
|
117
|
+
platform: 'android'
|
|
91
118
|
})
|
|
119
|
+
assert.strictEqual(expectState.success, true)
|
|
120
|
+
assert.strictEqual(uiTreeCallCount >= 2, true)
|
|
121
|
+
assert.ok(expectState.trace.steps.filter((step) => step.stage === 'stabilize').length >= 2)
|
|
122
|
+
assert.ok(expectState.trace.steps.filter((step) => step.stage === 'verify').length >= 2)
|
|
92
123
|
|
|
93
124
|
console.log('expect_* unit tests passed')
|
|
94
125
|
} finally {
|
|
95
126
|
;(Observe as any).ToolsObserve.getScreenFingerprintHandler = originalGetScreenFingerprintHandler
|
|
96
127
|
;(Observe as any).ToolsObserve.getCurrentScreenHandler = originalGetCurrentScreenHandler
|
|
128
|
+
;(Observe as any).ToolsObserve.getUITreeHandler = originalGetUITreeHandler
|
|
97
129
|
;(ToolsInteract as any).waitForUIHandler = originalWaitForUIHandler
|
|
98
130
|
}
|
|
99
131
|
}
|