mobile-debug-mcp 0.26.5 → 0.28.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,8 +6,9 @@ export { AndroidInteract, iOSInteract };
6
6
  import { resolveTargetDevice } from '../utils/resolve-device.js'
7
7
  import { ToolsObserve } from '../observe/index.js'
8
8
  import { computeSnapshotSignature } from '../observe/snapshot-metadata.js'
9
- import { buildActionExecutionResult } from '../server/common.js'
9
+ import { buildActionExecutionResult, createTraceStep, nextActionId } from '../server/common.js'
10
10
  import type {
11
+ ActionTrace,
11
12
  ActionFailureCode,
12
13
  ActionTargetResolved,
13
14
  AdjustControlResponse,
@@ -18,6 +19,7 @@ import type {
18
19
  WaitForUIChangeResponse,
19
20
  UIElementSemanticMetadata,
20
21
  UIElementState,
22
+ TraceStep,
21
23
  TapElementResponse
22
24
  } from '../types.js'
23
25
 
@@ -79,6 +81,39 @@ interface RankedResolutionCandidate {
79
81
  interactable: boolean
80
82
  }
81
83
 
84
+ function buildObservationTrace({
85
+ actionType,
86
+ stage,
87
+ success,
88
+ attempts,
89
+ metadata
90
+ }: {
91
+ actionType: string
92
+ stage: 'verify' | 'stabilize' | 'resolve'
93
+ success: boolean
94
+ attempts: number
95
+ metadata?: Record<string, unknown>
96
+ }): ActionTrace {
97
+ const now = Date.now()
98
+ const actionId = nextActionId(actionType, now)
99
+ const steps: TraceStep[] = [
100
+ createTraceStep({
101
+ stage,
102
+ timestamp: now,
103
+ result: success ? 'success' : 'failure',
104
+ attemptIndex: 0,
105
+ metadata
106
+ })
107
+ ]
108
+
109
+ return {
110
+ action_id: actionId,
111
+ steps,
112
+ final_outcome: success ? 'success' : 'failure',
113
+ attempts: Math.max(1, Math.floor(attempts || 1))
114
+ }
115
+ }
116
+
82
117
  interface FindElementResolutionSummary {
83
118
  confidence: number
84
119
  reason: string
@@ -417,6 +452,59 @@ export class ToolsInteract {
417
452
  return ToolsInteract._buildControlPoint(bounds, safeRatio, axis)
418
453
  }
419
454
 
455
+ private static _buildAdjustmentProbePoints(
456
+ bounds: [number, number, number, number],
457
+ targetValue: number,
458
+ currentValue: number | null,
459
+ min: number,
460
+ max: number,
461
+ axis: 'horizontal' | 'vertical'
462
+ ) {
463
+ const targetPoint = ToolsInteract._buildConservativeControlPoint(bounds, targetValue, currentValue, min, max, axis)
464
+ const currentPoint = currentValue !== null
465
+ ? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
466
+ : ToolsInteract._buildControlPoint(bounds, 0.5, axis)
467
+
468
+ const [left, top, right, bottom] = bounds
469
+ const width = Math.max(1, right - left)
470
+ const height = Math.max(1, bottom - top)
471
+ const crossAxisBumps = axis === 'horizontal'
472
+ ? [Math.max(24, Math.floor(height * 0.75)), Math.max(40, Math.floor(height * 1.5))]
473
+ : [Math.max(24, Math.floor(width * 0.75)), Math.max(40, Math.floor(width * 1.5))]
474
+
475
+ const clampPoint = (point: { x: number, y: number }) => ({
476
+ x: axis === 'horizontal'
477
+ ? Math.max(left, Math.min(right, point.x))
478
+ : Math.max(left, Math.min(right + Math.max(width, height), point.x)),
479
+ y: axis === 'vertical'
480
+ ? Math.max(top, Math.min(bottom, point.y))
481
+ : Math.max(top, Math.min(bottom + Math.max(height, width), point.y))
482
+ })
483
+
484
+ const probes = [targetPoint, currentPoint]
485
+ for (const bump of crossAxisBumps) {
486
+ if (axis === 'horizontal') {
487
+ probes.push(
488
+ { x: targetPoint.x, y: bottom + bump },
489
+ { x: currentPoint.x, y: bottom + bump }
490
+ )
491
+ } else {
492
+ probes.push(
493
+ { x: right + bump, y: targetPoint.y },
494
+ { x: right + bump, y: currentPoint.y }
495
+ )
496
+ }
497
+ }
498
+
499
+ return Array.from(
500
+ new Map(
501
+ probes
502
+ .map(clampPoint)
503
+ .map((point) => [`${point.x}:${point.y}`, point] as const)
504
+ ).values()
505
+ )
506
+ }
507
+
420
508
  private static _controlAxis(el: UiElement, bounds: [number, number, number, number]): 'horizontal' | 'vertical' {
421
509
  const type = ToolsInteract._normalize(el.type ?? el.class ?? '')
422
510
  const role = ToolsInteract._normalize(el.role ?? '')
@@ -689,6 +777,22 @@ export class ToolsInteract {
689
777
  let resolvedDeviceId = deviceId
690
778
  const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
691
779
  let semanticFallbackElement: FindElementResponse['element'] | null = null
780
+ const traceSteps: TraceStep[] = []
781
+ let traceAttemptIndex = 0
782
+
783
+ const recordTraceStep = (
784
+ stage: TraceStep['stage'],
785
+ result: TraceStep['result'],
786
+ metadata?: Record<string, unknown>
787
+ ) => {
788
+ traceSteps.push(createTraceStep({
789
+ stage,
790
+ timestamp: Date.now(),
791
+ result,
792
+ attemptIndex: traceAttemptIndex++,
793
+ metadata
794
+ }))
795
+ }
692
796
 
693
797
  const buildFailure = (
694
798
  failureCode: ActionFailureCode,
@@ -701,6 +805,22 @@ export class ToolsInteract {
701
805
  retryable = false,
702
806
  uiFingerprintAfter: string | null = null
703
807
  ): AdjustControlResponse => {
808
+ if (!traceSteps.some((step) => step.stage === 'resolve')) {
809
+ recordTraceStep('resolve', 'failure',
810
+ {
811
+ reason,
812
+ failure_code: failureCode
813
+ })
814
+ }
815
+ if (!traceSteps.some((step) => step.stage === 'recover')) {
816
+ recordTraceStep('recover', retryable ? 'retry' : 'failure', {
817
+ reason,
818
+ failure_code: failureCode,
819
+ retry_allowed: retryable,
820
+ recovery_attempts: attempts,
821
+ retry_depth: attempts
822
+ })
823
+ }
704
824
  const base = buildActionExecutionResult({
705
825
  actionType,
706
826
  sourceModule: 'interact',
@@ -721,7 +841,8 @@ export class ToolsInteract {
721
841
  converged: false,
722
842
  within_tolerance: false,
723
843
  reason
724
- }
844
+ },
845
+ traceSteps
725
846
  }) as AdjustControlResponse
726
847
 
727
848
  return {
@@ -946,11 +1067,25 @@ export class ToolsInteract {
946
1067
 
947
1068
  lastObservedState = actualState
948
1069
 
1070
+ if (!traceSteps.some((step) => step.stage === 'resolve')) {
1071
+ recordTraceStep('resolve', 'success', {
1072
+ resolved_target: resolvedTarget,
1073
+ current_value: currentValue,
1074
+ adjustment_mode: lastAdjustmentMode
1075
+ })
1076
+ }
1077
+
949
1078
  if (property !== 'value' && property !== 'raw_value') {
950
1079
  return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
951
1080
  }
952
1081
 
953
1082
  if (currentValue !== null && Math.abs(currentValue - targetValue) <= normalizedTolerance) {
1083
+ recordTraceStep('verify', 'success', {
1084
+ property,
1085
+ target_value: targetValue,
1086
+ actual_state: actualState,
1087
+ reason: 'control already within tolerance'
1088
+ })
954
1089
  const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
955
1090
  const base = buildActionExecutionResult({
956
1091
  actionType,
@@ -971,7 +1106,8 @@ export class ToolsInteract {
971
1106
  converged: true,
972
1107
  within_tolerance: true,
973
1108
  reason: 'control already within tolerance'
974
- }
1109
+ },
1110
+ traceSteps
975
1111
  }) as AdjustControlResponse
976
1112
 
977
1113
  return {
@@ -1012,6 +1148,7 @@ export class ToolsInteract {
1012
1148
  const currentPoint = currentValue !== null
1013
1149
  ? ToolsInteract._buildControlPoint(bounds, (currentValue - min) / (max - min), axis)
1014
1150
  : ToolsInteract._buildControlPoint(bounds, 0.5, axis)
1151
+ const probePoints = ToolsInteract._buildAdjustmentProbePoints(bounds, targetValue, currentValue, min, max, axis)
1015
1152
 
1016
1153
  const runVerification = async (): Promise<{
1017
1154
  verification: any
@@ -1047,42 +1184,98 @@ export class ToolsInteract {
1047
1184
  }
1048
1185
  }
1049
1186
 
1050
- lastAdjustmentMode = 'coordinate'
1051
- const primaryActionResult = await ToolsInteract.tapHandler({
1052
- platform: resolvedPlatform,
1053
- x: targetPoint.x,
1054
- y: targetPoint.y,
1055
- deviceId: resolvedDeviceId
1056
- })
1057
- let actionDevice = primaryActionResult.device ?? currentDevice
1058
- attemptCount++
1059
-
1060
- if (!primaryActionResult.success) {
1061
- lastAdjustmentMode = 'gesture'
1062
- const fallbackActionResult = await ToolsInteract.swipeHandler({
1187
+ let actionDevice: any = currentDevice
1188
+ let observedState: { property: string; value: number | null; raw_value?: number | null } | null = actualState
1189
+ let verification: any = null
1190
+ let verificationResult: any = { verification: null, observedState: actualState, withinTolerance: false }
1191
+
1192
+ for (let i = 0; i < probePoints.length; i++) {
1193
+ const probePoint = probePoints[i]
1194
+ lastAdjustmentMode = 'coordinate'
1195
+ recordTraceStep('execute', 'retry', {
1196
+ attempt: attemptCount + 1,
1197
+ mode: 'coordinate',
1198
+ point: probePoint
1199
+ })
1200
+ const actionResult = await ToolsInteract.tapHandler({
1063
1201
  platform: resolvedPlatform,
1064
- x1: currentPoint.x,
1065
- y1: currentPoint.y,
1066
- x2: targetPoint.x,
1067
- y2: targetPoint.y,
1068
- duration: 220,
1202
+ x: probePoint.x,
1203
+ y: probePoint.y,
1069
1204
  deviceId: resolvedDeviceId
1070
1205
  })
1071
1206
  attemptCount++
1072
-
1073
- if (!fallbackActionResult.success) {
1074
- return buildFailure('UNKNOWN', fallbackActionResult.error ?? primaryActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? primaryActionResult.device, actualState, attemptCount, lastAdjustmentMode, false)
1207
+ actionDevice = actionResult.device ?? actionDevice
1208
+
1209
+ if (!actionResult.success) {
1210
+ recordTraceStep('execute', 'retry', {
1211
+ attempt: attemptCount,
1212
+ mode: 'coordinate',
1213
+ point: probePoint,
1214
+ success: false
1215
+ })
1216
+ continue
1075
1217
  }
1076
1218
 
1077
- actionDevice = fallbackActionResult.device ?? actionDevice
1078
- }
1219
+ verificationResult = await runVerification()
1220
+ observedState = verificationResult.observedState
1221
+ lastObservedState = observedState
1222
+ recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1223
+ attempt: attemptCount,
1224
+ property,
1225
+ target_value: targetValue,
1226
+ actual_state: observedState,
1227
+ reason: verificationResult.verification?.reason ?? 'control did not converge yet'
1228
+ })
1079
1229
 
1080
- let verificationResult = await runVerification()
1081
- let observedState = verificationResult.observedState
1082
- lastObservedState = observedState
1230
+ if (verificationResult.withinTolerance) {
1231
+ const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1232
+ const base = buildActionExecutionResult({
1233
+ actionType,
1234
+ sourceModule: 'interact',
1235
+ device: actionDevice ?? currentDevice,
1236
+ selector: targetSelector,
1237
+ resolved: resolvedTarget,
1238
+ success: true,
1239
+ uiFingerprintBefore: fingerprintBefore,
1240
+ uiFingerprintAfter,
1241
+ details: {
1242
+ target_value: targetValue,
1243
+ tolerance: normalizedTolerance,
1244
+ property,
1245
+ attempts: attemptCount,
1246
+ adjustment_mode: lastAdjustmentMode,
1247
+ actual_state: observedState,
1248
+ converged: true,
1249
+ within_tolerance: true,
1250
+ reason: verificationResult.verification?.reason ?? 'control converged to target value'
1251
+ },
1252
+ traceSteps
1253
+ }) as AdjustControlResponse
1083
1254
 
1084
- if (!verificationResult.withinTolerance && currentValue !== null) {
1255
+ return {
1256
+ ...base,
1257
+ target_state: {
1258
+ property,
1259
+ target_value: targetValue,
1260
+ tolerance: normalizedTolerance
1261
+ },
1262
+ actual_state: observedState,
1263
+ within_tolerance: true,
1264
+ converged: true,
1265
+ attempts: attemptCount,
1266
+ adjustment_mode: lastAdjustmentMode
1267
+ }
1268
+ }
1269
+ }
1270
+
1271
+ if (currentValue !== null) {
1085
1272
  lastAdjustmentMode = 'gesture'
1273
+ recordTraceStep('execute', 'retry', {
1274
+ attempt: attemptCount + 1,
1275
+ mode: 'gesture',
1276
+ start: currentPoint,
1277
+ end: targetPoint
1278
+ })
1086
1279
  const fallbackActionResult = await ToolsInteract.swipeHandler({
1087
1280
  platform: resolvedPlatform,
1088
1281
  x1: currentPoint.x,
@@ -1094,17 +1287,80 @@ export class ToolsInteract {
1094
1287
  })
1095
1288
  attemptCount++
1096
1289
  if (!fallbackActionResult.success) {
1097
- return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device, observedState ?? actualState, attemptCount, lastAdjustmentMode, false)
1290
+ recordTraceStep('execute', 'failure', {
1291
+ attempt: attemptCount,
1292
+ mode: 'gesture',
1293
+ start: currentPoint,
1294
+ end: targetPoint,
1295
+ success: false
1296
+ })
1297
+ return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? actionDevice, observedState ?? actualState, attemptCount, lastAdjustmentMode, false)
1098
1298
  }
1099
1299
 
1300
+ actionDevice = fallbackActionResult.device ?? actionDevice
1100
1301
  verificationResult = await runVerification()
1101
1302
  observedState = verificationResult.observedState
1303
+ lastObservedState = observedState
1304
+ recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
1305
+ attempt: attemptCount,
1306
+ property,
1307
+ target_value: targetValue,
1308
+ actual_state: observedState,
1309
+ reason: verificationResult.verification?.reason ?? 'gesture adjustment did not converge yet'
1310
+ })
1311
+
1312
+ if (verificationResult.withinTolerance) {
1313
+ const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1314
+ const base = buildActionExecutionResult({
1315
+ actionType,
1316
+ sourceModule: 'interact',
1317
+ device: actionDevice ?? currentDevice,
1318
+ selector: targetSelector,
1319
+ resolved: resolvedTarget,
1320
+ success: true,
1321
+ uiFingerprintBefore: fingerprintBefore,
1322
+ uiFingerprintAfter,
1323
+ details: {
1324
+ target_value: targetValue,
1325
+ tolerance: normalizedTolerance,
1326
+ property,
1327
+ attempts: attemptCount,
1328
+ adjustment_mode: lastAdjustmentMode,
1329
+ actual_state: observedState,
1330
+ converged: true,
1331
+ within_tolerance: true,
1332
+ reason: verificationResult.verification?.reason ?? 'control converged to target value'
1333
+ },
1334
+ traceSteps
1335
+ }) as AdjustControlResponse
1336
+
1337
+ return {
1338
+ ...base,
1339
+ target_state: {
1340
+ property,
1341
+ target_value: targetValue,
1342
+ tolerance: normalizedTolerance
1343
+ },
1344
+ actual_state: observedState,
1345
+ within_tolerance: true,
1346
+ converged: true,
1347
+ attempts: attemptCount,
1348
+ adjustment_mode: lastAdjustmentMode
1349
+ }
1350
+ }
1102
1351
  }
1103
1352
 
1104
- const verification = verificationResult.verification
1353
+ verification = verificationResult.verification
1105
1354
  lastObservedState = observedState
1106
1355
 
1107
1356
  if (verificationResult.withinTolerance) {
1357
+ recordTraceStep('verify', 'success', {
1358
+ attempt: attemptCount,
1359
+ property,
1360
+ target_value: targetValue,
1361
+ actual_state: observedState,
1362
+ reason: verification?.reason ?? 'control converged to target value'
1363
+ })
1108
1364
  const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
1109
1365
  const base = buildActionExecutionResult({
1110
1366
  actionType,
@@ -1125,7 +1381,8 @@ export class ToolsInteract {
1125
1381
  converged: true,
1126
1382
  within_tolerance: true,
1127
1383
  reason: verification?.reason ?? 'control converged to target value'
1128
- }
1384
+ },
1385
+ traceSteps
1129
1386
  }) as AdjustControlResponse
1130
1387
 
1131
1388
  return {
@@ -1494,6 +1751,10 @@ export class ToolsInteract {
1494
1751
  let lastMatchedCount = 0
1495
1752
  let lastMatchedElement: ActionTargetResolved | null = null
1496
1753
  let lastConditionSatisfied = false
1754
+ let matchedAt: number | null = null
1755
+ let stableMatchCount = 0
1756
+ const stableObservationCount = 2
1757
+ const snapshotStaleThresholdMs = 500
1497
1758
 
1498
1759
  // Precompute normalized selector values and helpers (constant across polls)
1499
1760
  const normalize = ToolsInteract._normalize
@@ -1599,25 +1860,35 @@ export class ToolsInteract {
1599
1860
  lastMatchedCount = matchedCount
1600
1861
  lastConditionSatisfied = conditionMet
1601
1862
  lastMatchedElement = matchedElement ? ToolsInteract._buildResolvedElement(resolvedPlatform, resolvedDeviceId, matchedElement.el, matchedElement.idx) : null
1863
+ const now = Date.now()
1602
1864
 
1603
- if (conditionMet) {
1604
- const now = Date.now()
1605
- const latency_ms = now - overallStart
1606
- const outEl = lastMatchedElement
1865
+ const snapshotAgeMs = typeof tree?.captured_at_ms === 'number' ? now - tree.captured_at_ms : null
1866
+ const snapshotFresh = snapshotAgeMs === null || snapshotAgeMs <= snapshotStaleThresholdMs
1607
1867
 
1608
- return {
1609
- status: 'success',
1610
- matched: matchedCount,
1611
- element: outEl,
1612
- metrics: { latency_ms, poll_count: totalPollCount, attempts },
1613
- requested,
1614
- observed: {
1615
- matched_count: matchedCount,
1616
- condition_satisfied: true,
1617
- selected_index: outEl?.index ?? null,
1618
- last_matched_element: outEl
1868
+ if (conditionMet && snapshotFresh) {
1869
+ if (matchedAt === null) matchedAt = now
1870
+ stableMatchCount++
1871
+ if (stableMatchCount >= stableObservationCount) {
1872
+ const latency_ms = now - overallStart
1873
+ const outEl = lastMatchedElement
1874
+
1875
+ return {
1876
+ status: 'success',
1877
+ matched: matchedCount,
1878
+ element: outEl,
1879
+ metrics: { latency_ms, poll_count: totalPollCount, attempts },
1880
+ requested,
1881
+ observed: {
1882
+ matched_count: matchedCount,
1883
+ condition_satisfied: true,
1884
+ selected_index: outEl?.index ?? null,
1885
+ last_matched_element: outEl
1886
+ }
1619
1887
  }
1620
1888
  }
1889
+ } else {
1890
+ stableMatchCount = 0
1891
+ matchedAt = null
1621
1892
  }
1622
1893
 
1623
1894
  } catch (e) {
@@ -1883,7 +2154,24 @@ export class ToolsInteract {
1883
2154
  basis,
1884
2155
  matched: success,
1885
2156
  reason
1886
- }
2157
+ },
2158
+ trace: buildObservationTrace({
2159
+ actionType: 'expect_screen',
2160
+ stage: 'verify',
2161
+ success,
2162
+ attempts: 1,
2163
+ metadata: {
2164
+ expected_screen: expectedScreen,
2165
+ observed_screen: {
2166
+ fingerprint: observedScreen.fingerprint,
2167
+ screen: observedScreenLabel
2168
+ },
2169
+ comparison: {
2170
+ basis,
2171
+ matched: success
2172
+ }
2173
+ }
2174
+ })
1887
2175
  }
1888
2176
  }
1889
2177
 
@@ -1953,7 +2241,18 @@ export class ToolsInteract {
1953
2241
  semantic: (result.element as any).semantic ?? null
1954
2242
  }
1955
2243
  },
1956
- reason: 'selector is visible'
2244
+ reason: 'selector is visible',
2245
+ trace: buildObservationTrace({
2246
+ actionType: 'expect_element_visible',
2247
+ stage: 'verify',
2248
+ success: true,
2249
+ attempts: 1,
2250
+ metadata: {
2251
+ selector,
2252
+ element_id: result.element.elementId ?? element_id ?? null,
2253
+ status: result.status
2254
+ }
2255
+ })
1957
2256
  }
1958
2257
  }
1959
2258
 
@@ -1972,7 +2271,19 @@ export class ToolsInteract {
1972
2271
  },
1973
2272
  reason: result?.error?.message ?? 'selector is not visible',
1974
2273
  failure_code: errorCode,
1975
- retryable: errorCode === 'TIMEOUT'
2274
+ retryable: errorCode === 'TIMEOUT',
2275
+ trace: buildObservationTrace({
2276
+ actionType: 'expect_element_visible',
2277
+ stage: 'verify',
2278
+ success: false,
2279
+ attempts: 1,
2280
+ metadata: {
2281
+ selector,
2282
+ element_id: element_id ?? null,
2283
+ status: result?.status ?? null,
2284
+ reason: result?.error?.message ?? 'selector is not visible'
2285
+ }
2286
+ })
1976
2287
  }
1977
2288
  }
1978
2289
 
@@ -1982,189 +2293,341 @@ export class ToolsInteract {
1982
2293
  property,
1983
2294
  expected,
1984
2295
  platform,
1985
- deviceId
2296
+ deviceId,
2297
+ stabilization_window_ms = 1000,
2298
+ stable_observation_count = 2,
2299
+ snapshot_stale_threshold_ms = 500,
2300
+ poll_interval_ms = 150
1986
2301
  }: {
1987
2302
  selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean },
1988
2303
  element_id?: string,
1989
2304
  property: string,
1990
2305
  expected: boolean | number | string | Record<string, unknown>,
1991
2306
  platform?: 'android' | 'ios',
1992
- deviceId?: string
2307
+ deviceId?: string,
2308
+ stabilization_window_ms?: number,
2309
+ stable_observation_count?: number,
2310
+ snapshot_stale_threshold_ms?: number,
2311
+ poll_interval_ms?: number
1993
2312
  }): Promise<ExpectStateResponse> {
1994
- const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
1995
- const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
1996
- const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
1997
- const treeDeviceId = tree?.device?.id || deviceId
2313
+ const compareBoolean = (value: unknown) => typeof value === 'boolean' ? value : null
2314
+ const compareString = (value: unknown) => typeof value === 'string' ? value : null
2315
+ const compareNumber = (value: unknown) => typeof value === 'number' && Number.isFinite(value) ? value : null
2316
+ const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
2317
+ const start = Date.now()
2318
+ const deadline = start + Math.max(500, stabilization_window_ms)
2319
+ const stableTarget = Math.max(1, Math.floor(stable_observation_count || 2))
2320
+ const pollDelay = Math.max(100, Math.min(poll_interval_ms || 150, 200))
2321
+ const staleThreshold = Math.max(300, Math.min(snapshot_stale_threshold_ms || 500, 800))
1998
2322
 
1999
- let matched: { el: UiElement, idx: number } | null = null
2323
+ let attempts = 0
2324
+ let stableCount = 0
2325
+ let lastReason = 'element not found'
2326
+ let lastFailureCode: 'ELEMENT_NOT_FOUND' | 'UNKNOWN' = 'ELEMENT_NOT_FOUND'
2327
+ let lastObservedElement: (ActionTargetResolved & { state?: UIElementState | null }) | null = null
2328
+ let lastObservedValue: boolean | number | string | Record<string, unknown> | null = null
2329
+ let lastRawValue: boolean | number | string | null = null
2330
+ let lastResolvedElementId: string | null = element_id ?? null
2331
+ const traceSteps: TraceStep[] = []
2332
+ let traceAttemptIndex = 0
2333
+ let resolveRecorded = false
2334
+
2335
+ const recordTraceStep = (
2336
+ stage: TraceStep['stage'],
2337
+ result: TraceStep['result'],
2338
+ metadata?: Record<string, unknown>
2339
+ ) => {
2340
+ traceSteps.push(createTraceStep({
2341
+ stage,
2342
+ timestamp: Date.now(),
2343
+ result,
2344
+ attemptIndex: traceAttemptIndex++,
2345
+ metadata
2346
+ }))
2347
+ }
2348
+
2349
+ const buildStateTrace = (outcome: 'success' | 'failure'): ActionTrace => ({
2350
+ action_id: nextActionId('expect_state', Date.now()),
2351
+ steps: traceSteps,
2352
+ final_outcome: outcome,
2353
+ attempts
2354
+ })
2355
+
2356
+ while (Date.now() <= deadline) {
2357
+ attempts++
2358
+ const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
2359
+ const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
2360
+ const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
2361
+ const treeDeviceId = tree?.device?.id || deviceId
2362
+ const treeAgeMs = typeof tree?.captured_at_ms === 'number' ? Date.now() - tree.captured_at_ms : null
2363
+ let matched: { el: UiElement, idx: number } | null = null
2000
2364
 
2001
- if (element_id) {
2002
- const resolved = ToolsInteract._resolvedUiElements.get(element_id)
2003
- if (resolved) {
2004
- const current = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved)
2005
- if (current) matched = { el: current.el, idx: current.index }
2365
+ if (element_id) {
2366
+ const resolved = ToolsInteract._resolvedUiElements.get(element_id)
2367
+ if (resolved) {
2368
+ const current = ToolsInteract._findCurrentResolvedElement(elements, treePlatform, treeDeviceId, resolved)
2369
+ if (current) matched = { el: current.el, idx: current.index }
2370
+ }
2006
2371
  }
2007
- }
2008
2372
 
2009
- if (!matched && selector) {
2010
- matched = ToolsInteract._findFirstMatchingElement(elements, selector)
2011
- }
2373
+ if (!matched && selector) {
2374
+ matched = ToolsInteract._findFirstMatchingElement(elements, selector)
2375
+ }
2012
2376
 
2013
- if (!matched) {
2014
- return {
2015
- success: false,
2016
- selector,
2017
- element_id: element_id ?? null,
2018
- expected_state: { property, expected },
2019
- reason: 'element not found',
2020
- failure_code: 'ELEMENT_NOT_FOUND',
2021
- retryable: true
2377
+ if (!matched) {
2378
+ lastReason = 'element not found'
2379
+ lastFailureCode = 'ELEMENT_NOT_FOUND'
2380
+ stableCount = 0
2381
+ recordTraceStep('resolve', 'retry', {
2382
+ selector: selector ?? null,
2383
+ element_id: lastResolvedElementId,
2384
+ matched: false,
2385
+ reason: lastReason,
2386
+ attempt: attempts
2387
+ })
2388
+ recordTraceStep('stabilize', 'retry', {
2389
+ stabilization_attempts: attempts,
2390
+ stable_observation_count: stableCount,
2391
+ snapshot_freshness_ms: treeAgeMs,
2392
+ reason: lastReason
2393
+ })
2394
+ await sleep(pollDelay)
2395
+ continue
2022
2396
  }
2023
- }
2024
2397
 
2025
- const resolvedElement = ToolsInteract._resolvedTargetFromElement(
2026
- ToolsInteract._computeElementId(treePlatform, treeDeviceId, matched.el, matched.idx),
2027
- matched.el,
2028
- matched.idx
2029
- )
2030
- const observedState = matched.el.state ?? null
2031
- const actual = observedState?.[property as keyof UIElementState] ?? null
2398
+ const resolvedElement = ToolsInteract._resolvedTargetFromElement(
2399
+ ToolsInteract._computeElementId(treePlatform, treeDeviceId, matched.el, matched.idx),
2400
+ matched.el,
2401
+ matched.idx
2402
+ )
2403
+ lastResolvedElementId = resolvedElement.elementId
2404
+ lastObservedElement = { ...resolvedElement, state: matched.el.state ?? null }
2405
+
2406
+ if (treeAgeMs !== null && treeAgeMs > staleThreshold) {
2407
+ lastReason = 'stale snapshot'
2408
+ lastFailureCode = 'UNKNOWN'
2409
+ stableCount = 0
2410
+ recordTraceStep('resolve', 'retry', {
2411
+ selector: selector ?? null,
2412
+ element_id: lastResolvedElementId,
2413
+ matched: true,
2414
+ reason: lastReason,
2415
+ attempt: attempts
2416
+ })
2417
+ recordTraceStep('stabilize', 'retry', {
2418
+ stabilization_attempts: attempts,
2419
+ stable_observation_count: stableCount,
2420
+ snapshot_freshness_ms: treeAgeMs,
2421
+ reason: lastReason
2422
+ })
2423
+ await sleep(pollDelay)
2424
+ continue
2425
+ }
2032
2426
 
2033
- const compareBoolean = (value: unknown) => typeof value === 'boolean' ? value : null
2034
- const compareString = (value: unknown) => typeof value === 'string' ? value : null
2035
- const compareNumber = (value: unknown) => typeof value === 'number' && Number.isFinite(value) ? value : null
2427
+ if (!resolveRecorded) {
2428
+ recordTraceStep('resolve', 'success', {
2429
+ selector: selector ?? null,
2430
+ element_id: lastResolvedElementId,
2431
+ matched: true,
2432
+ reason: 'element resolved'
2433
+ })
2434
+ resolveRecorded = true
2435
+ }
2036
2436
 
2037
- let success = false
2038
- let reason = ''
2039
- let rawValue: boolean | number | string | null = null
2040
- let observedValue: boolean | number | string | Record<string, unknown> | null = actual as any
2041
-
2042
- switch (property) {
2043
- case 'checked':
2044
- case 'focused':
2045
- case 'expanded':
2046
- case 'enabled': {
2047
- const expectedBool = compareBoolean(expected)
2048
- const actualBool = compareBoolean(actual)
2049
- if (expectedBool === null) {
2050
- reason = `expected ${property} must be boolean`
2051
- } else if (actualBool === null) {
2052
- reason = `${property} state unavailable`
2053
- } else {
2054
- rawValue = actualBool
2055
- success = actualBool === expectedBool
2056
- reason = success ? `${property} matches expected value` : `expected ${property}=${expectedBool} but observed ${actualBool}`
2437
+ const observedState = matched.el.state ?? null
2438
+ const actual = observedState?.[property as keyof UIElementState] ?? null
2439
+
2440
+ let success = false
2441
+ let reason = ''
2442
+ let rawValue: boolean | number | string | null = null
2443
+ let observedValue: boolean | number | string | Record<string, unknown> | null = actual as any
2444
+
2445
+ switch (property) {
2446
+ case 'checked':
2447
+ case 'focused':
2448
+ case 'expanded':
2449
+ case 'enabled': {
2450
+ const expectedBool = compareBoolean(expected)
2451
+ const actualBool = compareBoolean(actual)
2452
+ if (expectedBool === null) {
2453
+ reason = `expected ${property} must be boolean`
2454
+ } else if (actualBool === null) {
2455
+ reason = `${property} state unavailable`
2456
+ } else {
2457
+ rawValue = actualBool
2458
+ success = actualBool === expectedBool
2459
+ reason = success ? `${property} matches expected value` : `expected ${property}=${expectedBool} but observed ${actualBool}`
2460
+ }
2461
+ observedValue = actualBool
2462
+ break
2057
2463
  }
2058
- observedValue = actualBool
2059
- break
2060
- }
2061
- case 'value':
2062
- case 'raw_value': {
2063
- const expectedNumber = compareNumber(expected)
2064
- const actualNumber = compareNumber(actual)
2065
- if (expectedNumber !== null && actualNumber !== null) {
2066
- success = actualNumber === expectedNumber
2067
- rawValue = actualNumber
2068
- observedValue = actualNumber
2069
- reason = success ? 'value matches expected value' : `expected value=${expectedNumber} but observed ${actualNumber}`
2464
+ case 'value':
2465
+ case 'raw_value': {
2466
+ const expectedNumber = compareNumber(expected)
2467
+ const actualNumber = compareNumber(actual)
2468
+ if (expectedNumber !== null && actualNumber !== null) {
2469
+ success = actualNumber === expectedNumber
2470
+ rawValue = actualNumber
2471
+ observedValue = actualNumber
2472
+ reason = success ? 'value matches expected value' : `expected value=${expectedNumber} but observed ${actualNumber}`
2473
+ break
2474
+ }
2475
+ const expectedString = typeof expected === 'string' ? expected : null
2476
+ const actualString = compareString(actual)
2477
+ if (expectedString !== null && actualString !== null) {
2478
+ success = actualString === expectedString
2479
+ rawValue = actualString
2480
+ observedValue = actualString
2481
+ reason = success ? 'value matches expected value' : `expected value=${expectedString} but observed ${actualString}`
2482
+ } else {
2483
+ reason = 'value state unavailable'
2484
+ }
2070
2485
  break
2071
2486
  }
2072
- const expectedString = typeof expected === 'string' ? expected : null
2073
- const actualString = compareString(actual)
2074
- if (expectedString !== null && actualString !== null) {
2075
- success = actualString === expectedString
2487
+ case 'selected': {
2488
+ const expectedBool = typeof expected === 'boolean' ? expected : null
2489
+ const expectedString = typeof expected === 'string'
2490
+ ? expected
2491
+ : expected && typeof expected === 'object'
2492
+ ? String((expected as { id?: unknown; label?: unknown }).id ?? (expected as { id?: unknown; label?: unknown }).label ?? '')
2493
+ : null
2494
+ if (!observedState || observedState.selected === undefined || observedState.selected === null) {
2495
+ reason = 'selected state unavailable'
2496
+ break
2497
+ }
2498
+ if (expectedBool !== null) {
2499
+ const actualBool = typeof observedState.selected === 'boolean' ? observedState.selected : null
2500
+ if (actualBool === null) {
2501
+ reason = 'selected state is not boolean'
2502
+ break
2503
+ }
2504
+ rawValue = actualBool
2505
+ observedValue = actualBool
2506
+ success = actualBool === expectedBool
2507
+ reason = success ? 'selected matches expected value' : `expected selected=${expectedBool} but observed ${actualBool}`
2508
+ break
2509
+ }
2510
+ const actualSelected = typeof observedState.selected === 'object' && observedState.selected !== null
2511
+ ? String((observedState.selected as { id?: unknown; label?: unknown }).id ?? (observedState.selected as { id?: unknown; label?: unknown }).label ?? '')
2512
+ : String(observedState.selected)
2513
+ const actualString = actualSelected.trim()
2514
+ if (!expectedString) {
2515
+ reason = 'expected selected must be boolean, string, or object with id/label'
2516
+ break
2517
+ }
2076
2518
  rawValue = actualString
2077
2519
  observedValue = actualString
2078
- reason = success ? 'value matches expected value' : `expected value=${expectedString} but observed ${actualString}`
2079
- } else {
2080
- reason = 'value state unavailable'
2081
- }
2082
- break
2083
- }
2084
- case 'selected': {
2085
- const expectedBool = typeof expected === 'boolean' ? expected : null
2086
- const expectedString = typeof expected === 'string'
2087
- ? expected
2088
- : expected && typeof expected === 'object'
2089
- ? String((expected as { id?: unknown; label?: unknown }).id ?? (expected as { id?: unknown; label?: unknown }).label ?? '')
2090
- : null
2091
- if (!observedState || observedState.selected === undefined || observedState.selected === null) {
2092
- reason = 'selected state unavailable'
2520
+ success = actualString === expectedString
2521
+ reason = success ? 'selected matches expected value' : `expected selected=${expectedString} but observed ${actualString}`
2093
2522
  break
2094
2523
  }
2095
- if (expectedBool !== null) {
2096
- const actualBool = typeof observedState.selected === 'boolean' ? observedState.selected : null
2097
- if (actualBool === null) {
2098
- reason = 'selected state is not boolean'
2099
- break
2524
+ case 'text_value': {
2525
+ const expectedString = typeof expected === 'string' ? expected : null
2526
+ const actualString = compareString(actual)
2527
+ if (!expectedString) {
2528
+ reason = 'expected text_value must be string'
2529
+ } else if (!actualString) {
2530
+ reason = 'text_value state unavailable'
2531
+ } else {
2532
+ success = actualString === expectedString
2533
+ rawValue = actualString
2534
+ observedValue = actualString
2535
+ reason = success ? 'text_value matches expected value' : `expected text_value=${expectedString} but observed ${actualString}`
2100
2536
  }
2101
- rawValue = actualBool
2102
- observedValue = actualBool
2103
- success = actualBool === expectedBool
2104
- reason = success ? 'selected matches expected value' : `expected selected=${expectedBool} but observed ${actualBool}`
2105
2537
  break
2106
2538
  }
2107
- const actualSelected = typeof observedState.selected === 'object' && observedState.selected !== null
2108
- ? String((observedState.selected as { id?: unknown; label?: unknown }).id ?? (observedState.selected as { id?: unknown; label?: unknown }).label ?? '')
2109
- : String(observedState.selected)
2110
- const actualString = actualSelected.trim()
2111
- if (!expectedString) {
2112
- reason = 'expected selected must be boolean, string, or object with id/label'
2113
- break
2539
+ default: {
2540
+ if (actual !== null && actual !== undefined) {
2541
+ success = actual === expected
2542
+ observedValue = actual as any
2543
+ rawValue = typeof actual === 'string' || typeof actual === 'number' || typeof actual === 'boolean' ? actual : null
2544
+ reason = success ? `${property} matches expected value` : `expected ${property} to match but observed ${String(actual)}`
2545
+ } else {
2546
+ reason = `unsupported or unavailable state property: ${property}`
2547
+ }
2114
2548
  }
2115
- rawValue = actualString
2116
- observedValue = actualString
2117
- success = actualString === expectedString
2118
- reason = success ? 'selected matches expected value' : `expected selected=${expectedString} but observed ${actualString}`
2119
- break
2120
2549
  }
2121
- case 'text_value': {
2122
- const expectedString = typeof expected === 'string' ? expected : null
2123
- const actualString = compareString(actual)
2124
- if (!expectedString) {
2125
- reason = 'expected text_value must be string'
2126
- } else if (!actualString) {
2127
- reason = 'text_value state unavailable'
2128
- } else {
2129
- success = actualString === expectedString
2130
- rawValue = actualString
2131
- observedValue = actualString
2132
- reason = success ? 'text_value matches expected value' : `expected text_value=${expectedString} but observed ${actualString}`
2550
+
2551
+ if (success) {
2552
+ stableCount++
2553
+ recordTraceStep('stabilize', 'success', {
2554
+ stabilization_attempts: attempts,
2555
+ stable_observation_count: stableCount,
2556
+ snapshot_freshness_ms: treeAgeMs,
2557
+ reason
2558
+ })
2559
+ recordTraceStep('verify', 'success', {
2560
+ property,
2561
+ expected,
2562
+ observed_value: observedValue,
2563
+ reason
2564
+ })
2565
+ if (stableCount >= stableTarget) {
2566
+ return {
2567
+ success: true,
2568
+ selector,
2569
+ element_id: lastResolvedElementId,
2570
+ expected_state: { property, expected },
2571
+ element: lastObservedElement,
2572
+ observed_state: {
2573
+ property,
2574
+ value: observedValue,
2575
+ ...(rawValue !== null ? { raw_value: rawValue } : {})
2576
+ },
2577
+ reason,
2578
+ stabilization_attempts: attempts,
2579
+ stabilization_window_ms: Date.now() - start,
2580
+ stable_observation_count: stableCount,
2581
+ snapshot_freshness_ms: treeAgeMs ?? undefined,
2582
+ trace: buildStateTrace('success')
2583
+ } as ExpectStateResponse & {
2584
+ stabilization_attempts?: number;
2585
+ stabilization_window_ms?: number;
2586
+ stable_observation_count?: number;
2587
+ snapshot_freshness_ms?: number;
2588
+ }
2133
2589
  }
2134
- break
2590
+ } else {
2591
+ stableCount = 0
2592
+ lastReason = reason || lastReason
2593
+ lastFailureCode = 'UNKNOWN'
2594
+ recordTraceStep('stabilize', 'retry', {
2595
+ stabilization_attempts: attempts,
2596
+ stable_observation_count: stableCount,
2597
+ snapshot_freshness_ms: treeAgeMs,
2598
+ reason: lastReason
2599
+ })
2600
+ recordTraceStep('verify', 'retry', {
2601
+ property,
2602
+ expected,
2603
+ observed_value: observedValue,
2604
+ reason: lastReason
2605
+ })
2135
2606
  }
2136
- default: {
2137
- if (actual !== null && actual !== undefined) {
2138
- success = actual === expected
2139
- observedValue = actual as any
2140
- rawValue = typeof actual === 'string' || typeof actual === 'number' || typeof actual === 'boolean' ? actual : null
2141
- reason = success ? `${property} matches expected value` : `expected ${property} to match but observed ${String(actual)}`
2142
- } else {
2143
- reason = `unsupported or unavailable state property: ${property}`
2144
- }
2607
+
2608
+ if (!success) {
2609
+ lastObservedValue = observedValue
2610
+ lastRawValue = rawValue
2145
2611
  }
2146
- }
2147
2612
 
2148
- if (!success && !reason) {
2149
- reason = `${property} did not match expected value`
2613
+ await sleep(pollDelay)
2150
2614
  }
2151
2615
 
2152
2616
  return {
2153
- success,
2617
+ success: false,
2154
2618
  selector,
2155
- element_id: element_id ?? resolvedElement.elementId,
2619
+ element_id: lastResolvedElementId,
2156
2620
  expected_state: { property, expected },
2157
- element: {
2158
- ...resolvedElement,
2159
- state: observedState
2160
- },
2621
+ element: lastObservedElement,
2161
2622
  observed_state: {
2162
2623
  property,
2163
- value: observedValue,
2164
- ...(rawValue !== null ? { raw_value: rawValue } : {})
2624
+ value: lastObservedValue,
2625
+ ...(lastRawValue !== null ? { raw_value: lastRawValue } : {})
2165
2626
  },
2166
- reason,
2167
- ...(success ? {} : { failure_code: 'UNKNOWN', retryable: false })
2627
+ reason: lastReason,
2628
+ failure_code: lastFailureCode,
2629
+ retryable: true,
2630
+ trace: buildStateTrace('failure')
2168
2631
  }
2169
2632
  }
2170
2633