mobile-debug-mcp 0.27.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +238 -11
- package/dist/server/common.js +134 -3
- package/dist/server-core.js +1 -1
- package/docs/CHANGELOG.md +3 -0
- package/docs/rfcs/{012.md → 012-action-trace-and-xecution-observability.md} +43 -4
- package/docs/specs/mcp-tooling-spec-v1.md +14 -0
- package/docs/tools/interact.md +44 -0
- package/package.json +1 -1
- package/src/interact/index.ts +268 -12
- package/src/server/common.ts +194 -4
- package/src/server-core.ts +1 -1
- package/src/types.ts +23 -0
- package/test/device/manual/observe/rfc012_trace.manual.ts +51 -0
- package/test/unit/interact/expect_tools.test.ts +57 -25
- package/test/unit/server/common.test.ts +24 -0
|
@@ -68,7 +68,7 @@ interface ActionTrace {
|
|
|
68
68
|
action_id: string;
|
|
69
69
|
steps: TraceStep[];
|
|
70
70
|
final_outcome: "success" | "failure";
|
|
71
|
-
attempts: number;
|
|
71
|
+
attempts: number; // total execution attempts including recovery-triggered retries
|
|
72
72
|
}
|
|
73
73
|
```
|
|
74
74
|
|
|
@@ -79,10 +79,25 @@ interface TraceStep {
|
|
|
79
79
|
stage: "resolve" | "execute" | "verify" | "stabilize" | "recover";
|
|
80
80
|
timestamp: number;
|
|
81
81
|
result: "success" | "failure" | "retry";
|
|
82
|
+
attempt_index: number; // monotonic per action execution
|
|
83
|
+
cycle_id?: number; // groups steps within a recovery cycle
|
|
82
84
|
metadata?: Record<string, any>;
|
|
83
85
|
}
|
|
84
86
|
```
|
|
85
87
|
|
|
88
|
+
### 6.3 Partial Trace Requirements
|
|
89
|
+
|
|
90
|
+
For actions that do not traverse the full lifecycle (resolve → execute → verify → stabilize → recover), implementations MUST emit a partial trace.
|
|
91
|
+
|
|
92
|
+
A partial trace MUST:
|
|
93
|
+
- include a valid action_id
|
|
94
|
+
- include final_outcome
|
|
95
|
+
- include at least one TraceStep with a valid stage and timestamp
|
|
96
|
+
|
|
97
|
+
Partial traces MUST still respect attempt_index semantics.
|
|
98
|
+
|
|
99
|
+
This ensures observability coverage even for legacy or bypass execution paths.
|
|
100
|
+
|
|
86
101
|
---
|
|
87
102
|
|
|
88
103
|
## 7. Stage Emission Rules
|
|
@@ -158,13 +173,32 @@ Metadata MUST remain lightweight.
|
|
|
158
173
|
- RFC 010: stabilization emits stabilize stage
|
|
159
174
|
- RFC 011: recovery emits recover stage
|
|
160
175
|
|
|
161
|
-
|
|
176
|
+
### 10.1 Compatibility with RFC 006 Observability Model
|
|
177
|
+
|
|
178
|
+
RFC 006 defines traceability as being assembled from distributed signals rather than a centralized event system.
|
|
179
|
+
|
|
180
|
+
This RFC does NOT replace that model; it standardizes a unified projection layer over those signals.
|
|
181
|
+
|
|
182
|
+
- Existing emitters (server, interact, stabilization, recovery) remain the source of truth
|
|
183
|
+
- RFC 012 defines how those signals are composed into a single ActionTrace
|
|
184
|
+
- Actions that bypass parts of the lifecycle MUST still emit partial traces reflecting the stages they execute
|
|
185
|
+
|
|
186
|
+
This ensures backward compatibility while enabling a coherent trace surface.
|
|
162
187
|
|
|
163
188
|
---
|
|
164
189
|
|
|
165
190
|
## 11. Output Behavior
|
|
166
191
|
|
|
167
|
-
Trace
|
|
192
|
+
Trace MUST be produced for all action flows (full or partial, depending on runtime capability).
|
|
193
|
+
|
|
194
|
+
Canonical contract:
|
|
195
|
+
- Trace SHOULD be included in ActionExecutionResult when the runtime path supports full trace emission
|
|
196
|
+
- Trace MAY also be stored internally for diagnostics
|
|
197
|
+
|
|
198
|
+
If a runtime path cannot yet emit a full trace (e.g. legacy or bypass actions), it MUST emit a partial trace containing at least:
|
|
199
|
+
- action_id
|
|
200
|
+
- final_outcome
|
|
201
|
+
- at least one TraceStep representing the executed stage
|
|
168
202
|
|
|
169
203
|
Example:
|
|
170
204
|
|
|
@@ -172,10 +206,15 @@ Example:
|
|
|
172
206
|
interface ActionExecutionResult {
|
|
173
207
|
success: boolean;
|
|
174
208
|
failure_code?: string;
|
|
175
|
-
trace?: ActionTrace;
|
|
209
|
+
trace?: ActionTrace; // optional in type, required by RFC behavior (full or partial)
|
|
176
210
|
}
|
|
177
211
|
```
|
|
178
212
|
|
|
213
|
+
|
|
214
|
+
Implementations MUST treat the absence of `trace` in the runtime type as a temporary compatibility constraint, not as an absence of trace generation. All execution paths MUST still generate a trace internally, even if only a partial trace is returned externally.
|
|
215
|
+
|
|
216
|
+
The optionality of `trace` in ActionExecutionResult is transitional. Implementations MUST treat the absence of `trace` as a compatibility constraint rather than a valid steady-state. Future versions of the runtime MAY require `trace` to be present on all ActionExecutionResult values once all execution paths support full trace emission.
|
|
217
|
+
|
|
179
218
|
---
|
|
180
219
|
|
|
181
220
|
## 12. Failure Analysis
|
|
@@ -80,6 +80,19 @@ MUST be returned in this structure:
|
|
|
80
80
|
ui_fingerprint_after: string | null,
|
|
81
81
|
failure_code?: string,
|
|
82
82
|
retryable?: boolean,
|
|
83
|
+
trace: {
|
|
84
|
+
action_id: string,
|
|
85
|
+
steps: Array<{
|
|
86
|
+
stage: 'resolve' | 'execute' | 'verify' | 'stabilize' | 'recover',
|
|
87
|
+
timestamp: number,
|
|
88
|
+
result: 'success' | 'failure' | 'retry',
|
|
89
|
+
attempt_index: number,
|
|
90
|
+
cycle_id?: number,
|
|
91
|
+
metadata?: Record<string, unknown>
|
|
92
|
+
}>,
|
|
93
|
+
final_outcome: 'success' | 'failure',
|
|
94
|
+
attempts: number
|
|
95
|
+
},
|
|
83
96
|
recovery?: {
|
|
84
97
|
failure_class: string,
|
|
85
98
|
runtime_code: string,
|
|
@@ -104,6 +117,7 @@ Rules:
|
|
|
104
117
|
- `source_module` identifies where the envelope was produced
|
|
105
118
|
- fingerprints represent observed pre/post UI state on a best-effort basis
|
|
106
119
|
- `failure_code` is optional but MUST be used when a structured mapping exists
|
|
120
|
+
- `trace` is required and carries the observable execution path
|
|
107
121
|
- `recovery` MAY be attached to failed actions to carry typed recovery metadata
|
|
108
122
|
|
|
109
123
|
### 4.4 Allowed Deviations
|
package/docs/tools/interact.md
CHANGED
|
@@ -40,6 +40,25 @@ Example response:
|
|
|
40
40
|
"source_module": "server",
|
|
41
41
|
"target": { "selector": { "x": 100, "y": 200 }, "resolved": null },
|
|
42
42
|
"success": true,
|
|
43
|
+
"trace": {
|
|
44
|
+
"action_id": "tap_element_1710000000002_3",
|
|
45
|
+
"steps": [
|
|
46
|
+
{
|
|
47
|
+
"stage": "resolve",
|
|
48
|
+
"timestamp": 1710000000002,
|
|
49
|
+
"result": "success",
|
|
50
|
+
"attempt_index": 0
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"stage": "execute",
|
|
54
|
+
"timestamp": 1710000000003,
|
|
55
|
+
"result": "success",
|
|
56
|
+
"attempt_index": 1
|
|
57
|
+
}
|
|
58
|
+
],
|
|
59
|
+
"final_outcome": "success",
|
|
60
|
+
"attempts": 1
|
|
61
|
+
},
|
|
43
62
|
"ui_fingerprint_before": "fp_before",
|
|
44
63
|
"ui_fingerprint_after": "fp_after"
|
|
45
64
|
}
|
|
@@ -395,6 +414,31 @@ Failure response:
|
|
|
395
414
|
"success": false,
|
|
396
415
|
"failure_code": "STALE_REFERENCE",
|
|
397
416
|
"retryable": true,
|
|
417
|
+
"trace": {
|
|
418
|
+
"action_id": "tap_element_1710000000003_4",
|
|
419
|
+
"steps": [
|
|
420
|
+
{
|
|
421
|
+
"stage": "resolve",
|
|
422
|
+
"timestamp": 1710000000003,
|
|
423
|
+
"result": "failure",
|
|
424
|
+
"attempt_index": 0
|
|
425
|
+
},
|
|
426
|
+
{
|
|
427
|
+
"stage": "execute",
|
|
428
|
+
"timestamp": 1710000000004,
|
|
429
|
+
"result": "failure",
|
|
430
|
+
"attempt_index": 1
|
|
431
|
+
},
|
|
432
|
+
{
|
|
433
|
+
"stage": "recover",
|
|
434
|
+
"timestamp": 1710000000005,
|
|
435
|
+
"result": "retry",
|
|
436
|
+
"attempt_index": 2
|
|
437
|
+
}
|
|
438
|
+
],
|
|
439
|
+
"final_outcome": "failure",
|
|
440
|
+
"attempts": 1
|
|
441
|
+
},
|
|
398
442
|
"recovery": {
|
|
399
443
|
"failure_class": "TargetResolutionFailure",
|
|
400
444
|
"runtime_code": "STALE_REFERENCE",
|
package/package.json
CHANGED
package/src/interact/index.ts
CHANGED
|
@@ -6,8 +6,9 @@ export { AndroidInteract, iOSInteract };
|
|
|
6
6
|
import { resolveTargetDevice } from '../utils/resolve-device.js'
|
|
7
7
|
import { ToolsObserve } from '../observe/index.js'
|
|
8
8
|
import { computeSnapshotSignature } from '../observe/snapshot-metadata.js'
|
|
9
|
-
import { buildActionExecutionResult } from '../server/common.js'
|
|
9
|
+
import { buildActionExecutionResult, createTraceStep, nextActionId } from '../server/common.js'
|
|
10
10
|
import type {
|
|
11
|
+
ActionTrace,
|
|
11
12
|
ActionFailureCode,
|
|
12
13
|
ActionTargetResolved,
|
|
13
14
|
AdjustControlResponse,
|
|
@@ -18,6 +19,7 @@ import type {
|
|
|
18
19
|
WaitForUIChangeResponse,
|
|
19
20
|
UIElementSemanticMetadata,
|
|
20
21
|
UIElementState,
|
|
22
|
+
TraceStep,
|
|
21
23
|
TapElementResponse
|
|
22
24
|
} from '../types.js'
|
|
23
25
|
|
|
@@ -79,6 +81,39 @@ interface RankedResolutionCandidate {
|
|
|
79
81
|
interactable: boolean
|
|
80
82
|
}
|
|
81
83
|
|
|
84
|
+
function buildObservationTrace({
|
|
85
|
+
actionType,
|
|
86
|
+
stage,
|
|
87
|
+
success,
|
|
88
|
+
attempts,
|
|
89
|
+
metadata
|
|
90
|
+
}: {
|
|
91
|
+
actionType: string
|
|
92
|
+
stage: 'verify' | 'stabilize' | 'resolve'
|
|
93
|
+
success: boolean
|
|
94
|
+
attempts: number
|
|
95
|
+
metadata?: Record<string, unknown>
|
|
96
|
+
}): ActionTrace {
|
|
97
|
+
const now = Date.now()
|
|
98
|
+
const actionId = nextActionId(actionType, now)
|
|
99
|
+
const steps: TraceStep[] = [
|
|
100
|
+
createTraceStep({
|
|
101
|
+
stage,
|
|
102
|
+
timestamp: now,
|
|
103
|
+
result: success ? 'success' : 'failure',
|
|
104
|
+
attemptIndex: 0,
|
|
105
|
+
metadata
|
|
106
|
+
})
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
return {
|
|
110
|
+
action_id: actionId,
|
|
111
|
+
steps,
|
|
112
|
+
final_outcome: success ? 'success' : 'failure',
|
|
113
|
+
attempts: Math.max(1, Math.floor(attempts || 1))
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
82
117
|
interface FindElementResolutionSummary {
|
|
83
118
|
confidence: number
|
|
84
119
|
reason: string
|
|
@@ -742,6 +777,22 @@ export class ToolsInteract {
|
|
|
742
777
|
let resolvedDeviceId = deviceId
|
|
743
778
|
const fingerprintBefore = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
744
779
|
let semanticFallbackElement: FindElementResponse['element'] | null = null
|
|
780
|
+
const traceSteps: TraceStep[] = []
|
|
781
|
+
let traceAttemptIndex = 0
|
|
782
|
+
|
|
783
|
+
const recordTraceStep = (
|
|
784
|
+
stage: TraceStep['stage'],
|
|
785
|
+
result: TraceStep['result'],
|
|
786
|
+
metadata?: Record<string, unknown>
|
|
787
|
+
) => {
|
|
788
|
+
traceSteps.push(createTraceStep({
|
|
789
|
+
stage,
|
|
790
|
+
timestamp: Date.now(),
|
|
791
|
+
result,
|
|
792
|
+
attemptIndex: traceAttemptIndex++,
|
|
793
|
+
metadata
|
|
794
|
+
}))
|
|
795
|
+
}
|
|
745
796
|
|
|
746
797
|
const buildFailure = (
|
|
747
798
|
failureCode: ActionFailureCode,
|
|
@@ -754,6 +805,22 @@ export class ToolsInteract {
|
|
|
754
805
|
retryable = false,
|
|
755
806
|
uiFingerprintAfter: string | null = null
|
|
756
807
|
): AdjustControlResponse => {
|
|
808
|
+
if (!traceSteps.some((step) => step.stage === 'resolve')) {
|
|
809
|
+
recordTraceStep('resolve', 'failure',
|
|
810
|
+
{
|
|
811
|
+
reason,
|
|
812
|
+
failure_code: failureCode
|
|
813
|
+
})
|
|
814
|
+
}
|
|
815
|
+
if (!traceSteps.some((step) => step.stage === 'recover')) {
|
|
816
|
+
recordTraceStep('recover', retryable ? 'retry' : 'failure', {
|
|
817
|
+
reason,
|
|
818
|
+
failure_code: failureCode,
|
|
819
|
+
retry_allowed: retryable,
|
|
820
|
+
recovery_attempts: attempts,
|
|
821
|
+
retry_depth: attempts
|
|
822
|
+
})
|
|
823
|
+
}
|
|
757
824
|
const base = buildActionExecutionResult({
|
|
758
825
|
actionType,
|
|
759
826
|
sourceModule: 'interact',
|
|
@@ -774,7 +841,8 @@ export class ToolsInteract {
|
|
|
774
841
|
converged: false,
|
|
775
842
|
within_tolerance: false,
|
|
776
843
|
reason
|
|
777
|
-
}
|
|
844
|
+
},
|
|
845
|
+
traceSteps
|
|
778
846
|
}) as AdjustControlResponse
|
|
779
847
|
|
|
780
848
|
return {
|
|
@@ -999,11 +1067,25 @@ export class ToolsInteract {
|
|
|
999
1067
|
|
|
1000
1068
|
lastObservedState = actualState
|
|
1001
1069
|
|
|
1070
|
+
if (!traceSteps.some((step) => step.stage === 'resolve')) {
|
|
1071
|
+
recordTraceStep('resolve', 'success', {
|
|
1072
|
+
resolved_target: resolvedTarget,
|
|
1073
|
+
current_value: currentValue,
|
|
1074
|
+
adjustment_mode: lastAdjustmentMode
|
|
1075
|
+
})
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1002
1078
|
if (property !== 'value' && property !== 'raw_value') {
|
|
1003
1079
|
return buildFailure('ELEMENT_NOT_INTERACTABLE', 'adjust_control currently supports numeric value and raw_value properties only', resolvedTarget, currentDevice, actualState, attemptCount, lastAdjustmentMode, false)
|
|
1004
1080
|
}
|
|
1005
1081
|
|
|
1006
1082
|
if (currentValue !== null && Math.abs(currentValue - targetValue) <= normalizedTolerance) {
|
|
1083
|
+
recordTraceStep('verify', 'success', {
|
|
1084
|
+
property,
|
|
1085
|
+
target_value: targetValue,
|
|
1086
|
+
actual_state: actualState,
|
|
1087
|
+
reason: 'control already within tolerance'
|
|
1088
|
+
})
|
|
1007
1089
|
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
1008
1090
|
const base = buildActionExecutionResult({
|
|
1009
1091
|
actionType,
|
|
@@ -1024,7 +1106,8 @@ export class ToolsInteract {
|
|
|
1024
1106
|
converged: true,
|
|
1025
1107
|
within_tolerance: true,
|
|
1026
1108
|
reason: 'control already within tolerance'
|
|
1027
|
-
}
|
|
1109
|
+
},
|
|
1110
|
+
traceSteps
|
|
1028
1111
|
}) as AdjustControlResponse
|
|
1029
1112
|
|
|
1030
1113
|
return {
|
|
@@ -1109,6 +1192,11 @@ export class ToolsInteract {
|
|
|
1109
1192
|
for (let i = 0; i < probePoints.length; i++) {
|
|
1110
1193
|
const probePoint = probePoints[i]
|
|
1111
1194
|
lastAdjustmentMode = 'coordinate'
|
|
1195
|
+
recordTraceStep('execute', 'retry', {
|
|
1196
|
+
attempt: attemptCount + 1,
|
|
1197
|
+
mode: 'coordinate',
|
|
1198
|
+
point: probePoint
|
|
1199
|
+
})
|
|
1112
1200
|
const actionResult = await ToolsInteract.tapHandler({
|
|
1113
1201
|
platform: resolvedPlatform,
|
|
1114
1202
|
x: probePoint.x,
|
|
@@ -1119,12 +1207,25 @@ export class ToolsInteract {
|
|
|
1119
1207
|
actionDevice = actionResult.device ?? actionDevice
|
|
1120
1208
|
|
|
1121
1209
|
if (!actionResult.success) {
|
|
1210
|
+
recordTraceStep('execute', 'retry', {
|
|
1211
|
+
attempt: attemptCount,
|
|
1212
|
+
mode: 'coordinate',
|
|
1213
|
+
point: probePoint,
|
|
1214
|
+
success: false
|
|
1215
|
+
})
|
|
1122
1216
|
continue
|
|
1123
1217
|
}
|
|
1124
1218
|
|
|
1125
1219
|
verificationResult = await runVerification()
|
|
1126
1220
|
observedState = verificationResult.observedState
|
|
1127
1221
|
lastObservedState = observedState
|
|
1222
|
+
recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
|
|
1223
|
+
attempt: attemptCount,
|
|
1224
|
+
property,
|
|
1225
|
+
target_value: targetValue,
|
|
1226
|
+
actual_state: observedState,
|
|
1227
|
+
reason: verificationResult.verification?.reason ?? 'control did not converge yet'
|
|
1228
|
+
})
|
|
1128
1229
|
|
|
1129
1230
|
if (verificationResult.withinTolerance) {
|
|
1130
1231
|
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
@@ -1147,7 +1248,8 @@ export class ToolsInteract {
|
|
|
1147
1248
|
converged: true,
|
|
1148
1249
|
within_tolerance: true,
|
|
1149
1250
|
reason: verificationResult.verification?.reason ?? 'control converged to target value'
|
|
1150
|
-
}
|
|
1251
|
+
},
|
|
1252
|
+
traceSteps
|
|
1151
1253
|
}) as AdjustControlResponse
|
|
1152
1254
|
|
|
1153
1255
|
return {
|
|
@@ -1168,6 +1270,12 @@ export class ToolsInteract {
|
|
|
1168
1270
|
|
|
1169
1271
|
if (currentValue !== null) {
|
|
1170
1272
|
lastAdjustmentMode = 'gesture'
|
|
1273
|
+
recordTraceStep('execute', 'retry', {
|
|
1274
|
+
attempt: attemptCount + 1,
|
|
1275
|
+
mode: 'gesture',
|
|
1276
|
+
start: currentPoint,
|
|
1277
|
+
end: targetPoint
|
|
1278
|
+
})
|
|
1171
1279
|
const fallbackActionResult = await ToolsInteract.swipeHandler({
|
|
1172
1280
|
platform: resolvedPlatform,
|
|
1173
1281
|
x1: currentPoint.x,
|
|
@@ -1179,6 +1287,13 @@ export class ToolsInteract {
|
|
|
1179
1287
|
})
|
|
1180
1288
|
attemptCount++
|
|
1181
1289
|
if (!fallbackActionResult.success) {
|
|
1290
|
+
recordTraceStep('execute', 'failure', {
|
|
1291
|
+
attempt: attemptCount,
|
|
1292
|
+
mode: 'gesture',
|
|
1293
|
+
start: currentPoint,
|
|
1294
|
+
end: targetPoint,
|
|
1295
|
+
success: false
|
|
1296
|
+
})
|
|
1182
1297
|
return buildFailure('UNKNOWN', fallbackActionResult.error ?? 'adjustment gesture failed', resolvedTarget, fallbackActionResult.device ?? actionDevice, observedState ?? actualState, attemptCount, lastAdjustmentMode, false)
|
|
1183
1298
|
}
|
|
1184
1299
|
|
|
@@ -1186,6 +1301,13 @@ export class ToolsInteract {
|
|
|
1186
1301
|
verificationResult = await runVerification()
|
|
1187
1302
|
observedState = verificationResult.observedState
|
|
1188
1303
|
lastObservedState = observedState
|
|
1304
|
+
recordTraceStep('verify', verificationResult.withinTolerance ? 'success' : 'retry', {
|
|
1305
|
+
attempt: attemptCount,
|
|
1306
|
+
property,
|
|
1307
|
+
target_value: targetValue,
|
|
1308
|
+
actual_state: observedState,
|
|
1309
|
+
reason: verificationResult.verification?.reason ?? 'gesture adjustment did not converge yet'
|
|
1310
|
+
})
|
|
1189
1311
|
|
|
1190
1312
|
if (verificationResult.withinTolerance) {
|
|
1191
1313
|
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
@@ -1208,7 +1330,8 @@ export class ToolsInteract {
|
|
|
1208
1330
|
converged: true,
|
|
1209
1331
|
within_tolerance: true,
|
|
1210
1332
|
reason: verificationResult.verification?.reason ?? 'control converged to target value'
|
|
1211
|
-
}
|
|
1333
|
+
},
|
|
1334
|
+
traceSteps
|
|
1212
1335
|
}) as AdjustControlResponse
|
|
1213
1336
|
|
|
1214
1337
|
return {
|
|
@@ -1231,6 +1354,13 @@ export class ToolsInteract {
|
|
|
1231
1354
|
lastObservedState = observedState
|
|
1232
1355
|
|
|
1233
1356
|
if (verificationResult.withinTolerance) {
|
|
1357
|
+
recordTraceStep('verify', 'success', {
|
|
1358
|
+
attempt: attemptCount,
|
|
1359
|
+
property,
|
|
1360
|
+
target_value: targetValue,
|
|
1361
|
+
actual_state: observedState,
|
|
1362
|
+
reason: verification?.reason ?? 'control converged to target value'
|
|
1363
|
+
})
|
|
1234
1364
|
const uiFingerprintAfter = await ToolsInteract._captureFingerprint(resolvedPlatform, resolvedDeviceId)
|
|
1235
1365
|
const base = buildActionExecutionResult({
|
|
1236
1366
|
actionType,
|
|
@@ -1251,7 +1381,8 @@ export class ToolsInteract {
|
|
|
1251
1381
|
converged: true,
|
|
1252
1382
|
within_tolerance: true,
|
|
1253
1383
|
reason: verification?.reason ?? 'control converged to target value'
|
|
1254
|
-
}
|
|
1384
|
+
},
|
|
1385
|
+
traceSteps
|
|
1255
1386
|
}) as AdjustControlResponse
|
|
1256
1387
|
|
|
1257
1388
|
return {
|
|
@@ -2023,7 +2154,24 @@ export class ToolsInteract {
|
|
|
2023
2154
|
basis,
|
|
2024
2155
|
matched: success,
|
|
2025
2156
|
reason
|
|
2026
|
-
}
|
|
2157
|
+
},
|
|
2158
|
+
trace: buildObservationTrace({
|
|
2159
|
+
actionType: 'expect_screen',
|
|
2160
|
+
stage: 'verify',
|
|
2161
|
+
success,
|
|
2162
|
+
attempts: 1,
|
|
2163
|
+
metadata: {
|
|
2164
|
+
expected_screen: expectedScreen,
|
|
2165
|
+
observed_screen: {
|
|
2166
|
+
fingerprint: observedScreen.fingerprint,
|
|
2167
|
+
screen: observedScreenLabel
|
|
2168
|
+
},
|
|
2169
|
+
comparison: {
|
|
2170
|
+
basis,
|
|
2171
|
+
matched: success
|
|
2172
|
+
}
|
|
2173
|
+
}
|
|
2174
|
+
})
|
|
2027
2175
|
}
|
|
2028
2176
|
}
|
|
2029
2177
|
|
|
@@ -2093,7 +2241,18 @@ export class ToolsInteract {
|
|
|
2093
2241
|
semantic: (result.element as any).semantic ?? null
|
|
2094
2242
|
}
|
|
2095
2243
|
},
|
|
2096
|
-
reason: 'selector is visible'
|
|
2244
|
+
reason: 'selector is visible',
|
|
2245
|
+
trace: buildObservationTrace({
|
|
2246
|
+
actionType: 'expect_element_visible',
|
|
2247
|
+
stage: 'verify',
|
|
2248
|
+
success: true,
|
|
2249
|
+
attempts: 1,
|
|
2250
|
+
metadata: {
|
|
2251
|
+
selector,
|
|
2252
|
+
element_id: result.element.elementId ?? element_id ?? null,
|
|
2253
|
+
status: result.status
|
|
2254
|
+
}
|
|
2255
|
+
})
|
|
2097
2256
|
}
|
|
2098
2257
|
}
|
|
2099
2258
|
|
|
@@ -2112,7 +2271,19 @@ export class ToolsInteract {
|
|
|
2112
2271
|
},
|
|
2113
2272
|
reason: result?.error?.message ?? 'selector is not visible',
|
|
2114
2273
|
failure_code: errorCode,
|
|
2115
|
-
retryable: errorCode === 'TIMEOUT'
|
|
2274
|
+
retryable: errorCode === 'TIMEOUT',
|
|
2275
|
+
trace: buildObservationTrace({
|
|
2276
|
+
actionType: 'expect_element_visible',
|
|
2277
|
+
stage: 'verify',
|
|
2278
|
+
success: false,
|
|
2279
|
+
attempts: 1,
|
|
2280
|
+
metadata: {
|
|
2281
|
+
selector,
|
|
2282
|
+
element_id: element_id ?? null,
|
|
2283
|
+
status: result?.status ?? null,
|
|
2284
|
+
reason: result?.error?.message ?? 'selector is not visible'
|
|
2285
|
+
}
|
|
2286
|
+
})
|
|
2116
2287
|
}
|
|
2117
2288
|
}
|
|
2118
2289
|
|
|
@@ -2157,6 +2328,30 @@ export class ToolsInteract {
|
|
|
2157
2328
|
let lastObservedValue: boolean | number | string | Record<string, unknown> | null = null
|
|
2158
2329
|
let lastRawValue: boolean | number | string | null = null
|
|
2159
2330
|
let lastResolvedElementId: string | null = element_id ?? null
|
|
2331
|
+
const traceSteps: TraceStep[] = []
|
|
2332
|
+
let traceAttemptIndex = 0
|
|
2333
|
+
let resolveRecorded = false
|
|
2334
|
+
|
|
2335
|
+
const recordTraceStep = (
|
|
2336
|
+
stage: TraceStep['stage'],
|
|
2337
|
+
result: TraceStep['result'],
|
|
2338
|
+
metadata?: Record<string, unknown>
|
|
2339
|
+
) => {
|
|
2340
|
+
traceSteps.push(createTraceStep({
|
|
2341
|
+
stage,
|
|
2342
|
+
timestamp: Date.now(),
|
|
2343
|
+
result,
|
|
2344
|
+
attemptIndex: traceAttemptIndex++,
|
|
2345
|
+
metadata
|
|
2346
|
+
}))
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2349
|
+
const buildStateTrace = (outcome: 'success' | 'failure'): ActionTrace => ({
|
|
2350
|
+
action_id: nextActionId('expect_state', Date.now()),
|
|
2351
|
+
steps: traceSteps,
|
|
2352
|
+
final_outcome: outcome,
|
|
2353
|
+
attempts
|
|
2354
|
+
})
|
|
2160
2355
|
|
|
2161
2356
|
while (Date.now() <= deadline) {
|
|
2162
2357
|
attempts++
|
|
@@ -2165,7 +2360,6 @@ export class ToolsInteract {
|
|
|
2165
2360
|
const treePlatform = tree?.device?.platform === 'ios' ? 'ios' : (platform || 'android')
|
|
2166
2361
|
const treeDeviceId = tree?.device?.id || deviceId
|
|
2167
2362
|
const treeAgeMs = typeof tree?.captured_at_ms === 'number' ? Date.now() - tree.captured_at_ms : null
|
|
2168
|
-
|
|
2169
2363
|
let matched: { el: UiElement, idx: number } | null = null
|
|
2170
2364
|
|
|
2171
2365
|
if (element_id) {
|
|
@@ -2184,6 +2378,19 @@ export class ToolsInteract {
|
|
|
2184
2378
|
lastReason = 'element not found'
|
|
2185
2379
|
lastFailureCode = 'ELEMENT_NOT_FOUND'
|
|
2186
2380
|
stableCount = 0
|
|
2381
|
+
recordTraceStep('resolve', 'retry', {
|
|
2382
|
+
selector: selector ?? null,
|
|
2383
|
+
element_id: lastResolvedElementId,
|
|
2384
|
+
matched: false,
|
|
2385
|
+
reason: lastReason,
|
|
2386
|
+
attempt: attempts
|
|
2387
|
+
})
|
|
2388
|
+
recordTraceStep('stabilize', 'retry', {
|
|
2389
|
+
stabilization_attempts: attempts,
|
|
2390
|
+
stable_observation_count: stableCount,
|
|
2391
|
+
snapshot_freshness_ms: treeAgeMs,
|
|
2392
|
+
reason: lastReason
|
|
2393
|
+
})
|
|
2187
2394
|
await sleep(pollDelay)
|
|
2188
2395
|
continue
|
|
2189
2396
|
}
|
|
@@ -2200,10 +2407,33 @@ export class ToolsInteract {
|
|
|
2200
2407
|
lastReason = 'stale snapshot'
|
|
2201
2408
|
lastFailureCode = 'UNKNOWN'
|
|
2202
2409
|
stableCount = 0
|
|
2410
|
+
recordTraceStep('resolve', 'retry', {
|
|
2411
|
+
selector: selector ?? null,
|
|
2412
|
+
element_id: lastResolvedElementId,
|
|
2413
|
+
matched: true,
|
|
2414
|
+
reason: lastReason,
|
|
2415
|
+
attempt: attempts
|
|
2416
|
+
})
|
|
2417
|
+
recordTraceStep('stabilize', 'retry', {
|
|
2418
|
+
stabilization_attempts: attempts,
|
|
2419
|
+
stable_observation_count: stableCount,
|
|
2420
|
+
snapshot_freshness_ms: treeAgeMs,
|
|
2421
|
+
reason: lastReason
|
|
2422
|
+
})
|
|
2203
2423
|
await sleep(pollDelay)
|
|
2204
2424
|
continue
|
|
2205
2425
|
}
|
|
2206
2426
|
|
|
2427
|
+
if (!resolveRecorded) {
|
|
2428
|
+
recordTraceStep('resolve', 'success', {
|
|
2429
|
+
selector: selector ?? null,
|
|
2430
|
+
element_id: lastResolvedElementId,
|
|
2431
|
+
matched: true,
|
|
2432
|
+
reason: 'element resolved'
|
|
2433
|
+
})
|
|
2434
|
+
resolveRecorded = true
|
|
2435
|
+
}
|
|
2436
|
+
|
|
2207
2437
|
const observedState = matched.el.state ?? null
|
|
2208
2438
|
const actual = observedState?.[property as keyof UIElementState] ?? null
|
|
2209
2439
|
|
|
@@ -2320,6 +2550,18 @@ export class ToolsInteract {
|
|
|
2320
2550
|
|
|
2321
2551
|
if (success) {
|
|
2322
2552
|
stableCount++
|
|
2553
|
+
recordTraceStep('stabilize', 'success', {
|
|
2554
|
+
stabilization_attempts: attempts,
|
|
2555
|
+
stable_observation_count: stableCount,
|
|
2556
|
+
snapshot_freshness_ms: treeAgeMs,
|
|
2557
|
+
reason
|
|
2558
|
+
})
|
|
2559
|
+
recordTraceStep('verify', 'success', {
|
|
2560
|
+
property,
|
|
2561
|
+
expected,
|
|
2562
|
+
observed_value: observedValue,
|
|
2563
|
+
reason
|
|
2564
|
+
})
|
|
2323
2565
|
if (stableCount >= stableTarget) {
|
|
2324
2566
|
return {
|
|
2325
2567
|
success: true,
|
|
@@ -2336,7 +2578,8 @@ export class ToolsInteract {
|
|
|
2336
2578
|
stabilization_attempts: attempts,
|
|
2337
2579
|
stabilization_window_ms: Date.now() - start,
|
|
2338
2580
|
stable_observation_count: stableCount,
|
|
2339
|
-
snapshot_freshness_ms: treeAgeMs ?? undefined
|
|
2581
|
+
snapshot_freshness_ms: treeAgeMs ?? undefined,
|
|
2582
|
+
trace: buildStateTrace('success')
|
|
2340
2583
|
} as ExpectStateResponse & {
|
|
2341
2584
|
stabilization_attempts?: number;
|
|
2342
2585
|
stabilization_window_ms?: number;
|
|
@@ -2348,6 +2591,18 @@ export class ToolsInteract {
|
|
|
2348
2591
|
stableCount = 0
|
|
2349
2592
|
lastReason = reason || lastReason
|
|
2350
2593
|
lastFailureCode = 'UNKNOWN'
|
|
2594
|
+
recordTraceStep('stabilize', 'retry', {
|
|
2595
|
+
stabilization_attempts: attempts,
|
|
2596
|
+
stable_observation_count: stableCount,
|
|
2597
|
+
snapshot_freshness_ms: treeAgeMs,
|
|
2598
|
+
reason: lastReason
|
|
2599
|
+
})
|
|
2600
|
+
recordTraceStep('verify', 'retry', {
|
|
2601
|
+
property,
|
|
2602
|
+
expected,
|
|
2603
|
+
observed_value: observedValue,
|
|
2604
|
+
reason: lastReason
|
|
2605
|
+
})
|
|
2351
2606
|
}
|
|
2352
2607
|
|
|
2353
2608
|
if (!success) {
|
|
@@ -2371,7 +2626,8 @@ export class ToolsInteract {
|
|
|
2371
2626
|
},
|
|
2372
2627
|
reason: lastReason,
|
|
2373
2628
|
failure_code: lastFailureCode,
|
|
2374
|
-
retryable: true
|
|
2629
|
+
retryable: true,
|
|
2630
|
+
trace: buildStateTrace('failure')
|
|
2375
2631
|
}
|
|
2376
2632
|
}
|
|
2377
2633
|
|