mobile-debug-mcp 0.25.1 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/interact/index.js +113 -0
- package/dist/observe/android.js +10 -1
- package/dist/observe/index.js +19 -1
- package/dist/observe/ios.js +15 -1
- package/dist/observe/snapshot-metadata.js +88 -0
- package/dist/server/tool-definitions.js +30 -2
- package/dist/server/tool-handlers.js +10 -0
- package/dist/server-core.js +1 -1
- package/docs/CHANGELOG.md +6 -0
- package/docs/rfcs/003-wait-and-synchronization-reliability.md +296 -0
- package/docs/specs/mcp-tooling-spec-v1.md +8 -0
- package/docs/tools/interact.md +21 -0
- package/docs/tools/observe.md +4 -2
- package/package.json +1 -1
- package/skills/rfc-review/SKILL.md +52 -0
- package/skills/rfc-review/references/rfc-review-checklist.md +12 -0
- package/skills/rfc-review/references/rfc-review-template.md +28 -0
- package/src/interact/index.ts +151 -0
- package/src/observe/android.ts +11 -1
- package/src/observe/index.ts +26 -1
- package/src/observe/ios.ts +28 -13
- package/src/observe/snapshot-metadata.ts +107 -0
- package/src/server/tool-definitions.ts +30 -2
- package/src/server/tool-handlers.ts +11 -0
- package/src/server-core.ts +1 -1
- package/src/types.ts +23 -0
- package/test/unit/interact/wait_for_ui_change.test.ts +76 -0
- package/test/unit/server/response_shapes.test.ts +37 -3
- package/docs/rfcs/003-wait-and-synchronization-reliability +0 -232
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
# RFC-003: Wait and Synchronization Reliability
|
|
2
|
+
|
|
3
|
+
Priority: 3
|
|
4
|
+
Depends on: RFC-001 (Stronger State Verification), RFC-002 (Platform-Native Element Metadata and Resolution Hints)
|
|
5
|
+
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# 1. Problem
|
|
9
|
+
|
|
10
|
+
Agents can often identify the right element (RFC-002) and verify the right state (RFC-001), but still fail because they act before the UI has reached the intended post-action state.
|
|
11
|
+
|
|
12
|
+
This causes:
|
|
13
|
+
|
|
14
|
+
- retries caused by racing the UI
|
|
15
|
+
- false failures from stale snapshots
|
|
16
|
+
- overuse of network/log verification when UI evidence should suffice
|
|
17
|
+
- flakiness in asynchronous and in-place update flows
|
|
18
|
+
- unreliable behaviour in Compose-heavy or thin accessibility trees
|
|
19
|
+
|
|
20
|
+
Current system limitations:
|
|
21
|
+
|
|
22
|
+
- wait_for_ui is underused after actions involving async state changes
|
|
23
|
+
- current waits focus on expected elements appearing, not general UI transition detection
|
|
24
|
+
- snapshot staleness is not explicitly surfaced
|
|
25
|
+
- loading state transitions are inconsistently observable
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
# 2. Goals
|
|
30
|
+
|
|
31
|
+
This RFC introduces:
|
|
32
|
+
|
|
33
|
+
1. UI-first synchronization policy after actions
|
|
34
|
+
2. Snapshot staleness and revision metadata
|
|
35
|
+
3. UI-change based waiting for in-place updates
|
|
36
|
+
4. Structured loading-state detection
|
|
37
|
+
5. Compose-aware synchronization hints
|
|
38
|
+
|
|
39
|
+
Success goals:
|
|
40
|
+
|
|
41
|
+
- reduce retries caused by premature actions
|
|
42
|
+
- increase successful post-action verification
|
|
43
|
+
- reduce unnecessary fallbacks to logs/network checks
|
|
44
|
+
- improve reliability in asynchronous UI flows
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
# 3. Non-Goals
|
|
49
|
+
|
|
50
|
+
This RFC does not:
|
|
51
|
+
|
|
52
|
+
- redefine state verification semantics (RFC-001)
|
|
53
|
+
- redefine element identity contracts (RFC-002)
|
|
54
|
+
- add new interaction primitives (long press, pinch, etc.)
|
|
55
|
+
- replace network or log verification where no UI outcome exists
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
# 4. Proposed Model
|
|
60
|
+
|
|
61
|
+
## 4.1 UI-First Synchronization Contract (v1)
|
|
62
|
+
|
|
63
|
+
Default post-action flow SHOULD be:
|
|
64
|
+
|
|
65
|
+
```text
|
|
66
|
+
action
|
|
67
|
+
→ wait_for_ui(expected outcome)
|
|
68
|
+
→ verify state
|
|
69
|
+
→ only fall back to network/logs when no UI outcome exists or wait fails
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Tool-level contract:
|
|
73
|
+
|
|
74
|
+
- After actions expected to cause visible UI changes, agents SHOULD invoke wait_for_ui or wait_for_ui_change before verification.
|
|
75
|
+
- wait_for_ui SHOULD be used when an expected element or explicit outcome is known.
|
|
76
|
+
- wait_for_ui_change SHOULD be used for in-place mutations where a specific element target is not known.
|
|
77
|
+
- wait_for_screen_change SHOULD remain preferred for full navigation transitions when available.
|
|
78
|
+
|
|
79
|
+
Rules:
|
|
80
|
+
|
|
81
|
+
- UI evidence MUST be preferred over network or log evidence when a UI outcome is expected.
|
|
82
|
+
- Actions that trigger navigation, async mutation, or visible state changes SHOULD be followed by a wait.
|
|
83
|
+
- Network/log checks are fallback signals, not primary synchronization mechanisms.
|
|
84
|
+
- This synchronization order is normative tool behavior for agents, not advisory prose.
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## 4.2 Snapshot Revision Contract
|
|
89
|
+
|
|
90
|
+
All snapshot responses MUST include revision metadata.
|
|
91
|
+
|
|
92
|
+
Emission scope:
|
|
93
|
+
|
|
94
|
+
- snapshot_revision and captured_at_ms MUST be emitted on snapshot responses.
|
|
95
|
+
- get_ui_tree responses SHOULD emit the same fields when backed by the same snapshot generation layer.
|
|
96
|
+
- If both surfaces exist, revision values MUST be consistent across them when derived from the same underlying snapshot.
|
|
97
|
+
|
|
98
|
+
Required snapshot envelope:
|
|
99
|
+
|
|
100
|
+
```json
|
|
101
|
+
{
|
|
102
|
+
"snapshot_revision": 184,
|
|
103
|
+
"captured_at_ms": 1714452012301
|
|
104
|
+
}
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Field requirements:
|
|
108
|
+
|
|
109
|
+
- snapshot_revision REQUIRED on every snapshot response.
|
|
110
|
+
- captured_at_ms REQUIRED on every snapshot response.
|
|
111
|
+
|
|
112
|
+
Source of truth:
|
|
113
|
+
|
|
114
|
+
- snapshot_revision originates in the snapshot generation layer.
|
|
115
|
+
- It MUST increment when a meaningful hierarchy delta is detected.
|
|
116
|
+
- Cosmetic-only changes MUST NOT increment revision.
|
|
117
|
+
|
|
118
|
+
Meaningful deltas include:
|
|
119
|
+
|
|
120
|
+
- node added or removed
|
|
121
|
+
- visible text mutation
|
|
122
|
+
- control state change
|
|
123
|
+
- list content mutation
|
|
124
|
+
- navigation or view transition
|
|
125
|
+
|
|
126
|
+
Cosmetic churn examples (must not increment):
|
|
127
|
+
|
|
128
|
+
- cursor blink
|
|
129
|
+
- focus-only changes
|
|
130
|
+
- animation-only transitions
|
|
131
|
+
- timestamp or unrelated ephemeral text changes
|
|
132
|
+
|
|
133
|
+
Rules:
|
|
134
|
+
|
|
135
|
+
- Agents SHOULD use revision changes as synchronization signals.
|
|
136
|
+
- Stale revisions SHOULD trigger reacquisition before verification.
|
|
137
|
+
- This extends the snapshot response contract defined by RFC-002.
|
|
138
|
+
|
|
139
|
+
- Snapshot responses are the normative required emission surface; get_ui_tree emission is recommended for consistency.
|
|
140
|
+
- snapshot_revision MUST be monotonically increasing within a session.
|
|
141
|
+
|
|
142
|
+
---
|
|
143
|
+
|
|
144
|
+
## 4.3 wait_for_ui_change API
|
|
145
|
+
|
|
146
|
+
Concrete API contract:
|
|
147
|
+
|
|
148
|
+
```ts
|
|
149
|
+
wait_for_ui_change({
|
|
150
|
+
expected_change?: "hierarchy_diff" | "text_change" | "state_change",
|
|
151
|
+
timeout_ms?: number,
|
|
152
|
+
stability_window_ms?: number
|
|
153
|
+
}) => {
|
|
154
|
+
success: boolean,
|
|
155
|
+
observed_change: "hierarchy_diff" | "text_change" | "state_change" | null,
|
|
156
|
+
snapshot_revision?: number,
|
|
157
|
+
timeout: boolean
|
|
158
|
+
}
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
Relationship to other wait primitives:
|
|
162
|
+
|
|
163
|
+
- wait_for_screen_change remains the preferred primitive for navigation-level transitions.
|
|
164
|
+
- wait_for_ui_change is the preferred primitive for non-navigation UI mutations and in-place updates.
|
|
165
|
+
- wait_for_ui_change is additive to wait_for_screen_change, not a replacement for it.
|
|
166
|
+
|
|
167
|
+
Rules:
|
|
168
|
+
|
|
169
|
+
- stability_window_ms represents time a detected change must remain stable before success.
|
|
170
|
+
- Meaningful delta semantics are inherited from Section 4.2.
|
|
171
|
+
- wait_for_ui_change complements wait_for_ui; it does not replace it.
|
|
172
|
+
|
|
173
|
+
- Agents SHOULD prefer wait_for_screen_change for navigation and wait_for_ui_change for non-navigation changes.
|
|
174
|
+
|
|
175
|
+
---
|
|
176
|
+
|
|
177
|
+
## 4.4 Structured Loading-State Contract
|
|
178
|
+
|
|
179
|
+
Loading signals are OPTIONAL overall, but when a detectable loading signal exists they SHOULD be surfaced on snapshot responses and UI tree responses, and if emitted they MUST conform to the contract below.
|
|
180
|
+
|
|
181
|
+
Required shape:
|
|
182
|
+
|
|
183
|
+
```json
|
|
184
|
+
{
|
|
185
|
+
"loading_state": {
|
|
186
|
+
"active": true,
|
|
187
|
+
"signal": "progress_indicator",
|
|
188
|
+
"source": "snapshot"
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
Required fields:
|
|
194
|
+
|
|
195
|
+
- active
|
|
196
|
+
- signal
|
|
197
|
+
- source
|
|
198
|
+
|
|
199
|
+
Rules:
|
|
200
|
+
|
|
201
|
+
- Loading signals are synchronization hints only.
|
|
202
|
+
- Loading completion MUST NOT alone be treated as success.
|
|
203
|
+
- If emitted, the shape above MUST be used.
|
|
204
|
+
- Absence of loading_state is valid when no reliable loading signal is detectable; malformed or partial loading_state emission is not valid.
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
## 4.5 Compose-Aware Synchronization Hints
|
|
209
|
+
|
|
210
|
+
For Compose or thin accessibility structures:
|
|
211
|
+
|
|
212
|
+
Systems SHOULD support:
|
|
213
|
+
|
|
214
|
+
- merged semantic node changes as wait signals
|
|
215
|
+
- text mutations within existing nodes
|
|
216
|
+
- in-place recomposition awareness
|
|
217
|
+
|
|
218
|
+
These are synchronization hints layered on top of standard wait behaviour.
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
222
|
+
# 5. Failure Modes
|
|
223
|
+
|
|
224
|
+
## 5.1 Premature Action Progression
|
|
225
|
+
|
|
226
|
+
If an action is followed immediately by verification without waiting:
|
|
227
|
+
|
|
228
|
+
- system SHOULD bias toward suggesting wait_for_ui
|
|
229
|
+
- retries SHOULD prefer synchronization correction before repeated action execution
|
|
230
|
+
|
|
231
|
+
---
|
|
232
|
+
|
|
233
|
+
## 5.2 Stale Snapshot Reads
|
|
234
|
+
|
|
235
|
+
If verification uses an old snapshot:
|
|
236
|
+
|
|
237
|
+
- revision metadata SHOULD expose staleness
|
|
238
|
+
- agents SHOULD reacquire snapshot before retrying verification
|
|
239
|
+
|
|
240
|
+
---
|
|
241
|
+
|
|
242
|
+
## 5.3 No Visible UI Outcome
|
|
243
|
+
|
|
244
|
+
If no UI outcome is expected:
|
|
245
|
+
|
|
246
|
+
- network/log verification MAY be primary evidence
|
|
247
|
+
- UI-first policy does not apply rigidly
|
|
248
|
+
|
|
249
|
+
---
|
|
250
|
+
|
|
251
|
+
## 5.4 False Positive UI Change Detection
|
|
252
|
+
|
|
253
|
+
If unrelated UI churn triggers early wait completion:
|
|
254
|
+
|
|
255
|
+
- systems SHOULD reject cosmetic-only changes using Section 4.2 rules
|
|
256
|
+
- agents SHOULD prefer stability windows before considering waits satisfied
|
|
257
|
+
|
|
258
|
+
---
|
|
259
|
+
|
|
260
|
+
# 6. Acceptance Criteria
|
|
261
|
+
|
|
262
|
+
RFC-003 specification is complete when:
|
|
263
|
+
|
|
264
|
+
- Snapshot Revision Contract is fully defined and mandatory.
|
|
265
|
+
- wait_for_ui_change API contract is fully defined.
|
|
266
|
+
- Loading-State Contract required schema is defined.
|
|
267
|
+
- Synchronization tool-selection rules are explicitly specified.
|
|
268
|
+
- False-positive change handling is specified.
|
|
269
|
+
|
|
270
|
+
Implementation readiness success is measured when:
|
|
271
|
+
|
|
272
|
+
- snapshot revisions reduce stale-read retries
|
|
273
|
+
- synchronization retries decrease
|
|
274
|
+
- post-action verification success increases
|
|
275
|
+
|
|
276
|
+
---
|
|
277
|
+
|
|
278
|
+
# 7. Success Metrics
|
|
279
|
+
|
|
280
|
+
- Fewer retries caused by timing/synchronization errors
|
|
281
|
+
- Higher post-action verification success rate
|
|
282
|
+
- Reduced unnecessary fallback to network/log evidence
|
|
283
|
+
- Improved stability in asynchronous and Compose-heavy flows
|
|
284
|
+
|
|
285
|
+
---
|
|
286
|
+
|
|
287
|
+
# 8. Deferred To Later RFCs
|
|
288
|
+
|
|
289
|
+
- Advanced subscriptions / notify-when-element-appears APIs
|
|
290
|
+
- Full action-to-ui trace correlation (Priority 7)
|
|
291
|
+
- Gesture-trigger-specific synchronization logic
|
|
292
|
+
- Element appearance subscription / notify-when-ready APIs
|
|
293
|
+
|
|
294
|
+
---
|
|
295
|
+
|
|
296
|
+
This RFC standardises temporal reliability and synchronization signals layered on top of state verification and element identity guarantees from RFC-001 and RFC-002.
|
|
@@ -151,6 +151,7 @@ Examples:
|
|
|
151
151
|
|
|
152
152
|
- `wait_for_ui`
|
|
153
153
|
- `wait_for_screen_change`
|
|
154
|
+
- `wait_for_ui_change`
|
|
154
155
|
|
|
155
156
|
### 6.2 Rules
|
|
156
157
|
|
|
@@ -239,6 +240,8 @@ Raw layer contents include:
|
|
|
239
240
|
- UI hierarchy or accessibility tree
|
|
240
241
|
- normalized readable element state where exposed by the platform
|
|
241
242
|
- platform-native identity hints such as stable identifiers, roles, and test tags
|
|
243
|
+
- snapshot metadata such as `snapshot_revision` and `captured_at_ms`
|
|
244
|
+
- `loading_state` when a reliable loading signal is detectable
|
|
242
245
|
- screenshot when available
|
|
243
246
|
- element-level attributes
|
|
244
247
|
- logs and fingerprint/activity observations
|
|
@@ -305,10 +308,15 @@ Canonical pattern:
|
|
|
305
308
|
|
|
306
309
|
`wait_for_ui -> tap_element -> wait_for_screen_change (optional) -> expect_screen`
|
|
307
310
|
|
|
311
|
+
For in-place UI mutations, agents SHOULD prefer:
|
|
312
|
+
|
|
313
|
+
`wait_for_ui_change -> expect_element_visible / expect_state`
|
|
314
|
+
|
|
308
315
|
Interpretation:
|
|
309
316
|
|
|
310
317
|
- `tap_element.success` = executed
|
|
311
318
|
- `wait_for_screen_change.success` = UI changed
|
|
319
|
+
- `wait_for_ui_change.success` = in-place UI mutation observed and stable
|
|
312
320
|
- `expect_screen.success` = correct outcome verified
|
|
313
321
|
|
|
314
322
|
## 12. Known Deviations
|
package/docs/tools/interact.md
CHANGED
|
@@ -58,6 +58,7 @@ Preferred verification:
|
|
|
58
58
|
|
|
59
59
|
Use `wait_for_screen_change` only when a visible transition is the expected outcome. If a button should trigger an API request but the screen should stay the same, rely on network activity and classification instead.
|
|
60
60
|
For backend-only actions, prefer comparing `get_screen_fingerprint` before/after and call `get_network_activity` immediately after the action; do not wait on `wait_for_screen_change` if no visible transition is expected.
|
|
61
|
+
Use `wait_for_ui_change` when the screen stays in place but visible text or element state should change.
|
|
61
62
|
|
|
62
63
|
---
|
|
63
64
|
|
|
@@ -148,6 +149,26 @@ Notes:
|
|
|
148
149
|
|
|
149
150
|
---
|
|
150
151
|
|
|
152
|
+
## wait_for_ui_change
|
|
153
|
+
|
|
154
|
+
Purpose:
|
|
155
|
+
|
|
156
|
+
- detect a stable in-place UI mutation without naming a target element first
|
|
157
|
+
|
|
158
|
+
Capabilities:
|
|
159
|
+
|
|
160
|
+
- waits for hierarchy, text, or state deltas
|
|
161
|
+
- uses snapshot revision metadata when available
|
|
162
|
+
- confirms the change remains stable before returning success
|
|
163
|
+
|
|
164
|
+
Guidance:
|
|
165
|
+
|
|
166
|
+
- prefer `wait_for_screen_change` for navigation
|
|
167
|
+
- prefer `wait_for_ui_change` for in-place updates and recomposition-style changes
|
|
168
|
+
- follow with `expect_*` when the expected final state is known
|
|
169
|
+
|
|
170
|
+
---
|
|
171
|
+
|
|
151
172
|
## find_element
|
|
152
173
|
|
|
153
174
|
Locate a UI element on the current screen using semantic matching and return an actionable element descriptor.
|
package/docs/tools/observe.md
CHANGED
|
@@ -83,13 +83,14 @@ Input:
|
|
|
83
83
|
Response (example):
|
|
84
84
|
|
|
85
85
|
```json
|
|
86
|
-
{ "device": { "platform": "android", "id": "emulator-5554" }, "screen": "", "resolution": { "width": 1080, "height": 2400 }, "elements": [ { "text": "Sign in", "type": "android.widget.Button", "resourceId": "com.example:id/signin", "clickable": true, "bounds": [0,0,100,50], "state": { "enabled": true }, "stable_id": "com.example:id/signin", "role": "button", "test_tag": "com.example:id/signin", "selector": { "value": "com.example:id/signin", "confidence": { "score": 1, "reason": "resource_id" } }, "semantic": { "is_clickable": true, "is_container": false } } ] }
|
|
86
|
+
{ "device": { "platform": "android", "id": "emulator-5554" }, "screen": "", "resolution": { "width": 1080, "height": 2400 }, "snapshot_revision": 12, "captured_at_ms": 1710000000123, "loading_state": { "active": true, "signal": "spinner", "source": "ui_tree" }, "elements": [ { "text": "Sign in", "type": "android.widget.Button", "resourceId": "com.example:id/signin", "clickable": true, "bounds": [0,0,100,50], "state": { "enabled": true }, "stable_id": "com.example:id/signin", "role": "button", "test_tag": "com.example:id/signin", "selector": { "value": "com.example:id/signin", "confidence": { "score": 1, "reason": "resource_id" } }, "semantic": { "is_clickable": true, "is_container": false } } ] }
|
|
87
87
|
```
|
|
88
88
|
|
|
89
89
|
Notes:
|
|
90
90
|
- Useful for inspection, selector development, and fallback debugging.
|
|
91
91
|
- Elements may include a normalized `state` object when the platform exposes readable state such as checked, selected, focused, expanded, text input, or slider values.
|
|
92
92
|
- Elements may also include platform-native identity hints such as `stable_id`, `role`, `test_tag`, `selector`, and `semantic`.
|
|
93
|
+
- The tree response may include `snapshot_revision`, `captured_at_ms`, and `loading_state` when a reliable signal is available.
|
|
93
94
|
- Prefer `wait_for_ui` for deterministic element resolution in interactive flows.
|
|
94
95
|
|
|
95
96
|
---
|
|
@@ -136,7 +137,8 @@ Behavior:
|
|
|
136
137
|
- Fast by default: does not wait for new logs and avoids long blocking operations.
|
|
137
138
|
- Returns a dual-layer payload:
|
|
138
139
|
- `raw` is authoritative and contains the underlying observation data unchanged.
|
|
139
|
-
|
|
140
|
+
- `semantic` is optional, derived from `raw`, and intended for planning only.
|
|
141
|
+
- `raw` now includes `snapshot_revision`, `captured_at_ms`, and `loading_state` when detectable.
|
|
140
142
|
|
|
141
143
|
Response (example):
|
|
142
144
|
|
package/package.json
CHANGED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# RFC Review skill
|
|
2
|
+
|
|
3
|
+
name: rfc-review
|
|
4
|
+
version: 0.1.1
|
|
5
|
+
summary: Reusable workflow for reviewing RFCs/specs in this repository with a consistent readiness rubric and output template.
|
|
6
|
+
|
|
7
|
+
# Purpose
|
|
8
|
+
Help an agent review an RFC for clarity, implementation readiness, and alignment with the current codebase. Use a common template so reviews stay consistent across documents and reviewers.
|
|
9
|
+
|
|
10
|
+
# Activation conditions
|
|
11
|
+
Activate when an agent needs to:
|
|
12
|
+
- review a new or revised RFC
|
|
13
|
+
- assess whether an RFC is implementation-ready
|
|
14
|
+
- identify whether feedback is an RFC issue or an implementation issue
|
|
15
|
+
- compare a spec against the current `src/` contract surface and docs
|
|
16
|
+
|
|
17
|
+
# Surface area (actions)
|
|
18
|
+
- locate-rfc
|
|
19
|
+
- compare-against-code
|
|
20
|
+
- assess-contract-completeness
|
|
21
|
+
- classify-gaps
|
|
22
|
+
- produce-review
|
|
23
|
+
|
|
24
|
+
# Core guidance
|
|
25
|
+
1. Read the RFC first, then compare it against the relevant code, docs, and tests.
|
|
26
|
+
2. Separate **spec gaps** from **implementation gaps**.
|
|
27
|
+
3. Check for: problem clarity, scope boundaries, explicit contracts, acceptance criteria, non-goals, and consistency with existing behavior.
|
|
28
|
+
4. Prefer precise feedback that names the missing contract, unclear rule, or inconsistent behavior.
|
|
29
|
+
5. Use the shared review template in `references/rfc-review-template.md` for the final output.
|
|
30
|
+
6. If the RFC is not ready, say exactly what must be clarified before implementation can start.
|
|
31
|
+
7. Classify each blocker as either a **spec gap** or an **implementation contract gap** and stop at that boundary.
|
|
32
|
+
|
|
33
|
+
# Inputs & outputs
|
|
34
|
+
- review-rfc(input: { rfcPath, relatedPaths?, focusAreas? }) -> { verdict, risks, specGaps, implementationGaps, recommendations }
|
|
35
|
+
- compare-against-code(input: { rfcPath, codePaths[] }) -> { matches, mismatches, notes }
|
|
36
|
+
- produce-review(input: { rfcPath, findings[] }) -> { summary, verdict, checklist, nextStep }
|
|
37
|
+
|
|
38
|
+
# Failure handling
|
|
39
|
+
- If the RFC file is missing, stop and report the missing path explicitly.
|
|
40
|
+
- If the RFC is ambiguous, classify each concern as either "spec" or "implementation" instead of blending them.
|
|
41
|
+
- If the review cannot be grounded in the current repo, state that the RFC is not reviewable yet.
|
|
42
|
+
|
|
43
|
+
# Progressive disclosure
|
|
44
|
+
- Keep this file short.
|
|
45
|
+
- Load the reference template only when writing the final review.
|
|
46
|
+
|
|
47
|
+
# References
|
|
48
|
+
- `references/rfc-review-template.md` — standard review format and verdict rubric
|
|
49
|
+
- `references/rfc-review-checklist.md` — questions to apply while reviewing an RFC
|
|
50
|
+
|
|
51
|
+
# License
|
|
52
|
+
Same as repository (MIT).
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# RFC Review Checklist
|
|
2
|
+
|
|
3
|
+
Ask these questions while reviewing:
|
|
4
|
+
|
|
5
|
+
1. Is the problem statement specific and grounded in current failures?
|
|
6
|
+
2. Are non-goals explicit?
|
|
7
|
+
3. Are contracts concrete enough to implement?
|
|
8
|
+
4. Are acceptance criteria testable?
|
|
9
|
+
5. Does the RFC define the source of truth for new fields or behaviors?
|
|
10
|
+
6. Does it match existing code paths and public tool surfaces?
|
|
11
|
+
7. Can each open concern be classified as a spec issue or an implementation issue?
|
|
12
|
+
8. Is the RFC ready to implement without further interpretation?
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# RFC Review Template
|
|
2
|
+
|
|
3
|
+
Use this structure for every RFC review:
|
|
4
|
+
|
|
5
|
+
## Verdict
|
|
6
|
+
- Ready / Needs clarification / Needs implementation contract / Not ready
|
|
7
|
+
|
|
8
|
+
## Summary
|
|
9
|
+
- One short paragraph on the RFC's current quality.
|
|
10
|
+
|
|
11
|
+
## What is good
|
|
12
|
+
- List the strongest parts of the RFC.
|
|
13
|
+
|
|
14
|
+
## Issues
|
|
15
|
+
For each issue, include:
|
|
16
|
+
- **Type:** spec / implementation / implementation contract / doc
|
|
17
|
+
- **Severity:** low / medium / high
|
|
18
|
+
- **Why it matters:** one sentence
|
|
19
|
+
- **Fix:** exact change needed
|
|
20
|
+
|
|
21
|
+
## Missing contract surfaces
|
|
22
|
+
- List any API shapes, response fields, state transitions, or invariants that are still undefined.
|
|
23
|
+
|
|
24
|
+
## Codebase alignment
|
|
25
|
+
- Note whether the RFC matches current `src/`, docs, and tests.
|
|
26
|
+
|
|
27
|
+
## Next step
|
|
28
|
+
- State the smallest next action needed to move the RFC forward.
|
package/src/interact/index.ts
CHANGED
|
@@ -5,6 +5,7 @@ export { AndroidInteract, iOSInteract };
|
|
|
5
5
|
|
|
6
6
|
import { resolveTargetDevice } from '../utils/resolve-device.js'
|
|
7
7
|
import { ToolsObserve } from '../observe/index.js'
|
|
8
|
+
import { computeSnapshotSignature } from '../observe/snapshot-metadata.js'
|
|
8
9
|
import { nextActionId } from '../server/common.js'
|
|
9
10
|
import type {
|
|
10
11
|
ActionFailureCode,
|
|
@@ -12,6 +13,7 @@ import type {
|
|
|
12
13
|
ExpectElementVisibleResponse,
|
|
13
14
|
ExpectStateResponse,
|
|
14
15
|
ExpectScreenResponse,
|
|
16
|
+
WaitForUIChangeResponse,
|
|
15
17
|
UIElementState,
|
|
16
18
|
TapElementResponse
|
|
17
19
|
} from '../types.js'
|
|
@@ -60,9 +62,16 @@ interface UiResolution {
|
|
|
60
62
|
height?: number
|
|
61
63
|
}
|
|
62
64
|
|
|
65
|
+
interface UiChangeSignatureSet {
|
|
66
|
+
hierarchy: string | null
|
|
67
|
+
text: string | null
|
|
68
|
+
state: string | null
|
|
69
|
+
}
|
|
70
|
+
|
|
63
71
|
|
|
64
72
|
export class ToolsInteract {
|
|
65
73
|
private static readonly _maxResolvedUiElements = 256
|
|
74
|
+
private static readonly _uiChangeKinds: Array<'hierarchy_diff' | 'text_change' | 'state_change'> = ['hierarchy_diff', 'text_change', 'state_change']
|
|
66
75
|
private static readonly _sliderSearchLookahead = 8
|
|
67
76
|
private static readonly _sliderNegativeGapTolerancePx = 32
|
|
68
77
|
private static readonly _sliderPositiveGapLimitPx = 640
|
|
@@ -85,6 +94,10 @@ export class ToolsInteract {
|
|
|
85
94
|
return normalized as [number, number, number, number]
|
|
86
95
|
}
|
|
87
96
|
|
|
97
|
+
private static _hash(value: unknown): string {
|
|
98
|
+
return createHash('sha256').update(JSON.stringify(value)).digest('hex')
|
|
99
|
+
}
|
|
100
|
+
|
|
88
101
|
private static _matchesSelector(el: UiElement, selector?: { text?: string, resource_id?: string, accessibility_id?: string, contains?: boolean }): boolean {
|
|
89
102
|
if (!selector) return false
|
|
90
103
|
const normalize = ToolsInteract._normalize
|
|
@@ -195,6 +208,66 @@ export class ToolsInteract {
|
|
|
195
208
|
}
|
|
196
209
|
}
|
|
197
210
|
|
|
211
|
+
private static _buildUiChangeSignatures(tree: any): UiChangeSignatureSet {
|
|
212
|
+
const elements = Array.isArray(tree?.elements) ? tree.elements as UiElement[] : []
|
|
213
|
+
const textPayload: Array<{ text: string, contentDescription: string, resourceId: string }> = []
|
|
214
|
+
const statePayload: Array<{
|
|
215
|
+
checked: boolean | null
|
|
216
|
+
selected: boolean | string | { id: string; label?: string } | null
|
|
217
|
+
focused: boolean | null
|
|
218
|
+
expanded: boolean | null
|
|
219
|
+
enabled: boolean | null
|
|
220
|
+
text_value: string | null
|
|
221
|
+
value: number | string | null
|
|
222
|
+
raw_value: number | string | null
|
|
223
|
+
value_range: UIElementState['value_range']
|
|
224
|
+
}> = []
|
|
225
|
+
|
|
226
|
+
for (const el of elements) {
|
|
227
|
+
textPayload.push({
|
|
228
|
+
text: ToolsInteract._normalize(el?.text ?? el?.label ?? el?.value ?? ''),
|
|
229
|
+
contentDescription: ToolsInteract._normalize(el?.contentDescription ?? el?.contentDesc ?? el?.accessibilityLabel ?? ''),
|
|
230
|
+
resourceId: ToolsInteract._normalize(el?.resourceId ?? el?.resourceID ?? el?.id ?? '')
|
|
231
|
+
})
|
|
232
|
+
|
|
233
|
+
statePayload.push({
|
|
234
|
+
checked: el?.state?.checked ?? null,
|
|
235
|
+
selected: el?.state?.selected ?? null,
|
|
236
|
+
focused: el?.state?.focused ?? null,
|
|
237
|
+
expanded: el?.state?.expanded ?? null,
|
|
238
|
+
enabled: el?.state?.enabled ?? null,
|
|
239
|
+
text_value: el?.state?.text_value ?? null,
|
|
240
|
+
value: el?.state?.value ?? null,
|
|
241
|
+
raw_value: el?.state?.raw_value ?? null,
|
|
242
|
+
value_range: el?.state?.value_range ?? null
|
|
243
|
+
})
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
return {
|
|
247
|
+
hierarchy: computeSnapshotSignature(tree),
|
|
248
|
+
text: ToolsInteract._hash({
|
|
249
|
+
screen: ToolsInteract._normalize(tree?.screen),
|
|
250
|
+
elements: textPayload
|
|
251
|
+
}),
|
|
252
|
+
state: ToolsInteract._hash({
|
|
253
|
+
screen: ToolsInteract._normalize(tree?.screen),
|
|
254
|
+
elements: statePayload
|
|
255
|
+
})
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
private static _matchesUiChange(expected: 'hierarchy_diff' | 'text_change' | 'state_change' | undefined, initial: UiChangeSignatureSet, current: UiChangeSignatureSet): 'hierarchy_diff' | 'text_change' | 'state_change' | null {
|
|
260
|
+
const candidates = expected ? [expected] : ToolsInteract._uiChangeKinds
|
|
261
|
+
|
|
262
|
+
for (const changeKind of candidates) {
|
|
263
|
+
if (changeKind === 'hierarchy_diff' && initial.hierarchy !== current.hierarchy) return changeKind
|
|
264
|
+
if (changeKind === 'text_change' && initial.text !== current.text) return changeKind
|
|
265
|
+
if (changeKind === 'state_change' && initial.state !== current.state) return changeKind
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
return null
|
|
269
|
+
}
|
|
270
|
+
|
|
198
271
|
private static _resolvedTargetFromElement(
|
|
199
272
|
elementId: string,
|
|
200
273
|
element: UiElement,
|
|
@@ -955,6 +1028,84 @@ export class ToolsInteract {
|
|
|
955
1028
|
}
|
|
956
1029
|
}
|
|
957
1030
|
|
|
1031
|
+
static async waitForUIChangeHandler({
|
|
1032
|
+
platform,
|
|
1033
|
+
deviceId,
|
|
1034
|
+
timeout_ms = 60000,
|
|
1035
|
+
stability_window_ms = 250,
|
|
1036
|
+
expected_change
|
|
1037
|
+
}: {
|
|
1038
|
+
platform?: 'android' | 'ios',
|
|
1039
|
+
deviceId?: string,
|
|
1040
|
+
timeout_ms?: number,
|
|
1041
|
+
stability_window_ms?: number,
|
|
1042
|
+
expected_change?: 'hierarchy_diff' | 'text_change' | 'state_change'
|
|
1043
|
+
}): Promise<WaitForUIChangeResponse> {
|
|
1044
|
+
const start = Date.now()
|
|
1045
|
+
const pollIntervalMs = 300
|
|
1046
|
+
const stabilityWindow = Math.max(0, typeof stability_window_ms === 'number' ? stability_window_ms : 250)
|
|
1047
|
+
let baseline: UiChangeSignatureSet | null = null
|
|
1048
|
+
let lastObservedRevision: number | null = null
|
|
1049
|
+
let lastLoadingState: any = null
|
|
1050
|
+
|
|
1051
|
+
while (Date.now() - start < timeout_ms) {
|
|
1052
|
+
try {
|
|
1053
|
+
const tree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
|
|
1054
|
+
const signatures = ToolsInteract._buildUiChangeSignatures(tree)
|
|
1055
|
+
lastObservedRevision = typeof tree?.snapshot_revision === 'number' ? tree.snapshot_revision : lastObservedRevision
|
|
1056
|
+
lastLoadingState = tree?.loading_state ?? lastLoadingState
|
|
1057
|
+
|
|
1058
|
+
if (!baseline) {
|
|
1059
|
+
baseline = signatures
|
|
1060
|
+
} else {
|
|
1061
|
+
const observedChange = ToolsInteract._matchesUiChange(expected_change, baseline, signatures)
|
|
1062
|
+
if (observedChange) {
|
|
1063
|
+
if (stabilityWindow > 0) {
|
|
1064
|
+
await new Promise(resolve => setTimeout(resolve, stabilityWindow))
|
|
1065
|
+
const confirmTree = await ToolsObserve.getUITreeHandler({ platform, deviceId }) as any
|
|
1066
|
+
const confirmSignatures = ToolsInteract._buildUiChangeSignatures(confirmTree)
|
|
1067
|
+
const confirmChange = ToolsInteract._matchesUiChange(expected_change, baseline, confirmSignatures)
|
|
1068
|
+
if (!confirmChange || confirmSignatures.hierarchy !== signatures.hierarchy || confirmSignatures.text !== signatures.text || confirmSignatures.state !== signatures.state) {
|
|
1069
|
+
lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision
|
|
1070
|
+
lastLoadingState = confirmTree?.loading_state ?? lastLoadingState
|
|
1071
|
+
await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
|
|
1072
|
+
continue
|
|
1073
|
+
}
|
|
1074
|
+
lastObservedRevision = typeof confirmTree?.snapshot_revision === 'number' ? confirmTree.snapshot_revision : lastObservedRevision
|
|
1075
|
+
lastLoadingState = confirmTree?.loading_state ?? lastLoadingState
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
return {
|
|
1079
|
+
success: true,
|
|
1080
|
+
observed_change: observedChange,
|
|
1081
|
+
snapshot_revision: lastObservedRevision ?? undefined,
|
|
1082
|
+
timeout: false,
|
|
1083
|
+
elapsed_ms: Date.now() - start,
|
|
1084
|
+
expected_change,
|
|
1085
|
+
loading_state: lastLoadingState ?? null,
|
|
1086
|
+
reason: 'UI change observed'
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
} catch {
|
|
1091
|
+
// Keep polling until timeout; the observable surface should be best-effort.
|
|
1092
|
+
}
|
|
1093
|
+
|
|
1094
|
+
await new Promise(resolve => setTimeout(resolve, pollIntervalMs))
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
return {
|
|
1098
|
+
success: false,
|
|
1099
|
+
observed_change: null,
|
|
1100
|
+
snapshot_revision: lastObservedRevision ?? undefined,
|
|
1101
|
+
timeout: true,
|
|
1102
|
+
elapsed_ms: Date.now() - start,
|
|
1103
|
+
expected_change,
|
|
1104
|
+
loading_state: lastLoadingState ?? null,
|
|
1105
|
+
reason: 'timeout'
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
|
|
958
1109
|
static async expectScreenHandler({
|
|
959
1110
|
platform,
|
|
960
1111
|
fingerprint,
|