mobile-automator 0.38.4 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +2 -0
  2. package/package.json +4 -2
  3. package/src/assertion/evaluator.js +116 -14
  4. package/src/assertion/fields.js +106 -0
  5. package/src/assertion/observe.js +17 -0
  6. package/src/cli.js +180 -174
  7. package/src/device/action-catalog.js +4 -2
  8. package/src/device/bridge.js +17 -1
  9. package/src/device/mobile-mcp-tools.js +1 -0
  10. package/src/device/resolve-connection.js +1 -1
  11. package/src/device/semantic-press/index.js +19 -0
  12. package/src/device/semantic-press/resolver.js +1 -1
  13. package/src/{device/semantic-press/labels.js → dialog/permission-labels.js} +6 -3
  14. package/src/guide/content/execute.agnostic.md +74 -21
  15. package/src/guide/content/execute.aware.md +69 -15
  16. package/src/guide/content/execute.invariants.md +1 -0
  17. package/src/guide/content/generate.agnostic.md +3 -3
  18. package/src/guide/content/generate.aware.md +5 -5
  19. package/src/guide/emitter.js +2 -0
  20. package/src/init/host-plan.js +1 -0
  21. package/src/observe/trace.js +1 -1
  22. package/src/observe/vocab.js +6 -0
  23. package/src/output/device-fail.js +41 -0
  24. package/src/output/envelope.js +3 -3
  25. package/src/output/run-human.js +37 -0
  26. package/src/result/capability-catalog.js +48 -0
  27. package/src/result/executor.js +58 -0
  28. package/src/result/finalize-against-scenario.js +103 -0
  29. package/src/result/flags.js +49 -6
  30. package/src/result/junit.js +39 -16
  31. package/src/result/outcome.js +24 -2
  32. package/src/result/pending-hint.js +67 -0
  33. package/src/result/scenario-check.js +58 -0
  34. package/src/result/status-guard.js +46 -0
  35. package/src/result/store.js +159 -16
  36. package/src/run/actions/capture.js +21 -0
  37. package/src/run/actions/gestures.js +89 -0
  38. package/src/run/actions/index.js +48 -0
  39. package/src/run/actions/launch.js +30 -0
  40. package/src/run/actions/preconditions.js +44 -0
  41. package/src/run/actions/scroll.js +22 -0
  42. package/src/run/actions/semantic.js +16 -0
  43. package/src/run/actions/wait.js +66 -0
  44. package/src/run/assert-reference.js +118 -0
  45. package/src/run/assertions.js +102 -0
  46. package/src/run/clock.js +10 -0
  47. package/src/run/condition.js +80 -0
  48. package/src/run/fatal.js +15 -0
  49. package/src/run/handle.js +151 -0
  50. package/src/run/needs-agent.js +39 -0
  51. package/src/run/poll.js +38 -0
  52. package/src/run/prepare.js +107 -0
  53. package/src/run/reference-assertion.js +172 -0
  54. package/src/run/reference-screenshot.js +92 -0
  55. package/src/run/resolve.js +95 -0
  56. package/src/run/run-id.js +14 -0
  57. package/src/run/runner.js +104 -0
  58. package/src/run/scale.js +52 -0
  59. package/src/run/screenshots.js +163 -0
  60. package/src/run/settings.js +35 -0
  61. package/src/run/state.js +22 -0
  62. package/src/run/step.js +138 -0
  63. package/src/run/summary.js +73 -0
  64. package/src/run/trace.js +30 -0
  65. package/src/run/tree.js +96 -0
  66. package/src/run/variables.js +56 -0
  67. package/src/scenario/load.js +78 -0
  68. package/src/schemas/config_schema.json +29 -0
  69. package/src/schemas/result_schema.json +45 -3
  70. package/src/schemas/scenario_schema.json +27 -0
  71. package/src/util/as-array.js +8 -0
  72. package/src/util/parse-positive-int.js +15 -0
  73. package/src/util/parse-unit-interval.js +17 -0
@@ -45,6 +45,7 @@ const MOBILE_MCP_TOOL_NAMES = Object.freeze([
45
45
  'mobile_uninstall_app',
46
46
  'mobile_open_url',
47
47
  'mobile_set_orientation',
48
+ 'mobile_get_orientation',
48
49
  // Diagnostic, not a scenario action — see DeviceBridge#listCrashes.
49
50
  'mobile_list_crashes',
50
51
  'mobile_get_crash',
@@ -135,7 +135,7 @@ async function resolveDeviceConnection({
135
135
  // what failed, the daemon's log is the only record of why, and transparent
136
136
  // autostart is the common path — most users never type `mauto session start`,
137
137
  // so without this the log stays invisible to almost everyone. The signal
138
- // rides err.hint because deviceFail() in cli.js already surfaces it on every
138
+ // rides err.hint because deviceFail() (src/output/device-fail.js) already surfaces it on every
139
139
  // device verb, so acquireConnection's deliberately narrow contract is
140
140
  // untouched (#163).
141
141
  const oneShotWithLogHint = async (afterFailedSpawn) => {
@@ -29,10 +29,29 @@ function selectResolver(platform, bridge) {
29
29
  return new Strategy(bridge);
30
30
  }
31
31
 
32
+ // Perform one semantic action on the bridge's device: ask the device its
33
+ // platform, pick that platform's resolver, run the action's method. Returns
34
+ // what was done, `{ platform, mechanism }`. The one implementation behind
35
+ // `mauto press <action>` and `mauto run`; a non-semantic token or an
36
+ // unsupported platform throws SemanticResolutionError before any device input.
37
+ async function pressSemantic(bridge, action) {
38
+ if (!isSemanticAction(action)) {
39
+ throw new SemanticResolutionError(
40
+ `"${action}" is not a semantic action.`,
41
+ `Semantic actions: ${SEMANTIC_ACTIONS.join(', ')}.`
42
+ );
43
+ }
44
+ const platform = await bridge.getPlatform();
45
+ const resolver = selectResolver(platform, bridge);
46
+ const { mechanism } = await resolver[ACTION_METHOD[action]]();
47
+ return { platform, mechanism };
48
+ }
49
+
32
50
  module.exports = {
33
51
  SEMANTIC_ACTIONS,
34
52
  ACTION_METHOD,
35
53
  isSemanticAction,
36
54
  selectResolver,
55
+ pressSemantic,
37
56
  SemanticResolutionError,
38
57
  };
@@ -1,6 +1,6 @@
1
1
  'use strict';
2
2
 
3
- const { ALLOW_LABELS, DENY_LABELS, findByLabels } = require('./labels');
3
+ const { ALLOW_LABELS, DENY_LABELS, findByLabels } = require('../../dialog/permission-labels');
4
4
 
5
5
  // Raised when a semantic action cannot be performed on the current screen/
6
6
  // platform. `.kind`/`.hint` let the CLI map it onto the `device` envelope.
@@ -1,6 +1,9 @@
1
1
  'use strict';
2
2
 
3
- // System permission-dialog button labels, per platform. Lowercase and
3
+ // System permission-dialog button labels, per platform. Neutral ground for
4
+ // both readers: semantic-press taps them (grant/deny_permission) and the
5
+ // assertion evaluator looks for them (permission_dialog_shown), so neither
6
+ // layer imports the other's internals. Lowercase and
4
7
  // apostrophe-normalized so matching is locale-tolerant within English.
5
8
  const ALLOW_LABELS = {
6
9
  android: ['allow', 'while using the app', 'allow only this time', 'ok', 'continue'],
@@ -16,8 +19,8 @@ function norm(s) {
16
19
  // Map the curly apostrophe (U+2019, the ’ glyph in the regex below) onto a
17
20
  // straight quote, so "Don’t Allow" matches the straight-quoted entries in the
18
21
  // label sets above. The U+2019 byte must be preserved literally here; the
19
- // guard assertion in labels.test.js fails loudly if it is ever flattened to a
20
- // straight quote by editor/transport tooling.
22
+ // guard assertion in permission-labels.test.js fails loudly if it is ever
23
+ // flattened to a straight quote by editor/transport tooling.
21
24
  return String(s == null ? '' : s).replace(/’/g, "'").trim().toLowerCase();
22
25
  }
23
26
 
@@ -74,7 +74,48 @@ Record each observation with `mauto result add-step --observation <type>:<messag
74
74
 
75
75
  The remaining precondition actions are not mechanically executable (see step execution): report them honestly or perform them out of band.
76
76
 
77
- ### 2. Semantic Action Resolution
77
+ ### 2. Replay with mauto run
78
+
79
+ Replay the scenario with the unattended runner before anything else:
80
+
81
+ ```bash
82
+ mauto run <scenario path> --run-id $MAUTO_RUN_ID
83
+ ```
84
+
85
+ Add `--junit <path>` when a CI job needs a JUnit XML report of the run (the file's directory must already exist). It is one invocation and one envelope: the runner performs the precondition `device_actions`, every step, and every assertion the CLI can decide, captures the evidence, and finalizes the result file itself. Nothing waits for you mid-run, and you never drive a step through it.
86
+
87
+ If pre-flight reported an `unsupported_action` warning, skip this section: `mauto run` refuses such a scenario with `invalid_input` before any device work. Replay it by hand instead (section 4).
88
+
89
+ `data` carries the same keys whatever the outcome: `run_id`, `scenario_id`, `status`, `result_path`, `duration_seconds`, `steps`, `assertions`, `failed_step`, `failures`, `pending`, `junit_path`, `executor`. Branch on the outcome:
90
+
91
+ - **Passed** — `ok: true`, exit code `0`, `data.status` is `passed`. Every step ran and the CLI decided every assertion. The result file is already finalized at `data.result_path`, so there is nothing left to record or finalize: present it (section 6 describes what it holds).
92
+ - **`test_failed`** — `ok: false` with `error.kind: "test_failed"` and exit code `6`; `data.status` is `failed` or `error`. That is the test result, not a tool error. Report the failure from `data`: `failed_step` is the first step that failed (`null` when only an assertion failed), and `failures` lists each failed step and assertion with its message. The runner stops at a failed step, so later steps did not run; the failed step's screenshot is recorded on its `steps_executed` entry in the result file at `data.result_path`. Do not re-judge a verdict the CLI decided, and do not retry the run or replay it by hand to get around the failure. The result is already finalized: present it.
93
+ - **`incomplete`** — `ok: false` with `error.kind: "incomplete"` and exit code `7`; `data.status` is `incomplete`. Nothing failed, but something is left for an agent to decide. Continue with section 3.
94
+ - **Anything else** is not a verdict about the app. `invalid_input` is a refusal before any device work (a bad flag, an unreadable file, or a scenario that cannot replay unattended): read `error.message` and the `hint`. `device` means no device could be driven: fix what the `hint` names and run again. An `environment` failure that still carries the run's `data` means only the JUnit report could not be written; the run itself is finalized and `data.status` is its outcome.
95
+
96
+ ### 3. Handle needs_agent
97
+
98
+ An `incomplete` run is waiting on an agent. Its `hint` says for what, and prints the exact commands that complete it.
99
+
100
+ **Pending assertions.** `data.pending` lists every assertion the CLI could not decide, each as `{assertion_id, step_id, type}`; the result file records them with status `needs_agent`. For each entry:
101
+
102
+ 1. **Find its screenshot.** Open the result file (`data.result_path`) and read the `screenshot` of the `steps_executed` entry with that `step_id`; the runner captures one for every step the scenario marks `checkpoint`. If none was recorded and `step_id` is the last step the run executed, the device is still showing that screen: capture it with `mauto screenshot <path>`. If neither holds, the screen is gone. Do not guess: leave that assertion pending.
103
+ 2. **Judge it** from that screenshot, per the Tier 2 list in section 5.
104
+ 3. **Record the verdict** with the `mauto result add-assertion` line the hint prints for that id, filling in only the verdict and the reason:
105
+ `mauto result add-assertion --run-id <run id> --step-id <step_id> --assertion-id <assertion_id> --type <type> --pass <true|false> --judge agent --message "<why>"`.
106
+ `--assertion-id` is what makes the verdict replace the pending entry in place, and `--message` IS the evidence, so omit neither.
107
+
108
+ A pending `screenshot_match` is the exception: it is pending because its reference screenshot does not exist yet, and it is never judged by eye. Ask the user whether the screen is correct; if they confirm, record the baselines by running the scenario again under a new run id with `--update-references` added. Otherwise leave it pending.
109
+
110
+ Then finalize again with the last line of the hint, `mauto result finalize --run-id <run id> --scenario <path>` (the hint prints the scenario's absolute path, so the command works from any directory). The status is derived again from what is now recorded: `passed` (exit code `0`), `failed` if any verdict was `false` (`test_failed`, exit code `6`), or still `incomplete` (exit code `7`) if you left anything pending. Report a run that is still `incomplete` as incomplete, never as passed.
111
+
112
+ **A pending step.** A hint line `Pending step <step_id>: <what to do>`, with `data.steps.needs_agent` above zero, means the runner reached a step it cannot perform itself (for example an install with no build path, or a condition it cannot evaluate). It stopped there, so no later step ran, and that recorded step keeps this run `incomplete` whatever is added to it afterwards. Do what the line asks, then replay the whole scenario by hand (section 4) under a new run id: export a new `MAUTO_RUN_ID`, perform that step with the ordinary one-shot verbs, and say in your report that the unattended run ended `incomplete` at that step.
113
+
114
+ ### 4. Manual per-step replay
115
+
116
+ This is the fallback: the per-step recipe an agent follows with the one-shot verbs. Use it only when the runner cannot be used for this scenario (section 2 refused it, or pre-flight reported an `unsupported_action`), or to perform a step the runner left `needs_agent` (section 3). Otherwise the runner has already done all of this.
117
+
118
+ #### Semantic Action Resolution
78
119
 
79
120
  When you encounter a semantic action — `press_back`, `dismiss_keyboard`, `grant_permission`, `deny_permission` — run `mauto press <action>` and let the CLI resolve it for the connected device's `platform`. Never record or hardcode the per-platform mechanics; the scenario stays portable.
80
121
 
@@ -85,17 +126,17 @@ When you encounter a semantic action — `press_back`, `dismiss_keyboard`, `gran
85
126
 
86
127
  If the device's platform has no resolution, halt the step and report it as unsupported on this device. If a resolution's primary attempt fails, the CLI falls back to the documented alternative before failing the step — record both the primary and fallback attempts in the step's `observations`.
87
128
 
88
- ### 3. Per-Step Execution
129
+ #### Per-Step Execution
89
130
 
90
131
  Before executing each step, initialize a **session variable map** (an empty object to store captured values).
91
132
 
92
133
  For each step in the scenario:
93
134
 
94
- **3.0 Pre-step: Evaluate condition**
135
+ **4.0 Pre-step: Evaluate condition**
95
136
 
96
137
  - If the step has a `condition` field, evaluate it (`device_property`, `previous_step_skipped`, `variable_value`, `element_visible`). If the condition is **false** → skip this step and all its `sub_steps`. Mark as `skipped`.
97
138
 
98
- **3.1 Execute the step**
139
+ **4.1 Execute the step**
99
140
 
100
141
  1. **Wait for prior stability:** poll `mauto elements` until loading indicators ({{loading_indicators}}) are absent before continuing.
101
142
  2. **Resolve the target:** for actions with a target, run `mauto elements` to locate by semantic description; never use stored coordinates. If `target_pattern` is set, use it as a regex to filter elements by text.
@@ -105,10 +146,10 @@ For each step in the scenario:
105
146
  - Semantic actions → run `mauto press <action>` and let the CLI resolve them for the connected device.
106
147
  - Not mechanically executable → `clear_app_data` (and the precondition device actions `enable_wifi` / `disable_wifi`) have no verb and no underlying primitive. Do NOT substitute a gesture: report them honestly as unsupported on this run, or perform them manually out of band if the run depends on them.
107
148
  4. **Capture screenshot:** Run `mauto screenshot .mobile-automator/screenshots/<scenario_id>/<run_id>/step_<step_id>.png`.
108
- 5. **Evaluate assertions** attached to the step (see section 4).
149
+ 5. **Evaluate assertions** attached to the step (see section 5).
109
150
  6. **Record** the step result with `mauto result add-step --step-id <id> --status <s> --screenshot <path> [--attempts <n>] [--error-message <text>] [--observation <type>:<message>] [--capture <name>=<value>]`, then record each assertion verdict with `mauto result add-assertion --step-id <id> --type <t> --pass <true|false> --message <text> [--expected <v> --actual <v>]`. `--message` carries the justification you already formed — for Tier-2 verdicts it IS the evidence, so never omit it; add `--expected`/`--actual` whenever the assertion compares a specific value.
110
151
 
111
- **3.2 Handle step outcome**
152
+ **4.2 Handle step outcome**
112
153
 
113
154
  - `"fail"` (default): capture a failure screenshot, apply the Flakiness Detector logic, record as failed via `mauto result add-step --status fail --error-message <text> --screenshot <path>`.
114
155
  - `"skip"`: mark `skipped`, continue silently, do NOT fail the scenario.
@@ -116,11 +157,11 @@ For each step in the scenario:
116
157
  - `optional: true` overrides a failure to `"skip"`.
117
158
  - `sub_steps`: when the parent succeeds, execute each sub-step in order under the same rules, then resume at the next top-level step.
118
159
 
119
- **3.3 Variable capture**
160
+ **4.3 Variable capture**
120
161
 
121
162
  When executing a `capture_value` action: locate the element with `mauto elements`, extract its text, and store it in the session variable map under the step's `capture_to` name (`variables[<capture_to>] = <extracted_text>`). Report: "Captured `capture_to` = '<value>'".
122
163
 
123
- ### When a step fails, find out whether the app is still alive
164
+ #### When a step fails, find out whether the app is still alive
124
165
 
125
166
  An action that fails, or an `elements` call that comes back empty, has two very
126
167
  different explanations: the UI changed, or the app died. They lead to opposite
@@ -160,7 +201,7 @@ default — a report from last week is not evidence about this step. And a repor
160
201
  whose own timestamp is unreadable is counted in `unattributed` rather than
161
202
  included; if that count is non-zero, widen `--since` and look yourself.
162
203
 
163
- ### When the element list looks wrong, check the engine
204
+ #### When the element list looks wrong, check the engine
164
205
 
165
206
  If the app is alive but `mauto elements` misses the label of a text field you
166
207
  can see on screen, or lists content from a tab that is not showing, the device
@@ -172,20 +213,34 @@ recover text-field labels; if the engine is already `legacy`, a missing
172
213
  text-field label is the device OS version, not the engine setting. Do not
173
214
  switch the engine mid-run yourself.
174
215
 
175
- ### 4. Validate Assertions (27 types)
216
+ ### 5. Validate Assertions (27 types)
176
217
 
177
- For each assertion, pick the type recorded in the scenario and evaluate it. The 27 types split into two tiers by **who decides pass/fail**:
218
+ For each assertion, pick the type recorded in the scenario and evaluate it. The 27 types split into two tiers by **who decides pass/fail**, plus one reference-screenshot type the CLI also decides:
178
219
 
179
- - **Tier 1 — mechanical (the CLI decides).** Deterministic checks over the on-screen element list (plus captured variables / a previous snapshot). Run `mauto elements`, then evaluate with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). A passing verdict arrives as a normal `ok: true` envelope. A failing one arrives as `ok: false` with `error.kind: "test_failed"` and exit code `6`, and the verdict (`pass: false`) is still in `data`. That is the test result, not a tool error: do not retry the verb or treat the run as broken. You record that verdict — you do not re-judge it.
220
+ - **Tier 1 — mechanical (the CLI decides).** Deterministic checks over the on-screen element list (plus captured variables / a previous snapshot / the screen size). Run `mauto elements`, then evaluate with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). A passing verdict arrives as a normal `ok: true` envelope. A failing one arrives as `ok: false` with `error.kind: "test_failed"` and exit code `6`, and the verdict (`pass: false`) is still in `data`. That is the test result, not a tool error: do not retry the verb or treat the run as broken. You record that verdict — you do not re-judge it.
221
+ - **Reference screenshot — `screenshot_match` (the CLI decides).** A deterministic pixel comparison against a stored reference screenshot (a baseline); see the reference-screenshot block below. You record its verdict — you do not judge it by eye.
180
222
  - **Tier 2 — visual / semantic (the agent decides).** These require judging pixels. `mauto assert <type> ...` reports them as `needs_agent` (with `pass: null`). For these you take a screenshot with `mauto screenshot <path>`, visually analyze it, and report pass/fail yourself with a clear justification.
181
223
 
224
+ `mauto run` decides the Tier 1 types and the reference-screenshot type itself and records each verdict; after a run you handle only what it left `needs_agent` (section 3). The `mauto assert` instructions below are for a manual replay (section 4).
225
+
182
226
  > One assertion type, `element_state`, names a Tier-1 intent but its per-attribute state inspection is not represented in the element model the CLI reads, so `mauto assert` returns it as `needs_agent` — judge it from the element list / screenshot like a Tier-2 check.
227
+ >
228
+ > Three Tier-1 types read the element list more narrowly than a screenshot would, so an agent never needs to re-judge them: `element_fully_visible` checks only that the element is not clipped by the screen edges (it cannot see overlap); `screen_title` passes when an element whose text or label equals `expected_text` starts in the top quarter of the screen; `permission_dialog_shown` passes when an allow button and a deny button are both on screen.
183
229
 
184
230
  ---
185
231
 
186
232
  **Tier 1 — mechanical, evaluated with `mauto assert <type>` over `mauto elements`**
187
233
 
188
- `element_exists`, `element_not_exists`, `element_visible` (`expected_visible: true/false`), `element_text` (`expected_value`), `text_contains` (`expected_substring`), `text_not_empty`, `element_hint` (`expected_text`), `pattern_match` (`pattern`), `element_state` (`state_property` — see note above; resolved by the agent), `element_count` / `list_item_count` (`operator` + `expected_count`), `list_is_empty`, `content_description` (`label_value`), `has_accessibility_label` (`label_value` optional), `value_matches_variable` (`variable_name`), `text_changed`.
234
+ `element_exists`, `element_not_exists`, `element_visible` (`expected_visible: true/false`), `element_text` (`expected_value`), `text_contains` (`expected_substring`), `text_not_empty`, `element_hint` (`expected_text`), `pattern_match` (`pattern`), `element_state` (`state_property` — see note above; resolved by the agent), `element_count` / `list_item_count` (`operator` + `expected_count`), `list_is_empty`, `content_description` (`label_value`), `has_accessibility_label` (`label_value` optional), `value_matches_variable` (`variable_name`), `text_changed`, `element_fully_visible` (`element_description`), `screen_title` (`expected_text`), `permission_dialog_shown` (`permission_name` optional, passed as `--permission`).
235
+
236
+ ---
237
+
238
+ **Reference screenshot — `screenshot_match`, decided by `mauto assert` against a baseline**
239
+
240
+ - Run `mauto assert screenshot_match --reference <reference_screenshot> --tolerance <tolerance>`, adding one `--mask <x,y,w,h>` or `--mask "<element description>"` per entry in the assertion's `masks`. Omit `--tolerance` when the scenario sets none (default `0.9`). The CLI captures a stable screenshot, compares it pixel by pixel, and decides: a pass is `ok: true`; a mismatch is `ok: false` with `error.kind: "test_failed"`, exit `6`, the similarity score and a diff image path in `data`. Record that verdict as-is — do not override it by looking at the screenshot.
241
+ - A dimension mismatch (a different device or orientation than the baseline was recorded on) is a failure too; report it as such and tell the user the baseline needs re-recording for this device.
242
+ - If the reference screenshot does not exist yet, the verdict is `needs_agent` (exit `0`), not a failure. Do not judge it by eye instead. Ask the user whether the current screen is correct; if they confirm, record the baseline with the same command plus `--update-references`, otherwise record the assertion as pending.
243
+ - Record the verdict with the `mauto result add-assertion` command in the envelope's `hint`; it carries the reference screenshot, the actual screenshot and the similarity score.
189
244
 
190
245
  ---
191
246
 
@@ -193,22 +248,18 @@ For each assertion, pick the type recorded in the scenario and evaluate it. The
193
248
 
194
249
  For each: take a screenshot, visually analyze it, and report pass/fail with a clear justification.
195
250
 
196
- - **`screenshot_match`:** compare against the reference baseline using **semantic visual comparison** — screen identity, key elements, text content, layout structure — within the `tolerance` threshold.
197
251
  - **`visual_state`:** assess `loaded` / `loading` / `empty` / `error` against `expected_visual_state`.
198
- - **`element_fully_visible`:** the element is fully in the viewport — not clipped, scrolled off, or overlapped.
199
252
  - **`color_style`:** assess the element's dominant color against `color_hex` or `expected_theme`.
200
- - **`screen_title`:** read the screen title and compare to `expected_text`.
201
253
  - **`alert_present` / `alert_text`:** a dialog is present (and its text matches `expected_text`).
202
254
  - **`toast_visible`:** look for a transient notification overlay; match `expected_text`, or mark `failed` if it already disappeared.
203
255
  - **`keyboard_visible`:** whether the soft keyboard is shown (`expected_visible`).
204
256
  - **`dark_mode_active`:** the screen appearance matches `expected_theme`.
205
- - **`permission_dialog_shown`:** a permission request dialog is visible (matching `permission_name` if set).
206
257
 
207
- **Dynamic content:** if the reference `expected_state` marks content as `[dynamic:...]`, ignore those regions during comparison.
258
+ **Dynamic content:** if the reference `expected_state` marks content as `[dynamic:...]`, ignore those regions when you judge a Tier-2 check. For `screenshot_match`, dynamic regions are excluded only through the assertion's `masks`.
208
259
 
209
260
  **Variable substitution in assertions:** before evaluating any assertion, replace `variable_name` references in string fields with values from the session variable map.
210
261
 
211
- ### 5. Generate the Result Report
262
+ ### 6. Generate the Result Report
212
263
 
213
264
  Obtain the result schema by running `mauto schema result`, then finalize the run with `mauto result finalize` (writing to `.mobile-automator/results/<run_id>.json`), passing the run metadata you already know (`--device-model`, `--api-level`, `--app-version`, `--environment`) — the timestamp is filled in for you, and omitted fields are recorded as `unknown`. You may supply your own narrative with `--summary <text>`; omitting it keeps the generated default summary line. A run that finalizes as `failed` or `error` returns `ok: false` with `error.kind: "test_failed"` and exit code `6`. The result file is still written and the full result is in `data`, so present it as usual; the non-zero exit is what lets CI see a red run. The finalized result assembles what the earlier `add-step`/`add-assertion` calls already recorded:
214
265
 
@@ -220,7 +271,7 @@ Obtain the result schema by running `mauto schema result`, then finalize the run
220
271
 
221
272
  `--duration` is now a cross-check rather than the recorded value: when a run trace exists the duration is measured from it, your figure is kept as `measurements.reported_duration_seconds`, and a disagreement beyond the tolerance is recorded as a `state_context` observation. Report the duration you believe is right and let the two be compared; do not reverse-engineer a number to match.
222
273
 
223
- ### 6. Flakiness & Resolution Reporting
274
+ ### 7. Flakiness & Resolution Reporting
224
275
 
225
276
  When a semantic action's primary resolution fails and the fallback succeeds, add a `flakiness` observation:
226
277
 
@@ -230,7 +281,7 @@ When neither attempt succeeds, fail the step and add a detailed observation desc
230
281
 
231
282
  ## Multi-Scenario Execution
232
283
 
233
- - Execute scenarios sequentially. Each scenario's leading `launch_app` restarts the app (`mauto launch <appId>`), so a scenario never inherits the previous one's screen and needs no separate reset; use `app_state: "fresh_install"` only when app data must be wiped too.
284
+ - Execute scenarios sequentially: one `mauto run` per scenario file, each under its own run id. Each scenario's leading `launch_app` restarts the app (`mauto launch <appId>`), so a scenario never inherits the previous one's screen and needs no separate reset; use `app_state: "fresh_install"` only when app data must be wiped too.
234
285
  - "run all smoke tests" → filter by the `smoke` tag. "run all" → every scenario in `.mobile-automator/scenarios/`. List `.mobile-automator/scenarios/` directly: the workspace folder is hidden, and recursive search tools (glob or grep style) skip hidden directories, so they will report it empty.
235
286
  - Present a combined summary with a per-scenario breakdown.
236
287
 
@@ -238,6 +289,7 @@ When neither attempt succeeds, fail the step and add a detailed observation desc
238
289
 
239
290
  ### DO
240
291
 
292
+ - Prefer `mauto run` (section 2); replay step by step (section 4) only when the runner cannot be used.
241
293
  - Execute exactly what the scenario specifies.
242
294
  - Run `mauto elements` before every interaction.
243
295
  - Capture a screenshot at every step, even on failure.
@@ -247,6 +299,7 @@ When neither attempt succeeds, fail the step and add a detailed observation desc
247
299
 
248
300
  ### DON'T
249
301
 
302
+ - Re-judge a verdict the CLI decided (a mechanical or reference-screenshot assertion), or replay by hand to get around a `test_failed` run.
250
303
  - Improvise or deviate from the scenario steps.
251
304
  - Modify app source code in {{protected_directories}}.
252
305
  - Hardcode coordinates — derive them from `mauto elements`.
@@ -62,20 +62,61 @@ Record each observation with `mauto result add-step --observation <type>:<messag
62
62
  - **Fresh install.** If the scenario's `preconditions.app_state` is `fresh_install` (from the scenario you read above), uninstall the app first — `mauto uninstall` with the app package ({{app_package}}) — and only then install, so the run starts with no app data.
63
63
  - State which path you took — installed a supplied artifact, or built — before continuing.
64
64
 
65
- ### 2. Compare Environment
65
+ ### Compare Environment
66
66
 
67
67
  - Read the `metadata` to compare the recording environment against the current execution environment. Note any differences (different device, API level, environment).
68
68
 
69
- ### 3. Verify Preconditions
69
+ ### Verify Preconditions
70
70
 
71
- - **Note:** an `app_state` of `fresh_install` is handled in pre-flight (uninstall before the install, see above), and precondition `device_actions` run per the table in step 4.
71
+ - **Note:** an `app_state` of `fresh_install` is handled in pre-flight (uninstall before the install, see above), and precondition `device_actions` are run by `mauto run` itself (section 2), or per the table in section 4 on a manual replay.
72
72
  - Review the scenario's `preconditions` array to understand the expected starting state.
73
73
  - If any preconditions are state-based (e.g., `"user_logged_out"`, `"no_network"`, `"dark_mode_enabled"`), verify or configure them now:
74
74
  - For app-state preconditions: launch the app and check/configure the state.
75
75
  - For device preconditions: configure device settings (e.g., airplane mode for `"no_network"`).
76
76
  - If a precondition cannot be verified or met, report it and ask the user how to proceed.
77
77
 
78
- ### 4. Step-by-Step Replay
78
+ ### 2. Replay with mauto run
79
+
80
+ Replay the scenario with the unattended runner before anything else:
81
+
82
+ ```bash
83
+ mauto run <scenario path> --run-id $MAUTO_RUN_ID
84
+ ```
85
+
86
+ Add `--junit <path>` when a CI job needs a JUnit XML report of the run (the file's directory must already exist). It is one invocation and one envelope: the runner performs the precondition `device_actions`, every step, and every assertion the CLI can decide, captures the evidence, and finalizes the result file itself. Nothing waits for you mid-run, and you never drive a step through it.
87
+
88
+ If pre-flight reported an `unsupported_action` warning, skip this section: `mauto run` refuses such a scenario with `invalid_input` before any device work. Replay it by hand instead (section 4).
89
+
90
+ `data` carries the same keys whatever the outcome: `run_id`, `scenario_id`, `status`, `result_path`, `duration_seconds`, `steps`, `assertions`, `failed_step`, `failures`, `pending`, `junit_path`, `executor`. Branch on the outcome:
91
+
92
+ - **Passed** — `ok: true`, exit code `0`, `data.status` is `passed`. Every step ran and the CLI decided every assertion. The result file is already finalized at `data.result_path`, so there is nothing left to record or finalize: present it (section 6 describes what it holds).
93
+ - **`test_failed`** — `ok: false` with `error.kind: "test_failed"` and exit code `6`; `data.status` is `failed` or `error`. That is the test result, not a tool error. Report the failure from `data`: `failed_step` is the first step that failed (`null` when only an assertion failed), and `failures` lists each failed step and assertion with its message. The runner stops at a failed step, so later steps did not run; the failed step's screenshot is recorded on its `steps_executed` entry in the result file at `data.result_path`. Do not re-judge a verdict the CLI decided, and do not retry the run or replay it by hand to get around the failure. The result is already finalized: present it.
94
+ - **`incomplete`** — `ok: false` with `error.kind: "incomplete"` and exit code `7`; `data.status` is `incomplete`. Nothing failed, but something is left for an agent to decide. Continue with section 3.
95
+ - **Anything else** is not a verdict about the app. `invalid_input` is a refusal before any device work (a bad flag, an unreadable file, or a scenario that cannot replay unattended): read `error.message` and the `hint`. `device` means no device could be driven: fix what the `hint` names and run again. An `environment` failure that still carries the run's `data` means only the JUnit report could not be written; the run itself is finalized and `data.status` is its outcome.
96
+
97
+ ### 3. Handle needs_agent
98
+
99
+ An `incomplete` run is waiting on an agent. Its `hint` says for what, and prints the exact commands that complete it.
100
+
101
+ **Pending assertions.** `data.pending` lists every assertion the CLI could not decide, each as `{assertion_id, step_id, type}`; the result file records them with status `needs_agent`. For each entry:
102
+
103
+ 1. **Find its screenshot.** Open the result file (`data.result_path`) and read the `screenshot` of the `steps_executed` entry with that `step_id`; the runner captures one for every step the scenario marks `checkpoint`. If none was recorded and `step_id` is the last step the run executed, the device is still showing that screen: capture it with `mauto screenshot <path>`. If neither holds, the screen is gone. Do not guess: leave that assertion pending.
104
+ 2. **Judge it** from that screenshot, per the Tier 2 list in section 5.
105
+ 3. **Record the verdict** with the `mauto result add-assertion` line the hint prints for that id, filling in only the verdict and the reason:
106
+ `mauto result add-assertion --run-id <run id> --step-id <step_id> --assertion-id <assertion_id> --type <type> --pass <true|false> --judge agent --message "<why>"`.
107
+ `--assertion-id` is what makes the verdict replace the pending entry in place, and `--message` IS the evidence, so omit neither.
108
+
109
+ A pending `screenshot_match` is the exception: it is pending because its reference screenshot does not exist yet, and it is never judged by eye. Ask the user whether the screen is correct; if they confirm, record the baselines by running the scenario again under a new run id with `--update-references` added. Otherwise leave it pending.
110
+
111
+ Then finalize again with the last line of the hint, `mauto result finalize --run-id <run id> --scenario <path>` (the hint prints the scenario's absolute path, so the command works from any directory). The status is derived again from what is now recorded: `passed` (exit code `0`), `failed` if any verdict was `false` (`test_failed`, exit code `6`), or still `incomplete` (exit code `7`) if you left anything pending. Report a run that is still `incomplete` as incomplete, never as passed.
112
+
113
+ **A pending step.** A hint line `Pending step <step_id>: <what to do>`, with `data.steps.needs_agent` above zero, means the runner reached a step it cannot perform itself (for example an install with no build path, or a condition it cannot evaluate). It stopped there, so no later step ran, and that recorded step keeps this run `incomplete` whatever is added to it afterwards. Do what the line asks, then replay the whole scenario by hand (section 4) under a new run id: export a new `MAUTO_RUN_ID`, perform that step with the ordinary one-shot verbs, and say in your report that the unattended run ended `incomplete` at that step.
114
+
115
+ ### 4. Manual per-step replay
116
+
117
+ This is the fallback: the per-step recipe an agent follows with the one-shot verbs. Use it only when the runner cannot be used for this scenario (section 2 refused it, or pre-flight reported an `unsupported_action`), or to perform a step the runner left `needs_agent` (section 3). Otherwise the runner has already done all of this.
118
+
119
+ #### Step-by-Step Replay
79
120
 
80
121
  Before executing each step, initialize a **session variable map** (an empty object to store captured values).
81
122
 
@@ -169,7 +210,7 @@ When executing a `capture_value` action:
169
210
  4. Also record it in the result file with `--capture <capture_to>=<extracted_text>` on that step's `mauto result add-step` call — the in-memory session variable map alone never reaches the result file; only the flag does.
170
211
  5. Report: "Captured `capture_to` = '<value>'".
171
212
 
172
- ### When a step fails, find out whether the app is still alive
213
+ #### When a step fails, find out whether the app is still alive
173
214
 
174
215
  An action that fails, or an `elements` call that comes back empty, has two very
175
216
  different explanations: the UI changed, or the app died. They lead to opposite
@@ -209,7 +250,7 @@ default — a report from last week is not evidence about this step. And a repor
209
250
  whose own timestamp is unreadable is counted in `unattributed` rather than
210
251
  included; if that count is non-zero, widen `--since` and look yourself.
211
252
 
212
- ### When the element list looks wrong, check the engine
253
+ #### When the element list looks wrong, check the engine
213
254
 
214
255
  If the app is alive but `mauto elements` misses the label of a text field you
215
256
  can see on screen, or lists content from a tab that is not showing (common in
@@ -224,11 +265,14 @@ setting. Do not switch the engine mid-run yourself.
224
265
 
225
266
  ### 5. Validate Assertions (27 types)
226
267
 
227
- For each assertion in the scenario (evaluated after the referenced step), pick the type recorded in the scenario and evaluate it. The 27 types split into two tiers by **who decides pass/fail**:
268
+ For each assertion in the scenario (evaluated after the referenced step), pick the type recorded in the scenario and evaluate it. The 27 types split into two tiers by **who decides pass/fail**, plus one reference-screenshot type the CLI also decides:
228
269
 
229
- - **Tier 1 — mechanical (the CLI decides).** These are deterministic checks over the on-screen element list (plus captured variables / a previous snapshot). Run `mauto elements` to get the current screen, then evaluate the assertion with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). A passing verdict arrives as a normal `ok: true` envelope. A failing one arrives as `ok: false` with `error.kind: "test_failed"` and exit code `6`, and the verdict (`pass: false`) is still in `data`. That is the test result, not a tool error: do not retry the verb or treat the run as broken. You record that verdict — you do not re-judge it.
270
+ - **Tier 1 — mechanical (the CLI decides).** These are deterministic checks over the on-screen element list (plus captured variables / a previous snapshot / the screen size). Run `mauto elements` to get the current screen, then evaluate the assertion with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). A passing verdict arrives as a normal `ok: true` envelope. A failing one arrives as `ok: false` with `error.kind: "test_failed"` and exit code `6`, and the verdict (`pass: false`) is still in `data`. That is the test result, not a tool error: do not retry the verb or treat the run as broken. You record that verdict — you do not re-judge it.
271
+ - **Reference screenshot — `screenshot_match` (the CLI decides).** A deterministic pixel comparison against a stored reference screenshot (a baseline); see the reference-screenshot block below. You record its verdict — you do not judge it by eye.
230
272
  - **Tier 2 — visual / semantic (the agent decides).** These require judging pixels. `mauto assert <type> ...` reports them as `needs_agent` (with `pass: null`) because the CLI cannot decide them. For these you take a screenshot with `mauto screenshot <path>`, visually analyze it, and report pass/fail yourself with a clear justification of what you observed.
231
273
 
274
+ `mauto run` decides the Tier 1 types and the reference-screenshot type itself and records each verdict; after a run you handle only what it left `needs_agent` (section 3). The `mauto assert` instructions below are for a manual replay (section 4).
275
+
232
276
  > One assertion type, `element_state`, names a Tier-1 intent but its per-attribute state inspection is not represented in the element model the CLI reads, so `mauto assert` returns it as `needs_agent` — judge it from the element list / screenshot like a Tier-2 check.
233
277
 
234
278
  ---
@@ -237,7 +281,7 @@ For each assertion in the scenario (evaluated after the referenced step), pick t
237
281
 
238
282
  - **`element_exists`:** the element described in `element_description` is present in the on-screen element list.
239
283
  - **`element_not_exists`:** the element is NOT present in the on-screen element list.
240
- - **`element_visible`:** pass if the element appears in the result. `expected_visible: true` = must be present, `false` = must be absent. Intended for elements that may be scrolled off or conditionally hidden, as distinct from fully absent elements (`element_exists` / `element_not_exists`). If you need to confirm the element is actually within the viewport (not clipped or scrolled off), use `element_fully_visible` (Tier 2) instead.
284
+ - **`element_visible`:** pass if the element appears in the result. `expected_visible: true` = must be present, `false` = must be absent. Intended for elements that may be scrolled off or conditionally hidden, as distinct from fully absent elements (`element_exists` / `element_not_exists`). If you need to confirm the element is actually within the viewport (not clipped or scrolled off), use `element_fully_visible` instead.
241
285
  - **`element_text`:** the element's text exactly matches `expected_value`.
242
286
  - **`text_contains`:** the element's text includes `expected_substring` as a substring.
243
287
  - **`text_not_empty`:** the target element's text has length > 0.
@@ -251,6 +295,18 @@ For each assertion in the scenario (evaluated after the referenced step), pick t
251
295
  - **`has_accessibility_label`:** the element's accessibility label is non-empty. If `label_value` is set, verify it matches exactly.
252
296
  - **`value_matches_variable`:** look up `variable_name` in the session variable map; verify the element described by `element_description` contains text that matches the captured value. If `pattern` is also set, use it as a format hint to locate the relevant portion.
253
297
  - **`text_changed`:** compare the element's current text with its text at the previous assertion checkpoint. Pass if the text differs.
298
+ - **`element_fully_visible`:** the element described in `element_description` lies fully inside the screen — not clipped by the screen edges or scrolled partly off. The CLI checks the element's bounds against the screen size; it cannot see overlap, so an element covered by another one still passes.
299
+ - **`screen_title`:** some element whose text or accessibility label equals `expected_text` (ignoring case and surrounding whitespace) starts in the top quarter of the screen. The same text lower on the screen fails.
300
+ - **`permission_dialog_shown`:** an OS-level permission dialog is up: an allow button and a deny button are both on screen. If `permission_name` is set (e.g., `"camera"`, `"location"`), pass it as `--permission <name>`; some element on screen must also mention it.
301
+
302
+ ---
303
+
304
+ **Reference screenshot — `screenshot_match`, decided by `mauto assert` against a baseline**
305
+
306
+ - Run `mauto assert screenshot_match --reference <reference_screenshot> --tolerance <tolerance>`, adding one `--mask <x,y,w,h>` or `--mask "<element description>"` per entry in the assertion's `masks`. Omit `--tolerance` when the scenario sets none (default `0.9`). The CLI captures a stable screenshot, compares it pixel by pixel, and decides: a pass is `ok: true`; a mismatch is `ok: false` with `error.kind: "test_failed"`, exit `6`, the similarity score and a diff image path in `data`. Record that verdict as-is — do not override it by looking at the screenshot.
307
+ - A dimension mismatch (a different device or orientation than the baseline was recorded on) is a failure too; report it as such and tell the user the baseline needs re-recording for this device.
308
+ - If the reference screenshot does not exist yet, the verdict is `needs_agent` (exit `0`), not a failure. Do not judge it by eye instead. Ask the user whether the current screen is correct; if they confirm, record the baseline with the same command plus `--update-references`, otherwise record the assertion as pending.
309
+ - Record the verdict with the `mauto result add-assertion` command in the envelope's `hint`; it carries the reference screenshot, the actual screenshot and the similarity score.
254
310
 
255
311
  ---
256
312
 
@@ -258,19 +314,15 @@ For each assertion in the scenario (evaluated after the referenced step), pick t
258
314
 
259
315
  For all Tier 2 assertions: take a screenshot, visually analyze it, and report pass/fail with a clear justification of what you observed.
260
316
 
261
- - **`screenshot_match`:** compare the captured screenshot against the reference baseline. Use **semantic visual comparison** — screen identity, presence/absence of key elements, text content, layout structure. Minor rendering differences (anti-aliasing, font smoothing) are acceptable within the `tolerance` threshold.
262
317
  - **`visual_state`:** assess whether the screen is in the `expected_visual_state`: `loaded` (content visible, no spinners), `loading` (loading indicator present), `empty` (empty-state UI), or `error` (error message/state visible).
263
- - **`element_fully_visible`:** the element described in `element_description` is fully visible in the viewport — not clipped, not scrolled off, nothing overlapping it.
264
318
  - **`color_style`:** locate the element and assess its dominant color. If `color_hex` is set, verify it matches approximately. If `expected_theme` is set (`dark`/`light`), verify the colors are consistent with that theme.
265
- - **`screen_title`:** read the navigation bar / screen title from the screenshot. Verify it matches `expected_text`.
266
319
  - **`alert_present`:** a system alert or modal dialog is visually present.
267
320
  - **`alert_text`:** an alert/dialog is on screen and its primary text matches `expected_text`.
268
321
  - **`toast_visible`:** the toast may be transient. Take a screenshot at the moment of the step and look for a floating notification overlay. If visible, verify its text matches `expected_text`. If it has already disappeared, mark `failed` and note it was transient.
269
322
  - **`keyboard_visible`:** whether the soft keyboard is visible at the bottom of the screen. `expected_visible: true` = must be shown, `false` = must be hidden.
270
323
  - **`dark_mode_active`:** assess the overall screen appearance. Pass if the background is dark (dark mode) or light (light mode) as specified by `expected_theme`.
271
- - **`permission_dialog_shown`:** an OS-level permission request dialog is visible. If `permission_name` is set (e.g., `"camera"`, `"location"`), verify the dialog requests that specific permission.
272
324
 
273
- **Dynamic content:** if the reference `expected_state` marks content as `[dynamic:...]`, ignore those regions during comparison.
325
+ **Dynamic content:** if the reference `expected_state` marks content as `[dynamic:...]`, ignore those regions when you judge a Tier-2 check. For `screenshot_match`, dynamic regions are excluded only through the assertion's `masks`.
274
326
 
275
327
  **Variable substitution in assertions:** before evaluating any assertion, replace `variable_name` references in string fields with values from the session variable map.
276
328
 
@@ -302,7 +354,7 @@ State context:
302
354
 
303
355
  When asked to run multiple scenarios:
304
356
 
305
- - Execute them sequentially.
357
+ - Execute them sequentially: one `mauto run` per scenario file, each under its own run id.
306
358
  - Each scenario's leading `launch_app` restarts the app (`mauto launch <appId>`), so a scenario never inherits the previous one's screen and needs no separate reset. Use `app_state: "fresh_install"` only when app data must be wiped too.
307
359
  - If the user says "run all smoke tests", filter scenarios by the `smoke` tag.
308
360
  - If the user says "run all", execute every scenario in `.mobile-automator/scenarios/`. List `.mobile-automator/scenarios/` directly: the workspace folder is hidden, and recursive search tools (glob or grep style) skip hidden directories, so they will report it empty.
@@ -312,6 +364,7 @@ When asked to run multiple scenarios:
312
364
 
313
365
  ### DO
314
366
 
367
+ - Prefer `mauto run` (section 2); replay step by step (section 4) only when the runner cannot be used.
315
368
  - Follow scenario steps exactly as written.
316
369
  - Run `mauto elements` before every interaction.
317
370
  - Retry once on suspected timing failures.
@@ -321,6 +374,7 @@ When asked to run multiple scenarios:
321
374
 
322
375
  ### DON'T
323
376
 
377
+ - Re-judge a verdict the CLI decided (a mechanical or reference-screenshot assertion), or replay by hand to get around a `test_failed` run.
324
378
  - Deviate from the scenario steps — if something unexpected happens, report it, don't work around it.
325
379
  - Modify app source code in {{protected_directories}}.
326
380
  - Hardcode coordinates — derive them from `mauto elements`.
@@ -1,4 +1,5 @@
1
1
  - Follow the scenario **exactly as written** — no improvisation, no skipped steps. If something unexpected happens, report it; do not work around it.
2
+ - Start with `mauto run <scenario>`: it replays the scenario unattended and exits `incomplete` when something needs an agent's judgment, which you then finish with the one-shot verbs.
2
3
  - Drive the device **only** through `mauto` verbs; never assume resource-ids or OS-specific element IDs (use the semantic actions in platform-agnostic projects).
3
4
  - Every assertion is backed by a screenshot recorded with `mauto result add-step --screenshot`; record its verdict with `mauto result add-assertion` and any observations with `--observation <type>:<message>`.
4
5
  - If this workspace has memory, read it first with `mauto memory show`; save durable app facts and user corrections with `mauto memory add`.
@@ -122,7 +122,7 @@ Translate user language to action types and `mauto` verbs:
122
122
 
123
123
  ### 5. Assertion Type Decision Table
124
124
 
125
- Use the most specific type that matches the user's intent. Tier 1 checks read structure from `mauto elements`; Tier 2 checks read pixels from `mauto screenshot <path>`. Evaluate each with `mauto assert <type> ...`.
125
+ Use the most specific type that matches the user's intent. Tier 1 checks read structure from `mauto elements`; Tier 2 checks read pixels from `mauto screenshot <path>`. Evaluate each with `mauto assert <type> ...`. Three types in the visual, navigation and system categories are Tier 1 too, decided by `mauto assert` over the element list: `element_fully_visible` (the element is not clipped by the screen edges; overlap is not checked), `screen_title` (an element equal to `expected_text` near the top of the screen) and `permission_dialog_shown` (an allow and a deny button on screen). `screenshot_match` is neither: `mauto assert` decides it with a pixel comparison against a reference screenshot. When you add one, the screen in front of you is the baseline: once the user confirms it looks right, record it with `mauto assert screenshot_match --reference <reference_screenshot> --update-references`, and add `masks` for regions that change between runs (clocks, timestamps, animations).
126
126
 
127
127
  **Element State**
128
128
 
@@ -157,8 +157,8 @@ Use the most specific type that matches the user's intent. Tier 1 checks read st
157
157
 
158
158
  | User intent | Type | Key fields |
159
159
  |---|---|---|
160
- | "the full element is visible, not clipped" | `element_fully_visible` | — |
161
- | "the screen looks the same as before" | `screenshot_match` | `reference_screenshot` |
160
+ | "the full element is visible, not clipped by the screen edges" | `element_fully_visible` | — |
161
+ | "the screen looks the same as before" | `screenshot_match` | `reference_screenshot`, `tolerance`, `masks` (regions that change between runs) |
162
162
  | "the screen is loaded / in error / empty state" | `visual_state` | `expected_visual_state` |
163
163
  | "the button color is X" | `color_style` | `color_hex` |
164
164
 
@@ -121,7 +121,7 @@ When the user provides a validation step, or when a declarative statement about
121
121
 
122
122
  #### Assertion Type Decision Table (27 types)
123
123
 
124
- Use the most specific type that matches the user's intent. Tier 1 checks read structure from `mauto elements`; Tier 2 checks read pixels from `mauto screenshot <path>`. Evaluate each with `mauto assert <type> ...`.
124
+ Use the most specific type that matches the user's intent. Tier 1 checks read structure from `mauto elements`; Tier 2 checks read pixels from `mauto screenshot <path>`. Evaluate each with `mauto assert <type> ...`. Three types in the visual, navigation and system categories are Tier 1 too, decided by `mauto assert` over the element list: `element_fully_visible` (the element is not clipped by the screen edges; overlap is not checked), `screen_title` (an element equal to `expected_text` near the top of the screen) and `permission_dialog_shown` (an allow and a deny button on screen). `screenshot_match` is neither: `mauto assert` decides it with a pixel comparison against a reference screenshot. When you add one, the screen in front of you is the baseline: once the user confirms it looks right, record it with `mauto assert screenshot_match --reference <reference_screenshot> --update-references`, and add `masks` for regions that change between runs (clocks, timestamps, animations).
125
125
 
126
126
  **Category 1 — Element State (Tier 1: `mauto elements`)**
127
127
 
@@ -162,8 +162,8 @@ Use the most specific type that matches the user's intent. Tier 1 checks read st
162
162
 
163
163
  | User intent / natural language | Type | Key fields |
164
164
  |---|---|---|
165
- | "the full element is visible, not clipped" | `element_fully_visible` | — |
166
- | "the element/screen looks the same as before" | `screenshot_match` | `reference_screenshot: "..."` |
165
+ | "the full element is visible, not clipped by the screen edges" | `element_fully_visible` (Tier 1: bounds vs screen size, by `mauto assert`) | — |
166
+ | "the element/screen looks the same as before" | `screenshot_match` (pixel comparison by `mauto assert`, not Tier 2) | `reference_screenshot: "..."`, `tolerance`, `masks: [{ "element_description": "clock" }]` |
167
167
  | "the screen is loaded / in error / empty state" | `visual_state` | `expected_visual_state: "loaded"` |
168
168
  | "the button color is blue / #0057FF" | `color_style` | `color_hex: "#0057FF"` |
169
169
 
@@ -171,7 +171,7 @@ Use the most specific type that matches the user's intent. Tier 1 checks read st
171
171
 
172
172
  | User intent / natural language | Type | Key fields |
173
173
  |---|---|---|
174
- | "the screen/page title is 'Settings'" | `screen_title` | `expected_text: "Settings"` |
174
+ | "the screen/page title is 'Settings'" | `screen_title` (Tier 1: by `mauto assert`) | `expected_text: "Settings"` |
175
175
  | "a dialog / alert appeared" | `alert_present` | — |
176
176
  | "the alert says 'Are you sure?'" | `alert_text` | `expected_text: "Are you sure?"` |
177
177
  | "a toast / snackbar appeared saying 'Saved'" | `toast_visible` | `expected_text: "Saved"` |
@@ -194,7 +194,7 @@ Use the most specific type that matches the user's intent. Tier 1 checks read st
194
194
 
195
195
  | User intent / natural language | Type | Key fields |
196
196
  |---|---|---|
197
- | "the app asked for camera / location / microphone permission" | `permission_dialog_shown` | `permission_name: "camera"` |
197
+ | "the app asked for camera / location / microphone permission" | `permission_dialog_shown` (Tier 1: by `mauto assert`) | `permission_name: "camera"` |
198
198
  | "dark mode is active / the screen is dark" | `dark_mode_active` | `expected_theme: "dark"` |
199
199
 
200
200
  #### Auto-Assertion Rule
@@ -18,6 +18,7 @@ const fs = require('fs');
18
18
  const path = require('path');
19
19
 
20
20
  const { interpolate } = require('./placeholders');
21
+ const { EXIT } = require('../output/envelope');
21
22
 
22
23
  // Topics with REAL ported prose live as markdown content files (carrying
23
24
  // `{{placeholder}}` tokens lifted from the Gemini SKILLs). They are emitted by
@@ -111,6 +112,7 @@ function emitBootstrap() {
111
112
  ['screenshot', 'save a screenshot to a path'],
112
113
  ['assert', 'evaluate an assertion against the current screen'],
113
114
  ['validate', 'validate a scenario file against the scenario schema'],
115
+ ['run', `replay a scenario unattended and finalize its result (exit ${EXIT.OK} passed, ${EXIT.TEST_FAILED} test_failed, ${EXIT.INCOMPLETE} incomplete)`],
114
116
  ['result', 'record step results and finalize a run'],
115
117
  ['setup', 'scaffold the workspace and write a config'],
116
118
  ['config', 'get/set values in the workspace config'],
@@ -83,6 +83,7 @@ function cursorRuleBody() {
83
83
  '---\n\n' +
84
84
  guideEmitter.emitBootstrap() +
85
85
  '\nRun `mauto guide <topic>` (generate|execute|setup) before a workflow. ' +
86
+ 'The execute guide starts with `mauto run`. ' +
86
87
  'Drive the device only through `mauto` verbs.\n'
87
88
  );
88
89
  }
@@ -15,7 +15,7 @@ const path = require('path');
15
15
  const { atCap } = require('./sinks/file');
16
16
  const { logsDir, runTracePath, RUN_TRACE_PREFIX } = require('./paths');
17
17
 
18
- // A device failure in the envelope's own taxonomy. `deviceFail` (src/cli.js) is
18
+ // A device failure in the envelope's own taxonomy. `deviceFail` (src/output/device-fail.js) is
19
19
  // the ONLY producer of these two kinds, so counting them needs no list of which
20
20
  // verbs are device verbs — a list that would have to be maintained by hand
21
21
  // alongside connectBridge and would be wrong the first time someone forgot.
@@ -48,6 +48,10 @@ const EVENT_NAMES = Object.freeze([
48
48
  'daemon.lock_conflict',
49
49
  'daemon.undeliverable',
50
50
  'daemon.handler_error',
51
+ // `mauto run`'s own stamps (src/cli.js RUN_TRACE_EVENTS), so a single
52
+ // in-process run's trace spans its steps before finalize measures it.
53
+ 'run.start',
54
+ 'run.step',
51
55
  // Emitted by the flush path, which lands in a later slice-5 task; declared
52
56
  // now because its event shape is already exercised by unit tests.
53
57
  'telemetry.flush',
@@ -88,6 +92,8 @@ const VERB_NAMES = Object.freeze([
88
92
  'crash',
89
93
  'mcp',
90
94
  'telemetry',
95
+ // The unattended replay runner (#278).
96
+ 'run',
91
97
  ]);
92
98
 
93
99
  // Why the daemon stopped. 'crash' is documented but not yet produced.