mobile-automator 0.38.4 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/package.json +4 -2
- package/src/assertion/evaluator.js +116 -14
- package/src/assertion/fields.js +106 -0
- package/src/assertion/observe.js +17 -0
- package/src/cli.js +180 -174
- package/src/device/action-catalog.js +4 -2
- package/src/device/bridge.js +17 -1
- package/src/device/mobile-mcp-tools.js +1 -0
- package/src/device/resolve-connection.js +1 -1
- package/src/device/semantic-press/index.js +19 -0
- package/src/device/semantic-press/resolver.js +1 -1
- package/src/{device/semantic-press/labels.js → dialog/permission-labels.js} +6 -3
- package/src/guide/content/execute.agnostic.md +74 -21
- package/src/guide/content/execute.aware.md +69 -15
- package/src/guide/content/execute.invariants.md +1 -0
- package/src/guide/content/generate.agnostic.md +3 -3
- package/src/guide/content/generate.aware.md +5 -5
- package/src/guide/emitter.js +2 -0
- package/src/init/host-plan.js +1 -0
- package/src/observe/trace.js +1 -1
- package/src/observe/vocab.js +6 -0
- package/src/output/device-fail.js +41 -0
- package/src/output/envelope.js +3 -3
- package/src/output/run-human.js +37 -0
- package/src/result/capability-catalog.js +48 -0
- package/src/result/executor.js +58 -0
- package/src/result/finalize-against-scenario.js +103 -0
- package/src/result/flags.js +49 -6
- package/src/result/junit.js +39 -16
- package/src/result/outcome.js +24 -2
- package/src/result/pending-hint.js +67 -0
- package/src/result/scenario-check.js +58 -0
- package/src/result/status-guard.js +46 -0
- package/src/result/store.js +159 -16
- package/src/run/actions/capture.js +21 -0
- package/src/run/actions/gestures.js +89 -0
- package/src/run/actions/index.js +48 -0
- package/src/run/actions/launch.js +30 -0
- package/src/run/actions/preconditions.js +44 -0
- package/src/run/actions/scroll.js +22 -0
- package/src/run/actions/semantic.js +16 -0
- package/src/run/actions/wait.js +66 -0
- package/src/run/assert-reference.js +118 -0
- package/src/run/assertions.js +102 -0
- package/src/run/clock.js +10 -0
- package/src/run/condition.js +80 -0
- package/src/run/fatal.js +15 -0
- package/src/run/handle.js +151 -0
- package/src/run/needs-agent.js +39 -0
- package/src/run/poll.js +38 -0
- package/src/run/prepare.js +107 -0
- package/src/run/reference-assertion.js +172 -0
- package/src/run/reference-screenshot.js +92 -0
- package/src/run/resolve.js +95 -0
- package/src/run/run-id.js +14 -0
- package/src/run/runner.js +104 -0
- package/src/run/scale.js +52 -0
- package/src/run/screenshots.js +163 -0
- package/src/run/settings.js +35 -0
- package/src/run/state.js +22 -0
- package/src/run/step.js +138 -0
- package/src/run/summary.js +73 -0
- package/src/run/trace.js +30 -0
- package/src/run/tree.js +96 -0
- package/src/run/variables.js +56 -0
- package/src/scenario/load.js +78 -0
- package/src/schemas/config_schema.json +29 -0
- package/src/schemas/result_schema.json +45 -3
- package/src/schemas/scenario_schema.json +27 -0
- package/src/util/as-array.js +8 -0
- package/src/util/parse-positive-int.js +15 -0
- package/src/util/parse-unit-interval.js +17 -0
|
@@ -45,6 +45,7 @@ const MOBILE_MCP_TOOL_NAMES = Object.freeze([
|
|
|
45
45
|
'mobile_uninstall_app',
|
|
46
46
|
'mobile_open_url',
|
|
47
47
|
'mobile_set_orientation',
|
|
48
|
+
'mobile_get_orientation',
|
|
48
49
|
// Diagnostic, not a scenario action — see DeviceBridge#listCrashes.
|
|
49
50
|
'mobile_list_crashes',
|
|
50
51
|
'mobile_get_crash',
|
|
@@ -135,7 +135,7 @@ async function resolveDeviceConnection({
|
|
|
135
135
|
// what failed, the daemon's log is the only record of why, and transparent
|
|
136
136
|
// autostart is the common path — most users never type `mauto session start`,
|
|
137
137
|
// so without this the log stays invisible to almost everyone. The signal
|
|
138
|
-
// rides err.hint because deviceFail()
|
|
138
|
+
// rides err.hint because deviceFail() (src/output/device-fail.js) already surfaces it on every
|
|
139
139
|
// device verb, so acquireConnection's deliberately narrow contract is
|
|
140
140
|
// untouched (#163).
|
|
141
141
|
const oneShotWithLogHint = async (afterFailedSpawn) => {
|
|
@@ -29,10 +29,29 @@ function selectResolver(platform, bridge) {
|
|
|
29
29
|
return new Strategy(bridge);
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
// Perform one semantic action on the bridge's device: ask the device its
|
|
33
|
+
// platform, pick that platform's resolver, run the action's method. Returns
|
|
34
|
+
// what was done, `{ platform, mechanism }`. The one implementation behind
|
|
35
|
+
// `mauto press <action>` and `mauto run`; a non-semantic token or an
|
|
36
|
+
// unsupported platform throws SemanticResolutionError before any device input.
|
|
37
|
+
async function pressSemantic(bridge, action) {
|
|
38
|
+
if (!isSemanticAction(action)) {
|
|
39
|
+
throw new SemanticResolutionError(
|
|
40
|
+
`"${action}" is not a semantic action.`,
|
|
41
|
+
`Semantic actions: ${SEMANTIC_ACTIONS.join(', ')}.`
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
const platform = await bridge.getPlatform();
|
|
45
|
+
const resolver = selectResolver(platform, bridge);
|
|
46
|
+
const { mechanism } = await resolver[ACTION_METHOD[action]]();
|
|
47
|
+
return { platform, mechanism };
|
|
48
|
+
}
|
|
49
|
+
|
|
32
50
|
module.exports = {
|
|
33
51
|
SEMANTIC_ACTIONS,
|
|
34
52
|
ACTION_METHOD,
|
|
35
53
|
isSemanticAction,
|
|
36
54
|
selectResolver,
|
|
55
|
+
pressSemantic,
|
|
37
56
|
SemanticResolutionError,
|
|
38
57
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
const { ALLOW_LABELS, DENY_LABELS, findByLabels } = require('
|
|
3
|
+
const { ALLOW_LABELS, DENY_LABELS, findByLabels } = require('../../dialog/permission-labels');
|
|
4
4
|
|
|
5
5
|
// Raised when a semantic action cannot be performed on the current screen/
|
|
6
6
|
// platform. `.kind`/`.hint` let the CLI map it onto the `device` envelope.
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
// System permission-dialog button labels, per platform.
|
|
3
|
+
// System permission-dialog button labels, per platform. Neutral ground for
|
|
4
|
+
// both readers: semantic-press taps them (grant/deny_permission) and the
|
|
5
|
+
// assertion evaluator looks for them (permission_dialog_shown), so neither
|
|
6
|
+
// layer imports the other's internals. Lowercase and
|
|
4
7
|
// apostrophe-normalized so matching is locale-tolerant within English.
|
|
5
8
|
const ALLOW_LABELS = {
|
|
6
9
|
android: ['allow', 'while using the app', 'allow only this time', 'ok', 'continue'],
|
|
@@ -16,8 +19,8 @@ function norm(s) {
|
|
|
16
19
|
// Map the curly apostrophe (U+2019, the ’ glyph in the regex below) onto a
|
|
17
20
|
// straight quote, so "Don’t Allow" matches the straight-quoted entries in the
|
|
18
21
|
// label sets above. The U+2019 byte must be preserved literally here; the
|
|
19
|
-
// guard assertion in labels.test.js fails loudly if it is ever
|
|
20
|
-
// straight quote by editor/transport tooling.
|
|
22
|
+
// guard assertion in permission-labels.test.js fails loudly if it is ever
|
|
23
|
+
// flattened to a straight quote by editor/transport tooling.
|
|
21
24
|
return String(s == null ? '' : s).replace(/’/g, "'").trim().toLowerCase();
|
|
22
25
|
}
|
|
23
26
|
|
|
@@ -74,7 +74,48 @@ Record each observation with `mauto result add-step --observation <type>:<messag
|
|
|
74
74
|
|
|
75
75
|
The remaining precondition actions are not mechanically executable (see step execution): report them honestly or perform them out of band.
|
|
76
76
|
|
|
77
|
-
### 2.
|
|
77
|
+
### 2. Replay with mauto run
|
|
78
|
+
|
|
79
|
+
Replay the scenario with the unattended runner before anything else:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
mauto run <scenario path> --run-id $MAUTO_RUN_ID
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Add `--junit <path>` when a CI job needs a JUnit XML report of the run (the file's directory must already exist). It is one invocation and one envelope: the runner performs the precondition `device_actions`, every step, and every assertion the CLI can decide, captures the evidence, and finalizes the result file itself. Nothing waits for you mid-run, and you never drive a step through it.
|
|
86
|
+
|
|
87
|
+
If pre-flight reported an `unsupported_action` warning, skip this section: `mauto run` refuses such a scenario with `invalid_input` before any device work. Replay it by hand instead (section 4).
|
|
88
|
+
|
|
89
|
+
`data` carries the same keys whatever the outcome: `run_id`, `scenario_id`, `status`, `result_path`, `duration_seconds`, `steps`, `assertions`, `failed_step`, `failures`, `pending`, `junit_path`, `executor`. Branch on the outcome:
|
|
90
|
+
|
|
91
|
+
- **Passed** — `ok: true`, exit code `0`, `data.status` is `passed`. Every step ran and the CLI decided every assertion. The result file is already finalized at `data.result_path`, so there is nothing left to record or finalize: present it (section 6 describes what it holds).
|
|
92
|
+
- **`test_failed`** — `ok: false` with `error.kind: "test_failed"` and exit code `6`; `data.status` is `failed` or `error`. That is the test result, not a tool error. Report the failure from `data`: `failed_step` is the first step that failed (`null` when only an assertion failed), and `failures` lists each failed step and assertion with its message. The runner stops at a failed step, so later steps did not run; the failed step's screenshot is recorded on its `steps_executed` entry in the result file at `data.result_path`. Do not re-judge a verdict the CLI decided, and do not retry the run or replay it by hand to get around the failure. The result is already finalized: present it.
|
|
93
|
+
- **`incomplete`** — `ok: false` with `error.kind: "incomplete"` and exit code `7`; `data.status` is `incomplete`. Nothing failed, but something is left for an agent to decide. Continue with section 3.
|
|
94
|
+
- **Anything else** is not a verdict about the app. `invalid_input` is a refusal before any device work (a bad flag, an unreadable file, or a scenario that cannot replay unattended): read `error.message` and the `hint`. `device` means no device could be driven: fix what the `hint` names and run again. An `environment` failure that still carries the run's `data` means only the JUnit report could not be written; the run itself is finalized and `data.status` is its outcome.
|
|
95
|
+
|
|
96
|
+
### 3. Handle needs_agent
|
|
97
|
+
|
|
98
|
+
An `incomplete` run is waiting on an agent. Its `hint` says for what, and prints the exact commands that complete it.
|
|
99
|
+
|
|
100
|
+
**Pending assertions.** `data.pending` lists every assertion the CLI could not decide, each as `{assertion_id, step_id, type}`; the result file records them with status `needs_agent`. For each entry:
|
|
101
|
+
|
|
102
|
+
1. **Find its screenshot.** Open the result file (`data.result_path`) and read the `screenshot` of the `steps_executed` entry with that `step_id`; the runner captures one for every step the scenario marks `checkpoint`. If none was recorded and `step_id` is the last step the run executed, the device is still showing that screen: capture it with `mauto screenshot <path>`. If neither holds, the screen is gone. Do not guess: leave that assertion pending.
|
|
103
|
+
2. **Judge it** from that screenshot, per the Tier 2 list in section 5.
|
|
104
|
+
3. **Record the verdict** with the `mauto result add-assertion` line the hint prints for that id, filling in only the verdict and the reason:
|
|
105
|
+
`mauto result add-assertion --run-id <run id> --step-id <step_id> --assertion-id <assertion_id> --type <type> --pass <true|false> --judge agent --message "<why>"`.
|
|
106
|
+
`--assertion-id` is what makes the verdict replace the pending entry in place, and `--message` IS the evidence, so omit neither.
|
|
107
|
+
|
|
108
|
+
A pending `screenshot_match` is the exception: it is pending because its reference screenshot does not exist yet, and it is never judged by eye. Ask the user whether the screen is correct; if they confirm, record the baselines by running the scenario again under a new run id with `--update-references` added. Otherwise leave it pending.
|
|
109
|
+
|
|
110
|
+
Then finalize again with the last line of the hint, `mauto result finalize --run-id <run id> --scenario <path>` (the hint prints the scenario's absolute path, so the command works from any directory). The status is derived again from what is now recorded: `passed` (exit code `0`), `failed` if any verdict was `false` (`test_failed`, exit code `6`), or still `incomplete` (exit code `7`) if you left anything pending. Report a run that is still `incomplete` as incomplete, never as passed.
|
|
111
|
+
|
|
112
|
+
**A pending step.** A hint line `Pending step <step_id>: <what to do>`, with `data.steps.needs_agent` above zero, means the runner reached a step it cannot perform itself (for example an install with no build path, or a condition it cannot evaluate). It stopped there, so no later step ran, and that recorded step keeps this run `incomplete` whatever is added to it afterwards. Do what the line asks, then replay the whole scenario by hand (section 4) under a new run id: export a new `MAUTO_RUN_ID`, perform that step with the ordinary one-shot verbs, and say in your report that the unattended run ended `incomplete` at that step.
|
|
113
|
+
|
|
114
|
+
### 4. Manual per-step replay
|
|
115
|
+
|
|
116
|
+
This is the fallback: the per-step recipe an agent follows with the one-shot verbs. Use it only when the runner cannot be used for this scenario (section 2 refused it, or pre-flight reported an `unsupported_action`), or to perform a step the runner left `needs_agent` (section 3). Otherwise the runner has already done all of this.
|
|
117
|
+
|
|
118
|
+
#### Semantic Action Resolution
|
|
78
119
|
|
|
79
120
|
When you encounter a semantic action — `press_back`, `dismiss_keyboard`, `grant_permission`, `deny_permission` — run `mauto press <action>` and let the CLI resolve it for the connected device's `platform`. Never record or hardcode the per-platform mechanics; the scenario stays portable.
|
|
80
121
|
|
|
@@ -85,17 +126,17 @@ When you encounter a semantic action — `press_back`, `dismiss_keyboard`, `gran
|
|
|
85
126
|
|
|
86
127
|
If the device's platform has no resolution, halt the step and report it as unsupported on this device. If a resolution's primary attempt fails, the CLI falls back to the documented alternative before failing the step — record both the primary and fallback attempts in the step's `observations`.
|
|
87
128
|
|
|
88
|
-
|
|
129
|
+
#### Per-Step Execution
|
|
89
130
|
|
|
90
131
|
Before executing each step, initialize a **session variable map** (an empty object to store captured values).
|
|
91
132
|
|
|
92
133
|
For each step in the scenario:
|
|
93
134
|
|
|
94
|
-
**
|
|
135
|
+
**4.0 Pre-step: Evaluate condition**
|
|
95
136
|
|
|
96
137
|
- If the step has a `condition` field, evaluate it (`device_property`, `previous_step_skipped`, `variable_value`, `element_visible`). If the condition is **false** → skip this step and all its `sub_steps`. Mark as `skipped`.
|
|
97
138
|
|
|
98
|
-
**
|
|
139
|
+
**4.1 Execute the step**
|
|
99
140
|
|
|
100
141
|
1. **Wait for prior stability:** poll `mauto elements` until loading indicators ({{loading_indicators}}) are absent before continuing.
|
|
101
142
|
2. **Resolve the target:** for actions with a target, run `mauto elements` to locate by semantic description; never use stored coordinates. If `target_pattern` is set, use it as a regex to filter elements by text.
|
|
@@ -105,10 +146,10 @@ For each step in the scenario:
|
|
|
105
146
|
- Semantic actions → run `mauto press <action>` and let the CLI resolve them for the connected device.
|
|
106
147
|
- Not mechanically executable → `clear_app_data` (and the precondition device actions `enable_wifi` / `disable_wifi`) have no verb and no underlying primitive. Do NOT substitute a gesture: report them honestly as unsupported on this run, or perform them manually out of band if the run depends on them.
|
|
107
148
|
4. **Capture screenshot:** Run `mauto screenshot .mobile-automator/screenshots/<scenario_id>/<run_id>/step_<step_id>.png`.
|
|
108
|
-
5. **Evaluate assertions** attached to the step (see section
|
|
149
|
+
5. **Evaluate assertions** attached to the step (see section 5).
|
|
109
150
|
6. **Record** the step result with `mauto result add-step --step-id <id> --status <s> --screenshot <path> [--attempts <n>] [--error-message <text>] [--observation <type>:<message>] [--capture <name>=<value>]`, then record each assertion verdict with `mauto result add-assertion --step-id <id> --type <t> --pass <true|false> --message <text> [--expected <v> --actual <v>]`. `--message` carries the justification you already formed — for Tier-2 verdicts it IS the evidence, so never omit it; add `--expected`/`--actual` whenever the assertion compares a specific value.
|
|
110
151
|
|
|
111
|
-
**
|
|
152
|
+
**4.2 Handle step outcome**
|
|
112
153
|
|
|
113
154
|
- `"fail"` (default): capture a failure screenshot, apply the Flakiness Detector logic, record as failed via `mauto result add-step --status fail --error-message <text> --screenshot <path>`.
|
|
114
155
|
- `"skip"`: mark `skipped`, continue silently, do NOT fail the scenario.
|
|
@@ -116,11 +157,11 @@ For each step in the scenario:
|
|
|
116
157
|
- `optional: true` overrides a failure to `"skip"`.
|
|
117
158
|
- `sub_steps`: when the parent succeeds, execute each sub-step in order under the same rules, then resume at the next top-level step.
|
|
118
159
|
|
|
119
|
-
**
|
|
160
|
+
**4.3 Variable capture**
|
|
120
161
|
|
|
121
162
|
When executing a `capture_value` action: locate the element with `mauto elements`, extract its text, and store it in the session variable map under the step's `capture_to` name (`variables[<capture_to>] = <extracted_text>`). Report: "Captured `capture_to` = '<value>'".
|
|
122
163
|
|
|
123
|
-
|
|
164
|
+
#### When a step fails, find out whether the app is still alive
|
|
124
165
|
|
|
125
166
|
An action that fails, or an `elements` call that comes back empty, has two very
|
|
126
167
|
different explanations: the UI changed, or the app died. They lead to opposite
|
|
@@ -160,7 +201,7 @@ default — a report from last week is not evidence about this step. And a repor
|
|
|
160
201
|
whose own timestamp is unreadable is counted in `unattributed` rather than
|
|
161
202
|
included; if that count is non-zero, widen `--since` and look yourself.
|
|
162
203
|
|
|
163
|
-
|
|
204
|
+
#### When the element list looks wrong, check the engine
|
|
164
205
|
|
|
165
206
|
If the app is alive but `mauto elements` misses the label of a text field you
|
|
166
207
|
can see on screen, or lists content from a tab that is not showing, the device
|
|
@@ -172,20 +213,34 @@ recover text-field labels; if the engine is already `legacy`, a missing
|
|
|
172
213
|
text-field label is the device OS version, not the engine setting. Do not
|
|
173
214
|
switch the engine mid-run yourself.
|
|
174
215
|
|
|
175
|
-
###
|
|
216
|
+
### 5. Validate Assertions (27 types)
|
|
176
217
|
|
|
177
|
-
For each assertion, pick the type recorded in the scenario and evaluate it. The 27 types split into two tiers by **who decides pass/fail
|
|
218
|
+
For each assertion, pick the type recorded in the scenario and evaluate it. The 27 types split into two tiers by **who decides pass/fail**, plus one reference-screenshot type the CLI also decides:
|
|
178
219
|
|
|
179
|
-
- **Tier 1 — mechanical (the CLI decides).** Deterministic checks over the on-screen element list (plus captured variables / a previous snapshot). Run `mauto elements`, then evaluate with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). A passing verdict arrives as a normal `ok: true` envelope. A failing one arrives as `ok: false` with `error.kind: "test_failed"` and exit code `6`, and the verdict (`pass: false`) is still in `data`. That is the test result, not a tool error: do not retry the verb or treat the run as broken. You record that verdict — you do not re-judge it.
|
|
220
|
+
- **Tier 1 — mechanical (the CLI decides).** Deterministic checks over the on-screen element list (plus captured variables / a previous snapshot / the screen size). Run `mauto elements`, then evaluate with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). A passing verdict arrives as a normal `ok: true` envelope. A failing one arrives as `ok: false` with `error.kind: "test_failed"` and exit code `6`, and the verdict (`pass: false`) is still in `data`. That is the test result, not a tool error: do not retry the verb or treat the run as broken. You record that verdict — you do not re-judge it.
|
|
221
|
+
- **Reference screenshot — `screenshot_match` (the CLI decides).** A deterministic pixel comparison against a stored reference screenshot (a baseline); see the reference-screenshot block below. You record its verdict — you do not judge it by eye.
|
|
180
222
|
- **Tier 2 — visual / semantic (the agent decides).** These require judging pixels. `mauto assert <type> ...` reports them as `needs_agent` (with `pass: null`). For these you take a screenshot with `mauto screenshot <path>`, visually analyze it, and report pass/fail yourself with a clear justification.
|
|
181
223
|
|
|
224
|
+
`mauto run` decides the Tier 1 types and the reference-screenshot type itself and records each verdict; after a run you handle only what it left `needs_agent` (section 3). The `mauto assert` instructions below are for a manual replay (section 4).
|
|
225
|
+
|
|
182
226
|
> One assertion type, `element_state`, names a Tier-1 intent but its per-attribute state inspection is not represented in the element model the CLI reads, so `mauto assert` returns it as `needs_agent` — judge it from the element list / screenshot like a Tier-2 check.
|
|
227
|
+
>
|
|
228
|
+
> Three Tier-1 types read the element list more narrowly than a screenshot would, so an agent never needs to re-judge them: `element_fully_visible` checks only that the element is not clipped by the screen edges (it cannot see overlap); `screen_title` passes when an element whose text or label equals `expected_text` starts in the top quarter of the screen; `permission_dialog_shown` passes when an allow button and a deny button are both on screen.
|
|
183
229
|
|
|
184
230
|
---
|
|
185
231
|
|
|
186
232
|
**Tier 1 — mechanical, evaluated with `mauto assert <type>` over `mauto elements`**
|
|
187
233
|
|
|
188
|
-
`element_exists`, `element_not_exists`, `element_visible` (`expected_visible: true/false`), `element_text` (`expected_value`), `text_contains` (`expected_substring`), `text_not_empty`, `element_hint` (`expected_text`), `pattern_match` (`pattern`), `element_state` (`state_property` — see note above; resolved by the agent), `element_count` / `list_item_count` (`operator` + `expected_count`), `list_is_empty`, `content_description` (`label_value`), `has_accessibility_label` (`label_value` optional), `value_matches_variable` (`variable_name`), `text_changed
|
|
234
|
+
`element_exists`, `element_not_exists`, `element_visible` (`expected_visible: true/false`), `element_text` (`expected_value`), `text_contains` (`expected_substring`), `text_not_empty`, `element_hint` (`expected_text`), `pattern_match` (`pattern`), `element_state` (`state_property` — see note above; resolved by the agent), `element_count` / `list_item_count` (`operator` + `expected_count`), `list_is_empty`, `content_description` (`label_value`), `has_accessibility_label` (`label_value` optional), `value_matches_variable` (`variable_name`), `text_changed`, `element_fully_visible` (`element_description`), `screen_title` (`expected_text`), `permission_dialog_shown` (`permission_name` optional, passed as `--permission`).
|
|
235
|
+
|
|
236
|
+
---
|
|
237
|
+
|
|
238
|
+
**Reference screenshot — `screenshot_match`, decided by `mauto assert` against a baseline**
|
|
239
|
+
|
|
240
|
+
- Run `mauto assert screenshot_match --reference <reference_screenshot> --tolerance <tolerance>`, adding one `--mask <x,y,w,h>` or `--mask "<element description>"` per entry in the assertion's `masks`. Omit `--tolerance` when the scenario sets none (default `0.9`). The CLI captures a stable screenshot, compares it pixel by pixel, and decides: a pass is `ok: true`; a mismatch is `ok: false` with `error.kind: "test_failed"`, exit `6`, the similarity score and a diff image path in `data`. Record that verdict as-is — do not override it by looking at the screenshot.
|
|
241
|
+
- A dimension mismatch (a different device or orientation than the baseline was recorded on) is a failure too; report it as such and tell the user the baseline needs re-recording for this device.
|
|
242
|
+
- If the reference screenshot does not exist yet, the verdict is `needs_agent` (exit `0`), not a failure. Do not judge it by eye instead. Ask the user whether the current screen is correct; if they confirm, record the baseline with the same command plus `--update-references`, otherwise record the assertion as pending.
|
|
243
|
+
- Record the verdict with the `mauto result add-assertion` command in the envelope's `hint`; it carries the reference screenshot, the actual screenshot and the similarity score.
|
|
189
244
|
|
|
190
245
|
---
|
|
191
246
|
|
|
@@ -193,22 +248,18 @@ For each assertion, pick the type recorded in the scenario and evaluate it. The
|
|
|
193
248
|
|
|
194
249
|
For each: take a screenshot, visually analyze it, and report pass/fail with a clear justification.
|
|
195
250
|
|
|
196
|
-
- **`screenshot_match`:** compare against the reference baseline using **semantic visual comparison** — screen identity, key elements, text content, layout structure — within the `tolerance` threshold.
|
|
197
251
|
- **`visual_state`:** assess `loaded` / `loading` / `empty` / `error` against `expected_visual_state`.
|
|
198
|
-
- **`element_fully_visible`:** the element is fully in the viewport — not clipped, scrolled off, or overlapped.
|
|
199
252
|
- **`color_style`:** assess the element's dominant color against `color_hex` or `expected_theme`.
|
|
200
|
-
- **`screen_title`:** read the screen title and compare to `expected_text`.
|
|
201
253
|
- **`alert_present` / `alert_text`:** a dialog is present (and its text matches `expected_text`).
|
|
202
254
|
- **`toast_visible`:** look for a transient notification overlay; match `expected_text`, or mark `failed` if it already disappeared.
|
|
203
255
|
- **`keyboard_visible`:** whether the soft keyboard is shown (`expected_visible`).
|
|
204
256
|
- **`dark_mode_active`:** the screen appearance matches `expected_theme`.
|
|
205
|
-
- **`permission_dialog_shown`:** a permission request dialog is visible (matching `permission_name` if set).
|
|
206
257
|
|
|
207
|
-
**Dynamic content:** if the reference `expected_state` marks content as `[dynamic:...]`, ignore those regions
|
|
258
|
+
**Dynamic content:** if the reference `expected_state` marks content as `[dynamic:...]`, ignore those regions when you judge a Tier-2 check. For `screenshot_match`, dynamic regions are excluded only through the assertion's `masks`.
|
|
208
259
|
|
|
209
260
|
**Variable substitution in assertions:** before evaluating any assertion, replace `variable_name` references in string fields with values from the session variable map.
|
|
210
261
|
|
|
211
|
-
###
|
|
262
|
+
### 6. Generate the Result Report
|
|
212
263
|
|
|
213
264
|
Obtain the result schema by running `mauto schema result`, then finalize the run with `mauto result finalize` (writing to `.mobile-automator/results/<run_id>.json`), passing the run metadata you already know (`--device-model`, `--api-level`, `--app-version`, `--environment`) — the timestamp is filled in for you, and omitted fields are recorded as `unknown`. You may supply your own narrative with `--summary <text>`; omitting it keeps the generated default summary line. A run that finalizes as `failed` or `error` returns `ok: false` with `error.kind: "test_failed"` and exit code `6`. The result file is still written and the full result is in `data`, so present it as usual; the non-zero exit is what lets CI see a red run. The finalized result assembles what the earlier `add-step`/`add-assertion` calls already recorded:
|
|
214
265
|
|
|
@@ -220,7 +271,7 @@ Obtain the result schema by running `mauto schema result`, then finalize the run
|
|
|
220
271
|
|
|
221
272
|
`--duration` is now a cross-check rather than the recorded value: when a run trace exists the duration is measured from it, your figure is kept as `measurements.reported_duration_seconds`, and a disagreement beyond the tolerance is recorded as a `state_context` observation. Report the duration you believe is right and let the two be compared; do not reverse-engineer a number to match.
|
|
222
273
|
|
|
223
|
-
###
|
|
274
|
+
### 7. Flakiness & Resolution Reporting
|
|
224
275
|
|
|
225
276
|
When a semantic action's primary resolution fails and the fallback succeeds, add a `flakiness` observation:
|
|
226
277
|
|
|
@@ -230,7 +281,7 @@ When neither attempt succeeds, fail the step and add a detailed observation desc
|
|
|
230
281
|
|
|
231
282
|
## Multi-Scenario Execution
|
|
232
283
|
|
|
233
|
-
- Execute scenarios sequentially. Each scenario's leading `launch_app` restarts the app (`mauto launch <appId>`), so a scenario never inherits the previous one's screen and needs no separate reset; use `app_state: "fresh_install"` only when app data must be wiped too.
|
|
284
|
+
- Execute scenarios sequentially: one `mauto run` per scenario file, each under its own run id. Each scenario's leading `launch_app` restarts the app (`mauto launch <appId>`), so a scenario never inherits the previous one's screen and needs no separate reset; use `app_state: "fresh_install"` only when app data must be wiped too.
|
|
234
285
|
- "run all smoke tests" → filter by the `smoke` tag. "run all" → every scenario in `.mobile-automator/scenarios/`. List `.mobile-automator/scenarios/` directly: the workspace folder is hidden, and recursive search tools (glob or grep style) skip hidden directories, so they will report it empty.
|
|
235
286
|
- Present a combined summary with a per-scenario breakdown.
|
|
236
287
|
|
|
@@ -238,6 +289,7 @@ When neither attempt succeeds, fail the step and add a detailed observation desc
|
|
|
238
289
|
|
|
239
290
|
### DO
|
|
240
291
|
|
|
292
|
+
- Prefer `mauto run` (section 2); replay step by step (section 4) only when the runner cannot be used.
|
|
241
293
|
- Execute exactly what the scenario specifies.
|
|
242
294
|
- Run `mauto elements` before every interaction.
|
|
243
295
|
- Capture a screenshot at every step, even on failure.
|
|
@@ -247,6 +299,7 @@ When neither attempt succeeds, fail the step and add a detailed observation desc
|
|
|
247
299
|
|
|
248
300
|
### DON'T
|
|
249
301
|
|
|
302
|
+
- Re-judge a verdict the CLI decided (a mechanical or reference-screenshot assertion), or replay by hand to get around a `test_failed` run.
|
|
250
303
|
- Improvise or deviate from the scenario steps.
|
|
251
304
|
- Modify app source code in {{protected_directories}}.
|
|
252
305
|
- Hardcode coordinates — derive them from `mauto elements`.
|
|
@@ -62,20 +62,61 @@ Record each observation with `mauto result add-step --observation <type>:<messag
|
|
|
62
62
|
- **Fresh install.** If the scenario's `preconditions.app_state` is `fresh_install` (from the scenario you read above), uninstall the app first — `mauto uninstall` with the app package ({{app_package}}) — and only then install, so the run starts with no app data.
|
|
63
63
|
- State which path you took — installed a supplied artifact, or built — before continuing.
|
|
64
64
|
|
|
65
|
-
###
|
|
65
|
+
### Compare Environment
|
|
66
66
|
|
|
67
67
|
- Read the `metadata` to compare the recording environment against the current execution environment. Note any differences (different device, API level, environment).
|
|
68
68
|
|
|
69
|
-
###
|
|
69
|
+
### Verify Preconditions
|
|
70
70
|
|
|
71
|
-
- **Note:** an `app_state` of `fresh_install` is handled in pre-flight (uninstall before the install, see above), and precondition `device_actions` run per the table in
|
|
71
|
+
- **Note:** an `app_state` of `fresh_install` is handled in pre-flight (uninstall before the install, see above), and precondition `device_actions` are run by `mauto run` itself (section 2), or per the table in section 4 on a manual replay.
|
|
72
72
|
- Review the scenario's `preconditions` array to understand the expected starting state.
|
|
73
73
|
- If any preconditions are state-based (e.g., `"user_logged_out"`, `"no_network"`, `"dark_mode_enabled"`), verify or configure them now:
|
|
74
74
|
- For app-state preconditions: launch the app and check/configure the state.
|
|
75
75
|
- For device preconditions: configure device settings (e.g., airplane mode for `"no_network"`).
|
|
76
76
|
- If a precondition cannot be verified or met, report it and ask the user how to proceed.
|
|
77
77
|
|
|
78
|
-
###
|
|
78
|
+
### 2. Replay with mauto run
|
|
79
|
+
|
|
80
|
+
Replay the scenario with the unattended runner before anything else:
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
mauto run <scenario path> --run-id $MAUTO_RUN_ID
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Add `--junit <path>` when a CI job needs a JUnit XML report of the run (the file's directory must already exist). It is one invocation and one envelope: the runner performs the precondition `device_actions`, every step, and every assertion the CLI can decide, captures the evidence, and finalizes the result file itself. Nothing waits for you mid-run, and you never drive a step through it.
|
|
87
|
+
|
|
88
|
+
If pre-flight reported an `unsupported_action` warning, skip this section: `mauto run` refuses such a scenario with `invalid_input` before any device work. Replay it by hand instead (section 4).
|
|
89
|
+
|
|
90
|
+
`data` carries the same keys whatever the outcome: `run_id`, `scenario_id`, `status`, `result_path`, `duration_seconds`, `steps`, `assertions`, `failed_step`, `failures`, `pending`, `junit_path`, `executor`. Branch on the outcome:
|
|
91
|
+
|
|
92
|
+
- **Passed** — `ok: true`, exit code `0`, `data.status` is `passed`. Every step ran and the CLI decided every assertion. The result file is already finalized at `data.result_path`, so there is nothing left to record or finalize: present it (section 6 describes what it holds).
|
|
93
|
+
- **`test_failed`** — `ok: false` with `error.kind: "test_failed"` and exit code `6`; `data.status` is `failed` or `error`. That is the test result, not a tool error. Report the failure from `data`: `failed_step` is the first step that failed (`null` when only an assertion failed), and `failures` lists each failed step and assertion with its message. The runner stops at a failed step, so later steps did not run; the failed step's screenshot is recorded on its `steps_executed` entry in the result file at `data.result_path`. Do not re-judge a verdict the CLI decided, and do not retry the run or replay it by hand to get around the failure. The result is already finalized: present it.
|
|
94
|
+
- **`incomplete`** — `ok: false` with `error.kind: "incomplete"` and exit code `7`; `data.status` is `incomplete`. Nothing failed, but something is left for an agent to decide. Continue with section 3.
|
|
95
|
+
- **Anything else** is not a verdict about the app. `invalid_input` is a refusal before any device work (a bad flag, an unreadable file, or a scenario that cannot replay unattended): read `error.message` and the `hint`. `device` means no device could be driven: fix what the `hint` names and run again. An `environment` failure that still carries the run's `data` means only the JUnit report could not be written; the run itself is finalized and `data.status` is its outcome.
|
|
96
|
+
|
|
97
|
+
### 3. Handle needs_agent
|
|
98
|
+
|
|
99
|
+
An `incomplete` run is waiting on an agent. Its `hint` says for what, and prints the exact commands that complete it.
|
|
100
|
+
|
|
101
|
+
**Pending assertions.** `data.pending` lists every assertion the CLI could not decide, each as `{assertion_id, step_id, type}`; the result file records them with status `needs_agent`. For each entry:
|
|
102
|
+
|
|
103
|
+
1. **Find its screenshot.** Open the result file (`data.result_path`) and read the `screenshot` of the `steps_executed` entry with that `step_id`; the runner captures one for every step the scenario marks `checkpoint`. If none was recorded and `step_id` is the last step the run executed, the device is still showing that screen: capture it with `mauto screenshot <path>`. If neither holds, the screen is gone. Do not guess: leave that assertion pending.
|
|
104
|
+
2. **Judge it** from that screenshot, per the Tier 2 list in section 5.
|
|
105
|
+
3. **Record the verdict** with the `mauto result add-assertion` line the hint prints for that id, filling in only the verdict and the reason:
|
|
106
|
+
`mauto result add-assertion --run-id <run id> --step-id <step_id> --assertion-id <assertion_id> --type <type> --pass <true|false> --judge agent --message "<why>"`.
|
|
107
|
+
`--assertion-id` is what makes the verdict replace the pending entry in place, and `--message` IS the evidence, so omit neither.
|
|
108
|
+
|
|
109
|
+
A pending `screenshot_match` is the exception: it is pending because its reference screenshot does not exist yet, and it is never judged by eye. Ask the user whether the screen is correct; if they confirm, record the baselines by running the scenario again under a new run id with `--update-references` added. Otherwise leave it pending.
|
|
110
|
+
|
|
111
|
+
Then finalize again with the last line of the hint, `mauto result finalize --run-id <run id> --scenario <path>` (the hint prints the scenario's absolute path, so the command works from any directory). The status is derived again from what is now recorded: `passed` (exit code `0`), `failed` if any verdict was `false` (`test_failed`, exit code `6`), or still `incomplete` (exit code `7`) if you left anything pending. Report a run that is still `incomplete` as incomplete, never as passed.
|
|
112
|
+
|
|
113
|
+
**A pending step.** A hint line `Pending step <step_id>: <what to do>`, with `data.steps.needs_agent` above zero, means the runner reached a step it cannot perform itself (for example an install with no build path, or a condition it cannot evaluate). It stopped there, so no later step ran, and that recorded step keeps this run `incomplete` whatever is added to it afterwards. Do what the line asks, then replay the whole scenario by hand (section 4) under a new run id: export a new `MAUTO_RUN_ID`, perform that step with the ordinary one-shot verbs, and say in your report that the unattended run ended `incomplete` at that step.
|
|
114
|
+
|
|
115
|
+
### 4. Manual per-step replay
|
|
116
|
+
|
|
117
|
+
This is the fallback: the per-step recipe an agent follows with the one-shot verbs. Use it only when the runner cannot be used for this scenario (section 2 refused it, or pre-flight reported an `unsupported_action`), or to perform a step the runner left `needs_agent` (section 3). Otherwise the runner has already done all of this.
|
|
118
|
+
|
|
119
|
+
#### Step-by-Step Replay
|
|
79
120
|
|
|
80
121
|
Before executing each step, initialize a **session variable map** (an empty object to store captured values).
|
|
81
122
|
|
|
@@ -169,7 +210,7 @@ When executing a `capture_value` action:
|
|
|
169
210
|
4. Also record it in the result file with `--capture <capture_to>=<extracted_text>` on that step's `mauto result add-step` call — the in-memory session variable map alone never reaches the result file; only the flag does.
|
|
170
211
|
5. Report: "Captured `capture_to` = '<value>'".
|
|
171
212
|
|
|
172
|
-
|
|
213
|
+
#### When a step fails, find out whether the app is still alive
|
|
173
214
|
|
|
174
215
|
An action that fails, or an `elements` call that comes back empty, has two very
|
|
175
216
|
different explanations: the UI changed, or the app died. They lead to opposite
|
|
@@ -209,7 +250,7 @@ default — a report from last week is not evidence about this step. And a repor
|
|
|
209
250
|
whose own timestamp is unreadable is counted in `unattributed` rather than
|
|
210
251
|
included; if that count is non-zero, widen `--since` and look yourself.
|
|
211
252
|
|
|
212
|
-
|
|
253
|
+
#### When the element list looks wrong, check the engine
|
|
213
254
|
|
|
214
255
|
If the app is alive but `mauto elements` misses the label of a text field you
|
|
215
256
|
can see on screen, or lists content from a tab that is not showing (common in
|
|
@@ -224,11 +265,14 @@ setting. Do not switch the engine mid-run yourself.
|
|
|
224
265
|
|
|
225
266
|
### 5. Validate Assertions (27 types)
|
|
226
267
|
|
|
227
|
-
For each assertion in the scenario (evaluated after the referenced step), pick the type recorded in the scenario and evaluate it. The 27 types split into two tiers by **who decides pass/fail
|
|
268
|
+
For each assertion in the scenario (evaluated after the referenced step), pick the type recorded in the scenario and evaluate it. The 27 types split into two tiers by **who decides pass/fail**, plus one reference-screenshot type the CLI also decides:
|
|
228
269
|
|
|
229
|
-
- **Tier 1 — mechanical (the CLI decides).** These are deterministic checks over the on-screen element list (plus captured variables / a previous snapshot). Run `mauto elements` to get the current screen, then evaluate the assertion with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). A passing verdict arrives as a normal `ok: true` envelope. A failing one arrives as `ok: false` with `error.kind: "test_failed"` and exit code `6`, and the verdict (`pass: false`) is still in `data`. That is the test result, not a tool error: do not retry the verb or treat the run as broken. You record that verdict — you do not re-judge it.
|
|
270
|
+
- **Tier 1 — mechanical (the CLI decides).** These are deterministic checks over the on-screen element list (plus captured variables / a previous snapshot / the screen size). Run `mauto elements` to get the current screen, then evaluate the assertion with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). A passing verdict arrives as a normal `ok: true` envelope. A failing one arrives as `ok: false` with `error.kind: "test_failed"` and exit code `6`, and the verdict (`pass: false`) is still in `data`. That is the test result, not a tool error: do not retry the verb or treat the run as broken. You record that verdict — you do not re-judge it.
|
|
271
|
+
- **Reference screenshot — `screenshot_match` (the CLI decides).** A deterministic pixel comparison against a stored reference screenshot (a baseline); see the reference-screenshot block below. You record its verdict — you do not judge it by eye.
|
|
230
272
|
- **Tier 2 — visual / semantic (the agent decides).** These require judging pixels. `mauto assert <type> ...` reports them as `needs_agent` (with `pass: null`) because the CLI cannot decide them. For these you take a screenshot with `mauto screenshot <path>`, visually analyze it, and report pass/fail yourself with a clear justification of what you observed.
|
|
231
273
|
|
|
274
|
+
`mauto run` decides the Tier 1 types and the reference-screenshot type itself and records each verdict; after a run you handle only what it left `needs_agent` (section 3). The `mauto assert` instructions below are for a manual replay (section 4).
|
|
275
|
+
|
|
232
276
|
> One assertion type, `element_state`, names a Tier-1 intent but its per-attribute state inspection is not represented in the element model the CLI reads, so `mauto assert` returns it as `needs_agent` — judge it from the element list / screenshot like a Tier-2 check.
|
|
233
277
|
|
|
234
278
|
---
|
|
@@ -237,7 +281,7 @@ For each assertion in the scenario (evaluated after the referenced step), pick t
|
|
|
237
281
|
|
|
238
282
|
- **`element_exists`:** the element described in `element_description` is present in the on-screen element list.
|
|
239
283
|
- **`element_not_exists`:** the element is NOT present in the on-screen element list.
|
|
240
|
-
- **`element_visible`:** pass if the element appears in the result. `expected_visible: true` = must be present, `false` = must be absent. Intended for elements that may be scrolled off or conditionally hidden, as distinct from fully absent elements (`element_exists` / `element_not_exists`). If you need to confirm the element is actually within the viewport (not clipped or scrolled off), use `element_fully_visible`
|
|
284
|
+
- **`element_visible`:** pass if the element appears in the result. `expected_visible: true` = must be present, `false` = must be absent. Intended for elements that may be scrolled off or conditionally hidden, as distinct from fully absent elements (`element_exists` / `element_not_exists`). If you need to confirm the element is actually within the viewport (not clipped or scrolled off), use `element_fully_visible` instead.
|
|
241
285
|
- **`element_text`:** the element's text exactly matches `expected_value`.
|
|
242
286
|
- **`text_contains`:** the element's text includes `expected_substring` as a substring.
|
|
243
287
|
- **`text_not_empty`:** the target element's text has length > 0.
|
|
@@ -251,6 +295,18 @@ For each assertion in the scenario (evaluated after the referenced step), pick t
|
|
|
251
295
|
- **`has_accessibility_label`:** the element's accessibility label is non-empty. If `label_value` is set, verify it matches exactly.
|
|
252
296
|
- **`value_matches_variable`:** look up `variable_name` in the session variable map; verify the element described by `element_description` contains text that matches the captured value. If `pattern` is also set, use it as a format hint to locate the relevant portion.
|
|
253
297
|
- **`text_changed`:** compare the element's current text with its text at the previous assertion checkpoint. Pass if the text differs.
|
|
298
|
+
- **`element_fully_visible`:** the element described in `element_description` lies fully inside the screen — not clipped by the screen edges or scrolled partly off. The CLI checks the element's bounds against the screen size; it cannot see overlap, so an element covered by another one still passes.
|
|
299
|
+
- **`screen_title`:** some element whose text or accessibility label equals `expected_text` (ignoring case and surrounding whitespace) starts in the top quarter of the screen. The same text lower on the screen fails.
|
|
300
|
+
- **`permission_dialog_shown`:** an OS-level permission dialog is up: an allow button and a deny button are both on screen. If `permission_name` is set (e.g., `"camera"`, `"location"`), pass it as `--permission <name>`; some element on screen must also mention it.
|
|
301
|
+
|
|
302
|
+
---
|
|
303
|
+
|
|
304
|
+
**Reference screenshot — `screenshot_match`, decided by `mauto assert` against a baseline**
|
|
305
|
+
|
|
306
|
+
- Run `mauto assert screenshot_match --reference <reference_screenshot> --tolerance <tolerance>`, adding one `--mask <x,y,w,h>` or `--mask "<element description>"` per entry in the assertion's `masks`. Omit `--tolerance` when the scenario sets none (default `0.9`). The CLI captures a stable screenshot, compares it pixel by pixel, and decides: a pass is `ok: true`; a mismatch is `ok: false` with `error.kind: "test_failed"`, exit `6`, the similarity score and a diff image path in `data`. Record that verdict as-is — do not override it by looking at the screenshot.
|
|
307
|
+
- A dimension mismatch (a different device or orientation than the baseline was recorded on) is a failure too; report it as such and tell the user the baseline needs re-recording for this device.
|
|
308
|
+
- If the reference screenshot does not exist yet, the verdict is `needs_agent` (exit `0`), not a failure. Do not judge it by eye instead. Ask the user whether the current screen is correct; if they confirm, record the baseline with the same command plus `--update-references`, otherwise record the assertion as pending.
|
|
309
|
+
- Record the verdict with the `mauto result add-assertion` command in the envelope's `hint`; it carries the reference screenshot, the actual screenshot and the similarity score.
|
|
254
310
|
|
|
255
311
|
---
|
|
256
312
|
|
|
@@ -258,19 +314,15 @@ For each assertion in the scenario (evaluated after the referenced step), pick t
|
|
|
258
314
|
|
|
259
315
|
For all Tier 2 assertions: take a screenshot, visually analyze it, and report pass/fail with a clear justification of what you observed.
|
|
260
316
|
|
|
261
|
-
- **`screenshot_match`:** compare the captured screenshot against the reference baseline. Use **semantic visual comparison** — screen identity, presence/absence of key elements, text content, layout structure. Minor rendering differences (anti-aliasing, font smoothing) are acceptable within the `tolerance` threshold.
|
|
262
317
|
- **`visual_state`:** assess whether the screen is in the `expected_visual_state`: `loaded` (content visible, no spinners), `loading` (loading indicator present), `empty` (empty-state UI), or `error` (error message/state visible).
|
|
263
|
-
- **`element_fully_visible`:** the element described in `element_description` is fully visible in the viewport — not clipped, not scrolled off, nothing overlapping it.
|
|
264
318
|
- **`color_style`:** locate the element and assess its dominant color. If `color_hex` is set, verify it matches approximately. If `expected_theme` is set (`dark`/`light`), verify the colors are consistent with that theme.
|
|
265
|
-
- **`screen_title`:** read the navigation bar / screen title from the screenshot. Verify it matches `expected_text`.
|
|
266
319
|
- **`alert_present`:** a system alert or modal dialog is visually present.
|
|
267
320
|
- **`alert_text`:** an alert/dialog is on screen and its primary text matches `expected_text`.
|
|
268
321
|
- **`toast_visible`:** the toast may be transient. Take a screenshot at the moment of the step and look for a floating notification overlay. If visible, verify its text matches `expected_text`. If it has already disappeared, mark `failed` and note it was transient.
|
|
269
322
|
- **`keyboard_visible`:** whether the soft keyboard is visible at the bottom of the screen. `expected_visible: true` = must be shown, `false` = must be hidden.
|
|
270
323
|
- **`dark_mode_active`:** assess the overall screen appearance. Pass if the background is dark (dark mode) or light (light mode) as specified by `expected_theme`.
|
|
271
|
-
- **`permission_dialog_shown`:** an OS-level permission request dialog is visible. If `permission_name` is set (e.g., `"camera"`, `"location"`), verify the dialog requests that specific permission.
|
|
272
324
|
|
|
273
|
-
**Dynamic content:** if the reference `expected_state` marks content as `[dynamic:...]`, ignore those regions
|
|
325
|
+
**Dynamic content:** if the reference `expected_state` marks content as `[dynamic:...]`, ignore those regions when you judge a Tier-2 check. For `screenshot_match`, dynamic regions are excluded only through the assertion's `masks`.
|
|
274
326
|
|
|
275
327
|
**Variable substitution in assertions:** before evaluating any assertion, replace `variable_name` references in string fields with values from the session variable map.
|
|
276
328
|
|
|
@@ -302,7 +354,7 @@ State context:
|
|
|
302
354
|
|
|
303
355
|
When asked to run multiple scenarios:
|
|
304
356
|
|
|
305
|
-
- Execute them sequentially.
|
|
357
|
+
- Execute them sequentially: one `mauto run` per scenario file, each under its own run id.
|
|
306
358
|
- Each scenario's leading `launch_app` restarts the app (`mauto launch <appId>`), so a scenario never inherits the previous one's screen and needs no separate reset. Use `app_state: "fresh_install"` only when app data must be wiped too.
|
|
307
359
|
- If the user says "run all smoke tests", filter scenarios by the `smoke` tag.
|
|
308
360
|
- If the user says "run all", execute every scenario in `.mobile-automator/scenarios/`. List `.mobile-automator/scenarios/` directly: the workspace folder is hidden, and recursive search tools (glob or grep style) skip hidden directories, so they will report it empty.
|
|
@@ -312,6 +364,7 @@ When asked to run multiple scenarios:
|
|
|
312
364
|
|
|
313
365
|
### DO
|
|
314
366
|
|
|
367
|
+
- Prefer `mauto run` (section 2); replay step by step (section 4) only when the runner cannot be used.
|
|
315
368
|
- Follow scenario steps exactly as written.
|
|
316
369
|
- Run `mauto elements` before every interaction.
|
|
317
370
|
- Retry once on suspected timing failures.
|
|
@@ -321,6 +374,7 @@ When asked to run multiple scenarios:
|
|
|
321
374
|
|
|
322
375
|
### DON'T
|
|
323
376
|
|
|
377
|
+
- Re-judge a verdict the CLI decided (a mechanical or reference-screenshot assertion), or replay by hand to get around a `test_failed` run.
|
|
324
378
|
- Deviate from the scenario steps — if something unexpected happens, report it, don't work around it.
|
|
325
379
|
- Modify app source code in {{protected_directories}}.
|
|
326
380
|
- Hardcode coordinates — derive them from `mauto elements`.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
- Follow the scenario **exactly as written** — no improvisation, no skipped steps. If something unexpected happens, report it; do not work around it.
|
|
2
|
+
- Start with `mauto run <scenario>`: it replays the scenario unattended and exits `incomplete` when something needs an agent's judgment, which you then finish with the one-shot verbs.
|
|
2
3
|
- Drive the device **only** through `mauto` verbs; never assume resource-ids or OS-specific element IDs (use the semantic actions in platform-agnostic projects).
|
|
3
4
|
- Every assertion is backed by a screenshot recorded with `mauto result add-step --screenshot`; record its verdict with `mauto result add-assertion` and any observations with `--observation <type>:<message>`.
|
|
4
5
|
- If this workspace has memory, read it first with `mauto memory show`; save durable app facts and user corrections with `mauto memory add`.
|
|
@@ -122,7 +122,7 @@ Translate user language to action types and `mauto` verbs:
|
|
|
122
122
|
|
|
123
123
|
### 5. Assertion Type Decision Table
|
|
124
124
|
|
|
125
|
-
Use the most specific type that matches the user's intent. Tier 1 checks read structure from `mauto elements`; Tier 2 checks read pixels from `mauto screenshot <path>`. Evaluate each with `mauto assert <type> ...`.
|
|
125
|
+
Use the most specific type that matches the user's intent. Tier 1 checks read structure from `mauto elements`; Tier 2 checks read pixels from `mauto screenshot <path>`. Evaluate each with `mauto assert <type> ...`. Three types in the visual, navigation and system categories are Tier 1 too, decided by `mauto assert` over the element list: `element_fully_visible` (the element is not clipped by the screen edges; overlap is not checked), `screen_title` (an element equal to `expected_text` near the top of the screen) and `permission_dialog_shown` (an allow and a deny button on screen). `screenshot_match` is neither: `mauto assert` decides it with a pixel comparison against a reference screenshot. When you add one, the screen in front of you is the baseline: once the user confirms it looks right, record it with `mauto assert screenshot_match --reference <reference_screenshot> --update-references`, and add `masks` for regions that change between runs (clocks, timestamps, animations).
|
|
126
126
|
|
|
127
127
|
**Element State**
|
|
128
128
|
|
|
@@ -157,8 +157,8 @@ Use the most specific type that matches the user's intent. Tier 1 checks read st
|
|
|
157
157
|
|
|
158
158
|
| User intent | Type | Key fields |
|
|
159
159
|
|---|---|---|
|
|
160
|
-
| "the full element is visible, not clipped" | `element_fully_visible` | — |
|
|
161
|
-
| "the screen looks the same as before" | `screenshot_match` | `reference_screenshot` |
|
|
160
|
+
| "the full element is visible, not clipped by the screen edges" | `element_fully_visible` | — |
|
|
161
|
+
| "the screen looks the same as before" | `screenshot_match` | `reference_screenshot`, `tolerance`, `masks` (regions that change between runs) |
|
|
162
162
|
| "the screen is loaded / in error / empty state" | `visual_state` | `expected_visual_state` |
|
|
163
163
|
| "the button color is X" | `color_style` | `color_hex` |
|
|
164
164
|
|
|
@@ -121,7 +121,7 @@ When the user provides a validation step, or when a declarative statement about
|
|
|
121
121
|
|
|
122
122
|
#### Assertion Type Decision Table (27 types)
|
|
123
123
|
|
|
124
|
-
Use the most specific type that matches the user's intent. Tier 1 checks read structure from `mauto elements`; Tier 2 checks read pixels from `mauto screenshot <path>`. Evaluate each with `mauto assert <type> ...`.
|
|
124
|
+
Use the most specific type that matches the user's intent. Tier 1 checks read structure from `mauto elements`; Tier 2 checks read pixels from `mauto screenshot <path>`. Evaluate each with `mauto assert <type> ...`. Three types in the visual, navigation and system categories are Tier 1 too, decided by `mauto assert` over the element list: `element_fully_visible` (the element is not clipped by the screen edges; overlap is not checked), `screen_title` (an element equal to `expected_text` near the top of the screen) and `permission_dialog_shown` (an allow and a deny button on screen). `screenshot_match` is neither: `mauto assert` decides it with a pixel comparison against a reference screenshot. When you add one, the screen in front of you is the baseline: once the user confirms it looks right, record it with `mauto assert screenshot_match --reference <reference_screenshot> --update-references`, and add `masks` for regions that change between runs (clocks, timestamps, animations).
|
|
125
125
|
|
|
126
126
|
**Category 1 — Element State (Tier 1: `mauto elements`)**
|
|
127
127
|
|
|
@@ -162,8 +162,8 @@ Use the most specific type that matches the user's intent. Tier 1 checks read st
|
|
|
162
162
|
|
|
163
163
|
| User intent / natural language | Type | Key fields |
|
|
164
164
|
|---|---|---|
|
|
165
|
-
| "the full element is visible, not clipped" | `element_fully_visible` | — |
|
|
166
|
-
| "the element/screen looks the same as before" | `screenshot_match` | `reference_screenshot: "..."` |
|
|
165
|
+
| "the full element is visible, not clipped by the screen edges" | `element_fully_visible` (Tier 1: bounds vs screen size, by `mauto assert`) | — |
|
|
166
|
+
| "the element/screen looks the same as before" | `screenshot_match` (pixel comparison by `mauto assert`, not Tier 2) | `reference_screenshot: "..."`, `tolerance`, `masks: [{ "element_description": "clock" }]` |
|
|
167
167
|
| "the screen is loaded / in error / empty state" | `visual_state` | `expected_visual_state: "loaded"` |
|
|
168
168
|
| "the button color is blue / #0057FF" | `color_style` | `color_hex: "#0057FF"` |
|
|
169
169
|
|
|
@@ -171,7 +171,7 @@ Use the most specific type that matches the user's intent. Tier 1 checks read st
|
|
|
171
171
|
|
|
172
172
|
| User intent / natural language | Type | Key fields |
|
|
173
173
|
|---|---|---|
|
|
174
|
-
| "the screen/page title is 'Settings'" | `screen_title` | `expected_text: "Settings"` |
|
|
174
|
+
| "the screen/page title is 'Settings'" | `screen_title` (Tier 1: by `mauto assert`) | `expected_text: "Settings"` |
|
|
175
175
|
| "a dialog / alert appeared" | `alert_present` | — |
|
|
176
176
|
| "the alert says 'Are you sure?'" | `alert_text` | `expected_text: "Are you sure?"` |
|
|
177
177
|
| "a toast / snackbar appeared saying 'Saved'" | `toast_visible` | `expected_text: "Saved"` |
|
|
@@ -194,7 +194,7 @@ Use the most specific type that matches the user's intent. Tier 1 checks read st
|
|
|
194
194
|
|
|
195
195
|
| User intent / natural language | Type | Key fields |
|
|
196
196
|
|---|---|---|
|
|
197
|
-
| "the app asked for camera / location / microphone permission" | `permission_dialog_shown` | `permission_name: "camera"` |
|
|
197
|
+
| "the app asked for camera / location / microphone permission" | `permission_dialog_shown` (Tier 1: by `mauto assert`) | `permission_name: "camera"` |
|
|
198
198
|
| "dark mode is active / the screen is dark" | `dark_mode_active` | `expected_theme: "dark"` |
|
|
199
199
|
|
|
200
200
|
#### Auto-Assertion Rule
|
package/src/guide/emitter.js
CHANGED
|
@@ -18,6 +18,7 @@ const fs = require('fs');
|
|
|
18
18
|
const path = require('path');
|
|
19
19
|
|
|
20
20
|
const { interpolate } = require('./placeholders');
|
|
21
|
+
const { EXIT } = require('../output/envelope');
|
|
21
22
|
|
|
22
23
|
// Topics with REAL ported prose live as markdown content files (carrying
|
|
23
24
|
// `{{placeholder}}` tokens lifted from the Gemini SKILLs). They are emitted by
|
|
@@ -111,6 +112,7 @@ function emitBootstrap() {
|
|
|
111
112
|
['screenshot', 'save a screenshot to a path'],
|
|
112
113
|
['assert', 'evaluate an assertion against the current screen'],
|
|
113
114
|
['validate', 'validate a scenario file against the scenario schema'],
|
|
115
|
+
['run', `replay a scenario unattended and finalize its result (exit ${EXIT.OK} passed, ${EXIT.TEST_FAILED} test_failed, ${EXIT.INCOMPLETE} incomplete)`],
|
|
114
116
|
['result', 'record step results and finalize a run'],
|
|
115
117
|
['setup', 'scaffold the workspace and write a config'],
|
|
116
118
|
['config', 'get/set values in the workspace config'],
|
package/src/init/host-plan.js
CHANGED
|
@@ -83,6 +83,7 @@ function cursorRuleBody() {
|
|
|
83
83
|
'---\n\n' +
|
|
84
84
|
guideEmitter.emitBootstrap() +
|
|
85
85
|
'\nRun `mauto guide <topic>` (generate|execute|setup) before a workflow. ' +
|
|
86
|
+
'The execute guide starts with `mauto run`. ' +
|
|
86
87
|
'Drive the device only through `mauto` verbs.\n'
|
|
87
88
|
);
|
|
88
89
|
}
|
package/src/observe/trace.js
CHANGED
|
@@ -15,7 +15,7 @@ const path = require('path');
|
|
|
15
15
|
const { atCap } = require('./sinks/file');
|
|
16
16
|
const { logsDir, runTracePath, RUN_TRACE_PREFIX } = require('./paths');
|
|
17
17
|
|
|
18
|
-
// A device failure in the envelope's own taxonomy. `deviceFail` (src/
|
|
18
|
+
// A device failure in the envelope's own taxonomy. `deviceFail` (src/output/device-fail.js) is
|
|
19
19
|
// the ONLY producer of these two kinds, so counting them needs no list of which
|
|
20
20
|
// verbs are device verbs — a list that would have to be maintained by hand
|
|
21
21
|
// alongside connectBridge and would be wrong the first time someone forgot.
|
package/src/observe/vocab.js
CHANGED
|
@@ -48,6 +48,10 @@ const EVENT_NAMES = Object.freeze([
|
|
|
48
48
|
'daemon.lock_conflict',
|
|
49
49
|
'daemon.undeliverable',
|
|
50
50
|
'daemon.handler_error',
|
|
51
|
+
// `mauto run`'s own stamps (src/cli.js RUN_TRACE_EVENTS), so a single
|
|
52
|
+
// in-process run's trace spans its steps before finalize measures it.
|
|
53
|
+
'run.start',
|
|
54
|
+
'run.step',
|
|
51
55
|
// Emitted by the flush path, which lands in a later slice-5 task; declared
|
|
52
56
|
// now because its event shape is already exercised by unit tests.
|
|
53
57
|
'telemetry.flush',
|
|
@@ -88,6 +92,8 @@ const VERB_NAMES = Object.freeze([
|
|
|
88
92
|
'crash',
|
|
89
93
|
'mcp',
|
|
90
94
|
'telemetry',
|
|
95
|
+
// The unattended replay runner (#278).
|
|
96
|
+
'run',
|
|
91
97
|
]);
|
|
92
98
|
|
|
93
99
|
// Why the daemon stopped. 'crash' is documented but not yet produced.
|