mobile-automator 0.39.0 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +2 -0
  2. package/package.json +4 -2
  3. package/src/assertion/evaluator.js +116 -14
  4. package/src/assertion/fields.js +106 -0
  5. package/src/assertion/observe.js +17 -0
  6. package/src/cli.js +177 -177
  7. package/src/device/action-catalog.js +4 -2
  8. package/src/device/bridge.js +17 -1
  9. package/src/device/mobile-mcp-tools.js +1 -0
  10. package/src/device/resolve-connection.js +1 -1
  11. package/src/device/semantic-press/index.js +19 -0
  12. package/src/device/semantic-press/resolver.js +1 -1
  13. package/src/{device/semantic-press/labels.js → dialog/permission-labels.js} +6 -3
  14. package/src/guide/content/execute.agnostic.md +74 -21
  15. package/src/guide/content/execute.aware.md +69 -15
  16. package/src/guide/content/execute.invariants.md +1 -0
  17. package/src/guide/content/generate.agnostic.md +3 -3
  18. package/src/guide/content/generate.aware.md +5 -5
  19. package/src/guide/emitter.js +2 -0
  20. package/src/init/host-plan.js +1 -0
  21. package/src/observe/trace.js +1 -1
  22. package/src/observe/vocab.js +6 -0
  23. package/src/output/device-fail.js +41 -0
  24. package/src/output/envelope.js +3 -3
  25. package/src/output/run-human.js +37 -0
  26. package/src/result/capability-catalog.js +48 -0
  27. package/src/result/executor.js +58 -0
  28. package/src/result/finalize-against-scenario.js +103 -0
  29. package/src/result/flags.js +49 -6
  30. package/src/result/junit.js +39 -16
  31. package/src/result/outcome.js +24 -2
  32. package/src/result/pending-hint.js +67 -0
  33. package/src/result/scenario-check.js +58 -0
  34. package/src/result/status-guard.js +46 -0
  35. package/src/result/store.js +135 -9
  36. package/src/run/actions/capture.js +21 -0
  37. package/src/run/actions/gestures.js +89 -0
  38. package/src/run/actions/index.js +48 -0
  39. package/src/run/actions/launch.js +30 -0
  40. package/src/run/actions/preconditions.js +44 -0
  41. package/src/run/actions/scroll.js +22 -0
  42. package/src/run/actions/semantic.js +16 -0
  43. package/src/run/actions/wait.js +66 -0
  44. package/src/run/assert-reference.js +118 -0
  45. package/src/run/assertions.js +102 -0
  46. package/src/run/clock.js +10 -0
  47. package/src/run/condition.js +80 -0
  48. package/src/run/fatal.js +15 -0
  49. package/src/run/handle.js +151 -0
  50. package/src/run/needs-agent.js +39 -0
  51. package/src/run/poll.js +38 -0
  52. package/src/run/prepare.js +107 -0
  53. package/src/run/reference-assertion.js +172 -0
  54. package/src/run/reference-screenshot.js +92 -0
  55. package/src/run/resolve.js +95 -0
  56. package/src/run/run-id.js +14 -0
  57. package/src/run/runner.js +104 -0
  58. package/src/run/scale.js +52 -0
  59. package/src/run/screenshots.js +163 -0
  60. package/src/run/settings.js +35 -0
  61. package/src/run/state.js +22 -0
  62. package/src/run/step.js +138 -0
  63. package/src/run/summary.js +73 -0
  64. package/src/run/trace.js +30 -0
  65. package/src/run/tree.js +96 -0
  66. package/src/run/variables.js +56 -0
  67. package/src/scenario/load.js +78 -0
  68. package/src/schemas/config_schema.json +29 -0
  69. package/src/schemas/result_schema.json +45 -3
  70. package/src/schemas/scenario_schema.json +27 -0
  71. package/src/util/as-array.js +8 -0
  72. package/src/util/parse-positive-int.js +15 -0
  73. package/src/util/parse-unit-interval.js +17 -0
package/README.md CHANGED
@@ -83,6 +83,7 @@ That's it. Scenarios land in `.mobile-automator/scenarios/`, results in `.mobile
83
83
 
84
84
  - **The agent (brain)** resolves targets ("the Login button"), judges visual assertions, and assembles scenarios. It reads workflow guidance on demand with `mauto guide <topic>`.
85
85
  - **`mauto` (hands)** performs deterministic, scriptable actions and mechanical assertions, driving the device via mobile-mcp. Every verb emits a uniform JSON envelope (`{ok, data, error, hint}`); add `--human` for readable output.
86
+ - **Unattended replay** needs no agent: `mauto run <scenario>` replays a saved scenario in one command and exits `0` (passed), `6` (test failed) or `7` (incomplete: something still needs an agent's judgment); `--junit <path>` writes a JUnit report for CI. See the [CI recipe](https://sh3lan93.github.io/mobile-automator/faq/#how-do-i-integrate-with-cicd).
86
87
 
87
88
  Targets are always **visible text + semantic role + coordinates** — never brittle resource-ids — which is what makes scenarios portable across Android and iOS.
88
89
 
@@ -149,6 +150,7 @@ Most low-level verbs are called **by the agent**, not by you. The ones you'll ru
149
150
  | `mauto memory add <text> --kind <app-knowledge\|preferences>` | Record a durable agent-learned app fact or user preference (`[asserted]`, de-duped) |
150
151
  | `mauto memory forget --kind <k> --match <substr>` | Remove agent-authored memory entries matching a substring (the correction path) |
151
152
  | `mauto validate <file> [--strict]` | Validate a scenario JSON against the schema; warns on actions that cannot replay and on broken step/variable references (`--strict` fails) |
153
+ | `mauto run <scenario> [--junit <path>] [--update-references]` | Replay a scenario unattended and finalize its result; exits `0` passed, `6` test failed, `7` incomplete |
152
154
  | `mauto config get\|set <key> [value]` | Read / update workspace config |
153
155
  | **Agent-driven verbs** | `elements`, `tap`, `type`, `swipe`, `press`, `screenshot`, `assert`, `result`, `schema` |
154
156
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobile-automator",
3
- "version": "0.39.0",
3
+ "version": "0.40.0",
4
4
  "description": "Host-agnostic mauto CLI for AI-driven mobile QA automation — analyzes a mobile project and drives devices through mobile-mcp.",
5
5
  "homepage": "https://github.com/sh3lan93/mobile-automator#readme",
6
6
  "repository": {
@@ -37,7 +37,9 @@
37
37
  "@mobilenext/mobile-mcp": "1.0.15",
38
38
  "@modelcontextprotocol/sdk": "^1.29.0",
39
39
  "ajv": "^8.20.0",
40
- "commander": "^12.1.0"
40
+ "commander": "^12.1.0",
41
+ "pixelmatch": "5.3.0",
42
+ "pngjs": "^6.0.0"
41
43
  },
42
44
  "devDependencies": {
43
45
  "@eslint/js": "^9.39.5",
@@ -3,21 +3,35 @@
3
3
  // Pure assertion evaluator.
4
4
  //
5
5
  // The `mauto guide execute` reasoning floor splits the 27 assertion types into
6
- // two tiers:
6
+ // tiers:
7
7
  //
8
8
  // Tier 1 — element-list-powered. These are purely MECHANICAL: a deterministic
9
9
  // check over the agnostic element list (+ captured variables / a
10
- // previous snapshot). The CLI owns them.
10
+ // previous snapshot / the screen size). The CLI owns them.
11
11
  //
12
12
  // Tier 2 — screenshot + AI visual analysis. These require an agent to judge a
13
13
  // screenshot, so the CLI cannot decide pass/fail. We surface them as
14
14
  // needs_agent with pass = null.
15
15
  //
16
- // MECHANICAL_TYPES is the verbatim Tier-1 list from that guide section,
17
- // minus `element_state` whose per-attribute state
18
- // inspection is not represented in the agnostic element model (no enabled/
19
- // focused/checked fields survive normalization), so it cannot be decided
20
- // mechanically here and falls through to needs_agent.
16
+ // (screenshot_match is a third class, REFERENCE_SCREENSHOT_TYPES below.)
17
+ //
18
+ // MECHANICAL_TYPES is the Tier-1 list from that guide section. It leaves out
19
+ // `element_state`, whose per-attribute state inspection is not represented in
20
+ // the agnostic element model (no enabled/focused/checked fields survive
21
+ // normalization), and `alert_present` / `alert_text`, for which the element
22
+ // list carries no platform-agnostic signal that a dialog is up. Three types
23
+ // were promoted from Tier 2 by #276, each decided over what the element list
24
+ // does carry, and each narrower than a screenshot judgment would be:
25
+ // element_fully_visible — the bounds lie inside the screen (not clipped by
26
+ // its edges; overlap by another element is NOT
27
+ // detected);
28
+ // screen_title — an element whose text/label equals the title
29
+ // starts in the top quarter of the screen;
30
+ // permission_dialog_shown — a system allow button and a deny button are both
31
+ // on screen (+ the permission named, if given).
32
+
33
+ const { ALLOW_LABELS, DENY_LABELS, findByLabels } = require('../dialog/permission-labels');
34
+
21
35
  const MECHANICAL_TYPES = new Set([
22
36
  'element_exists',
23
37
  'element_not_exists',
@@ -34,8 +48,32 @@ const MECHANICAL_TYPES = new Set([
34
48
  'has_accessibility_label',
35
49
  'value_matches_variable',
36
50
  'text_changed',
51
+ 'element_fully_visible',
52
+ 'screen_title',
53
+ 'permission_dialog_shown',
37
54
  ]);
38
55
 
56
+ // The mechanical types that need `ctx.screenSize` ({width, height}, in the
57
+ // element bounds' coordinate space). Callers fetch the screen size only for
58
+ // these, so every other assertion costs no extra device call.
59
+ const SCREEN_SIZE_TYPES = new Set(['element_fully_visible', 'screen_title']);
60
+
61
+ // A title sits in the top quarter of the screen.
62
+ const TITLE_BAND = 0.25;
63
+
64
+ // The types the CLI decides by comparing a stable screenshot against a stored
65
+ // reference image (src/run/reference-assertion.js, #281) rather than over the
66
+ // element list. Disjoint from MECHANICAL_TYPES: `evaluate()` below stays pure
67
+ // and keeps answering needs_agent for them — the comparison needs files and a
68
+ // device, so it lives in the runner and `mauto assert`, which branch on this
69
+ // set before they reach `evaluate()`.
70
+ const REFERENCE_SCREENSHOT_TYPES = new Set(['screenshot_match']);
71
+
72
+ // Both platforms' system-dialog button labels: the assertion is agnostic, so a
73
+ // dialog counts whichever platform drew it.
74
+ const ANY_ALLOW = [...new Set(Object.values(ALLOW_LABELS).flat())];
75
+ const ANY_DENY = [...new Set(Object.values(DENY_LABELS).flat())];
76
+
39
77
  function labelOf(e) {
40
78
  return (e && typeof e.accessibility_label === 'string') ? e.accessibility_label : '';
41
79
  }
@@ -115,18 +153,32 @@ function needsAgent(type) {
115
153
  );
116
154
  }
117
155
 
156
+ // A usable {width, height} or null.
157
+ function validScreenSize(s) {
158
+ if (!s || typeof s !== 'object') return null;
159
+ const width = Number(s.width);
160
+ const height = Number(s.height);
161
+ return width > 0 && height > 0 ? { width, height } : null;
162
+ }
163
+
164
+ const sizeOf = (s) => `${s.width}x${s.height}`;
165
+
118
166
  // Evaluate a single assertion.
119
- // assertion: { type, target?, expected?, operator?, count?, pattern?, variable_name? }
120
- // ctx: { elements, variables = {}, previous = null }
167
+ // assertion: { type, target?, expected?, operator?, count?, pattern?, variable_name?, permission_name? }
168
+ // ctx: { elements, variables = {}, previous = null, screenSize = null }
121
169
  function evaluate(assertion = {}, ctx = {}) {
122
170
  const type = assertion.type;
123
171
  const elements = Array.isArray(ctx.elements) ? ctx.elements : [];
124
172
  const variables = ctx.variables || {};
125
173
  const previous = Array.isArray(ctx.previous) ? ctx.previous : null;
174
+ const screenSize = validScreenSize(ctx.screenSize);
126
175
 
127
176
  if (!MECHANICAL_TYPES.has(type)) {
128
177
  return needsAgent(type);
129
178
  }
179
+ if (SCREEN_SIZE_TYPES.has(type) && !screenSize) {
180
+ return result(type, true, false, `No screen size to check '${type}' against.`);
181
+ }
130
182
 
131
183
  const target = assertion.target;
132
184
  let pass = false;
@@ -274,6 +326,55 @@ function evaluate(assertion = {}, ctx = {}) {
274
326
  : `Element '${target}' text unchanged ('${currText}').`;
275
327
  break;
276
328
  }
329
+ case 'element_fully_visible': {
330
+ const found = findFirst(elements, target);
331
+ if (!found) {
332
+ return result(type, true, false, `No element matching '${target}' found on screen.`);
333
+ }
334
+ const [x1, y1, x2, y2] = found.bounds;
335
+ const inside = [x1, y1, x2, y2].every(Number.isFinite) &&
336
+ x1 >= 0 && y1 >= 0 && x2 <= screenSize.width && y2 <= screenSize.height;
337
+ pass = inside;
338
+ message = pass
339
+ ? `Element '${target}' at [${found.bounds}] lies inside the ${sizeOf(screenSize)} screen.`
340
+ : `Element '${target}' at [${found.bounds}] is clipped by the edges of the ${sizeOf(screenSize)} screen.`;
341
+ break;
342
+ }
343
+ case 'screen_title': {
344
+ const expected = String(assertion.expected == null ? '' : assertion.expected).trim().toLowerCase();
345
+ const band = TITLE_BAND * screenSize.height;
346
+ const carriers = elements.filter((e) =>
347
+ [e.text, e.accessibility_label].some((v) => typeof v === 'string' && v.trim().toLowerCase() === expected));
348
+ const title = carriers.find((e) => Array.isArray(e.bounds) && e.bounds[1] < band);
349
+ pass = expected !== '' && Boolean(title);
350
+ if (pass) {
351
+ message = `Screen title '${assertion.expected}' found at top edge y=${title.bounds[1]} (title band y < ${band}).`;
352
+ } else if (carriers.length) {
353
+ const tops = carriers.map((e) => (Array.isArray(e.bounds) ? e.bounds[1] : '?')).join(', ');
354
+ message = `'${assertion.expected}' is on screen but below the title band (top edge y=${tops}, band y < ${band}).`;
355
+ } else {
356
+ message = `No element carries the screen title '${assertion.expected}'.`;
357
+ }
358
+ break;
359
+ }
360
+ case 'permission_dialog_shown': {
361
+ const allow = findByLabels(elements, ANY_ALLOW);
362
+ const deny = findByLabels(elements, ANY_DENY);
363
+ const name = assertion.permission_name;
364
+ const named = name === undefined || name === null || name === ''
365
+ ? true
366
+ : elements.some((e) => matches(e, name));
367
+ pass = Boolean(allow && deny) && named;
368
+ if (pass) {
369
+ message = `Permission dialog shown ('${allow.label}' / '${deny.label}')${name ? ` for '${name}'` : ''}.`;
370
+ } else if (!allow || !deny) {
371
+ const missing = [!allow && 'an allow button', !deny && 'a deny button'].filter(Boolean).join(' and ');
372
+ message = `No permission dialog: ${missing} not found on screen.`;
373
+ } else {
374
+ message = `A permission dialog is shown, but no element names '${name}'.`;
375
+ }
376
+ break;
377
+ }
277
378
  default:
278
379
  // Should be unreachable given the MECHANICAL_TYPES guard above.
279
380
  return needsAgent(type);
@@ -282,8 +383,9 @@ function evaluate(assertion = {}, ctx = {}) {
282
383
  return result(type, true, pass, message);
283
384
  }
284
385
 
285
- // `matches`/`findFirst` are exported for the real-device E2E harness
286
- // (sample-app/scripts/lib/resolve.js), which resolves tap targets with THIS
287
- // matcher so the harness and `mauto assert` can never disagree about what a
288
- // target string means (#94).
289
- module.exports = { MECHANICAL_TYPES, evaluate, matches, findFirst };
386
+ // `matches`/`findFirst` are exported for `mauto run`'s target resolver
387
+ // (src/run/resolve.js), which resolves tap targets with THIS matcher so a run
388
+ // and `mauto assert` can never disagree about what a target string means
389
+ // (#94). The real-device E2E harness had its own copy of that resolver until
390
+ // replay R7 (#283); it now calls `mauto run`.
391
+ module.exports = { MECHANICAL_TYPES, REFERENCE_SCREENSHOT_TYPES, SCREEN_SIZE_TYPES, evaluate, matches, findFirst, textOf, compare };
@@ -0,0 +1,106 @@
1
+ 'use strict';
2
+
3
+ // The single scenario <-> evaluator <-> CLI mapping for assertions.
4
+ //
5
+ // A scenario assertion names its fields one way (`element_description`,
6
+ // `expected_value`, `expected_count`, ...), `mauto assert` takes flags
7
+ // (`--target`, `--expected`, `--count`, ...), and the pure evaluator in
8
+ // ./evaluator.js reads a third shape (`target`, `expected`, `count`, ...).
9
+ // Every path to a verdict goes through this module, so an agent driving
10
+ // `mauto assert` and `mauto run` evaluating in-process cannot build different
11
+ // evaluator inputs from the same scenario assertion.
12
+
13
+ // Which scenario field carries the expected value for each mechanical type,
14
+ // per the execute guide's assertion table. `--expected` is the one flag
15
+ // `mauto assert` reads it from.
16
+ const EXPECTED_FIELD = Object.freeze({
17
+ element_text: 'expected_value',
18
+ text_contains: 'expected_substring',
19
+ element_hint: 'expected_text',
20
+ content_description: 'label_value',
21
+ has_accessibility_label: 'label_value',
22
+ element_visible: 'expected_visible',
23
+ screen_title: 'expected_text',
24
+ });
25
+
26
+ // One row per evaluator input, in `mauto assert` argv order:
27
+ // scenario the scenario field (or a function of the assertion naming it)
28
+ // flag the `mauto assert` flag, without `--` (also commander's option key)
29
+ // input the evaluator input key
30
+ // coerce how the evaluator wants the flag's string value, when not as-is
31
+ const EXPECTED = Object.freeze({ scenario: (a) => EXPECTED_FIELD[a.type], flag: 'expected', input: 'expected' });
32
+ const FIELDS = Object.freeze([
33
+ Object.freeze({ scenario: 'element_description', flag: 'target', input: 'target' }),
34
+ EXPECTED,
35
+ Object.freeze({ scenario: 'operator', flag: 'operator', input: 'operator' }),
36
+ Object.freeze({ scenario: 'expected_count', flag: 'count', input: 'count', coerce: Number }),
37
+ Object.freeze({ scenario: 'pattern', flag: 'pattern', input: 'pattern' }),
38
+ Object.freeze({ scenario: 'variable_name', flag: 'variable', input: 'variable_name' }),
39
+ // screenshot_match (#281): the baseline and the similarity it must reach.
40
+ // The pure evaluator never reads them; the shared reference-screenshot
41
+ // decision (src/run/reference-assertion.js) reads them from this input on
42
+ // both paths (`mauto run` via toEvaluatorInput, `mauto assert` via
43
+ // fromCliFlags). `tolerance` stays the string a command line carries: the
44
+ // decision's toleranceOf parses and range-checks it, so both paths refuse
45
+ // the same values with the same message.
46
+ // `masks` is deliberately not a row: it is an array of objects in a
47
+ // scenario but a repeatable `--mask <spec>` flag, so each path passes it to
48
+ // the decision on its own (scenario `masks` / parseMaskSpec per flag).
49
+ Object.freeze({ scenario: 'reference_screenshot', flag: 'reference', input: 'reference_screenshot' }),
50
+ Object.freeze({ scenario: 'tolerance', flag: 'tolerance', input: 'tolerance' }),
51
+ // permission_dialog_shown (#276): the permission the dialog must name.
52
+ Object.freeze({ scenario: 'permission_name', flag: 'permission', input: 'permission_name' }),
53
+ ]);
54
+
55
+ // A row's value in a scenario assertion, stringified as it would reach the
56
+ // verb on a command line; undefined when the assertion does not carry it (a
57
+ // null, which the schema allows for `reference_screenshot`, carries nothing).
58
+ function scenarioValue(row, assertion) {
59
+ const key = typeof row.scenario === 'function' ? row.scenario(assertion) : row.scenario;
60
+ return key && assertion[key] !== undefined && assertion[key] !== null ? String(assertion[key]) : undefined;
61
+ }
62
+
63
+ // Scenario assertion -> the option object commander would parse from its
64
+ // `mauto assert` flags. Only provided keys.
65
+ function cliOptionsOf(assertion) {
66
+ const options = {};
67
+ for (const row of FIELDS) {
68
+ const value = scenarioValue(row, assertion);
69
+ if (value !== undefined) options[row.flag] = value;
70
+ }
71
+ return options;
72
+ }
73
+
74
+ // Scenario assertion -> `mauto assert` argv (after the type). Every value is
75
+ // stringified: that is what reaches the verb on a command line.
76
+ function toCliFlags(assertion) {
77
+ return Object.entries(cliOptionsOf(assertion)).flatMap(([flag, value]) => [`--${flag}`, value]);
78
+ }
79
+
80
+ // `mauto assert` parsed options -> evaluator input. Only provided keys.
81
+ function fromCliFlags(type, flags = {}) {
82
+ const input = { type };
83
+ for (const row of FIELDS) {
84
+ const value = flags[row.flag];
85
+ if (value !== undefined) input[row.input] = row.coerce ? row.coerce(value) : value;
86
+ }
87
+ return input;
88
+ }
89
+
90
+ // Scenario assertion -> evaluator input, defined as the CLI path composed
91
+ // (scenario -> parsed flags -> evaluator input) so the in-process path
92
+ // stringifies exactly as a command line would. `expected_visible: false`
93
+ // reaches the evaluator as the string 'false' either way.
94
+ function toEvaluatorInput(assertion) {
95
+ return fromCliFlags(assertion.type, cliOptionsOf(assertion));
96
+ }
97
+
98
+ // The expected value a result records for this assertion (the `--expected`
99
+ // of `mauto result add-assertion`), stringified; null when its type has no
100
+ // expected field or the assertion does not carry one.
101
+ function expectedOf(assertion) {
102
+ const value = scenarioValue(EXPECTED, assertion);
103
+ return value === undefined ? null : value;
104
+ }
105
+
106
+ module.exports = { EXPECTED_FIELD, FIELDS, toEvaluatorInput, fromCliFlags, toCliFlags, expectedOf };
@@ -0,0 +1,17 @@
1
+ 'use strict';
2
+
3
+ // What the evaluator reads off the device for one assertion: the element
4
+ // list, plus the screen size for only the types whose rule needs it (#276),
5
+ // so every other assertion costs no extra device call. The one place both
6
+ // `mauto assert` and `mauto run` build it; evaluator.js stays pure. A device
7
+ // error propagates — each caller decides what it means.
8
+
9
+ const { SCREEN_SIZE_TYPES } = require('./evaluator');
10
+
11
+ async function observe(bridge, type) {
12
+ const elements = await bridge.listElements();
13
+ const screenSize = SCREEN_SIZE_TYPES.has(type) ? await bridge.getScreenSize() : null;
14
+ return { elements, screenSize };
15
+ }
16
+
17
+ module.exports = { observe };