mobile-automator 0.37.0 → 0.38.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -148,7 +148,7 @@ Most low-level verbs are called **by the agent**, not by you. The ones you'll ru
148
148
  | `mauto memory show [--kind <k>] [--scenario <id>]` | Print cross-session memory (run history auto-harvested from results) as raw markdown for the agent to read at run start |
149
149
  | `mauto memory add <text> --kind <app-knowledge\|preferences>` | Record a durable agent-learned app fact or user preference (`[asserted]`, de-duped) |
150
150
  | `mauto memory forget --kind <k> --match <substr>` | Remove agent-authored memory entries matching a substring (the correction path) |
151
- | `mauto validate <file> [--strict]` | Validate a scenario JSON against the schema; warns on actions that cannot replay (`--strict` fails) |
151
+ | `mauto validate <file> [--strict]` | Validate a scenario JSON against the schema; warns on actions that cannot replay and on broken step/variable references (`--strict` fails) |
152
152
  | `mauto config get\|set <key> [value]` | Read / update workspace config |
153
153
  | **Agent-driven verbs** | `elements`, `tap`, `type`, `swipe`, `press`, `screenshot`, `assert`, `result`, `schema` |
154
154
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobile-automator",
3
- "version": "0.37.0",
3
+ "version": "0.38.2",
4
4
  "description": "Host-agnostic mauto CLI for AI-driven mobile QA automation — analyzes a mobile project and drives devices through mobile-mcp.",
5
5
  "homepage": "https://github.com/sh3lan93/mobile-automator#readme",
6
6
  "repository": {
@@ -34,7 +34,7 @@
34
34
  "test:all": "jest"
35
35
  },
36
36
  "dependencies": {
37
- "@mobilenext/mobile-mcp": "1.0.13",
37
+ "@mobilenext/mobile-mcp": "1.0.15",
38
38
  "@modelcontextprotocol/sdk": "^1.29.0",
39
39
  "ajv": "^8.20.0",
40
40
  "commander": "^12.1.0"
package/src/cli.js CHANGED
@@ -13,7 +13,7 @@ const { ScenarioValidator } = require('./scenario/validator');
13
13
  const { findReplayWarnings, summarize } = require('./scenario/replayability');
14
14
  const { evaluate, MECHANICAL_TYPES } = require('./assertion/evaluator');
15
15
  const { ResultStore } = require('./result/store');
16
- const { OBSERVATION_TYPES, parseObservation, parseCapture, parseBool } = require('./result/flags');
16
+ const { OBSERVATION_TYPES, parseObservation, parseCapture, parseBool, parseRunStatus, parseStepStatus } = require('./result/flags');
17
17
  const { MemoryStore } = require('./memory/store');
18
18
  const { FILES: MEMORY_FILES, AGENT_KINDS } = require('./memory/paths');
19
19
  const { validateEntryText, MAX_ENTRY_LEN } = require('./memory/entries');
@@ -258,7 +258,7 @@ function handleValidate({ validator }, file, opts = {}) {
258
258
  }
259
259
  const { blocking, hint } = summarize(warnings);
260
260
  if (opts.strict && blocking.length > 0) {
261
- const env = fail('invalid_input', 'scenario uses actions with no execution path', hint);
261
+ const env = fail('invalid_input', 'scenario cannot replay unattended; see data.warnings', hint);
262
262
  env.data = { valid: false, errors: [], warnings };
263
263
  return { envelope: env, exitKind: 'invalid_input' };
264
264
  }
@@ -566,8 +566,11 @@ async function handlePress({ deviceBridge }, button) {
566
566
  }
567
567
 
568
568
  // Build an assertion object from CLI flags, fetch the current element list,
569
- // and run the pure evaluator. A failed assertion is a valid RESULT (exit 0);
570
- // only an unknown assertion type is a structural (invalid_input) error.
569
+ // and run the pure evaluator. A decided failure is a test outcome, not a tool
570
+ // fault: it exits 6 (`test_failed`, #273) so CI sees red without parsing JSON,
571
+ // with the verdict still in `data`. A pass and an undecided Tier-2 verdict
572
+ // (`needs_agent`) stay ok / exit 0. Only an unknown assertion type is a
573
+ // structural (invalid_input) error.
571
574
  async function handleAssert({ deviceBridge }, type, flags = {}) {
572
575
  if (!KNOWN_ASSERTION_TYPES.has(type)) {
573
576
  return {
@@ -600,19 +603,24 @@ async function handleAssert({ deviceBridge }, type, flags = {}) {
600
603
  `Record it: mauto result add-assertion --run-id <run_id> --step-id <step_id> ` +
601
604
  `--type ${r.type} --pass ${verdict} --message "<why>"`;
602
605
 
603
- return {
604
- envelope: ok(
605
- {
606
- type: r.type,
607
- mechanical: r.mechanical,
608
- pass: r.pass,
609
- needs_agent: r.needs_agent,
610
- message: r.message,
611
- },
612
- hint
613
- ),
614
- exitKind: 'ok',
606
+ const verdictData = {
607
+ type: r.type,
608
+ mechanical: r.mechanical,
609
+ pass: r.pass,
610
+ needs_agent: r.needs_agent,
611
+ message: r.message,
615
612
  };
613
+
614
+ // Same data, same hint on both paths: an agent that reads `data` and copies
615
+ // the hint keeps working; only the exit code and `ok` change on a failure.
616
+ if (!r.needs_agent && r.pass === false) {
617
+ return {
618
+ envelope: fail('test_failed', `assertion failed: ${r.message}`, hint, verdictData),
619
+ exitKind: 'test_failed',
620
+ };
621
+ }
622
+
623
+ return { envelope: ok(verdictData, hint), exitKind: 'ok' };
616
624
  }
617
625
 
618
626
  // Collect a repeatable flag's values into an array (commander's collector).
@@ -627,6 +635,19 @@ function handleResultAddStep({ resultStoreFactory, projectRoot }, opts) {
627
635
  };
628
636
  }
629
637
 
638
+ // The status flag is a schema enum, not free text: an unknown value used to
639
+ // be silently recorded as 'error' — a mislabelled step with a green exit —
640
+ // because 'error' was the catch-all of the normalizer rather than an explicit
641
+ // acceptance (#286 review). Same all-or-nothing rule as the composite flags
642
+ // below: reject BEFORE the store is touched, so nothing is half-recorded.
643
+ const parsedStatus = parseStepStatus(status);
644
+ if (parsedStatus.error) {
645
+ return {
646
+ envelope: fail('invalid_input', parsedStatus.error, 'Valid values: pass | fail | skipped | error.'),
647
+ exitKind: 'invalid_input',
648
+ };
649
+ }
650
+
630
651
  // Parse every composite flag BEFORE touching the store: a half-recorded step
631
652
  // (screenshot persisted, observation rejected) leaves the agent unable to
632
653
  // tell which half landed, so validation is all-or-nothing.
@@ -657,7 +678,9 @@ function handleResultAddStep({ resultStoreFactory, projectRoot }, opts) {
657
678
  const store = resultStoreFactory({ runId, scenarioId, projectRoot });
658
679
  const step = store.addStep({
659
680
  step_id: stepId,
660
- status,
681
+ // The VALIDATED status: validated AND normalized at the boundary, so the
682
+ // store only ever records schema values.
683
+ status: parsedStatus.value,
661
684
  attempts: attempts === undefined ? 1 : Number(attempts),
662
685
  screenshot: screenshot === undefined ? null : screenshot,
663
686
  error_message: errorMessage === undefined ? null : errorMessage,
@@ -801,6 +824,26 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
801
824
  };
802
825
  }
803
826
 
827
+ // The status flag is a schema enum, not free text: an unlisted value (a case
828
+ // variant of a documented word, a typo) used to be persisted verbatim — a
829
+ // schema-invalid result file with ok:true and exit 0 — directly under the
830
+ // exit-6 contract keyed on that status (#286 review). Reject before anything
831
+ // is written. A provided-but-empty value is NOT an omitted one: only absence
832
+ // derives the status from the recorded assertions.
833
+ if (status !== undefined) {
834
+ const parsed = parseRunStatus(status);
835
+ if (parsed.error) {
836
+ return {
837
+ envelope: fail(
838
+ 'invalid_input',
839
+ parsed.error,
840
+ 'Valid values: passed | failed | error, or omit --status to derive it from the recorded assertions.'
841
+ ),
842
+ exitKind: 'invalid_input',
843
+ };
844
+ }
845
+ }
846
+
804
847
  // Only provided keys: `defaultMetadata` fills the rest with 'unknown', and
805
848
  // the result schema requires all five fields to be strings, so threading
806
849
  // `undefined` through would break conformance.
@@ -854,6 +897,24 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
854
897
  if (memHint) hints.push(memHint);
855
898
  }
856
899
 
900
+ // The exit code is keyed on the FINALIZED status, not the --status flag:
901
+ // with the flag omitted, the store derives 'failed' from a failed assertion,
902
+ // and the exit code must agree with the file either way (#273). 'error' takes
903
+ // the same path — a run that errored is not a pass for CI either; data.status
904
+ // tells the two apart. Everything above (write, retention, harvest) has
905
+ // already run: exit 6 is a verdict about the app, not a reason to lose the
906
+ // record of it.
907
+ if (result.status === 'failed' || result.status === 'error') {
908
+ const hint = hints.length
909
+ ? hints.join(' ')
910
+ : 'See data.steps_executed and data.assertion_results for what failed; ' +
911
+ `the result is saved to ${path.relative(projectRoot, store.file)}.`;
912
+ return {
913
+ envelope: fail('test_failed', `run ${runId} ${result.status}`, hint, result),
914
+ exitKind: 'test_failed',
915
+ };
916
+ }
917
+
857
918
  return { envelope: ok(result, hints.length ? hints.join(' ') : null), exitKind: 'ok' };
858
919
  }
859
920
 
@@ -1958,7 +2019,7 @@ function buildProgram(deps = {}) {
1958
2019
  program
1959
2020
  .command('validate <file>')
1960
2021
  .description('Validate a scenario JSON file against the scenario schema')
1961
- .option('--strict', 'fail (instead of warn) when the scenario uses actions that cannot replay unattended')
2022
+ .option('--strict', 'fail (instead of warn) when the scenario cannot replay unattended')
1962
2023
  .action(withEnvelope((file, opts) => {
1963
2024
  const r = handleValidate({ validator }, file, { strict: Boolean(opts && opts.strict) });
1964
2025
  emit(r, humanFlag());
@@ -18,7 +18,8 @@
18
18
  // 'composed' — no dedicated verb; the agent composes it from existing verbs
19
19
  // (e.g. poll `mauto elements`, repeat `mauto swipe`). The
20
20
  // execute guide documents the composition.
21
- // 'unsupported' — mobile-mcp 1.0.5 exposes no primitive for it. Not
21
+ // 'unsupported' — no `mauto` verb reaches it: the engine has no primitive, or
22
+ // (clear_app_data, since 1.0.15) mauto has not wired it. Not
22
23
  // mechanically executable; the guide must NOT promise a verb,
23
24
  // and the agent reports it honestly / handles it manually.
24
25
  //
@@ -26,6 +27,11 @@
26
27
  // method must pass — deliberately explicit because the keys are inconsistent
27
28
  // across the engine (launch=packageName, uninstall=packageName, install=path).
28
29
  //
30
+ // `requiresTarget` (every step-enum action; precondition-only actions omit it)
31
+ // says whether the step locates an element, so needs a `target` or
32
+ // `target_pattern`. `waitAction: true` marks the actions the schema's
33
+ // `wait_config` configures. targetActions()/waitActions() derive both sets.
34
+ //
29
35
  // `stepFlags` records a scenario step field that changes how the verb runs:
30
36
  // `{ field, value, flag }` means "when step[field] === value, pass flag". This
31
37
  // is the one place the mapping lives; the E2E harness derives the flags via
@@ -41,31 +47,38 @@
41
47
  const ACTION_CATALOG = {
42
48
  // --- direct verbs (engine-backed) --------------------------------------
43
49
  tap: {
50
+ requiresTarget: true,
44
51
  resolution: 'verb', verb: 'tap', bridge: 'tap',
45
52
  primitive: 'mobile_click_on_screen_at_coordinates', args: ['x', 'y'],
46
53
  },
47
54
  long_press: {
55
+ requiresTarget: true,
48
56
  resolution: 'verb', verb: 'long-press', bridge: 'longPress',
49
57
  primitive: 'mobile_long_press_on_screen_at_coordinates', args: ['x', 'y', 'duration?'],
50
58
  },
51
59
  double_tap: {
60
+ requiresTarget: true,
52
61
  resolution: 'verb', verb: 'double-tap', bridge: 'doubleTap',
53
62
  primitive: 'mobile_double_tap_on_screen', args: ['x', 'y'],
54
63
  },
55
64
  type: {
65
+ requiresTarget: false,
56
66
  resolution: 'verb', verb: 'type', bridge: 'type',
57
67
  primitive: 'mobile_type_keys', args: ['text', 'submit'],
58
68
  },
59
69
  swipe: {
70
+ requiresTarget: false,
60
71
  resolution: 'verb', verb: 'swipe', bridge: 'swipe',
61
72
  primitive: 'mobile_swipe_on_screen', args: ['direction'],
62
73
  },
63
74
  press_button: {
75
+ requiresTarget: false,
64
76
  resolution: 'verb', verb: 'press', bridge: 'pressButton',
65
77
  primitive: 'mobile_press_button', args: ['button'],
66
78
  },
67
79
  launch_app: {
68
80
  // Cold start (#255): restartApp terminates, then launches.
81
+ requiresTarget: false,
69
82
  resolution: 'verb', verb: 'launch', bridge: 'restartApp',
70
83
  primitives: ['mobile_terminate_app', 'mobile_launch_app'], args: ['packageName'],
71
84
  // `cold: false` opts out of the restart (#255).
@@ -80,6 +93,7 @@ const ACTION_CATALOG = {
80
93
  primitive: 'mobile_uninstall_app', args: ['packageName'],
81
94
  },
82
95
  open_url: {
96
+ requiresTarget: false,
83
97
  resolution: 'verb', verb: 'open-url', bridge: 'openUrl',
84
98
  primitive: 'mobile_open_url', args: ['url'],
85
99
  },
@@ -89,24 +103,35 @@ const ACTION_CATALOG = {
89
103
  },
90
104
 
91
105
  // --- semantic (mauto press <action>, platform-resolved) ----------------
92
- press_back: { resolution: 'semantic', verb: 'press' },
93
- dismiss_keyboard: { resolution: 'semantic', verb: 'press' },
94
- grant_permission: { resolution: 'semantic', verb: 'press' },
95
- deny_permission: { resolution: 'semantic', verb: 'press' },
106
+ press_back: { requiresTarget: false, resolution: 'semantic', verb: 'press' },
107
+ dismiss_keyboard: { requiresTarget: false, resolution: 'semantic', verb: 'press' },
108
+ grant_permission: { requiresTarget: false, resolution: 'semantic', verb: 'press' },
109
+ deny_permission: { requiresTarget: false, resolution: 'semantic', verb: 'press' },
96
110
 
97
111
  // --- composed (agent builds from existing verbs; no dedicated verb) -----
98
- scroll_to_element: { resolution: 'composed' },
99
- wait_for_element: { resolution: 'composed' },
100
- wait_for_element_gone: { resolution: 'composed' },
101
- wait_for_loading_complete: { resolution: 'composed' },
102
- capture_value: { resolution: 'composed' },
112
+ scroll_to_element: { requiresTarget: true, resolution: 'composed' },
113
+ wait_for_element: { requiresTarget: true, waitAction: true, resolution: 'composed' },
114
+ wait_for_element_gone: { requiresTarget: true, waitAction: true, resolution: 'composed' },
115
+ wait_for_loading_complete: { requiresTarget: false, waitAction: true, resolution: 'composed' },
116
+ capture_value: { requiresTarget: true, resolution: 'composed' },
103
117
 
104
118
  // --- unsupported (no mobile-mcp 1.0.5 primitive) -----------------------
105
- clear_app_data: { resolution: 'unsupported' },
119
+ clear_app_data: { requiresTarget: false, resolution: 'unsupported' },
106
120
  enable_wifi: { resolution: 'unsupported' },
107
121
  disable_wifi: { resolution: 'unsupported' },
108
122
  };
109
123
 
124
+ // Steps that locate an element on screen (tap, waits, scroll, capture) need a
125
+ // `target` or `target_pattern`; the scenario referential checks read this set.
126
+ function targetActions() {
127
+ return Object.keys(ACTION_CATALOG).filter((a) => ACTION_CATALOG[a].requiresTarget === true);
128
+ }
129
+
130
+ // Steps the schema's `wait_config` configures.
131
+ function waitActions() {
132
+ return Object.keys(ACTION_CATALOG).filter((a) => ACTION_CATALOG[a].waitAction === true);
133
+ }
134
+
110
135
  const RESOLUTIONS = ['verb', 'semantic', 'composed', 'unsupported'];
111
136
 
112
137
  function actionsByResolution(resolution) {
@@ -140,4 +165,5 @@ function implicitStepFlags(action, step) {
140
165
 
141
166
  module.exports = {
142
167
  ACTION_CATALOG, RESOLUTIONS, actionsByResolution, primitivesOf, stepFlagsFor, implicitStepFlags,
168
+ targetActions, waitActions,
143
169
  };
@@ -86,6 +86,8 @@ const PROBE_EXEMPT_VERBS = new Set(['devices', 'crash']);
86
86
  // Failure kinds that can plausibly mean "the app died". invalid_input is a bad
87
87
  // flag, target_not_found/environment/internal never reached (or never
88
88
  // involved) the device — none of them are worth a device round trip.
89
+ // test_failed/incomplete are outcomes: the device answered and the app failed;
90
+ // the evidence is the step screenshot, not a crash list.
89
91
  const PROBE_KINDS = new Set(['device', 'timeout']);
90
92
 
91
93
  function shouldProbe({ envelope, verb } = {}) {
@@ -54,15 +54,15 @@ Record each observation with `mauto result add-step --observation <type>:<messag
54
54
 
55
55
  1. **Name the run first.** Choose the run identifier for this execution (format `run_YYYYMMDD_HHMMSS`) and export it as `MAUTO_RUN_ID` before running any other verb — `export MAUTO_RUN_ID=run_20260905_141500`. Every verb then records into a trace for that run, and `mauto result finalize --run-id <the same id>` measures the run's duration from it instead of taking your word for it. Use the *same* id in both places; a different one means no trace to measure.
56
56
  2. Read the scenario JSON.
57
- 3. Verify a device is available with `mauto devices`. Read the device's `platform` field — store it for the duration of execution.
58
- 4. **App under test.**
57
+ 3. **Validate schema version:** read the `$schema_version` field; it must be `"2.1"`. If absent or unrecognized, report an error and halt.
58
+ 4. Validate the scenario with `mauto validate <path>` against `mauto schema scenario`. If `validate` itself fails (`ok: false`), stop and report its errors, and run no device verbs. If the result lists `data.warnings`, branch on each entry's `code` (its `path` names the step or precondition): `unsupported_action` means it uses an action that is unsupported on this run: report it as unsupported in the result (see "Not mechanically executable" under step execution) and do not substitute a gesture for it. `launch_cold_implicit` is advisory: replay that step exactly as written (a step that does not set `"cold": false` is a cold start, per the step table below) and do not substitute your own reading of what the author meant; record the warning's `message` as a `state_context` observation on that step (`mauto result add-step ... --observation state_context:<message>`). `duplicate_step_id`, `dangling_after_step`, `missing_target`, `undeclared_capture_variable` and `dangling_condition_step` mean the scenario itself is broken (a step id used twice, a reference to a step or variable that does not exist, a step with no element to resolve): run no further verbs against the device and do not guess the missing reference; finalize the run with `mauto result finalize --run-id <the run id> --scenario-id <scenario_id> --status error --summary "<each warning's path and message>"`, and ask the user to fix the scenario. `missing_wait_config` is advisory: replay that wait step with the default timeout and record the warning's `message` as a `state_context` observation on that step.
59
+ 5. Verify a device is available with `mauto devices`. Read the device's `platform` field — store it for the duration of execution.
60
+ 6. **App under test.**
59
61
  - If the user pointed you at a prebuilt app artifact, install it with `mauto install <path>`.
60
62
  - Otherwise verify the app is already installed; if it is not, halt and ask the user to install it. This guide never builds.
61
63
  - **Fresh install.** If the scenario's `preconditions.app_state` is `fresh_install` and the user pointed you at a prebuilt app artifact, uninstall the app first with `mauto uninstall <appId>` (the app identifier from `.mobile-automator/config.json` — `android_package` / `ios_bundle_id`) and only then install that artifact, so the run starts with no app data. Without an artifact, do not uninstall — you could not reinstall the app. Leave it in place and report that the app state was not reset, so a failure that depends on existing app data is not mistaken for a regression.
62
64
  - If the install fails because a different build is already on the device, uninstall it with `mauto uninstall <appId>` (the app identifier from `.mobile-automator/config.json` — `android_package` / `ios_bundle_id`), install again, and report that app data was wiped — a scenario whose preconditions assume existing app state may now fail for that reason.
63
65
  - State which path you took before continuing.
64
- 5. **Validate schema version:** read the `$schema_version` field; it must be `"2.1"`. If absent or unrecognized, report an error and halt.
65
- 6. Validate the scenario with `mauto validate <path>` against `mauto schema scenario`. If the result lists `data.warnings`, branch on each entry's `code` (its `path` names the step or precondition): `unsupported_action` means it uses an action that is unsupported on this run: report it as unsupported in the result (see "Not mechanically executable" under step execution) and do not substitute a gesture for it. `launch_cold_implicit` is advisory: replay that step exactly as written (a step that does not set `"cold": false` is a cold start, per the step table below) and do not substitute your own reading of what the author meant; record the warning's `message` as a `state_context` observation on that step (`mauto result add-step ... --observation state_context:<message>`).
66
66
 
67
67
  **Precondition `device_actions`** (from the scenario's `preconditions`, run before step replay):
68
68
 
@@ -176,7 +176,7 @@ switch the engine mid-run yourself.
176
176
 
177
177
  For each assertion, pick the type recorded in the scenario and evaluate it. The 27 types split into two tiers by **who decides pass/fail**:
178
178
 
179
- - **Tier 1 — mechanical (the CLI decides).** Deterministic checks over the on-screen element list (plus captured variables / a previous snapshot). Run `mauto elements`, then evaluate with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). You record that verdict — you do not re-judge it.
179
+ - **Tier 1 — mechanical (the CLI decides).** Deterministic checks over the on-screen element list (plus captured variables / a previous snapshot). Run `mauto elements`, then evaluate with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). A passing verdict arrives as a normal `ok: true` envelope. A failing one arrives as `ok: false` with `error.kind: "test_failed"` and exit code `6`, and the verdict (`pass: false`) is still in `data`. That is the test result, not a tool error: do not retry the verb or treat the run as broken. You record that verdict — you do not re-judge it.
180
180
  - **Tier 2 — visual / semantic (the agent decides).** These require judging pixels. `mauto assert <type> ...` reports them as `needs_agent` (with `pass: null`). For these you take a screenshot with `mauto screenshot <path>`, visually analyze it, and report pass/fail yourself with a clear justification.
181
181
 
182
182
  > One assertion type, `element_state`, names a Tier-1 intent but its per-attribute state inspection is not represented in the element model the CLI reads, so `mauto assert` returns it as `needs_agent` — judge it from the element list / screenshot like a Tier-2 check.
@@ -210,7 +210,7 @@ For each: take a screenshot, visually analyze it, and report pass/fail with a cl
210
210
 
211
211
  ### 5. Generate the Result Report
212
212
 
213
- Obtain the result schema by running `mauto schema result`, then finalize the run with `mauto result finalize` (writing to `.mobile-automator/results/<run_id>.json`), passing the run metadata you already know (`--device-model`, `--api-level`, `--app-version`, `--environment`) — the timestamp is filled in for you, and omitted fields are recorded as `unknown`. You may supply your own narrative with `--summary <text>`; omitting it keeps the generated default summary line. The finalized result assembles what the earlier `add-step`/`add-assertion` calls already recorded:
213
+ Obtain the result schema by running `mauto schema result`, then finalize the run with `mauto result finalize` (writing to `.mobile-automator/results/<run_id>.json`), passing the run metadata you already know (`--device-model`, `--api-level`, `--app-version`, `--environment`) — the timestamp is filled in for you, and omitted fields are recorded as `unknown`. You may supply your own narrative with `--summary <text>`; omitting it keeps the generated default summary line. A run that finalizes as `failed` or `error` returns `ok: false` with `error.kind: "test_failed"` and exit code `6`. The result file is still written and the full result is in `data`, so present it as usual; the non-zero exit is what lets CI see a red run. The finalized result assembles what the earlier `add-step`/`add-assertion` calls already recorded:
214
214
 
215
215
  - `run_id`, `scenario_id`, `status`, `duration_seconds`.
216
216
  - `steps_executed[]`: per-step status, retry_count, screenshot, error_message.
@@ -52,18 +52,18 @@ Record each observation with `mauto result add-step --observation <type>:<messag
52
52
  ### 1. Pre-flight
53
53
 
54
54
  - **Name the run first.** Choose the run identifier for this execution (format `run_YYYYMMDD_HHMMSS`) and export it as `MAUTO_RUN_ID` before running any other verb — `export MAUTO_RUN_ID=run_20260905_141500`. Every verb then records into a trace for that run, and `mauto result finalize --run-id <the same id>` measures the run's duration from it instead of taking your word for it. Use the *same* id in both places; a different one means no trace to measure.
55
+ - Read the JSON scenario file from `.mobile-automator/scenarios/`.
56
+ - **Validate schema version:** Read the `$schema_version` field. It must be `"2.1"`. If absent or unrecognized, report an error and halt.
57
+ - Validate the scenario with `mauto validate <path>` against `mauto schema scenario`. If `validate` itself fails (`ok: false`), stop and report its errors, and run no device verbs. If the result lists `data.warnings`, branch on each entry's `code` (its `path` names the step or precondition): `unsupported_action` means it uses an action that is unsupported on this run: report it as unsupported in the result, as described under "Not mechanically executable" below, and do not substitute a gesture for it. `launch_cold_implicit` is advisory: replay that step exactly as written (a step that does not set `"cold": false` is a cold start, per the step table below) and do not substitute your own reading of what the author meant; record the warning's `message` as a `state_context` observation on that step (`mauto result add-step ... --observation state_context:<message>`). `duplicate_step_id`, `dangling_after_step`, `missing_target`, `undeclared_capture_variable` and `dangling_condition_step` mean the scenario itself is broken (a step id used twice, a reference to a step or variable that does not exist, a step with no element to resolve): run no further verbs against the device and do not guess the missing reference; finalize the run with `mauto result finalize --run-id <the run id> --scenario-id <scenario_id> --status error --summary "<each warning's path and message>"`, and ask the user to fix the scenario. `missing_wait_config` is advisory: replay that wait step with the default timeout and record the warning's `message` as a `state_context` observation on that step.
55
58
  - Verify a device is available with `mauto devices`.
56
59
  - **App under test.** If the user pointed you at a prebuilt artifact, install it with `mauto install <path>` and do NOT build — reusing a ready artifact is the point.
57
60
  - If no artifact was named: check whether the app is already installed. If it is, say so and ask whether to rebuild before running the build command ({{build_command}}) — a previous run may have just built it. If it is not installed, build and install with the build command ({{build_command}}).
58
61
  - If that install fails because a different build of the app is already on the device, uninstall it — run `mauto uninstall` with the app package ({{app_package}}) — then install again, and report that app data was wiped — a scenario whose preconditions assume existing app state may now fail for that reason.
59
- - **Fresh install.** If the scenario's `preconditions.app_state` is `fresh_install` (read it from the scenario file now), uninstall the app first — `mauto uninstall` with the app package ({{app_package}}) — and only then install, so the run starts with no app data.
62
+ - **Fresh install.** If the scenario's `preconditions.app_state` is `fresh_install` (from the scenario you read above), uninstall the app first — `mauto uninstall` with the app package ({{app_package}}) — and only then install, so the run starts with no app data.
60
63
  - State which path you took — installed a supplied artifact, or built — before continuing.
61
64
 
62
- ### 2. Load Scenario
65
+ ### 2. Compare Environment
63
66
 
64
- - Read the JSON scenario file from `.mobile-automator/scenarios/`.
65
- - **Validate schema version:** Read the `$schema_version` field. It must be `"2.1"`. If absent or unrecognized, report an error and halt.
66
- - Validate the scenario with `mauto validate <path>` against `mauto schema scenario`. If the result lists `data.warnings`, branch on each entry's `code` (its `path` names the step or precondition): `unsupported_action` means it uses an action that is unsupported on this run: report it as unsupported in the result, as described under "Not mechanically executable" below, and do not substitute a gesture for it. `launch_cold_implicit` is advisory: replay that step exactly as written (a step that does not set `"cold": false` is a cold start, per the step table below) and do not substitute your own reading of what the author meant; record the warning's `message` as a `state_context` observation on that step (`mauto result add-step ... --observation state_context:<message>`).
67
67
  - Read the `metadata` to compare the recording environment against the current execution environment. Note any differences (different device, API level, environment).
68
68
 
69
69
  ### 3. Verify Preconditions
@@ -226,7 +226,7 @@ setting. Do not switch the engine mid-run yourself.
226
226
 
227
227
  For each assertion in the scenario (evaluated after the referenced step), pick the type recorded in the scenario and evaluate it. The 27 types split into two tiers by **who decides pass/fail**:
228
228
 
229
- - **Tier 1 — mechanical (the CLI decides).** These are deterministic checks over the on-screen element list (plus captured variables / a previous snapshot). Run `mauto elements` to get the current screen, then evaluate the assertion with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). You record that verdict — you do not re-judge it.
229
+ - **Tier 1 — mechanical (the CLI decides).** These are deterministic checks over the on-screen element list (plus captured variables / a previous snapshot). Run `mauto elements` to get the current screen, then evaluate the assertion with `mauto assert <type> ...`; the CLI returns a JSON verdict (`{ mechanical: true, pass: true|false, message }`). A passing verdict arrives as a normal `ok: true` envelope. A failing one arrives as `ok: false` with `error.kind: "test_failed"` and exit code `6`, and the verdict (`pass: false`) is still in `data`. That is the test result, not a tool error: do not retry the verb or treat the run as broken. You record that verdict — you do not re-judge it.
230
230
  - **Tier 2 — visual / semantic (the agent decides).** These require judging pixels. `mauto assert <type> ...` reports them as `needs_agent` (with `pass: null`) because the CLI cannot decide them. For these you take a screenshot with `mauto screenshot <path>`, visually analyze it, and report pass/fail yourself with a clear justification of what you observed.
231
231
 
232
232
  > One assertion type, `element_state`, names a Tier-1 intent but its per-attribute state inspection is not represented in the element model the CLI reads, so `mauto assert` returns it as `needs_agent` — judge it from the element list / screenshot like a Tier-2 check.
@@ -276,7 +276,7 @@ For all Tier 2 assertions: take a screenshot, visually analyze it, and report pa
276
276
 
277
277
  ### 6. Generate the Result Report
278
278
 
279
- Obtain the result schema by running `mauto schema result`, then finalize the run with `mauto result finalize` (writing to `.mobile-automator/results/<run_id>.json`), passing the run metadata you already know (`--device-model`, `--api-level`, `--app-version`, `--environment`) — the timestamp is filled in for you, and omitted fields are recorded as `unknown`. You may supply your own narrative with `--summary <text>`; omitting it keeps the generated default summary line. The result carries typed `observations` (`regression`, `flakiness`, `state_context`) gathered by the Observer traits above. `--duration` is now a cross-check rather than the recorded value: when a run trace exists the duration is measured from it, your figure is kept as `measurements.reported_duration_seconds`, and a disagreement beyond the tolerance is recorded as a `state_context` observation. Report the duration you believe is right and let the two be compared; do not reverse-engineer a number to match.
279
+ Obtain the result schema by running `mauto schema result`, then finalize the run with `mauto result finalize` (writing to `.mobile-automator/results/<run_id>.json`), passing the run metadata you already know (`--device-model`, `--api-level`, `--app-version`, `--environment`) — the timestamp is filled in for you, and omitted fields are recorded as `unknown`. You may supply your own narrative with `--summary <text>`; omitting it keeps the generated default summary line. The result carries typed `observations` (`regression`, `flakiness`, `state_context`) gathered by the Observer traits above. `--duration` is now a cross-check rather than the recorded value: when a run trace exists the duration is measured from it, your figure is kept as `measurements.reported_duration_seconds`, and a disagreement beyond the tolerance is recorded as a `state_context` observation. Report the duration you believe is right and let the two be compared; do not reverse-engineer a number to match. A run that finalizes as `failed` or `error` returns `ok: false` with `error.kind: "test_failed"` and exit code `6`. The result file is still written and the full result is in `data`, so present it as usual; the non-zero exit is what lets CI see a red run.
280
280
 
281
281
  ### 7. Present the Summary
282
282
 
@@ -203,7 +203,7 @@ After all steps are executed and recorded:
203
203
  2. Obtain the scenario schema by running `mauto schema scenario`, and assemble the JSON scenario to match it. Include the `tags` array.
204
204
  3. **Always include `"$schema_version": "2.1"` as the first field** in agnostic-mode scenarios.
205
205
  4. **Use named string IDs** for all steps and assertions (snake_case, e.g., `"id": "tap_login"`).
206
- 5. Validate the assembled scenario with `mauto validate <path>` against `mauto schema scenario`. If it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly (`false` when it means returning to the already-open app, `true` when it means a restart).
206
+ 5. Validate the assembled scenario with `mauto validate <path>` against `mauto schema scenario`. If it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly (`false` when it means returning to the already-open app, `true` when it means a restart). Fix every reference finding before you save, using its `path`: `duplicate_step_id` — rename the step so every id is unique; `dangling_after_step` — point the assertion's `after_step` at a step that exists; `missing_target` — give the step the `target` it acts on; `undeclared_capture_variable` — declare the variable in the root `variables` block; `dangling_condition_step` — point the `previous_step_skipped` condition's `step_id` at an earlier step; `missing_wait_config` — add a `wait_config` with an explicit `timeout_ms`.
207
207
  6. Save to `.mobile-automator/scenarios/<scenario_id>.json`.
208
208
  7. Present summary:
209
209
  > "Scenario saved: `.mobile-automator/scenarios/<scenario_id>.json`
@@ -217,7 +217,7 @@ After all steps are executed and recorded:
217
217
  - `app_version` — from the installed app
218
218
  - `environment` — ask the user or use `default_environment` from `.mobile-automator/config.json`
219
219
  - Do NOT include `device_model`, `api_level`, or `timestamp` in metadata — these belong in the result schema, not the scenario.
220
- 7. Validate the assembled scenario with `mauto validate <path>` against `mauto schema scenario`. If it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly (`false` when it means returning to the already-open app, `true` when it means a restart).
220
+ 7. Validate the assembled scenario with `mauto validate <path>` against `mauto schema scenario`. If it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly (`false` when it means returning to the already-open app, `true` when it means a restart). Fix every reference finding before you save, using its `path`: `duplicate_step_id` — rename the step so every id is unique; `dangling_after_step` — point the assertion's `after_step` at a step that exists; `missing_target` — give the step the `target` it acts on; `undeclared_capture_variable` — declare the variable in the root `variables` block; `dangling_condition_step` — point the `previous_step_skipped` condition's `step_id` at an earlier step; `missing_wait_config` — add a `wait_config` with an explicit `timeout_ms`.
221
221
  8. Save to `.mobile-automator/scenarios/<scenario_id>.json`.
222
222
  9. Present summary:
223
223
  > "Scenario saved: `.mobile-automator/scenarios/<scenario_id>.json`
@@ -269,7 +269,7 @@ Translate user language to `mauto` verbs and schema actions:
269
269
  - Create assertion step using mapped type from Pass 2b
270
270
  - Evaluate via `mauto assert <type> ...` and record with appropriate fields
271
271
  6. Combine steps and assertions into ordered scenario
272
- 7. Generate scenario JSON, then `mauto validate <path>` (if it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly)
272
+ 7. Generate scenario JSON, then `mauto validate <path>` (if it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly; fix each `duplicate_step_id`, `dangling_after_step`, `missing_target`, `undeclared_capture_variable`, `dangling_condition_step` and `missing_wait_config` entry as "Save Scenario" describes)
273
273
  ```
274
274
 
275
275
  ### Detecting and Encoding Patterns
@@ -41,7 +41,9 @@ const { runTracePath } = require('./paths');
41
41
  // `invalid_input` is excluded: nothing reached the device, so the picture would
42
42
  // show the state BEFORE the verb and be evidence of nothing. `internal` is
43
43
  // excluded for the same reason, and because a crashed CLI has a stack trace,
44
- // which is the artifact that actually helps.
44
+ // which is the artifact that actually helps. `test_failed` / `incomplete` are
45
+ // excluded too: the device answered and the app failed, and the evidence is the
46
+ // step's own screenshot — a second capture would only duplicate it.
45
47
  const CAPTURED_KINDS = new Set(['device', 'timeout']);
46
48
 
47
49
  function isCapturable(result) {
@@ -14,6 +14,14 @@ const EXIT = {
14
14
  // disk) — an environment fault the caller can fix, distinct from exit 1
15
15
  // ("the CLI itself broke"). Kept separate so agents/CI can branch on it.
16
16
  ENVIRONMENT: 5,
17
+ // Test outcomes, not faults: the device and the tooling worked, and the app
18
+ // under test failed. Distinct from 1-5 so CI can tell "the app is broken"
19
+ // from "the tooling is broken" by exit code alone, without parsing JSON.
20
+ TEST_FAILED: 6,
21
+ // Reserved for replay (`mauto run`, and `result finalize` once it computes
22
+ // it): the run is not decided yet because something still needs agent
23
+ // judgment. Nothing produces it yet.
24
+ INCOMPLETE: 7,
17
25
  CANCEL: 130,
18
26
  };
19
27
 
@@ -29,6 +37,8 @@ const KIND_TO_CODE = {
29
37
  invalid_input: EXIT.INVALID_INPUT,
30
38
  target_not_found: EXIT.TARGET_NOT_FOUND,
31
39
  environment: EXIT.ENVIRONMENT,
40
+ test_failed: EXIT.TEST_FAILED,
41
+ incomplete: EXIT.INCOMPLETE,
32
42
  cancel: EXIT.CANCEL,
33
43
  internal: 1,
34
44
  // `init --agent all` partial failure: some hosts wired, some failed. Non-zero
@@ -43,10 +43,70 @@ function parseCapture(spec) {
43
43
  return { value: { name, value } };
44
44
  }
45
45
 
46
+ // Mirrors the enum at /properties/steps_executed/items/properties/status in
47
+ // src/schemas/result_schema.json; tests/lint/result-coverage.test.js asserts
48
+ // the mapping's outputs never drift from that enum (every accepted spelling
49
+ // lands on a schema value, every schema value is reachable). The CLI accepts
50
+ // the friendly spellings an agent types (pass/fail/skip) alongside the
51
+ // schema's own values; anything else is rejected instead of being silently
52
+ // recorded as 'error' (#286 review).
53
+ const STEP_STATUS_ALIASES = {
54
+ pass: 'passed',
55
+ passed: 'passed',
56
+ fail: 'failed',
57
+ failed: 'failed',
58
+ skipped: 'skipped',
59
+ skip: 'skipped',
60
+ error: 'error',
61
+ };
62
+
63
+ // The throwing form, for the store: addStep's input is already validated at
64
+ // the CLI boundary (parseStepStatus), so reaching this with an unknown value
65
+ // means a CLI bug — it escapes as `internal` (exit 1), never as a
66
+ // mislabelled-but-green step.
67
+ function stepStatus(raw) {
68
+ const normalized = STEP_STATUS_ALIASES[raw];
69
+ if (!normalized) throw new Error(`unknown step status "${raw}"`);
70
+ return normalized;
71
+ }
72
+
73
+ function parseStepStatus(raw) {
74
+ const normalized = STEP_STATUS_ALIASES[raw];
75
+ if (!normalized) {
76
+ return { error: `unknown --status "${raw}" (expected pass | fail | skipped | error)` };
77
+ }
78
+ return { value: normalized };
79
+ }
80
+
46
81
  function parseBool(raw, flagName) {
47
82
  if (raw === 'true') return { value: true };
48
83
  if (raw === 'false') return { value: false };
49
84
  return { error: `${flagName} must be "true" or "false" (got "${raw}")` };
50
85
  }
51
86
 
52
- module.exports = { OBSERVATION_TYPES, parseObservation, parseCapture, parseBool };
87
+ // Mirrors the enum at /properties/status in src/schemas/result_schema.json;
88
+ // tests/lint/result-coverage.test.js asserts the two never drift. `result
89
+ // finalize --status` must be one of these or omitted (omission derives the
90
+ // status from the recorded assertions): an unlisted value used to be persisted
91
+ // verbatim — a schema-invalid result file with a green exit — directly under
92
+ // the exit-6 contract keyed on that status (#286 review).
93
+ const RUN_STATUSES = ['passed', 'failed', 'error'];
94
+
95
+ function parseRunStatus(raw) {
96
+ if (!RUN_STATUSES.includes(raw)) {
97
+ return { error: `unknown --status "${raw}" (expected ${RUN_STATUSES.join(' | ')})` };
98
+ }
99
+ return { value: raw };
100
+ }
101
+
102
+ module.exports = {
103
+ OBSERVATION_TYPES,
104
+ parseObservation,
105
+ parseCapture,
106
+ parseBool,
107
+ RUN_STATUSES,
108
+ parseRunStatus,
109
+ STEP_STATUS_ALIASES,
110
+ stepStatus,
111
+ parseStepStatus,
112
+ };
@@ -5,7 +5,7 @@ const path = require('path');
5
5
 
6
6
  const { atomicWrite } = require('../util/atomic');
7
7
  const { withLock } = require('../util/lock');
8
- const { OBSERVATION_TYPES } = require('./flags');
8
+ const { OBSERVATION_TYPES, stepStatus } = require('./flags');
9
9
  const { workspaceDir } = require('../workspace/layout');
10
10
 
11
11
  // Incremental result accumulator that persists to
@@ -33,14 +33,6 @@ function defaultMetadata(overrides = {}) {
33
33
  };
34
34
  }
35
35
 
36
- // Normalize the caller-friendly status ('pass'/'fail') to the schema enums.
37
- function stepStatus(status) {
38
- if (status === 'pass' || status === 'passed') return 'passed';
39
- if (status === 'fail' || status === 'failed') return 'failed';
40
- if (status === 'skipped' || status === 'skip') return 'skipped';
41
- return 'error';
42
- }
43
-
44
36
  function assertionStatus(pass) {
45
37
  return pass ? 'passed' : 'failed';
46
38
  }
@@ -70,6 +62,12 @@ class ResultStore {
70
62
  this._refreshFromDisk();
71
63
  }
72
64
 
65
+ // The result file this store reads and writes. Read-only: the store owns the
66
+ // location, so callers name the file through it rather than rebuilding it.
67
+ get file() {
68
+ return this._file;
69
+ }
70
+
73
71
  // Re-load the on-disk accumulator and refresh the in-memory fields. The
74
72
  // constructor eager-loads once (read-only paths and back-compat), but every
75
73
  // persisting mutator re-runs this UNDER the per-runId lock so it operates on
@@ -0,0 +1,160 @@
1
+ 'use strict';
2
+
3
+ const { targetActions, waitActions } = require('../device/action-catalog');
4
+
5
+ // Referential checks for scenarios (#272). The schema types every reference as
6
+ // a bare string, so a scenario whose ids, targets and variables do not line up
7
+ // still validates -- and the gap surfaces mid-run, on a device. This module
8
+ // reports those gaps as finding descriptors -- `{ code, fields }`, plus `path`
9
+ // and `ids` for assertions -- and `replayability.js` turns them into warnings
10
+ // with its one constructor, during its one walk over the steps, so `validate`
11
+ // sees one finding list in one shape.
12
+ //
13
+ // Every code but `missing_wait_config` is blocking: the step or assertion has
14
+ // nothing to resolve against, so it cannot replay unattended. A wait without
15
+ // `wait_config` still runs on the default timeout, so that one is advisory.
16
+
17
+ // Which step actions need a target, and which take `wait_config`, are catalog
18
+ // facts (src/device/action-catalog.js), not restated here.
19
+ const TARGET_ACTIONS = new Set(targetActions());
20
+ const WAIT_ACTIONS = new Set(waitActions());
21
+
22
+ // A condition or assertion of these types reads a variable, so it must name one.
23
+ const CONDITION_TYPES_NEEDING_VARIABLE = new Set(['variable_value']);
24
+ const ASSERTION_TYPES_NEEDING_VARIABLE = new Set(['value_matches_variable']);
25
+
26
+ // Finding fields for a variable name that is undeclared, or absent where its
27
+ // type requires one (an empty name counts as absent there); null when fine.
28
+ function variableFields(field, name, required, index) {
29
+ const ref = required && name === '' ? undefined : name;
30
+ if (ref === undefined ? !required : index.variables.has(ref)) return null;
31
+ return { field, ref };
32
+ }
33
+
34
+ const REFERENCE_CODES = {
35
+ duplicate_step_id: {
36
+ blocking: true,
37
+ clause: (n) => `${n} step id(s) are declared more than once`,
38
+ message: (f) => `step id "${f.ref}" is already used by ${f.firstPath}; `
39
+ + 'step ids must be unique so assertions and conditions can name one step',
40
+ },
41
+ dangling_after_step: {
42
+ blocking: true,
43
+ clause: (n) => `${n} assertion(s) run after a step that does not exist`,
44
+ message: (f) => (f.ref === undefined
45
+ ? 'assertion has no "after_step"; name the step it runs after'
46
+ : `assertion runs after step "${f.ref}", which no step declares`),
47
+ },
48
+ missing_target: {
49
+ blocking: true,
50
+ clause: (n) => `${n} step(s) have no target to resolve`,
51
+ message: (f) => `${f.action} has no "target" (or "target_pattern"), so there is no element to resolve`,
52
+ },
53
+ undeclared_capture_variable: {
54
+ blocking: true,
55
+ clause: (n) => `${n} variable reference(s) are not declared in "variables"`,
56
+ message: (f) => (f.ref === undefined
57
+ ? `"${f.field}" is required here but has no variable name`
58
+ : `"${f.field}" names variable "${f.ref}", which the root "variables" block does not declare`),
59
+ },
60
+ dangling_condition_step: {
61
+ blocking: true,
62
+ clause: (n) => `${n} condition(s) refer to a step that has not run`,
63
+ message: (f) => (f.ref === undefined
64
+ ? 'previous_step_skipped condition has no "step_id"; name an earlier step'
65
+ : `previous_step_skipped condition names step "${f.ref}", which is not an earlier step`),
66
+ },
67
+ missing_wait_config: {
68
+ blocking: false,
69
+ clause: (n) => `${n} wait step(s) have no wait_config; the default timeout applies`,
70
+ message: (f) => `${f.action} has no "wait_config", so it waits for the default timeout; `
71
+ + 'set "wait_config" to make the wait explicit',
72
+ },
73
+ };
74
+
75
+ const isNonEmptyString = (v) => typeof v === 'string' && v.length > 0;
76
+
77
+ // Indexes the flattened step list (see `steps.js`): `after_step` and duplicate
78
+ // detection need every step, and a condition's step must come earlier, i.e.
79
+ // at a lower position in `entries`.
80
+ function referenceIndex(scenario, entries) {
81
+ const firstPathById = new Map();
82
+ const orderById = new Map();
83
+ entries.forEach(({ step, path }, position) => {
84
+ if (typeof step.id === 'string' && !firstPathById.has(step.id)) {
85
+ firstPathById.set(step.id, path);
86
+ orderById.set(step.id, position);
87
+ }
88
+ });
89
+
90
+ const vars = scenario && scenario.variables;
91
+ const variables = new Set(vars && typeof vars === 'object' && !Array.isArray(vars) ? Object.keys(vars) : []);
92
+ return { firstPathById, orderById, variables };
93
+ }
94
+
95
+ // Descriptors for one step; the caller supplies its path, action and id.
96
+ // `position` is its index in the flattened step list, the same numbering
97
+ // `orderById` uses.
98
+ function stepReferences(step, position, index) {
99
+ const findings = [];
100
+ const add = (code, fields) => findings.push({ code, fields });
101
+
102
+ const firstPath = index.firstPathById.get(step.id);
103
+ if (firstPath !== undefined && index.orderById.get(step.id) !== position) {
104
+ add('duplicate_step_id', { ref: step.id, firstPath });
105
+ }
106
+
107
+ if (TARGET_ACTIONS.has(step.action) && !isNonEmptyString(step.target) && !isNonEmptyString(step.target_pattern)) {
108
+ add('missing_target', {});
109
+ }
110
+
111
+ if (WAIT_ACTIONS.has(step.action) && !step.wait_config) {
112
+ add('missing_wait_config', {});
113
+ }
114
+
115
+ if (step.capture_to !== undefined && !index.variables.has(step.capture_to)) {
116
+ add('undeclared_capture_variable', { field: 'capture_to', ref: step.capture_to });
117
+ }
118
+
119
+ const cond = step.condition;
120
+ if (cond && typeof cond === 'object') {
121
+ const fields = variableFields('condition.variable_name', cond.variable_name,
122
+ CONDITION_TYPES_NEEDING_VARIABLE.has(cond.type), index);
123
+ if (fields) add('undeclared_capture_variable', fields);
124
+ if (cond.type === 'previous_step_skipped') {
125
+ const order = index.orderById.get(cond.step_id);
126
+ if (order === undefined || order >= position) {
127
+ add('dangling_condition_step', { ref: cond.step_id });
128
+ }
129
+ }
130
+ }
131
+
132
+ return findings;
133
+ }
134
+
135
+ // Descriptors for the root `assertions` array, in order, each with its path
136
+ // and `assertion_id`.
137
+ function assertionReferences(scenario, index) {
138
+ const assertions = scenario && scenario.assertions;
139
+ if (!Array.isArray(assertions)) return [];
140
+ const findings = [];
141
+ assertions.forEach((a, i) => {
142
+ if (!a) return;
143
+ const path = `assertions[${i}]`;
144
+ const ids = a.id !== undefined ? { assertion_id: a.id } : {};
145
+ if (!index.firstPathById.has(a.after_step)) {
146
+ findings.push({ code: 'dangling_after_step', fields: { ref: a.after_step }, path, ids });
147
+ }
148
+ const fields = variableFields('variable_name', a.variable_name,
149
+ ASSERTION_TYPES_NEEDING_VARIABLE.has(a.type), index);
150
+ if (fields) findings.push({ code: 'undeclared_capture_variable', fields, path, ids });
151
+ });
152
+ return findings;
153
+ }
154
+
155
+ module.exports = {
156
+ REFERENCE_CODES,
157
+ referenceIndex,
158
+ stepReferences,
159
+ assertionReferences,
160
+ };
@@ -1,27 +1,41 @@
1
1
  'use strict';
2
2
 
3
3
  // Replayability check for scenarios. Schema validity alone overstates what a
4
- // scenario will do on replay; this module reports two kinds of gap:
4
+ // scenario will do on replay; this module reports three kinds of gap:
5
5
  //
6
6
  // - `unsupported_action` (#166): the schema accepts every action in its enum,
7
- // but the action catalog marks some `unsupported`. mobile-mcp exposes no
8
- // primitive for them, so no `mauto` verb can run them and the step can never
7
+ // but the action catalog marks some `unsupported`. no `mauto` verb reaches
8
+ // them, whether or not the engine has a primitive, so the step can never
9
9
  // replay unattended. Blocking: `mauto validate --strict` fails on it.
10
10
  // - `launch_cold_implicit` (#255): a catalog `stepFlags` entry marked
11
11
  // `warnWhenUnset`. A step after the first that leaves the marked field to its
12
12
  // schema default is ambiguous about the author's intent. Advisory: it never
13
13
  // fails `--strict`.
14
+ // - Referential gaps (#272): duplicate step ids, dangling references, steps
15
+ // with no target, undeclared variables, waits with no `wait_config`. The
16
+ // `references` module owns those codes; this one merges its findings into
17
+ // the single walk over `flattenSteps`, so `CODES` stays the one table
18
+ // every consumer reads.
14
19
  //
15
20
  // The unsupported set is derived from the catalog, never restated here, so it
16
21
  // moves when an action gains (or loses) a device primitive.
17
22
 
18
23
  const { actionsByResolution, implicitStepFlags } = require('../device/action-catalog');
24
+ const {
25
+ REFERENCE_CODES,
26
+ referenceIndex,
27
+ stepReferences,
28
+ assertionReferences,
29
+ } = require('./references');
30
+ const { flattenSteps } = require('./steps');
19
31
 
20
- // Severity and wording for every finding code live here and nowhere else.
21
- // `blocking` is the `--strict` policy: only a step that cannot run at all fails
22
- // validation. `clause(n)` is the hint fragment for n findings of the code;
23
- // `message(f)` is the per-finding text. Advisory codes other than
24
- // `unsupported_action` are the catalog's `warnWhenUnset` markers.
32
+ // Severity and wording for every finding code: the replay codes are defined
33
+ // here, the referential ones in `references.js`, and `CODES` assembles both, so
34
+ // it is the one table every consumer reads.
35
+ // `blocking` is the `--strict` policy: a blocking finding means the scenario
36
+ // cannot replay unattended as written: a step that cannot run or a broken
37
+ // reference. `clause(n)` is the hint fragment for n findings of the code;
38
+ // `message(f)` is the per-finding text.
25
39
  const CODES = {
26
40
  unsupported_action: {
27
41
  blocking: true,
@@ -35,61 +49,55 @@ const CODES = {
35
49
  + `set "${f.field}" explicitly (${JSON.stringify(f.value)} to return to the running app, `
36
50
  + `${JSON.stringify(!f.value)} to confirm the restart)`,
37
51
  },
52
+ ...REFERENCE_CODES,
38
53
  };
39
54
 
40
- function makeFinding(code, fields, path, stepId) {
41
- const f = { code, action: fields.action, path, message: CODES[code].message(fields) };
42
- if (stepId !== undefined) f.step_id = stepId;
43
- return f;
55
+ // The one constructor for a `data.warnings` entry: `{ code, action, path,
56
+ // message }` plus `step_id` or `assertion_id` from `ids`. `action` is omitted
57
+ // when the finding has none (an assertion).
58
+ function makeFinding(code, fields, path, ids = {}) {
59
+ const f = { code };
60
+ if (fields.action !== undefined) f.action = fields.action;
61
+ f.path = path;
62
+ f.message = CODES[code].message(fields);
63
+ return Object.assign(f, ids);
44
64
  }
45
65
 
46
- function unsupportedFinding(action, path, stepId) {
47
- return makeFinding('unsupported_action', { action }, path, stepId);
48
- }
66
+ const stepIds = (step) => (step.id !== undefined ? { step_id: step.id } : {});
49
67
 
50
- // One advisory per catalog marker: the step leaves `sf.field` unset, so the
51
- // field's default decides. The catalog owns action/field/value.
52
- function implicitFinding(action, sf, path, stepId) {
53
- return makeFinding(sf.warnWhenUnset, { action, field: sf.field, value: sf.value }, path, stepId);
54
- }
68
+ // Returns every finding in document order: steps first (including nested
69
+ // sub_steps, depth-first), then precondition device actions, then assertions.
70
+ function findReplayWarnings(scenario) {
71
+ const unsupported = new Set(actionsByResolution('unsupported'));
72
+ const entries = flattenSteps(scenario);
73
+ const index = referenceIndex(scenario, entries);
74
+ const findings = [];
55
75
 
56
- // Walks a step list depth-first in document order, descending into each
57
- // step's `sub_steps` (the schema types them as steps, so they nest freely).
58
- function scanSteps(steps, basePath, unsupported, findings, topLevel) {
59
- if (!Array.isArray(steps)) return;
60
- steps.forEach((step, i) => {
61
- if (!step) return;
62
- const path = `${basePath}[${i}]`;
63
- if (unsupported.has(step.action)) {
64
- findings.push(unsupportedFinding(step.action, path, step.id));
65
- }
76
+ entries.forEach(({ step, path, isFirstTopLevel }, position) => {
77
+ const ids = stepIds(step);
78
+ const add = (code, fields) => findings.push(makeFinding(code, { action: step.action, ...fields }, path, ids));
79
+ if (unsupported.has(step.action)) add('unsupported_action', {});
66
80
  // Only the scenario's first top-level step is unambiguously a fresh
67
81
  // start; any other step that leaves a marked field unset is worth a look.
68
- if (!(topLevel && i === 0)) {
82
+ // The catalog owns the marker's code, field and default value.
83
+ if (!isFirstTopLevel) {
69
84
  for (const sf of implicitStepFlags(step.action, step)) {
70
- findings.push(implicitFinding(step.action, sf, path, step.id));
85
+ add(sf.warnWhenUnset, { field: sf.field, value: sf.value });
71
86
  }
72
87
  }
73
- scanSteps(step.sub_steps, `${path}.sub_steps`, unsupported, findings, false);
88
+ for (const r of stepReferences(step, position, index)) add(r.code, r.fields);
74
89
  });
75
- }
76
-
77
- // Returns every finding in document order: steps first (including nested
78
- // sub_steps, depth-first), then precondition device actions.
79
- function findReplayWarnings(scenario) {
80
- const unsupported = new Set(actionsByResolution('unsupported'));
81
- const findings = [];
82
-
83
- scanSteps(scenario && scenario.steps, 'steps', unsupported, findings, true);
84
90
 
85
91
  const pre = scenario && scenario.preconditions;
86
92
  const deviceActions = pre && Array.isArray(pre.device_actions) ? pre.device_actions : [];
87
93
  deviceActions.forEach((da, i) => {
88
94
  if (da && unsupported.has(da.action)) {
89
- findings.push(unsupportedFinding(da.action, `preconditions.device_actions[${i}]`));
95
+ findings.push(makeFinding('unsupported_action', { action: da.action }, `preconditions.device_actions[${i}]`));
90
96
  }
91
97
  });
92
98
 
99
+ for (const r of assertionReferences(scenario, index)) findings.push(makeFinding(r.code, r.fields, r.path, r.ids));
100
+
93
101
  return findings;
94
102
  }
95
103
 
@@ -0,0 +1,27 @@
1
+ 'use strict';
2
+
3
+ // The one walk over a scenario's step tree. Steps nest freely through
4
+ // `sub_steps` (the schema types them as steps), and every check that cares
5
+ // about step order reads it from this list: an entry's index is its position
6
+ // in depth-first document order, so "earlier step" means a lower index.
7
+
8
+ // Returns `[{ step, path, isFirstTopLevel }]`, parents before their sub_steps.
9
+ // Null steps and non-array step lists are skipped; a skipped step keeps its
10
+ // slot in the paths of its siblings. Only the scenario's first top-level step
11
+ // is `isFirstTopLevel`.
12
+ function flattenSteps(scenario) {
13
+ const entries = [];
14
+ const walk = (steps, basePath, topLevel) => {
15
+ if (!Array.isArray(steps)) return;
16
+ steps.forEach((step, i) => {
17
+ if (!step) return;
18
+ const path = `${basePath}[${i}]`;
19
+ entries.push({ step, path, isFirstTopLevel: topLevel && i === 0 });
20
+ walk(step.sub_steps, `${path}.sub_steps`, false);
21
+ });
22
+ };
23
+ walk(scenario && scenario.steps, 'steps', true);
24
+ return entries;
25
+ }
26
+
27
+ module.exports = { flattenSteps };
@@ -140,7 +140,7 @@
140
140
  "disable_wifi",
141
141
  "set_orientation"
142
142
  ],
143
- "description": "The setup action to perform. enable_wifi, disable_wifi and clear_app_data are accepted by the schema but are not mechanically executable (no device primitive), so they cannot replay unattended; mauto validate warns."
143
+ "description": "The setup action to perform. enable_wifi, disable_wifi and clear_app_data are accepted by the schema but are not mechanically executable (no mauto verb runs them), so they cannot replay unattended; mauto validate warns."
144
144
  },
145
145
  "target_package": {
146
146
  "type": "string",
@@ -247,7 +247,7 @@
247
247
  "grant_permission",
248
248
  "deny_permission"
249
249
  ],
250
- "description": "The automation action to perform. Supported types: wait_for_element, wait_for_element_gone, wait_for_loading_complete, capture_value, long_press, double_tap, scroll_to_element, press_back, dismiss_keyboard, grant_permission, deny_permission. clear_app_data is accepted by the schema but is not mechanically executable: mobile-mcp exposes no primitive for it, so it cannot replay unattended (mauto validate warns)."
250
+ "description": "The automation action to perform. Supported types: wait_for_element, wait_for_element_gone, wait_for_loading_complete, capture_value, long_press, double_tap, scroll_to_element, press_back, dismiss_keyboard, grant_permission, deny_permission. clear_app_data is accepted by the schema but is not mechanically executable: no mauto verb runs it, so it cannot replay unattended (mauto validate warns)."
251
251
  },
252
252
  "description": {
253
253
  "type": "string",