mobile-automator 0.38.1 → 0.38.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -148,7 +148,7 @@ Most low-level verbs are called **by the agent**, not by you. The ones you'll ru
148
148
  | `mauto memory show [--kind <k>] [--scenario <id>]` | Print cross-session memory (run history auto-harvested from results) as raw markdown for the agent to read at run start |
149
149
  | `mauto memory add <text> --kind <app-knowledge\|preferences>` | Record a durable agent-learned app fact or user preference (`[asserted]`, de-duped) |
150
150
  | `mauto memory forget --kind <k> --match <substr>` | Remove agent-authored memory entries matching a substring (the correction path) |
151
- | `mauto validate <file> [--strict]` | Validate a scenario JSON against the schema; warns on actions that cannot replay (`--strict` fails) |
151
+ | `mauto validate <file> [--strict]` | Validate a scenario JSON against the schema; warns on actions that cannot replay and on broken step/variable references (`--strict` fails) |
152
152
  | `mauto config get\|set <key> [value]` | Read / update workspace config |
153
153
  | **Agent-driven verbs** | `elements`, `tap`, `type`, `swipe`, `press`, `screenshot`, `assert`, `result`, `schema` |
154
154
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mobile-automator",
3
- "version": "0.38.1",
3
+ "version": "0.38.4",
4
4
  "description": "Host-agnostic mauto CLI for AI-driven mobile QA automation — analyzes a mobile project and drives devices through mobile-mcp.",
5
5
  "homepage": "https://github.com/sh3lan93/mobile-automator#readme",
6
6
  "repository": {
package/src/cli.js CHANGED
@@ -12,8 +12,10 @@ const { engineHint } = require('./device/engine-hints');
12
12
  const { ScenarioValidator } = require('./scenario/validator');
13
13
  const { findReplayWarnings, summarize } = require('./scenario/replayability');
14
14
  const { evaluate, MECHANICAL_TYPES } = require('./assertion/evaluator');
15
- const { ResultStore } = require('./result/store');
15
+ const { ResultStore, readFinalizedResult } = require('./result/store');
16
16
  const { OBSERVATION_TYPES, parseObservation, parseCapture, parseBool, parseRunStatus, parseStepStatus } = require('./result/flags');
17
+ const { toJUnit } = require('./result/junit');
18
+ const { atomicWrite } = require('./util/atomic');
17
19
  const { MemoryStore } = require('./memory/store');
18
20
  const { FILES: MEMORY_FILES, AGENT_KINDS } = require('./memory/paths');
19
21
  const { validateEntryText, MAX_ENTRY_LEN } = require('./memory/entries');
@@ -258,7 +260,7 @@ function handleValidate({ validator }, file, opts = {}) {
258
260
  }
259
261
  const { blocking, hint } = summarize(warnings);
260
262
  if (opts.strict && blocking.length > 0) {
261
- const env = fail('invalid_input', 'scenario uses actions with no execution path', hint);
263
+ const env = fail('invalid_input', 'scenario cannot replay unattended; see data.warnings', hint);
262
264
  env.data = { valid: false, errors: [], warnings };
263
265
  return { envelope: env, exitKind: 'invalid_input' };
264
266
  }
@@ -829,7 +831,7 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
829
831
  // schema-invalid result file with ok:true and exit 0 — directly under the
830
832
  // exit-6 contract keyed on that status (#286 review). Reject before anything
831
833
  // is written. A provided-but-empty value is NOT an omitted one: only absence
832
- // derives the status from the recorded assertions.
834
+ // derives the status from the recorded steps and assertions.
833
835
  if (status !== undefined) {
834
836
  const parsed = parseRunStatus(status);
835
837
  if (parsed.error) {
@@ -837,7 +839,7 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
837
839
  envelope: fail(
838
840
  'invalid_input',
839
841
  parsed.error,
840
- 'Valid values: passed | failed | error, or omit --status to derive it from the recorded assertions.'
842
+ 'Valid values: passed | failed | error, or omit --status to derive it from the recorded steps and assertions.'
841
843
  ),
842
844
  exitKind: 'invalid_input',
843
845
  };
@@ -898,10 +900,11 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
898
900
  }
899
901
 
900
902
  // The exit code is keyed on the FINALIZED status, not the --status flag:
901
- // with the flag omitted, the store derives 'failed' from a failed assertion,
902
- // and the exit code must agree with the file either way (#273). 'error' takes
903
- // the same path — a run that errored is not a pass for CI either; data.status
904
- // tells the two apart. Everything above (write, retention, harvest) has
903
+ // with the flag omitted, the store derives it from the recorded steps and
904
+ // assertions ('error' for an errored step, else 'failed' for a failed step or
905
+ // assertion, #296), and the exit code must agree with the file either way
906
+ // (#273). 'error' takes the same path — a run that errored is not a pass for
907
+ // CI either; data.status tells the two apart. Everything above (write, retention, harvest) has
905
908
  // already run: exit 6 is a verdict about the app, not a reason to lose the
906
909
  // record of it.
907
910
  if (result.status === 'failed' || result.status === 'error') {
@@ -918,6 +921,58 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
918
921
  return { envelope: ok(result, hints.length ? hints.join(' ') : null), exitKind: 'ok' };
919
922
  }
920
923
 
924
+ // Formats `mauto result export --format` accepts. The only one, junit, is
925
+ // rendered by toJUnit (src/result/junit.js).
926
+ const REPORT_FORMATS = Object.freeze(['junit']);
927
+
928
+ // `mauto result export` (#274): render finalized result files as a CI report.
929
+ // Read-only by contract — tests/lint/result-coverage.test.js exempts it from
930
+ // the capability catalog on that basis and checks the results directory is
931
+ // unchanged afterwards. Reading (and that read-only guarantee) is
932
+ // readFinalizedResult's job in src/result/store.js.
933
+ //
934
+ // The export succeeds (exit 0) whatever the runs' outcome: it is a report, not
935
+ // a gate. A single unusable run id fails the whole export, so a CI job never
936
+ // uploads a report silently missing a run.
937
+ function handleResultExport({ projectRoot, cwd = process.cwd() }, opts = {}) {
938
+ const invalid = (message, hint) => ({ envelope: fail('invalid_input', message, hint), exitKind: 'invalid_input' });
939
+ const formatsHint = `Formats: ${REPORT_FORMATS.join(', ')}.`;
940
+
941
+ if (!opts.format) return invalid('--format is required', formatsHint);
942
+ if (!REPORT_FORMATS.includes(opts.format)) return invalid(`unknown report format "${opts.format}"`, formatsHint);
943
+
944
+ const runIds = opts.runIds || [];
945
+ if (runIds.length === 0) return invalid('at least one --run-id is required', 'Pass --run-id <id>; repeat it to export several runs.');
946
+
947
+ const results = [];
948
+ for (const runId of runIds) {
949
+ const { result, error } = readFinalizedResult(projectRoot, runId);
950
+ if (error) return { envelope: fail(error.kind, error.message, error.hint), exitKind: error.kind };
951
+ results.push(result);
952
+ }
953
+
954
+ const { xml, ...counts } = toJUnit(results);
955
+
956
+ if (!opts.out) {
957
+ return { envelope: ok({ format: opts.format, path: null, ...counts, xml }), exitKind: 'ok' };
958
+ }
959
+
960
+ const outPath = path.resolve(cwd, opts.out);
961
+ try {
962
+ atomicWrite(outPath, xml);
963
+ } catch (err) {
964
+ return {
965
+ envelope: fail(
966
+ 'environment',
967
+ `could not write the report to ${outPath}: ${err.message}`,
968
+ 'Check the directory exists and is writable.'
969
+ ),
970
+ exitKind: 'environment',
971
+ };
972
+ }
973
+ return { envelope: ok({ format: opts.format, path: outPath, ...counts }), exitKind: 'ok' };
974
+ }
975
+
921
976
  // Collapse any recovery warnings a ResultStore accumulated (e.g. a corrupt
922
977
  // prior file preserved as a sidecar) into a single envelope hint, or null.
923
978
  function storeHint(store) {
@@ -2019,7 +2074,7 @@ function buildProgram(deps = {}) {
2019
2074
  program
2020
2075
  .command('validate <file>')
2021
2076
  .description('Validate a scenario JSON file against the scenario schema')
2022
- .option('--strict', 'fail (instead of warn) when the scenario uses actions that cannot replay unattended')
2077
+ .option('--strict', 'fail (instead of warn) when the scenario cannot replay unattended')
2023
2078
  .action(withEnvelope((file, opts) => {
2024
2079
  const r = handleValidate({ validator }, file, { strict: Boolean(opts && opts.strict) });
2025
2080
  emit(r, humanFlag());
@@ -2258,6 +2313,20 @@ function buildProgram(deps = {}) {
2258
2313
  emit(r, humanFlag());
2259
2314
  })));
2260
2315
 
2316
+ result
2317
+ .command('export')
2318
+ .description('Render finalized result files as a CI report (read-only)')
2319
+ .requiredOption('--run-id <id>', 'run to export; repeatable', collect, [])
2320
+ .requiredOption('--format <f>', REPORT_FORMATS.join(' | '))
2321
+ .option('--out <path>', 'write the report here instead of returning it in data.xml')
2322
+ .action(withEnvelope(requireWorkspace((opts) => {
2323
+ const r = handleResultExport(
2324
+ { projectRoot, cwd },
2325
+ { runIds: opts.runId, format: opts.format, out: opts.out }
2326
+ );
2327
+ emit(r, humanFlag());
2328
+ })));
2329
+
2261
2330
  // --- Slice 3 verbs --------------------------------------------------------
2262
2331
 
2263
2332
  // Emit a handler result that may be either an envelope (success/error) OR
@@ -2726,6 +2795,7 @@ module.exports = {
2726
2795
  handleResultAddAssertion,
2727
2796
  handleResultAddCrash,
2728
2797
  handleResultFinalize,
2798
+ handleResultExport,
2729
2799
  handleSetup,
2730
2800
  handleConfigGet,
2731
2801
  handleConfigSet,
@@ -27,6 +27,11 @@
27
27
  // method must pass — deliberately explicit because the keys are inconsistent
28
28
  // across the engine (launch=packageName, uninstall=packageName, install=path).
29
29
  //
30
+ // `requiresTarget` (every step-enum action; precondition-only actions omit it)
31
+ // says whether the step locates an element, so needs a `target` or
32
+ // `target_pattern`. `waitAction: true` marks the actions the schema's
33
+ // `wait_config` configures. targetActions()/waitActions() derive both sets.
34
+ //
30
35
  // `stepFlags` records a scenario step field that changes how the verb runs:
31
36
  // `{ field, value, flag }` means "when step[field] === value, pass flag". This
32
37
  // is the one place the mapping lives; the E2E harness derives the flags via
@@ -42,31 +47,38 @@
42
47
  const ACTION_CATALOG = {
43
48
  // --- direct verbs (engine-backed) --------------------------------------
44
49
  tap: {
50
+ requiresTarget: true,
45
51
  resolution: 'verb', verb: 'tap', bridge: 'tap',
46
52
  primitive: 'mobile_click_on_screen_at_coordinates', args: ['x', 'y'],
47
53
  },
48
54
  long_press: {
55
+ requiresTarget: true,
49
56
  resolution: 'verb', verb: 'long-press', bridge: 'longPress',
50
57
  primitive: 'mobile_long_press_on_screen_at_coordinates', args: ['x', 'y', 'duration?'],
51
58
  },
52
59
  double_tap: {
60
+ requiresTarget: true,
53
61
  resolution: 'verb', verb: 'double-tap', bridge: 'doubleTap',
54
62
  primitive: 'mobile_double_tap_on_screen', args: ['x', 'y'],
55
63
  },
56
64
  type: {
65
+ requiresTarget: false,
57
66
  resolution: 'verb', verb: 'type', bridge: 'type',
58
67
  primitive: 'mobile_type_keys', args: ['text', 'submit'],
59
68
  },
60
69
  swipe: {
70
+ requiresTarget: false,
61
71
  resolution: 'verb', verb: 'swipe', bridge: 'swipe',
62
72
  primitive: 'mobile_swipe_on_screen', args: ['direction'],
63
73
  },
64
74
  press_button: {
75
+ requiresTarget: false,
65
76
  resolution: 'verb', verb: 'press', bridge: 'pressButton',
66
77
  primitive: 'mobile_press_button', args: ['button'],
67
78
  },
68
79
  launch_app: {
69
80
  // Cold start (#255): restartApp terminates, then launches.
81
+ requiresTarget: false,
70
82
  resolution: 'verb', verb: 'launch', bridge: 'restartApp',
71
83
  primitives: ['mobile_terminate_app', 'mobile_launch_app'], args: ['packageName'],
72
84
  // `cold: false` opts out of the restart (#255).
@@ -81,6 +93,7 @@ const ACTION_CATALOG = {
81
93
  primitive: 'mobile_uninstall_app', args: ['packageName'],
82
94
  },
83
95
  open_url: {
96
+ requiresTarget: false,
84
97
  resolution: 'verb', verb: 'open-url', bridge: 'openUrl',
85
98
  primitive: 'mobile_open_url', args: ['url'],
86
99
  },
@@ -90,24 +103,35 @@ const ACTION_CATALOG = {
90
103
  },
91
104
 
92
105
  // --- semantic (mauto press <action>, platform-resolved) ----------------
93
- press_back: { resolution: 'semantic', verb: 'press' },
94
- dismiss_keyboard: { resolution: 'semantic', verb: 'press' },
95
- grant_permission: { resolution: 'semantic', verb: 'press' },
96
- deny_permission: { resolution: 'semantic', verb: 'press' },
106
+ press_back: { requiresTarget: false, resolution: 'semantic', verb: 'press' },
107
+ dismiss_keyboard: { requiresTarget: false, resolution: 'semantic', verb: 'press' },
108
+ grant_permission: { requiresTarget: false, resolution: 'semantic', verb: 'press' },
109
+ deny_permission: { requiresTarget: false, resolution: 'semantic', verb: 'press' },
97
110
 
98
111
  // --- composed (agent builds from existing verbs; no dedicated verb) -----
99
- scroll_to_element: { resolution: 'composed' },
100
- wait_for_element: { resolution: 'composed' },
101
- wait_for_element_gone: { resolution: 'composed' },
102
- wait_for_loading_complete: { resolution: 'composed' },
103
- capture_value: { resolution: 'composed' },
112
+ scroll_to_element: { requiresTarget: true, resolution: 'composed' },
113
+ wait_for_element: { requiresTarget: true, waitAction: true, resolution: 'composed' },
114
+ wait_for_element_gone: { requiresTarget: true, waitAction: true, resolution: 'composed' },
115
+ wait_for_loading_complete: { requiresTarget: false, waitAction: true, resolution: 'composed' },
116
+ capture_value: { requiresTarget: true, resolution: 'composed' },
104
117
 
105
118
  // --- unsupported (no mobile-mcp 1.0.5 primitive) -----------------------
106
- clear_app_data: { resolution: 'unsupported' },
119
+ clear_app_data: { requiresTarget: false, resolution: 'unsupported' },
107
120
  enable_wifi: { resolution: 'unsupported' },
108
121
  disable_wifi: { resolution: 'unsupported' },
109
122
  };
110
123
 
124
+ // Steps that locate an element on screen (tap, waits, scroll, capture) need a
125
+ // `target` or `target_pattern`; the scenario referential checks read this set.
126
+ function targetActions() {
127
+ return Object.keys(ACTION_CATALOG).filter((a) => ACTION_CATALOG[a].requiresTarget === true);
128
+ }
129
+
130
+ // Steps the schema's `wait_config` configures.
131
+ function waitActions() {
132
+ return Object.keys(ACTION_CATALOG).filter((a) => ACTION_CATALOG[a].waitAction === true);
133
+ }
134
+
111
135
  const RESOLUTIONS = ['verb', 'semantic', 'composed', 'unsupported'];
112
136
 
113
137
  function actionsByResolution(resolution) {
@@ -141,4 +165,5 @@ function implicitStepFlags(action, step) {
141
165
 
142
166
  module.exports = {
143
167
  ACTION_CATALOG, RESOLUTIONS, actionsByResolution, primitivesOf, stepFlagsFor, implicitStepFlags,
168
+ targetActions, waitActions,
144
169
  };
@@ -54,15 +54,15 @@ Record each observation with `mauto result add-step --observation <type>:<messag
54
54
 
55
55
  1. **Name the run first.** Choose the run identifier for this execution (format `run_YYYYMMDD_HHMMSS`) and export it as `MAUTO_RUN_ID` before running any other verb — `export MAUTO_RUN_ID=run_20260905_141500`. Every verb then records into a trace for that run, and `mauto result finalize --run-id <the same id>` measures the run's duration from it instead of taking your word for it. Use the *same* id in both places; a different one means no trace to measure.
56
56
  2. Read the scenario JSON.
57
- 3. Verify a device is available with `mauto devices`. Read the device's `platform` field — store it for the duration of execution.
58
- 4. **App under test.**
57
+ 3. **Validate schema version:** read the `$schema_version` field; it must be `"2.1"`. If absent or unrecognized, report an error and halt.
58
+ 4. Validate the scenario with `mauto validate <path>` against `mauto schema scenario`. If `validate` itself fails (`ok: false`), stop and report its errors, and run no device verbs. If the result lists `data.warnings`, branch on each entry's `code` (its `path` names the step or precondition): `unsupported_action` means it uses an action that is unsupported on this run: report it as unsupported in the result (see "Not mechanically executable" under step execution) and do not substitute a gesture for it. `launch_cold_implicit` is advisory: replay that step exactly as written (a step that does not set `"cold": false` is a cold start, per the step table below) and do not substitute your own reading of what the author meant; record the warning's `message` as a `state_context` observation on that step (`mauto result add-step ... --observation state_context:<message>`). `duplicate_step_id`, `dangling_after_step`, `missing_target`, `undeclared_capture_variable` and `dangling_condition_step` mean the scenario itself is broken (a step id used twice, a reference to a step or variable that does not exist, a step with no element to resolve): run no further verbs against the device and do not guess the missing reference; finalize the run with `mauto result finalize --run-id <the run id> --scenario-id <scenario_id> --status error --summary "<each warning's path and message>"`, and ask the user to fix the scenario. `missing_wait_config` is advisory: replay that wait step with the default timeout and record the warning's `message` as a `state_context` observation on that step.
59
+ 5. Verify a device is available with `mauto devices`. Read the device's `platform` field — store it for the duration of execution.
60
+ 6. **App under test.**
59
61
  - If the user pointed you at a prebuilt app artifact, install it with `mauto install <path>`.
60
62
  - Otherwise verify the app is already installed; if it is not, halt and ask the user to install it. This guide never builds.
61
63
  - **Fresh install.** If the scenario's `preconditions.app_state` is `fresh_install` and the user pointed you at a prebuilt app artifact, uninstall the app first with `mauto uninstall <appId>` (the app identifier from `.mobile-automator/config.json` — `android_package` / `ios_bundle_id`) and only then install that artifact, so the run starts with no app data. Without an artifact, do not uninstall — you could not reinstall the app. Leave it in place and report that the app state was not reset, so a failure that depends on existing app data is not mistaken for a regression.
62
64
  - If the install fails because a different build is already on the device, uninstall it with `mauto uninstall <appId>` (the app identifier from `.mobile-automator/config.json` — `android_package` / `ios_bundle_id`), install again, and report that app data was wiped — a scenario whose preconditions assume existing app state may now fail for that reason.
63
65
  - State which path you took before continuing.
64
- 5. **Validate schema version:** read the `$schema_version` field; it must be `"2.1"`. If absent or unrecognized, report an error and halt.
65
- 6. Validate the scenario with `mauto validate <path>` against `mauto schema scenario`. If the result lists `data.warnings`, branch on each entry's `code` (its `path` names the step or precondition): `unsupported_action` means it uses an action that is unsupported on this run: report it as unsupported in the result (see "Not mechanically executable" under step execution) and do not substitute a gesture for it. `launch_cold_implicit` is advisory: replay that step exactly as written (a step that does not set `"cold": false` is a cold start, per the step table below) and do not substitute your own reading of what the author meant; record the warning's `message` as a `state_context` observation on that step (`mauto result add-step ... --observation state_context:<message>`).
66
66
 
67
67
  **Precondition `device_actions`** (from the scenario's `preconditions`, run before step replay):
68
68
 
@@ -52,18 +52,18 @@ Record each observation with `mauto result add-step --observation <type>:<messag
52
52
  ### 1. Pre-flight
53
53
 
54
54
  - **Name the run first.** Choose the run identifier for this execution (format `run_YYYYMMDD_HHMMSS`) and export it as `MAUTO_RUN_ID` before running any other verb — `export MAUTO_RUN_ID=run_20260905_141500`. Every verb then records into a trace for that run, and `mauto result finalize --run-id <the same id>` measures the run's duration from it instead of taking your word for it. Use the *same* id in both places; a different one means no trace to measure.
55
+ - Read the JSON scenario file from `.mobile-automator/scenarios/`.
56
+ - **Validate schema version:** Read the `$schema_version` field. It must be `"2.1"`. If absent or unrecognized, report an error and halt.
57
+ - Validate the scenario with `mauto validate <path>` against `mauto schema scenario`. If `validate` itself fails (`ok: false`), stop and report its errors, and run no device verbs. If the result lists `data.warnings`, branch on each entry's `code` (its `path` names the step or precondition): `unsupported_action` means it uses an action that is unsupported on this run: report it as unsupported in the result, as described under "Not mechanically executable" below, and do not substitute a gesture for it. `launch_cold_implicit` is advisory: replay that step exactly as written (a step that does not set `"cold": false` is a cold start, per the step table below) and do not substitute your own reading of what the author meant; record the warning's `message` as a `state_context` observation on that step (`mauto result add-step ... --observation state_context:<message>`). `duplicate_step_id`, `dangling_after_step`, `missing_target`, `undeclared_capture_variable` and `dangling_condition_step` mean the scenario itself is broken (a step id used twice, a reference to a step or variable that does not exist, a step with no element to resolve): run no further verbs against the device and do not guess the missing reference; finalize the run with `mauto result finalize --run-id <the run id> --scenario-id <scenario_id> --status error --summary "<each warning's path and message>"`, and ask the user to fix the scenario. `missing_wait_config` is advisory: replay that wait step with the default timeout and record the warning's `message` as a `state_context` observation on that step.
55
58
  - Verify a device is available with `mauto devices`.
56
59
  - **App under test.** If the user pointed you at a prebuilt artifact, install it with `mauto install <path>` and do NOT build — reusing a ready artifact is the point.
57
60
  - If no artifact was named: check whether the app is already installed. If it is, say so and ask whether to rebuild before running the build command ({{build_command}}) — a previous run may have just built it. If it is not installed, build and install with the build command ({{build_command}}).
58
61
  - If that install fails because a different build of the app is already on the device, uninstall it — run `mauto uninstall` with the app package ({{app_package}}) — then install again, and report that app data was wiped — a scenario whose preconditions assume existing app state may now fail for that reason.
59
- - **Fresh install.** If the scenario's `preconditions.app_state` is `fresh_install` (read it from the scenario file now), uninstall the app first — `mauto uninstall` with the app package ({{app_package}}) — and only then install, so the run starts with no app data.
62
+ - **Fresh install.** If the scenario's `preconditions.app_state` is `fresh_install` (from the scenario you read above), uninstall the app first — `mauto uninstall` with the app package ({{app_package}}) — and only then install, so the run starts with no app data.
60
63
  - State which path you took — installed a supplied artifact, or built — before continuing.
61
64
 
62
- ### 2. Load Scenario
65
+ ### 2. Compare Environment
63
66
 
64
- - Read the JSON scenario file from `.mobile-automator/scenarios/`.
65
- - **Validate schema version:** Read the `$schema_version` field. It must be `"2.1"`. If absent or unrecognized, report an error and halt.
66
- - Validate the scenario with `mauto validate <path>` against `mauto schema scenario`. If the result lists `data.warnings`, branch on each entry's `code` (its `path` names the step or precondition): `unsupported_action` means it uses an action that is unsupported on this run: report it as unsupported in the result, as described under "Not mechanically executable" below, and do not substitute a gesture for it. `launch_cold_implicit` is advisory: replay that step exactly as written (a step that does not set `"cold": false` is a cold start, per the step table below) and do not substitute your own reading of what the author meant; record the warning's `message` as a `state_context` observation on that step (`mauto result add-step ... --observation state_context:<message>`).
67
67
  - Read the `metadata` to compare the recording environment against the current execution environment. Note any differences (different device, API level, environment).
68
68
 
69
69
  ### 3. Verify Preconditions
@@ -203,7 +203,7 @@ After all steps are executed and recorded:
203
203
  2. Obtain the scenario schema by running `mauto schema scenario`, and assemble the JSON scenario to match it. Include the `tags` array.
204
204
  3. **Always include `"$schema_version": "2.1"` as the first field** in agnostic-mode scenarios.
205
205
  4. **Use named string IDs** for all steps and assertions (snake_case, e.g., `"id": "tap_login"`).
206
- 5. Validate the assembled scenario with `mauto validate <path>` against `mauto schema scenario`. If it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly (`false` when it means returning to the already-open app, `true` when it means a restart).
206
+ 5. Validate the assembled scenario with `mauto validate <path>` against `mauto schema scenario`. If it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly (`false` when it means returning to the already-open app, `true` when it means a restart). Fix every reference finding before you save, using its `path`: `duplicate_step_id` — rename the step so every id is unique; `dangling_after_step` — point the assertion's `after_step` at a step that exists; `missing_target` — give the step the `target` it acts on; `undeclared_capture_variable` — declare the variable in the root `variables` block; `dangling_condition_step` — point the `previous_step_skipped` condition's `step_id` at an earlier step; `missing_wait_config` — add a `wait_config` with an explicit `timeout_ms`.
207
207
  6. Save to `.mobile-automator/scenarios/<scenario_id>.json`.
208
208
  7. Present summary:
209
209
  > "Scenario saved: `.mobile-automator/scenarios/<scenario_id>.json`
@@ -217,7 +217,7 @@ After all steps are executed and recorded:
217
217
  - `app_version` — from the installed app
218
218
  - `environment` — ask the user or use `default_environment` from `.mobile-automator/config.json`
219
219
  - Do NOT include `device_model`, `api_level`, or `timestamp` in metadata — these belong in the result schema, not the scenario.
220
- 7. Validate the assembled scenario with `mauto validate <path>` against `mauto schema scenario`. If it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly (`false` when it means returning to the already-open app, `true` when it means a restart).
220
+ 7. Validate the assembled scenario with `mauto validate <path>` against `mauto schema scenario`. If it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly (`false` when it means returning to the already-open app, `true` when it means a restart). Fix every reference finding before you save, using its `path`: `duplicate_step_id` — rename the step so every id is unique; `dangling_after_step` — point the assertion's `after_step` at a step that exists; `missing_target` — give the step the `target` it acts on; `undeclared_capture_variable` — declare the variable in the root `variables` block; `dangling_condition_step` — point the `previous_step_skipped` condition's `step_id` at an earlier step; `missing_wait_config` — add a `wait_config` with an explicit `timeout_ms`.
221
221
  8. Save to `.mobile-automator/scenarios/<scenario_id>.json`.
222
222
  9. Present summary:
223
223
  > "Scenario saved: `.mobile-automator/scenarios/<scenario_id>.json`
@@ -269,7 +269,7 @@ Translate user language to `mauto` verbs and schema actions:
269
269
  - Create assertion step using mapped type from Pass 2b
270
270
  - Evaluate via `mauto assert <type> ...` and record with appropriate fields
271
271
  6. Combine steps and assertions into ordered scenario
272
- 7. Generate scenario JSON, then `mauto validate <path>` (if it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly)
272
+ 7. Generate scenario JSON, then `mauto validate <path>` (if it returns `data.warnings`, revise the scenario to drop each `unsupported_action` step, or tell the user the step needs manual handling; for a `launch_cold_implicit` entry, set the step's `cold` explicitly; fix each `duplicate_step_id`, `dangling_after_step`, `missing_target`, `undeclared_capture_variable`, `dangling_condition_step` and `missing_wait_config` entry as "Save Scenario" describes)
273
273
  ```
274
274
 
275
275
  ### Detecting and Encoding Patterns
@@ -87,7 +87,7 @@ function parseBool(raw, flagName) {
87
87
  // Mirrors the enum at /properties/status in src/schemas/result_schema.json;
88
88
  // tests/lint/result-coverage.test.js asserts the two never drift. `result
89
89
  // finalize --status` must be one of these or omitted (omission derives the
90
- // status from the recorded assertions): an unlisted value used to be persisted
90
+ // status from the recorded steps and assertions): an unlisted value used to be persisted
91
91
  // verbatim — a schema-invalid result file with a green exit — directly under
92
92
  // the exit-6 contract keyed on that status (#286 review).
93
93
  const RUN_STATUSES = ['passed', 'failed', 'error'];
@@ -0,0 +1,203 @@
1
+ 'use strict';
2
+
3
+ // JUnit XML rendering of finalized result files (#274). Its input is what
4
+ // readFinalizedResult (src/result/store.js) guarantees: an object finalize wrote
5
+ // for the run id — NOT one valid against the result schema (a run recorded
6
+ // without --scenario-id has scenario_id null; a pre-0.38.0 file has an
7
+ // upper-case status). So the renderer tolerates: the suite is named by the run
8
+ // id when there is no scenario_id, any status other than `passed` reads red, a
9
+ // missing metadata/container is empty, and non-object entries are dropped.
10
+ // Pure: no fs, no clock, no input mutation, so the same results always render
11
+ // to the same bytes. It is consumed by `mauto result export`.
12
+ //
13
+ // Mapping, one result at a time:
14
+ // result -> <testsuite name=scenario_id, else run_id>
15
+ // assertion_results[] -> <testcase name=assertion_id>; failed -> <failure>
16
+ // failed / error step -> <testcase name="step: <id>"> with <failure>
17
+ // run id, status, metadata -> <properties>
18
+ //
19
+ // It renders only what current writers produce: assertions are passed or
20
+ // failed, and only failed or errored steps get a testcase, so there is no
21
+ // JUnit skip element and no skip count. A step's `error` status is reported as
22
+ // a <failure> of type step_error rather than a JUnit <error>, so the report's
23
+ // counts stay {tests, failures} and `errors` is always 0.
24
+
25
+ const { isErroredStep, isFailingAssertion, isFailingStep } = require('./outcome');
26
+
27
+ // Characters XML 1.0 forbids anywhere in a document: C0 controls other than
28
+ // tab/LF/CR, lone surrogates, U+FFFE and U+FFFF. A crash excerpt or an OCR'd
29
+ // label can carry any of them, and one is enough to make the whole report
30
+ // unparseable, so they are dropped rather than escaped (escaping does not make
31
+ // them legal).
32
+ // eslint-disable-next-line no-control-regex
33
+ const ILLEGAL_XML_CHARS = /[\u0000-\u0008\u000B\u000C\u000E-\u001F￾￿]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;
34
+
35
+ const METADATA_KEYS = ['app_version', 'device_model', 'api_level', 'environment', 'timestamp'];
36
+
37
+ function clean(value) {
38
+ return String(value).replace(ILLEGAL_XML_CHARS, '');
39
+ }
40
+
41
+ function escapeText(value) {
42
+ return clean(value).replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
43
+ }
44
+
45
+ // Attribute values additionally escape quotes, and encode whitespace a parser
46
+ // would otherwise normalise to a space, so a multi-line message survives.
47
+ function escapeAttr(value) {
48
+ return escapeText(value)
49
+ .replace(/"/g, '&quot;')
50
+ .replace(/'/g, '&apos;')
51
+ .replace(/\n/g, '&#10;')
52
+ .replace(/\r/g, '&#13;')
53
+ .replace(/\t/g, '&#9;');
54
+ }
55
+
56
+ function attrs(pairs) {
57
+ return pairs
58
+ .filter(([, v]) => v !== undefined && v !== null)
59
+ .map(([k, v]) => ` ${k}="${escapeAttr(v)}"`)
60
+ .join('');
61
+ }
62
+
63
+ const present = (v) => v !== undefined && v !== null && v !== '';
64
+
65
+ // A container as an array of its object entries; anything else is empty.
66
+ const isObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
67
+ const asArray = (value) => (Array.isArray(value) ? value.filter(isObject) : []);
68
+
69
+ // Seconds as a plain decimal, rounded to the millisecond so summed floats do
70
+ // not render as 50.50000000001.
71
+ function seconds(value) {
72
+ const n = Number(value);
73
+ return String(Number.isFinite(n) && n > 0 ? Math.round(n * 1000) / 1000 : 0);
74
+ }
75
+
76
+ function properties(result, metadata) {
77
+ const props = [
78
+ ['run_id', result.run_id],
79
+ ['status', result.status],
80
+ ];
81
+ for (const key of METADATA_KEYS) {
82
+ if (present(metadata[key])) props.push([`metadata.${key}`, metadata[key]]);
83
+ }
84
+ return props.filter(([, v]) => present(v));
85
+ }
86
+
87
+ function assertionCase(assertion) {
88
+ const testcase = { name: assertion.assertion_id };
89
+ if (isFailingAssertion(assertion)) {
90
+ const body = [];
91
+ if (present(assertion.expected)) body.push(`expected: ${assertion.expected}`);
92
+ if (present(assertion.actual)) body.push(`actual: ${assertion.actual}`);
93
+ testcase.failure = {
94
+ type: assertion.type ? `${assertion.type}_failed` : 'assertion_failed',
95
+ message: assertion.message || `${assertion.assertion_id} failed`,
96
+ body: body.join('\n'),
97
+ };
98
+ }
99
+ return testcase;
100
+ }
101
+
102
+ function stepCase(step) {
103
+ const name = `step: ${step.step_id}`;
104
+ if (isFailingStep(step)) {
105
+ const body = [];
106
+ if (present(step.error_message)) body.push(step.error_message);
107
+ if (present(step.screenshot)) body.push(`screenshot: ${step.screenshot}`);
108
+ return {
109
+ name,
110
+ failure: {
111
+ type: isErroredStep(step) ? 'step_error' : 'step_failed',
112
+ message: step.error_message || `step ${step.step_id} ${step.status}`,
113
+ body: body.join('\n'),
114
+ },
115
+ };
116
+ }
117
+ return null;
118
+ }
119
+
120
+ function suiteFor(result) {
121
+ const scenario = result.scenario_id ?? result.run_id;
122
+ const metadata = isObject(result.metadata) ? result.metadata : {};
123
+ const cases = [
124
+ ...asArray(result.assertion_results).map(assertionCase),
125
+ ...asArray(result.steps_executed).map(stepCase).filter(Boolean),
126
+ ];
127
+
128
+ // A red result must never render as a green report. When the run says
129
+ // anything but passed yet recorded nothing that fails (e.g. an agent
130
+ // finalized with --status failed after an abort, or a pre-0.38.0 "FAILED"),
131
+ // the status itself becomes the failure.
132
+ const isRed = result.status !== 'passed';
133
+ if (isRed && !cases.some((c) => c.failure)) {
134
+ cases.push({
135
+ name: 'run status',
136
+ failure: {
137
+ type: 'run_status',
138
+ message: `result status is ${result.status} but no failed step or assertion was recorded`,
139
+ body: result.summary,
140
+ },
141
+ });
142
+ }
143
+
144
+ const failures = cases.filter((c) => c.failure).length;
145
+
146
+ const lines = [];
147
+ lines.push(
148
+ ` <testsuite${attrs([
149
+ ['name', scenario],
150
+ ['tests', cases.length],
151
+ ['failures', failures],
152
+ ['errors', 0],
153
+ ['time', seconds(result.duration_seconds)],
154
+ ['timestamp', present(metadata.timestamp) ? metadata.timestamp : null],
155
+ ])}>`
156
+ );
157
+ const props = properties(result, metadata);
158
+ if (props.length) {
159
+ lines.push(' <properties>');
160
+ for (const [name, value] of props) lines.push(` <property${attrs([['name', name], ['value', value]])}/>`);
161
+ lines.push(' </properties>');
162
+ }
163
+ for (const c of cases) {
164
+ const open = ` <testcase${attrs([['name', c.name], ['classname', scenario]])}`;
165
+ if (c.failure) {
166
+ const { type, message, body } = c.failure;
167
+ lines.push(`${open}>`);
168
+ lines.push(` <failure${attrs([['message', message], ['type', type]])}>${escapeText(body)}</failure>`);
169
+ lines.push(' </testcase>');
170
+ } else {
171
+ lines.push(`${open}/>`);
172
+ }
173
+ }
174
+ lines.push(' </testsuite>');
175
+
176
+ return { lines, tests: cases.length, failures, time: Number(seconds(result.duration_seconds)) };
177
+ }
178
+
179
+ // toJUnit(results) -> { xml, suites, tests, failures }
180
+ function toJUnit(results) {
181
+ const suites = results.map(suiteFor);
182
+ const total = (key) => suites.reduce((sum, s) => sum + s[key], 0);
183
+ const tests = total('tests');
184
+ const failures = total('failures');
185
+
186
+ const xml = [
187
+ '<?xml version="1.0" encoding="UTF-8"?>',
188
+ `<testsuites${attrs([
189
+ ['name', 'mauto'],
190
+ ['tests', tests],
191
+ ['failures', failures],
192
+ ['errors', 0],
193
+ ['time', seconds(total('time'))],
194
+ ])}>`,
195
+ ...suites.flatMap((s) => s.lines),
196
+ '</testsuites>',
197
+ '',
198
+ ].join('\n');
199
+
200
+ return { xml, suites: suites.length, tests, failures };
201
+ }
202
+
203
+ module.exports = { toJUnit };
@@ -0,0 +1,33 @@
1
+ 'use strict';
2
+
3
+ // What counts as a failure in a result file — the one definition shared by
4
+ // `result finalize` (src/result/store.js, which derives the run status and
5
+ // counts) and the JUnit export (src/result/junit.js, which decides which
6
+ // entries render a <failure>). Keeping both on these predicates is what stops
7
+ // the result file and its export disagreeing (#296); the cross-check in
8
+ // tests/unit/result/outcome.test.js renders derived statuses through toJUnit.
9
+ //
10
+ // The predicates take anything: the JUnit renderer reads older and
11
+ // hand-edited files, so a null or non-object entry is simply not failing.
12
+
13
+ const statusOf = (entry) => (entry !== null && typeof entry === 'object' ? entry.status : undefined);
14
+
15
+ const isFailedStep = (step) => statusOf(step) === 'failed';
16
+ const isErroredStep = (step) => statusOf(step) === 'error';
17
+ const isFailingStep = (step) => isFailedStep(step) || isErroredStep(step);
18
+ const isFailingAssertion = (assertion) => statusOf(assertion) === 'failed';
19
+
20
+ // The run status `result finalize` derives when `--status` is omitted (#296).
21
+ // It used to read assertions only, so a run whose step failed finalized
22
+ // `passed` while its JUnit export was red. Every entry counts — a step
23
+ // recorded twice (fail, then pass) still failed once, matching the export.
24
+ // Skipped steps count for nothing; a run with no steps and no assertions
25
+ // stays `passed` (#275 owns that case).
26
+ function deriveRunStatus(steps = [], assertions = []) {
27
+ if (steps.some(isErroredStep)) return 'error';
28
+ if (steps.some(isFailedStep)) return 'failed';
29
+ if (assertions.some(isFailingAssertion)) return 'failed';
30
+ return 'passed';
31
+ }
32
+
33
+ module.exports = { isFailedStep, isErroredStep, isFailingStep, isFailingAssertion, deriveRunStatus };
@@ -6,6 +6,7 @@ const path = require('path');
6
6
  const { atomicWrite } = require('../util/atomic');
7
7
  const { withLock } = require('../util/lock');
8
8
  const { OBSERVATION_TYPES, stepStatus } = require('./flags');
9
+ const { deriveRunStatus, isErroredStep, isFailedStep, isFailingAssertion } = require('./outcome');
9
10
  const { workspaceDir } = require('../workspace/layout');
10
11
 
11
12
  // Incremental result accumulator that persists to
@@ -18,6 +19,15 @@ const { workspaceDir } = require('../workspace/layout');
18
19
 
19
20
  const SCHEMA_VERSION = '2.0';
20
21
 
22
+ // The one place that knows where a run's result file lives.
23
+ function resultsDir(projectRoot) {
24
+ return path.join(workspaceDir(projectRoot), 'results');
25
+ }
26
+
27
+ function resultFile(projectRoot, runId) {
28
+ return path.join(resultsDir(projectRoot), `${runId}.json`);
29
+ }
30
+
21
31
  // A crash report is read into an agent's context and routinely committed to a
22
32
  // repo. A native tombstone is tens of kilobytes, so the excerpt is bounded HERE
23
33
  // rather than trusted from the caller — the full report's home is report_path.
@@ -33,6 +43,15 @@ function defaultMetadata(overrides = {}) {
33
43
  };
34
44
  }
35
45
 
46
+ // The default summary's tail naming the failing steps, so a run a step turned
47
+ // red does not read `failed: 0/0 assertion(s) passed` with no reason (#296).
48
+ // Empty when no step failed, which keeps the pre-#296 text for those runs.
49
+ function stepFailureSuffix(steps) {
50
+ const failed = steps.filter(isFailedStep).length;
51
+ const errored = steps.filter(isErroredStep).length;
52
+ return `${failed ? `, ${failed} failed` : ''}${errored ? `, ${errored} errored` : ''}`;
53
+ }
54
+
36
55
  function assertionStatus(pass) {
37
56
  return pass ? 'passed' : 'failed';
38
57
  }
@@ -47,8 +66,8 @@ class ResultStore {
47
66
  this.projectRoot = projectRoot;
48
67
  this._metadataOverrides = metadata || {};
49
68
 
50
- this._dir = path.join(workspaceDir(projectRoot), 'results');
51
- this._file = path.join(this._dir, `${runId}.json`);
69
+ this._dir = resultsDir(projectRoot);
70
+ this._file = resultFile(projectRoot, runId);
52
71
  // Per-runId advisory lock guarding every read-modify-write mutator. Lives
53
72
  // in the same directory as the result file, so the lock is scoped to this
54
73
  // run only — different runs never contend on it.
@@ -147,6 +166,8 @@ class ResultStore {
147
166
  atomicWrite(this._file, contents);
148
167
  }
149
168
 
169
+ // The in-progress marker below is private to this module; readers ask
170
+ // isInProgress() instead of keying on the field themselves.
150
171
  _persistInProgress() {
151
172
  const snapshot = {
152
173
  run_id: this.runId,
@@ -326,10 +347,10 @@ class ResultStore {
326
347
  }
327
348
 
328
349
  const passed = this._assertions.filter((a) => a.status === 'passed').length;
329
- const failed = this._assertions.filter((a) => a.status === 'failed').length;
350
+ const failed = this._assertions.filter(isFailingAssertion).length;
330
351
  const total = this._assertions.length;
331
352
 
332
- const resolvedStatus = status || (failed > 0 ? 'failed' : 'passed');
353
+ const resolvedStatus = status || deriveRunStatus(this._steps, this._assertions);
333
354
 
334
355
  const result = {
335
356
  run_id: this.runId,
@@ -347,7 +368,8 @@ class ResultStore {
347
368
  captured_variables: this._capturedVariables,
348
369
  summary:
349
370
  summary ||
350
- `${resolvedStatus}: ${passed}/${total} assertion(s) passed across ${this._steps.length} step(s).`,
371
+ `${resolvedStatus}: ${passed}/${total} assertion(s) passed across ${this._steps.length} step(s)` +
372
+ `${stepFailureSuffix(this._steps)}.`,
351
373
  };
352
374
 
353
375
  // Additive and OMITTED when absent: a result file from a run with no
@@ -366,4 +388,80 @@ class ResultStore {
366
388
  }
367
389
  }
368
390
 
369
- module.exports = { ResultStore };
391
+ function isInProgress(parsed) {
392
+ return Boolean(parsed && parsed._in_progress);
393
+ }
394
+
395
+ // A run id names a file in results/ — `<runId>.json` — so it must be a single
396
+ // path segment: a non-empty string with no separator and no NUL. `.` and `..`
397
+ // pass, harmlessly: the appended `.json` keeps them inside results/. This is
398
+ // deliberately NOT the result schema's run_id pattern nor observe/paths.js's
399
+ // isValidRunId: the writer (add-step / finalize) accepts any id, so a stricter
400
+ // reader would refuse runs finalize wrote successfully.
401
+ function isResultRunId(runId) {
402
+ return (
403
+ typeof runId === 'string' &&
404
+ runId.length > 0 &&
405
+ !runId.includes('\0') &&
406
+ path.basename(runId) === runId
407
+ );
408
+ }
409
+
410
+ // Read a finalized result file for a report, WITHOUT going through the
411
+ // ResultStore constructor: that constructor renames a corrupt file aside as a
412
+ // `.corrupt.<ts>` sidecar, and a reader must never mutate the workspace it
413
+ // reads. This function performs no writes, renames or locks.
414
+ //
415
+ // "Finalized" is defined by its writer, `ResultStore#finalize` above: a JSON
416
+ // object with no `_in_progress` marker whose run_id is the requested id. The
417
+ // reader checks exactly that and nothing more — it does not hold files to the
418
+ // result schema, because finalize itself does not (a run recorded without
419
+ // --scenario-id is written with scenario_id null, and pre-0.38.0 files carry
420
+ // an upper-case status). Consumers tolerate shape beyond that object.
421
+ //
422
+ // Returns `{ result }` or `{ error: { kind, message, hint } }` where kind is
423
+ // `invalid_input` (bad id, no file, not JSON, in progress, not a finalized
424
+ // result) or `environment` (the file exists but could not be read).
425
+ function readFinalizedResult(projectRoot, runId) {
426
+ const invalid = (message, hint) => ({ error: { kind: 'invalid_input', message, hint } });
427
+ if (!isResultRunId(runId)) {
428
+ return invalid(
429
+ `invalid run id "${runId}"`,
430
+ `A run id names a file in ${resultsDir(projectRoot)}; it cannot contain a path separator.`
431
+ );
432
+ }
433
+ const file = resultFile(projectRoot, runId);
434
+ let raw;
435
+ try {
436
+ raw = fs.readFileSync(file, 'utf8');
437
+ } catch (err) {
438
+ if (err && err.code === 'ENOENT') {
439
+ return invalid(
440
+ `no result file for ${runId}`,
441
+ `Expected ${file}; list the run ids with \`ls ${resultsDir(projectRoot)}\`.`
442
+ );
443
+ }
444
+ return { error: { kind: 'environment', message: `could not read ${file}: ${err.message}`, hint: null } };
445
+ }
446
+ let parsed;
447
+ try {
448
+ parsed = JSON.parse(raw);
449
+ } catch (err) {
450
+ return invalid(`result file for ${runId} is not valid JSON (${err.message})`, `Inspect ${file}.`);
451
+ }
452
+ const notAResult = (reason) =>
453
+ invalid(
454
+ `result file for ${runId} is not a finalized result: ${reason}`,
455
+ `Inspect ${file}; result files are written by \`mauto result finalize\`.`
456
+ );
457
+ if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
458
+ return notAResult('(root) must be object');
459
+ }
460
+ if (isInProgress(parsed)) {
461
+ return invalid(`run ${runId} is not finalized`, `Run \`mauto result finalize --run-id ${runId}\` first.`);
462
+ }
463
+ if (parsed.run_id !== runId) return notAResult(`its run_id is ${JSON.stringify(parsed.run_id)}`);
464
+ return { result: parsed };
465
+ }
466
+
467
+ module.exports = { ResultStore, readFinalizedResult, resultFile, isResultRunId };
@@ -0,0 +1,160 @@
1
+ 'use strict';
2
+
3
+ const { targetActions, waitActions } = require('../device/action-catalog');
4
+
5
+ // Referential checks for scenarios (#272). The schema types every reference as
6
+ // a bare string, so a scenario whose ids, targets and variables do not line up
7
+ // still validates -- and the gap surfaces mid-run, on a device. This module
8
+ // reports those gaps as finding descriptors -- `{ code, fields }`, plus `path`
9
+ // and `ids` for assertions -- and `replayability.js` turns them into warnings
10
+ // with its one constructor, during its one walk over the steps, so `validate`
11
+ // sees one finding list in one shape.
12
+ //
13
+ // Every code but `missing_wait_config` is blocking: the step or assertion has
14
+ // nothing to resolve against, so it cannot replay unattended. A wait without
15
+ // `wait_config` still runs on the default timeout, so that one is advisory.
16
+
17
+ // Which step actions need a target, and which take `wait_config`, are catalog
18
+ // facts (src/device/action-catalog.js), not restated here.
19
+ const TARGET_ACTIONS = new Set(targetActions());
20
+ const WAIT_ACTIONS = new Set(waitActions());
21
+
22
+ // A condition or assertion of these types reads a variable, so it must name one.
23
+ const CONDITION_TYPES_NEEDING_VARIABLE = new Set(['variable_value']);
24
+ const ASSERTION_TYPES_NEEDING_VARIABLE = new Set(['value_matches_variable']);
25
+
26
+ // Finding fields for a variable name that is undeclared, or absent where its
27
+ // type requires one (an empty name counts as absent there); null when fine.
28
+ function variableFields(field, name, required, index) {
29
+ const ref = required && name === '' ? undefined : name;
30
+ if (ref === undefined ? !required : index.variables.has(ref)) return null;
31
+ return { field, ref };
32
+ }
33
+
34
+ const REFERENCE_CODES = {
35
+ duplicate_step_id: {
36
+ blocking: true,
37
+ clause: (n) => `${n} step id(s) are declared more than once`,
38
+ message: (f) => `step id "${f.ref}" is already used by ${f.firstPath}; `
39
+ + 'step ids must be unique so assertions and conditions can name one step',
40
+ },
41
+ dangling_after_step: {
42
+ blocking: true,
43
+ clause: (n) => `${n} assertion(s) run after a step that does not exist`,
44
+ message: (f) => (f.ref === undefined
45
+ ? 'assertion has no "after_step"; name the step it runs after'
46
+ : `assertion runs after step "${f.ref}", which no step declares`),
47
+ },
48
+ missing_target: {
49
+ blocking: true,
50
+ clause: (n) => `${n} step(s) have no target to resolve`,
51
+ message: (f) => `${f.action} has no "target" (or "target_pattern"), so there is no element to resolve`,
52
+ },
53
+ undeclared_capture_variable: {
54
+ blocking: true,
55
+ clause: (n) => `${n} variable reference(s) are not declared in "variables"`,
56
+ message: (f) => (f.ref === undefined
57
+ ? `"${f.field}" is required here but has no variable name`
58
+ : `"${f.field}" names variable "${f.ref}", which the root "variables" block does not declare`),
59
+ },
60
+ dangling_condition_step: {
61
+ blocking: true,
62
+ clause: (n) => `${n} condition(s) refer to a step that has not run`,
63
+ message: (f) => (f.ref === undefined
64
+ ? 'previous_step_skipped condition has no "step_id"; name an earlier step'
65
+ : `previous_step_skipped condition names step "${f.ref}", which is not an earlier step`),
66
+ },
67
+ missing_wait_config: {
68
+ blocking: false,
69
+ clause: (n) => `${n} wait step(s) have no wait_config; the default timeout applies`,
70
+ message: (f) => `${f.action} has no "wait_config", so it waits for the default timeout; `
71
+ + 'set "wait_config" to make the wait explicit',
72
+ },
73
+ };
74
+
75
+ const isNonEmptyString = (v) => typeof v === 'string' && v.length > 0;
76
+
77
+ // Indexes the flattened step list (see `steps.js`): `after_step` and duplicate
78
+ // detection need every step, and a condition's step must come earlier, i.e.
79
+ // at a lower position in `entries`.
80
+ function referenceIndex(scenario, entries) {
81
+ const firstPathById = new Map();
82
+ const orderById = new Map();
83
+ entries.forEach(({ step, path }, position) => {
84
+ if (typeof step.id === 'string' && !firstPathById.has(step.id)) {
85
+ firstPathById.set(step.id, path);
86
+ orderById.set(step.id, position);
87
+ }
88
+ });
89
+
90
+ const vars = scenario && scenario.variables;
91
+ const variables = new Set(vars && typeof vars === 'object' && !Array.isArray(vars) ? Object.keys(vars) : []);
92
+ return { firstPathById, orderById, variables };
93
+ }
94
+
95
+ // Descriptors for one step; the caller supplies its path, action and id.
96
+ // `position` is its index in the flattened step list, the same numbering
97
+ // `orderById` uses.
98
+ function stepReferences(step, position, index) {
99
+ const findings = [];
100
+ const add = (code, fields) => findings.push({ code, fields });
101
+
102
+ const firstPath = index.firstPathById.get(step.id);
103
+ if (firstPath !== undefined && index.orderById.get(step.id) !== position) {
104
+ add('duplicate_step_id', { ref: step.id, firstPath });
105
+ }
106
+
107
+ if (TARGET_ACTIONS.has(step.action) && !isNonEmptyString(step.target) && !isNonEmptyString(step.target_pattern)) {
108
+ add('missing_target', {});
109
+ }
110
+
111
+ if (WAIT_ACTIONS.has(step.action) && !step.wait_config) {
112
+ add('missing_wait_config', {});
113
+ }
114
+
115
+ if (step.capture_to !== undefined && !index.variables.has(step.capture_to)) {
116
+ add('undeclared_capture_variable', { field: 'capture_to', ref: step.capture_to });
117
+ }
118
+
119
+ const cond = step.condition;
120
+ if (cond && typeof cond === 'object') {
121
+ const fields = variableFields('condition.variable_name', cond.variable_name,
122
+ CONDITION_TYPES_NEEDING_VARIABLE.has(cond.type), index);
123
+ if (fields) add('undeclared_capture_variable', fields);
124
+ if (cond.type === 'previous_step_skipped') {
125
+ const order = index.orderById.get(cond.step_id);
126
+ if (order === undefined || order >= position) {
127
+ add('dangling_condition_step', { ref: cond.step_id });
128
+ }
129
+ }
130
+ }
131
+
132
+ return findings;
133
+ }
134
+
135
+ // Descriptors for the root `assertions` array, in order, each with its path
136
+ // and `assertion_id`.
137
+ function assertionReferences(scenario, index) {
138
+ const assertions = scenario && scenario.assertions;
139
+ if (!Array.isArray(assertions)) return [];
140
+ const findings = [];
141
+ assertions.forEach((a, i) => {
142
+ if (!a) return;
143
+ const path = `assertions[${i}]`;
144
+ const ids = a.id !== undefined ? { assertion_id: a.id } : {};
145
+ if (!index.firstPathById.has(a.after_step)) {
146
+ findings.push({ code: 'dangling_after_step', fields: { ref: a.after_step }, path, ids });
147
+ }
148
+ const fields = variableFields('variable_name', a.variable_name,
149
+ ASSERTION_TYPES_NEEDING_VARIABLE.has(a.type), index);
150
+ if (fields) findings.push({ code: 'undeclared_capture_variable', fields, path, ids });
151
+ });
152
+ return findings;
153
+ }
154
+
155
+ module.exports = {
156
+ REFERENCE_CODES,
157
+ referenceIndex,
158
+ stepReferences,
159
+ assertionReferences,
160
+ };
@@ -1,7 +1,7 @@
1
1
  'use strict';
2
2
 
3
3
  // Replayability check for scenarios. Schema validity alone overstates what a
4
- // scenario will do on replay; this module reports two kinds of gap:
4
+ // scenario will do on replay; this module reports three kinds of gap:
5
5
  //
6
6
  // - `unsupported_action` (#166): the schema accepts every action in its enum,
7
7
  // but the action catalog marks some `unsupported`. no `mauto` verb reaches
@@ -11,17 +11,31 @@
11
11
  // `warnWhenUnset`. A step after the first that leaves the marked field to its
12
12
  // schema default is ambiguous about the author's intent. Advisory: it never
13
13
  // fails `--strict`.
14
+ // - Referential gaps (#272): duplicate step ids, dangling references, steps
15
+ // with no target, undeclared variables, waits with no `wait_config`. The
16
+ // `references` module owns those codes; this one merges its findings into
17
+ // the single walk over `flattenSteps`, so `CODES` stays the one table
18
+ // every consumer reads.
14
19
  //
15
20
  // The unsupported set is derived from the catalog, never restated here, so it
16
21
  // moves when an action gains (or loses) a device primitive.
17
22
 
18
23
  const { actionsByResolution, implicitStepFlags } = require('../device/action-catalog');
24
+ const {
25
+ REFERENCE_CODES,
26
+ referenceIndex,
27
+ stepReferences,
28
+ assertionReferences,
29
+ } = require('./references');
30
+ const { flattenSteps } = require('./steps');
19
31
 
20
- // Severity and wording for every finding code live here and nowhere else.
21
- // `blocking` is the `--strict` policy: only a step that cannot run at all fails
22
- // validation. `clause(n)` is the hint fragment for n findings of the code;
23
- // `message(f)` is the per-finding text. Advisory codes other than
24
- // `unsupported_action` are the catalog's `warnWhenUnset` markers.
32
+ // Severity and wording for every finding code: the replay codes are defined
33
+ // here, the referential ones in `references.js`, and `CODES` assembles both, so
34
+ // it is the one table every consumer reads.
35
+ // `blocking` is the `--strict` policy: a blocking finding means the scenario
36
+ // cannot replay unattended as written: a step that cannot run or a broken
37
+ // reference. `clause(n)` is the hint fragment for n findings of the code;
38
+ // `message(f)` is the per-finding text.
25
39
  const CODES = {
26
40
  unsupported_action: {
27
41
  blocking: true,
@@ -35,61 +49,55 @@ const CODES = {
35
49
  + `set "${f.field}" explicitly (${JSON.stringify(f.value)} to return to the running app, `
36
50
  + `${JSON.stringify(!f.value)} to confirm the restart)`,
37
51
  },
52
+ ...REFERENCE_CODES,
38
53
  };
39
54
 
40
- function makeFinding(code, fields, path, stepId) {
41
- const f = { code, action: fields.action, path, message: CODES[code].message(fields) };
42
- if (stepId !== undefined) f.step_id = stepId;
43
- return f;
55
+ // The one constructor for a `data.warnings` entry: `{ code, action, path,
56
+ // message }` plus `step_id` or `assertion_id` from `ids`. `action` is omitted
57
+ // when the finding has none (an assertion).
58
+ function makeFinding(code, fields, path, ids = {}) {
59
+ const f = { code };
60
+ if (fields.action !== undefined) f.action = fields.action;
61
+ f.path = path;
62
+ f.message = CODES[code].message(fields);
63
+ return Object.assign(f, ids);
44
64
  }
45
65
 
46
- function unsupportedFinding(action, path, stepId) {
47
- return makeFinding('unsupported_action', { action }, path, stepId);
48
- }
66
+ const stepIds = (step) => (step.id !== undefined ? { step_id: step.id } : {});
49
67
 
50
- // One advisory per catalog marker: the step leaves `sf.field` unset, so the
51
- // field's default decides. The catalog owns action/field/value.
52
- function implicitFinding(action, sf, path, stepId) {
53
- return makeFinding(sf.warnWhenUnset, { action, field: sf.field, value: sf.value }, path, stepId);
54
- }
68
+ // Returns every finding in document order: steps first (including nested
69
+ // sub_steps, depth-first), then precondition device actions, then assertions.
70
+ function findReplayWarnings(scenario) {
71
+ const unsupported = new Set(actionsByResolution('unsupported'));
72
+ const entries = flattenSteps(scenario);
73
+ const index = referenceIndex(scenario, entries);
74
+ const findings = [];
55
75
 
56
- // Walks a step list depth-first in document order, descending into each
57
- // step's `sub_steps` (the schema types them as steps, so they nest freely).
58
- function scanSteps(steps, basePath, unsupported, findings, topLevel) {
59
- if (!Array.isArray(steps)) return;
60
- steps.forEach((step, i) => {
61
- if (!step) return;
62
- const path = `${basePath}[${i}]`;
63
- if (unsupported.has(step.action)) {
64
- findings.push(unsupportedFinding(step.action, path, step.id));
65
- }
76
+ entries.forEach(({ step, path, isFirstTopLevel }, position) => {
77
+ const ids = stepIds(step);
78
+ const add = (code, fields) => findings.push(makeFinding(code, { action: step.action, ...fields }, path, ids));
79
+ if (unsupported.has(step.action)) add('unsupported_action', {});
66
80
  // Only the scenario's first top-level step is unambiguously a fresh
67
81
  // start; any other step that leaves a marked field unset is worth a look.
68
- if (!(topLevel && i === 0)) {
82
+ // The catalog owns the marker's code, field and default value.
83
+ if (!isFirstTopLevel) {
69
84
  for (const sf of implicitStepFlags(step.action, step)) {
70
- findings.push(implicitFinding(step.action, sf, path, step.id));
85
+ add(sf.warnWhenUnset, { field: sf.field, value: sf.value });
71
86
  }
72
87
  }
73
- scanSteps(step.sub_steps, `${path}.sub_steps`, unsupported, findings, false);
88
+ for (const r of stepReferences(step, position, index)) add(r.code, r.fields);
74
89
  });
75
- }
76
-
77
- // Returns every finding in document order: steps first (including nested
78
- // sub_steps, depth-first), then precondition device actions.
79
- function findReplayWarnings(scenario) {
80
- const unsupported = new Set(actionsByResolution('unsupported'));
81
- const findings = [];
82
-
83
- scanSteps(scenario && scenario.steps, 'steps', unsupported, findings, true);
84
90
 
85
91
  const pre = scenario && scenario.preconditions;
86
92
  const deviceActions = pre && Array.isArray(pre.device_actions) ? pre.device_actions : [];
87
93
  deviceActions.forEach((da, i) => {
88
94
  if (da && unsupported.has(da.action)) {
89
- findings.push(unsupportedFinding(da.action, `preconditions.device_actions[${i}]`));
95
+ findings.push(makeFinding('unsupported_action', { action: da.action }, `preconditions.device_actions[${i}]`));
90
96
  }
91
97
  });
92
98
 
99
+ for (const r of assertionReferences(scenario, index)) findings.push(makeFinding(r.code, r.fields, r.path, r.ids));
100
+
93
101
  return findings;
94
102
  }
95
103
 
@@ -0,0 +1,27 @@
1
+ 'use strict';
2
+
3
+ // The one walk over a scenario's step tree. Steps nest freely through
4
+ // `sub_steps` (the schema types them as steps), and every check that cares
5
+ // about step order reads it from this list: an entry's index is its position
6
+ // in depth-first document order, so "earlier step" means a lower index.
7
+
8
+ // Returns `[{ step, path, isFirstTopLevel }]`, parents before their sub_steps.
9
+ // Null steps and non-array step lists are skipped; a skipped step keeps its
10
+ // slot in the paths of its siblings. Only the scenario's first top-level step
11
+ // is `isFirstTopLevel`.
12
+ function flattenSteps(scenario) {
13
+ const entries = [];
14
+ const walk = (steps, basePath, topLevel) => {
15
+ if (!Array.isArray(steps)) return;
16
+ steps.forEach((step, i) => {
17
+ if (!step) return;
18
+ const path = `${basePath}[${i}]`;
19
+ entries.push({ step, path, isFirstTopLevel: topLevel && i === 0 });
20
+ walk(step.sub_steps, `${path}.sub_steps`, false);
21
+ });
22
+ };
23
+ walk(scenario && scenario.steps, 'steps', true);
24
+ return entries;
25
+ }
26
+
27
+ module.exports = { flattenSteps };