mobile-automator 0.38.4 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +2 -0
  2. package/package.json +4 -2
  3. package/src/assertion/evaluator.js +116 -14
  4. package/src/assertion/fields.js +106 -0
  5. package/src/assertion/observe.js +17 -0
  6. package/src/cli.js +180 -174
  7. package/src/device/action-catalog.js +4 -2
  8. package/src/device/bridge.js +17 -1
  9. package/src/device/mobile-mcp-tools.js +1 -0
  10. package/src/device/resolve-connection.js +1 -1
  11. package/src/device/semantic-press/index.js +19 -0
  12. package/src/device/semantic-press/resolver.js +1 -1
  13. package/src/{device/semantic-press/labels.js → dialog/permission-labels.js} +6 -3
  14. package/src/guide/content/execute.agnostic.md +74 -21
  15. package/src/guide/content/execute.aware.md +69 -15
  16. package/src/guide/content/execute.invariants.md +1 -0
  17. package/src/guide/content/generate.agnostic.md +3 -3
  18. package/src/guide/content/generate.aware.md +5 -5
  19. package/src/guide/emitter.js +2 -0
  20. package/src/init/host-plan.js +1 -0
  21. package/src/observe/trace.js +1 -1
  22. package/src/observe/vocab.js +6 -0
  23. package/src/output/device-fail.js +41 -0
  24. package/src/output/envelope.js +3 -3
  25. package/src/output/run-human.js +37 -0
  26. package/src/result/capability-catalog.js +48 -0
  27. package/src/result/executor.js +58 -0
  28. package/src/result/finalize-against-scenario.js +103 -0
  29. package/src/result/flags.js +49 -6
  30. package/src/result/junit.js +39 -16
  31. package/src/result/outcome.js +24 -2
  32. package/src/result/pending-hint.js +67 -0
  33. package/src/result/scenario-check.js +58 -0
  34. package/src/result/status-guard.js +46 -0
  35. package/src/result/store.js +159 -16
  36. package/src/run/actions/capture.js +21 -0
  37. package/src/run/actions/gestures.js +89 -0
  38. package/src/run/actions/index.js +48 -0
  39. package/src/run/actions/launch.js +30 -0
  40. package/src/run/actions/preconditions.js +44 -0
  41. package/src/run/actions/scroll.js +22 -0
  42. package/src/run/actions/semantic.js +16 -0
  43. package/src/run/actions/wait.js +66 -0
  44. package/src/run/assert-reference.js +118 -0
  45. package/src/run/assertions.js +102 -0
  46. package/src/run/clock.js +10 -0
  47. package/src/run/condition.js +80 -0
  48. package/src/run/fatal.js +15 -0
  49. package/src/run/handle.js +151 -0
  50. package/src/run/needs-agent.js +39 -0
  51. package/src/run/poll.js +38 -0
  52. package/src/run/prepare.js +107 -0
  53. package/src/run/reference-assertion.js +172 -0
  54. package/src/run/reference-screenshot.js +92 -0
  55. package/src/run/resolve.js +95 -0
  56. package/src/run/run-id.js +14 -0
  57. package/src/run/runner.js +104 -0
  58. package/src/run/scale.js +52 -0
  59. package/src/run/screenshots.js +163 -0
  60. package/src/run/settings.js +35 -0
  61. package/src/run/state.js +22 -0
  62. package/src/run/step.js +138 -0
  63. package/src/run/summary.js +73 -0
  64. package/src/run/trace.js +30 -0
  65. package/src/run/tree.js +96 -0
  66. package/src/run/variables.js +56 -0
  67. package/src/scenario/load.js +78 -0
  68. package/src/schemas/config_schema.json +29 -0
  69. package/src/schemas/result_schema.json +45 -3
  70. package/src/schemas/scenario_schema.json +27 -0
  71. package/src/util/as-array.js +8 -0
  72. package/src/util/parse-positive-int.js +15 -0
  73. package/src/util/parse-unit-interval.js +17 -0
@@ -0,0 +1,41 @@
1
+ 'use strict';
2
+
3
+ // The device-kind failure envelope, shared by every verb handler that touches
4
+ // the device (src/cli.js and the handlers split out of it, such as
5
+ // src/run/assert-reference.js), so a device fault reads the same from all of
6
+ // them.
7
+
8
+ const { fail } = require('./envelope');
9
+ const { engineHint } = require('../device/engine-hints');
10
+
11
+ // fs error codes that signal an environment/filesystem problem (perms, missing
12
+ // path, read-only FS, disk full) rather than a logic bug — surfaced as
13
+ // `environment` with a filesystem-specific hint.
14
+ const FS_ERROR_CODES = new Set(['EACCES', 'ENOENT', 'EROFS', 'ENOSPC', 'EPERM']);
15
+
16
+ // The hint for any device-kind failure. Precedence: an explicit err.hint wins
17
+ // (our own code attached it knowing the exact cause — a DeviceResolutionError,
18
+ // the daemon-log pointer, a timeout); then engineHint, which recognises engine
19
+ // messages that are useless on their own (the 1.0.5 engine's missing iOS agent
20
+ // and changed Android device ids, #199); then the handler's generic fallback.
21
+ // Every device fail() goes through here so no handler can miss it.
22
+ function deviceHint(err, fallback) {
23
+ return (err && err.hint) || engineHint(err && err.message) || fallback;
24
+ }
25
+
26
+ function deviceFail(err) {
27
+ // Respect a typed error's kind (e.g. the daemon's 'timeout') so the envelope
28
+ // and exit code reflect the real failure class; anything else defaults to
29
+ // 'device' (exit 2) as before.
30
+ const kind = err.kind || 'device';
31
+ return {
32
+ envelope: fail(
33
+ kind,
34
+ err.message || String(err),
35
+ deviceHint(err, 'Ensure a device or simulator is connected and the app is running.')
36
+ ),
37
+ exitKind: kind,
38
+ };
39
+ }
40
+
41
+ module.exports = { FS_ERROR_CODES, deviceHint, deviceFail };
@@ -18,9 +18,9 @@ const EXIT = {
18
18
  // under test failed. Distinct from 1-5 so CI can tell "the app is broken"
19
19
  // from "the tooling is broken" by exit code alone, without parsing JSON.
20
20
  TEST_FAILED: 6,
21
- // Reserved for replay (`mauto run`, and `result finalize` once it computes
22
- // it): the run is not decided yet because something still needs agent
23
- // judgment. Nothing produces it yet.
21
+ // The run is not decided yet because something still needs agent judgment:
22
+ // `result finalize` (and `mauto run`) when the finalized status is
23
+ // `incomplete` (#275).
24
24
  INCOMPLETE: 7,
25
25
  CANCEL: 130,
26
26
  };
@@ -0,0 +1,37 @@
1
+ 'use strict';
2
+
3
+ // `mauto run --human` (replay R6, #282): the run's envelope as a short summary
4
+ // a person reads in a CI log. src/cli.js attaches it only to what handleRun
5
+ // returns, and every such envelope carries the run's frozen data
6
+ // (src/run/summary.js); a refusal, a connect failure or a thrown error never
7
+ // reaches it and takes the generic render. JSON output never comes through
8
+ // here.
9
+
10
+ const secondsOf = (value) => `${Math.round((Number(value) || 0) * 10) / 10}s`;
11
+
12
+ function pendingLine({ assertion_id: id, type, step_id: step }) {
13
+ const detail = [type, step == null ? null : `after ${step}`].filter((v) => v != null);
14
+ return ` - ${id}${detail.length ? ` (${detail.join(', ')})` : ''}`;
15
+ }
16
+
17
+ function renderRunHuman(envelope) {
18
+ const d = envelope.data;
19
+ const s = d.steps;
20
+ const a = d.assertions;
21
+ const lines = [];
22
+ if (!envelope.ok && envelope.error) lines.push(`error [${envelope.error.kind}]: ${envelope.error.message}`);
23
+ lines.push(`run ${d.run_id} (${d.scenario_id}): ${d.status} in ${secondsOf(d.duration_seconds)}`);
24
+ lines.push(
25
+ `steps: ${s.total} total, ${s.passed} passed, ${s.failed} failed, ${s.error} error, ${s.skipped} skipped, ${s.needs_agent} needs agent`
26
+ );
27
+ lines.push(`assertions: ${a.total} total, ${a.passed} passed, ${a.failed} failed, ${a.needs_agent} needs agent`);
28
+ if (d.failed_step) lines.push(`failed step: ${d.failed_step}`);
29
+ if (d.failures.length) lines.push('failures:', ...d.failures.map((f) => ` - ${f}`));
30
+ if (d.pending.length) lines.push('pending:', ...d.pending.map(pendingLine));
31
+ lines.push(`result: ${d.result_path}`);
32
+ if (d.junit_path) lines.push(`junit: ${d.junit_path}`);
33
+ if (envelope.hint) lines.push(`hint: ${envelope.hint}`);
34
+ return lines.join('\n');
35
+ }
36
+
37
+ module.exports = { renderRunHuman };
@@ -82,6 +82,44 @@ const RESULT_CAPABILITIES = {
82
82
  writeCheck: 'substring',
83
83
  schemaPointer: '/properties/assertion_results',
84
84
  },
85
+ // Who decided a verdict (#275). `--pass pending` reuses assertion_verdict's
86
+ // --pass; `type`, `step_id` and the derived `mechanical` ride on its
87
+ // --type/--step-id. Behavioral: `judge` is also a parameter name.
88
+ assertion_provenance: {
89
+ verb: 'add-assertion',
90
+ flags: ['--judge'],
91
+ store: 'addAssertion',
92
+ writes: 'judge',
93
+ writeCheck: 'behavioral',
94
+ schemaPointer: '/properties/assertion_results/items/properties/judge',
95
+ },
96
+ // A screenshot_match's comparison (#281): the images compared and the
97
+ // similarity measured. `mauto run` writes them in-process; an agent copies
98
+ // them from `mauto assert screenshot_match`'s hint.
99
+ assertion_reference_screenshot: {
100
+ verb: 'add-assertion',
101
+ flags: ['--reference-screenshot'],
102
+ store: 'addAssertion',
103
+ writes: 'entry.reference_screenshot = String(reference_screenshot)',
104
+ writeCheck: 'substring',
105
+ schemaPointer: '/properties/assertion_results/items/properties/reference_screenshot',
106
+ },
107
+ assertion_actual_screenshot: {
108
+ verb: 'add-assertion',
109
+ flags: ['--actual-screenshot'],
110
+ store: 'addAssertion',
111
+ writes: 'entry.actual_screenshot = String(actual_screenshot)',
112
+ writeCheck: 'substring',
113
+ schemaPointer: '/properties/assertion_results/items/properties/actual_screenshot',
114
+ },
115
+ assertion_similarity_score: {
116
+ verb: 'add-assertion',
117
+ flags: ['--similarity-score'],
118
+ store: 'addAssertion',
119
+ writes: 'entry.similarity_score = Number(similarity_score)',
120
+ writeCheck: 'substring',
121
+ schemaPointer: '/properties/assertion_results/items/properties/similarity_score',
122
+ },
85
123
  run_status: {
86
124
  verb: 'finalize',
87
125
  flags: ['--status'],
@@ -119,6 +157,16 @@ const RESULT_CAPABILITIES = {
119
157
  writeCheck: 'behavioral',
120
158
  schemaPointer: '/properties/summary',
121
159
  },
160
+ // Who produced the result and from what (#275). `finalize` always writes
161
+ // one; --scenario supplies the checked scenario path and its hash.
162
+ executor: {
163
+ verb: 'finalize',
164
+ flags: ['--scenario'],
165
+ store: 'finalize',
166
+ writes: 'executor',
167
+ writeCheck: 'behavioral',
168
+ schemaPointer: '/properties/executor',
169
+ },
122
170
  crash_record: {
123
171
  verb: 'add-crash',
124
172
  flags: ['--crash-id', '--process', '--crash-timestamp', '--excerpt', '--report-path'],
@@ -0,0 +1,58 @@
1
+ 'use strict';
2
+
3
+ // Who produced a result and from what (#275): the `executor` block a finalize
4
+ // records. Pure apart from reading the workspace config (for `engine` and
5
+ // `mode`) and package.json; shared by `result finalize` and `mauto run`.
6
+
7
+ const path = require('path');
8
+
9
+ const { version: MAUTO_VERSION } = require('../../package.json');
10
+ const manager = require('../config/manager');
11
+ const { readEngine } = require('../device/engine');
12
+ const resultSchema = require('../schemas/result_schema.json');
13
+
14
+ // The schema's executor.kind enum, read rather than restated.
15
+ const EXECUTOR_KINDS = resultSchema.properties.executor.properties.kind.enum;
16
+
17
+ // What executor.scenario_path means, the one place it is decided: relative
18
+ // to the project root when the file is inside it (so the result stays
19
+ // meaningful on another checkout), else absolute. `absolute` must already be
20
+ // resolved — against the caller's cwd — since a relative path here could mean
21
+ // either directory.
22
+ function recordedScenarioPath(projectRoot, absolute) {
23
+ if (!path.isAbsolute(absolute)) {
24
+ throw new Error(`scenario path must be absolute, got "${absolute}" (resolve it against the cwd first)`);
25
+ }
26
+ const relative = path.relative(projectRoot, absolute);
27
+ return relative && !relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)
28
+ ? relative
29
+ : absolute;
30
+ }
31
+
32
+ // The `executor` block a finalize records. `engine` resolves as the device
33
+ // layer resolves it (an exported MOBILEMCP_LEGACY_ROBOT wins over config), and
34
+ // `mode` as the guides resolve it (unset is platform-aware). The device fields
35
+ // are whatever the caller connected to — null when it connected to none.
36
+ // `scenarioPath` is absolute (or null); it is recorded per
37
+ // recordedScenarioPath, so `mauto run` and `result finalize --scenario` record
38
+ // the same value for the same file.
39
+ function buildExecutor({
40
+ kind, projectRoot, platform = null, deviceId = null, scenarioPath = null, scenarioSha256 = null, env = process.env,
41
+ }) {
42
+ if (!EXECUTOR_KINDS.includes(kind)) {
43
+ throw new Error(`unknown executor kind "${kind}" (expected ${EXECUTOR_KINDS.join(' | ')})`);
44
+ }
45
+ const recordedPath = scenarioPath == null ? null : recordedScenarioPath(projectRoot, scenarioPath);
46
+ return {
47
+ kind,
48
+ mauto_version: MAUTO_VERSION,
49
+ engine: readEngine(projectRoot, { env }).engine,
50
+ platform,
51
+ device_id: deviceId,
52
+ scenario_path: recordedPath,
53
+ scenario_sha256: scenarioSha256,
54
+ mode: manager.resolveMode(manager.load(projectRoot)),
55
+ };
56
+ }
57
+
58
+ module.exports = { buildExecutor, recordedScenarioPath };
@@ -0,0 +1,103 @@
1
+ 'use strict';
2
+
3
+ // The scenario-aware half of `mauto result finalize` (#275): load and hash the
4
+ // scenario, cross-check the recorded run against it, guard an explicit
5
+ // --status, build the executor, finalize, and map the finalized status to an
6
+ // envelope. Flag parsing stays with the caller. `finalizeRun` is injected so
7
+ // this module never requires src/cli.js, which owns it.
8
+
9
+ const path = require('path');
10
+
11
+ const { ok, fail } = require('../output/envelope');
12
+ const { loadScenario } = require('../scenario/load');
13
+ const { scenarioMismatch } = require('./scenario-check');
14
+ const { buildExecutor } = require('./executor');
15
+ const { incompleteEnvelope } = require('./pending-hint');
16
+ const { checkExplicitStatus } = require('./status-guard');
17
+
18
+ function finalizeAgainstScenario(
19
+ { validator, resultStoreFactory, memoryStoreFactory, projectRoot, cwd = process.cwd(), finalizeRun },
20
+ opts
21
+ ) {
22
+ const { runId, scenarioId, status, duration } = opts;
23
+
24
+ // --scenario (#275): the result is checked against the file it claims and
25
+ // the file is hashed into `executor`. Refused before anything is written.
26
+ // Resolved against the caller's cwd, as handleScreenshot resolves its path;
27
+ // buildExecutor decides how it is recorded.
28
+ let scenarioPath = null;
29
+ let scenarioSha256 = null;
30
+ let scenarioAssertionCount;
31
+ if (opts.scenario !== undefined) {
32
+ const scenarioFile = path.resolve(cwd, opts.scenario);
33
+ const loaded = loadScenario(validator, scenarioFile, { strict: false });
34
+ if (loaded.failure) return loaded.failure;
35
+ const mismatch = scenarioMismatch(resultStoreFactory({ runId, scenarioId, projectRoot }).recorded(), loaded);
36
+ if (mismatch) {
37
+ return { envelope: fail('invalid_input', mismatch.message, mismatch.hint, mismatch.data), exitKind: 'invalid_input' };
38
+ }
39
+ scenarioPath = scenarioFile;
40
+ scenarioSha256 = loaded.sha256;
41
+ scenarioAssertionCount = Array.isArray(loaded.scenario.assertions) ? loaded.scenario.assertions.length : 0;
42
+ }
43
+
44
+ // An explicit --status may equal the derived one or make the run worse,
45
+ // never greener (src/result/status-guard.js). Refused before anything is
46
+ // written.
47
+ if (status !== undefined) {
48
+ const recorded = resultStoreFactory({ runId, scenarioId, projectRoot }).recorded();
49
+ const refusal = checkExplicitStatus({
50
+ status, steps: recorded.steps, assertions: recorded.assertions, scenarioAssertionCount,
51
+ });
52
+ if (refusal) {
53
+ return { envelope: fail('invalid_input', refusal.message, refusal.hint), exitKind: 'invalid_input' };
54
+ }
55
+ }
56
+
57
+ // Only provided keys: `defaultMetadata` fills the rest with 'unknown', and
58
+ // the result schema requires all five fields to be strings, so threading
59
+ // `undefined` through would break conformance.
60
+ const metadata = {};
61
+ if (opts.appVersion !== undefined) metadata.app_version = opts.appVersion;
62
+ if (opts.deviceModel !== undefined) metadata.device_model = opts.deviceModel;
63
+ if (opts.apiLevel !== undefined) metadata.api_level = opts.apiLevel;
64
+ if (opts.environment !== undefined) metadata.environment = opts.environment;
65
+
66
+ const { result, store, hints } = finalizeRun(
67
+ { resultStoreFactory, memoryStoreFactory, projectRoot },
68
+ {
69
+ runId, scenarioId, status, duration, metadata, summary: opts.summary,
70
+ // No device is connected here, so none is claimed (plan decision 5).
71
+ executor: buildExecutor({ kind: 'agent', projectRoot, scenarioPath, scenarioSha256 }),
72
+ }
73
+ );
74
+
75
+ // Exit 7: nothing failed, but an agent still has to judge something. The
76
+ // file is written; the hint is the exact commands that complete it.
77
+ if (result.status === 'incomplete') {
78
+ return { envelope: incompleteEnvelope(result, { hints, projectRoot, data: result }), exitKind: 'incomplete' };
79
+ }
80
+
81
+ // The exit code is keyed on the FINALIZED status, not the --status flag:
82
+ // with the flag omitted, the store derives it from the recorded steps and
83
+ // assertions ('error' for an errored step, else 'failed' for a failed step or
84
+ // assertion, #296), and the exit code must agree with the file either way
85
+ // (#273). 'error' takes the same path — a run that errored is not a pass for
86
+ // CI either; data.status tells the two apart. Everything above (write, retention, harvest) has
87
+ // already run: exit 6 is a verdict about the app, not a reason to lose the
88
+ // record of it.
89
+ if (result.status === 'failed' || result.status === 'error') {
90
+ const hint = hints.length
91
+ ? hints.join(' ')
92
+ : 'See data.steps_executed and data.assertion_results for what failed; ' +
93
+ `the result is saved to ${path.relative(projectRoot, store.file)}.`;
94
+ return {
95
+ envelope: fail('test_failed', `run ${runId} ${result.status}`, hint, result),
96
+ exitKind: 'test_failed',
97
+ };
98
+ }
99
+
100
+ return { envelope: ok(result, hints.length ? hints.join(' ') : null), exitKind: 'ok' };
101
+ }
102
+
103
+ module.exports = { finalizeAgainstScenario };
@@ -7,6 +7,8 @@
7
7
  // `error` into a fail('invalid_input', …) envelope so an unknown observation
8
8
  // type is rejected loudly instead of being silently dropped.
9
9
 
10
+ const { parseUnitInterval } = require('../util/parse-unit-interval');
11
+
10
12
  // Mirrors the enum at /properties/observations/items/properties/type in
11
13
  // src/schemas/result_schema.json. tests/lint/result-coverage.test.js asserts
12
14
  // the two never drift.
@@ -78,10 +80,49 @@ function parseStepStatus(raw) {
78
80
  return { value: normalized };
79
81
  }
80
82
 
81
- function parseBool(raw, flagName) {
82
- if (raw === 'true') return { value: true };
83
- if (raw === 'false') return { value: false };
84
- return { error: `${flagName} must be "true" or "false" (got "${raw}")` };
83
+ // `result add-assertion --pass` / `--judge` (#275, B3), parsed as one unit
84
+ // because the two flags constrain each other: `--pass` is a decided verdict or
85
+ // `pending` (recorded `needs_agent`, to be decided later by an agent).
86
+ //
87
+ // Returns `{ value: { status, judge } }` or `{ error }`. `status` is the
88
+ // assertion status to record; `judge` is undefined (no claim made — a caller
89
+ // that predates the flag writes exactly what it always wrote) or `{by}`. A
90
+ // pending verdict was decided by nobody, so it records `{by: 'none'}` and
91
+ // refuses `agent`; a decided verdict refuses `none`.
92
+ const VERDICT_STATUSES = { true: 'passed', false: 'failed', pending: 'needs_agent' };
93
+
94
+ // Who an agent may say decided a verdict (`--judge`). The schema's judge.by
95
+ // enum also has `mauto`, which only the in-process runner writes: an agent
96
+ // claiming mauto decided would be the self-attestation the field exists to
97
+ // expose.
98
+ const JUDGES = ['agent', 'none'];
99
+
100
+ function parseAssertionVerdict(passRaw, judgeRaw) {
101
+ if (!Object.prototype.hasOwnProperty.call(VERDICT_STATUSES, passRaw)) {
102
+ return { error: `--pass must be true | false | pending (got "${passRaw}")` };
103
+ }
104
+ if (judgeRaw !== undefined && !JUDGES.includes(judgeRaw)) {
105
+ return { error: `--judge must be ${JUDGES.join(' | ')} (got "${judgeRaw}")` };
106
+ }
107
+ const status = VERDICT_STATUSES[passRaw];
108
+ if (status === 'needs_agent') {
109
+ if (judgeRaw === 'agent') return { error: '--judge agent needs a verdict: use --pass true or --pass false' };
110
+ return { value: { status, judge: { by: 'none' } } };
111
+ }
112
+ if (judgeRaw === 'none') {
113
+ return { error: `--judge none means nobody decided, but --pass is ${passRaw}: use --pass pending or --judge agent` };
114
+ }
115
+ return { value: { status, judge: judgeRaw === 'agent' ? { by: 'agent' } : undefined } };
116
+ }
117
+
118
+ // `result add-assertion --similarity-score` (#281): a screenshot_match's
119
+ // measured similarity, as `mauto assert` reports it. Absent is `{ value: null }`
120
+ // (nothing to record); otherwise a number in [0, 1].
121
+ function parseSimilarityScore(raw) {
122
+ if (raw === undefined) return { value: null };
123
+ const value = parseUnitInterval(raw);
124
+ if (value === null) return { error: `--similarity-score must be a number from 0 to 1 (got "${raw}")` };
125
+ return { value };
85
126
  }
86
127
 
87
128
  // Mirrors the enum at /properties/status in src/schemas/result_schema.json;
@@ -90,7 +131,7 @@ function parseBool(raw, flagName) {
90
131
  // status from the recorded steps and assertions): an unlisted value used to be persisted
91
132
  // verbatim — a schema-invalid result file with a green exit — directly under
92
133
  // the exit-6 contract keyed on that status (#286 review).
93
- const RUN_STATUSES = ['passed', 'failed', 'error'];
134
+ const RUN_STATUSES = ['passed', 'failed', 'error', 'incomplete'];
94
135
 
95
136
  function parseRunStatus(raw) {
96
137
  if (!RUN_STATUSES.includes(raw)) {
@@ -103,7 +144,9 @@ module.exports = {
103
144
  OBSERVATION_TYPES,
104
145
  parseObservation,
105
146
  parseCapture,
106
- parseBool,
147
+ JUDGES,
148
+ parseAssertionVerdict,
149
+ parseSimilarityScore,
107
150
  RUN_STATUSES,
108
151
  parseRunStatus,
109
152
  STEP_STATUS_ALIASES,
@@ -12,17 +12,20 @@
12
12
  //
13
13
  // Mapping, one result at a time:
14
14
  // result -> <testsuite name=scenario_id, else run_id>
15
- // assertion_results[] -> <testcase name=assertion_id>; failed -> <failure>
15
+ // assertion_results[] -> <testcase name=assertion_id>; failed -> <failure>,
16
+ // needs_agent -> <skipped>
16
17
  // failed / error step -> <testcase name="step: <id>"> with <failure>
18
+ // needs_agent step -> <testcase name="step: <id>"> with <skipped>
17
19
  // run id, status, metadata -> <properties>
18
20
  //
19
- // It renders only what current writers produce: assertions are passed or
20
- // failed, and only failed or errored steps get a testcase, so there is no
21
- // JUnit skip element and no skip count. A step's `error` status is reported as
22
- // a <failure> of type step_error rather than a JUnit <error>, so the report's
23
- // counts stay {tests, failures} and `errors` is always 0.
21
+ // Only failed, errored or pending steps get a testcase. Pending (#275) is
22
+ // rendered skipped, not failed: CI gates on `incomplete`'s exit code 7, and
23
+ // the report shows what still needs an agent. A step's `error` status is
24
+ // reported as a <failure> of type step_error rather than a JUnit <error>, so
25
+ // the counts are {tests, failures, skipped} and `errors` is always 0.
24
26
 
25
- const { isErroredStep, isFailingAssertion, isFailingStep } = require('./outcome');
27
+ const { isErroredStep, isFailingAssertion, isFailingStep, isPendingAssertion, isPendingStep } = require('./outcome');
28
+ const { asArray } = require('../util/as-array');
26
29
 
27
30
  // Characters XML 1.0 forbids anywhere in a document: C0 controls other than
28
31
  // tab/LF/CR, lone surrogates, U+FFFE and U+FFFF. A crash excerpt or an OCR'd
@@ -64,7 +67,7 @@ const present = (v) => v !== undefined && v !== null && v !== '';
64
67
 
65
68
  // A container as an array of its object entries; anything else is empty.
66
69
  const isObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
67
- const asArray = (value) => (Array.isArray(value) ? value.filter(isObject) : []);
70
+ const objectsOf = (value) => asArray(value).filter(isObject);
68
71
 
69
72
  // Seconds as a plain decimal, rounded to the millisecond so summed floats do
70
73
  // not render as 50.50000000001.
@@ -95,6 +98,9 @@ function assertionCase(assertion) {
95
98
  message: assertion.message || `${assertion.assertion_id} failed`,
96
99
  body: body.join('\n'),
97
100
  };
101
+ } else if (isPendingAssertion(assertion)) {
102
+ const type = assertion.type ? ` (${assertion.type})` : '';
103
+ testcase.skipped = { message: `${assertion.assertion_id} needs agent judgment${type}` };
98
104
  }
99
105
  return testcase;
100
106
  }
@@ -114,6 +120,7 @@ function stepCase(step) {
114
120
  },
115
121
  };
116
122
  }
123
+ if (isPendingStep(step)) return { name, skipped: { message: `step ${step.step_id} needs agent judgment` } };
117
124
  return null;
118
125
  }
119
126
 
@@ -121,16 +128,24 @@ function suiteFor(result) {
121
128
  const scenario = result.scenario_id ?? result.run_id;
122
129
  const metadata = isObject(result.metadata) ? result.metadata : {};
123
130
  const cases = [
124
- ...asArray(result.assertion_results).map(assertionCase),
125
- ...asArray(result.steps_executed).map(stepCase).filter(Boolean),
131
+ ...objectsOf(result.assertion_results).map(assertionCase),
132
+ ...objectsOf(result.steps_executed).map(stepCase).filter(Boolean),
126
133
  ];
127
134
 
128
135
  // A red result must never render as a green report. When the run says
129
136
  // anything but passed yet recorded nothing that fails (e.g. an agent
130
137
  // finalized with --status failed after an abort, or a pre-0.38.0 "FAILED"),
131
- // the status itself becomes the failure.
132
- const isRed = result.status !== 'passed';
133
- if (isRed && !cases.some((c) => c.failure)) {
138
+ // the status itself becomes the failure. `incomplete` is not red: its
139
+ // pending cases are already skipped, and only when none was recorded does
140
+ // the status itself become a skipped case, so it never reads green either.
141
+ if (result.status === 'incomplete') {
142
+ if (!cases.some((c) => c.failure || c.skipped)) {
143
+ cases.push({
144
+ name: 'run status',
145
+ skipped: { message: 'result status is incomplete but no pending step or assertion was recorded' },
146
+ });
147
+ }
148
+ } else if (result.status !== 'passed' && !cases.some((c) => c.failure)) {
134
149
  cases.push({
135
150
  name: 'run status',
136
151
  failure: {
@@ -142,6 +157,7 @@ function suiteFor(result) {
142
157
  }
143
158
 
144
159
  const failures = cases.filter((c) => c.failure).length;
160
+ const skipped = cases.filter((c) => c.skipped).length;
145
161
 
146
162
  const lines = [];
147
163
  lines.push(
@@ -149,6 +165,7 @@ function suiteFor(result) {
149
165
  ['name', scenario],
150
166
  ['tests', cases.length],
151
167
  ['failures', failures],
168
+ ['skipped', skipped],
152
169
  ['errors', 0],
153
170
  ['time', seconds(result.duration_seconds)],
154
171
  ['timestamp', present(metadata.timestamp) ? metadata.timestamp : null],
@@ -167,21 +184,26 @@ function suiteFor(result) {
167
184
  lines.push(`${open}>`);
168
185
  lines.push(` <failure${attrs([['message', message], ['type', type]])}>${escapeText(body)}</failure>`);
169
186
  lines.push(' </testcase>');
187
+ } else if (c.skipped) {
188
+ lines.push(`${open}>`);
189
+ lines.push(` <skipped${attrs([['message', c.skipped.message]])}/>`);
190
+ lines.push(' </testcase>');
170
191
  } else {
171
192
  lines.push(`${open}/>`);
172
193
  }
173
194
  }
174
195
  lines.push(' </testsuite>');
175
196
 
176
- return { lines, tests: cases.length, failures, time: Number(seconds(result.duration_seconds)) };
197
+ return { lines, tests: cases.length, failures, skipped, time: Number(seconds(result.duration_seconds)) };
177
198
  }
178
199
 
179
- // toJUnit(results) -> { xml, suites, tests, failures }
200
+ // toJUnit(results) -> { xml, suites, tests, failures, skipped }
180
201
  function toJUnit(results) {
181
202
  const suites = results.map(suiteFor);
182
203
  const total = (key) => suites.reduce((sum, s) => sum + s[key], 0);
183
204
  const tests = total('tests');
184
205
  const failures = total('failures');
206
+ const skipped = total('skipped');
185
207
 
186
208
  const xml = [
187
209
  '<?xml version="1.0" encoding="UTF-8"?>',
@@ -189,6 +211,7 @@ function toJUnit(results) {
189
211
  ['name', 'mauto'],
190
212
  ['tests', tests],
191
213
  ['failures', failures],
214
+ ['skipped', skipped],
192
215
  ['errors', 0],
193
216
  ['time', seconds(total('time'))],
194
217
  ])}>`,
@@ -197,7 +220,7 @@ function toJUnit(results) {
197
220
  '',
198
221
  ].join('\n');
199
222
 
200
- return { xml, suites: suites.length, tests, failures };
223
+ return { xml, suites: suites.length, tests, failures, skipped };
201
224
  }
202
225
 
203
226
  module.exports = { toJUnit };
@@ -12,22 +12,44 @@
12
12
 
13
13
  const statusOf = (entry) => (entry !== null && typeof entry === 'object' ? entry.status : undefined);
14
14
 
15
+ const isPassedStep = (step) => statusOf(step) === 'passed';
16
+ const isSkippedStep = (step) => statusOf(step) === 'skipped';
15
17
  const isFailedStep = (step) => statusOf(step) === 'failed';
16
18
  const isErroredStep = (step) => statusOf(step) === 'error';
17
19
  const isFailingStep = (step) => isFailedStep(step) || isErroredStep(step);
20
+ const isPassedAssertion = (assertion) => statusOf(assertion) === 'passed';
18
21
  const isFailingAssertion = (assertion) => statusOf(assertion) === 'failed';
19
22
 
23
+ // Pending (#275): recorded, but not decided, because only an agent can judge
24
+ // it. An assertion says so with its status; a step keeps its own status and
25
+ // carries `needs_agent: true` beside it.
26
+ const isPendingAssertion = (assertion) => statusOf(assertion) === 'needs_agent';
27
+ const isPendingStep = (step) => step !== null && typeof step === 'object' && step.needs_agent === true;
28
+
20
29
  // The run status `result finalize` derives when `--status` is omitted (#296).
21
30
  // It used to read assertions only, so a run whose step failed finalized
22
31
  // `passed` while its JUnit export was red. Every entry counts — a step
23
32
  // recorded twice (fail, then pass) still failed once, matching the export.
24
33
  // Skipped steps count for nothing; a run with no steps and no assertions
25
- // stays `passed` (#275 owns that case).
34
+ // stays `passed`. Anything failing outranks anything pending: a run that is
35
+ // already red is decided, whatever an agent later says (#275).
26
36
  function deriveRunStatus(steps = [], assertions = []) {
27
37
  if (steps.some(isErroredStep)) return 'error';
28
38
  if (steps.some(isFailedStep)) return 'failed';
29
39
  if (assertions.some(isFailingAssertion)) return 'failed';
40
+ if (steps.some(isPendingStep) || assertions.some(isPendingAssertion)) return 'incomplete';
30
41
  return 'passed';
31
42
  }
32
43
 
33
- module.exports = { isFailedStep, isErroredStep, isFailingStep, isFailingAssertion, deriveRunStatus };
44
+ module.exports = {
45
+ isPassedStep,
46
+ isSkippedStep,
47
+ isFailedStep,
48
+ isErroredStep,
49
+ isFailingStep,
50
+ isPassedAssertion,
51
+ isFailingAssertion,
52
+ isPendingAssertion,
53
+ isPendingStep,
54
+ deriveRunStatus,
55
+ };
@@ -0,0 +1,67 @@
1
+ 'use strict';
2
+
3
+ // What an agent must do to complete a run that is waiting for its judgment
4
+ // (#275): the pending assertions, the `incomplete` message and hint, and the
5
+ // envelope built from them. Shared by `result finalize` and `mauto run`.
6
+
7
+ const path = require('path');
8
+
9
+ const { fail } = require('../output/envelope');
10
+ const { isPendingAssertion, isPendingStep } = require('./outcome');
11
+
12
+ // The assertions a finalized result is still waiting on, as `{assertion_id,
13
+ // step_id, type}` — what `mauto run` puts in data.pending.
14
+ function pendingAssertions(result) {
15
+ return (result.assertion_results || [])
16
+ .filter(isPendingAssertion)
17
+ .map(({ assertion_id, step_id = null, type = null }) => ({ assertion_id, step_id, type }));
18
+ }
19
+
20
+ // The error message of an `incomplete` run, shared by `result finalize` and
21
+ // `mauto run`. Counts the pending assertions; a run pending only on a step
22
+ // names no count.
23
+ function incompleteMessage(result) {
24
+ const pending = pendingAssertions(result).length;
25
+ return `run ${result.run_id} incomplete${pending ? `: ${pending} assertion(s) need agent judgment` : ''}`;
26
+ }
27
+
28
+ // A flag value as it must be typed into a shell: bare when that is safe, else
29
+ // JSON-quoted.
30
+ const arg = (value) => (/^[\w./@:+-]+$/.test(String(value)) ? String(value) : JSON.stringify(String(value)));
31
+
32
+ // The hint for an `incomplete` run: the pending ids, the exact
33
+ // `result add-assertion` line per pending assertion (only the verdict and the
34
+ // reason left to fill), then the re-finalize. The scenario path is the one
35
+ // the executor recorded, so the re-finalize is checked against the same file,
36
+ // printed absolute (resolved against `projectRoot`): --scenario resolves
37
+ // against the cwd while the workspace is discovered by walking up from it, so
38
+ // a project-relative path would be wrong from any subdirectory.
39
+ function pendingHint(result, { projectRoot }) {
40
+ const pending = pendingAssertions(result);
41
+ const pendingSteps = (result.steps_executed || []).filter(isPendingStep);
42
+ const runId = arg(result.run_id);
43
+ const recorded = result.executor && result.executor.scenario_path;
44
+ const scenarioPath = recorded ? path.resolve(projectRoot, recorded) : null;
45
+ return [
46
+ ...pendingSteps.map((s) => `Pending step ${s.step_id}: ${s.error_message || 'needs an agent'}`),
47
+ pending.length
48
+ ? `Pending agent judgment: ${pending.map((p) => p.assertion_id).join(', ')}. Decide each, then re-finalize:`
49
+ : 'No assertion is pending; re-finalize once the run is decided:',
50
+ ...pending.map(
51
+ (p) =>
52
+ `mauto result add-assertion --run-id ${runId} --step-id ${p.step_id == null ? '<step_id>' : arg(p.step_id)} ` +
53
+ `--assertion-id ${arg(p.assertion_id)} --type ${p.type == null ? '<type>' : arg(p.type)} ` +
54
+ '--pass <true|false> --judge agent --message "<why>"'
55
+ ),
56
+ `mauto result finalize --run-id ${runId} --scenario ${scenarioPath ? arg(scenarioPath) : '<scenario.json>'}`,
57
+ ].join('\n');
58
+ }
59
+
60
+ // The `incomplete` (exit 7) envelope, shared by `result finalize` and
61
+ // `mauto run`: the message, the store's hints, then the commands that complete
62
+ // the run. `data` is what the envelope carries.
63
+ function incompleteEnvelope(result, { hints, projectRoot, data }) {
64
+ return fail('incomplete', incompleteMessage(result), [...hints, pendingHint(result, { projectRoot })].join('\n'), data);
65
+ }
66
+
67
+ module.exports = { arg, pendingAssertions, pendingHint, incompleteMessage, incompleteEnvelope };