mobile-automator 0.38.4 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/package.json +4 -2
- package/src/assertion/evaluator.js +116 -14
- package/src/assertion/fields.js +106 -0
- package/src/assertion/observe.js +17 -0
- package/src/cli.js +180 -174
- package/src/device/action-catalog.js +4 -2
- package/src/device/bridge.js +17 -1
- package/src/device/mobile-mcp-tools.js +1 -0
- package/src/device/resolve-connection.js +1 -1
- package/src/device/semantic-press/index.js +19 -0
- package/src/device/semantic-press/resolver.js +1 -1
- package/src/{device/semantic-press/labels.js → dialog/permission-labels.js} +6 -3
- package/src/guide/content/execute.agnostic.md +74 -21
- package/src/guide/content/execute.aware.md +69 -15
- package/src/guide/content/execute.invariants.md +1 -0
- package/src/guide/content/generate.agnostic.md +3 -3
- package/src/guide/content/generate.aware.md +5 -5
- package/src/guide/emitter.js +2 -0
- package/src/init/host-plan.js +1 -0
- package/src/observe/trace.js +1 -1
- package/src/observe/vocab.js +6 -0
- package/src/output/device-fail.js +41 -0
- package/src/output/envelope.js +3 -3
- package/src/output/run-human.js +37 -0
- package/src/result/capability-catalog.js +48 -0
- package/src/result/executor.js +58 -0
- package/src/result/finalize-against-scenario.js +103 -0
- package/src/result/flags.js +49 -6
- package/src/result/junit.js +39 -16
- package/src/result/outcome.js +24 -2
- package/src/result/pending-hint.js +67 -0
- package/src/result/scenario-check.js +58 -0
- package/src/result/status-guard.js +46 -0
- package/src/result/store.js +159 -16
- package/src/run/actions/capture.js +21 -0
- package/src/run/actions/gestures.js +89 -0
- package/src/run/actions/index.js +48 -0
- package/src/run/actions/launch.js +30 -0
- package/src/run/actions/preconditions.js +44 -0
- package/src/run/actions/scroll.js +22 -0
- package/src/run/actions/semantic.js +16 -0
- package/src/run/actions/wait.js +66 -0
- package/src/run/assert-reference.js +118 -0
- package/src/run/assertions.js +102 -0
- package/src/run/clock.js +10 -0
- package/src/run/condition.js +80 -0
- package/src/run/fatal.js +15 -0
- package/src/run/handle.js +151 -0
- package/src/run/needs-agent.js +39 -0
- package/src/run/poll.js +38 -0
- package/src/run/prepare.js +107 -0
- package/src/run/reference-assertion.js +172 -0
- package/src/run/reference-screenshot.js +92 -0
- package/src/run/resolve.js +95 -0
- package/src/run/run-id.js +14 -0
- package/src/run/runner.js +104 -0
- package/src/run/scale.js +52 -0
- package/src/run/screenshots.js +163 -0
- package/src/run/settings.js +35 -0
- package/src/run/state.js +22 -0
- package/src/run/step.js +138 -0
- package/src/run/summary.js +73 -0
- package/src/run/trace.js +30 -0
- package/src/run/tree.js +96 -0
- package/src/run/variables.js +56 -0
- package/src/scenario/load.js +78 -0
- package/src/schemas/config_schema.json +29 -0
- package/src/schemas/result_schema.json +45 -3
- package/src/schemas/scenario_schema.json +27 -0
- package/src/util/as-array.js +8 -0
- package/src/util/parse-positive-int.js +15 -0
- package/src/util/parse-unit-interval.js +17 -0
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// The device-kind failure envelope, shared by every verb handler that touches
|
|
4
|
+
// the device (src/cli.js and the handlers split out of it, such as
|
|
5
|
+
// src/run/assert-reference.js), so a device fault reads the same from all of
|
|
6
|
+
// them.
|
|
7
|
+
|
|
8
|
+
const { fail } = require('./envelope');
|
|
9
|
+
const { engineHint } = require('../device/engine-hints');
|
|
10
|
+
|
|
11
|
+
// fs error codes that signal an environment/filesystem problem (perms, missing
|
|
12
|
+
// path, read-only FS, disk full) rather than a logic bug — surfaced as
|
|
13
|
+
// `environment` with a filesystem-specific hint.
|
|
14
|
+
const FS_ERROR_CODES = new Set(['EACCES', 'ENOENT', 'EROFS', 'ENOSPC', 'EPERM']);
|
|
15
|
+
|
|
16
|
+
// The hint for any device-kind failure. Precedence: an explicit err.hint wins
|
|
17
|
+
// (our own code attached it knowing the exact cause — a DeviceResolutionError,
|
|
18
|
+
// the daemon-log pointer, a timeout); then engineHint, which recognises engine
|
|
19
|
+
// messages that are useless on their own (the 1.0.5 engine's missing iOS agent
|
|
20
|
+
// and changed Android device ids, #199); then the handler's generic fallback.
|
|
21
|
+
// Every device fail() goes through here so no handler can miss it.
|
|
22
|
+
function deviceHint(err, fallback) {
|
|
23
|
+
return (err && err.hint) || engineHint(err && err.message) || fallback;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function deviceFail(err) {
|
|
27
|
+
// Respect a typed error's kind (e.g. the daemon's 'timeout') so the envelope
|
|
28
|
+
// and exit code reflect the real failure class; anything else defaults to
|
|
29
|
+
// 'device' (exit 2) as before.
|
|
30
|
+
const kind = err.kind || 'device';
|
|
31
|
+
return {
|
|
32
|
+
envelope: fail(
|
|
33
|
+
kind,
|
|
34
|
+
err.message || String(err),
|
|
35
|
+
deviceHint(err, 'Ensure a device or simulator is connected and the app is running.')
|
|
36
|
+
),
|
|
37
|
+
exitKind: kind,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
module.exports = { FS_ERROR_CODES, deviceHint, deviceFail };
|
package/src/output/envelope.js
CHANGED
|
@@ -18,9 +18,9 @@ const EXIT = {
|
|
|
18
18
|
// under test failed. Distinct from 1-5 so CI can tell "the app is broken"
|
|
19
19
|
// from "the tooling is broken" by exit code alone, without parsing JSON.
|
|
20
20
|
TEST_FAILED: 6,
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
21
|
+
// The run is not decided yet because something still needs agent judgment:
|
|
22
|
+
// `result finalize` (and `mauto run`) when the finalized status is
|
|
23
|
+
// `incomplete` (#275).
|
|
24
24
|
INCOMPLETE: 7,
|
|
25
25
|
CANCEL: 130,
|
|
26
26
|
};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// `mauto run --human` (replay R6, #282): the run's envelope as a short summary
|
|
4
|
+
// a person reads in a CI log. src/cli.js attaches it only to what handleRun
|
|
5
|
+
// returns, and every such envelope carries the run's frozen data
|
|
6
|
+
// (src/run/summary.js); a refusal, a connect failure or a thrown error never
|
|
7
|
+
// reaches it and takes the generic render. JSON output never comes through
|
|
8
|
+
// here.
|
|
9
|
+
|
|
10
|
+
const secondsOf = (value) => `${Math.round((Number(value) || 0) * 10) / 10}s`;
|
|
11
|
+
|
|
12
|
+
function pendingLine({ assertion_id: id, type, step_id: step }) {
|
|
13
|
+
const detail = [type, step == null ? null : `after ${step}`].filter((v) => v != null);
|
|
14
|
+
return ` - ${id}${detail.length ? ` (${detail.join(', ')})` : ''}`;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function renderRunHuman(envelope) {
|
|
18
|
+
const d = envelope.data;
|
|
19
|
+
const s = d.steps;
|
|
20
|
+
const a = d.assertions;
|
|
21
|
+
const lines = [];
|
|
22
|
+
if (!envelope.ok && envelope.error) lines.push(`error [${envelope.error.kind}]: ${envelope.error.message}`);
|
|
23
|
+
lines.push(`run ${d.run_id} (${d.scenario_id}): ${d.status} in ${secondsOf(d.duration_seconds)}`);
|
|
24
|
+
lines.push(
|
|
25
|
+
`steps: ${s.total} total, ${s.passed} passed, ${s.failed} failed, ${s.error} error, ${s.skipped} skipped, ${s.needs_agent} needs agent`
|
|
26
|
+
);
|
|
27
|
+
lines.push(`assertions: ${a.total} total, ${a.passed} passed, ${a.failed} failed, ${a.needs_agent} needs agent`);
|
|
28
|
+
if (d.failed_step) lines.push(`failed step: ${d.failed_step}`);
|
|
29
|
+
if (d.failures.length) lines.push('failures:', ...d.failures.map((f) => ` - ${f}`));
|
|
30
|
+
if (d.pending.length) lines.push('pending:', ...d.pending.map(pendingLine));
|
|
31
|
+
lines.push(`result: ${d.result_path}`);
|
|
32
|
+
if (d.junit_path) lines.push(`junit: ${d.junit_path}`);
|
|
33
|
+
if (envelope.hint) lines.push(`hint: ${envelope.hint}`);
|
|
34
|
+
return lines.join('\n');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
module.exports = { renderRunHuman };
|
|
@@ -82,6 +82,44 @@ const RESULT_CAPABILITIES = {
|
|
|
82
82
|
writeCheck: 'substring',
|
|
83
83
|
schemaPointer: '/properties/assertion_results',
|
|
84
84
|
},
|
|
85
|
+
// Who decided a verdict (#275). `--pass pending` reuses assertion_verdict's
|
|
86
|
+
// --pass; `type`, `step_id` and the derived `mechanical` ride on its
|
|
87
|
+
// --type/--step-id. Behavioral: `judge` is also a parameter name.
|
|
88
|
+
assertion_provenance: {
|
|
89
|
+
verb: 'add-assertion',
|
|
90
|
+
flags: ['--judge'],
|
|
91
|
+
store: 'addAssertion',
|
|
92
|
+
writes: 'judge',
|
|
93
|
+
writeCheck: 'behavioral',
|
|
94
|
+
schemaPointer: '/properties/assertion_results/items/properties/judge',
|
|
95
|
+
},
|
|
96
|
+
// A screenshot_match's comparison (#281): the images compared and the
|
|
97
|
+
// similarity measured. `mauto run` writes them in-process; an agent copies
|
|
98
|
+
// them from `mauto assert screenshot_match`'s hint.
|
|
99
|
+
assertion_reference_screenshot: {
|
|
100
|
+
verb: 'add-assertion',
|
|
101
|
+
flags: ['--reference-screenshot'],
|
|
102
|
+
store: 'addAssertion',
|
|
103
|
+
writes: 'entry.reference_screenshot = String(reference_screenshot)',
|
|
104
|
+
writeCheck: 'substring',
|
|
105
|
+
schemaPointer: '/properties/assertion_results/items/properties/reference_screenshot',
|
|
106
|
+
},
|
|
107
|
+
assertion_actual_screenshot: {
|
|
108
|
+
verb: 'add-assertion',
|
|
109
|
+
flags: ['--actual-screenshot'],
|
|
110
|
+
store: 'addAssertion',
|
|
111
|
+
writes: 'entry.actual_screenshot = String(actual_screenshot)',
|
|
112
|
+
writeCheck: 'substring',
|
|
113
|
+
schemaPointer: '/properties/assertion_results/items/properties/actual_screenshot',
|
|
114
|
+
},
|
|
115
|
+
assertion_similarity_score: {
|
|
116
|
+
verb: 'add-assertion',
|
|
117
|
+
flags: ['--similarity-score'],
|
|
118
|
+
store: 'addAssertion',
|
|
119
|
+
writes: 'entry.similarity_score = Number(similarity_score)',
|
|
120
|
+
writeCheck: 'substring',
|
|
121
|
+
schemaPointer: '/properties/assertion_results/items/properties/similarity_score',
|
|
122
|
+
},
|
|
85
123
|
run_status: {
|
|
86
124
|
verb: 'finalize',
|
|
87
125
|
flags: ['--status'],
|
|
@@ -119,6 +157,16 @@ const RESULT_CAPABILITIES = {
|
|
|
119
157
|
writeCheck: 'behavioral',
|
|
120
158
|
schemaPointer: '/properties/summary',
|
|
121
159
|
},
|
|
160
|
+
// Who produced the result and from what (#275). `finalize` always writes
|
|
161
|
+
// one; --scenario supplies the checked scenario path and its hash.
|
|
162
|
+
executor: {
|
|
163
|
+
verb: 'finalize',
|
|
164
|
+
flags: ['--scenario'],
|
|
165
|
+
store: 'finalize',
|
|
166
|
+
writes: 'executor',
|
|
167
|
+
writeCheck: 'behavioral',
|
|
168
|
+
schemaPointer: '/properties/executor',
|
|
169
|
+
},
|
|
122
170
|
crash_record: {
|
|
123
171
|
verb: 'add-crash',
|
|
124
172
|
flags: ['--crash-id', '--process', '--crash-timestamp', '--excerpt', '--report-path'],
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Who produced a result and from what (#275): the `executor` block a finalize
|
|
4
|
+
// records. Pure apart from reading the workspace config (for `engine` and
|
|
5
|
+
// `mode`) and package.json; shared by `result finalize` and `mauto run`.
|
|
6
|
+
|
|
7
|
+
const path = require('path');
|
|
8
|
+
|
|
9
|
+
const { version: MAUTO_VERSION } = require('../../package.json');
|
|
10
|
+
const manager = require('../config/manager');
|
|
11
|
+
const { readEngine } = require('../device/engine');
|
|
12
|
+
const resultSchema = require('../schemas/result_schema.json');
|
|
13
|
+
|
|
14
|
+
// The schema's executor.kind enum, read rather than restated.
|
|
15
|
+
const EXECUTOR_KINDS = resultSchema.properties.executor.properties.kind.enum;
|
|
16
|
+
|
|
17
|
+
// What executor.scenario_path means, the one place it is decided: relative
|
|
18
|
+
// to the project root when the file is inside it (so the result stays
|
|
19
|
+
// meaningful on another checkout), else absolute. `absolute` must already be
|
|
20
|
+
// resolved — against the caller's cwd — since a relative path here could mean
|
|
21
|
+
// either directory.
|
|
22
|
+
function recordedScenarioPath(projectRoot, absolute) {
|
|
23
|
+
if (!path.isAbsolute(absolute)) {
|
|
24
|
+
throw new Error(`scenario path must be absolute, got "${absolute}" (resolve it against the cwd first)`);
|
|
25
|
+
}
|
|
26
|
+
const relative = path.relative(projectRoot, absolute);
|
|
27
|
+
return relative && !relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)
|
|
28
|
+
? relative
|
|
29
|
+
: absolute;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// The `executor` block a finalize records. `engine` resolves as the device
|
|
33
|
+
// layer resolves it (an exported MOBILEMCP_LEGACY_ROBOT wins over config), and
|
|
34
|
+
// `mode` as the guides resolve it (unset is platform-aware). The device fields
|
|
35
|
+
// are whatever the caller connected to — null when it connected to none.
|
|
36
|
+
// `scenarioPath` is absolute (or null); it is recorded per
|
|
37
|
+
// recordedScenarioPath, so `mauto run` and `result finalize --scenario` record
|
|
38
|
+
// the same value for the same file.
|
|
39
|
+
function buildExecutor({
|
|
40
|
+
kind, projectRoot, platform = null, deviceId = null, scenarioPath = null, scenarioSha256 = null, env = process.env,
|
|
41
|
+
}) {
|
|
42
|
+
if (!EXECUTOR_KINDS.includes(kind)) {
|
|
43
|
+
throw new Error(`unknown executor kind "${kind}" (expected ${EXECUTOR_KINDS.join(' | ')})`);
|
|
44
|
+
}
|
|
45
|
+
const recordedPath = scenarioPath == null ? null : recordedScenarioPath(projectRoot, scenarioPath);
|
|
46
|
+
return {
|
|
47
|
+
kind,
|
|
48
|
+
mauto_version: MAUTO_VERSION,
|
|
49
|
+
engine: readEngine(projectRoot, { env }).engine,
|
|
50
|
+
platform,
|
|
51
|
+
device_id: deviceId,
|
|
52
|
+
scenario_path: recordedPath,
|
|
53
|
+
scenario_sha256: scenarioSha256,
|
|
54
|
+
mode: manager.resolveMode(manager.load(projectRoot)),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
module.exports = { buildExecutor, recordedScenarioPath };
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// The scenario-aware half of `mauto result finalize` (#275): load and hash the
|
|
4
|
+
// scenario, cross-check the recorded run against it, guard an explicit
|
|
5
|
+
// --status, build the executor, finalize, and map the finalized status to an
|
|
6
|
+
// envelope. Flag parsing stays with the caller. `finalizeRun` is injected so
|
|
7
|
+
// this module never requires src/cli.js, which owns it.
|
|
8
|
+
|
|
9
|
+
const path = require('path');
|
|
10
|
+
|
|
11
|
+
const { ok, fail } = require('../output/envelope');
|
|
12
|
+
const { loadScenario } = require('../scenario/load');
|
|
13
|
+
const { scenarioMismatch } = require('./scenario-check');
|
|
14
|
+
const { buildExecutor } = require('./executor');
|
|
15
|
+
const { incompleteEnvelope } = require('./pending-hint');
|
|
16
|
+
const { checkExplicitStatus } = require('./status-guard');
|
|
17
|
+
|
|
18
|
+
function finalizeAgainstScenario(
|
|
19
|
+
{ validator, resultStoreFactory, memoryStoreFactory, projectRoot, cwd = process.cwd(), finalizeRun },
|
|
20
|
+
opts
|
|
21
|
+
) {
|
|
22
|
+
const { runId, scenarioId, status, duration } = opts;
|
|
23
|
+
|
|
24
|
+
// --scenario (#275): the result is checked against the file it claims and
|
|
25
|
+
// the file is hashed into `executor`. Refused before anything is written.
|
|
26
|
+
// Resolved against the caller's cwd, as handleScreenshot resolves its path;
|
|
27
|
+
// buildExecutor decides how it is recorded.
|
|
28
|
+
let scenarioPath = null;
|
|
29
|
+
let scenarioSha256 = null;
|
|
30
|
+
let scenarioAssertionCount;
|
|
31
|
+
if (opts.scenario !== undefined) {
|
|
32
|
+
const scenarioFile = path.resolve(cwd, opts.scenario);
|
|
33
|
+
const loaded = loadScenario(validator, scenarioFile, { strict: false });
|
|
34
|
+
if (loaded.failure) return loaded.failure;
|
|
35
|
+
const mismatch = scenarioMismatch(resultStoreFactory({ runId, scenarioId, projectRoot }).recorded(), loaded);
|
|
36
|
+
if (mismatch) {
|
|
37
|
+
return { envelope: fail('invalid_input', mismatch.message, mismatch.hint, mismatch.data), exitKind: 'invalid_input' };
|
|
38
|
+
}
|
|
39
|
+
scenarioPath = scenarioFile;
|
|
40
|
+
scenarioSha256 = loaded.sha256;
|
|
41
|
+
scenarioAssertionCount = Array.isArray(loaded.scenario.assertions) ? loaded.scenario.assertions.length : 0;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// An explicit --status may equal the derived one or make the run worse,
|
|
45
|
+
// never greener (src/result/status-guard.js). Refused before anything is
|
|
46
|
+
// written.
|
|
47
|
+
if (status !== undefined) {
|
|
48
|
+
const recorded = resultStoreFactory({ runId, scenarioId, projectRoot }).recorded();
|
|
49
|
+
const refusal = checkExplicitStatus({
|
|
50
|
+
status, steps: recorded.steps, assertions: recorded.assertions, scenarioAssertionCount,
|
|
51
|
+
});
|
|
52
|
+
if (refusal) {
|
|
53
|
+
return { envelope: fail('invalid_input', refusal.message, refusal.hint), exitKind: 'invalid_input' };
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Only provided keys: `defaultMetadata` fills the rest with 'unknown', and
|
|
58
|
+
// the result schema requires all five fields to be strings, so threading
|
|
59
|
+
// `undefined` through would break conformance.
|
|
60
|
+
const metadata = {};
|
|
61
|
+
if (opts.appVersion !== undefined) metadata.app_version = opts.appVersion;
|
|
62
|
+
if (opts.deviceModel !== undefined) metadata.device_model = opts.deviceModel;
|
|
63
|
+
if (opts.apiLevel !== undefined) metadata.api_level = opts.apiLevel;
|
|
64
|
+
if (opts.environment !== undefined) metadata.environment = opts.environment;
|
|
65
|
+
|
|
66
|
+
const { result, store, hints } = finalizeRun(
|
|
67
|
+
{ resultStoreFactory, memoryStoreFactory, projectRoot },
|
|
68
|
+
{
|
|
69
|
+
runId, scenarioId, status, duration, metadata, summary: opts.summary,
|
|
70
|
+
// No device is connected here, so none is claimed (plan decision 5).
|
|
71
|
+
executor: buildExecutor({ kind: 'agent', projectRoot, scenarioPath, scenarioSha256 }),
|
|
72
|
+
}
|
|
73
|
+
);
|
|
74
|
+
|
|
75
|
+
// Exit 7: nothing failed, but an agent still has to judge something. The
|
|
76
|
+
// file is written; the hint is the exact commands that complete it.
|
|
77
|
+
if (result.status === 'incomplete') {
|
|
78
|
+
return { envelope: incompleteEnvelope(result, { hints, projectRoot, data: result }), exitKind: 'incomplete' };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// The exit code is keyed on the FINALIZED status, not the --status flag:
|
|
82
|
+
// with the flag omitted, the store derives it from the recorded steps and
|
|
83
|
+
// assertions ('error' for an errored step, else 'failed' for a failed step or
|
|
84
|
+
// assertion, #296), and the exit code must agree with the file either way
|
|
85
|
+
// (#273). 'error' takes the same path — a run that errored is not a pass for
|
|
86
|
+
// CI either; data.status tells the two apart. Everything above (write, retention, harvest) has
|
|
87
|
+
// already run: exit 6 is a verdict about the app, not a reason to lose the
|
|
88
|
+
// record of it.
|
|
89
|
+
if (result.status === 'failed' || result.status === 'error') {
|
|
90
|
+
const hint = hints.length
|
|
91
|
+
? hints.join(' ')
|
|
92
|
+
: 'See data.steps_executed and data.assertion_results for what failed; ' +
|
|
93
|
+
`the result is saved to ${path.relative(projectRoot, store.file)}.`;
|
|
94
|
+
return {
|
|
95
|
+
envelope: fail('test_failed', `run ${runId} ${result.status}`, hint, result),
|
|
96
|
+
exitKind: 'test_failed',
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return { envelope: ok(result, hints.length ? hints.join(' ') : null), exitKind: 'ok' };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
module.exports = { finalizeAgainstScenario };
|
package/src/result/flags.js
CHANGED
|
@@ -7,6 +7,8 @@
|
|
|
7
7
|
// `error` into a fail('invalid_input', …) envelope so an unknown observation
|
|
8
8
|
// type is rejected loudly instead of being silently dropped.
|
|
9
9
|
|
|
10
|
+
const { parseUnitInterval } = require('../util/parse-unit-interval');
|
|
11
|
+
|
|
10
12
|
// Mirrors the enum at /properties/observations/items/properties/type in
|
|
11
13
|
// src/schemas/result_schema.json. tests/lint/result-coverage.test.js asserts
|
|
12
14
|
// the two never drift.
|
|
@@ -78,10 +80,49 @@ function parseStepStatus(raw) {
|
|
|
78
80
|
return { value: normalized };
|
|
79
81
|
}
|
|
80
82
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
83
|
+
// `result add-assertion --pass` / `--judge` (#275, B3), parsed as one unit
|
|
84
|
+
// because the two flags constrain each other: `--pass` is a decided verdict or
|
|
85
|
+
// `pending` (recorded `needs_agent`, to be decided later by an agent).
|
|
86
|
+
//
|
|
87
|
+
// Returns `{ value: { status, judge } }` or `{ error }`. `status` is the
|
|
88
|
+
// assertion status to record; `judge` is undefined (no claim made — a caller
|
|
89
|
+
// that predates the flag writes exactly what it always wrote) or `{by}`. A
|
|
90
|
+
// pending verdict was decided by nobody, so it records `{by: 'none'}` and
|
|
91
|
+
// refuses `agent`; a decided verdict refuses `none`.
|
|
92
|
+
const VERDICT_STATUSES = { true: 'passed', false: 'failed', pending: 'needs_agent' };
|
|
93
|
+
|
|
94
|
+
// Who an agent may say decided a verdict (`--judge`). The schema's judge.by
|
|
95
|
+
// enum also has `mauto`, which only the in-process runner writes: an agent
|
|
96
|
+
// claiming mauto decided would be the self-attestation the field exists to
|
|
97
|
+
// expose.
|
|
98
|
+
const JUDGES = ['agent', 'none'];
|
|
99
|
+
|
|
100
|
+
function parseAssertionVerdict(passRaw, judgeRaw) {
|
|
101
|
+
if (!Object.prototype.hasOwnProperty.call(VERDICT_STATUSES, passRaw)) {
|
|
102
|
+
return { error: `--pass must be true | false | pending (got "${passRaw}")` };
|
|
103
|
+
}
|
|
104
|
+
if (judgeRaw !== undefined && !JUDGES.includes(judgeRaw)) {
|
|
105
|
+
return { error: `--judge must be ${JUDGES.join(' | ')} (got "${judgeRaw}")` };
|
|
106
|
+
}
|
|
107
|
+
const status = VERDICT_STATUSES[passRaw];
|
|
108
|
+
if (status === 'needs_agent') {
|
|
109
|
+
if (judgeRaw === 'agent') return { error: '--judge agent needs a verdict: use --pass true or --pass false' };
|
|
110
|
+
return { value: { status, judge: { by: 'none' } } };
|
|
111
|
+
}
|
|
112
|
+
if (judgeRaw === 'none') {
|
|
113
|
+
return { error: `--judge none means nobody decided, but --pass is ${passRaw}: use --pass pending or --judge agent` };
|
|
114
|
+
}
|
|
115
|
+
return { value: { status, judge: judgeRaw === 'agent' ? { by: 'agent' } : undefined } };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// `result add-assertion --similarity-score` (#281): a screenshot_match's
|
|
119
|
+
// measured similarity, as `mauto assert` reports it. Absent is `{ value: null }`
|
|
120
|
+
// (nothing to record); otherwise a number in [0, 1].
|
|
121
|
+
function parseSimilarityScore(raw) {
|
|
122
|
+
if (raw === undefined) return { value: null };
|
|
123
|
+
const value = parseUnitInterval(raw);
|
|
124
|
+
if (value === null) return { error: `--similarity-score must be a number from 0 to 1 (got "${raw}")` };
|
|
125
|
+
return { value };
|
|
85
126
|
}
|
|
86
127
|
|
|
87
128
|
// Mirrors the enum at /properties/status in src/schemas/result_schema.json;
|
|
@@ -90,7 +131,7 @@ function parseBool(raw, flagName) {
|
|
|
90
131
|
// status from the recorded steps and assertions): an unlisted value used to be persisted
|
|
91
132
|
// verbatim — a schema-invalid result file with a green exit — directly under
|
|
92
133
|
// the exit-6 contract keyed on that status (#286 review).
|
|
93
|
-
const RUN_STATUSES = ['passed', 'failed', 'error'];
|
|
134
|
+
const RUN_STATUSES = ['passed', 'failed', 'error', 'incomplete'];
|
|
94
135
|
|
|
95
136
|
function parseRunStatus(raw) {
|
|
96
137
|
if (!RUN_STATUSES.includes(raw)) {
|
|
@@ -103,7 +144,9 @@ module.exports = {
|
|
|
103
144
|
OBSERVATION_TYPES,
|
|
104
145
|
parseObservation,
|
|
105
146
|
parseCapture,
|
|
106
|
-
|
|
147
|
+
JUDGES,
|
|
148
|
+
parseAssertionVerdict,
|
|
149
|
+
parseSimilarityScore,
|
|
107
150
|
RUN_STATUSES,
|
|
108
151
|
parseRunStatus,
|
|
109
152
|
STEP_STATUS_ALIASES,
|
package/src/result/junit.js
CHANGED
|
@@ -12,17 +12,20 @@
|
|
|
12
12
|
//
|
|
13
13
|
// Mapping, one result at a time:
|
|
14
14
|
// result -> <testsuite name=scenario_id, else run_id>
|
|
15
|
-
// assertion_results[] -> <testcase name=assertion_id>; failed -> <failure
|
|
15
|
+
// assertion_results[] -> <testcase name=assertion_id>; failed -> <failure>,
|
|
16
|
+
// needs_agent -> <skipped>
|
|
16
17
|
// failed / error step -> <testcase name="step: <id>"> with <failure>
|
|
18
|
+
// needs_agent step -> <testcase name="step: <id>"> with <skipped>
|
|
17
19
|
// run id, status, metadata -> <properties>
|
|
18
20
|
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
// a <failure> of type step_error rather than a JUnit <error>, so
|
|
23
|
-
// counts
|
|
21
|
+
// Only failed, errored or pending steps get a testcase. Pending (#275) is
|
|
22
|
+
// rendered skipped, not failed: CI gates on `incomplete`'s exit code 7, and
|
|
23
|
+
// the report shows what still needs an agent. A step's `error` status is
|
|
24
|
+
// reported as a <failure> of type step_error rather than a JUnit <error>, so
|
|
25
|
+
// the counts are {tests, failures, skipped} and `errors` is always 0.
|
|
24
26
|
|
|
25
|
-
const { isErroredStep, isFailingAssertion, isFailingStep } = require('./outcome');
|
|
27
|
+
const { isErroredStep, isFailingAssertion, isFailingStep, isPendingAssertion, isPendingStep } = require('./outcome');
|
|
28
|
+
const { asArray } = require('../util/as-array');
|
|
26
29
|
|
|
27
30
|
// Characters XML 1.0 forbids anywhere in a document: C0 controls other than
|
|
28
31
|
// tab/LF/CR, lone surrogates, U+FFFE and U+FFFF. A crash excerpt or an OCR'd
|
|
@@ -64,7 +67,7 @@ const present = (v) => v !== undefined && v !== null && v !== '';
|
|
|
64
67
|
|
|
65
68
|
// A container as an array of its object entries; anything else is empty.
|
|
66
69
|
const isObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
67
|
-
const
|
|
70
|
+
const objectsOf = (value) => asArray(value).filter(isObject);
|
|
68
71
|
|
|
69
72
|
// Seconds as a plain decimal, rounded to the millisecond so summed floats do
|
|
70
73
|
// not render as 50.50000000001.
|
|
@@ -95,6 +98,9 @@ function assertionCase(assertion) {
|
|
|
95
98
|
message: assertion.message || `${assertion.assertion_id} failed`,
|
|
96
99
|
body: body.join('\n'),
|
|
97
100
|
};
|
|
101
|
+
} else if (isPendingAssertion(assertion)) {
|
|
102
|
+
const type = assertion.type ? ` (${assertion.type})` : '';
|
|
103
|
+
testcase.skipped = { message: `${assertion.assertion_id} needs agent judgment${type}` };
|
|
98
104
|
}
|
|
99
105
|
return testcase;
|
|
100
106
|
}
|
|
@@ -114,6 +120,7 @@ function stepCase(step) {
|
|
|
114
120
|
},
|
|
115
121
|
};
|
|
116
122
|
}
|
|
123
|
+
if (isPendingStep(step)) return { name, skipped: { message: `step ${step.step_id} needs agent judgment` } };
|
|
117
124
|
return null;
|
|
118
125
|
}
|
|
119
126
|
|
|
@@ -121,16 +128,24 @@ function suiteFor(result) {
|
|
|
121
128
|
const scenario = result.scenario_id ?? result.run_id;
|
|
122
129
|
const metadata = isObject(result.metadata) ? result.metadata : {};
|
|
123
130
|
const cases = [
|
|
124
|
-
...
|
|
125
|
-
...
|
|
131
|
+
...objectsOf(result.assertion_results).map(assertionCase),
|
|
132
|
+
...objectsOf(result.steps_executed).map(stepCase).filter(Boolean),
|
|
126
133
|
];
|
|
127
134
|
|
|
128
135
|
// A red result must never render as a green report. When the run says
|
|
129
136
|
// anything but passed yet recorded nothing that fails (e.g. an agent
|
|
130
137
|
// finalized with --status failed after an abort, or a pre-0.38.0 "FAILED"),
|
|
131
|
-
// the status itself becomes the failure.
|
|
132
|
-
|
|
133
|
-
|
|
138
|
+
// the status itself becomes the failure. `incomplete` is not red: its
|
|
139
|
+
// pending cases are already skipped, and only when none was recorded does
|
|
140
|
+
// the status itself become a skipped case, so it never reads green either.
|
|
141
|
+
if (result.status === 'incomplete') {
|
|
142
|
+
if (!cases.some((c) => c.failure || c.skipped)) {
|
|
143
|
+
cases.push({
|
|
144
|
+
name: 'run status',
|
|
145
|
+
skipped: { message: 'result status is incomplete but no pending step or assertion was recorded' },
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
} else if (result.status !== 'passed' && !cases.some((c) => c.failure)) {
|
|
134
149
|
cases.push({
|
|
135
150
|
name: 'run status',
|
|
136
151
|
failure: {
|
|
@@ -142,6 +157,7 @@ function suiteFor(result) {
|
|
|
142
157
|
}
|
|
143
158
|
|
|
144
159
|
const failures = cases.filter((c) => c.failure).length;
|
|
160
|
+
const skipped = cases.filter((c) => c.skipped).length;
|
|
145
161
|
|
|
146
162
|
const lines = [];
|
|
147
163
|
lines.push(
|
|
@@ -149,6 +165,7 @@ function suiteFor(result) {
|
|
|
149
165
|
['name', scenario],
|
|
150
166
|
['tests', cases.length],
|
|
151
167
|
['failures', failures],
|
|
168
|
+
['skipped', skipped],
|
|
152
169
|
['errors', 0],
|
|
153
170
|
['time', seconds(result.duration_seconds)],
|
|
154
171
|
['timestamp', present(metadata.timestamp) ? metadata.timestamp : null],
|
|
@@ -167,21 +184,26 @@ function suiteFor(result) {
|
|
|
167
184
|
lines.push(`${open}>`);
|
|
168
185
|
lines.push(` <failure${attrs([['message', message], ['type', type]])}>${escapeText(body)}</failure>`);
|
|
169
186
|
lines.push(' </testcase>');
|
|
187
|
+
} else if (c.skipped) {
|
|
188
|
+
lines.push(`${open}>`);
|
|
189
|
+
lines.push(` <skipped${attrs([['message', c.skipped.message]])}/>`);
|
|
190
|
+
lines.push(' </testcase>');
|
|
170
191
|
} else {
|
|
171
192
|
lines.push(`${open}/>`);
|
|
172
193
|
}
|
|
173
194
|
}
|
|
174
195
|
lines.push(' </testsuite>');
|
|
175
196
|
|
|
176
|
-
return { lines, tests: cases.length, failures, time: Number(seconds(result.duration_seconds)) };
|
|
197
|
+
return { lines, tests: cases.length, failures, skipped, time: Number(seconds(result.duration_seconds)) };
|
|
177
198
|
}
|
|
178
199
|
|
|
179
|
-
// toJUnit(results) -> { xml, suites, tests, failures }
|
|
200
|
+
// toJUnit(results) -> { xml, suites, tests, failures, skipped }
|
|
180
201
|
function toJUnit(results) {
|
|
181
202
|
const suites = results.map(suiteFor);
|
|
182
203
|
const total = (key) => suites.reduce((sum, s) => sum + s[key], 0);
|
|
183
204
|
const tests = total('tests');
|
|
184
205
|
const failures = total('failures');
|
|
206
|
+
const skipped = total('skipped');
|
|
185
207
|
|
|
186
208
|
const xml = [
|
|
187
209
|
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
@@ -189,6 +211,7 @@ function toJUnit(results) {
|
|
|
189
211
|
['name', 'mauto'],
|
|
190
212
|
['tests', tests],
|
|
191
213
|
['failures', failures],
|
|
214
|
+
['skipped', skipped],
|
|
192
215
|
['errors', 0],
|
|
193
216
|
['time', seconds(total('time'))],
|
|
194
217
|
])}>`,
|
|
@@ -197,7 +220,7 @@ function toJUnit(results) {
|
|
|
197
220
|
'',
|
|
198
221
|
].join('\n');
|
|
199
222
|
|
|
200
|
-
return { xml, suites: suites.length, tests, failures };
|
|
223
|
+
return { xml, suites: suites.length, tests, failures, skipped };
|
|
201
224
|
}
|
|
202
225
|
|
|
203
226
|
module.exports = { toJUnit };
|
package/src/result/outcome.js
CHANGED
|
@@ -12,22 +12,44 @@
|
|
|
12
12
|
|
|
13
13
|
const statusOf = (entry) => (entry !== null && typeof entry === 'object' ? entry.status : undefined);
|
|
14
14
|
|
|
15
|
+
const isPassedStep = (step) => statusOf(step) === 'passed';
|
|
16
|
+
const isSkippedStep = (step) => statusOf(step) === 'skipped';
|
|
15
17
|
const isFailedStep = (step) => statusOf(step) === 'failed';
|
|
16
18
|
const isErroredStep = (step) => statusOf(step) === 'error';
|
|
17
19
|
const isFailingStep = (step) => isFailedStep(step) || isErroredStep(step);
|
|
20
|
+
const isPassedAssertion = (assertion) => statusOf(assertion) === 'passed';
|
|
18
21
|
const isFailingAssertion = (assertion) => statusOf(assertion) === 'failed';
|
|
19
22
|
|
|
23
|
+
// Pending (#275): recorded, but not decided, because only an agent can judge
|
|
24
|
+
// it. An assertion says so with its status; a step keeps its own status and
|
|
25
|
+
// carries `needs_agent: true` beside it.
|
|
26
|
+
const isPendingAssertion = (assertion) => statusOf(assertion) === 'needs_agent';
|
|
27
|
+
const isPendingStep = (step) => step !== null && typeof step === 'object' && step.needs_agent === true;
|
|
28
|
+
|
|
20
29
|
// The run status `result finalize` derives when `--status` is omitted (#296).
|
|
21
30
|
// It used to read assertions only, so a run whose step failed finalized
|
|
22
31
|
// `passed` while its JUnit export was red. Every entry counts — a step
|
|
23
32
|
// recorded twice (fail, then pass) still failed once, matching the export.
|
|
24
33
|
// Skipped steps count for nothing; a run with no steps and no assertions
|
|
25
|
-
// stays `passed
|
|
34
|
+
// stays `passed`. Anything failing outranks anything pending: a run that is
|
|
35
|
+
// already red is decided, whatever an agent later says (#275).
|
|
26
36
|
function deriveRunStatus(steps = [], assertions = []) {
|
|
27
37
|
if (steps.some(isErroredStep)) return 'error';
|
|
28
38
|
if (steps.some(isFailedStep)) return 'failed';
|
|
29
39
|
if (assertions.some(isFailingAssertion)) return 'failed';
|
|
40
|
+
if (steps.some(isPendingStep) || assertions.some(isPendingAssertion)) return 'incomplete';
|
|
30
41
|
return 'passed';
|
|
31
42
|
}
|
|
32
43
|
|
|
33
|
-
module.exports = {
|
|
44
|
+
module.exports = {
|
|
45
|
+
isPassedStep,
|
|
46
|
+
isSkippedStep,
|
|
47
|
+
isFailedStep,
|
|
48
|
+
isErroredStep,
|
|
49
|
+
isFailingStep,
|
|
50
|
+
isPassedAssertion,
|
|
51
|
+
isFailingAssertion,
|
|
52
|
+
isPendingAssertion,
|
|
53
|
+
isPendingStep,
|
|
54
|
+
deriveRunStatus,
|
|
55
|
+
};
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// What an agent must do to complete a run that is waiting for its judgment
|
|
4
|
+
// (#275): the pending assertions, the `incomplete` message and hint, and the
|
|
5
|
+
// envelope built from them. Shared by `result finalize` and `mauto run`.
|
|
6
|
+
|
|
7
|
+
const path = require('path');
|
|
8
|
+
|
|
9
|
+
const { fail } = require('../output/envelope');
|
|
10
|
+
const { isPendingAssertion, isPendingStep } = require('./outcome');
|
|
11
|
+
|
|
12
|
+
// The assertions a finalized result is still waiting on, as `{assertion_id,
|
|
13
|
+
// step_id, type}` — what `mauto run` puts in data.pending.
|
|
14
|
+
function pendingAssertions(result) {
|
|
15
|
+
return (result.assertion_results || [])
|
|
16
|
+
.filter(isPendingAssertion)
|
|
17
|
+
.map(({ assertion_id, step_id = null, type = null }) => ({ assertion_id, step_id, type }));
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// The error message of an `incomplete` run, shared by `result finalize` and
|
|
21
|
+
// `mauto run`. Counts the pending assertions; a run pending only on a step
|
|
22
|
+
// names no count.
|
|
23
|
+
function incompleteMessage(result) {
|
|
24
|
+
const pending = pendingAssertions(result).length;
|
|
25
|
+
return `run ${result.run_id} incomplete${pending ? `: ${pending} assertion(s) need agent judgment` : ''}`;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// A flag value as it must be typed into a shell: bare when that is safe, else
|
|
29
|
+
// JSON-quoted.
|
|
30
|
+
const arg = (value) => (/^[\w./@:+-]+$/.test(String(value)) ? String(value) : JSON.stringify(String(value)));
|
|
31
|
+
|
|
32
|
+
// The hint for an `incomplete` run: the pending ids, the exact
|
|
33
|
+
// `result add-assertion` line per pending assertion (only the verdict and the
|
|
34
|
+
// reason left to fill), then the re-finalize. The scenario path is the one
|
|
35
|
+
// the executor recorded, so the re-finalize is checked against the same file,
|
|
36
|
+
// printed absolute (resolved against `projectRoot`): --scenario resolves
|
|
37
|
+
// against the cwd while the workspace is discovered by walking up from it, so
|
|
38
|
+
// a project-relative path would be wrong from any subdirectory.
|
|
39
|
+
function pendingHint(result, { projectRoot }) {
|
|
40
|
+
const pending = pendingAssertions(result);
|
|
41
|
+
const pendingSteps = (result.steps_executed || []).filter(isPendingStep);
|
|
42
|
+
const runId = arg(result.run_id);
|
|
43
|
+
const recorded = result.executor && result.executor.scenario_path;
|
|
44
|
+
const scenarioPath = recorded ? path.resolve(projectRoot, recorded) : null;
|
|
45
|
+
return [
|
|
46
|
+
...pendingSteps.map((s) => `Pending step ${s.step_id}: ${s.error_message || 'needs an agent'}`),
|
|
47
|
+
pending.length
|
|
48
|
+
? `Pending agent judgment: ${pending.map((p) => p.assertion_id).join(', ')}. Decide each, then re-finalize:`
|
|
49
|
+
: 'No assertion is pending; re-finalize once the run is decided:',
|
|
50
|
+
...pending.map(
|
|
51
|
+
(p) =>
|
|
52
|
+
`mauto result add-assertion --run-id ${runId} --step-id ${p.step_id == null ? '<step_id>' : arg(p.step_id)} ` +
|
|
53
|
+
`--assertion-id ${arg(p.assertion_id)} --type ${p.type == null ? '<type>' : arg(p.type)} ` +
|
|
54
|
+
'--pass <true|false> --judge agent --message "<why>"'
|
|
55
|
+
),
|
|
56
|
+
`mauto result finalize --run-id ${runId} --scenario ${scenarioPath ? arg(scenarioPath) : '<scenario.json>'}`,
|
|
57
|
+
].join('\n');
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// The `incomplete` (exit 7) envelope, shared by `result finalize` and
|
|
61
|
+
// `mauto run`: the message, the store's hints, then the commands that complete
|
|
62
|
+
// the run. `data` is what the envelope carries.
|
|
63
|
+
function incompleteEnvelope(result, { hints, projectRoot, data }) {
|
|
64
|
+
return fail('incomplete', incompleteMessage(result), [...hints, pendingHint(result, { projectRoot })].join('\n'), data);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
module.exports = { arg, pendingAssertions, pendingHint, incompleteMessage, incompleteEnvelope };
|