mobile-automator 0.38.2 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli.js +83 -7
- package/src/result/flags.js +1 -1
- package/src/result/junit.js +203 -0
- package/src/result/outcome.js +33 -0
- package/src/result/store.js +122 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mobile-automator",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.39.0",
|
|
4
4
|
"description": "Host-agnostic mauto CLI for AI-driven mobile QA automation — analyzes a mobile project and drives devices through mobile-mcp.",
|
|
5
5
|
"homepage": "https://github.com/sh3lan93/mobile-automator#readme",
|
|
6
6
|
"repository": {
|
package/src/cli.js
CHANGED
|
@@ -12,8 +12,10 @@ const { engineHint } = require('./device/engine-hints');
|
|
|
12
12
|
const { ScenarioValidator } = require('./scenario/validator');
|
|
13
13
|
const { findReplayWarnings, summarize } = require('./scenario/replayability');
|
|
14
14
|
const { evaluate, MECHANICAL_TYPES } = require('./assertion/evaluator');
|
|
15
|
-
const { ResultStore } = require('./result/store');
|
|
15
|
+
const { ResultStore, readFinalizedResult, InvalidRunIdError } = require('./result/store');
|
|
16
16
|
const { OBSERVATION_TYPES, parseObservation, parseCapture, parseBool, parseRunStatus, parseStepStatus } = require('./result/flags');
|
|
17
|
+
const { toJUnit } = require('./result/junit');
|
|
18
|
+
const { atomicWrite } = require('./util/atomic');
|
|
17
19
|
const { MemoryStore } = require('./memory/store');
|
|
18
20
|
const { FILES: MEMORY_FILES, AGENT_KINDS } = require('./memory/paths');
|
|
19
21
|
const { validateEntryText, MAX_ENTRY_LEN } = require('./memory/entries');
|
|
@@ -128,6 +130,7 @@ function usageLine(cmd) {
|
|
|
128
130
|
// is the single boundary that guarantees a calling agent always receives one
|
|
129
131
|
// JSON line + a meaningful exit code instead of a raw stack trace on stderr.
|
|
130
132
|
// CommanderError (parse failure) -> invalid_input, with the usage line as hint
|
|
133
|
+
// InvalidRunIdError -> invalid_input, with the store's hint
|
|
131
134
|
// SyntaxError -> invalid_input (a malformed JSON file)
|
|
132
135
|
// fs EACCES/ENOENT/EROFS/... -> environment, with a filesystem hint
|
|
133
136
|
// anything else -> internal
|
|
@@ -142,6 +145,11 @@ function toEnvelope(err) {
|
|
|
142
145
|
exitKind: 'invalid_input',
|
|
143
146
|
};
|
|
144
147
|
}
|
|
148
|
+
// A --run-id the ResultStore refuses (#295): the store owns the rule and
|
|
149
|
+
// the wording, so every result verb reports it identically.
|
|
150
|
+
if (err instanceof InvalidRunIdError) {
|
|
151
|
+
return { envelope: fail(err.kind, err.message, err.hint), exitKind: err.kind };
|
|
152
|
+
}
|
|
145
153
|
if (err instanceof SyntaxError) {
|
|
146
154
|
return {
|
|
147
155
|
envelope: fail(
|
|
@@ -829,7 +837,7 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
|
|
|
829
837
|
// schema-invalid result file with ok:true and exit 0 — directly under the
|
|
830
838
|
// exit-6 contract keyed on that status (#286 review). Reject before anything
|
|
831
839
|
// is written. A provided-but-empty value is NOT an omitted one: only absence
|
|
832
|
-
// derives the status from the recorded assertions.
|
|
840
|
+
// derives the status from the recorded steps and assertions.
|
|
833
841
|
if (status !== undefined) {
|
|
834
842
|
const parsed = parseRunStatus(status);
|
|
835
843
|
if (parsed.error) {
|
|
@@ -837,7 +845,7 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
|
|
|
837
845
|
envelope: fail(
|
|
838
846
|
'invalid_input',
|
|
839
847
|
parsed.error,
|
|
840
|
-
'Valid values: passed | failed | error, or omit --status to derive it from the recorded assertions.'
|
|
848
|
+
'Valid values: passed | failed | error, or omit --status to derive it from the recorded steps and assertions.'
|
|
841
849
|
),
|
|
842
850
|
exitKind: 'invalid_input',
|
|
843
851
|
};
|
|
@@ -898,10 +906,11 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
|
|
|
898
906
|
}
|
|
899
907
|
|
|
900
908
|
// The exit code is keyed on the FINALIZED status, not the --status flag:
|
|
901
|
-
// with the flag omitted, the store derives
|
|
902
|
-
//
|
|
903
|
-
//
|
|
904
|
-
//
|
|
909
|
+
// with the flag omitted, the store derives it from the recorded steps and
|
|
910
|
+
// assertions ('error' for an errored step, else 'failed' for a failed step or
|
|
911
|
+
// assertion, #296), and the exit code must agree with the file either way
|
|
912
|
+
// (#273). 'error' takes the same path — a run that errored is not a pass for
|
|
913
|
+
// CI either; data.status tells the two apart. Everything above (write, retention, harvest) has
|
|
905
914
|
// already run: exit 6 is a verdict about the app, not a reason to lose the
|
|
906
915
|
// record of it.
|
|
907
916
|
if (result.status === 'failed' || result.status === 'error') {
|
|
@@ -918,6 +927,58 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
|
|
|
918
927
|
return { envelope: ok(result, hints.length ? hints.join(' ') : null), exitKind: 'ok' };
|
|
919
928
|
}
|
|
920
929
|
|
|
930
|
+
// Formats `mauto result export --format` accepts. The only one, junit, is
|
|
931
|
+
// rendered by toJUnit (src/result/junit.js).
|
|
932
|
+
const REPORT_FORMATS = Object.freeze(['junit']);
|
|
933
|
+
|
|
934
|
+
// `mauto result export` (#274): render finalized result files as a CI report.
|
|
935
|
+
// Read-only by contract — tests/lint/result-coverage.test.js exempts it from
|
|
936
|
+
// the capability catalog on that basis and checks the results directory is
|
|
937
|
+
// unchanged afterwards. Reading (and that read-only guarantee) is
|
|
938
|
+
// readFinalizedResult's job in src/result/store.js.
|
|
939
|
+
//
|
|
940
|
+
// The export succeeds (exit 0) whatever the runs' outcome: it is a report, not
|
|
941
|
+
// a gate. A single unusable run id fails the whole export, so a CI job never
|
|
942
|
+
// uploads a report silently missing a run.
|
|
943
|
+
function handleResultExport({ projectRoot, cwd = process.cwd() }, opts = {}) {
|
|
944
|
+
const invalid = (message, hint) => ({ envelope: fail('invalid_input', message, hint), exitKind: 'invalid_input' });
|
|
945
|
+
const formatsHint = `Formats: ${REPORT_FORMATS.join(', ')}.`;
|
|
946
|
+
|
|
947
|
+
if (!opts.format) return invalid('--format is required', formatsHint);
|
|
948
|
+
if (!REPORT_FORMATS.includes(opts.format)) return invalid(`unknown report format "${opts.format}"`, formatsHint);
|
|
949
|
+
|
|
950
|
+
const runIds = opts.runIds || [];
|
|
951
|
+
if (runIds.length === 0) return invalid('at least one --run-id is required', 'Pass --run-id <id>; repeat it to export several runs.');
|
|
952
|
+
|
|
953
|
+
const results = [];
|
|
954
|
+
for (const runId of runIds) {
|
|
955
|
+
const { result, error } = readFinalizedResult(projectRoot, runId);
|
|
956
|
+
if (error) return { envelope: fail(error.kind, error.message, error.hint), exitKind: error.kind };
|
|
957
|
+
results.push(result);
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
const { xml, ...counts } = toJUnit(results);
|
|
961
|
+
|
|
962
|
+
if (!opts.out) {
|
|
963
|
+
return { envelope: ok({ format: opts.format, path: null, ...counts, xml }), exitKind: 'ok' };
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
const outPath = path.resolve(cwd, opts.out);
|
|
967
|
+
try {
|
|
968
|
+
atomicWrite(outPath, xml);
|
|
969
|
+
} catch (err) {
|
|
970
|
+
return {
|
|
971
|
+
envelope: fail(
|
|
972
|
+
'environment',
|
|
973
|
+
`could not write the report to ${outPath}: ${err.message}`,
|
|
974
|
+
'Check the directory exists and is writable.'
|
|
975
|
+
),
|
|
976
|
+
exitKind: 'environment',
|
|
977
|
+
};
|
|
978
|
+
}
|
|
979
|
+
return { envelope: ok({ format: opts.format, path: outPath, ...counts }), exitKind: 'ok' };
|
|
980
|
+
}
|
|
981
|
+
|
|
921
982
|
// Collapse any recovery warnings a ResultStore accumulated (e.g. a corrupt
|
|
922
983
|
// prior file preserved as a sidecar) into a single envelope hint, or null.
|
|
923
984
|
function storeHint(store) {
|
|
@@ -2258,6 +2319,20 @@ function buildProgram(deps = {}) {
|
|
|
2258
2319
|
emit(r, humanFlag());
|
|
2259
2320
|
})));
|
|
2260
2321
|
|
|
2322
|
+
result
|
|
2323
|
+
.command('export')
|
|
2324
|
+
.description('Render finalized result files as a CI report (read-only)')
|
|
2325
|
+
.requiredOption('--run-id <id>', 'run to export; repeatable', collect, [])
|
|
2326
|
+
.requiredOption('--format <f>', REPORT_FORMATS.join(' | '))
|
|
2327
|
+
.option('--out <path>', 'write the report here instead of returning it in data.xml')
|
|
2328
|
+
.action(withEnvelope(requireWorkspace((opts) => {
|
|
2329
|
+
const r = handleResultExport(
|
|
2330
|
+
{ projectRoot, cwd },
|
|
2331
|
+
{ runIds: opts.runId, format: opts.format, out: opts.out }
|
|
2332
|
+
);
|
|
2333
|
+
emit(r, humanFlag());
|
|
2334
|
+
})));
|
|
2335
|
+
|
|
2261
2336
|
// --- Slice 3 verbs --------------------------------------------------------
|
|
2262
2337
|
|
|
2263
2338
|
// Emit a handler result that may be either an envelope (success/error) OR
|
|
@@ -2726,6 +2801,7 @@ module.exports = {
|
|
|
2726
2801
|
handleResultAddAssertion,
|
|
2727
2802
|
handleResultAddCrash,
|
|
2728
2803
|
handleResultFinalize,
|
|
2804
|
+
handleResultExport,
|
|
2729
2805
|
handleSetup,
|
|
2730
2806
|
handleConfigGet,
|
|
2731
2807
|
handleConfigSet,
|
package/src/result/flags.js
CHANGED
|
@@ -87,7 +87,7 @@ function parseBool(raw, flagName) {
|
|
|
87
87
|
// Mirrors the enum at /properties/status in src/schemas/result_schema.json;
|
|
88
88
|
// tests/lint/result-coverage.test.js asserts the two never drift. `result
|
|
89
89
|
// finalize --status` must be one of these or omitted (omission derives the
|
|
90
|
-
// status from the recorded assertions): an unlisted value used to be persisted
|
|
90
|
+
// status from the recorded steps and assertions): an unlisted value used to be persisted
|
|
91
91
|
// verbatim — a schema-invalid result file with a green exit — directly under
|
|
92
92
|
// the exit-6 contract keyed on that status (#286 review).
|
|
93
93
|
const RUN_STATUSES = ['passed', 'failed', 'error'];
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// JUnit XML rendering of finalized result files (#274). Its input is what
|
|
4
|
+
// readFinalizedResult (src/result/store.js) guarantees: an object finalize wrote
|
|
5
|
+
// for the run id — NOT one valid against the result schema (a run recorded
|
|
6
|
+
// without --scenario-id has scenario_id null; a pre-0.38.0 file has an
|
|
7
|
+
// upper-case status). So the renderer tolerates: the suite is named by the run
|
|
8
|
+
// id when there is no scenario_id, any status other than `passed` reads red, a
|
|
9
|
+
// missing metadata/container is empty, and non-object entries are dropped.
|
|
10
|
+
// Pure: no fs, no clock, no input mutation, so the same results always render
|
|
11
|
+
// to the same bytes. It is consumed by `mauto result export`.
|
|
12
|
+
//
|
|
13
|
+
// Mapping, one result at a time:
|
|
14
|
+
// result -> <testsuite name=scenario_id, else run_id>
|
|
15
|
+
// assertion_results[] -> <testcase name=assertion_id>; failed -> <failure>
|
|
16
|
+
// failed / error step -> <testcase name="step: <id>"> with <failure>
|
|
17
|
+
// run id, status, metadata -> <properties>
|
|
18
|
+
//
|
|
19
|
+
// It renders only what current writers produce: assertions are passed or
|
|
20
|
+
// failed, and only failed or errored steps get a testcase, so there is no
|
|
21
|
+
// JUnit skip element and no skip count. A step's `error` status is reported as
|
|
22
|
+
// a <failure> of type step_error rather than a JUnit <error>, so the report's
|
|
23
|
+
// counts stay {tests, failures} and `errors` is always 0.
|
|
24
|
+
|
|
25
|
+
const { isErroredStep, isFailingAssertion, isFailingStep } = require('./outcome');
|
|
26
|
+
|
|
27
|
+
// Characters XML 1.0 forbids anywhere in a document: C0 controls other than
|
|
28
|
+
// tab/LF/CR, lone surrogates, U+FFFE and U+FFFF. A crash excerpt or an OCR'd
|
|
29
|
+
// label can carry any of them, and one is enough to make the whole report
|
|
30
|
+
// unparseable, so they are dropped rather than escaped (escaping does not make
|
|
31
|
+
// them legal).
|
|
32
|
+
// eslint-disable-next-line no-control-regex
|
|
33
|
+
const ILLEGAL_XML_CHARS = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;
|
|
34
|
+
|
|
35
|
+
const METADATA_KEYS = ['app_version', 'device_model', 'api_level', 'environment', 'timestamp'];
|
|
36
|
+
|
|
37
|
+
function clean(value) {
|
|
38
|
+
return String(value).replace(ILLEGAL_XML_CHARS, '');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function escapeText(value) {
|
|
42
|
+
return clean(value).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Attribute values additionally escape quotes, and encode whitespace a parser
|
|
46
|
+
// would otherwise normalise to a space, so a multi-line message survives.
|
|
47
|
+
function escapeAttr(value) {
|
|
48
|
+
return escapeText(value)
|
|
49
|
+
.replace(/"/g, '"')
|
|
50
|
+
.replace(/'/g, ''')
|
|
51
|
+
.replace(/\n/g, ' ')
|
|
52
|
+
.replace(/\r/g, ' ')
|
|
53
|
+
.replace(/\t/g, '	');
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function attrs(pairs) {
|
|
57
|
+
return pairs
|
|
58
|
+
.filter(([, v]) => v !== undefined && v !== null)
|
|
59
|
+
.map(([k, v]) => ` ${k}="${escapeAttr(v)}"`)
|
|
60
|
+
.join('');
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const present = (v) => v !== undefined && v !== null && v !== '';
|
|
64
|
+
|
|
65
|
+
// A container as an array of its object entries; anything else is empty.
|
|
66
|
+
const isObject = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
67
|
+
const asArray = (value) => (Array.isArray(value) ? value.filter(isObject) : []);
|
|
68
|
+
|
|
69
|
+
// Seconds as a plain decimal, rounded to the millisecond so summed floats do
|
|
70
|
+
// not render as 50.50000000001.
|
|
71
|
+
function seconds(value) {
|
|
72
|
+
const n = Number(value);
|
|
73
|
+
return String(Number.isFinite(n) && n > 0 ? Math.round(n * 1000) / 1000 : 0);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function properties(result, metadata) {
|
|
77
|
+
const props = [
|
|
78
|
+
['run_id', result.run_id],
|
|
79
|
+
['status', result.status],
|
|
80
|
+
];
|
|
81
|
+
for (const key of METADATA_KEYS) {
|
|
82
|
+
if (present(metadata[key])) props.push([`metadata.${key}`, metadata[key]]);
|
|
83
|
+
}
|
|
84
|
+
return props.filter(([, v]) => present(v));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function assertionCase(assertion) {
|
|
88
|
+
const testcase = { name: assertion.assertion_id };
|
|
89
|
+
if (isFailingAssertion(assertion)) {
|
|
90
|
+
const body = [];
|
|
91
|
+
if (present(assertion.expected)) body.push(`expected: ${assertion.expected}`);
|
|
92
|
+
if (present(assertion.actual)) body.push(`actual: ${assertion.actual}`);
|
|
93
|
+
testcase.failure = {
|
|
94
|
+
type: assertion.type ? `${assertion.type}_failed` : 'assertion_failed',
|
|
95
|
+
message: assertion.message || `${assertion.assertion_id} failed`,
|
|
96
|
+
body: body.join('\n'),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
return testcase;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function stepCase(step) {
|
|
103
|
+
const name = `step: ${step.step_id}`;
|
|
104
|
+
if (isFailingStep(step)) {
|
|
105
|
+
const body = [];
|
|
106
|
+
if (present(step.error_message)) body.push(step.error_message);
|
|
107
|
+
if (present(step.screenshot)) body.push(`screenshot: ${step.screenshot}`);
|
|
108
|
+
return {
|
|
109
|
+
name,
|
|
110
|
+
failure: {
|
|
111
|
+
type: isErroredStep(step) ? 'step_error' : 'step_failed',
|
|
112
|
+
message: step.error_message || `step ${step.step_id} ${step.status}`,
|
|
113
|
+
body: body.join('\n'),
|
|
114
|
+
},
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function suiteFor(result) {
|
|
121
|
+
const scenario = result.scenario_id ?? result.run_id;
|
|
122
|
+
const metadata = isObject(result.metadata) ? result.metadata : {};
|
|
123
|
+
const cases = [
|
|
124
|
+
...asArray(result.assertion_results).map(assertionCase),
|
|
125
|
+
...asArray(result.steps_executed).map(stepCase).filter(Boolean),
|
|
126
|
+
];
|
|
127
|
+
|
|
128
|
+
// A red result must never render as a green report. When the run says
|
|
129
|
+
// anything but passed yet recorded nothing that fails (e.g. an agent
|
|
130
|
+
// finalized with --status failed after an abort, or a pre-0.38.0 "FAILED"),
|
|
131
|
+
// the status itself becomes the failure.
|
|
132
|
+
const isRed = result.status !== 'passed';
|
|
133
|
+
if (isRed && !cases.some((c) => c.failure)) {
|
|
134
|
+
cases.push({
|
|
135
|
+
name: 'run status',
|
|
136
|
+
failure: {
|
|
137
|
+
type: 'run_status',
|
|
138
|
+
message: `result status is ${result.status} but no failed step or assertion was recorded`,
|
|
139
|
+
body: result.summary,
|
|
140
|
+
},
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const failures = cases.filter((c) => c.failure).length;
|
|
145
|
+
|
|
146
|
+
const lines = [];
|
|
147
|
+
lines.push(
|
|
148
|
+
` <testsuite${attrs([
|
|
149
|
+
['name', scenario],
|
|
150
|
+
['tests', cases.length],
|
|
151
|
+
['failures', failures],
|
|
152
|
+
['errors', 0],
|
|
153
|
+
['time', seconds(result.duration_seconds)],
|
|
154
|
+
['timestamp', present(metadata.timestamp) ? metadata.timestamp : null],
|
|
155
|
+
])}>`
|
|
156
|
+
);
|
|
157
|
+
const props = properties(result, metadata);
|
|
158
|
+
if (props.length) {
|
|
159
|
+
lines.push(' <properties>');
|
|
160
|
+
for (const [name, value] of props) lines.push(` <property${attrs([['name', name], ['value', value]])}/>`);
|
|
161
|
+
lines.push(' </properties>');
|
|
162
|
+
}
|
|
163
|
+
for (const c of cases) {
|
|
164
|
+
const open = ` <testcase${attrs([['name', c.name], ['classname', scenario]])}`;
|
|
165
|
+
if (c.failure) {
|
|
166
|
+
const { type, message, body } = c.failure;
|
|
167
|
+
lines.push(`${open}>`);
|
|
168
|
+
lines.push(` <failure${attrs([['message', message], ['type', type]])}>${escapeText(body)}</failure>`);
|
|
169
|
+
lines.push(' </testcase>');
|
|
170
|
+
} else {
|
|
171
|
+
lines.push(`${open}/>`);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
lines.push(' </testsuite>');
|
|
175
|
+
|
|
176
|
+
return { lines, tests: cases.length, failures, time: Number(seconds(result.duration_seconds)) };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// toJUnit(results) -> { xml, suites, tests, failures }
|
|
180
|
+
function toJUnit(results) {
|
|
181
|
+
const suites = results.map(suiteFor);
|
|
182
|
+
const total = (key) => suites.reduce((sum, s) => sum + s[key], 0);
|
|
183
|
+
const tests = total('tests');
|
|
184
|
+
const failures = total('failures');
|
|
185
|
+
|
|
186
|
+
const xml = [
|
|
187
|
+
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
188
|
+
`<testsuites${attrs([
|
|
189
|
+
['name', 'mauto'],
|
|
190
|
+
['tests', tests],
|
|
191
|
+
['failures', failures],
|
|
192
|
+
['errors', 0],
|
|
193
|
+
['time', seconds(total('time'))],
|
|
194
|
+
])}>`,
|
|
195
|
+
...suites.flatMap((s) => s.lines),
|
|
196
|
+
'</testsuites>',
|
|
197
|
+
'',
|
|
198
|
+
].join('\n');
|
|
199
|
+
|
|
200
|
+
return { xml, suites: suites.length, tests, failures };
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
module.exports = { toJUnit };
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// What counts as a failure in a result file — the one definition shared by
|
|
4
|
+
// `result finalize` (src/result/store.js, which derives the run status and
|
|
5
|
+
// counts) and the JUnit export (src/result/junit.js, which decides which
|
|
6
|
+
// entries render a <failure>). Keeping both on these predicates is what stops
|
|
7
|
+
// the result file and its export disagreeing (#296); the cross-check in
|
|
8
|
+
// tests/unit/result/outcome.test.js renders derived statuses through toJUnit.
|
|
9
|
+
//
|
|
10
|
+
// The predicates take anything: the JUnit renderer reads older and
|
|
11
|
+
// hand-edited files, so a null or non-object entry is simply not failing.
|
|
12
|
+
|
|
13
|
+
const statusOf = (entry) => (entry !== null && typeof entry === 'object' ? entry.status : undefined);
|
|
14
|
+
|
|
15
|
+
const isFailedStep = (step) => statusOf(step) === 'failed';
|
|
16
|
+
const isErroredStep = (step) => statusOf(step) === 'error';
|
|
17
|
+
const isFailingStep = (step) => isFailedStep(step) || isErroredStep(step);
|
|
18
|
+
const isFailingAssertion = (assertion) => statusOf(assertion) === 'failed';
|
|
19
|
+
|
|
20
|
+
// The run status `result finalize` derives when `--status` is omitted (#296).
|
|
21
|
+
// It used to read assertions only, so a run whose step failed finalized
|
|
22
|
+
// `passed` while its JUnit export was red. Every entry counts — a step
|
|
23
|
+
// recorded twice (fail, then pass) still failed once, matching the export.
|
|
24
|
+
// Skipped steps count for nothing; a run with no steps and no assertions
|
|
25
|
+
// stays `passed` (#275 owns that case).
|
|
26
|
+
function deriveRunStatus(steps = [], assertions = []) {
|
|
27
|
+
if (steps.some(isErroredStep)) return 'error';
|
|
28
|
+
if (steps.some(isFailedStep)) return 'failed';
|
|
29
|
+
if (assertions.some(isFailingAssertion)) return 'failed';
|
|
30
|
+
return 'passed';
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
module.exports = { isFailedStep, isErroredStep, isFailingStep, isFailingAssertion, deriveRunStatus };
|
package/src/result/store.js
CHANGED
|
@@ -6,6 +6,7 @@ const path = require('path');
|
|
|
6
6
|
const { atomicWrite } = require('../util/atomic');
|
|
7
7
|
const { withLock } = require('../util/lock');
|
|
8
8
|
const { OBSERVATION_TYPES, stepStatus } = require('./flags');
|
|
9
|
+
const { deriveRunStatus, isErroredStep, isFailedStep, isFailingAssertion } = require('./outcome');
|
|
9
10
|
const { workspaceDir } = require('../workspace/layout');
|
|
10
11
|
|
|
11
12
|
// Incremental result accumulator that persists to
|
|
@@ -18,6 +19,15 @@ const { workspaceDir } = require('../workspace/layout');
|
|
|
18
19
|
|
|
19
20
|
const SCHEMA_VERSION = '2.0';
|
|
20
21
|
|
|
22
|
+
// The one place that knows where a run's result file lives.
|
|
23
|
+
function resultsDir(projectRoot) {
|
|
24
|
+
return path.join(workspaceDir(projectRoot), 'results');
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function resultFile(projectRoot, runId) {
|
|
28
|
+
return path.join(resultsDir(projectRoot), `${runId}.json`);
|
|
29
|
+
}
|
|
30
|
+
|
|
21
31
|
// A crash report is read into an agent's context and routinely committed to a
|
|
22
32
|
// repo. A native tombstone is tens of kilobytes, so the excerpt is bounded HERE
|
|
23
33
|
// rather than trusted from the caller — the full report's home is report_path.
|
|
@@ -33,22 +43,37 @@ function defaultMetadata(overrides = {}) {
|
|
|
33
43
|
};
|
|
34
44
|
}
|
|
35
45
|
|
|
46
|
+
// The default summary's tail naming the failing steps, so a run a step turned
|
|
47
|
+
// red does not read `failed: 0/0 assertion(s) passed` with no reason (#296).
|
|
48
|
+
// Empty when no step failed, which keeps the pre-#296 text for those runs.
|
|
49
|
+
function stepFailureSuffix(steps) {
|
|
50
|
+
const failed = steps.filter(isFailedStep).length;
|
|
51
|
+
const errored = steps.filter(isErroredStep).length;
|
|
52
|
+
return `${failed ? `, ${failed} failed` : ''}${errored ? `, ${errored} errored` : ''}`;
|
|
53
|
+
}
|
|
54
|
+
|
|
36
55
|
function assertionStatus(pass) {
|
|
37
56
|
return pass ? 'passed' : 'failed';
|
|
38
57
|
}
|
|
39
58
|
|
|
40
59
|
class ResultStore {
|
|
41
60
|
constructor({ runId, scenarioId, projectRoot, metadata } = {}) {
|
|
42
|
-
if (!runId) throw new Error('ResultStore requires a runId');
|
|
43
61
|
if (!projectRoot) throw new Error('ResultStore requires a projectRoot');
|
|
62
|
+
// The one gate for every writer (#295): the id becomes `<runId>.json` and
|
|
63
|
+
// `<runId>.lock` below, and atomicWrite creates parent directories, so an
|
|
64
|
+
// id with a separator would write outside results/. Same rule as the
|
|
65
|
+
// reader, readFinalizedResult, so the two cannot disagree on valid ids.
|
|
66
|
+
// Thrown before anything is read or written; the CLI's toEnvelope maps it
|
|
67
|
+
// to invalid_input (exit 3).
|
|
68
|
+
if (!isResultRunId(runId)) throw new InvalidRunIdError(projectRoot, runId);
|
|
44
69
|
|
|
45
70
|
this.runId = runId;
|
|
46
71
|
this.scenarioId = scenarioId || null;
|
|
47
72
|
this.projectRoot = projectRoot;
|
|
48
73
|
this._metadataOverrides = metadata || {};
|
|
49
74
|
|
|
50
|
-
this._dir =
|
|
51
|
-
this._file =
|
|
75
|
+
this._dir = resultsDir(projectRoot);
|
|
76
|
+
this._file = resultFile(projectRoot, runId);
|
|
52
77
|
// Per-runId advisory lock guarding every read-modify-write mutator. Lives
|
|
53
78
|
// in the same directory as the result file, so the lock is scoped to this
|
|
54
79
|
// run only — different runs never contend on it.
|
|
@@ -147,6 +172,8 @@ class ResultStore {
|
|
|
147
172
|
atomicWrite(this._file, contents);
|
|
148
173
|
}
|
|
149
174
|
|
|
175
|
+
// The in-progress marker below is private to this module; readers ask
|
|
176
|
+
// isInProgress() instead of keying on the field themselves.
|
|
150
177
|
_persistInProgress() {
|
|
151
178
|
const snapshot = {
|
|
152
179
|
run_id: this.runId,
|
|
@@ -326,10 +353,10 @@ class ResultStore {
|
|
|
326
353
|
}
|
|
327
354
|
|
|
328
355
|
const passed = this._assertions.filter((a) => a.status === 'passed').length;
|
|
329
|
-
const failed = this._assertions.filter(
|
|
356
|
+
const failed = this._assertions.filter(isFailingAssertion).length;
|
|
330
357
|
const total = this._assertions.length;
|
|
331
358
|
|
|
332
|
-
const resolvedStatus = status || (
|
|
359
|
+
const resolvedStatus = status || deriveRunStatus(this._steps, this._assertions);
|
|
333
360
|
|
|
334
361
|
const result = {
|
|
335
362
|
run_id: this.runId,
|
|
@@ -347,7 +374,8 @@ class ResultStore {
|
|
|
347
374
|
captured_variables: this._capturedVariables,
|
|
348
375
|
summary:
|
|
349
376
|
summary ||
|
|
350
|
-
`${resolvedStatus}: ${passed}/${total} assertion(s) passed across ${this._steps.length} step(s)
|
|
377
|
+
`${resolvedStatus}: ${passed}/${total} assertion(s) passed across ${this._steps.length} step(s)` +
|
|
378
|
+
`${stepFailureSuffix(this._steps)}.`,
|
|
351
379
|
};
|
|
352
380
|
|
|
353
381
|
// Additive and OMITTED when absent: a result file from a run with no
|
|
@@ -366,4 +394,91 @@ class ResultStore {
|
|
|
366
394
|
}
|
|
367
395
|
}
|
|
368
396
|
|
|
369
|
-
|
|
397
|
+
function isInProgress(parsed) {
|
|
398
|
+
return Boolean(parsed && parsed._in_progress);
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
// A run id names a file in results/ — `<runId>.json` — so it must be a single
|
|
402
|
+
// path segment: a non-empty string with no separator and no NUL. `.` and `..`
|
|
403
|
+
// pass, harmlessly: the appended `.json` keeps them inside results/. This is
|
|
404
|
+
// deliberately NOT the result schema's run_id pattern nor observe/paths.js's
|
|
405
|
+
// isValidRunId: it is the one rule shared by the writers (the ResultStore
|
|
406
|
+
// constructor and the `result` write verbs, #295) and the reader
|
|
407
|
+
// (readFinalizedResult), so an id finalize accepted is always exportable.
|
|
408
|
+
function isResultRunId(runId) {
|
|
409
|
+
return (
|
|
410
|
+
typeof runId === 'string' &&
|
|
411
|
+
runId.length > 0 &&
|
|
412
|
+
!runId.includes('\0') &&
|
|
413
|
+
path.basename(runId) === runId
|
|
414
|
+
);
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
// Typed error for a run id isResultRunId refuses, carrying the envelope's kind
|
|
418
|
+
// and hint (the DeviceResolutionError pattern). Thrown by the ResultStore
|
|
419
|
+
// constructor, returned as data by readFinalizedResult — one wording for both.
|
|
420
|
+
class InvalidRunIdError extends Error {
|
|
421
|
+
constructor(projectRoot, runId) {
|
|
422
|
+
super(`invalid run id "${runId}"`);
|
|
423
|
+
this.name = 'InvalidRunIdError';
|
|
424
|
+
this.kind = 'invalid_input';
|
|
425
|
+
this.hint = `A run id names a file in ${resultsDir(projectRoot)}; it cannot contain a path separator.`;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// Read a finalized result file for a report, WITHOUT going through the
|
|
430
|
+
// ResultStore constructor: that constructor renames a corrupt file aside as a
|
|
431
|
+
// `.corrupt.<ts>` sidecar, and a reader must never mutate the workspace it
|
|
432
|
+
// reads. This function performs no writes, renames or locks.
|
|
433
|
+
//
|
|
434
|
+
// "Finalized" is defined by its writer, `ResultStore#finalize` above: a JSON
|
|
435
|
+
// object with no `_in_progress` marker whose run_id is the requested id. The
|
|
436
|
+
// reader checks exactly that and nothing more — it does not hold files to the
|
|
437
|
+
// result schema, because finalize itself does not (a run recorded without
|
|
438
|
+
// --scenario-id is written with scenario_id null, and pre-0.38.0 files carry
|
|
439
|
+
// an upper-case status). Consumers tolerate shape beyond that object.
|
|
440
|
+
//
|
|
441
|
+
// Returns `{ result }` or `{ error: { kind, message, hint } }` where kind is
|
|
442
|
+
// `invalid_input` (bad id, no file, not JSON, in progress, not a finalized
|
|
443
|
+
// result) or `environment` (the file exists but could not be read).
|
|
444
|
+
function readFinalizedResult(projectRoot, runId) {
|
|
445
|
+
const invalid = (message, hint) => ({ error: { kind: 'invalid_input', message, hint } });
|
|
446
|
+
if (!isResultRunId(runId)) {
|
|
447
|
+
const { kind, message, hint } = new InvalidRunIdError(projectRoot, runId);
|
|
448
|
+
return { error: { kind, message, hint } };
|
|
449
|
+
}
|
|
450
|
+
const file = resultFile(projectRoot, runId);
|
|
451
|
+
let raw;
|
|
452
|
+
try {
|
|
453
|
+
raw = fs.readFileSync(file, 'utf8');
|
|
454
|
+
} catch (err) {
|
|
455
|
+
if (err && err.code === 'ENOENT') {
|
|
456
|
+
return invalid(
|
|
457
|
+
`no result file for ${runId}`,
|
|
458
|
+
`Expected ${file}; list the run ids with \`ls ${resultsDir(projectRoot)}\`.`
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
return { error: { kind: 'environment', message: `could not read ${file}: ${err.message}`, hint: null } };
|
|
462
|
+
}
|
|
463
|
+
let parsed;
|
|
464
|
+
try {
|
|
465
|
+
parsed = JSON.parse(raw);
|
|
466
|
+
} catch (err) {
|
|
467
|
+
return invalid(`result file for ${runId} is not valid JSON (${err.message})`, `Inspect ${file}.`);
|
|
468
|
+
}
|
|
469
|
+
const notAResult = (reason) =>
|
|
470
|
+
invalid(
|
|
471
|
+
`result file for ${runId} is not a finalized result: ${reason}`,
|
|
472
|
+
`Inspect ${file}; result files are written by \`mauto result finalize\`.`
|
|
473
|
+
);
|
|
474
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
475
|
+
return notAResult('(root) must be object');
|
|
476
|
+
}
|
|
477
|
+
if (isInProgress(parsed)) {
|
|
478
|
+
return invalid(`run ${runId} is not finalized`, `Run \`mauto result finalize --run-id ${runId}\` first.`);
|
|
479
|
+
}
|
|
480
|
+
if (parsed.run_id !== runId) return notAResult(`its run_id is ${JSON.stringify(parsed.run_id)}`);
|
|
481
|
+
return { result: parsed };
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
module.exports = { ResultStore, readFinalizedResult, resultFile, isResultRunId, InvalidRunIdError };
|