mobile-automator 0.38.4 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/package.json +4 -2
- package/src/assertion/evaluator.js +116 -14
- package/src/assertion/fields.js +106 -0
- package/src/assertion/observe.js +17 -0
- package/src/cli.js +180 -174
- package/src/device/action-catalog.js +4 -2
- package/src/device/bridge.js +17 -1
- package/src/device/mobile-mcp-tools.js +1 -0
- package/src/device/resolve-connection.js +1 -1
- package/src/device/semantic-press/index.js +19 -0
- package/src/device/semantic-press/resolver.js +1 -1
- package/src/{device/semantic-press/labels.js → dialog/permission-labels.js} +6 -3
- package/src/guide/content/execute.agnostic.md +74 -21
- package/src/guide/content/execute.aware.md +69 -15
- package/src/guide/content/execute.invariants.md +1 -0
- package/src/guide/content/generate.agnostic.md +3 -3
- package/src/guide/content/generate.aware.md +5 -5
- package/src/guide/emitter.js +2 -0
- package/src/init/host-plan.js +1 -0
- package/src/observe/trace.js +1 -1
- package/src/observe/vocab.js +6 -0
- package/src/output/device-fail.js +41 -0
- package/src/output/envelope.js +3 -3
- package/src/output/run-human.js +37 -0
- package/src/result/capability-catalog.js +48 -0
- package/src/result/executor.js +58 -0
- package/src/result/finalize-against-scenario.js +103 -0
- package/src/result/flags.js +49 -6
- package/src/result/junit.js +39 -16
- package/src/result/outcome.js +24 -2
- package/src/result/pending-hint.js +67 -0
- package/src/result/scenario-check.js +58 -0
- package/src/result/status-guard.js +46 -0
- package/src/result/store.js +159 -16
- package/src/run/actions/capture.js +21 -0
- package/src/run/actions/gestures.js +89 -0
- package/src/run/actions/index.js +48 -0
- package/src/run/actions/launch.js +30 -0
- package/src/run/actions/preconditions.js +44 -0
- package/src/run/actions/scroll.js +22 -0
- package/src/run/actions/semantic.js +16 -0
- package/src/run/actions/wait.js +66 -0
- package/src/run/assert-reference.js +118 -0
- package/src/run/assertions.js +102 -0
- package/src/run/clock.js +10 -0
- package/src/run/condition.js +80 -0
- package/src/run/fatal.js +15 -0
- package/src/run/handle.js +151 -0
- package/src/run/needs-agent.js +39 -0
- package/src/run/poll.js +38 -0
- package/src/run/prepare.js +107 -0
- package/src/run/reference-assertion.js +172 -0
- package/src/run/reference-screenshot.js +92 -0
- package/src/run/resolve.js +95 -0
- package/src/run/run-id.js +14 -0
- package/src/run/runner.js +104 -0
- package/src/run/scale.js +52 -0
- package/src/run/screenshots.js +163 -0
- package/src/run/settings.js +35 -0
- package/src/run/state.js +22 -0
- package/src/run/step.js +138 -0
- package/src/run/summary.js +73 -0
- package/src/run/trace.js +30 -0
- package/src/run/tree.js +96 -0
- package/src/run/variables.js +56 -0
- package/src/scenario/load.js +78 -0
- package/src/schemas/config_schema.json +29 -0
- package/src/schemas/result_schema.json +45 -3
- package/src/schemas/scenario_schema.json +27 -0
- package/src/util/as-array.js +8 -0
- package/src/util/parse-positive-int.js +15 -0
- package/src/util/parse-unit-interval.js +17 -0
package/src/cli.js
CHANGED
|
@@ -8,12 +8,18 @@ const { version: PKG_VERSION } = require('../package.json');
|
|
|
8
8
|
|
|
9
9
|
const { ok, fail, render, exitCodeFor } = require('./output/envelope');
|
|
10
10
|
const selectionStore = require('./device/selection');
|
|
11
|
-
const {
|
|
11
|
+
const { FS_ERROR_CODES, deviceHint, deviceFail } = require('./output/device-fail');
|
|
12
12
|
const { ScenarioValidator } = require('./scenario/validator');
|
|
13
|
-
const {
|
|
14
|
-
const { evaluate, MECHANICAL_TYPES } = require('./assertion/evaluator');
|
|
15
|
-
const {
|
|
16
|
-
const {
|
|
13
|
+
const { loadScenario } = require('./scenario/load');
|
|
14
|
+
const { evaluate, MECHANICAL_TYPES, REFERENCE_SCREENSHOT_TYPES } = require('./assertion/evaluator');
|
|
15
|
+
const { observe } = require('./assertion/observe');
|
|
16
|
+
const { fromCliFlags } = require('./assertion/fields');
|
|
17
|
+
const { ResultStore, readFinalizedResult, InvalidRunIdError, AlreadyDecidedError } = require('./result/store');
|
|
18
|
+
const {
|
|
19
|
+
OBSERVATION_TYPES, parseObservation, parseCapture, parseAssertionVerdict, parseSimilarityScore, RUN_STATUSES,
|
|
20
|
+
parseRunStatus, parseStepStatus,
|
|
21
|
+
} = require('./result/flags');
|
|
22
|
+
const { finalizeAgainstScenario } = require('./result/finalize-against-scenario');
|
|
17
23
|
const { toJUnit } = require('./result/junit');
|
|
18
24
|
const { atomicWrite } = require('./util/atomic');
|
|
19
25
|
const { MemoryStore } = require('./memory/store');
|
|
@@ -28,7 +34,7 @@ const { ADAPTERS } = require('./init/adapters');
|
|
|
28
34
|
const { inspectHosts, ADVISORY_FS_CODES } = require('./init/host-plan');
|
|
29
35
|
const { renderStaleNotice, installedBefore, refreshHint } = require('./init/stale-notice');
|
|
30
36
|
const { doctorReport } = require('./init/doctor');
|
|
31
|
-
const { isSemanticAction,
|
|
37
|
+
const { isSemanticAction, pressSemantic } = require('./device/semantic-press');
|
|
32
38
|
const connection = require('./device/connection');
|
|
33
39
|
const { isSupportedHost, unsupportedHostError } = require('./device/host-support');
|
|
34
40
|
const { record } = require('./observe/recorder');
|
|
@@ -43,6 +49,13 @@ const observeTelemetry = require('./observe/telemetry');
|
|
|
43
49
|
const observeSpool = require('./observe/spool');
|
|
44
50
|
const observeTransport = require('./observe/transport');
|
|
45
51
|
const { makeFlusher } = require('./observe/flush');
|
|
52
|
+
const { prepareRun } = require('./run/prepare');
|
|
53
|
+
const { handleRun } = require('./run/handle');
|
|
54
|
+
const { renderRunHuman } = require('./output/run-human');
|
|
55
|
+
const { realClock } = require('./run/clock');
|
|
56
|
+
const { handleAssertReference } = require('./run/assert-reference');
|
|
57
|
+
const { makeRunTrace } = require('./run/trace');
|
|
58
|
+
const { parsePositiveInt } = require('./util/parse-positive-int');
|
|
46
59
|
const {
|
|
47
60
|
findWorkspaceRoot,
|
|
48
61
|
workspaceFailure,
|
|
@@ -92,29 +105,22 @@ const MODE_ALIASES = {
|
|
|
92
105
|
|
|
93
106
|
const KNOWN_ASSERTION_TYPES = new Set([
|
|
94
107
|
...MECHANICAL_TYPES,
|
|
108
|
+
// Decided by comparing a stable screenshot against a reference image (#281).
|
|
109
|
+
...REFERENCE_SCREENSHOT_TYPES,
|
|
95
110
|
// Tier-2 visual types the agent must judge.
|
|
96
|
-
'screenshot_match',
|
|
97
111
|
'visual_state',
|
|
98
|
-
'element_fully_visible',
|
|
99
112
|
'color_style',
|
|
100
|
-
'screen_title',
|
|
101
113
|
'alert_present',
|
|
102
114
|
'alert_text',
|
|
103
115
|
'toast_visible',
|
|
104
116
|
'keyboard_visible',
|
|
105
117
|
'dark_mode_active',
|
|
106
|
-
'permission_dialog_shown',
|
|
107
118
|
'element_state',
|
|
108
119
|
]);
|
|
109
120
|
|
|
110
121
|
const SWIPE_DIRECTIONS = new Set(['up', 'down', 'left', 'right']);
|
|
111
122
|
const ORIENTATIONS = new Set(['portrait', 'landscape']);
|
|
112
123
|
|
|
113
|
-
// fs error codes that signal an environment/filesystem problem (perms, missing
|
|
114
|
-
// path, read-only FS, disk full) rather than a logic bug — surfaced as
|
|
115
|
-
// `environment` with a filesystem-specific hint.
|
|
116
|
-
const FS_ERROR_CODES = new Set(['EACCES', 'ENOENT', 'EROFS', 'ENOSPC', 'EPERM']);
|
|
117
|
-
|
|
118
124
|
// Build a human-readable usage line for a commander command, e.g.
|
|
119
125
|
// "Usage: mauto result add-step [options]". Walks the parent chain so a
|
|
120
126
|
// subcommand's hint names the full invocation path.
|
|
@@ -130,6 +136,7 @@ function usageLine(cmd) {
|
|
|
130
136
|
// is the single boundary that guarantees a calling agent always receives one
|
|
131
137
|
// JSON line + a meaningful exit code instead of a raw stack trace on stderr.
|
|
132
138
|
// CommanderError (parse failure) -> invalid_input, with the usage line as hint
|
|
139
|
+
// InvalidRunIdError -> invalid_input, with the store's hint
|
|
133
140
|
// SyntaxError -> invalid_input (a malformed JSON file)
|
|
134
141
|
// fs EACCES/ENOENT/EROFS/... -> environment, with a filesystem hint
|
|
135
142
|
// anything else -> internal
|
|
@@ -144,6 +151,12 @@ function toEnvelope(err) {
|
|
|
144
151
|
exitKind: 'invalid_input',
|
|
145
152
|
};
|
|
146
153
|
}
|
|
154
|
+
// A --run-id the ResultStore refuses (#295), or a pending verdict over a
|
|
155
|
+
// decided one (#275): the store owns the rule and the wording, so every
|
|
156
|
+
// result verb reports it identically.
|
|
157
|
+
if (err instanceof InvalidRunIdError || err instanceof AlreadyDecidedError) {
|
|
158
|
+
return { envelope: fail(err.kind, err.message, err.hint), exitKind: err.kind };
|
|
159
|
+
}
|
|
147
160
|
if (err instanceof SyntaxError) {
|
|
148
161
|
return {
|
|
149
162
|
envelope: fail(
|
|
@@ -170,16 +183,6 @@ function toEnvelope(err) {
|
|
|
170
183
|
};
|
|
171
184
|
}
|
|
172
185
|
|
|
173
|
-
// The hint for any device-kind failure. Precedence: an explicit err.hint wins
|
|
174
|
-
// (our own code attached it knowing the exact cause — a DeviceResolutionError,
|
|
175
|
-
// the daemon-log pointer, a timeout); then engineHint, which recognises engine
|
|
176
|
-
// messages that are useless on their own (the 1.0.5 engine's missing iOS agent
|
|
177
|
-
// and changed Android device ids, #199); then the handler's generic fallback.
|
|
178
|
-
// Every device fail() below goes through here so no handler can miss it.
|
|
179
|
-
function deviceHint(err, fallback) {
|
|
180
|
-
return (err && err.hint) || engineHint(err && err.message) || fallback;
|
|
181
|
-
}
|
|
182
|
-
|
|
183
186
|
// ---------------------------------------------------------------------------
|
|
184
187
|
// Handlers — pure-ish: accept injected deps, return { envelope, exitKind }.
|
|
185
188
|
// No process.exit / printing here so they are trivially unit-testable.
|
|
@@ -221,59 +224,12 @@ async function handleScreenshot({ deviceBridge, cwd = process.cwd() }, destPath)
|
|
|
221
224
|
}
|
|
222
225
|
|
|
223
226
|
function handleValidate({ validator }, file, opts = {}) {
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
return {
|
|
229
|
-
envelope: fail(
|
|
230
|
-
'invalid_input',
|
|
231
|
-
`cannot read file: ${err.message}`,
|
|
232
|
-
'Check that the path exists and is readable.'
|
|
233
|
-
),
|
|
234
|
-
exitKind: 'invalid_input',
|
|
235
|
-
};
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
let scenario;
|
|
239
|
-
try {
|
|
240
|
-
scenario = JSON.parse(raw);
|
|
241
|
-
} catch (err) {
|
|
242
|
-
return {
|
|
243
|
-
envelope: fail(
|
|
244
|
-
'invalid_input',
|
|
245
|
-
`file is not valid JSON: ${err.message}`,
|
|
246
|
-
'Fix the JSON syntax and try again.'
|
|
247
|
-
),
|
|
248
|
-
exitKind: 'invalid_input',
|
|
249
|
-
};
|
|
227
|
+
const loaded = loadScenario(validator, file, opts);
|
|
228
|
+
if (loaded.failure) return loaded.failure;
|
|
229
|
+
if (loaded.warnings.length === 0) {
|
|
230
|
+
return { envelope: ok({ valid: true }), exitKind: 'ok' };
|
|
250
231
|
}
|
|
251
|
-
|
|
252
|
-
const result = validator.validate(scenario);
|
|
253
|
-
if (result.valid) {
|
|
254
|
-
// Schema-valid is not the same as replayable: surface findings by default
|
|
255
|
-
// so existing scenarios keep validating; `--strict` fails only on the
|
|
256
|
-
// blocking ones. Codes, severity and wording live in the replayability module.
|
|
257
|
-
const warnings = findReplayWarnings(scenario);
|
|
258
|
-
if (warnings.length === 0) {
|
|
259
|
-
return { envelope: ok({ valid: true }), exitKind: 'ok' };
|
|
260
|
-
}
|
|
261
|
-
const { blocking, hint } = summarize(warnings);
|
|
262
|
-
if (opts.strict && blocking.length > 0) {
|
|
263
|
-
const env = fail('invalid_input', 'scenario cannot replay unattended; see data.warnings', hint);
|
|
264
|
-
env.data = { valid: false, errors: [], warnings };
|
|
265
|
-
return { envelope: env, exitKind: 'invalid_input' };
|
|
266
|
-
}
|
|
267
|
-
return { envelope: ok({ valid: true, warnings }, hint), exitKind: 'ok' };
|
|
268
|
-
}
|
|
269
|
-
|
|
270
|
-
const env = fail(
|
|
271
|
-
'invalid_input',
|
|
272
|
-
'scenario failed schema validation',
|
|
273
|
-
'See data.errors for the specific schema violations.'
|
|
274
|
-
);
|
|
275
|
-
env.data = { valid: false, errors: result.errors };
|
|
276
|
-
return { envelope: env, exitKind: 'invalid_input' };
|
|
232
|
+
return { envelope: ok({ valid: true, warnings: loaded.warnings }, loaded.hint), exitKind: 'ok' };
|
|
277
233
|
}
|
|
278
234
|
|
|
279
235
|
// The unsupported-host gate (#165), shared by every site that reaches the
|
|
@@ -289,21 +245,6 @@ function hostGate(platform) {
|
|
|
289
245
|
return { envelope: fail(err.kind, err.message, err.hint), exitKind: err.kind };
|
|
290
246
|
}
|
|
291
247
|
|
|
292
|
-
function deviceFail(err) {
|
|
293
|
-
// Respect a typed error's kind (e.g. the daemon's 'timeout') so the envelope
|
|
294
|
-
// and exit code reflect the real failure class; anything else defaults to
|
|
295
|
-
// 'device' (exit 2) as before.
|
|
296
|
-
const kind = err.kind || 'device';
|
|
297
|
-
return {
|
|
298
|
-
envelope: fail(
|
|
299
|
-
kind,
|
|
300
|
-
err.message || String(err),
|
|
301
|
-
deviceHint(err, 'Ensure a device or simulator is connected and the app is running.')
|
|
302
|
-
),
|
|
303
|
-
exitKind: kind,
|
|
304
|
-
};
|
|
305
|
-
}
|
|
306
|
-
|
|
307
248
|
// Parse a "x,y" coordinate string shared by tap / long-press / double-tap.
|
|
308
249
|
// Returns { x, y } (rounded ints) on success, or { error } carrying the
|
|
309
250
|
// invalid_input fail envelope so callers can short-circuit uniformly.
|
|
@@ -368,8 +309,8 @@ async function handleLongPress({ deviceBridge }, raw, { duration } = {}) {
|
|
|
368
309
|
const { x, y } = coords;
|
|
369
310
|
let ms;
|
|
370
311
|
if (duration !== undefined) {
|
|
371
|
-
ms =
|
|
372
|
-
if (
|
|
312
|
+
ms = parsePositiveInt(duration);
|
|
313
|
+
if (ms === null) {
|
|
373
314
|
return {
|
|
374
315
|
envelope: fail('invalid_input', `duration must be a positive integer in ms, got "${duration}"`, 'Pass --duration <ms>, e.g. --duration 800.'),
|
|
375
316
|
exitKind: 'invalid_input',
|
|
@@ -546,9 +487,7 @@ async function handlePress({ deviceBridge }, button) {
|
|
|
546
487
|
// resolution is a hard error, never a bogus token forwarded to mobile-mcp.
|
|
547
488
|
if (isSemanticAction(b)) {
|
|
548
489
|
try {
|
|
549
|
-
const platform = await deviceBridge
|
|
550
|
-
const resolver = selectResolver(platform, deviceBridge);
|
|
551
|
-
const { mechanism } = await resolver[ACTION_METHOD[b]]();
|
|
490
|
+
const { platform, mechanism } = await pressSemantic(deviceBridge, b);
|
|
552
491
|
return {
|
|
553
492
|
envelope: ok({ pressed: b, resolved: { platform, mechanism } }),
|
|
554
493
|
exitKind: 'ok',
|
|
@@ -573,34 +512,33 @@ async function handlePress({ deviceBridge }, button) {
|
|
|
573
512
|
// with the verdict still in `data`. A pass and an undecided Tier-2 verdict
|
|
574
513
|
// (`needs_agent`) stay ok / exit 0. Only an unknown assertion type is a
|
|
575
514
|
// structural (invalid_input) error.
|
|
576
|
-
async function handleAssert(
|
|
515
|
+
async function handleAssert(deps, type, flags = {}) {
|
|
516
|
+
const { deviceBridge } = deps;
|
|
577
517
|
if (!KNOWN_ASSERTION_TYPES.has(type)) {
|
|
578
518
|
return {
|
|
579
519
|
envelope: fail('invalid_input', `unknown assertion type "${type}"`, 'Run `mauto schema scenario` for the supported assertion types.'),
|
|
580
520
|
exitKind: 'invalid_input',
|
|
581
521
|
};
|
|
582
522
|
}
|
|
523
|
+
if (REFERENCE_SCREENSHOT_TYPES.has(type)) return handleAssertReference(deps, type, flags);
|
|
583
524
|
|
|
584
|
-
let
|
|
525
|
+
let observed;
|
|
585
526
|
try {
|
|
586
|
-
|
|
527
|
+
observed = await observe(deviceBridge, type);
|
|
587
528
|
} catch (err) {
|
|
588
529
|
return deviceFail(err);
|
|
589
530
|
}
|
|
531
|
+
const { elements, screenSize } = observed;
|
|
590
532
|
|
|
591
|
-
const assertion =
|
|
592
|
-
if (flags.target !== undefined) assertion.target = flags.target;
|
|
593
|
-
if (flags.expected !== undefined) assertion.expected = flags.expected;
|
|
594
|
-
if (flags.operator !== undefined) assertion.operator = flags.operator;
|
|
595
|
-
if (flags.count !== undefined) assertion.count = Number(flags.count);
|
|
596
|
-
if (flags.pattern !== undefined) assertion.pattern = flags.pattern;
|
|
597
|
-
if (flags.variable !== undefined) assertion.variable_name = flags.variable;
|
|
533
|
+
const assertion = fromCliFlags(type, flags);
|
|
598
534
|
|
|
599
|
-
const r = evaluate(assertion, { elements });
|
|
535
|
+
const r = evaluate(assertion, { elements, screenSize });
|
|
600
536
|
// The verdict has to be transcribed into `result add-assertion` by the agent
|
|
601
537
|
// (assert has no run context, and Tier-2 types are not decided here at all).
|
|
602
538
|
// Emitting the exact command turns transcription into copying (#140 D1).
|
|
603
|
-
|
|
539
|
+
// A Tier-2 verdict is the agent's, so it is recorded `--judge agent` (#275);
|
|
540
|
+
// a mechanical one is the CLI's and claims no judge.
|
|
541
|
+
const verdict = r.needs_agent ? '<true|false> --judge agent' : String(r.pass);
|
|
604
542
|
const hint =
|
|
605
543
|
`Record it: mauto result add-assertion --run-id <run_id> --step-id <step_id> ` +
|
|
606
544
|
`--type ${r.type} --pass ${verdict} --message "<why>"`;
|
|
@@ -730,20 +668,31 @@ function handleResultAddAssertion({ resultStoreFactory, projectRoot }, opts) {
|
|
|
730
668
|
exitKind: 'invalid_input',
|
|
731
669
|
};
|
|
732
670
|
}
|
|
733
|
-
const verdict =
|
|
671
|
+
const verdict = parseAssertionVerdict(pass, opts.judge);
|
|
734
672
|
if (verdict.error) {
|
|
735
673
|
return { envelope: fail('invalid_input', verdict.error, null), exitKind: 'invalid_input' };
|
|
736
674
|
}
|
|
675
|
+
// A screenshot_match's comparison (#281), as `mauto assert` reports it.
|
|
676
|
+
const similarity = parseSimilarityScore(opts.similarityScore);
|
|
677
|
+
if (similarity.error) {
|
|
678
|
+
return { envelope: fail('invalid_input', similarity.error, null), exitKind: 'invalid_input' };
|
|
679
|
+
}
|
|
737
680
|
|
|
738
681
|
const store = resultStoreFactory({ runId, scenarioId, projectRoot });
|
|
739
682
|
const entry = store.addAssertion({
|
|
740
683
|
step_id: stepId,
|
|
741
684
|
assertion_id: assertionId,
|
|
742
685
|
type,
|
|
743
|
-
|
|
686
|
+
mechanical: MECHANICAL_TYPES.has(type),
|
|
687
|
+
status: verdict.value.status,
|
|
688
|
+
judge: verdict.value.judge,
|
|
744
689
|
message,
|
|
745
690
|
expected: expected === undefined ? null : expected,
|
|
746
691
|
actual: actual === undefined ? null : actual,
|
|
692
|
+
// The store records these only when supplied (it skips null/undefined).
|
|
693
|
+
reference_screenshot: opts.referenceScreenshot,
|
|
694
|
+
actual_screenshot: opts.actualScreenshot,
|
|
695
|
+
similarity_score: similarity.value,
|
|
747
696
|
});
|
|
748
697
|
return { envelope: ok({ run_id: runId, assertion: entry }, storeHint(store)), exitKind: 'ok' };
|
|
749
698
|
}
|
|
@@ -817,44 +766,16 @@ function measureRun({ projectRoot, runId, reported }) {
|
|
|
817
766
|
};
|
|
818
767
|
}
|
|
819
768
|
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
}
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
// variant of a documented word, a typo) used to be persisted verbatim — a
|
|
831
|
-
// schema-invalid result file with ok:true and exit 0 — directly under the
|
|
832
|
-
// exit-6 contract keyed on that status (#286 review). Reject before anything
|
|
833
|
-
// is written. A provided-but-empty value is NOT an omitted one: only absence
|
|
834
|
-
// derives the status from the recorded steps and assertions.
|
|
835
|
-
if (status !== undefined) {
|
|
836
|
-
const parsed = parseRunStatus(status);
|
|
837
|
-
if (parsed.error) {
|
|
838
|
-
return {
|
|
839
|
-
envelope: fail(
|
|
840
|
-
'invalid_input',
|
|
841
|
-
parsed.error,
|
|
842
|
-
'Valid values: passed | failed | error, or omit --status to derive it from the recorded steps and assertions.'
|
|
843
|
-
),
|
|
844
|
-
exitKind: 'invalid_input',
|
|
845
|
-
};
|
|
846
|
-
}
|
|
847
|
-
}
|
|
848
|
-
|
|
849
|
-
// Only provided keys: `defaultMetadata` fills the rest with 'unknown', and
|
|
850
|
-
// the result schema requires all five fields to be strings, so threading
|
|
851
|
-
// `undefined` through would break conformance.
|
|
852
|
-
const metadata = {};
|
|
853
|
-
if (opts.appVersion !== undefined) metadata.app_version = opts.appVersion;
|
|
854
|
-
if (opts.deviceModel !== undefined) metadata.device_model = opts.deviceModel;
|
|
855
|
-
if (opts.apiLevel !== undefined) metadata.api_level = opts.apiLevel;
|
|
856
|
-
if (opts.environment !== undefined) metadata.environment = opts.environment;
|
|
857
|
-
|
|
769
|
+
// Finalize a run: measure its duration from the run trace, write the result
|
|
770
|
+
// file (status derived when omitted), prune other runs' traces, and harvest
|
|
771
|
+
// it into cross-session memory. Returns `{ result, store, hints }`; hints
|
|
772
|
+
// collects the store's and memory's best-effort warnings. Shared by
|
|
773
|
+
// `result finalize` and `mauto run`; flag validation and the envelope stay
|
|
774
|
+
// with the callers.
|
|
775
|
+
function finalizeRun(
|
|
776
|
+
{ resultStoreFactory, memoryStoreFactory, projectRoot },
|
|
777
|
+
{ runId, scenarioId, status, duration, metadata, summary, executor }
|
|
778
|
+
) {
|
|
858
779
|
// Measure BEFORE constructing the store: the trace is read-only here, and
|
|
859
780
|
// deriving first keeps the store's lock window as small as it already is.
|
|
860
781
|
const measured = measureRun({ projectRoot, runId, reported: duration });
|
|
@@ -873,7 +794,9 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
|
|
|
873
794
|
// `summary || <generated default>` already treats that as "no override",
|
|
874
795
|
// so no provided-keys-only guard is needed here (unlike metadata, whose
|
|
875
796
|
// sub-fields default independently to 'unknown').
|
|
876
|
-
summary
|
|
797
|
+
summary,
|
|
798
|
+
// Who produced the result (#275); the store keeps the first one it saw.
|
|
799
|
+
executor,
|
|
877
800
|
});
|
|
878
801
|
|
|
879
802
|
// Retention, at the only moment a run is known to be over and the only place
|
|
@@ -899,26 +822,42 @@ function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectR
|
|
|
899
822
|
if (memHint) hints.push(memHint);
|
|
900
823
|
}
|
|
901
824
|
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
// already run: exit 6 is a verdict about the app, not a reason to lose the
|
|
909
|
-
// record of it.
|
|
910
|
-
if (result.status === 'failed' || result.status === 'error') {
|
|
911
|
-
const hint = hints.length
|
|
912
|
-
? hints.join(' ')
|
|
913
|
-
: 'See data.steps_executed and data.assertion_results for what failed; ' +
|
|
914
|
-
`the result is saved to ${path.relative(projectRoot, store.file)}.`;
|
|
825
|
+
return { result, store, hints };
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
function handleResultFinalize({ resultStoreFactory, memoryStoreFactory, projectRoot, validator, cwd = process.cwd() }, opts) {
|
|
829
|
+
const { runId, status } = opts;
|
|
830
|
+
if (!runId) {
|
|
915
831
|
return {
|
|
916
|
-
envelope: fail('
|
|
917
|
-
exitKind: '
|
|
832
|
+
envelope: fail('invalid_input', '--run-id is required', null),
|
|
833
|
+
exitKind: 'invalid_input',
|
|
918
834
|
};
|
|
919
835
|
}
|
|
920
836
|
|
|
921
|
-
|
|
837
|
+
// The status flag is a schema enum, not free text: an unlisted value (a case
|
|
838
|
+
// variant of a documented word, a typo) used to be persisted verbatim — a
|
|
839
|
+
// schema-invalid result file with ok:true and exit 0 — directly under the
|
|
840
|
+
// exit-6 contract keyed on that status (#286 review). Reject before anything
|
|
841
|
+
// is written. A provided-but-empty value is NOT an omitted one: only absence
|
|
842
|
+
// derives the status from the recorded steps and assertions.
|
|
843
|
+
if (status !== undefined) {
|
|
844
|
+
const parsed = parseRunStatus(status);
|
|
845
|
+
if (parsed.error) {
|
|
846
|
+
return {
|
|
847
|
+
envelope: fail(
|
|
848
|
+
'invalid_input',
|
|
849
|
+
parsed.error,
|
|
850
|
+
`Valid values: ${RUN_STATUSES.join(' | ')}, or omit --status to derive it from the recorded steps and assertions.`
|
|
851
|
+
),
|
|
852
|
+
exitKind: 'invalid_input',
|
|
853
|
+
};
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
return finalizeAgainstScenario(
|
|
858
|
+
{ validator, resultStoreFactory, memoryStoreFactory, projectRoot, cwd, finalizeRun },
|
|
859
|
+
opts
|
|
860
|
+
);
|
|
922
861
|
}
|
|
923
862
|
|
|
924
863
|
// Formats `mauto result export --format` accepts. The only one, junit, is
|
|
@@ -1867,6 +1806,9 @@ function buildProgram(deps = {}) {
|
|
|
1867
1806
|
emit = emitters.emit,
|
|
1868
1807
|
// Host OS, for the unsupported-host gate (#165). Overridable in tests.
|
|
1869
1808
|
platform = process.platform,
|
|
1809
|
+
// `{ now, sleep }` for `mauto run`'s polls and retry backoff. Overridable
|
|
1810
|
+
// in tests, which run on a fake clock.
|
|
1811
|
+
clock = realClock,
|
|
1870
1812
|
} = deps;
|
|
1871
1813
|
|
|
1872
1814
|
const program = new Command();
|
|
@@ -2085,6 +2027,45 @@ function buildProgram(deps = {}) {
|
|
|
2085
2027
|
const withBridge = (explicitDevice, fn) =>
|
|
2086
2028
|
connectBridge(resolveVerbDevice(explicitDevice), fn);
|
|
2087
2029
|
|
|
2030
|
+
// `mauto run <scenario>` (replay R1, #278; graduated in #284). prepareRun
|
|
2031
|
+
// (src/run/prepare.js) makes every refusal before connectBridge, so none
|
|
2032
|
+
// touches a device.
|
|
2033
|
+
program
|
|
2034
|
+
.command('run <scenario>')
|
|
2035
|
+
.description('Replay a scenario unattended and finalize its result')
|
|
2036
|
+
.option('--device <id>', 'target device id')
|
|
2037
|
+
.option('--run-id <id>', 'run identifier (run_YYYYMMDD_HHMMSS); minted when omitted')
|
|
2038
|
+
.option('--timeout-ms <ms>', 'cap, in milliseconds, on how long any one step waits (a positive integer)')
|
|
2039
|
+
.option('--update-references', 'record each screenshot_match capture as its reference image instead of comparing')
|
|
2040
|
+
.option('--judge <who>', 'who decides the assertions only an agent can judge (accepted: none)', 'none')
|
|
2041
|
+
.option('--junit <path>', 'also write the run as a JUnit XML report to this file (its directory must exist)')
|
|
2042
|
+
.action(withEnvelope(requireWorkspace(async (file, opts) => {
|
|
2043
|
+
const prepared = prepareRun({
|
|
2044
|
+
validator, file: path.resolve(cwd, file), runId: opts.runId, timeoutMs: opts.timeoutMs, judge: opts.judge,
|
|
2045
|
+
junit: opts.junit, cwd, env: process.env,
|
|
2046
|
+
});
|
|
2047
|
+
if (prepared.failure) return emit(prepared.failure, humanFlag());
|
|
2048
|
+
// The preAction hook saw only the flag and the environment; this covers
|
|
2049
|
+
// a minted id, so verb.end lands in the same trace as the run's stamps.
|
|
2050
|
+
emitters.setRunId(prepared.runId);
|
|
2051
|
+
// Resolved here, not inside withBridge, so the executor records the
|
|
2052
|
+
// device the run connected to (null: mobile-mcp auto-selected one).
|
|
2053
|
+
const deviceId = resolveVerbDevice(opts.device);
|
|
2054
|
+
// Every envelope handleRun returns carries the run's frozen data, so
|
|
2055
|
+
// each one renders as the run summary under --human. A refusal above, a
|
|
2056
|
+
// connect failure or a thrown error take the generic render.
|
|
2057
|
+
return connectBridge(deviceId, async (bridge) => ({
|
|
2058
|
+
...(await handleRun(
|
|
2059
|
+
{
|
|
2060
|
+
bridge, resultStoreFactory, memoryStoreFactory, projectRoot, finalizeRun, clock, cwd,
|
|
2061
|
+
trace: makeRunTrace({ projectRoot, runId: prepared.runId }),
|
|
2062
|
+
},
|
|
2063
|
+
{ ...prepared, deviceId, updateReferences: opts.updateReferences === true }
|
|
2064
|
+
)),
|
|
2065
|
+
renderHuman: renderRunHuman,
|
|
2066
|
+
}));
|
|
2067
|
+
})));
|
|
2068
|
+
|
|
2088
2069
|
program
|
|
2089
2070
|
.command('tap')
|
|
2090
2071
|
.description('Tap at absolute screen coordinates')
|
|
@@ -2165,23 +2146,33 @@ function buildProgram(deps = {}) {
|
|
|
2165
2146
|
|
|
2166
2147
|
program
|
|
2167
2148
|
.command('assert <type>')
|
|
2168
|
-
.description('Evaluate an assertion against the current screen (mechanical types decided by the CLI; visual types deferred to the agent)')
|
|
2149
|
+
.description('Evaluate an assertion against the current screen (mechanical types decided by the CLI; screenshot_match compared against --reference; other visual types deferred to the agent)')
|
|
2169
2150
|
.option('--target <s>', 'element/target description')
|
|
2170
2151
|
.option('--expected <s>', 'expected value')
|
|
2171
2152
|
.option('--operator <op>', 'comparison operator for count assertions')
|
|
2172
2153
|
.option('--count <n>', 'expected count')
|
|
2173
2154
|
.option('--pattern <re>', 'regex pattern')
|
|
2174
2155
|
.option('--variable <name>', 'captured variable name')
|
|
2156
|
+
.option('--permission <name>', 'permission_dialog_shown: the permission the dialog must name (e.g. camera)')
|
|
2157
|
+
.option('--reference <path>', 'screenshot_match: reference image, relative to the workspace directory')
|
|
2158
|
+
.option('--tolerance <n>', 'screenshot_match: similarity it must reach, 0..1 (default 0.9)')
|
|
2159
|
+
.option('--mask <spec>', 'screenshot_match: region to ignore, "x,y,width,height" or an element description; repeatable', collect, [])
|
|
2160
|
+
.option('--update-references', 'screenshot_match: record this screen as the reference instead of comparing')
|
|
2175
2161
|
.option('--device <id>', 'target device id')
|
|
2176
2162
|
.action(withEnvelope((type, opts) =>
|
|
2177
2163
|
withBridge(opts.device, (bridge) =>
|
|
2178
|
-
handleAssert({ deviceBridge: bridge }, type, {
|
|
2164
|
+
handleAssert({ deviceBridge: bridge, projectRoot: workspace.root || null, clock }, type, {
|
|
2179
2165
|
target: opts.target,
|
|
2180
2166
|
expected: opts.expected,
|
|
2181
2167
|
operator: opts.operator,
|
|
2182
2168
|
count: opts.count,
|
|
2183
2169
|
pattern: opts.pattern,
|
|
2184
2170
|
variable: opts.variable,
|
|
2171
|
+
permission: opts.permission,
|
|
2172
|
+
reference: opts.reference,
|
|
2173
|
+
tolerance: opts.tolerance,
|
|
2174
|
+
mask: opts.mask,
|
|
2175
|
+
updateReferences: opts.updateReferences === true,
|
|
2185
2176
|
})
|
|
2186
2177
|
)
|
|
2187
2178
|
));
|
|
@@ -2232,11 +2223,15 @@ function buildProgram(deps = {}) {
|
|
|
2232
2223
|
.option('--scenario-id <id>', 'scenario identifier')
|
|
2233
2224
|
.requiredOption('--step-id <id>', 'step this assertion belongs to')
|
|
2234
2225
|
.requiredOption('--type <t>', 'assertion type (see `mauto schema scenario`)')
|
|
2235
|
-
.requiredOption('--pass <
|
|
2226
|
+
.requiredOption('--pass <verdict>', 'true | false | pending — the verdict, or pending for an agent to decide later')
|
|
2227
|
+
.option('--judge <who>', 'agent | none — who decided (pending implies none)')
|
|
2236
2228
|
.option('--assertion-id <id>', 'stable assertion identifier (derived when omitted)')
|
|
2237
2229
|
.option('--message <text>', 'human-readable justification for the verdict')
|
|
2238
2230
|
.option('--expected <v>', 'expected value')
|
|
2239
2231
|
.option('--actual <v>', 'observed value')
|
|
2232
|
+
.option('--reference-screenshot <path>', 'screenshot_match: the reference image compared against')
|
|
2233
|
+
.option('--actual-screenshot <path>', 'screenshot_match: the screenshot captured for it')
|
|
2234
|
+
.option('--similarity-score <n>', 'screenshot_match: the similarity measured (0..1)')
|
|
2240
2235
|
.action(withEnvelope(requireWorkspace((opts) => {
|
|
2241
2236
|
const r = handleResultAddAssertion(
|
|
2242
2237
|
{ resultStoreFactory, projectRoot },
|
|
@@ -2246,10 +2241,14 @@ function buildProgram(deps = {}) {
|
|
|
2246
2241
|
stepId: opts.stepId,
|
|
2247
2242
|
type: opts.type,
|
|
2248
2243
|
pass: opts.pass,
|
|
2244
|
+
judge: opts.judge,
|
|
2249
2245
|
assertionId: opts.assertionId,
|
|
2250
2246
|
message: opts.message,
|
|
2251
2247
|
expected: opts.expected,
|
|
2252
2248
|
actual: opts.actual,
|
|
2249
|
+
referenceScreenshot: opts.referenceScreenshot,
|
|
2250
|
+
actualScreenshot: opts.actualScreenshot,
|
|
2251
|
+
similarityScore: opts.similarityScore,
|
|
2253
2252
|
}
|
|
2254
2253
|
);
|
|
2255
2254
|
emit(r, humanFlag());
|
|
@@ -2288,7 +2287,8 @@ function buildProgram(deps = {}) {
|
|
|
2288
2287
|
.description('Assemble and write the final result file')
|
|
2289
2288
|
.requiredOption('--run-id <id>', 'run identifier')
|
|
2290
2289
|
.option('--scenario-id <id>', 'scenario identifier')
|
|
2291
|
-
.option('--status <s>', '
|
|
2290
|
+
.option('--status <s>', RUN_STATUSES.join(' | '))
|
|
2291
|
+
.option('--scenario <path>', 'scenario file to check the recorded ids against and hash into executor')
|
|
2292
2292
|
.option('--duration <secs>', 'total duration in seconds')
|
|
2293
2293
|
.option('--app-version <v>', 'app version under test')
|
|
2294
2294
|
.option('--device-model <v>', 'device the run executed on')
|
|
@@ -2297,11 +2297,12 @@ function buildProgram(deps = {}) {
|
|
|
2297
2297
|
.option('--summary <text>', 'override the generated one-line result summary')
|
|
2298
2298
|
.action(withEnvelope(requireWorkspace((opts) => {
|
|
2299
2299
|
const r = handleResultFinalize(
|
|
2300
|
-
{ resultStoreFactory, memoryStoreFactory, projectRoot },
|
|
2300
|
+
{ resultStoreFactory, memoryStoreFactory, projectRoot, validator, cwd },
|
|
2301
2301
|
{
|
|
2302
2302
|
runId: opts.runId,
|
|
2303
2303
|
scenarioId: opts.scenarioId,
|
|
2304
2304
|
status: opts.status,
|
|
2305
|
+
scenario: opts.scenario,
|
|
2305
2306
|
duration: opts.duration,
|
|
2306
2307
|
appVersion: opts.appVersion,
|
|
2307
2308
|
deviceModel: opts.deviceModel,
|
|
@@ -2671,8 +2672,11 @@ function makeEmitters({ projectRoot = process.cwd() } = {}) {
|
|
|
2671
2672
|
process.exit(exitCode);
|
|
2672
2673
|
}
|
|
2673
2674
|
|
|
2674
|
-
|
|
2675
|
-
|
|
2675
|
+
// `renderHuman` is a handler's own --human rendering of its envelope (only
|
|
2676
|
+
// `mauto run` has one, src/output/run-human.js); without --human, or
|
|
2677
|
+
// without one, the generic render. JSON output is the same either way.
|
|
2678
|
+
const fromEnvelope = (envelope, exitKind, human, renderHuman) => ({
|
|
2679
|
+
text: human && typeof renderHuman === 'function' ? renderHuman(envelope) : render(envelope, { human }),
|
|
2676
2680
|
exitKind,
|
|
2677
2681
|
ok: Boolean(envelope && envelope.ok),
|
|
2678
2682
|
errorKind: envelope && envelope.error ? envelope.error.kind : undefined,
|
|
@@ -2697,7 +2701,7 @@ function makeEmitters({ projectRoot = process.cwd() } = {}) {
|
|
|
2697
2701
|
getRunId: () => resolvedRunId,
|
|
2698
2702
|
getVerb: () => resolvedVerb,
|
|
2699
2703
|
|
|
2700
|
-
emit: ({ envelope, exitKind }, human) => finish(fromEnvelope(envelope, exitKind, human)),
|
|
2704
|
+
emit: ({ envelope, exitKind, renderHuman }, human) => finish(fromEnvelope(envelope, exitKind, human, renderHuman)),
|
|
2701
2705
|
|
|
2702
2706
|
// Raw content (markdown / JSON schema / text) verbatim, no envelope
|
|
2703
2707
|
// wrapping. Used by the guide/schema/bootstrap success paths.
|
|
@@ -2795,6 +2799,8 @@ module.exports = {
|
|
|
2795
2799
|
handleResultAddAssertion,
|
|
2796
2800
|
handleResultAddCrash,
|
|
2797
2801
|
handleResultFinalize,
|
|
2802
|
+
finalizeRun,
|
|
2803
|
+
handleRun,
|
|
2798
2804
|
handleResultExport,
|
|
2799
2805
|
handleSetup,
|
|
2800
2806
|
handleConfigGet,
|
|
@@ -16,8 +16,10 @@
|
|
|
16
16
|
// 'semantic' — invoked via `mauto press <action>`; resolved to per-platform
|
|
17
17
|
// mechanics by src/device/semantic-press (see ACTION_METHOD).
|
|
18
18
|
// 'composed' — no dedicated verb; the agent composes it from existing verbs
|
|
19
|
-
// (e.g. poll `mauto elements`, repeat `mauto swipe`)
|
|
20
|
-
//
|
|
19
|
+
// (e.g. poll `mauto elements`, repeat `mauto swipe`), or
|
|
20
|
+
// `mauto run` composes it in-process from the same bridge
|
|
21
|
+
// calls (src/run/actions/). The execute guide documents the
|
|
22
|
+
// agent's composition.
|
|
21
23
|
// 'unsupported' — no `mauto` verb reaches it: the engine has no primitive, or
|
|
22
24
|
// (clear_app_data, since 1.0.15) mauto has not wired it. Not
|
|
23
25
|
// mechanically executable; the guide must NOT promise a verb,
|
package/src/device/bridge.js
CHANGED
|
@@ -9,6 +9,9 @@ const { isAppNotRunning } = require('./app-not-running');
|
|
|
9
9
|
|
|
10
10
|
// Thin wrapper over an injected mobile-mcp `call(toolName, args)` function.
|
|
11
11
|
// Returns the agnostic element model and exposes only the primitives the CLI needs.
|
|
12
|
+
// The error code of the bridge's own "cannot report the platform" failure.
|
|
13
|
+
const PLATFORM_UNKNOWN = 'platform_unknown';
|
|
14
|
+
|
|
12
15
|
class DeviceBridge {
|
|
13
16
|
constructor({ call, device = null }) {
|
|
14
17
|
if (typeof call !== 'function') {
|
|
@@ -108,6 +111,9 @@ class DeviceBridge {
|
|
|
108
111
|
_unknownPlatform(id) {
|
|
109
112
|
const err = new Error(`Cannot determine the platform of device "${id}".`);
|
|
110
113
|
err.hint = 'Ensure the device is listed by `mauto devices`.';
|
|
114
|
+
// Callers that can live without a platform (mauto run's executor record)
|
|
115
|
+
// match on this, so every other failure still propagates.
|
|
116
|
+
err.code = PLATFORM_UNKNOWN;
|
|
111
117
|
return err;
|
|
112
118
|
}
|
|
113
119
|
|
|
@@ -199,6 +205,16 @@ class DeviceBridge {
|
|
|
199
205
|
return await this._call('mobile_set_orientation', { orientation });
|
|
200
206
|
}
|
|
201
207
|
|
|
208
|
+
// The device's current orientation, 'portrait' or 'landscape'. mobile-mcp
|
|
209
|
+
// answers in prose ("Current device orientation is portrait"), so the word
|
|
210
|
+
// is parsed out; anything else is a hard error rather than a guess.
|
|
211
|
+
async getOrientation() {
|
|
212
|
+
const r = await this._call('mobile_get_orientation', {});
|
|
213
|
+
const m = /\b(portrait|landscape)\b/i.exec(r && typeof r === 'object' ? JSON.stringify(r) : String(r));
|
|
214
|
+
if (!m) throw new Error(`Could not read the device orientation from "${String(r)}".`);
|
|
215
|
+
return m[1].toLowerCase();
|
|
216
|
+
}
|
|
217
|
+
|
|
202
218
|
// Crash reports currently readable on the device, as the agnostic crash
|
|
203
219
|
// model. DIAGNOSTIC only — never invoked by a scenario action, which is why
|
|
204
220
|
// `crash` has no src/device/action-catalog.js entry (that catalog holds
|
|
@@ -247,4 +263,4 @@ class DeviceBridge {
|
|
|
247
263
|
}
|
|
248
264
|
}
|
|
249
265
|
|
|
250
|
-
module.exports = { DeviceBridge };
|
|
266
|
+
module.exports = { DeviceBridge, PLATFORM_UNKNOWN };
|