mobile-debug-mcp 0.32.2 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/.agents/plugins/marketplace.json +20 -0
  2. package/{skills → .agents/skills}/mcp-builder/SKILL.md +1 -2
  3. package/{skills → .agents/skills}/spec-review/SKILL.md +1 -1
  4. package/{skills → .agents/skills}/test-authoring/SKILL.md +1 -1
  5. package/.claude-plugin/marketplace.json +14 -0
  6. package/.cursor-plugin/marketplace.json +16 -0
  7. package/.github/copilot-instructions.md +7 -0
  8. package/AGENTS.md +21 -10
  9. package/README.md +67 -5
  10. package/agent-plugin/.claude-plugin/plugin.json +20 -0
  11. package/agent-plugin/.codex-plugin/plugin.json +34 -0
  12. package/agent-plugin/.cursor-plugin/plugin.json +23 -0
  13. package/agent-plugin/.mcp.json +14 -0
  14. package/agent-plugin/LICENSE +201 -0
  15. package/agent-plugin/README.md +29 -0
  16. package/agent-plugin/assets/mobile-debug-tools.svg +28 -0
  17. package/agent-plugin/mcp.json +11 -0
  18. package/agent-plugin/plugin.json +29 -0
  19. package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +124 -0
  20. package/dist/interact/android.js +10 -4
  21. package/dist/interact/conditional-action.js +68 -0
  22. package/dist/interact/index.js +54 -14
  23. package/dist/interact/ios.js +14 -8
  24. package/dist/manage/android.js +4 -3
  25. package/dist/manage/ios.js +3 -2
  26. package/dist/observe/android.js +27 -22
  27. package/dist/observe/evidence.js +273 -0
  28. package/dist/observe/index.js +33 -6
  29. package/dist/observe/ios.js +25 -6
  30. package/dist/observe/readiness-service.js +115 -0
  31. package/dist/observe/readiness.js +81 -0
  32. package/dist/observe/recording.js +165 -0
  33. package/dist/observe/snapshot-metadata.js +10 -2
  34. package/dist/server/common.js +9 -0
  35. package/dist/server/condition-schema.js +32 -0
  36. package/dist/server/journey-schema.js +3 -2
  37. package/dist/server/readiness-schema.js +12 -0
  38. package/dist/server/tool-definitions.js +46 -0
  39. package/dist/server/tool-handlers.js +99 -17
  40. package/dist/server-core.js +1 -1
  41. package/dist/utils/ios/utils.js +4 -4
  42. package/dist/utils/operation-budget.js +24 -2
  43. package/dist/utils/resolve-device.js +7 -0
  44. package/docs/CHANGELOG.md +16 -2
  45. package/docs/benchmarks/ui-tree-performance/README.md +4 -3
  46. package/docs/specs/020-conditional-actions/intent.md +62 -0
  47. package/docs/specs/020-conditional-actions/plan.md +214 -0
  48. package/docs/specs/020-conditional-actions/spec.md +301 -0
  49. package/docs/specs/021-app-readiness-signals/intent.md +40 -0
  50. package/docs/specs/021-app-readiness-signals/plan.md +102 -0
  51. package/docs/specs/021-app-readiness-signals/spec.md +144 -0
  52. package/docs/specs/022-correlated-evidence/device-evidence.json +60 -0
  53. package/docs/specs/022-correlated-evidence/intent.md +37 -0
  54. package/docs/specs/022-correlated-evidence/modul8-evidence.json +1062 -0
  55. package/docs/specs/022-correlated-evidence/modul8-selector-fix-evidence.json +336 -0
  56. package/docs/specs/022-correlated-evidence/plan.md +171 -0
  57. package/docs/specs/022-correlated-evidence/spec.md +123 -0
  58. package/docs/specs/process.md +213 -0
  59. package/docs/tools/TOOLS.md +3 -0
  60. package/docs/tools/evidence.md +79 -0
  61. package/docs/tools/interact.md +125 -2
  62. package/docs/tools/observe.md +64 -2
  63. package/package.json +4 -2
  64. package/scripts/agent-plugin.mjs +98 -0
  65. package/src/interact/android.ts +10 -4
  66. package/src/interact/conditional-action.ts +75 -0
  67. package/src/interact/index.ts +42 -14
  68. package/src/interact/ios.ts +14 -8
  69. package/src/manage/android.ts +4 -3
  70. package/src/manage/ios.ts +3 -2
  71. package/src/observe/android.ts +35 -21
  72. package/src/observe/evidence.ts +255 -0
  73. package/src/observe/index.ts +32 -8
  74. package/src/observe/ios.ts +32 -8
  75. package/src/observe/readiness-service.ts +89 -0
  76. package/src/observe/readiness.ts +78 -0
  77. package/src/observe/snapshot-metadata.ts +12 -1
  78. package/src/server/common.ts +7 -0
  79. package/src/server/condition-schema.ts +32 -0
  80. package/src/server/journey-schema.ts +3 -2
  81. package/src/server/readiness-schema.ts +12 -0
  82. package/src/server/tool-definitions.ts +44 -0
  83. package/src/server/tool-handlers.ts +93 -17
  84. package/src/server-core.ts +1 -1
  85. package/src/types.ts +30 -1
  86. package/src/utils/ios/utils.ts +4 -4
  87. package/src/utils/operation-budget.ts +22 -2
  88. package/src/utils/resolve-device.ts +8 -0
  89. package/test/device/automated/observe/correlated-evidence.smoke.ts +70 -0
  90. package/test/device/manual/interact/conditional_actions.manual.ts +95 -0
  91. package/test/device/manual/interact/modul8_selector_tap.manual.ts +63 -0
  92. package/test/device/manual/interact/readiness.manual.ts +99 -0
  93. package/test/fixtures/readiness-app/MainActivity.java +23 -0
  94. package/test/fixtures/readiness-app/README.md +12 -0
  95. package/test/fixtures/readiness-app/build.ts +37 -0
  96. package/test/fixtures/readiness-app/main.m +27 -0
  97. package/test/unit/observe/android_tree_acquisition.test.ts +6 -0
  98. package/test/unit/observe/capture_debug_snapshot.test.ts +42 -32
  99. package/test/unit/observe/compact_tree_error.test.ts +24 -0
  100. package/test/unit/observe/conditional_snapshots.test.ts +33 -0
  101. package/test/unit/observe/correlated-evidence.test.ts +128 -0
  102. package/test/unit/observe/ios-getlogs.test.ts +3 -3
  103. package/test/unit/observe/ios_tree_acquisition.test.ts +2 -0
  104. package/test/unit/observe/readiness.test.ts +39 -0
  105. package/test/unit/observe/readiness_native.test.ts +68 -0
  106. package/test/unit/server/conditional_actions.test.ts +299 -0
  107. package/test/unit/server/conditional_schema.test.ts +59 -0
  108. package/test/unit/server/correlated-evidence.test.ts +194 -0
  109. package/test/unit/server/readiness.test.ts +170 -0
  110. package/test/unit/server/selector_tap_target.test.ts +79 -0
  111. package/test/unit/server/tap_selector.test.ts +2 -0
  112. package/skills/mobile-debug-tools-usage/SKILL.md +0 -74
  113. package/src/observe/android-tree-bridge.ts +0 -178
  114. package/test/device/automated/ui-tree-performance/bridge-probe.ts +0 -197
  115. package/test/fixtures/ui-tree-bridge/AndroidManifest.xml +0 -6
  116. package/test/fixtures/ui-tree-bridge/build.ts +0 -34
  117. package/test/fixtures/ui-tree-bridge/src/TreeInstrumentation.java +0 -126
  118. package/test/unit/observe/android_tree_bridge_cleanup.test.ts +0 -95
  119. /package/{skills → .agents/skills}/README.md +0 -0
  120. /package/{skills → .agents/skills}/mcp-builder/references/build-flags.md +0 -0
  121. /package/{skills → .agents/skills}/mcp-builder/references/diagnostics-schema.md +0 -0
  122. /package/{skills → .agents/skills}/mcp-builder/references/toolchain-details.md +0 -0
  123. /package/{skills → .agents/skills}/spec-review/references/spec-review-checklist.md +0 -0
  124. /package/{skills → .agents/skills}/spec-review/references/spec-review-template.md +0 -0
  125. /package/{skills → .agents/skills}/test-authoring/references/repo-test-layout.md +0 -0
  126. /package/{skills → .agents/skills}/test-authoring/references/test-authoring-workflow.md +0 -0
  127. /package/{skills → .agents/skills}/test-authoring/references/test-quality-checklist.md +0 -0
@@ -0,0 +1,115 @@
1
+ import { resolveTargetDevice, withResolvedDevice } from '../utils/resolve-device.js';
2
+ import { remainingBudget, runWithBudget } from '../utils/operation-budget.js';
3
+ import { ToolsObserve } from './index.js';
4
+ import { isDeviceUnavailable, ReadinessTracker, withReadinessTracker } from './readiness.js';
5
+ function budgetExpired(error) {
6
+ if (/ACTION_TIMEOUT/.test(String(error)))
7
+ return true;
8
+ try {
9
+ remainingBudget();
10
+ return false;
11
+ }
12
+ catch {
13
+ return true;
14
+ }
15
+ }
16
+ async function resolve(target) {
17
+ if (target.platform)
18
+ return resolveTargetDevice({ ...target, platform: target.platform });
19
+ try {
20
+ return await resolveTargetDevice({ ...target, platform: 'android' });
21
+ }
22
+ catch {
23
+ return resolveTargetDevice({ ...target, platform: 'ios' });
24
+ }
25
+ }
26
+ export async function readinessDelay(ms) {
27
+ await new Promise(resolve => setTimeout(resolve, Math.min(ms, remainingBudget(ms))));
28
+ remainingBudget();
29
+ }
30
+ export async function getAppReadiness(args, wait = false) {
31
+ const started = performance.now();
32
+ let device = null;
33
+ let tracker = new ReadinessTracker(args.platform ?? 'android', args.deviceId ?? '');
34
+ let failureCode;
35
+ await runWithBudget(args.timeoutMs ?? (wait ? 30000 : 15000), async () => {
36
+ try {
37
+ device = await resolve(args);
38
+ remainingBudget();
39
+ tracker = new ReadinessTracker(device.platform, device.id, args.stabilityMs ?? 500);
40
+ await withResolvedDevice(device, async () => {
41
+ let count = 0;
42
+ while (wait || count < 2) {
43
+ remainingBudget();
44
+ const captureStarted = Date.now();
45
+ try {
46
+ const tree = await ToolsObserve.getUITreeHandler({ platform: device.platform, deviceId: device.id });
47
+ remainingBudget();
48
+ tracker.observe(tree, captureStarted);
49
+ if (isDeviceUnavailable(tree?.error)) {
50
+ failureCode = 'DEVICE_UNAVAILABLE';
51
+ return;
52
+ }
53
+ }
54
+ catch (error) {
55
+ if (/ACTION_TIMEOUT/.test(String(error)))
56
+ throw error;
57
+ tracker.unavailable();
58
+ if (isDeviceUnavailable(error)) {
59
+ failureCode = 'DEVICE_UNAVAILABLE';
60
+ return;
61
+ }
62
+ }
63
+ count++;
64
+ if (tracker.summary.state === 'settled' || (!wait && tracker.summary.state === 'loading'))
65
+ return;
66
+ if (!wait && count === 2)
67
+ return;
68
+ await readinessDelay(wait ? args.pollIntervalMs ?? 500 : args.stabilityMs ?? 500);
69
+ }
70
+ });
71
+ }
72
+ catch (error) {
73
+ const expired = budgetExpired(error);
74
+ if (expired)
75
+ tracker.deadline();
76
+ else
77
+ tracker.unavailable();
78
+ failureCode = !expired && (!device || isDeviceUnavailable(error)) ? 'DEVICE_UNAVAILABLE' : wait ? 'READINESS_TIMEOUT' : 'READINESS_UNAVAILABLE';
79
+ }
80
+ });
81
+ const state = tracker.summary.state;
82
+ const success = !failureCode && (wait ? state === 'settled' : state !== 'unknown');
83
+ return { success, device, readiness: tracker.summary, elapsed_ms: performance.now() - started,
84
+ ...(!success ? { failure_code: failureCode ?? (wait ? 'READINESS_TIMEOUT' : 'READINESS_UNAVAILABLE') } : {}) };
85
+ }
86
+ export async function withReadyWait(target, timeoutMs, operation, failure) {
87
+ const started = performance.now();
88
+ let tracker = new ReadinessTracker(target.platform ?? 'android', target.deviceId ?? '');
89
+ let resolved = false;
90
+ return runWithBudget(timeoutMs, async () => {
91
+ try {
92
+ const device = await resolve(target);
93
+ resolved = true;
94
+ remainingBudget();
95
+ tracker = new ReadinessTracker(device.platform, device.id);
96
+ const result = await withResolvedDevice(device, () => withReadinessTracker(tracker, () => operation(device)));
97
+ try {
98
+ remainingBudget();
99
+ }
100
+ catch {
101
+ const alreadyFailed = result && (('status' in result && result.status !== 'success') || ('success' in result && result.success === false));
102
+ return { ...(alreadyFailed ? result : failure('TIMEOUT', performance.now() - started, tracker.summary.sample_count)), readiness: tracker.deadline() };
103
+ }
104
+ return { ...result, readiness: tracker.summary };
105
+ }
106
+ catch (error) {
107
+ const expired = budgetExpired(error);
108
+ if (expired)
109
+ tracker.deadline();
110
+ else
111
+ tracker.unavailable();
112
+ return { ...failure(!expired && (!resolved || isDeviceUnavailable(error)) ? 'DEVICE_UNAVAILABLE' : 'TIMEOUT', performance.now() - started, tracker.summary.sample_count), readiness: tracker.summary };
113
+ }
114
+ });
115
+ }
@@ -0,0 +1,81 @@
1
+ import { AsyncLocalStorage } from 'node:async_hooks';
2
+ import { remainingBudget } from '../utils/operation-budget.js';
3
+ import { computeSnapshotSignature, getSnapshotSessionId } from './snapshot-metadata.js';
4
+ const baselineLimitations = ['NORMALIZED_UI_ONLY', 'FOREGROUND_APP_UNVERIFIED', 'UNRESPONSIVE_SIGNAL_UNSUPPORTED', 'DEBUGGER_SIGNAL_UNSUPPORTED'];
5
+ const capabilities = { loading: true, transitioning: true, settled: true, unresponsive: false, debugger_paused: false };
6
+ export function isDeviceUnavailable(error) {
7
+ return /device.*(?:not found|offline|disconnected)|no devices found|READINESS_DEVICE_UNAVAILABLE/i.test(String(error));
8
+ }
9
+ export class ReadinessTracker {
10
+ platform;
11
+ deviceId;
12
+ stabilityMs;
13
+ signature = null;
14
+ stableSince = 0;
15
+ summary = {
16
+ state: 'unknown', scope: 'device_ui', basis: 'insufficient', captured_at_ms: null,
17
+ sample_count: 0, stable_for_ms: 0, snapshot_session_id: null, snapshot_revision: null,
18
+ evidence: [], limitations: [...baselineLimitations, 'INSUFFICIENT_SAMPLES'], capabilities: { ...capabilities }
19
+ };
20
+ constructor(platform, deviceId, stabilityMs = 500) {
21
+ this.platform = platform;
22
+ this.deviceId = deviceId;
23
+ this.stabilityMs = stabilityMs;
24
+ }
25
+ unavailable(code = 'OBSERVATION_FAILED') {
26
+ this.signature = null;
27
+ this.summary = { ...this.summary, state: 'unknown', basis: 'insufficient', stable_for_ms: 0,
28
+ sample_count: this.summary.sample_count + 1, limitations: [...baselineLimitations, code],
29
+ evidence: [...this.summary.evidence, { code, source: 'ui_tree', captured_at_ms: null, snapshot_revision: null }].slice(-8) };
30
+ return this.summary;
31
+ }
32
+ deadline() {
33
+ this.summary = { ...this.summary, limitations: [...new Set([...this.summary.limitations, 'DEADLINE_EXCEEDED'])] };
34
+ return this.summary;
35
+ }
36
+ observe(tree, acquisitionStartedAtMs, now = performance.now()) {
37
+ if (tree?.error)
38
+ return this.unavailable('OBSERVATION_FAILED');
39
+ if (!tree || tree.device?.platform !== this.platform || tree.device?.id !== this.deviceId ||
40
+ tree.snapshot_session_id !== getSnapshotSessionId() || !Number.isSafeInteger(tree.snapshot_revision) || tree.snapshot_revision <= 0 ||
41
+ !Number.isFinite(tree.captured_at_ms) || tree.captured_at_ms < acquisitionStartedAtMs || tree.captured_at_ms <= 0 ||
42
+ !Array.isArray(tree.elements) || !tree.elements.length || tree.elements.some((element) => !element || typeof element !== 'object') ||
43
+ !Number.isFinite(tree.resolution?.width) || !Number.isFinite(tree.resolution?.height) || tree.resolution.width <= 0 || tree.resolution.height <= 0)
44
+ return this.unavailable('OBSERVATION_INVALID');
45
+ const signature = computeSnapshotSignature(tree);
46
+ if (!signature)
47
+ return this.unavailable('OBSERVATION_INVALID');
48
+ const nativeLoading = tree.elements.some((element) => element.visible === true && ['android.widget.ProgressBar', 'ActivityIndicator', 'ProgressIndicator'].includes(element.type));
49
+ const textLoading = tree.elements.some((element) => element.visible === true && [element.text, element.contentDescription].some(label => typeof label === 'string' && /^(loading|please\s+wait)[.\s…]*$/i.test(label.trim())));
50
+ const previous = this.signature;
51
+ const unchanged = previous === signature;
52
+ if (!unchanged || nativeLoading || textLoading)
53
+ this.stableSince = now;
54
+ this.signature = signature;
55
+ const stableFor = unchanged && !nativeLoading && !textLoading ? Math.max(0, now - this.stableSince) : 0;
56
+ const state = nativeLoading || textLoading ? 'loading' : !previous ? 'unknown' : !unchanged ? 'transitioning' : stableFor >= this.stabilityMs ? 'settled' : 'unknown';
57
+ const code = nativeLoading ? 'NATIVE_LOADING_INDICATOR' : textLoading ? 'LOADING_LABEL' : state === 'transitioning' ? 'SNAPSHOT_CHANGED' : state === 'settled' ? 'SNAPSHOT_STABLE' : 'INSUFFICIENT_SAMPLES';
58
+ this.summary = {
59
+ state, scope: 'device_ui', basis: textLoading && !nativeLoading ? 'heuristic' : state === 'unknown' ? 'insufficient' : 'observed',
60
+ captured_at_ms: tree.captured_at_ms, snapshot_session_id: tree.snapshot_session_id, snapshot_revision: tree.snapshot_revision,
61
+ sample_count: this.summary.sample_count + 1, stable_for_ms: stableFor,
62
+ evidence: [...this.summary.evidence, { code, source: 'ui_tree', captured_at_ms: tree.captured_at_ms, snapshot_revision: tree.snapshot_revision }].slice(-8),
63
+ limitations: [...baselineLimitations, ...(state === 'unknown' ? ['INSUFFICIENT_SAMPLES'] : [])], capabilities: { ...capabilities }
64
+ };
65
+ return this.summary;
66
+ }
67
+ }
68
+ const waits = new AsyncLocalStorage();
69
+ export function currentReadinessWait() { return waits.getStore(); }
70
+ export function withReadinessTracker(tracker, operation) { return waits.run(tracker, operation); }
71
+ export function observeReadinessWait(tree, startedAtMs) {
72
+ const tracker = waits.getStore();
73
+ if (!tracker)
74
+ return;
75
+ remainingBudget();
76
+ const summary = tracker.observe(tree, startedAtMs);
77
+ if (isDeviceUnavailable(tree?.error))
78
+ throw new Error('READINESS_DEVICE_UNAVAILABLE');
79
+ if (summary.limitations.some(code => ['OBSERVATION_INVALID', 'OBSERVATION_FAILED'].includes(code)))
80
+ throw new Error('READINESS_SAMPLE_INVALID');
81
+ }
@@ -0,0 +1,165 @@
1
+ import { spawn } from 'node:child_process';
2
+ import { randomUUID } from 'node:crypto';
3
+ import { mkdir, readdir, rm, stat } from 'node:fs/promises';
4
+ import { tmpdir } from 'node:os';
5
+ import path from 'node:path';
6
+ import { fileURLToPath } from 'node:url';
7
+ import { execCmd } from '../utils/exec.js';
8
+ import { getAdbCmd } from '../utils/android/utils.js';
9
+ import { resolveTargetDevice } from '../utils/resolve-device.js';
10
+ const recordings = new Map();
11
+ const activeDevices = new Map();
12
+ const adb = (deviceId, args, timeout = 15000) => execCmd(getAdbCmd(), ['-s', deviceId, ...args], { timeout });
13
+ const tool = (name, args, timeout = 30000) => execCmd(name, args, { timeout });
14
+ const swiftScript = fileURLToPath(new URL('../../scripts/recording-frames.swift', import.meta.url));
15
+ async function swiftDecode(video, directory, startMs, endMs) {
16
+ const result = await tool('xcrun', ['swift', swiftScript, video, directory, String(startMs), String(endMs)], 120000);
17
+ if (result.exitCode !== 0)
18
+ throw new Error(`Swift frame decoder failed: ${result.stderr}`);
19
+ return JSON.parse(result.stdout);
20
+ }
21
+ function positiveInteger(value, name, max) {
22
+ if (typeof value !== 'number' || !Number.isInteger(value) || value < 1 || value > max)
23
+ throw new Error(`${name} must be an integer from 1 to ${max}`);
24
+ return value;
25
+ }
26
+ async function probe(video) {
27
+ let result;
28
+ try {
29
+ result = await tool('ffprobe', ['-v', 'error', '-select_streams', 'v:0', '-show_entries', 'stream=width,height,nb_frames:format=duration', '-of', 'json', video]);
30
+ }
31
+ catch (error) {
32
+ if (process.platform !== 'darwin')
33
+ throw error;
34
+ const metadata = await swiftDecode(video, path.join(path.dirname(video), 'metadata'), 1, 0);
35
+ const { frames: _frames, ...rest } = metadata;
36
+ return rest;
37
+ }
38
+ if (result.exitCode !== 0)
39
+ throw new Error(`ffprobe failed: ${result.stderr || 'ffprobe is required for recording metadata and frame extraction'}`);
40
+ const data = JSON.parse(result.stdout);
41
+ const stream = data.streams?.[0];
42
+ if (!stream)
43
+ throw new Error('Recording contains no video stream');
44
+ const frames = await frameTimes(video);
45
+ return { durationMs: Math.round(Number(data.format.duration) * 1000), resolution: { width: stream.width, height: stream.height }, frameCount: frames.length, frameTimestampsMs: frames };
46
+ }
47
+ async function frameTimes(video) {
48
+ const result = await tool('ffprobe', ['-v', 'error', '-select_streams', 'v:0', '-show_entries', 'frame=best_effort_timestamp_time', '-of', 'json', video], 60000);
49
+ if (result.exitCode !== 0)
50
+ throw new Error(`ffprobe failed: ${result.stderr || 'ffprobe is required'}`);
51
+ return (JSON.parse(result.stdout).frames ?? []).map((frame) => Math.round(Number(frame.best_effort_timestamp_time) * 1000)).filter((value) => Number.isFinite(value));
52
+ }
53
+ export async function startScreenRecording(deviceId, maxDurationSeconds = 15) {
54
+ if (deviceId !== undefined && (typeof deviceId !== 'string' || !deviceId))
55
+ throw new Error('deviceId must be a nonempty string');
56
+ const durationLimit = positiveInteger(maxDurationSeconds, 'maxDurationSeconds', 180);
57
+ const device = await resolveTargetDevice({ platform: 'android', deviceId: deviceId });
58
+ if (activeDevices.has(device.id))
59
+ throw new Error(`A recording is already active for device ${device.id}`);
60
+ const id = randomUUID();
61
+ const directory = path.join(tmpdir(), 'mobile-debug-mcp-recordings', id);
62
+ const remote = `/sdcard/mobile-debug-mcp-${id}.mp4`;
63
+ await mkdir(directory, { recursive: true });
64
+ let finish;
65
+ const finished = new Promise(resolve => { finish = resolve; });
66
+ const child = spawn(getAdbCmd(), ['-s', device.id, 'shell', 'screenrecord', '--time-limit', String(durationLimit), remote], { stdio: 'ignore' });
67
+ const timer = setTimeout(() => { void stopScreenRecording(id).catch(() => undefined); }, (durationLimit + 2) * 1000);
68
+ timer.unref();
69
+ const recording = { id, deviceId: device.id, remote, directory, video: path.join(directory, 'recording.mp4'), startedAt: new Date().toISOString(), durationLimit, process: child, timer, state: 'recording', finished, finish };
70
+ recordings.set(id, recording);
71
+ activeDevices.set(device.id, id);
72
+ child.once('close', () => { finish(); if (recording.state === 'recording')
73
+ void stopScreenRecording(id).catch(() => undefined); });
74
+ child.once('error', () => { finish(); if (recording.state === 'recording')
75
+ void stopScreenRecording(id).catch(() => undefined); });
76
+ return { recordingId: id, deviceId: device.id, startedAt: recording.startedAt, maxDurationSeconds: durationLimit };
77
+ }
78
+ export async function stopScreenRecording(recordingId) {
79
+ const recording = typeof recordingId === 'string' ? recordings.get(recordingId) : undefined;
80
+ if (!recording)
81
+ throw new Error('Unknown recordingId');
82
+ if (recording.stopPromise)
83
+ return recording.stopPromise;
84
+ recording.state = 'stopping';
85
+ clearTimeout(recording.timer);
86
+ recording.stopPromise = (async () => {
87
+ try {
88
+ if (recording.process.exitCode === null) {
89
+ const processes = await adb(recording.deviceId, ['shell', 'ps', '-A', '-o', 'PID,ARGS']).catch(() => null);
90
+ const pid = processes?.stdout.split('\n').find(line => line.includes('screenrecord') && line.includes(recording.remote))?.trim().match(/^\d+/)?.[0];
91
+ if (pid)
92
+ await adb(recording.deviceId, ['shell', 'kill', '-2', pid]).catch(() => undefined);
93
+ }
94
+ await Promise.race([recording.finished, new Promise(resolve => setTimeout(resolve, 3000))]);
95
+ if (recording.process.exitCode === null)
96
+ recording.process.kill('SIGKILL');
97
+ const pulled = await adb(recording.deviceId, ['pull', recording.remote, recording.video], 30000);
98
+ if (pulled.exitCode !== 0)
99
+ throw new Error(`Could not pull recording: ${pulled.stderr}`);
100
+ if ((await stat(recording.video)).size === 0)
101
+ throw new Error('Recording is empty');
102
+ let metadata;
103
+ try {
104
+ metadata = await probe(recording.video);
105
+ }
106
+ catch (error) {
107
+ metadata = { metadataError: String(error), durationMs: null, resolution: null, frameCount: null, frameTimestampsMs: [] };
108
+ }
109
+ recording.result = { recordingId: recording.id, deviceId: recording.deviceId, videoPath: recording.video, startedAt: recording.startedAt, stoppedAt: new Date().toISOString(), ...metadata };
110
+ return recording.result;
111
+ }
112
+ finally {
113
+ await adb(recording.deviceId, ['shell', 'rm', '-f', recording.remote]).catch(() => undefined);
114
+ activeDevices.delete(recording.deviceId);
115
+ recording.state = 'stopped';
116
+ if (!recording.result) {
117
+ recordings.delete(recording.id);
118
+ await rm(recording.directory, { recursive: true, force: true }).catch(() => undefined);
119
+ }
120
+ }
121
+ })();
122
+ return recording.stopPromise;
123
+ }
124
+ export async function extractRecordingFrames(recordingId, startMs, endMs) {
125
+ const recording = typeof recordingId === 'string' ? recordings.get(recordingId) : undefined;
126
+ if (!recording || recording.state !== 'stopped' || !recording.result)
127
+ throw new Error('Unknown or unfinished recordingId');
128
+ if (typeof startMs !== 'number' || !Number.isInteger(startMs) || startMs < 0 || typeof endMs !== 'number' || !Number.isInteger(endMs) || endMs <= startMs || endMs - startMs > 10000)
129
+ throw new Error('Window must be 1 to 10000 ms with nonnegative integer timestamps');
130
+ const directory = path.join(recording.directory, `frames-${startMs}-${endMs}`);
131
+ await rm(directory, { recursive: true, force: true });
132
+ await mkdir(directory, { recursive: true });
133
+ let allTimes;
134
+ let frames;
135
+ try {
136
+ allTimes = await frameTimes(recording.video);
137
+ const selected = allTimes.filter(time => time >= startMs && time <= endMs);
138
+ const result = await tool('ffmpeg', ['-v', 'error', '-i', recording.video, '-vf', `select=between(t\,${startMs / 1000}\,${endMs / 1000})`, '-vsync', '0', path.join(directory, '%06d.png')], 60000);
139
+ if (result.exitCode !== 0)
140
+ throw new Error(`ffmpeg failed: ${result.stderr || 'ffmpeg is required for frame extraction'}`);
141
+ const files = (await readdir(directory)).filter(file => file.endsWith('.png')).sort();
142
+ if (files.length !== selected.length)
143
+ throw new Error(`Frame count mismatch: ffprobe found ${selected.length}, ffmpeg extracted ${files.length}`);
144
+ frames = files.map((file, index) => ({ timestampMs: selected[index], path: path.join(directory, file) }));
145
+ }
146
+ catch (error) {
147
+ if (process.platform !== 'darwin')
148
+ throw error;
149
+ const decoded = await swiftDecode(recording.video, directory, startMs, endMs);
150
+ allTimes = decoded.frameTimestampsMs;
151
+ frames = decoded.frames;
152
+ }
153
+ const selected = frames.map(frame => frame.timestampMs);
154
+ const deltas = allTimes.slice(1).map((time, index) => time - allTimes[index]).filter(delta => delta > 0).sort((a, b) => a - b);
155
+ const nominalMs = deltas.length ? deltas[Math.floor((deltas.length - 1) / 2)] : null;
156
+ const gaps = allTimes.slice(1).flatMap((time, index) => nominalMs && time - allTimes[index] > nominalMs * 1.5 && time >= startMs && allTimes[index] <= endMs ? [{ startMs: allTimes[index], endMs: time, durationMs: time - allTimes[index] }] : []);
157
+ if (selected.length === 0 || selected[0] > startMs + (nominalMs ?? 0) * 1.5)
158
+ gaps.unshift({ startMs, endMs: selected[0] ?? endMs, durationMs: (selected[0] ?? endMs) - startMs });
159
+ if (selected.length && selected[selected.length - 1] < endMs - (nominalMs ?? 0) * 1.5)
160
+ gaps.push({ startMs: selected[selected.length - 1], endMs, durationMs: endMs - selected[selected.length - 1] });
161
+ return { recordingId, startMs, endMs, frames, frameCount: frames.length, nominalFrameIntervalMs: nominalMs, captureGaps: gaps, continuousCoverage: gaps.length === 0, note: 'Frames are encoded samples. A gap may hide a brief flash; absence cannot be established across gaps.' };
162
+ }
163
+ export async function shutdownRecordings() {
164
+ await Promise.allSettled([...activeDevices.values()].map(id => stopScreenRecording(id)));
165
+ }
@@ -1,6 +1,14 @@
1
1
  import crypto from 'crypto';
2
2
  let nextRevision = 1;
3
+ const snapshotSessionId = crypto.randomUUID();
3
4
  const snapshotStateByDevice = new Map();
5
+ export function getSnapshotSessionId() { return snapshotSessionId; }
6
+ export function hasSnapshotBaseline(deviceKey, revision) {
7
+ const state = snapshotStateByDevice.get(deviceKey);
8
+ const capturedAt = state?.historyTimes.get(revision);
9
+ return !!state && capturedAt !== undefined && state.history.has(revision) &&
10
+ Date.now() - capturedAt <= 10 * 60 * 1000 && Date.now() - state.updatedAt <= 10 * 60 * 1000;
11
+ }
4
12
  function normalize(value) {
5
13
  if (value === null || value === undefined)
6
14
  return '';
@@ -135,8 +143,7 @@ export function deriveSnapshotMetadata(deviceKey, tree, source, signatureOverrid
135
143
  const oversized = previous?.oversized ?? new Map();
136
144
  if (hasValidTree) {
137
145
  history.set(revision, currentElements);
138
- if (!historyTimes.has(revision))
139
- historyTimes.set(revision, Date.now());
146
+ historyTimes.set(revision, Date.now());
140
147
  if ((tree.elements?.length ?? 0) > 500)
141
148
  oversized.set(revision, (tree.elements?.length ?? 0) - 500);
142
149
  }
@@ -164,6 +171,7 @@ export function deriveSnapshotMetadata(deviceKey, tree, source, signatureOverrid
164
171
  updatedAt: Date.now()
165
172
  });
166
173
  return {
174
+ snapshot_session_id: snapshotSessionId,
167
175
  snapshot_revision: revision,
168
176
  captured_at_ms: Date.now(),
169
177
  snapshot_delta: hasValidTree ? summarizeSnapshotDelta(previous, currentElementSignatures) : null,
@@ -1,5 +1,7 @@
1
+ import { currentEvidenceActionId } from '../observe/evidence.js';
1
2
  import { recordTiming } from '../utils/timing.js';
2
3
  import { ToolsObserve } from '../observe/index.js';
4
+ import { isConditionBlocked } from '../interact/conditional-action.js';
3
5
  export const DEFAULT_MAX_RECOVERY_ATTEMPTS = 3;
4
6
  export const DEFAULT_MAX_RETRY_DEPTH = 3;
5
7
  export function wrapResponse(data, includeSerializationTiming = true) {
@@ -62,10 +64,15 @@ export function getArrayArg(args, key) {
62
64
  }
63
65
  let actionSequence = 0;
64
66
  export function nextActionId(actionType, timestamp) {
67
+ const current = currentEvidenceActionId();
68
+ if (current)
69
+ return current;
65
70
  actionSequence += 1;
66
71
  return `${actionType}_${timestamp}_${actionSequence}`;
67
72
  }
68
73
  export async function captureActionFingerprint(platform, deviceId) {
74
+ if (isConditionBlocked())
75
+ return null;
69
76
  if (!platform)
70
77
  return null;
71
78
  try {
@@ -235,6 +242,8 @@ function mapFailureCodeToFailureClass(code) {
235
242
  case 'STALE_REFERENCE':
236
243
  return 'TargetResolutionFailure';
237
244
  case 'ELEMENT_NOT_INTERACTABLE':
245
+ case 'PRECONDITION_FAILED':
246
+ case 'PRECONDITION_UNAVAILABLE':
238
247
  case 'ACTION_TIMEOUT':
239
248
  return 'ExecutionFailure';
240
249
  case 'TIMEOUT':
@@ -0,0 +1,32 @@
1
+ import { z } from 'zod';
2
+ export const conditionalTools = new Set(['tap', 'tap_element', 'swipe', 'type_text', 'press_back']);
3
+ export const expectedStateSchema = z.object({
4
+ snapshotSessionId: z.string().min(1),
5
+ snapshotRevision: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
6
+ timeoutMs: z.number().int().min(100).max(60000).optional()
7
+ }).strict();
8
+ export const expectedStateJson = {
9
+ type: 'object', additionalProperties: false,
10
+ properties: {
11
+ snapshotSessionId: { type: 'string', minLength: 1 },
12
+ snapshotRevision: { type: 'integer', minimum: 1, maximum: Number.MAX_SAFE_INTEGER },
13
+ timeoutMs: { type: 'integer', minimum: 100, maximum: 60000, default: 15000 }
14
+ },
15
+ required: ['snapshotSessionId', 'snapshotRevision']
16
+ };
17
+ export function validateExpectedState(name, args) {
18
+ if (args.expectedState === undefined)
19
+ return undefined;
20
+ if (!conditionalTools.has(name))
21
+ throw new Error('INVALID_ARGUMENT: expectedState is not supported on this tool');
22
+ const parsed = expectedStateSchema.safeParse(args.expectedState);
23
+ if (!parsed.success)
24
+ throw new Error('INVALID_ARGUMENT: invalid expectedState');
25
+ if ((args.platform !== 'android' && args.platform !== 'ios') ||
26
+ typeof args.deviceId !== 'string' || !args.deviceId.trim() ||
27
+ ['booted', 'default'].includes(args.deviceId))
28
+ throw new Error('INVALID_ARGUMENT: expectedState requires an explicit platform and concrete deviceId');
29
+ if (['type_text', 'press_back'].includes(name) && args.platform !== 'android')
30
+ throw new Error('INVALID_ARGUMENT: this action supports only Android');
31
+ return parsed.data;
32
+ }
@@ -1,4 +1,5 @@
1
1
  import { z } from 'zod';
2
+ import { expectedStateSchema, expectedStateJson } from './condition-schema.js';
2
3
  const selector = z.object({
3
4
  text: z.string().min(1).optional(), resource_id: z.string().min(1).optional(),
4
5
  accessibility_id: z.string().min(1).optional(), contains: z.boolean().optional()
@@ -22,7 +23,7 @@ const assertion = z.union([
22
23
  const id = z.string().min(1);
23
24
  const step = z.union([
24
25
  z.object({ id, type: z.literal('start_app'), appId: z.string().min(1), verificationMode: verificationMode.optional() }).strict(),
25
- z.object({ id, type: z.literal('tap'), ...target, waitFor: waitFor.optional(), verificationMode: verificationMode.optional() }).strict().refine(exactlyOneTarget),
26
+ z.object({ id, type: z.literal('tap'), ...target, waitFor: waitFor.optional(), verificationMode: verificationMode.optional(), expectedState: expectedStateSchema.optional() }).strict().refine(exactlyOneTarget),
26
27
  z.object({ id, type: z.literal('wait'), selector, condition: z.enum(['exists', 'not_exists', 'visible', 'clickable']).optional(), timeoutMs, pollIntervalMs }).strict(),
27
28
  z.object({ id, type: z.literal('assert'), assertion, timeoutMs, pollIntervalMs }).strict()
28
29
  ]);
@@ -46,7 +47,7 @@ const objectJson = (properties, required, extra = {}) => ({ type: 'object', addi
46
47
  export const journeyStepsJson = {
47
48
  type: 'array', minItems: 1, maxItems: 50, items: { oneOf: [
48
49
  objectJson({ id: { type: 'string', minLength: 1 }, type: { const: 'start_app' }, appId: { type: 'string', minLength: 1 }, verificationMode: modeJson }, ['id', 'type', 'appId']),
49
- objectJson({ id: { type: 'string', minLength: 1 }, type: { const: 'tap' }, ...targetJson, verificationMode: modeJson,
50
+ objectJson({ id: { type: 'string', minLength: 1 }, type: { const: 'tap' }, ...targetJson, verificationMode: modeJson, expectedState: expectedStateJson,
50
51
  waitFor: objectJson({ ...intervalJson, timeoutMs: { type: 'integer', minimum: 100, maximum: 10000 }, condition: { enum: ['exists', 'visible', 'clickable'] }, match: objectJson({ index: { type: 'integer', minimum: 0 } }, []) }, [])
51
52
  }, ['id', 'type'], { oneOf: oneTargetJson }),
52
53
  objectJson({ id: { type: 'string', minLength: 1 }, type: { const: 'wait' }, selector: selectorJson, condition: { enum: ['exists', 'not_exists', 'visible', 'clickable'] }, ...intervalJson }, ['id', 'type', 'selector']),
@@ -0,0 +1,12 @@
1
+ import { z } from 'zod';
2
+ const timeout = z.number().int().min(100).max(60000);
3
+ export const readinessSchema = z.object({
4
+ platform: z.enum(['android', 'ios']), deviceId: z.string().min(1).optional(),
5
+ timeoutMs: timeout.optional(), stabilityMs: z.number().int().min(100).max(10000).optional()
6
+ }).strict();
7
+ export const readinessWaitSchema = readinessSchema.extend({ pollIntervalMs: z.number().int().min(100).max(1000).optional() }).strict();
8
+ export const readinessPropertiesJson = {
9
+ platform: { type: 'string', enum: ['android', 'ios'] }, deviceId: { type: 'string', minLength: 1 },
10
+ timeoutMs: { type: 'integer', minimum: 100, maximum: 60000 },
11
+ stabilityMs: { type: 'integer', minimum: 100, maximum: 10000, default: 500 }
12
+ };
@@ -1,5 +1,16 @@
1
1
  import { journeyStepsJson } from './journey-schema.js';
2
+ import { conditionalTools, expectedStateJson } from './condition-schema.js';
3
+ import { readinessPropertiesJson } from './readiness-schema.js';
2
4
  export const toolDefinitions = [
5
+ ...['get_app_readiness', 'wait_for_app_ready'].map(name => ({
6
+ name,
7
+ description: `${name === 'get_app_readiness' ? 'Observe readiness with at most two fresh UI-tree samples.' : 'Wait within a single bounded budget for settled UI evidence.'} Reports loading, transitioning, settled or unknown with evidence and limitations. Unresponsive/debugger-paused detection is unsupported. Settled normalized foreground UI is not proof of app identity, responsiveness or backend completion.`,
8
+ inputSchema: { type: 'object', additionalProperties: false, required: ['platform'], properties: {
9
+ ...readinessPropertiesJson,
10
+ timeoutMs: { ...readinessPropertiesJson.timeoutMs, default: name === 'get_app_readiness' ? 15000 : 30000 },
11
+ ...(name === 'wait_for_app_ready' ? { pollIntervalMs: { type: 'integer', minimum: 100, maximum: 1000, default: 500 } } : {})
12
+ } }
13
+ })),
3
14
  {
4
15
  name: 'start_app',
5
16
  description: `Purpose:
@@ -1128,6 +1139,29 @@ Returns { requests: [], count: 0 } when no credible network signals are found.`,
1128
1139
  ];
1129
1140
  // Keep the additive execution controls consistent across all action tools.
1130
1141
  for (const tool of toolDefinitions) {
1142
+ if (['wait_for_ui', 'wait_for_ui_change'].includes(tool.name)) {
1143
+ tool.inputSchema.properties.requireReady = { type: 'boolean', default: false };
1144
+ tool.description += '\nSet requireReady=true to require both the existing predicate and settled UI readiness, using the same fresh samples and one timeout budget. The readiness summary explains uncertainty. Default predicate semantics are unchanged.';
1145
+ }
1146
+ if (tool.name === 'get_ui_tree')
1147
+ tool.description += '\nSuccessful observations include snapshot_session_id and snapshot_revision for same-device conditional input. Compact errors preserve the error and cannot authorize an expectation.';
1148
+ if (tool.name === 'run_journey')
1149
+ tool.description += '\nTap steps may include expectedState from a successful same-device get_ui_tree. A failed condition blocks that input, retains fresh condition evidence even in compact mode, and stops subsequent steps. Conditions are not journey defaults or supported on other step types.';
1150
+ if (conditionalTools.has(tool.name)) {
1151
+ tool.inputSchema.properties.expectedState = expectedStateJson;
1152
+ tool.inputSchema.properties.platform ??= { type: 'string', enum: ['android'] };
1153
+ tool.inputSchema.allOf = [...(tool.inputSchema.allOf ?? []), {
1154
+ if: { required: ['expectedState'] },
1155
+ then: { required: ['platform', 'deviceId'], properties: {
1156
+ platform: { enum: ['type_text', 'press_back'].includes(tool.name) ? ['android'] : ['android', 'ios'] },
1157
+ deviceId: { type: 'string', minLength: 1, pattern: '\\S', not: { enum: ['booted', 'default'] } }
1158
+ } }
1159
+ }];
1160
+ tool.description += '\n\nConditional input: optionally supply expectedState { snapshotSessionId, snapshotRevision, timeoutMs? } from a successful get_ui_tree on the same explicit platform/deviceId. A fresh normalized UI-tree check runs immediately before input, even with verification disabled. PRECONDITION_FAILED or PRECONDITION_UNAVAILABLE means no input was dispatched; inspect condition evidence and re-observe before deciding what to do next. Compact failures include the fresh UI tree. This is not atomic with OS input delivery.';
1161
+ }
1162
+ else {
1163
+ tool.inputSchema.allOf = [...(tool.inputSchema.allOf ?? []), { not: { required: ['expectedState'] } }];
1164
+ }
1131
1165
  if (['start_app', 'restart_app', 'tap', 'tap_element', 'swipe', 'scroll_to_element', 'type_text', 'press_back'].includes(tool.name)) {
1132
1166
  Object.assign(tool.inputSchema.properties, {
1133
1167
  verificationMode: { type: 'string', enum: ['none', 'light', 'full'], default: 'full' },
@@ -1148,3 +1182,15 @@ for (const tool of toolDefinitions) {
1148
1182
  } }
1149
1183
  });
1150
1184
  }
1185
+ // Correlation is explicit; an omitted reference creates an independent observation.
1186
+ for (const tool of toolDefinitions) {
1187
+ if (['capture_screenshot', 'get_ui_tree', 'get_screen_fingerprint', 'capture_debug_snapshot', 'get_logs', 'read_log_stream'].includes(tool.name)) {
1188
+ Object.assign(tool.inputSchema.properties, { actionRef: {
1189
+ type: 'object', additionalProperties: false, required: ['sessionId', 'actionId'], properties: {
1190
+ sessionId: { type: 'string', minLength: 1, maxLength: 128 },
1191
+ actionId: { type: 'string', minLength: 1, maxLength: 128 }
1192
+ }
1193
+ } });
1194
+ tool.description += '\nReturns evidence metadata with acquisition intervals and conservative freshness. Optional actionRef links a known same-session/device action; unavailable references collect independently. Device log times are not comparable with host dispatch times.';
1195
+ }
1196
+ }