codeloop-mcp-server 0.1.88 → 0.1.95
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/evidence/change_coverage.d.ts +41 -0
- package/dist/evidence/change_coverage.d.ts.map +1 -1
- package/dist/evidence/change_coverage.js +69 -2
- package/dist/evidence/change_coverage.js.map +1 -1
- package/dist/evidence/interaction_coverage.d.ts.map +1 -1
- package/dist/evidence/interaction_coverage.js +2 -0
- package/dist/evidence/interaction_coverage.js.map +1 -1
- package/dist/evidence/interaction_evidence.d.ts +38 -0
- package/dist/evidence/interaction_evidence.d.ts.map +1 -1
- package/dist/evidence/interaction_evidence.js +60 -0
- package/dist/evidence/interaction_evidence.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +311 -57
- package/dist/index.js.map +1 -1
- package/dist/runners/active_target.d.ts +31 -0
- package/dist/runners/active_target.d.ts.map +1 -0
- package/dist/runners/active_target.js +65 -0
- package/dist/runners/active_target.js.map +1 -0
- package/dist/runners/android_sdk.d.ts +50 -0
- package/dist/runners/android_sdk.d.ts.map +1 -0
- package/dist/runners/android_sdk.js +123 -0
- package/dist/runners/android_sdk.js.map +1 -0
- package/dist/runners/app_launcher.d.ts +59 -1
- package/dist/runners/app_launcher.d.ts.map +1 -1
- package/dist/runners/app_launcher.js +184 -33
- package/dist/runners/app_launcher.js.map +1 -1
- package/dist/runners/app_logger.d.ts +1 -1
- package/dist/runners/app_logger.d.ts.map +1 -1
- package/dist/runners/app_logger.js +9 -8
- package/dist/runners/app_logger.js.map +1 -1
- package/dist/runners/device_probe.d.ts +18 -0
- package/dist/runners/device_probe.d.ts.map +1 -1
- package/dist/runners/device_probe.js +52 -22
- package/dist/runners/device_probe.js.map +1 -1
- package/dist/runners/interaction_engine.d.ts +6 -0
- package/dist/runners/interaction_engine.d.ts.map +1 -1
- package/dist/runners/interaction_engine.js +71 -21
- package/dist/runners/interaction_engine.js.map +1 -1
- package/dist/runners/ios_sim_input.d.ts +106 -0
- package/dist/runners/ios_sim_input.d.ts.map +1 -0
- package/dist/runners/ios_sim_input.js +453 -0
- package/dist/runners/ios_sim_input.js.map +1 -0
- package/dist/runners/journey_to_maestro.d.ts +27 -0
- package/dist/runners/journey_to_maestro.d.ts.map +1 -1
- package/dist/runners/journey_to_maestro.js +54 -0
- package/dist/runners/journey_to_maestro.js.map +1 -1
- package/dist/runners/logging_readiness.d.ts.map +1 -1
- package/dist/runners/logging_readiness.js +38 -7
- package/dist/runners/logging_readiness.js.map +1 -1
- package/dist/runners/maestro.d.ts +27 -2
- package/dist/runners/maestro.d.ts.map +1 -1
- package/dist/runners/maestro.js +57 -7
- package/dist/runners/maestro.js.map +1 -1
- package/dist/runners/maestro_generator.d.ts +1 -1
- package/dist/runners/maestro_generator.d.ts.map +1 -1
- package/dist/runners/maestro_generator.js +3 -2
- package/dist/runners/maestro_generator.js.map +1 -1
- package/dist/runners/screenshot.d.ts +6 -0
- package/dist/runners/screenshot.d.ts.map +1 -1
- package/dist/runners/screenshot.js +20 -10
- package/dist/runners/screenshot.js.map +1 -1
- package/dist/runners/semantics_audit.d.ts +32 -0
- package/dist/runners/semantics_audit.d.ts.map +1 -0
- package/dist/runners/semantics_audit.js +140 -0
- package/dist/runners/semantics_audit.js.map +1 -0
- package/dist/runners/video_recorder.d.ts +3 -1
- package/dist/runners/video_recorder.d.ts.map +1 -1
- package/dist/runners/video_recorder.js +44 -12
- package/dist/runners/video_recorder.js.map +1 -1
- package/dist/runners/wayland.d.ts +32 -0
- package/dist/runners/wayland.d.ts.map +1 -0
- package/dist/runners/wayland.js +49 -0
- package/dist/runners/wayland.js.map +1 -0
- package/dist/runners/window_manager.d.ts +55 -25
- package/dist/runners/window_manager.d.ts.map +1 -1
- package/dist/runners/window_manager.js +171 -71
- package/dist/runners/window_manager.js.map +1 -1
- package/dist/tools/discover_interactions.d.ts +22 -0
- package/dist/tools/discover_interactions.d.ts.map +1 -1
- package/dist/tools/discover_interactions.js +283 -6
- package/dist/tools/discover_interactions.js.map +1 -1
- package/dist/tools/discover_screens.d.ts +2 -0
- package/dist/tools/discover_screens.d.ts.map +1 -1
- package/dist/tools/discover_screens.js +139 -1
- package/dist/tools/discover_screens.js.map +1 -1
- package/dist/tools/gate_check.d.ts.map +1 -1
- package/dist/tools/gate_check.js +16 -2
- package/dist/tools/gate_check.js.map +1 -1
- package/dist/tools/plan_change_journey.d.ts.map +1 -1
- package/dist/tools/plan_change_journey.js +15 -2
- package/dist/tools/plan_change_journey.js.map +1 -1
- package/dist/tools/plan_user_journey.d.ts.map +1 -1
- package/dist/tools/plan_user_journey.js +5 -2
- package/dist/tools/plan_user_journey.js.map +1 -1
- package/dist/tools/run_journey.d.ts +70 -0
- package/dist/tools/run_journey.d.ts.map +1 -1
- package/dist/tools/run_journey.js +209 -21
- package/dist/tools/run_journey.js.map +1 -1
- package/dist/tools/verify.d.ts.map +1 -1
- package/dist/tools/verify.js +55 -11
- package/dist/tools/verify.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -216,6 +216,14 @@ function autoCloseStuckModalsEnabled() {
|
|
|
216
216
|
}
|
|
217
217
|
/** Consecutive same-dialog detections that mark a file dialog as genuinely stuck. */
|
|
218
218
|
const MODAL_AUTOCLOSE_THRESHOLD = 3;
|
|
219
|
+
// P2.1 — Android Studio installs the SDK without putting platform-tools on
|
|
220
|
+
// the shell PATH. Resolve ANDROID_HOME / ANDROID_SDK_ROOT / the default
|
|
221
|
+
// install location ONCE at startup and append the tool dirs to PATH so every
|
|
222
|
+
// adb/emulator call (ours and Maestro's) just works. Resolution details are
|
|
223
|
+
// surfaced by `codeloop doctor` and Android launch errors.
|
|
224
|
+
import { ensureAndroidToolsOnPath } from "./runners/android_sdk.js";
|
|
225
|
+
const androidSdkResolution = ensureAndroidToolsOnPath();
|
|
226
|
+
export { androidSdkResolution };
|
|
219
227
|
const server = new McpServer({
|
|
220
228
|
name: "codeloop",
|
|
221
229
|
version: "0.1.14",
|
|
@@ -1495,6 +1503,7 @@ Returns: confirmation + the captured image as an MCP ImageContent block so you c
|
|
|
1495
1503
|
workspace_root: z.string().optional().describe("[Alias for project_dir] Same semantics; accepted because many agents reach for this conventional name. Pass either `project_dir` OR `workspace_root` — they're equivalent."),
|
|
1496
1504
|
target_change_entry: z.string().optional().describe("0.1.52 C7 — verbatim display name of the change-manifest entry this screenshot exercises (e.g. 'datagrid_column: \"Product Code\"' or 'PhotometricConfigurations.ProductCode'). When present, the screenshot file is auto-anchored: the filename is prefixed with a slugged form of the entry so the change_coverage_evidence (C3) gate's screenshot scan can credit this evidence to the correct manifest entry without fuzzy matching. The value is also persisted alongside the screenshot path in the response so downstream tools (interaction_replay, gate_check) can use it."),
|
|
1497
1505
|
target_type: z.string().optional().describe("Capture source. Omit for desktop/window capture (default). Set 'android_emulator' (or 'android') to capture the BOOTED Android emulator via adb, or 'ios_simulator' (or 'ios', macOS only) to capture the booted iOS simulator via simctl. Use this to screenshot a Flutter/native mobile app running on a simulator instead of the host desktop."),
|
|
1506
|
+
device_id: z.string().optional().describe("Pinned mobile device for emulator/simulator capture: adb serial or simulator UDID. Defaults to the device the last journey/recording targeted, then adb default / simctl 'booted'."),
|
|
1498
1507
|
}, async (params) => {
|
|
1499
1508
|
const authResult = await withAuth(async () => {
|
|
1500
1509
|
const { captureScreenshot } = await import("./runners/screenshot.js");
|
|
@@ -1543,7 +1552,14 @@ Returns: confirmation + the captured image as an MCP ImageContent block so you c
|
|
|
1543
1552
|
const ttRaw = normalizeTargetType(params.target_type);
|
|
1544
1553
|
const explicitTargetType = ttRaw === "android_emulator" || ttRaw === "ios_simulator" ? ttRaw : undefined;
|
|
1545
1554
|
const isMobileCapture = explicitTargetType !== undefined;
|
|
1546
|
-
|
|
1555
|
+
// Pin the capture to the journey/recording device when known.
|
|
1556
|
+
let captureDevice;
|
|
1557
|
+
if (isMobileCapture) {
|
|
1558
|
+
const { resolveDeviceId } = await import("./runners/active_target.js");
|
|
1559
|
+
const vr = await import("./runners/video_recorder.js");
|
|
1560
|
+
captureDevice = resolveDeviceId(params.device_id ?? vr.getActiveRecordingDevice() ?? undefined, cwd, explicitTargetType);
|
|
1561
|
+
}
|
|
1562
|
+
const result = await captureScreenshot(screenshotsDir, finalScreenName, isMobileCapture ? undefined : targetApp, explicitTargetType, { desktopAppMode: isMobileCapture ? false : desktopApp, device: captureDevice });
|
|
1547
1563
|
// Photometry-DB E2E 8 follow-on: when we capture a desktop app
|
|
1548
1564
|
// window, also resolve its on-screen bounds so the agent can
|
|
1549
1565
|
// (a) compute window-relative coords from the returned image
|
|
@@ -1849,20 +1865,72 @@ screens_captured[], screenshots[], unsupported_count, manual_followups[], direct
|
|
|
1849
1865
|
target_type: targetTypeSchema.optional().describe("Override the auto-detected interaction target. Accepts synonyms (web→browser, android→android_emulator, ios→ios_simulator, *_desktop→desktop)."),
|
|
1850
1866
|
web_url: z.string().optional().describe("URL to open for browser targets (e.g. http://localhost:3000). Defaults to e2e.web_url from config. Start your dev server first."),
|
|
1851
1867
|
max_duration_seconds: z.number().int().min(10).max(600).optional().describe("Max video recording length. Default 180s."),
|
|
1852
|
-
}, async (params) => {
|
|
1868
|
+
}, async (params, extra) => {
|
|
1869
|
+
// P5.5 (0.1.92) — stream journey progress (boot, build heartbeat lines,
|
|
1870
|
+
// drive, capture) the same way verify streams phases, so a cold first
|
|
1871
|
+
// build doesn't look like a hang. Same safety gate as verify: only when
|
|
1872
|
+
// the client passed a progressToken AND is safe to stream to.
|
|
1873
|
+
const progressToken = extra?._meta?.progressToken;
|
|
1874
|
+
let journeyClientName;
|
|
1875
|
+
try {
|
|
1876
|
+
journeyClientName = server.server.getClientVersion?.()?.name;
|
|
1877
|
+
}
|
|
1878
|
+
catch {
|
|
1879
|
+
journeyClientName = undefined;
|
|
1880
|
+
}
|
|
1853
1881
|
const result = await withAuth(async () => {
|
|
1854
1882
|
const cwd = resolveCwd(params);
|
|
1855
1883
|
const { loadConfig } = await import("./config.js");
|
|
1856
1884
|
const cfg = loadConfig(cwd);
|
|
1857
|
-
const {
|
|
1858
|
-
|
|
1885
|
+
const { runJourneyForPlatforms } = await import("./tools/run_journey.js");
|
|
1886
|
+
const { progressClientDecision } = await import("./tools/verify.js");
|
|
1887
|
+
const progressSafe = progressClientDecision(journeyClientName, process.env.CODELOOP_PROGRESS);
|
|
1888
|
+
let progressCount = 0;
|
|
1889
|
+
const onProgress = !progressSafe || progressToken === undefined
|
|
1890
|
+
? undefined
|
|
1891
|
+
: (u) => {
|
|
1892
|
+
progressCount += 1;
|
|
1893
|
+
void extra
|
|
1894
|
+
.sendNotification({
|
|
1895
|
+
method: "notifications/progress",
|
|
1896
|
+
params: {
|
|
1897
|
+
progressToken,
|
|
1898
|
+
progress: progressCount,
|
|
1899
|
+
message: `${u.phase}${u.detail ? ` — ${u.detail.slice(0, 120)}` : ""} (${Math.round(u.elapsedMs / 1000)}s)`,
|
|
1900
|
+
},
|
|
1901
|
+
})
|
|
1902
|
+
.catch(() => { });
|
|
1903
|
+
};
|
|
1904
|
+
// P5.3 — e2e.platforms drives every listed platform sequentially; an
|
|
1905
|
+
// explicit target_type argument still forces a single-platform run.
|
|
1906
|
+
return runJourneyForPlatforms({
|
|
1859
1907
|
cwd,
|
|
1860
1908
|
e2e: { ...cfg.e2e, web_url: params.web_url ?? cfg.e2e?.web_url },
|
|
1861
1909
|
targetApp: cfg.evidence?.target_app,
|
|
1862
1910
|
targetType: params.target_type,
|
|
1863
1911
|
maxDurationSeconds: params.max_duration_seconds,
|
|
1912
|
+
onProgress,
|
|
1864
1913
|
});
|
|
1865
1914
|
}, { tool: "codeloop_run_journey", cwd: resolveCwd(params), input: params });
|
|
1915
|
+
// Single-platform runs keep the legacy flat result shape; multi-platform
|
|
1916
|
+
// runs return the per-platform outcome list plus a combined directive.
|
|
1917
|
+
const isWrapped = result && typeof result === "object" && "outcomes" in result;
|
|
1918
|
+
const wrapped = isWrapped ? result : null;
|
|
1919
|
+
if (wrapped && !wrapped.multi) {
|
|
1920
|
+
const single = wrapped.outcomes[0].result;
|
|
1921
|
+
const directive = `\n\n${single.directive}`;
|
|
1922
|
+
return {
|
|
1923
|
+
content: withInitHint([{ type: "text", text: JSON.stringify(single, null, 2) + directive }]),
|
|
1924
|
+
};
|
|
1925
|
+
}
|
|
1926
|
+
if (wrapped) {
|
|
1927
|
+
const directives = wrapped.outcomes
|
|
1928
|
+
.map((o) => `── ${o.platform} ──\n${o.result.directive}`)
|
|
1929
|
+
.join("\n\n");
|
|
1930
|
+
return {
|
|
1931
|
+
content: withInitHint([{ type: "text", text: JSON.stringify(result, null, 2) + `\n\n${directives}` }]),
|
|
1932
|
+
};
|
|
1933
|
+
}
|
|
1866
1934
|
const directive = result && typeof result === "object" && "directive" in result ? `\n\n${result.directive}` : "";
|
|
1867
1935
|
return {
|
|
1868
1936
|
content: withInitHint([{ type: "text", text: JSON.stringify(result, null, 2) + directive }]),
|
|
@@ -2028,6 +2096,7 @@ App logs (stdout, logcat, simctl log) are automatically captured alongside the v
|
|
|
2028
2096
|
target_type: targetTypeSchema.optional()
|
|
2029
2097
|
.describe("Capture method. Auto-detected from project if omitted. desktop=ffmpeg screen, android_emulator=adb screenrecord, ios_simulator=simctl recordVideo, browser=ffmpeg/Playwright"),
|
|
2030
2098
|
auto_launch: z.boolean().default(true).describe("When target_type=desktop and the app isn't already running, auto-launch it from the project's build output via evidence.target_app. Set false to skip (e.g. when the app is started by another process)."),
|
|
2099
|
+
device_id: z.string().optional().describe("Pinned mobile device to record: adb serial (emulator-5554) or simulator UDID. Defaults to the device the last journey targeted, then the first booted device. The pinned device is persisted so follow-up codeloop_interact calls hit the same one."),
|
|
2031
2100
|
project_dir: z.string().optional().describe("Absolute path to the project root. Defaults to CODELOOP_PROJECT_DIR env var or auto-discovered project directory. MUST be an actual project folder — passing the user's home directory is rejected. If your IDE launches the MCP server from the wrong cwd (common on Windows where Cursor uses C:\\Users\\<name> as cwd), set CODELOOP_PROJECT_DIR or pass this param explicitly."),
|
|
2032
2101
|
workspace_root: z.string().optional().describe("[Alias for project_dir] Same semantics; accepted because many agents reach for this conventional name. Pass either `project_dir` OR `workspace_root` — they're equivalent."),
|
|
2033
2102
|
}, async (params) => {
|
|
@@ -2125,6 +2194,7 @@ App logs (stdout, logcat, simctl log) are automatically captured alongside the v
|
|
|
2125
2194
|
// concrete commands rather than letting it guess. CodeLoop does not boot
|
|
2126
2195
|
// devices itself (heavy + can hijack the desktop), it directs the agent.
|
|
2127
2196
|
let deviceReadinessDirective;
|
|
2197
|
+
let recordingDevice;
|
|
2128
2198
|
if (targetType === "android_emulator" || targetType === "ios_simulator") {
|
|
2129
2199
|
try {
|
|
2130
2200
|
const { probeBootedDevices, detectMobileTargets, buildOpenSimulatorsDirective } = await import("./runners/device_probe.js");
|
|
@@ -2139,10 +2209,22 @@ App logs (stdout, logcat, simctl log) are automatically captured alongside the v
|
|
|
2139
2209
|
context: "recording",
|
|
2140
2210
|
});
|
|
2141
2211
|
}
|
|
2212
|
+
else {
|
|
2213
|
+
// Pin the recording to ONE device: explicit param → persisted
|
|
2214
|
+
// active target → first booted device of this platform. Persist
|
|
2215
|
+
// the choice so follow-up codeloop_interact calls hit the same one.
|
|
2216
|
+
const { resolveDeviceId, saveActiveTarget } = await import("./runners/active_target.js");
|
|
2217
|
+
recordingDevice =
|
|
2218
|
+
resolveDeviceId(params.device_id, cwd, targetType) ??
|
|
2219
|
+
(targetType === "android_emulator" ? booted.android[0] : booted.ios_devices[0]?.udid);
|
|
2220
|
+
if (recordingDevice) {
|
|
2221
|
+
saveActiveTarget(cwd, { target_type: targetType, device_id: recordingDevice });
|
|
2222
|
+
}
|
|
2223
|
+
}
|
|
2142
2224
|
}
|
|
2143
2225
|
catch { /* best-effort */ }
|
|
2144
2226
|
}
|
|
2145
|
-
const result = await startBackgroundRecording(videosDir, appName ?? "", params.max_duration_seconds, targetType);
|
|
2227
|
+
const result = await startBackgroundRecording(videosDir, appName ?? "", params.max_duration_seconds, targetType, recordingDevice);
|
|
2146
2228
|
if (autoLaunchSummary) {
|
|
2147
2229
|
result.auto_launch = autoLaunchSummary;
|
|
2148
2230
|
}
|
|
@@ -2930,14 +3012,18 @@ scroll works on desktop via CGEvent (macOS), user32 mouse_event (Windows), xdoto
|
|
|
2930
3012
|
Falls back to arrow key presses if CGEvent fails (permissions).
|
|
2931
3013
|
Browser-specific: Uses Playwright selectors (CSS/text) when target_type is "browser".
|
|
2932
3014
|
Mobile-specific: swipe, back_button, home_button, deep_link, grant_permission, rotate_device,
|
|
2933
|
-
biometric_auth, launch_app, clear_app_data, mock_location, simulate_network
|
|
3015
|
+
biometric_auth, launch_app, clear_app_data, mock_location, simulate_network,
|
|
3016
|
+
push_notification (iOS: simctl push — package_id + value as APNs JSON or alert text),
|
|
3017
|
+
status_bar (iOS: simctl status_bar — value as JSON or "time=9:41,batteryLevel=100", or "clear").
|
|
3018
|
+
iOS (0.1.91): grant_permission → simctl privacy; mock_location → simctl location;
|
|
3019
|
+
clear_app_data → simctl uninstall (REINSTALL needed after); rotate_device → Simulator Cmd+arrow.
|
|
2934
3020
|
Maestro: maestro_flow — generate and run a Maestro YAML flow from high-level steps.
|
|
2935
3021
|
Windows: win_ui_inspect, win_ui_automate — PowerShell UI Automation for UWP/WinUI apps.
|
|
2936
3022
|
|
|
2937
3023
|
MANDATORY for web apps: You MUST type into form fields, fill login/signup forms, test
|
|
2938
3024
|
validation errors, and click submit buttons. Just navigating pages is NOT enough.
|
|
2939
3025
|
Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
2940
|
-
action: z.string().describe("Action to perform: click, double_click, right_click, hover, type, keystroke, hotkey, scroll, drag_drop, long_press, type_and_submit, type_and_tab, fill_form, select_option, toggle, get_text, upload_file, navigate_url, navigate_back, navigate_forward, wait, sequence, swipe, back_button, home_button, deep_link, grant_permission, rotate_device, biometric_auth, launch_app, clear_app_data, mock_location, simulate_network, maestro_flow, win_ui_inspect, win_ui_automate"),
|
|
3026
|
+
action: z.string().describe("Action to perform: click, double_click, right_click, hover, type, keystroke, hotkey, scroll, drag_drop, long_press, type_and_submit, type_and_tab, fill_form, select_option, toggle, get_text, upload_file, navigate_url, navigate_back, navigate_forward, wait, sequence, swipe, back_button, home_button, deep_link, grant_permission, rotate_device, biometric_auth, launch_app, clear_app_data, mock_location, simulate_network, push_notification, status_bar, maestro_flow, win_ui_inspect, win_ui_automate"),
|
|
2941
3027
|
expect_contains: z.string().optional().describe("[get_text] Optional assertion: the read-back text must contain this substring (case-insensitive). When set, success is false if the substring is absent. Use to confirm an AI chatbot answer / dynamic response actually rendered the expected content."),
|
|
2942
3028
|
target_type: targetTypeSchema.optional()
|
|
2943
3029
|
.describe("Interaction target. Auto-detected if omitted. Accepts synonyms: `windows_desktop`/`mac_desktop`/`linux_desktop` → `desktop`; `web` → `browser`; `android` → `android_emulator`; `ios` → `ios_simulator`."),
|
|
@@ -2956,7 +3042,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
2956
3042
|
direction: z.enum(["up", "down", "left", "right"]).optional().describe("Scroll/swipe direction"),
|
|
2957
3043
|
amount: z.number().optional().describe("Scroll amount or other numeric value"),
|
|
2958
3044
|
duration_ms: z.number().optional().describe("Duration for wait, long_press, swipe"),
|
|
2959
|
-
value: z.string().optional().describe("Value for select_option, permission name, network mode, package ID"),
|
|
3045
|
+
value: z.string().optional().describe("Value for select_option, permission name, network mode, package ID, push_notification payload (APNs JSON or alert text), status_bar overrides (JSON or key=value list, or \"clear\")"),
|
|
2960
3046
|
file_path: z.string().optional().describe("File path for upload_file"),
|
|
2961
3047
|
fields: z.array(z.object({
|
|
2962
3048
|
selector: z.string(),
|
|
@@ -2979,6 +3065,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
2979
3065
|
.describe("For win_ui_automate"),
|
|
2980
3066
|
app_name: z.string().optional().describe("App name to bring to front before interaction. Auto-detected from active recording if omitted. Also used for launch_app, win_ui_inspect, win_ui_automate."),
|
|
2981
3067
|
package_id: z.string().optional().describe("Package/bundle ID for mobile actions"),
|
|
3068
|
+
device_id: z.string().optional().describe("Pinned mobile device: adb serial (e.g. emulator-5554) or iOS simulator UDID. When omitted, falls back to the device the last journey/recording targeted (.codeloop/active_target.json), then to adb's default / simctl 'booted'. Pass explicitly when several emulators/simulators are attached."),
|
|
2982
3069
|
intent: z.string().optional().describe("Semantic label for what this interaction is doing — short verb-phrase like 'edit product row', 'confirm delete dialog', 'save form', 'create new range'. STRONGLY RECOMMENDED for desktop coordinate-based clicks (target_type='desktop' with only x/y) because the CRUD classifier in user_journey_evidence can't infer the meaning of a raw (x, y) pair the way it can from a Playwright selector or DOM aria_label. Without `intent`, a sequence of coordinate clicks that delete a record will score `delete_actions: 0` and fail the gate. The classifier reads `intent` (plus `description` / `purpose` / `step` as aliases) into the same target-text bucket as selectors and aria labels, so the SAME keywords that work for browsers (edit / delete / save / submit / create / new) also work here. Example: when clicking the 'Yes' button of a Windows MessageBox at (2640, 820), pass intent='confirm delete'."),
|
|
2983
3070
|
description: z.string().optional().describe("[Alias for intent] Same semantics."),
|
|
2984
3071
|
purpose: z.string().optional().describe("[Alias for intent] Same semantics."),
|
|
@@ -3015,6 +3102,19 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3015
3102
|
tt = await detectTargetType(cwd);
|
|
3016
3103
|
}
|
|
3017
3104
|
}
|
|
3105
|
+
// Pin mobile commands to one device: explicit device_id param →
|
|
3106
|
+
// the device the active recording is capturing → the device the
|
|
3107
|
+
// last journey persisted (active_target.json) → undefined (adb
|
|
3108
|
+
// default / simctl "booted").
|
|
3109
|
+
let deviceId;
|
|
3110
|
+
if (tt === "android_emulator" || tt === "ios_simulator") {
|
|
3111
|
+
const { resolveDeviceId } = await import("./runners/active_target.js");
|
|
3112
|
+
deviceId = resolveDeviceId(params.device_id ?? vr.getActiveRecordingDevice() ?? undefined, cwd, tt);
|
|
3113
|
+
}
|
|
3114
|
+
// iOS sim input backend (CGEvent window mapping / idb) — simctl
|
|
3115
|
+
// has NO tap/type/swipe; see runners/ios_sim_input.ts.
|
|
3116
|
+
const sim = await import("./runners/ios_sim_input.js");
|
|
3117
|
+
const iosUdid = deviceId || "booted";
|
|
3018
3118
|
// For browser target, ensure Playwright headed browser is running
|
|
3019
3119
|
if (tt === "browser" && action !== "wait") {
|
|
3020
3120
|
await bi.ensureBrowserPage();
|
|
@@ -3023,6 +3123,18 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3023
3123
|
let windowOriginOffset = null;
|
|
3024
3124
|
let screenshotDims = null;
|
|
3025
3125
|
if (tt === "desktop") {
|
|
3126
|
+
// P2.2 — Wayland-only Linux sessions can't be driven by xdotool.
|
|
3127
|
+
// Fail fast with the remediation directive instead of letting every
|
|
3128
|
+
// individual action return an unexplained success: false.
|
|
3129
|
+
const { waylandDirective } = await import("./runners/wayland.js");
|
|
3130
|
+
const waylandBlock = waylandDirective();
|
|
3131
|
+
if (waylandBlock && action !== "wait") {
|
|
3132
|
+
return {
|
|
3133
|
+
content: [
|
|
3134
|
+
{ type: "text", text: JSON.stringify({ success: false, action, error: waylandBlock }, null, 2) },
|
|
3135
|
+
],
|
|
3136
|
+
};
|
|
3137
|
+
}
|
|
3026
3138
|
const appName = params.app_name || vr.getActiveRecordingAppName();
|
|
3027
3139
|
if (appName && action !== "wait") {
|
|
3028
3140
|
await wm.bringAppToFront(appName);
|
|
@@ -3161,10 +3273,13 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3161
3273
|
success = await bi.browserClick(params.selector);
|
|
3162
3274
|
}
|
|
3163
3275
|
else if (tt === "android_emulator" && params.x != null && params.y != null) {
|
|
3164
|
-
success = await wm.adbTap(params.x, params.y);
|
|
3276
|
+
success = await wm.adbTap(params.x, params.y, deviceId);
|
|
3165
3277
|
}
|
|
3166
3278
|
else if (tt === "ios_simulator" && params.x != null && params.y != null) {
|
|
3167
|
-
|
|
3279
|
+
const r = await sim.iosSimTap(params.x, params.y, iosUdid);
|
|
3280
|
+
success = r.success;
|
|
3281
|
+
detail = `click at (${params.x},${params.y}) — ${r.detail}`;
|
|
3282
|
+
break;
|
|
3168
3283
|
}
|
|
3169
3284
|
else if (params.selector) {
|
|
3170
3285
|
if (process.platform === "win32") {
|
|
@@ -3249,10 +3364,13 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3249
3364
|
success = await bi.browserType(params.selector, params.text);
|
|
3250
3365
|
}
|
|
3251
3366
|
else if (tt === "android_emulator" && params.text) {
|
|
3252
|
-
success = await wm.adbType(params.text);
|
|
3367
|
+
success = await wm.adbType(params.text, deviceId);
|
|
3253
3368
|
}
|
|
3254
3369
|
else if (tt === "ios_simulator" && params.text) {
|
|
3255
|
-
|
|
3370
|
+
const r = await sim.iosSimType(params.text, iosUdid);
|
|
3371
|
+
success = r.success;
|
|
3372
|
+
detail = `type "${(params.text || "").substring(0, 50)}" — ${r.detail}`;
|
|
3373
|
+
break;
|
|
3256
3374
|
}
|
|
3257
3375
|
else if (params.text) {
|
|
3258
3376
|
success = await wm.typeText(params.text);
|
|
@@ -3279,7 +3397,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3279
3397
|
// uiautomator hierarchy — lets the agent analyze a Flutter/native
|
|
3280
3398
|
// AI chatbot's answer without OCR. `selector` is treated as a
|
|
3281
3399
|
// text/resource-id substring filter when provided.
|
|
3282
|
-
extractedText = await wm.adbGetText(params.selector);
|
|
3400
|
+
extractedText = await wm.adbGetText(params.selector, deviceId);
|
|
3283
3401
|
success = extractedText !== null && extractedText.length > 0;
|
|
3284
3402
|
const preview = (extractedText ?? "").replace(/\s+/g, " ").trim().slice(0, 120);
|
|
3285
3403
|
detail = success
|
|
@@ -3287,11 +3405,14 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3287
3405
|
: "get_text (android) found no visible text — is the emulator booted and the app foregrounded?";
|
|
3288
3406
|
}
|
|
3289
3407
|
else if (tt === "ios_simulator") {
|
|
3290
|
-
//
|
|
3291
|
-
// screenshot via its own vision.
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3408
|
+
// idb (when installed) exposes the accessibility tree; otherwise
|
|
3409
|
+
// the agent reads the answer from a screenshot via its own vision.
|
|
3410
|
+
extractedText = await sim.iosSimGetText(iosUdid);
|
|
3411
|
+
success = extractedText !== null && extractedText.length > 0;
|
|
3412
|
+
const preview = (extractedText ?? "").replace(/\s+/g, " ").trim().slice(0, 120);
|
|
3413
|
+
detail = success
|
|
3414
|
+
? `get_text (ios via idb) → "${preview}${(extractedText ?? "").length > 120 ? "…" : ""}"`
|
|
3415
|
+
: "get_text on iOS needs idb (brew install idb-companion && pipx install fb-idb). Without it, call codeloop_capture_screenshot with target_type=\"ios_simulator\" and read the text from the image with your vision.";
|
|
3295
3416
|
}
|
|
3296
3417
|
else {
|
|
3297
3418
|
success = false;
|
|
@@ -3327,22 +3448,22 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3327
3448
|
up: "KEYCODE_DPAD_UP", down: "KEYCODE_DPAD_DOWN",
|
|
3328
3449
|
left: "KEYCODE_DPAD_LEFT", right: "KEYCODE_DPAD_RIGHT",
|
|
3329
3450
|
};
|
|
3330
|
-
success = await wm.adbKey(adbKeyMap[params.key.toLowerCase()] || `KEYCODE_${params.key.toUpperCase()}
|
|
3451
|
+
success = await wm.adbKey(adbKeyMap[params.key.toLowerCase()] || `KEYCODE_${params.key.toUpperCase()}`, deviceId);
|
|
3331
3452
|
}
|
|
3332
3453
|
else if (tt === "ios_simulator") {
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
const mapped = simKeyMap[params.key.toLowerCase()];
|
|
3340
|
-
if (mapped) {
|
|
3341
|
-
success = await wm.simctlKey(mapped);
|
|
3454
|
+
// Named keys (return/tab/arrows/…) go through the shared
|
|
3455
|
+
// key-name map; single printable characters are typed.
|
|
3456
|
+
if (params.key.length === 1 && !/\s/.test(params.key)) {
|
|
3457
|
+
const r = await sim.iosSimType(params.key, iosUdid);
|
|
3458
|
+
success = r.success;
|
|
3459
|
+
detail = `keystroke "${params.key}" — ${r.detail}`;
|
|
3342
3460
|
}
|
|
3343
3461
|
else {
|
|
3344
|
-
|
|
3462
|
+
const r = await sim.iosSimKey(params.key, iosUdid);
|
|
3463
|
+
success = r.success;
|
|
3464
|
+
detail = `keystroke "${params.key}" — ${r.detail}`;
|
|
3345
3465
|
}
|
|
3466
|
+
break;
|
|
3346
3467
|
}
|
|
3347
3468
|
else {
|
|
3348
3469
|
success = await wm.sendKeyByName(params.key);
|
|
@@ -3370,15 +3491,20 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3370
3491
|
const sx = params.x || 540, sy = params.y || 1200;
|
|
3371
3492
|
const ey = dir === "down" ? sy - 600 : dir === "up" ? sy + 600 : sy;
|
|
3372
3493
|
const ex = dir === "left" ? sx + 600 : dir === "right" ? sx - 600 : sx;
|
|
3373
|
-
success = await wm.adbSwipe(sx, sy, ex, ey, 300);
|
|
3494
|
+
success = await wm.adbSwipe(sx, sy, ex, ey, 300, deviceId);
|
|
3374
3495
|
}
|
|
3375
3496
|
else if (tt === "ios_simulator") {
|
|
3376
3497
|
const dir = params.direction || "down";
|
|
3377
|
-
|
|
3378
|
-
|
|
3498
|
+
// Defaults are device PIXELS on a screenshot — centre-ish of a
|
|
3499
|
+
// modern iPhone (~1179×2556).
|
|
3500
|
+
const sx = params.x || 590, sy = params.y || 1280;
|
|
3501
|
+
const dist = params.amount || 600;
|
|
3379
3502
|
const ey = dir === "down" ? sy - dist : dir === "up" ? sy + dist : sy;
|
|
3380
3503
|
const ex = dir === "left" ? sx + dist : dir === "right" ? sx - dist : sx;
|
|
3381
|
-
|
|
3504
|
+
const r = await sim.iosSimSwipe(sx, sy, ex, ey, 300, iosUdid);
|
|
3505
|
+
success = r.success;
|
|
3506
|
+
detail = `scroll ${dir} — ${r.detail}`;
|
|
3507
|
+
break;
|
|
3382
3508
|
}
|
|
3383
3509
|
else {
|
|
3384
3510
|
const t = translateXY(params.x || 500, params.y || 400);
|
|
@@ -3392,7 +3518,13 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3392
3518
|
}
|
|
3393
3519
|
else if (params.x != null && params.y != null && params.x2 != null && params.y2 != null) {
|
|
3394
3520
|
if (tt === "android_emulator") {
|
|
3395
|
-
success = await wm.adbSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 500);
|
|
3521
|
+
success = await wm.adbSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 500, deviceId);
|
|
3522
|
+
}
|
|
3523
|
+
else if (tt === "ios_simulator") {
|
|
3524
|
+
const r = await sim.iosSimSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 500, iosUdid);
|
|
3525
|
+
success = r.success;
|
|
3526
|
+
detail = `drag_drop — ${r.detail}`;
|
|
3527
|
+
break;
|
|
3396
3528
|
}
|
|
3397
3529
|
else {
|
|
3398
3530
|
const a = translateXY(params.x, params.y);
|
|
@@ -3404,7 +3536,13 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3404
3536
|
break;
|
|
3405
3537
|
case "long_press":
|
|
3406
3538
|
if (tt === "android_emulator" && params.x != null && params.y != null) {
|
|
3407
|
-
success = await wm.adbLongPress(params.x, params.y, params.duration_ms || 1000);
|
|
3539
|
+
success = await wm.adbLongPress(params.x, params.y, params.duration_ms || 1000, deviceId);
|
|
3540
|
+
}
|
|
3541
|
+
else if (tt === "ios_simulator" && params.x != null && params.y != null) {
|
|
3542
|
+
const r = await sim.iosSimLongPress(params.x, params.y, params.duration_ms || 1000, iosUdid);
|
|
3543
|
+
success = r.success;
|
|
3544
|
+
detail = `long_press at (${params.x},${params.y}) — ${r.detail}`;
|
|
3545
|
+
break;
|
|
3408
3546
|
}
|
|
3409
3547
|
else if (params.x != null && params.y != null) {
|
|
3410
3548
|
const t = translateXY(params.x, params.y);
|
|
@@ -3416,6 +3554,21 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3416
3554
|
if (tt === "browser" && params.selector && params.text) {
|
|
3417
3555
|
success = await bi.browserTypeAndSubmit(params.selector, params.text);
|
|
3418
3556
|
}
|
|
3557
|
+
else if (tt === "android_emulator" && params.text) {
|
|
3558
|
+
success = await wm.adbType(params.text, deviceId);
|
|
3559
|
+
if (success) {
|
|
3560
|
+
await new Promise(r => setTimeout(r, 100));
|
|
3561
|
+
success = await wm.adbKey("KEYCODE_ENTER", deviceId);
|
|
3562
|
+
}
|
|
3563
|
+
}
|
|
3564
|
+
else if (tt === "ios_simulator" && params.text) {
|
|
3565
|
+
const typed = await sim.iosSimType(params.text, iosUdid);
|
|
3566
|
+
success = typed.success;
|
|
3567
|
+
if (success) {
|
|
3568
|
+
await new Promise(r => setTimeout(r, 100));
|
|
3569
|
+
success = (await sim.iosSimKey("return", iosUdid)).success;
|
|
3570
|
+
}
|
|
3571
|
+
}
|
|
3419
3572
|
else if (params.text) {
|
|
3420
3573
|
success = await wm.typeText(params.text);
|
|
3421
3574
|
if (success) {
|
|
@@ -3429,6 +3582,21 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3429
3582
|
if (tt === "browser" && params.selector && params.text) {
|
|
3430
3583
|
success = await bi.browserTypeAndTab(params.selector, params.text);
|
|
3431
3584
|
}
|
|
3585
|
+
else if (tt === "android_emulator" && params.text) {
|
|
3586
|
+
success = await wm.adbType(params.text, deviceId);
|
|
3587
|
+
if (success) {
|
|
3588
|
+
await new Promise(r => setTimeout(r, 50));
|
|
3589
|
+
success = await wm.adbKey("KEYCODE_TAB", deviceId);
|
|
3590
|
+
}
|
|
3591
|
+
}
|
|
3592
|
+
else if (tt === "ios_simulator" && params.text) {
|
|
3593
|
+
const typed = await sim.iosSimType(params.text, iosUdid);
|
|
3594
|
+
success = typed.success;
|
|
3595
|
+
if (success) {
|
|
3596
|
+
await new Promise(r => setTimeout(r, 50));
|
|
3597
|
+
success = (await sim.iosSimKey("tab", iosUdid)).success;
|
|
3598
|
+
}
|
|
3599
|
+
}
|
|
3432
3600
|
else if (params.text) {
|
|
3433
3601
|
success = await wm.typeText(params.text);
|
|
3434
3602
|
if (success) {
|
|
@@ -3500,10 +3668,10 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3500
3668
|
success = await bi.browserNavigate(params.url);
|
|
3501
3669
|
}
|
|
3502
3670
|
else if (tt === "android_emulator") {
|
|
3503
|
-
success = await wm.adbDeepLink(params.url);
|
|
3671
|
+
success = await wm.adbDeepLink(params.url, deviceId);
|
|
3504
3672
|
}
|
|
3505
3673
|
else if (tt === "ios_simulator") {
|
|
3506
|
-
success = await wm.simctlOpenUrl(params.url);
|
|
3674
|
+
success = await wm.simctlOpenUrl(params.url, iosUdid);
|
|
3507
3675
|
}
|
|
3508
3676
|
else {
|
|
3509
3677
|
const { navigateDesktopBrowser } = await import("./runners/window_manager.js");
|
|
@@ -3514,7 +3682,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3514
3682
|
break;
|
|
3515
3683
|
case "navigate_back":
|
|
3516
3684
|
if (tt === "android_emulator") {
|
|
3517
|
-
success = await wm.adbBackButton();
|
|
3685
|
+
success = await wm.adbBackButton(deviceId);
|
|
3518
3686
|
}
|
|
3519
3687
|
else if (tt === "browser") {
|
|
3520
3688
|
success = await bi.browserGoBack();
|
|
@@ -3542,7 +3710,13 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3542
3710
|
break;
|
|
3543
3711
|
case "swipe":
|
|
3544
3712
|
if (tt === "android_emulator" && params.x != null && params.y != null && params.x2 != null && params.y2 != null) {
|
|
3545
|
-
success = await wm.adbSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 300);
|
|
3713
|
+
success = await wm.adbSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 300, deviceId);
|
|
3714
|
+
}
|
|
3715
|
+
else if (tt === "ios_simulator" && params.x != null && params.y != null && params.x2 != null && params.y2 != null) {
|
|
3716
|
+
const r = await sim.iosSimSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 300, iosUdid);
|
|
3717
|
+
success = r.success;
|
|
3718
|
+
detail = `swipe from (${params.x},${params.y}) to (${params.x2},${params.y2}) — ${r.detail}`;
|
|
3719
|
+
break;
|
|
3546
3720
|
}
|
|
3547
3721
|
else if (params.x != null && params.y != null && params.x2 != null && params.y2 != null) {
|
|
3548
3722
|
success = await wm.dragDrop(params.x, params.y, params.x2, params.y2, params.duration_ms || 300);
|
|
@@ -3551,49 +3725,67 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3551
3725
|
break;
|
|
3552
3726
|
case "back_button":
|
|
3553
3727
|
if (tt === "android_emulator")
|
|
3554
|
-
success = await wm.adbBackButton();
|
|
3728
|
+
success = await wm.adbBackButton(deviceId);
|
|
3555
3729
|
detail = "back_button";
|
|
3556
3730
|
break;
|
|
3557
3731
|
case "home_button":
|
|
3558
3732
|
if (tt === "android_emulator")
|
|
3559
|
-
success = await wm.adbHomeButton();
|
|
3733
|
+
success = await wm.adbHomeButton(deviceId);
|
|
3560
3734
|
detail = "home_button";
|
|
3561
3735
|
break;
|
|
3562
3736
|
case "deep_link":
|
|
3563
3737
|
if (params.url) {
|
|
3564
3738
|
if (tt === "android_emulator")
|
|
3565
|
-
success = await wm.adbDeepLink(params.url);
|
|
3739
|
+
success = await wm.adbDeepLink(params.url, deviceId);
|
|
3566
3740
|
else if (tt === "ios_simulator")
|
|
3567
|
-
success = await wm.simctlOpenUrl(params.url);
|
|
3741
|
+
success = await wm.simctlOpenUrl(params.url, iosUdid);
|
|
3568
3742
|
}
|
|
3569
3743
|
detail = `deep_link "${params.url}"`;
|
|
3570
3744
|
break;
|
|
3571
3745
|
case "grant_permission":
|
|
3572
3746
|
if (tt === "android_emulator" && params.package_id && params.value) {
|
|
3573
|
-
success = await wm.adbPermission(params.package_id, params.value, params.grant !== false);
|
|
3747
|
+
success = await wm.adbPermission(params.package_id, params.value, params.grant !== false, deviceId);
|
|
3748
|
+
detail = `grant_permission "${params.value}"`;
|
|
3749
|
+
}
|
|
3750
|
+
else if (tt === "ios_simulator" && params.package_id && params.value) {
|
|
3751
|
+
// P1.2: simctl privacy grant|revoke <service> <bundle>
|
|
3752
|
+
success = await wm.simctlPermission(params.package_id, params.value, params.grant !== false, iosUdid);
|
|
3753
|
+
detail = `grant_permission "${params.value}" → simctl privacy ${params.grant !== false ? "grant" : "revoke"} (services: photos, location, microphone, contacts, calendar, …)`;
|
|
3754
|
+
}
|
|
3755
|
+
else {
|
|
3756
|
+
detail = `grant_permission "${params.value}" — needs package_id + value (permission/service name)`;
|
|
3574
3757
|
}
|
|
3575
|
-
detail = `grant_permission "${params.value}"`;
|
|
3576
3758
|
break;
|
|
3577
3759
|
case "rotate_device":
|
|
3578
3760
|
if (tt === "android_emulator") {
|
|
3579
|
-
success = await wm.adbRotate(params.orientation === "landscape");
|
|
3761
|
+
success = await wm.adbRotate(params.orientation === "landscape", deviceId);
|
|
3762
|
+
detail = `rotate_device ${params.orientation}`;
|
|
3763
|
+
}
|
|
3764
|
+
else if (tt === "ios_simulator") {
|
|
3765
|
+
// P1.2: no simctl rotation exists — drive the Simulator app's
|
|
3766
|
+
// Device menu shortcut (Cmd+Left/Right) via CGEvent.
|
|
3767
|
+
const r = await sim.iosSimRotate(params.orientation === "landscape" ? "landscape" : "portrait", iosUdid);
|
|
3768
|
+
success = r.success;
|
|
3769
|
+
detail = `rotate_device ${params.orientation} — ${r.detail}`;
|
|
3770
|
+
}
|
|
3771
|
+
else {
|
|
3772
|
+
detail = `rotate_device ${params.orientation}`;
|
|
3580
3773
|
}
|
|
3581
|
-
detail = `rotate_device ${params.orientation}`;
|
|
3582
3774
|
break;
|
|
3583
3775
|
case "biometric_auth":
|
|
3584
3776
|
if (tt === "ios_simulator") {
|
|
3585
|
-
success = await wm.simctlBiometric(params.accept !== false);
|
|
3777
|
+
success = await wm.simctlBiometric(params.accept !== false, iosUdid);
|
|
3586
3778
|
}
|
|
3587
3779
|
detail = `biometric_auth ${params.accept !== false ? "accept" : "reject"}`;
|
|
3588
3780
|
break;
|
|
3589
3781
|
case "launch_app":
|
|
3590
3782
|
if (tt === "android_emulator" && params.package_id) {
|
|
3591
|
-
const r = await import("./runners/base.js").then(m => m.runCommand("adb",
|
|
3783
|
+
const r = await import("./runners/base.js").then(m => m.runCommand("adb", wm.adbArgs(deviceId, "shell", "am", "start", "-n", params.package_id), process.cwd()));
|
|
3592
3784
|
success = r.exit_code === 0;
|
|
3593
3785
|
detail = `launch_app "${params.package_id}"`;
|
|
3594
3786
|
}
|
|
3595
3787
|
else if (tt === "ios_simulator" && params.package_id) {
|
|
3596
|
-
success = await wm.simctlLaunch(params.package_id);
|
|
3788
|
+
success = await wm.simctlLaunch(params.package_id, iosUdid);
|
|
3597
3789
|
detail = `launch_app "${params.package_id}"`;
|
|
3598
3790
|
}
|
|
3599
3791
|
else if (tt === "desktop") {
|
|
@@ -3624,21 +3816,83 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3624
3816
|
break;
|
|
3625
3817
|
case "clear_app_data":
|
|
3626
3818
|
if (tt === "android_emulator" && params.package_id) {
|
|
3627
|
-
success = await wm.adbClearData(params.package_id);
|
|
3819
|
+
success = await wm.adbClearData(params.package_id, deviceId);
|
|
3820
|
+
detail = `clear_app_data "${params.package_id}"`;
|
|
3821
|
+
}
|
|
3822
|
+
else if (tt === "ios_simulator" && params.package_id) {
|
|
3823
|
+
// P1.2: iOS has no `pm clear` — uninstall wipes data AND the
|
|
3824
|
+
// binary, so the app must be reinstalled before the next step.
|
|
3825
|
+
success = await wm.simctlUninstall(params.package_id, iosUdid);
|
|
3826
|
+
detail = success
|
|
3827
|
+
? `clear_app_data "${params.package_id}": app uninstalled (data wiped). REINSTALL required — re-run codeloop_run_journey or \`flutter run\`/xcodebuild install before interacting again.`
|
|
3828
|
+
: `clear_app_data "${params.package_id}": simctl uninstall failed — is the bundle id correct and the simulator booted?`;
|
|
3829
|
+
}
|
|
3830
|
+
else {
|
|
3831
|
+
detail = `clear_app_data "${params.package_id}"`;
|
|
3628
3832
|
}
|
|
3629
|
-
detail = `clear_app_data "${params.package_id}"`;
|
|
3630
3833
|
break;
|
|
3631
3834
|
case "mock_location":
|
|
3632
3835
|
if (tt === "android_emulator" && params.latitude != null && params.longitude != null) {
|
|
3633
|
-
|
|
3836
|
+
// P1.3: `adb emu geo` talks to the EMULATOR CONSOLE — it can
|
|
3837
|
+
// never work on physical hardware, so fail with a usable
|
|
3838
|
+
// directive instead of a cryptic adb error.
|
|
3839
|
+
if (!wm.isEmulatorSerial(deviceId)) {
|
|
3840
|
+
success = false;
|
|
3841
|
+
detail = `mock_location is emulator-only (\`adb emu geo\` drives the emulator console; "${deviceId}" is a physical device). On hardware: enable Developer options → "Select mock location app" and use a mock-location app, or test this journey on an emulator.`;
|
|
3842
|
+
break;
|
|
3843
|
+
}
|
|
3844
|
+
success = await wm.adbMockLocation(params.latitude, params.longitude, deviceId);
|
|
3845
|
+
}
|
|
3846
|
+
else if (tt === "ios_simulator" && params.latitude != null && params.longitude != null) {
|
|
3847
|
+
// P1.2: simctl location <udid> set <lat>,<lon>
|
|
3848
|
+
success = await wm.simctlSetLocation(params.latitude, params.longitude, iosUdid);
|
|
3634
3849
|
}
|
|
3635
3850
|
detail = `mock_location (${params.latitude},${params.longitude})`;
|
|
3636
3851
|
break;
|
|
3637
3852
|
case "simulate_network":
|
|
3638
3853
|
if (tt === "android_emulator" && params.value) {
|
|
3639
|
-
|
|
3854
|
+
// P1.3: `adb emu network` is emulator-console-only as well.
|
|
3855
|
+
if (!wm.isEmulatorSerial(deviceId)) {
|
|
3856
|
+
success = false;
|
|
3857
|
+
detail = `simulate_network is emulator-only (\`adb emu network\` drives the emulator console; "${deviceId}" is a physical device). On hardware: toggle airplane mode / Wi-Fi via adb shell, or test this journey on an emulator.`;
|
|
3858
|
+
break;
|
|
3859
|
+
}
|
|
3860
|
+
success = await wm.adbNetworkCondition(params.value, deviceId);
|
|
3861
|
+
detail = `simulate_network "${params.value}"`;
|
|
3862
|
+
}
|
|
3863
|
+
else if (tt === "ios_simulator") {
|
|
3864
|
+
// Honest unsupported: simctl has no network conditioner; the
|
|
3865
|
+
// status_bar override only changes the INDICATOR, not traffic.
|
|
3866
|
+
detail = `simulate_network is not supported on the iOS simulator (simctl has no network conditioner). Install Apple's "Network Link Conditioner" prefpane for host-level shaping, or use action "status_bar" to fake the indicator for screenshots.`;
|
|
3867
|
+
}
|
|
3868
|
+
else {
|
|
3869
|
+
detail = `simulate_network "${params.value}"`;
|
|
3870
|
+
}
|
|
3871
|
+
break;
|
|
3872
|
+
case "push_notification":
|
|
3873
|
+
// P1.2: simulated APNs push — iOS simulator only.
|
|
3874
|
+
if (tt === "ios_simulator" && params.package_id && params.value) {
|
|
3875
|
+
const r = await wm.simctlPush(params.package_id, params.value, iosUdid);
|
|
3876
|
+
success = r.success;
|
|
3877
|
+
detail = `push_notification → ${r.detail}`;
|
|
3878
|
+
}
|
|
3879
|
+
else if (tt === "android_emulator") {
|
|
3880
|
+
detail = `push_notification is iOS-simulator-only (simctl push). On Android, drive the app's own notification trigger or use \`adb shell am broadcast\` for app-specific receivers.`;
|
|
3881
|
+
}
|
|
3882
|
+
else {
|
|
3883
|
+
detail = `push_notification needs package_id (bundle id) + value (APNs JSON payload or plain alert text).`;
|
|
3884
|
+
}
|
|
3885
|
+
break;
|
|
3886
|
+
case "status_bar":
|
|
3887
|
+
// P1.2: simctl status_bar override for demo-clean screenshots.
|
|
3888
|
+
if (tt === "ios_simulator") {
|
|
3889
|
+
const r = await wm.simctlStatusBar(params.value ?? "", iosUdid);
|
|
3890
|
+
success = r.success;
|
|
3891
|
+
detail = `status_bar → ${r.detail}`;
|
|
3892
|
+
}
|
|
3893
|
+
else {
|
|
3894
|
+
detail = `status_bar is iOS-simulator-only (simctl status_bar). Value: JSON or "key=value,…" with time, dataNetwork, wifiMode, wifiBars, cellularMode, cellularBars, operatorName, batteryState, batteryLevel — or "clear".`;
|
|
3640
3895
|
}
|
|
3641
|
-
detail = `simulate_network "${params.value}"`;
|
|
3642
3896
|
break;
|
|
3643
3897
|
case "maestro_flow":
|
|
3644
3898
|
if (params.maestro_steps) {
|
|
@@ -3648,7 +3902,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3648
3902
|
if ("error" in genResult) {
|
|
3649
3903
|
return { success: false, action, detail: genResult.error };
|
|
3650
3904
|
}
|
|
3651
|
-
const runResult = await mg.runGeneratedFlow(genResult.flowPath, cwd);
|
|
3905
|
+
const runResult = await mg.runGeneratedFlow(genResult.flowPath, cwd, deviceId);
|
|
3652
3906
|
success = runResult.success;
|
|
3653
3907
|
detail = `maestro_flow (${params.maestro_steps.length} steps) → ${runResult.success ? "passed" : runResult.error}`;
|
|
3654
3908
|
}
|
|
@@ -3820,7 +4074,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3820
4074
|
}
|
|
3821
4075
|
break;
|
|
3822
4076
|
default:
|
|
3823
|
-
detail = `Unknown action: "${action}". Available: click, double_click, right_click, hover, type, keystroke, hotkey, scroll, drag_drop, long_press, type_and_submit, type_and_tab, fill_form, select_option, toggle, upload_file, navigate_url, navigate_back, navigate_forward, wait, swipe, back_button, home_button, deep_link, grant_permission, rotate_device, biometric_auth, launch_app, clear_app_data, mock_location, simulate_network, maestro_flow, win_ui_inspect, win_ui_automate, sequence`;
|
|
4077
|
+
detail = `Unknown action: "${action}". Available: click, double_click, right_click, hover, type, keystroke, hotkey, scroll, drag_drop, long_press, type_and_submit, type_and_tab, fill_form, select_option, toggle, upload_file, navigate_url, navigate_back, navigate_forward, wait, swipe, back_button, home_button, deep_link, grant_permission, rotate_device, biometric_auth, launch_app, clear_app_data, mock_location, simulate_network, push_notification, status_bar, maestro_flow, win_ui_inspect, win_ui_automate, sequence`;
|
|
3824
4078
|
return { success: false, action, detail };
|
|
3825
4079
|
}
|
|
3826
4080
|
await trackUsage(apiKey, "interaction");
|