codeloop-mcp-server 0.1.88 → 0.1.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/evidence/interaction_coverage.d.ts.map +1 -1
- package/dist/evidence/interaction_coverage.js +2 -0
- package/dist/evidence/interaction_coverage.js.map +1 -1
- package/dist/index.js +236 -54
- package/dist/index.js.map +1 -1
- package/dist/runners/active_target.d.ts +31 -0
- package/dist/runners/active_target.d.ts.map +1 -0
- package/dist/runners/active_target.js +65 -0
- package/dist/runners/active_target.js.map +1 -0
- package/dist/runners/app_launcher.d.ts +37 -1
- package/dist/runners/app_launcher.d.ts.map +1 -1
- package/dist/runners/app_launcher.js +114 -15
- package/dist/runners/app_launcher.js.map +1 -1
- package/dist/runners/app_logger.d.ts +1 -1
- package/dist/runners/app_logger.d.ts.map +1 -1
- package/dist/runners/app_logger.js +9 -8
- package/dist/runners/app_logger.js.map +1 -1
- package/dist/runners/device_probe.d.ts +18 -0
- package/dist/runners/device_probe.d.ts.map +1 -1
- package/dist/runners/device_probe.js +47 -20
- package/dist/runners/device_probe.js.map +1 -1
- package/dist/runners/interaction_engine.d.ts +6 -0
- package/dist/runners/interaction_engine.d.ts.map +1 -1
- package/dist/runners/interaction_engine.js +64 -21
- package/dist/runners/interaction_engine.js.map +1 -1
- package/dist/runners/ios_sim_input.d.ts +106 -0
- package/dist/runners/ios_sim_input.d.ts.map +1 -0
- package/dist/runners/ios_sim_input.js +453 -0
- package/dist/runners/ios_sim_input.js.map +1 -0
- package/dist/runners/maestro.d.ts +27 -2
- package/dist/runners/maestro.d.ts.map +1 -1
- package/dist/runners/maestro.js +57 -7
- package/dist/runners/maestro.js.map +1 -1
- package/dist/runners/maestro_generator.d.ts +1 -1
- package/dist/runners/maestro_generator.d.ts.map +1 -1
- package/dist/runners/maestro_generator.js +3 -2
- package/dist/runners/maestro_generator.js.map +1 -1
- package/dist/runners/screenshot.d.ts +6 -0
- package/dist/runners/screenshot.d.ts.map +1 -1
- package/dist/runners/screenshot.js +10 -9
- package/dist/runners/screenshot.js.map +1 -1
- package/dist/runners/video_recorder.d.ts +3 -1
- package/dist/runners/video_recorder.d.ts.map +1 -1
- package/dist/runners/video_recorder.js +19 -6
- package/dist/runners/video_recorder.js.map +1 -1
- package/dist/runners/window_manager.d.ts +55 -25
- package/dist/runners/window_manager.d.ts.map +1 -1
- package/dist/runners/window_manager.js +171 -71
- package/dist/runners/window_manager.js.map +1 -1
- package/dist/tools/discover_interactions.d.ts +22 -0
- package/dist/tools/discover_interactions.d.ts.map +1 -1
- package/dist/tools/discover_interactions.js +278 -5
- package/dist/tools/discover_interactions.js.map +1 -1
- package/dist/tools/discover_screens.d.ts +2 -0
- package/dist/tools/discover_screens.d.ts.map +1 -1
- package/dist/tools/discover_screens.js +139 -1
- package/dist/tools/discover_screens.js.map +1 -1
- package/dist/tools/run_journey.d.ts +6 -0
- package/dist/tools/run_journey.d.ts.map +1 -1
- package/dist/tools/run_journey.js +75 -16
- package/dist/tools/run_journey.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1495,6 +1495,7 @@ Returns: confirmation + the captured image as an MCP ImageContent block so you c
|
|
|
1495
1495
|
workspace_root: z.string().optional().describe("[Alias for project_dir] Same semantics; accepted because many agents reach for this conventional name. Pass either `project_dir` OR `workspace_root` — they're equivalent."),
|
|
1496
1496
|
target_change_entry: z.string().optional().describe("0.1.52 C7 — verbatim display name of the change-manifest entry this screenshot exercises (e.g. 'datagrid_column: \"Product Code\"' or 'PhotometricConfigurations.ProductCode'). When present, the screenshot file is auto-anchored: the filename is prefixed with a slugged form of the entry so the change_coverage_evidence (C3) gate's screenshot scan can credit this evidence to the correct manifest entry without fuzzy matching. The value is also persisted alongside the screenshot path in the response so downstream tools (interaction_replay, gate_check) can use it."),
|
|
1497
1497
|
target_type: z.string().optional().describe("Capture source. Omit for desktop/window capture (default). Set 'android_emulator' (or 'android') to capture the BOOTED Android emulator via adb, or 'ios_simulator' (or 'ios', macOS only) to capture the booted iOS simulator via simctl. Use this to screenshot a Flutter/native mobile app running on a simulator instead of the host desktop."),
|
|
1498
|
+
device_id: z.string().optional().describe("Pinned mobile device for emulator/simulator capture: adb serial or simulator UDID. Defaults to the device the last journey/recording targeted, then adb default / simctl 'booted'."),
|
|
1498
1499
|
}, async (params) => {
|
|
1499
1500
|
const authResult = await withAuth(async () => {
|
|
1500
1501
|
const { captureScreenshot } = await import("./runners/screenshot.js");
|
|
@@ -1543,7 +1544,14 @@ Returns: confirmation + the captured image as an MCP ImageContent block so you c
|
|
|
1543
1544
|
const ttRaw = normalizeTargetType(params.target_type);
|
|
1544
1545
|
const explicitTargetType = ttRaw === "android_emulator" || ttRaw === "ios_simulator" ? ttRaw : undefined;
|
|
1545
1546
|
const isMobileCapture = explicitTargetType !== undefined;
|
|
1546
|
-
|
|
1547
|
+
// Pin the capture to the journey/recording device when known.
|
|
1548
|
+
let captureDevice;
|
|
1549
|
+
if (isMobileCapture) {
|
|
1550
|
+
const { resolveDeviceId } = await import("./runners/active_target.js");
|
|
1551
|
+
const vr = await import("./runners/video_recorder.js");
|
|
1552
|
+
captureDevice = resolveDeviceId(params.device_id ?? vr.getActiveRecordingDevice() ?? undefined, cwd, explicitTargetType);
|
|
1553
|
+
}
|
|
1554
|
+
const result = await captureScreenshot(screenshotsDir, finalScreenName, isMobileCapture ? undefined : targetApp, explicitTargetType, { desktopAppMode: isMobileCapture ? false : desktopApp, device: captureDevice });
|
|
1547
1555
|
// Photometry-DB E2E 8 follow-on: when we capture a desktop app
|
|
1548
1556
|
// window, also resolve its on-screen bounds so the agent can
|
|
1549
1557
|
// (a) compute window-relative coords from the returned image
|
|
@@ -2028,6 +2036,7 @@ App logs (stdout, logcat, simctl log) are automatically captured alongside the v
|
|
|
2028
2036
|
target_type: targetTypeSchema.optional()
|
|
2029
2037
|
.describe("Capture method. Auto-detected from project if omitted. desktop=ffmpeg screen, android_emulator=adb screenrecord, ios_simulator=simctl recordVideo, browser=ffmpeg/Playwright"),
|
|
2030
2038
|
auto_launch: z.boolean().default(true).describe("When target_type=desktop and the app isn't already running, auto-launch it from the project's build output via evidence.target_app. Set false to skip (e.g. when the app is started by another process)."),
|
|
2039
|
+
device_id: z.string().optional().describe("Pinned mobile device to record: adb serial (emulator-5554) or simulator UDID. Defaults to the device the last journey targeted, then the first booted device. The pinned device is persisted so follow-up codeloop_interact calls hit the same one."),
|
|
2031
2040
|
project_dir: z.string().optional().describe("Absolute path to the project root. Defaults to CODELOOP_PROJECT_DIR env var or auto-discovered project directory. MUST be an actual project folder — passing the user's home directory is rejected. If your IDE launches the MCP server from the wrong cwd (common on Windows where Cursor uses C:\\Users\\<name> as cwd), set CODELOOP_PROJECT_DIR or pass this param explicitly."),
|
|
2032
2041
|
workspace_root: z.string().optional().describe("[Alias for project_dir] Same semantics; accepted because many agents reach for this conventional name. Pass either `project_dir` OR `workspace_root` — they're equivalent."),
|
|
2033
2042
|
}, async (params) => {
|
|
@@ -2125,6 +2134,7 @@ App logs (stdout, logcat, simctl log) are automatically captured alongside the v
|
|
|
2125
2134
|
// concrete commands rather than letting it guess. CodeLoop does not boot
|
|
2126
2135
|
// devices itself (heavy + can hijack the desktop), it directs the agent.
|
|
2127
2136
|
let deviceReadinessDirective;
|
|
2137
|
+
let recordingDevice;
|
|
2128
2138
|
if (targetType === "android_emulator" || targetType === "ios_simulator") {
|
|
2129
2139
|
try {
|
|
2130
2140
|
const { probeBootedDevices, detectMobileTargets, buildOpenSimulatorsDirective } = await import("./runners/device_probe.js");
|
|
@@ -2139,10 +2149,22 @@ App logs (stdout, logcat, simctl log) are automatically captured alongside the v
|
|
|
2139
2149
|
context: "recording",
|
|
2140
2150
|
});
|
|
2141
2151
|
}
|
|
2152
|
+
else {
|
|
2153
|
+
// Pin the recording to ONE device: explicit param → persisted
|
|
2154
|
+
// active target → first booted device of this platform. Persist
|
|
2155
|
+
// the choice so follow-up codeloop_interact calls hit the same one.
|
|
2156
|
+
const { resolveDeviceId, saveActiveTarget } = await import("./runners/active_target.js");
|
|
2157
|
+
recordingDevice =
|
|
2158
|
+
resolveDeviceId(params.device_id, cwd, targetType) ??
|
|
2159
|
+
(targetType === "android_emulator" ? booted.android[0] : booted.ios_devices[0]?.udid);
|
|
2160
|
+
if (recordingDevice) {
|
|
2161
|
+
saveActiveTarget(cwd, { target_type: targetType, device_id: recordingDevice });
|
|
2162
|
+
}
|
|
2163
|
+
}
|
|
2142
2164
|
}
|
|
2143
2165
|
catch { /* best-effort */ }
|
|
2144
2166
|
}
|
|
2145
|
-
const result = await startBackgroundRecording(videosDir, appName ?? "", params.max_duration_seconds, targetType);
|
|
2167
|
+
const result = await startBackgroundRecording(videosDir, appName ?? "", params.max_duration_seconds, targetType, recordingDevice);
|
|
2146
2168
|
if (autoLaunchSummary) {
|
|
2147
2169
|
result.auto_launch = autoLaunchSummary;
|
|
2148
2170
|
}
|
|
@@ -2930,14 +2952,18 @@ scroll works on desktop via CGEvent (macOS), user32 mouse_event (Windows), xdoto
|
|
|
2930
2952
|
Falls back to arrow key presses if CGEvent fails (permissions).
|
|
2931
2953
|
Browser-specific: Uses Playwright selectors (CSS/text) when target_type is "browser".
|
|
2932
2954
|
Mobile-specific: swipe, back_button, home_button, deep_link, grant_permission, rotate_device,
|
|
2933
|
-
biometric_auth, launch_app, clear_app_data, mock_location, simulate_network
|
|
2955
|
+
biometric_auth, launch_app, clear_app_data, mock_location, simulate_network,
|
|
2956
|
+
push_notification (iOS: simctl push — package_id + value as APNs JSON or alert text),
|
|
2957
|
+
status_bar (iOS: simctl status_bar — value as JSON or "time=9:41,batteryLevel=100", or "clear").
|
|
2958
|
+
iOS (0.1.91): grant_permission → simctl privacy; mock_location → simctl location;
|
|
2959
|
+
clear_app_data → simctl uninstall (REINSTALL needed after); rotate_device → Simulator Cmd+arrow.
|
|
2934
2960
|
Maestro: maestro_flow — generate and run a Maestro YAML flow from high-level steps.
|
|
2935
2961
|
Windows: win_ui_inspect, win_ui_automate — PowerShell UI Automation for UWP/WinUI apps.
|
|
2936
2962
|
|
|
2937
2963
|
MANDATORY for web apps: You MUST type into form fields, fill login/signup forms, test
|
|
2938
2964
|
validation errors, and click submit buttons. Just navigating pages is NOT enough.
|
|
2939
2965
|
Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
2940
|
-
action: z.string().describe("Action to perform: click, double_click, right_click, hover, type, keystroke, hotkey, scroll, drag_drop, long_press, type_and_submit, type_and_tab, fill_form, select_option, toggle, get_text, upload_file, navigate_url, navigate_back, navigate_forward, wait, sequence, swipe, back_button, home_button, deep_link, grant_permission, rotate_device, biometric_auth, launch_app, clear_app_data, mock_location, simulate_network, maestro_flow, win_ui_inspect, win_ui_automate"),
|
|
2966
|
+
action: z.string().describe("Action to perform: click, double_click, right_click, hover, type, keystroke, hotkey, scroll, drag_drop, long_press, type_and_submit, type_and_tab, fill_form, select_option, toggle, get_text, upload_file, navigate_url, navigate_back, navigate_forward, wait, sequence, swipe, back_button, home_button, deep_link, grant_permission, rotate_device, biometric_auth, launch_app, clear_app_data, mock_location, simulate_network, push_notification, status_bar, maestro_flow, win_ui_inspect, win_ui_automate"),
|
|
2941
2967
|
expect_contains: z.string().optional().describe("[get_text] Optional assertion: the read-back text must contain this substring (case-insensitive). When set, success is false if the substring is absent. Use to confirm an AI chatbot answer / dynamic response actually rendered the expected content."),
|
|
2942
2968
|
target_type: targetTypeSchema.optional()
|
|
2943
2969
|
.describe("Interaction target. Auto-detected if omitted. Accepts synonyms: `windows_desktop`/`mac_desktop`/`linux_desktop` → `desktop`; `web` → `browser`; `android` → `android_emulator`; `ios` → `ios_simulator`."),
|
|
@@ -2956,7 +2982,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
2956
2982
|
direction: z.enum(["up", "down", "left", "right"]).optional().describe("Scroll/swipe direction"),
|
|
2957
2983
|
amount: z.number().optional().describe("Scroll amount or other numeric value"),
|
|
2958
2984
|
duration_ms: z.number().optional().describe("Duration for wait, long_press, swipe"),
|
|
2959
|
-
value: z.string().optional().describe("Value for select_option, permission name, network mode, package ID"),
|
|
2985
|
+
value: z.string().optional().describe("Value for select_option, permission name, network mode, package ID, push_notification payload (APNs JSON or alert text), status_bar overrides (JSON or key=value list, or \"clear\")"),
|
|
2960
2986
|
file_path: z.string().optional().describe("File path for upload_file"),
|
|
2961
2987
|
fields: z.array(z.object({
|
|
2962
2988
|
selector: z.string(),
|
|
@@ -2979,6 +3005,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
2979
3005
|
.describe("For win_ui_automate"),
|
|
2980
3006
|
app_name: z.string().optional().describe("App name to bring to front before interaction. Auto-detected from active recording if omitted. Also used for launch_app, win_ui_inspect, win_ui_automate."),
|
|
2981
3007
|
package_id: z.string().optional().describe("Package/bundle ID for mobile actions"),
|
|
3008
|
+
device_id: z.string().optional().describe("Pinned mobile device: adb serial (e.g. emulator-5554) or iOS simulator UDID. When omitted, falls back to the device the last journey/recording targeted (.codeloop/active_target.json), then to adb's default / simctl 'booted'. Pass explicitly when several emulators/simulators are attached."),
|
|
2982
3009
|
intent: z.string().optional().describe("Semantic label for what this interaction is doing — short verb-phrase like 'edit product row', 'confirm delete dialog', 'save form', 'create new range'. STRONGLY RECOMMENDED for desktop coordinate-based clicks (target_type='desktop' with only x/y) because the CRUD classifier in user_journey_evidence can't infer the meaning of a raw (x, y) pair the way it can from a Playwright selector or DOM aria_label. Without `intent`, a sequence of coordinate clicks that delete a record will score `delete_actions: 0` and fail the gate. The classifier reads `intent` (plus `description` / `purpose` / `step` as aliases) into the same target-text bucket as selectors and aria labels, so the SAME keywords that work for browsers (edit / delete / save / submit / create / new) also work here. Example: when clicking the 'Yes' button of a Windows MessageBox at (2640, 820), pass intent='confirm delete'."),
|
|
2983
3010
|
description: z.string().optional().describe("[Alias for intent] Same semantics."),
|
|
2984
3011
|
purpose: z.string().optional().describe("[Alias for intent] Same semantics."),
|
|
@@ -3015,6 +3042,19 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3015
3042
|
tt = await detectTargetType(cwd);
|
|
3016
3043
|
}
|
|
3017
3044
|
}
|
|
3045
|
+
// Pin mobile commands to one device: explicit device_id param →
|
|
3046
|
+
// the device the active recording is capturing → the device the
|
|
3047
|
+
// last journey persisted (active_target.json) → undefined (adb
|
|
3048
|
+
// default / simctl "booted").
|
|
3049
|
+
let deviceId;
|
|
3050
|
+
if (tt === "android_emulator" || tt === "ios_simulator") {
|
|
3051
|
+
const { resolveDeviceId } = await import("./runners/active_target.js");
|
|
3052
|
+
deviceId = resolveDeviceId(params.device_id ?? vr.getActiveRecordingDevice() ?? undefined, cwd, tt);
|
|
3053
|
+
}
|
|
3054
|
+
// iOS sim input backend (CGEvent window mapping / idb) — simctl
|
|
3055
|
+
// has NO tap/type/swipe; see runners/ios_sim_input.ts.
|
|
3056
|
+
const sim = await import("./runners/ios_sim_input.js");
|
|
3057
|
+
const iosUdid = deviceId || "booted";
|
|
3018
3058
|
// For browser target, ensure Playwright headed browser is running
|
|
3019
3059
|
if (tt === "browser" && action !== "wait") {
|
|
3020
3060
|
await bi.ensureBrowserPage();
|
|
@@ -3161,10 +3201,13 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3161
3201
|
success = await bi.browserClick(params.selector);
|
|
3162
3202
|
}
|
|
3163
3203
|
else if (tt === "android_emulator" && params.x != null && params.y != null) {
|
|
3164
|
-
success = await wm.adbTap(params.x, params.y);
|
|
3204
|
+
success = await wm.adbTap(params.x, params.y, deviceId);
|
|
3165
3205
|
}
|
|
3166
3206
|
else if (tt === "ios_simulator" && params.x != null && params.y != null) {
|
|
3167
|
-
|
|
3207
|
+
const r = await sim.iosSimTap(params.x, params.y, iosUdid);
|
|
3208
|
+
success = r.success;
|
|
3209
|
+
detail = `click at (${params.x},${params.y}) — ${r.detail}`;
|
|
3210
|
+
break;
|
|
3168
3211
|
}
|
|
3169
3212
|
else if (params.selector) {
|
|
3170
3213
|
if (process.platform === "win32") {
|
|
@@ -3249,10 +3292,13 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3249
3292
|
success = await bi.browserType(params.selector, params.text);
|
|
3250
3293
|
}
|
|
3251
3294
|
else if (tt === "android_emulator" && params.text) {
|
|
3252
|
-
success = await wm.adbType(params.text);
|
|
3295
|
+
success = await wm.adbType(params.text, deviceId);
|
|
3253
3296
|
}
|
|
3254
3297
|
else if (tt === "ios_simulator" && params.text) {
|
|
3255
|
-
|
|
3298
|
+
const r = await sim.iosSimType(params.text, iosUdid);
|
|
3299
|
+
success = r.success;
|
|
3300
|
+
detail = `type "${(params.text || "").substring(0, 50)}" — ${r.detail}`;
|
|
3301
|
+
break;
|
|
3256
3302
|
}
|
|
3257
3303
|
else if (params.text) {
|
|
3258
3304
|
success = await wm.typeText(params.text);
|
|
@@ -3279,7 +3325,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3279
3325
|
// uiautomator hierarchy — lets the agent analyze a Flutter/native
|
|
3280
3326
|
// AI chatbot's answer without OCR. `selector` is treated as a
|
|
3281
3327
|
// text/resource-id substring filter when provided.
|
|
3282
|
-
extractedText = await wm.adbGetText(params.selector);
|
|
3328
|
+
extractedText = await wm.adbGetText(params.selector, deviceId);
|
|
3283
3329
|
success = extractedText !== null && extractedText.length > 0;
|
|
3284
3330
|
const preview = (extractedText ?? "").replace(/\s+/g, " ").trim().slice(0, 120);
|
|
3285
3331
|
detail = success
|
|
@@ -3287,11 +3333,14 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3287
3333
|
: "get_text (android) found no visible text — is the emulator booted and the app foregrounded?";
|
|
3288
3334
|
}
|
|
3289
3335
|
else if (tt === "ios_simulator") {
|
|
3290
|
-
//
|
|
3291
|
-
// screenshot via its own vision.
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3336
|
+
// idb (when installed) exposes the accessibility tree; otherwise
|
|
3337
|
+
// the agent reads the answer from a screenshot via its own vision.
|
|
3338
|
+
extractedText = await sim.iosSimGetText(iosUdid);
|
|
3339
|
+
success = extractedText !== null && extractedText.length > 0;
|
|
3340
|
+
const preview = (extractedText ?? "").replace(/\s+/g, " ").trim().slice(0, 120);
|
|
3341
|
+
detail = success
|
|
3342
|
+
? `get_text (ios via idb) → "${preview}${(extractedText ?? "").length > 120 ? "…" : ""}"`
|
|
3343
|
+
: "get_text on iOS needs idb (brew install idb-companion && pipx install fb-idb). Without it, call codeloop_capture_screenshot with target_type=\"ios_simulator\" and read the text from the image with your vision.";
|
|
3295
3344
|
}
|
|
3296
3345
|
else {
|
|
3297
3346
|
success = false;
|
|
@@ -3327,22 +3376,22 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3327
3376
|
up: "KEYCODE_DPAD_UP", down: "KEYCODE_DPAD_DOWN",
|
|
3328
3377
|
left: "KEYCODE_DPAD_LEFT", right: "KEYCODE_DPAD_RIGHT",
|
|
3329
3378
|
};
|
|
3330
|
-
success = await wm.adbKey(adbKeyMap[params.key.toLowerCase()] || `KEYCODE_${params.key.toUpperCase()}
|
|
3379
|
+
success = await wm.adbKey(adbKeyMap[params.key.toLowerCase()] || `KEYCODE_${params.key.toUpperCase()}`, deviceId);
|
|
3331
3380
|
}
|
|
3332
3381
|
else if (tt === "ios_simulator") {
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
const mapped = simKeyMap[params.key.toLowerCase()];
|
|
3340
|
-
if (mapped) {
|
|
3341
|
-
success = await wm.simctlKey(mapped);
|
|
3382
|
+
// Named keys (return/tab/arrows/…) go through the shared
|
|
3383
|
+
// key-name map; single printable characters are typed.
|
|
3384
|
+
if (params.key.length === 1 && !/\s/.test(params.key)) {
|
|
3385
|
+
const r = await sim.iosSimType(params.key, iosUdid);
|
|
3386
|
+
success = r.success;
|
|
3387
|
+
detail = `keystroke "${params.key}" — ${r.detail}`;
|
|
3342
3388
|
}
|
|
3343
3389
|
else {
|
|
3344
|
-
|
|
3390
|
+
const r = await sim.iosSimKey(params.key, iosUdid);
|
|
3391
|
+
success = r.success;
|
|
3392
|
+
detail = `keystroke "${params.key}" — ${r.detail}`;
|
|
3345
3393
|
}
|
|
3394
|
+
break;
|
|
3346
3395
|
}
|
|
3347
3396
|
else {
|
|
3348
3397
|
success = await wm.sendKeyByName(params.key);
|
|
@@ -3370,15 +3419,20 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3370
3419
|
const sx = params.x || 540, sy = params.y || 1200;
|
|
3371
3420
|
const ey = dir === "down" ? sy - 600 : dir === "up" ? sy + 600 : sy;
|
|
3372
3421
|
const ex = dir === "left" ? sx + 600 : dir === "right" ? sx - 600 : sx;
|
|
3373
|
-
success = await wm.adbSwipe(sx, sy, ex, ey, 300);
|
|
3422
|
+
success = await wm.adbSwipe(sx, sy, ex, ey, 300, deviceId);
|
|
3374
3423
|
}
|
|
3375
3424
|
else if (tt === "ios_simulator") {
|
|
3376
3425
|
const dir = params.direction || "down";
|
|
3377
|
-
|
|
3378
|
-
|
|
3426
|
+
// Defaults are device PIXELS on a screenshot — centre-ish of a
|
|
3427
|
+
// modern iPhone (~1179×2556).
|
|
3428
|
+
const sx = params.x || 590, sy = params.y || 1280;
|
|
3429
|
+
const dist = params.amount || 600;
|
|
3379
3430
|
const ey = dir === "down" ? sy - dist : dir === "up" ? sy + dist : sy;
|
|
3380
3431
|
const ex = dir === "left" ? sx + dist : dir === "right" ? sx - dist : sx;
|
|
3381
|
-
|
|
3432
|
+
const r = await sim.iosSimSwipe(sx, sy, ex, ey, 300, iosUdid);
|
|
3433
|
+
success = r.success;
|
|
3434
|
+
detail = `scroll ${dir} — ${r.detail}`;
|
|
3435
|
+
break;
|
|
3382
3436
|
}
|
|
3383
3437
|
else {
|
|
3384
3438
|
const t = translateXY(params.x || 500, params.y || 400);
|
|
@@ -3392,7 +3446,13 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3392
3446
|
}
|
|
3393
3447
|
else if (params.x != null && params.y != null && params.x2 != null && params.y2 != null) {
|
|
3394
3448
|
if (tt === "android_emulator") {
|
|
3395
|
-
success = await wm.adbSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 500);
|
|
3449
|
+
success = await wm.adbSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 500, deviceId);
|
|
3450
|
+
}
|
|
3451
|
+
else if (tt === "ios_simulator") {
|
|
3452
|
+
const r = await sim.iosSimSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 500, iosUdid);
|
|
3453
|
+
success = r.success;
|
|
3454
|
+
detail = `drag_drop — ${r.detail}`;
|
|
3455
|
+
break;
|
|
3396
3456
|
}
|
|
3397
3457
|
else {
|
|
3398
3458
|
const a = translateXY(params.x, params.y);
|
|
@@ -3404,7 +3464,13 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3404
3464
|
break;
|
|
3405
3465
|
case "long_press":
|
|
3406
3466
|
if (tt === "android_emulator" && params.x != null && params.y != null) {
|
|
3407
|
-
success = await wm.adbLongPress(params.x, params.y, params.duration_ms || 1000);
|
|
3467
|
+
success = await wm.adbLongPress(params.x, params.y, params.duration_ms || 1000, deviceId);
|
|
3468
|
+
}
|
|
3469
|
+
else if (tt === "ios_simulator" && params.x != null && params.y != null) {
|
|
3470
|
+
const r = await sim.iosSimLongPress(params.x, params.y, params.duration_ms || 1000, iosUdid);
|
|
3471
|
+
success = r.success;
|
|
3472
|
+
detail = `long_press at (${params.x},${params.y}) — ${r.detail}`;
|
|
3473
|
+
break;
|
|
3408
3474
|
}
|
|
3409
3475
|
else if (params.x != null && params.y != null) {
|
|
3410
3476
|
const t = translateXY(params.x, params.y);
|
|
@@ -3416,6 +3482,21 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3416
3482
|
if (tt === "browser" && params.selector && params.text) {
|
|
3417
3483
|
success = await bi.browserTypeAndSubmit(params.selector, params.text);
|
|
3418
3484
|
}
|
|
3485
|
+
else if (tt === "android_emulator" && params.text) {
|
|
3486
|
+
success = await wm.adbType(params.text, deviceId);
|
|
3487
|
+
if (success) {
|
|
3488
|
+
await new Promise(r => setTimeout(r, 100));
|
|
3489
|
+
success = await wm.adbKey("KEYCODE_ENTER", deviceId);
|
|
3490
|
+
}
|
|
3491
|
+
}
|
|
3492
|
+
else if (tt === "ios_simulator" && params.text) {
|
|
3493
|
+
const typed = await sim.iosSimType(params.text, iosUdid);
|
|
3494
|
+
success = typed.success;
|
|
3495
|
+
if (success) {
|
|
3496
|
+
await new Promise(r => setTimeout(r, 100));
|
|
3497
|
+
success = (await sim.iosSimKey("return", iosUdid)).success;
|
|
3498
|
+
}
|
|
3499
|
+
}
|
|
3419
3500
|
else if (params.text) {
|
|
3420
3501
|
success = await wm.typeText(params.text);
|
|
3421
3502
|
if (success) {
|
|
@@ -3429,6 +3510,21 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3429
3510
|
if (tt === "browser" && params.selector && params.text) {
|
|
3430
3511
|
success = await bi.browserTypeAndTab(params.selector, params.text);
|
|
3431
3512
|
}
|
|
3513
|
+
else if (tt === "android_emulator" && params.text) {
|
|
3514
|
+
success = await wm.adbType(params.text, deviceId);
|
|
3515
|
+
if (success) {
|
|
3516
|
+
await new Promise(r => setTimeout(r, 50));
|
|
3517
|
+
success = await wm.adbKey("KEYCODE_TAB", deviceId);
|
|
3518
|
+
}
|
|
3519
|
+
}
|
|
3520
|
+
else if (tt === "ios_simulator" && params.text) {
|
|
3521
|
+
const typed = await sim.iosSimType(params.text, iosUdid);
|
|
3522
|
+
success = typed.success;
|
|
3523
|
+
if (success) {
|
|
3524
|
+
await new Promise(r => setTimeout(r, 50));
|
|
3525
|
+
success = (await sim.iosSimKey("tab", iosUdid)).success;
|
|
3526
|
+
}
|
|
3527
|
+
}
|
|
3432
3528
|
else if (params.text) {
|
|
3433
3529
|
success = await wm.typeText(params.text);
|
|
3434
3530
|
if (success) {
|
|
@@ -3500,10 +3596,10 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3500
3596
|
success = await bi.browserNavigate(params.url);
|
|
3501
3597
|
}
|
|
3502
3598
|
else if (tt === "android_emulator") {
|
|
3503
|
-
success = await wm.adbDeepLink(params.url);
|
|
3599
|
+
success = await wm.adbDeepLink(params.url, deviceId);
|
|
3504
3600
|
}
|
|
3505
3601
|
else if (tt === "ios_simulator") {
|
|
3506
|
-
success = await wm.simctlOpenUrl(params.url);
|
|
3602
|
+
success = await wm.simctlOpenUrl(params.url, iosUdid);
|
|
3507
3603
|
}
|
|
3508
3604
|
else {
|
|
3509
3605
|
const { navigateDesktopBrowser } = await import("./runners/window_manager.js");
|
|
@@ -3514,7 +3610,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3514
3610
|
break;
|
|
3515
3611
|
case "navigate_back":
|
|
3516
3612
|
if (tt === "android_emulator") {
|
|
3517
|
-
success = await wm.adbBackButton();
|
|
3613
|
+
success = await wm.adbBackButton(deviceId);
|
|
3518
3614
|
}
|
|
3519
3615
|
else if (tt === "browser") {
|
|
3520
3616
|
success = await bi.browserGoBack();
|
|
@@ -3542,7 +3638,13 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3542
3638
|
break;
|
|
3543
3639
|
case "swipe":
|
|
3544
3640
|
if (tt === "android_emulator" && params.x != null && params.y != null && params.x2 != null && params.y2 != null) {
|
|
3545
|
-
success = await wm.adbSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 300);
|
|
3641
|
+
success = await wm.adbSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 300, deviceId);
|
|
3642
|
+
}
|
|
3643
|
+
else if (tt === "ios_simulator" && params.x != null && params.y != null && params.x2 != null && params.y2 != null) {
|
|
3644
|
+
const r = await sim.iosSimSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 300, iosUdid);
|
|
3645
|
+
success = r.success;
|
|
3646
|
+
detail = `swipe from (${params.x},${params.y}) to (${params.x2},${params.y2}) — ${r.detail}`;
|
|
3647
|
+
break;
|
|
3546
3648
|
}
|
|
3547
3649
|
else if (params.x != null && params.y != null && params.x2 != null && params.y2 != null) {
|
|
3548
3650
|
success = await wm.dragDrop(params.x, params.y, params.x2, params.y2, params.duration_ms || 300);
|
|
@@ -3551,49 +3653,67 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3551
3653
|
break;
|
|
3552
3654
|
case "back_button":
|
|
3553
3655
|
if (tt === "android_emulator")
|
|
3554
|
-
success = await wm.adbBackButton();
|
|
3656
|
+
success = await wm.adbBackButton(deviceId);
|
|
3555
3657
|
detail = "back_button";
|
|
3556
3658
|
break;
|
|
3557
3659
|
case "home_button":
|
|
3558
3660
|
if (tt === "android_emulator")
|
|
3559
|
-
success = await wm.adbHomeButton();
|
|
3661
|
+
success = await wm.adbHomeButton(deviceId);
|
|
3560
3662
|
detail = "home_button";
|
|
3561
3663
|
break;
|
|
3562
3664
|
case "deep_link":
|
|
3563
3665
|
if (params.url) {
|
|
3564
3666
|
if (tt === "android_emulator")
|
|
3565
|
-
success = await wm.adbDeepLink(params.url);
|
|
3667
|
+
success = await wm.adbDeepLink(params.url, deviceId);
|
|
3566
3668
|
else if (tt === "ios_simulator")
|
|
3567
|
-
success = await wm.simctlOpenUrl(params.url);
|
|
3669
|
+
success = await wm.simctlOpenUrl(params.url, iosUdid);
|
|
3568
3670
|
}
|
|
3569
3671
|
detail = `deep_link "${params.url}"`;
|
|
3570
3672
|
break;
|
|
3571
3673
|
case "grant_permission":
|
|
3572
3674
|
if (tt === "android_emulator" && params.package_id && params.value) {
|
|
3573
|
-
success = await wm.adbPermission(params.package_id, params.value, params.grant !== false);
|
|
3675
|
+
success = await wm.adbPermission(params.package_id, params.value, params.grant !== false, deviceId);
|
|
3676
|
+
detail = `grant_permission "${params.value}"`;
|
|
3677
|
+
}
|
|
3678
|
+
else if (tt === "ios_simulator" && params.package_id && params.value) {
|
|
3679
|
+
// P1.2: simctl privacy grant|revoke <service> <bundle>
|
|
3680
|
+
success = await wm.simctlPermission(params.package_id, params.value, params.grant !== false, iosUdid);
|
|
3681
|
+
detail = `grant_permission "${params.value}" → simctl privacy ${params.grant !== false ? "grant" : "revoke"} (services: photos, location, microphone, contacts, calendar, …)`;
|
|
3682
|
+
}
|
|
3683
|
+
else {
|
|
3684
|
+
detail = `grant_permission "${params.value}" — needs package_id + value (permission/service name)`;
|
|
3574
3685
|
}
|
|
3575
|
-
detail = `grant_permission "${params.value}"`;
|
|
3576
3686
|
break;
|
|
3577
3687
|
case "rotate_device":
|
|
3578
3688
|
if (tt === "android_emulator") {
|
|
3579
|
-
success = await wm.adbRotate(params.orientation === "landscape");
|
|
3689
|
+
success = await wm.adbRotate(params.orientation === "landscape", deviceId);
|
|
3690
|
+
detail = `rotate_device ${params.orientation}`;
|
|
3691
|
+
}
|
|
3692
|
+
else if (tt === "ios_simulator") {
|
|
3693
|
+
// P1.2: no simctl rotation exists — drive the Simulator app's
|
|
3694
|
+
// Device menu shortcut (Cmd+Left/Right) via CGEvent.
|
|
3695
|
+
const r = await sim.iosSimRotate(params.orientation === "landscape" ? "landscape" : "portrait", iosUdid);
|
|
3696
|
+
success = r.success;
|
|
3697
|
+
detail = `rotate_device ${params.orientation} — ${r.detail}`;
|
|
3698
|
+
}
|
|
3699
|
+
else {
|
|
3700
|
+
detail = `rotate_device ${params.orientation}`;
|
|
3580
3701
|
}
|
|
3581
|
-
detail = `rotate_device ${params.orientation}`;
|
|
3582
3702
|
break;
|
|
3583
3703
|
case "biometric_auth":
|
|
3584
3704
|
if (tt === "ios_simulator") {
|
|
3585
|
-
success = await wm.simctlBiometric(params.accept !== false);
|
|
3705
|
+
success = await wm.simctlBiometric(params.accept !== false, iosUdid);
|
|
3586
3706
|
}
|
|
3587
3707
|
detail = `biometric_auth ${params.accept !== false ? "accept" : "reject"}`;
|
|
3588
3708
|
break;
|
|
3589
3709
|
case "launch_app":
|
|
3590
3710
|
if (tt === "android_emulator" && params.package_id) {
|
|
3591
|
-
const r = await import("./runners/base.js").then(m => m.runCommand("adb",
|
|
3711
|
+
const r = await import("./runners/base.js").then(m => m.runCommand("adb", wm.adbArgs(deviceId, "shell", "am", "start", "-n", params.package_id), process.cwd()));
|
|
3592
3712
|
success = r.exit_code === 0;
|
|
3593
3713
|
detail = `launch_app "${params.package_id}"`;
|
|
3594
3714
|
}
|
|
3595
3715
|
else if (tt === "ios_simulator" && params.package_id) {
|
|
3596
|
-
success = await wm.simctlLaunch(params.package_id);
|
|
3716
|
+
success = await wm.simctlLaunch(params.package_id, iosUdid);
|
|
3597
3717
|
detail = `launch_app "${params.package_id}"`;
|
|
3598
3718
|
}
|
|
3599
3719
|
else if (tt === "desktop") {
|
|
@@ -3624,21 +3744,83 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3624
3744
|
break;
|
|
3625
3745
|
case "clear_app_data":
|
|
3626
3746
|
if (tt === "android_emulator" && params.package_id) {
|
|
3627
|
-
success = await wm.adbClearData(params.package_id);
|
|
3747
|
+
success = await wm.adbClearData(params.package_id, deviceId);
|
|
3748
|
+
detail = `clear_app_data "${params.package_id}"`;
|
|
3749
|
+
}
|
|
3750
|
+
else if (tt === "ios_simulator" && params.package_id) {
|
|
3751
|
+
// P1.2: iOS has no `pm clear` — uninstall wipes data AND the
|
|
3752
|
+
// binary, so the app must be reinstalled before the next step.
|
|
3753
|
+
success = await wm.simctlUninstall(params.package_id, iosUdid);
|
|
3754
|
+
detail = success
|
|
3755
|
+
? `clear_app_data "${params.package_id}": app uninstalled (data wiped). REINSTALL required — re-run codeloop_run_journey or \`flutter run\`/xcodebuild install before interacting again.`
|
|
3756
|
+
: `clear_app_data "${params.package_id}": simctl uninstall failed — is the bundle id correct and the simulator booted?`;
|
|
3757
|
+
}
|
|
3758
|
+
else {
|
|
3759
|
+
detail = `clear_app_data "${params.package_id}"`;
|
|
3628
3760
|
}
|
|
3629
|
-
detail = `clear_app_data "${params.package_id}"`;
|
|
3630
3761
|
break;
|
|
3631
3762
|
case "mock_location":
|
|
3632
3763
|
if (tt === "android_emulator" && params.latitude != null && params.longitude != null) {
|
|
3633
|
-
|
|
3764
|
+
// P1.3: `adb emu geo` talks to the EMULATOR CONSOLE — it can
|
|
3765
|
+
// never work on physical hardware, so fail with a usable
|
|
3766
|
+
// directive instead of a cryptic adb error.
|
|
3767
|
+
if (!wm.isEmulatorSerial(deviceId)) {
|
|
3768
|
+
success = false;
|
|
3769
|
+
detail = `mock_location is emulator-only (\`adb emu geo\` drives the emulator console; "${deviceId}" is a physical device). On hardware: enable Developer options → "Select mock location app" and use a mock-location app, or test this journey on an emulator.`;
|
|
3770
|
+
break;
|
|
3771
|
+
}
|
|
3772
|
+
success = await wm.adbMockLocation(params.latitude, params.longitude, deviceId);
|
|
3773
|
+
}
|
|
3774
|
+
else if (tt === "ios_simulator" && params.latitude != null && params.longitude != null) {
|
|
3775
|
+
// P1.2: simctl location <udid> set <lat>,<lon>
|
|
3776
|
+
success = await wm.simctlSetLocation(params.latitude, params.longitude, iosUdid);
|
|
3634
3777
|
}
|
|
3635
3778
|
detail = `mock_location (${params.latitude},${params.longitude})`;
|
|
3636
3779
|
break;
|
|
3637
3780
|
case "simulate_network":
|
|
3638
3781
|
if (tt === "android_emulator" && params.value) {
|
|
3639
|
-
|
|
3782
|
+
// P1.3: `adb emu network` is emulator-console-only as well.
|
|
3783
|
+
if (!wm.isEmulatorSerial(deviceId)) {
|
|
3784
|
+
success = false;
|
|
3785
|
+
detail = `simulate_network is emulator-only (\`adb emu network\` drives the emulator console; "${deviceId}" is a physical device). On hardware: toggle airplane mode / Wi-Fi via adb shell, or test this journey on an emulator.`;
|
|
3786
|
+
break;
|
|
3787
|
+
}
|
|
3788
|
+
success = await wm.adbNetworkCondition(params.value, deviceId);
|
|
3789
|
+
detail = `simulate_network "${params.value}"`;
|
|
3790
|
+
}
|
|
3791
|
+
else if (tt === "ios_simulator") {
|
|
3792
|
+
// Honest unsupported: simctl has no network conditioner; the
|
|
3793
|
+
// status_bar override only changes the INDICATOR, not traffic.
|
|
3794
|
+
detail = `simulate_network is not supported on the iOS simulator (simctl has no network conditioner). Install Apple's "Network Link Conditioner" prefpane for host-level shaping, or use action "status_bar" to fake the indicator for screenshots.`;
|
|
3795
|
+
}
|
|
3796
|
+
else {
|
|
3797
|
+
detail = `simulate_network "${params.value}"`;
|
|
3798
|
+
}
|
|
3799
|
+
break;
|
|
3800
|
+
case "push_notification":
|
|
3801
|
+
// P1.2: simulated APNs push — iOS simulator only.
|
|
3802
|
+
if (tt === "ios_simulator" && params.package_id && params.value) {
|
|
3803
|
+
const r = await wm.simctlPush(params.package_id, params.value, iosUdid);
|
|
3804
|
+
success = r.success;
|
|
3805
|
+
detail = `push_notification → ${r.detail}`;
|
|
3806
|
+
}
|
|
3807
|
+
else if (tt === "android_emulator") {
|
|
3808
|
+
detail = `push_notification is iOS-simulator-only (simctl push). On Android, drive the app's own notification trigger or use \`adb shell am broadcast\` for app-specific receivers.`;
|
|
3809
|
+
}
|
|
3810
|
+
else {
|
|
3811
|
+
detail = `push_notification needs package_id (bundle id) + value (APNs JSON payload or plain alert text).`;
|
|
3812
|
+
}
|
|
3813
|
+
break;
|
|
3814
|
+
case "status_bar":
|
|
3815
|
+
// P1.2: simctl status_bar override for demo-clean screenshots.
|
|
3816
|
+
if (tt === "ios_simulator") {
|
|
3817
|
+
const r = await wm.simctlStatusBar(params.value ?? "", iosUdid);
|
|
3818
|
+
success = r.success;
|
|
3819
|
+
detail = `status_bar → ${r.detail}`;
|
|
3820
|
+
}
|
|
3821
|
+
else {
|
|
3822
|
+
detail = `status_bar is iOS-simulator-only (simctl status_bar). Value: JSON or "key=value,…" with time, dataNetwork, wifiMode, wifiBars, cellularMode, cellularBars, operatorName, batteryState, batteryLevel — or "clear".`;
|
|
3640
3823
|
}
|
|
3641
|
-
detail = `simulate_network "${params.value}"`;
|
|
3642
3824
|
break;
|
|
3643
3825
|
case "maestro_flow":
|
|
3644
3826
|
if (params.maestro_steps) {
|
|
@@ -3648,7 +3830,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3648
3830
|
if ("error" in genResult) {
|
|
3649
3831
|
return { success: false, action, detail: genResult.error };
|
|
3650
3832
|
}
|
|
3651
|
-
const runResult = await mg.runGeneratedFlow(genResult.flowPath, cwd);
|
|
3833
|
+
const runResult = await mg.runGeneratedFlow(genResult.flowPath, cwd, deviceId);
|
|
3652
3834
|
success = runResult.success;
|
|
3653
3835
|
detail = `maestro_flow (${params.maestro_steps.length} steps) → ${runResult.success ? "passed" : runResult.error}`;
|
|
3654
3836
|
}
|
|
@@ -3820,7 +4002,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
|
|
|
3820
4002
|
}
|
|
3821
4003
|
break;
|
|
3822
4004
|
default:
|
|
3823
|
-
detail = `Unknown action: "${action}". Available: click, double_click, right_click, hover, type, keystroke, hotkey, scroll, drag_drop, long_press, type_and_submit, type_and_tab, fill_form, select_option, toggle, upload_file, navigate_url, navigate_back, navigate_forward, wait, swipe, back_button, home_button, deep_link, grant_permission, rotate_device, biometric_auth, launch_app, clear_app_data, mock_location, simulate_network, maestro_flow, win_ui_inspect, win_ui_automate, sequence`;
|
|
4005
|
+
detail = `Unknown action: "${action}". Available: click, double_click, right_click, hover, type, keystroke, hotkey, scroll, drag_drop, long_press, type_and_submit, type_and_tab, fill_form, select_option, toggle, upload_file, navigate_url, navigate_back, navigate_forward, wait, swipe, back_button, home_button, deep_link, grant_permission, rotate_device, biometric_auth, launch_app, clear_app_data, mock_location, simulate_network, push_notification, status_bar, maestro_flow, win_ui_inspect, win_ui_automate, sequence`;
|
|
3824
4006
|
return { success: false, action, detail };
|
|
3825
4007
|
}
|
|
3826
4008
|
await trackUsage(apiKey, "interaction");
|