codeloop-mcp-server 0.1.47 → 0.1.48

package/dist/auth/critical_floors.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"critical_floors.d.ts","sourceRoot":"","sources":["../../src/auth/critical_floors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,MAAM,WAAW,aAAa;IAC5B,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,eAAe,EAAE,aAAa,EAqB1C,CAAC"}
+{"version":3,"file":"critical_floors.d.ts","sourceRoot":"","sources":["../../src/auth/critical_floors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,MAAM,WAAW,aAAa;IAC5B,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,eAAe,EAAE,aAAa,EAyB1C,CAAC"}

package/dist/auth/critical_floors.js

@@ -56,5 +56,9 @@ export const CRITICAL_FLOORS = [
         min_version: "0.1.47",
         reason: "Desktop-app capture-screenshot honesty + auto-detected target_app — pre-0.1.47 builds applied the desktop-app honesty refusal ONLY inside codeloop_verify's auto-capture path, so the standalone codeloop_capture_screenshot tool still silently grabbed the IDE when called without app_name during recording. Also pre-0.1.47, evidence.target_app required manual user setup — codeloop_init_project now auto-extracts it from .csproj AssemblyName / Xcode PRODUCT_NAME / AndroidManifest android:label so desktop projects work without manual config",
     },
+    {
+        min_version: "0.1.48",
+        reason: "Desktop interaction reliability — pre-0.1.48 builds rejected target_type: 'windows_desktop' / 'mac_desktop' / 'linux_desktop' with a hard Zod error (agents had to know to type 'desktop' instead), didn't translate window-relative click coordinates to screen-absolute (so coords computed from a captured window-screenshot missed targets by hundreds of pixels), had no codeloop_launch_app tool / auto-launch in start_recording (agents had to hand-roll Get-Process + Start-Process PowerShell every recording), refused agent-natural flat sequence step shapes like { action: 'click', x: 100, y: 200 }, and treated cross-run weak design_compare matches as critical 0% failures of the gate-run screen. All four broke the autonomous loop on Photometry-DB E2E session 8",
+    },
 ];
 //# sourceMappingURL=critical_floors.js.map

package/dist/auth/critical_floors.js.map

@@ -1 +1 @@
-{"version":3,"file":"critical_floors.js","sourceRoot":"","sources":["../../src/auth/critical_floors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AASH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,eAAe,GAAoB;IAC9C;QACE,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,8JAA8J;KACvK;IACD;QACE,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,yNAAyN;KAClO;IACD;QACE,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,oYAAoY;KAC7Y;IACD;QACE,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,sbAAsb;KAC/b;IACD;QACE,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,4hBAA4hB;KACriB;CACF,CAAC"}
+{"version":3,"file":"critical_floors.js","sourceRoot":"","sources":["../../src/auth/critical_floors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AASH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,eAAe,GAAoB;IAC9C;QACE,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,8JAA8J;KACvK;IACD;QACE,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,yNAAyN;KAClO;IACD;QACE,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,oYAAoY;KAC7Y;IACD;QACE,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,sbAAsb;KAC/b;IACD;QACE,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,4hBAA4hB;KACriB;IACD;QACE,WAAW,EAAE,QAAQ;QACrB,MAAM,EAAE,yvBAAyvB;KAClwB;CACF,CAAC"}

package/dist/index.js

@@ -135,6 +135,12 @@ const server = new McpServer({
 async function withAuth(fn, tracker) {
     const started = Date.now();
     let outcome = { success: false };
+    // Photometry-DB E2E 8 regression: when the agent passes
+    // `project_dir`/`workspace_root` to a tool, remember that dir for
+    // the lifetime of this MCP process so the init-hint check no
+    // longer false-positives later calls where the call site didn't
+    // forward `dir` to `withInitHint`.
+    rememberInitializedDir(tracker?.cwd);
     try {
         // Local / self-hosted mode (CODELOOP_MODE=local): skip API-key validation
         // entirely. All cloud-side checks are bypassed; usage events are queued
@@ -286,6 +292,26 @@ function buildVersionBanner() {
         text: `[CodeLoop server v${v}]`,
     };
 }
+/**
+ * Last project directory observed to be initialized via an actual
+ * tool call (set by `withAuth`'s tracker — see the .cwd field). The
+ * Photometry-DB E2E session 8 regression was that the agent passed
+ * `project_dir: "D:\\Work\\Photometry DB"` (which IS initialized) on
+ * every call, but the init hint was checking the server's startup
+ * `projectDir` (C:\Users\jiq on Windows) and incorrectly prepending
+ * "This project has not been initialized" to every response. With
+ * this cache, the first authenticated call that hits an initialized
+ * dir silences the hint for the rest of the session — independent of
+ * whether the specific call site forwarded `dir` to `withInitHint`.
+ */
+let lastInitializedDir = null;
+function rememberInitializedDir(dir) {
+    if (!dir)
+        return;
+    if (isProjectInitialized(dir)) {
+        lastInitializedDir = dir;
+    }
+}
 function withInitHint(content, dir) {
     // Order matters:
     //   1. Update notice (most actionable signal — CRITICAL stays at top).
@@ -296,7 +322,22 @@ function withInitHint(content, dir) {
     const banner = buildVersionBanner();
     const withUpdate = withUpdateNotice(content);
     const head = [];
-    if (!isProjectInitialized(dir || projectDir)) {
+    // Candidate dirs in priority order:
+    //   1. Explicit `dir` arg from the call site (when the handler
+    //      bothered to thread it through — the verify / capture /
+    //      design_compare etc. handlers do).
+    //   2. `lastInitializedDir` — any dir the agent has already passed
+    //      to *any* CodeLoop tool this session that resolved to an
+    //      initialized config. This catches the long tail of handlers
+    //      that don't forward `dir` (e.g. `recommend_tool`,
+    //      `release_readiness`, `section_status`) so they no longer
+    //      false-positive the hint after init has succeeded once.
+    //   3. The MCP server's startup `projectDir` (often the user's
+    //      home folder on Windows / Cursor — see CODELOOP_PROJECT_DIR
+    //      auto-injection notes in setup-project.ts).
+    const candidates = [dir, lastInitializedDir, projectDir].filter((d) => typeof d === "string" && d.length > 0);
+    const anyInitialized = candidates.some((d) => isProjectInitialized(d));
+    if (!anyInitialized) {
         head.push({ type: "text", text: INIT_HINT });
     }
     const result = [...head, ...withUpdate];
@@ -305,6 +346,42 @@ function withInitHint(content, dir) {
     return result;
 }
 const TOOL_BOOTSTRAP = "[CodeLoop] You MUST call codeloop_verify after every code change. If .codeloop/config.json is missing, call codeloop_init_project FIRST.\n\n";
+/**
+ * Zod preprocessor for `target_type` that accepts common synonyms.
+ *
+ * Photometry-DB E2E 8 regression: the agent (reasonably) tried
+ * `target_type: "windows_desktop"` because the project is a WPF
+ * Windows app, and got a hard schema-validation rejection. Same
+ * pattern for `mac_desktop`, `linux_desktop`, `web`, `android`,
+ * `ios`. Coerce them to the canonical 4-value enum so the agent
+ * isn't blocked by a thin naming convention.
+ */
+const TARGET_TYPE_SYNONYMS = {
+    desktop: "desktop",
+    windows_desktop: "desktop",
+    win_desktop: "desktop",
+    win32_desktop: "desktop",
+    mac_desktop: "desktop",
+    macos_desktop: "desktop",
+    osx_desktop: "desktop",
+    linux_desktop: "desktop",
+    native_desktop: "desktop",
+    browser: "browser",
+    web: "browser",
+    webapp: "browser",
+    chrome: "browser",
+    android_emulator: "android_emulator",
+    android: "android_emulator",
+    ios_simulator: "ios_simulator",
+    ios: "ios_simulator",
+};
+function normalizeTargetType(v) {
+    if (typeof v !== "string")
+        return v;
+    const key = v.toLowerCase().trim();
+    return TARGET_TYPE_SYNONYMS[key] ?? v;
+}
+const targetTypeSchema = z.preprocess(normalizeTargetType, z.enum(["desktop", "browser", "android_emulator", "ios_simulator"]));
 // ── Implemented Tools ────────────────────────────────────────────
 server.tool("codeloop_verify", TOOL_BOOTSTRAP + `Run the CodeLoop verification suite on the current project. Use this tool when:
 - You have implemented or modified code and need to check if it works correctly
@@ -1136,8 +1213,28 @@ Returns: confirmation + the captured image as an MCP ImageContent block so you c
         const cfg = loadConfig(cwd);
         const targetApp = params.app_name ?? cfg.evidence?.target_app;
         const result = await captureScreenshot(screenshotsDir, params.screen_name, targetApp, undefined, { desktopAppMode: isDesktopAppProject });
+        // Photometry-DB E2E 8 follow-on: when we capture a desktop app
+        // window, also resolve its on-screen bounds so the agent can
+        // (a) compute window-relative coords from the returned image
+        //     dimensions, and
+        // (b) pass coords:"window" to codeloop_interact to get them
+        //     translated to screen-absolute automatically.
+        // Without this, agents reasoned from a downscaled vision view
+        // of the image and clicked tens or hundreds of pixels off the
+        // intended target.
+        let windowBounds = null;
+        if (isDesktopAppProject && targetApp && result.captured) {
+            try {
+                const wm = await import("./runners/window_manager.js");
+                const b = await wm.getWindowBounds(targetApp);
+                if (b && b.width > 0 && b.height > 0) {
+                    windowBounds = { x: b.x, y: b.y, width: b.width, height: b.height };
+                }
+            }
+            catch { /* best-effort */ }
+        }
         await trackUsage(apiKey, "visual_review");
-        return result;
+        return { ...result, windowBounds };
     }, { tool: "codeloop_capture_screenshot", cwd: (params.project_dir || params.workspace_root || projectDir), input: params });
     if (typeof authResult === "object" && authResult !== null && "error" in authResult) {
         return { content: [{ type: "text", text: JSON.stringify(authResult, null, 2) }] };
@@ -1145,12 +1242,18 @@ Returns: confirmation + the captured image as an MCP ImageContent block so you c
     const result = authResult;
     const content = [];
     if (result.captured && result.paths.length > 0) {
-        content.push({ type: "text", text: JSON.stringify({
-                captured: true,
-                screen_name: params.screen_name,
-                path: result.paths[0],
-                method: result.method,
-            }, null, 2) });
+        const payload = {
+            captured: true,
+            screen_name: params.screen_name,
+            path: result.paths[0],
+            method: result.method,
+        };
+        if (result.windowBounds) {
+            payload.window_bounds = result.windowBounds;
+            payload.coordinate_hint =
+                "This screenshot captures the named window. When you compute click coordinates from the image, pass them to codeloop_interact with `coords: \"window\"` so they're translated to screen-absolute automatically. (Default `coords: \"auto\"` also works when the coord fits inside the window — but `\"window\"` is unambiguous.)";
+        }
+        content.push({ type: "text", text: JSON.stringify(payload, null, 2) });
         const data = readImageAsBase64(result.paths[0]);
         if (data) {
             content.push({ type: "image", data, mimeType: mimeForPath(result.paths[0]) });
@@ -1321,6 +1424,45 @@ After recording, call codeloop_interaction_replay to extract frames and analyze
         content: withInitHint([{ type: "text", text: JSON.stringify(result, null, 2) }]),
     };
 });
+server.tool("codeloop_launch_app", TOOL_BOOTSTRAP + `Launch a desktop application by name from a project's build output. Use when:
+- You need the app under test to be running before codeloop_start_recording / codeloop_interact.
+- The agent doesn't know where the executable lives and shouldn't have to hand-roll Start-Process / open -a.
+
+This tool is the canonical replacement for hand-coding PowerShell Start-Process / osascript / xdg-open
+from the agent. For Android / iOS, use codeloop_interact action="launch_app" with package_id instead.
+
+Search order on Windows:  publish/**/*.exe → bin/Release/**/*.exe → bin/Debug/**/*.exe (newest first).
+On macOS:                  publish/**/*.app → build/**/*.app → /Applications/<name>.app → open -a.
+On Linux:                  build/**/<name>  → bin/**/<name>  → dist/**/<name>.
+
+If app_name is omitted, falls back to evidence.target_app from .codeloop/config.json (auto-detected at
+init for .NET/Xcode/Android projects via detect-target-app).`, {
+    app_name: z.string().optional().describe("Window title / executable name of the app to launch. Defaults to evidence.target_app from .codeloop/config.json. Required if target_app is unset."),
+    project_dir: z.string().optional().describe("Absolute path to the project root. Defaults to CODELOOP_PROJECT_DIR / discovered project dir."),
+    workspace_root: z.string().optional().describe("[Alias for project_dir]"),
+}, async (params) => {
+    const authResult = await withAuth(async () => {
+        const wm = await import("./runners/window_manager.js");
+        const { loadConfig } = await import("./config.js");
+        const cwd = (params.project_dir || params.workspace_root || projectDir);
+        const cfg = loadConfig(cwd);
+        const appName = params.app_name || cfg.evidence?.target_app;
+        if (!appName) {
+            return {
+                launched: false,
+                reason: "No app_name provided and evidence.target_app is unset in .codeloop/config.json. Set it (e.g. \"evidence\": { \"target_app\": \"My App\" }) or pass app_name explicitly. For .NET/Xcode/Android projects, codeloop_init_project auto-detects this — re-run init or edit the config by hand.",
+            };
+        }
+        const r = await wm.launchDesktopApp(appName, cwd);
+        return { app_name: appName, ...r };
+    }, { tool: "codeloop_launch_app", cwd: (params.project_dir || params.workspace_root || projectDir), input: params });
+    if (typeof authResult === "object" && authResult !== null && "error" in authResult) {
+        return { content: [{ type: "text", text: JSON.stringify(authResult, null, 2) }] };
+    }
+    return {
+        content: withInitHint([{ type: "text", text: JSON.stringify(authResult, null, 2) }], params.project_dir || params.workspace_root),
+    };
+});
 server.tool("codeloop_start_recording", TOOL_BOOTSTRAP + `Start recording the app window in the background. The app is brought to the front automatically
 (un-minimized if needed). Recording continues while you interact with the app. Call codeloop_stop_recording when done.
 This is the PREFERRED recording method because it lets you actively operate the app during capture.
@@ -1347,11 +1489,12 @@ Flow: start_recording → codeloop_interact with ALL app elements → stop_recor
 Supports desktop apps, Android emulator, iOS Simulator, and browser targets.
 Multi-monitor: on macOS, automatically detects which screen the app window is on.
 App logs (stdout, logcat, simctl log) are automatically captured alongside the video.`, {
-    app_name: z.string().describe("The name of the app to record (used to find and focus its window)"),
+    app_name: z.string().optional().describe("The name of the app to record (used to find and focus its window). For desktop projects, defaults to evidence.target_app from .codeloop/config.json — set during init via detect-target-app for .NET/Xcode/Android projects, or settable manually."),
     run_id: z.string().optional().describe("Existing run ID to store the video in"),
     max_duration_seconds: z.number().default(120).describe("Safety timeout — recording stops automatically after this many seconds"),
-    target_type: z.enum(["desktop", "android_emulator", "ios_simulator", "browser"]).optional()
+    target_type: targetTypeSchema.optional()
         .describe("Capture method. Auto-detected from project if omitted. desktop=ffmpeg screen, android_emulator=adb screenrecord, ios_simulator=simctl recordVideo, browser=ffmpeg/Playwright"),
+    auto_launch: z.boolean().default(true).describe("When target_type=desktop and the app isn't already running, auto-launch it from the project's build output via evidence.target_app. Set false to skip (e.g. when the app is started by another process)."),
     project_dir: z.string().optional().describe("Absolute path to the project root. Defaults to CODELOOP_PROJECT_DIR env var or auto-discovered project directory. MUST be an actual project folder — passing the user's home directory is rejected. If your IDE launches the MCP server from the wrong cwd (common on Windows where Cursor uses C:\\Users\\<name> as cwd), set CODELOOP_PROJECT_DIR or pass this param explicitly."),
     workspace_root: z.string().optional().describe("[Alias for project_dir] Same semantics; accepted because many agents reach for this conventional name. Pass either `project_dir` OR `workspace_root` — they're equivalent."),
 }, async (params) => {
@@ -1359,6 +1502,7 @@ App logs (stdout, logcat, simctl log) are automatically captured alongside the v
         const { startBackgroundRecording } = await import("./runners/video_recorder.js");
         const { createRunDir, getRunDir, getArtifactsBaseDir } = await import("./evidence/artifacts.js");
         const { detectTargetType } = await import("./runners/platform_detect.js");
+        const { loadConfig } = await import("./config.js");
         const cwd = (params.project_dir || params.workspace_root || projectDir);
         let videosDir;
         if (params.run_id) {
@@ -1370,7 +1514,11 @@ App logs (stdout, logcat, simctl log) are automatically captured alongside the v
             videosDir = join(runDir, "videos");
         }
         const targetType = params.target_type || (await detectTargetType(cwd));
+        const cfg = loadConfig(cwd);
         let appName = params.app_name;
+        if (!appName && (targetType === "desktop")) {
+            appName = cfg.evidence?.target_app;
+        }
         if (targetType === "browser") {
             const bi = await import("./runners/browser_interaction.js");
             await bi.ensureBrowserPage();
@@ -1379,7 +1527,42 @@ App logs (stdout, logcat, simctl log) are automatically captured alongside the v
                 appName = pwAppName;
             }
         }
-        const result = await startBackgroundRecording(videosDir, appName, params.max_duration_seconds, targetType);
+        // Photometry-DB E2E 8: agents spent many turns manually
+        // probing `Get-Process` / `Start-Process` to launch the app
+        // because nothing in CodeLoop did it for them. Now, when
+        // recording a desktop app, we auto-launch from the build
+        // output if the app isn't already running.
+        const wm = await import("./runners/window_manager.js");
+        let autoLaunchSummary;
+        if (targetType === "desktop" && params.auto_launch !== false && appName) {
+            try {
+                const bounds = await wm.getWindowBounds(appName);
+                if (!bounds) {
+                    const r = await wm.launchDesktopApp(appName, cwd);
+                    autoLaunchSummary = {
+                        attempted: true,
+                        launched: r.launched,
+                        command: r.command,
+                        reason: r.reason,
+                    };
+                    if (r.launched) {
+                        // Give the window time to appear / paint before the
+                        // recorder starts capturing frames.
+                        await new Promise((res) => setTimeout(res, 2000));
+                    }
+                }
+                else {
+                    autoLaunchSummary = { attempted: false, launched: true, reason: "already running" };
+                }
+            }
+            catch (e) {
+                autoLaunchSummary = { attempted: true, launched: false, reason: e.message };
+            }
+        }
+        const result = await startBackgroundRecording(videosDir, appName ?? "", params.max_duration_seconds, targetType);
+        if (autoLaunchSummary) {
+            result.auto_launch = autoLaunchSummary;
+        }
         await trackUsage(apiKey, "visual_review");
         return result;
     }, { tool: "codeloop_start_recording", cwd: (params.project_dir || params.workspace_root || projectDir), input: params });
@@ -1955,8 +2138,8 @@ MANDATORY for web apps: You MUST type into form fields, fill login/signup forms,
 validation errors, and click submit buttons. Just navigating pages is NOT enough.
 Wait 1-2 seconds between interactions so video frames capture state changes.`, {
     action: z.string().describe("Action to perform: click, double_click, right_click, hover, type, keystroke, hotkey, scroll, drag_drop, long_press, type_and_submit, type_and_tab, fill_form, select_option, toggle, upload_file, navigate_url, navigate_back, navigate_forward, wait, sequence, swipe, back_button, home_button, deep_link, grant_permission, rotate_device, biometric_auth, launch_app, clear_app_data, mock_location, simulate_network, maestro_flow, win_ui_inspect, win_ui_automate"),
-    target_type: z.enum(["desktop", "browser", "android_emulator", "ios_simulator"]).optional()
-        .describe("Interaction target. Auto-detected if omitted."),
+    target_type: targetTypeSchema.optional()
+        .describe("Interaction target. Auto-detected if omitted. Accepts synonyms: `windows_desktop`/`mac_desktop`/`linux_desktop` → `desktop`; `web` → `browser`; `android` → `android_emulator`; `ios` → `ios_simulator`."),
     x: z.number().optional().describe("X coordinate for click/scroll/drag/swipe"),
     y: z.number().optional().describe("Y coordinate for click/scroll/drag/swipe"),
     x2: z.number().optional().describe("End X for drag_drop/swipe"),
@@ -1987,7 +2170,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
         action: z.string(),
         params: z.record(z.unknown()).optional(),
         delay_ms: z.number().optional(),
-    })).optional().describe("Steps for sequence action"),
+    }).passthrough()).optional().describe("Steps for sequence action. Accepts BOTH nested form `{ action, params: { x, y, … }, delay_ms? }` and flat form `{ action, x, y, … }` — the flat form is what agents naturally write (mirrors the top-level codeloop_interact shape). Supports inside desktop sequences: click, double_click, right_click, hover, scroll, type, hotkey, keystroke, navigate_url, wait, win_ui_automate."),
     maestro_steps: z.array(z.string()).optional().describe("High-level steps for maestro_flow"),
     automation_action: z.enum(["invoke", "setValue", "toggle", "select", "scroll"]).optional()
         .describe("For win_ui_automate"),
@@ -1997,6 +2180,7 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
     description: z.string().optional().describe("[Alias for intent] Same semantics."),
     purpose: z.string().optional().describe("[Alias for intent] Same semantics."),
     step: z.string().optional().describe("Plan-step name when this interaction is driving a codeloop_plan_user_journey arc (e.g. 'edit', 'delete', 'create', 'save', 'verify'). Logged alongside `intent` and read by the CRUD classifier."),
+    coords: z.enum(["auto", "window", "screen"]).optional().describe("How to interpret x/y for desktop click/double_click/right_click/hover/scroll/drag/long_press. `auto` (default): if `app_name` resolves to a visible window AND (x, y) fits inside the window's client area, treat as window-relative and auto-offset by the window origin; otherwise leave as raw screen-absolute coords. `window`: ALWAYS add the window origin offset (errors if the window isn't found). `screen`: ALWAYS pass through (legacy behaviour, matches CGEvent / user32.dll / xdotool semantics). Fixes the Photometry-DB E2E 8 failure mode where the agent captured a 1600×900 window screenshot, computed click coords against the image, and missed the sidebar because the window's actual top-left was (286, 286) on a 5120×1440 screen."),
     project_dir: z.string().optional().describe("Absolute path to project root."),
     workspace_root: z.string().optional().describe("[Alias for project_dir] Pass either; they're equivalent."),
 }, async (params) => {
@@ -2026,13 +2210,58 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
             await bi.ensureBrowserPage();
         }
         // Bring the app to front before desktop interactions (non-browser, non-mobile).
+        let windowOriginOffset = null;
         if (tt === "desktop") {
             const appName = params.app_name || vr.getActiveRecordingAppName();
             if (appName && action !== "wait") {
                 await wm.bringAppToFront(appName);
                 await new Promise(r => setTimeout(r, 300));
+                // Photometry-DB E2E 8: agents commonly compute click coords
+                // from a window-cropped screenshot (which is what
+                // codeloop_capture_screenshot returns when `app_name` is
+                // set), then pass those coords to codeloop_interact —
+                // which expects raw SCREEN coordinates. On a multi-monitor
+                // / DPI-scaled setup that mismatch silently dropped clicks
+                // 100s of pixels off-target. When `coords` is `auto` (the
+                // default) we look up the window's actual screen origin
+                // and add it to x/y, but ONLY if (x, y) fits inside the
+                // window — that keeps legacy callers passing raw screen
+                // coords working unchanged. `coords: "window"` forces the
+                // offset; `coords: "screen"` opts out.
+                const coordsMode = params.coords ?? "auto";
+                if (coordsMode !== "screen") {
+                    try {
+                        const b = await wm.getWindowBounds(appName);
+                        if (b && b.width > 0 && b.height > 0) {
+                            windowOriginOffset = { dx: b.x, dy: b.y, width: b.width, height: b.height };
+                        }
+                    }
+                    catch { /* best-effort */ }
+                }
             }
         }
+        // Helper used by every coordinate-driven desktop action below.
+        const translateXY = (x, y) => {
+            if (tt !== "desktop" || x == null || y == null || !windowOriginOffset) {
+                return { x, y };
+            }
+            const mode = params.coords ?? "auto";
+            if (mode === "screen")
+                return { x, y };
+            if (mode === "window") {
+                return { x: x + windowOriginOffset.dx, y: y + windowOriginOffset.dy };
+            }
+            // auto: if (x, y) fits inside the window's client area,
+            // assume the agent computed against a window-cropped
+            // screenshot and add the origin. Otherwise pass through
+            // (likely a raw screen coord from a manual workflow).
+            const inside = x >= 0 && x <= windowOriginOffset.width &&
+                y >= 0 && y <= windowOriginOffset.height;
+            if (inside) {
+                return { x: x + windowOriginOffset.dx, y: y + windowOriginOffset.dy };
+            }
+            return { x, y };
+        };
         switch (action) {
             case "click":
                 if (tt === "browser" && params.selector) {
@@ -2054,7 +2283,8 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
                     }
                 }
                 else if (params.x != null && params.y != null) {
-                    success = await wm.clickAtPosition(params.x, params.y);
+                    const t = translateXY(params.x, params.y);
+                    success = await wm.clickAtPosition(t.x, t.y);
                 }
                 detail = `click at ${params.selector || `(${params.x},${params.y})`}`;
                 break;
@@ -2063,7 +2293,8 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
                     success = await bi.browserDoubleClick(params.selector);
                 }
                 else if (params.x != null && params.y != null) {
-                    success = await wm.doubleClickAtPosition(params.x, params.y);
+                    const t = translateXY(params.x, params.y);
+                    success = await wm.doubleClickAtPosition(t.x, t.y);
                 }
                 detail = `double_click at ${params.selector || `(${params.x},${params.y})`}`;
                 break;
@@ -2072,7 +2303,8 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
                     success = await bi.browserRightClick(params.selector);
                 }
                 else if (params.x != null && params.y != null) {
-                    success = await wm.rightClickAtPosition(params.x, params.y);
+                    const t = translateXY(params.x, params.y);
+                    success = await wm.rightClickAtPosition(t.x, t.y);
                 }
                 detail = `right_click at ${params.selector || `(${params.x},${params.y})`}`;
                 break;
@@ -2081,7 +2313,8 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
                     success = await bi.browserHover(params.selector);
                 }
                 else if (params.x != null && params.y != null) {
-                    success = await wm.hoverAtPosition(params.x, params.y);
+                    const t = translateXY(params.x, params.y);
+                    success = await wm.hoverAtPosition(t.x, t.y);
                 }
                 detail = `hover at ${params.selector || `(${params.x},${params.y})`}`;
                 break;
@@ -2166,7 +2399,8 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
                     success = await wm.simctlSwipe(sx, sy, ex, ey);
                 }
                 else {
-                    success = await wm.scrollAtPosition(params.x || 500, params.y || 400, params.direction || "down", params.amount || 3);
+                    const t = translateXY(params.x || 500, params.y || 400);
+                    success = await wm.scrollAtPosition(t.x, t.y, params.direction || "down", params.amount || 3);
                 }
                 detail = `scroll ${params.direction || "down"}`;
                 break;
@@ -2179,7 +2413,9 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
                         success = await wm.adbSwipe(params.x, params.y, params.x2, params.y2, params.duration_ms || 500);
                     }
                     else {
-                        success = await wm.dragDrop(params.x, params.y, params.x2, params.y2, params.duration_ms || 500);
+                        const a = translateXY(params.x, params.y);
+                        const b = translateXY(params.x2, params.y2);
+                        success = await wm.dragDrop(a.x, a.y, b.x, b.y, params.duration_ms || 500);
                     }
                 }
                 detail = `drag_drop`;
@@ -2189,7 +2425,8 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
                     success = await wm.adbLongPress(params.x, params.y, params.duration_ms || 1000);
                 }
                 else if (params.x != null && params.y != null) {
-                    success = await wm.longPressAtPosition(params.x, params.y, params.duration_ms || 1000);
+                    const t = translateXY(params.x, params.y);
+                    success = await wm.longPressAtPosition(t.x, t.y, params.duration_ms || 1000);
                 }
                 detail = `long_press at (${params.x},${params.y})`;
                 break;
@@ -2371,11 +2608,37 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
                 if (tt === "android_emulator" && params.package_id) {
                     const r = await import("./runners/base.js").then(m => m.runCommand("adb", ["shell", "am", "start", "-n", params.package_id], process.cwd()));
                     success = r.exit_code === 0;
+                    detail = `launch_app "${params.package_id}"`;
                 }
                 else if (tt === "ios_simulator" && params.package_id) {
                     success = await wm.simctlLaunch(params.package_id);
+                    detail = `launch_app "${params.package_id}"`;
+                }
+                else if (tt === "desktop") {
+                    // Photometry-DB E2E 8: desktop launch was completely
+                    // missing — agents had to hand-roll PowerShell
+                    // Start-Process / `open -a` calls. Now resolves via
+                    // evidence.target_app from the project config when
+                    // app_name is omitted.
+                    const { loadConfig } = await import("./config.js");
+                    const cfg = loadConfig(cwd);
+                    const appName = params.app_name || cfg.evidence?.target_app;
+                    if (!appName) {
+                        success = false;
+                        detail = "launch_app desktop: no app_name provided and evidence.target_app is unset in .codeloop/config.json. Set it (e.g. \"target_app\": \"Photometry DB\") or pass app_name explicitly.";
+                    }
+                    else {
+                        const r = await wm.launchDesktopApp(appName, cwd);
+                        success = r.launched;
+                        detail = r.launched
+                            ? `launch_app "${appName}" via ${r.command}${r.pid ? ` (pid ${r.pid})` : ""}`
+                            : `launch_app "${appName}" failed: ${r.reason || "unknown error"}`;
+                    }
+                }
+                else {
+                    success = false;
+                    detail = `launch_app "${params.package_id || params.app_name || ""}": target ${tt} not supported in this action`;
                 }
-                detail = `launch_app "${params.package_id}"`;
                 break;
             case "clear_app_data":
                 if (tt === "android_emulator" && params.package_id) {
@@ -2439,7 +2702,22 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
                         if (step.delay_ms)
                             await new Promise(r => setTimeout(r, step.delay_ms));
                         const stepAction = step.action;
-                        const sp = (step.params || {});
+                        // Photometry-DB E2E 8: agents naturally write steps in
+                        // FLAT form `{ action, x, y, ms, … }` because that
+                        // mirrors the top-level codeloop_interact shape. The
+                        // schema documented the NESTED form `{ action,
+                        // params: { … } }`. Now we accept both: prefer
+                        // `step.params` if present, otherwise fall back to the
+                        // step object itself minus the wrapper keys.
+                        const stepObj = step;
+                        const nested = (step.params || {});
+                        const sp = Object.keys(nested).length > 0
+                            ? nested
+                            : Object.fromEntries(Object.entries(stepObj).filter(([k]) => k !== "action" && k !== "params" && k !== "delay_ms"));
+                        // Convenient aliases: agents wrote `ms` for wait
+                        // duration in the log; accept that as `duration_ms`.
+                        if (sp.ms != null && sp.duration_ms == null)
+                            sp.duration_ms = sp.ms;
                         let stepOk = false;
                         let stepReason;
                         try {
@@ -2447,7 +2725,20 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
                                 stepOk = await bi.browserClick(sp.selector);
                             }
                             else if (stepAction === "click" && sp.x != null && sp.y != null) {
-                                stepOk = await wm.clickAtPosition(sp.x, sp.y);
+                                const t = translateXY(sp.x, sp.y);
+                                stepOk = await wm.clickAtPosition(t.x, t.y);
+                            }
+                            else if (stepAction === "double_click" && tt !== "browser" && sp.x != null && sp.y != null) {
+                                const t = translateXY(sp.x, sp.y);
+                                stepOk = await wm.doubleClickAtPosition(t.x, t.y);
+                            }
+                            else if (stepAction === "right_click" && tt !== "browser" && sp.x != null && sp.y != null) {
+                                const t = translateXY(sp.x, sp.y);
+                                stepOk = await wm.rightClickAtPosition(t.x, t.y);
+                            }
+                            else if (stepAction === "hover" && tt !== "browser" && sp.x != null && sp.y != null) {
+                                const t = translateXY(sp.x, sp.y);
+                                stepOk = await wm.hoverAtPosition(t.x, t.y);
                             }
                             else if (stepAction === "type" && tt === "browser" && sp.selector && sp.text) {
                                 stepOk = await bi.browserType(sp.selector, sp.text);
@@ -2483,9 +2774,13 @@ Wait 1-2 seconds between interactions so video frames capture state changes.`, {
                                 stepOk = tt === "browser" ? await bi.browserKeystroke(sp.key) : await wm.sendKeyByName(sp.key);
                             }
                             else if (stepAction === "scroll") {
-                                stepOk = tt === "browser"
-                                    ? await bi.browserScroll(sp.direction || "down", sp.amount || 300)
-                                    : await wm.scrollAtPosition(sp.x || 500, sp.y || 400, sp.direction || "down", sp.amount || 3);
+                                if (tt === "browser") {
+                                    stepOk = await bi.browserScroll(sp.direction || "down", sp.amount || 300);
+                                }
+                                else {
+                                    const t = translateXY(sp.x || 500, sp.y || 400);
+                                    stepOk = await wm.scrollAtPosition(t.x, t.y, sp.direction || "down", sp.amount || 3);
+                                }
                             }
                             else if (stepAction === "wait") {
                                 await new Promise(r => setTimeout(r, sp.duration_ms || 1000));