@houwert/conductor 0.33.2 → 0.33.3

package/dist/commands/screenshot.js

@@ -5,6 +5,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HELP = void 0;
 exports.captureScreen = captureScreen;
+exports.iosHierarchySize = iosHierarchySize;
+exports.cropRect = cropRect;
 exports.screenshot = screenshot;
 exports.HELP = `  take-screenshot [<element>] [--output <path>] [--full-page] [--display <panel>]
                                        Take screenshot (--full-page: web only, capture entire scrollable page)
@@ -75,7 +77,8 @@ async function captureScreen(driver, opts, displayOverride) {
     const primary = displays.find((d) => d.primary);
     // Nothing to redirect: the driver already captures the primary panel.
     if (choice.displayId === null || (primary && choice.displayId === primary.displayId)) {
-        return { buffer: await driver.screenshot(opts), redirected: false };
+        const live = choice.displayId === null || primary?.active !== false;
+        return { buffer: await driver.screenshot(opts), redirected: false, live };
     }
     const file = path_1.default.join(os_1.default.tmpdir(), `conductor-shot-${Date.now()}.png`);
     try {
@@ -88,7 +91,13 @@ async function captureScreen(driver, opts, displayOverride) {
             String(choice.displayId),
             file,
         ]);
-        return { buffer: await promises_1.default.readFile(file), redirected: true };
+        const panel = displays.find((d) => d.displayId === choice.displayId);
+        return {
+            buffer: await promises_1.default.readFile(file),
+            redirected: true,
+            scale: panel?.pointScale,
+            live: panel?.active,
+        };
     }
     catch (err) {
         throw new Error(`could not capture display ${choice.displayId}: ${err instanceof Error ? err.message : String(err)}`);
@@ -97,6 +106,38 @@ async function captureScreen(driver, opts, displayOverride) {
         await promises_1.default.unlink(file).catch(() => { });
     }
 }
+/**
+ * The size of the coordinate space the view hierarchy reports, in its own
+ * units, for an iOS capture.
+ *
+ * `deviceInfo()` describes `XCUIScreen.main`, which is the wrong panel for a
+ * redirected capture: on an unfolded foldable the hierarchy is in the inner
+ * panel's points while deviceInfo still reports the cover's. Fall back to the
+ * captured image divided by that panel's own scale, which is exact.
+ */
+function iosHierarchySize(capture, png, deviceInfo) {
+    if (capture.redirected && capture.scale) {
+        return { width: png.width / capture.scale, height: png.height / capture.scale };
+    }
+    return { width: deviceInfo.widthPoints, height: deviceInfo.heightPoints };
+}
+/**
+ * Map element bounds onto screenshot pixels. The margin is in the same logical
+ * units as the bounds (points on iOS, pixels on Android/Web — what `inspect`
+ * prints), so it scales alongside them.
+ */
+function cropRect(bounds, hierarchy, png, margin) {
+    const scaleX = hierarchy.width > 0 ? png.width / hierarchy.width : 1;
+    const scaleY = hierarchy.height > 0 ? png.height / hierarchy.height : 1;
+    const marginX = margin * scaleX;
+    const marginY = margin * scaleY;
+    return {
+        x: Math.round(bounds.x * scaleX - marginX),
+        y: Math.round(bounds.y * scaleY - marginY),
+        width: Math.round(bounds.width * scaleX + marginX * 2),
+        height: Math.round(bounds.height * scaleY + marginY * 2),
+    };
+}
 async function screenshot(outputPath, opts = {}, sessionName = 'default', fullPage = false, query = '', flags = {}) {
     const timestamp = Date.now();
     const defaultName = `screenshot-${timestamp}.png`;
@@ -127,12 +168,12 @@ async function screenshot(outputPath, opts = {}, sessionName = 'default', fullPa
                 : '';
     const margin = flags.margin ?? DEFAULT_MARGIN_PX;
     const result = await (0, runner_js_1.runDirect)(async (driver) => {
-        const { buffer: buf, redirected } = await captureScreen(driver, { fullPage }, flags.display);
+        const { buffer: buf, redirected, scale, live, } = await captureScreen(driver, { fullPage }, flags.display);
         let out = buf;
-        if (sel && redirected) {
-            throw new Error('cropping to an element is not supported on this display yet — the panel is ' +
-                'rotated relative to the accessibility coordinate space. Re-run with ' +
-                '--display cover, or fold the device, to crop against the main panel.');
+        if (sel && live === false) {
+            throw new Error(`cannot crop to ${label} on this display: it is powered off, and the view ` +
+                'hierarchy describes whichever panel is live. Drop --display to capture ' +
+                'the live panel, or fold the device so this one takes over.');
         }
         if (sel) {
             let el;
@@ -143,10 +184,12 @@ async function screenshot(outputPath, opts = {}, sessionName = 'default', fullPa
                 // around the foreground app + status bars and has frame=.zero, so
                 // reading scale from it would always collapse to 1× and crop the
                 // wrong region on retina/4K screens. Use deviceInfo, which reports
-                // both points (AX space) and pixels (screenshot space).
-                const info = await driver.deviceInfo();
-                hierarchyW = info.widthPoints;
-                hierarchyH = info.heightPoints;
+                // both points (AX space) and pixels (screenshot space) — except for a
+                // redirected capture, where deviceInfo describes XCUIScreen.main and
+                // not the panel we captured, so derive the points from its own scale.
+                const size = iosHierarchySize({ redirected, scale }, (0, png_crop_js_1.readPngDimensions)(buf), await driver.deviceInfo());
+                hierarchyW = size.width;
+                hierarchyH = size.height;
                 el = await (0, wait_js_1.waitForIOSElement)((o) => driver.viewHierarchy(false, [], { cache: o?.cached }).then((x) => x.axElement), sel, undefined, undefined, (0, direct_ios_selector_js_1.makeIOSDirectResolver)(driver, sel));
             }
             else if (driver instanceof web_js_1.WebDriver) {
@@ -170,21 +213,14 @@ async function screenshot(outputPath, opts = {}, sessionName = 'default', fullPa
                 throw new Error('selector cropping is not supported for this driver');
             }
             const { width: pngW, height: pngH } = (0, png_crop_js_1.readPngDimensions)(buf);
-            const scaleX = hierarchyW > 0 ? pngW / hierarchyW : 1;
-            const scaleY = hierarchyH > 0 ? pngH / hierarchyH : 1;
-            // Margin is in the same logical units as the bounds (points on iOS,
-            // pixels on Android/Web — same units the `inspect` command prints),
-            // so scale it into screenshot pixels alongside the bounds.
-            const marginX = margin * scaleX;
-            const marginY = margin * scaleY;
-            const rectX = Math.round(el.bounds.x * scaleX - marginX);
-            const rectY = Math.round(el.bounds.y * scaleY - marginY);
-            const rectW = Math.round(el.bounds.width * scaleX + marginX * 2);
-            const rectH = Math.round(el.bounds.height * scaleY + marginY * 2);
-            if (rectX + rectW <= 0 || rectY + rectH <= 0 || rectX >= pngW || rectY >= pngH) {
-                throw new Error(`element ${label} bounds [${rectX},${rectY} ${rectW}x${rectH}] are outside the screenshot (${pngW}x${pngH})`);
+            const rect = cropRect(el.bounds, { width: hierarchyW, height: hierarchyH }, { width: pngW, height: pngH }, margin);
+            if (rect.x + rect.width <= 0 ||
+                rect.y + rect.height <= 0 ||
+                rect.x >= pngW ||
+                rect.y >= pngH) {
+                throw new Error(`element ${label} bounds [${rect.x},${rect.y} ${rect.width}x${rect.height}] are outside the screenshot (${pngW}x${pngH})`);
             }
-            out = (0, png_crop_js_1.cropPng)(buf, { x: rectX, y: rectY, width: rectW, height: rectH });
+            out = (0, png_crop_js_1.cropPng)(buf, rect);
         }
         await promises_1.default.writeFile(resolvedPath, out);
     }, sessionName);

package/dist/drivers/devicectl.js

@@ -290,6 +290,7 @@ async function listDisplays(deviceId) {
             primary: d.primary === true,
             kind,
             integrated: kind === 'integrated',
+            pointScale: typeof d.pointScale === 'number' && d.pointScale > 0 ? d.pointScale : 1,
         };
     });
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@houwert/conductor",
-  "version": "0.33.2",
+  "version": "0.33.3",
   "description": "CLI tool for mobile app interactions — optimized for AI agents",
   "license": "MIT",
   "repository": {

package/skills/conductor-device-setup/SKILL.md

@@ -45,10 +45,10 @@ Pass `--display cover` or `--display inner` to pin it to one panel, or a display
 id for anything else the device has attached (CarPlay, an external screen). An
 unknown value lists that device's displays with their ids.
 
-Cropping a screenshot to an element (`take-screenshot <element>`) only works
-against the main panel: the inner panel's framebuffer is rotated relative to the
-accessibility coordinate space, so conductor refuses the crop there instead of
-returning the wrong region. Fold the device or pass `--display cover` to crop.
+Cropping to an element (`take-screenshot <element>`) works on either panel. The
+one thing it cannot do is crop against a panel that is powered off — the view
+hierarchy always describes the live one — so combining a selector with a
+`--display` that points at the dark panel is an error rather than a wrong crop.
 
 Angles are in degrees, 0 (shut) to 180 (flat). `closed`/`book`/`open` map to
 0/130/180. Named poses always swap the display; an arbitrary mid-way angle sets