@houwert/conductor 0.24.0 → 0.25.0

package/dist/daemon/web-server.js

@@ -4,6 +4,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.parseAriaSnapshot = parseAriaSnapshot;
+exports.mergeCanvasNodes = mergeCanvasNodes;
 exports.startWebServer = startWebServer;
 exports.stopWebServer = stopWebServer;
 exports.getCdpPort = getCdpPort;
@@ -225,11 +226,22 @@ async function resolveBoundingBoxes(page, elements) {
         const el = queue.shift();
         if (el.ref) {
             try {
-                const locator = page.locator(`aria-ref=${el.ref}`);
-                const box = await locator.boundingBox({ timeout: 750 });
-                if (box && box.width > 0 && box.height > 0) {
-                    el.bounds = { x: box.x, y: box.y, width: box.width, height: box.height };
+                // One round-trip per ref for bounds + the element's own `data-testid`. Reading identity
+                // from the element itself (not by geometric overlap) is what lets canvas focus later be
+                // joined to the right node by identity — see mergeCanvasNodes.
+                const info = (await page
+                    .locator(`aria-ref=${el.ref}`)
+                    .first()
+                    .evaluate(`(node) => {
+              const r = node.getBoundingClientRect();
+              return { x: r.x, y: r.y, width: r.width, height: r.height,
+                       testId: node.getAttribute('data-testid') || '' };
+            }`, undefined, { timeout: 750 }));
+                if (info.width > 0 && info.height > 0) {
+                    el.bounds = { x: info.x, y: info.y, width: info.width, height: info.height };
                 }
+                if (info.testId)
+                    el.testId = info.testId;
             }
             catch {
                 // Element not visible or locator failed — skip
@@ -469,44 +481,56 @@ async function stampFocusFromDocumentActiveElement(page, elements) {
         rectPick.best.focused = true;
     }
 }
-/** IoU + center-inside score of a candidate node's bounds against a mirror rect. */
-function overlapScore(bounds, m) {
-    const x1 = Math.max(m.x, bounds.x);
-    const y1 = Math.max(m.y, bounds.y);
-    const x2 = Math.min(m.x + m.width, bounds.x + bounds.width);
-    const y2 = Math.min(m.y + m.height, bounds.y + bounds.height);
-    const inter = Math.max(0, x2 - x1) * Math.max(0, y2 - y1);
-    if (inter <= 0)
-        return 0;
-    const union = m.width * m.height + bounds.width * bounds.height - inter;
-    const iou = union > 0 ? inter / union : 0;
-    const mcx = m.x + m.width / 2;
-    const mcy = m.y + m.height / 2;
-    const centerInside = mcx >= bounds.x &&
-        mcx <= bounds.x + bounds.width &&
-        mcy >= bounds.y &&
-        mcy <= bounds.y + bounds.height;
-    return centerInside ? iou + 1 : iou;
-}
-/** Find the existing tree node whose bounds best overlap a mirror rect (>0.5 score). */
-function bestOverlappingNode(elements, m) {
-    let best = null;
-    let bestScore = 0.5;
-    const walk = (els) => {
+/**
+ * Merge canvas scene-graph nodes into the ARIA tree by identity (`data-testid`), never by
+ * geometry. A mirror node that shares a testId with an existing (natively-read) tree node
+ * enriches it and, authoritatively, carries its `data-focused` state onto it. Mirror nodes with
+ * no counterpart (canvas-only / off-screen scene nodes) are appended as their own nodes.
+ *
+ * Because focus rides identity — not bounds — an open drawer overlapping a focused tile can no
+ * longer steal the tile's focus, and a tile's identity can't smear onto a neighbour it happens
+ * to overlap.
+ */
+function mergeCanvasNodes(elements, mirror) {
+    const byTestId = new Map();
+    const index = (els) => {
         for (const el of els) {
-            if (el.bounds) {
-                const s = overlapScore(el.bounds, m);
-                if (s > bestScore) {
-                    bestScore = s;
-                    best = el;
-                }
-            }
+            if (el.testId)
+                byTestId.set(el.testId, el);
             if (el.children)
-                walk(el.children);
+                index(el.children);
         }
     };
-    walk(elements);
-    return best;
+    index(elements);
+    for (const m of mirror) {
+        const bounds = m.width > 0 && m.height > 0
+            ? { x: m.x, y: m.y, width: m.width, height: m.height }
+            : undefined;
+        const existing = m.testId ? byTestId.get(m.testId) : undefined;
+        if (existing) {
+            if (m.focused)
+                existing.focused = true;
+            if (!existing.name && m.name)
+                existing.name = m.name;
+            if (bounds && !existing.bounds)
+                existing.bounds = bounds;
+        }
+        else {
+            const node = {
+                role: m.role,
+                name: m.name,
+                ref: '',
+                testId: m.testId || undefined,
+                bounds,
+                enabled: !m.disabled,
+                focused: m.focused,
+                children: [],
+            };
+            elements.push(node);
+            if (m.testId)
+                byTestId.set(m.testId, node);
+        }
+    }
 }
 /**
  * Harvest a canvas DOM-inspector mirror into the hierarchy. Canvas webtv frameworks
@@ -514,10 +538,11 @@ function bestOverlappingNode(elements, m) {
  * off-screen `<div>`s carrying `data-testid` (real identity) and `data-focused="true"` (focus
  * — the canvas owns `document.activeElement`, so the normal focus path can't see it).
  *
- * Each mirror node is matched to an existing ARIA node by bounds overlap and used to enrich it
- * (testId + focus); unmatched mirror nodes are appended as new nodes. Mirror rects come from
- * `getBoundingClientRect`, i.e. the same viewport-CSS-pixel space taps use — drive TV apps at
- * the app's native resolution (e.g. `set-viewport 1920 1080`) so lower nodes aren't off-screen.
+ * Nodes are joined to the ARIA tree by identity (see mergeCanvasNodes); canvas-only nodes are
+ * appended. Mirror rects come from `getBoundingClientRect`, i.e. the same viewport-CSS-pixel
+ * space taps use — drive TV apps at the app's native resolution (e.g. `set-viewport 1920 1080`)
+ * so lower scene nodes aren't off-screen. Returns true when a mirror was present (a canvas app),
+ * so the caller can skip the DOM `activeElement` focus fallback.
  */
 async function harvestDomMirror(page, elements) {
     let mirror = null;
@@ -541,40 +566,12 @@ async function harvestDomMirror(page, elements) {
     })()`));
     }
     catch {
-        return;
+        return false;
     }
     if (!mirror || mirror.length === 0)
-        return;
-    // data-focused is authoritative for canvas apps (activeElement is the <canvas>, never the
-    // focused scene node), so clear any focus already inferred from the ARIA snapshot.
-    if (mirror.some((m) => m.focused))
-        clearFocusedFlags(elements);
-    for (const m of mirror) {
-        const bounds = m.width > 0 && m.height > 0
-            ? { x: m.x, y: m.y, width: m.width, height: m.height }
-            : undefined;
-        const target = bounds ? bestOverlappingNode(elements, m) : null;
-        if (target) {
-            if (m.testId)
-                target.testId = m.testId;
-            if (m.focused)
-                target.focused = true;
-            if (!target.name && m.name)
-                target.name = m.name;
-        }
-        else {
-            elements.push({
-                role: m.role,
-                name: m.name,
-                ref: '',
-                testId: m.testId || undefined,
-                bounds,
-                enabled: !m.disabled,
-                focused: m.focused,
-                children: [],
-            });
-        }
-    }
+        return false;
+    mergeCanvasNodes(elements, mirror);
+    return true;
 }
 // ── Web server ───────────────────────────────────────────────────────────────
 const DEFAULT_VIEWPORT = { width: 1280, height: 720 };
@@ -983,12 +980,16 @@ async function handleRequest(req, res, dlog) {
                 await resolveBoundingBoxes(p, elements);
                 await resolveBoundingBoxesBatch(p, elements);
                 await resolveBoundingBoxesByRole(p, elements);
+                // Focus source depends on the app, and the two are mutually exclusive:
+                //  • Canvas/inspector apps (Lightning/WPE) own document.activeElement on the <canvas>,
+                //    so focus lives on scene nodes as `data-focused` — merged by identity.
+                //  • Plain DOM apps expose focus via ARIA `[active]` / document.activeElement.
                 clearFocusedFlags(elements);
-                applyFocusFromAriaSnapshotYaml(ariaSnapshot, elements);
-                // Canvas webtv: merge the data-testid/data-focused mirror before falling back to
-                // document.activeElement (which is the <canvas>, not the focused scene node).
-                await harvestDomMirror(p, elements);
-                await stampFocusFromDocumentActiveElement(p, elements);
+                const isCanvasApp = await harvestDomMirror(p, elements);
+                if (!isCanvasApp) {
+                    applyFocusFromAriaSnapshotYaml(ariaSnapshot, elements);
+                    await stampFocusFromDocumentActiveElement(p, elements);
+                }
                 jsonResponse(res, {
                     url: p.url(),
                     title: await p.title(),

package/dist/drivers/bootstrap.js

@@ -5,6 +5,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.detectPlatform = detectPlatform;
 exports.getDriverPort = getDriverPort;
+exports.getInputPort = getInputPort;
+exports.getInprocDylibPath = getInprocDylibPath;
+exports.getHidBinaryPath = getHidBinaryPath;
 exports.installDriver = installDriver;
 exports.isSimulatorBooted = isSimulatorBooted;
 exports.isPortOpen = isPortOpen;
@@ -89,6 +92,7 @@ const TVOS_BASE_PORT = 2075;
 const ANDROID_BASE_PORT = 3763;
 const WEB_BASE_PORT = 4075;
 const VEGA_BASE_PORT = 5075;
+const INPUT_BASE_PORT = 7075;
 const PORT_FILE = path_1.default.join(os_1.default.homedir(), '.conductor', 'ports.json');
 const PORT_LOCK = PORT_FILE + '.lock';
 const PORT_LOCK_TIMEOUT_MS = 5000;
@@ -175,6 +179,27 @@ async function getDriverPort(platform, deviceId) {
         return port;
     });
 }
+/**
+ * Assign and persist a streaming-input WebSocket port for a device. Kept in a
+ * separate namespace from the driver port (a device has both): the driver port
+ * serves the XCUITest/gRPC HTTP surface, this one the persistent input socket.
+ */
+async function getInputPort(deviceId) {
+    return withPortLock(() => {
+        const state = readPortState();
+        if (!state.inputAssignments)
+            state.inputAssignments = {};
+        if (state.nextInputPort === undefined)
+            state.nextInputPort = INPUT_BASE_PORT;
+        if (state.inputAssignments[deviceId] !== undefined) {
+            return state.inputAssignments[deviceId];
+        }
+        const port = state.nextInputPort++;
+        state.inputAssignments[deviceId] = port;
+        writePortState(state);
+        return port;
+    });
+}
 // ── Driver paths (bundled dev fallback + runtime download cache) ──────────────
 /**
  * Walk up from __dirname to find the package root (the directory containing
@@ -225,6 +250,30 @@ async function getDriversDir() {
     });
     return _driversDirPromise;
 }
+/**
+ * Absolute path to the injectable in-process control library
+ * (`<platform>-inproc/Conductor.framework/Conductor`), built by
+ * `packages/ios-inproc/tools/build-inproc-dylib.sh`. Passed to the target app
+ * via SIMCTL_CHILD_DYLD_INSERT_LIBRARIES at launch. iOS and tvOS ship separate
+ * builds (different simulator SDK).
+ */
+async function getInprocDylibPath(platform = 'ios') {
+    const dir = await getDriversDir();
+    return path_1.default.join(dir, `${platform}-inproc`, 'Conductor.framework', 'Conductor');
+}
+/**
+ * Absolute path to the host-side CoreSimulator HID injector (`ios-hid/conductor-hid`),
+ * built by `packages/ios-hid/tools/build-hid.sh`. Optional: only used for live
+ * held-touch drags when CONDUCTOR_IOS_HID=1 and the binary is present. Returns
+ * null if it hasn't been built.
+ */
+async function getHidBinaryPath() {
+    const dir = await getDriversDir().catch(() => null);
+    if (!dir)
+        return null;
+    const p = path_1.default.join(dir, 'ios-hid', 'conductor-hid');
+    return fs_1.default.existsSync(p) ? p : null;
+}
 async function ensureDriversCache(pkgRoot) {
     const pkgJsonPath = path_1.default.join(pkgRoot, 'package.json');
     const pkg = JSON.parse(fs_1.default.readFileSync(pkgJsonPath, 'utf-8'));

package/dist/drivers/eval-compiler.js

@@ -0,0 +1,131 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.compileEval = compileEval;
+/**
+ * Compiles user Swift into a fresh dylib exporting `conductor_eval`, then drops
+ * it into the target app's container so the injected library can dlopen it.
+ *
+ * `expr` mode wraps an expression; `full` mode takes the whole function body.
+ * Each build is a uniquely-named dylib to avoid dlopen caching.
+ */
+const child_process_1 = require("child_process");
+const util_1 = require("util");
+const crypto_1 = __importDefault(require("crypto"));
+const promises_1 = __importDefault(require("fs/promises"));
+const os_1 = __importDefault(require("os"));
+const path_1 = __importDefault(require("path"));
+const run = (0, util_1.promisify)(child_process_1.execFile);
+const EXPR_TEMPLATE = (code) => `import Foundation
+import UIKit
+import SwiftUI
+
+@_cdecl("conductor_eval")
+public func conductor_eval() -> UnsafePointer<CChar> {
+    let __result: Any = {
+${code}
+    }()
+    return UnsafePointer(strdup(String(describing: __result))!)
+}
+`;
+const FULL_TEMPLATE = (code) => `import Foundation
+import UIKit
+import SwiftUI
+
+@_cdecl("conductor_eval")
+public func conductor_eval() -> UnsafePointer<CChar> {
+${code}
+}
+`;
+async function detectTarget(platform) {
+    const sdkName = platform === 'tvos' ? 'appletvsimulator' : 'iphonesimulator';
+    const sdkPlatform = platform === 'tvos' ? 'tvos' : 'ios';
+    const [{ stdout: sdkPath }, { stdout: sdkVer }] = await Promise.all([
+        run('xcrun', ['--sdk', sdkName, '--show-sdk-path']),
+        run('xcrun', ['--sdk', sdkName, '--show-sdk-version']),
+    ]);
+    const major = sdkVer.trim().split('.')[0];
+    const arch = os_1.default.arch() === 'arm64' ? 'arm64' : 'x86_64';
+    return {
+        sdkName,
+        sdkPath: sdkPath.trim(),
+        target: `${arch}-apple-${sdkPlatform}${major}.0-simulator`,
+    };
+}
+/**
+ * Compile `code` and place the dylib inside the app container's tmp so the
+ * sandboxed app can dlopen it. Returns the container-relative host path (which,
+ * on the simulator, is the same path the app opens).
+ */
+async function compileEval(code, mode, platform, deviceId, bundleId) {
+    const start = Date.now();
+    const { sdkName, sdkPath, target } = await detectTarget(platform);
+    const source = (mode === 'full' ? FULL_TEMPLATE : EXPR_TEMPLATE)(code);
+    const hash = crypto_1.default.createHash('md5').update(source).digest('hex').slice(0, 10);
+    const workDir = path_1.default.join(os_1.default.tmpdir(), 'conductor-eval');
+    await promises_1.default.mkdir(workDir, { recursive: true });
+    const swiftFile = path_1.default.join(workDir, `eval_${hash}.swift`);
+    const dylibName = `eval_${hash}_${Date.now()}.dylib`;
+    const dylibPath = path_1.default.join(workDir, dylibName);
+    await promises_1.default.writeFile(swiftFile, source, 'utf-8');
+    try {
+        await run('xcrun', [
+            '-sdk',
+            sdkName,
+            'swiftc',
+            '-target',
+            target,
+            '-sdk',
+            sdkPath,
+            '-emit-library',
+            '-Onone',
+            '-enable-testing',
+            '-o',
+            dylibPath,
+            // Unresolved symbols (app/system) resolve at dlopen time in the host process.
+            '-Xlinker',
+            '-undefined',
+            '-Xlinker',
+            'dynamic_lookup',
+            swiftFile,
+        ]);
+    }
+    catch (err) {
+        return {
+            ok: false,
+            error: `compile failed:\n${err.stderr ?? String(err)}`,
+        };
+    }
+    // Ad-hoc sign (sim rejects unsigned dylibs for dlopen on newer runtimes).
+    await run('codesign', ['--force', '--sign', '-', dylibPath]).catch(() => { });
+    // Copy into the app's data container tmp so the sandboxed app can read it.
+    try {
+        const { stdout } = await run('xcrun', [
+            'simctl',
+            'get_app_container',
+            deviceId,
+            bundleId,
+            'data',
+        ]);
+        const container = stdout.trim();
+        // System apps (and any without a data container) print "(null)"; use the raw
+        // path there — simulator apps are host processes and can dlopen it directly.
+        if (!container.startsWith('/'))
+            throw new Error('no data container');
+        const containerTmp = path_1.default.join(container, 'tmp');
+        await promises_1.default.mkdir(containerTmp, { recursive: true });
+        const dest = path_1.default.join(containerTmp, dylibName);
+        await promises_1.default.copyFile(dylibPath, dest);
+        return { ok: true, dylibPath: dest, info: `compiled in ${Date.now() - start}ms` };
+    }
+    catch {
+        // Fall back to the raw temp path — simulator apps can usually open it directly.
+        return {
+            ok: true,
+            dylibPath,
+            info: `compiled in ${Date.now() - start}ms (container copy skipped)`,
+        };
+    }
+}

package/dist/drivers/ios-hid.js

@@ -0,0 +1,95 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IOSHidClient = void 0;
+/**
+ * Node client for the host-side CoreSimulator HID injector
+ * (`packages/ios-hid`, built to `drivers/ios-hid/conductor-hid`).
+ *
+ * This is the streaming pointer backend: unlike XCUITest's atomic
+ * `_XCT_synthesizeEvent`, it holds a touch DOWN and streams moves, so a live
+ * drag animates on-device as the finger moves. Opt-in (CONDUCTOR_IOS_HID=1) and
+ * single-touch — multitouch/discrete gestures stay on the XCUITest path.
+ *
+ * Talks newline-delimited JSON over the binary's stdio, one request in flight
+ * at a time (FIFO correlation, mirroring the XCTest driver's simplicity).
+ */
+const child_process_1 = require("child_process");
+const readline_1 = __importDefault(require("readline"));
+const PHASE_TO_TYPE = {
+    down: 0,
+    move: 1,
+    up: 2,
+    cancel: 2, // release the held touch
+};
+class IOSHidClient {
+    constructor(binaryPath, udid) {
+        this.binaryPath = binaryPath;
+        this.udid = udid;
+        this.proc = null;
+        this.rl = null;
+        this.pending = [];
+    }
+    start() {
+        if (this.proc)
+            return;
+        this.proc = (0, child_process_1.spawn)(this.binaryPath, [], { stdio: ['pipe', 'pipe', 'inherit'] });
+        this.rl = readline_1.default.createInterface({ input: this.proc.stdout });
+        this.rl.on('line', (line) => {
+            const resolve = this.pending.shift();
+            if (!resolve)
+                return;
+            try {
+                resolve(JSON.parse(line));
+            }
+            catch {
+                resolve({ ok: false, error: `bad response: ${line}` });
+            }
+        });
+        this.proc.on('exit', () => {
+            this.proc = null;
+            this.rl = null;
+            // Fail any in-flight requests so callers don't hang.
+            while (this.pending.length)
+                this.pending.shift()({ ok: false, error: 'hid process exited' });
+        });
+    }
+    stop() {
+        this.proc?.kill();
+        this.proc = null;
+        this.rl = null;
+    }
+    send(req) {
+        if (!this.proc)
+            this.start();
+        return new Promise((resolve) => {
+            this.pending.push(resolve);
+            this.proc.stdin.write(JSON.stringify(req) + '\n');
+        });
+    }
+    async ping() {
+        const r = await this.send({ cmd: 'ping' });
+        return r.ok;
+    }
+    /** Inject a touch phase at normalized coords. */
+    async touch(nx, ny, phase) {
+        const r = await this.send({
+            cmd: 'touch',
+            udid: this.udid,
+            x: nx,
+            y: ny,
+            type: PHASE_TO_TYPE[phase],
+        });
+        if (!r.ok)
+            throw new Error(`hid touch failed (rc=${r.rc ?? '?'}) ${r.error ?? ''}`.trim());
+    }
+    /** Adapter for the router's live-pointer path (single finger; id ignored). */
+    asLivePointer() {
+        return {
+            pointer: (_id, phase, nx, ny) => this.touch(nx, ny, phase),
+        };
+    }
+}
+exports.IOSHidClient = IOSHidClient;