@houwert/conductor 0.24.1 → 0.25.0

package/dist/daemon/input-backends.js

@@ -0,0 +1,203 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.iosBackend = iosBackend;
+exports.androidBackend = androidBackend;
+const clampNorm = (v) => (v < 0 ? 0 : v > 1 ? 1 : v);
+// ── iOS / tvOS (XCUITest driver) ──────────────────────────────────────────────
+// key-name → XCUITest pressKey value
+const IOS_KEY = {
+    Backspace: 'delete',
+    Delete: 'delete',
+    Enter: 'enter',
+    Return: 'enter',
+    Tab: 'tab',
+    Space: 'space',
+    ' ': 'space',
+};
+// button/key-name → XCUITest pressButton value
+const IOS_BUTTON = {
+    home: 'home',
+    Home: 'home',
+    lock: 'lock',
+    Lock: 'lock',
+    power: 'lock',
+    Power: 'lock',
+    // tvOS remote (also reachable via the tvremote frame)
+    up: 'up',
+    down: 'down',
+    left: 'left',
+    right: 'right',
+    select: 'select',
+    menu: 'menu',
+    playPause: 'playPause',
+    ArrowUp: 'up',
+    ArrowDown: 'down',
+    ArrowLeft: 'left',
+    ArrowRight: 'right',
+};
+function iosBackend(driver) {
+    let size = null;
+    const dims = async () => {
+        if (!size) {
+            const info = await driver.deviceInfo();
+            size = { w: info.widthPoints, h: info.heightPoints };
+        }
+        return size;
+    };
+    const tvos = driver.platform === 'tvos';
+    return {
+        platform: tvos ? 'tvos' : 'ios',
+        capabilities(liveDrag) {
+            return {
+                touch: true,
+                drag: true,
+                multitouch: true,
+                buttons: ['home', 'lock'],
+                keyboard: true,
+                text: true,
+                tvRemote: tvos,
+                springboard: true,
+                liveDrag,
+                binaryPointer: false,
+                coord: 'normalized',
+            };
+        },
+        async tap(nx, ny, durationMs) {
+            const { w, h } = await dims();
+            await driver.tap(clampNorm(nx) * w, clampNorm(ny) * h, durationMs ? durationMs / 1000 : undefined);
+        },
+        async gesture(paths) {
+            const { w, h } = await dims();
+            const converted = paths.map((p) => ({
+                steps: p.steps.map((s) => ({
+                    x: clampNorm(s.nx) * w,
+                    y: clampNorm(s.ny) * h,
+                    dt: s.tMs / 1000,
+                })),
+            }));
+            await driver.gesturePath(converted);
+        },
+        async swipe(nsx, nsy, nex, ney, durationMs) {
+            const { w, h } = await dims();
+            await driver.swipe(clampNorm(nsx) * w, clampNorm(nsy) * h, clampNorm(nex) * w, clampNorm(ney) * h, durationMs / 1000);
+        },
+        async text(value) {
+            await driver.inputText(value);
+        },
+        async key(code, opts) {
+            // XCUITest pressKey is atomic — act on key-down, ignore the paired up.
+            if (opts && opts.down === false)
+                return;
+            const k = IOS_KEY[code];
+            if (k) {
+                await driver.pressKey(k);
+                return;
+            }
+            const b = IOS_BUTTON[code];
+            if (b)
+                await driver.pressButton(b);
+        },
+        async button(name, holdMs) {
+            const b = IOS_BUTTON[name];
+            if (b)
+                await driver.pressButton(b, holdMs ? holdMs / 1000 : undefined);
+        },
+    };
+}
+// ── Android (gRPC APK + adb) ───────────────────────────────────────────────────
+const ANDROID_KEYCODE = {
+    Home: 3,
+    home: 3,
+    Back: 4,
+    back: 4,
+    Enter: 66,
+    Return: 66,
+    Backspace: 67,
+    Delete: 67,
+    Tab: 61,
+    Space: 62,
+    ' ': 62,
+    Escape: 111,
+    Lock: 26,
+    Power: 26,
+    power: 26,
+    VolumeUp: 24,
+    volumeUp: 24,
+    VolumeDown: 25,
+    volumeDown: 25,
+    Menu: 82,
+    menu: 82,
+    Search: 84,
+    ArrowUp: 19,
+    ArrowDown: 20,
+    ArrowLeft: 21,
+    ArrowRight: 22,
+    up: 19,
+    down: 20,
+    left: 21,
+    right: 22,
+    select: 23,
+    playPause: 85,
+};
+function androidBackend(driver) {
+    let size = null;
+    const dims = async () => {
+        if (!size) {
+            const info = await driver.deviceInfo();
+            size = { w: info.widthPixels, h: info.heightPixels };
+        }
+        return size;
+    };
+    return {
+        platform: 'android',
+        capabilities(liveDrag) {
+            return {
+                touch: true,
+                drag: true,
+                multitouch: true,
+                buttons: ['home', 'back', 'menu', 'power', 'volumeUp', 'volumeDown'],
+                keyboard: true,
+                text: true,
+                tvRemote: true, // Android-TV dpad via keyevents
+                springboard: false,
+                liveDrag,
+                binaryPointer: false,
+                coord: 'normalized',
+            };
+        },
+        async tap(nx, ny) {
+            const { w, h } = await dims();
+            await driver.tap(clampNorm(nx) * w, clampNorm(ny) * h);
+        },
+        async gesture(paths) {
+            const { w, h } = await dims();
+            const converted = paths.map((p) => ({
+                steps: p.steps.map((s) => ({
+                    x: clampNorm(s.nx) * w,
+                    y: clampNorm(s.ny) * h,
+                    dt_ms: Math.round(s.tMs),
+                })),
+            }));
+            await driver.gesturePath(converted);
+        },
+        async swipe(nsx, nsy, nex, ney, durationMs) {
+            const { w, h } = await dims();
+            await driver.swipe(clampNorm(nsx) * w, clampNorm(nsy) * h, clampNorm(nex) * w, clampNorm(ney) * h, durationMs);
+        },
+        async text(value) {
+            await driver.inputText(value);
+        },
+        async key(code, opts) {
+            if (opts && opts.down === false)
+                return;
+            const kc = ANDROID_KEYCODE[code];
+            if (kc !== undefined)
+                await driver.pressKeyEvent(kc);
+        },
+        async button(name) {
+            const kc = ANDROID_KEYCODE[name];
+            if (kc !== undefined)
+                await driver.pressKeyEvent(kc);
+        },
+    };
+}

package/dist/daemon/input-protocol.js

@@ -0,0 +1,40 @@
+"use strict";
+/**
+ * Streaming device-input protocol (conductor ⇄ Argus).
+ *
+ * A persistent per-device WebSocket carries pointer/key/button/scroll/remote
+ * frames so continuous drags and fast typing stay low-latency, instead of the
+ * per-event HTTP the web driver uses. Conductor owns coord→device translation
+ * and keymaps; the client streams normalized 0..1 coordinates.
+ *
+ * See docs/device-input-migration.md for the design.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.INPUT_PROTOCOL_VERSION = void 0;
+exports.decodeClientFrame = decodeClientFrame;
+exports.INPUT_PROTOCOL_VERSION = 1;
+const CLIENT_FRAME_TYPES = new Set([
+    'select',
+    'pointer',
+    'key',
+    'text',
+    'button',
+    'scroll',
+    'tvremote',
+]);
+/** Parse and shallow-validate one text frame. Returns null on malformed input. */
+function decodeClientFrame(raw) {
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+    if (typeof parsed !== 'object' || parsed === null)
+        return null;
+    const t = parsed.t;
+    if (typeof t !== 'string' || !CLIENT_FRAME_TYPES.has(t))
+        return null;
+    return parsed;
+}

package/dist/daemon/input-router.js

@@ -0,0 +1,110 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InputRouter = void 0;
+/** Movement below this (normalized) counts as a tap, not a drag. */
+const TAP_EPSILON = 0.01;
+/** Default scroll gesture duration. */
+const SCROLL_DURATION_MS = 200;
+class InputRouter {
+    constructor(backend, opts = {}) {
+        this.backend = backend;
+        this.opts = opts;
+        this.sessions = new Map();
+        this.liveOpen = new Set();
+        this.now = opts.now ?? (() => Date.now());
+    }
+    get liveDrag() {
+        return this.opts.livePointer ? 'native' : 'buffered';
+    }
+    capabilities() {
+        return this.backend.capabilities(this.liveDrag);
+    }
+    /** Dispatch one frame. `select` is handled by the server, not here. */
+    async dispatch(frame) {
+        switch (frame.t) {
+            case 'pointer':
+                return this.handlePointer(frame.id ?? 0, frame.phase, frame.x, frame.y);
+            case 'key':
+                return this.backend.key(frame.code, { down: frame.down, mods: frame.mods });
+            case 'text':
+                return this.backend.text(frame.value);
+            case 'button':
+                return this.backend.button(frame.name, frame.holdMs);
+            case 'scroll':
+                return this.backend.swipe(frame.x, frame.y, frame.x - frame.dx, frame.y - frame.dy, SCROLL_DURATION_MS);
+            case 'tvremote':
+                return this.backend.button(frame.button, frame.holdMs);
+            case 'select':
+                return;
+        }
+    }
+    async handlePointer(id, phase, nx, ny) {
+        // Native held-touch path: stream straight through.
+        if (this.opts.livePointer) {
+            if (phase === 'down')
+                this.liveOpen.add(id);
+            if (phase === 'up' || phase === 'cancel')
+                this.liveOpen.delete(id);
+            return this.opts.livePointer.pointer(id, phase, nx, ny);
+        }
+        // Buffered path.
+        const t = this.now();
+        if (phase === 'down') {
+            this.sessions.set(id, {
+                steps: [{ nx, ny, tMs: 0 }],
+                lastStepAt: t,
+                startNx: nx,
+                startNy: ny,
+                moved: false,
+                done: false,
+                canceled: false,
+            });
+            return;
+        }
+        const s = this.sessions.get(id);
+        if (!s)
+            return; // move/up without a down — ignore
+        s.steps.push({ nx, ny, tMs: Math.max(0, t - s.lastStepAt) });
+        s.lastStepAt = t;
+        if (Math.hypot(nx - s.startNx, ny - s.startNy) > TAP_EPSILON)
+            s.moved = true;
+        if (phase === 'up' || phase === 'cancel') {
+            s.done = true;
+            s.canceled = phase === 'cancel';
+            if ([...this.sessions.values()].every((x) => x.done))
+                await this.flush();
+        }
+    }
+    async flush() {
+        const sessions = [...this.sessions.values()];
+        this.sessions.clear();
+        const active = sessions.filter((s) => !s.canceled);
+        if (active.length === 0)
+            return;
+        if (active.length === 1 && !active[0].moved) {
+            await this.backend.tap(active[0].startNx, active[0].startNy);
+            return;
+        }
+        await this.backend.gesture(active.map((s) => ({ steps: s.steps })));
+    }
+    /**
+     * Release any touch still held when the socket closes, so no finger is left
+     * stuck down. Buffered sessions flush as-is; live touches get a cancel.
+     */
+    async onClose() {
+        if (this.opts.livePointer) {
+            const open = [...this.liveOpen];
+            this.liveOpen.clear();
+            for (const id of open) {
+                await this.opts.livePointer.pointer(id, 'cancel', 0, 0).catch(() => { });
+            }
+            return;
+        }
+        if (this.sessions.size > 0) {
+            for (const s of this.sessions.values())
+                s.done = true;
+            await this.flush().catch(() => { });
+        }
+    }
+}
+exports.InputRouter = InputRouter;

package/dist/daemon/input-server.js

@@ -0,0 +1,124 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startInputServer = startInputServer;
+/**
+ * Streaming input WebSocket server (one per device daemon).
+ *
+ * Bound to loopback; the host IDE opens one long-lived socket per active device
+ * and streams pointer/key/button frames. Frames are processed in-order on a
+ * single-consumer queue per connection; consecutive pointer moves for the same
+ * finger are coalesced so a fast drag never backs up the injector. Phase
+ * transitions (down/up/cancel) are never dropped.
+ */
+const ws_1 = require("ws");
+const input_protocol_js_1 = require("./input-protocol.js");
+const STAT_INTERVAL_MS = 500;
+function send(ws, frame) {
+    if (ws.readyState === ws_1.WebSocket.OPEN)
+        ws.send(JSON.stringify(frame));
+}
+function startInputServer(opts) {
+    const host = opts.host ?? '127.0.0.1';
+    const dlog = opts.dlog ?? (() => { });
+    return new Promise((resolve, reject) => {
+        const wss = new ws_1.WebSocketServer({ host, port: opts.port });
+        wss.on('error', reject);
+        wss.on('listening', () => {
+            const addr = wss.address();
+            const boundPort = typeof addr === 'object' && addr ? addr.port : opts.port;
+            dlog(`input socket ready at ws://${host}:${boundPort}/input (${opts.platform})`);
+            resolve({
+                port: boundPort,
+                close: () => new Promise((res) => {
+                    for (const client of wss.clients)
+                        client.close();
+                    wss.close(() => res());
+                }),
+            });
+        });
+        wss.on('connection', (ws) => {
+            handleConnection(ws, opts, dlog).catch((err) => {
+                dlog(`input connection error: ${err instanceof Error ? err.message : String(err)}`);
+                try {
+                    ws.close();
+                }
+                catch {
+                    /* ok */
+                }
+            });
+        });
+    });
+}
+async function handleConnection(ws, opts, dlog) {
+    const router = await opts.makeRouter();
+    send(ws, {
+        t: 'hello',
+        protocol: input_protocol_js_1.INPUT_PROTOCOL_VERSION,
+        device: opts.device,
+        platform: opts.platform,
+        capabilities: router.capabilities(),
+    });
+    const queue = [];
+    let draining = false;
+    let dropped = 0;
+    const statTimer = setInterval(() => {
+        if (dropped > 0) {
+            send(ws, { t: 'stat', dropped });
+            dropped = 0;
+        }
+    }, STAT_INTERVAL_MS);
+    statTimer.unref?.();
+    const pump = async () => {
+        if (draining)
+            return;
+        draining = true;
+        while (queue.length > 0) {
+            const frame = queue.shift();
+            try {
+                await router.dispatch(frame);
+                if (frame.t !== 'select' && frame.ack && typeof frame.seq === 'number') {
+                    send(ws, { t: 'ok', seq: frame.seq });
+                }
+            }
+            catch (err) {
+                send(ws, {
+                    t: 'error',
+                    seq: frame.t !== 'select' ? frame.seq : undefined,
+                    code: 'dispatch_failed',
+                    msg: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+        draining = false;
+    };
+    ws.on('message', (data) => {
+        const frame = (0, input_protocol_js_1.decodeClientFrame)(data.toString());
+        if (!frame) {
+            send(ws, { t: 'error', code: 'bad_frame', msg: 'malformed or unknown frame' });
+            return;
+        }
+        if (frame.t === 'select')
+            return; // handshake reply; nothing to enqueue
+        // Coalesce consecutive moves for the same finger — keep only the latest.
+        if (frame.t === 'pointer' && frame.phase === 'move') {
+            const last = queue[queue.length - 1];
+            if (last &&
+                last.t === 'pointer' &&
+                last.phase === 'move' &&
+                (last.id ?? 0) === (frame.id ?? 0)) {
+                queue[queue.length - 1] = frame;
+                dropped++;
+                return;
+            }
+        }
+        queue.push(frame);
+        void pump();
+    });
+    ws.on('close', () => {
+        clearInterval(statTimer);
+        void router.onClose();
+    });
+    ws.on('error', (err) => {
+        dlog(`input socket error: ${err instanceof Error ? err.message : String(err)}`);
+    });
+}

package/dist/daemon/server.js

@@ -22,7 +22,12 @@ const protocol_js_1 = require("./protocol.js");
 const sdk_js_1 = require("../android/sdk.js");
 const bootstrap_js_1 = require("../drivers/bootstrap.js");
 const android_js_1 = require("../drivers/android.js");
+const ios_js_1 = require("../drivers/ios.js");
 const web_server_js_1 = require("./web-server.js");
+const input_server_js_1 = require("./input-server.js");
+const input_router_js_1 = require("./input-router.js");
+const input_backends_js_1 = require("./input-backends.js");
+const ios_hid_js_1 = require("../drivers/ios-hid.js");
 const log_collector_js_1 = require("./log-collector.js");
 const session_js_1 = require("../session.js");
 const sessionName = process.argv[2] ?? 'default';
@@ -78,6 +83,62 @@ function dlog(msg) {
 let driverPort = 1075;
 let driverPlatform = 'ios';
 let logCollector = null;
+let inputServer = null;
+let inputPort = null;
+let hidClient = null;
+/**
+ * Start the streaming-input WebSocket server for the current device, once its
+ * driver is up. iOS/tvOS/Android only — web keeps its per-event REST path.
+ * Each connection gets a fresh router (its own pointer state) over a shared
+ * driver instance.
+ */
+async function startInputServerForPlatform() {
+    if (inputServer)
+        return;
+    if (driverPlatform !== 'ios' && driverPlatform !== 'tvos' && driverPlatform !== 'android')
+        return;
+    let makeBackend;
+    let livePointer;
+    if (driverPlatform === 'android') {
+        const driver = new android_js_1.AndroidDriver(sessionName, driverPort);
+        await driver.connect();
+        makeBackend = () => (0, input_backends_js_1.androidBackend)(driver);
+    }
+    else {
+        const driver = new ios_js_1.IOSDriver(driverPort, '127.0.0.1', sessionName, driverPlatform);
+        makeBackend = () => (0, input_backends_js_1.iosBackend)(driver);
+        // Opt-in native held-touch backend for live drags (iOS only; single-touch).
+        if (driverPlatform === 'ios' && process.env.CONDUCTOR_IOS_HID === '1') {
+            const bin = await (0, bootstrap_js_1.getHidBinaryPath)();
+            if (bin) {
+                const client = new ios_hid_js_1.IOSHidClient(bin, sessionName);
+                client.start();
+                if (await client.ping().catch(() => false)) {
+                    hidClient = client;
+                    livePointer = client.asLivePointer();
+                    dlog('iOS HID injector active — live drags use CoreSimulator HID');
+                }
+                else {
+                    client.stop();
+                    dlog('iOS HID injector present but not responding — falling back to buffered drag');
+                }
+            }
+            else {
+                dlog('CONDUCTOR_IOS_HID=1 but no HID binary built — falling back to buffered drag');
+            }
+        }
+    }
+    const port = await (0, bootstrap_js_1.getInputPort)(sessionName);
+    inputServer = await (0, input_server_js_1.startInputServer)({
+        port,
+        device: sessionName,
+        platform: driverPlatform,
+        makeRouter: () => new input_router_js_1.InputRouter(makeBackend(), { livePointer }),
+        dlog,
+    });
+    inputPort = port;
+    dlog(`input server listening on ${port}`);
+}
 const DRIVER_HEALTH_INTERVAL_MS = 10000; // Check driver health every 10s
 let _restartInProgress = false;
 let _driverStarted = false;
@@ -169,6 +230,19 @@ async function main() {
             logCollector.stop();
             logCollector = null;
         }
+        if (inputServer) {
+            try {
+                await inputServer.close();
+            }
+            catch {
+                /* ok */
+            }
+            inputServer = null;
+        }
+        if (hidClient) {
+            hidClient.stop();
+            hidClient = null;
+        }
         try {
             fs_1.default.unlinkSync(SOCKET_PATH);
         }
@@ -291,6 +365,7 @@ async function main() {
                 ok: true,
                 platform: driverPlatform,
                 driverPort,
+                inputPort,
                 cdpUrl: cdpUrl ?? null,
                 cdpTargetId: cdpTargetId ?? null,
                 chromiumCdpPort: driverPlatform === 'web' ? (0, web_server_js_1.getCdpPort)() : null,
@@ -336,6 +411,15 @@ async function main() {
                 else {
                     await startDriverForPlatform(platform);
                 }
+                // Start the streaming-input socket once the driver is up.
+                if (_driverStarted) {
+                    try {
+                        await startInputServerForPlatform();
+                    }
+                    catch (err) {
+                        dlog(`Input server startup error: ${err instanceof Error ? err.message : String(err)}`);
+                    }
+                }
                 // Start collecting logs once the driver is (or was already) running.
                 if (_driverStarted) {
                     try {

package/dist/drivers/bootstrap.js

@@ -5,6 +5,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.detectPlatform = detectPlatform;
 exports.getDriverPort = getDriverPort;
+exports.getInputPort = getInputPort;
+exports.getInprocDylibPath = getInprocDylibPath;
+exports.getHidBinaryPath = getHidBinaryPath;
 exports.installDriver = installDriver;
 exports.isSimulatorBooted = isSimulatorBooted;
 exports.isPortOpen = isPortOpen;
@@ -89,6 +92,7 @@ const TVOS_BASE_PORT = 2075;
 const ANDROID_BASE_PORT = 3763;
 const WEB_BASE_PORT = 4075;
 const VEGA_BASE_PORT = 5075;
+const INPUT_BASE_PORT = 7075;
 const PORT_FILE = path_1.default.join(os_1.default.homedir(), '.conductor', 'ports.json');
 const PORT_LOCK = PORT_FILE + '.lock';
 const PORT_LOCK_TIMEOUT_MS = 5000;
@@ -175,6 +179,27 @@ async function getDriverPort(platform, deviceId) {
         return port;
     });
 }
+/**
+ * Assign and persist a streaming-input WebSocket port for a device. Kept in a
+ * separate namespace from the driver port (a device has both): the driver port
+ * serves the XCUITest/gRPC HTTP surface, this one the persistent input socket.
+ */
+async function getInputPort(deviceId) {
+    return withPortLock(() => {
+        const state = readPortState();
+        if (!state.inputAssignments)
+            state.inputAssignments = {};
+        if (state.nextInputPort === undefined)
+            state.nextInputPort = INPUT_BASE_PORT;
+        if (state.inputAssignments[deviceId] !== undefined) {
+            return state.inputAssignments[deviceId];
+        }
+        const port = state.nextInputPort++;
+        state.inputAssignments[deviceId] = port;
+        writePortState(state);
+        return port;
+    });
+}
 // ── Driver paths (bundled dev fallback + runtime download cache) ──────────────
 /**
  * Walk up from __dirname to find the package root (the directory containing
@@ -225,6 +250,30 @@ async function getDriversDir() {
     });
     return _driversDirPromise;
 }
+/**
+ * Absolute path to the injectable in-process control library
+ * (`<platform>-inproc/Conductor.framework/Conductor`), built by
+ * `packages/ios-inproc/tools/build-inproc-dylib.sh`. Passed to the target app
+ * via SIMCTL_CHILD_DYLD_INSERT_LIBRARIES at launch. iOS and tvOS ship separate
+ * builds (different simulator SDK).
+ */
+async function getInprocDylibPath(platform = 'ios') {
+    const dir = await getDriversDir();
+    return path_1.default.join(dir, `${platform}-inproc`, 'Conductor.framework', 'Conductor');
+}
+/**
+ * Absolute path to the host-side CoreSimulator HID injector (`ios-hid/conductor-hid`),
+ * built by `packages/ios-hid/tools/build-hid.sh`. Optional: only used for live
+ * held-touch drags when CONDUCTOR_IOS_HID=1 and the binary is present. Returns
+ * null if it hasn't been built.
+ */
+async function getHidBinaryPath() {
+    const dir = await getDriversDir().catch(() => null);
+    if (!dir)
+        return null;
+    const p = path_1.default.join(dir, 'ios-hid', 'conductor-hid');
+    return fs_1.default.existsSync(p) ? p : null;
+}
 async function ensureDriversCache(pkgRoot) {
     const pkgJsonPath = path_1.default.join(pkgRoot, 'package.json');
     const pkg = JSON.parse(fs_1.default.readFileSync(pkgJsonPath, 'utf-8'));