@houwert/conductor 0.24.1 → 0.26.0

package/dist/daemon/input-server.js

@@ -0,0 +1,124 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startInputServer = startInputServer;
+/**
+ * Streaming input WebSocket server (one per device daemon).
+ *
+ * Bound to loopback; the host IDE opens one long-lived socket per active device
+ * and streams pointer/key/button frames. Frames are processed in-order on a
+ * single-consumer queue per connection; consecutive pointer moves for the same
+ * finger are coalesced so a fast drag never backs up the injector. Phase
+ * transitions (down/up/cancel) are never dropped.
+ */
+const ws_1 = require("ws");
+const input_protocol_js_1 = require("./input-protocol.js");
+const STAT_INTERVAL_MS = 500;
+function send(ws, frame) {
+    if (ws.readyState === ws_1.WebSocket.OPEN)
+        ws.send(JSON.stringify(frame));
+}
+function startInputServer(opts) {
+    const host = opts.host ?? '127.0.0.1';
+    const dlog = opts.dlog ?? (() => { });
+    return new Promise((resolve, reject) => {
+        const wss = new ws_1.WebSocketServer({ host, port: opts.port });
+        wss.on('error', reject);
+        wss.on('listening', () => {
+            const addr = wss.address();
+            const boundPort = typeof addr === 'object' && addr ? addr.port : opts.port;
+            dlog(`input socket ready at ws://${host}:${boundPort}/input (${opts.platform})`);
+            resolve({
+                port: boundPort,
+                close: () => new Promise((res) => {
+                    for (const client of wss.clients)
+                        client.close();
+                    wss.close(() => res());
+                }),
+            });
+        });
+        wss.on('connection', (ws) => {
+            handleConnection(ws, opts, dlog).catch((err) => {
+                dlog(`input connection error: ${err instanceof Error ? err.message : String(err)}`);
+                try {
+                    ws.close();
+                }
+                catch {
+                    /* ok */
+                }
+            });
+        });
+    });
+}
+async function handleConnection(ws, opts, dlog) {
+    const router = await opts.makeRouter();
+    send(ws, {
+        t: 'hello',
+        protocol: input_protocol_js_1.INPUT_PROTOCOL_VERSION,
+        device: opts.device,
+        platform: opts.platform,
+        capabilities: router.capabilities(),
+    });
+    const queue = [];
+    let draining = false;
+    let dropped = 0;
+    const statTimer = setInterval(() => {
+        if (dropped > 0) {
+            send(ws, { t: 'stat', dropped });
+            dropped = 0;
+        }
+    }, STAT_INTERVAL_MS);
+    statTimer.unref?.();
+    const pump = async () => {
+        if (draining)
+            return;
+        draining = true;
+        while (queue.length > 0) {
+            const frame = queue.shift();
+            try {
+                await router.dispatch(frame);
+                if (frame.t !== 'select' && frame.ack && typeof frame.seq === 'number') {
+                    send(ws, { t: 'ok', seq: frame.seq });
+                }
+            }
+            catch (err) {
+                send(ws, {
+                    t: 'error',
+                    seq: frame.t !== 'select' ? frame.seq : undefined,
+                    code: 'dispatch_failed',
+                    msg: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+        draining = false;
+    };
+    ws.on('message', (data) => {
+        const frame = (0, input_protocol_js_1.decodeClientFrame)(data.toString());
+        if (!frame) {
+            send(ws, { t: 'error', code: 'bad_frame', msg: 'malformed or unknown frame' });
+            return;
+        }
+        if (frame.t === 'select')
+            return; // handshake reply; nothing to enqueue
+        // Coalesce consecutive moves for the same finger — keep only the latest.
+        if (frame.t === 'pointer' && frame.phase === 'move') {
+            const last = queue[queue.length - 1];
+            if (last &&
+                last.t === 'pointer' &&
+                last.phase === 'move' &&
+                (last.id ?? 0) === (frame.id ?? 0)) {
+                queue[queue.length - 1] = frame;
+                dropped++;
+                return;
+            }
+        }
+        queue.push(frame);
+        void pump();
+    });
+    ws.on('close', () => {
+        clearInterval(statTimer);
+        void router.onClose();
+    });
+    ws.on('error', (err) => {
+        dlog(`input socket error: ${err instanceof Error ? err.message : String(err)}`);
+    });
+}

package/dist/daemon/server.js

@@ -22,7 +22,14 @@ const protocol_js_1 = require("./protocol.js");
 const sdk_js_1 = require("../android/sdk.js");
 const bootstrap_js_1 = require("../drivers/bootstrap.js");
 const android_js_1 = require("../drivers/android.js");
+const ios_js_1 = require("../drivers/ios.js");
 const web_server_js_1 = require("./web-server.js");
+const input_server_js_1 = require("./input-server.js");
+const input_router_js_1 = require("./input-router.js");
+const input_backends_js_1 = require("./input-backends.js");
+const ios_hid_js_1 = require("../drivers/ios-hid.js");
+const video_server_js_1 = require("./video-server.js");
+const video_source_js_1 = require("./video-source.js");
 const log_collector_js_1 = require("./log-collector.js");
 const session_js_1 = require("../session.js");
 const sessionName = process.argv[2] ?? 'default';
@@ -78,6 +85,91 @@ function dlog(msg) {
 let driverPort = 1075;
 let driverPlatform = 'ios';
 let logCollector = null;
+let inputServer = null;
+let inputPort = null;
+let hidClient = null;
+let videoServer = null;
+let streamPort = null;
+/**
+ * Start the streaming-input WebSocket server for the current device, once its
+ * driver is up. iOS/tvOS/Android only — web keeps its per-event REST path.
+ * Each connection gets a fresh router (its own pointer state) over a shared
+ * driver instance.
+ */
+async function startInputServerForPlatform() {
+    if (inputServer)
+        return;
+    if (driverPlatform !== 'ios' && driverPlatform !== 'tvos' && driverPlatform !== 'android')
+        return;
+    let makeBackend;
+    let livePointer;
+    if (driverPlatform === 'android') {
+        const driver = new android_js_1.AndroidDriver(sessionName, driverPort);
+        await driver.connect();
+        makeBackend = () => (0, input_backends_js_1.androidBackend)(driver);
+    }
+    else {
+        const driver = new ios_js_1.IOSDriver(driverPort, '127.0.0.1', sessionName, driverPlatform);
+        makeBackend = () => (0, input_backends_js_1.iosBackend)(driver);
+        // Opt-in native held-touch backend for live drags (iOS only; single-touch).
+        if (driverPlatform === 'ios' && process.env.CONDUCTOR_IOS_HID === '1') {
+            const bin = await (0, bootstrap_js_1.getHidBinaryPath)();
+            if (bin) {
+                const client = new ios_hid_js_1.IOSHidClient(bin, sessionName);
+                client.start();
+                if (await client.ping().catch(() => false)) {
+                    hidClient = client;
+                    livePointer = client.asLivePointer();
+                    dlog('iOS HID injector active — live drags use CoreSimulator HID');
+                }
+                else {
+                    client.stop();
+                    dlog('iOS HID injector present but not responding — falling back to buffered drag');
+                }
+            }
+            else {
+                dlog('CONDUCTOR_IOS_HID=1 but no HID binary built — falling back to buffered drag');
+            }
+        }
+    }
+    const port = await (0, bootstrap_js_1.getInputPort)(sessionName);
+    inputServer = await (0, input_server_js_1.startInputServer)({
+        port,
+        device: sessionName,
+        platform: driverPlatform,
+        makeRouter: () => new input_router_js_1.InputRouter(makeBackend(), { livePointer }),
+        dlog,
+    });
+    inputPort = port;
+    dlog(`input server listening on ${port}`);
+}
+/**
+ * Start the streaming-video WebSocket server for the current device, once its
+ * driver is up. iOS/tvOS only for now (host-side SimulatorKit capture); the
+ * binary must be built/downloaded. Capture is lazy: it doesn't spawn until the
+ * first subscriber connects, and stops when the last one leaves.
+ */
+async function startVideoServerForPlatform() {
+    if (videoServer)
+        return;
+    if (driverPlatform !== 'ios' && driverPlatform !== 'tvos')
+        return;
+    const binary = await (0, bootstrap_js_1.getCaptureBinaryPath)();
+    if (!binary) {
+        dlog('video capture binary not present — stream server disabled for this device');
+        return;
+    }
+    const port = await (0, bootstrap_js_1.getStreamPort)(sessionName);
+    videoServer = await (0, video_server_js_1.startVideoServer)({
+        port,
+        device: sessionName,
+        platform: driverPlatform,
+        makeSource: (hub) => new video_source_js_1.IOSCaptureSource(binary, sessionName, hub, dlog),
+        dlog,
+    });
+    streamPort = port;
+    dlog(`video server listening on ${port}`);
+}
 const DRIVER_HEALTH_INTERVAL_MS = 10000; // Check driver health every 10s
 let _restartInProgress = false;
 let _driverStarted = false;
@@ -169,6 +261,28 @@ async function main() {
             logCollector.stop();
             logCollector = null;
         }
+        if (inputServer) {
+            try {
+                await inputServer.close();
+            }
+            catch {
+                /* ok */
+            }
+            inputServer = null;
+        }
+        if (videoServer) {
+            try {
+                await videoServer.close();
+            }
+            catch {
+                /* ok */
+            }
+            videoServer = null;
+        }
+        if (hidClient) {
+            hidClient.stop();
+            hidClient = null;
+        }
         try {
             fs_1.default.unlinkSync(SOCKET_PATH);
         }
@@ -291,6 +405,8 @@ async function main() {
                 ok: true,
                 platform: driverPlatform,
                 driverPort,
+                inputPort,
+                streamPort,
                 cdpUrl: cdpUrl ?? null,
                 cdpTargetId: cdpTargetId ?? null,
                 chromiumCdpPort: driverPlatform === 'web' ? (0, web_server_js_1.getCdpPort)() : null,
@@ -336,6 +452,21 @@ async function main() {
                 else {
                     await startDriverForPlatform(platform);
                 }
+                // Start the streaming-input socket once the driver is up.
+                if (_driverStarted) {
+                    try {
+                        await startInputServerForPlatform();
+                    }
+                    catch (err) {
+                        dlog(`Input server startup error: ${err instanceof Error ? err.message : String(err)}`);
+                    }
+                    try {
+                        await startVideoServerForPlatform();
+                    }
+                    catch (err) {
+                        dlog(`Video server startup error: ${err instanceof Error ? err.message : String(err)}`);
+                    }
+                }
                 // Start collecting logs once the driver is (or was already) running.
                 if (_driverStarted) {
                     try {

package/dist/daemon/video-hub.js

@@ -0,0 +1,58 @@
+"use strict";
+/**
+ * Fan-out hub for a single device's H.264 stream: one capture, N subscribers.
+ *
+ * Mirrors Argus's videoForward.ts. The last-seen config and the last keyframe
+ * access unit are cached so a late subscriber can configure its decoder and
+ * start rendering immediately, without waiting for the next natural keyframe.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VideoHub = void 0;
+class VideoHub {
+    constructor() {
+        this.listeners = new Set();
+        this.lastConfig = null;
+        this.lastKeyframe = null;
+    }
+    get subscriberCount() {
+        return this.listeners.size;
+    }
+    /**
+     * Subscribe. If a config (and keyframe) are already cached they are delivered
+     * synchronously so the decoder starts without waiting. Returns an unsubscribe.
+     */
+    subscribe(listener) {
+        this.listeners.add(listener);
+        if (this.lastConfig) {
+            listener.onConfig(this.lastConfig);
+            if (this.lastKeyframe)
+                listener.onFrame(this.lastKeyframe, true);
+        }
+        return () => this.listeners.delete(listener);
+    }
+    emitConfig(config) {
+        this.lastConfig = config;
+        // A config change invalidates the cached keyframe (dimensions may differ).
+        this.lastKeyframe = null;
+        for (const l of this.listeners)
+            l.onConfig(config);
+    }
+    emitFrame(annexB, keyFrame) {
+        if (keyFrame)
+            this.lastKeyframe = annexB;
+        for (const l of this.listeners)
+            l.onFrame(annexB, keyFrame);
+    }
+    /** Current coded dimensions, if known. */
+    dims() {
+        return this.lastConfig
+            ? { width: this.lastConfig.width, height: this.lastConfig.height }
+            : null;
+    }
+    /** Drop cached state when capture ends. */
+    clear() {
+        this.lastConfig = null;
+        this.lastKeyframe = null;
+    }
+}
+exports.VideoHub = VideoHub;

package/dist/daemon/video-protocol.js

@@ -0,0 +1,38 @@
+"use strict";
+/**
+ * Streaming device-video protocol (conductor ⇄ subscribers).
+ *
+ * The capture counterpart to input-protocol.ts. A persistent per-device
+ * WebSocket carries a low-latency H.264 stream: on connect the server sends one
+ * JSON `config` frame (codec + dimensions + SPS/PPS), then a sequence of
+ * **binary** frames, each a single H.264 Annex B access unit (keyframe-led). One
+ * capture feeds N subscribers (fan-out); a late joiner always gets the cached
+ * config + a fresh keyframe immediately.
+ *
+ * Web devices advertise a different codec (JPEG over CDP screencast) in the same
+ * `config` frame so clients pick a decoder from the wire, not the URL.
+ *
+ * See docs/device-video-stream.md for the design.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VIDEO_PROTOCOL_VERSION = void 0;
+exports.toConfigFrame = toConfigFrame;
+exports.VIDEO_PROTOCOL_VERSION = 1;
+/** Serialize an in-daemon config into the JSON frame sent to a subscriber. */
+function toConfigFrame(cfg, device, platform) {
+    return {
+        t: 'config',
+        protocol: exports.VIDEO_PROTOCOL_VERSION,
+        device,
+        platform,
+        codec: cfg.codec,
+        width: cfg.width,
+        height: cfg.height,
+        rotation: cfg.rotation,
+        fps: cfg.fps,
+        codecString: cfg.codecString,
+        sps: cfg.sps ? cfg.sps.toString('base64') : undefined,
+        pps: cfg.pps ? cfg.pps.toString('base64') : undefined,
+        avcC: cfg.avcC ? cfg.avcC.toString('base64') : undefined,
+    };
+}

package/dist/daemon/video-server.js

@@ -0,0 +1,88 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.startVideoServer = startVideoServer;
+/**
+ * Streaming video WebSocket server (one per device daemon).
+ *
+ * The capture counterpart to input-server.ts. Bound to loopback; N clients open
+ * a socket and receive one JSON `config` frame followed by binary H.264 Annex B
+ * access units. All subscribers share one capture via a VideoHub — the first
+ * connection starts the capture backend, the last one out stops it. A late
+ * joiner gets the cached config + keyframe immediately (hub responsibility).
+ */
+const ws_1 = require("ws");
+const video_protocol_js_1 = require("./video-protocol.js");
+const video_hub_js_1 = require("./video-hub.js");
+function sendJson(ws, frame) {
+    if (ws.readyState === ws_1.WebSocket.OPEN)
+        ws.send(JSON.stringify(frame));
+}
+function startVideoServer(opts) {
+    const host = opts.host ?? '127.0.0.1';
+    const dlog = opts.dlog ?? (() => { });
+    const hub = new video_hub_js_1.VideoHub();
+    const source = opts.makeSource(hub);
+    // Serialize start/stop so a rapid connect/disconnect churn can't race the
+    // capture backend into an inconsistent state.
+    let lifecycle = Promise.resolve();
+    const runExclusive = (fn) => {
+        lifecycle = lifecycle.then(fn, fn);
+        return lifecycle;
+    };
+    return new Promise((resolve, reject) => {
+        const wss = new ws_1.WebSocketServer({ host, port: opts.port });
+        wss.on('error', reject);
+        wss.on('listening', () => {
+            const addr = wss.address();
+            const boundPort = typeof addr === 'object' && addr ? addr.port : opts.port;
+            dlog(`video socket ready at ws://${host}:${boundPort}/stream (${opts.platform})`);
+            resolve({
+                port: boundPort,
+                hub,
+                close: () => runExclusive(async () => {
+                    for (const client of wss.clients)
+                        client.close();
+                    await source.stop().catch(() => { });
+                    await new Promise((res) => wss.close(() => res()));
+                }),
+            });
+        });
+        wss.on('connection', (ws) => {
+            const listener = {
+                onConfig: (config) => sendJson(ws, (0, video_protocol_js_1.toConfigFrame)(config, opts.device, opts.platform)),
+                onFrame: (annexB, _keyFrame) => {
+                    if (ws.readyState === ws_1.WebSocket.OPEN)
+                        ws.send(annexB);
+                },
+            };
+            const unsubscribe = hub.subscribe(listener);
+            const wasFirst = hub.subscriberCount === 1;
+            if (wasFirst) {
+                void runExclusive(async () => {
+                    try {
+                        await source.start();
+                    }
+                    catch (err) {
+                        dlog(`capture start error: ${err instanceof Error ? err.message : String(err)}`);
+                        sendJson(ws, {
+                            t: 'notice',
+                            code: 'capture_failed',
+                            msg: 'capture backend failed to start',
+                        });
+                    }
+                });
+            }
+            const teardown = () => {
+                unsubscribe();
+                if (hub.subscriberCount === 0) {
+                    void runExclusive(() => source.stop().catch(() => { }));
+                }
+            };
+            ws.on('close', teardown);
+            ws.on('error', (err) => {
+                dlog(`video socket error: ${err instanceof Error ? err.message : String(err)}`);
+            });
+            // Clients are pure subscribers; inbound messages are ignored.
+        });
+    });
+}

package/dist/daemon/video-source.js

@@ -0,0 +1,142 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IOSCaptureSource = void 0;
+/**
+ * Capture sources — the producer half of the video stream. A source drives one
+ * platform's capture and feeds decoded config/access-units into a VideoHub. The
+ * video server ref-counts subscribers and calls start()/stop() so exactly one
+ * capture runs while ≥1 subscriber is connected (last one out stops it).
+ *
+ * iOS/tvOS: spawns the host-side `conductor-capture` binary (packages/ios-capture),
+ * which captures the Simulator framebuffer via SimulatorKit, VideoToolbox-encodes
+ * H.264, and serves a raw Annex B TCP stream. We connect to that port, parse the
+ * elementary stream, and forward.
+ */
+const child_process_1 = require("child_process");
+const net_1 = __importDefault(require("net"));
+const readline_1 = __importDefault(require("readline"));
+const h264_annexb_js_1 = require("./h264-annexb.js");
+/** Restart the capture backend at most this often if the TCP stream drops. */
+const RESTART_BACKOFF_MS = 1000;
+class IOSCaptureSource {
+    constructor(binaryPath, udid, hub, dlog = () => { }) {
+        this.binaryPath = binaryPath;
+        this.udid = udid;
+        this.hub = hub;
+        this.dlog = dlog;
+        this.proc = null;
+        this.rl = null;
+        this.socket = null;
+        this.parser = null;
+        this.pending = [];
+        this.active = false;
+        this.restarting = false;
+    }
+    async start() {
+        if (this.active)
+            return;
+        this.active = true;
+        await this.spawnAndCapture();
+    }
+    async spawnAndCapture() {
+        this.proc = (0, child_process_1.spawn)(this.binaryPath, [], { stdio: ['pipe', 'pipe', 'inherit'] });
+        this.rl = readline_1.default.createInterface({ input: this.proc.stdout });
+        this.rl.on('line', (line) => {
+            const resolve = this.pending.shift();
+            if (!resolve)
+                return;
+            try {
+                resolve(JSON.parse(line));
+            }
+            catch {
+                resolve({ ok: false, error: `bad response: ${line}` });
+            }
+        });
+        this.proc.on('exit', (code) => {
+            this.dlog(`capture backend exited (code=${code ?? '?'})`);
+            this.proc = null;
+            this.rl = null;
+            while (this.pending.length)
+                this.pending.shift()({ ok: false, error: 'capture exited' });
+            this.maybeRestart();
+        });
+        const res = await this.send({ cmd: 'start_capture', udid: this.udid });
+        if (!res.ok || typeof res.port !== 'number') {
+            throw new Error(`start_capture failed: ${res.error ?? 'no port returned'}`);
+        }
+        this.connectStream(res.port);
+        this.dlog(`capture started, streaming H.264 from 127.0.0.1:${res.port}`);
+    }
+    connectStream(port) {
+        const parser = new h264_annexb_js_1.H264AnnexBParser({
+            onConfig: (config) => {
+                this.dlog(`H.264 config ${config.codecString} ${config.width}x${config.height}`);
+                this.hub.emitConfig(config);
+            },
+            onFrame: (annexB, keyFrame) => this.hub.emitFrame(annexB, keyFrame),
+        });
+        this.parser = parser;
+        const socket = net_1.default.connect(port, '127.0.0.1', () => {
+            this.dlog(`connected to capture stream on ${port}`);
+        });
+        socket.on('data', (chunk) => parser.push(chunk));
+        socket.on('error', (err) => this.dlog(`capture stream error: ${err.message}`));
+        socket.on('close', () => {
+            this.dlog('capture stream closed');
+            this.maybeRestart();
+        });
+        this.socket = socket;
+    }
+    /** If capture died while subscribers still want it, relaunch after a short backoff. */
+    maybeRestart() {
+        if (!this.active || this.restarting)
+            return;
+        this.restarting = true;
+        this.teardownTransport();
+        setTimeout(() => {
+            this.restarting = false;
+            if (!this.active)
+                return;
+            this.dlog('restarting capture backend');
+            this.spawnAndCapture().catch((err) => this.dlog(`capture restart failed: ${err.message}`));
+        }, RESTART_BACKOFF_MS).unref?.();
+    }
+    teardownTransport() {
+        this.socket?.destroy();
+        this.socket = null;
+        this.parser?.reset();
+        this.parser = null;
+    }
+    async stop() {
+        if (!this.active)
+            return;
+        this.active = false;
+        try {
+            if (this.proc)
+                await this.send({ cmd: 'stop_capture' }).catch(() => { });
+        }
+        finally {
+            this.teardownTransport();
+            this.rl?.close();
+            this.rl = null;
+            this.proc?.kill();
+            this.proc = null;
+            this.hub.clear();
+            this.dlog('capture stopped');
+        }
+    }
+    send(req) {
+        return new Promise((resolve) => {
+            if (!this.proc || !this.proc.stdin?.writable) {
+                resolve({ ok: false, error: 'capture process not running' });
+                return;
+            }
+            this.pending.push(resolve);
+            this.proc.stdin.write(JSON.stringify(req) + '\n');
+        });
+    }
+}
+exports.IOSCaptureSource = IOSCaptureSource;