@houwert/conductor 0.24.1 → 0.26.0

package/dist/daemon/h264-annexb.js

@@ -0,0 +1,264 @@
+"use strict";
+/**
+ * H.264 Annex B streaming parser (daemon side).
+ *
+ * Splits the raw Annex B byte stream from the capture backend into access units,
+ * extracts SPS/PPS to build the stream config, and preserves each access unit
+ * **in Annex B form** (start codes intact) for forwarding to subscribers — the
+ * wire contract is "config + a sequence of Annex B access units, keyframe-led".
+ *
+ * Ported from Argus's electron/services/simulator/h264-parser.ts; the SPS
+ * bit-reader is identical, but access units are emitted as Annex B rather than
+ * converted to AVCC length-prefix (that conversion is left to WebCodecs clients).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.H264AnnexBParser = void 0;
+const NAL_SLICE = 1;
+const NAL_IDR = 5;
+const NAL_SPS = 7;
+const NAL_PPS = 8;
+// ── Exp-Golomb reader (minimal, for SPS parsing) ────────────────────────────
+class ExpGolombReader {
+    constructor(data) {
+        this.data = data;
+        this.byteOffset = 0;
+        this.bitOffset = 0;
+    }
+    readBit() {
+        if (this.byteOffset >= this.data.length)
+            return 0;
+        const bit = (this.data[this.byteOffset] >> (7 - this.bitOffset)) & 1;
+        this.bitOffset++;
+        if (this.bitOffset === 8) {
+            this.bitOffset = 0;
+            this.byteOffset++;
+        }
+        return bit;
+    }
+    readBits(n) {
+        let val = 0;
+        for (let i = 0; i < n; i++)
+            val = (val << 1) | this.readBit();
+        return val;
+    }
+    readUE() {
+        let zeros = 0;
+        while (this.readBit() === 0 && zeros < 31)
+            zeros++;
+        if (zeros === 0)
+            return 0;
+        return (1 << zeros) - 1 + this.readBits(zeros);
+    }
+    readSE() {
+        const val = this.readUE();
+        return val % 2 === 0 ? -(val >> 1) : (val + 1) >> 1;
+    }
+}
+/** Parse the SPS fields needed for config. `body` is the NAL after the header byte. */
+function parseSps(body) {
+    const r = new ExpGolombReader(body);
+    const profileIdc = r.readBits(8);
+    const constraintFlags = r.readBits(8);
+    const levelIdc = r.readBits(8);
+    r.readUE(); // seq_parameter_set_id
+    if ([100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134].includes(profileIdc)) {
+        const chromaFormatIdc = r.readUE();
+        if (chromaFormatIdc === 3)
+            r.readBits(1);
+        r.readUE(); // bit_depth_luma_minus8
+        r.readUE(); // bit_depth_chroma_minus8
+        r.readBits(1); // qpprime_y_zero_transform_bypass_flag
+        if (r.readBits(1)) {
+            const count = chromaFormatIdc !== 3 ? 8 : 12;
+            for (let i = 0; i < count; i++) {
+                if (r.readBits(1)) {
+                    const size = i < 6 ? 16 : 64;
+                    let lastScale = 8;
+                    let nextScale = 8;
+                    for (let j = 0; j < size; j++) {
+                        if (nextScale !== 0)
+                            nextScale = (lastScale + r.readSE() + 256) % 256;
+                        lastScale = nextScale === 0 ? lastScale : nextScale;
+                    }
+                }
+            }
+        }
+    }
+    r.readUE(); // log2_max_frame_num_minus4
+    const picOrderCntType = r.readUE();
+    if (picOrderCntType === 0) {
+        r.readUE();
+    }
+    else if (picOrderCntType === 1) {
+        r.readBits(1);
+        r.readSE();
+        r.readSE();
+        const n = r.readUE();
+        for (let i = 0; i < n; i++)
+            r.readSE();
+    }
+    r.readUE(); // max_num_ref_frames
+    r.readBits(1); // gaps_in_frame_num_value_allowed_flag
+    const picWidthInMbsMinus1 = r.readUE();
+    const picHeightInMapUnitsMinus1 = r.readUE();
+    const frameMbsOnlyFlag = r.readBits(1);
+    let width = (picWidthInMbsMinus1 + 1) * 16;
+    let height = (2 - frameMbsOnlyFlag) * (picHeightInMapUnitsMinus1 + 1) * 16;
+    if (!frameMbsOnlyFlag)
+        r.readBits(1);
+    r.readBits(1); // direct_8x8_inference_flag
+    if (r.readBits(1)) {
+        // frame_cropping_flag — 4:2:0 crop units are 2px
+        const cl = r.readUE();
+        const cr = r.readUE();
+        const ct = r.readUE();
+        const cb = r.readUE();
+        width -= (cl + cr) * 2;
+        height -= (ct + cb) * 2;
+    }
+    return { profileIdc, constraintFlags, levelIdc, width, height };
+}
+function buildCodecString(info) {
+    const pp = info.profileIdc.toString(16).padStart(2, '0');
+    const cc = info.constraintFlags.toString(16).padStart(2, '0');
+    const ll = info.levelIdc.toString(16).padStart(2, '0');
+    return `avc1.${pp}${cc}${ll}`;
+}
+/** AVCDecoderConfigurationRecord from raw SPS/PPS NAL bodies (with NAL header byte). */
+function buildAvcC(sps, pps) {
+    const buf = Buffer.alloc(11 + sps.length + pps.length);
+    buf[0] = 1; // configurationVersion
+    buf[1] = sps[1]; // profile
+    buf[2] = sps[2]; // compatibility
+    buf[3] = sps[3]; // level
+    buf[4] = 0xff; // lengthSizeMinusOne = 3
+    buf[5] = 0xe1; // numOfSPS = 1
+    buf.writeUInt16BE(sps.length, 6);
+    Buffer.from(sps).copy(buf, 8);
+    const off = 8 + sps.length;
+    buf[off] = 1; // numOfPPS
+    buf.writeUInt16BE(pps.length, off + 1);
+    Buffer.from(pps).copy(buf, off + 3);
+    return buf;
+}
+/** Find the next Annex B start code (00 00 01 or 00 00 00 01). */
+function findStartCode(buf, offset) {
+    for (let i = offset; i < buf.length - 2; i++) {
+        if (buf[i] === 0 && buf[i + 1] === 0) {
+            if (buf[i + 2] === 1)
+                return { index: i, length: 3 };
+            if (buf[i + 2] === 0 && i + 3 < buf.length && buf[i + 3] === 1)
+                return { index: i, length: 4 };
+        }
+    }
+    return null;
+}
+/**
+ * Streaming Annex B parser. Feed raw capture bytes with push(); emits config
+ * once SPS+PPS are seen, then one Annex B access unit per VCL frame.
+ */
+class H264AnnexBParser {
+    constructor(cbs, fps = 30) {
+        this.cbs = cbs;
+        this.buffer = Buffer.alloc(0);
+        this.sps = null;
+        this.pps = null;
+        this.config = null;
+        // NALs (each Annex-B, start-code prefixed) buffered for the current access unit.
+        this.au = [];
+        this.auIsKey = false;
+        this.fps = fps;
+    }
+    reset() {
+        this.buffer = Buffer.alloc(0);
+        this.sps = null;
+        this.pps = null;
+        this.config = null;
+        this.au = [];
+        this.auIsKey = false;
+    }
+    push(chunk) {
+        this.buffer = this.buffer.length === 0 ? chunk : Buffer.concat([this.buffer, chunk]);
+        this.processBuffer();
+    }
+    processBuffer() {
+        let sc = findStartCode(this.buffer, 0);
+        if (!sc)
+            return;
+        while (true) {
+            const next = findStartCode(this.buffer, sc.index + sc.length);
+            if (!next)
+                break; // incomplete NAL — wait for more data
+            const nal = this.buffer.subarray(sc.index + sc.length, next.index);
+            // Keep the whole unit including start code so subscribers get Annex B.
+            const withStartCode = this.buffer.subarray(sc.index, next.index);
+            this.processNal(nal, withStartCode);
+            sc = next;
+        }
+        // Retain the trailing (unterminated) NAL for the next push.
+        if (sc.index > 0)
+            this.buffer = Buffer.from(this.buffer.subarray(sc.index));
+        // Guard against unbounded growth on a stuck stream.
+        if (this.buffer.length > 4 * 1024 * 1024) {
+            this.buffer = Buffer.from(this.buffer.subarray(-1024 * 1024));
+        }
+    }
+    processNal(nal, withStartCode) {
+        if (nal.length === 0)
+            return;
+        const nalType = nal[0] & 0x1f;
+        if (nalType === NAL_SPS) {
+            this.sps = Buffer.from(nal);
+            this.emitConfigIfReady();
+            return;
+        }
+        if (nalType === NAL_PPS) {
+            this.pps = Buffer.from(nal);
+            this.emitConfigIfReady();
+            return;
+        }
+        if (nalType === NAL_IDR || nalType === NAL_SLICE) {
+            // Flush any previously buffered AU before starting this VCL frame.
+            if (this.au.length > 0)
+                this.emitAccessUnit();
+            this.auIsKey = nalType === NAL_IDR;
+            this.au.push(Buffer.from(withStartCode));
+            this.emitAccessUnit();
+            return;
+        }
+        // SEI / AUD / other — attach to the current access unit.
+        this.au.push(Buffer.from(withStartCode));
+    }
+    emitAccessUnit() {
+        if (this.au.length === 0)
+            return;
+        if (!this.config) {
+            // Can't decode without config yet — drop until SPS/PPS arrive.
+            this.au = [];
+            this.auIsKey = false;
+            return;
+        }
+        const data = this.au.length === 1 ? this.au[0] : Buffer.concat(this.au);
+        this.cbs.onFrame(data, this.auIsKey);
+        this.au = [];
+        this.auIsKey = false;
+    }
+    emitConfigIfReady() {
+        if (this.config || !this.sps || !this.pps)
+            return;
+        const info = parseSps(this.sps.subarray(1));
+        this.config = {
+            codec: 'h264',
+            width: info.width,
+            height: info.height,
+            rotation: 0,
+            fps: this.fps,
+            codecString: buildCodecString(info),
+            sps: this.sps,
+            pps: this.pps,
+            avcC: buildAvcC(this.sps, this.pps),
+        };
+        this.cbs.onConfig(this.config);
+    }
+}
+exports.H264AnnexBParser = H264AnnexBParser;

package/dist/daemon/input-backends.js

@@ -0,0 +1,203 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.iosBackend = iosBackend;
+exports.androidBackend = androidBackend;
+const clampNorm = (v) => (v < 0 ? 0 : v > 1 ? 1 : v);
+// ── iOS / tvOS (XCUITest driver) ──────────────────────────────────────────────
+// key-name → XCUITest pressKey value
+const IOS_KEY = {
+    Backspace: 'delete',
+    Delete: 'delete',
+    Enter: 'enter',
+    Return: 'enter',
+    Tab: 'tab',
+    Space: 'space',
+    ' ': 'space',
+};
+// button/key-name → XCUITest pressButton value
+const IOS_BUTTON = {
+    home: 'home',
+    Home: 'home',
+    lock: 'lock',
+    Lock: 'lock',
+    power: 'lock',
+    Power: 'lock',
+    // tvOS remote (also reachable via the tvremote frame)
+    up: 'up',
+    down: 'down',
+    left: 'left',
+    right: 'right',
+    select: 'select',
+    menu: 'menu',
+    playPause: 'playPause',
+    ArrowUp: 'up',
+    ArrowDown: 'down',
+    ArrowLeft: 'left',
+    ArrowRight: 'right',
+};
+function iosBackend(driver) {
+    let size = null;
+    const dims = async () => {
+        if (!size) {
+            const info = await driver.deviceInfo();
+            size = { w: info.widthPoints, h: info.heightPoints };
+        }
+        return size;
+    };
+    const tvos = driver.platform === 'tvos';
+    return {
+        platform: tvos ? 'tvos' : 'ios',
+        capabilities(liveDrag) {
+            return {
+                touch: true,
+                drag: true,
+                multitouch: true,
+                buttons: ['home', 'lock'],
+                keyboard: true,
+                text: true,
+                tvRemote: tvos,
+                springboard: true,
+                liveDrag,
+                binaryPointer: false,
+                coord: 'normalized',
+            };
+        },
+        async tap(nx, ny, durationMs) {
+            const { w, h } = await dims();
+            await driver.tap(clampNorm(nx) * w, clampNorm(ny) * h, durationMs ? durationMs / 1000 : undefined);
+        },
+        async gesture(paths) {
+            const { w, h } = await dims();
+            const converted = paths.map((p) => ({
+                steps: p.steps.map((s) => ({
+                    x: clampNorm(s.nx) * w,
+                    y: clampNorm(s.ny) * h,
+                    dt: s.tMs / 1000,
+                })),
+            }));
+            await driver.gesturePath(converted);
+        },
+        async swipe(nsx, nsy, nex, ney, durationMs) {
+            const { w, h } = await dims();
+            await driver.swipe(clampNorm(nsx) * w, clampNorm(nsy) * h, clampNorm(nex) * w, clampNorm(ney) * h, durationMs / 1000);
+        },
+        async text(value) {
+            await driver.inputText(value);
+        },
+        async key(code, opts) {
+            // XCUITest pressKey is atomic — act on key-down, ignore the paired up.
+            if (opts && opts.down === false)
+                return;
+            const k = IOS_KEY[code];
+            if (k) {
+                await driver.pressKey(k);
+                return;
+            }
+            const b = IOS_BUTTON[code];
+            if (b)
+                await driver.pressButton(b);
+        },
+        async button(name, holdMs) {
+            const b = IOS_BUTTON[name];
+            if (b)
+                await driver.pressButton(b, holdMs ? holdMs / 1000 : undefined);
+        },
+    };
+}
+// ── Android (gRPC APK + adb) ───────────────────────────────────────────────────
+const ANDROID_KEYCODE = {
+    Home: 3,
+    home: 3,
+    Back: 4,
+    back: 4,
+    Enter: 66,
+    Return: 66,
+    Backspace: 67,
+    Delete: 67,
+    Tab: 61,
+    Space: 62,
+    ' ': 62,
+    Escape: 111,
+    Lock: 26,
+    Power: 26,
+    power: 26,
+    VolumeUp: 24,
+    volumeUp: 24,
+    VolumeDown: 25,
+    volumeDown: 25,
+    Menu: 82,
+    menu: 82,
+    Search: 84,
+    ArrowUp: 19,
+    ArrowDown: 20,
+    ArrowLeft: 21,
+    ArrowRight: 22,
+    up: 19,
+    down: 20,
+    left: 21,
+    right: 22,
+    select: 23,
+    playPause: 85,
+};
+function androidBackend(driver) {
+    let size = null;
+    const dims = async () => {
+        if (!size) {
+            const info = await driver.deviceInfo();
+            size = { w: info.widthPixels, h: info.heightPixels };
+        }
+        return size;
+    };
+    return {
+        platform: 'android',
+        capabilities(liveDrag) {
+            return {
+                touch: true,
+                drag: true,
+                multitouch: true,
+                buttons: ['home', 'back', 'menu', 'power', 'volumeUp', 'volumeDown'],
+                keyboard: true,
+                text: true,
+                tvRemote: true, // Android-TV dpad via keyevents
+                springboard: false,
+                liveDrag,
+                binaryPointer: false,
+                coord: 'normalized',
+            };
+        },
+        async tap(nx, ny) {
+            const { w, h } = await dims();
+            await driver.tap(clampNorm(nx) * w, clampNorm(ny) * h);
+        },
+        async gesture(paths) {
+            const { w, h } = await dims();
+            const converted = paths.map((p) => ({
+                steps: p.steps.map((s) => ({
+                    x: clampNorm(s.nx) * w,
+                    y: clampNorm(s.ny) * h,
+                    dt_ms: Math.round(s.tMs),
+                })),
+            }));
+            await driver.gesturePath(converted);
+        },
+        async swipe(nsx, nsy, nex, ney, durationMs) {
+            const { w, h } = await dims();
+            await driver.swipe(clampNorm(nsx) * w, clampNorm(nsy) * h, clampNorm(nex) * w, clampNorm(ney) * h, durationMs);
+        },
+        async text(value) {
+            await driver.inputText(value);
+        },
+        async key(code, opts) {
+            if (opts && opts.down === false)
+                return;
+            const kc = ANDROID_KEYCODE[code];
+            if (kc !== undefined)
+                await driver.pressKeyEvent(kc);
+        },
+        async button(name) {
+            const kc = ANDROID_KEYCODE[name];
+            if (kc !== undefined)
+                await driver.pressKeyEvent(kc);
+        },
+    };
+}

package/dist/daemon/input-protocol.js

@@ -0,0 +1,40 @@
+"use strict";
+/**
+ * Streaming device-input protocol (conductor ⇄ Argus).
+ *
+ * A persistent per-device WebSocket carries pointer/key/button/scroll/remote
+ * frames so continuous drags and fast typing stay low-latency, instead of the
+ * per-event HTTP the web driver uses. Conductor owns coord→device translation
+ * and keymaps; the client streams normalized 0..1 coordinates.
+ *
+ * See docs/device-input-migration.md for the design.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.INPUT_PROTOCOL_VERSION = void 0;
+exports.decodeClientFrame = decodeClientFrame;
+exports.INPUT_PROTOCOL_VERSION = 1;
+const CLIENT_FRAME_TYPES = new Set([
+    'select',
+    'pointer',
+    'key',
+    'text',
+    'button',
+    'scroll',
+    'tvremote',
+]);
+/** Parse and shallow-validate one text frame. Returns null on malformed input. */
+function decodeClientFrame(raw) {
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+    if (typeof parsed !== 'object' || parsed === null)
+        return null;
+    const t = parsed.t;
+    if (typeof t !== 'string' || !CLIENT_FRAME_TYPES.has(t))
+        return null;
+    return parsed;
+}

package/dist/daemon/input-router.js

@@ -0,0 +1,110 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InputRouter = void 0;
+/** Movement below this (normalized) counts as a tap, not a drag. */
+const TAP_EPSILON = 0.01;
+/** Default scroll gesture duration. */
+const SCROLL_DURATION_MS = 200;
+class InputRouter {
+    constructor(backend, opts = {}) {
+        this.backend = backend;
+        this.opts = opts;
+        this.sessions = new Map();
+        this.liveOpen = new Set();
+        this.now = opts.now ?? (() => Date.now());
+    }
+    get liveDrag() {
+        return this.opts.livePointer ? 'native' : 'buffered';
+    }
+    capabilities() {
+        return this.backend.capabilities(this.liveDrag);
+    }
+    /** Dispatch one frame. `select` is handled by the server, not here. */
+    async dispatch(frame) {
+        switch (frame.t) {
+            case 'pointer':
+                return this.handlePointer(frame.id ?? 0, frame.phase, frame.x, frame.y);
+            case 'key':
+                return this.backend.key(frame.code, { down: frame.down, mods: frame.mods });
+            case 'text':
+                return this.backend.text(frame.value);
+            case 'button':
+                return this.backend.button(frame.name, frame.holdMs);
+            case 'scroll':
+                return this.backend.swipe(frame.x, frame.y, frame.x - frame.dx, frame.y - frame.dy, SCROLL_DURATION_MS);
+            case 'tvremote':
+                return this.backend.button(frame.button, frame.holdMs);
+            case 'select':
+                return;
+        }
+    }
+    async handlePointer(id, phase, nx, ny) {
+        // Native held-touch path: stream straight through.
+        if (this.opts.livePointer) {
+            if (phase === 'down')
+                this.liveOpen.add(id);
+            if (phase === 'up' || phase === 'cancel')
+                this.liveOpen.delete(id);
+            return this.opts.livePointer.pointer(id, phase, nx, ny);
+        }
+        // Buffered path.
+        const t = this.now();
+        if (phase === 'down') {
+            this.sessions.set(id, {
+                steps: [{ nx, ny, tMs: 0 }],
+                lastStepAt: t,
+                startNx: nx,
+                startNy: ny,
+                moved: false,
+                done: false,
+                canceled: false,
+            });
+            return;
+        }
+        const s = this.sessions.get(id);
+        if (!s)
+            return; // move/up without a down — ignore
+        s.steps.push({ nx, ny, tMs: Math.max(0, t - s.lastStepAt) });
+        s.lastStepAt = t;
+        if (Math.hypot(nx - s.startNx, ny - s.startNy) > TAP_EPSILON)
+            s.moved = true;
+        if (phase === 'up' || phase === 'cancel') {
+            s.done = true;
+            s.canceled = phase === 'cancel';
+            if ([...this.sessions.values()].every((x) => x.done))
+                await this.flush();
+        }
+    }
+    async flush() {
+        const sessions = [...this.sessions.values()];
+        this.sessions.clear();
+        const active = sessions.filter((s) => !s.canceled);
+        if (active.length === 0)
+            return;
+        if (active.length === 1 && !active[0].moved) {
+            await this.backend.tap(active[0].startNx, active[0].startNy);
+            return;
+        }
+        await this.backend.gesture(active.map((s) => ({ steps: s.steps })));
+    }
+    /**
+     * Release any touch still held when the socket closes, so no finger is left
+     * stuck down. Buffered sessions flush as-is; live touches get a cancel.
+     */
+    async onClose() {
+        if (this.opts.livePointer) {
+            const open = [...this.liveOpen];
+            this.liveOpen.clear();
+            for (const id of open) {
+                await this.opts.livePointer.pointer(id, 'cancel', 0, 0).catch(() => { });
+            }
+            return;
+        }
+        if (this.sessions.size > 0) {
+            for (const s of this.sessions.values())
+                s.done = true;
+            await this.flush().catch(() => { });
+        }
+    }
+}
+exports.InputRouter = InputRouter;