@alivelabs/expo-orchestrator-react-client 0.1.0 → 0.2.0

package/README.md

@@ -28,39 +28,48 @@ function App() {
 
 ## Hook usage
 
+The live screen is an **H.264 stream decoded with WebCodecs** onto a `<canvas>`.
+The hook owns the decoder — pass `attachCanvas` to a `<canvas ref>`:
+
 ```tsx
 import { useExpoCiSession } from "@alivelabs/expo-orchestrator-react-client";
 
 function BuildPage() {
-  const { status, logs, frame, connected, error, sendInput, reconnect } = useExpoCiSession({
-    sessionId: "abc-123",
-    apiToken: "your-api-token",
-    baseUrl: "https://your-orchestrator.example.com",
-    autoConnect: true, // default
-    maxLogs: 2000,     // default
-  });
+  const { status, logs, attachCanvas, videoSupported, connected, sendInput, reconnect } =
+    useExpoCiSession({
+      sessionId: "abc-123",
+      apiToken: "your-api-token",
+      baseUrl: "https://your-orchestrator.example.com",
+      autoConnect: true, // default
+      maxLogs: 2000,     // default
+    });
 
   return (
     <div>
       <p>Status: {status} — {connected ? "live" : "disconnected"}</p>
-      {frame && <img src={frame} alt="simulator" />}
-      <button onClick={() => sendInput({ type: "keypress", key: "home" })}>Home</button>
+      {videoSupported ? <canvas ref={attachCanvas} /> : <p>Video not supported in this browser.</p>}
+      <button onClick={() => sendInput({ type: "button", button: "home" })}>Home</button>
       {!connected && <button onClick={reconnect}>Reconnect</button>}
     </div>
   );
 }
 ```
 
+WebCodecs needs a modern browser (Chrome/Edge, Safari 16.4+, recent Firefox) and
+a secure context (HTTPS or localhost); `videoSupported` is `false` otherwise. For
+full control you can import `SimulatorDecoder` and drive your own canvas.
+
 ## Exports reference
 
 | Export | Kind | Description |
 |---|---|---|
 | `SessionViewer` | Component | All-in-one: status badge, simulator screen, log console, interaction controls |
-| `SimulatorScreen` | Component | Renders the current frame; click = tap, click-drag = swipe |
+| `SimulatorScreen` | Component | Renders the live H.264 stream to a canvas; click = tap, click-drag = swipe |
 | `LogConsole` | Component | Scrollable, color-coded, auto-scrolling log viewer |
 | `StatusBadge` | Component | Colored pill showing session status |
-| `useExpoCiSession` | Hook | React hook wrapping `ExpoCiClient` with state management |
+| `useExpoCiSession` | Hook | React hook wrapping `ExpoCiClient` + the WebCodecs decoder |
 | `ExpoCiClient` | Class | Framework-agnostic client: REST + WebSocket with auto-reconnect |
+| `SimulatorDecoder` | Class | WebCodecs H.264 decoder painting to a `<canvas>` (advanced) |
 
 ### `<SessionViewer>` props
 
@@ -75,9 +84,13 @@ function BuildPage() {
 
 | Prop | Type | Required | Description |
 |---|---|---|---|
-| `frame` | `string \| null` | yes | Current frame as a `data:image/jpeg;base64,...` string |
+| `canvasRef` | `(canvas: HTMLCanvasElement \| null) => void` | yes | Ref callback wiring the canvas to the decoder (pass `attachCanvas`) |
+| `videoSupported` | `boolean` | yes | Whether WebCodecs is available; shows a notice when `false` |
+| `hasFrame` | `boolean` | yes | Whether a frame has painted yet (drives the placeholder) |
 | `onTap` | `(input: { type: "tap"; x: number; y: number }) => void` | no | Called on click |
-| `onSwipe` | `(input: { type: "swipe"; fromX: number; fromY: number; toX: number; toY: number }) => void` | no | Called on drag |
+| `onSwipe` | `(input: { type: "swipe"; startX; startY; endX; endY }) => void` | no | Called on drag |
+| `onKeyInput` | `(input: { type: "type" \| "key"; … }) => void` | no | Called for keystrokes while focused |
+| `maxHeight` | `number \| string` | no | Cap on rendered height (default `80vh`) |
 
 ### `<LogConsole>` props
 
@@ -115,9 +128,10 @@ client.getScreenshotObjectUrl()  // GET /api/sessions/:id/screenshot → object
 client.connect()                 // Open WebSocket (auto-reconnect with exponential backoff)
 client.disconnect()              // Close WebSocket and remove all listeners
 
-// Typed event emitter
-const off = client.on("video-frame", (msg) => console.log(msg.data));
+// Typed event emitter. Video arrives as raw encoded chunks (ArrayBuffer) —
+// feed them to a SimulatorDecoder, or let the hook do it for you.
+const off = client.on("video-chunk", (chunk) => decoder.push(chunk));
 off(); // unsubscribe
 ```
 
-Events: `open`, `close`, `connected`, `log`, `video-frame`, `status`, `error`.
+Events: `open`, `close`, `connected`, `log`, `video-chunk`, `status`, `error`.

package/dist/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,cAAc,EACd,sBAAsB,EAEvB,MAAM,YAAY,CAAC;AAIpB,KAAK,QAAQ,CAAC,CAAC,IAAI,CAAC,SAAS,SAAS,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,CAAC;AAGxE,KAAK,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAEpC,cAAM,iBAAiB,CAAC,MAAM,SAAS,QAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAwD;IAElF,EAAE,CAAC,CAAC,SAAS,MAAM,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI;IASxF,GAAG,CAAC,CAAC,SAAS,MAAM,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI;IAInF,IAAI,CAAC,CAAC,SAAS,MAAM,MAAM,GAAG,MAAM,EAClC,KAAK,EAAE,CAAC,EACR,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GACtD,IAAI;IAQP,kBAAkB,IAAI,IAAI;CAG3B;AAUD,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,YAAa,SAAQ,iBAAiB,CAAC,cAAc,CAAC;IACjE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAElC,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,SAAS,CAAS;gBAEd,EAAE,OAAiC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,mBAAmB;YAU7E,SAAS;IAgBvB,UAAU,IAAI,OAAO,CAAC,aAAa,CAAC;IAIpC,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IAI/B,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAU3D,sBAAsB,IAAI,OAAO,CAAC,MAAM,CAAC;IAc/C,OAAO,KAAK,KAAK,GAKhB;IAED,OAAO,IAAI,IAAI;IAMf,OAAO,CAAC,UAAU;IAyDlB,OAAO,CAAC,iBAAiB;IASzB,UAAU,IAAI,IAAI;CAYnB"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,cAAc,EACd,sBAAsB,EAEvB,MAAM,YAAY,CAAC;AAIpB,KAAK,QAAQ,CAAC,CAAC,IAAI,CAAC,SAAS,SAAS,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,CAAC;AAGxE,KAAK,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAEpC,cAAM,iBAAiB,CAAC,MAAM,SAAS,QAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAwD;IAElF,EAAE,CAAC,CAAC,SAAS,MAAM,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI;IASxF,GAAG,CAAC,CAAC,SAAS,MAAM,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI;IAInF,IAAI,CAAC,CAAC,SAAS,MAAM,MAAM,GAAG,MAAM,EAClC,KAAK,EAAE,CAAC,EACR,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GACtD,IAAI;IAQP,kBAAkB,IAAI,IAAI;CAG3B;AAUD,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,YAAa,SAAQ,iBAAiB,CAAC,cAAc,CAAC;IACjE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAElC,OAAO,CAAC,EAAE,CAA0B;IACpC,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,SAAS,CAAS;gBAEd,EAAE,OAAiC,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,mBAAmB;YAU7E,SAAS;IAgBvB,UAAU,IAAI,OAAO,CAAC,aAAa,CAAC;IAIpC,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC;IAI/B,SAAS,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAU3D,sBAAsB,IAAI,OAAO,CAAC,MAAM,CAAC;IAc/C,OAAO,KAAK,KAAK,GAKhB;IAED,OAAO,IAAI,IAAI;IAMf,OAAO,CAAC,UAAU;IA6DlB,OAAO,CAAC,iBAAiB;IASzB,UAAU,IAAI,IAAI;CAYnB"}

package/dist/client.js

@@ -97,6 +97,8 @@ export class ExpoCiClient extends TypedEventEmitter {
         if (this.destroyed)
             return;
         const socket = new WebSocket(this.wsUrl);
+        // Video arrives as binary frames; logs/status as JSON text.
+        socket.binaryType = "arraybuffer";
         this.ws = socket;
         socket.addEventListener("open", () => {
             if (this.destroyed) {
@@ -109,6 +111,11 @@ export class ExpoCiClient extends TypedEventEmitter {
         socket.addEventListener("message", (event) => {
             if (this.destroyed)
                 return;
+            // Binary = an encoded H.264 chunk; hand it straight to the decoder.
+            if (event.data instanceof ArrayBuffer) {
+                this.emit("video-chunk", event.data);
+                return;
+            }
             let msg;
             try {
                 msg = JSON.parse(event.data);
@@ -123,9 +130,6 @@ export class ExpoCiClient extends TypedEventEmitter {
                 case "log":
                     this.emit("log", msg);
                     break;
-                case "video-frame":
-                    this.emit("video-frame", msg);
-                    break;
                 case "status":
                     this.emit("status", msg);
                     break;

package/dist/components/SessionViewer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SessionViewer.d.ts","sourceRoot":"","sources":["../../src/components/SessionViewer.tsx"],"names":[],"mappings":"AAOA,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAyHD,wBAAgB,aAAa,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,kBAAkB,2CAgJ5F"}
+{"version":3,"file":"SessionViewer.d.ts","sourceRoot":"","sources":["../../src/components/SessionViewer.tsx"],"names":[],"mappings":"AAOA,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAyHD,wBAAgB,aAAa,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,kBAAkB,2CAwK5F"}

package/dist/components/SessionViewer.js

@@ -111,7 +111,7 @@ const PRESETS = [
     { label: "Enter", input: { type: "key", code: "Enter" } },
 ];
 export function SessionViewer({ sessionId, apiToken, baseUrl, className }) {
-    const { status, logs, frame, connected, error, sendInput, reconnect } = useExpoCiSession({
+    const { status, logs, attachCanvas, videoSupported, frameSize, connected, error, sendInput, reconnect, } = useExpoCiSession({
         baseUrl,
         sessionId,
         apiToken,
@@ -153,11 +153,14 @@ export function SessionViewer({ sessionId, apiToken, baseUrl, className }) {
     const handleSwipe = useCallback((input) => {
         sendInput(input).catch(() => { });
     }, [sendInput]);
+    const handleKeyInput = useCallback((input) => {
+        sendInput(input).catch(() => { });
+    }, [sendInput]);
     return (_jsxs("div", { style: rootStyle, className: className, children: [_jsxs("div", { style: headerStyle, children: [_jsxs("h2", { style: titleStyle, children: ["Session ", sessionId] }), _jsxs("div", { style: { display: "flex", alignItems: "center", gap: "8px", flexWrap: "wrap" }, children: [_jsx(StatusBadge, { status: status }), _jsx("span", { style: {
                                     fontSize: "11px",
                                     color: connected ? "#4ade80" : "#6b7280",
                                     fontWeight: 500,
-                                }, children: connected ? "● live" : "○ disconnected" }), !connected && (_jsx("button", { type: "button", style: btnStyle("secondary"), onClick: reconnect, children: "Reconnect" }))] })] }), error && _jsxs("div", { style: errorBannerStyle, children: ["\u26A0 ", error] }), _jsxs("div", { style: bodyStyle, children: [_jsx("div", { style: simulatorColStyle, children: _jsx(SimulatorScreen, { frame: frame, onTap: handleTap, onSwipe: handleSwipe }) }), _jsxs("div", { style: rightColStyle, children: [_jsxs("div", { style: controlsStyle, children: [_jsx("div", { style: {
+                                }, children: connected ? "● live" : "○ disconnected" }), !connected && (_jsx("button", { type: "button", style: btnStyle("secondary"), onClick: reconnect, children: "Reconnect" }))] })] }), error && _jsxs("div", { style: errorBannerStyle, children: ["\u26A0 ", error] }), _jsxs("div", { style: bodyStyle, children: [_jsx("div", { style: simulatorColStyle, children: _jsx(SimulatorScreen, { canvasRef: attachCanvas, videoSupported: videoSupported, hasFrame: frameSize !== null, onTap: handleTap, onSwipe: handleSwipe, onKeyInput: handleKeyInput }) }), _jsxs("div", { style: rightColStyle, children: [_jsxs("div", { style: controlsStyle, children: [_jsx("div", { style: {
                                             fontSize: "11px",
                                             fontWeight: 600,
                                             color: "#8b949e",

package/dist/components/SimulatorScreen.d.ts

@@ -1,17 +1,28 @@
 import type { SimulatorInput } from "../types.js";
 interface SimulatorScreenProps {
-    frame: string | null;
+    /** Ref callback wiring the rendering `<canvas>` to the decoder (from the hook). */
+    canvasRef: (canvas: HTMLCanvasElement | null) => void;
+    /** Whether this browser can decode the live H.264 stream (WebCodecs present). */
+    videoSupported: boolean;
+    /** True once the first frame has painted — drives the placeholder/canvas swap. */
+    hasFrame: boolean;
     onTap?: (input: Extract<SimulatorInput, {
         type: "tap";
     }>) => void;
     onSwipe?: (input: Extract<SimulatorInput, {
         type: "swipe";
     }>) => void;
-    /** Cap on the rendered simulator height; the image scales down proportionally
+    /** Forward the focused viewer's keystrokes to the simulator (text + key events). */
+    onKeyInput?: (input: Extract<SimulatorInput, {
+        type: "type";
+    } | {
+        type: "key";
+    }>) => void;
+    /** Cap on the rendered simulator height; the canvas scales down proportionally
      *  to fit. Accepts any CSS length. Defaults to `80vh` so the device never
      *  exceeds the viewport on a tall phone like iPhone 17 Pro Max. */
     maxHeight?: number | string;
 }
-export declare function SimulatorScreen({ frame, onTap, onSwipe, maxHeight, }: SimulatorScreenProps): import("react/jsx-runtime").JSX.Element;
+export declare function SimulatorScreen({ canvasRef, videoSupported, hasFrame, onTap, onSwipe, onKeyInput, maxHeight, }: SimulatorScreenProps): import("react/jsx-runtime").JSX.Element;
 export {};
 //# sourceMappingURL=SimulatorScreen.d.ts.map

package/dist/components/SimulatorScreen.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SimulatorScreen.d.ts","sourceRoot":"","sources":["../../src/components/SimulatorScreen.tsx"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,UAAU,oBAAoB;IAC5B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE;QAAE,IAAI,EAAE,KAAK,CAAA;KAAE,CAAC,KAAK,IAAI,CAAC;IAClE,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE;QAAE,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,KAAK,IAAI,CAAC;IACtE;;uEAEmE;IACnE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CAC7B;AAgFD,wBAAgB,eAAe,CAAC,EAC9B,KAAK,EACL,KAAK,EACL,OAAO,EACP,SAAkB,GACnB,EAAE,oBAAoB,2CA2GtB"}
+{"version":3,"file":"SimulatorScreen.d.ts","sourceRoot":"","sources":["../../src/components/SimulatorScreen.tsx"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,cAAc,EAA0C,MAAM,aAAa,CAAC;AAE1F,UAAU,oBAAoB;IAC5B,mFAAmF;IACnF,SAAS,EAAE,CAAC,MAAM,EAAE,iBAAiB,GAAG,IAAI,KAAK,IAAI,CAAC;IACtD,iFAAiF;IACjF,cAAc,EAAE,OAAO,CAAC;IACxB,kFAAkF;IAClF,QAAQ,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE;QAAE,IAAI,EAAE,KAAK,CAAA;KAAE,CAAC,KAAK,IAAI,CAAC;IAClE,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE;QAAE,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,KAAK,IAAI,CAAC;IACtE,oFAAoF;IACpF,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,cAAc,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,IAAI,EAAE,KAAK,CAAA;KAAE,CAAC,KAAK,IAAI,CAAC;IAC1F;;uEAEmE;IACnE,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;CAC7B;AAmJD,wBAAgB,eAAe,CAAC,EAC9B,SAAS,EACT,cAAc,EACd,QAAQ,EACR,KAAK,EACL,OAAO,EACP,UAAU,EACV,SAAkB,GACnB,EAAE,oBAAoB,2CA2JtB"}

package/dist/components/SimulatorScreen.js

@@ -1,6 +1,66 @@
 import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
-import { useCallback, useRef, useState } from "react";
+// biome-ignore-all lint/a11y/noNoninteractiveTabindex: the simulator screen is an application-role surface that forwards keyboard + pointer input and must be focusable.
+import { useCallback, useRef, useState, } from "react";
 const SWIPE_THRESHOLD_PX = 5;
+// Browser KeyboardEvent.code values baguette accepts (mirrors KeyCodeSchema in
+// @alivelabs/expo-orchestrator-schemas). Built at module load to stay compact.
+const SUPPORTED_KEY_CODES = (() => {
+    const set = new Set([
+        "Enter",
+        "Escape",
+        "Backspace",
+        "Tab",
+        "Space",
+        "ArrowUp",
+        "ArrowDown",
+        "ArrowLeft",
+        "ArrowRight",
+        "Minus",
+        "Equal",
+        "BracketLeft",
+        "BracketRight",
+        "Backslash",
+        "Semicolon",
+        "Quote",
+        "Backquote",
+        "Comma",
+        "Period",
+        "Slash",
+    ]);
+    for (let i = 0; i < 26; i += 1)
+        set.add(`Key${String.fromCharCode(65 + i)}`);
+    for (let i = 0; i < 10; i += 1)
+        set.add(`Digit${i}`);
+    return set;
+})();
+function isSupportedKeyCode(code) {
+    return SUPPORTED_KEY_CODES.has(code);
+}
+/**
+ * Translate a browser keydown into a simulator input. A printable character
+ * (no Ctrl/Cmd) is sent as `type` so layout/shift resolve correctly; everything
+ * else (Enter, Backspace, arrows, shortcuts like Cmd+A) goes as a `key` event
+ * with modifiers. Returns null for keys baguette can't take (e.g. F-keys).
+ */
+function mapKeyboardEvent(e) {
+    const usesCommand = e.ctrlKey || e.metaKey;
+    if (e.key.length === 1 && !usesCommand) {
+        return { type: "type", text: e.key };
+    }
+    if (isSupportedKeyCode(e.code)) {
+        const modifiers = [];
+        if (e.shiftKey)
+            modifiers.push("shift");
+        if (e.ctrlKey)
+            modifiers.push("control");
+        if (e.altKey)
+            modifiers.push("option");
+        if (e.metaKey)
+            modifiers.push("command");
+        return { type: "key", code: e.code, modifiers: modifiers.length > 0 ? modifiers : undefined };
+    }
+    return null;
+}
 // The outer titanium-style bezel that makes the screen read as a physical
 // device. The screen (wrapper) sits inside it with its own rounded corners.
 const deviceFrameStyle = {
@@ -11,6 +71,12 @@ const deviceFrameStyle = {
     boxShadow: "0 24px 60px -18px rgba(0,0,0,0.75), 0 4px 14px rgba(0,0,0,0.4)," +
         " inset 0 0 0 1px rgba(255,255,255,0.05), inset 0 1.5px 1px rgba(255,255,255,0.12)",
     lineHeight: 0,
+    transition: "box-shadow 0.18s ease",
+};
+// Extra ring + soft glow layered onto the bezel shadow while the screen is
+// focused, so it's clear keystrokes are being forwarded to the simulator.
+const focusedFrameStyle = {
+    boxShadow: `${deviceFrameStyle.boxShadow}, 0 0 0 2px rgba(59,130,246,0.7), 0 0 18px -4px rgba(59,130,246,0.5)`,
 };
 const wrapperStyle = {
     position: "relative",
@@ -22,6 +88,9 @@ const wrapperStyle = {
     userSelect: "none",
     lineHeight: 0,
     boxShadow: "inset 0 0 0 2px rgba(0,0,0,0.85)",
+    // The bezel renders the focus ring; suppress the default outline on the
+    // focusable screen itself.
+    outline: "none",
 };
 const basePlaceholderStyle = {
     // iPhone 17 logical resolution (402×874 pt). Using aspect-ratio keeps the
@@ -43,34 +112,44 @@ const basePlaceholderStyle = {
     // here or the stacked glyph + text collapse onto each other.
     lineHeight: 1.4,
 };
-const baseImgStyle = {
-    display: "block",
+const baseCanvasStyle = {
     maxWidth: "100%",
     width: "auto",
     height: "auto",
     pointerEvents: "none",
 };
-/** Convert a rendered-canvas pixel position to the image's natural pixel coords */
+/** Convert a rendered pixel position to the canvas's natural (device) pixel coords. */
 function toNatural(el, clientX, clientY) {
     const rect = el.getBoundingClientRect();
-    const scaleX = el.naturalWidth / rect.width;
-    const scaleY = el.naturalHeight / rect.height;
+    const scaleX = el.width / rect.width;
+    const scaleY = el.height / rect.height;
     return {
         x: Math.round((clientX - rect.left) * scaleX),
         y: Math.round((clientY - rect.top) * scaleY),
     };
 }
-export function SimulatorScreen({ frame, onTap, onSwipe, maxHeight = "80vh", }) {
-    const imgRef = useRef(null);
+export function SimulatorScreen({ canvasRef, videoSupported, hasFrame, onTap, onSwipe, onKeyInput, maxHeight = "80vh", }) {
+    const canvasElRef = useRef(null);
     const dragRef = useRef(null);
-    const imgStyle = { ...baseImgStyle, maxHeight };
-    // Drive the placeholder by height so its width derives from the iPhone 17
-    // aspect ratio (a block div would otherwise fill the column width and distort).
+    const [focused, setFocused] = useState(false);
+    // The canvas only drives layout once a frame has painted (it's 0×0 before);
+    // until then the placeholder gives the device its iPhone-aspect footprint.
+    const canvasStyle = {
+        ...baseCanvasStyle,
+        maxHeight,
+        display: hasFrame ? "block" : "none",
+    };
     const placeholderStyle = { ...basePlaceholderStyle, height: maxHeight };
+    // Wire the canvas into both the decoder (parent callback) and our local ref
+    // for pointer-coordinate math.
+    const setCanvas = useCallback((el) => {
+        canvasElRef.current = el;
+        canvasRef(el);
+    }, [canvasRef]);
     const handleMouseDown = useCallback((e) => {
-        if (!imgRef.current)
+        if (!canvasElRef.current)
             return;
-        const natural = toNatural(imgRef.current, e.clientX, e.clientY);
+        const natural = toNatural(canvasElRef.current, e.clientX, e.clientY);
         dragRef.current = {
             startX: e.clientX,
             startY: e.clientY,
@@ -81,7 +160,7 @@ export function SimulatorScreen({ frame, onTap, onSwipe, maxHeight = "80vh", })
     const [ripple, setRipple] = useState(null);
     const rippleCounter = useRef(0);
     const handleMouseUp = useCallback((e) => {
-        if (!dragRef.current || !imgRef.current)
+        if (!dragRef.current || !canvasElRef.current)
             return;
         const drag = dragRef.current;
         dragRef.current = null;
@@ -90,17 +169,17 @@ export function SimulatorScreen({ frame, onTap, onSwipe, maxHeight = "80vh", })
         const dist = Math.sqrt(dx * dx + dy * dy);
         if (dist < SWIPE_THRESHOLD_PX) {
             // Tap
-            const natural = toNatural(imgRef.current, e.clientX, e.clientY);
+            const natural = toNatural(canvasElRef.current, e.clientX, e.clientY);
             onTap?.({ type: "tap", x: natural.x, y: natural.y });
             // Show ripple at rendered coords
-            const rect = imgRef.current.getBoundingClientRect();
+            const rect = canvasElRef.current.getBoundingClientRect();
             const id = ++rippleCounter.current;
             setRipple({ x: e.clientX - rect.left, y: e.clientY - rect.top, id });
             setTimeout(() => setRipple((r) => (r?.id === id ? null : r)), 500);
         }
         else {
             // Swipe
-            const naturalEnd = toNatural(imgRef.current, e.clientX, e.clientY);
+            const naturalEnd = toNatural(canvasElRef.current, e.clientX, e.clientY);
             onSwipe?.({
                 type: "swipe",
                 startX: drag.naturalStartX,
@@ -110,13 +189,25 @@ export function SimulatorScreen({ frame, onTap, onSwipe, maxHeight = "80vh", })
             });
         }
     }, [onTap, onSwipe]);
-    return (_jsx("div", { style: deviceFrameStyle, children: _jsxs("div", { style: wrapperStyle, role: "application", "aria-label": "iOS simulator \u2014 click to tap, drag to swipe", onMouseDown: handleMouseDown, onMouseUp: handleMouseUp, children: [frame ? (_jsx("img", { ref: imgRef, src: frame, alt: "iOS Simulator", style: imgStyle, draggable: false })) : (_jsxs("div", { style: placeholderStyle, children: [_jsx("span", { style: { fontWeight: 500 }, children: "Waiting for the simulator" }), _jsx("div", { style: { display: "flex", gap: "6px" }, children: [0, 1, 2].map((i) => (_jsx("span", { style: {
+    const handleKeyDown = useCallback((e) => {
+        if (!onKeyInput)
+            return;
+        const input = mapKeyboardEvent(e);
+        if (!input)
+            return;
+        // Keep the keystroke off the page (no scrolling on space/arrows, no focus
+        // loss on Tab) — it goes to the simulator instead.
+        e.preventDefault();
+        onKeyInput(input);
+    }, [onKeyInput]);
+    const frameStyle = focused ? { ...deviceFrameStyle, ...focusedFrameStyle } : deviceFrameStyle;
+    return (_jsx("div", { style: frameStyle, children: _jsxs("div", { style: wrapperStyle, role: "application", "aria-label": "iOS simulator \u2014 click to focus; tap, drag to swipe, type to send keys", tabIndex: 0, onMouseDown: handleMouseDown, onMouseUp: handleMouseUp, onKeyDown: handleKeyDown, onFocus: () => setFocused(true), onBlur: () => setFocused(false), children: [videoSupported && (_jsx("canvas", { ref: setCanvas, style: canvasStyle, "aria-label": "iOS Simulator" })), !videoSupported ? (_jsxs("div", { style: placeholderStyle, children: [_jsx("span", { style: { fontWeight: 500 }, children: "Live video isn't supported here" }), _jsx("span", { style: { fontSize: "11px", textAlign: "center", maxWidth: "80%" }, children: "This browser can't decode the stream (WebCodecs unavailable). Try a recent Chrome, Edge, or Safari 16.4+." })] })) : (!hasFrame && (_jsxs("div", { style: placeholderStyle, children: [_jsx("span", { style: { fontWeight: 500 }, children: "Waiting for the simulator" }), _jsx("div", { style: { display: "flex", gap: "6px" }, children: [0, 1, 2].map((i) => (_jsx("span", { style: {
                                     width: 6,
                                     height: 6,
                                     borderRadius: "50%",
                                     backgroundColor: "currentColor",
                                     animation: `expo-pulse 1.2s ease-in-out ${i * 0.2}s infinite`,
-                                } }, i))) })] })), ripple && (_jsx("span", { style: {
+                                } }, i))) })] }))), ripple && (_jsx("span", { style: {
                         position: "absolute",
                         left: ripple.x - 14,
                         top: ripple.y - 14,

package/dist/decoder.d.ts

@@ -0,0 +1,34 @@
+export interface FrameSize {
+    width: number;
+    height: number;
+}
+/**
+ * Owns a `VideoDecoder` and paints decoded frames onto a caller-provided
+ * `<canvas>`. Feed it raw chunks via {@link push}; attach the canvas with
+ * {@link setCanvas}. One instance per session; `dispose()` when done.
+ */
+export declare class SimulatorDecoder {
+    /** Whether this environment can decode the stream at all (needs WebCodecs). */
+    static isSupported(): boolean;
+    private decoder;
+    private canvas;
+    private ctx;
+    private size;
+    private onResizeCb;
+    private nextTimestamp;
+    private awaitingKeyframe;
+    private disposed;
+    setCanvas(canvas: HTMLCanvasElement | null): void;
+    onResize(cb: ((size: FrameSize) => void) | null): void;
+    get frameSize(): FrameSize | null;
+    push(chunk: ArrayBuffer): void;
+    dispose(): void;
+    private configure;
+    private decode;
+    private paintFrame;
+    private paintJpeg;
+    private updateSize;
+    private onDecoderError;
+    private closeDecoder;
+}
+//# sourceMappingURL=decoder.d.ts.map

package/dist/decoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"decoder.d.ts","sourceRoot":"","sources":["../src/decoder.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AACH,qBAAa,gBAAgB;IAC3B,+EAA+E;IAC/E,MAAM,CAAC,WAAW,IAAI,OAAO;IAI7B,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,MAAM,CAAkC;IAChD,OAAO,CAAC,GAAG,CAAyC;IACpD,OAAO,CAAC,IAAI,CAA0B;IACtC,OAAO,CAAC,UAAU,CAA4C;IAG9D,OAAO,CAAC,aAAa,CAAK;IAG1B,OAAO,CAAC,gBAAgB,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAS;IAEzB,SAAS,CAAC,MAAM,EAAE,iBAAiB,GAAG,IAAI,GAAG,IAAI;IASjD,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,SAAS,KAAK,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI;IAItD,IAAI,SAAS,IAAI,SAAS,GAAG,IAAI,CAEhC;IAED,IAAI,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAqB9B,OAAO,IAAI,IAAI;IAUf,OAAO,CAAC,SAAS;IAsBjB,OAAO,CAAC,MAAM;IAsBd,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,SAAS;IAiBjB,OAAO,CAAC,UAAU;IAWlB,OAAO,CAAC,cAAc;IAKtB,OAAO,CAAC,YAAY;CAWrB"}

package/dist/decoder.js

@@ -0,0 +1,187 @@
+// WebCodecs H.264 decoder for the simulator stream. The server forwards
+// `baguette serve`'s AVCC chunks verbatim: each is `[tag byte][payload]` with
+//   0x01 avcC description — feed to VideoDecoder.configure
+//   0x02 keyframe (IDR)
+//   0x03 delta frame
+//   0x04 standalone JPEG — painted immediately for an instant first frame
+// We decode H.264 with a VideoDecoder and paint each frame onto a <canvas>.
+const TAG_DESCRIPTION = 0x01;
+const TAG_KEYFRAME = 0x02;
+const TAG_DELTA = 0x03;
+const TAG_JPEG_SEED = 0x04;
+/**
+ * Owns a `VideoDecoder` and paints decoded frames onto a caller-provided
+ * `<canvas>`. Feed it raw chunks via {@link push}; attach the canvas with
+ * {@link setCanvas}. One instance per session; `dispose()` when done.
+ */
+export class SimulatorDecoder {
+    constructor() {
+        this.decoder = null;
+        this.canvas = null;
+        this.ctx = null;
+        this.size = null;
+        this.onResizeCb = null;
+        // Monotonic, strictly-increasing microsecond timestamps. The values are
+        // arbitrary (we render immediately) but EncodedVideoChunk requires them.
+        this.nextTimestamp = 0;
+        // Until the first keyframe arrives after (re)configuring, deltas reference
+        // frames we never decoded — skip them to avoid decoder errors.
+        this.awaitingKeyframe = true;
+        this.disposed = false;
+    }
+    /** Whether this environment can decode the stream at all (needs WebCodecs). */
+    static isSupported() {
+        return typeof globalThis !== "undefined" && "VideoDecoder" in globalThis;
+    }
+    setCanvas(canvas) {
+        this.canvas = canvas;
+        this.ctx = canvas ? canvas.getContext("2d") : null;
+        if (canvas && this.size) {
+            canvas.width = this.size.width;
+            canvas.height = this.size.height;
+        }
+    }
+    onResize(cb) {
+        this.onResizeCb = cb;
+    }
+    get frameSize() {
+        return this.size;
+    }
+    push(chunk) {
+        if (this.disposed || chunk.byteLength === 0)
+            return;
+        const bytes = new Uint8Array(chunk);
+        const tag = bytes[0];
+        const payload = bytes.subarray(1);
+        switch (tag) {
+            case TAG_DESCRIPTION:
+                this.configure(payload);
+                break;
+            case TAG_JPEG_SEED:
+                this.paintJpeg(payload);
+                break;
+            case TAG_KEYFRAME:
+                this.decode(payload, "key");
+                break;
+            case TAG_DELTA:
+                this.decode(payload, "delta");
+                break;
+        }
+    }
+    dispose() {
+        this.disposed = true;
+        this.closeDecoder();
+        this.canvas = null;
+        this.ctx = null;
+        this.onResizeCb = null;
+    }
+    // ── internals ────────────────────────────────────────────────────────────
+    configure(description) {
+        if (!SimulatorDecoder.isSupported() || description.byteLength < 4)
+            return;
+        this.closeDecoder();
+        const decoder = new VideoDecoder({
+            output: (frame) => this.paintFrame(frame),
+            error: (err) => this.onDecoderError(err),
+        });
+        try {
+            decoder.configure({
+                codec: codecFromAvcC(description),
+                description,
+                optimizeForLatency: true,
+            });
+        }
+        catch {
+            // Unsupported profile/level on this platform — leave video unsupported.
+            decoder.close();
+            return;
+        }
+        this.decoder = decoder;
+        this.awaitingKeyframe = true;
+    }
+    decode(data, type) {
+        const decoder = this.decoder;
+        if (!decoder || decoder.state !== "configured")
+            return;
+        if (type === "key") {
+            this.awaitingKeyframe = false;
+        }
+        else if (this.awaitingKeyframe) {
+            return; // can't decode a delta before its keyframe
+        }
+        try {
+            decoder.decode(new EncodedVideoChunk({
+                type,
+                timestamp: this.nextTimestamp++,
+                data,
+            }));
+        }
+        catch {
+            // A bad/out-of-order chunk — wait for the next keyframe to resync.
+            this.awaitingKeyframe = true;
+        }
+    }
+    paintFrame(frame) {
+        const width = frame.displayWidth;
+        const height = frame.displayHeight;
+        this.updateSize(width, height);
+        if (this.canvas && this.ctx) {
+            this.ctx.drawImage(frame, 0, 0, this.canvas.width, this.canvas.height);
+        }
+        frame.close();
+    }
+    paintJpeg(jpeg) {
+        if (jpeg.byteLength === 0 || typeof createImageBitmap !== "function")
+            return;
+        // Copy into a fresh buffer so the Blob owns its bytes.
+        const blob = new Blob([jpeg.slice()], { type: "image/jpeg" });
+        createImageBitmap(blob)
+            .then((bitmap) => {
+            this.updateSize(bitmap.width, bitmap.height);
+            if (this.canvas && this.ctx) {
+                this.ctx.drawImage(bitmap, 0, 0, this.canvas.width, this.canvas.height);
+            }
+            bitmap.close();
+        })
+            .catch(() => {
+            /* a stray/partial seed — ignore */
+        });
+    }
+    updateSize(width, height) {
+        if (width === 0 || height === 0)
+            return;
+        if (this.size && this.size.width === width && this.size.height === height)
+            return;
+        this.size = { width, height };
+        if (this.canvas) {
+            this.canvas.width = width;
+            this.canvas.height = height;
+        }
+        this.onResizeCb?.(this.size);
+    }
+    onDecoderError(_err) {
+        // Reset and wait for the next server-forced keyframe to recover.
+        this.awaitingKeyframe = true;
+    }
+    closeDecoder() {
+        const decoder = this.decoder;
+        this.decoder = null;
+        if (decoder && decoder.state !== "closed") {
+            try {
+                decoder.close();
+            }
+            catch {
+                /* already closed */
+            }
+        }
+    }
+}
+/**
+ * Derive the WebCodecs codec string from an avcC configuration record. Bytes
+ * 1–3 are AVCProfileIndication / profile_compatibility / AVCLevelIndication,
+ * which form the `avc1.PPCCLL` suffix VideoDecoder.configure expects.
+ */
+function codecFromAvcC(avcc) {
+    const hex = (b) => b.toString(16).padStart(2, "0");
+    return `avc1.${hex(avcc[1])}${hex(avcc[2])}${hex(avcc[3])}`;
+}

package/dist/index.d.ts

@@ -5,6 +5,8 @@ export type { SessionViewerProps } from "./components/SessionViewer.js";
 export { SessionViewer } from "./components/SessionViewer.js";
 export { SimulatorScreen } from "./components/SimulatorScreen.js";
 export { StatusBadge } from "./components/StatusBadge.js";
-export type { ClientEventMap, ListSimulatorsResponse, LogEntry, LogLevel, SessionDetail, SessionLogs, SessionStatus, SimulatorButton, SimulatorDevice, SimulatorInput, SimulatorInputResponse, SimulatorKeyCode, SimulatorKeyModifier, UseExpoCiSessionOptions, UseExpoCiSessionResult, WsConnectedMessage, WsErrorMessage, WsLogMessage, WsMessage, WsStatusMessage, WsVideoFrameMessage, } from "./types.js";
+export type { FrameSize } from "./decoder.js";
+export { SimulatorDecoder } from "./decoder.js";
+export type { ClientEventMap, ListSimulatorsResponse, LogEntry, LogLevel, SessionDetail, SessionLogs, SessionStatus, SimulatorButton, SimulatorDevice, SimulatorInput, SimulatorInputResponse, SimulatorKeyCode, SimulatorKeyModifier, UseExpoCiSessionOptions, UseExpoCiSessionResult, WsConnectedMessage, WsErrorMessage, WsLogMessage, WsMessage, WsStatusMessage, } from "./types.js";
 export { useExpoCiSession } from "./useExpoCiSession.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,YAAY,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAExE,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAE1D,YAAY,EACV,cAAc,EACd,sBAAsB,EACtB,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,WAAW,EACX,aAAa,EACb,eAAe,EACf,eAAe,EACf,cAAc,EACd,sBAAsB,EACtB,gBAAgB,EAChB,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,kBAAkB,EAClB,cAAc,EACd,YAAY,EACZ,SAAS,EACT,eAAe,EACf,mBAAmB,GACpB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,YAAY,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,YAAY,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AAExE,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAC;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,iCAAiC,CAAC;AAClE,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,YAAY,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD,YAAY,EACV,cAAc,EACd,sBAAsB,EACtB,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,WAAW,EACX,aAAa,EACb,eAAe,EACf,eAAe,EACf,cAAc,EACd,sBAAsB,EACtB,gBAAgB,EAChB,oBAAoB,EACpB,uBAAuB,EACvB,sBAAsB,EACtB,kBAAkB,EAClB,cAAc,EACd,YAAY,EACZ,SAAS,EACT,eAAe,GAChB,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC"}

package/dist/index.js

@@ -5,5 +5,7 @@ export { LogConsole } from "./components/LogConsole.js";
 export { SessionViewer } from "./components/SessionViewer.js";
 export { SimulatorScreen } from "./components/SimulatorScreen.js";
 export { StatusBadge } from "./components/StatusBadge.js";
+// Decoder (advanced: drive your own canvas)
+export { SimulatorDecoder } from "./decoder.js";
 // Hook
 export { useExpoCiSession } from "./useExpoCiSession.js";

package/dist/types.d.ts

@@ -20,13 +20,6 @@ export interface WsLogMessage {
         message: string;
     };
 }
-export interface WsVideoFrameMessage {
-    type: "video-frame";
-    sessionId: string;
-    timestamp: string;
-    /** base64 JPEG with NO data-URI prefix */
-    data: string;
-}
 export interface WsStatusMessage {
     type: "status";
     sessionId: string;
@@ -43,13 +36,16 @@ export interface WsErrorMessage {
         message: string;
     };
 }
-export type WsMessage = WsConnectedMessage | WsLogMessage | WsVideoFrameMessage | WsStatusMessage | WsErrorMessage;
+/** JSON messages pushed down the WebSocket. Live video is sent as binary
+ *  frames instead — see the `video-chunk` client event. */
+export type WsMessage = WsConnectedMessage | WsLogMessage | WsStatusMessage | WsErrorMessage;
 export interface ClientEventMap {
     open: undefined;
     close: undefined;
     connected: WsConnectedMessage;
     log: WsLogMessage;
-    "video-frame": WsVideoFrameMessage;
+    /** One raw encoded H.264/AVCC chunk (`[tag][payload]`) for the decoder. */
+    "video-chunk": ArrayBuffer;
     status: WsStatusMessage;
     error: WsErrorMessage;
 }
@@ -65,8 +61,16 @@ export interface UseExpoCiSessionOptions {
 export interface UseExpoCiSessionResult {
     status: SessionStatus | null;
     logs: LogEntry[];
-    /** Current frame as a data:image/jpeg;base64,... string, or null */
-    frame: string | null;
+    /** Attach the `<canvas>` the decoder paints the live H.264 stream onto.
+     *  Pass via a ref callback; pass `null` on unmount. */
+    attachCanvas: (canvas: HTMLCanvasElement | null) => void;
+    /** Whether this browser can decode the stream (WebCodecs available). */
+    videoSupported: boolean;
+    /** Natural pixel size of the decoded video, once the first frame paints. */
+    frameSize: {
+        width: number;
+        height: number;
+    } | null;
     connected: boolean;
     error: string | null;
     sendInput: (input: SimulatorInput) => Promise<import("@alivelabs/expo-orchestrator-schemas").SimulatorInputResponse>;

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAE/F,YAAY,EACV,sBAAsB,EACtB,GAAG,EACH,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,aAAa,EACb,eAAe,EACf,eAAe,EACf,cAAc,EACd,sBAAsB,EACtB,gBAAgB,EAChB,oBAAoB,EACpB,UAAU,GACX,MAAM,sCAAsC,CAAC;AAE9C,0CAA0C;AAC1C,MAAM,MAAM,QAAQ,GAAG,GAAG,CAAC;AAE3B,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,GAAG,EAAE,CAAC;CACb;AAMD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,KAAK,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE;QAAE,KAAK,EAAE,OAAO,sCAAsC,EAAE,QAAQ,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3F;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,aAAa,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE;QAAE,MAAM,EAAE,aAAa,CAAA;KAAE,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3B;AAED,MAAM,MAAM,SAAS,GACjB,kBAAkB,GAClB,YAAY,GACZ,mBAAmB,GACnB,eAAe,GACf,cAAc,CAAC;AAInB,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,SAAS,CAAC;IACjB,SAAS,EAAE,kBAAkB,CAAC;IAC9B,GAAG,EAAE,YAAY,CAAC;IAClB,aAAa,EAAE,mBAAmB,CAAC;IACnC,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,cAAc,CAAC;CACvB;AAID,MAAM,WAAW,uBAAuB;IACtC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,wCAAwC;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,QAAQ,EAAE,CAAC;IACjB,oEAAoE;IACpE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,EAAE,CACT,KAAK,EAAE,cAAc,KAClB,OAAO,CAAC,OAAO,sCAAsC,EAAE,sBAAsB,CAAC,CAAC;IACpF,SAAS,EAAE,MAAM,IAAI,CAAC;CACvB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,sCAAsC,CAAC;AAE/F,YAAY,EACV,sBAAsB,EACtB,GAAG,EACH,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,aAAa,EACb,eAAe,EACf,eAAe,EACf,cAAc,EACd,sBAAsB,EACtB,gBAAgB,EAChB,oBAAoB,EACpB,UAAU,GACX,MAAM,sCAAsC,CAAC;AAE9C,0CAA0C;AAC1C,MAAM,MAAM,QAAQ,GAAG,GAAG,CAAC;AAE3B,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,GAAG,EAAE,CAAC;CACb;AAMD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,KAAK,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE;QAAE,KAAK,EAAE,OAAO,sCAAsC,EAAE,QAAQ,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3F;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE;QAAE,MAAM,EAAE,aAAa,CAAA;KAAE,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;CAC3B;AAED;2DAC2D;AAC3D,MAAM,MAAM,SAAS,GAAG,kBAAkB,GAAG,YAAY,GAAG,eAAe,GAAG,cAAc,CAAC;AAI7F,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,SAAS,CAAC;IAChB,KAAK,EAAE,SAAS,CAAC;IACjB,SAAS,EAAE,kBAAkB,CAAC;IAC9B,GAAG,EAAE,YAAY,CAAC;IAClB,2EAA2E;IAC3E,aAAa,EAAE,WAAW,CAAC;IAC3B,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,cAAc,CAAC;CACvB;AAID,MAAM,WAAW,uBAAuB;IACtC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,wCAAwC;IACxC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC;IAC7B,IAAI,EAAE,QAAQ,EAAE,CAAC;IACjB;2DACuD;IACvD,YAAY,EAAE,CAAC,MAAM,EAAE,iBAAiB,GAAG,IAAI,KAAK,IAAI,CAAC;IACzD,wEAAwE;IACxE,cAAc,EAAE,OAAO,CAAC;IACxB,4EAA4E;IAC5E,SAAS,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACpD,SAAS,EAAE,OAAO,CAAC;IACnB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,EAAE,CACT,KAAK,EAAE,cAAc,KAClB,OAAO,CAAC,OAAO,sCAAsC,EAAE,sBAAsB,CAAC,CAAC;IACpF,SAAS,EAAE,MAAM,IAAI,CAAC;CACvB"}

package/dist/useExpoCiSession.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useExpoCiSession.d.ts","sourceRoot":"","sources":["../src/useExpoCiSession.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAKV,uBAAuB,EACvB,sBAAsB,EACvB,MAAM,YAAY,CAAC;AAIpB,wBAAgB,gBAAgB,CAAC,EAC/B,OAAO,EACP,SAAS,EACT,QAAQ,EACR,WAAkB,EAClB,OAA0B,GAC3B,EAAE,uBAAuB,GAAG,sBAAsB,CA+FlD"}
+{"version":3,"file":"useExpoCiSession.d.ts","sourceRoot":"","sources":["../src/useExpoCiSession.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAKV,uBAAuB,EACvB,sBAAsB,EACvB,MAAM,YAAY,CAAC;AAIpB,wBAAgB,gBAAgB,CAAC,EAC/B,OAAO,EACP,SAAS,EACT,QAAQ,EACR,WAAkB,EAClB,OAA0B,GAC3B,EAAE,uBAAuB,GAAG,sBAAsB,CA8HlD"}

package/dist/useExpoCiSession.js

@@ -1,15 +1,27 @@
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { ExpoCiClient } from "./client.js";
+import { SimulatorDecoder } from "./decoder.js";
 const DEFAULT_MAX_LOGS = 2000;
 export function useExpoCiSession({ baseUrl, sessionId, apiToken, autoConnect = true, maxLogs = DEFAULT_MAX_LOGS, }) {
     const [status, setStatus] = useState(null);
     const [logs, setLogs] = useState([]);
-    const [frame, setFrame] = useState(null);
+    const [frameSize, setFrameSize] = useState(null);
     const [connected, setConnected] = useState(false);
     const [error, setError] = useState(null);
+    const videoSupported = useMemo(() => SimulatorDecoder.isSupported(), []);
     // We keep the client in a ref so reconnect() can call client.connect()
     // without causing a re-render or stale closure issues.
     const clientRef = useRef(null);
+    // One decoder per hook instance; lives across reconnects within a session.
+    const decoderRef = useRef(null);
+    if (decoderRef.current === null) {
+        decoderRef.current = new SimulatorDecoder();
+        decoderRef.current.onResize(setFrameSize);
+    }
+    // Stable callback to wire the rendering <canvas> into the decoder.
+    const attachCanvas = useCallback((canvas) => {
+        decoderRef.current?.setCanvas(canvas);
+    }, []);
     // Stable reconnect callback
     const reconnect = useCallback(() => {
         clientRef.current?.connect();
@@ -22,6 +34,13 @@ export function useExpoCiSession({ baseUrl, sessionId, apiToken, autoConnect = t
         }
         return client.sendInput(input);
     }, []);
+    // Tear the decoder down only on unmount — it is reused across reconnects.
+    useEffect(() => {
+        return () => {
+            decoderRef.current?.dispose();
+            decoderRef.current = null;
+        };
+    }, []);
     useEffect(() => {
         const client = new ExpoCiClient({ baseUrl, sessionId, apiToken });
         clientRef.current = client;
@@ -56,8 +75,8 @@ export function useExpoCiSession({ baseUrl, sessionId, apiToken, autoConnect = t
                 return next.length > maxLogs ? next.slice(next.length - maxLogs) : next;
             });
         });
-        const offFrame = client.on("video-frame", (msg) => {
-            setFrame(`data:image/jpeg;base64,${msg.data}`);
+        const offFrame = client.on("video-chunk", (chunk) => {
+            decoderRef.current?.push(chunk);
         });
         const offStatus = client.on("status", (msg) => {
             setStatus(msg.data.status);
@@ -80,5 +99,15 @@ export function useExpoCiSession({ baseUrl, sessionId, apiToken, autoConnect = t
         };
         // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [baseUrl, sessionId, apiToken, autoConnect, maxLogs]);
-    return { status, logs, frame, connected, error, sendInput, reconnect };
+    return {
+        status,
+        logs,
+        attachCanvas,
+        videoSupported,
+        frameSize,
+        connected,
+        error,
+        sendInput,
+        reconnect,
+    };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alivelabs/expo-orchestrator-react-client",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "React client for Expo CI Orchestrator — streaming logs, live simulator video, and interactive controls.",
   "type": "module",
   "license": "MIT",
@@ -30,7 +30,7 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@alivelabs/expo-orchestrator-schemas": "^0.1.0"
+    "@alivelabs/expo-orchestrator-schemas": "^0.2.0"
   },
   "peerDependencies": {
     "react": ">=18.0.0",