@sailfish-ai/recorder 1.12.5 → 1.12.6

package/dist/websocket.js

@@ -2,6 +2,7 @@ import ReconnectingWebSocket from "reconnecting-websocket";
 import { buildEventTimeEnvelope, getClockSyncManager, monotonicNow, timeOriginMs, } from "./clockSync";
 import { readDebugFlag } from "./env";
 import { deleteEventsByIds, getAllIndexedEvents, saveEventsToIDB, } from "./eventStore";
+import { deleteCanvasFramesByIds, persistCanvasFrame, readAllCanvasFrames, } from "./canvasFrameStore";
 import { deleteNotifyMessageById, getAllNotifyMessages, saveNotifyMessageToIDB, } from "./notifyEventStore";
 import { getOrSetSessionId } from "./session";
 import { buildBatches, eventSize } from "./utils";
@@ -40,6 +41,18 @@ function drain() {
   }
 }
 
+// Backpressure (A11): report the live socket buffer depth to the main thread on
+// a light interval so canvas video capture can throttle frame production when a
+// slow-but-open socket falls behind. Cheap (one number) and off the main thread.
+var bufferedReportTimer = null;
+function startBufferedReporting() {
+  if (bufferedReportTimer) return;
+  bufferedReportTimer = setInterval(function() {
+    var amt = (ws && ws.readyState === 1) ? (ws.bufferedAmount || 0) : 0;
+    postMessage({ type: "buffered", bufferedAmount: amt });
+  }, 250);
+}
+
 function connect() {
   if (ws && (ws.readyState === 0 || ws.readyState === 1)) return;
   try { ws = new WebSocket(wsUrl); } catch (e) { scheduleReconnect(); return; }
@@ -48,11 +61,14 @@ function connect() {
     clearTimeout(tid);
     reconnectDelay = 1000;
     drain();
+    startBufferedReporting();
     postMessage({ type: "open" });
   };
   ws.onclose = function() {
     clearTimeout(tid);
     ws = null;
+    if (bufferedReportTimer) { clearInterval(bufferedReportTimer); bufferedReportTimer = null; }
+    postMessage({ type: "buffered", bufferedAmount: 0 });
     postMessage({ type: "close" });
     scheduleReconnect();
   };
@@ -86,6 +102,13 @@ self.onmessage = function(e) {
     if (ws && ws.readyState === 1) {
       try { ws.send(msg.payload); } catch (e) { enqueue(msg.payload); }
     } else { enqueue(msg.payload); }
+  } else if (msg.type === "sendBinary") {
+    // Binary frames (canvas SFCV). The main thread only posts these while the
+    // socket is open; if it closed in the meantime the frame is dropped (the
+    // main-thread ring buffer covers the not-open case).
+    if (ws && ws.readyState === 1) {
+      try { ws.send(msg.payload); } catch (e) {}
+    }
   } else if (msg.type === "close") {
     if (reconnectTimer) { clearTimeout(reconnectTimer); reconnectTimer = null; }
     if (ws) { ws.close(); ws = null; }
@@ -94,9 +117,176 @@ self.onmessage = function(e) {
 `;
 let wsWorker = null;
 let webSocket = null;
+/** raw send on the direct (non-worker) socket — set in initializeWebSocket */
+let directSend = null;
+/** live socket buffer depth on the direct (non-worker) path */
+let directBufferedAmount = null;
+/** worker path: last reported socket buffer depth (worker posts on an interval) */
+let _workerBufferedAmount = 0;
 let isDraining = false;
 let inFlightFlush = null;
 let flushIntervalId = null;
+// ─── Binary frame buffering (canvas SFCV frames) ─────────────────────────────
+// While the socket is down, binary frames are held durably in IndexedDB
+// (canvasFrameStore — survives reload, 64 MiB at-rest budget) and drained in
+// capture order on reconnect. The 8 MiB in-memory ring below is the fallback
+// for environments where IDB is unavailable (SSR/private mode/exotic runtimes)
+// and for the rare frame whose async IDB write fails — there it keeps today's
+// behavior (bounded heap, oldest-dropped). A frame lives in exactly ONE of the
+// two (IDB on success, ring on failure), so reconnect can flush both without
+// double-sending.
+const BINARY_RING_MAX_BYTES = 8 * 1024 * 1024;
+const _binaryRing = [];
+let _binaryRingBytes = 0;
+let _binaryDropCount = 0;
+// True once any frame has been buffered offline — gates the periodic IDB
+// backlog drain so the always-connected common case never touches IDB.
+let _hadOfflineBinary = false;
+function pushBinaryRing(buffer) {
+    _binaryRing.push(buffer);
+    _binaryRingBytes += buffer.byteLength;
+    while (_binaryRingBytes > BINARY_RING_MAX_BYTES && _binaryRing.length > 0) {
+        const dropped = _binaryRing.shift();
+        _binaryRingBytes -= dropped.byteLength;
+        _binaryDropCount++;
+    }
+}
+function flushBinaryRing() {
+    while (_binaryRing.length > 0 && isWebSocketOpen(webSocket)) {
+        const buffer = _binaryRing.shift();
+        _binaryRingBytes -= buffer.byteLength;
+        if (!_wsSendBinaryNow(buffer)) {
+            // failed — put it back at the front and stop
+            _binaryRing.unshift(buffer);
+            _binaryRingBytes += buffer.byteLength;
+            break;
+        }
+    }
+}
+/**
+ * Drain the IDB-persisted binary backlog in capture order, deleting each frame
+ * only after it is handed to the socket. Stops at the first send failure so the
+ * remainder stays durable for the next reconnect. No-op (and cheap) when the
+ * store is empty or unavailable.
+ */
+async function flushPersistedBinary() {
+    if (!isWebSocketOpen(webSocket))
+        return;
+    const frames = await readAllCanvasFrames();
+    if (frames.length === 0)
+        return;
+    const sentIds = [];
+    for (const f of frames) {
+        if (!isWebSocketOpen(webSocket))
+            break;
+        if (!_wsSendBinaryNow(f.buf))
+            break; // stop; rest remain in IDB
+        if (typeof f.id === "number")
+            sentIds.push(f.id);
+    }
+    if (sentIds.length > 0)
+        void deleteCanvasFramesByIds(sentIds).catch(() => { });
+}
+/**
+ * Reconnect/top-up drain of all offline binary frames. Flushes the in-memory
+ * ring (IDB-unavailable fallback frames) then the durable IDB backlog. Each
+ * frame is in exactly one place, so this never double-sends.
+ */
+async function flushBinaryBacklog() {
+    flushBinaryRing();
+    await flushPersistedBinary();
+}
+/** Count of binary frames dropped from the ring (diagnostics). */
+export function getBinaryDropCount() {
+    return _binaryDropCount;
+}
+// ─── Transport backpressure (A11) ────────────────────────────────────────────
+// The canvas video encoder can outpace a slow-but-open socket, growing
+// WebSocket.bufferedAmount without bound (the data never lands and memory
+// climbs). videoCapture polls isTransportSaturated() in afterRender and SKIPS
+// producing a new VideoFrame while the socket is behind, resuming once it
+// drains. This is best-effort throttling, never a hard block of the main thread.
+/** ~16 MiB: above this the socket is treated as saturated and frame production
+ * pauses. Chosen above the 8 MiB max single frame so one large keyframe in
+ * flight does not by itself trip the gate. */
+const TRANSPORT_SATURATION_BYTES = 16 * 1024 * 1024;
+/**
+ * Current socket send-buffer depth in bytes (best effort). Worker path: the
+ * worker posts `bufferedAmount` on a light interval and we cache it. Direct
+ * path: read live. 0 when the socket is closed/unavailable.
+ */
+export function getBufferedAmount() {
+    if (wsWorker)
+        return _workerBufferedAmount;
+    if (directBufferedAmount && isWebSocketOpen(webSocket)) {
+        try {
+            return directBufferedAmount();
+        }
+        catch {
+            return 0;
+        }
+    }
+    return 0;
+}
+/** True when the socket's send buffer is over the saturation threshold —
+ * callers (canvas video capture) should pause producing new frames. */
+export function isTransportSaturated() {
+    return getBufferedAmount() > TRANSPORT_SATURATION_BYTES;
+}
+/**
+ * Buffer a binary frame while offline: durable IDB first, falling back to the
+ * in-memory ring if the IDB write fails or IDB is unavailable. Sync-returning;
+ * the IDB write is awaited internally and routed to the ring on failure.
+ */
+function bufferBinaryOffline(buffer) {
+    _hadOfflineBinary = true;
+    persistCanvasFrame(buffer)
+        .then((ok) => {
+        if (!ok)
+            pushBinaryRing(buffer);
+    })
+        .catch(() => pushBinaryRing(buffer));
+}
+/** Direct/worker send of a binary frame; assumes the socket is open. */
+function _wsSendBinaryNow(buffer) {
+    if (wsWorker) {
+        try {
+            wsWorker.postMessage({ type: "sendBinary", payload: buffer }, [buffer]);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    try {
+        if (directSend) {
+            directSend(buffer);
+            return true;
+        }
+        webSocket.send(buffer);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Send a binary frame (canvas SFCV) over the recording WebSocket.
+ * Direct path: webSocket.send(ArrayBuffer); worker path: transferred
+ * postMessage. When the socket is not open, the frame is buffered durably in
+ * IndexedDB (surviving reload) with a memory-ring fallback, and flushed on
+ * reconnect.
+ */
+export function wsSendBinary(buffer) {
+    if (isDraining || !isWebSocketOpen(webSocket)) {
+        bufferBinaryOffline(buffer);
+        return false;
+    }
+    if (_wsSendBinaryNow(buffer))
+        return true;
+    bufferBinaryOffline(buffer);
+    return false;
+}
 // ─── Time sync state ─────────────────────────────────────────────────────────
 // Periodic re-sync of the browser→server clock offset. The first burst
 // happens in handleWsOpen so events flushed from IDB on reconnect get a
@@ -558,12 +748,19 @@ function handleWsOpen() {
         finally {
             isDraining = false; //  resume live sends
         }
+        // drain any binary frames (canvas SFCV) buffered while offline —
+        // memory-ring fallback then the durable IDB backlog (survives reload)
+        await flushBinaryBacklog();
         // start periodic top-up flush after initial drain
         if (flushIntervalId != null)
             clearInterval(flushIntervalId);
         flushIntervalId = window.setInterval(() => {
             // fire-and-forget; single-flight guard prevents overlaps
             void flushBufferedEvents();
+            // Top up any binary frames whose async IDB write landed after the
+            // reconnect drain. Gated so the always-connected case never hits IDB.
+            if (_hadOfflineBinary)
+                void flushBinaryBacklog();
         }, 2000);
     })();
     // Seed the clock-offset estimate immediately so subsequent events get a
@@ -731,6 +928,9 @@ export function initializeWebSocket(backendApi, apiKey, sessionId, envValue, use
         wsUrl += `&envValue=${encodeURIComponent(envValue)}`;
     }
     // ── Try Web Worker path (moves JSON.stringify + WS I/O off main thread) ──
+    directSend = null;
+    directBufferedAmount = null;
+    _workerBufferedAmount = 0;
     wsWorker = useWsWorker ? tryCreateWsWorker() : null;
     if (wsWorker) {
         // Proxy object so callers can check .readyState (e.g. g.ws.readyState === 1)
@@ -757,11 +957,17 @@ export function initializeWebSocket(backendApi, apiKey, sessionId, envValue, use
             }
             else if (msg.type === "close") {
                 proxy.readyState = WebSocket.CLOSED;
+                _workerBufferedAmount = 0;
                 handleWsClose();
             }
             else if (msg.type === "message") {
                 handleWsMessage(msg.data);
             }
+            else if (msg.type === "buffered") {
+                // A11 backpressure: cache the socket's send-buffer depth reported by
+                // the worker (off the main thread) for isTransportSaturated().
+                _workerBufferedAmount = typeof msg.bufferedAmount === "number" ? msg.bufferedAmount : 0;
+            }
         };
         wsWorker.onerror = () => {
             // Worker crashed — fall through, events will buffer to IDB
@@ -782,6 +988,10 @@ export function initializeWebSocket(backendApi, apiKey, sessionId, envValue, use
         connectionTimeout: 30000,
     };
     const directWs = new ReconnectingWebSocket(wsUrl, [], options);
+    directWs.binaryType = "arraybuffer";
+    directSend = (data) => directWs.send(data);
+    // A11 backpressure: expose the live socket send-buffer depth.
+    directBufferedAmount = () => directWs.bufferedAmount || 0;
     const handle = {
         get readyState() { return directWs.readyState; },
         close: () => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sailfish-ai/recorder",
-  "version": "1.12.5",
+  "version": "1.12.6",
   "publishPublicly": true,
   "publishToCdn": {
     "jsdelivr": true,
@@ -37,6 +37,12 @@
       "require": "./dist/babel-plugin-sailfish-source.cjs",
       "default": "./dist/babel-plugin-sailfish-source.js"
     },
+    "./canvas-worker": {
+      "types": "./dist/types/canvas/workerShim.d.ts",
+      "import": "./dist/canvas-worker.js",
+      "require": "./dist/canvas-worker.cjs",
+      "default": "./dist/canvas-worker.js"
+    },
     "./*": {
       "types": "./dist/types/*.d.ts",
       "import": "./dist/*.js",