@relaymessenger/livekit 0.1.0-staging.11 → 0.1.0-staging.12

package/dist/video.js

@@ -0,0 +1,228 @@
+/**
+ * Video tracks for Relay calls, with LiveKit's public API shape.
+ *
+ * Names and semantics copied from `@livekit/rtc-node` 0.13.35
+ * (node_modules/@livekit/rtc-node/src):
+ * - video_source.ts:18-75: `new VideoSource(width, height)`,
+ *   `captureFrame(frame, timestampUs = BigInt(0), rotation = VIDEO_ROTATION_0)`
+ *   which throws "VideoSource is closed" after `close()`.
+ * - track.ts:241-268: `LocalVideoTrack.createVideoTrack(name, source)` and
+ *   `close(closeSource = true)`.
+ * - video_stream.ts:12-16 and :94-98: `VideoFrameEvent { frame, timestampUs,
+ *   rotation }` and `class VideoStream extends ReadableStream<VideoFrameEvent>`
+ *   built from a remote track, read with `for await`.
+ * - participant.ts:737 `publishTrack(track, options)` and :783
+ *   `unpublishTrack(...)`; the publish options `videoCodec` and
+ *   `videoEncoding { maxBitrate, maxFramerate }` from
+ *   @livekit/rtc-ffi-bindings proto/room_pb.d.ts:1792-1859, and `VideoCodec`
+ *   (VP8 = 0, H264 = 1) from proto/video_frame_pb.d.ts:28-50.
+ * The optional `{ capacity, format }` of `VideoStream` is LiveKit's Python SDK
+ * (livekit-rtc/livekit/rtc/video_stream.py:42-60, `RingQueue` in _utils.py:78-94:
+ * capacity 0 keeps every frame, a positive capacity drops the oldest).
+ *
+ * Relay semantics (PROTOCOL.md sections 1-2, _artifacts/calls-facetime-20260922):
+ * a participant publishes at most one `video` track; publishing it the first
+ * time negotiates it with an add-track offer and announces `userUpdate
+ * { video: true }`; unpublishing stops sending and announces `video: false`
+ * but leaves the track negotiated, so publishing again only resumes.
+ */
+import { ReadableStream } from "node:stream/web";
+import { VideoBufferType, VideoFrame, VideoRotation } from "./video-frame.js";
+export { VideoBufferType, VideoFrame, VideoRotation } from "./video-frame.js";
+/** Same names and numbers as LiveKit's `VideoCodec`; Relay sends these two. */
+export var VideoCodec;
+(function (VideoCodec) {
+    VideoCodec[VideoCodec["VP8"] = 0] = "VP8";
+    VideoCodec[VideoCodec["H264"] = 1] = "H264";
+})(VideoCodec || (VideoCodec = {}));
+export class VideoSource {
+    width;
+    height;
+    /** @internal */
+    closed = false;
+    #sinks = new Set();
+    constructor(width, height) {
+        this.width = width;
+        this.height = height;
+    }
+    /**
+     * Hand one raw frame to the published track. RGBA, BGRA and I420 (LiveKit
+     * `VideoBufferType`) go to the encoder as they are; the other packed RGB
+     * layouts are converted to I420 first. `timestampUs` 0 means "now".
+     * Frames captured while the track is not published are dropped.
+     */
+    captureFrame(frame, timestampUs = BigInt(0), rotation = VideoRotation.VIDEO_ROTATION_0) {
+        if (this.closed)
+            throw new Error("VideoSource is closed");
+        for (const sink of this.#sinks)
+            sink(frame, timestampUs, rotation);
+    }
+    async close() {
+        this.closed = true;
+        this.#sinks.clear();
+    }
+    /** @internal */
+    _attach(sink) {
+        this.#sinks.add(sink);
+        return () => this.#sinks.delete(sink);
+    }
+}
+export class LocalVideoTrack {
+    kind = "video";
+    name;
+    /** @internal */
+    source;
+    constructor(name, source) {
+        this.name = name;
+        this.source = source;
+    }
+    static createVideoTrack(name, source) {
+        return new LocalVideoTrack(name, source);
+    }
+    async close(closeSource = true) {
+        if (closeSource)
+            await this.source.close();
+    }
+}
+/**
+ * The other participant's camera. Emitted once per call by the transport's
+ * `trackSubscribed` event; it survives SFU restarts (the transport moves it
+ * onto the new session's receiver), so a `VideoStream` read from it keeps going.
+ */
+export class RemoteVideoTrack {
+    kind = "video";
+    name = "video";
+    #receiver;
+    #retired;
+    #consumers = new Set();
+    #ended = false;
+    /** Counters summed over every receiver this track has had. */
+    stats() {
+        const current = this.#receiver?.stats();
+        const retired = this.#retired;
+        if (!retired)
+            return current;
+        if (!current)
+            return { ...retired, recentFrames: 0 };
+        return {
+            ...current,
+            rtpPackets: retired.rtpPackets + current.rtpPackets,
+            framesAssembled: retired.framesAssembled + current.framesAssembled,
+            framesDropped: retired.framesDropped + current.framesDropped,
+            framesDecoded: retired.framesDecoded + current.framesDecoded,
+            decodeErrors: retired.decodeErrors + current.decodeErrors,
+            keyframeRequests: retired.keyframeRequests + current.keyframeRequests,
+            firstFrameAt: retired.firstFrameAt ?? current.firstFrameAt,
+            lastFrameAt: current.lastFrameAt ?? retired.lastFrameAt,
+        };
+    }
+    /** @internal The transport moves the track onto a new session's receiver. */
+    _attach(receiver) {
+        this._detach();
+        this.#receiver = receiver;
+        receiver.onframe = (event) => {
+            for (const consumer of this.#consumers)
+                consumer.push(event);
+        };
+        receiver.setActive(this.#consumers.size > 0);
+    }
+    /** @internal */
+    _detach() {
+        const receiver = this.#receiver;
+        if (!receiver)
+            return;
+        this.#receiver = undefined;
+        const stats = receiver.stats();
+        const retired = this.#retired;
+        this.#retired = retired
+            ? {
+                ...stats,
+                rtpPackets: retired.rtpPackets + stats.rtpPackets,
+                framesAssembled: retired.framesAssembled + stats.framesAssembled,
+                framesDropped: retired.framesDropped + stats.framesDropped,
+                framesDecoded: retired.framesDecoded + stats.framesDecoded,
+                decodeErrors: retired.decodeErrors + stats.decodeErrors,
+                keyframeRequests: retired.keyframeRequests + stats.keyframeRequests,
+                firstFrameAt: retired.firstFrameAt ?? stats.firstFrameAt,
+                lastFrameAt: stats.lastFrameAt ?? retired.lastFrameAt,
+            }
+            : stats;
+        receiver.onframe = null;
+        receiver.stop();
+    }
+    /** @internal The call is over: every `VideoStream` ends. */
+    _end() {
+        this._detach();
+        this.#ended = true;
+        for (const consumer of [...this.#consumers])
+            consumer.end();
+        this.#consumers.clear();
+    }
+    /** @internal */
+    _subscribe(consumer) {
+        if (this.#ended) {
+            consumer.end();
+            return () => undefined;
+        }
+        this.#consumers.add(consumer);
+        if (this.#consumers.size === 1)
+            this.#receiver?.setActive(true);
+        return () => {
+            this.#consumers.delete(consumer);
+            if (this.#consumers.size === 0)
+                this.#receiver?.setActive(false);
+        };
+    }
+}
+/**
+ * Decoded frames of a remote video track. Decoding starts when the first
+ * stream opens and stops when the last one is cancelled.
+ */
+export class VideoStream extends ReadableStream {
+    constructor(track, options = {}) {
+        const capacity = options.capacity ?? 0;
+        const format = options.format;
+        const queue = [];
+        let waiting;
+        let ended = false;
+        let unsubscribe;
+        const wake = () => {
+            const resolve = waiting;
+            waiting = undefined;
+            resolve?.();
+        };
+        super({
+            start: () => {
+                unsubscribe = track._subscribe({
+                    push: (event) => {
+                        if (capacity > 0 && queue.length >= capacity)
+                            queue.shift();
+                        queue.push(event);
+                        wake();
+                    },
+                    end: () => {
+                        ended = true;
+                        wake();
+                    },
+                });
+            },
+            pull: async (controller) => {
+                while (queue.length === 0 && !ended)
+                    await new Promise((resolve) => { waiting = resolve; });
+                const event = queue.shift();
+                if (!event) {
+                    controller.close();
+                    return;
+                }
+                const frame = format === undefined || format === event.frame.type ? event.frame : event.frame.convert(format);
+                controller.enqueue({ ...event, frame });
+            },
+            cancel: () => {
+                unsubscribe?.();
+                ended = true;
+                wake();
+            },
+        }, { highWaterMark: 0 });
+    }
+}
+//# sourceMappingURL=video.js.map

package/dist/video.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"video.js","sourceRoot":"","sources":["../src/video.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAE9E,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAE9E,+EAA+E;AAC/E,MAAM,CAAN,IAAY,UAGX;AAHD,WAAY,UAAU;IACpB,yCAAO,CAAA;IACP,2CAAQ,CAAA;AACV,CAAC,EAHW,UAAU,KAAV,UAAU,QAGrB;AAiGD,MAAM,OAAO,WAAW;IACtB,KAAK,CAAS;IACd,MAAM,CAAS;IACf,gBAAgB;IAChB,MAAM,GAAG,KAAK,CAAC;IACN,MAAM,GAAG,IAAI,GAAG,EAAa,CAAC;IAEvC,YAAY,KAAa,EAAE,MAAc;QACvC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;;;OAKG;IACH,YAAY,CAAC,KAAiB,EAAE,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,GAAG,aAAa,CAAC,gBAAgB;QAChG,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC1D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,KAAK,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC;IACrE,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;IAED,gBAAgB;IAChB,OAAO,CAAC,IAAe;QACrB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtB,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;CACF;AAED,MAAM,OAAO,eAAe;IACjB,IAAI,GAAG,OAAgB,CAAC;IACxB,IAAI,CAAS;IACtB,gBAAgB;IACP,MAAM,CAAc;IAE7B,YAAY,IAAY,EAAE,MAAmB;QAC3C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,IAAY,EAAE,MAAmB;QACvD,OAAO,IAAI,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,WAAW,GAAG,IAAI;QAC5B,IAAI,WAAW;YAAE,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC7C,CAAC;CACF;AAOD;;;;GAIG;AACH,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,OAAgB,CAAC;IACxB,IAAI,GAAG,OAAO,CAAC;IACxB,SAAS,CAAqC;IAC9C,QAAQ,CAAsC;IACrC,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,MAAM,GAAG,KAAK,CAAC;IAEf,8DAA8D;IAC9D,KAAK;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC;QACxC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,OAAO;YAAE,OAAO,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,GAAG,OAAO,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;QACrD,OAAO;YACL,GAAG,OAAO;YACV,UAAU,EAAE,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU;YACnD,eAAe,EAAE,OAAO,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe;YAClE,aAAa,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa;YAC5D,aAAa,EAAE,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa;YAC5D,YAAY,EAAE,OAAO,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY;YACzD,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB;YACrE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,YAAY;YAC1D,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW;SACxD,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,OAAO,CAAC,QAAgC;QACtC,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,QAAQ,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;YAC3B,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,UAAU;gBAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/D,CAAC,CAAC;QACF,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,gBAAgB;IAChB,OAAO;QACL,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,OAAO;YACrB,CAAC,CAAC;gBACA,GAAG,KAAK;gBACR,UAAU,EAAE,OAAO,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU;gBACjD,eAAe,EAAE,OAAO,CAAC,eAAe,GAAG,KAAK,CAAC,eAAe;gBAChE,aAAa,EAAE,OAAO,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa;gBAC1D,aAAa,EAAE,OAAO,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa;gBAC1D,YAAY,EAAE,OAAO,CAAC,YAAY,GAAG,KAAK,CAAC,YAAY;gBACvD,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,GAAG,KAAK,CAAC,gBAAgB;gBACnE,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC,YAAY;gBACxD,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW;aACtD;YACD,CAAC,CAAC,KAAK,CAAC;QACV,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,QAAQ,CAAC,IAAI,EAAE,CAAC;IAClB,CAAC;IAED,4DAA4D;IAC5D,IAAI;QACF,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,KAAK,MAAM,QAAQ,IAAI,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;YAAE,QAAQ,CAAC,GAAG,EAAE,CAAC;QAC5D,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;IAED,gBAAgB;IAChB,UAAU,CAAC,QAAwB;QACjC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,QAAQ,CAAC,GAAG,EAAE,CAAC;YACf,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;QACzB,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC9B,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;YAAE,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QAChE,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;gBAAE,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;QACnE,CAAC,CAAC;IACJ,CAAC;CACF;AASD;;;GAGG;AACH,MAAM,OAAO,WAAY,SAAQ,cAA+B;IAC9D,YAAY,KAAuB,EAAE,UAA8B,EAAE;QACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,MAAM,KAAK,GAAsB,EAAE,CAAC;QACpC,IAAI,OAA4C,CAAC;QACjD,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,IAAI,WAAqC,CAAC;QAC1C,MAAM,IAAI,GAAG,GAAS,EAAE;YACtB,MAAM,OAAO,GAAG,OAAO,CAAC;YACxB,OAAO,GAAG,SAAS,CAAC;YACpB,OAAO,EAAE,EAAE,CAAC;QACd,CAAC,CAAC;QACF,KAAK,CAAC;YACJ,KAAK,EAAE,GAAG,EAAE;gBACV,WAAW,GAAG,KAAK,CAAC,UAAU,CAAC;oBAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE;wBACd,IAAI,QAAQ,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ;4BAAE,KAAK,CAAC,KAAK,EAAE,CAAC;wBAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAClB,IAAI,EAAE,CAAC;oBACT,CAAC;oBACD,GAAG,EAAE,GAAG,EAAE;wBACR,KAAK,GAAG,IAAI,CAAC;wBACb,IAAI,EAAE,CAAC;oBACT,CAAC;iBACF,CAAC,CAAC;YACL,CAAC;YACD,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE;gBACzB,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK;oBAAE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,GAAG,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;gBAClG,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,UAAU,CAAC,KAAK,EAAE,CAAC;oBACnB,OAAO;gBACT,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC9G,UAAU,CAAC,OAAO,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;YAC1C,CAAC;YACD,MAAM,EAAE,GAAG,EAAE;gBACX,WAAW,EAAE,EAAE,CAAC;gBAChB,KAAK,GAAG,IAAI,CAAC;gBACb,IAAI,EAAE,CAAC;YACT,CAAC;SACF,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;CACF"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@relaymessenger/livekit",
-  "version": "0.1.0-staging.11",
-  "description": "Relay WebRTC audio transport and LiveKit Agents integration for Node.js.",
+  "version": "0.1.0-staging.12",
+  "description": "Relay WebRTC audio and video transport and LiveKit Agents integration for Node.js.",
   "type": "module",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -39,10 +39,12 @@
   "dependencies": {
     "@evan/opus": "1.0.3",
     "@relaymessenger/sdk": "0.3.6-staging.19",
+    "rtp-packet": "0.4.2",
     "werift": "0.24.4"
   },
   "optionalDependencies": {
-    "@roamhq/wrtc": "^0.10.0"
+    "@roamhq/wrtc": "^0.10.0",
+    "node-webcodecs": "1.3.0"
   },
   "peerDependencies": {
     "@livekit/agents": "^1.9.0",
@@ -72,6 +74,7 @@
     "livekit",
     "webrtc",
     "voice",
+    "video",
     "agents"
   ]
 }