@relaymessenger/livekit 0.1.0-staging.11 → 0.1.0-staging.13

package/README.md

@@ -3,7 +3,7 @@
 `@relaymessenger/livekit` connects Relay Calls to TypeScript LiveKit Agents.
 Relay stays responsible for the Call resource and signaling; the package owns
 the Node WebRTC peer and translates audio between Relay and LiveKit `AudioFrame`
-objects. Application code does not handle Cloudflare, SDP, ICE, or SFU
+objects, and sends and receives video with LiveKit's video API. Application code does not handle Cloudflare, SDP, ICE, or SFU
 credentials.
 
 Install it next to the Relay SDK and the LiveKit Agents runtime:
@@ -87,6 +87,59 @@ await transport.writeAudio({
 await transport.waitForPlayout();
 ```
 
+## Video
+
+Relay calls carry video whenever a camera is on. The video API copies
+LiveKit's `@livekit/rtc-node` names: `VideoSource`, `LocalVideoTrack`,
+`VideoFrame`, `VideoBufferType`, `VideoStream`. Video needs the default
+`"werift"` engine and the optional dependency `node-webcodecs` (prebuilt for
+macOS arm64 and Linux x64/arm64).
+
+Send a video feed:
+
+```ts
+import {
+  LocalVideoTrack,
+  VideoBufferType,
+  VideoFrame,
+  VideoSource,
+} from "@relaymessenger/livekit/transport";
+
+await transport.connect();
+
+const source = new VideoSource(640, 480);
+const track = LocalVideoTrack.createVideoTrack("camera", source);
+await transport.publishTrack(track, {
+  videoEncoding: { maxBitrate: 800_000, maxFramerate: 15 },
+});
+
+// RGBA, BGRA or I420 bytes, tightly packed.
+source.captureFrame(new VideoFrame(rgba, 640, 480, VideoBufferType.RGBA));
+
+// Camera off, then on again; the track stays negotiated.
+await transport.unpublishTrack(track);
+await transport.publishTrack(track);
+```
+
+Receive the other participant's video:
+
+```ts
+import { VideoBufferType, VideoStream } from "@relaymessenger/livekit/transport";
+
+transport.on("trackSubscribed", async (track) => {
+  const stream = new VideoStream(track, { format: VideoBufferType.RGBA, capacity: 2 });
+  for await (const { frame, timestampUs } of stream) {
+    // frame.data is width x height x 4 bytes of RGBA.
+  }
+});
+transport.on("remoteVideo", (on) => {
+  // The other participant's camera started or stopped sending.
+});
+```
+
+Frames are decoded only while a `VideoStream` is open. `transport.videoStats()`
+reports frames, packets, keyframes and decode errors in both directions.
+
 ## ICE servers, TURN, and diagnostics
 
 By default the peer uses Cloudflare's STUN server

package/dist/engine-werift-video.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Video for the werift engine: `node-webcodecs` 1.3.0 encodes and decodes
+ * (its static build: LGPL FFmpeg with openh264 and libvpx), `rtp-packet`
+ * packetizes, `video-rtp.ts` depacketizes, werift carries RTP and RTCP.
+ *
+ * Choices measured on 2026-09-22 (/tmp/video-spike-webcodecs, the spike that
+ * picked this engine), each kept here:
+ * - `NODE_WEBCODECS_FORCE=static` selects the bundled FFmpeg
+ *   (node-webcodecs dist/native.js `load()`; the default order tries a
+ *   dynamic build linked against system FFmpeg first).
+ * - `hardwareAcceleration: "prefer-software"` on encoder and decoder: on a
+ *   Mac, VideoToolbox otherwise takes the stream.
+ * - H.264 is sent as constrained baseline `avc1.42e01f`, Annex-B; the codec
+ *   string FFmpeg reports for H.264 is "MPEG4/ISO/AVC", so the codec is never
+ *   matched by FFmpeg's name, only by the negotiated MIME type.
+ * - A keyframe every 1 s, and on every PLI or FIR from the SFU.
+ * - The send codec is the first codec of the SFU's answer (werift
+ *   rtpSender.js:408 `this.codec = params.codecs[0]`).
+ * - The receive codec is learned from the payload type of the first RTP
+ *   packet, looked up in the pulled m-line's codecs: the SFU's pull offer
+ *   lists every codec (spike receiver.mjs).
+ * - werift sends no PLI on loss by itself: the receiver sends one when the
+ *   stream starts, when a frame is dropped and on a decode error, then every
+ *   1 s until a keyframe decodes (spike receiver.mjs `pliTimer`).
+ */
+import { MediaStreamTrack, type RTCRtpTransceiver } from "werift";
+import type { RelayMediaStreamTrackLike } from "./transport.js";
+import { type RelayVideoFactory, type RelayVideoReceiverLike, type RelayVideoReceiverStats, type RelayVideoSenderLike, type RelayVideoSenderStats, type TrackPublishOptions } from "./video.js";
+import { VideoFrame, VideoRotation } from "./video-frame.js";
+type WebCodecs = typeof import("node-webcodecs");
+export declare const KEYFRAME_INTERVAL_MS = 1000;
+export declare const KEYFRAME_REQUEST_INTERVAL_MS = 1000;
+export declare const DEFAULT_VIDEO_BITRATE = 800000;
+export declare const DEFAULT_VIDEO_FRAMERATE = 30;
+/** Loads node-webcodecs once, on first video use, so audio-only calls never load FFmpeg. */
+export declare const loadWebCodecs: () => Promise<WebCodecs>;
+/**
+ * Raw frames in, H.264 or VP8 RTP out on one werift track. One sender lives
+ * for the whole call; a restart binds it to the new peer's transceiver.
+ */
+export declare class WeriftVideoSender implements RelayVideoSenderLike {
+    #private;
+    readonly track: RelayMediaStreamTrackLike;
+    constructor(wc: WebCodecs, options?: TrackPublishOptions);
+    /**
+     * Offer the codecs this sender can encode, the preferred one first (werift
+     * fills an empty `transceiver.codecs` from the peer's config at
+     * `createOffer`, peerConnection.js:520-523, so setting it here wins).
+     */
+    bind(transceiver: unknown, peer: unknown): void;
+    setEnabled(enabled: boolean): void;
+    capture(frame: VideoFrame, timestampUs: bigint, rotation: VideoRotation): void;
+    stats(): RelayVideoSenderStats;
+    close(): void;
+}
+/**
+ * One pulled video m-line: RTP in, decoded I420 frames out. RTP is counted
+ * always; it is reordered, assembled and decoded only while active.
+ */
+export declare class WeriftVideoReceiver implements RelayVideoReceiverLike {
+    #private;
+    onframe: RelayVideoReceiverLike["onframe"];
+    constructor(track: MediaStreamTrack, transceiver: RTCRtpTransceiver | undefined);
+    setActive(active: boolean): void;
+    stats(): RelayVideoReceiverStats;
+    stop(): void;
+}
+export declare const createWeriftVideoFactory: () => RelayVideoFactory;
+export {};
+//# sourceMappingURL=engine-werift-video.d.ts.map

package/dist/engine-werift-video.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine-werift-video.d.ts","sourceRoot":"","sources":["../src/engine-werift-video.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,EACL,gBAAgB,EAEhB,KAAK,iBAAiB,EAGvB,MAAM,QAAQ,CAAC;AAEhB,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,oBAAoB,EACzB,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EAGzB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAmB,UAAU,EAAE,aAAa,EAAoB,MAAM,kBAAkB,CAAC;AAWhG,KAAK,SAAS,GAAG,cAAc,gBAAgB,CAAC,CAAC;AAMjD,eAAO,MAAM,oBAAoB,OAAQ,CAAC;AAC1C,eAAO,MAAM,4BAA4B,OAAQ,CAAC;AAClD,eAAO,MAAM,qBAAqB,SAAU,CAAC;AAC7C,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAU1C,4FAA4F;AAC5F,eAAO,MAAM,aAAa,QAAO,OAAO,CAAC,SAAS,CAgBjD,CAAC;AA0CF;;;GAGG;AACH,qBAAa,iBAAkB,YAAW,oBAAoB;;IAC5D,QAAQ,CAAC,KAAK,EAAE,yBAAyB,CAAC;gBA8B9B,EAAE,EAAE,SAAS,EAAE,OAAO,GAAE,mBAAwB;IAS5D;;;;OAIG;IACH,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,GAAG,IAAI;IAqB/C,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKlC,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,GAAG,IAAI;IAqD9E,KAAK,IAAI,qBAAqB;IAmB9B,KAAK,IAAI,IAAI;CAwEd;AAED;;;GAGG;AACH,qBAAa,mBAAoB,YAAW,sBAAsB;;IAChE,OAAO,EAAE,sBAAsB,CAAC,SAAS,CAAC,CAAQ;gBA+BtC,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,iBAAiB,GAAG,SAAS;IAW/E,SAAS,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI;IAehC,KAAK,IAAI,uBAAuB;IAiBhC,IAAI,IAAI,IAAI;CA8Kb;AAED,eAAO,MAAM,wBAAwB,QAAO,iBAI1C,CAAC"}