@relaymessenger/livekit 0.1.0-staging.10 → 0.1.0-staging.12

package/README.md

@@ -3,7 +3,7 @@
 `@relaymessenger/livekit` connects Relay Calls to TypeScript LiveKit Agents.
 Relay stays responsible for the Call resource and signaling; the package owns
 the Node WebRTC peer and translates audio between Relay and LiveKit `AudioFrame`
-objects. Application code does not handle Cloudflare, SDP, ICE, or SFU
+objects, and sends and receives video with LiveKit's video API. Application code does not handle Cloudflare, SDP, ICE, or SFU
 credentials.
 
 Install it next to the Relay SDK and the LiveKit Agents runtime:
@@ -87,10 +87,67 @@ await transport.writeAudio({
 await transport.waitForPlayout();
 ```
 
+## Video
+
+Relay calls carry video whenever a camera is on. The video API copies
+LiveKit's `@livekit/rtc-node` names: `VideoSource`, `LocalVideoTrack`,
+`VideoFrame`, `VideoBufferType`, `VideoStream`. Video needs the default
+`"werift"` engine and the optional dependency `node-webcodecs` (prebuilt for
+macOS arm64 and Linux x64/arm64).
+
+Send a video feed:
+
+```ts
+import {
+  LocalVideoTrack,
+  VideoBufferType,
+  VideoFrame,
+  VideoSource,
+} from "@relaymessenger/livekit/transport";
+
+await transport.connect();
+
+const source = new VideoSource(640, 480);
+const track = LocalVideoTrack.createVideoTrack("camera", source);
+await transport.publishTrack(track, {
+  videoEncoding: { maxBitrate: 800_000, maxFramerate: 15 },
+});
+
+// RGBA, BGRA or I420 bytes, tightly packed.
+source.captureFrame(new VideoFrame(rgba, 640, 480, VideoBufferType.RGBA));
+
+// Camera off, then on again; the track stays negotiated.
+await transport.unpublishTrack(track);
+await transport.publishTrack(track);
+```
+
+Receive the other participant's video:
+
+```ts
+import { VideoBufferType, VideoStream } from "@relaymessenger/livekit/transport";
+
+transport.on("trackSubscribed", async (track) => {
+  const stream = new VideoStream(track, { format: VideoBufferType.RGBA, capacity: 2 });
+  for await (const { frame, timestampUs } of stream) {
+    // frame.data is width x height x 4 bytes of RGBA.
+  }
+});
+transport.on("remoteVideo", (on) => {
+  // The other participant's camera started or stopped sending.
+});
+```
+
+Frames are decoded only while a `VideoStream` is open. `transport.videoStats()`
+reports frames, packets, keyframes and decode errors in both directions.
+
 ## ICE servers, TURN, and diagnostics
 
-By default the peer gathers host candidates only and Cloudflare's SFU supplies
-its own candidates in the answer. Inside a container or behind a firewall that
+By default the peer uses Cloudflare's STUN server
+(`stun:stun.cloudflare.com:3478`), as Cloudflare's own Realtime echo example
+does, and Cloudflare's SFU supplies its own candidates in the answer. The offer
+leaves as soon as the first local candidate exists, without waiting for ICE
+gathering to finish: the SFU is ICE-lite and learns the agent's address from
+its connectivity checks. Inside a container or behind a firewall that
 blocks outbound UDP, pass TURN servers in the standard `RTCIceServer` shape and,
 if every path must go through TURN, `iceTransportPolicy: "relay"`. Both options
 are accepted by `RelayLiveKitCall.connect()` and `RelayCallTransport`.
@@ -150,7 +207,10 @@ Packet counts come from the `werift` engine; `wrtc` reports zero packets and
 `pacer n/a`.
 
 Like a live microphone, the `werift` engine's published track sends one Opus
-packet every 20 ms from the moment media connects until the transport closes:
+packet every 20 ms from the moment media connects until the transport closes,
+paced by the monotonic clock: a timer that fires late sends every packet due
+by then, so the wire carries exactly 50 packets a second; a stall longer than
+200 ms restarts the clock instead of bursting. The track carries
 the caller's audio when some is queued, Opus silence otherwise. Cloudflare's
 SFU will not let the person's side pull a track that has carried no RTP, so a
 silent agent track would never be heard. Silence never counts toward

package/dist/engine-werift-video.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Video for the werift engine: `node-webcodecs` 1.3.0 encodes and decodes
+ * (its static build: LGPL FFmpeg with openh264 and libvpx), `rtp-packet`
+ * packetizes, `video-rtp.ts` depacketizes, werift carries RTP and RTCP.
+ *
+ * Choices measured on 2026-09-22 (/tmp/video-spike-webcodecs, the spike that
+ * picked this engine), each kept here:
+ * - `NODE_WEBCODECS_FORCE=static` selects the bundled FFmpeg
+ *   (node-webcodecs dist/native.js `load()`; the default order tries a
+ *   dynamic build linked against system FFmpeg first).
+ * - `hardwareAcceleration: "prefer-software"` on encoder and decoder: on a
+ *   Mac, VideoToolbox otherwise takes the stream.
+ * - H.264 is sent as constrained baseline `avc1.42e01f`, Annex-B; the codec
+ *   string FFmpeg reports for H.264 is "MPEG4/ISO/AVC", so the codec is never
+ *   matched by FFmpeg's name, only by the negotiated MIME type.
+ * - A keyframe every 1 s, and on every PLI or FIR from the SFU.
+ * - The send codec is the first codec of the SFU's answer (werift
+ *   rtpSender.js:408 `this.codec = params.codecs[0]`).
+ * - The receive codec is learned from the payload type of the first RTP
+ *   packet, looked up in the pulled m-line's codecs: the SFU's pull offer
+ *   lists every codec (spike receiver.mjs).
+ * - werift sends no PLI on loss by itself: the receiver sends one when the
+ *   stream starts, when a frame is dropped and on a decode error, then every
+ *   1 s until a keyframe decodes (spike receiver.mjs `pliTimer`).
+ */
+import { MediaStreamTrack, type RTCRtpTransceiver } from "werift";
+import type { RelayMediaStreamTrackLike } from "./transport.js";
+import { type RelayVideoFactory, type RelayVideoReceiverLike, type RelayVideoReceiverStats, type RelayVideoSenderLike, type RelayVideoSenderStats, type TrackPublishOptions } from "./video.js";
+import { VideoFrame, VideoRotation } from "./video-frame.js";
+type WebCodecs = typeof import("node-webcodecs");
+export declare const KEYFRAME_INTERVAL_MS = 1000;
+export declare const KEYFRAME_REQUEST_INTERVAL_MS = 1000;
+export declare const DEFAULT_VIDEO_BITRATE = 800000;
+export declare const DEFAULT_VIDEO_FRAMERATE = 30;
+/** Loads node-webcodecs once, on first video use, so audio-only calls never load FFmpeg. */
+export declare const loadWebCodecs: () => Promise<WebCodecs>;
+/**
+ * Raw frames in, H.264 or VP8 RTP out on one werift track. One sender lives
+ * for the whole call; a restart binds it to the new peer's transceiver.
+ */
+export declare class WeriftVideoSender implements RelayVideoSenderLike {
+    #private;
+    readonly track: RelayMediaStreamTrackLike;
+    constructor(wc: WebCodecs, options?: TrackPublishOptions);
+    /**
+     * Offer the codecs this sender can encode, the preferred one first (werift
+     * fills an empty `transceiver.codecs` from the peer's config at
+     * `createOffer`, peerConnection.js:520-523, so setting it here wins).
+     */
+    bind(transceiver: unknown, peer: unknown): void;
+    setEnabled(enabled: boolean): void;
+    capture(frame: VideoFrame, timestampUs: bigint, rotation: VideoRotation): void;
+    stats(): RelayVideoSenderStats;
+    close(): void;
+}
+/**
+ * One pulled video m-line: RTP in, decoded I420 frames out. RTP is counted
+ * always; it is reordered, assembled and decoded only while active.
+ */
+export declare class WeriftVideoReceiver implements RelayVideoReceiverLike {
+    #private;
+    onframe: RelayVideoReceiverLike["onframe"];
+    constructor(track: MediaStreamTrack, transceiver: RTCRtpTransceiver | undefined);
+    setActive(active: boolean): void;
+    stats(): RelayVideoReceiverStats;
+    stop(): void;
+}
+export declare const createWeriftVideoFactory: () => RelayVideoFactory;
+export {};
+//# sourceMappingURL=engine-werift-video.d.ts.map

package/dist/engine-werift-video.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"engine-werift-video.d.ts","sourceRoot":"","sources":["../src/engine-werift-video.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,EACL,gBAAgB,EAEhB,KAAK,iBAAiB,EAGvB,MAAM,QAAQ,CAAC;AAEhB,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,oBAAoB,EACzB,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EAGzB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAmB,UAAU,EAAE,aAAa,EAAoB,MAAM,kBAAkB,CAAC;AAWhG,KAAK,SAAS,GAAG,cAAc,gBAAgB,CAAC,CAAC;AAMjD,eAAO,MAAM,oBAAoB,OAAQ,CAAC;AAC1C,eAAO,MAAM,4BAA4B,OAAQ,CAAC;AAClD,eAAO,MAAM,qBAAqB,SAAU,CAAC;AAC7C,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAU1C,4FAA4F;AAC5F,eAAO,MAAM,aAAa,QAAO,OAAO,CAAC,SAAS,CAgBjD,CAAC;AA0CF;;;GAGG;AACH,qBAAa,iBAAkB,YAAW,oBAAoB;;IAC5D,QAAQ,CAAC,KAAK,EAAE,yBAAyB,CAAC;gBA8B9B,EAAE,EAAE,SAAS,EAAE,OAAO,GAAE,mBAAwB;IAS5D;;;;OAIG;IACH,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,GAAG,IAAI;IAqB/C,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKlC,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,GAAG,IAAI;IAqD9E,KAAK,IAAI,qBAAqB;IAmB9B,KAAK,IAAI,IAAI;CAwEd;AAED;;;GAGG;AACH,qBAAa,mBAAoB,YAAW,sBAAsB;;IAChE,OAAO,EAAE,sBAAsB,CAAC,SAAS,CAAC,CAAQ;gBA+BtC,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,iBAAiB,GAAG,SAAS;IAW/E,SAAS,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI;IAehC,KAAK,IAAI,uBAAuB;IAiBhC,IAAI,IAAI,IAAI;CA8Kb;AAED,eAAO,MAAM,wBAAwB,QAAO,iBAI1C,CAAC"}