@relaymessenger/livekit 0.1.0-staging.10 → 0.1.0-staging.12

package/dist/transport.js

@@ -1,3 +1,5 @@
+import { RemoteVideoTrack, } from "./video.js";
+export { LocalVideoTrack, RemoteVideoTrack, VideoBufferType, VideoCodec, VideoFrame, VideoRotation, VideoSource, VideoStream, } from "./video.js";
 /** Opus's native rate and Relay's wire channel count. */
 export const DEFAULT_INBOUND_AUDIO = Object.freeze({
     sampleRate: 48_000,
@@ -41,6 +43,11 @@ export class RelayCallTransportError extends Error {
     }
 }
 const DEFAULT_PEER_CONFIG = { iceServers: [], iceTransportPolicy: "all" };
+/**
+ * Cloudflare Realtime's echo example builds its peer with exactly this list
+ * (cloudflare/realtime-examples echo/index.html `createPeerConnection`).
+ */
+const DEFAULT_ICE_SERVERS = [{ urls: "stun:stun.cloudflare.com:3478" }];
 const loadWebRTCFactory = async (engine) => {
     if (engine === "werift") {
         const { createWeriftWebRTCFactory } = await import("./engine-werift.js");
@@ -68,7 +75,8 @@ const seconds = (milliseconds) => `${(milliseconds / 1000).toFixed(1)}s`;
 const summarizeIce = (diagnostics) => {
     const { local, remote, transitions, connected } = diagnostics;
     const localPart = `local: host ${local.host}, srflx ${local.srflx}, relay ${local.relay}`
-        + (local.other ? `, other ${local.other}` : "");
+        + (local.other ? `, other ${local.other}` : "")
+        + (diagnostics.selectedPair ? `, pair ${diagnostics.selectedPair}` : "");
     const remotePart = remote.length
         ? `remote: ${remote.map((candidate) => `${candidate.transport} ${candidate.port}`).join(", ")}`
         : "remote: none";
@@ -215,6 +223,7 @@ export class RelayCallTransport {
     #stallWarned = false;
     #iceLocal = { host: 0, srflx: 0, relay: 0, other: 0 };
     #iceRemote = [];
+    #selectedPair;
     #iceTransitions = [];
     #listeners = new Map();
     #factory;
@@ -225,7 +234,11 @@ export class RelayCallTransport {
     #peerConnected = false;
     #audioSource;
     #localTrack;
+    /** Set while an offer waits for its first local candidate. */
+    #firstLocalCandidate;
     #remoteSink;
+    /** The track `#remoteSink` decodes; a repeated `ontrack` for it keeps the sink. */
+    #remoteSinkTrack;
     #publishTransceiver;
     #publishFrame;
     #initialAnswerSdp;
@@ -254,6 +267,17 @@ export class RelayCallTransport {
     #ready;
     #handlersAttached = false;
     #closed = false;
+    #muted = false;
+    /** The published camera: one per call, kept across restarts (PROTOCOL.md section 1). */
+    #video;
+    #videoTransceiver;
+    #remoteVideoTrack;
+    /** The engine track the current video receiver reads. */
+    #remoteVideoEngineTrack;
+    /** An add-track offer is out; a pull offer that crosses it waits for its answer. */
+    #addTrackPending = false;
+    #deferredOffer;
+    #lastAnswerSdp;
     constructor(options) {
         if (!options.callId.trim())
             throw new Error("callId is required.");
@@ -289,7 +313,7 @@ export class RelayCallTransport {
             throw new Error('iceTransportPolicy must be "all" or "relay".');
         }
         this.#iceTransportPolicy = policy;
-        const iceServers = options.iceServers ?? [];
+        const iceServers = options.iceServers ?? DEFAULT_ICE_SERVERS;
         this.#iceServers = typeof iceServers === "function" ? iceServers : copyIceServers(iceServers);
         if (!Number.isFinite(this.#iceGatheringTimeoutMs) || this.#iceGatheringTimeoutMs <= 0) {
             throw new Error("iceGatheringTimeoutMs must be greater than zero.");
@@ -383,6 +407,7 @@ export class RelayCallTransport {
             local: { ...this.#iceLocal },
             remote: this.#iceRemote.map((candidate) => ({ ...candidate })),
             transitions: this.#iceTransitions.map((transition) => ({ ...transition })),
+            selectedPair: this.#selectedPair,
             connected: this.#peerConnected,
             inbound: this.#inboundDiagnostics(),
             outbound: this.#outboundDiagnostics(),
@@ -566,8 +591,83 @@ export class RelayCallTransport {
             waiter.reject(error);
     }
     setMuted(muted) {
+        this.#muted = muted;
         this.#room.userUpdate({ muted });
     }
+    /**
+     * Publish the camera (LiveKit `LocalParticipant.publishTrack`). The first
+     * call adds a `video` track to the SFU session with an add-track offer, then
+     * announces `userUpdate { video: true }`; later calls, after
+     * `unpublishTrack`, only resume sending and announce it again
+     * (PROTOCOL.md sections 1-2). Requires `connect()` and the werift engine.
+     */
+    async publishTrack(track, options = {}) {
+        if (this.#closed || this.#ended)
+            throw new Error("Relay Call transport is closed.");
+        if (!this.#factory || !this.#reportedConnected)
+            throw new Error("Relay Call transport is not connected.");
+        if (this.#video) {
+            if (this.#video.track !== track)
+                throw new RelayCallTransportError("A Relay call publishes one video track.");
+            this.#video.sender.setEnabled(true);
+            this.#room.userUpdate({ muted: this.#muted, video: true });
+            return;
+        }
+        const factory = this.#factory;
+        if (!factory.createVideoSender) {
+            throw new RelayCallTransportError('This engine cannot send video; use the "werift" engine.');
+        }
+        const sender = await factory.createVideoSender(options);
+        if (this.#closed || this.#ended) {
+            sender.close();
+            throw new Error("Relay Call transport is closed.");
+        }
+        const detach = track.source._attach((frame, timestampUs, rotation) => sender.capture(frame, timestampUs, rotation));
+        this.#video = { track, sender, detach };
+        await this.#negotiate(async () => {
+            // No peer: a restart is pending, and the next peer publishes video with audio.
+            const peer = this.#peer;
+            if (!peer)
+                return;
+            this.#addVideoTransceiver(peer);
+            await this.#publishLocalAudio(peer, false);
+            this.#addTrackPending = this.#peer === peer;
+        });
+        this.#room.userUpdate({ muted: this.#muted, video: true });
+    }
+    /**
+     * Stop sending the camera and announce `userUpdate { video: false }`. The
+     * track stays negotiated on the SFU (PROTOCOL.md section 1), so a later
+     * `publishTrack` with the same track resumes it without renegotiating.
+     */
+    async unpublishTrack(track) {
+        if (!this.#video || this.#video.track !== track)
+            return;
+        this.#video.sender.setEnabled(false);
+        if (!this.#closed && !this.#ended)
+            this.#room.userUpdate({ muted: this.#muted, video: false });
+    }
+    /** The person's video track, once it has reached this peer. */
+    get remoteVideoTrack() {
+        return this.#remoteVideoTrack;
+    }
+    /** Frame, packet and keyframe counters for both video directions. */
+    videoStats() {
+        return { outbound: this.#video?.sender.stats(), inbound: this.#remoteVideoTrack?.stats() };
+    }
+    /** Runs `work` in the negotiation queue and hands its result or error to the caller. */
+    #negotiate(work) {
+        const run = this.#negotiationTail.then(work);
+        this.#negotiationTail = run.then(() => undefined, () => undefined);
+        return run;
+    }
+    #addVideoTransceiver(peer) {
+        const video = this.#video;
+        if (!video)
+            return;
+        this.#videoTransceiver = peer.addTransceiver(video.sender.track, { direction: "sendonly" });
+        video.sender.bind(this.#videoTransceiver, peer);
+    }
     end() {
         this.#room.end();
     }
@@ -604,6 +704,7 @@ export class RelayCallTransport {
         this.#peerConnected = false;
         this.#initialAnswerSdp = undefined;
         this.#publishTransceiver = peer.addTransceiver(track, { direction: "sendonly" });
+        this.#addVideoTransceiver(peer);
         this.#observeIce(peer);
         peer.onconnectionstatechange = () => {
             if (this.#peer !== peer)
@@ -613,24 +714,36 @@ export class RelayCallTransport {
         };
         peer.ontrack = (event) => {
             if (this.#peer === peer)
-                this.#remoteTrack(event.track);
+                this.#remoteTrack(event.track, event.transceiver);
         };
         await this.#publishLocalAudio(peer, restarts > 0);
     }
+    /**
+     * Sends the offer without waiting for ICE gathering, as Cloudflare's echo
+     * example does (`setLocalDescription(offer)`, then `tracks/new` at once): the
+     * SFU is ICE-lite and learns this peer's address from its connectivity
+     * checks. werift's `setLocalDescription` resolves only after every host,
+     * STUN and TURN candidate has settled (peerConnection.js `await
+     * this.gatherCandidates()`; ice.js `await Promise.allSettled(candidatePromises)`),
+     * which took 6.6 s on a staging call on 2026-09-22, so the offer leaves as
+     * soon as the description is applied and the first local candidate exists.
+     */
     async #publishLocalAudio(peer, restart) {
         const offer = await peer.createOffer();
-        await peer.setLocalDescription(offer);
-        await this.#waitForIceGathering(peer);
+        await this.#applyLocalOffer(peer, offer);
         if (this.#peer !== peer)
             return;
         const description = this.#localDescription(peer, "offer");
         const mid = this.#publishTransceiver?.mid;
         if (!mid)
             throw new RelayCallTransportError("Relay audio publication has no WebRTC MID.");
+        const videoMid = this.#video ? this.#videoTransceiver?.mid : undefined;
+        if (this.#video && !videoMid)
+            throw new RelayCallTransportError("Relay video publication has no WebRTC MID.");
         this.#publishFrame = {
             type: "offer",
             session_description: description,
-            tracks: [{ mid, name: "audio" }],
+            tracks: videoMid ? [{ mid, name: "audio" }, { mid: videoMid, name: "video" }] : [{ mid, name: "audio" }],
             ...(restart ? { restart: true } : {}),
         };
         this.#room.send(this.#publishFrame);
@@ -702,7 +815,7 @@ export class RelayCallTransport {
         // Reconnecting the signaling socket replays the exact initial offer. Relay
         // returns its cached answer; applying that answer again in stable state is
         // invalid WebRTC signaling, so recognize and ignore the replay.
-        if (peer.signalingState === "stable" && this.#initialAnswerSdp === sdp)
+        if (peer.signalingState === "stable" && (this.#initialAnswerSdp === sdp || this.#lastAnswerSdp === sdp))
             return;
         const initial = this.#initialAnswerSdp === undefined;
         if (initial)
@@ -711,22 +824,35 @@ export class RelayCallTransport {
         if (this.#peer !== peer)
             return;
         this.#initialAnswerSdp ??= sdp;
+        this.#lastAnswerSdp = sdp;
         if (initial && !this.#peerConnected)
             this.#armConnectTimer(peer);
+        if (this.#addTrackPending) {
+            this.#addTrackPending = false;
+            const deferred = this.#deferredOffer;
+            this.#deferredOffer = undefined;
+            if (deferred)
+                await this.#serverOffer(deferred);
+        }
     }
     async #serverOffer(frame) {
         const peer = this.#peer;
+        // A pull offer that crosses this participant's add-track offer is for the
+        // live session: it is answered once the add-track answer is applied.
+        if (peer && this.#addTrackPending) {
+            this.#deferredOffer = frame;
+            return;
+        }
         // A pull offer that arrives while this participant's restart offer is
         // unanswered was sent for the replaced session: the room clears those
         // pulls on restart and pulls again after the new session connects.
         if (!peer || peer.signalingState === "have-local-offer")
             return;
-        // `video` m-lines are answered receive-only by the engine and never decoded
-        // (`#remoteTrack` takes audio only).
+        // `video` m-lines are answered receive-only; `#remoteTrack` decodes them
+        // when the engine has video.
         await peer.setRemoteDescription(frame.session_description);
         const answer = await peer.createAnswer();
         await peer.setLocalDescription(answer);
-        await this.#waitForIceGathering(peer);
         if (this.#peer !== peer)
             return;
         this.#room.send({ type: "answer", session_description: this.#localDescription(peer, "answer") });
@@ -746,6 +872,7 @@ export class RelayCallTransport {
                 return;
             this.#peerConnected = true;
             this.#failedAttempts = 0;
+            void this.#readSelectedPair(peer);
             // The SFU will not pull a track that has carried no RTP, so the source
             // sends silence from here on until it is closed.
             this.#audioSource?.start?.();
@@ -860,11 +987,18 @@ export class RelayCallTransport {
         this.#peer = undefined;
         this.#peerConnected = false;
         this.#publishTransceiver = undefined;
+        this.#videoTransceiver = undefined;
         this.#initialAnswerSdp = undefined;
+        this.#lastAnswerSdp = undefined;
+        this.#addTrackPending = false;
+        this.#deferredOffer = undefined;
+        this.#remoteVideoTrack?._detach();
+        this.#remoteVideoEngineTrack = undefined;
         if (this.#remoteSink) {
             this.#retiredSinkStats = this.#sinkStats();
             this.#remoteSink.stop();
             this.#remoteSink = undefined;
+            this.#remoteSinkTrack = undefined;
         }
         if (!peer)
             return;
@@ -875,12 +1009,29 @@ export class RelayCallTransport {
         peer.oniceconnectionstatechange = null;
         closePeer(peer);
     }
-    #remoteTrack(track) {
+    /**
+     * werift fires `ontrack` again for every sending m-line on each
+     * `setRemoteDescription` (transceiverManager.js `setRemoteRTP`), so a pull
+     * offer that adds the person's video re-announces the audio track already
+     * decoded. That track keeps its sink, decoder and packet counts; a different
+     * audio track retires the old sink's counts into the call totals, as a restart does.
+     */
+    #remoteTrack(track, transceiver) {
+        if (track.kind === "video") {
+            this.#subscribeRemoteVideo(track, transceiver);
+            return;
+        }
         if (track.kind !== "audio" || !this.#factory)
             return;
-        this.#remoteSink?.stop();
+        if (this.#remoteSink && this.#remoteSinkTrack === track)
+            return;
+        if (this.#remoteSink) {
+            this.#retiredSinkStats = this.#sinkStats();
+            this.#remoteSink.stop();
+        }
         const sink = this.#factory.createAudioSink(track, this.#inboundAudio);
         this.#remoteSink = sink;
+        this.#remoteSinkTrack = track;
         sink.ondata = (data) => {
             if (this.#closed || this.#remoteSink !== sink)
                 return;
@@ -901,6 +1052,27 @@ export class RelayCallTransport {
             }
         };
     }
+    /**
+     * One `RemoteVideoTrack` per call; each new session's receiver is attached
+     * to it. werift fires `ontrack` again for every sending m-line on each
+     * `setRemoteDescription` (transceiverManager.js `setRemoteRTP`), so the
+     * track already being decoded keeps its receiver.
+     */
+    #subscribeRemoteVideo(track, transceiver) {
+        const factory = this.#factory;
+        if (!factory?.createVideoReceiver || this.#closed)
+            return;
+        if (this.#remoteVideoEngineTrack === track)
+            return;
+        this.#remoteVideoEngineTrack = track;
+        const receiver = factory.createVideoReceiver(track, transceiver);
+        const existing = this.#remoteVideoTrack;
+        const remote = existing ?? new RemoteVideoTrack();
+        this.#remoteVideoTrack = remote;
+        remote._attach(receiver);
+        if (!existing)
+            this.#emit("trackSubscribed", remote);
+    }
     #writeAudio(frame, generation) {
         const source = this.#audioSource;
         if (!source || generation !== this.#audioGeneration)
@@ -927,28 +1099,47 @@ export class RelayCallTransport {
         for (const release of [...this.#playoutWaiters])
             release();
     }
-    async #waitForIceGathering(peer) {
-        if (peer.iceGatheringState === "complete")
-            return;
-        await new Promise((resolve, reject) => {
-            const timeout = setTimeout(() => {
-                cleanup();
-                reject(new RelayCallTransportError("Timed out gathering Relay WebRTC ICE candidates."));
-            }, this.#iceGatheringTimeoutMs);
-            timeout.unref?.();
-            const changed = () => {
-                if (peer.iceGatheringState !== "complete")
-                    return;
-                cleanup();
-                resolve();
-            };
-            const cleanup = () => {
-                clearTimeout(timeout);
-                peer.removeEventListener("icegatheringstatechange", changed);
-            };
-            peer.addEventListener("icegatheringstatechange", changed);
-            changed();
+    /**
+     * Resolves once the offer is applied and has a first local candidate, or
+     * once `setLocalDescription` resolves (W3C engines resolve before gathering),
+     * whichever comes first.
+     */
+    async #applyLocalOffer(peer, offer) {
+        const applied = peer.setLocalDescription(offer);
+        let settled = false;
+        applied.catch((error) => {
+            if (!settled || this.#peer !== peer)
+                return;
+            const parsed = error instanceof Error ? error : new Error(String(error));
+            this.#rejectReady(parsed);
+            this.#emit("error", parsed);
         });
+        try {
+            await new Promise((resolve, reject) => {
+                const timeout = setTimeout(() => {
+                    cleanup();
+                    reject(new RelayCallTransportError("Timed out gathering Relay WebRTC ICE candidates."));
+                }, this.#iceGatheringTimeoutMs);
+                timeout.unref?.();
+                const cleanup = () => {
+                    clearTimeout(timeout);
+                    if (this.#firstLocalCandidate === done)
+                        this.#firstLocalCandidate = undefined;
+                };
+                const done = () => {
+                    cleanup();
+                    resolve();
+                };
+                this.#firstLocalCandidate = done;
+                applied.then(done, (error) => {
+                    cleanup();
+                    reject(error);
+                });
+            });
+        }
+        finally {
+            settled = true;
+        }
     }
     /**
      * werift emits these as W3C-style handler calls (peerConnection.js:314-341:
@@ -965,6 +1156,8 @@ export class RelayCallTransport {
                 this.#iceLocal[type] += 1;
             else
                 this.#iceLocal.other += 1;
+            if (this.#peer === peer && peer.localDescription)
+                this.#firstLocalCandidate?.();
         };
         peer.onicegatheringstatechange = () => this.#recordTransition("gathering", peer.iceGatheringState);
         peer.oniceconnectionstatechange = () => {
@@ -972,6 +1165,35 @@ export class RelayCallTransport {
                 this.#recordTransition("ice", peer.iceConnectionState);
         };
     }
+    /**
+     * W3C stats path: `transport.selectedCandidatePairId` -> `candidate-pair`
+     * -> `local-candidate.candidateType` (werift transport/dtls.js and
+     * transport/ice.js `getStats`); a nominated, succeeded pair when the
+     * transport names none.
+     */
+    async #readSelectedPair(peer) {
+        let report;
+        try {
+            report = await peer.getStats?.();
+        }
+        catch {
+            return;
+        }
+        if (!report || this.#peer !== peer)
+            return;
+        const stats = new Map();
+        report.forEach((stat) => stats.set(String(stat.id), stat));
+        const all = [...stats.values()];
+        const selectedId = all.find((stat) => stat.type === "transport" && stat.selectedCandidatePairId)
+            ?.selectedCandidatePairId;
+        const pair = selectedId !== undefined
+            ? stats.get(String(selectedId))
+            : all.find((stat) => stat.type === "candidate-pair" && stat.nominated === true && stat.state === "succeeded");
+        const local = pair ? stats.get(String(pair.localCandidateId)) : undefined;
+        if (typeof local?.candidateType !== "string")
+            return;
+        this.#selectedPair = `${local.candidateType} ${typeof local.protocol === "string" ? local.protocol : "?"}`;
+    }
     #recordTransition(kind, state) {
         this.#iceTransitions.push({ kind, state, atMs: Date.now() - this.#connectStartedAt });
     }
@@ -1012,8 +1234,18 @@ export class RelayCallTransport {
         this.#finalSourceStats ??= this.#audioSource?.stats?.();
         this.#remoteSink?.stop();
         this.#remoteSink = undefined;
+        this.#remoteSinkTrack = undefined;
         this.#localTrack?.stop();
         this.#localTrack = undefined;
+        if (this.#video) {
+            this.#video.detach();
+            this.#video.sender.close();
+        }
+        const remoteVideo = this.#remoteVideoTrack;
+        if (remoteVideo) {
+            remoteVideo._end();
+            this.#emit("trackUnsubscribed", remoteVideo);
+        }
         if (this.#peer)
             closePeer(this.#peer);
         this.#peer = undefined;