@relaymessenger/livekit 0.1.0-staging.15 → 0.1.0-staging.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -187
- package/dist/index.d.ts +0 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -3
- package/dist/index.js.map +1 -1
- package/dist/livekit.d.ts +7 -1
- package/dist/livekit.d.ts.map +1 -1
- package/dist/livekit.js +19 -7
- package/dist/livekit.js.map +1 -1
- package/package.json +3 -8
- package/dist/engine-werift-video.d.ts +0 -70
- package/dist/engine-werift-video.d.ts.map +0 -1
- package/dist/engine-werift-video.js +0 -594
- package/dist/engine-werift-video.js.map +0 -1
- package/dist/engine-werift.d.ts +0 -48
- package/dist/engine-werift.d.ts.map +0 -1
- package/dist/engine-werift.js +0 -457
- package/dist/engine-werift.js.map +0 -1
- package/dist/transport.d.ts +0 -314
- package/dist/transport.d.ts.map +0 -1
- package/dist/transport.js +0 -1303
- package/dist/transport.js.map +0 -1
- package/dist/video-frame.d.ts +0 -56
- package/dist/video-frame.d.ts.map +0 -1
- package/dist/video-frame.js +0 -245
- package/dist/video-frame.js.map +0 -1
- package/dist/video-rtp.d.ts +0 -51
- package/dist/video-rtp.d.ts.map +0 -1
- package/dist/video-rtp.js +0 -294
- package/dist/video-rtp.js.map +0 -1
- package/dist/video.d.ts +0 -144
- package/dist/video.d.ts.map +0 -1
- package/dist/video.js +0 -228
- package/dist/video.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# `@relaymessenger/livekit`
|
|
2
2
|
|
|
3
3
|
`@relaymessenger/livekit` connects Relay Calls to TypeScript LiveKit Agents.
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
objects
|
|
7
|
-
credentials.
|
|
4
|
+
It translates audio between the Relay SDK's call transport
|
|
5
|
+
(`RelayCallTransport` from `@relaymessenger/sdk/calls`) and LiveKit
|
|
6
|
+
`AudioFrame` objects. The transport owns the Node WebRTC peer, so application
|
|
7
|
+
code does not handle Cloudflare, SDP, ICE, or SFU credentials.
|
|
8
8
|
|
|
9
9
|
Install it next to the Relay SDK and the LiveKit Agents runtime:
|
|
10
10
|
|
|
@@ -43,197 +43,26 @@ reached `connected`. `RelayAudioInput` hands the AgentSession the remote
|
|
|
43
43
|
participant's audio as 24 kHz mono PCM16 frames, the format of LiveKit's own
|
|
44
44
|
room input; the Opus decoder produces that format directly.
|
|
45
45
|
|
|
46
|
-
`RelayAudioOutput` has the shape of LiveKit's own `ParticipantAudioOutput
|
|
47
|
-
|
|
46
|
+
`RelayAudioOutput` has the shape of LiveKit's own `ParticipantAudioOutput`.
|
|
47
|
+
Like LiveKit's, it waits until the person is receiving the agent's audio: the
|
|
48
|
+
transport holds each frame until then, so a greeting that starts early is not
|
|
49
|
+
cut, sends silence meanwhile, and skips nothing. Then `captureFrame()` hands each
|
|
50
|
+
frame to the transport and returns at once, so the
|
|
48
51
|
AgentSession may push a whole reply faster than real time; the engine's 20 ms
|
|
49
52
|
pump paces the wire. `flush()` closes the segment and reports
|
|
50
53
|
`playbackFinished` only after the transport has drained. `clearBuffer()` drops
|
|
51
54
|
audio that has not reached the wire and reports the segment as interrupted at
|
|
52
55
|
the position that actually played.
|
|
53
56
|
|
|
54
|
-
On the transport, `writeAudio()` resolves once its 10 ms slices are queued,
|
|
55
|
-
`queuedAudioMs()` is what has not left yet, and `waitForPlayout()` resolves when
|
|
56
|
-
the queue is empty and the pump is idle (early on `clearAudio()`).
|
|
57
|
-
|
|
58
57
|
Use `setMuted(true)` to publish participant mute state, `end()` to end the Relay
|
|
59
58
|
Call, and `close()` for local cleanup. If the signaling connection is replaced,
|
|
60
59
|
`call.transport.reconnect()` keeps the existing media peer and replays the same
|
|
61
60
|
audio publication so Relay can return its cached answer.
|
|
62
61
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
const room = relay.calls.room(callId);
|
|
72
|
-
const transport = new RelayCallTransport({ room });
|
|
73
|
-
|
|
74
|
-
transport.on("audio", ({ samples, sampleRate, channelCount }) => {
|
|
75
|
-
// Interleaved signed PCM16 from the remote Relay participant.
|
|
76
|
-
});
|
|
77
|
-
// Inbound audio is 48 kHz stereo unless you pass
|
|
78
|
-
// `inboundAudio: { sampleRate: 8000 | 12000 | 16000 | 24000 | 48000, channelCount: 1 | 2 }`;
|
|
79
|
-
// the `"wrtc"` engine accepts only the default.
|
|
80
|
-
|
|
81
|
-
await transport.connect();
|
|
82
|
-
await transport.writeAudio({
|
|
83
|
-
samples,
|
|
84
|
-
sampleRate: 48_000,
|
|
85
|
-
channelCount: 1,
|
|
86
|
-
});
|
|
87
|
-
await transport.waitForPlayout();
|
|
88
|
-
```
|
|
89
|
-
|
|
90
|
-
## Video
|
|
91
|
-
|
|
92
|
-
Relay calls carry video whenever a camera is on. The video API copies
|
|
93
|
-
LiveKit's `@livekit/rtc-node` names: `VideoSource`, `LocalVideoTrack`,
|
|
94
|
-
`VideoFrame`, `VideoBufferType`, `VideoStream`. Video needs the default
|
|
95
|
-
`"werift"` engine and the optional dependency `node-webcodecs` (prebuilt for
|
|
96
|
-
macOS arm64 and Linux x64/arm64).
|
|
97
|
-
|
|
98
|
-
Send a video feed:
|
|
99
|
-
|
|
100
|
-
```ts
|
|
101
|
-
import {
|
|
102
|
-
LocalVideoTrack,
|
|
103
|
-
VideoBufferType,
|
|
104
|
-
VideoFrame,
|
|
105
|
-
VideoSource,
|
|
106
|
-
} from "@relaymessenger/livekit/transport";
|
|
107
|
-
|
|
108
|
-
await transport.connect();
|
|
109
|
-
|
|
110
|
-
const source = new VideoSource(640, 480);
|
|
111
|
-
const track = LocalVideoTrack.createVideoTrack("camera", source);
|
|
112
|
-
await transport.publishTrack(track, {
|
|
113
|
-
videoEncoding: { maxBitrate: 800_000, maxFramerate: 15 },
|
|
114
|
-
});
|
|
115
|
-
|
|
116
|
-
// RGBA, BGRA or I420 bytes, tightly packed.
|
|
117
|
-
source.captureFrame(new VideoFrame(rgba, 640, 480, VideoBufferType.RGBA));
|
|
118
|
-
|
|
119
|
-
// Camera off, then on again; the track stays negotiated.
|
|
120
|
-
await transport.unpublishTrack(track);
|
|
121
|
-
await transport.publishTrack(track);
|
|
122
|
-
```
|
|
123
|
-
|
|
124
|
-
Receive the other participant's video:
|
|
125
|
-
|
|
126
|
-
```ts
|
|
127
|
-
import { VideoBufferType, VideoStream } from "@relaymessenger/livekit/transport";
|
|
128
|
-
|
|
129
|
-
transport.on("trackSubscribed", async (track) => {
|
|
130
|
-
const stream = new VideoStream(track, { format: VideoBufferType.RGBA, capacity: 2 });
|
|
131
|
-
for await (const { frame, timestampUs } of stream) {
|
|
132
|
-
// frame.data is width x height x 4 bytes of RGBA.
|
|
133
|
-
}
|
|
134
|
-
});
|
|
135
|
-
transport.on("remoteVideo", (on) => {
|
|
136
|
-
// The other participant's camera started or stopped sending.
|
|
137
|
-
});
|
|
138
|
-
```
|
|
139
|
-
|
|
140
|
-
Frames are decoded only while a `VideoStream` is open. `transport.videoStats()`
|
|
141
|
-
reports frames, packets, keyframes and decode errors in both directions.
|
|
142
|
-
|
|
143
|
-
## ICE servers, TURN, and diagnostics
|
|
144
|
-
|
|
145
|
-
By default the peer uses the servers the Call room sends after it joins:
|
|
146
|
-
Cloudflare's STUN server and TURN credentials that Relay mints for the call.
|
|
147
|
-
An agent in a container or behind a firewall that blocks outbound UDP connects
|
|
148
|
-
through TURN with no setup. Each restart uses the room's latest servers. A room
|
|
149
|
-
that sends none falls back to Cloudflare's STUN server
|
|
150
|
-
(`stun:stun.cloudflare.com:3478`), as Cloudflare's own Realtime echo example
|
|
151
|
-
does.
|
|
152
|
-
|
|
153
|
-
The offer leaves as soon as the first local candidate exists, without waiting
|
|
154
|
-
for ICE gathering to finish: the SFU is ICE-lite and learns the agent's address
|
|
155
|
-
from its connectivity checks, TURN relay checks included.
|
|
156
|
-
|
|
157
|
-
To use your own servers instead, pass `iceServers` in the standard
|
|
158
|
-
`RTCIceServer` shape and, if every path must go through TURN,
|
|
159
|
-
`iceTransportPolicy: "relay"`. Your value replaces the room's. Both options are
|
|
160
|
-
accepted by `RelayLiveKitCall.connect()` and `RelayCallTransport`:
|
|
161
|
-
|
|
162
|
-
```ts
|
|
163
|
-
const call = await RelayLiveKitCall.connect({
|
|
164
|
-
relay,
|
|
165
|
-
callId,
|
|
166
|
-
iceServers: [
|
|
167
|
-
{
|
|
168
|
-
urls: [
|
|
169
|
-
"turn:turn.example.com:3478?transport=udp",
|
|
170
|
-
"turn:turn.example.com:3478?transport=tcp",
|
|
171
|
-
"turns:turn.example.com:5349?transport=tcp",
|
|
172
|
-
],
|
|
173
|
-
username: process.env.TURN_USERNAME!,
|
|
174
|
-
credential: process.env.TURN_CREDENTIAL!,
|
|
175
|
-
},
|
|
176
|
-
],
|
|
177
|
-
iceTransportPolicy: "all",
|
|
178
|
-
});
|
|
179
|
-
```
|
|
180
|
-
|
|
181
|
-
`iceServers` may also be a function; it is called before every peer
|
|
182
|
-
connection, so it can mint fresh TURN credentials for each restart. The room's
|
|
183
|
-
servers are on `room.iceServers` and its `iceServers` event.
|
|
184
|
-
|
|
185
|
-
`connect()` resolves when media first reaches `connected`. It has no overall
|
|
186
|
-
deadline: when an SFU session is not `connected` within
|
|
187
|
-
`sessionConnectTimeoutMs` (5 seconds by default) of its answer, becomes
|
|
188
|
-
`failed`, or stays `disconnected` for 7 seconds, the transport closes that
|
|
189
|
-
peer, waits 250 ms (x1.1 per further attempt, at most 10 s), and publishes
|
|
190
|
-
from a new peer on a new session, for as long as the Call is ringing or in
|
|
191
|
-
progress. Outgoing audio keeps flowing into the new peer. `connect()` rejects
|
|
192
|
-
only when the Call ends, the room or transport closes, the room reports an
|
|
193
|
-
error, or the `signal` passed to it aborts. Each replacement emits
|
|
194
|
-
`restarted` with the reason and the replaced session's summary, for example
|
|
195
|
-
`local: host 2, srflx 0, relay 0; remote: udp 1473; states: new→complete 0.2s, connecting 0.3s, no connected; …`,
|
|
196
|
-
and `diagnostics().restarts` counts them. `mediaConnectTimeoutMs` and
|
|
197
|
-
`connectionTimeoutMs` are deprecated names for `sessionConnectTimeoutMs`.
|
|
198
|
-
|
|
199
|
-
`call.waitForPeerAudio(timeoutMs)` resolves once the person's audio has
|
|
200
|
-
arrived and the room shows them connected (the transport's `peerAudio` event);
|
|
201
|
-
start the agent's session after it so the first words are heard.
|
|
202
|
-
|
|
203
|
-
The same facts are available at any time from `call.diagnostics()` or
|
|
204
|
-
`transport.diagnostics()`: local candidate counts by type, the remote
|
|
205
|
-
candidates' transport and port (never their address), the ICE gathering, ICE
|
|
206
|
-
connection and peer connection state changes with their offsets from
|
|
207
|
-
`connect()`, packet counts in both directions (`inbound`: RTP received, Opus
|
|
208
|
-
decode failures, PCM frames delivered, first and last packet offsets, packets
|
|
209
|
-
in the last 5 s; `outbound`: PCM frames accepted, Opus packets, RTP written
|
|
210
|
-
with the caller's audio, RTP written with silence, first and last packet
|
|
211
|
-
offsets, packets in the last 5 s, paced queue size, pacer state), the room frames seen (`roomState` count, pull `offer` count,
|
|
212
|
-
`ended` reason, `error` messages), and the one-line `summary`, for example
|
|
213
|
-
`…; in: 1234 rtp, 0 bad, 1234 frames, first 0.9s last 41.2s, 250/5s; out: 2600 frames, 1300 opus, 1300 rtp, silence 700, first 1.1s last 41.0s, 250/5s, queue 0, pacer alive; room: 3 roomState, 1 offer`.
|
|
214
|
-
Packet counts come from the `werift` engine; `wrtc` reports zero packets and
|
|
215
|
-
`pacer n/a`.
|
|
216
|
-
|
|
217
|
-
Like a live microphone, the `werift` engine's published track sends one Opus
|
|
218
|
-
packet every 20 ms from the moment media connects until the transport closes,
|
|
219
|
-
paced by the monotonic clock: a timer that fires late sends every packet due
|
|
220
|
-
by then, so the wire carries exactly 50 packets a second; a stall longer than
|
|
221
|
-
200 ms restarts the clock instead of bursting. The track carries
|
|
222
|
-
the caller's audio when some is queued, Opus silence otherwise. Cloudflare's
|
|
223
|
-
SFU will not let the person's side pull a track that has carried no RTP, so a
|
|
224
|
-
silent agent track would never be heard. Silence never counts toward
|
|
225
|
-
`queuedAudioMs()` or `waitForPlayout()`. The `wrtc` engine sends only the
|
|
226
|
-
audio written to it; write silence yourself if you use it.
|
|
227
|
-
|
|
228
|
-
`onWarning` is called once per call, with the summary, when outbound audio is
|
|
229
|
-
queued but no RTP packet has been written for 2 s while media is connected.
|
|
230
|
-
Nothing is restarted; the callback exists so the failing direction is named in
|
|
231
|
-
the agent's logs.
|
|
232
|
-
|
|
233
|
-
`RelayCallTransport` consumes the SDK's `CallRoom`; it does not duplicate the
|
|
234
|
-
room protocol. The default engine is `werift` (pure TypeScript WebRTC) with
|
|
235
|
-
`@evan/opus` (prebuilt Opus, WASM fallback), so no native WebRTC binding is
|
|
236
|
-
loaded. Pass `engine: "wrtc"` to use the optional `@roamhq/wrtc` binding
|
|
237
|
-
instead. The transport boundary remains provider-neutral for additional adapters. `@livekit/agents`
|
|
238
|
-
and `@livekit/rtc-node` are peer dependencies so the host agent process owns
|
|
239
|
-
those runtimes.
|
|
62
|
+
`RelayLiveKitCall.connect()` accepts the transport's `iceServers`,
|
|
63
|
+
`iceTransportPolicy` and `sessionConnectTimeoutMs`; `call.waitForPeerAudio()`
|
|
64
|
+
and `call.diagnostics()` read the transport. Video, ICE servers, restarts and
|
|
65
|
+
diagnostics are documented with the transport in the
|
|
66
|
+
[`@relaymessenger/sdk` README](https://github.com/RelayMessenger/Relay-SDK/tree/main/packages/sdk#join-a-call-as-the-agent).
|
|
67
|
+
`@livekit/agents` and `@livekit/rtc-node` are peer dependencies so the host
|
|
68
|
+
agent process owns those runtimes.
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,2 @@
|
|
|
1
1
|
export { RelayAudioInput, RelayAudioOutput, LIVEKIT_ROOM_INPUT_AUDIO, RelayLiveKitCall, createRelayLiveKitAudio, type RelayLiveKitAudio, type RelayLiveKitAudioOptions, type RelayLiveKitConnectOptions, } from "./livekit.js";
|
|
2
|
-
export { RelayCallTransport, RelayCallTransportError, type RelayAudioFrame, type RelayCallVideoStats, type RelayCallEngine, type RelayCallIceDiagnostics, type RelayCallInboundDiagnostics, type RelayCallOutboundDiagnostics, type RelayCallRestartEvent, type RelayCallRestartReason, type RelayCallRoomDiagnostics, type RelayCallTransportCloseEvent, type RelayCallTransportOptions, type RelayIceServer, type RelayIceServersProvider, type RelayIceTransportPolicy, type RelayInboundAudioFormat, } from "./transport.js";
|
|
3
|
-
export { createWeriftWebRTCFactory } from "./engine-werift.js";
|
|
4
|
-
export { LocalVideoTrack, RemoteVideoTrack, VideoBufferType, VideoCodec, VideoFrame, VideoRotation, VideoSource, VideoStream, type RelayVideoReceiverStats, type RelayVideoSenderStats, type TrackPublishOptions, type VideoEncoding, type VideoFrameEvent, type VideoStreamOptions, } from "./video.js";
|
|
5
2
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,gBAAgB,EAChB,uBAAuB,EACvB,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,KAAK,0BAA0B,GAChC,MAAM,cAAc,CAAC
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,gBAAgB,EAChB,uBAAuB,EACvB,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,KAAK,0BAA0B,GAChC,MAAM,cAAc,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,2 @@
|
|
|
1
1
|
export { RelayAudioInput, RelayAudioOutput, LIVEKIT_ROOM_INPUT_AUDIO, RelayLiveKitCall, createRelayLiveKitAudio, } from "./livekit.js";
|
|
2
|
-
export { RelayCallTransport, RelayCallTransportError, } from "./transport.js";
|
|
3
|
-
export { createWeriftWebRTCFactory } from "./engine-werift.js";
|
|
4
|
-
export { LocalVideoTrack, RemoteVideoTrack, VideoBufferType, VideoCodec, VideoFrame, VideoRotation, VideoSource, VideoStream, } from "./video.js";
|
|
5
2
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,gBAAgB,EAChB,uBAAuB,GAIxB,MAAM,cAAc,CAAC
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,gBAAgB,EAChB,uBAAuB,GAIxB,MAAM,cAAc,CAAC"}
|
package/dist/livekit.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { AudioInput, AudioOutput, type AgentSession } from "@livekit/agents";
|
|
|
2
2
|
import { AudioFrame } from "@livekit/rtc-node";
|
|
3
3
|
import type Relay from "@relaymessenger/sdk";
|
|
4
4
|
import type { CallRoomOptions } from "@relaymessenger/sdk";
|
|
5
|
-
import { RelayCallTransport, type RelayCallIceDiagnostics, type RelayIceServer, type RelayIceServersProvider, type RelayInboundAudioFormat, type RelayIceTransportPolicy } from "
|
|
5
|
+
import { RelayCallTransport, type RelayCallIceDiagnostics, type RelayIceServer, type RelayIceServersProvider, type RelayInboundAudioFormat, type RelayIceTransportPolicy } from "@relaymessenger/sdk/calls";
|
|
6
6
|
/**
|
|
7
7
|
* The format LiveKit's own room input hands an AgentSession:
|
|
8
8
|
* `@livekit/agents` dist/voice/room_io/room_io.js:46-48
|
|
@@ -34,6 +34,12 @@ export declare class RelayAudioInput extends AudioInput {
|
|
|
34
34
|
export declare class RelayAudioOutput extends AudioOutput {
|
|
35
35
|
#private;
|
|
36
36
|
constructor(transport: RelayCallTransport, sampleRate?: number);
|
|
37
|
+
/**
|
|
38
|
+
* Hands the frame to the transport, which holds it until the person receives
|
|
39
|
+
* the agent's audio (PROTOCOL.md section 6b) and resolves once it is queued,
|
|
40
|
+
* so this waits as LiveKit's room audio output waits for the track's
|
|
41
|
+
* subscription. A frame dropped by `clearBuffer()` meanwhile is not counted.
|
|
42
|
+
*/
|
|
37
43
|
captureFrame(frame: AudioFrame): Promise<void>;
|
|
38
44
|
/** Mark the segment complete; `onPlaybackFinished` fires once the transport has drained. */
|
|
39
45
|
flush(): void;
|
package/dist/livekit.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"livekit.d.ts","sourceRoot":"","sources":["../src/livekit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAU,KAAK,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAC;AAC7C,OAAO,KAAK,EAAY,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EACL,kBAAkB,EAElB,KAAK,uBAAuB,EAE5B,KAAK,cAAc,EACnB,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAE7B,MAAM,
|
|
1
|
+
{"version":3,"file":"livekit.d.ts","sourceRoot":"","sources":["../src/livekit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAU,KAAK,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAC;AAC7C,OAAO,KAAK,EAAY,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EACL,kBAAkB,EAElB,KAAK,uBAAuB,EAE5B,KAAK,cAAc,EACnB,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAE7B,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;GASG;AACH,eAAO,MAAM,wBAAwB,EAAE,QAAQ,CAAC,uBAAuB,CAGrE,CAAC;AAEH,kFAAkF;AAClF,qBAAa,eAAgB,SAAQ,UAAU;;gBASjC,SAAS,EAAE,kBAAkB;IA4BhC,WAAW,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI;IAI9B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAQtC;AAED;;;;;;;;;GASG;AACH,qBAAa,gBAAiB,SAAQ,WAAW;;gBAanC,SAAS,EAAE,kBAAkB,EAAE,UAAU,SAAS;IAK9D;;;;;OAKG;IACY,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAsB7D,4FAA4F;IACnF,KAAK,IAAI,IAAI;IAWb,WAAW,IAAI,IAAI;IAU5B,KAAK,IAAI,IAAI;CAmBd;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,eAAe,CAAC;IACvB,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED,MAAM,WAAW,wBAAwB;IACvC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,iFAAiF;AACjF,eAAO,MAAM,uBAAuB,GAClC,WAAW,kBAAkB,EAC7B,UAAS,wBAA6B,KACrC,iBAGD,CAAC;AAEH,MAAM,WAAW,0BAA2B,SAAQ,wBAAwB;IAC1E,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,eAAe,CAAC;IAKvB;;;;OAIG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,GAAG,uBAAuB,CAAC;IACxD,kDAAkD;IAClD,kBAAkB,CAAC,EAAE,uBAAuB,CAAC;IAG7C;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uGAAuG;IACvG,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,iDAAiD;IACjD,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,yFAAyF;IACzF,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,KAAK,uBAAuB,GAAG,IAAI,CAAC,YAAY,EAAE,OAAO,GAAG,QAAQ,CAAC,CAAC;AAEtE;;;;GAIG;AACH,qBAAa,gBAAgB;;IAC3B,QAAQ,CAAC,SAAS,EAAE,kBAAkB,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAIlC,OAAO;WAMM,OAAO,CAAC,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA4BpF,MAAM,CAAC,OAAO,EAAE,uBAAuB,GAAG,IAAI;IAQ9C,MAAM,IAAI,IAAI;IAOd;;;;OAIG;IACH,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIlD,4FAA4F;IAC5F,WAAW,IAAI,uBAAuB;IAItC,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAI9B,GAAG,IAAI,IAAI;IAIL,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAQ7B"}
|
package/dist/livekit.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AudioInput, AudioOutput, Future } from "@livekit/agents";
|
|
2
2
|
import { AudioFrame } from "@livekit/rtc-node";
|
|
3
3
|
import { TransformStream } from "node:stream/web";
|
|
4
|
-
import { RelayCallTransport, } from "
|
|
4
|
+
import { RelayCallTransport, } from "@relaymessenger/sdk/calls";
|
|
5
5
|
/**
|
|
6
6
|
* The format LiveKit's own room input hands an AgentSession:
|
|
7
7
|
* `@livekit/agents` dist/voice/room_io/room_io.js:46-48
|
|
@@ -80,11 +80,19 @@ export class RelayAudioOutput extends AudioOutput {
|
|
|
80
80
|
#flushDone = true;
|
|
81
81
|
/** Resolved by `clearBuffer()` with the milliseconds still queued at that moment. */
|
|
82
82
|
#interruptedFuture = new Future();
|
|
83
|
+
/** Bumped by `clearBuffer()`; a frame the transport dropped across it is not counted (agents-js `interruptionGeneration`). */
|
|
84
|
+
#interruptionGeneration = 0;
|
|
83
85
|
#closed = false;
|
|
84
86
|
constructor(transport, sampleRate = 48_000) {
|
|
85
87
|
super(sampleRate, undefined, { pause: false });
|
|
86
88
|
this.#transport = transport;
|
|
87
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* Hands the frame to the transport, which holds it until the person receives
|
|
92
|
+
* the agent's audio (PROTOCOL.md section 6b) and resolves once it is queued,
|
|
93
|
+
* so this waits as LiveKit's room audio output waits for the track's
|
|
94
|
+
* subscription. A frame dropped by `clearBuffer()` meanwhile is not counted.
|
|
95
|
+
*/
|
|
88
96
|
async captureFrame(frame) {
|
|
89
97
|
if (this.#closed)
|
|
90
98
|
throw new Error("Relay LiveKit audio output is closed.");
|
|
@@ -92,18 +100,21 @@ export class RelayAudioOutput extends AudioOutput {
|
|
|
92
100
|
this.logger.error("captureFrame called while flush is in progress");
|
|
93
101
|
await this.#flushTask;
|
|
94
102
|
}
|
|
95
|
-
|
|
96
|
-
if (!this.#firstFrameEmitted) {
|
|
97
|
-
this.#firstFrameEmitted = true;
|
|
98
|
-
this.onPlaybackStarted(Date.now());
|
|
99
|
-
}
|
|
100
|
-
this.#pushedDuration += frame.samplesPerChannel / frame.sampleRate;
|
|
103
|
+
const interruptionGeneration = this.#interruptionGeneration;
|
|
101
104
|
// Resolves once the slices are queued; the engine's pump paces the wire.
|
|
102
105
|
await this.#transport.writeAudio({
|
|
103
106
|
samples: frame.data,
|
|
104
107
|
sampleRate: frame.sampleRate,
|
|
105
108
|
channelCount: frame.channels,
|
|
106
109
|
});
|
|
110
|
+
if (interruptionGeneration !== this.#interruptionGeneration)
|
|
111
|
+
return;
|
|
112
|
+
await super.captureFrame(frame);
|
|
113
|
+
if (!this.#firstFrameEmitted) {
|
|
114
|
+
this.#firstFrameEmitted = true;
|
|
115
|
+
this.onPlaybackStarted(Date.now());
|
|
116
|
+
}
|
|
117
|
+
this.#pushedDuration += frame.samplesPerChannel / frame.sampleRate;
|
|
107
118
|
}
|
|
108
119
|
/** Mark the segment complete; `onPlaybackFinished` fires once the transport has drained. */
|
|
109
120
|
flush() {
|
|
@@ -119,6 +130,7 @@ export class RelayAudioOutput extends AudioOutput {
|
|
|
119
130
|
void this.#flushTask.catch(() => undefined);
|
|
120
131
|
}
|
|
121
132
|
clearBuffer() {
|
|
133
|
+
this.#interruptionGeneration += 1;
|
|
122
134
|
const queuedMs = this.#transport.queuedAudioMs();
|
|
123
135
|
this.#transport.clearAudio();
|
|
124
136
|
if (this.#interruptedFuture.done)
|
package/dist/livekit.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"livekit.js","sourceRoot":"","sources":["../src/livekit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAqB,MAAM,iBAAiB,CAAC;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAoC,MAAM,iBAAiB,CAAC;AAGpF,OAAO,EACL,kBAAkB,GASnB,MAAM,
|
|
1
|
+
{"version":3,"file":"livekit.js","sourceRoot":"","sources":["../src/livekit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAqB,MAAM,iBAAiB,CAAC;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAoC,MAAM,iBAAiB,CAAC;AAGpF,OAAO,EACL,kBAAkB,GASnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAsC,MAAM,CAAC,MAAM,CAAC;IACvF,UAAU,EAAE,MAAM;IAClB,YAAY,EAAE,CAAC;CAChB,CAAC,CAAC;AAEH,kFAAkF;AAClF,MAAM,OAAO,eAAgB,SAAQ,UAAU;IACpC,UAAU,CAAqB;IAC/B,OAAO,CAA0C;IACjD,QAAQ,CAAmC;IACpD,gGAAgG;IAChG,OAAO,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3C,SAAS,GAAG,KAAK,CAAC;IAClB,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAY,SAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,MAAM,OAAO,GAAG,IAAI,eAAe,EAA0B,CAAC;QAC9D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,CAAC,QAAQ,GAAG,CAAC,KAAK,EAAE,EAAE;YACxB,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS;gBAAE,OAAO;YAC5C,MAAM,KAAK,GAAG,IAAI,UAAU,CAC1B,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,YAAY,CAC1C,CAAC;YACF,uEAAuE;YACvE,sEAAsE;YACtE,sEAAsE;YACtE,8DAA8D;YAC9D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO;iBACxB,IAAI,CAAC,KAAK,IAAI,EAAE;gBACf,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;gBACzB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC,CAAC;iBACD,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC,CAAC;QACF,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAEQ,WAAW,CAAC,QAAiB;QACpC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAEQ,KAAK,CAAC,KAAK;QAClB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,OAAO,CAAC;QACnB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;CACF;AAED;;;;;;;;;GASG;AACH,MAAM,OAAO,gBAAiB,SAAQ,WAAW;IACtC,UAAU,CAAqB;IACxC,2DAA2D;IAC3D,eAAe,GAAG,CAAC,CAAC;IACpB,kBAAkB,GAAG,KAAK,CAAC;IAC3B,UAAU,CAA4B;IACtC,UAAU,GAAG,IAAI,CAAC;IAClB,qFAAqF;IACrF,kBAAkB,GAAG,IAAI,MAAM,EAAU,CAAC;IAC1C,8HAA8H;IAC9H,uBAAuB,GAAG,CAAC,CAAC;IAC5B,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAY,SAA6B,EAAE,UAAU,GAAG,MAAM;QAC5D,KAAK,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACM,KAAK,CAAC,YAAY,CAAC,KAAiB;QAC3C,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3E,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACpE,MAAM,IAAI,CAAC,UAAU,CAAC;QACxB,CAAC;QACD,MAAM,sBAAsB,GAAG,IAAI,CAAC,uBAAuB,CAAC;QAC5D,yEAAyE;QACzE,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,OAAO,EAAE,KAAK,CAAC,IAAI;YACnB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,YAAY,EAAE,KAAK,CAAC,QAAQ;SAC7B,CAAC,CAAC;QACH,IAAI,sBAAsB,KAAK,IAAI,CAAC,uBAAuB;YAAE,OAAO;QACpE,MAAM,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC7B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;YAC/B,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC,iBAAiB,GAAG,KAAK,CAAC,UAAU,CAAC;IACrE,CAAC;IAED,4FAA4F;IACnF,KAAK;QACZ,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO;QAClC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAChD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACxD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,KAAK,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;IAEQ,WAAW;QAClB,IAAI,CAAC,uBAAuB,IAAI,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC;QACjD,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI;YAAE,OAAO;QACzC,IAAI,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,IAAI,CAAC,sBAAsB,KAAK,CAAC;YAAE,OAAO;QAC5E,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU;YAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QACtD,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI;YAAE,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/E,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,iBAAiB,GAAG,IAAI,CAAC,kBAAkB,CAAC;QAClD,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,EAAE,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;QAChF,MAAM,WAAW,GAAG,iBAAiB,CAAC,IAAI,CAAC;QAC3C,IAAI,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5C,IAAI,WAAW,EAAE,CAAC;YAChB,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,gBAAgB,GAAG,iBAAiB,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;QAChC,IAAI,IAAI,CAAC,kBAAkB,KAAK,iBAAiB;YAAE,IAAI,CAAC,kBAAkB,GAAG,IAAI,MAAM,EAAU,CAAC;QAClG,IAAI,CAAC,kBAAkB,CAAC,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAC,CAAC;IAC7D,CAAC;CACF;AAWD,iFAAiF;AACjF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,SAA6B,EAC7B,UAAoC,EAAE,EACnB,EAAE,CAAC,CAAC;IACvB,KAAK,EAAE,IAAI,eAAe,CAAC,SAAS,CAAC;IACrC,MAAM,EAAE,IAAI,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,gBAAgB,CAAC;CAClE,CAAC,CAAC;AAmCH;;;;GAIG;AACH,MAAM,OAAO,gBAAgB;IAClB,SAAS,CAAqB;IAC9B,KAAK,CAAkB;IACvB,MAAM,CAAmB;IAClC,QAAQ,CAAsC;IAC9C,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAoB,SAA6B,EAAE,KAAwB;QACzE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAmC;QACtD,MAAM,gBAAgB,GAA8B;YAClD,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,YAAY,EAAE,wBAAwB;YACtC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,GAAG,CAAC,OAAO,CAAC,qBAAqB,KAAK,SAAS;gBAC7C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC;YAC7D,GAAG,CAAC,OAAO,CAAC,uBAAuB,KAAK,SAAS;gBAC/C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,uBAAuB,EAAE,OAAO,CAAC,uBAAuB,EAAE,CAAC;YACjE,GAAG,CAAC,OAAO,CAAC,qBAAqB,KAAK,SAAS;gBAC7C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC;YAC7D,GAAG,CAAC,OAAO,CAAC,mBAAmB,KAAK,SAAS;gBAC3C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC;SAC1D,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;QAC3D,MAAM,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1E,OAAO,IAAI,gBAAgB,CAAC,SAAS,EAAE,uBAAuB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IACtF,CAAC;IAED,MAAM,CAAC,OAAgC;QACrC,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnE,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO;YAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QAC9D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACjC,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,MAAM;QACJ,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC3B,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK;YAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC;QAC/E,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;QAClF,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACH,gBAAgB,CAAC,SAAiB;QAChC,OAAO,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;IAED,4FAA4F;IAC5F,WAAW;QACT,OAAO,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;IACtC,CAAC;IAED,QAAQ,CAAC,KAAc;QACrB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,GAAG;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@relaymessenger/livekit",
|
|
3
|
-
"version": "0.1.0-staging.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.1.0-staging.17",
|
|
4
|
+
"description": "LiveKit Agents audio input and output for Relay Calls, over @relaymessenger/sdk/calls.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
7
7
|
"main": "./dist/index.js",
|
|
@@ -12,11 +12,6 @@
|
|
|
12
12
|
"import": "./dist/index.js",
|
|
13
13
|
"default": "./dist/index.js"
|
|
14
14
|
},
|
|
15
|
-
"./transport": {
|
|
16
|
-
"types": "./dist/transport.d.ts",
|
|
17
|
-
"import": "./dist/transport.js",
|
|
18
|
-
"default": "./dist/transport.js"
|
|
19
|
-
},
|
|
20
15
|
"./package.json": "./package.json"
|
|
21
16
|
},
|
|
22
17
|
"files": [
|
|
@@ -38,7 +33,7 @@
|
|
|
38
33
|
"license": "MIT",
|
|
39
34
|
"dependencies": {
|
|
40
35
|
"@evan/opus": "1.0.3",
|
|
41
|
-
"@relaymessenger/sdk": "0.3.6-staging.
|
|
36
|
+
"@relaymessenger/sdk": "0.3.6-staging.24",
|
|
42
37
|
"rtp-packet": "0.4.2",
|
|
43
38
|
"werift": "0.24.4"
|
|
44
39
|
},
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Video for the werift engine: `node-webcodecs` 1.3.0 encodes and decodes
|
|
3
|
-
* (its static build: LGPL FFmpeg with openh264 and libvpx), `rtp-packet`
|
|
4
|
-
* packetizes, `video-rtp.ts` depacketizes, werift carries RTP and RTCP.
|
|
5
|
-
*
|
|
6
|
-
* Choices measured on 2026-09-22 (/tmp/video-spike-webcodecs, the spike that
|
|
7
|
-
* picked this engine), each kept here:
|
|
8
|
-
* - `NODE_WEBCODECS_FORCE=static` selects the bundled FFmpeg
|
|
9
|
-
* (node-webcodecs dist/native.js `load()`; the default order tries a
|
|
10
|
-
* dynamic build linked against system FFmpeg first).
|
|
11
|
-
* - `hardwareAcceleration: "prefer-software"` on encoder and decoder: on a
|
|
12
|
-
* Mac, VideoToolbox otherwise takes the stream.
|
|
13
|
-
* - H.264 is sent as constrained baseline `avc1.42e01f`, Annex-B; the codec
|
|
14
|
-
* string FFmpeg reports for H.264 is "MPEG4/ISO/AVC", so the codec is never
|
|
15
|
-
* matched by FFmpeg's name, only by the negotiated MIME type.
|
|
16
|
-
* - A keyframe every 1 s, and on every PLI or FIR from the SFU.
|
|
17
|
-
* - The send codec is the first codec of the SFU's answer (werift
|
|
18
|
-
* rtpSender.js:408 `this.codec = params.codecs[0]`).
|
|
19
|
-
* - The receive codec is learned from the payload type of the first RTP
|
|
20
|
-
* packet, looked up in the pulled m-line's codecs: the SFU's pull offer
|
|
21
|
-
* lists every codec (spike receiver.mjs).
|
|
22
|
-
* - werift sends no PLI on loss by itself: the receiver sends one when the
|
|
23
|
-
* stream starts, when a frame is dropped and on a decode error, then every
|
|
24
|
-
* 1 s until a keyframe decodes (spike receiver.mjs `pliTimer`).
|
|
25
|
-
*/
|
|
26
|
-
import { MediaStreamTrack, type RTCRtpTransceiver } from "werift";
|
|
27
|
-
import type { RelayMediaStreamTrackLike } from "./transport.js";
|
|
28
|
-
import { type RelayVideoFactory, type RelayVideoReceiverLike, type RelayVideoReceiverStats, type RelayVideoSenderLike, type RelayVideoSenderStats, type TrackPublishOptions } from "./video.js";
|
|
29
|
-
import { VideoFrame, VideoRotation } from "./video-frame.js";
|
|
30
|
-
type WebCodecs = typeof import("node-webcodecs");
|
|
31
|
-
export declare const KEYFRAME_INTERVAL_MS = 1000;
|
|
32
|
-
export declare const KEYFRAME_REQUEST_INTERVAL_MS = 1000;
|
|
33
|
-
export declare const DEFAULT_VIDEO_BITRATE = 800000;
|
|
34
|
-
export declare const DEFAULT_VIDEO_FRAMERATE = 30;
|
|
35
|
-
/** Loads node-webcodecs once, on first video use, so audio-only calls never load FFmpeg. */
|
|
36
|
-
export declare const loadWebCodecs: () => Promise<WebCodecs>;
|
|
37
|
-
/**
|
|
38
|
-
* Raw frames in, H.264 or VP8 RTP out on one werift track. One sender lives
|
|
39
|
-
* for the whole call; a restart binds it to the new peer's transceiver.
|
|
40
|
-
*/
|
|
41
|
-
export declare class WeriftVideoSender implements RelayVideoSenderLike {
|
|
42
|
-
#private;
|
|
43
|
-
readonly track: RelayMediaStreamTrackLike;
|
|
44
|
-
constructor(wc: WebCodecs, options?: TrackPublishOptions);
|
|
45
|
-
/**
|
|
46
|
-
* Offer the codecs this sender can encode, the preferred one first (werift
|
|
47
|
-
* fills an empty `transceiver.codecs` from the peer's config at
|
|
48
|
-
* `createOffer`, peerConnection.js:520-523, so setting it here wins).
|
|
49
|
-
*/
|
|
50
|
-
bind(transceiver: unknown, peer: unknown): void;
|
|
51
|
-
setEnabled(enabled: boolean): void;
|
|
52
|
-
capture(frame: VideoFrame, timestampUs: bigint, rotation: VideoRotation): void;
|
|
53
|
-
stats(): RelayVideoSenderStats;
|
|
54
|
-
close(): void;
|
|
55
|
-
}
|
|
56
|
-
/**
|
|
57
|
-
* One pulled video m-line: RTP in, decoded I420 frames out. RTP is counted
|
|
58
|
-
* always; it is reordered, assembled and decoded only while active.
|
|
59
|
-
*/
|
|
60
|
-
export declare class WeriftVideoReceiver implements RelayVideoReceiverLike {
|
|
61
|
-
#private;
|
|
62
|
-
onframe: RelayVideoReceiverLike["onframe"];
|
|
63
|
-
constructor(track: MediaStreamTrack, transceiver: RTCRtpTransceiver | undefined);
|
|
64
|
-
setActive(active: boolean): void;
|
|
65
|
-
stats(): RelayVideoReceiverStats;
|
|
66
|
-
stop(): void;
|
|
67
|
-
}
|
|
68
|
-
export declare const createWeriftVideoFactory: () => RelayVideoFactory;
|
|
69
|
-
export {};
|
|
70
|
-
//# sourceMappingURL=engine-werift-video.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"engine-werift-video.d.ts","sourceRoot":"","sources":["../src/engine-werift-video.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,EACL,gBAAgB,EAEhB,KAAK,iBAAiB,EAGvB,MAAM,QAAQ,CAAC;AAEhB,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,gBAAgB,CAAC;AAChE,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,oBAAoB,EACzB,KAAK,qBAAqB,EAC1B,KAAK,mBAAmB,EAGzB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAmB,UAAU,EAAE,aAAa,EAAoB,MAAM,kBAAkB,CAAC;AAWhG,KAAK,SAAS,GAAG,cAAc,gBAAgB,CAAC,CAAC;AAMjD,eAAO,MAAM,oBAAoB,OAAQ,CAAC;AAC1C,eAAO,MAAM,4BAA4B,OAAQ,CAAC;AAClD,eAAO,MAAM,qBAAqB,SAAU,CAAC;AAC7C,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAU1C,4FAA4F;AAC5F,eAAO,MAAM,aAAa,QAAO,OAAO,CAAC,SAAS,CAgBjD,CAAC;AA0CF;;;GAGG;AACH,qBAAa,iBAAkB,YAAW,oBAAoB;;IAC5D,QAAQ,CAAC,KAAK,EAAE,yBAAyB,CAAC;gBA8B9B,EAAE,EAAE,SAAS,EAAE,OAAO,GAAE,mBAAwB;IAS5D;;;;OAIG;IACH,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,GAAG,IAAI;IAqB/C,UAAU,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKlC,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,GAAG,IAAI;IAqD9E,KAAK,IAAI,qBAAqB;IAmB9B,KAAK,IAAI,IAAI;CAwEd;AAED;;;GAGG;AACH,qBAAa,mBAAoB,YAAW,sBAAsB;;IAChE,OAAO,EAAE,sBAAsB,CAAC,SAAS,CAAC,CAAQ;gBA+BtC,KAAK,EAAE,gBAAgB,EAAE,WAAW,EAAE,iBAAiB,GAAG,SAAS;IAW/E,SAAS,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI;IAehC,KAAK,IAAI,uBAAuB;IAiBhC,IAAI,IAAI,IAAI;CA8Kb;AAED,eAAO,MAAM,wBAAwB,QAAO,iBAI1C,CAAC"}
|