@relaymessenger/livekit 0.1.0-staging.2 → 0.1.0-staging.21

package/README.md

@@ -1,10 +1,10 @@
 # `@relaymessenger/livekit`
 
 `@relaymessenger/livekit` connects Relay Calls to TypeScript LiveKit Agents.
-Relay stays responsible for the Call resource and signaling; the package owns
-the Node WebRTC peer and translates audio between Relay and LiveKit `AudioFrame`
-objects. Application code does not handle Cloudflare, SDP, ICE, or SFU
-credentials.
+It translates audio between the Relay SDK's call transport
+(`RelayCallTransport` from `@relaymessenger/sdk/calls`) and LiveKit
+`AudioFrame` objects. The transport owns the Node WebRTC peer, so application
+code does not handle Cloudflare, SDP, ICE, or SFU credentials.
 
 Install it next to the Relay SDK and the LiveKit Agents runtime:
 
@@ -39,94 +39,30 @@ async function answerCall(callId: string, session: AgentSession) {
 ```
 
 `RelayLiveKitCall.connect()` does not resolve until the WebRTC media peer has
-reached `connected`. `RelayAudioInput` converts the remote participant's PCM16
-audio to LiveKit frames. `RelayAudioOutput` publishes LiveKit TTS frames to
-Relay, pacing larger frames into 10 ms WebRTC source slices. `clearBuffer()`
-drops queued outbound audio that has not yet reached WebRTC.
+reached `connected`. `RelayAudioInput` hands the AgentSession the remote
+participant's audio as 24 kHz mono PCM16 frames, the format of LiveKit's own
+room input; the Opus decoder produces that format directly.
+
+`RelayAudioOutput` has the shape of LiveKit's own `ParticipantAudioOutput`.
+Like LiveKit's, it waits until the person is receiving the agent's audio: the
+transport holds each frame until then, so a greeting that starts early is not
+cut, sends silence meanwhile, and skips nothing. Then `captureFrame()` hands each
+frame to the transport and returns at once, so the
+AgentSession may push a whole reply faster than real time; the engine's 20 ms
+pump paces the wire. `flush()` closes the segment and reports
+`playbackFinished` only after the transport has drained. `clearBuffer()` drops
+audio that has not reached the wire and reports the segment as interrupted at
+the position that actually played.
 
 Use `setMuted(true)` to publish participant mute state, `end()` to end the Relay
 Call, and `close()` for local cleanup. If the signaling connection is replaced,
 `call.transport.reconnect()` keeps the existing media peer and replays the same
 audio publication so Relay can return its cached answer.
 
-## Provider-neutral transport
-
-The WebRTC/PCM layer is a separate public entry point for future adapters that
-do not use LiveKit Agents:
-
-```ts
-import { RelayCallTransport } from "@relaymessenger/livekit/transport";
-
-const room = relay.calls.room(callId);
-const transport = new RelayCallTransport({ room });
-
-transport.on("audio", ({ samples, sampleRate, channelCount }) => {
-  // Interleaved signed PCM16 from the remote Relay participant.
-});
-
-await transport.connect();
-await transport.writeAudio({
-  samples,
-  sampleRate: 48_000,
-  channelCount: 1,
-});
-```
-
-## ICE servers, TURN, and diagnostics
-
-By default the peer gathers host candidates only and Cloudflare's SFU supplies
-its own candidates in the answer. Inside a container or behind a firewall that
-blocks outbound UDP, pass TURN servers in the standard `RTCIceServer` shape and,
-if every path must go through TURN, `iceTransportPolicy: "relay"`. Both options
-are accepted by `RelayLiveKitCall.connect()` and `RelayCallTransport`.
-
-```ts
-const call = await RelayLiveKitCall.connect({
-  relay,
-  callId,
-  iceServers: [
-    {
-      urls: [
-        "turn:turn.cloudflare.com:3478?transport=udp",
-        "turn:turn.cloudflare.com:3478?transport=tcp",
-        "turns:turn.cloudflare.com:5349?transport=tcp",
-      ],
-      username: process.env.TURN_USERNAME!,
-      credential: process.env.TURN_CREDENTIAL!,
-    },
-  ],
-  iceTransportPolicy: "all",
-  mediaConnectTimeoutMs: 15_000,
-});
-```
-
-`mediaConnectTimeoutMs` bounds the wait for the media peer to reach
-`connected` (15 seconds by default). When that wait expires, the error message
-carries what ICE saw, for example
-`Timed out connecting Relay WebRTC media (local: host 2, srflx 0, relay 0; remote: udp 1473; states: new→complete 0.2s, connecting 0.3s, no connected)`.
-The same facts are available at any time from `call.diagnostics()` or
-`transport.diagnostics()`: local candidate counts by type, the remote
-candidates' transport and port (never their address), the ICE gathering, ICE
-connection and peer connection state changes with their offsets from
-`connect()`, packet counts in both directions (`inbound`: RTP received, Opus
-decode failures, PCM frames delivered, first and last packet offsets, packets
-in the last 5 s; `outbound`: PCM frames accepted, Opus packets, RTP written,
-first and last packet offsets, packets in the last 5 s, paced queue size,
-pacer state), the room frames seen (`roomState` count, pull `offer` count,
-`ended` reason, `error` messages), and the one-line `summary`, for example
-`…; in: 1234 rtp, 0 bad, 1234 frames, first 0.9s last 41.2s, 250/5s; out: 2050 frames, 2050 opus, 2050 rtp, first 1.1s last 41.0s, 250/5s, queue 0, pacer alive; room: 3 roomState, 1 offer`.
-Packet counts come from the `werift` engine; `wrtc` reports zero packets and
-`pacer n/a`.
-
-`onWarning` is called once per call, with the summary, when outbound audio is
-queued but no RTP packet has been written for 2 s while media is connected.
-Nothing is restarted; the callback exists so the failing direction is named in
-the agent's logs.
-
-`RelayCallTransport` consumes the SDK's `CallRoom`; it does not duplicate the
-room protocol. The default engine is `werift` (pure TypeScript WebRTC) with
-`@evan/opus` (prebuilt Opus, WASM fallback), so no native WebRTC binding is
-loaded. Pass `engine: "wrtc"` to use the optional `@roamhq/wrtc` binding
-instead. The transport boundary remains provider-neutral for additional adapters. `@livekit/agents`
-and `@livekit/rtc-node` are peer dependencies so the host agent process owns
-those runtimes.
+`RelayLiveKitCall.connect()` accepts the transport's `iceServers`,
+`iceTransportPolicy` and `sessionConnectTimeoutMs`; `call.waitForPeerAudio()`
+and `call.diagnostics()` read the transport. Video, ICE servers, restarts and
+diagnostics are documented with the transport in the
+[`@relaymessenger/sdk` README](https://github.com/RelayMessenger/Relay-SDK/tree/main/packages/sdk#join-a-call-as-the-agent).
+`@livekit/agents` and `@livekit/rtc-node` are peer dependencies so the host
+agent process owns those runtimes.

package/dist/index.d.ts

@@ -1,4 +1,2 @@
-export { RelayAudioInput, RelayAudioOutput, RelayLiveKitCall, createRelayLiveKitAudio, type RelayLiveKitAudio, type RelayLiveKitAudioOptions, type RelayLiveKitConnectOptions, } from "./livekit.js";
-export { RelayCallTransport, RelayCallTransportError, type RelayAudioFrame, type RelayCallEngine, type RelayCallIceDiagnostics, type RelayCallInboundDiagnostics, type RelayCallOutboundDiagnostics, type RelayCallRoomDiagnostics, type RelayCallTransportCloseEvent, type RelayCallTransportOptions, type RelayIceServer, type RelayIceTransportPolicy, } from "./transport.js";
-export { createWeriftWebRTCFactory } from "./engine-werift.js";
+export { RelayAudioInput, RelayAudioOutput, LIVEKIT_ROOM_INPUT_AUDIO, RelayLiveKitCall, createRelayLiveKitAudio, type RelayLiveKitAudio, type RelayLiveKitAudioOptions, type RelayLiveKitConnectOptions, } from "./livekit.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,uBAAuB,EACvB,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,KAAK,0BAA0B,GAChC,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,kBAAkB,EAClB,uBAAuB,EACvB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,KAAK,4BAA4B,EACjC,KAAK,wBAAwB,EAC7B,KAAK,4BAA4B,EACjC,KAAK,yBAAyB,EAC9B,KAAK,cAAc,EACnB,KAAK,uBAAuB,GAC7B,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,gBAAgB,EAChB,uBAAuB,EACvB,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,KAAK,0BAA0B,GAChC,MAAM,cAAc,CAAC"}

package/dist/index.js

@@ -1,4 +1,2 @@
-export { RelayAudioInput, RelayAudioOutput, RelayLiveKitCall, createRelayLiveKitAudio, } from "./livekit.js";
-export { RelayCallTransport, RelayCallTransportError, } from "./transport.js";
-export { createWeriftWebRTCFactory } from "./engine-werift.js";
+export { RelayAudioInput, RelayAudioOutput, LIVEKIT_ROOM_INPUT_AUDIO, RelayLiveKitCall, createRelayLiveKitAudio, } from "./livekit.js";
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,gBAAgB,EAChB,uBAAuB,GAIxB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,kBAAkB,EAClB,uBAAuB,GAWxB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,yBAAyB,EAAE,MAAM,oBAAoB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,wBAAwB,EACxB,gBAAgB,EAChB,uBAAuB,GAIxB,MAAM,cAAc,CAAC"}

package/dist/livekit.d.ts

@@ -2,7 +2,18 @@ import { AudioInput, AudioOutput, type AgentSession } from "@livekit/agents";
 import { AudioFrame } from "@livekit/rtc-node";
 import type Relay from "@relaymessenger/sdk";
 import type { CallRoomOptions } from "@relaymessenger/sdk";
-import { RelayCallTransport, type RelayCallIceDiagnostics, type RelayIceServer, type RelayIceTransportPolicy } from "./transport.js";
+import { RelayCallTransport, type RelayCallIceDiagnostics, type RelayIceServer, type RelayIceServersProvider, type RelayInboundAudioFormat, type RelayIceTransportPolicy } from "@relaymessenger/sdk/calls";
+/**
+ * The format LiveKit's own room input hands an AgentSession:
+ * `@livekit/agents` dist/voice/room_io/room_io.js:46-48
+ * `DEFAULT_ROOM_INPUT_OPTIONS = { audioSampleRate: 24e3, audioNumChannels: 1 }`,
+ * passed to the participant `AudioStream` (room_io.js:373-374). Plugins
+ * resample assuming the channel count they construct with
+ * (`@livekit/agents-plugin-google` 1.9.0 realtime_api.js:1307-1329
+ * `new AudioResampler(frame.sampleRate, 16000, 1)`; rtc-node's resampler reads
+ * raw bytes and ignores `frame.channels`), so the session must receive mono.
+ */
+export declare const LIVEKIT_ROOM_INPUT_AUDIO: Readonly<RelayInboundAudioFormat>;
 /** LiveKit Agents input backed by the remote Relay participant's WebRTC audio. */
 export declare class RelayAudioInput extends AudioInput {
     #private;
@@ -10,11 +21,27 @@ export declare class RelayAudioInput extends AudioInput {
     setAttached(attached: boolean): void;
     close(): Promise<void>;
 }
-/** LiveKit Agents output that publishes TTS PCM to the Relay participant. */
+/**
+ * LiveKit Agents output that publishes TTS PCM to the Relay participant.
+ *
+ * Shape copied from `@livekit/agents` `ParticipantAudioOutput`
+ * (dist/voice/room_io/_output.js): `captureFrame` hands the frame to the
+ * transport and returns without waiting for playout, `flush()` starts a playout
+ * task that reports `onPlaybackFinished` only once the transport has drained,
+ * and `clearBuffer()` resolves the interruption future so that task reports
+ * `interrupted: true` with the position actually played.
+ */
 export declare class RelayAudioOutput extends AudioOutput {
     #private;
     constructor(transport: RelayCallTransport, sampleRate?: number);
+    /**
+     * Hands the frame to the transport, which holds it until the person receives
+     * the agent's audio (PROTOCOL.md section 6b) and resolves once it is queued,
+     * so this waits as LiveKit's room audio output waits for the track's
+     * subscription. A frame dropped by `clearBuffer()` meanwhile is not counted.
+     */
     captureFrame(frame: AudioFrame): Promise<void>;
+    /** Mark the segment complete; `onPlaybackFinished` fires once the transport has drained. */
     flush(): void;
     clearBuffer(): void;
     close(): void;
@@ -32,14 +59,25 @@ export interface RelayLiveKitConnectOptions extends RelayLiveKitAudioOptions {
     relay: Relay;
     callId: string;
     room?: CallRoomOptions;
-    /** STUN and TURN servers for the agent's WebRTC peer. Defaults to none. */
-    iceServers?: RelayIceServer[];
+    /**
+     * STUN and TURN servers for the agent's WebRTC peer. Defaults to the servers
+     * the Call room sends (Cloudflare STUN plus TURN credentials Relay mints). A
+     * function is called again before every restart, to mint fresh TURN credentials.
+     */
+    iceServers?: RelayIceServer[] | RelayIceServersProvider;
     /** `"relay"` forces TURN. Defaults to `"all"`. */
     iceTransportPolicy?: RelayIceTransportPolicy;
-    /** Maximum wait for WebRTC media to connect. Defaults to 15 seconds. */
+    /**
+     * How long one SFU session has to connect before the transport restarts onto
+     * a new one. Defaults to 5 seconds. `connect()` has no overall deadline.
+     */
+    sessionConnectTimeoutMs?: number;
+    /** @deprecated Use `sessionConnectTimeoutMs`; now the per-session wait, not a `connect()` deadline. */
     mediaConnectTimeoutMs?: number;
-    /** @deprecated Use `mediaConnectTimeoutMs`. */
+    /** @deprecated Use `sessionConnectTimeoutMs`. */
     connectionTimeoutMs?: number;
+    /** Aborting stops `connect()` and closes the transport; the Call itself is not ended. */
+    signal?: AbortSignal;
 }
 type AgentSessionAudioTarget = Pick<AgentSession, "input" | "output">;
 /**
@@ -56,6 +94,12 @@ export declare class RelayLiveKitCall {
     static connect(options: RelayLiveKitConnectOptions): Promise<RelayLiveKitCall>;
     attach(session: AgentSessionAudioTarget): void;
     detach(): void;
+    /**
+     * Resolves once the person's audio has arrived and the room shows them
+     * connected (the transport's `peerAudio`). Await it before starting the
+     * AgentSession so the greeting is heard.
+     */
+    waitForPeerAudio(timeoutMs: number): Promise<void>;
     /** ICE candidates and state transitions for this call, with a one-line summary for logs. */
     diagnostics(): RelayCallIceDiagnostics;
     setMuted(muted: boolean): void;

package/dist/livekit.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"livekit.d.ts","sourceRoot":"","sources":["../src/livekit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,KAAK,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAC;AAC7C,OAAO,KAAK,EAAY,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EACL,kBAAkB,EAElB,KAAK,uBAAuB,EAE5B,KAAK,cAAc,EACnB,KAAK,uBAAuB,EAE7B,MAAM,gBAAgB,CAAC;AAExB,kFAAkF;AAClF,qBAAa,eAAgB,SAAQ,UAAU;;gBAOjC,SAAS,EAAE,kBAAkB;IAmBhC,WAAW,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI;IAI9B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAOtC;AAED,6EAA6E;AAC7E,qBAAa,gBAAiB,SAAQ,WAAW;;gBAOnC,SAAS,EAAE,kBAAkB,EAAE,UAAU,SAAS;IAK/C,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBpD,KAAK,IAAI,IAAI;IAUb,WAAW,IAAI,IAAI;IAa5B,KAAK,IAAI,IAAI;CAWd;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,eAAe,CAAC;IACvB,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED,MAAM,WAAW,wBAAwB;IACvC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,iFAAiF;AACjF,eAAO,MAAM,uBAAuB,GAClC,WAAW,kBAAkB,EAC7B,UAAS,wBAA6B,KACrC,iBAGD,CAAC;AAEH,MAAM,WAAW,0BAA2B,SAAQ,wBAAwB;IAC1E,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,eAAe,CAAC;IAKvB,2EAA2E;IAC3E,UAAU,CAAC,EAAE,cAAc,EAAE,CAAC;IAC9B,kDAAkD;IAClD,kBAAkB,CAAC,EAAE,uBAAuB,CAAC;IAG7C,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,+CAA+C;IAC/C,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,KAAK,uBAAuB,GAAG,IAAI,CAAC,YAAY,EAAE,OAAO,GAAG,QAAQ,CAAC,CAAC;AAEtE;;;;GAIG;AACH,qBAAa,gBAAgB;;IAC3B,QAAQ,CAAC,SAAS,EAAE,kBAAkB,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAIlC,OAAO;WAMM,OAAO,CAAC,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwBpF,MAAM,CAAC,OAAO,EAAE,uBAAuB,GAAG,IAAI;IAQ9C,MAAM,IAAI,IAAI;IAOd,4FAA4F;IAC5F,WAAW,IAAI,uBAAuB;IAItC,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAI9B,GAAG,IAAI,IAAI;IAIL,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAQ7B"}
+{"version":3,"file":"livekit.d.ts","sourceRoot":"","sources":["../src/livekit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAU,KAAK,YAAY,EAAE,MAAM,iBAAiB,CAAC;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAC;AAC7C,OAAO,KAAK,EAAY,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,EACL,kBAAkB,EAElB,KAAK,uBAAuB,EAE5B,KAAK,cAAc,EACnB,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAE7B,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;GASG;AACH,eAAO,MAAM,wBAAwB,EAAE,QAAQ,CAAC,uBAAuB,CAGrE,CAAC;AAEH,kFAAkF;AAClF,qBAAa,eAAgB,SAAQ,UAAU;;gBASjC,SAAS,EAAE,kBAAkB;IA4BhC,WAAW,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI;IAI9B,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAQtC;AAED;;;;;;;;;GASG;AACH,qBAAa,gBAAiB,SAAQ,WAAW;;gBAanC,SAAS,EAAE,kBAAkB,EAAE,UAAU,SAAS;IAK9D;;;;;OAKG;IACY,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAsB7D,4FAA4F;IACnF,KAAK,IAAI,IAAI;IAWb,WAAW,IAAI,IAAI;IAU5B,KAAK,IAAI,IAAI;CAmBd;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,eAAe,CAAC;IACvB,MAAM,EAAE,gBAAgB,CAAC;CAC1B;AAED,MAAM,WAAW,wBAAwB;IACvC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,iFAAiF;AACjF,eAAO,MAAM,uBAAuB,GAClC,WAAW,kBAAkB,EAC7B,UAAS,wBAA6B,KACrC,iBAGD,CAAC;AAEH,MAAM,WAAW,0BAA2B,SAAQ,wBAAwB;IAC1E,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,eAAe,CAAC;IAKvB;;;;OAIG;IACH,UAAU,CAAC,EAAE,cAAc,EAAE,GAAG,uBAAuB,CAAC;IACxD,kDAAkD;IAClD,kBAAkB,CAAC,EAAE,uBAAuB,CAAC;IAG7C;;;OAGG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uGAAuG;IACvG,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,iDAAiD;IACjD,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,yFAAyF;IACzF,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,KAAK,uBAAuB,GAAG,IAAI,CAAC,YAAY,EAAE,OAAO,GAAG,QAAQ,CAAC,CAAC;AAEtE;;;;GAIG;AACH,qBAAa,gBAAgB;;IAC3B,QAAQ,CAAC,SAAS,EAAE,kBAAkB,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAIlC,OAAO;WAMM,OAAO,CAAC,OAAO,EAAE,0BAA0B,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA4BpF,MAAM,CAAC,OAAO,EAAE,uBAAuB,GAAG,IAAI;IAQ9C,MAAM,IAAI,IAAI;IAOd;;;;OAIG;IACH,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIlD,4FAA4F;IAC5F,WAAW,IAAI,uBAAuB;IAItC,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAI9B,GAAG,IAAI,IAAI;IAIL,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAQ7B"}

package/dist/livekit.js

@@ -1,12 +1,28 @@
-import { AudioInput, AudioOutput } from "@livekit/agents";
+import { AudioInput, AudioOutput, Future } from "@livekit/agents";
 import { AudioFrame } from "@livekit/rtc-node";
 import { TransformStream } from "node:stream/web";
-import { RelayCallTransport, } from "./transport.js";
+import { RelayCallTransport, } from "@relaymessenger/sdk/calls";
+/**
+ * The format LiveKit's own room input hands an AgentSession:
+ * `@livekit/agents` dist/voice/room_io/room_io.js:46-48
+ * `DEFAULT_ROOM_INPUT_OPTIONS = { audioSampleRate: 24e3, audioNumChannels: 1 }`,
+ * passed to the participant `AudioStream` (room_io.js:373-374). Plugins
+ * resample assuming the channel count they construct with
+ * (`@livekit/agents-plugin-google` 1.9.0 realtime_api.js:1307-1329
+ * `new AudioResampler(frame.sampleRate, 16000, 1)`; rtc-node's resampler reads
+ * raw bytes and ignores `frame.channels`), so the session must receive mono.
+ */
+export const LIVEKIT_ROOM_INPUT_AUDIO = Object.freeze({
+    sampleRate: 24_000,
+    channelCount: 1,
+});
 /** LiveKit Agents input backed by the remote Relay participant's WebRTC audio. */
 export class RelayAudioInput extends AudioInput {
     #transport;
     #writer;
     #onAudio;
+    /** Serial write chain: each frame waits for the writer, as `pipeTo` does in LiveKit's input. */
+    #writes = Promise.resolve();
     #attached = false;
     #closed = false;
     constructor(transport) {
@@ -19,7 +35,16 @@ export class RelayAudioInput extends AudioInput {
             if (this.#closed || !this.#attached)
                 return;
             const audio = new AudioFrame(frame.samples, frame.sampleRate, frame.channelCount, frame.samples.length / frame.channelCount);
-            void this.#writer.write(audio).catch(() => undefined);
+            // LiveKit's ParticipantAudioInputStream pipes its AudioStream into the
+            // session with `pipeTo(output.writable)` (room_io/_input.js:161-162),
+            // which awaits the writer before each chunk. The transport pushes and
+            // cannot be paused, so the frames wait here in order instead.
+            this.#writes = this.#writes
+                .then(async () => {
+                await this.#writer.ready;
+                await this.#writer.write(audio);
+            })
+                .catch(() => undefined);
         };
         this.#transport.on("audio", this.#onAudio);
     }
@@ -31,56 +56,91 @@ export class RelayAudioInput extends AudioInput {
             return;
         this.#closed = true;
         this.#transport.off("audio", this.#onAudio);
+        await this.#writes;
         await this.#writer.close().catch(() => undefined);
         await super.close();
     }
 }
-/** LiveKit Agents output that publishes TTS PCM to the Relay participant. */
+/**
+ * LiveKit Agents output that publishes TTS PCM to the Relay participant.
+ *
+ * Shape copied from `@livekit/agents` `ParticipantAudioOutput`
+ * (dist/voice/room_io/_output.js): `captureFrame` hands the frame to the
+ * transport and returns without waiting for playout, `flush()` starts a playout
+ * task that reports `onPlaybackFinished` only once the transport has drained,
+ * and `clearBuffer()` resolves the interruption future so that task reports
+ * `interrupted: true` with the position actually played.
+ */
 export class RelayAudioOutput extends AudioOutput {
     #transport;
-    #segmentDuration = 0;
-    #segmentStarted = false;
-    #segmentStartedAt = 0;
+    /** Seconds pushed to the transport in the open segment. */
+    #pushedDuration = 0;
+    #firstFrameEmitted = false;
+    #flushTask;
+    #flushDone = true;
+    /** Resolved by `clearBuffer()` with the milliseconds still queued at that moment. */
+    #interruptedFuture = new Future();
+    /** Bumped by `clearBuffer()`; a frame the transport dropped across it is not counted (agents-js `interruptionGeneration`). */
+    #interruptionGeneration = 0;
     #closed = false;
     constructor(transport, sampleRate = 48_000) {
         super(sampleRate, undefined, { pause: false });
         this.#transport = transport;
     }
+    /**
+     * Hands the frame to the transport, which holds it until the person receives
+     * the agent's audio (PROTOCOL.md section 6b) and resolves once it is queued,
+     * so this waits as LiveKit's room audio output waits for the track's
+     * subscription. A frame dropped by `clearBuffer()` meanwhile is not counted.
+     */
     async captureFrame(frame) {
         if (this.#closed)
             throw new Error("Relay LiveKit audio output is closed.");
-        await super.captureFrame(frame);
-        if (!this.#segmentStarted) {
-            this.#segmentStarted = true;
-            this.#segmentStartedAt = Date.now();
-            this.onPlaybackStarted(this.#segmentStartedAt);
+        if (this.#flushTask && !this.#flushDone) {
+            this.logger.error("captureFrame called while flush is in progress");
+            await this.#flushTask;
         }
-        this.#segmentDuration += frame.samplesPerChannel / frame.sampleRate;
+        const interruptionGeneration = this.#interruptionGeneration;
+        // Resolves once the slices are queued; the engine's pump paces the wire.
         await this.#transport.writeAudio({
             samples: frame.data,
             sampleRate: frame.sampleRate,
             channelCount: frame.channels,
         });
+        if (interruptionGeneration !== this.#interruptionGeneration)
+            return;
+        await super.captureFrame(frame);
+        if (!this.#firstFrameEmitted) {
+            this.#firstFrameEmitted = true;
+            this.onPlaybackStarted(Date.now());
+        }
+        this.#pushedDuration += frame.samplesPerChannel / frame.sampleRate;
     }
+    /** Mark the segment complete; `onPlaybackFinished` fires once the transport has drained. */
     flush() {
         super.flush();
-        if (!this.#segmentStarted)
+        if (!this.#pushedDuration)
+            return;
+        if (this.#flushTask && !this.#flushDone)
             return;
-        this.onPlaybackFinished({
-            playbackPosition: this.#segmentDuration,
-            interrupted: false,
+        this.#flushDone = false;
+        this.#flushTask = this.#waitForPlayoutTask().finally(() => {
+            this.#flushDone = true;
         });
-        this.#resetSegment();
+        void this.#flushTask.catch(() => undefined);
     }
     clearBuffer() {
+        this.#interruptionGeneration += 1;
+        const queuedMs = this.#transport.queuedAudioMs();
         this.#transport.clearAudio();
-        if (!this.#segmentStarted)
+        if (this.#interruptedFuture.done)
             return;
-        this.onPlaybackFinished({
-            playbackPosition: Math.min(this.#segmentDuration, Math.max(0, (Date.now() - this.#segmentStartedAt) / 1_000)),
-            interrupted: true,
-        });
-        this.#resetSegment();
+        if (this.#pushedDuration === 0 && this.pendingPlayoutSegments === 0)
+            return;
+        if (!this.#flushTask || this.#flushDone)
+            this.flush();
+        if (!this.#interruptedFuture.done)
+            this.#interruptedFuture.resolve(queuedMs);
     }
     close() {
         if (this.#closed)
@@ -88,10 +148,19 @@ export class RelayAudioOutput extends AudioOutput {
         this.#closed = true;
         this.clearBuffer();
     }
-    #resetSegment() {
-        this.#segmentDuration = 0;
-        this.#segmentStarted = false;
-        this.#segmentStartedAt = 0;
+    async #waitForPlayoutTask() {
+        const interruptedFuture = this.#interruptedFuture;
+        await Promise.race([this.#transport.waitForPlayout(), interruptedFuture.await]);
+        const interrupted = interruptedFuture.done;
+        let playbackPosition = this.#pushedDuration;
+        if (interrupted) {
+            playbackPosition = Math.max(0, playbackPosition - interruptedFuture.result / 1_000);
+        }
+        this.#pushedDuration = 0;
+        this.#firstFrameEmitted = false;
+        if (this.#interruptedFuture === interruptedFuture)
+            this.#interruptedFuture = new Future();
+        this.onPlaybackFinished({ playbackPosition, interrupted });
     }
 }
 /** Build LiveKit Agents audio IO around an already-connected Relay transport. */
@@ -119,6 +188,7 @@ export class RelayLiveKitCall {
         const transportOptions = {
             relay: options.relay,
             callId: options.callId,
+            inboundAudio: LIVEKIT_ROOM_INPUT_AUDIO,
             ...(options.room ? { room: options.room } : {}),
             ...(options.roomClient ? { roomClient: options.roomClient } : {}),
             ...(options.webRTC ? { webRTC: options.webRTC } : {}),
@@ -127,6 +197,9 @@ export class RelayLiveKitCall {
             ...(options.iceGatheringTimeoutMs === undefined
                 ? {}
                 : { iceGatheringTimeoutMs: options.iceGatheringTimeoutMs }),
+            ...(options.sessionConnectTimeoutMs === undefined
+                ? {}
+                : { sessionConnectTimeoutMs: options.sessionConnectTimeoutMs }),
             ...(options.mediaConnectTimeoutMs === undefined
                 ? {}
                 : { mediaConnectTimeoutMs: options.mediaConnectTimeoutMs }),
@@ -135,7 +208,7 @@ export class RelayLiveKitCall {
                 : { connectionTimeoutMs: options.connectionTimeoutMs }),
         };
         const transport = new RelayCallTransport(transportOptions);
-        await transport.connect();
+        await transport.connect(options.signal ? { signal: options.signal } : {});
         return new RelayLiveKitCall(transport, createRelayLiveKitAudio(transport, options));
     }
     attach(session) {
@@ -156,6 +229,14 @@ export class RelayLiveKitCall {
             this.#session.output.audio = null;
         this.#session = undefined;
     }
+    /**
+     * Resolves once the person's audio has arrived and the room shows them
+     * connected (the transport's `peerAudio`). Await it before starting the
+     * AgentSession so the greeting is heard.
+     */
+    waitForPeerAudio(timeoutMs) {
+        return this.transport.waitForPeerAudio(timeoutMs);
+    }
     /** ICE candidates and state transitions for this call, with a one-line summary for logs. */
     diagnostics() {
         return this.transport.diagnostics();

package/dist/livekit.js.map

@@ -1 +1 @@
-{"version":3,"file":"livekit.js","sourceRoot":"","sources":["../src/livekit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAqB,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAoC,MAAM,iBAAiB,CAAC;AAGpF,OAAO,EACL,kBAAkB,GAOnB,MAAM,gBAAgB,CAAC;AAExB,kFAAkF;AAClF,MAAM,OAAO,eAAgB,SAAQ,UAAU;IACpC,UAAU,CAAqB;IAC/B,OAAO,CAA0C;IACjD,QAAQ,CAAmC;IACpD,SAAS,GAAG,KAAK,CAAC;IAClB,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAY,SAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,MAAM,OAAO,GAAG,IAAI,eAAe,EAA0B,CAAC;QAC9D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,CAAC,QAAQ,GAAG,CAAC,KAAK,EAAE,EAAE;YACxB,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS;gBAAE,OAAO;YAC5C,MAAM,KAAK,GAAG,IAAI,UAAU,CAC1B,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,YAAY,CAC1C,CAAC;YACF,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACxD,CAAC,CAAC;QACF,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAEQ,WAAW,CAAC,QAAiB;QACpC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAEQ,KAAK,CAAC,KAAK;QAClB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;CACF;AAED,6EAA6E;AAC7E,MAAM,OAAO,gBAAiB,SAAQ,WAAW;IACtC,UAAU,CAAqB;IACxC,gBAAgB,GAAG,CAAC,CAAC;IACrB,eAAe,GAAG,KAAK,CAAC;IACxB,iBAAiB,GAAG,CAAC,CAAC;IACtB,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAY,SAA6B,EAAE,UAAU,GAAG,MAAM;QAC5D,KAAK,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAEQ,KAAK,CAAC,YAAY,CAAC,KAAiB;QAC3C,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3E,MAAM,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAC5B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACpC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,CAAC,gBAAgB,IAAI,KAAK,CAAC,iBAAiB,GAAG,KAAK,CAAC,UAAU,CAAC;QACpE,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,OAAO,EAAE,KAAK,CAAC,IAAI;YACnB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,YAAY,EAAE,KAAK,CAAC,QAAQ;SAC7B,CAAC,CAAC;IACL,CAAC;IAEQ,KAAK;QACZ,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO;QAClC,IAAI,CAAC,kBAAkB,CAAC;YACtB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,WAAW,EAAE,KAAK;SACnB,CAAC,CAAC;QACH,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC;IAEQ,WAAW;QAClB,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO;QAClC,IAAI,CAAC,kBAAkB,CAAC;YACtB,gBAAgB,EAAE,IAAI,CAAC,GAAG,CACxB,IAAI,CAAC,gBAAgB,EACrB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,KAAK,CAAC,CAC3D;YACD,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QACH,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAED,aAAa;QACX,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;IAC7B,CAAC;CACF;AAWD,iFAAiF;AACjF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,SAA6B,EAC7B,UAAoC,EAAE,EACnB,EAAE,CAAC,CAAC;IACvB,KAAK,EAAE,IAAI,eAAe,CAAC,SAAS,CAAC;IACrC,MAAM,EAAE,IAAI,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,gBAAgB,CAAC;CAClE,CAAC,CAAC;AAwBH;;;;GAIG;AACH,MAAM,OAAO,gBAAgB;IAClB,SAAS,CAAqB;IAC9B,KAAK,CAAkB;IACvB,MAAM,CAAmB;IAClC,QAAQ,CAAsC;IAC9C,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAoB,SAA6B,EAAE,KAAwB;QACzE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAmC;QACtD,MAAM,gBAAgB,GAA8B;YAClD,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,GAAG,CAAC,OAAO,CAAC,qBAAqB,KAAK,SAAS;gBAC7C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC;YAC7D,GAAG,CAAC,OAAO,CAAC,qBAAqB,KAAK,SAAS;gBAC7C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC;YAC7D,GAAG,CAAC,OAAO,CAAC,mBAAmB,KAAK,SAAS;gBAC3C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC;SAC1D,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;QAC3D,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;QAC1B,OAAO,IAAI,gBAAgB,CAAC,SAAS,EAAE,uBAAuB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IACtF,CAAC;IAED,MAAM,CAAC,OAAgC;QACrC,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnE,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO;YAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QAC9D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACjC,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,MAAM;QACJ,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC3B,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK;YAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC;QAC/E,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;QAClF,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED,4FAA4F;IAC5F,WAAW;QACT,OAAO,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;IACtC,CAAC;IAED,QAAQ,CAAC,KAAc;QACrB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,GAAG;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACF"}
+{"version":3,"file":"livekit.js","sourceRoot":"","sources":["../src/livekit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAqB,MAAM,iBAAiB,CAAC;AACrF,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAoC,MAAM,iBAAiB,CAAC;AAGpF,OAAO,EACL,kBAAkB,GASnB,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAsC,MAAM,CAAC,MAAM,CAAC;IACvF,UAAU,EAAE,MAAM;IAClB,YAAY,EAAE,CAAC;CAChB,CAAC,CAAC;AAEH,kFAAkF;AAClF,MAAM,OAAO,eAAgB,SAAQ,UAAU;IACpC,UAAU,CAAqB;IAC/B,OAAO,CAA0C;IACjD,QAAQ,CAAmC;IACpD,gGAAgG;IAChG,OAAO,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3C,SAAS,GAAG,KAAK,CAAC;IAClB,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAY,SAA6B;QACvC,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,MAAM,OAAO,GAAG,IAAI,eAAe,EAA0B,CAAC;QAC9D,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QAC5C,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,CAAC,QAAQ,GAAG,CAAC,KAAK,EAAE,EAAE;YACxB,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS;gBAAE,OAAO;YAC5C,MAAM,KAAK,GAAG,IAAI,UAAU,CAC1B,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,YAAY,CAC1C,CAAC;YACF,uEAAuE;YACvE,sEAAsE;YACtE,sEAAsE;YACtE,8DAA8D;YAC9D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO;iBACxB,IAAI,CAAC,KAAK,IAAI,EAAE;gBACf,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;gBACzB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAClC,CAAC,CAAC;iBACD,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC,CAAC;QACF,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC7C,CAAC;IAEQ,WAAW,CAAC,QAAiB;QACpC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAEQ,KAAK,CAAC,KAAK;QAClB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,OAAO,CAAC;QACnB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAClD,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;CACF;AAED;;;;;;;;;GASG;AACH,MAAM,OAAO,gBAAiB,SAAQ,WAAW;IACtC,UAAU,CAAqB;IACxC,2DAA2D;IAC3D,eAAe,GAAG,CAAC,CAAC;IACpB,kBAAkB,GAAG,KAAK,CAAC;IAC3B,UAAU,CAA4B;IACtC,UAAU,GAAG,IAAI,CAAC;IAClB,qFAAqF;IACrF,kBAAkB,GAAG,IAAI,MAAM,EAAU,CAAC;IAC1C,8HAA8H;IAC9H,uBAAuB,GAAG,CAAC,CAAC;IAC5B,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAY,SAA6B,EAAE,UAAU,GAAG,MAAM;QAC5D,KAAK,CAAC,UAAU,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED;;;;;OAKG;IACM,KAAK,CAAC,YAAY,CAAC,KAAiB;QAC3C,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3E,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACxC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACpE,MAAM,IAAI,CAAC,UAAU,CAAC;QACxB,CAAC;QACD,MAAM,sBAAsB,GAAG,IAAI,CAAC,uBAAuB,CAAC;QAC5D,yEAAyE;QACzE,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;YAC/B,OAAO,EAAE,KAAK,CAAC,IAAI;YACnB,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,YAAY,EAAE,KAAK,CAAC,QAAQ;SAC7B,CAAC,CAAC;QACH,IAAI,sBAAsB,KAAK,IAAI,CAAC,uBAAuB;YAAE,OAAO;QACpE,MAAM,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC7B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;YAC/B,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,CAAC,eAAe,IAAI,KAAK,CAAC,iBAAiB,GAAG,KAAK,CAAC,UAAU,CAAC;IACrE,CAAC;IAED,4FAA4F;IACnF,KAAK;QACZ,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO;QAClC,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAChD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE;YACxD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC,CAAC,CAAC;QACH,KAAK,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;IAEQ,WAAW;QAClB,IAAI,CAAC,uBAAuB,IAAI,CAAC,CAAC;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC;QACjD,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;QAC7B,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI;YAAE,OAAO;QACzC,IAAI,IAAI,CAAC,eAAe,KAAK,CAAC,IAAI,IAAI,CAAC,sBAAsB,KAAK,CAAC;YAAE,OAAO;QAC5E,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,UAAU;YAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QACtD,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI;YAAE,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/E,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,iBAAiB,GAAG,IAAI,CAAC,kBAAkB,CAAC;QAClD,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,EAAE,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;QAChF,MAAM,WAAW,GAAG,iBAAiB,CAAC,IAAI,CAAC;QAC3C,IAAI,gBAAgB,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5C,IAAI,WAAW,EAAE,CAAC;YAChB,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,gBAAgB,GAAG,iBAAiB,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC;QAChC,IAAI,IAAI,CAAC,kBAAkB,KAAK,iBAAiB;YAAE,IAAI,CAAC,kBAAkB,GAAG,IAAI,MAAM,EAAU,CAAC;QAClG,IAAI,CAAC,kBAAkB,CAAC,EAAE,gBAAgB,EAAE,WAAW,EAAE,CAAC,CAAC;IAC7D,CAAC;CACF;AAWD,iFAAiF;AACjF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,SAA6B,EAC7B,UAAoC,EAAE,EACnB,EAAE,CAAC,CAAC;IACvB,KAAK,EAAE,IAAI,eAAe,CAAC,SAAS,CAAC;IACrC,MAAM,EAAE,IAAI,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,gBAAgB,CAAC;CAClE,CAAC,CAAC;AAmCH;;;;GAIG;AACH,MAAM,OAAO,gBAAgB;IAClB,SAAS,CAAqB;IAC9B,KAAK,CAAkB;IACvB,MAAM,CAAmB;IAClC,QAAQ,CAAsC;IAC9C,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAoB,SAA6B,EAAE,KAAwB;QACzE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAmC;QACtD,MAAM,gBAAgB,GAA8B;YAClD,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,YAAY,EAAE,wBAAwB;YACtC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/C,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,GAAG,CAAC,OAAO,CAAC,qBAAqB,KAAK,SAAS;gBAC7C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC;YAC7D,GAAG,CAAC,OAAO,CAAC,uBAAuB,KAAK,SAAS;gBAC/C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,uBAAuB,EAAE,OAAO,CAAC,uBAAuB,EAAE,CAAC;YACjE,GAAG,CAAC,OAAO,CAAC,qBAAqB,KAAK,SAAS;gBAC7C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,qBAAqB,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC;YAC7D,GAAG,CAAC,OAAO,CAAC,mBAAmB,KAAK,SAAS;gBAC3C,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,mBAAmB,EAAE,OAAO,CAAC,mBAAmB,EAAE,CAAC;SAC1D,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;QAC3D,MAAM,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1E,OAAO,IAAI,gBAAgB,CAAC,SAAS,EAAE,uBAAuB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IACtF,CAAC;IAED,MAAM,CAAC,OAAgC;QACrC,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnE,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO;YAAE,IAAI,CAAC,MAAM,EAAE,CAAC;QAC9D,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACjC,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;IACrC,CAAC;IAED,MAAM;QACJ,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC3B,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK;YAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC;QAC/E,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;QAClF,IAAI,CAAC,QAAQ,GAAG,SAAS,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACH,gBAAgB,CAAC,SAAiB;QAChC,OAAO,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;IAED,4FAA4F;IAC5F,WAAW;QACT,OAAO,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;IACtC,CAAC;IAED,QAAQ,CAAC,KAAc;QACrB,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;IAED,GAAG;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;CACF"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@relaymessenger/livekit",
-  "version": "0.1.0-staging.2",
-  "description": "Relay WebRTC audio transport and LiveKit Agents integration for Node.js.",
+  "version": "0.1.0-staging.21",
+  "description": "LiveKit Agents audio input and output for Relay Calls, over @relaymessenger/sdk/calls.",
   "type": "module",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -12,11 +12,6 @@
       "import": "./dist/index.js",
       "default": "./dist/index.js"
     },
-    "./transport": {
-      "types": "./dist/transport.d.ts",
-      "import": "./dist/transport.js",
-      "default": "./dist/transport.js"
-    },
     "./package.json": "./package.json"
   },
   "files": [
@@ -38,11 +33,13 @@
   "license": "MIT",
   "dependencies": {
     "@evan/opus": "1.0.3",
-    "@relaymessenger/sdk": "0.3.6-staging.14",
+    "@relaymessenger/sdk": "0.3.6-staging.28",
+    "rtp-packet": "0.4.2",
     "werift": "0.24.4"
   },
   "optionalDependencies": {
-    "@roamhq/wrtc": "^0.10.0"
+    "@roamhq/wrtc": "^0.10.0",
+    "node-webcodecs": "1.3.0"
   },
   "peerDependencies": {
     "@livekit/agents": "^1.9.0",
@@ -72,6 +69,7 @@
     "livekit",
     "webrtc",
     "voice",
+    "video",
     "agents"
   ]
 }

package/dist/engine-werift.d.ts

@@ -1,19 +0,0 @@
-import { RTCPeerConnection } from "werift";
-import type { RelayAudioSourceLike, RelayPeerConnectionConfig, RelayWebRTCFactory } from "./transport.js";
-/** Opus on the wire is always 48 kHz; Relay sends stereo so callers may pass either channel count. */
-export declare const WERIFT_SAMPLE_RATE = 48000;
-export declare const WERIFT_CHANNEL_COUNT = 2;
-/** One RTP packet carries 20 ms: 960 frames at 48 kHz, timestamp advances by 960. */
-export declare const WERIFT_PACKET_MS = 20;
-export type WeriftAudioSourceData = Parameters<RelayAudioSourceLike["onData"]>[0];
-/** Packets counted for `diagnostics()`; the recent window is the last 5 s. */
-export declare const STATS_WINDOW_MS = 5000;
-/** Convert any PCM16 frame to interleaved 48 kHz stereo by linear interpolation. */
-export declare const toWireFormat: (data: {
-    samples: Int16Array;
-    sampleRate: number;
-    channelCount: number;
-}) => Int16Array;
-export declare const createWeriftPeerConnection: (config?: RelayPeerConnectionConfig) => RTCPeerConnection;
-export declare const createWeriftWebRTCFactory: () => RelayWebRTCFactory;
-//# sourceMappingURL=engine-werift.d.ts.map

package/dist/engine-werift.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"engine-werift.d.ts","sourceRoot":"","sources":["../src/engine-werift.ts"],"names":[],"mappings":"AAiCA,OAAO,EAGL,iBAAiB,EAIlB,MAAM,QAAQ,CAAC;AAChB,OAAO,KAAK,EAGV,oBAAoB,EAGpB,yBAAyB,EAEzB,kBAAkB,EACnB,MAAM,gBAAgB,CAAC;AAExB,sGAAsG;AACtG,eAAO,MAAM,kBAAkB,QAAS,CAAC;AACzC,eAAO,MAAM,oBAAoB,IAAI,CAAC;AACtC,qFAAqF;AACrF,eAAO,MAAM,gBAAgB,KAAK,CAAC;AAMnC,MAAM,MAAM,qBAAqB,GAAG,UAAU,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAElF,8EAA8E;AAC9E,eAAO,MAAM,eAAe,OAAQ,CAAC;AA8BrC,oFAAoF;AACpF,eAAO,MAAM,YAAY,GAAI,MAAM;IACjC,OAAO,EAAE,UAAU,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,KAAG,UAwBH,CAAC;AAuJF,eAAO,MAAM,0BAA0B,GACrC,SAAQ,yBAAyE,KAChF,iBAcC,CAAC;AAEL,eAAO,MAAM,yBAAyB,QAAO,kBAI3C,CAAC"}