@memberjunction/ai-bridge-livekit 0.0.1 → 5.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +128 -43
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/livekit-bridge.d.ts +164 -0
- package/dist/livekit-bridge.d.ts.map +1 -0
- package/dist/livekit-bridge.js +353 -0
- package/dist/livekit-bridge.js.map +1 -0
- package/dist/livekit-meeting-controls.d.ts +88 -0
- package/dist/livekit-meeting-controls.d.ts.map +1 -0
- package/dist/livekit-meeting-controls.js +143 -0
- package/dist/livekit-meeting-controls.js.map +1 -0
- package/dist/livekit-sdk.d.ts +182 -0
- package/dist/livekit-sdk.d.ts.map +1 -0
- package/dist/livekit-sdk.js +45 -0
- package/dist/livekit-sdk.js.map +1 -0
- package/package.json +32 -7
package/README.md
CHANGED
|
@@ -1,45 +1,130 @@
|
|
|
1
1
|
# @memberjunction/ai-bridge-livekit
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
3
|
+
The **LiveKit** Realtime Bridge driver — the **MJ-native multi-party room**. Unlike every other bridge
|
|
4
|
+
in MemberJunction's Realtime Bridges program (which connect **out** to a 3rd-party meeting platform like
|
|
5
|
+
Zoom/Teams/Meet or a telephony carrier), LiveKit is a **self-hosted WebRTC SFU that MJ runs itself**. The
|
|
6
|
+
architecture treats a Zoom meeting and an MJ-native LiveKit room **identically** — both are multi-party
|
|
7
|
+
media transports — so this is "*another bridge, not a special build.*"
|
|
8
|
+
|
|
9
|
+
It connects the one realtime agent engine to a **LiveKit room** as a bot participant: bidirectional
|
|
10
|
+
audio (and full video/screen), a per-participant diarized roster, room-admin mute, data-channel chat, and
|
|
11
|
+
a **Meeting Controls** facilitator channel — all behind an injectable LiveKit room SDK seam so the driver
|
|
12
|
+
builds and unit-tests with **no network and no real LiveKit SDK**.
|
|
13
|
+
|
|
14
|
+
See the [Realtime Bridges Guide](../../../../guides/REALTIME_BRIDGES_GUIDE.md) and
|
|
15
|
+
[`/plans/realtime/realtime-bridges-architecture.md`](../../../../plans/realtime/realtime-bridges-architecture.md)
|
|
16
|
+
(§4c multi-party / MJ-native room, §3 provider abstraction, §4b channels) for the full architecture.
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
npm install @memberjunction/ai-bridge-livekit
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Self-hosted vs. connecting out
|
|
25
|
+
|
|
26
|
+
| | The other bridges (Zoom, Teams, Meet, Webex, Slack, Discord, Twilio…) | **LiveKit (this package)** |
|
|
27
|
+
|---|---|---|
|
|
28
|
+
| Who owns the room | a 3rd-party platform | **MJ — self-hosted SFU** |
|
|
29
|
+
| How you join | a join URL / number you were handed | a room URL + a **token MJ mints** |
|
|
30
|
+
| Bot admission | per-platform review/quirks | none — MJ controls the room |
|
|
31
|
+
| Use case | meet the customer where they already are | an MJ-native multi-party experience (e.g. embedded in Explorer) |
|
|
32
|
+
|
|
33
|
+
The same multi-party machinery (§4c) works either way: put **1+ agents into a shared room** and the room
|
|
34
|
+
itself is the shared media plane.
|
|
35
|
+
|
|
36
|
+
## What it provides
|
|
37
|
+
|
|
38
|
+
- **`LiveKitBridge`** — `@RegisterClass(BaseRealtimeBridge, 'LiveKitBridge')`. The `MJ: AI Bridge
|
|
39
|
+
Providers` row with `DriverClass = 'LiveKitBridge'` resolves to this driver via the `ClassFactory`.
|
|
40
|
+
Implements the four `BaseRealtimeBridge` abstracts (`Connect` / `Disconnect` / `SendMedia` / `OnMedia`)
|
|
41
|
+
and the capability-gated virtuals LiveKit supports (`GetParticipants`, `OnParticipantChange`), plus
|
|
42
|
+
`GetMeetingControlsEventSource` for the facilitator channel and a `SendDataMessage` helper.
|
|
43
|
+
- **`ILiveKitRoomSdk`** — the **injectable seam** the driver depends on instead of the real SDK.
|
|
44
|
+
- **`LiveKitMeetingControlsEventSource`** — adapts the seam's roster / speaking / mute into the bridge's
|
|
45
|
+
`IBridgeMeetingControlsEventSource`, so the engine wires the Meeting Controls channel.
|
|
46
|
+
|
|
47
|
+
## Capability coverage (the LiveKit seed row)
|
|
48
|
+
|
|
49
|
+
| Capability | Status |
|
|
50
|
+
|---|---|
|
|
51
|
+
| On-demand join | ✅ |
|
|
52
|
+
| Audio in / out | ✅ |
|
|
53
|
+
| Video in / out | ✅ (LiveKit does full A/V) |
|
|
54
|
+
| Screen in / out | ✅ (full screen share) |
|
|
55
|
+
| Speaker diarization (per-participant tracks → labels) | ✅ — **native**, the SFU delivers tracks per participant |
|
|
56
|
+
| Room-admin mute (Meeting Controls) | ✅ |
|
|
57
|
+
| Data-channel chat | ✅ |
|
|
58
|
+
| Scheduled / invite / native-invite join, DTMF / transfer / recording | ➖ not LiveKit-room features — the gated base methods throw `BridgeCapabilityNotSupportedError` |
|
|
59
|
+
|
|
60
|
+
Capability gating is two-layer (defense-in-depth): the engine checks the provider's `SupportedFeatures`
|
|
61
|
+
first, and the driver re-asserts each flag with `RequireFeature` at the top of its overrides.
|
|
62
|
+
|
|
63
|
+
## The LiveKit room SDK seam (`ILiveKitRoomSdk`)
|
|
64
|
+
|
|
65
|
+
The driver never imports the real LiveKit SDK. It depends only on this minimal interface:
|
|
66
|
+
|
|
67
|
+
```typescript
|
|
68
|
+
export interface ILiveKitRoomSdk {
|
|
69
|
+
connect(args: LiveKitConnectArgs): Promise<LiveKitConnectResult>; // roomUrl + signed token → join as bot
|
|
70
|
+
disconnect(): Promise<void>;
|
|
71
|
+
publishAudioFrame(pcm: ArrayBuffer): void; // agent's voice out
|
|
72
|
+
onAudioTrack(cb: (frame: LiveKitAudioFrame) => void): void; // per-participant audio in (diarization)
|
|
73
|
+
publishVideoFrame(frame: ArrayBuffer): void; // full video out
|
|
74
|
+
publishScreenFrame(frame: ArrayBuffer): void; // full screen share out
|
|
75
|
+
onParticipantJoin(cb: (p: LiveKitParticipant) => void): void;
|
|
76
|
+
onParticipantLeave(cb: (id: string) => void): void;
|
|
77
|
+
getParticipants(): Promise<LiveKitParticipant[]>;
|
|
78
|
+
sendDataMessage(text: string): Promise<void>; // data-channel chat
|
|
79
|
+
onDisconnected(cb: () => void): void;
|
|
80
|
+
}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
**Production binding (TODO at deployment):** bind this to `livekit-server-sdk` (token minting / room
|
|
84
|
+
admin) plus a room client (the Node WebRTC participant, e.g. `@livekit/rtc-node`). The adapter is thin
|
|
85
|
+
and the driver/tests do not change. None of the SDK types leak into this package. Until a real factory is
|
|
86
|
+
bound via `LiveKitBridge.SetSdkFactory(...)`, `Connect` throws an explicit "bind the real LiveKit SDK"
|
|
87
|
+
error.
|
|
88
|
+
|
|
89
|
+
## Echo / self-audio
|
|
90
|
+
|
|
91
|
+
A LiveKit SFU **never delivers a participant its own published track back** — the bot does not hear its
|
|
92
|
+
own voice, so no echo gate is needed. This is exactly the property the multi-party model relies on
|
|
93
|
+
(§4c): each agent in a room hears the *others'* mix natively, never itself, so two agents can converse
|
|
94
|
+
without a transcript-relay hack.
|
|
95
|
+
|
|
96
|
+
## Usage
|
|
97
|
+
|
|
98
|
+
```typescript
|
|
99
|
+
import { LiveKitBridge } from '@memberjunction/ai-bridge-livekit';
|
|
100
|
+
import { AIBridgeEngine } from '@memberjunction/ai-bridge-server';
|
|
101
|
+
|
|
102
|
+
// The engine resolves the driver by DriverClass via the ClassFactory; you do not new it up directly.
|
|
103
|
+
// In production, bind the real SDK factory once at boot:
|
|
104
|
+
// (resolved per provider config) — LiveKitBridge instances call SetSdkFactory with a real adapter.
|
|
105
|
+
|
|
106
|
+
const active = await AIBridgeEngine.Instance.StartBridgeSession({
|
|
107
|
+
AgentSessionID: sessionId,
|
|
108
|
+
Provider: liveKitProvider, // MJ: AI Bridge Providers row, DriverClass='LiveKitBridge'
|
|
109
|
+
RealtimeSession: realtimeSession, // injected IRealtimeSession
|
|
110
|
+
Address: 'wss://livekit.myorg.com', // the MJ-native room server
|
|
111
|
+
Configuration: { AccessToken: signedToken, BotDisplayName: 'Sage' },
|
|
112
|
+
MetadataProvider: provider,
|
|
113
|
+
ContextUser: user,
|
|
114
|
+
});
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
For multiple agents in one LiveKit room, register each session with the engine's room coordinator — see
|
|
118
|
+
the [Multi-party section of the guide](../../../../guides/REALTIME_BRIDGES_GUIDE.md) and
|
|
119
|
+
`MultiAgentRoomCoordinator` in `@memberjunction/ai-bridge-server`.
|
|
120
|
+
|
|
121
|
+
## Testing
|
|
122
|
+
|
|
123
|
+
`FakeLiveKitRoomSdk` (in the test file) implements `ILiveKitRoomSdk` in memory with drive helpers and
|
|
124
|
+
capture sinks — connect/disconnect, audio in→`OnMedia` (speaker labels) + out→track, video/screen out,
|
|
125
|
+
participant join/leave→roster, speaking attribution, data-channel chat, and capability gating. **24
|
|
126
|
+
tests, no network.** Run with `npm test`.
|
|
127
|
+
|
|
128
|
+
## License
|
|
129
|
+
|
|
130
|
+
ISC
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,4BAA4B,CAAC;AAC3C,cAAc,kBAAkB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export * from './livekit-sdk.js';
|
|
2
|
+
export * from './livekit-meeting-controls.js';
|
|
3
|
+
export * from './livekit-bridge.js';
|
|
4
|
+
import { LoadLiveKitBridge } from './livekit-bridge.js';
|
|
5
|
+
// Static reference so bundlers cannot tree-shake the @RegisterClass(BaseRealtimeBridge, 'LiveKitBridge')
|
|
6
|
+
// registration. Calling the no-op here keeps the driver resolvable by the engine's ClassFactory.
|
|
7
|
+
LoadLiveKitBridge();
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAC;AAC9B,cAAc,4BAA4B,CAAC;AAC3C,cAAc,kBAAkB,CAAC;AAEjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,yGAAyG;AACzG,iGAAiG;AACjG,iBAAiB,EAAE,CAAC"}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview `LiveKitBridge` — the **MJ-native multi-party room** Realtime Bridge driver. Unlike
|
|
3
|
+
* every other bridge (which connects OUT to a 3rd-party meeting/telephony platform), LiveKit is a
|
|
4
|
+
* **self-hosted WebRTC SFU that MJ runs itself** — the "MJ-native room"
|
|
5
|
+
* (`/plans/realtime/realtime-bridges-architecture.md` §4c). The architecture treats a Zoom meeting and
|
|
6
|
+
* an MJ-native LiveKit room **identically** — both are multi-party media transports — so this is
|
|
7
|
+
* "*another bridge, not a special build*."
|
|
8
|
+
*
|
|
9
|
+
* Implements the {@link BaseRealtimeBridge} contract against an injectable {@link ILiveKitRoomSdk} seam
|
|
10
|
+
* so it builds + unit-tests with NO network and NO real LiveKit SDK (the Zoom `SetSdkFactory`
|
|
11
|
+
* testability pattern).
|
|
12
|
+
*
|
|
13
|
+
* LiveKit capability coverage (per the §4c / §8 seed row): on-demand join, **full** audio/video/screen
|
|
14
|
+
* in+out (LiveKit does the lot), and per-participant diarization (`SpeakerDiarization`) — the SFU
|
|
15
|
+
* delivers tracks per participant, so speaker labels come free. No scheduled/invite/telephony features.
|
|
16
|
+
* Those virtual base methods keep throwing `BridgeCapabilityNotSupportedError`.
|
|
17
|
+
*
|
|
18
|
+
* ## Echo / self-audio
|
|
19
|
+
* A LiveKit SFU **never delivers a participant its own published track back**, so the bot does not hear
|
|
20
|
+
* its own voice — no echo gate is required (see {@link wireInboundAudio}). This is exactly the property
|
|
21
|
+
* §4c relies on for putting **multiple agents in one room**: each agent hears the *others'* mixed audio
|
|
22
|
+
* natively, not its own, so two agents converse without a transcript-relay hack.
|
|
23
|
+
*
|
|
24
|
+
* @module @memberjunction/ai-bridge-livekit
|
|
25
|
+
* @author MemberJunction.com
|
|
26
|
+
*/
|
|
27
|
+
import { BaseRealtimeBridge, BridgeConnectResult, BridgeDisconnectReason, BridgeMediaFrame, BridgeMediaTrackKind, BridgeParticipantInfo, RealtimeBridgeContext, IBridgeMeetingControlsEventSource } from '@memberjunction/ai-bridge-base';
|
|
28
|
+
import { LiveKitRoomSdkFactory } from './livekit-sdk.js';
|
|
29
|
+
/**
|
|
30
|
+
* The `DriverClass` key {@link LiveKitBridge} registers under. A `MJ: AI Bridge Providers` row with
|
|
31
|
+
* `DriverClass = 'LiveKitBridge'` resolves to this driver via the `ClassFactory`.
|
|
32
|
+
*/
|
|
33
|
+
export declare const LIVEKIT_BRIDGE_DRIVER_CLASS = "LiveKitBridge";
|
|
34
|
+
/**
|
|
35
|
+
* Realtime Bridge driver for the **MJ-native LiveKit room**.
|
|
36
|
+
*
|
|
37
|
+
* Construct the driver with the default constructor (the engine's `ClassFactory` path) — it lazily
|
|
38
|
+
* builds the LiveKit room SDK from the {@link sdkFactory} at {@link Connect} time. Tests inject a
|
|
39
|
+
* `FakeLiveKitRoomSdk` by overriding the factory via {@link SetSdkFactory} (the creation seam) before
|
|
40
|
+
* connecting.
|
|
41
|
+
*
|
|
42
|
+
* Registered via `@RegisterClass(BaseRealtimeBridge, 'LiveKitBridge')`.
|
|
43
|
+
*/
|
|
44
|
+
export declare class LiveKitBridge extends BaseRealtimeBridge {
|
|
45
|
+
/** The live LiveKit SDK seam for this session, created at {@link Connect}. */
|
|
46
|
+
private sdk;
|
|
47
|
+
/** The bot's own participant identity in the joined room (set at {@link Connect}). */
|
|
48
|
+
private botIdentity;
|
|
49
|
+
/** The inbound-media handler registered via {@link OnMedia}; raw audio frames are forwarded to it. */
|
|
50
|
+
private mediaHandler?;
|
|
51
|
+
/** The roster-change handler registered via {@link OnParticipantChange}. */
|
|
52
|
+
private participantHandler?;
|
|
53
|
+
/** The Meeting Controls event source for this session (only when diarization is supported). */
|
|
54
|
+
private meetingControls;
|
|
55
|
+
/**
|
|
56
|
+
* The SDK creation seam. Defaults to a factory that throws an explicit "bind the real LiveKit SDK"
|
|
57
|
+
* error, since this package ships WITHOUT the real SDK adapter (a deployment concern). Production
|
|
58
|
+
* sets a real factory via {@link SetSdkFactory}; tests inject a `FakeLiveKitRoomSdk`.
|
|
59
|
+
*/
|
|
60
|
+
private sdkFactory;
|
|
61
|
+
/**
|
|
62
|
+
* Sets the {@link LiveKitRoomSdkFactory} this driver uses to construct its SDK seam at connect — the
|
|
63
|
+
* creation seam (mirroring Zoom's `SetSdkFactory`). Production binds the real LiveKit room SDK
|
|
64
|
+
* adapter here; tests inject a `FakeLiveKitRoomSdk`.
|
|
65
|
+
*
|
|
66
|
+
* @param factory The factory that builds the {@link ILiveKitRoomSdk} for a session.
|
|
67
|
+
*/
|
|
68
|
+
SetSdkFactory(factory: LiveKitRoomSdkFactory): void;
|
|
69
|
+
/**
|
|
70
|
+
* Connects to the MJ-native LiveKit room and brings the bot online. Captures the capability context,
|
|
71
|
+
* builds the SDK from the {@link sdkFactory}, wires the inbound per-participant audio path and
|
|
72
|
+
* room-disconnected callback, connects, and (when the provider diarizes) constructs + seeds the
|
|
73
|
+
* Meeting Controls event source.
|
|
74
|
+
*
|
|
75
|
+
* @param ctx The bridge context (features, provider name, the room URL as `Address`, config carrying
|
|
76
|
+
* the signed access token).
|
|
77
|
+
* @returns The bot identity + room (external connection) identifiers, persisted by the engine.
|
|
78
|
+
*/
|
|
79
|
+
Connect(ctx: RealtimeBridgeContext): Promise<BridgeConnectResult>;
|
|
80
|
+
/**
|
|
81
|
+
* Disconnects from the LiveKit room and releases all SDK resources. Tolerant of teardown errors.
|
|
82
|
+
*
|
|
83
|
+
* @param _reason Why the disconnect happened (LiveKit teardown is uniform; the bot simply leaves).
|
|
84
|
+
*/
|
|
85
|
+
Disconnect(_reason: BridgeDisconnectReason): Promise<void>;
|
|
86
|
+
/**
|
|
87
|
+
* Sends an outbound media frame into the room. LiveKit does the FULL media set, so audio, video, and
|
|
88
|
+
* screen are all published on their respective tracks (gated by the directional capability flags —
|
|
89
|
+
* the realtime models light audio first, video/screen ride the same path once a model emits them).
|
|
90
|
+
*
|
|
91
|
+
* @param track The outbound track the frame targets.
|
|
92
|
+
* @param frame The media frame to send.
|
|
93
|
+
*/
|
|
94
|
+
SendMedia(track: BridgeMediaTrackKind, frame: BridgeMediaFrame): void;
|
|
95
|
+
/**
|
|
96
|
+
* Registers the inbound-media handler. The driver forwards each raw per-participant audio frame
|
|
97
|
+
* (with its speaker identity) to this handler; the engine routes it to `IRealtimeSession.SendInput`.
|
|
98
|
+
*
|
|
99
|
+
* @param handler Invoked with each inbound media frame.
|
|
100
|
+
*/
|
|
101
|
+
OnMedia(handler: (frame: BridgeMediaFrame) => void): void;
|
|
102
|
+
/**
|
|
103
|
+
* Returns the current LiveKit participant roster (gated by `SpeakerDiarization`). Re-asserts the
|
|
104
|
+
* flag (defense-in-depth) so even an engine-bypassing caller cannot pull a roster a disabled
|
|
105
|
+
* provider forbids.
|
|
106
|
+
*
|
|
107
|
+
* @returns The current participants.
|
|
108
|
+
* @throws {BridgeCapabilityNotSupportedError} when diarization is not enabled for the provider.
|
|
109
|
+
*/
|
|
110
|
+
GetParticipants(): Promise<BridgeParticipantInfo[]>;
|
|
111
|
+
/**
|
|
112
|
+
* Registers a roster-change handler (gated by `SpeakerDiarization`). The driver fires it from the
|
|
113
|
+
* SDK's participant join/leave stream with the full current roster.
|
|
114
|
+
*
|
|
115
|
+
* @param handler Invoked with the updated participant list on each change.
|
|
116
|
+
* @throws {BridgeCapabilityNotSupportedError} when diarization is not enabled.
|
|
117
|
+
*/
|
|
118
|
+
OnParticipantChange(handler: (participants: BridgeParticipantInfo[]) => void): void;
|
|
119
|
+
/**
|
|
120
|
+
* Returns the LiveKit Meeting Controls event source for this session (roster · speaking · mute), or
|
|
121
|
+
* `null` when diarization is off (no roster to facilitate). The engine wires the Meeting Controls
|
|
122
|
+
* channel from this.
|
|
123
|
+
*/
|
|
124
|
+
GetMeetingControlsEventSource(): IBridgeMeetingControlsEventSource | null;
|
|
125
|
+
/**
|
|
126
|
+
* Sends a message on the LiveKit data channel — the room-native "chat". Exposed for the channel
|
|
127
|
+
* plane / turn-taking hybrid mode (the social-cost-free "raise hand"). Best-effort — a failure is
|
|
128
|
+
* logged, never fatal.
|
|
129
|
+
*
|
|
130
|
+
* @param text The data/chat message to send.
|
|
131
|
+
*/
|
|
132
|
+
SendDataMessage(text: string): Promise<void>;
|
|
133
|
+
/** The bot's participant identity in the joined room (or `null` before {@link Connect}). */
|
|
134
|
+
get BotIdentity(): string | null;
|
|
135
|
+
/**
|
|
136
|
+
* Wires the SDK's inbound per-participant audio callback to a diarized inbound
|
|
137
|
+
* {@link BridgeMediaFrame}. The SFU never echoes the bot's own audio, so no self-audio gate is
|
|
138
|
+
* needed; the frame's `SpeakerLabel` is the source participant identity.
|
|
139
|
+
*/
|
|
140
|
+
private wireInboundAudio;
|
|
141
|
+
/**
|
|
142
|
+
* Wires the SDK's participant join/leave streams ONCE (the driver is the single owner; the SDK seam
|
|
143
|
+
* is latest-handler-wins). Each roster change fans out to both the driver's own roster-change
|
|
144
|
+
* handler and the Meeting Controls source.
|
|
145
|
+
*/
|
|
146
|
+
private wireRoster;
|
|
147
|
+
/** Pulls the current roster from the SDK and fans it out to both the participant handler + Meeting Controls. */
|
|
148
|
+
private refreshRoster;
|
|
149
|
+
/** Handles the SDK's room-disconnected signal: surface an empty roster so the engine sees everyone gone. */
|
|
150
|
+
private handleRoomDisconnected;
|
|
151
|
+
/** Builds the SDK connect args from the bridge context (Address is the room URL; token from config). */
|
|
152
|
+
private buildConnectArgs;
|
|
153
|
+
/** Returns the bytes of an outbound media frame, preferring the binary payload. */
|
|
154
|
+
private framePcm;
|
|
155
|
+
/** Decodes a base64 payload to an `ArrayBuffer` (Node `Buffer` fast path, `atob` fallback). */
|
|
156
|
+
private decodeBase64;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Tree-shaking-prevention loader. Modern bundlers cannot see the `@RegisterClass` dynamic registration
|
|
160
|
+
* of {@link LiveKitBridge} and may eliminate it. Import and call this no-op from a static code path
|
|
161
|
+
* (the package entry point does) so the `ClassFactory` can resolve `'LiveKitBridge'`.
|
|
162
|
+
*/
|
|
163
|
+
export declare function LoadLiveKitBridge(): void;
|
|
164
|
+
//# sourceMappingURL=livekit-bridge.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"livekit-bridge.d.ts","sourceRoot":"","sources":["../src/livekit-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAIH,OAAO,EACH,kBAAkB,EAClB,mBAAmB,EACnB,sBAAsB,EACtB,gBAAgB,EAChB,oBAAoB,EACpB,qBAAqB,EAErB,qBAAqB,EACrB,iCAAiC,EACpC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAEH,qBAAqB,EAKxB,MAAM,eAAe,CAAC;AAGvB;;;GAGG;AACH,eAAO,MAAM,2BAA2B,kBAAkB,CAAC;AA8B3D;;;;;;;;;GASG;AACH,qBACa,aAAc,SAAQ,kBAAkB;IACjD,8EAA8E;IAC9E,OAAO,CAAC,GAAG,CAAgC;IAE3C,sFAAsF;IACtF,OAAO,CAAC,WAAW,CAAuB;IAE1C,sGAAsG;IACtG,OAAO,CAAC,YAAY,CAAC,CAAoC;IAEzD,4EAA4E;IAC5E,OAAO,CAAC,kBAAkB,CAAC,CAAkD;IAE7E,+FAA+F;IAC/F,OAAO,CAAC,eAAe,CAAkD;IAEzE;;;;OAIG;IACH,OAAO,CAAC,UAAU,CAKhB;IAEF;;;;;;OAMG;IACI,aAAa,CAAC,OAAO,EAAE,qBAAqB,GAAG,IAAI;IAM1D;;;;;;;;;OASG;IACU,OAAO,CAAC,GAAG,EAAE,qBAAqB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IA2B9E;;;;OAIG;IACU,UAAU,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBvE;;;;;;;OAOG;IACI,SAAS,CAAC,KAAK,EAAE,oBAAoB,EAAE,KAAK,EAAE,gBAAgB,GAAG,IAAI;IA4B5E;;;;;OAKG;IACI,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,GAAG,IAAI;IAMhE;;;;;;;OAOG;IACmB,eAAe,IAAI,OAAO,CAAC,qBAAqB,EAAE,CAAC;IASzE;;;;;;OAMG;IACa,mBAAmB,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,qBAAqB,EAAE,KAAK,IAAI,GAAG,IAAI;IAKnG;;;;OAIG;IACa,6BAA6B,IAAI,iCAAiC,GAAG,IAAI;IAMzF;;;;;;OAMG;IACU,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzD,4FAA4F;IAC5F,IAAW,WAAW,IAAI,MAAM,GAAG,IAAI,CAEtC;IAID;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAaxB;;;;OAIG;IACH,OAAO,CAAC,UAAU;IAKlB,gHAAgH;YAClG,aAAa;IAU3B,4GAA4G;IAC5G,OAAO,CAAC,sBAAsB;IAK9B,wGAAwG;IACxG,OAAO,CAAC,gBAAgB;IASxB,mFAAmF;IACnF,OAAO,CAAC,QAAQ;IAUhB,+FAA+F;IAC/F,OAAO,CAAC,YAAY;CAcvB;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,IAAI,IAAI,CAExC"}
|
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview `LiveKitBridge` — the **MJ-native multi-party room** Realtime Bridge driver. Unlike
|
|
3
|
+
* every other bridge (which connects OUT to a 3rd-party meeting/telephony platform), LiveKit is a
|
|
4
|
+
* **self-hosted WebRTC SFU that MJ runs itself** — the "MJ-native room"
|
|
5
|
+
* (`/plans/realtime/realtime-bridges-architecture.md` §4c). The architecture treats a Zoom meeting and
|
|
6
|
+
* an MJ-native LiveKit room **identically** — both are multi-party media transports — so this is
|
|
7
|
+
* "*another bridge, not a special build*."
|
|
8
|
+
*
|
|
9
|
+
* Implements the {@link BaseRealtimeBridge} contract against an injectable {@link ILiveKitRoomSdk} seam
|
|
10
|
+
* so it builds + unit-tests with NO network and NO real LiveKit SDK (the Zoom `SetSdkFactory`
|
|
11
|
+
* testability pattern).
|
|
12
|
+
*
|
|
13
|
+
* LiveKit capability coverage (per the §4c / §8 seed row): on-demand join, **full** audio/video/screen
|
|
14
|
+
* in+out (LiveKit does the lot), and per-participant diarization (`SpeakerDiarization`) — the SFU
|
|
15
|
+
* delivers tracks per participant, so speaker labels come free. No scheduled/invite/telephony features.
|
|
16
|
+
* Those virtual base methods keep throwing `BridgeCapabilityNotSupportedError`.
|
|
17
|
+
*
|
|
18
|
+
* ## Echo / self-audio
|
|
19
|
+
* A LiveKit SFU **never delivers a participant its own published track back**, so the bot does not hear
|
|
20
|
+
* its own voice — no echo gate is required (see {@link wireInboundAudio}). This is exactly the property
|
|
21
|
+
* §4c relies on for putting **multiple agents in one room**: each agent hears the *others'* mixed audio
|
|
22
|
+
* natively, not its own, so two agents converse without a transcript-relay hack.
|
|
23
|
+
*
|
|
24
|
+
* @module @memberjunction/ai-bridge-livekit
|
|
25
|
+
* @author MemberJunction.com
|
|
26
|
+
*/
|
|
27
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
28
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
29
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
30
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
31
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
32
|
+
};
|
|
33
|
+
import { RegisterClass } from '@memberjunction/global';
|
|
34
|
+
import { LogError } from '@memberjunction/core';
|
|
35
|
+
import { BaseRealtimeBridge, } from '@memberjunction/ai-bridge-base';
|
|
36
|
+
import { LiveKitMeetingControlsEventSource } from './livekit-meeting-controls.js';
|
|
37
|
+
/**
|
|
38
|
+
* The `DriverClass` key {@link LiveKitBridge} registers under. A `MJ: AI Bridge Providers` row with
|
|
39
|
+
* `DriverClass = 'LiveKitBridge'` resolves to this driver via the `ClassFactory`.
|
|
40
|
+
*/
|
|
41
|
+
export const LIVEKIT_BRIDGE_DRIVER_CLASS = 'LiveKitBridge';
|
|
42
|
+
/**
|
|
43
|
+
* Maps a LiveKit participant role to the bridge's {@link BridgeParticipantRole}. The bot (`IsLocal`) is
|
|
44
|
+
* surfaced as `'Agent'`.
|
|
45
|
+
*/
|
|
46
|
+
function mapParticipantRole(role, isLocal) {
|
|
47
|
+
if (isLocal) {
|
|
48
|
+
return 'Agent';
|
|
49
|
+
}
|
|
50
|
+
switch (role) {
|
|
51
|
+
case 'Host':
|
|
52
|
+
return 'Host';
|
|
53
|
+
case 'CoHost':
|
|
54
|
+
return 'CoHost';
|
|
55
|
+
default:
|
|
56
|
+
return 'Participant';
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/** Maps a LiveKit participant onto the bridge's {@link BridgeParticipantInfo}. */
|
|
60
|
+
function toBridgeParticipant(p) {
|
|
61
|
+
return {
|
|
62
|
+
ExternalId: p.Identity,
|
|
63
|
+
DisplayName: p.DisplayName,
|
|
64
|
+
Role: mapParticipantRole(p.Role, p.IsLocal),
|
|
65
|
+
IsAgent: p.IsLocal === true,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Realtime Bridge driver for the **MJ-native LiveKit room**.
|
|
70
|
+
*
|
|
71
|
+
* Construct the driver with the default constructor (the engine's `ClassFactory` path) — it lazily
|
|
72
|
+
* builds the LiveKit room SDK from the {@link sdkFactory} at {@link Connect} time. Tests inject a
|
|
73
|
+
* `FakeLiveKitRoomSdk` by overriding the factory via {@link SetSdkFactory} (the creation seam) before
|
|
74
|
+
* connecting.
|
|
75
|
+
*
|
|
76
|
+
* Registered via `@RegisterClass(BaseRealtimeBridge, 'LiveKitBridge')`.
|
|
77
|
+
*/
|
|
78
|
+
let LiveKitBridge = class LiveKitBridge extends BaseRealtimeBridge {
|
|
79
|
+
constructor() {
|
|
80
|
+
super(...arguments);
|
|
81
|
+
/** The live LiveKit SDK seam for this session, created at {@link Connect}. */
|
|
82
|
+
this.sdk = null;
|
|
83
|
+
/** The bot's own participant identity in the joined room (set at {@link Connect}). */
|
|
84
|
+
this.botIdentity = null;
|
|
85
|
+
/** The Meeting Controls event source for this session (only when diarization is supported). */
|
|
86
|
+
this.meetingControls = null;
|
|
87
|
+
/**
|
|
88
|
+
* The SDK creation seam. Defaults to a factory that throws an explicit "bind the real LiveKit SDK"
|
|
89
|
+
* error, since this package ships WITHOUT the real SDK adapter (a deployment concern). Production
|
|
90
|
+
* sets a real factory via {@link SetSdkFactory}; tests inject a `FakeLiveKitRoomSdk`.
|
|
91
|
+
*/
|
|
92
|
+
this.sdkFactory = () => {
|
|
93
|
+
throw new Error('LiveKitBridge has no LiveKit room SDK bound. Call LiveKitBridge.SetSdkFactory(...) with a factory ' +
|
|
94
|
+
'that builds an ILiveKitRoomSdk over the real livekit-server-sdk + a room client, or inject a fake in tests.');
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Sets the {@link LiveKitRoomSdkFactory} this driver uses to construct its SDK seam at connect — the
|
|
99
|
+
* creation seam (mirroring Zoom's `SetSdkFactory`). Production binds the real LiveKit room SDK
|
|
100
|
+
* adapter here; tests inject a `FakeLiveKitRoomSdk`.
|
|
101
|
+
*
|
|
102
|
+
* @param factory The factory that builds the {@link ILiveKitRoomSdk} for a session.
|
|
103
|
+
*/
|
|
104
|
+
SetSdkFactory(factory) {
|
|
105
|
+
this.sdkFactory = factory;
|
|
106
|
+
}
|
|
107
|
+
// ── Abstract — every bridge MUST implement ───────────────────────────────────────
|
|
108
|
+
/**
|
|
109
|
+
* Connects to the MJ-native LiveKit room and brings the bot online. Captures the capability context,
|
|
110
|
+
* builds the SDK from the {@link sdkFactory}, wires the inbound per-participant audio path and
|
|
111
|
+
* room-disconnected callback, connects, and (when the provider diarizes) constructs + seeds the
|
|
112
|
+
* Meeting Controls event source.
|
|
113
|
+
*
|
|
114
|
+
* @param ctx The bridge context (features, provider name, the room URL as `Address`, config carrying
|
|
115
|
+
* the signed access token).
|
|
116
|
+
* @returns The bot identity + room (external connection) identifiers, persisted by the engine.
|
|
117
|
+
*/
|
|
118
|
+
async Connect(ctx) {
|
|
119
|
+
this.applyContext(ctx);
|
|
120
|
+
this.RequireFeature('AudioIn'); // a LiveKit room bridge requires bidirectional audio at minimum
|
|
121
|
+
this.RequireFeature('AudioOut');
|
|
122
|
+
this.sdk = this.sdkFactory(ctx.Configuration);
|
|
123
|
+
this.wireInboundAudio(this.sdk);
|
|
124
|
+
this.sdk.onDisconnected(() => this.handleRoomDisconnected());
|
|
125
|
+
// Roster diarization is native to LiveKit (per-participant tracks); only stand up the Meeting
|
|
126
|
+
// Controls source when the provider advertises it (the engine also gates on this flag).
|
|
127
|
+
if (this.features.SpeakerDiarization === true) {
|
|
128
|
+
this.meetingControls = new LiveKitMeetingControlsEventSource(this.sdk);
|
|
129
|
+
}
|
|
130
|
+
// The driver owns the single SDK subscription set and fans out to BOTH its own roster handler
|
|
131
|
+
// and the Meeting Controls source (the SDK seam is latest-handler-wins, so there is one owner).
|
|
132
|
+
this.wireRoster(this.sdk);
|
|
133
|
+
const result = await this.sdk.connect(this.buildConnectArgs(ctx));
|
|
134
|
+
this.botIdentity = result.BotIdentity;
|
|
135
|
+
// Seed the initial roster now that the bot has joined.
|
|
136
|
+
await this.refreshRoster(this.sdk);
|
|
137
|
+
return { BotParticipantId: result.BotIdentity, ExternalConnectionId: result.RoomName };
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Disconnects from the LiveKit room and releases all SDK resources. Tolerant of teardown errors.
|
|
141
|
+
*
|
|
142
|
+
* @param _reason Why the disconnect happened (LiveKit teardown is uniform; the bot simply leaves).
|
|
143
|
+
*/
|
|
144
|
+
async Disconnect(_reason) {
|
|
145
|
+
const sdk = this.sdk;
|
|
146
|
+
this.sdk = null;
|
|
147
|
+
this.botIdentity = null;
|
|
148
|
+
this.meetingControls = null;
|
|
149
|
+
this.mediaHandler = undefined;
|
|
150
|
+
this.participantHandler = undefined;
|
|
151
|
+
if (sdk) {
|
|
152
|
+
try {
|
|
153
|
+
await sdk.disconnect();
|
|
154
|
+
}
|
|
155
|
+
catch (err) {
|
|
156
|
+
LogError(`[LiveKitBridge] disconnect() failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Sends an outbound media frame into the room. LiveKit does the FULL media set, so audio, video, and
|
|
162
|
+
* screen are all published on their respective tracks (gated by the directional capability flags —
|
|
163
|
+
* the realtime models light audio first, video/screen ride the same path once a model emits them).
|
|
164
|
+
*
|
|
165
|
+
* @param track The outbound track the frame targets.
|
|
166
|
+
* @param frame The media frame to send.
|
|
167
|
+
*/
|
|
168
|
+
SendMedia(track, frame) {
|
|
169
|
+
if (!this.sdk) {
|
|
170
|
+
return; // not connected — drop
|
|
171
|
+
}
|
|
172
|
+
const bytes = this.framePcm(frame);
|
|
173
|
+
if (!bytes) {
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
switch (track) {
|
|
177
|
+
case 'audio-out':
|
|
178
|
+
this.sdk.publishAudioFrame(bytes);
|
|
179
|
+
break;
|
|
180
|
+
case 'video-out':
|
|
181
|
+
if (this.features.VideoOut === true) {
|
|
182
|
+
this.sdk.publishVideoFrame(bytes);
|
|
183
|
+
}
|
|
184
|
+
break;
|
|
185
|
+
case 'screen-out':
|
|
186
|
+
if (this.features.ScreenOut === true) {
|
|
187
|
+
this.sdk.publishScreenFrame(bytes);
|
|
188
|
+
}
|
|
189
|
+
break;
|
|
190
|
+
default:
|
|
191
|
+
// An inbound track was passed to SendMedia — ignore (defensive).
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Registers the inbound-media handler. The driver forwards each raw per-participant audio frame
|
|
197
|
+
* (with its speaker identity) to this handler; the engine routes it to `IRealtimeSession.SendInput`.
|
|
198
|
+
*
|
|
199
|
+
* @param handler Invoked with each inbound media frame.
|
|
200
|
+
*/
|
|
201
|
+
OnMedia(handler) {
|
|
202
|
+
this.mediaHandler = handler;
|
|
203
|
+
}
|
|
204
|
+
// ── Capability-gated virtuals LiveKit supports (gated by SupportedFeatures) ───────
|
|
205
|
+
/**
|
|
206
|
+
* Returns the current LiveKit participant roster (gated by `SpeakerDiarization`). Re-asserts the
|
|
207
|
+
* flag (defense-in-depth) so even an engine-bypassing caller cannot pull a roster a disabled
|
|
208
|
+
* provider forbids.
|
|
209
|
+
*
|
|
210
|
+
* @returns The current participants.
|
|
211
|
+
* @throws {BridgeCapabilityNotSupportedError} when diarization is not enabled for the provider.
|
|
212
|
+
*/
|
|
213
|
+
async GetParticipants() {
|
|
214
|
+
this.RequireFeature('SpeakerDiarization');
|
|
215
|
+
if (!this.sdk) {
|
|
216
|
+
return [];
|
|
217
|
+
}
|
|
218
|
+
const participants = await this.sdk.getParticipants();
|
|
219
|
+
return participants.map(toBridgeParticipant);
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Registers a roster-change handler (gated by `SpeakerDiarization`). The driver fires it from the
|
|
223
|
+
* SDK's participant join/leave stream with the full current roster.
|
|
224
|
+
*
|
|
225
|
+
* @param handler Invoked with the updated participant list on each change.
|
|
226
|
+
* @throws {BridgeCapabilityNotSupportedError} when diarization is not enabled.
|
|
227
|
+
*/
|
|
228
|
+
OnParticipantChange(handler) {
|
|
229
|
+
this.RequireFeature('SpeakerDiarization');
|
|
230
|
+
this.participantHandler = handler;
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* Returns the LiveKit Meeting Controls event source for this session (roster · speaking · mute), or
|
|
234
|
+
* `null` when diarization is off (no roster to facilitate). The engine wires the Meeting Controls
|
|
235
|
+
* channel from this.
|
|
236
|
+
*/
|
|
237
|
+
GetMeetingControlsEventSource() {
|
|
238
|
+
return this.meetingControls;
|
|
239
|
+
}
|
|
240
|
+
// ── LiveKit-native surfaces (used by the channel plane / turn-taking) ────────────
|
|
241
|
+
/**
|
|
242
|
+
* Sends a message on the LiveKit data channel — the room-native "chat". Exposed for the channel
|
|
243
|
+
* plane / turn-taking hybrid mode (the social-cost-free "raise hand"). Best-effort — a failure is
|
|
244
|
+
* logged, never fatal.
|
|
245
|
+
*
|
|
246
|
+
* @param text The data/chat message to send.
|
|
247
|
+
*/
|
|
248
|
+
async SendDataMessage(text) {
|
|
249
|
+
if (!this.sdk) {
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
try {
|
|
253
|
+
await this.sdk.sendDataMessage(text);
|
|
254
|
+
}
|
|
255
|
+
catch (err) {
|
|
256
|
+
LogError(`[LiveKitBridge] sendDataMessage failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
/** The bot's participant identity in the joined room (or `null` before {@link Connect}). */
|
|
260
|
+
get BotIdentity() {
|
|
261
|
+
return this.botIdentity;
|
|
262
|
+
}
|
|
263
|
+
// ── internals ────────────────────────────────────────────────────────────────────
|
|
264
|
+
/**
|
|
265
|
+
* Wires the SDK's inbound per-participant audio callback to a diarized inbound
|
|
266
|
+
* {@link BridgeMediaFrame}. The SFU never echoes the bot's own audio, so no self-audio gate is
|
|
267
|
+
* needed; the frame's `SpeakerLabel` is the source participant identity.
|
|
268
|
+
*/
|
|
269
|
+
wireInboundAudio(sdk) {
|
|
270
|
+
sdk.onAudioTrack((frame) => {
|
|
271
|
+
// Mark the speaking participant on the Meeting Controls source for who's-speaking perception.
|
|
272
|
+
this.meetingControls?.NotifySpeaking([frame.ParticipantIdentity]);
|
|
273
|
+
this.mediaHandler?.({
|
|
274
|
+
Track: 'audio-in',
|
|
275
|
+
Bytes: frame.Pcm,
|
|
276
|
+
SpeakerLabel: frame.ParticipantIdentity,
|
|
277
|
+
TimestampMs: frame.TimestampMs ?? Date.now(),
|
|
278
|
+
});
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Wires the SDK's participant join/leave streams ONCE (the driver is the single owner; the SDK seam
|
|
283
|
+
* is latest-handler-wins). Each roster change fans out to both the driver's own roster-change
|
|
284
|
+
* handler and the Meeting Controls source.
|
|
285
|
+
*/
|
|
286
|
+
wireRoster(sdk) {
|
|
287
|
+
sdk.onParticipantJoin(() => void this.refreshRoster(sdk));
|
|
288
|
+
sdk.onParticipantLeave(() => void this.refreshRoster(sdk));
|
|
289
|
+
}
|
|
290
|
+
/** Pulls the current roster from the SDK and fans it out to both the participant handler + Meeting Controls. */
|
|
291
|
+
async refreshRoster(sdk) {
|
|
292
|
+
try {
|
|
293
|
+
const participants = await sdk.getParticipants();
|
|
294
|
+
this.participantHandler?.(participants.map(toBridgeParticipant));
|
|
295
|
+
this.meetingControls?.IngestRoster(participants);
|
|
296
|
+
}
|
|
297
|
+
catch (err) {
|
|
298
|
+
LogError(`[LiveKitBridge] roster refresh failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
/** Handles the SDK's room-disconnected signal: surface an empty roster so the engine sees everyone gone. */
|
|
302
|
+
handleRoomDisconnected() {
|
|
303
|
+
this.participantHandler?.([]);
|
|
304
|
+
this.meetingControls?.IngestRoster([]);
|
|
305
|
+
}
|
|
306
|
+
/** Builds the SDK connect args from the bridge context (Address is the room URL; token from config). */
|
|
307
|
+
buildConnectArgs(ctx) {
|
|
308
|
+
const config = ctx.Configuration ?? {};
|
|
309
|
+
return {
|
|
310
|
+
RoomUrl: ctx.Address,
|
|
311
|
+
AccessToken: typeof config.AccessToken === 'string' ? config.AccessToken : '',
|
|
312
|
+
BotDisplayName: typeof config.BotDisplayName === 'string' ? config.BotDisplayName : `${ctx.ProviderName} Agent`,
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
/** Returns the bytes of an outbound media frame, preferring the binary payload. */
|
|
316
|
+
framePcm(frame) {
|
|
317
|
+
if (frame.Bytes) {
|
|
318
|
+
return frame.Bytes;
|
|
319
|
+
}
|
|
320
|
+
if (frame.Base64) {
|
|
321
|
+
return this.decodeBase64(frame.Base64);
|
|
322
|
+
}
|
|
323
|
+
return undefined;
|
|
324
|
+
}
|
|
325
|
+
/** Decodes a base64 payload to an `ArrayBuffer` (Node `Buffer` fast path, `atob` fallback). */
|
|
326
|
+
decodeBase64(base64) {
|
|
327
|
+
if (typeof Buffer !== 'undefined') {
|
|
328
|
+
const buf = Buffer.from(base64, 'base64');
|
|
329
|
+
const out = new Uint8Array(buf.byteLength);
|
|
330
|
+
out.set(buf);
|
|
331
|
+
return out.buffer;
|
|
332
|
+
}
|
|
333
|
+
const binary = atob(base64);
|
|
334
|
+
const bytes = new Uint8Array(binary.length);
|
|
335
|
+
for (let i = 0; i < binary.length; i++) {
|
|
336
|
+
bytes[i] = binary.charCodeAt(i);
|
|
337
|
+
}
|
|
338
|
+
return bytes.buffer;
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
LiveKitBridge = __decorate([
|
|
342
|
+
RegisterClass(BaseRealtimeBridge, LIVEKIT_BRIDGE_DRIVER_CLASS)
|
|
343
|
+
], LiveKitBridge);
|
|
344
|
+
export { LiveKitBridge };
|
|
345
|
+
/**
|
|
346
|
+
* Tree-shaking-prevention loader. Modern bundlers cannot see the `@RegisterClass` dynamic registration
|
|
347
|
+
* of {@link LiveKitBridge} and may eliminate it. Import and call this no-op from a static code path
|
|
348
|
+
* (the package entry point does) so the `ClassFactory` can resolve `'LiveKitBridge'`.
|
|
349
|
+
*/
|
|
350
|
+
export function LoadLiveKitBridge() {
|
|
351
|
+
// Intentionally empty — referencing the module is what prevents tree-shaking.
|
|
352
|
+
}
|
|
353
|
+
//# sourceMappingURL=livekit-bridge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"livekit-bridge.js","sourceRoot":"","sources":["../src/livekit-bridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;;;;;;;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EACH,kBAAkB,GASrB,MAAM,gCAAgC,CAAC;AASxC,OAAO,EAAE,iCAAiC,EAAE,MAAM,4BAA4B,CAAC;AAE/E;;;GAGG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,eAAe,CAAC;AAE3D;;;GAGG;AACH,SAAS,kBAAkB,CAAC,IAA4B,EAAE,OAA4B;IAClF,IAAI,OAAO,EAAE,CAAC;QACV,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,QAAQ,IAAI,EAAE,CAAC;QACX,KAAK,MAAM;YACP,OAAO,MAAM,CAAC;QAClB,KAAK,QAAQ;YACT,OAAO,QAAQ,CAAC;QACpB;YACI,OAAO,aAAa,CAAC;IAC7B,CAAC;AACL,CAAC;AAED,kFAAkF;AAClF,SAAS,mBAAmB,CAAC,CAAqB;IAC9C,OAAO;QACH,UAAU,EAAE,CAAC,CAAC,QAAQ;QACtB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC;QAC3C,OAAO,EAAE,CAAC,CAAC,OAAO,KAAK,IAAI;KAC9B,CAAC;AACN,CAAC;AAED;;;;;;;;;GASG;AAEI,IAAM,aAAa,GAAnB,MAAM,aAAc,SAAQ,kBAAkB;IAA9C;;QACH,8EAA8E;QACtE,QAAG,GAA2B,IAAI,CAAC;QAE3C,sFAAsF;QAC9E,gBAAW,GAAkB,IAAI,CAAC;QAQ1C,+FAA+F;QACvF,oBAAe,GAA6C,IAAI,CAAC;QAEzE;;;;WAIG;QACK,eAAU,GAA0B,GAAG,EAAE;YAC7C,MAAM,IAAI,KAAK,CACX,oGAAoG;gBAChG,6GAA6G,CACpH,CAAC;QACN,CAAC,CAAC;IA2QN,CAAC;IAzQG;;;;;;OAMG;IACI,aAAa,CAAC,OAA8B;QAC/C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC;IAC9B,CAAC;IAED,oFAAoF;IAEpF;;;;;;;;;OASG;IACI,KAAK,CAAC,OAAO,CAAC,GAA0B;QAC3C,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,gEAAgE;QAChG,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,CAAC;QAEhC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;QAC9C,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC,CAAC;QAE7D,8FAA8F;QAC9F,wFAAwF;QACxF,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,KAAK,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,eAAe,GAAG,IAAI,iCAAiC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3E,CAAC;QACD,8FAA8F;QAC9F,gGAAgG;QAChG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QAEtC,uDAAuD;QACvD,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEnC,OAAO,EAAE,gBAAgB,EAAE,MAAM,CAAC,WAAW,EAAE,oBAAoB,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC3F,CAAC;IAED;;;;OAIG;IACI,KAAK,CAAC,UAAU,CAAC,OAA+B;QACnD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACrB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC;QAChB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5B,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC;QAC9B,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAC;QACpC,IAAI,GAAG,EAAE,CAAC;YACN,IAAI,CAAC;gBACD,MAAM,GAAG,CAAC,UAAU,EAAE,CAAC;YAC3B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,QAAQ,CAAC,wCAAwC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACzG,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACI,SAAS,CAAC,KAA2B,EAAE,KAAuB;QACjE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACZ,OAAO,CAAC,uBAAuB;QACnC,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,OAAO;QACX,CAAC;QACD,QAAQ,KAAK,EAAE,CAAC;YACZ,KAAK,WAAW;gBACZ,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBAClC,MAAM;YACV,KAAK,WAAW;gBACZ,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;oBAClC,IAAI,CAAC,GAAG,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBACtC,CAAC;gBACD,MAAM;YACV,KAAK,YAAY;gBACb,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC;oBACnC,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;gBACvC,CAAC;gBACD,MAAM;YACV;gBACI,iEAAiE;gBACjE,MAAM;QACd,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACI,OAAO,CAAC,OAA0C;QACrD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED,qFAAqF;IAErF;;;;;;;OAOG;IACa,KAAK,CAAC,eAAe;QACjC,IAAI,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACZ,OAAO,EAAE,CAAC;QACd,CAAC;QACD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;QACtD,OAAO,YAAY,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjD,CAAC;IAED;;;;;;OAMG;IACa,mBAAmB,CAAC,OAAwD;QACxF,IAAI,CAAC,cAAc,CAAC,oBAAoB,CAAC,CAAC;QAC1C,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACtC,CAAC;IAED;;;;OAIG;IACa,6BAA6B;QACzC,OAAO,IAAI,CAAC,eAAe,CAAC;IAChC,CAAC;IAED,oFAAoF;IAEpF;;;;;;OAMG;IACI,KAAK,CAAC,eAAe,CAAC,IAAY;QACrC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;YACZ,OAAO;QACX,CAAC;QACD,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,QAAQ,CAAC,2CAA2C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5G,CAAC;IACL,CAAC;IAED,4FAA4F;IAC5F,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,WAAW,CAAC;IAC5B,CAAC;IAED,oFAAoF;IAEpF;;;;OAIG;IACK,gBAAgB,CAAC,GAAoB;QACzC,GAAG,CAAC,YAAY,CAAC,CAAC,KAAwB,EAAE,EAAE;YAC1C,8FAA8F;YAC9F,IAAI,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;YAClE,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChB,KAAK,EAAE,UAAU;gBACjB,KAAK,EAAE,KAAK,CAAC,GAAG;gBAChB,YAAY,EAAE,KAAK,CAAC,mBAAmB;gBACvC,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,IAAI,CAAC,GAAG,EAAE;aAC/C,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACK,UAAU,CAAC,GAAoB;QACnC,GAAG,CAAC,iBAAiB,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;QAC1D,GAAG,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,gHAAgH;IACxG,KAAK,CAAC,aAAa,CAAC,GAAoB;QAC5C,IAAI,CAAC;YACD,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YACjD,IAAI,CAAC,kBAAkB,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC;YACjE,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,YAAY,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,QAAQ,CAAC,0CAA0C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3G,CAAC;IACL,CAAC;IAED,4GAA4G;IACpG,sBAAsB;QAC1B,IAAI,CAAC,kBAAkB,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9B,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;IAC3C,CAAC;IAED,wGAAwG;IAChG,gBAAgB,CAAC,GAA0B;QAC/C,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC;QACvC,OAAO;YACH,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,WAAW,EAAE,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE;YAC7E,cAAc,EAAE,OAAO,MAAM,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,YAAY,QAAQ;SAClH,CAAC;IACN,CAAC;IAED,mFAAmF;IAC3E,QAAQ,CAAC,KAAuB;QACpC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO,KAAK,CAAC,KAAK,CAAC;QACvB,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;YACf,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,+FAA+F;IACvF,YAAY,CAAC,MAAc;QAC/B,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAC1C,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC3C,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACb,OAAO,GAAG,CAAC,MAAM,CAAC;QACtB,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,KAAK,CAAC,MAAM,CAAC;IACxB,CAAC;CACJ,CAAA;AArSY,aAAa;IADzB,aAAa,CAAC,kBAAkB,EAAE,2BAA2B,CAAC;GAClD,aAAa,CAqSzB;;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB;IAC7B,8EAA8E;AAClF,CAAC"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adapts the {@link ILiveKitRoomSdk} participant / speaking stream into an
|
|
3
|
+
* {@link IBridgeMeetingControlsEventSource} so the bridge engine can wire the **Meeting Controls**
|
|
4
|
+
* facilitator channel (roster · who's-speaking · hand-raise queue) for a LiveKit room session. This is
|
|
5
|
+
* the §4b "the bridge contributes a channel" pattern realized for the MJ-native room — no channel logic
|
|
6
|
+
* lives here, only the platform→channel signal mapping.
|
|
7
|
+
*
|
|
8
|
+
* ## What LiveKit contributes (vs. Zoom)
|
|
9
|
+
* - **Roster + speaking**: native (per-participant tracks → diarization → speaking attribution).
|
|
10
|
+
* - **Hand-raise**: LiveKit has **no built-in raise-hand**. The channel still offers the facilitator
|
|
11
|
+
* hand-raise *queue* (an MJ construct), but no PLATFORM hand-raise signal arrives, so
|
|
12
|
+
* {@link IngestHandRaise} is available for an app that layers raise-hand over data messages — it is
|
|
13
|
+
* simply never fired by the room itself.
|
|
14
|
+
* - **Mute**: LiveKit room-admin can mute a participant's published track, so the `Mute` capability is
|
|
15
|
+
* advertised and {@link MuteParticipant} actuates it through the SDK adapter.
|
|
16
|
+
*/
|
|
17
|
+
import { IBridgeMeetingControlsEventSource, BridgeMeetingParticipant, BridgeMeetingControlsCapability } from '@memberjunction/ai-bridge-base';
|
|
18
|
+
import { ILiveKitRoomSdk, LiveKitParticipant } from './livekit-sdk.js';
|
|
19
|
+
/** Maps one LiveKit participant onto the channel's {@link BridgeMeetingParticipant} shape. */
|
|
20
|
+
export declare function toMeetingParticipant(p: LiveKitParticipant): BridgeMeetingParticipant;
|
|
21
|
+
/**
|
|
22
|
+
* LiveKit's adapter to the Meeting Controls channel's event source. It maintains a live roster and
|
|
23
|
+
* mirrors diarized speaking signals, and actuates mute through the SDK adapter.
|
|
24
|
+
*
|
|
25
|
+
* **The driver owns the SDK subscriptions** (the SDK seam is "latest handler wins") and **feeds this
|
|
26
|
+
* source imperatively** via {@link IngestRoster} / {@link IngestSpeaking} / {@link IngestHandRaise}.
|
|
27
|
+
* This source only needs the SDK for the one *action* it actuates — {@link MuteParticipant}. The
|
|
28
|
+
* Meeting Controls channel subscribes the three `On*` streams in turn.
|
|
29
|
+
*
|
|
30
|
+
* The driver constructs ONE of these per session (only when the provider has `SpeakerDiarization`, i.e.
|
|
31
|
+
* there is a roster to facilitate) and returns it from `LiveKitBridge.GetMeetingControlsEventSource`.
|
|
32
|
+
*/
|
|
33
|
+
export declare class LiveKitMeetingControlsEventSource implements IBridgeMeetingControlsEventSource {
|
|
34
|
+
private readonly sdk;
|
|
35
|
+
/** Live roster keyed by participant identity (lowercased), kept current by {@link IngestRoster}. */
|
|
36
|
+
private readonly roster;
|
|
37
|
+
private rosterHandler?;
|
|
38
|
+
private speakingHandler?;
|
|
39
|
+
private handRaiseHandler?;
|
|
40
|
+
/** LiveKit room-admin can mute a published track, so the facilitator `MuteParticipant` tool is offered. */
|
|
41
|
+
readonly Capabilities: ReadonlyArray<BridgeMeetingControlsCapability>;
|
|
42
|
+
/**
|
|
43
|
+
* @param sdk The LiveKit SDK seam used only to actuate {@link MuteParticipant} — all *perception* is
|
|
44
|
+
* fed in by the driver via the `Ingest*` methods so the SDK keeps a single subscriber per event.
|
|
45
|
+
*/
|
|
46
|
+
constructor(sdk: ILiveKitRoomSdk);
|
|
47
|
+
/**
|
|
48
|
+
* Replaces the roster with a fresh snapshot from the driver and emits it to the channel. Called by
|
|
49
|
+
* the driver after the initial connect and on every participant join/leave.
|
|
50
|
+
*
|
|
51
|
+
* @param participants The full current LiveKit roster.
|
|
52
|
+
*/
|
|
53
|
+
IngestRoster(participants: LiveKitParticipant[]): void;
|
|
54
|
+
/**
|
|
55
|
+
* Forwards a hand-raise/lower signal to the channel. LiveKit emits none natively; this exists for an
|
|
56
|
+
* app that layers raise-hand over data messages.
|
|
57
|
+
*
|
|
58
|
+
* @param participantId The participant whose hand changed.
|
|
59
|
+
* @param raised Whether the hand is now raised.
|
|
60
|
+
*/
|
|
61
|
+
IngestHandRaise(participantId: string, raised: boolean): void;
|
|
62
|
+
/**
|
|
63
|
+
* Forwards a diarized speaking-set change from the driver to the channel.
|
|
64
|
+
*
|
|
65
|
+
* @param participantIds The identities currently speaking.
|
|
66
|
+
*/
|
|
67
|
+
IngestSpeaking(participantIds: string[]): void;
|
|
68
|
+
/**
|
|
69
|
+
* Convenience alias for {@link IngestSpeaking}, kept because "notify" reads naturally at the
|
|
70
|
+
* driver's inbound-audio call site.
|
|
71
|
+
*
|
|
72
|
+
* @param participantIds The identities currently speaking.
|
|
73
|
+
*/
|
|
74
|
+
NotifySpeaking(participantIds: string[]): void;
|
|
75
|
+
/** @inheritdoc */
|
|
76
|
+
OnRosterChange(handler: (participants: BridgeMeetingParticipant[]) => void): void;
|
|
77
|
+
/** @inheritdoc */
|
|
78
|
+
OnSpeakingChange(handler: (participantIds: string[]) => void): void;
|
|
79
|
+
/** @inheritdoc */
|
|
80
|
+
OnHandRaiseChange(handler: (participantId: string, raised: boolean) => void): void;
|
|
81
|
+
/** @inheritdoc */
|
|
82
|
+
MuteParticipant(participantId: string): Promise<void>;
|
|
83
|
+
/** Actuates a mute through the SDK adapter's data-channel admin path. */
|
|
84
|
+
private muteViaSdk;
|
|
85
|
+
private emitRoster;
|
|
86
|
+
private key;
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=livekit-meeting-controls.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"livekit-meeting-controls.d.ts","sourceRoot":"","sources":["../src/livekit-meeting-controls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EACH,iCAAiC,EACjC,wBAAwB,EACxB,+BAA+B,EAElC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAA0B,MAAM,eAAe,CAAC;AAoB5F,8FAA8F;AAC9F,wBAAgB,oBAAoB,CAAC,CAAC,EAAE,kBAAkB,GAAG,wBAAwB,CAOpF;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,iCAAkC,YAAW,iCAAiC;IAe3E,OAAO,CAAC,QAAQ,CAAC,GAAG;IAdhC,oGAAoG;IACpG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA+C;IAEtE,OAAO,CAAC,aAAa,CAAC,CAAqD;IAC3E,OAAO,CAAC,eAAe,CAAC,CAAqC;IAC7D,OAAO,CAAC,gBAAgB,CAAC,CAAmD;IAE5E,2GAA2G;IAC3G,SAAgB,YAAY,EAAE,aAAa,CAAC,+BAA+B,CAAC,CAAY;IAExF;;;OAGG;gBAC0B,GAAG,EAAE,eAAe;IAIjD;;;;;OAKG;IACI,YAAY,CAAC,YAAY,EAAE,kBAAkB,EAAE,GAAG,IAAI;IAQ7D;;;;;;OAMG;IACI,eAAe,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,IAAI;IAIpE;;;;OAIG;IACI,cAAc,CAAC,cAAc,EAAE,MAAM,EAAE,GAAG,IAAI;IAIrD;;;;;OAKG;IACI,cAAc,CAAC,cAAc,EAAE,MAAM,EAAE,GAAG,IAAI;IAMrD,kBAAkB;IACX,cAAc,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,wBAAwB,EAAE,KAAK,IAAI,GAAG,IAAI;IAMxF,kBAAkB;IACX,gBAAgB,CAAC,OAAO,EAAE,CAAC,cAAc,EAAE,MAAM,EAAE,KAAK,IAAI,GAAG,IAAI;IAI1E,kBAAkB;IACX,iBAAiB,CAAC,OAAO,EAAE,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,KAAK,IAAI,GAAG,IAAI;IAIzF,kBAAkB;IACL,eAAe,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQlE,yEAAyE;YAC3D,UAAU;IAMxB,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,GAAG;CAGd"}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adapts the {@link ILiveKitRoomSdk} participant / speaking stream into an
|
|
3
|
+
* {@link IBridgeMeetingControlsEventSource} so the bridge engine can wire the **Meeting Controls**
|
|
4
|
+
* facilitator channel (roster · who's-speaking · hand-raise queue) for a LiveKit room session. This is
|
|
5
|
+
* the §4b "the bridge contributes a channel" pattern realized for the MJ-native room — no channel logic
|
|
6
|
+
* lives here, only the platform→channel signal mapping.
|
|
7
|
+
*
|
|
8
|
+
* ## What LiveKit contributes (vs. Zoom)
|
|
9
|
+
* - **Roster + speaking**: native (per-participant tracks → diarization → speaking attribution).
|
|
10
|
+
* - **Hand-raise**: LiveKit has **no built-in raise-hand**. The channel still offers the facilitator
|
|
11
|
+
* hand-raise *queue* (an MJ construct), but no PLATFORM hand-raise signal arrives, so
|
|
12
|
+
* {@link IngestHandRaise} is available for an app that layers raise-hand over data messages — it is
|
|
13
|
+
* simply never fired by the room itself.
|
|
14
|
+
* - **Mute**: LiveKit room-admin can mute a participant's published track, so the `Mute` capability is
|
|
15
|
+
* advertised and {@link MuteParticipant} actuates it through the SDK adapter.
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* Maps a LiveKit participant role to the bridge's meeting-participant role. LiveKit has no distinct
|
|
19
|
+
* "Agent" role; the bot is flagged via `IsLocal` and surfaced as `'Agent'`.
|
|
20
|
+
*/
|
|
21
|
+
function mapRole(role, isLocal) {
|
|
22
|
+
if (isLocal) {
|
|
23
|
+
return 'Agent';
|
|
24
|
+
}
|
|
25
|
+
switch (role) {
|
|
26
|
+
case 'Host':
|
|
27
|
+
return 'Host';
|
|
28
|
+
case 'CoHost':
|
|
29
|
+
return 'CoHost';
|
|
30
|
+
default:
|
|
31
|
+
return 'Participant';
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/** Maps one LiveKit participant onto the channel's {@link BridgeMeetingParticipant} shape. */
|
|
35
|
+
export function toMeetingParticipant(p) {
|
|
36
|
+
return {
|
|
37
|
+
ParticipantId: p.Identity,
|
|
38
|
+
DisplayName: p.DisplayName,
|
|
39
|
+
Role: mapRole(p.Role, p.IsLocal),
|
|
40
|
+
IsAgent: p.IsLocal === true,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* LiveKit's adapter to the Meeting Controls channel's event source. It maintains a live roster and
|
|
45
|
+
* mirrors diarized speaking signals, and actuates mute through the SDK adapter.
|
|
46
|
+
*
|
|
47
|
+
* **The driver owns the SDK subscriptions** (the SDK seam is "latest handler wins") and **feeds this
|
|
48
|
+
* source imperatively** via {@link IngestRoster} / {@link IngestSpeaking} / {@link IngestHandRaise}.
|
|
49
|
+
* This source only needs the SDK for the one *action* it actuates — {@link MuteParticipant}. The
|
|
50
|
+
* Meeting Controls channel subscribes the three `On*` streams in turn.
|
|
51
|
+
*
|
|
52
|
+
* The driver constructs ONE of these per session (only when the provider has `SpeakerDiarization`, i.e.
|
|
53
|
+
* there is a roster to facilitate) and returns it from `LiveKitBridge.GetMeetingControlsEventSource`.
|
|
54
|
+
*/
|
|
55
|
+
export class LiveKitMeetingControlsEventSource {
|
|
56
|
+
/**
|
|
57
|
+
* @param sdk The LiveKit SDK seam used only to actuate {@link MuteParticipant} — all *perception* is
|
|
58
|
+
* fed in by the driver via the `Ingest*` methods so the SDK keeps a single subscriber per event.
|
|
59
|
+
*/
|
|
60
|
+
constructor(sdk) {
|
|
61
|
+
this.sdk = sdk;
|
|
62
|
+
/** Live roster keyed by participant identity (lowercased), kept current by {@link IngestRoster}. */
|
|
63
|
+
this.roster = new Map();
|
|
64
|
+
/** LiveKit room-admin can mute a published track, so the facilitator `MuteParticipant` tool is offered. */
|
|
65
|
+
this.Capabilities = ['Mute'];
|
|
66
|
+
}
|
|
67
|
+
// ── Driver-fed perception (the driver owns the SDK subscriptions) ─────────────────
|
|
68
|
+
/**
|
|
69
|
+
* Replaces the roster with a fresh snapshot from the driver and emits it to the channel. Called by
|
|
70
|
+
* the driver after the initial connect and on every participant join/leave.
|
|
71
|
+
*
|
|
72
|
+
* @param participants The full current LiveKit roster.
|
|
73
|
+
*/
|
|
74
|
+
IngestRoster(participants) {
|
|
75
|
+
this.roster.clear();
|
|
76
|
+
for (const p of participants) {
|
|
77
|
+
this.roster.set(this.key(p.Identity), toMeetingParticipant(p));
|
|
78
|
+
}
|
|
79
|
+
this.emitRoster();
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Forwards a hand-raise/lower signal to the channel. LiveKit emits none natively; this exists for an
|
|
83
|
+
* app that layers raise-hand over data messages.
|
|
84
|
+
*
|
|
85
|
+
* @param participantId The participant whose hand changed.
|
|
86
|
+
* @param raised Whether the hand is now raised.
|
|
87
|
+
*/
|
|
88
|
+
IngestHandRaise(participantId, raised) {
|
|
89
|
+
this.handRaiseHandler?.(participantId, raised);
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Forwards a diarized speaking-set change from the driver to the channel.
|
|
93
|
+
*
|
|
94
|
+
* @param participantIds The identities currently speaking.
|
|
95
|
+
*/
|
|
96
|
+
IngestSpeaking(participantIds) {
|
|
97
|
+
this.speakingHandler?.(participantIds);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Convenience alias for {@link IngestSpeaking}, kept because "notify" reads naturally at the
|
|
101
|
+
* driver's inbound-audio call site.
|
|
102
|
+
*
|
|
103
|
+
* @param participantIds The identities currently speaking.
|
|
104
|
+
*/
|
|
105
|
+
NotifySpeaking(participantIds) {
|
|
106
|
+
this.IngestSpeaking(participantIds);
|
|
107
|
+
}
|
|
108
|
+
// ── IBridgeMeetingControlsEventSource ────────────────────────────────────────────
|
|
109
|
+
/** @inheritdoc */
|
|
110
|
+
OnRosterChange(handler) {
|
|
111
|
+
this.rosterHandler = handler;
|
|
112
|
+
// Emit the current roster immediately so a late subscriber is not blank.
|
|
113
|
+
this.emitRoster();
|
|
114
|
+
}
|
|
115
|
+
/** @inheritdoc */
|
|
116
|
+
OnSpeakingChange(handler) {
|
|
117
|
+
this.speakingHandler = handler;
|
|
118
|
+
}
|
|
119
|
+
/** @inheritdoc */
|
|
120
|
+
OnHandRaiseChange(handler) {
|
|
121
|
+
this.handRaiseHandler = handler;
|
|
122
|
+
}
|
|
123
|
+
/** @inheritdoc */
|
|
124
|
+
async MuteParticipant(participantId) {
|
|
125
|
+
// LiveKit mute is actuated by the room-admin path; the SDK adapter maps this onto its
|
|
126
|
+
// mute-published-track call. The fake captures it for tests.
|
|
127
|
+
await this.muteViaSdk(participantId);
|
|
128
|
+
}
|
|
129
|
+
// ── internals ────────────────────────────────────────────────────────────────────
|
|
130
|
+
/** Actuates a mute through the SDK adapter's data-channel admin path. */
|
|
131
|
+
async muteViaSdk(participantId) {
|
|
132
|
+
// The SDK seam does not surface a dedicated mute primitive (room-admin mute is a server-SDK
|
|
133
|
+
// concern bound at deployment); we signal it as a structured data message the adapter recognizes.
|
|
134
|
+
await this.sdk.sendDataMessage(`__mj_mute:${participantId}`);
|
|
135
|
+
}
|
|
136
|
+
emitRoster() {
|
|
137
|
+
this.rosterHandler?.(Array.from(this.roster.values()));
|
|
138
|
+
}
|
|
139
|
+
key(identity) {
|
|
140
|
+
return identity.trim().toLowerCase();
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
//# sourceMappingURL=livekit-meeting-controls.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"livekit-meeting-controls.js","sourceRoot":"","sources":["../src/livekit-meeting-controls.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAUH;;;GAGG;AACH,SAAS,OAAO,CAAC,IAA4B,EAAE,OAA4B;IACvE,IAAI,OAAO,EAAE,CAAC;QACV,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,QAAQ,IAAI,EAAE,CAAC;QACX,KAAK,MAAM;YACP,OAAO,MAAM,CAAC;QAClB,KAAK,QAAQ;YACT,OAAO,QAAQ,CAAC;QACpB;YACI,OAAO,aAAa,CAAC;IAC7B,CAAC;AACL,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,oBAAoB,CAAC,CAAqB;IACtD,OAAO;QACH,aAAa,EAAE,CAAC,CAAC,QAAQ;QACzB,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC;QAChC,OAAO,EAAE,CAAC,CAAC,OAAO,KAAK,IAAI;KAC9B,CAAC;AACN,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,iCAAiC;IAW1C;;;OAGG;IACH,YAA6B,GAAoB;QAApB,QAAG,GAAH,GAAG,CAAiB;QAdjD,oGAAoG;QACnF,WAAM,GAAG,IAAI,GAAG,EAAoC,CAAC;QAMtE,2GAA2G;QAC3F,iBAAY,GAAmD,CAAC,MAAM,CAAC,CAAC;IAMpC,CAAC;IAErD,qFAAqF;IAErF;;;;;OAKG;IACI,YAAY,CAAC,YAAkC;QAClD,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC;QACnE,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC;IACtB,CAAC;IAED;;;;;;OAMG;IACI,eAAe,CAAC,aAAqB,EAAE,MAAe;QACzD,IAAI,CAAC,gBAAgB,EAAE,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IACnD,CAAC;IAED;;;;OAIG;IACI,cAAc,CAAC,cAAwB;QAC1C,IAAI,CAAC,eAAe,EAAE,CAAC,cAAc,CAAC,CAAC;IAC3C,CAAC;IAED;;;;;OAKG;IACI,cAAc,CAAC,cAAwB;QAC1C,IAAI,CAAC,cAAc,CAAC,cAAc,CAAC,CAAC;IACxC,CAAC;IAED,oFAAoF;IAEpF,kBAAkB;IACX,cAAc,CAAC,OAA2D;QAC7E,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;QAC7B,yEAAyE;QACzE,IAAI,CAAC,UAAU,EAAE,CAAC;IACtB,CAAC;IAED,kBAAkB;IACX,gBAAgB,CAAC,OAA2C;QAC/D,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;IACnC,CAAC;IAED,kBAAkB;IACX,iBAAiB,CAAC,OAAyD;QAC9E,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;IACpC,CAAC;IAED,kBAAkB;IACX,KAAK,CAAC,eAAe,CAAC,aAAqB;QAC9C,sFAAsF;QACtF,6DAA6D;QAC7D,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IACzC,CAAC;IAED,oFAAoF;IAEpF,yEAAyE;IACjE,KAAK,CAAC,UAAU,CAAC,aAAqB;QAC1C,4FAA4F;QAC5F,kGAAkG;QAClG,MAAM,IAAI,CAAC,GAAG,CAAC,eAAe,CAAC,aAAa,aAAa,EAAE,CAAC,CAAC;IACjE,CAAC;IAEO,UAAU;QACd,IAAI,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAEO,GAAG,CAAC,QAAgB;QACxB,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzC,CAAC;CACJ"}
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The injectable **LiveKit room SDK seam** — the minimal set of operations
|
|
3
|
+
* {@link import('./livekit-bridge.js').LiveKitBridge} needs from LiveKit, declared as an interface so the
|
|
4
|
+
* driver builds and unit-tests against an in-memory fake with **no network and no real LiveKit SDK**.
|
|
5
|
+
*
|
|
6
|
+
* ## LiveKit is the MJ-NATIVE room (self-hosted), not a 3rd-party platform
|
|
7
|
+
* Every OTHER bridge in this program connects **out** to a meeting hosted by someone else (Zoom,
|
|
8
|
+
* Teams, Meet, Webex, Slack, Discord) or to a telephony carrier (Twilio, Vonage, RingCentral). LiveKit
|
|
9
|
+
* is the opposite: it is an **open-source WebRTC SFU that MJ runs itself** — the "MJ-native room"
|
|
10
|
+
* (`/plans/realtime/realtime-bridges-architecture.md` §4c). The architecture deliberately treats a Zoom
|
|
11
|
+
* meeting, a Teams meeting, and an MJ-native LiveKit room **identically** — all are multi-party media
|
|
12
|
+
* transports — so LiveKit is "*another bridge, not a special build*." The only practical differences:
|
|
13
|
+
* - **We own the room.** No marketplace review, no per-platform bot-admission quirks; MJ mints the
|
|
14
|
+
* access token and the room exists because MJ stood up the SFU. An MJ-native multi-party experience
|
|
15
|
+
* (e.g. embedded in Explorer) is just *this* bridge.
|
|
16
|
+
* - **`connect(roomUrl, token)`** takes a LiveKit room URL + a signed access token (minted upstream by
|
|
17
|
+
* MJ's credential/token layer — never inline secrets), rather than a join URL we were handed.
|
|
18
|
+
*
|
|
19
|
+
* ## Production binding (TODO at deployment)
|
|
20
|
+
* In production this interface is bound to **`livekit-server-sdk`** (token minting / room admin) plus a
|
|
21
|
+
* **room client** (the Node WebRTC participant — e.g. `@livekit/rtc-node`) so the bot can publish/
|
|
22
|
+
* subscribe media. The named operations map to the SDK as follows:
|
|
23
|
+
* - {@link connect} / {@link disconnect} → the room client `connect()` / `disconnect()` lifecycle.
|
|
24
|
+
* - {@link publishAudioFrame} → publishing PCM on the bot's audio track (the agent's voice).
|
|
25
|
+
* - {@link onAudioTrack} → subscribing each remote participant's audio track; LiveKit delivers tracks
|
|
26
|
+
* **per participant**, which is the native source of speaker labels for diarization (no extra mixer).
|
|
27
|
+
* - {@link publishVideoFrame} / {@link publishScreenFrame} → publishing the bot's camera / screen-share
|
|
28
|
+
* tracks (LiveKit does full A/V/screen; the realtime models light audio first).
|
|
29
|
+
* - {@link onParticipantJoin} / {@link onParticipantLeave} / {@link getParticipants} → the room's
|
|
30
|
+
* `ParticipantConnected` / `ParticipantDisconnected` events + the participant list.
|
|
31
|
+
* - {@link sendDataMessage} → the LiveKit **data channel** (reliable data publish) — used for chat.
|
|
32
|
+
* - {@link onDisconnected} → the room `Disconnected` event (the SFU closed / the bot was removed).
|
|
33
|
+
*
|
|
34
|
+
* ## Echo / self-audio
|
|
35
|
+
* A LiveKit SFU **never delivers a participant its own published track back** — the bot does not hear
|
|
36
|
+
* its own voice, so no echo gate is needed here (documented in {@link onAudioTrack}). The bridge still
|
|
37
|
+
* flags its own track via {@link LiveKitParticipant.IsLocal} defensively.
|
|
38
|
+
*
|
|
39
|
+
* Binding the real SDK is a thin adapter that implements this interface; the driver and its tests do
|
|
40
|
+
* not change. **None of the SDK types leak into this package.**
|
|
41
|
+
*
|
|
42
|
+
* See `/plans/realtime/realtime-bridges-architecture.md` §4c and `/guides/REALTIME_BRIDGES_GUIDE.md`.
|
|
43
|
+
*/
|
|
44
|
+
/** The role a LiveKit participant holds, normalized to the bridge's participant roles. */
|
|
45
|
+
export type LiveKitParticipantRole = 'Host' | 'CoHost' | 'Participant';
|
|
46
|
+
/**
|
|
47
|
+
* One LiveKit room participant as the seam reports it. Platform-native and minimal — the driver maps
|
|
48
|
+
* this onto `BridgeParticipantInfo` / `BridgeMeetingParticipant`.
|
|
49
|
+
*/
|
|
50
|
+
export interface LiveKitParticipant {
|
|
51
|
+
/** The LiveKit participant identity (stable, application-assigned — the SID's human-facing key). */
|
|
52
|
+
Identity: string;
|
|
53
|
+
/** The participant's display name as LiveKit reports it (the `name` attribute), when set. */
|
|
54
|
+
DisplayName?: string;
|
|
55
|
+
/** The participant's room role (derived from participant metadata / permissions by the adapter). */
|
|
56
|
+
Role: LiveKitParticipantRole;
|
|
57
|
+
/** Whether this participant is the bridge's own bot (LiveKit's local participant). */
|
|
58
|
+
IsLocal?: boolean;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* One frame of raw per-participant audio the seam surfaces for diarization + the agent's "hearing".
|
|
62
|
+
* Because LiveKit subscribes tracks **per participant**, every inbound frame is already attributed to a
|
|
63
|
+
* single speaker — diarization comes free from the SFU.
|
|
64
|
+
*/
|
|
65
|
+
export interface LiveKitAudioFrame {
|
|
66
|
+
/** Raw PCM audio bytes for this frame. */
|
|
67
|
+
Pcm: ArrayBuffer;
|
|
68
|
+
/** The LiveKit participant identity this audio came from (the diarization speaker label). */
|
|
69
|
+
ParticipantIdentity: string;
|
|
70
|
+
/** The participant's display name at capture time, when known. */
|
|
71
|
+
DisplayName?: string;
|
|
72
|
+
/** Optional epoch-ms capture timestamp. */
|
|
73
|
+
TimestampMs?: number;
|
|
74
|
+
}
|
|
75
|
+
/** Arguments to {@link ILiveKitRoomSdk.connect} — what the bot needs to join an MJ-native room. */
|
|
76
|
+
export interface LiveKitConnectArgs {
|
|
77
|
+
/** The LiveKit room server URL (e.g. `wss://livekit.myorg.com`). MJ-owned, self-hosted. */
|
|
78
|
+
RoomUrl: string;
|
|
79
|
+
/**
|
|
80
|
+
* The signed LiveKit access token authorizing the bot to join a specific room as a participant.
|
|
81
|
+
* Minted upstream by MJ's token/credential layer (it encodes the room name + grants). Never inline
|
|
82
|
+
* secrets — this arrives already-signed.
|
|
83
|
+
*/
|
|
84
|
+
AccessToken: string;
|
|
85
|
+
/** The display name the bot appears as in the participant list. */
|
|
86
|
+
BotDisplayName: string;
|
|
87
|
+
}
|
|
88
|
+
/** The handles the seam returns after a successful {@link ILiveKitRoomSdk.connect}. */
|
|
89
|
+
export interface LiveKitConnectResult {
|
|
90
|
+
/** The bot's own participant identity in the joined room. */
|
|
91
|
+
BotIdentity: string;
|
|
92
|
+
/** The LiveKit room name the bot joined (the durable external connection id). */
|
|
93
|
+
RoomName: string;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* The minimal LiveKit room SDK surface the {@link import('./livekit-bridge.js').LiveKitBridge} depends on.
|
|
97
|
+
* Production binds this to `livekit-server-sdk` + a room client; tests inject a `FakeLiveKitRoomSdk`.
|
|
98
|
+
*/
|
|
99
|
+
export interface ILiveKitRoomSdk {
|
|
100
|
+
/**
|
|
101
|
+
* Connects to the MJ-native LiveKit room as a bot participant and brings the bot online. Returns
|
|
102
|
+
* the bot's identity + the room name.
|
|
103
|
+
*
|
|
104
|
+
* @param args Connect parameters (room URL, signed access token, bot name).
|
|
105
|
+
* @returns The bot identity + room handles.
|
|
106
|
+
*/
|
|
107
|
+
connect(args: LiveKitConnectArgs): Promise<LiveKitConnectResult>;
|
|
108
|
+
/** Disconnects from the room and releases SDK resources. */
|
|
109
|
+
disconnect(): Promise<void>;
|
|
110
|
+
/**
|
|
111
|
+
* Publishes one raw PCM audio frame on the bot's audio track (the agent's voice into the room).
|
|
112
|
+
*
|
|
113
|
+
* @param pcm The PCM audio bytes to publish.
|
|
114
|
+
*/
|
|
115
|
+
publishAudioFrame(pcm: ArrayBuffer): void;
|
|
116
|
+
/**
|
|
117
|
+
* Subscribes inbound per-participant audio frames (what the agent hears, carrying the speaker
|
|
118
|
+
* identity for diarization). "Latest handler wins." The SFU never echoes the bot's own published
|
|
119
|
+
* audio back, so frames here are always from OTHER participants.
|
|
120
|
+
*
|
|
121
|
+
* @param cb Invoked with each inbound, per-participant audio frame.
|
|
122
|
+
*/
|
|
123
|
+
onAudioTrack(cb: (frame: LiveKitAudioFrame) => void): void;
|
|
124
|
+
/**
|
|
125
|
+
* Publishes one raw video frame on the bot's camera track. LiveKit does full video; the realtime
|
|
126
|
+
* models light audio first, so this is wired but typically unused until a model emits video.
|
|
127
|
+
*
|
|
128
|
+
* @param frame The encoded/raw video frame bytes to publish.
|
|
129
|
+
*/
|
|
130
|
+
publishVideoFrame(frame: ArrayBuffer): void;
|
|
131
|
+
/**
|
|
132
|
+
* Publishes one raw screen-share frame on the bot's screen track (e.g. a Remote Browser channel's
|
|
133
|
+
* viewport). LiveKit does full screen share.
|
|
134
|
+
*
|
|
135
|
+
* @param frame The encoded/raw screen frame bytes to publish.
|
|
136
|
+
*/
|
|
137
|
+
publishScreenFrame(frame: ArrayBuffer): void;
|
|
138
|
+
/**
|
|
139
|
+
* Registers a callback fired when a participant connects. "Latest handler wins."
|
|
140
|
+
*
|
|
141
|
+
* @param cb Invoked with the participant who joined.
|
|
142
|
+
*/
|
|
143
|
+
onParticipantJoin(cb: (participant: LiveKitParticipant) => void): void;
|
|
144
|
+
/**
|
|
145
|
+
* Registers a callback fired when a participant disconnects. "Latest handler wins."
|
|
146
|
+
*
|
|
147
|
+
* @param cb Invoked with the participant identity that left.
|
|
148
|
+
*/
|
|
149
|
+
onParticipantLeave(cb: (participantIdentity: string) => void): void;
|
|
150
|
+
/**
|
|
151
|
+
* Returns the current participant list (including the bot).
|
|
152
|
+
*
|
|
153
|
+
* @returns The current participants.
|
|
154
|
+
*/
|
|
155
|
+
getParticipants(): Promise<LiveKitParticipant[]>;
|
|
156
|
+
/**
|
|
157
|
+
* Sends a text message on the LiveKit data channel (reliable publish to all participants) — the
|
|
158
|
+
* room-native "chat".
|
|
159
|
+
*
|
|
160
|
+
* @param text The chat/data message text.
|
|
161
|
+
*/
|
|
162
|
+
sendDataMessage(text: string): Promise<void>;
|
|
163
|
+
/**
|
|
164
|
+
* Registers a callback fired when the room disconnects the bot (SFU closed / bot removed). "Latest
|
|
165
|
+
* handler wins."
|
|
166
|
+
*
|
|
167
|
+
* @param cb Invoked when the room has disconnected.
|
|
168
|
+
*/
|
|
169
|
+
onDisconnected(cb: () => void): void;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* A factory that constructs an {@link ILiveKitRoomSdk} for a session — the creation seam (mirroring
|
|
173
|
+
* Zoom's `ZoomMeetingSdkFactory`). Production supplies a factory that builds the real LiveKit adapter
|
|
174
|
+
* (`livekit-server-sdk` + a room client) from resolved config; tests supply one that returns a
|
|
175
|
+
* `FakeLiveKitRoomSdk`.
|
|
176
|
+
*
|
|
177
|
+
* @param config The resolved provider/session configuration (room URL, region, credential refs already
|
|
178
|
+
* resolved upstream into a signed token).
|
|
179
|
+
* @returns The LiveKit room SDK instance to drive the room with.
|
|
180
|
+
*/
|
|
181
|
+
export type LiveKitRoomSdkFactory = (config?: Record<string, unknown>) => ILiveKitRoomSdk;
|
|
182
|
+
//# sourceMappingURL=livekit-sdk.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"livekit-sdk.d.ts","sourceRoot":"","sources":["../src/livekit-sdk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH,0FAA0F;AAC1F,MAAM,MAAM,sBAAsB,GAAG,MAAM,GAAG,QAAQ,GAAG,aAAa,CAAC;AAEvE;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAC/B,oGAAoG;IACpG,QAAQ,EAAE,MAAM,CAAC;IACjB,6FAA6F;IAC7F,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oGAAoG;IACpG,IAAI,EAAE,sBAAsB,CAAC;IAC7B,sFAAsF;IACtF,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAC9B,0CAA0C;IAC1C,GAAG,EAAE,WAAW,CAAC;IACjB,6FAA6F;IAC7F,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kEAAkE;IAClE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,mGAAmG;AACnG,MAAM,WAAW,kBAAkB;IAC/B,2FAA2F;IAC3F,OAAO,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,cAAc,EAAE,MAAM,CAAC;CAC1B;AAED,uFAAuF;AACvF,MAAM,WAAW,oBAAoB;IACjC,6DAA6D;IAC7D,WAAW,EAAE,MAAM,CAAC;IACpB,iFAAiF;IACjF,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC5B;;;;;;OAMG;IACH,OAAO,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjE,4DAA4D;IAC5D,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B;;;;OAIG;IACH,iBAAiB,CAAC,GAAG,EAAE,WAAW,GAAG,IAAI,CAAC;IAE1C;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI,GAAG,IAAI,CAAC;IAE3D;;;;;OAKG;IACH,iBAAiB,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI,CAAC;IAE5C;;;;;OAKG;IACH,kBAAkB,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI,CAAC;IAE7C;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,EAAE,CAAC,WAAW,EAAE,kBAAkB,KAAK,IAAI,GAAG,IAAI,CAAC;IAEvE;;;;OAIG;IACH,kBAAkB,CAAC,EAAE,EAAE,CAAC,mBAAmB,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI,CAAC;IAEpE;;;;OAIG;IACH,eAAe,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAEjD;;;;;OAKG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7C;;;;;OAKG;IACH,cAAc,CAAC,EAAE,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;CACxC;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,qBAAqB,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,eAAe,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The injectable **LiveKit room SDK seam** — the minimal set of operations
|
|
3
|
+
* {@link import('./livekit-bridge.js').LiveKitBridge} needs from LiveKit, declared as an interface so the
|
|
4
|
+
* driver builds and unit-tests against an in-memory fake with **no network and no real LiveKit SDK**.
|
|
5
|
+
*
|
|
6
|
+
* ## LiveKit is the MJ-NATIVE room (self-hosted), not a 3rd-party platform
|
|
7
|
+
* Every OTHER bridge in this program connects **out** to a meeting hosted by someone else (Zoom,
|
|
8
|
+
* Teams, Meet, Webex, Slack, Discord) or to a telephony carrier (Twilio, Vonage, RingCentral). LiveKit
|
|
9
|
+
* is the opposite: it is an **open-source WebRTC SFU that MJ runs itself** — the "MJ-native room"
|
|
10
|
+
* (`/plans/realtime/realtime-bridges-architecture.md` §4c). The architecture deliberately treats a Zoom
|
|
11
|
+
* meeting, a Teams meeting, and an MJ-native LiveKit room **identically** — all are multi-party media
|
|
12
|
+
* transports — so LiveKit is "*another bridge, not a special build*." The only practical differences:
|
|
13
|
+
* - **We own the room.** No marketplace review, no per-platform bot-admission quirks; MJ mints the
|
|
14
|
+
* access token and the room exists because MJ stood up the SFU. An MJ-native multi-party experience
|
|
15
|
+
* (e.g. embedded in Explorer) is just *this* bridge.
|
|
16
|
+
* - **`connect(roomUrl, token)`** takes a LiveKit room URL + a signed access token (minted upstream by
|
|
17
|
+
* MJ's credential/token layer — never inline secrets), rather than a join URL we were handed.
|
|
18
|
+
*
|
|
19
|
+
* ## Production binding (TODO at deployment)
|
|
20
|
+
* In production this interface is bound to **`livekit-server-sdk`** (token minting / room admin) plus a
|
|
21
|
+
* **room client** (the Node WebRTC participant — e.g. `@livekit/rtc-node`) so the bot can publish/
|
|
22
|
+
* subscribe media. The named operations map to the SDK as follows:
|
|
23
|
+
* - {@link connect} / {@link disconnect} → the room client `connect()` / `disconnect()` lifecycle.
|
|
24
|
+
* - {@link publishAudioFrame} → publishing PCM on the bot's audio track (the agent's voice).
|
|
25
|
+
* - {@link onAudioTrack} → subscribing each remote participant's audio track; LiveKit delivers tracks
|
|
26
|
+
* **per participant**, which is the native source of speaker labels for diarization (no extra mixer).
|
|
27
|
+
* - {@link publishVideoFrame} / {@link publishScreenFrame} → publishing the bot's camera / screen-share
|
|
28
|
+
* tracks (LiveKit does full A/V/screen; the realtime models light audio first).
|
|
29
|
+
* - {@link onParticipantJoin} / {@link onParticipantLeave} / {@link getParticipants} → the room's
|
|
30
|
+
* `ParticipantConnected` / `ParticipantDisconnected` events + the participant list.
|
|
31
|
+
* - {@link sendDataMessage} → the LiveKit **data channel** (reliable data publish) — used for chat.
|
|
32
|
+
* - {@link onDisconnected} → the room `Disconnected` event (the SFU closed / the bot was removed).
|
|
33
|
+
*
|
|
34
|
+
* ## Echo / self-audio
|
|
35
|
+
* A LiveKit SFU **never delivers a participant its own published track back** — the bot does not hear
|
|
36
|
+
* its own voice, so no echo gate is needed here (documented in {@link onAudioTrack}). The bridge still
|
|
37
|
+
* flags its own track via {@link LiveKitParticipant.IsLocal} defensively.
|
|
38
|
+
*
|
|
39
|
+
* Binding the real SDK is a thin adapter that implements this interface; the driver and its tests do
|
|
40
|
+
* not change. **None of the SDK types leak into this package.**
|
|
41
|
+
*
|
|
42
|
+
* See `/plans/realtime/realtime-bridges-architecture.md` §4c and `/guides/REALTIME_BRIDGES_GUIDE.md`.
|
|
43
|
+
*/
|
|
44
|
+
export {};
|
|
45
|
+
//# sourceMappingURL=livekit-sdk.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"livekit-sdk.js","sourceRoot":"","sources":["../src/livekit-sdk.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG"}
|
package/package.json
CHANGED
|
@@ -1,10 +1,35 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@memberjunction/ai-bridge-livekit",
|
|
3
|
-
"
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
3
|
+
"type": "module",
|
|
4
|
+
"version": "5.41.0",
|
|
5
|
+
"description": "MemberJunction: LiveKit Realtime Bridge driver — the MJ-native multi-party room. Unlike the other bridges (which connect OUT to a 3rd-party meeting platform), LiveKit is a SELF-HOSTED WebRTC SFU that MJ runs itself. Connects the realtime agent engine to a LiveKit room as a bot participant (audio/video/screen in+out, per-participant diarization, data-channel chat) via an injectable LiveKit room SDK seam, and contributes a Meeting Controls facilitator channel.",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"files": [
|
|
9
|
+
"/dist"
|
|
10
|
+
],
|
|
11
|
+
"scripts": {
|
|
12
|
+
"start": "ts-node-dev src/index.ts",
|
|
13
|
+
"build": "tsc && tsc-alias -f",
|
|
14
|
+
"test": "vitest run",
|
|
15
|
+
"test:watch": "vitest"
|
|
16
|
+
},
|
|
17
|
+
"author": "MemberJunction.com",
|
|
18
|
+
"license": "ISC",
|
|
19
|
+
"dependencies": {
|
|
20
|
+
"@memberjunction/core": "5.41.0",
|
|
21
|
+
"@memberjunction/global": "5.41.0",
|
|
22
|
+
"@memberjunction/core-entities": "5.41.0",
|
|
23
|
+
"@memberjunction/ai-bridge-base": "5.41.0"
|
|
24
|
+
},
|
|
25
|
+
"devDependencies": {
|
|
26
|
+
"@types/node": "24.10.11",
|
|
27
|
+
"ts-node-dev": "^2.0.0",
|
|
28
|
+
"typescript": "^5.9.3",
|
|
29
|
+
"vitest": "^4.0.18"
|
|
30
|
+
},
|
|
31
|
+
"repository": {
|
|
32
|
+
"type": "git",
|
|
33
|
+
"url": "https://github.com/MemberJunction/MJ"
|
|
34
|
+
}
|
|
10
35
|
}
|