@pipecat-ai/moq-transport 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +24 -0
- package/README.md +125 -0
- package/dist/index.d.ts +156 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +574 -0
- package/dist/index.js.map +1 -0
- package/dist/index.module.js +557 -0
- package/dist/index.module.js.map +1 -0
- package/package.json +46 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,574 @@
|
|
|
1
|
+
var $9t2yL$moqjson = require("@moq/json");
|
|
2
|
+
var $9t2yL$moqnet = require("@moq/net");
|
|
3
|
+
var $9t2yL$moqpublish = require("@moq/publish");
|
|
4
|
+
var $9t2yL$moqsignals = require("@moq/signals");
|
|
5
|
+
var $9t2yL$moqwatch = require("@moq/watch");
|
|
6
|
+
var $9t2yL$pipecataiclientjs = require("@pipecat-ai/client-js");
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
function $parcel$exportWildcard(dest, source) {
|
|
10
|
+
Object.keys(source).forEach(function(key) {
|
|
11
|
+
if (key === 'default' || key === '__esModule' || Object.prototype.hasOwnProperty.call(dest, key)) {
|
|
12
|
+
return;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
Object.defineProperty(dest, key, {
|
|
16
|
+
enumerable: true,
|
|
17
|
+
get: function get() {
|
|
18
|
+
return source[key];
|
|
19
|
+
}
|
|
20
|
+
});
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
return dest;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function $parcel$export(e, n, v, s) {
|
|
27
|
+
Object.defineProperty(e, n, {get: v, set: s, enumerable: true, configurable: true});
|
|
28
|
+
}
|
|
29
|
+
var $c3a8005c91f7c1e1$exports = {};
|
|
30
|
+
|
|
31
|
+
$parcel$export($c3a8005c91f7c1e1$exports, "MoqTransport", () => $c3a8005c91f7c1e1$export$739cde4b077543d1);
|
|
32
|
+
/*
|
|
33
|
+
* Copyright (c) 2024-2026, Daily
|
|
34
|
+
*
|
|
35
|
+
* SPDX-License-Identifier: BSD-2-Clause
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
const $c3a8005c91f7c1e1$var$DEFAULT_NAMESPACE = "pipecat";
|
|
43
|
+
const $c3a8005c91f7c1e1$var$DEFAULT_CLIENT_ID = "client0";
|
|
44
|
+
const $c3a8005c91f7c1e1$var$DEFAULT_BOT_ID = "bot0";
|
|
45
|
+
const $c3a8005c91f7c1e1$var$DEFAULT_TRANSCRIPT_TRACK = "transcript.json.z";
|
|
46
|
+
// Bounded jitter buffer on the audio decoder. Lower = more interactive
|
|
47
|
+
// but more drops on bad networks. Matches the bot's audio_in_max_latency_ms
|
|
48
|
+
// in spirit (each side enforces its own deadline).
|
|
49
|
+
const $c3a8005c91f7c1e1$var$DEFAULT_AUDIO_LATENCY_MS = 80;
|
|
50
|
+
// Latency ceiling for buffered TTS playback (ms). A finite cap (vs
|
|
51
|
+
// uncapped) because the player's group/jitter buffer needs a concrete
|
|
52
|
+
// maximum span to retain before dropping. The bot paces its writes a
|
|
53
|
+
// little under this (~25s, see the Python transport's
|
|
54
|
+
// `audio_out_max_buffer_ms`) so the producer self-limits below this drop
|
|
55
|
+
// ceiling and the player never actually has to drop.
|
|
56
|
+
const $c3a8005c91f7c1e1$var$DEFAULT_AUDIO_BUFFER_MAX_MS = 30000;
|
|
57
|
+
// Sample rate we pin on the publish-side Opus encoder. Opus only supports
|
|
58
|
+
// {8, 12, 16, 24, 48} kHz; 48 kHz is the upstream default and what most
|
|
59
|
+
// browsers report as the mic's native rate. The bot reads this from our
|
|
60
|
+
// catalog and resamples to its `audio_in_sample_rate` after decode, so we
|
|
61
|
+
// don't need to match it exactly — but pinning it keeps the catalog
|
|
62
|
+
// unambiguous.
|
|
63
|
+
const $c3a8005c91f7c1e1$var$DEFAULT_AUDIO_SAMPLE_RATE = 48000;
|
|
64
|
+
function $c3a8005c91f7c1e1$var$applyDefaults(opts) {
|
|
65
|
+
return {
|
|
66
|
+
relayUrl: opts.relayUrl,
|
|
67
|
+
serverCertificateHashes: opts.serverCertificateHashes,
|
|
68
|
+
clientId: opts.clientId ?? $c3a8005c91f7c1e1$var$DEFAULT_CLIENT_ID,
|
|
69
|
+
botId: opts.botId ?? $c3a8005c91f7c1e1$var$DEFAULT_BOT_ID,
|
|
70
|
+
namespace: opts.namespace ?? $c3a8005c91f7c1e1$var$DEFAULT_NAMESPACE,
|
|
71
|
+
transcriptTrack: opts.transcriptTrack ?? $c3a8005c91f7c1e1$var$DEFAULT_TRANSCRIPT_TRACK,
|
|
72
|
+
audioLatencyMs: opts.audioLatencyMs ?? $c3a8005c91f7c1e1$var$DEFAULT_AUDIO_LATENCY_MS,
|
|
73
|
+
audioBufferMaxMs: opts.audioBufferMaxMs ?? $c3a8005c91f7c1e1$var$DEFAULT_AUDIO_BUFFER_MAX_MS,
|
|
74
|
+
audioSampleRate: opts.audioSampleRate ?? $c3a8005c91f7c1e1$var$DEFAULT_AUDIO_SAMPLE_RATE
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
function $c3a8005c91f7c1e1$var$isMoqStartResponse(value) {
|
|
78
|
+
return "moq" in value;
|
|
79
|
+
}
|
|
80
|
+
/** Decode the bot's base64 SHA-256 cert hash into the ArrayBuffer shape
|
|
81
|
+
* ``WebTransport.serverCertificateHashes`` expects. */ function $c3a8005c91f7c1e1$var$decodeCertHash(b64) {
|
|
82
|
+
const bin = atob(b64);
|
|
83
|
+
const bytes = new Uint8Array(bin.length);
|
|
84
|
+
for(let i = 0; i < bin.length; i++)bytes[i] = bin.charCodeAt(i);
|
|
85
|
+
return bytes.buffer;
|
|
86
|
+
}
|
|
87
|
+
class $c3a8005c91f7c1e1$export$739cde4b077543d1 extends (0, $9t2yL$pipecataiclientjs.Transport) {
|
|
88
|
+
constructor(options){
|
|
89
|
+
super();
|
|
90
|
+
// Connection + reactive root.
|
|
91
|
+
this._reload = null;
|
|
92
|
+
this._signals = null;
|
|
93
|
+
// Publish side (mic → bot).
|
|
94
|
+
this._publishBroadcast = null;
|
|
95
|
+
this._microphone = null;
|
|
96
|
+
// RTVI JSON side-channel from client → bot, symmetric with the bot's
|
|
97
|
+
// outbound `transcript` track. Published as a lossless `@moq/json`
|
|
98
|
+
// append-stream (compression on); `sendMessage` appends each RTVI
|
|
99
|
+
// message and the bot's `_forward_peer_transcript` on the Python side
|
|
100
|
+
// drains it. The stream Producer binds to a single track with no
|
|
101
|
+
// built-in fan-out, and `publishTrack` serves each subscriber its own
|
|
102
|
+
// track, so we hold one Producer per subscription (`_transcriptOut`)
|
|
103
|
+
// and replay the message log (`_transcriptLog`) into each — so a bot
|
|
104
|
+
// that subscribes late still gets every message, in order.
|
|
105
|
+
this._transcriptOut = null;
|
|
106
|
+
this._transcriptLog = null;
|
|
107
|
+
this._micEnabled = new (0, $9t2yL$moqsignals.Signal)(true);
|
|
108
|
+
this._micConstraints = new (0, $9t2yL$moqsignals.Signal)(undefined);
|
|
109
|
+
this._preferredMicId = new (0, $9t2yL$moqsignals.Signal)(undefined);
|
|
110
|
+
// Encoder sample rate is wired into the catalog the bot reads. We
|
|
111
|
+
// expose this as a Signal so `_connect` can update it from the resolved
|
|
112
|
+
// options at connect time.
|
|
113
|
+
this._audioSampleRate = new (0, $9t2yL$moqsignals.Signal)(undefined);
|
|
114
|
+
// Watch side (bot → playback). Buffered playback via @moq/watch:
|
|
115
|
+
// Broadcast -> Audio.Source -> Decoder -> Emitter, with `Sync.latencyMax`
|
|
116
|
+
// letting faster-than-real-time TTS build up instead of the player
|
|
117
|
+
// skipping ahead, and `reset()` flushing the buffer on interruption.
|
|
118
|
+
this._watchBroadcast = null;
|
|
119
|
+
this._sync = null;
|
|
120
|
+
this._audioSource = null;
|
|
121
|
+
this._audioDecoder = null;
|
|
122
|
+
this._audioEmitter = null;
|
|
123
|
+
this._moqOptions = $c3a8005c91f7c1e1$var$applyDefaults(options);
|
|
124
|
+
this._state = "disconnected";
|
|
125
|
+
}
|
|
126
|
+
// --------------------------------------------------------------------
|
|
127
|
+
// Lifecycle
|
|
128
|
+
// --------------------------------------------------------------------
|
|
129
|
+
initialize(options, messageHandler) {
|
|
130
|
+
this._options = options;
|
|
131
|
+
this._callbacks = options.callbacks ?? {};
|
|
132
|
+
this._onMessage = messageHandler;
|
|
133
|
+
this.state = "initialized";
|
|
134
|
+
}
|
|
135
|
+
async initDevices() {
|
|
136
|
+
// Spin up the Microphone with permission requested so labels are
|
|
137
|
+
// available for the picker before the user clicks Connect. Real
|
|
138
|
+
// capture constraints (incl. sample rate) get applied in `_connect`
|
|
139
|
+
// once we have the resolved options.
|
|
140
|
+
if (!this._microphone) this._microphone = new $9t2yL$moqpublish.Source.Microphone({
|
|
141
|
+
enabled: this._micEnabled,
|
|
142
|
+
constraints: this._micConstraints,
|
|
143
|
+
device: {
|
|
144
|
+
preferred: this._preferredMicId
|
|
145
|
+
}
|
|
146
|
+
});
|
|
147
|
+
this._microphone.device.requestPermission();
|
|
148
|
+
}
|
|
149
|
+
_validateConnectionParams(connectParams) {
|
|
150
|
+
if (connectParams === undefined || connectParams === null) return this._optionsAsInput();
|
|
151
|
+
if (typeof connectParams !== "object") throw new (0, $9t2yL$pipecataiclientjs.RTVIError)("MoqTransport connect params must be an object");
|
|
152
|
+
// The bot's `/start` response is `{ moq: {...} }`, not already-shaped
|
|
153
|
+
// `MoqTransportOptions` — unwrap and decode it here so apps can pass
|
|
154
|
+
// the raw response straight through with no transport-specific glue.
|
|
155
|
+
if ($c3a8005c91f7c1e1$var$isMoqStartResponse(connectParams)) {
|
|
156
|
+
const { moq: moq } = connectParams;
|
|
157
|
+
if (!moq) throw new (0, $9t2yL$pipecataiclientjs.RTVIError)("MoqTransport connect params did not include `moq` config (server may not be running with `-t moq`)");
|
|
158
|
+
return {
|
|
159
|
+
...this._optionsAsInput(),
|
|
160
|
+
relayUrl: moq.relayUrl,
|
|
161
|
+
clientId: moq.clientId,
|
|
162
|
+
botId: moq.botId,
|
|
163
|
+
namespace: moq.namespace,
|
|
164
|
+
transcriptTrack: moq.transcriptTrack,
|
|
165
|
+
serverCertificateHashes: moq.certHash ? [
|
|
166
|
+
{
|
|
167
|
+
algorithm: "sha-256",
|
|
168
|
+
value: $c3a8005c91f7c1e1$var$decodeCertHash(moq.certHash)
|
|
169
|
+
}
|
|
170
|
+
] : undefined
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
return {
|
|
174
|
+
...this._optionsAsInput(),
|
|
175
|
+
...connectParams
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
/** Return the resolved options re-shaped to the public input type
|
|
179
|
+
* (used when merging with connect-time params). */ _optionsAsInput() {
|
|
180
|
+
return {
|
|
181
|
+
...this._moqOptions
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
async _connect(connectParams) {
|
|
185
|
+
const merged = connectParams ? {
|
|
186
|
+
...this._moqOptions,
|
|
187
|
+
...$c3a8005c91f7c1e1$var$applyDefaults({
|
|
188
|
+
...this._moqOptions,
|
|
189
|
+
...connectParams
|
|
190
|
+
})
|
|
191
|
+
} : this._moqOptions;
|
|
192
|
+
if (!merged.relayUrl) throw new (0, $9t2yL$pipecataiclientjs.RTVIError)("MoqTransport requires `relayUrl`");
|
|
193
|
+
this._moqOptions = merged;
|
|
194
|
+
this.state = "connecting";
|
|
195
|
+
let url;
|
|
196
|
+
try {
|
|
197
|
+
url = new URL(merged.relayUrl);
|
|
198
|
+
} catch (err) {
|
|
199
|
+
this.state = "error";
|
|
200
|
+
throw new (0, $9t2yL$pipecataiclientjs.RTVIError)(`MoqTransport invalid relayUrl ${JSON.stringify(merged.relayUrl)}: ${err instanceof Error ? err.message : String(err)}`);
|
|
201
|
+
}
|
|
202
|
+
// @moq/net narrows `algorithm` to the literal `"sha-256"`, while
|
|
203
|
+
// the browser's `WebTransportHash` types it as `string`. Cast at
|
|
204
|
+
// the boundary; SHA-256 is the only algorithm WebTransport accepts.
|
|
205
|
+
const webtransport = merged.serverCertificateHashes ? {
|
|
206
|
+
serverCertificateHashes: merged.serverCertificateHashes.flatMap((h)=>h.value ? [
|
|
207
|
+
{
|
|
208
|
+
algorithm: "sha-256",
|
|
209
|
+
value: h.value
|
|
210
|
+
}
|
|
211
|
+
] : [])
|
|
212
|
+
} : undefined;
|
|
213
|
+
// Reload auto-reconnects on disconnect; Publish.Broadcast and
|
|
214
|
+
// Watch.Broadcast both react to its `established` signal.
|
|
215
|
+
this._reload = new $9t2yL$moqnet.Connection.Reload({
|
|
216
|
+
enabled: new (0, $9t2yL$moqsignals.Signal)(true),
|
|
217
|
+
url: new (0, $9t2yL$moqsignals.Signal)(url),
|
|
218
|
+
webtransport: webtransport
|
|
219
|
+
});
|
|
220
|
+
// One reactive root for status mirroring. Connect/Watch are
|
|
221
|
+
// self-driving via their own internal effects.
|
|
222
|
+
this._signals = new (0, $9t2yL$moqsignals.Effect)();
|
|
223
|
+
this._signals.run((eff)=>{
|
|
224
|
+
const status = eff.get(this._reload.status);
|
|
225
|
+
if (status === "connected") {
|
|
226
|
+
if (this._state === "connecting") this.state = "connected";
|
|
227
|
+
} else if (status === "connecting") this.state = "connecting";
|
|
228
|
+
else if (status === "disconnected") {
|
|
229
|
+
if (this._state !== "disconnecting" && this._state !== "disconnected") this.state = "disconnected";
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
const ourPath = $9t2yL$moqnet.Path.from(merged.namespace, merged.clientId);
|
|
233
|
+
const botPath = $9t2yL$moqnet.Path.from(merged.namespace, merged.botId);
|
|
234
|
+
// ----------------------------------------------------------------
|
|
235
|
+
// Publish — Microphone owns getUserMedia, device selection, and
|
|
236
|
+
// produces a reactive `source` signal that Publish.Broadcast
|
|
237
|
+
// consumes. Constraints (channelCount:1, sampleRate) and preferred
|
|
238
|
+
// deviceId flow through the signals we hold a reference to, so
|
|
239
|
+
// updateMic() re-routes audio without re-creating the broadcast.
|
|
240
|
+
// The encoder's sampleRate Signal pins what the catalog advertises.
|
|
241
|
+
// ----------------------------------------------------------------
|
|
242
|
+
this._micConstraints.set({
|
|
243
|
+
channelCount: {
|
|
244
|
+
exact: 1
|
|
245
|
+
},
|
|
246
|
+
sampleRate: {
|
|
247
|
+
ideal: merged.audioSampleRate
|
|
248
|
+
},
|
|
249
|
+
echoCancellation: true,
|
|
250
|
+
noiseSuppression: true
|
|
251
|
+
});
|
|
252
|
+
this._audioSampleRate.set(merged.audioSampleRate);
|
|
253
|
+
if (!this._microphone) this._microphone = new $9t2yL$moqpublish.Source.Microphone({
|
|
254
|
+
enabled: this._micEnabled,
|
|
255
|
+
constraints: this._micConstraints,
|
|
256
|
+
device: {
|
|
257
|
+
preferred: this._preferredMicId
|
|
258
|
+
}
|
|
259
|
+
});
|
|
260
|
+
this._publishBroadcast = new $9t2yL$moqpublish.Broadcast({
|
|
261
|
+
connection: this._reload.established,
|
|
262
|
+
enabled: new (0, $9t2yL$moqsignals.Signal)(true),
|
|
263
|
+
name: new (0, $9t2yL$moqsignals.Signal)(ourPath),
|
|
264
|
+
audio: {
|
|
265
|
+
source: this._microphone.source,
|
|
266
|
+
enabled: this._micEnabled,
|
|
267
|
+
sampleRate: this._audioSampleRate
|
|
268
|
+
}
|
|
269
|
+
});
|
|
270
|
+
// Client-side transcript: `sendMessage` appends each RTVI message to
|
|
271
|
+
// a lossless JSON append-stream. `publishTrack` hands us a fresh track
|
|
272
|
+
// per subscription (only one, in the normal single-bot flow), so we
|
|
273
|
+
// spin up a `Json.Stream.Producer` per subscriber and replay the
|
|
274
|
+
// message log into it — a bot that subscribes after we've already sent
|
|
275
|
+
// messages still gets the full log, in order. The bot subscribes to
|
|
276
|
+
// this track by its name — same convention as the bot's own transcript
|
|
277
|
+
// track — so no catalog entry is needed.
|
|
278
|
+
this._transcriptLog = [];
|
|
279
|
+
this._transcriptOut = new Set();
|
|
280
|
+
this._publishBroadcast.publishTrack(merged.transcriptTrack, (track, effect)=>{
|
|
281
|
+
const producer = new $9t2yL$moqjson.Stream.Producer(track, {
|
|
282
|
+
compression: true
|
|
283
|
+
});
|
|
284
|
+
for (const msg of this._transcriptLog ?? [])producer.append(msg);
|
|
285
|
+
this._transcriptOut?.add(producer);
|
|
286
|
+
effect.cleanup(()=>{
|
|
287
|
+
this._transcriptOut?.delete(producer);
|
|
288
|
+
producer.finish();
|
|
289
|
+
});
|
|
290
|
+
});
|
|
291
|
+
// Log the mic settings the browser actually granted, so we can see
|
|
292
|
+
// when a UA ignores the constraint (e.g. macOS often pins 48k
|
|
293
|
+
// regardless of `sampleRate.ideal`).
|
|
294
|
+
this._signals.run((eff)=>{
|
|
295
|
+
const src = eff.get(this._microphone.source);
|
|
296
|
+
if (!src) return;
|
|
297
|
+
const track = "track" in src ? src.track : src;
|
|
298
|
+
const s = track.getSettings();
|
|
299
|
+
console.log(`[MoqTransport] publish: requested=${merged.audioSampleRate}Hz, ` + `mic granted=${s.sampleRate}Hz, channels=${s.channelCount}, ` + `deviceId=${s.deviceId}`);
|
|
300
|
+
});
|
|
301
|
+
// ----------------------------------------------------------------
|
|
302
|
+
// Watch — Broadcast handles catalog discovery and rendition
|
|
303
|
+
// tracking; Audio.Source picks the active audio rendition;
|
|
304
|
+
// Audio.Decoder runs the WebCodecs decode loop and feeds an
|
|
305
|
+
// AudioWorklet ring buffer; Audio.Emitter routes that to the
|
|
306
|
+
// speakers. `Sync.latencyMax` lets faster-than-real-time TTS build
|
|
307
|
+
// up a buffer instead of the player skipping ahead; `reset()`
|
|
308
|
+
// (invoked on `user-started-speaking`) flushes it on interruption.
|
|
309
|
+
// We also tap Decoder.root → MediaStreamAudioDestinationNode so
|
|
310
|
+
// tracks().bot.audio returns a MediaStreamTrack.
|
|
311
|
+
//
|
|
312
|
+
// `catalogFormat: "hang"` is pinned because the pipecat bot publishes
|
|
313
|
+
// a hang-format catalog (camelCase `sampleRate`). The auto-detector
|
|
314
|
+
// would also land on "hang" here (no suffix on the broadcast name,
|
|
315
|
+
// and hang is the DEFAULT_FORMAT) but pinning removes ambiguity if
|
|
316
|
+
// a future publisher adds a suffix or a different default ships.
|
|
317
|
+
// ----------------------------------------------------------------
|
|
318
|
+
this._watchBroadcast = new $9t2yL$moqwatch.Broadcast({
|
|
319
|
+
connection: this._reload.established,
|
|
320
|
+
enabled: new (0, $9t2yL$moqsignals.Signal)(true),
|
|
321
|
+
name: new (0, $9t2yL$moqsignals.Signal)(botPath),
|
|
322
|
+
catalogFormat: new (0, $9t2yL$moqsignals.Signal)("hang")
|
|
323
|
+
});
|
|
324
|
+
// Latency range: floor = interactive jitter buffer (audioLatencyMs),
|
|
325
|
+
// ceiling = how much faster-than-real-time TTS the player will hold
|
|
326
|
+
// before dropping (audioBufferMaxMs). A number-typed max opens the
|
|
327
|
+
// buffer; "real-time" collapses to the floor (skip-ahead behavior).
|
|
328
|
+
const sync = new $9t2yL$moqwatch.Sync({
|
|
329
|
+
connection: this._reload.established,
|
|
330
|
+
latency: new (0, $9t2yL$moqsignals.Signal)({
|
|
331
|
+
min: merged.audioLatencyMs,
|
|
332
|
+
max: merged.audioBufferMaxMs === "real-time" ? "real-time" : merged.audioBufferMaxMs
|
|
333
|
+
})
|
|
334
|
+
});
|
|
335
|
+
this._sync = sync;
|
|
336
|
+
this._audioSource = new $9t2yL$moqwatch.Audio.Source(sync, {
|
|
337
|
+
broadcast: this._watchBroadcast
|
|
338
|
+
});
|
|
339
|
+
this._audioDecoder = new $9t2yL$moqwatch.Audio.Decoder(this._audioSource, {
|
|
340
|
+
enabled: new (0, $9t2yL$moqsignals.Signal)(true)
|
|
341
|
+
});
|
|
342
|
+
this._audioEmitter = new $9t2yL$moqwatch.Audio.Emitter(this._audioDecoder);
|
|
343
|
+
// Bridge Decoder.root (AudioNode) → MediaStreamTrack for tracks().bot.audio.
|
|
344
|
+
this._signals.run((eff)=>{
|
|
345
|
+
const ctx = eff.get(this._audioDecoder.context);
|
|
346
|
+
const root = eff.get(this._audioDecoder.root);
|
|
347
|
+
if (!ctx || !root) return;
|
|
348
|
+
const dest = ctx.createMediaStreamDestination();
|
|
349
|
+
root.connect(dest);
|
|
350
|
+
this._botAudioTrack = dest.stream.getAudioTracks()[0];
|
|
351
|
+
eff.cleanup(()=>{
|
|
352
|
+
try {
|
|
353
|
+
root.disconnect(dest);
|
|
354
|
+
} catch {
|
|
355
|
+
// best-effort.
|
|
356
|
+
}
|
|
357
|
+
this._botAudioTrack?.stop();
|
|
358
|
+
this._botAudioTrack = undefined;
|
|
359
|
+
});
|
|
360
|
+
});
|
|
361
|
+
// Log the bot's announced audio config + the AudioContext rate we
|
|
362
|
+
// ended up at. If these disagree the voice will play at the wrong
|
|
363
|
+
// pitch — the catalog drives the AudioContext, so a mismatch means
|
|
364
|
+
// either a parser bug or the bot is advertising a rate that doesn't
|
|
365
|
+
// match its actual Opus stream.
|
|
366
|
+
this._signals.run((eff)=>{
|
|
367
|
+
const config = eff.get(this._audioSource.config);
|
|
368
|
+
const ctx = eff.get(this._audioDecoder.context);
|
|
369
|
+
if (!config && !ctx) return;
|
|
370
|
+
console.log(`[MoqTransport] consume: catalog codec=${config?.codec}, ` + `catalog sampleRate=${config?.sampleRate}Hz, ` + `catalog channels=${config?.numberOfChannels}, ` + `AudioContext rate=${ctx?.sampleRate}Hz`);
|
|
371
|
+
});
|
|
372
|
+
// Transcript — a lossless JSON append-stream over a single track. We
|
|
373
|
+
// re-subscribe on each (re)connect; @moq/json's stream Consumer yields
|
|
374
|
+
// every appended record in order, losslessly.
|
|
375
|
+
this._signals.run((eff)=>{
|
|
376
|
+
const conn = eff.get(this._reload.established);
|
|
377
|
+
if (!conn) return;
|
|
378
|
+
const botBroadcast = conn.consume(botPath);
|
|
379
|
+
const track = botBroadcast.subscribe(merged.transcriptTrack, 0);
|
|
380
|
+
const consumer = new $9t2yL$moqjson.Stream.Consumer(track, {
|
|
381
|
+
compression: true
|
|
382
|
+
});
|
|
383
|
+
const ac = new AbortController();
|
|
384
|
+
this._drainTranscript(consumer, ac.signal).catch((e)=>{
|
|
385
|
+
if (!ac.signal.aborted) console.warn("MoqTransport bot-transcript loop:", e);
|
|
386
|
+
});
|
|
387
|
+
eff.cleanup(()=>{
|
|
388
|
+
ac.abort();
|
|
389
|
+
try {
|
|
390
|
+
track.close();
|
|
391
|
+
} catch {
|
|
392
|
+
// best-effort.
|
|
393
|
+
}
|
|
394
|
+
});
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
async _disconnect() {
|
|
398
|
+
if (this._state === "disconnected") return;
|
|
399
|
+
this.state = "disconnecting";
|
|
400
|
+
try {
|
|
401
|
+
this._audioEmitter?.close();
|
|
402
|
+
this._audioDecoder?.close();
|
|
403
|
+
this._audioSource?.close();
|
|
404
|
+
this._sync?.close();
|
|
405
|
+
this._watchBroadcast?.close();
|
|
406
|
+
// Each producer's own effect.cleanup (registered in the `publishTrack`
|
|
407
|
+
// callback above) already calls producer.finish() when this cascades
|
|
408
|
+
// through the broadcast's internal effect scope — no need to also
|
|
409
|
+
// finish them explicitly here.
|
|
410
|
+
this._publishBroadcast?.close();
|
|
411
|
+
this._microphone?.close();
|
|
412
|
+
this._signals?.close();
|
|
413
|
+
this._reload?.close();
|
|
414
|
+
} finally{
|
|
415
|
+
this._audioEmitter = null;
|
|
416
|
+
this._audioDecoder = null;
|
|
417
|
+
this._audioSource = null;
|
|
418
|
+
this._sync = null;
|
|
419
|
+
this._watchBroadcast = null;
|
|
420
|
+
this._transcriptOut = null;
|
|
421
|
+
this._transcriptLog = null;
|
|
422
|
+
this._publishBroadcast = null;
|
|
423
|
+
this._microphone = null;
|
|
424
|
+
this._signals = null;
|
|
425
|
+
this._reload = null;
|
|
426
|
+
this.state = "disconnected";
|
|
427
|
+
// Fire the SDK's disconnected callback. `state = "disconnected"`
|
|
428
|
+
// above already triggers `onTransportStateChanged`, but the SDK
|
|
429
|
+
// wires UI teardown (voice-ui-kit's connection-state hook, the
|
|
430
|
+
// `Disconnected` event, message-dispatcher cleanup) through
|
|
431
|
+
// `onDisconnected` — without this call the transport goes idle
|
|
432
|
+
// but the UI stays stuck in "connected" and never restores its
|
|
433
|
+
// "Connect" button. Matches the pattern used by
|
|
434
|
+
// small-webrtc-transport, openai-realtime-webrtc-transport, and
|
|
435
|
+
// gemini-live-websocket-transport.
|
|
436
|
+
this._callbacks?.onDisconnected?.();
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
sendReadyMessage() {
|
|
440
|
+
// Flip transport state to `ready` and dispatch the RTVI `client-ready`
|
|
441
|
+
// message on the client→bot transcript track so the bot's RTVIProcessor
|
|
442
|
+
// can negotiate protocol version. Mirrors the pattern used by
|
|
443
|
+
// small-webrtc-transport and daily-transport.
|
|
444
|
+
this.state = "ready";
|
|
445
|
+
this.sendMessage((0, $9t2yL$pipecataiclientjs.RTVIMessage).clientReady());
|
|
446
|
+
}
|
|
447
|
+
// --------------------------------------------------------------------
|
|
448
|
+
// State
|
|
449
|
+
// --------------------------------------------------------------------
|
|
450
|
+
get state() {
|
|
451
|
+
return this._state;
|
|
452
|
+
}
|
|
453
|
+
set state(next) {
|
|
454
|
+
if (this._state === next) return;
|
|
455
|
+
this._state = next;
|
|
456
|
+
this._callbacks?.onTransportStateChanged?.(next);
|
|
457
|
+
}
|
|
458
|
+
// --------------------------------------------------------------------
|
|
459
|
+
// Devices
|
|
460
|
+
// --------------------------------------------------------------------
|
|
461
|
+
async getAllMics() {
|
|
462
|
+
return this._microphone?.device.available.peek() ?? [];
|
|
463
|
+
}
|
|
464
|
+
async getAllCams() {
|
|
465
|
+
return [];
|
|
466
|
+
}
|
|
467
|
+
async getAllSpeakers() {
|
|
468
|
+
return [];
|
|
469
|
+
}
|
|
470
|
+
updateMic(micId) {
|
|
471
|
+
this._preferredMicId.set(micId);
|
|
472
|
+
}
|
|
473
|
+
updateCam(_camId) {}
|
|
474
|
+
updateSpeaker(_speakerId) {}
|
|
475
|
+
get selectedMic() {
|
|
476
|
+
const id = this._microphone?.device.active.peek();
|
|
477
|
+
if (!id) return {};
|
|
478
|
+
return this._microphone?.device.available.peek()?.find((d)=>d.deviceId === id) ?? {};
|
|
479
|
+
}
|
|
480
|
+
get selectedCam() {
|
|
481
|
+
return {};
|
|
482
|
+
}
|
|
483
|
+
get selectedSpeaker() {
|
|
484
|
+
return {};
|
|
485
|
+
}
|
|
486
|
+
enableMic(enable) {
|
|
487
|
+
this._micEnabled.set(enable);
|
|
488
|
+
}
|
|
489
|
+
enableCam(_enable) {}
|
|
490
|
+
enableScreenShare(_enable) {}
|
|
491
|
+
get isCamEnabled() {
|
|
492
|
+
return false;
|
|
493
|
+
}
|
|
494
|
+
get isMicEnabled() {
|
|
495
|
+
return this._micEnabled.get();
|
|
496
|
+
}
|
|
497
|
+
get isSharingScreen() {
|
|
498
|
+
return false;
|
|
499
|
+
}
|
|
500
|
+
// --------------------------------------------------------------------
|
|
501
|
+
// Messaging + tracks
|
|
502
|
+
// --------------------------------------------------------------------
|
|
503
|
+
sendMessage(message) {
|
|
504
|
+
if (!this._transcriptOut || !this._transcriptLog) {
|
|
505
|
+
console.warn("[MoqTransport] sendMessage called before connect; dropping", message);
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
// Append to the lossless stream. Keep a log too, so a bot that
|
|
509
|
+
// subscribes after this point is replayed every message in order
|
|
510
|
+
// (see the `publishTrack` serve callback above).
|
|
511
|
+
this._transcriptLog.push(message);
|
|
512
|
+
for (const producer of this._transcriptOut)producer.append(message);
|
|
513
|
+
}
|
|
514
|
+
tracks() {
|
|
515
|
+
const localSource = this._microphone?.source.peek();
|
|
516
|
+
const localAudio = localSource && "track" in localSource ? localSource.track : localSource;
|
|
517
|
+
return {
|
|
518
|
+
local: localAudio ? {
|
|
519
|
+
audio: localAudio
|
|
520
|
+
} : {},
|
|
521
|
+
bot: this._botAudioTrack ? {
|
|
522
|
+
audio: this._botAudioTrack
|
|
523
|
+
} : {}
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
// --------------------------------------------------------------------
|
|
527
|
+
// Internals — transcript
|
|
528
|
+
// --------------------------------------------------------------------
|
|
529
|
+
/** Pull RTVI messages off the @moq/json stream consumer and hand each
|
|
530
|
+
* one to `PipecatClient` via `_onMessage`. The handler resolves the
|
|
531
|
+
* SDK's connect promise when bot-ready arrives. */ async _drainTranscript(consumer, signal) {
|
|
532
|
+
for(;;){
|
|
533
|
+
const message = await consumer.next();
|
|
534
|
+
if (!message || signal.aborted) break;
|
|
535
|
+
if (typeof message !== "object") continue;
|
|
536
|
+
// Intra-transport shutdown signal from the bot: bot is about to
|
|
537
|
+
// close its MoQ session. Disable auto-reconnect, tear down the
|
|
538
|
+
// audio decoder + subscribers on our side while WebTransport is
|
|
539
|
+
// still healthy — without this the connection just vanishes
|
|
540
|
+
// underneath Chrome's WebCodecs decoder and can crash the
|
|
541
|
+
// renderer. Not an RTVI event; don't forward it to `_onMessage`.
|
|
542
|
+
const label = message.label;
|
|
543
|
+
const type = message.type;
|
|
544
|
+
if (label === "moq-transport" && type === "session-ending") {
|
|
545
|
+
console.log("[MoqTransport] bot signaled session-ending \u2014 closing cleanly");
|
|
546
|
+
// Fire-and-forget; we're returning from the drain loop right
|
|
547
|
+
// after, and _disconnect() aborts the consumer that owns us.
|
|
548
|
+
this._disconnect();
|
|
549
|
+
return;
|
|
550
|
+
}
|
|
551
|
+
// Interruption: the user started talking, so flush the
|
|
552
|
+
// already-buffered TTS for the bot's previous utterance instead
|
|
553
|
+
// of letting it drain. With the bot writing faster than
|
|
554
|
+
// real-time, that buffer can be seconds long.
|
|
555
|
+
if (type === "user-started-speaking") this._resetBotAudio();
|
|
556
|
+
this._onMessage?.(message);
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
/** Flush buffered bot audio and re-anchor playback at an utterance
|
|
560
|
+
* boundary. Called on interruption (`user-started-speaking`) so the
|
|
561
|
+
* already-buffered TTS for the previous utterance stops immediately
|
|
562
|
+
* instead of draining: re-anchor the sync reference and flush the
|
|
563
|
+
* decoder's ring buffer. */ _resetBotAudio() {
|
|
564
|
+
this._sync?.reset();
|
|
565
|
+
this._audioDecoder?.reset();
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
$c3a8005c91f7c1e1$export$739cde4b077543d1.SERVICE_NAME = "moq-transport";
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
$parcel$exportWildcard(module.exports, $c3a8005c91f7c1e1$exports);
|
|
572
|
+
|
|
573
|
+
|
|
574
|
+
//# sourceMappingURL=index.js.map
|