@livekit/agents 1.0.12 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio.cjs +88 -2
- package/dist/audio.cjs.map +1 -1
- package/dist/audio.d.cts +35 -0
- package/dist/audio.d.ts +35 -0
- package/dist/audio.d.ts.map +1 -1
- package/dist/audio.js +75 -1
- package/dist/audio.js.map +1 -1
- package/dist/stream/stream_channel.test.cjs +27 -0
- package/dist/stream/stream_channel.test.cjs.map +1 -1
- package/dist/stream/stream_channel.test.js +27 -0
- package/dist/stream/stream_channel.test.js.map +1 -1
- package/dist/voice/background_audio.cjs +326 -0
- package/dist/voice/background_audio.cjs.map +1 -0
- package/dist/voice/background_audio.d.cts +114 -0
- package/dist/voice/background_audio.d.ts +114 -0
- package/dist/voice/background_audio.d.ts.map +1 -0
- package/dist/voice/background_audio.js +301 -0
- package/dist/voice/background_audio.js.map +1 -0
- package/dist/voice/index.cjs +2 -0
- package/dist/voice/index.cjs.map +1 -1
- package/dist/voice/index.d.cts +1 -0
- package/dist/voice/index.d.ts +1 -0
- package/dist/voice/index.d.ts.map +1 -1
- package/dist/voice/index.js +1 -0
- package/dist/voice/index.js.map +1 -1
- package/package.json +7 -3
- package/resources/NOTICE +2 -0
- package/resources/keyboard-typing.ogg +0 -0
- package/resources/keyboard-typing2.ogg +0 -0
- package/resources/office-ambience.ogg +0 -0
- package/src/audio.ts +131 -0
- package/src/stream/stream_channel.test.ts +37 -0
- package/src/voice/background_audio.ts +451 -0
- package/src/voice/index.ts +1 -1
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
var background_audio_exports = {};
|
|
20
|
+
__export(background_audio_exports, {
|
|
21
|
+
BackgroundAudioPlayer: () => BackgroundAudioPlayer,
|
|
22
|
+
BuiltinAudioClip: () => BuiltinAudioClip,
|
|
23
|
+
PlayHandle: () => PlayHandle,
|
|
24
|
+
getBuiltinAudioPath: () => getBuiltinAudioPath,
|
|
25
|
+
isBuiltinAudioClip: () => isBuiltinAudioClip
|
|
26
|
+
});
|
|
27
|
+
module.exports = __toCommonJS(background_audio_exports);
|
|
28
|
+
var getImportMetaUrl = () => typeof document === "undefined" ? new URL(`file:${__filename}`).href : document.currentScript && document.currentScript.src || new URL("main.js", document.baseURI).href;
|
|
29
|
+
var importMetaUrl = /* @__PURE__ */ getImportMetaUrl();
|
|
30
|
+
var import_rtc_node = require("@livekit/rtc-node");
|
|
31
|
+
var import_node_path = require("node:path");
|
|
32
|
+
var import_node_url = require("node:url");
|
|
33
|
+
var import_audio = require("../audio.cjs");
|
|
34
|
+
var import_log = require("../log.cjs");
|
|
35
|
+
var import_utils = require("../utils.cjs");
|
|
36
|
+
var import_events = require("./events.cjs");
|
|
37
|
+
const TASK_TIMEOUT_MS = 500;
|
|
38
|
+
var BuiltinAudioClip = /* @__PURE__ */ ((BuiltinAudioClip2) => {
|
|
39
|
+
BuiltinAudioClip2["OFFICE_AMBIENCE"] = "office-ambience.ogg";
|
|
40
|
+
BuiltinAudioClip2["KEYBOARD_TYPING"] = "keyboard-typing.ogg";
|
|
41
|
+
BuiltinAudioClip2["KEYBOARD_TYPING2"] = "keyboard-typing2.ogg";
|
|
42
|
+
return BuiltinAudioClip2;
|
|
43
|
+
})(BuiltinAudioClip || {});
|
|
44
|
+
function isBuiltinAudioClip(source) {
|
|
45
|
+
return typeof source === "string" && Object.values(BuiltinAudioClip).includes(source);
|
|
46
|
+
}
|
|
47
|
+
function getBuiltinAudioPath(clip) {
|
|
48
|
+
const resourcesPath = (0, import_node_path.join)((0, import_node_path.dirname)((0, import_node_url.fileURLToPath)(importMetaUrl)), "../../resources");
|
|
49
|
+
return (0, import_node_path.join)(resourcesPath, clip);
|
|
50
|
+
}
|
|
51
|
+
const AUDIO_SOURCE_BUFFER_MS = 400;
|
|
52
|
+
class PlayHandle {
|
|
53
|
+
doneFuture = new import_utils.Future();
|
|
54
|
+
stopFuture = new import_utils.Future();
|
|
55
|
+
done() {
|
|
56
|
+
return this.doneFuture.done;
|
|
57
|
+
}
|
|
58
|
+
stop() {
|
|
59
|
+
if (this.done()) return;
|
|
60
|
+
if (!this.stopFuture.done) {
|
|
61
|
+
this.stopFuture.resolve();
|
|
62
|
+
}
|
|
63
|
+
this._markPlayoutDone();
|
|
64
|
+
}
|
|
65
|
+
async waitForPlayout() {
|
|
66
|
+
return this.doneFuture.await;
|
|
67
|
+
}
|
|
68
|
+
_markPlayoutDone() {
|
|
69
|
+
if (!this.doneFuture.done) {
|
|
70
|
+
this.doneFuture.resolve();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
class BackgroundAudioPlayer {
|
|
75
|
+
ambientSound;
|
|
76
|
+
thinkingSound;
|
|
77
|
+
playTasks = [];
|
|
78
|
+
audioSource = new import_rtc_node.AudioSource(48e3, 1, AUDIO_SOURCE_BUFFER_MS);
|
|
79
|
+
room;
|
|
80
|
+
agentSession;
|
|
81
|
+
publication;
|
|
82
|
+
trackPublishOptions;
|
|
83
|
+
republishTask;
|
|
84
|
+
ambientHandle;
|
|
85
|
+
thinkingHandle;
|
|
86
|
+
// TODO (Brian): add lock
|
|
87
|
+
#logger = (0, import_log.log)();
|
|
88
|
+
constructor(options) {
|
|
89
|
+
const { ambientSound, thinkingSound } = options || {};
|
|
90
|
+
this.ambientSound = ambientSound;
|
|
91
|
+
this.thinkingSound = thinkingSound;
|
|
92
|
+
if (this.thinkingSound) {
|
|
93
|
+
this.#logger.warn("thinkingSound is not yet supported");
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Select a sound from a list of background sound based on probability weights
|
|
98
|
+
* Return undefined if no sound is selected (when sum of probabilities < 1.0).
|
|
99
|
+
*/
|
|
100
|
+
selectSoundFromList(sounds) {
|
|
101
|
+
const totalProbability = sounds.reduce((sum, sound) => sum + (sound.probability ?? 1), 0);
|
|
102
|
+
if (totalProbability <= 0) {
|
|
103
|
+
return void 0;
|
|
104
|
+
}
|
|
105
|
+
if (totalProbability < 1 && Math.random() > totalProbability) {
|
|
106
|
+
return void 0;
|
|
107
|
+
}
|
|
108
|
+
const normalizeFactor = totalProbability <= 1 ? 1 : totalProbability;
|
|
109
|
+
const r = Math.random() * Math.min(totalProbability, 1);
|
|
110
|
+
let cumulative = 0;
|
|
111
|
+
for (const sound of sounds) {
|
|
112
|
+
const prob = sound.probability ?? 1;
|
|
113
|
+
if (prob <= 0) {
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
const normProb = prob / normalizeFactor;
|
|
117
|
+
cumulative += normProb;
|
|
118
|
+
if (r <= cumulative) {
|
|
119
|
+
return sound;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return sounds[sounds.length - 1];
|
|
123
|
+
}
|
|
124
|
+
normalizeSoundSource(source) {
|
|
125
|
+
if (source === void 0) {
|
|
126
|
+
return void 0;
|
|
127
|
+
}
|
|
128
|
+
if (typeof source === "string") {
|
|
129
|
+
return {
|
|
130
|
+
source: this.normalizeBuiltinAudio(source),
|
|
131
|
+
volume: 1
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
if (Array.isArray(source)) {
|
|
135
|
+
const selected = this.selectSoundFromList(source);
|
|
136
|
+
if (selected === void 0) {
|
|
137
|
+
return void 0;
|
|
138
|
+
}
|
|
139
|
+
return {
|
|
140
|
+
source: selected.source,
|
|
141
|
+
volume: selected.volume ?? 1
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
if (typeof source === "object" && "source" in source) {
|
|
145
|
+
return {
|
|
146
|
+
source: this.normalizeBuiltinAudio(source.source),
|
|
147
|
+
volume: source.volume ?? 1
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
return { source, volume: 1 };
|
|
151
|
+
}
|
|
152
|
+
normalizeBuiltinAudio(source) {
|
|
153
|
+
if (isBuiltinAudioClip(source)) {
|
|
154
|
+
return getBuiltinAudioPath(source);
|
|
155
|
+
}
|
|
156
|
+
return source;
|
|
157
|
+
}
|
|
158
|
+
play(audio, loop = false) {
|
|
159
|
+
const normalized = this.normalizeSoundSource(audio);
|
|
160
|
+
if (normalized === void 0) {
|
|
161
|
+
const handle = new PlayHandle();
|
|
162
|
+
handle._markPlayoutDone();
|
|
163
|
+
return handle;
|
|
164
|
+
}
|
|
165
|
+
const { source, volume } = normalized;
|
|
166
|
+
const playHandle = new PlayHandle();
|
|
167
|
+
const task = import_utils.Task.from(async ({ signal }) => {
|
|
168
|
+
await this.playTask({ playHandle, sound: source, volume, loop, signal });
|
|
169
|
+
});
|
|
170
|
+
task.addDoneCallback(() => {
|
|
171
|
+
playHandle._markPlayoutDone();
|
|
172
|
+
this.playTasks.splice(this.playTasks.indexOf(task), 1);
|
|
173
|
+
});
|
|
174
|
+
this.playTasks.push(task);
|
|
175
|
+
return playHandle;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Start the background audio system, publishing the audio track
|
|
179
|
+
* and beginning playback of any configured ambient sound.
|
|
180
|
+
*
|
|
181
|
+
* If `ambientSound` is provided (and contains file paths), they will loop
|
|
182
|
+
* automatically. If `ambientSound` contains AsyncIterators, they are assumed
|
|
183
|
+
* to be already infinite or looped.
|
|
184
|
+
*
|
|
185
|
+
* @param options - Options for starting background audio playback
|
|
186
|
+
*/
|
|
187
|
+
async start(options) {
|
|
188
|
+
var _a;
|
|
189
|
+
const { room, agentSession, trackPublishOptions } = options;
|
|
190
|
+
this.room = room;
|
|
191
|
+
this.agentSession = agentSession;
|
|
192
|
+
this.trackPublishOptions = trackPublishOptions;
|
|
193
|
+
await this.publishTrack();
|
|
194
|
+
this.room.on("reconnected", this.onReconnected);
|
|
195
|
+
(_a = this.agentSession) == null ? void 0 : _a.on(import_events.AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);
|
|
196
|
+
if (!this.ambientSound) return;
|
|
197
|
+
const normalized = this.normalizeSoundSource(this.ambientSound);
|
|
198
|
+
if (!normalized) return;
|
|
199
|
+
const { source, volume } = normalized;
|
|
200
|
+
const selectedSound = { source, volume, probability: 1 };
|
|
201
|
+
this.ambientHandle = this.play(selectedSound, typeof source === "string");
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Close and cleanup the background audio system
|
|
205
|
+
*/
|
|
206
|
+
async close() {
|
|
207
|
+
var _a, _b, _c, _d;
|
|
208
|
+
await (0, import_utils.cancelAndWait)(this.playTasks, TASK_TIMEOUT_MS);
|
|
209
|
+
if (this.republishTask) {
|
|
210
|
+
await this.republishTask.cancelAndWait(TASK_TIMEOUT_MS);
|
|
211
|
+
}
|
|
212
|
+
await this.audioSource.close();
|
|
213
|
+
(_a = this.agentSession) == null ? void 0 : _a.off(import_events.AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);
|
|
214
|
+
(_b = this.room) == null ? void 0 : _b.off("reconnected", this.onReconnected);
|
|
215
|
+
if (this.publication && this.publication.sid) {
|
|
216
|
+
await ((_d = (_c = this.room) == null ? void 0 : _c.localParticipant) == null ? void 0 : _d.unpublishTrack(this.publication.sid));
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Get the current track publication
|
|
221
|
+
*/
|
|
222
|
+
getPublication() {
|
|
223
|
+
return this.publication;
|
|
224
|
+
}
|
|
225
|
+
async publishTrack() {
|
|
226
|
+
var _a;
|
|
227
|
+
if (this.publication !== void 0) {
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
const track = import_rtc_node.LocalAudioTrack.createAudioTrack("background_audio", this.audioSource);
|
|
231
|
+
if (((_a = this.room) == null ? void 0 : _a.localParticipant) === void 0) {
|
|
232
|
+
throw new Error("Local participant not available");
|
|
233
|
+
}
|
|
234
|
+
const publication = await this.room.localParticipant.publishTrack(
|
|
235
|
+
track,
|
|
236
|
+
this.trackPublishOptions ?? new import_rtc_node.TrackPublishOptions()
|
|
237
|
+
);
|
|
238
|
+
this.publication = publication;
|
|
239
|
+
this.#logger.debug(`Background audio track published: ${this.publication.sid}`);
|
|
240
|
+
}
|
|
241
|
+
onReconnected = () => {
|
|
242
|
+
if (this.republishTask) {
|
|
243
|
+
this.republishTask.cancel();
|
|
244
|
+
}
|
|
245
|
+
this.publication = void 0;
|
|
246
|
+
this.republishTask = import_utils.Task.from(async () => {
|
|
247
|
+
await this.republishTrackTask();
|
|
248
|
+
});
|
|
249
|
+
};
|
|
250
|
+
async republishTrackTask() {
|
|
251
|
+
await this.publishTrack();
|
|
252
|
+
}
|
|
253
|
+
onAgentStateChanged = (ev) => {
|
|
254
|
+
var _a;
|
|
255
|
+
if (!this.thinkingSound) {
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
if (ev.newState === "thinking") {
|
|
259
|
+
if (this.thinkingHandle && !this.thinkingHandle.done()) {
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
} else {
|
|
263
|
+
(_a = this.thinkingHandle) == null ? void 0 : _a.stop();
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
async playTask({
|
|
267
|
+
playHandle,
|
|
268
|
+
sound,
|
|
269
|
+
volume,
|
|
270
|
+
loop,
|
|
271
|
+
signal
|
|
272
|
+
}) {
|
|
273
|
+
if (isBuiltinAudioClip(sound)) {
|
|
274
|
+
sound = getBuiltinAudioPath(sound);
|
|
275
|
+
}
|
|
276
|
+
if (typeof sound === "string") {
|
|
277
|
+
sound = loop ? (0, import_audio.loopAudioFramesFromFile)(sound, { abortSignal: signal }) : (0, import_audio.audioFramesFromFile)(sound, { abortSignal: signal });
|
|
278
|
+
}
|
|
279
|
+
try {
|
|
280
|
+
for await (const frame of sound) {
|
|
281
|
+
if (signal.aborted || playHandle.done()) break;
|
|
282
|
+
let processedFrame;
|
|
283
|
+
if (volume !== 1) {
|
|
284
|
+
const int16Data = new Int16Array(
|
|
285
|
+
frame.data.buffer,
|
|
286
|
+
frame.data.byteOffset,
|
|
287
|
+
frame.data.byteLength / 2
|
|
288
|
+
);
|
|
289
|
+
const float32Data = new Float32Array(int16Data.length);
|
|
290
|
+
for (let i = 0; i < int16Data.length; i++) {
|
|
291
|
+
float32Data[i] = int16Data[i];
|
|
292
|
+
}
|
|
293
|
+
const volumeFactor = 10 ** Math.log10(volume);
|
|
294
|
+
for (let i = 0; i < float32Data.length; i++) {
|
|
295
|
+
float32Data[i] *= volumeFactor;
|
|
296
|
+
}
|
|
297
|
+
const outputData = new Int16Array(float32Data.length);
|
|
298
|
+
for (let i = 0; i < float32Data.length; i++) {
|
|
299
|
+
const clipped = Math.max(-32768, Math.min(32767, float32Data[i]));
|
|
300
|
+
outputData[i] = Math.round(clipped);
|
|
301
|
+
}
|
|
302
|
+
processedFrame = new import_rtc_node.AudioFrame(
|
|
303
|
+
outputData,
|
|
304
|
+
frame.sampleRate,
|
|
305
|
+
frame.channels,
|
|
306
|
+
frame.samplesPerChannel
|
|
307
|
+
);
|
|
308
|
+
} else {
|
|
309
|
+
processedFrame = frame;
|
|
310
|
+
}
|
|
311
|
+
await this.audioSource.captureFrame(processedFrame);
|
|
312
|
+
}
|
|
313
|
+
} finally {
|
|
314
|
+
playHandle._markPlayoutDone();
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
319
|
+
0 && (module.exports = {
|
|
320
|
+
BackgroundAudioPlayer,
|
|
321
|
+
BuiltinAudioClip,
|
|
322
|
+
PlayHandle,
|
|
323
|
+
getBuiltinAudioPath,
|
|
324
|
+
isBuiltinAudioClip
|
|
325
|
+
});
|
|
326
|
+
//# sourceMappingURL=background_audio.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/voice/background_audio.ts","../../../node_modules/.pnpm/tsup@8.4.0_@microsoft+api-extractor@7.43.7_@types+node@22.15.30__postcss@8.4.38_tsx@4.20.4_typescript@5.4.5/node_modules/tsup/assets/cjs_shims.js"],"sourcesContent":["// SPDX-FileCopyrightText: 2025 LiveKit, Inc.\n//\n// SPDX-License-Identifier: Apache-2.0\nimport {\n AudioFrame,\n AudioSource,\n LocalAudioTrack,\n type LocalTrackPublication,\n type Room,\n TrackPublishOptions,\n} from '@livekit/rtc-node';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath } from 'node:url';\nimport { audioFramesFromFile, loopAudioFramesFromFile } from '../audio.js';\nimport { log } from '../log.js';\nimport { Future, Task, cancelAndWait } from '../utils.js';\nimport type { AgentSession } from './agent_session.js';\nimport { AgentSessionEventTypes, type AgentStateChangedEvent } from './events.js';\n\nconst TASK_TIMEOUT_MS = 500;\n\nexport enum BuiltinAudioClip {\n OFFICE_AMBIENCE = 'office-ambience.ogg',\n KEYBOARD_TYPING = 'keyboard-typing.ogg',\n KEYBOARD_TYPING2 = 'keyboard-typing2.ogg',\n}\n\nexport function isBuiltinAudioClip(\n source: AudioSourceType | AudioConfig | AudioConfig[],\n): source is BuiltinAudioClip {\n return (\n typeof source === 'string' &&\n Object.values(BuiltinAudioClip).includes(source as BuiltinAudioClip)\n );\n}\n\nexport function getBuiltinAudioPath(clip: BuiltinAudioClip): string {\n const resourcesPath = join(dirname(fileURLToPath(import.meta.url)), '../../resources');\n return join(resourcesPath, clip);\n}\n\nexport type AudioSourceType = string | BuiltinAudioClip | AsyncIterable<AudioFrame>;\n\nexport interface AudioConfig {\n source: AudioSourceType;\n volume?: number;\n probability?: number;\n}\n\nexport interface BackgroundAudioPlayerOptions {\n /**\n * Ambient sound to play continuously in the background.\n * Can be a file path, BuiltinAudioClip, or AudioConfig.\n * File paths will be looped automatically.\n */\n ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];\n\n /**\n * Sound to play when the agent is thinking.\n * TODO (Brian): Implement thinking sound when AudioMixer becomes available\n */\n thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];\n\n /**\n * Stream timeout in milliseconds\n * @defaultValue 200\n */\n streamTimeoutMs?: number;\n}\n\nexport interface BackgroundAudioStartOptions {\n room: Room;\n agentSession?: AgentSession;\n trackPublishOptions?: TrackPublishOptions;\n}\n\n// Queue size for AudioSource buffer (400ms)\n// Kept small to avoid abrupt cutoffs when removing sounds\nconst AUDIO_SOURCE_BUFFER_MS = 400;\n\nexport class PlayHandle {\n private doneFuture = new Future<void>();\n private stopFuture = new Future<void>();\n\n done(): boolean {\n return this.doneFuture.done;\n }\n\n stop(): void {\n if (this.done()) return;\n\n if (!this.stopFuture.done) {\n this.stopFuture.resolve();\n }\n\n this._markPlayoutDone();\n }\n\n async waitForPlayout(): Promise<void> {\n return this.doneFuture.await;\n }\n\n _markPlayoutDone(): void {\n if (!this.doneFuture.done) {\n this.doneFuture.resolve();\n }\n }\n}\n\n/**\n * Manages background audio playback for LiveKit agent sessions\n *\n * This class handles playing ambient sounds and manages audio track publishing.\n * It supports:\n * - Continuous ambient sound playback with looping\n * - Volume control and probability-based sound selection\n * - Integration with LiveKit rooms and agent sessions\n *\n * Note: Thinking sound not yet supported\n *\n * @example\n * ```typescript\n * const player = new BackgroundAudioPlayer({\n * ambientSound: { source: BuiltinAudioClip.OFFICE_AMBIENCE, volume: 0.8 },\n * });\n *\n * await player.start({ room, agentSession });\n * ```\n */\nexport class BackgroundAudioPlayer {\n private ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];\n private thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];\n\n private playTasks: Task<void>[] = [];\n private audioSource = new AudioSource(48000, 1, AUDIO_SOURCE_BUFFER_MS);\n\n private room?: Room;\n private agentSession?: AgentSession;\n private publication?: LocalTrackPublication;\n private trackPublishOptions?: TrackPublishOptions;\n private republishTask?: Task<void>;\n\n private ambientHandle?: PlayHandle;\n private thinkingHandle?: PlayHandle;\n\n // TODO (Brian): add lock\n\n #logger = log();\n\n constructor(options?: BackgroundAudioPlayerOptions) {\n const { ambientSound, thinkingSound } = options || {};\n\n this.ambientSound = ambientSound;\n this.thinkingSound = thinkingSound;\n\n if (this.thinkingSound) {\n this.#logger.warn('thinkingSound is not yet supported');\n // TODO: Implement thinking sound when AudioMixer becomes available\n }\n }\n\n /**\n * Select a sound from a list of background sound based on probability weights\n * Return undefined if no sound is selected (when sum of probabilities < 1.0).\n */\n private selectSoundFromList(sounds: AudioConfig[]): AudioConfig | undefined {\n const totalProbability = sounds.reduce((sum, sound) => sum + (sound.probability ?? 1.0), 0);\n\n if (totalProbability <= 0) {\n return undefined;\n }\n\n if (totalProbability < 1.0 && Math.random() > totalProbability) {\n return undefined;\n }\n\n const normalizeFactor = totalProbability <= 1.0 ? 1.0 : totalProbability;\n const r = Math.random() * Math.min(totalProbability, 1.0);\n let cumulative = 0.0;\n\n for (const sound of sounds) {\n const prob = sound.probability ?? 1.0;\n if (prob <= 0) {\n continue;\n }\n\n const normProb = prob / normalizeFactor;\n cumulative += normProb;\n\n if (r <= cumulative) {\n return sound;\n }\n }\n\n return sounds[sounds.length - 1];\n }\n\n private normalizeSoundSource(\n source?: AudioSourceType | AudioConfig | AudioConfig[],\n ): { source: AudioSourceType; volume: number } | undefined {\n if (source === undefined) {\n return undefined;\n }\n\n if (typeof source === 'string') {\n return {\n source: this.normalizeBuiltinAudio(source),\n volume: 1.0,\n };\n }\n\n if (Array.isArray(source)) {\n const selected = this.selectSoundFromList(source);\n if (selected === undefined) {\n return undefined;\n }\n\n return {\n source: selected.source,\n volume: selected.volume ?? 1.0,\n };\n }\n\n if (typeof source === 'object' && 'source' in source) {\n return {\n source: this.normalizeBuiltinAudio(source.source),\n volume: source.volume ?? 1.0,\n };\n }\n\n return { source, volume: 1.0 };\n }\n\n private normalizeBuiltinAudio(source: AudioSourceType): AudioSourceType {\n if (isBuiltinAudioClip(source)) {\n return getBuiltinAudioPath(source);\n }\n return source;\n }\n\n play(audio: AudioSourceType | AudioConfig | AudioConfig[], loop = false): PlayHandle {\n const normalized = this.normalizeSoundSource(audio);\n if (normalized === undefined) {\n const handle = new PlayHandle();\n handle._markPlayoutDone();\n return handle;\n }\n\n const { source, volume } = normalized;\n const playHandle = new PlayHandle();\n\n const task = Task.from(async ({ signal }) => {\n await this.playTask({ playHandle, sound: source, volume, loop, signal });\n });\n\n task.addDoneCallback(() => {\n playHandle._markPlayoutDone();\n this.playTasks.splice(this.playTasks.indexOf(task), 1);\n });\n\n this.playTasks.push(task);\n return playHandle;\n }\n\n /**\n * Start the background audio system, publishing the audio track\n * and beginning playback of any configured ambient sound.\n *\n * If `ambientSound` is provided (and contains file paths), they will loop\n * automatically. If `ambientSound` contains AsyncIterators, they are assumed\n * to be already infinite or looped.\n *\n * @param options - Options for starting background audio playback\n */\n async start(options: BackgroundAudioStartOptions): Promise<void> {\n const { room, agentSession, trackPublishOptions } = options;\n this.room = room;\n this.agentSession = agentSession;\n this.trackPublishOptions = trackPublishOptions;\n\n await this.publishTrack();\n\n // TODO (Brian): check job context is not fake\n\n // TODO (Brian): start audio mixer task\n this.room.on('reconnected', this.onReconnected);\n\n this.agentSession?.on(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);\n\n if (!this.ambientSound) return;\n\n const normalized = this.normalizeSoundSource(this.ambientSound);\n if (!normalized) return;\n\n const { source, volume } = normalized;\n const selectedSound: AudioConfig = { source, volume, probability: 1.0 };\n this.ambientHandle = this.play(selectedSound, typeof source === 'string');\n }\n\n /**\n * Close and cleanup the background audio system\n */\n async close(): Promise<void> {\n await cancelAndWait(this.playTasks, TASK_TIMEOUT_MS);\n\n if (this.republishTask) {\n await this.republishTask.cancelAndWait(TASK_TIMEOUT_MS);\n }\n\n // TODO (Brian): cancel audio mixer task and close audio mixer\n\n await this.audioSource.close();\n\n this.agentSession?.off(AgentSessionEventTypes.AgentStateChanged, this.onAgentStateChanged);\n this.room?.off('reconnected', this.onReconnected);\n\n if (this.publication && this.publication.sid) {\n await this.room?.localParticipant?.unpublishTrack(this.publication.sid);\n }\n }\n\n /**\n * Get the current track publication\n */\n getPublication(): LocalTrackPublication | undefined {\n return this.publication;\n }\n\n private async publishTrack(): Promise<void> {\n if (this.publication !== undefined) {\n return;\n }\n\n const track = LocalAudioTrack.createAudioTrack('background_audio', this.audioSource);\n\n if (this.room?.localParticipant === undefined) {\n throw new Error('Local participant not available');\n }\n\n const publication = await this.room.localParticipant.publishTrack(\n track,\n this.trackPublishOptions ?? new TrackPublishOptions(),\n );\n\n this.publication = publication;\n this.#logger.debug(`Background audio track published: ${this.publication.sid}`);\n }\n\n private onReconnected = (): void => {\n if (this.republishTask) {\n this.republishTask.cancel();\n }\n\n this.publication = undefined;\n this.republishTask = Task.from(async () => {\n await this.republishTrackTask();\n });\n };\n\n private async republishTrackTask(): Promise<void> {\n // TODO (Brian): add lock protection when implementing lock\n await this.publishTrack();\n }\n\n private onAgentStateChanged = (ev: AgentStateChangedEvent): void => {\n if (!this.thinkingSound) {\n return;\n }\n\n if (ev.newState === 'thinking') {\n if (this.thinkingHandle && !this.thinkingHandle.done()) {\n return;\n }\n\n // TODO (Brian): play thinking sound and assign to thinkingHandle\n } else {\n this.thinkingHandle?.stop();\n }\n };\n\n private async playTask({\n playHandle,\n sound,\n volume,\n loop,\n signal,\n }: {\n playHandle: PlayHandle;\n sound: AudioSourceType;\n volume: number;\n loop: boolean;\n signal: AbortSignal;\n }): Promise<void> {\n if (isBuiltinAudioClip(sound)) {\n sound = getBuiltinAudioPath(sound);\n }\n\n if (typeof sound === 'string') {\n sound = loop\n ? loopAudioFramesFromFile(sound, { abortSignal: signal })\n : audioFramesFromFile(sound, { abortSignal: signal });\n }\n\n try {\n for await (const frame of sound) {\n if (signal.aborted || playHandle.done()) break;\n\n let processedFrame: AudioFrame;\n\n if (volume !== 1.0) {\n const int16Data = new Int16Array(\n frame.data.buffer,\n frame.data.byteOffset,\n frame.data.byteLength / 2,\n );\n const float32Data = new Float32Array(int16Data.length);\n\n for (let i = 0; i < int16Data.length; i++) {\n float32Data[i] = int16Data[i]!;\n }\n\n const volumeFactor = 10 ** Math.log10(volume);\n for (let i = 0; i < float32Data.length; i++) {\n float32Data[i]! *= volumeFactor;\n }\n\n const outputData = new Int16Array(float32Data.length);\n for (let i = 0; i < float32Data.length; i++) {\n const clipped = Math.max(-32768, Math.min(32767, float32Data[i]!));\n outputData[i] = Math.round(clipped);\n }\n\n processedFrame = new AudioFrame(\n outputData,\n frame.sampleRate,\n frame.channels,\n frame.samplesPerChannel,\n );\n } else {\n processedFrame = frame;\n }\n\n // TODO (Brian): use AudioMixer to add/remove frame streams\n await this.audioSource.captureFrame(processedFrame);\n }\n } finally {\n // TODO: the waitForPlayout() may be innaccurate by 400ms\n playHandle._markPlayoutDone();\n }\n }\n}\n","// Shim globals in cjs bundle\n// There's a weird bug that esbuild will always inject importMetaUrl\n// if we export it as `const importMetaUrl = ... __filename ...`\n// But using a function will not cause this issue\n\nconst getImportMetaUrl = () =>\n typeof document === 'undefined'\n ? new URL(`file:${__filename}`).href\n : (document.currentScript && document.currentScript.src) ||\n new URL('main.js', document.baseURI).href\n\nexport const importMetaUrl = /* @__PURE__ */ getImportMetaUrl()\n"],"mappings":";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;ACKA,IAAM,mBAAmB,MACvB,OAAO,aAAa,cAChB,IAAI,IAAI,QAAQ,UAAU,EAAE,EAAE,OAC7B,SAAS,iBAAiB,SAAS,cAAc,OAClD,IAAI,IAAI,WAAW,SAAS,OAAO,EAAE;AAEpC,IAAM,gBAAgC,iCAAiB;ADR9D,sBAOO;AACP,uBAA8B;AAC9B,sBAA8B;AAC9B,mBAA6D;AAC7D,iBAAoB;AACpB,mBAA4C;AAE5C,oBAAoE;AAEpE,MAAM,kBAAkB;AAEjB,IAAK,mBAAL,kBAAKA,sBAAL;AACL,EAAAA,kBAAA,qBAAkB;AAClB,EAAAA,kBAAA,qBAAkB;AAClB,EAAAA,kBAAA,sBAAmB;AAHT,SAAAA;AAAA,GAAA;AAML,SAAS,mBACd,QAC4B;AAC5B,SACE,OAAO,WAAW,YAClB,OAAO,OAAO,gBAAgB,EAAE,SAAS,MAA0B;AAEvE;AAEO,SAAS,oBAAoB,MAAgC;AAClE,QAAM,oBAAgB,2BAAK,8BAAQ,+BAAc,aAAe,CAAC,GAAG,iBAAiB;AACrF,aAAO,uBAAK,eAAe,IAAI;AACjC;AAuCA,MAAM,yBAAyB;AAExB,MAAM,WAAW;AAAA,EACd,aAAa,IAAI,oBAAa;AAAA,EAC9B,aAAa,IAAI,oBAAa;AAAA,EAEtC,OAAgB;AACd,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,OAAa;AACX,QAAI,KAAK,KAAK,EAAG;AAEjB,QAAI,CAAC,KAAK,WAAW,MAAM;AACzB,WAAK,WAAW,QAAQ;AAAA,IAC1B;AAEA,SAAK,iBAAiB;AAAA,EACxB;AAAA,EAEA,MAAM,iBAAgC;AACpC,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,mBAAyB;AACvB,QAAI,CAAC,KAAK,WAAW,MAAM;AACzB,WAAK,WAAW,QAAQ;AAAA,IAC1B;AAAA,EACF;AACF;AAsBO,MAAM,sBAAsB;AAAA,EACzB;AAAA,EACA;AAAA,EAEA,YAA0B,CAAC;AAAA,EAC3B,cAAc,IAAI,4BAAY,MAAO,GAAG,sBAAsB;AAAA,EAE9D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEA;AAAA,EACA;AAAA;AAAA,EAIR,cAAU,gBAAI;AAAA,EAEd,YAAY,SAAwC;AAClD,UAAM,EAAE,cAAc,cAAc,IAAI,WAAW,CAAC;AAEpD,SAAK,eAAe;AACpB,SAAK,gBAAgB;AAErB,QAAI,KAAK,eAAe;AACtB,WAAK,QAAQ,KAAK,oCAAoC;AAAA,IAExD;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMQ,oBAAoB,QAAgD;AAC1E,UAAM,mBAAmB,OAAO,OAAO,CAAC,KAAK,UAAU,OAAO,MAAM,eAAe,IAAM,CAAC;AAE1F,QAAI,oBAAoB,GAAG;AACzB,aAAO;AAAA,IACT;AAEA,QAAI,mBAAmB,KAAO,KAAK,OAAO,IAAI,kBAAkB;AAC9D,aAAO;AAAA,IACT;AAEA,UAAM,kBAAkB,oBAAoB,IAAM,IAAM;AACxD,UAAM,IAAI,KAAK,OAAO,IAAI,KAAK,IAAI,kBAAkB,CAAG;AACxD,QAAI,aAAa;AAEjB,eAAW,SAAS,QAAQ;AAC1B,YAAM,OAAO,MAAM,eAAe;AAClC,UAAI,QAAQ,GAAG;AACb;AAAA,MACF;AAEA,YAAM,WAAW,OAAO;AACxB,oBAAc;AAEd,UAAI,KAAK,YAAY;AACnB,eAAO;AAAA,MACT;AAAA,IACF;AAEA,WAAO,OAAO,OAAO,SAAS,CAAC;AAAA,EACjC;AAAA,EAEQ,qBACN,QACyD;AACzD,QAAI,WAAW,QAAW;AACxB,aAAO;AAAA,IACT;AAEA,QAAI,OAAO,WAAW,UAAU;AAC9B,aAAO;AAAA,QACL,QAAQ,KAAK,sBAAsB,MAAM;AAAA,QACzC,QAAQ;AAAA,MACV;AAAA,IACF;AAEA,QAAI,MAAM,QAAQ,MAAM,GAAG;AACzB,YAAM,WAAW,KAAK,oBAAoB,MAAM;AAChD,UAAI,aAAa,QAAW;AAC1B,eAAO;AAAA,MACT;AAEA,aAAO;AAAA,QACL,QAAQ,SAAS;AAAA,QACjB,QAAQ,SAAS,UAAU;AAAA,MAC7B;AAAA,IACF;AAEA,QAAI,OAAO,WAAW,YAAY,YAAY,QAAQ;AACpD,aAAO;AAAA,QACL,QAAQ,KAAK,sBAAsB,OAAO,MAAM;AAAA,QAChD,QAAQ,OAAO,UAAU;AAAA,MAC3B;AAAA,IACF;AAEA,WAAO,EAAE,QAAQ,QAAQ,EAAI;AAAA,EAC/B;AAAA,EAEQ,sBAAsB,QAA0C;AACtE,QAAI,mBAAmB,MAAM,GAAG;AAC9B,aAAO,oBAAoB,MAAM;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,KAAK,OAAsD,OAAO,OAAmB;AACnF,UAAM,aAAa,KAAK,qBAAqB,KAAK;AAClD,QAAI,eAAe,QAAW;AAC5B,YAAM,SAAS,IAAI,WAAW;AAC9B,aAAO,iBAAiB;AACxB,aAAO;AAAA,IACT;AAEA,UAAM,EAAE,QAAQ,OAAO,IAAI;AAC3B,UAAM,aAAa,IAAI,WAAW;AAElC,UAAM,OAAO,kBAAK,KAAK,OAAO,EAAE,OAAO,MAAM;AAC3C,YAAM,KAAK,SAAS,EAAE,YAAY,OAAO,QAAQ,QAAQ,MAAM,OAAO,CAAC;AAAA,IACzE,CAAC;AAED,SAAK,gBAAgB,MAAM;AACzB,iBAAW,iBAAiB;AAC5B,WAAK,UAAU,OAAO,KAAK,UAAU,QAAQ,IAAI,GAAG,CAAC;AAAA,IACvD,CAAC;AAED,SAAK,UAAU,KAAK,IAAI;AACxB,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,MAAM,SAAqD;AAlRnE;AAmRI,UAAM,EAAE,MAAM,cAAc,oBAAoB,IAAI;AACpD,SAAK,OAAO;AACZ,SAAK,eAAe;AACpB,SAAK,sBAAsB;AAE3B,UAAM,KAAK,aAAa;AAKxB,SAAK,KAAK,GAAG,eAAe,KAAK,aAAa;AAE9C,eAAK,iBAAL,mBAAmB,GAAG,qCAAuB,mBAAmB,KAAK;AAErE,QAAI,CAAC,KAAK,aAAc;AAExB,UAAM,aAAa,KAAK,qBAAqB,KAAK,YAAY;AAC9D,QAAI,CAAC,WAAY;AAEjB,UAAM,EAAE,QAAQ,OAAO,IAAI;AAC3B,UAAM,gBAA6B,EAAE,QAAQ,QAAQ,aAAa,EAAI;AACtE,SAAK,gBAAgB,KAAK,KAAK,eAAe,OAAO,WAAW,QAAQ;AAAA,EAC1E;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,QAAuB;AA9S/B;AA+SI,cAAM,4BAAc,KAAK,WAAW,eAAe;AAEnD,QAAI,KAAK,eAAe;AACtB,YAAM,KAAK,cAAc,cAAc,eAAe;AAAA,IACxD;AAIA,UAAM,KAAK,YAAY,MAAM;AAE7B,eAAK,iBAAL,mBAAmB,IAAI,qCAAuB,mBAAmB,KAAK;AACtE,eAAK,SAAL,mBAAW,IAAI,eAAe,KAAK;AAEnC,QAAI,KAAK,eAAe,KAAK,YAAY,KAAK;AAC5C,cAAM,gBAAK,SAAL,mBAAW,qBAAX,mBAA6B,eAAe,KAAK,YAAY;AAAA,IACrE;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,iBAAoD;AAClD,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAc,eAA8B;AAxU9C;AAyUI,QAAI,KAAK,gBAAgB,QAAW;AAClC;AAAA,IACF;AAEA,UAAM,QAAQ,gCAAgB,iBAAiB,oBAAoB,KAAK,WAAW;AAEnF,UAAI,UAAK,SAAL,mBAAW,sBAAqB,QAAW;AAC7C,YAAM,IAAI,MAAM,iCAAiC;AAAA,IACnD;AAEA,UAAM,cAAc,MAAM,KAAK,KAAK,iBAAiB;AAAA,MACnD;AAAA,MACA,KAAK,uBAAuB,IAAI,oCAAoB;AAAA,IACtD;AAEA,SAAK,cAAc;AACnB,SAAK,QAAQ,MAAM,qCAAqC,KAAK,YAAY,GAAG,EAAE;AAAA,EAChF;AAAA,EAEQ,gBAAgB,MAAY;AAClC,QAAI,KAAK,eAAe;AACtB,WAAK,cAAc,OAAO;AAAA,IAC5B;AAEA,SAAK,cAAc;AACnB,SAAK,gBAAgB,kBAAK,KAAK,YAAY;AACzC,YAAM,KAAK,mBAAmB;AAAA,IAChC,CAAC;AAAA,EACH;AAAA,EAEA,MAAc,qBAAoC;AAEhD,UAAM,KAAK,aAAa;AAAA,EAC1B;AAAA,EAEQ,sBAAsB,CAAC,OAAqC;AA5WtE;AA6WI,QAAI,CAAC,KAAK,eAAe;AACvB;AAAA,IACF;AAEA,QAAI,GAAG,aAAa,YAAY;AAC9B,UAAI,KAAK,kBAAkB,CAAC,KAAK,eAAe,KAAK,GAAG;AACtD;AAAA,MACF;AAAA,IAGF,OAAO;AACL,iBAAK,mBAAL,mBAAqB;AAAA,IACvB;AAAA,EACF;AAAA,EAEA,MAAc,SAAS;AAAA,IACrB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAMkB;AAChB,QAAI,mBAAmB,KAAK,GAAG;AAC7B,cAAQ,oBAAoB,KAAK;AAAA,IACnC;AAEA,QAAI,OAAO,UAAU,UAAU;AAC7B,cAAQ,WACJ,sCAAwB,OAAO,EAAE,aAAa,OAAO,CAAC,QACtD,kCAAoB,OAAO,EAAE,aAAa,OAAO,CAAC;AAAA,IACxD;AAEA,QAAI;AACF,uBAAiB,SAAS,OAAO;AAC/B,YAAI,OAAO,WAAW,WAAW,KAAK,EAAG;AAEzC,YAAI;AAEJ,YAAI,WAAW,GAAK;AAClB,gBAAM,YAAY,IAAI;AAAA,YACpB,MAAM,KAAK;AAAA,YACX,MAAM,KAAK;AAAA,YACX,MAAM,KAAK,aAAa;AAAA,UAC1B;AACA,gBAAM,cAAc,IAAI,aAAa,UAAU,MAAM;AAErD,mBAAS,IAAI,GAAG,IAAI,UAAU,QAAQ,KAAK;AACzC,wBAAY,CAAC,IAAI,UAAU,CAAC;AAAA,UAC9B;AAEA,gBAAM,eAAe,MAAM,KAAK,MAAM,MAAM;AAC5C,mBAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AAC3C,wBAAY,CAAC,KAAM;AAAA,UACrB;AAEA,gBAAM,aAAa,IAAI,WAAW,YAAY,MAAM;AACpD,mBAAS,IAAI,GAAG,IAAI,YAAY,QAAQ,KAAK;AAC3C,kBAAM,UAAU,KAAK,IAAI,QAAQ,KAAK,IAAI,OAAO,YAAY,CAAC,CAAE,CAAC;AACjE,uBAAW,CAAC,IAAI,KAAK,MAAM,OAAO;AAAA,UACpC;AAEA,2BAAiB,IAAI;AAAA,YACnB;AAAA,YACA,MAAM;AAAA,YACN,MAAM;AAAA,YACN,MAAM;AAAA,UACR;AAAA,QACF,OAAO;AACL,2BAAiB;AAAA,QACnB;AAGA,cAAM,KAAK,YAAY,aAAa,cAAc;AAAA,MACpD;AAAA,IACF,UAAE;AAEA,iBAAW,iBAAiB;AAAA,IAC9B;AAAA,EACF;AACF;","names":["BuiltinAudioClip"]}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { AudioFrame, type LocalTrackPublication, type Room, TrackPublishOptions } from '@livekit/rtc-node';
|
|
2
|
+
import type { AgentSession } from './agent_session.js';
|
|
3
|
+
export declare enum BuiltinAudioClip {
|
|
4
|
+
OFFICE_AMBIENCE = "office-ambience.ogg",
|
|
5
|
+
KEYBOARD_TYPING = "keyboard-typing.ogg",
|
|
6
|
+
KEYBOARD_TYPING2 = "keyboard-typing2.ogg"
|
|
7
|
+
}
|
|
8
|
+
export declare function isBuiltinAudioClip(source: AudioSourceType | AudioConfig | AudioConfig[]): source is BuiltinAudioClip;
|
|
9
|
+
export declare function getBuiltinAudioPath(clip: BuiltinAudioClip): string;
|
|
10
|
+
export type AudioSourceType = string | BuiltinAudioClip | AsyncIterable<AudioFrame>;
|
|
11
|
+
export interface AudioConfig {
|
|
12
|
+
source: AudioSourceType;
|
|
13
|
+
volume?: number;
|
|
14
|
+
probability?: number;
|
|
15
|
+
}
|
|
16
|
+
export interface BackgroundAudioPlayerOptions {
|
|
17
|
+
/**
|
|
18
|
+
* Ambient sound to play continuously in the background.
|
|
19
|
+
* Can be a file path, BuiltinAudioClip, or AudioConfig.
|
|
20
|
+
* File paths will be looped automatically.
|
|
21
|
+
*/
|
|
22
|
+
ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];
|
|
23
|
+
/**
|
|
24
|
+
* Sound to play when the agent is thinking.
|
|
25
|
+
* TODO (Brian): Implement thinking sound when AudioMixer becomes available
|
|
26
|
+
*/
|
|
27
|
+
thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];
|
|
28
|
+
/**
|
|
29
|
+
* Stream timeout in milliseconds
|
|
30
|
+
* @defaultValue 200
|
|
31
|
+
*/
|
|
32
|
+
streamTimeoutMs?: number;
|
|
33
|
+
}
|
|
34
|
+
export interface BackgroundAudioStartOptions {
|
|
35
|
+
room: Room;
|
|
36
|
+
agentSession?: AgentSession;
|
|
37
|
+
trackPublishOptions?: TrackPublishOptions;
|
|
38
|
+
}
|
|
39
|
+
export declare class PlayHandle {
|
|
40
|
+
private doneFuture;
|
|
41
|
+
private stopFuture;
|
|
42
|
+
done(): boolean;
|
|
43
|
+
stop(): void;
|
|
44
|
+
waitForPlayout(): Promise<void>;
|
|
45
|
+
_markPlayoutDone(): void;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Manages background audio playback for LiveKit agent sessions
|
|
49
|
+
*
|
|
50
|
+
* This class handles playing ambient sounds and manages audio track publishing.
|
|
51
|
+
* It supports:
|
|
52
|
+
* - Continuous ambient sound playback with looping
|
|
53
|
+
* - Volume control and probability-based sound selection
|
|
54
|
+
* - Integration with LiveKit rooms and agent sessions
|
|
55
|
+
*
|
|
56
|
+
* Note: Thinking sound not yet supported
|
|
57
|
+
*
|
|
58
|
+
* @example
|
|
59
|
+
* ```typescript
|
|
60
|
+
* const player = new BackgroundAudioPlayer({
|
|
61
|
+
* ambientSound: { source: BuiltinAudioClip.OFFICE_AMBIENCE, volume: 0.8 },
|
|
62
|
+
* });
|
|
63
|
+
*
|
|
64
|
+
* await player.start({ room, agentSession });
|
|
65
|
+
* ```
|
|
66
|
+
*/
|
|
67
|
+
export declare class BackgroundAudioPlayer {
|
|
68
|
+
#private;
|
|
69
|
+
private ambientSound?;
|
|
70
|
+
private thinkingSound?;
|
|
71
|
+
private playTasks;
|
|
72
|
+
private audioSource;
|
|
73
|
+
private room?;
|
|
74
|
+
private agentSession?;
|
|
75
|
+
private publication?;
|
|
76
|
+
private trackPublishOptions?;
|
|
77
|
+
private republishTask?;
|
|
78
|
+
private ambientHandle?;
|
|
79
|
+
private thinkingHandle?;
|
|
80
|
+
constructor(options?: BackgroundAudioPlayerOptions);
|
|
81
|
+
/**
|
|
82
|
+
* Select a sound from a list of background sound based on probability weights
|
|
83
|
+
* Return undefined if no sound is selected (when sum of probabilities < 1.0).
|
|
84
|
+
*/
|
|
85
|
+
private selectSoundFromList;
|
|
86
|
+
private normalizeSoundSource;
|
|
87
|
+
private normalizeBuiltinAudio;
|
|
88
|
+
play(audio: AudioSourceType | AudioConfig | AudioConfig[], loop?: boolean): PlayHandle;
|
|
89
|
+
/**
|
|
90
|
+
* Start the background audio system, publishing the audio track
|
|
91
|
+
* and beginning playback of any configured ambient sound.
|
|
92
|
+
*
|
|
93
|
+
* If `ambientSound` is provided (and contains file paths), they will loop
|
|
94
|
+
* automatically. If `ambientSound` contains AsyncIterators, they are assumed
|
|
95
|
+
* to be already infinite or looped.
|
|
96
|
+
*
|
|
97
|
+
* @param options - Options for starting background audio playback
|
|
98
|
+
*/
|
|
99
|
+
start(options: BackgroundAudioStartOptions): Promise<void>;
|
|
100
|
+
/**
|
|
101
|
+
* Close and cleanup the background audio system
|
|
102
|
+
*/
|
|
103
|
+
close(): Promise<void>;
|
|
104
|
+
/**
|
|
105
|
+
* Get the current track publication
|
|
106
|
+
*/
|
|
107
|
+
getPublication(): LocalTrackPublication | undefined;
|
|
108
|
+
private publishTrack;
|
|
109
|
+
private onReconnected;
|
|
110
|
+
private republishTrackTask;
|
|
111
|
+
private onAgentStateChanged;
|
|
112
|
+
private playTask;
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=background_audio.d.ts.map
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { AudioFrame, type LocalTrackPublication, type Room, TrackPublishOptions } from '@livekit/rtc-node';
|
|
2
|
+
import type { AgentSession } from './agent_session.js';
|
|
3
|
+
export declare enum BuiltinAudioClip {
|
|
4
|
+
OFFICE_AMBIENCE = "office-ambience.ogg",
|
|
5
|
+
KEYBOARD_TYPING = "keyboard-typing.ogg",
|
|
6
|
+
KEYBOARD_TYPING2 = "keyboard-typing2.ogg"
|
|
7
|
+
}
|
|
8
|
+
export declare function isBuiltinAudioClip(source: AudioSourceType | AudioConfig | AudioConfig[]): source is BuiltinAudioClip;
|
|
9
|
+
export declare function getBuiltinAudioPath(clip: BuiltinAudioClip): string;
|
|
10
|
+
export type AudioSourceType = string | BuiltinAudioClip | AsyncIterable<AudioFrame>;
|
|
11
|
+
export interface AudioConfig {
|
|
12
|
+
source: AudioSourceType;
|
|
13
|
+
volume?: number;
|
|
14
|
+
probability?: number;
|
|
15
|
+
}
|
|
16
|
+
export interface BackgroundAudioPlayerOptions {
|
|
17
|
+
/**
|
|
18
|
+
* Ambient sound to play continuously in the background.
|
|
19
|
+
* Can be a file path, BuiltinAudioClip, or AudioConfig.
|
|
20
|
+
* File paths will be looped automatically.
|
|
21
|
+
*/
|
|
22
|
+
ambientSound?: AudioSourceType | AudioConfig | AudioConfig[];
|
|
23
|
+
/**
|
|
24
|
+
* Sound to play when the agent is thinking.
|
|
25
|
+
* TODO (Brian): Implement thinking sound when AudioMixer becomes available
|
|
26
|
+
*/
|
|
27
|
+
thinkingSound?: AudioSourceType | AudioConfig | AudioConfig[];
|
|
28
|
+
/**
|
|
29
|
+
* Stream timeout in milliseconds
|
|
30
|
+
* @defaultValue 200
|
|
31
|
+
*/
|
|
32
|
+
streamTimeoutMs?: number;
|
|
33
|
+
}
|
|
34
|
+
export interface BackgroundAudioStartOptions {
|
|
35
|
+
room: Room;
|
|
36
|
+
agentSession?: AgentSession;
|
|
37
|
+
trackPublishOptions?: TrackPublishOptions;
|
|
38
|
+
}
|
|
39
|
+
export declare class PlayHandle {
|
|
40
|
+
private doneFuture;
|
|
41
|
+
private stopFuture;
|
|
42
|
+
done(): boolean;
|
|
43
|
+
stop(): void;
|
|
44
|
+
waitForPlayout(): Promise<void>;
|
|
45
|
+
_markPlayoutDone(): void;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Manages background audio playback for LiveKit agent sessions
|
|
49
|
+
*
|
|
50
|
+
* This class handles playing ambient sounds and manages audio track publishing.
|
|
51
|
+
* It supports:
|
|
52
|
+
* - Continuous ambient sound playback with looping
|
|
53
|
+
* - Volume control and probability-based sound selection
|
|
54
|
+
* - Integration with LiveKit rooms and agent sessions
|
|
55
|
+
*
|
|
56
|
+
* Note: Thinking sound not yet supported
|
|
57
|
+
*
|
|
58
|
+
* @example
|
|
59
|
+
* ```typescript
|
|
60
|
+
* const player = new BackgroundAudioPlayer({
|
|
61
|
+
* ambientSound: { source: BuiltinAudioClip.OFFICE_AMBIENCE, volume: 0.8 },
|
|
62
|
+
* });
|
|
63
|
+
*
|
|
64
|
+
* await player.start({ room, agentSession });
|
|
65
|
+
* ```
|
|
66
|
+
*/
|
|
67
|
+
export declare class BackgroundAudioPlayer {
|
|
68
|
+
#private;
|
|
69
|
+
private ambientSound?;
|
|
70
|
+
private thinkingSound?;
|
|
71
|
+
private playTasks;
|
|
72
|
+
private audioSource;
|
|
73
|
+
private room?;
|
|
74
|
+
private agentSession?;
|
|
75
|
+
private publication?;
|
|
76
|
+
private trackPublishOptions?;
|
|
77
|
+
private republishTask?;
|
|
78
|
+
private ambientHandle?;
|
|
79
|
+
private thinkingHandle?;
|
|
80
|
+
constructor(options?: BackgroundAudioPlayerOptions);
|
|
81
|
+
/**
|
|
82
|
+
* Select a sound from a list of background sound based on probability weights
|
|
83
|
+
* Return undefined if no sound is selected (when sum of probabilities < 1.0).
|
|
84
|
+
*/
|
|
85
|
+
private selectSoundFromList;
|
|
86
|
+
private normalizeSoundSource;
|
|
87
|
+
private normalizeBuiltinAudio;
|
|
88
|
+
play(audio: AudioSourceType | AudioConfig | AudioConfig[], loop?: boolean): PlayHandle;
|
|
89
|
+
/**
|
|
90
|
+
* Start the background audio system, publishing the audio track
|
|
91
|
+
* and beginning playback of any configured ambient sound.
|
|
92
|
+
*
|
|
93
|
+
* If `ambientSound` is provided (and contains file paths), they will loop
|
|
94
|
+
* automatically. If `ambientSound` contains AsyncIterators, they are assumed
|
|
95
|
+
* to be already infinite or looped.
|
|
96
|
+
*
|
|
97
|
+
* @param options - Options for starting background audio playback
|
|
98
|
+
*/
|
|
99
|
+
start(options: BackgroundAudioStartOptions): Promise<void>;
|
|
100
|
+
/**
|
|
101
|
+
* Close and cleanup the background audio system
|
|
102
|
+
*/
|
|
103
|
+
close(): Promise<void>;
|
|
104
|
+
/**
|
|
105
|
+
* Get the current track publication
|
|
106
|
+
*/
|
|
107
|
+
getPublication(): LocalTrackPublication | undefined;
|
|
108
|
+
private publishTrack;
|
|
109
|
+
private onReconnected;
|
|
110
|
+
private republishTrackTask;
|
|
111
|
+
private onAgentStateChanged;
|
|
112
|
+
private playTask;
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=background_audio.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"background_audio.d.ts","sourceRoot":"","sources":["../../src/voice/background_audio.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,UAAU,EAGV,KAAK,qBAAqB,EAC1B,KAAK,IAAI,EACT,mBAAmB,EACpB,MAAM,mBAAmB,CAAC;AAM3B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAKvD,oBAAY,gBAAgB;IAC1B,eAAe,wBAAwB;IACvC,eAAe,wBAAwB;IACvC,gBAAgB,yBAAyB;CAC1C;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,GACpD,MAAM,IAAI,gBAAgB,CAK5B;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,gBAAgB,GAAG,MAAM,CAGlE;AAED,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,gBAAgB,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;AAEpF,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,eAAe,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,4BAA4B;IAC3C;;;;OAIG;IACH,YAAY,CAAC,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,CAAC;IAE7D;;;OAGG;IACH,aAAa,CAAC,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,CAAC;IAE9D;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,IAAI,CAAC;IACX,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,mBAAmB,CAAC,EAAE,mBAAmB,CAAC;CAC3C;AAMD,qBAAa,UAAU;IACrB,OAAO,CAAC,UAAU,CAAsB;IACxC,OAAO,CAAC,UAAU,CAAsB;IAExC,IAAI,IAAI,OAAO;IAIf,IAAI,IAAI,IAAI;IAUN,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IAIrC,gBAAgB,IAAI,IAAI;CAKzB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,qBAAqB;;IAChC,OAAO,CAAC,YAAY,CAAC,CAAgD;IACrE,OAAO,CAAC,aAAa,CAAC,CAAgD;IAEtE,OAAO,CAAC,SAAS,CAAoB;IACrC,OAAO,CAAC,WAAW,CAAqD;IAExE,OAAO,CAAC,IAAI,CAAC,CAAO;IACpB,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,WAAW,CAAC,CAAwB;IAC5C,OAAO,CAAC,mBAAmB,CAAC,CAAsB;IAClD,OAAO,CAAC,aAAa,CAAC,CAAa;IAEnC,OAAO,CAAC,aAAa,CAAC,CAAa;IACnC,OAAO,CAAC,cAAc,CAAC,CAAa;gBAMxB,OAAO,CAAC,EAAE,4BAA4B;IAYlD;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAgC3B,OAAO,CAAC,oBAAoB;IAoC5B,OAAO,CAAC,qBAAqB;IAO7B,IAAI,CAAC,KAAK,EAAE,eAAe,GAAG,WAAW,GAAG,WAAW,EAAE,EAAE,IAAI,UAAQ,GAAG,UAAU;IAwBpF;;;;;;;;;OASG;IACG,KAAK,CAAC,OAAO,EAAE,2BAA2B,GAAG,OAAO,CAAC,IAAI,CAAC;IAyBhE;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB5B;;OAEG;IACH,cAAc,IAAI,qBAAqB,GAAG,SAAS;YAIrC,YAAY;IAoB1B,OAAO,CAAC,aAAa,CASnB;YAEY,kBAAkB;IAKhC,OAAO,CAAC,mBAAmB,CAczB;YAEY,QAAQ;CAsEvB"}
|