@livekit/agents 0.1.0 → 0.3.0

package/dist/multimodal/agent_playout.js

@@ -0,0 +1,221 @@
+var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
+    if (kind === "m") throw new TypeError("Private method is not writable");
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
+    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
+};
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var _PlayoutHandle_audioSource, _PlayoutHandle_sampleRate, _PlayoutHandle_itemId, _PlayoutHandle_contentIndex, _PlayoutHandle_interrupted, _AgentPlayout_instances, _AgentPlayout_audioSource, _AgentPlayout_playoutTask, _AgentPlayout_sampleRate, _AgentPlayout_numChannels, _AgentPlayout_inFrameSize, _AgentPlayout_outFrameSize, _AgentPlayout_makePlayoutTask;
+import { EventEmitter } from 'events';
+import { AudioByteStream } from '../audio.js';
+import { CancellablePromise, Future, gracefullyCancel } from '../utils.js';
+export const proto = {};
+export class PlayoutHandle extends EventEmitter {
+    constructor(audioSource, sampleRate, itemId, contentIndex, transcriptionFwd) {
+        super();
+        _PlayoutHandle_audioSource.set(this, void 0);
+        _PlayoutHandle_sampleRate.set(this, void 0);
+        _PlayoutHandle_itemId.set(this, void 0);
+        _PlayoutHandle_contentIndex.set(this, void 0);
+        /** @internal */
+        _PlayoutHandle_interrupted.set(this, void 0);
+        __classPrivateFieldSet(this, _PlayoutHandle_audioSource, audioSource, "f");
+        __classPrivateFieldSet(this, _PlayoutHandle_sampleRate, sampleRate, "f");
+        __classPrivateFieldSet(this, _PlayoutHandle_itemId, itemId, "f");
+        __classPrivateFieldSet(this, _PlayoutHandle_contentIndex, contentIndex, "f");
+        this.transcriptionFwd = transcriptionFwd;
+        this.doneFut = new Future();
+        this.intFut = new Future();
+        __classPrivateFieldSet(this, _PlayoutHandle_interrupted, false, "f");
+        this.pushedDuration = 0;
+        this.totalPlayedTime = undefined;
+    }
+    get itemId() {
+        return __classPrivateFieldGet(this, _PlayoutHandle_itemId, "f");
+    }
+    get audioSamples() {
+        if (this.totalPlayedTime !== undefined) {
+            return Math.floor(this.totalPlayedTime * __classPrivateFieldGet(this, _PlayoutHandle_sampleRate, "f"));
+        }
+        return Math.floor(this.pushedDuration - __classPrivateFieldGet(this, _PlayoutHandle_audioSource, "f").queuedDuration * __classPrivateFieldGet(this, _PlayoutHandle_sampleRate, "f"));
+    }
+    get textChars() {
+        return this.transcriptionFwd.currentCharacterIndex;
+    }
+    get contentIndex() {
+        return __classPrivateFieldGet(this, _PlayoutHandle_contentIndex, "f");
+    }
+    get interrupted() {
+        return __classPrivateFieldGet(this, _PlayoutHandle_interrupted, "f");
+    }
+    get done() {
+        return this.doneFut.done || __classPrivateFieldGet(this, _PlayoutHandle_interrupted, "f");
+    }
+    interrupt() {
+        if (this.doneFut.done)
+            return;
+        this.intFut.resolve();
+        __classPrivateFieldSet(this, _PlayoutHandle_interrupted, true, "f");
+    }
+}
+_PlayoutHandle_audioSource = new WeakMap(), _PlayoutHandle_sampleRate = new WeakMap(), _PlayoutHandle_itemId = new WeakMap(), _PlayoutHandle_contentIndex = new WeakMap(), _PlayoutHandle_interrupted = new WeakMap();
+export class AgentPlayout {
+    constructor(audioSource, sampleRate, numChannels, inFrameSize, outFrameSize) {
+        _AgentPlayout_instances.add(this);
+        _AgentPlayout_audioSource.set(this, void 0);
+        _AgentPlayout_playoutTask.set(this, void 0);
+        _AgentPlayout_sampleRate.set(this, void 0);
+        _AgentPlayout_numChannels.set(this, void 0);
+        _AgentPlayout_inFrameSize.set(this, void 0);
+        _AgentPlayout_outFrameSize.set(this, void 0);
+        __classPrivateFieldSet(this, _AgentPlayout_audioSource, audioSource, "f");
+        __classPrivateFieldSet(this, _AgentPlayout_playoutTask, null, "f");
+        __classPrivateFieldSet(this, _AgentPlayout_sampleRate, sampleRate, "f");
+        __classPrivateFieldSet(this, _AgentPlayout_numChannels, numChannels, "f");
+        __classPrivateFieldSet(this, _AgentPlayout_inFrameSize, inFrameSize, "f");
+        __classPrivateFieldSet(this, _AgentPlayout_outFrameSize, outFrameSize, "f");
+    }
+    play(itemId, contentIndex, transcriptionFwd, textStream, audioStream) {
+        const handle = new PlayoutHandle(__classPrivateFieldGet(this, _AgentPlayout_audioSource, "f"), __classPrivateFieldGet(this, _AgentPlayout_sampleRate, "f"), itemId, contentIndex, transcriptionFwd);
+        __classPrivateFieldSet(this, _AgentPlayout_playoutTask, __classPrivateFieldGet(this, _AgentPlayout_instances, "m", _AgentPlayout_makePlayoutTask).call(this, __classPrivateFieldGet(this, _AgentPlayout_playoutTask, "f"), handle, textStream, audioStream), "f");
+        return handle;
+    }
+}
+_AgentPlayout_audioSource = new WeakMap(), _AgentPlayout_playoutTask = new WeakMap(), _AgentPlayout_sampleRate = new WeakMap(), _AgentPlayout_numChannels = new WeakMap(), _AgentPlayout_inFrameSize = new WeakMap(), _AgentPlayout_outFrameSize = new WeakMap(), _AgentPlayout_instances = new WeakSet(), _AgentPlayout_makePlayoutTask = function _AgentPlayout_makePlayoutTask(oldTask, handle, textStream, audioStream) {
+    return new CancellablePromise((resolve, reject, onCancel) => {
+        let cancelled = false;
+        onCancel(() => {
+            cancelled = true;
+        });
+        (async () => {
+            try {
+                if (oldTask) {
+                    await gracefullyCancel(oldTask);
+                }
+                let firstFrame = true;
+                const readText = () => new CancellablePromise((resolveText, rejectText, onCancelText) => {
+                    let cancelledText = false;
+                    onCancelText(() => {
+                        cancelledText = true;
+                    });
+                    (async () => {
+                        var _a, e_1, _b, _c;
+                        try {
+                            try {
+                                for (var _d = true, textStream_1 = __asyncValues(textStream), textStream_1_1; textStream_1_1 = await textStream_1.next(), _a = textStream_1_1.done, !_a; _d = true) {
+                                    _c = textStream_1_1.value;
+                                    _d = false;
+                                    const text = _c;
+                                    if (cancelledText || cancelled) {
+                                        break;
+                                    }
+                                    handle.transcriptionFwd.pushText(text);
+                                }
+                            }
+                            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                            finally {
+                                try {
+                                    if (!_d && !_a && (_b = textStream_1.return)) await _b.call(textStream_1);
+                                }
+                                finally { if (e_1) throw e_1.error; }
+                            }
+                            resolveText();
+                        }
+                        catch (error) {
+                            rejectText(error);
+                        }
+                    })();
+                });
+                const capture = () => new CancellablePromise((resolveCapture, rejectCapture, onCancelCapture) => {
+                    let cancelledCapture = false;
+                    onCancelCapture(() => {
+                        cancelledCapture = true;
+                    });
+                    (async () => {
+                        var _a, e_2, _b, _c;
+                        try {
+                            const samplesPerChannel = __classPrivateFieldGet(this, _AgentPlayout_outFrameSize, "f");
+                            const bstream = new AudioByteStream(__classPrivateFieldGet(this, _AgentPlayout_sampleRate, "f"), __classPrivateFieldGet(this, _AgentPlayout_numChannels, "f"), samplesPerChannel);
+                            try {
+                                for (var _d = true, audioStream_1 = __asyncValues(audioStream), audioStream_1_1; audioStream_1_1 = await audioStream_1.next(), _a = audioStream_1_1.done, !_a; _d = true) {
+                                    _c = audioStream_1_1.value;
+                                    _d = false;
+                                    const frame = _c;
+                                    if (cancelledCapture || cancelled) {
+                                        break;
+                                    }
+                                    if (firstFrame) {
+                                        handle.transcriptionFwd.start();
+                                        firstFrame = false;
+                                    }
+                                    handle.transcriptionFwd.pushAudio(frame);
+                                    for (const f of bstream.write(frame.data.buffer)) {
+                                        handle.pushedDuration += f.samplesPerChannel / f.sampleRate;
+                                        await __classPrivateFieldGet(this, _AgentPlayout_audioSource, "f").captureFrame(f);
+                                    }
+                                }
+                            }
+                            catch (e_2_1) { e_2 = { error: e_2_1 }; }
+                            finally {
+                                try {
+                                    if (!_d && !_a && (_b = audioStream_1.return)) await _b.call(audioStream_1);
+                                }
+                                finally { if (e_2) throw e_2.error; }
+                            }
+                            if (!cancelledCapture && !cancelled) {
+                                for (const f of bstream.flush()) {
+                                    handle.pushedDuration += f.samplesPerChannel / f.sampleRate;
+                                    await __classPrivateFieldGet(this, _AgentPlayout_audioSource, "f").captureFrame(f);
+                                }
+                                handle.transcriptionFwd.markAudioComplete();
+                                await __classPrivateFieldGet(this, _AgentPlayout_audioSource, "f").waitForPlayout();
+                            }
+                            resolveCapture();
+                        }
+                        catch (error) {
+                            rejectCapture(error);
+                        }
+                    })();
+                });
+                const readTextTask = readText();
+                const captureTask = capture();
+                try {
+                    await Promise.race([captureTask, handle.intFut.await]);
+                }
+                finally {
+                    if (!captureTask.isCancelled) {
+                        await gracefullyCancel(captureTask);
+                    }
+                    handle.totalPlayedTime = handle.pushedDuration - __classPrivateFieldGet(this, _AgentPlayout_audioSource, "f").queuedDuration;
+                    if (handle.interrupted || captureTask.error) {
+                        __classPrivateFieldGet(this, _AgentPlayout_audioSource, "f").clearQueue(); // make sure to remove any queued frames
+                    }
+                    if (!readTextTask.isCancelled) {
+                        await gracefullyCancel(readTextTask);
+                    }
+                    if (!firstFrame && !handle.interrupted) {
+                        handle.transcriptionFwd.markTextComplete();
+                    }
+                    handle.doneFut.resolve();
+                    await handle.transcriptionFwd.close(handle.interrupted);
+                }
+                resolve();
+            }
+            catch (error) {
+                reject(error);
+            }
+        })();
+    });
+};
+//# sourceMappingURL=agent_playout.js.map

package/dist/multimodal/agent_playout.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent_playout.js","sourceRoot":"","sources":["../../src/multimodal/agent_playout.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAKA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,EAA2B,kBAAkB,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEpG,MAAM,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;AAExB,MAAM,OAAO,aAAc,SAAQ,YAAY;IAkB7C,YACE,WAAwB,EACxB,UAAkB,EAClB,MAAc,EACd,YAAoB,EACpB,gBAAwC;QAExC,KAAK,EAAE,CAAC;QAxBV,6CAA0B;QAC1B,4CAAoB;QACpB,wCAAgB;QAChB,8CAAsB;QAOtB,gBAAgB;QAChB,6CAAsB;QAcpB,uBAAA,IAAI,8BAAgB,WAAW,MAAA,CAAC;QAChC,uBAAA,IAAI,6BAAe,UAAU,MAAA,CAAC;QAC9B,uBAAA,IAAI,yBAAW,MAAM,MAAA,CAAC;QACtB,uBAAA,IAAI,+BAAiB,YAAY,MAAA,CAAC;QAClC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,OAAO,GAAG,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC3B,uBAAA,IAAI,8BAAgB,KAAK,MAAA,CAAC;QAC1B,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;IACnC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,uBAAA,IAAI,6BAAQ,CAAC;IACtB,CAAC;IAED,IAAI,YAAY;QACd,IAAI,IAAI,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACvC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,GAAG,uBAAA,IAAI,iCAAY,CAAC,CAAC;QAC7D,CAAC;QAED,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,GAAG,uBAAA,IAAI,kCAAa,CAAC,cAAc,GAAG,uBAAA,IAAI,iCAAY,CAAC,CAAC;IAC/F,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC;IACrD,CAAC;IAED,IAAI,YAAY;QACd,OAAO,uBAAA,IAAI,mCAAc,CAAC;IAC5B,CAAC;IAED,IAAI,WAAW;QACb,OAAO,uBAAA,IAAI,kCAAa,CAAC;IAC3B,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,uBAAA,IAAI,kCAAa,CAAC;IAChD,CAAC;IAED,SAAS;QACP,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI;YAAE,OAAO;QAC9B,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACtB,uBAAA,IAAI,8BAAgB,IAAI,MAAA,CAAC;IAC3B,CAAC;CACF;;AAED,MAAM,OAAO,YAAY;IAOvB,YACE,WAAwB,EACxB,UAAkB,EAClB,WAAmB,EACnB,WAAmB,EACnB,YAAoB;;QAXtB,4CAA0B;QAC1B,4CAA8C;QAC9C,2CAAoB;QACpB,4CAAqB;QACrB,4CAAqB;QACrB,6CAAsB;QAQpB,uBAAA,IAAI,6BAAgB,WAAW,MAAA,CAAC;QAChC,uBAAA,IAAI,6BAAgB,IAAI,MAAA,CAAC;QACzB,uBAAA,IAAI,4BAAe,UAAU,MAAA,CAAC;QAC9B,uBAAA,IAAI,6BAAgB,WAAW,MAAA,CAAC;QAChC,uBAAA,IAAI,6BAAgB,WAAW,MAAA,CAAC;QAChC,uBAAA,IAAI,8BAAiB,YAAY,MAAA,CAAC;IACpC,CAAC;IAED,IAAI,CACF,MAAc,EACd,YAAoB,EACpB,gBAAwC,EACxC,UAAsC,EACtC,WAA2C;QAE3C,MAAM,MAAM,GAAG,IAAI,aAAa,CAC9B,uBAAA,IAAI,iCAAa,EACjB,uBAAA,IAAI,gCAAY,EAChB,MAAM,EACN,YAAY,EACZ,gBAAgB,CACjB,CAAC;QACF,uBAAA,IAAI,6BAAgB,uBAAA,IAAI,8DAAiB,MAArB,IAAI,EAAkB,uBAAA,IAAI,iCAAa,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,CAAC,MAAA,CAAC;QAC9F,OAAO,MAAM,CAAC;IAChB,CAAC;CAkIF;kXA/HG,OAAwC,EACxC,MAAqB,EACrB,UAAsC,EACtC,WAA2C;IAE3C,OAAO,IAAI,kBAAkB,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE;QAChE,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,QAAQ,CAAC,GAAG,EAAE;YACZ,SAAS,GAAG,IAAI,CAAC;QACnB,CAAC,CAAC,CAAC;QAEH,CAAC,KAAK,IAAI,EAAE;YACV,IAAI,CAAC;gBACH,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAClC,CAAC;gBAED,IAAI,UAAU,GAAG,IAAI,CAAC;gBAEtB,MAAM,QAAQ,GAAG,GAAG,EAAE,CACpB,IAAI,kBAAkB,CAAO,CAAC,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE;oBACrE,IAAI,aAAa,GAAG,KAAK,CAAC;oBAC1B,YAAY,CAAC,GAAG,EAAE;wBAChB,aAAa,GAAG,IAAI,CAAC;oBACvB,CAAC,CAAC,CAAC;oBAEH,CAAC,KAAK,IAAI,EAAE;;wBACV,IAAI,CAAC;;gCACH,KAAyB,eAAA,eAAA,cAAA,UAAU,CAAA,gBAAA,wFAAE,CAAC;oCAAb,0BAAU;oCAAV,WAAU;oCAAxB,MAAM,IAAI,KAAA,CAAA;oCACnB,IAAI,aAAa,IAAI,SAAS,EAAE,CAAC;wCAC/B,MAAM;oCACR,CAAC;oCACD,MAAM,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gCACzC,CAAC;;;;;;;;;4BACD,WAAW,EAAE,CAAC;wBAChB,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,UAAU,CAAC,KAAK,CAAC,CAAC;wBACpB,CAAC;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,CAAC,CAAC,CAAC;gBAEL,MAAM,OAAO,GAAG,GAAG,EAAE,CACnB,IAAI,kBAAkB,CAAO,CAAC,cAAc,EAAE,aAAa,EAAE,eAAe,EAAE,EAAE;oBAC9E,IAAI,gBAAgB,GAAG,KAAK,CAAC;oBAC7B,eAAe,CAAC,GAAG,EAAE;wBACnB,gBAAgB,GAAG,IAAI,CAAC;oBAC1B,CAAC,CAAC,CAAC;oBAEH,CAAC,KAAK,IAAI,EAAE;;wBACV,IAAI,CAAC;4BACH,MAAM,iBAAiB,GAAG,uBAAA,IAAI,kCAAc,CAAC;4BAC7C,MAAM,OAAO,GAAG,IAAI,eAAe,CACjC,uBAAA,IAAI,gCAAY,EAChB,uBAAA,IAAI,iCAAa,EACjB,iBAAiB,CAClB,CAAC;;gCAEF,KAA0B,eAAA,gBAAA,cAAA,WAAW,CAAA,iBAAA,2FAAE,CAAC;oCAAd,2BAAW;oCAAX,WAAW;oCAA1B,MAAM,KAAK,KAAA,CAAA;oCACpB,IAAI,gBAAgB,IAAI,SAAS,EAAE,CAAC;wCAClC,MAAM;oCACR,CAAC;oCACD,IAAI,UAAU,EAAE,CAAC;wCACf,MAAM,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC;wCAChC,UAAU,GAAG,KAAK,CAAC;oCACrB,CAAC;oCAED,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;oCAEzC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;wCACjD,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,UAAU,CAAC;wCAC5D,MAAM,uBAAA,IAAI,iCAAa,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oCAC1C,CAAC;gCACH,CAAC;;;;;;;;;4BAED,IAAI,CAAC,gBAAgB,IAAI,CAAC,SAAS,EAAE,CAAC;gCACpC,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;oCAChC,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC,iBAAiB,GAAG,CAAC,CAAC,UAAU,CAAC;oCAC5D,MAAM,uBAAA,IAAI,iCAAa,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;gCAC1C,CAAC;gCAED,MAAM,CAAC,gBAAgB,CAAC,iBAAiB,EAAE,CAAC;gCAE5C,MAAM,uBAAA,IAAI,iCAAa,CAAC,cAAc,EAAE,CAAC;4BAC3C,CAAC;4BAED,cAAc,EAAE,CAAC;wBACnB,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,aAAa,CAAC,KAAK,CAAC,CAAC;wBACvB,CAAC;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,CAAC,CAAC,CAAC;gBAEL,MAAM,YAAY,GAAG,QAAQ,EAAE,CAAC;gBAChC,MAAM,WAAW,GAAG,OAAO,EAAE,CAAC;gBAE9B,IAAI,CAAC;oBACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBACzD,CAAC;wBAAS,CAAC;oBACT,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,CAAC;wBAC7B,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAC;oBACtC,CAAC;oBAED,MAAM,CAAC,eAAe,GAAG,MAAM,CAAC,cAAc,GAAG,uBAAA,IAAI,iCAAa,CAAC,cAAc,CAAC;oBAElF,IAAI,MAAM,CAAC,WAAW,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;wBAC5C,uBAAA,IAAI,iCAAa,CAAC,UAAU,EAAE,CAAC,CAAC,wCAAwC;oBAC1E,CAAC;oBAED,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;wBAC9B,MAAM,gBAAgB,CAAC,YAAY,CAAC,CAAC;oBACvC,CAAC;oBAED,IAAI,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;wBACvC,MAAM,CAAC,gBAAgB,CAAC,gBAAgB,EAAE,CAAC;oBAC7C,CAAC;oBAED,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;oBACzB,MAAM,MAAM,CAAC,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBAC1D,CAAC;gBAED,OAAO,EAAE,CAAC;YACZ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/multimodal/index.d.ts

@@ -0,0 +1,3 @@
+export * from './multimodal_agent.js';
+export * from './agent_playout.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/multimodal/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/multimodal/index.ts"],"names":[],"mappings":"AAGA,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC"}

package/dist/multimodal/index.js

@@ -0,0 +1,6 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export * from './multimodal_agent.js';
+export * from './agent_playout.js';
+//# sourceMappingURL=index.js.map

package/dist/multimodal/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/multimodal/index.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC"}

package/dist/multimodal/multimodal_agent.d.ts

@@ -0,0 +1,47 @@
+/// <reference types="node" />
+import type { RemoteAudioTrack, RemoteParticipant, Room } from '@livekit/rtc-node';
+import { EventEmitter } from 'events';
+import type * as llm from '../llm/index.js';
+/**
+ * @internal
+ * @beta
+ */
+export declare abstract class RealtimeSession extends EventEmitter {
+    abstract defaultConversation: any;
+    abstract inputAudioBuffer: any;
+    abstract fncCtx: llm.FunctionContext | undefined;
+}
+/**
+ * @internal
+ * @beta
+ */
+export declare abstract class RealtimeModel {
+    abstract session(options: any): RealtimeSession;
+    abstract close(): Promise<void>;
+    abstract sampleRate: number;
+    abstract numChannels: number;
+    abstract inFrameSize: number;
+    abstract outFrameSize: number;
+}
+export type AgentState = 'initializing' | 'thinking' | 'listening' | 'speaking';
+export declare const AGENT_STATE_ATTRIBUTE = "lk.agent.state";
+/** @beta */
+export declare class MultimodalAgent {
+    #private;
+    model: RealtimeModel;
+    room: Room | null;
+    linkedParticipant: RemoteParticipant | null;
+    subscribedTrack: RemoteAudioTrack | null;
+    readMicroTask: {
+        promise: Promise<void>;
+        cancel: () => void;
+    } | null;
+    constructor({ model, fncCtx, }: {
+        model: RealtimeModel;
+        fncCtx?: llm.FunctionContext | undefined;
+    });
+    get fncCtx(): llm.FunctionContext | undefined;
+    set fncCtx(ctx: llm.FunctionContext | undefined);
+    start(room: Room, participant?: RemoteParticipant | string | null): Promise<RealtimeSession>;
+}
+//# sourceMappingURL=multimodal_agent.d.ts.map

package/dist/multimodal/multimodal_agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"multimodal_agent.d.ts","sourceRoot":"","sources":["../../src/multimodal/multimodal_agent.ts"],"names":[],"mappings":";AAGA,OAAO,KAAK,EAEV,gBAAgB,EAChB,iBAAiB,EACjB,IAAI,EACL,MAAM,mBAAmB,CAAC;AAS3B,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC,OAAO,KAAK,KAAK,GAAG,MAAM,iBAAiB,CAAC;AAM5C;;;GAGG;AACH,8BAAsB,eAAgB,SAAQ,YAAY;IAExD,QAAQ,CAAC,mBAAmB,EAAE,GAAG,CAAC;IAElC,QAAQ,CAAC,gBAAgB,EAAE,GAAG,CAAC;IAC/B,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,CAAC;CAClD;AAED;;;GAGG;AACH,8BAAsB,aAAa;IAEjC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,GAAG,eAAe;IAC/C,QAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,MAAM,UAAU,GAAG,cAAc,GAAG,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC;AAChF,eAAO,MAAM,qBAAqB,mBAAmB,CAAC;AAEtD,YAAY;AACZ,qBAAa,eAAe;;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,IAAI,EAAE,IAAI,GAAG,IAAI,CAAQ;IACzB,iBAAiB,EAAE,iBAAiB,GAAG,IAAI,CAAQ;IACnD,eAAe,EAAE,gBAAgB,GAAG,IAAI,CAAQ;IAChD,aAAa,EAAE;QAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAAC,MAAM,EAAE,MAAM,IAAI,CAAA;KAAE,GAAG,IAAI,CAAQ;gBAEhE,EACV,KAAK,EACL,MAAM,GACP,EAAE;QACD,KAAK,EAAE,aAAa,CAAC;QACrB,MAAM,CAAC,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,CAAC;KAC1C;IAmBD,IAAI,MAAM,IAAI,GAAG,CAAC,eAAe,GAAG,SAAS,CAE5C;IAED,IAAI,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,eAAe,GAAG,SAAS,EAK9C;IA6BD,KAAK,CACH,IAAI,EAAE,IAAI,EACV,WAAW,GAAE,iBAAiB,GAAG,MAAM,GAAG,IAAW,GACpD,OAAO,CAAC,eAAe,CAAC;CA2S5B"}

package/dist/multimodal/multimodal_agent.js

@@ -0,0 +1,331 @@
+var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
+    if (kind === "m") throw new TypeError("Private method is not writable");
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
+    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
+};
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var _MultimodalAgent_instances, _MultimodalAgent_participant, _MultimodalAgent_agentPublication, _MultimodalAgent_localTrackSid, _MultimodalAgent_localSource, _MultimodalAgent_agentPlayout, _MultimodalAgent_playingHandle, _MultimodalAgent_logger, _MultimodalAgent_session, _MultimodalAgent_fncCtx, _MultimodalAgent__started, _MultimodalAgent__pendingFunctionCalls, _MultimodalAgent__speaking, _MultimodalAgent_pendingFunctionCalls_get, _MultimodalAgent_pendingFunctionCalls_set, _MultimodalAgent_speaking_get, _MultimodalAgent_speaking_set, _MultimodalAgent_started_get, _MultimodalAgent_started_set, _MultimodalAgent_linkParticipant, _MultimodalAgent_subscribeToMicrophone, _MultimodalAgent_getLocalTrackSid, _MultimodalAgent_publishTranscription, _MultimodalAgent_updateState, _MultimodalAgent_setState;
+import { AudioSource, AudioStream, LocalAudioTrack, RoomEvent, TrackPublishOptions, TrackSource, } from '@livekit/rtc-node';
+import { EventEmitter } from 'events';
+import { AudioByteStream } from '../audio.js';
+import { log } from '../log.js';
+import { BasicTranscriptionForwarder } from '../transcription.js';
+import { findMicroTrackId } from '../utils.js';
+import { AgentPlayout } from './agent_playout.js';
+/**
+ * @internal
+ * @beta
+ */
+export class RealtimeSession extends EventEmitter {
+}
+/**
+ * @internal
+ * @beta
+ */
+export class RealtimeModel {
+}
+export const AGENT_STATE_ATTRIBUTE = 'lk.agent.state';
+/** @beta */
+export class MultimodalAgent {
+    constructor({ model, fncCtx, }) {
+        _MultimodalAgent_instances.add(this);
+        this.room = null;
+        this.linkedParticipant = null;
+        this.subscribedTrack = null;
+        this.readMicroTask = null;
+        _MultimodalAgent_participant.set(this, null);
+        _MultimodalAgent_agentPublication.set(this, null);
+        _MultimodalAgent_localTrackSid.set(this, null);
+        _MultimodalAgent_localSource.set(this, null);
+        _MultimodalAgent_agentPlayout.set(this, null);
+        _MultimodalAgent_playingHandle.set(this, undefined);
+        _MultimodalAgent_logger.set(this, log());
+        _MultimodalAgent_session.set(this, null);
+        _MultimodalAgent_fncCtx.set(this, undefined);
+        _MultimodalAgent__started.set(this, false);
+        _MultimodalAgent__pendingFunctionCalls.set(this, new Set());
+        _MultimodalAgent__speaking.set(this, false);
+        this.model = model;
+        __classPrivateFieldSet(this, _MultimodalAgent_fncCtx, fncCtx, "f");
+    }
+    get fncCtx() {
+        return __classPrivateFieldGet(this, _MultimodalAgent_fncCtx, "f");
+    }
+    set fncCtx(ctx) {
+        __classPrivateFieldSet(this, _MultimodalAgent_fncCtx, ctx, "f");
+        if (__classPrivateFieldGet(this, _MultimodalAgent_session, "f")) {
+            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").fncCtx = ctx;
+        }
+    }
+    start(room, participant = null) {
+        return new Promise(async (resolve, reject) => {
+            var _a;
+            if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_started_get)) {
+                __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").warn('MultimodalAgent already started');
+                resolve(__classPrivateFieldGet(this, _MultimodalAgent_session, "f")); // TODO: throw error?
+                return;
+            }
+            __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
+            room.on(RoomEvent.ParticipantConnected, (participant) => {
+                if (!this.linkedParticipant) {
+                    return;
+                }
+                __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant.identity);
+            });
+            this.room = room;
+            __classPrivateFieldSet(this, _MultimodalAgent_participant, participant, "f");
+            __classPrivateFieldSet(this, _MultimodalAgent_localSource, new AudioSource(this.model.sampleRate, this.model.numChannels), "f");
+            __classPrivateFieldSet(this, _MultimodalAgent_agentPlayout, new AgentPlayout(__classPrivateFieldGet(this, _MultimodalAgent_localSource, "f"), this.model.sampleRate, this.model.numChannels, this.model.inFrameSize, this.model.outFrameSize), "f");
+            const track = LocalAudioTrack.createAudioTrack('assistant_voice', __classPrivateFieldGet(this, _MultimodalAgent_localSource, "f"));
+            const options = new TrackPublishOptions();
+            options.source = TrackSource.SOURCE_MICROPHONE;
+            __classPrivateFieldSet(this, _MultimodalAgent_agentPublication, (await ((_a = room.localParticipant) === null || _a === void 0 ? void 0 : _a.publishTrack(track, options))) || null, "f");
+            if (!__classPrivateFieldGet(this, _MultimodalAgent_agentPublication, "f")) {
+                __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Failed to publish track');
+                reject(new Error('Failed to publish track'));
+                return;
+            }
+            await __classPrivateFieldGet(this, _MultimodalAgent_agentPublication, "f").waitForSubscription();
+            if (participant) {
+                if (typeof participant === 'string') {
+                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant);
+                }
+                else {
+                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant.identity);
+                }
+            }
+            else {
+                // No participant specified, try to find the first participant in the room
+                for (const participant of room.remoteParticipants.values()) {
+                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_linkParticipant).call(this, participant.identity);
+                    break;
+                }
+            }
+            __classPrivateFieldSet(this, _MultimodalAgent_session, this.model.session({ fncCtx: __classPrivateFieldGet(this, _MultimodalAgent_fncCtx, "f") }), "f");
+            __classPrivateFieldSet(this, _MultimodalAgent_instances, true, "a", _MultimodalAgent_started_set);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('response_content_added', (message) => {
+                var _a;
+                // openai.realtime.RealtimeContent
+                const trFwd = new BasicTranscriptionForwarder(this.room, this.room.localParticipant.identity, __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_getLocalTrackSid).call(this), message.responseId);
+                const handle = (_a = __classPrivateFieldGet(this, _MultimodalAgent_agentPlayout, "f")) === null || _a === void 0 ? void 0 : _a.play(message.itemId, message.contentIndex, trFwd, message.textStream, message.audioStream);
+                if (handle) {
+                    __classPrivateFieldSet(this, _MultimodalAgent_instances, true, "a", _MultimodalAgent_speaking_set);
+                    handle.on('done', () => {
+                        if (__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f") == handle) {
+                            __classPrivateFieldSet(this, _MultimodalAgent_instances, false, "a", _MultimodalAgent_speaking_set);
+                        }
+                    });
+                }
+                __classPrivateFieldSet(this, _MultimodalAgent_playingHandle, handle, "f");
+            });
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('input_speech_committed', (ev) => {
+                var _a, _b;
+                // openai.realtime.InputSpeechCommittedEvent
+                const participantIdentity = (_a = this.linkedParticipant) === null || _a === void 0 ? void 0 : _a.identity;
+                const trackSid = (_b = this.subscribedTrack) === null || _b === void 0 ? void 0 : _b.sid;
+                if (participantIdentity && trackSid) {
+                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_publishTranscription).call(this, participantIdentity, trackSid, '…', false, ev.itemId);
+                }
+                else {
+                    __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Participant or track not set');
+                }
+            });
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('input_speech_transcription_completed', (ev) => {
+                var _a, _b;
+                // openai.realtime.InputSpeechTranscriptionCompletedEvent
+                const transcription = ev.transcript;
+                const participantIdentity = (_a = this.linkedParticipant) === null || _a === void 0 ? void 0 : _a.identity;
+                const trackSid = (_b = this.subscribedTrack) === null || _b === void 0 ? void 0 : _b.sid;
+                if (participantIdentity && trackSid) {
+                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_publishTranscription).call(this, participantIdentity, trackSid, transcription, true, ev.itemId);
+                }
+                else {
+                    __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Participant or track not set');
+                }
+            });
+            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('input_speech_started', (ev) => {
+                var _a, _b;
+                if (__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f") && !__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").done) {
+                    __classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").interrupt();
+                    __classPrivateFieldGet(this, _MultimodalAgent_session, "f").defaultConversation.item.truncate(__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").itemId, __classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").contentIndex, Math.floor((__classPrivateFieldGet(this, _MultimodalAgent_playingHandle, "f").audioSamples / 24000) * 1000));
+                    __classPrivateFieldSet(this, _MultimodalAgent_playingHandle, undefined, "f");
+                }
+                const participantIdentity = (_a = this.linkedParticipant) === null || _a === void 0 ? void 0 : _a.identity;
+                const trackSid = (_b = this.subscribedTrack) === null || _b === void 0 ? void 0 : _b.sid;
+                if (participantIdentity && trackSid) {
+                    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_publishTranscription).call(this, participantIdentity, trackSid, '…', false, ev.itemId);
+                }
+            });
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('function_call_started', (ev) => {
+                __classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).add(ev.callId);
+                __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
+            });
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('function_call_completed', (ev) => {
+                __classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).delete(ev.callId);
+                __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
+            });
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            __classPrivateFieldGet(this, _MultimodalAgent_session, "f").on('function_call_failed', (ev) => {
+                __classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).delete(ev.callId);
+                __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
+            });
+            resolve(__classPrivateFieldGet(this, _MultimodalAgent_session, "f"));
+        });
+    }
+}
+_MultimodalAgent_participant = new WeakMap(), _MultimodalAgent_agentPublication = new WeakMap(), _MultimodalAgent_localTrackSid = new WeakMap(), _MultimodalAgent_localSource = new WeakMap(), _MultimodalAgent_agentPlayout = new WeakMap(), _MultimodalAgent_playingHandle = new WeakMap(), _MultimodalAgent_logger = new WeakMap(), _MultimodalAgent_session = new WeakMap(), _MultimodalAgent_fncCtx = new WeakMap(), _MultimodalAgent__started = new WeakMap(), _MultimodalAgent__pendingFunctionCalls = new WeakMap(), _MultimodalAgent__speaking = new WeakMap(), _MultimodalAgent_instances = new WeakSet(), _MultimodalAgent_pendingFunctionCalls_get = function _MultimodalAgent_pendingFunctionCalls_get() {
+    return __classPrivateFieldGet(this, _MultimodalAgent__pendingFunctionCalls, "f");
+}, _MultimodalAgent_pendingFunctionCalls_set = function _MultimodalAgent_pendingFunctionCalls_set(calls) {
+    __classPrivateFieldSet(this, _MultimodalAgent__pendingFunctionCalls, calls, "f");
+    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
+}, _MultimodalAgent_speaking_get = function _MultimodalAgent_speaking_get() {
+    return __classPrivateFieldGet(this, _MultimodalAgent__speaking, "f");
+}, _MultimodalAgent_speaking_set = function _MultimodalAgent_speaking_set(isSpeaking) {
+    __classPrivateFieldSet(this, _MultimodalAgent__speaking, isSpeaking, "f");
+    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
+}, _MultimodalAgent_started_get = function _MultimodalAgent_started_get() {
+    return __classPrivateFieldGet(this, _MultimodalAgent__started, "f");
+}, _MultimodalAgent_started_set = function _MultimodalAgent_started_set(started) {
+    __classPrivateFieldSet(this, _MultimodalAgent__started, started, "f");
+    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_updateState).call(this);
+}, _MultimodalAgent_linkParticipant = function _MultimodalAgent_linkParticipant(participantIdentity) {
+    if (!this.room) {
+        __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Room is not set');
+        return;
+    }
+    this.linkedParticipant = this.room.remoteParticipants.get(participantIdentity) || null;
+    if (!this.linkedParticipant) {
+        __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error(`Participant with identity ${participantIdentity} not found`);
+        return;
+    }
+    if (this.linkedParticipant.trackPublications.size > 0) {
+        __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_subscribeToMicrophone).call(this);
+    }
+    else {
+        this.room.on(RoomEvent.TrackPublished, () => {
+            __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_subscribeToMicrophone).call(this);
+        });
+    }
+}, _MultimodalAgent_subscribeToMicrophone = function _MultimodalAgent_subscribeToMicrophone() {
+    const readAudioStreamTask = async (audioStream) => {
+        var _a, e_1, _b, _c;
+        const bstream = new AudioByteStream(this.model.sampleRate, this.model.numChannels, this.model.inFrameSize);
+        try {
+            for (var _d = true, audioStream_1 = __asyncValues(audioStream), audioStream_1_1; audioStream_1_1 = await audioStream_1.next(), _a = audioStream_1_1.done, !_a; _d = true) {
+                _c = audioStream_1_1.value;
+                _d = false;
+                const frame = _c;
+                const audioData = frame.data;
+                for (const frame of bstream.write(audioData.buffer)) {
+                    __classPrivateFieldGet(this, _MultimodalAgent_session, "f").inputAudioBuffer.append(frame);
+                }
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (!_d && !_a && (_b = audioStream_1.return)) await _b.call(audioStream_1);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+    };
+    if (!this.linkedParticipant) {
+        __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Participant is not set');
+        return;
+    }
+    for (const publication of this.linkedParticipant.trackPublications.values()) {
+        if (publication.source !== TrackSource.SOURCE_MICROPHONE) {
+            continue;
+        }
+        if (!publication.subscribed) {
+            publication.setSubscribed(true);
+        }
+        const track = publication.track;
+        if (track && track !== this.subscribedTrack) {
+            this.subscribedTrack = track;
+            if (this.readMicroTask) {
+                this.readMicroTask.cancel();
+            }
+            let cancel;
+            this.readMicroTask = {
+                promise: new Promise((resolve, reject) => {
+                    cancel = () => {
+                        reject(new Error('Task cancelled'));
+                    };
+                    readAudioStreamTask(new AudioStream(track, this.model.sampleRate, this.model.numChannels))
+                        .then(resolve)
+                        .catch(reject);
+                }),
+                cancel: () => cancel(),
+            };
+        }
+    }
+}, _MultimodalAgent_getLocalTrackSid = function _MultimodalAgent_getLocalTrackSid() {
+    var _a;
+    if (!__classPrivateFieldGet(this, _MultimodalAgent_localTrackSid, "f") && this.room && this.room.localParticipant) {
+        __classPrivateFieldSet(this, _MultimodalAgent_localTrackSid, findMicroTrackId(this.room, (_a = this.room.localParticipant) === null || _a === void 0 ? void 0 : _a.identity), "f");
+    }
+    return __classPrivateFieldGet(this, _MultimodalAgent_localTrackSid, "f");
+}, _MultimodalAgent_publishTranscription = function _MultimodalAgent_publishTranscription(participantIdentity, trackSid, text, isFinal, id) {
+    var _a;
+    __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").debug(`Publishing transcription ${participantIdentity} ${trackSid} ${text} ${isFinal} ${id}`);
+    if (!((_a = this.room) === null || _a === void 0 ? void 0 : _a.localParticipant)) {
+        __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").error('Room or local participant not set');
+        return;
+    }
+    this.room.localParticipant.publishTranscription({
+        participantIdentity,
+        trackSid,
+        segments: [
+            {
+                text,
+                final: isFinal,
+                id,
+                startTime: BigInt(0),
+                endTime: BigInt(0),
+                language: '',
+            },
+        ],
+    });
+}, _MultimodalAgent_updateState = function _MultimodalAgent_updateState() {
+    let newState = 'initializing';
+    if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_pendingFunctionCalls_get).size > 0) {
+        newState = 'thinking';
+    }
+    else if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_speaking_get)) {
+        newState = 'speaking';
+    }
+    else if (__classPrivateFieldGet(this, _MultimodalAgent_instances, "a", _MultimodalAgent_started_get)) {
+        newState = 'listening';
+    }
+    __classPrivateFieldGet(this, _MultimodalAgent_instances, "m", _MultimodalAgent_setState).call(this, newState);
+}, _MultimodalAgent_setState = function _MultimodalAgent_setState(state) {
+    var _a;
+    if (((_a = this.room) === null || _a === void 0 ? void 0 : _a.isConnected) && this.room.localParticipant) {
+        const currentState = this.room.localParticipant.attributes[AGENT_STATE_ATTRIBUTE];
+        if (currentState !== state) {
+            this.room.localParticipant.setAttributes({
+                [AGENT_STATE_ATTRIBUTE]: state,
+            });
+            __classPrivateFieldGet(this, _MultimodalAgent_logger, "f").debug(`${AGENT_STATE_ATTRIBUTE}: ${currentState} ->${state}`);
+        }
+    }
+};
+//# sourceMappingURL=multimodal_agent.js.map