@livekit/agents-plugin-openai 0.2.0

package/dist/omni_assistant/proto.d.ts

@@ -0,0 +1,218 @@
+export declare enum Voice {
+    ALLOY = "alloy",
+    SHIMMER = "shimmer",
+    ECHO = "echo"
+}
+export declare enum TurnEndType {
+    SERVER_DETECTION = "server_detection",
+    CLIENT_DECISION = "client_decision"
+}
+export declare enum AudioFormat {
+    PCM16 = "pcm16"
+}
+export declare enum ServerEventType {
+    START_SESSION = "start_session",
+    ERROR = "error",
+    ADD_MESSAGE = "add_message",
+    ADD_CONTENT = "add_content",
+    MESSAGE_ADDED = "message_added",
+    VAD_SPEECH_STARTED = "vad_speech_started",
+    VAD_SPEECH_STOPPED = "vad_speech_stopped",
+    INPUT_TRANSCRIBED = "input_transcribed",
+    GENERATION_CANCELED = "generation_canceled",
+    SEND_STATE = "send_state",
+    GENERATION_FINISHED = "generation_finished"
+}
+export type ServerEvent = {
+    event: ServerEventType.START_SESSION;
+    session_id: string;
+    model: string;
+    system_fingerprint: string;
+} | {
+    event: ServerEventType.ERROR;
+    error: string;
+} | {
+    event: ServerEventType.ADD_MESSAGE;
+    previous_id: string;
+    conversation_label: string;
+    message: {
+        role: 'assistant';
+        content: ({
+            type: 'text';
+            text: string;
+        } | {
+            type: 'audio';
+            audio: string;
+        } | {
+            type: 'tool_call';
+            name: string;
+            arguments: string;
+            tool_call_id: string;
+        })[];
+    };
+} | {
+    event: ServerEventType.ADD_CONTENT;
+    message_id: string;
+    type: 'text' | 'audio' | 'tool_call';
+    data: string;
+} | {
+    event: ServerEventType.MESSAGE_ADDED;
+    id: string;
+    previous_id: string;
+    conversation_label: string;
+    content: {
+        type: 'tool_call';
+        name: string;
+        tool_call_id: string;
+        arguments: string;
+    }[] | null;
+} | {
+    event: ServerEventType.GENERATION_FINISHED;
+    reason: 'stop' | 'max_tokens' | 'content_filter' | 'interrupt';
+    conversation_label: string;
+    message_ids: string[];
+} | {
+    event: ServerEventType.SEND_STATE;
+    session_id: string;
+    input_audio_format: AudioFormat;
+    vad_active: boolean;
+    audio_buffer: string;
+    conversations: any;
+    session_config: SessionConfig;
+} | {
+    event: ServerEventType.VAD_SPEECH_STARTED | ServerEventType.VAD_SPEECH_STOPPED | ServerEventType.GENERATION_CANCELED;
+    sample_index: number;
+    message_id: string;
+} | {
+    event: ServerEventType.INPUT_TRANSCRIBED;
+    message_id: string;
+    transcript: string;
+};
+export declare enum ClientEventType {
+    UPDATE_SESSION_CONFIG = "update_session_config",
+    UPDATE_CONVERSATION_CONFIG = "update_conversation_config",
+    ADD_MESSAGE = "add_message",
+    DELETE_MESSAGE = "delete_message",
+    ADD_USER_AUDIO = "add_user_audio",
+    COMMIT_USER_AUDIO = "commit_user_audio",
+    CANCEL_GENERATION = "cancel_generation",
+    GENERATE = "generate",
+    CREATE_CONVERSATION = "create_conversation",
+    DELETE_CONVERSATION = "delete_conversation",
+    TRUNCATE_CONTENT = "truncate_content",
+    REQUEST_STATE = "request_state"
+}
+export type ClientEvent = ({
+    event: ClientEventType.UPDATE_SESSION_CONFIG;
+} & SessionConfig) | ({
+    event: ClientEventType.UPDATE_CONVERSATION_CONFIG;
+} & ConversationConfig) | {
+    event: ClientEventType.ADD_MESSAGE;
+    message: ({
+        role: 'tool';
+        tool_call_id: string;
+    } | {
+        role: 'user' | 'assistant' | 'system';
+    }) & ({
+        content: ({
+            type: 'text';
+            text: string;
+        } | {
+            type: 'tool_call';
+            name: string;
+            arguments: string;
+            tool_call_id: string;
+        })[];
+    } | {
+        role: 'user' | 'tool';
+        content: ({
+            type: 'text';
+            text: string;
+        } | {
+            type: 'tool_call';
+            name: string;
+            arguments: string;
+            tool_call_id: string;
+        } | {
+            type: 'audio';
+            audio: string;
+        })[];
+    });
+} | {
+    event: ClientEventType.DELETE_MESSAGE;
+    id: string;
+    conversation_label?: string;
+} | {
+    event: ClientEventType.ADD_USER_AUDIO;
+    data: string;
+} | {
+    event: ClientEventType.COMMIT_USER_AUDIO | ClientEventType.CANCEL_GENERATION;
+} | {
+    event: ClientEventType.GENERATE;
+    conversation_label?: string;
+} | {
+    event: ClientEventType.CREATE_CONVERSATION | ClientEventType.DELETE_CONVERSATION | ClientEventType.REQUEST_STATE;
+    label: string;
+} | {
+    event: ClientEventType.TRUNCATE_CONTENT;
+    message_id: string;
+    index: number;
+    text_chars?: number;
+    audio_samples?: number;
+};
+export declare enum ToolChoice {
+    AUTO = "auto",
+    NONE = "none",
+    REQUIRED = "required"
+}
+export interface Tool {
+    type: 'function';
+    function: {
+        name: string;
+        description: string;
+        parameters: {
+            type: 'object';
+            properties: {
+                [prop: string]: {
+                    [prop: string]: any;
+                };
+            };
+            required_properties: string[];
+        };
+    };
+}
+export declare const API_URL = "wss://api.openai.com/v1/realtime";
+export declare const SAMPLE_RATE = 24000;
+export declare const NUM_CHANNELS = 1;
+export declare const INPUT_PCM_FRAME_SIZE = 2400;
+export declare const OUTPUT_PCM_FRAME_SIZE = 1200;
+export type SessionConfig = Partial<{
+    turn_detection: 'disabled' | 'server_vad';
+    input_audio_format: AudioFormat;
+    transcribe_input: boolean;
+    vad: Partial<{
+        threshold: number;
+        prefix_padding_ms: number;
+        silence_duration_ms: number;
+    }>;
+}>;
+export type ConversationConfig = Partial<{
+    system_message: string;
+    voice: Voice;
+    subscribe_to_user_audio: boolean;
+    output_audio_format: AudioFormat;
+    tools: Tool[];
+    tool_choice: ToolChoice;
+    temperature: number;
+    max_tokens: number;
+    disable_audio: boolean;
+    transcribe_input: boolean;
+    conversation_label: string;
+}>;
+export declare enum State {
+    INITIALIZING = "initializing",
+    LISTENING = "listening",
+    THINKING = "thinking",
+    SPEAKING = "speaking"
+}
+//# sourceMappingURL=proto.d.ts.map

package/dist/omni_assistant/proto.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"proto.d.ts","sourceRoot":"","sources":["../../src/omni_assistant/proto.ts"],"names":[],"mappings":"AAGA,oBAAY,KAAK;IACf,KAAK,UAAU;IACf,OAAO,YAAY;IACnB,IAAI,SAAS;CACd;AAED,oBAAY,WAAW;IACrB,gBAAgB,qBAAqB;IACrC,eAAe,oBAAoB;CACpC;AAED,oBAAY,WAAW;IACrB,KAAK,UAAU;CAGhB;AAED,oBAAY,eAAe;IACzB,aAAa,kBAAkB;IAC/B,KAAK,UAAU;IACf,WAAW,gBAAgB;IAC3B,WAAW,gBAAgB;IAC3B,aAAa,kBAAkB;IAC/B,kBAAkB,uBAAuB;IACzC,kBAAkB,uBAAuB;IACzC,iBAAiB,sBAAsB;IACvC,mBAAmB,wBAAwB;IAC3C,UAAU,eAAe;IACzB,mBAAmB,wBAAwB;CAC5C;AAED,MAAM,MAAM,WAAW,GACnB;IACE,KAAK,EAAE,eAAe,CAAC,aAAa,CAAC;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,kBAAkB,EAAE,MAAM,CAAC;CAC5B,GACD;IACE,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;CACf,GACD;IACE,KAAK,EAAE,eAAe,CAAC,WAAW,CAAC;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,EAAE;QACP,IAAI,EAAE,WAAW,CAAC;QAClB,OAAO,EAAE,CACL;YACE,IAAI,EAAE,MAAM,CAAC;YACb,IAAI,EAAE,MAAM,CAAC;SACd,GACD;YACE,IAAI,EAAE,OAAO,CAAC;YACd,KAAK,EAAE,MAAM,CAAC;SACf,GACD;YACE,IAAI,EAAE,WAAW,CAAC;YAClB,IAAI,EAAE,MAAM,CAAC;YACb,SAAS,EAAE,MAAM,CAAC;YAClB,YAAY,EAAE,MAAM,CAAC;SACtB,CACJ,EAAE,CAAC;KACL,CAAC;CACH,GACD;IACE,KAAK,EAAE,eAAe,CAAC,WAAW,CAAC;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,WAAW,CAAC;IACrC,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,KAAK,EAAE,eAAe,CAAC,aAAa,CAAC;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,EACH;QACE,IAAI,EAAE,WAAW,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,EAAE,MAAM,CAAC;KACnB,EAAE,GACH,IAAI,CAAC;CACV,GACD;IACE,KAAK,EAAE,eAAe,CAAC,mBAAmB,CAAC;IAC3C,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,gBAAgB,GAAG,WAAW,CAAC;IAC/D,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB,GACD;IACE,KAAK,EAAE,eAAe,CAAC,UAAU,CAAC;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,WAAW,CAAC;IAChC,UAAU,EAAE,OAAO,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,GAAG,CAAC;IACnB,cAAc,EAAE,aAAa,CAAC;CAC/B,GACD;IACE,KAAK,EACD,eAAe,CAAC,kBAAkB,GAClC,eAAe,CAAC,kBAAkB,GAClC,eAAe,CAAC,mBAAmB,CAAC;IACxC,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB,GACD;IACE,KAAK,EAAE,eAAe,CAAC,iBAAiB,CAAC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEN,oBAAY,eAAe;IACzB,qBAAqB,0BAA0B;IAC/C,0BAA0B,+BAA+B;IACzD,WAAW,gBAAgB;IAC3B,cAAc,mBAAmB;IACjC,cAAc,mBAAmB;IACjC,iBAAiB,sBAAsB;IACvC,iBAAiB,sBAAsB;IACvC,QAAQ,aAAa;IACrB,mBAAmB,wBAAwB;IAC3C,mBAAmB,wBAAwB;IAC3C,gBAAgB,qBAAqB;IACrC,aAAa,kBAAkB;CAChC;AAED,MAAM,MAAM,WAAW,GACnB,CAAC;IACC,KAAK,EAAE,eAAe,CAAC,qBAAqB,CAAC;CAC9C,GAAG,aAAa,CAAC,GAClB,CAAC;IACC,KAAK,EAAE,eAAe,CAAC,0BAA0B,CAAC;CACnD,GAAG,kBAAkB,CAAC,GACvB;IACE,KAAK,EAAE,eAAe,CAAC,WAAW,CAAC;IAEnC,OAAO,EAAE,CACL;QACE,IAAI,EAAE,MAAM,CAAC;QACb,YAAY,EAAE,MAAM,CAAC;KACtB,GACD;QACE,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;KACvC,CACJ,GACC,CACI;QACE,OAAO,EAAE,CACL;YACE,IAAI,EAAE,MAAM,CAAC;YACb,IAAI,EAAE,MAAM,CAAC;SACd,GACD;YACE,IAAI,EAAE,WAAW,CAAC;YAClB,IAAI,EAAE,MAAM,CAAC;YACb,SAAS,EAAE,MAAM,CAAC;YAClB,YAAY,EAAE,MAAM,CAAC;SACtB,CACJ,EAAE,CAAC;KACL,GACD;QACE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;QACtB,OAAO,EAAE,CACL;YACE,IAAI,EAAE,MAAM,CAAC;YACb,IAAI,EAAE,MAAM,CAAC;SACd,GACD;YACE,IAAI,EAAE,WAAW,CAAC;YAClB,IAAI,EAAE,MAAM,CAAC;YACb,SAAS,EAAE,MAAM,CAAC;YAClB,YAAY,EAAE,MAAM,CAAC;SACtB,GACD;YACE,IAAI,EAAE,OAAO,CAAC;YACd,KAAK,EAAE,MAAM,CAAC;SACf,CACJ,EAAE,CAAC;KACL,CACJ,CAAC;CACL,GACD;IACE,KAAK,EAAE,eAAe,CAAC,cAAc,CAAC;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B,GACD;IACE,KAAK,EAAE,eAAe,CAAC,cAAc,CAAC;IACtC,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,KAAK,EAAE,eAAe,CAAC,iBAAiB,GAAG,eAAe,CAAC,iBAAiB,CAAC;CAC9E,GACD;IACE,KAAK,EAAE,eAAe,CAAC,QAAQ,CAAC;IAChC,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B,GACD;IACE,KAAK,EACD,eAAe,CAAC,mBAAmB,GACnC,eAAe,CAAC,mBAAmB,GACnC,eAAe,CAAC,aAAa,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC;CACf,GACD;IACE,KAAK,EAAE,eAAe,CAAC,gBAAgB,CAAC;IACxC,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC;AAEN,oBAAY,UAAU;IACpB,IAAI,SAAS;IACb,IAAI,SAAS;IACb,QAAQ,aAAa;CACtB;AAED,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE;YACV,IAAI,EAAE,QAAQ,CAAC;YACf,UAAU,EAAE;gBACV,CAAC,IAAI,EAAE,MAAM,GAAG;oBACd,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAAC;iBACrB,CAAC;aACH,CAAC;YACF,mBAAmB,EAAE,MAAM,EAAE,CAAC;SAC/B,CAAC;KACH,CAAC;CACH;AAED,eAAO,MAAM,OAAO,qCAAqC,CAAC;AAC1D,eAAO,MAAM,WAAW,QAAQ,CAAC;AACjC,eAAO,MAAM,YAAY,IAAI,CAAC;AAE9B,eAAO,MAAM,oBAAoB,OAAO,CAAC;AACzC,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAE1C,MAAM,MAAM,aAAa,GAAG,OAAO,CAAC;IAClC,cAAc,EAAE,UAAU,GAAG,YAAY,CAAC;IAC1C,kBAAkB,EAAE,WAAW,CAAC;IAChC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,GAAG,EAAE,OAAO,CAAC;QACX,SAAS,EAAE,MAAM,CAAC;QAClB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,mBAAmB,EAAE,MAAM,CAAC;KAC7B,CAAC,CAAC;CACJ,CAAC,CAAC;AAEH,MAAM,MAAM,kBAAkB,GAAG,OAAO,CAAC;IACvC,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,KAAK,CAAC;IACb,uBAAuB,EAAE,OAAO,CAAC;IACjC,mBAAmB,EAAE,WAAW,CAAC;IACjC,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,WAAW,EAAE,UAAU,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;CAC5B,CAAC,CAAC;AAEH,oBAAY,KAAK;IACf,YAAY,iBAAiB;IAC7B,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,QAAQ,aAAa;CACtB"}

package/dist/omni_assistant/proto.js

@@ -0,0 +1,68 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export var Voice;
+(function (Voice) {
+    Voice["ALLOY"] = "alloy";
+    Voice["SHIMMER"] = "shimmer";
+    Voice["ECHO"] = "echo";
+})(Voice || (Voice = {}));
+export var TurnEndType;
+(function (TurnEndType) {
+    TurnEndType["SERVER_DETECTION"] = "server_detection";
+    TurnEndType["CLIENT_DECISION"] = "client_decision";
+})(TurnEndType || (TurnEndType = {}));
+export var AudioFormat;
+(function (AudioFormat) {
+    AudioFormat["PCM16"] = "pcm16";
+    // G711_ULAW = 'g711-ulaw',
+    // G711_ALAW = 'g711-alaw',
+})(AudioFormat || (AudioFormat = {}));
+export var ServerEventType;
+(function (ServerEventType) {
+    ServerEventType["START_SESSION"] = "start_session";
+    ServerEventType["ERROR"] = "error";
+    ServerEventType["ADD_MESSAGE"] = "add_message";
+    ServerEventType["ADD_CONTENT"] = "add_content";
+    ServerEventType["MESSAGE_ADDED"] = "message_added";
+    ServerEventType["VAD_SPEECH_STARTED"] = "vad_speech_started";
+    ServerEventType["VAD_SPEECH_STOPPED"] = "vad_speech_stopped";
+    ServerEventType["INPUT_TRANSCRIBED"] = "input_transcribed";
+    ServerEventType["GENERATION_CANCELED"] = "generation_canceled";
+    ServerEventType["SEND_STATE"] = "send_state";
+    ServerEventType["GENERATION_FINISHED"] = "generation_finished";
+})(ServerEventType || (ServerEventType = {}));
+export var ClientEventType;
+(function (ClientEventType) {
+    ClientEventType["UPDATE_SESSION_CONFIG"] = "update_session_config";
+    ClientEventType["UPDATE_CONVERSATION_CONFIG"] = "update_conversation_config";
+    ClientEventType["ADD_MESSAGE"] = "add_message";
+    ClientEventType["DELETE_MESSAGE"] = "delete_message";
+    ClientEventType["ADD_USER_AUDIO"] = "add_user_audio";
+    ClientEventType["COMMIT_USER_AUDIO"] = "commit_user_audio";
+    ClientEventType["CANCEL_GENERATION"] = "cancel_generation";
+    ClientEventType["GENERATE"] = "generate";
+    ClientEventType["CREATE_CONVERSATION"] = "create_conversation";
+    ClientEventType["DELETE_CONVERSATION"] = "delete_conversation";
+    ClientEventType["TRUNCATE_CONTENT"] = "truncate_content";
+    ClientEventType["REQUEST_STATE"] = "request_state";
+})(ClientEventType || (ClientEventType = {}));
+export var ToolChoice;
+(function (ToolChoice) {
+    ToolChoice["AUTO"] = "auto";
+    ToolChoice["NONE"] = "none";
+    ToolChoice["REQUIRED"] = "required";
+})(ToolChoice || (ToolChoice = {}));
+export const API_URL = 'wss://api.openai.com/v1/realtime';
+export const SAMPLE_RATE = 24000;
+export const NUM_CHANNELS = 1;
+export const INPUT_PCM_FRAME_SIZE = 2400; // 100ms
+export const OUTPUT_PCM_FRAME_SIZE = 1200; // 50ms
+export var State;
+(function (State) {
+    State["INITIALIZING"] = "initializing";
+    State["LISTENING"] = "listening";
+    State["THINKING"] = "thinking";
+    State["SPEAKING"] = "speaking";
+})(State || (State = {}));
+//# sourceMappingURL=proto.js.map

package/dist/omni_assistant/proto.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"proto.js","sourceRoot":"","sources":["../../src/omni_assistant/proto.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,MAAM,CAAN,IAAY,KAIX;AAJD,WAAY,KAAK;IACf,wBAAe,CAAA;IACf,4BAAmB,CAAA;IACnB,sBAAa,CAAA;AACf,CAAC,EAJW,KAAK,KAAL,KAAK,QAIhB;AAED,MAAM,CAAN,IAAY,WAGX;AAHD,WAAY,WAAW;IACrB,oDAAqC,CAAA;IACrC,kDAAmC,CAAA;AACrC,CAAC,EAHW,WAAW,KAAX,WAAW,QAGtB;AAED,MAAM,CAAN,IAAY,WAIX;AAJD,WAAY,WAAW;IACrB,8BAAe,CAAA;IACf,2BAA2B;IAC3B,2BAA2B;AAC7B,CAAC,EAJW,WAAW,KAAX,WAAW,QAItB;AAED,MAAM,CAAN,IAAY,eAYX;AAZD,WAAY,eAAe;IACzB,kDAA+B,CAAA;IAC/B,kCAAe,CAAA;IACf,8CAA2B,CAAA;IAC3B,8CAA2B,CAAA;IAC3B,kDAA+B,CAAA;IAC/B,4DAAyC,CAAA;IACzC,4DAAyC,CAAA;IACzC,0DAAuC,CAAA;IACvC,8DAA2C,CAAA;IAC3C,4CAAyB,CAAA;IACzB,8DAA2C,CAAA;AAC7C,CAAC,EAZW,eAAe,KAAf,eAAe,QAY1B;AAsFD,MAAM,CAAN,IAAY,eAaX;AAbD,WAAY,eAAe;IACzB,kEAA+C,CAAA;IAC/C,4EAAyD,CAAA;IACzD,8CAA2B,CAAA;IAC3B,oDAAiC,CAAA;IACjC,oDAAiC,CAAA;IACjC,0DAAuC,CAAA;IACvC,0DAAuC,CAAA;IACvC,wCAAqB,CAAA;IACrB,8DAA2C,CAAA;IAC3C,8DAA2C,CAAA;IAC3C,wDAAqC,CAAA;IACrC,kDAA+B,CAAA;AACjC,CAAC,EAbW,eAAe,KAAf,eAAe,QAa1B;AAwFD,MAAM,CAAN,IAAY,UAIX;AAJD,WAAY,UAAU;IACpB,2BAAa,CAAA;IACb,2BAAa,CAAA;IACb,mCAAqB,CAAA;AACvB,CAAC,EAJW,UAAU,KAAV,UAAU,QAIrB;AAmBD,MAAM,CAAC,MAAM,OAAO,GAAG,kCAAkC,CAAC;AAC1D,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,CAAC;AACjC,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,CAAC;AAE9B,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,CAAC,CAAC,QAAQ;AAClD,MAAM,CAAC,MAAM,qBAAqB,GAAG,IAAI,CAAC,CAAC,OAAO;AA2BlD,MAAM,CAAN,IAAY,KAKX;AALD,WAAY,KAAK;IACf,sCAA6B,CAAA;IAC7B,gCAAuB,CAAA;IACvB,8BAAqB,CAAA;IACrB,8BAAqB,CAAA;AACvB,CAAC,EALW,KAAK,KAAL,KAAK,QAKhB"}

package/dist/omni_assistant/transcription_forwarder.d.ts

@@ -0,0 +1,28 @@
+import type { AudioFrame, Room } from '@livekit/rtc-node';
+export interface TranscriptionForwarder {
+    start(): void;
+    pushAudio(frame: AudioFrame): void;
+    pushText(text: string): void;
+    markTextComplete(): void;
+    markAudioComplete(): void;
+    close(interrupt: boolean): Promise<void>;
+    currentCharacterIndex: number;
+}
+export declare class BasicTranscriptionForwarder implements TranscriptionForwarder {
+    #private;
+    currentCharacterIndex: number;
+    constructor(room: Room, participantIdentity: string, trackSid: string, messageId: string);
+    start(): void;
+    pushAudio(frame: AudioFrame): void;
+    pushText(text: string): void;
+    private textIsComplete;
+    private audioIsComplete;
+    markTextComplete(): void;
+    markAudioComplete(): void;
+    private adjustTimingIfBothFinished;
+    private computeSleepInterval;
+    private startPublishingLoop;
+    private publishTranscription;
+    close(interrupt: boolean): Promise<void>;
+}
+//# sourceMappingURL=transcription_forwarder.d.ts.map

package/dist/omni_assistant/transcription_forwarder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcription_forwarder.d.ts","sourceRoot":"","sources":["../../src/omni_assistant/transcription_forwarder.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,mBAAmB,CAAC;AAE1D,MAAM,WAAW,sBAAsB;IACrC,KAAK,IAAI,IAAI,CAAC;IACd,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,IAAI,IAAI,CAAC;IACzB,iBAAiB,IAAI,IAAI,CAAC;IAC1B,KAAK,CAAC,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzC,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,qBAAa,2BAA4B,YAAW,sBAAsB;;IAWxE,qBAAqB,EAAE,MAAM,CAAK;gBAEtB,IAAI,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IAOxF,KAAK,IAAI,IAAI;IAUb,SAAS,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAIlC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAI5B,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,eAAe,CAAkB;IAEzC,gBAAgB,IAAI,IAAI;IAKxB,iBAAiB,IAAI,IAAI;IAKzB,OAAO,CAAC,0BAA0B;IASlC,OAAO,CAAC,oBAAoB;YAId,mBAAmB;YAoBnB,oBAAoB;IAkB5B,KAAK,CAAC,SAAS,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;CAS/C"}

package/dist/omni_assistant/transcription_forwarder.js

@@ -0,0 +1,117 @@
+var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
+    if (kind === "m") throw new TypeError("Private method is not writable");
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
+    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
+};
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var _BasicTranscriptionForwarder_room, _BasicTranscriptionForwarder_participantIdentity, _BasicTranscriptionForwarder_trackSid, _BasicTranscriptionForwarder_currentText, _BasicTranscriptionForwarder_totalAudioDuration, _BasicTranscriptionForwarder_currentPlayoutTime, _BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND, _BasicTranscriptionForwarder_charsPerSecond, _BasicTranscriptionForwarder_messageId, _BasicTranscriptionForwarder_isRunning;
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log } from '@livekit/agents';
+export class BasicTranscriptionForwarder {
+    constructor(room, participantIdentity, trackSid, messageId) {
+        _BasicTranscriptionForwarder_room.set(this, void 0);
+        _BasicTranscriptionForwarder_participantIdentity.set(this, void 0);
+        _BasicTranscriptionForwarder_trackSid.set(this, void 0);
+        _BasicTranscriptionForwarder_currentText.set(this, '');
+        _BasicTranscriptionForwarder_totalAudioDuration.set(this, 0);
+        _BasicTranscriptionForwarder_currentPlayoutTime.set(this, 0);
+        _BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND.set(this, 16);
+        _BasicTranscriptionForwarder_charsPerSecond.set(this, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND, "f"));
+        _BasicTranscriptionForwarder_messageId.set(this, void 0);
+        _BasicTranscriptionForwarder_isRunning.set(this, false);
+        this.currentCharacterIndex = 0;
+        this.textIsComplete = false;
+        this.audioIsComplete = false;
+        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_room, room, "f");
+        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_participantIdentity, participantIdentity, "f");
+        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_trackSid, trackSid, "f");
+        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_messageId, messageId, "f");
+    }
+    start() {
+        if (!__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f")) {
+            __classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, true, "f");
+            this.startPublishingLoop().catch((error) => {
+                log().error('Error in publishing loop:', error);
+                __classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, false, "f");
+            });
+        }
+    }
+    pushAudio(frame) {
+        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_totalAudioDuration, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_totalAudioDuration, "f") + frame.samplesPerChannel / frame.sampleRate, "f");
+    }
+    pushText(text) {
+        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_currentText, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f") + text, "f");
+    }
+    markTextComplete() {
+        this.textIsComplete = true;
+        this.adjustTimingIfBothFinished();
+    }
+    markAudioComplete() {
+        this.audioIsComplete = true;
+        this.adjustTimingIfBothFinished();
+    }
+    adjustTimingIfBothFinished() {
+        if (this.textIsComplete && this.audioIsComplete) {
+            const actualDuration = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_totalAudioDuration, "f");
+            if (actualDuration > 0 && __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length > 0) {
+                __classPrivateFieldSet(this, _BasicTranscriptionForwarder_charsPerSecond, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length / actualDuration, "f");
+            }
+        }
+    }
+    computeSleepInterval() {
+        return Math.min(Math.max(1 / __classPrivateFieldGet(this, _BasicTranscriptionForwarder_charsPerSecond, "f"), 0.0625), 0.5);
+    }
+    async startPublishingLoop() {
+        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, true, "f");
+        let sleepInterval = this.computeSleepInterval();
+        let isComplete = false;
+        while (__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f") && !isComplete) {
+            __classPrivateFieldSet(this, _BasicTranscriptionForwarder_currentPlayoutTime, __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentPlayoutTime, "f") + sleepInterval, "f");
+            this.currentCharacterIndex = Math.floor(__classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentPlayoutTime, "f") * __classPrivateFieldGet(this, _BasicTranscriptionForwarder_charsPerSecond, "f"));
+            isComplete = this.textIsComplete && this.currentCharacterIndex >= __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length;
+            await this.publishTranscription(false);
+            if (__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f") && !isComplete) {
+                sleepInterval = this.computeSleepInterval();
+                await new Promise((resolve) => setTimeout(resolve, sleepInterval * 1000));
+            }
+        }
+        if (__classPrivateFieldGet(this, _BasicTranscriptionForwarder_isRunning, "f")) {
+            this.close(false);
+        }
+    }
+    async publishTranscription(final) {
+        var _a;
+        const textToPublish = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").slice(0, this.currentCharacterIndex);
+        await ((_a = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_room, "f").localParticipant) === null || _a === void 0 ? void 0 : _a.publishTranscription({
+            participantIdentity: __classPrivateFieldGet(this, _BasicTranscriptionForwarder_participantIdentity, "f"),
+            trackSid: __classPrivateFieldGet(this, _BasicTranscriptionForwarder_trackSid, "f"),
+            segments: [
+                {
+                    text: textToPublish,
+                    final: final,
+                    id: __classPrivateFieldGet(this, _BasicTranscriptionForwarder_messageId, "f"),
+                    startTime: BigInt(0),
+                    endTime: BigInt(0),
+                    language: '',
+                },
+            ],
+        }));
+    }
+    async close(interrupt) {
+        __classPrivateFieldSet(this, _BasicTranscriptionForwarder_isRunning, false, "f");
+        // Publish whatever we had as final
+        if (!interrupt) {
+            this.currentCharacterIndex = __classPrivateFieldGet(this, _BasicTranscriptionForwarder_currentText, "f").length;
+        }
+        await this.publishTranscription(true);
+    }
+}
+_BasicTranscriptionForwarder_room = new WeakMap(), _BasicTranscriptionForwarder_participantIdentity = new WeakMap(), _BasicTranscriptionForwarder_trackSid = new WeakMap(), _BasicTranscriptionForwarder_currentText = new WeakMap(), _BasicTranscriptionForwarder_totalAudioDuration = new WeakMap(), _BasicTranscriptionForwarder_currentPlayoutTime = new WeakMap(), _BasicTranscriptionForwarder_DEFAULT_CHARS_PER_SECOND = new WeakMap(), _BasicTranscriptionForwarder_charsPerSecond = new WeakMap(), _BasicTranscriptionForwarder_messageId = new WeakMap(), _BasicTranscriptionForwarder_isRunning = new WeakMap();
+//# sourceMappingURL=transcription_forwarder.js.map

package/dist/omni_assistant/transcription_forwarder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcription_forwarder.js","sourceRoot":"","sources":["../../src/omni_assistant/transcription_forwarder.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,6CAA6C;AAC7C,EAAE;AACF,sCAAsC;AACtC,OAAO,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAC;AAatC,MAAM,OAAO,2BAA2B;IAatC,YAAY,IAAU,EAAE,mBAA2B,EAAE,QAAgB,EAAE,SAAiB;QAZxF,oDAAY;QACZ,mEAA6B;QAC7B,wDAAkB;QAClB,mDAAuB,EAAE,EAAC;QAC1B,0DAA8B,CAAC,EAAC;QAChC,0DAA8B,CAAC,EAAC;QAChC,gEAA4B,EAAE,EAAC;QAC/B,sDAA0B,uBAAA,IAAI,6DAA0B,EAAC;QACzD,yDAAmB;QACnB,iDAAsB,KAAK,EAAC;QAC5B,0BAAqB,GAAW,CAAC,CAAC;QA2B1B,mBAAc,GAAY,KAAK,CAAC;QAChC,oBAAe,GAAY,KAAK,CAAC;QAzBvC,uBAAA,IAAI,qCAAS,IAAI,MAAA,CAAC;QAClB,uBAAA,IAAI,oDAAwB,mBAAmB,MAAA,CAAC;QAChD,uBAAA,IAAI,yCAAa,QAAQ,MAAA,CAAC;QAC1B,uBAAA,IAAI,0CAAc,SAAS,MAAA,CAAC;IAC9B,CAAC;IAED,KAAK;QACH,IAAI,CAAC,uBAAA,IAAI,8CAAW,EAAE,CAAC;YACrB,uBAAA,IAAI,0CAAc,IAAI,MAAA,CAAC;YACvB,IAAI,CAAC,mBAAmB,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACzC,GAAG,EAAE,CAAC,KAAK,CAAC,2BAA2B,EAAE,KAAK,CAAC,CAAC;gBAChD,uBAAA,IAAI,0CAAc,KAAK,MAAA,CAAC;YAC1B,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,SAAS,CAAC,KAAiB;QACzB,mKAA4B,KAAK,CAAC,iBAAiB,GAAG,KAAK,CAAC,UAAU,MAAA,CAAC;IACzE,CAAC;IAED,QAAQ,CAAC,IAAY;QACnB,qJAAqB,IAAI,MAAA,CAAC;IAC5B,CAAC;IAKD,gBAAgB;QACd,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,0BAA0B,EAAE,CAAC;IACpC,CAAC;IAED,iBAAiB;QACf,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5B,IAAI,CAAC,0BAA0B,EAAE,CAAC;IACpC,CAAC;IAEO,0BAA0B;QAChC,IAAI,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,uBAAA,IAAI,uDAAoB,CAAC;YAChD,IAAI,cAAc,GAAG,CAAC,IAAI,uBAAA,IAAI,gDAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvD,uBAAA,IAAI,+CAAmB,uBAAA,IAAI,gDAAa,CAAC,MAAM,GAAG,cAAc,MAAA,CAAC;YACnE,CAAC;QACH,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,uBAAA,IAAI,mDAAgB,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,uBAAA,IAAI,0CAAc,IAAI,MAAA,CAAC;QACvB,IAAI,aAAa,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QAChD,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,OAAO,uBAAA,IAAI,8CAAW,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,mKAA4B,aAAa,MAAA,CAAC;YAC1C,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC,KAAK,CAAC,uBAAA,IAAI,uDAAoB,GAAG,uBAAA,IAAI,mDAAgB,CAAC,CAAC;YACzF,UAAU,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,qBAAqB,IAAI,uBAAA,IAAI,gDAAa,CAAC,MAAM,CAAC;YAC3F,MAAM,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;YACvC,IAAI,uBAAA,IAAI,8CAAW,IAAI,CAAC,UAAU,EAAE,CAAC;gBACnC,aAAa,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;gBAC5C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QAED,IAAI,uBAAA,IAAI,8CAAW,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,oBAAoB,CAAC,KAAc;;QAC/C,MAAM,aAAa,GAAG,uBAAA,IAAI,gDAAa,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAC7E,MAAM,CAAA,MAAA,uBAAA,IAAI,yCAAM,CAAC,gBAAgB,0CAAE,oBAAoB,CAAC;YACtD,mBAAmB,EAAE,uBAAA,IAAI,wDAAqB;YAC9C,QAAQ,EAAE,uBAAA,IAAI,6CAAU;YACxB,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,aAAa;oBACnB,KAAK,EAAE,KAAK;oBACZ,EAAE,EAAE,uBAAA,IAAI,8CAAW;oBACnB,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;oBACpB,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;oBAClB,QAAQ,EAAE,EAAE;iBACb;aACF;SACF,CAAC,CAAA,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,SAAkB;QAC5B,uBAAA,IAAI,0CAAc,KAAK,MAAA,CAAC;QAExB,mCAAmC;QACnC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,IAAI,CAAC,qBAAqB,GAAG,uBAAA,IAAI,gDAAa,CAAC,MAAM,CAAC;QACxD,CAAC;QACD,MAAM,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;CACF"}

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@livekit/agents-plugin-openai",
+  "version": "0.2.0",
+  "description": "OpenAI plugin for LiveKit Node Agents",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "author": "LiveKit",
+  "type": "module",
+  "devDependencies": {
+    "@microsoft/api-extractor": "^7.35.0",
+    "@types/ws": "^8.5.10",
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "@livekit/rtc-node": "^0.8.1",
+    "ws": "^8.16.0",
+    "@livekit/agents": "0.2.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  }
+}

package/src/index.ts

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export * from './omni_assistant/index.js';

package/src/omni_assistant/agent_playout.ts

@@ -0,0 +1,127 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { AudioByteStream } from '@livekit/agents';
+import { Queue } from '@livekit/agents';
+import { AudioFrame, type AudioSource } from '@livekit/rtc-node';
+import { EventEmitter } from 'events';
+import * as proto from './proto.js';
+import type { TranscriptionForwarder } from './transcription_forwarder';
+
+export class AgentPlayout {
+  #audioSource: AudioSource;
+  #currentPlayoutHandle: PlayoutHandle | null;
+  #currentPlayoutTask: Promise<void> | null;
+
+  constructor(audioSource: AudioSource) {
+    this.#audioSource = audioSource;
+    this.#currentPlayoutHandle = null;
+    this.#currentPlayoutTask = null;
+  }
+
+  play(messageId: string, transcriptionFwd: TranscriptionForwarder): PlayoutHandle {
+    if (this.#currentPlayoutHandle) {
+      this.#currentPlayoutHandle.interrupt();
+    }
+    this.#currentPlayoutHandle = new PlayoutHandle(messageId, transcriptionFwd);
+    this.#currentPlayoutTask = this.playoutTask(
+      this.#currentPlayoutTask,
+      this.#currentPlayoutHandle,
+    );
+    return this.#currentPlayoutHandle;
+  }
+
+  private async playoutTask(oldTask: Promise<void> | null, handle: PlayoutHandle): Promise<void> {
+    let firstFrame = true;
+    try {
+      const bstream = new AudioByteStream(
+        proto.SAMPLE_RATE,
+        proto.NUM_CHANNELS,
+        proto.OUTPUT_PCM_FRAME_SIZE,
+      );
+
+      while (!handle.interrupted) {
+        const frame = await handle.playoutQueue.get();
+        if (frame === null) break;
+        if (firstFrame) {
+          handle.transcriptionFwd.start();
+          firstFrame = false;
+        }
+
+        for (const f of bstream.write(frame.data.buffer)) {
+          handle.playedAudioSamples += f.samplesPerChannel;
+          if (handle.interrupted) break;
+
+          await this.#audioSource.captureFrame(f);
+        }
+      }
+
+      if (!handle.interrupted) {
+        for (const f of bstream.flush()) {
+          await this.#audioSource.captureFrame(f);
+        }
+      }
+    } finally {
+      if (!firstFrame && !handle.interrupted) {
+        handle.transcriptionFwd.markTextComplete();
+      }
+      await handle.transcriptionFwd.close(handle.interrupted);
+      handle.complete();
+    }
+  }
+}
+
+export class PlayoutHandle extends EventEmitter {
+  messageId: string;
+  transcriptionFwd: TranscriptionForwarder;
+  playedAudioSamples: number;
+  done: boolean;
+  interrupted: boolean;
+  playoutQueue: Queue<AudioFrame | null>;
+
+  constructor(messageId: string, transcriptionFwd: TranscriptionForwarder) {
+    super();
+    this.messageId = messageId;
+    this.transcriptionFwd = transcriptionFwd;
+    this.playedAudioSamples = 0;
+    this.done = false;
+    this.interrupted = false;
+    this.playoutQueue = new Queue<AudioFrame | null>();
+  }
+
+  pushAudio(data: Uint8Array) {
+    const frame = new AudioFrame(
+      new Int16Array(data.buffer),
+      proto.SAMPLE_RATE,
+      proto.NUM_CHANNELS,
+      data.length / 2,
+    );
+    this.transcriptionFwd.pushAudio(frame);
+    this.playoutQueue.put(frame);
+  }
+
+  pushText(text: string) {
+    this.transcriptionFwd.pushText(text);
+  }
+
+  endInput() {
+    this.transcriptionFwd.markAudioComplete();
+    this.transcriptionFwd.markTextComplete();
+    this.playoutQueue.put(null);
+  }
+
+  interrupt() {
+    if (this.done) return;
+    this.interrupted = true;
+  }
+
+  publishedTextChars(): number {
+    return this.transcriptionFwd.currentCharacterIndex;
+  }
+
+  complete() {
+    if (this.done) return;
+    this.done = true;
+    this.emit('complete', this.interrupted);
+  }
+}