@livekit/agents-plugin-openai 0.2.0

package/src/omni_assistant/proto.ts

@@ -0,0 +1,280 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+export enum Voice {
+  ALLOY = 'alloy',
+  SHIMMER = 'shimmer',
+  ECHO = 'echo',
+}
+
+export enum TurnEndType {
+  SERVER_DETECTION = 'server_detection',
+  CLIENT_DECISION = 'client_decision',
+}
+
+export enum AudioFormat {
+  PCM16 = 'pcm16',
+  // G711_ULAW = 'g711-ulaw',
+  // G711_ALAW = 'g711-alaw',
+}
+
+export enum ServerEventType {
+  START_SESSION = 'start_session',
+  ERROR = 'error',
+  ADD_MESSAGE = 'add_message',
+  ADD_CONTENT = 'add_content',
+  MESSAGE_ADDED = 'message_added',
+  VAD_SPEECH_STARTED = 'vad_speech_started',
+  VAD_SPEECH_STOPPED = 'vad_speech_stopped',
+  INPUT_TRANSCRIBED = 'input_transcribed',
+  GENERATION_CANCELED = 'generation_canceled',
+  SEND_STATE = 'send_state',
+  GENERATION_FINISHED = 'generation_finished',
+}
+
+export type ServerEvent =
+  | {
+      event: ServerEventType.START_SESSION;
+      session_id: string;
+      model: string;
+      system_fingerprint: string;
+    }
+  | {
+      event: ServerEventType.ERROR;
+      error: string;
+    }
+  | {
+      event: ServerEventType.ADD_MESSAGE;
+      previous_id: string;
+      conversation_label: string;
+      message: {
+        role: 'assistant';
+        content: (
+          | {
+              type: 'text';
+              text: string;
+            }
+          | {
+              type: 'audio';
+              audio: string;
+            }
+          | {
+              type: 'tool_call';
+              name: string;
+              arguments: string;
+              tool_call_id: string;
+            }
+        )[];
+      };
+    }
+  | {
+      event: ServerEventType.ADD_CONTENT;
+      message_id: string;
+      type: 'text' | 'audio' | 'tool_call';
+      data: string; // text or base64 audio or JSON stringified object
+    }
+  | {
+      event: ServerEventType.MESSAGE_ADDED;
+      id: string;
+      previous_id: string;
+      conversation_label: string;
+      content:
+        | {
+            type: 'tool_call';
+            name: string;
+            tool_call_id: string;
+            arguments: string; // JSON stringified object
+          }[]
+        | null;
+    }
+  | {
+      event: ServerEventType.GENERATION_FINISHED;
+      reason: 'stop' | 'max_tokens' | 'content_filter' | 'interrupt';
+      conversation_label: string;
+      message_ids: string[];
+    }
+  | {
+      event: ServerEventType.SEND_STATE;
+      session_id: string;
+      input_audio_format: AudioFormat;
+      vad_active: boolean;
+      audio_buffer: string;
+      conversations: any; // TODO(nbsp): get this
+      session_config: SessionConfig;
+    }
+  | {
+      event:
+        | ServerEventType.VAD_SPEECH_STARTED
+        | ServerEventType.VAD_SPEECH_STOPPED
+        | ServerEventType.GENERATION_CANCELED;
+      sample_index: number;
+      message_id: string;
+    }
+  | {
+      event: ServerEventType.INPUT_TRANSCRIBED;
+      message_id: string;
+      transcript: string;
+    };
+
+export enum ClientEventType {
+  UPDATE_SESSION_CONFIG = 'update_session_config',
+  UPDATE_CONVERSATION_CONFIG = 'update_conversation_config',
+  ADD_MESSAGE = 'add_message',
+  DELETE_MESSAGE = 'delete_message',
+  ADD_USER_AUDIO = 'add_user_audio',
+  COMMIT_USER_AUDIO = 'commit_user_audio',
+  CANCEL_GENERATION = 'cancel_generation',
+  GENERATE = 'generate',
+  CREATE_CONVERSATION = 'create_conversation',
+  DELETE_CONVERSATION = 'delete_conversation',
+  TRUNCATE_CONTENT = 'truncate_content',
+  REQUEST_STATE = 'request_state',
+}
+
+export type ClientEvent =
+  | ({
+      event: ClientEventType.UPDATE_SESSION_CONFIG;
+    } & SessionConfig)
+  | ({
+      event: ClientEventType.UPDATE_CONVERSATION_CONFIG;
+    } & ConversationConfig)
+  | {
+      event: ClientEventType.ADD_MESSAGE;
+      // id, previous_id, conversation_label are unused by us
+      message: (
+        | {
+            role: 'tool';
+            tool_call_id: string;
+          }
+        | {
+            role: 'user' | 'assistant' | 'system';
+          }
+      ) &
+        (
+          | {
+              content: (
+                | {
+                    type: 'text';
+                    text: string;
+                  }
+                | {
+                    type: 'tool_call';
+                    name: string;
+                    arguments: string;
+                    tool_call_id: string;
+                  }
+              )[];
+            }
+          | {
+              role: 'user' | 'tool';
+              content: (
+                | {
+                    type: 'text';
+                    text: string;
+                  }
+                | {
+                    type: 'tool_call';
+                    name: string;
+                    arguments: string;
+                    tool_call_id: string;
+                  }
+                | {
+                    type: 'audio';
+                    audio: string; // base64 encoded buffer
+                  }
+              )[];
+            }
+        );
+    }
+  | {
+      event: ClientEventType.DELETE_MESSAGE;
+      id: string;
+      conversation_label?: string; // defaults to 'default'
+    }
+  | {
+      event: ClientEventType.ADD_USER_AUDIO;
+      data: string; // base64 encoded buffer
+    }
+  | {
+      event: ClientEventType.COMMIT_USER_AUDIO | ClientEventType.CANCEL_GENERATION;
+    }
+  | {
+      event: ClientEventType.GENERATE;
+      conversation_label?: string; // defaults to 'default'
+    }
+  | {
+      event:
+        | ClientEventType.CREATE_CONVERSATION
+        | ClientEventType.DELETE_CONVERSATION
+        | ClientEventType.REQUEST_STATE;
+      label: string;
+    }
+  | {
+      event: ClientEventType.TRUNCATE_CONTENT;
+      message_id: string;
+      index: number; // integer, ignored
+      text_chars?: number; // integer
+      audio_samples?: number; // integer
+    };
+
+export enum ToolChoice {
+  AUTO = 'auto',
+  NONE = 'none',
+  REQUIRED = 'required',
+}
+
+export interface Tool {
+  type: 'function';
+  function: {
+    name: string;
+    description: string;
+    parameters: {
+      type: 'object';
+      properties: {
+        [prop: string]: {
+          [prop: string]: any;
+        };
+      };
+      required_properties: string[];
+    };
+  };
+}
+
+export const API_URL = 'wss://api.openai.com/v1/realtime';
+export const SAMPLE_RATE = 24000;
+export const NUM_CHANNELS = 1;
+
+export const INPUT_PCM_FRAME_SIZE = 2400; // 100ms
+export const OUTPUT_PCM_FRAME_SIZE = 1200; // 50ms
+
+export type SessionConfig = Partial<{
+  turn_detection: 'disabled' | 'server_vad';
+  input_audio_format: AudioFormat;
+  transcribe_input: boolean;
+  vad: Partial<{
+    threshold: number; // 0..1 inclusive, default 0.5
+    prefix_padding_ms: number; // default 300
+    silence_duration_ms: number; // default 200
+  }>;
+}>;
+
+export type ConversationConfig = Partial<{
+  system_message: string;
+  voice: Voice;
+  subscribe_to_user_audio: boolean;
+  output_audio_format: AudioFormat;
+  tools: Tool[];
+  tool_choice: ToolChoice;
+  temperature: number; // 0.6..1.2 inclusive, default 0.8
+  max_tokens: number; // 1..4096, default 2048;
+  disable_audio: boolean;
+  transcribe_input: boolean;
+  conversation_label: string; // default "default"
+}>;
+
+export enum State {
+  INITIALIZING = 'initializing',
+  LISTENING = 'listening',
+  THINKING = 'thinking',
+  SPEAKING = 'speaking',
+}

package/src/omni_assistant/transcription_forwarder.ts

@@ -0,0 +1,128 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { log } from '@livekit/agents';
+import type { AudioFrame, Room } from '@livekit/rtc-node';
+
+export interface TranscriptionForwarder {
+  start(): void;
+  pushAudio(frame: AudioFrame): void;
+  pushText(text: string): void;
+  markTextComplete(): void;
+  markAudioComplete(): void;
+  close(interrupt: boolean): Promise<void>;
+  currentCharacterIndex: number;
+}
+
+export class BasicTranscriptionForwarder implements TranscriptionForwarder {
+  #room: Room;
+  #participantIdentity: string;
+  #trackSid: string;
+  #currentText: string = '';
+  #totalAudioDuration: number = 0;
+  #currentPlayoutTime: number = 0;
+  #DEFAULT_CHARS_PER_SECOND = 16;
+  #charsPerSecond: number = this.#DEFAULT_CHARS_PER_SECOND;
+  #messageId: string;
+  #isRunning: boolean = false;
+  currentCharacterIndex: number = 0;
+
+  constructor(room: Room, participantIdentity: string, trackSid: string, messageId: string) {
+    this.#room = room;
+    this.#participantIdentity = participantIdentity;
+    this.#trackSid = trackSid;
+    this.#messageId = messageId;
+  }
+
+  start(): void {
+    if (!this.#isRunning) {
+      this.#isRunning = true;
+      this.startPublishingLoop().catch((error) => {
+        log().error('Error in publishing loop:', error);
+        this.#isRunning = false;
+      });
+    }
+  }
+
+  pushAudio(frame: AudioFrame): void {
+    this.#totalAudioDuration += frame.samplesPerChannel / frame.sampleRate;
+  }
+
+  pushText(text: string): void {
+    this.#currentText += text;
+  }
+
+  private textIsComplete: boolean = false;
+  private audioIsComplete: boolean = false;
+
+  markTextComplete(): void {
+    this.textIsComplete = true;
+    this.adjustTimingIfBothFinished();
+  }
+
+  markAudioComplete(): void {
+    this.audioIsComplete = true;
+    this.adjustTimingIfBothFinished();
+  }
+
+  private adjustTimingIfBothFinished(): void {
+    if (this.textIsComplete && this.audioIsComplete) {
+      const actualDuration = this.#totalAudioDuration;
+      if (actualDuration > 0 && this.#currentText.length > 0) {
+        this.#charsPerSecond = this.#currentText.length / actualDuration;
+      }
+    }
+  }
+
+  private computeSleepInterval(): number {
+    return Math.min(Math.max(1 / this.#charsPerSecond, 0.0625), 0.5);
+  }
+
+  private async startPublishingLoop(): Promise<void> {
+    this.#isRunning = true;
+    let sleepInterval = this.computeSleepInterval();
+    let isComplete = false;
+    while (this.#isRunning && !isComplete) {
+      this.#currentPlayoutTime += sleepInterval;
+      this.currentCharacterIndex = Math.floor(this.#currentPlayoutTime * this.#charsPerSecond);
+      isComplete = this.textIsComplete && this.currentCharacterIndex >= this.#currentText.length;
+      await this.publishTranscription(false);
+      if (this.#isRunning && !isComplete) {
+        sleepInterval = this.computeSleepInterval();
+        await new Promise((resolve) => setTimeout(resolve, sleepInterval * 1000));
+      }
+    }
+
+    if (this.#isRunning) {
+      this.close(false);
+    }
+  }
+
+  private async publishTranscription(final: boolean): Promise<void> {
+    const textToPublish = this.#currentText.slice(0, this.currentCharacterIndex);
+    await this.#room.localParticipant?.publishTranscription({
+      participantIdentity: this.#participantIdentity,
+      trackSid: this.#trackSid,
+      segments: [
+        {
+          text: textToPublish,
+          final: final,
+          id: this.#messageId,
+          startTime: BigInt(0),
+          endTime: BigInt(0),
+          language: '',
+        },
+      ],
+    });
+  }
+
+  async close(interrupt: boolean): Promise<void> {
+    this.#isRunning = false;
+
+    // Publish whatever we had as final
+    if (!interrupt) {
+      this.currentCharacterIndex = this.#currentText.length;
+    }
+    await this.publishTranscription(true);
+  }
+}

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.json",
+  "include": ["./src"],
+  "compilerOptions": {
+    // match output dir to input dir. e.g. dist/index instead of dist/src/index
+    "rootDir": "./src",
+    "declarationDir": "./dist",
+    "outDir": "./dist"
+  },
+  "typedocOptions": {
+    "name": "plugins/agents-plugin-openai",
+    "entryPointStrategy": "resolve",
+    "entryPoints": ["src/index.ts"]
+  }
+}