@kernl-sdk/openai 0.1.0

package/src/convert/types.ts

@@ -0,0 +1,344 @@
+import type { JSONSchema7 } from "json-schema";
+
+/**
+ * OpenAI Realtime API wire types (GA).
+ */
+
+// =============================================================================
+// Client Events
+// =============================================================================
+
+export type OpenAIClientEvent =
+  | OpenAISessionUpdate
+  | OpenAIInputAudioBufferAppend
+  | OpenAIInputAudioBufferCommit
+  | OpenAIInputAudioBufferClear
+  | OpenAIConversationItemCreate
+  | OpenAIConversationItemDelete
+  | OpenAIConversationItemTruncate
+  | OpenAIResponseCreate
+  | OpenAIResponseCancel;
+
+export interface OpenAISessionUpdate {
+  type: "session.update";
+  session: OpenAISessionConfig;
+}
+
+export interface OpenAIInputAudioBufferAppend {
+  type: "input_audio_buffer.append";
+  audio: string;
+}
+
+export interface OpenAIInputAudioBufferCommit {
+  type: "input_audio_buffer.commit";
+}
+
+export interface OpenAIInputAudioBufferClear {
+  type: "input_audio_buffer.clear";
+}
+
+export interface OpenAIConversationItemCreate {
+  type: "conversation.item.create";
+  item: OpenAIItem;
+  previous_item_id?: string;
+}
+
+export interface OpenAIConversationItemDelete {
+  type: "conversation.item.delete";
+  item_id: string;
+}
+
+export interface OpenAIConversationItemTruncate {
+  type: "conversation.item.truncate";
+  item_id: string;
+  content_index: number;
+  audio_end_ms: number;
+}
+
+export interface OpenAIResponseCreate {
+  type: "response.create";
+  response?: {
+    instructions?: string;
+    modalities?: ("text" | "audio")[];
+  };
+}
+
+export interface OpenAIResponseCancel {
+  type: "response.cancel";
+  response_id?: string;
+}
+
+// =============================================================================
+// Server Events
+// =============================================================================
+
+export type OpenAIServerEvent =
+  | OpenAISessionCreated
+  | OpenAISessionUpdated
+  | OpenAIError
+  | OpenAIInputAudioBufferCommitted
+  | OpenAIInputAudioBufferCleared
+  | OpenAIInputAudioBufferSpeechStarted
+  | OpenAIInputAudioBufferSpeechStopped
+  | OpenAIConversationItemCreated
+  | OpenAIConversationItemDone
+  | OpenAIConversationItemDeleted
+  | OpenAIConversationItemTruncated
+  | OpenAIResponseCreated
+  | OpenAIResponseDone
+  | OpenAIResponseOutputAudioDelta
+  | OpenAIResponseOutputAudioDone
+  | OpenAIResponseTextDelta
+  | OpenAIResponseTextDone
+  | OpenAIInputAudioTranscriptionDelta
+  | OpenAIInputAudioTranscriptionCompleted
+  | OpenAIResponseOutputAudioTranscriptDelta
+  | OpenAIResponseOutputAudioTranscriptDone
+  | OpenAIResponseOutputItemAdded
+  | OpenAIResponseOutputItemDone
+  | OpenAIResponseFunctionCallArgumentsDelta
+  | OpenAIResponseFunctionCallArgumentsDone;
+
+export interface OpenAISessionCreated {
+  type: "session.created";
+  session: OpenAISession;
+}
+
+export interface OpenAISessionUpdated {
+  type: "session.updated";
+  session: OpenAISession;
+}
+
+export interface OpenAIError {
+  type: "error";
+  error: {
+    code: string;
+    message: string;
+  };
+}
+
+export interface OpenAIInputAudioBufferCommitted {
+  type: "input_audio_buffer.committed";
+  item_id: string;
+}
+
+export interface OpenAIInputAudioBufferCleared {
+  type: "input_audio_buffer.cleared";
+}
+
+export interface OpenAIInputAudioBufferSpeechStarted {
+  type: "input_audio_buffer.speech_started";
+  audio_start_ms: number;
+  item_id: string;
+}
+
+export interface OpenAIInputAudioBufferSpeechStopped {
+  type: "input_audio_buffer.speech_stopped";
+  audio_end_ms: number;
+  item_id: string;
+}
+
+export interface OpenAIConversationItemCreated {
+  type: "conversation.item.created";
+  item: OpenAIItem;
+  previous_item_id?: string;
+}
+
+export interface OpenAIConversationItemDone {
+  type: "conversation.item.done";
+  item: OpenAIItem;
+}
+
+export interface OpenAIConversationItemDeleted {
+  type: "conversation.item.deleted";
+  item_id: string;
+}
+
+export interface OpenAIConversationItemTruncated {
+  type: "conversation.item.truncated";
+  item_id: string;
+  audio_end_ms: number;
+  content_index: number;
+}
+
+export interface OpenAIResponseCreated {
+  type: "response.created";
+  response: {
+    id: string;
+  };
+}
+
+export interface OpenAIResponseDone {
+  type: "response.done";
+  response: {
+    id: string;
+    status: "completed" | "cancelled" | "failed" | "incomplete" | "in_progress";
+    usage?: {
+      input_tokens: number;
+      output_tokens: number;
+      total_tokens?: number;
+    };
+  };
+}
+
+export interface OpenAIResponseOutputAudioDelta {
+  type: "response.output_audio.delta";
+  response_id: string;
+  item_id: string;
+  content_index: number;
+  delta: string;
+}
+
+export interface OpenAIResponseOutputAudioDone {
+  type: "response.output_audio.done";
+  response_id: string;
+  item_id: string;
+  content_index: number;
+}
+
+export interface OpenAIResponseTextDelta {
+  type: "response.text.delta";
+  response_id: string;
+  item_id: string;
+  content_index: number;
+  delta: string;
+}
+
+export interface OpenAIResponseTextDone {
+  type: "response.text.done";
+  response_id: string;
+  item_id: string;
+  content_index: number;
+  text: string;
+}
+
+export interface OpenAIInputAudioTranscriptionDelta {
+  type: "conversation.item.input_audio_transcription.delta";
+  item_id: string;
+  content_index?: number;
+  delta: string;
+}
+
+export interface OpenAIInputAudioTranscriptionCompleted {
+  type: "conversation.item.input_audio_transcription.completed";
+  item_id: string;
+  content_index: number;
+  transcript: string;
+}
+
+export interface OpenAIResponseOutputAudioTranscriptDelta {
+  type: "response.output_audio_transcript.delta";
+  response_id: string;
+  item_id: string;
+  content_index: number;
+  delta: string;
+}
+
+export interface OpenAIResponseOutputAudioTranscriptDone {
+  type: "response.output_audio_transcript.done";
+  response_id: string;
+  item_id: string;
+  content_index: number;
+  transcript: string;
+}
+
+export interface OpenAIResponseOutputItemAdded {
+  type: "response.output_item.added";
+  response_id: string;
+  output_index: number;
+  item: OpenAIFunctionCallItem | OpenAIItem;
+}
+
+export interface OpenAIResponseOutputItemDone {
+  type: "response.output_item.done";
+  response_id: string;
+  output_index: number;
+  item: OpenAIFunctionCallItem | OpenAIItem;
+}
+
+export interface OpenAIResponseFunctionCallArgumentsDelta {
+  type: "response.function_call_arguments.delta";
+  response_id: string;
+  item_id: string;
+  call_id: string;
+  delta: string;
+}
+
+export interface OpenAIResponseFunctionCallArgumentsDone {
+  type: "response.function_call_arguments.done";
+  response_id: string;
+  item_id: string;
+  call_id: string;
+  name: string;
+  arguments: string;
+}
+
+// =============================================================================
+// Shared Types
+// =============================================================================
+
+export interface OpenAISession {
+  id: string;
+  instructions?: string;
+  voice?: string;
+  modalities?: ("text" | "audio")[];
+  turn_detection?: OpenAITurnDetection;
+  input_audio_format?: string;
+  output_audio_format?: string;
+}
+
+export interface OpenAISessionConfig {
+  instructions?: string;
+  voice?: string;
+  modalities?: ("text" | "audio")[];
+  tools?: OpenAITool[];
+  turn_detection?: OpenAITurnDetection;
+  input_audio_format?: string;
+  output_audio_format?: string;
+}
+
+export interface OpenAITurnDetection {
+  type: "server_vad" | "none";
+  threshold?: number;
+  silence_duration_ms?: number;
+  prefix_padding_ms?: number;
+  create_response?: boolean;
+  interrupt_response?: boolean;
+}
+
+export interface OpenAITool {
+  type: "function";
+  name: string;
+  description?: string;
+  parameters?: JSONSchema7;
+}
+
+export type OpenAIItem =
+  | OpenAIMessageItem
+  | OpenAIFunctionCallItem
+  | OpenAIFunctionCallOutputItem;
+
+export interface OpenAIMessageItem {
+  type: "message";
+  role: "user" | "assistant" | "system";
+  content: OpenAIContentPart[];
+}
+
+export interface OpenAIFunctionCallItem {
+  type: "function_call";
+  call_id: string;
+  name: string;
+  arguments: string;
+}
+
+export interface OpenAIFunctionCallOutputItem {
+  type: "function_call_output";
+  call_id: string;
+  output: string;
+}
+
+export type OpenAIContentPart =
+  | { type: "input_text"; text: string }
+  | { type: "input_audio"; audio: string }
+  | { type: "output_text"; text: string }
+  | { type: "output_audio"; audio: string; transcript?: string };

package/src/index.ts

@@ -0,0 +1,41 @@
+import { OpenAIRealtimeModel, type OpenAIRealtimeOptions } from "./realtime";
+
+/**
+ * OpenAI provider interface.
+ */
+export interface OpenAIProvider {
+  /**
+   * Create a realtime model.
+   *
+   * @example
+   * ```ts
+   * import { openai } from '@kernl-sdk/openai';
+   *
+   * const model = openai.realtime('gpt-4o-realtime-preview');
+   * ```
+   *
+   * @example
+   * ```ts
+   * const model = openai.realtime('gpt-4o-realtime-preview', {
+   *   apiKey: 'sk-...',
+   * });
+   * ```
+   */
+  realtime(modelId: string, options?: OpenAIRealtimeOptions): OpenAIRealtimeModel;
+}
+
+/**
+ * OpenAI provider.
+ *
+ * @example
+ * ```ts
+ * import { openai } from '@kernl-sdk/openai';
+ *
+ * const model = openai.realtime('gpt-4o-realtime-preview');
+ * ```
+ */
+export const openai: OpenAIProvider = {
+  realtime(modelId: string, options?: OpenAIRealtimeOptions) {
+    return new OpenAIRealtimeModel(modelId, options);
+  },
+};

package/src/realtime.ts

@@ -0,0 +1,276 @@
+import { EventEmitter } from "node:events";
+import WebSocket from "ws";
+
+import type {
+  RealtimeModel,
+  RealtimeConnection,
+  RealtimeConnectOptions,
+  RealtimeClientEvent,
+  TransportStatus,
+} from "@kernl-sdk/protocol";
+
+import { CLIENT_EVENT, SERVER_EVENT } from "./convert/event";
+import type { OpenAIServerEvent } from "./convert/types";
+
+const OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime";
+
+/**
+ * Options for creating an OpenAI realtime model.
+ */
+export interface OpenAIRealtimeOptions {
+  /**
+   * OpenAI API key. Defaults to OPENAI_API_KEY env var.
+   */
+  apiKey?: string;
+
+  /**
+   * Base URL for the realtime API.
+   */
+  baseUrl?: string;
+}
+
+/**
+ * OpenAI realtime model implementation.
+ */
+export class OpenAIRealtimeModel implements RealtimeModel {
+  readonly spec = "1.0" as const;
+  readonly provider = "openai";
+  readonly modelId: string;
+
+  private apiKey: string;
+  private baseUrl: string;
+
+  constructor(modelId: string, options?: OpenAIRealtimeOptions) {
+    this.modelId = modelId;
+    this.apiKey = options?.apiKey ?? process.env.OPENAI_API_KEY ?? "";
+    this.baseUrl = options?.baseUrl ?? OPENAI_REALTIME_URL;
+
+    if (!this.apiKey) {
+      throw new Error("OpenAI API key is required");
+    }
+  }
+
+  /**
+   * Establish a WebSocket connection to the OpenAI realtime API.
+   */
+  async connect(options?: RealtimeConnectOptions): Promise<RealtimeConnection> {
+    const url = `${this.baseUrl}?model=${this.modelId}`;
+
+    const ws = new WebSocket(url, {
+      headers: {
+        Authorization: `Bearer ${this.apiKey}`,
+        "OpenAI-Beta": "realtime=v1",
+      },
+    });
+
+    const connection = new OpenAIRealtimeConnection(ws);
+
+    await new Promise<void>((resolve, reject) => {
+      if (options?.abort?.aborted) {
+        return reject(new Error("Connection aborted"));
+      }
+
+      const cleanup = () => {
+        ws.off("open", onOpen);
+        ws.off("error", onError);
+        options?.abort?.removeEventListener("abort", onAbort);
+      };
+
+      const onOpen = () => {
+        cleanup();
+        resolve();
+      };
+      const onError = (err: Error) => {
+        cleanup();
+        reject(err);
+      };
+      const onAbort = () => {
+        cleanup();
+        ws.close();
+        reject(new Error("Connection aborted"));
+      };
+
+      ws.on("open", onOpen);
+      ws.on("error", onError);
+      options?.abort?.addEventListener("abort", onAbort);
+    });
+
+    if (options?.sessionConfig) {
+      connection.send({
+        kind: "session.update",
+        config: options.sessionConfig,
+      });
+    }
+
+    return connection;
+  }
+}
+
+/**
+ * OpenAI realtime connection implementation.
+ */
+class OpenAIRealtimeConnection
+  extends EventEmitter
+  implements RealtimeConnection
+{
+  private ws: WebSocket;
+  private _status: TransportStatus = "connecting";
+  private _muted = false;
+  private _sessionId: string | null = null;
+
+  // audio state tracking for interruption
+  private currid: string | undefined;
+  private curridx: number | undefined;
+  private faudtime: number | undefined; /* first audio timestamp */
+  private audlenms: number = 0;
+  private responding: boolean = false;
+
+  constructor(socket: WebSocket) {
+    super();
+    this.ws = socket;
+
+    socket.on("message", (data) => {
+      try {
+        const raw = JSON.parse(data.toString()) as OpenAIServerEvent;
+
+        // track audio state for interruption handling
+        if (raw.type === "response.output_audio.delta") {
+          this.currid = raw.item_id;
+          this.curridx = raw.content_index;
+          if (this.faudtime === undefined) {
+            this.faudtime = Date.now();
+            this.audlenms = 0;
+          }
+          // calculate audio length assuming 24kHz PCM16
+          // TODO: support g711 (8kHz, 1 byte/sample) and configurable PCM rates
+          const bytes = base64ByteLength(raw.delta);
+          this.audlenms += (bytes / 2 / 24000) * 1000;
+        } else if (raw.type === "response.created") {
+          this.responding = true;
+        } else if (raw.type === "response.done") {
+          this.responding = false;
+          this.reset();
+        } else if (raw.type === "input_audio_buffer.speech_started") {
+          this.interrupt();
+        }
+
+        const event = SERVER_EVENT.decode(raw);
+        if (event) {
+          if (event.kind === "session.created") {
+            this._sessionId = event.session.id;
+          }
+          this.emit("event", event);
+        }
+      } catch (err) {
+        this.emit("error", err instanceof Error ? err : new Error(String(err)));
+      }
+    });
+
+    socket.on("open", () => {
+      this._status = "connected";
+      this.emit("status", this._status);
+    });
+
+    socket.on("close", () => {
+      this._status = "closed";
+      this.reset();
+      this.emit("status", this._status);
+    });
+
+    socket.on("error", (err) => {
+      this.emit("error", err);
+    });
+  }
+
+  get status(): TransportStatus {
+    return this._status;
+  }
+
+  get muted(): boolean {
+    return this._muted;
+  }
+
+  get sessionId(): string | null {
+    return this._sessionId;
+  }
+
+  /**
+   * Send a client event to the OpenAI realtime API.
+   */
+  send(event: RealtimeClientEvent): void {
+    const encoded = CLIENT_EVENT.encode(event);
+    if (encoded && this.ws.readyState === WebSocket.OPEN) {
+      this.ws.send(JSON.stringify(encoded));
+    }
+  }
+
+  /**
+   * Close the WebSocket connection.
+   */
+  close(): void {
+    this.reset();
+    this.ws.close();
+  }
+
+  /**
+   * Mute audio input.
+   */
+  mute(): void {
+    this._muted = true;
+  }
+
+  /**
+   * Unmute audio input.
+   */
+  unmute(): void {
+    this._muted = false;
+  }
+
+  /**
+   * Interrupt the current response.
+   */
+  interrupt(): void {
+    // cancel ongoing response
+    if (this.responding) {
+      this.send({ kind: "response.cancel" });
+      this.responding = false;
+    }
+
+    // truncate if we have audio state
+    if (this.currid && this.faudtime !== undefined) {
+      const elapsed = Date.now() - this.faudtime;
+      const endms = Math.max(0, Math.floor(Math.min(elapsed, this.audlenms)));
+
+      if (this.ws.readyState === WebSocket.OPEN) {
+        this.ws.send(
+          JSON.stringify({
+            type: "conversation.item.truncate",
+            item_id: this.currid,
+            content_index: this.curridx ?? 0,
+            audio_end_ms: endms,
+          }),
+        );
+      }
+    }
+
+    this.reset();
+  }
+
+  /**
+   * Reset audio tracking state.
+   */
+  private reset(): void {
+    this.currid = undefined;
+    this.curridx = undefined;
+    this.faudtime = undefined;
+    this.audlenms = 0;
+  }
+}
+
+/**
+ * Get byte length from base64 string without decoding.
+ */
+function base64ByteLength(b64: string): number {
+  const padding = b64.endsWith("==") ? 2 : b64.endsWith("=") ? 1 : 0;
+  return (b64.length * 3) / 4 - padding;
+}

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}