@kernl-sdk/openai 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,36 @@
+import { OpenAIRealtimeModel, type OpenAIRealtimeOptions } from "./realtime.js";
+/**
+ * OpenAI provider interface.
+ */
+export interface OpenAIProvider {
+    /**
+     * Create a realtime model.
+     *
+     * @example
+     * ```ts
+     * import { openai } from '@kernl-sdk/openai';
+     *
+     * const model = openai.realtime('gpt-4o-realtime-preview');
+     * ```
+     *
+     * @example
+     * ```ts
+     * const model = openai.realtime('gpt-4o-realtime-preview', {
+     *   apiKey: 'sk-...',
+     * });
+     * ```
+     */
+    realtime(modelId: string, options?: OpenAIRealtimeOptions): OpenAIRealtimeModel;
+}
+/**
+ * OpenAI provider.
+ *
+ * @example
+ * ```ts
+ * import { openai } from '@kernl-sdk/openai';
+ *
+ * const model = openai.realtime('gpt-4o-realtime-preview');
+ * ```
+ */
+export declare const openai: OpenAIProvider;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,KAAK,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAE7E;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;;;;;;;;;;;;;OAgBG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,qBAAqB,GAAG,mBAAmB,CAAC;CACjF;AAED;;;;;;;;;GASG;AACH,eAAO,MAAM,MAAM,EAAE,cAIpB,CAAC"}

package/dist/index.js

@@ -0,0 +1,16 @@
+import { OpenAIRealtimeModel } from "./realtime.js";
+/**
+ * OpenAI provider.
+ *
+ * @example
+ * ```ts
+ * import { openai } from '@kernl-sdk/openai';
+ *
+ * const model = openai.realtime('gpt-4o-realtime-preview');
+ * ```
+ */
+export const openai = {
+    realtime(modelId, options) {
+        return new OpenAIRealtimeModel(modelId, options);
+    },
+};

package/dist/realtime.d.ts

@@ -0,0 +1,30 @@
+import type { RealtimeModel, RealtimeConnection, RealtimeConnectOptions } from "@kernl-sdk/protocol";
+/**
+ * Options for creating an OpenAI realtime model.
+ */
+export interface OpenAIRealtimeOptions {
+    /**
+     * OpenAI API key. Defaults to OPENAI_API_KEY env var.
+     */
+    apiKey?: string;
+    /**
+     * Base URL for the realtime API.
+     */
+    baseUrl?: string;
+}
+/**
+ * OpenAI realtime model implementation.
+ */
+export declare class OpenAIRealtimeModel implements RealtimeModel {
+    readonly spec: "1.0";
+    readonly provider = "openai";
+    readonly modelId: string;
+    private apiKey;
+    private baseUrl;
+    constructor(modelId: string, options?: OpenAIRealtimeOptions);
+    /**
+     * Establish a WebSocket connection to the OpenAI realtime API.
+     */
+    connect(options?: RealtimeConnectOptions): Promise<RealtimeConnection>;
+}
+//# sourceMappingURL=realtime.d.ts.map

package/dist/realtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime.d.ts","sourceRoot":"","sources":["../src/realtime.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,aAAa,EACb,kBAAkB,EAClB,sBAAsB,EAGvB,MAAM,qBAAqB,CAAC;AAO7B;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,qBAAa,mBAAoB,YAAW,aAAa;IACvD,QAAQ,CAAC,IAAI,EAAG,KAAK,CAAU;IAC/B,QAAQ,CAAC,QAAQ,YAAY;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,qBAAqB;IAU5D;;OAEG;IACG,OAAO,CAAC,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAmD7E"}

package/dist/realtime.js

@@ -0,0 +1,214 @@
+import { EventEmitter } from "node:events";
+import WebSocket from "ws";
+import { CLIENT_EVENT, SERVER_EVENT } from "./convert/event.js";
+const OPENAI_REALTIME_URL = "wss://api.openai.com/v1/realtime";
+/**
+ * OpenAI realtime model implementation.
+ */
+export class OpenAIRealtimeModel {
+    spec = "1.0";
+    provider = "openai";
+    modelId;
+    apiKey;
+    baseUrl;
+    constructor(modelId, options) {
+        this.modelId = modelId;
+        this.apiKey = options?.apiKey ?? process.env.OPENAI_API_KEY ?? "";
+        this.baseUrl = options?.baseUrl ?? OPENAI_REALTIME_URL;
+        if (!this.apiKey) {
+            throw new Error("OpenAI API key is required");
+        }
+    }
+    /**
+     * Establish a WebSocket connection to the OpenAI realtime API.
+     */
+    async connect(options) {
+        const url = `${this.baseUrl}?model=${this.modelId}`;
+        const ws = new WebSocket(url, {
+            headers: {
+                Authorization: `Bearer ${this.apiKey}`,
+                "OpenAI-Beta": "realtime=v1",
+            },
+        });
+        const connection = new OpenAIRealtimeConnection(ws);
+        await new Promise((resolve, reject) => {
+            if (options?.abort?.aborted) {
+                return reject(new Error("Connection aborted"));
+            }
+            const cleanup = () => {
+                ws.off("open", onOpen);
+                ws.off("error", onError);
+                options?.abort?.removeEventListener("abort", onAbort);
+            };
+            const onOpen = () => {
+                cleanup();
+                resolve();
+            };
+            const onError = (err) => {
+                cleanup();
+                reject(err);
+            };
+            const onAbort = () => {
+                cleanup();
+                ws.close();
+                reject(new Error("Connection aborted"));
+            };
+            ws.on("open", onOpen);
+            ws.on("error", onError);
+            options?.abort?.addEventListener("abort", onAbort);
+        });
+        if (options?.sessionConfig) {
+            connection.send({
+                kind: "session.update",
+                config: options.sessionConfig,
+            });
+        }
+        return connection;
+    }
+}
+/**
+ * OpenAI realtime connection implementation.
+ */
+class OpenAIRealtimeConnection extends EventEmitter {
+    ws;
+    _status = "connecting";
+    _muted = false;
+    _sessionId = null;
+    // audio state tracking for interruption
+    currid;
+    curridx;
+    faudtime; /* first audio timestamp */
+    audlenms = 0;
+    responding = false;
+    constructor(socket) {
+        super();
+        this.ws = socket;
+        socket.on("message", (data) => {
+            try {
+                const raw = JSON.parse(data.toString());
+                // track audio state for interruption handling
+                if (raw.type === "response.output_audio.delta") {
+                    this.currid = raw.item_id;
+                    this.curridx = raw.content_index;
+                    if (this.faudtime === undefined) {
+                        this.faudtime = Date.now();
+                        this.audlenms = 0;
+                    }
+                    // calculate audio length assuming 24kHz PCM16
+                    // TODO: support g711 (8kHz, 1 byte/sample) and configurable PCM rates
+                    const bytes = base64ByteLength(raw.delta);
+                    this.audlenms += (bytes / 2 / 24000) * 1000;
+                }
+                else if (raw.type === "response.created") {
+                    this.responding = true;
+                }
+                else if (raw.type === "response.done") {
+                    this.responding = false;
+                    this.reset();
+                }
+                else if (raw.type === "input_audio_buffer.speech_started") {
+                    this.interrupt();
+                }
+                const event = SERVER_EVENT.decode(raw);
+                if (event) {
+                    if (event.kind === "session.created") {
+                        this._sessionId = event.session.id;
+                    }
+                    this.emit("event", event);
+                }
+            }
+            catch (err) {
+                this.emit("error", err instanceof Error ? err : new Error(String(err)));
+            }
+        });
+        socket.on("open", () => {
+            this._status = "connected";
+            this.emit("status", this._status);
+        });
+        socket.on("close", () => {
+            this._status = "closed";
+            this.reset();
+            this.emit("status", this._status);
+        });
+        socket.on("error", (err) => {
+            this.emit("error", err);
+        });
+    }
+    get status() {
+        return this._status;
+    }
+    get muted() {
+        return this._muted;
+    }
+    get sessionId() {
+        return this._sessionId;
+    }
+    /**
+     * Send a client event to the OpenAI realtime API.
+     */
+    send(event) {
+        const encoded = CLIENT_EVENT.encode(event);
+        if (encoded && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(JSON.stringify(encoded));
+        }
+    }
+    /**
+     * Close the WebSocket connection.
+     */
+    close() {
+        this.reset();
+        this.ws.close();
+    }
+    /**
+     * Mute audio input.
+     */
+    mute() {
+        this._muted = true;
+    }
+    /**
+     * Unmute audio input.
+     */
+    unmute() {
+        this._muted = false;
+    }
+    /**
+     * Interrupt the current response.
+     */
+    interrupt() {
+        // cancel ongoing response
+        if (this.responding) {
+            this.send({ kind: "response.cancel" });
+            this.responding = false;
+        }
+        // truncate if we have audio state
+        if (this.currid && this.faudtime !== undefined) {
+            const elapsed = Date.now() - this.faudtime;
+            const endms = Math.max(0, Math.floor(Math.min(elapsed, this.audlenms)));
+            if (this.ws.readyState === WebSocket.OPEN) {
+                this.ws.send(JSON.stringify({
+                    type: "conversation.item.truncate",
+                    item_id: this.currid,
+                    content_index: this.curridx ?? 0,
+                    audio_end_ms: endms,
+                }));
+            }
+        }
+        this.reset();
+    }
+    /**
+     * Reset audio tracking state.
+     */
+    reset() {
+        this.currid = undefined;
+        this.curridx = undefined;
+        this.faudtime = undefined;
+        this.audlenms = 0;
+    }
+}
+/**
+ * Get byte length from base64 string without decoding.
+ */
+function base64ByteLength(b64) {
+    const padding = b64.endsWith("==") ? 2 : b64.endsWith("=") ? 1 : 0;
+    return (b64.length * 3) / 4 - padding;
+}

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@kernl-sdk/openai",
+  "version": "0.1.0",
+  "description": "OpenAI provider for kernl",
+  "keywords": [
+    "kernl",
+    "openai",
+    "realtime",
+    "voice",
+    "ai"
+  ],
+  "author": "dremnik",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/kernl-sdk/kernl.git",
+    "directory": "packages/providers/openai"
+  },
+  "homepage": "https://github.com/kernl-sdk/kernl#readme",
+  "bugs": {
+    "url": "https://github.com/kernl-sdk/kernl/issues"
+  },
+  "type": "module",
+  "publishConfig": {
+    "access": "public"
+  },
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc && tsc-alias --resolve-full-paths",
+    "dev": "tsc --watch",
+    "check-types": "tsc --noEmit",
+    "test": "vitest",
+    "test:watch": "vitest --watch",
+    "test:run": "vitest run"
+  },
+  "dependencies": {
+    "@kernl-sdk/protocol": "workspace:*",
+    "@kernl-sdk/shared": "workspace:*",
+    "ws": "^8.18.0"
+  },
+  "devDependencies": {
+    "@types/json-schema": "^7.0.15",
+    "@types/node": "^24.10.0",
+    "@types/ws": "^8.18.0",
+    "tsc-alias": "^1.8.10",
+    "typescript": "5.9.2",
+    "vitest": "^4.0.8"
+  }
+}

package/src/__tests__/realtime.integration.test.ts

@@ -0,0 +1,217 @@
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+
+import type {
+  RealtimeServerEvent,
+  RealtimeConnection,
+} from "@kernl-sdk/protocol";
+import { OpenAIRealtimeModel } from "../realtime";
+
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+
+describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime Integration", () => {
+  let model: OpenAIRealtimeModel;
+
+  beforeAll(() => {
+    model = new OpenAIRealtimeModel("gpt-realtime", {
+      apiKey: OPENAI_API_KEY,
+    });
+  });
+
+  it("should connect and receive session.created", async () => {
+    const conn = await model.connect();
+    const events: RealtimeServerEvent[] = [];
+
+    const sessionCreated = new Promise<void>((resolve, reject) => {
+      const timeout = setTimeout(() => reject(new Error("timeout")), 10000);
+      conn.on("event", (e: RealtimeServerEvent) => {
+        events.push(e);
+        if (e.kind === "session.created") {
+          clearTimeout(timeout);
+          resolve();
+        }
+      });
+    });
+
+    await sessionCreated;
+    conn.close();
+
+    expect(events.some((e) => e.kind === "session.created")).toBe(true);
+    expect(conn.sessionId).toBeTruthy();
+  });
+
+  it("should complete text round-trip", async () => {
+    const conn = await model.connect();
+    const events: RealtimeServerEvent[] = [];
+
+    conn.on("event", (e: RealtimeServerEvent) => {
+      events.push(e);
+    });
+
+    // wait for session
+    await waitFor(conn, "session.created");
+
+    // configure text-only mode
+    conn.send({
+      kind: "session.update",
+      config: {
+        modalities: ["text"],
+        instructions: "You are a helpful assistant. Be very brief.",
+      },
+    });
+
+    await waitFor(conn, "session.updated");
+
+    // add user message
+    conn.send({
+      kind: "item.create",
+      item: {
+        kind: "message",
+        id: "test-msg-1",
+        role: "user",
+        content: [{ kind: "text", text: "Say exactly: hello world" }],
+      },
+    });
+
+    // trigger response
+    conn.send({ kind: "response.create" });
+
+    // wait for response to complete
+    await waitFor(conn, "response.done", 15000);
+
+    conn.close();
+
+    // verify event flow
+    const kinds = events.map((e) => e.kind);
+    expect(kinds).toContain("session.created");
+    expect(kinds).toContain("session.updated");
+    expect(kinds).toContain("response.created");
+    expect(kinds).toContain("response.done");
+
+    // verify we got text output
+    const textOutput = events.find((e) => e.kind === "text.output");
+    expect(textOutput).toBeDefined();
+    if (textOutput?.kind === "text.output") {
+      expect(textOutput.text.toLowerCase()).toContain("hello");
+    }
+
+    // verify response completed successfully
+    const done = events.find((e) => e.kind === "response.done");
+    if (done?.kind === "response.done") {
+      expect(done.status).toBe("completed");
+    }
+  });
+
+  it("should handle tool calling", { timeout: 10000 }, async () => {
+    const conn = await model.connect();
+    const events: RealtimeServerEvent[] = [];
+
+    conn.on("event", (e: RealtimeServerEvent) => {
+      events.push(e);
+    });
+
+    await waitFor(conn, "session.created");
+
+    // configure with a tool
+    conn.send({
+      kind: "session.update",
+      config: {
+        modalities: ["text"],
+        instructions: "You have access to tools. Use them when appropriate.",
+        tools: [
+          {
+            kind: "function",
+            name: "get_weather",
+            description: "Get the current weather for a location",
+            parameters: {
+              type: "object",
+              properties: {
+                location: { type: "string", description: "City name" },
+              },
+              required: ["location"],
+            },
+          },
+        ],
+      },
+    });
+
+    await waitFor(conn, "session.updated");
+
+    // ask about weather
+    conn.send({
+      kind: "item.create",
+      item: {
+        kind: "message",
+        id: "test-msg-2",
+        role: "user",
+        content: [{ kind: "text", text: "What is the weather in Tokyo?" }],
+      },
+    });
+
+    conn.send({ kind: "response.create" });
+
+    // wait for tool call
+    const toolCall = await waitFor(conn, "tool.call", 15000);
+
+    expect(toolCall.kind).toBe("tool.call");
+    if (toolCall.kind !== "tool.call") {
+      throw new Error("Expected tool.call");
+    }
+
+    expect(toolCall.toolId).toBe("get_weather");
+    const args = JSON.parse(toolCall.arguments);
+    expect(args.location.toLowerCase()).toContain("tokyo");
+
+    // wait for first response to complete before sending tool result
+    await waitFor(conn, "response.done", 15000);
+
+    // send tool result
+    conn.send({
+      kind: "tool.result",
+      callId: toolCall.callId,
+      result: JSON.stringify({ temperature: 22, condition: "sunny" }),
+    });
+
+    // trigger follow-up response
+    conn.send({ kind: "response.create" });
+
+    // wait for second response to complete
+    await waitFor(conn, "response.done", 15000);
+
+    conn.close();
+
+    // verify we got text mentioning the weather
+    const textEvents = events.filter((e) => e.kind === "text.output");
+    const allText = textEvents
+      .map((e) => (e.kind === "text.output" ? e.text : ""))
+      .join(" ")
+      .toLowerCase();
+
+    expect(allText).toMatch(/sunny|22|tokyo/i);
+  });
+});
+
+/**
+ * Wait for a specific event kind.
+ */
+function waitFor(
+  conn: RealtimeConnection,
+  kind: RealtimeServerEvent["kind"],
+  timeout = 10000,
+): Promise<RealtimeServerEvent> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      conn.off("event", handler);
+      reject(new Error(`timeout waiting for ${kind}`));
+    }, timeout);
+
+    const handler = (e: RealtimeServerEvent) => {
+      if (e.kind === kind) {
+        clearTimeout(timer);
+        conn.off("event", handler);
+        resolve(e);
+      }
+    };
+
+    conn.on("event", handler);
+  });
+}