@kernl-sdk/openai 0.1.0

package/.turbo/turbo-build.log

@@ -0,0 +1,5 @@
+
+
+> @kernl-sdk/openai@0.1.0 build /Users/andjones/Documents/projects/kernl/packages/providers/openai
+> tsc && tsc-alias --resolve-full-paths
+

package/CHANGELOG.md

@@ -0,0 +1,15 @@
+# @kernl-sdk/openai
+
+## 0.1.0
+
+### Minor Changes
+
+- 572ae80: Add realtime voice agent support with memory capabilities.
+  - **protocol**: Add realtime model and event types for voice agents
+  - **kernl**: Extract BaseAgent class shared by Agent and RealtimeAgent, enabling memory support for realtime agents. Add `kind` discriminator for type narrowing.
+  - **openai**: Add OpenAI realtime voice provider with WebSocket-based streaming
+
+### Patch Changes
+
+- Updated dependencies [572ae80]
+  - @kernl-sdk/protocol@0.3.0

package/dist/__tests__/realtime.integration.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=realtime.integration.test.d.ts.map

package/dist/__tests__/realtime.integration.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime.integration.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/realtime.integration.test.ts"],"names":[],"mappings":""}

package/dist/__tests__/realtime.integration.test.js

@@ -0,0 +1,169 @@
+import { describe, it, expect, beforeAll } from "vitest";
+import { OpenAIRealtimeModel } from "../realtime.js";
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
+describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime Integration", () => {
+    let model;
+    beforeAll(() => {
+        model = new OpenAIRealtimeModel("gpt-realtime", {
+            apiKey: OPENAI_API_KEY,
+        });
+    });
+    it("should connect and receive session.created", async () => {
+        const conn = await model.connect();
+        const events = [];
+        const sessionCreated = new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => reject(new Error("timeout")), 10000);
+            conn.on("event", (e) => {
+                events.push(e);
+                if (e.kind === "session.created") {
+                    clearTimeout(timeout);
+                    resolve();
+                }
+            });
+        });
+        await sessionCreated;
+        conn.close();
+        expect(events.some((e) => e.kind === "session.created")).toBe(true);
+        expect(conn.sessionId).toBeTruthy();
+    });
+    it("should complete text round-trip", async () => {
+        const conn = await model.connect();
+        const events = [];
+        conn.on("event", (e) => {
+            events.push(e);
+        });
+        // wait for session
+        await waitFor(conn, "session.created");
+        // configure text-only mode
+        conn.send({
+            kind: "session.update",
+            config: {
+                modalities: ["text"],
+                instructions: "You are a helpful assistant. Be very brief.",
+            },
+        });
+        await waitFor(conn, "session.updated");
+        // add user message
+        conn.send({
+            kind: "item.create",
+            item: {
+                kind: "message",
+                id: "test-msg-1",
+                role: "user",
+                content: [{ kind: "text", text: "Say exactly: hello world" }],
+            },
+        });
+        // trigger response
+        conn.send({ kind: "response.create" });
+        // wait for response to complete
+        await waitFor(conn, "response.done", 15000);
+        conn.close();
+        // verify event flow
+        const kinds = events.map((e) => e.kind);
+        expect(kinds).toContain("session.created");
+        expect(kinds).toContain("session.updated");
+        expect(kinds).toContain("response.created");
+        expect(kinds).toContain("response.done");
+        // verify we got text output
+        const textOutput = events.find((e) => e.kind === "text.output");
+        expect(textOutput).toBeDefined();
+        if (textOutput?.kind === "text.output") {
+            expect(textOutput.text.toLowerCase()).toContain("hello");
+        }
+        // verify response completed successfully
+        const done = events.find((e) => e.kind === "response.done");
+        if (done?.kind === "response.done") {
+            expect(done.status).toBe("completed");
+        }
+    });
+    it("should handle tool calling", { timeout: 10000 }, async () => {
+        const conn = await model.connect();
+        const events = [];
+        conn.on("event", (e) => {
+            events.push(e);
+        });
+        await waitFor(conn, "session.created");
+        // configure with a tool
+        conn.send({
+            kind: "session.update",
+            config: {
+                modalities: ["text"],
+                instructions: "You have access to tools. Use them when appropriate.",
+                tools: [
+                    {
+                        kind: "function",
+                        name: "get_weather",
+                        description: "Get the current weather for a location",
+                        parameters: {
+                            type: "object",
+                            properties: {
+                                location: { type: "string", description: "City name" },
+                            },
+                            required: ["location"],
+                        },
+                    },
+                ],
+            },
+        });
+        await waitFor(conn, "session.updated");
+        // ask about weather
+        conn.send({
+            kind: "item.create",
+            item: {
+                kind: "message",
+                id: "test-msg-2",
+                role: "user",
+                content: [{ kind: "text", text: "What is the weather in Tokyo?" }],
+            },
+        });
+        conn.send({ kind: "response.create" });
+        // wait for tool call
+        const toolCall = await waitFor(conn, "tool.call", 15000);
+        expect(toolCall.kind).toBe("tool.call");
+        if (toolCall.kind !== "tool.call") {
+            throw new Error("Expected tool.call");
+        }
+        expect(toolCall.toolId).toBe("get_weather");
+        const args = JSON.parse(toolCall.arguments);
+        expect(args.location.toLowerCase()).toContain("tokyo");
+        // wait for first response to complete before sending tool result
+        await waitFor(conn, "response.done", 15000);
+        // send tool result
+        conn.send({
+            kind: "tool.result",
+            callId: toolCall.callId,
+            result: JSON.stringify({ temperature: 22, condition: "sunny" }),
+        });
+        // trigger follow-up response
+        conn.send({ kind: "response.create" });
+        // wait for second response to complete
+        await waitFor(conn, "response.done", 15000);
+        conn.close();
+        // verify we got text mentioning the weather
+        const textEvents = events.filter((e) => e.kind === "text.output");
+        const allText = textEvents
+            .map((e) => (e.kind === "text.output" ? e.text : ""))
+            .join(" ")
+            .toLowerCase();
+        expect(allText).toMatch(/sunny|22|tokyo/i);
+    });
+});
+/**
+ * Wait for a specific event kind.
+ */
+function waitFor(conn, kind, timeout = 10000) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            conn.off("event", handler);
+            reject(new Error(`timeout waiting for ${kind}`));
+        }, timeout);
+        const handler = (e) => {
+            if (e.kind === kind) {
+                clearTimeout(timer);
+                conn.off("event", handler);
+                resolve(e);
+            }
+        };
+        conn.on("event", handler);
+    });
+}

package/dist/__tests__/realtime.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=realtime.test.d.ts.map

package/dist/__tests__/realtime.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/realtime.test.ts"],"names":[],"mappings":""}

package/dist/__tests__/realtime.test.js

@@ -0,0 +1,314 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { EventEmitter } from "node:events";
+// Track mock WebSocket instances
+const wsInstances = [];
+function createMockWebSocket() {
+    const emitter = new EventEmitter();
+    emitter.send = vi.fn();
+    emitter.close = vi.fn();
+    emitter.readyState = 1; // OPEN
+    emitter.OPEN = 1;
+    return emitter;
+}
+// Mock WebSocket with a proper constructor function
+vi.mock("ws", () => {
+    const MockWebSocket = function () {
+        const instance = createMockWebSocket();
+        wsInstances.push(instance);
+        return instance;
+    };
+    MockWebSocket.OPEN = 1;
+    return {
+        default: MockWebSocket,
+        WebSocket: MockWebSocket,
+    };
+});
+// Import after mock
+import { OpenAIRealtimeModel } from "../realtime.js";
+describe("OpenAIRealtimeModel", () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+        wsInstances.length = 0;
+    });
+    it("should require API key", () => {
+        const originalEnv = process.env.OPENAI_API_KEY;
+        delete process.env.OPENAI_API_KEY;
+        expect(() => new OpenAIRealtimeModel("gpt-4o-realtime")).toThrow("OpenAI API key is required");
+        process.env.OPENAI_API_KEY = originalEnv;
+    });
+    it("should accept API key via options", () => {
+        const model = new OpenAIRealtimeModel("gpt-4o-realtime", {
+            apiKey: "test-key",
+        });
+        expect(model.modelId).toBe("gpt-4o-realtime");
+        expect(model.provider).toBe("openai");
+        expect(model.spec).toBe("1.0");
+    });
+    it("should use OPENAI_API_KEY env var", () => {
+        const originalEnv = process.env.OPENAI_API_KEY;
+        process.env.OPENAI_API_KEY = "env-key";
+        const model = new OpenAIRealtimeModel("gpt-4o-realtime");
+        expect(model.modelId).toBe("gpt-4o-realtime");
+        process.env.OPENAI_API_KEY = originalEnv;
+    });
+});
+describe("base64ByteLength", () => {
+    // Test the helper function indirectly through the module
+    // The actual function is not exported, but we can verify the audio length calculation works
+    it("should calculate correct byte length for base64 without padding", () => {
+        // "SGVsbG8" = "Hello" (5 bytes), no padding
+        // base64 length = 8, padding = 0
+        // bytes = (8 * 3) / 4 - 0 = 6 (actually "Hello" is 5 bytes, but base64 of "Hello" is "SGVsbG8=" with padding)
+        // Let's use a known value: "AAAA" = 3 bytes (no padding needed for 3 bytes)
+        // Actually "AAA=" is 2 bytes, "AAAA" is 3 bytes
+        const b64NoPadding = "AAAA"; // 3 bytes
+        const expectedBytes = 3;
+        const padding = 0;
+        const calculated = (b64NoPadding.length * 3) / 4 - padding;
+        expect(calculated).toBe(expectedBytes);
+    });
+    it("should calculate correct byte length for base64 with single padding", () => {
+        // "AAA=" represents 2 bytes
+        const b64SinglePad = "AAA=";
+        const expectedBytes = 2;
+        const padding = 1;
+        const calculated = (b64SinglePad.length * 3) / 4 - padding;
+        expect(calculated).toBe(expectedBytes);
+    });
+    it("should calculate correct byte length for base64 with double padding", () => {
+        // "AA==" represents 1 byte
+        const b64DoublePad = "AA==";
+        const expectedBytes = 1;
+        const padding = 2;
+        const calculated = (b64DoublePad.length * 3) / 4 - padding;
+        expect(calculated).toBe(expectedBytes);
+    });
+});
+describe("audio length calculation", () => {
+    it("should calculate correct duration for 24kHz PCM16", () => {
+        // 24kHz PCM16 = 24000 samples/sec, 2 bytes/sample = 48000 bytes/sec
+        // 48000 bytes = 1000ms
+        // 4800 bytes = 100ms
+        const bytes = 4800;
+        const expectedMs = (bytes / 2 / 24000) * 1000;
+        expect(expectedMs).toBe(100);
+    });
+    it("should accumulate audio length from multiple chunks", () => {
+        // Simulate multiple audio chunks
+        const chunk1Bytes = 2400; // 50ms
+        const chunk2Bytes = 2400; // 50ms
+        const chunk3Bytes = 2400; // 50ms
+        let totalMs = 0;
+        totalMs += (chunk1Bytes / 2 / 24000) * 1000;
+        totalMs += (chunk2Bytes / 2 / 24000) * 1000;
+        totalMs += (chunk3Bytes / 2 / 24000) * 1000;
+        expect(totalMs).toBe(150);
+    });
+});
+describe("interruption timing", () => {
+    it("should calculate audio_end_ms as min of elapsed and total length", () => {
+        const firstAudioTimestamp = 1000;
+        const currentTime = 1150; // 150ms elapsed
+        const audioLengthMs = 200; // but only 200ms of audio received
+        const elapsed = currentTime - firstAudioTimestamp;
+        const audioEndMs = Math.max(0, Math.floor(Math.min(elapsed, audioLengthMs)));
+        expect(audioEndMs).toBe(150); // elapsed is less than total
+    });
+    it("should cap audio_end_ms at total audio length", () => {
+        const firstAudioTimestamp = 1000;
+        const currentTime = 1500; // 500ms elapsed
+        const audioLengthMs = 200; // but only 200ms of audio received
+        const elapsed = currentTime - firstAudioTimestamp;
+        const audioEndMs = Math.max(0, Math.floor(Math.min(elapsed, audioLengthMs)));
+        expect(audioEndMs).toBe(200); // capped at audio length
+    });
+    it("should handle zero elapsed time", () => {
+        const firstAudioTimestamp = 1000;
+        const currentTime = 1000; // 0ms elapsed
+        const audioLengthMs = 200;
+        const elapsed = currentTime - firstAudioTimestamp;
+        const audioEndMs = Math.max(0, Math.floor(Math.min(elapsed, audioLengthMs)));
+        expect(audioEndMs).toBe(0);
+    });
+});
+describe("OpenAIRealtimeConnection (mocked WebSocket)", () => {
+    const apiKey = "test-key";
+    beforeEach(() => {
+        vi.clearAllMocks();
+        wsInstances.length = 0;
+    });
+    const getLastSocket = () => {
+        if (wsInstances.length === 0) {
+            throw new Error("No WebSocket instances were created");
+        }
+        return wsInstances[wsInstances.length - 1];
+    };
+    const emitServerEvent = (socket, event) => {
+        const payload = Buffer.from(JSON.stringify(event));
+        socket.emit("message", payload);
+    };
+    const createConnectedRealtime = async () => {
+        // Ensure env key is set so constructor does not throw.
+        const originalEnv = process.env.OPENAI_API_KEY;
+        process.env.OPENAI_API_KEY = apiKey;
+        const model = new OpenAIRealtimeModel("gpt-4o-realtime");
+        const connectPromise = model.connect();
+        const socket = getLastSocket();
+        // Simulate successful WebSocket open.
+        socket.emit("open");
+        const connection = await connectPromise;
+        // Restore env to avoid side effects for other tests.
+        process.env.OPENAI_API_KEY = originalEnv;
+        return { connection, socket };
+    };
+    it("should process a basic conversation flow and emit events", async () => {
+        const { connection, socket } = await createConnectedRealtime();
+        const statusEvents = [];
+        const realtimeEvents = [];
+        connection.on("status", (status) => {
+            statusEvents.push(status);
+        });
+        connection.on("event", (event) => {
+            realtimeEvents.push(event);
+        });
+        // Verify initial status after open.
+        expect(connection.status).toBe("connected");
+        // session.created
+        emitServerEvent(socket, {
+            type: "session.created",
+            session: { id: "sess-1", instructions: "Be helpful" },
+        });
+        // response.created
+        emitServerEvent(socket, {
+            type: "response.created",
+            response: { id: "resp-1" },
+        });
+        // small audio delta then done
+        emitServerEvent(socket, {
+            type: "response.output_audio.delta",
+            response_id: "resp-1",
+            item_id: "item-1",
+            content_index: 0,
+            delta: "AAAA",
+        });
+        emitServerEvent(socket, {
+            type: "response.output_audio.done",
+            response_id: "resp-1",
+            item_id: "item-1",
+            content_index: 0,
+        });
+        // text delta then done
+        emitServerEvent(socket, {
+            type: "response.text.delta",
+            response_id: "resp-1",
+            item_id: "item-1",
+            content_index: 0,
+            delta: "Hello",
+        });
+        emitServerEvent(socket, {
+            type: "response.text.done",
+            response_id: "resp-1",
+            item_id: "item-1",
+            content_index: 0,
+            text: "Hello world",
+        });
+        // transcripts
+        emitServerEvent(socket, {
+            type: "conversation.item.input_audio_transcription.completed",
+            item_id: "item-1",
+            content_index: 0,
+            transcript: "User said hello",
+        });
+        emitServerEvent(socket, {
+            type: "response.output_audio_transcript.done",
+            response_id: "resp-1",
+            item_id: "item-1",
+            content_index: 0,
+            transcript: "Assistant said hi",
+        });
+        // response.done with usage
+        emitServerEvent(socket, {
+            type: "response.done",
+            response: {
+                id: "resp-1",
+                status: "completed",
+                usage: {
+                    input_tokens: 10,
+                    output_tokens: 20,
+                    total_tokens: 30,
+                },
+            },
+        });
+        // Close socket to trigger status change and reset.
+        socket.emit("close");
+        // Status events should include closed (connected is emitted before we subscribe).
+        expect(statusEvents).toContain("closed");
+        // We should have seen several realtime events in a sensible order.
+        const kinds = realtimeEvents.map((e) => e?.kind);
+        expect(kinds).toContain("session.created");
+        expect(kinds).toContain("response.created");
+        expect(kinds).toContain("audio.output.delta");
+        expect(kinds).toContain("audio.output.done");
+        expect(kinds).toContain("text.output.delta");
+        expect(kinds).toContain("text.output");
+        expect(kinds).toContain("transcript.input");
+        expect(kinds).toContain("transcript.output");
+        expect(kinds).toContain("response.done");
+    });
+    it("should cancel and truncate correctly on speech start (interrupt)", async () => {
+        const { connection, socket } = await createConnectedRealtime();
+        const connectionWithInterrupt = connection;
+        // Control time so we can reason about audio_end_ms.
+        let now = 1000;
+        const dateSpy = vi
+            .spyOn(Date, "now")
+            .mockImplementation(() => now);
+        // Mark that a response is in progress with some audio.
+        emitServerEvent(socket, {
+            type: "response.created",
+            response: { id: "resp-1" },
+        });
+        // Single audio delta chunk; compute its duration with the same formula.
+        const deltaAudio = "AAAA";
+        emitServerEvent(socket, {
+            type: "response.output_audio.delta",
+            response_id: "resp-1",
+            item_id: "item-1",
+            content_index: 0,
+            delta: deltaAudio,
+        });
+        const base64ByteLength = (b64) => {
+            const padding = b64.endsWith("==")
+                ? 2
+                : b64.endsWith("=")
+                    ? 1
+                    : 0;
+            return (b64.length * 3) / 4 - padding;
+        };
+        const bytes = base64ByteLength(deltaAudio);
+        const totalAudioMs = (bytes / 2 / 24000) * 1000;
+        // Advance time so that some time has elapsed since first audio.
+        now = 1150; // 150ms elapsed
+        // speech_started should trigger interrupt logic.
+        emitServerEvent(socket, {
+            type: "input_audio_buffer.speech_started",
+            audio_start_ms: 0,
+            item_id: "item-2",
+        });
+        // We expect two outbound sends: response.cancel and item.truncate.
+        const sendMock = socket.send;
+        expect(sendMock.mock.calls.length).toBe(2);
+        const cancelPayload = JSON.parse(sendMock.mock.calls[0][0]);
+        expect(cancelPayload).toEqual({ type: "response.cancel" });
+        const truncatePayload = JSON.parse(sendMock.mock.calls[1][0]);
+        expect(truncatePayload.type).toBe("conversation.item.truncate");
+        expect(truncatePayload.item_id).toBe("item-1");
+        expect(truncatePayload.content_index).toBe(0);
+        const expectedEndMs = Math.max(0, Math.floor(Math.min(150, totalAudioMs)));
+        expect(truncatePayload.audio_end_ms).toBe(expectedEndMs);
+        // Calling interrupt again should be a no-op (state was reset).
+        connectionWithInterrupt.interrupt();
+        expect(sendMock.mock.calls.length).toBe(2);
+        dateSpy.mockRestore();
+    });
+});

package/dist/convert/__tests__/event.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=event.test.d.ts.map

package/dist/convert/__tests__/event.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"event.test.d.ts","sourceRoot":"","sources":["../../../src/convert/__tests__/event.test.ts"],"names":[],"mappings":""}