@kernl-sdk/xai 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -0
- package/dist/__tests__/realtime.integration.test.d.ts +2 -0
- package/dist/__tests__/realtime.integration.test.d.ts.map +1 -0
- package/dist/__tests__/realtime.integration.test.js +157 -0
- package/dist/__tests__/realtime.test.d.ts +2 -0
- package/dist/__tests__/realtime.test.d.ts.map +1 -0
- package/dist/__tests__/realtime.test.js +263 -0
- package/dist/connection.d.ts +47 -0
- package/dist/connection.d.ts.map +1 -0
- package/dist/connection.js +138 -0
- package/dist/convert/event.d.ts +28 -0
- package/dist/convert/event.d.ts.map +1 -0
- package/dist/convert/event.js +314 -0
- package/dist/convert/types.d.ts +212 -0
- package/dist/convert/types.d.ts.map +1 -0
- package/dist/convert/types.js +1 -0
- package/dist/index.d.ts +36 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +16 -0
- package/dist/model.d.ts +36 -0
- package/dist/model.d.ts.map +1 -0
- package/dist/model.js +112 -0
- package/dist/protocol.d.ts +212 -0
- package/dist/protocol.d.ts.map +1 -0
- package/dist/protocol.js +1 -0
- package/dist/realtime/connection.d.ts +47 -0
- package/dist/realtime/connection.d.ts.map +1 -0
- package/dist/realtime/connection.js +138 -0
- package/dist/realtime/convert/event.d.ts +28 -0
- package/dist/realtime/convert/event.d.ts.map +1 -0
- package/dist/realtime/convert/event.js +314 -0
- package/dist/realtime/model.d.ts +36 -0
- package/dist/realtime/model.d.ts.map +1 -0
- package/dist/realtime/model.js +111 -0
- package/dist/realtime/protocol.d.ts +212 -0
- package/dist/realtime/protocol.d.ts.map +1 -0
- package/dist/realtime/protocol.js +1 -0
- package/dist/realtime.d.ts +36 -0
- package/dist/realtime.d.ts.map +1 -0
- package/dist/realtime.js +250 -0
- package/package.json +55 -0
- package/src/__tests__/realtime.integration.test.ts +203 -0
- package/src/__tests__/realtime.test.ts +350 -0
- package/src/index.ts +41 -0
- package/src/realtime/connection.ts +167 -0
- package/src/realtime/convert/event.ts +388 -0
- package/src/realtime/model.ts +162 -0
- package/src/realtime/protocol.ts +286 -0
- package/tsconfig.json +13 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"realtime.integration.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/realtime.integration.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { describe, it, expect, beforeAll } from "vitest";
|
|
2
|
+
import { GrokRealtimeModel } from "../realtime/model.js";
|
|
3
|
+
const XAI_API_KEY = process.env.XAI_API_KEY;
|
|
4
|
+
describe.skipIf(!XAI_API_KEY)("Grok Realtime Integration", () => {
|
|
5
|
+
let model;
|
|
6
|
+
beforeAll(() => {
|
|
7
|
+
model = new GrokRealtimeModel({
|
|
8
|
+
apiKey: XAI_API_KEY,
|
|
9
|
+
});
|
|
10
|
+
});
|
|
11
|
+
it("should connect and receive conversation.created", async () => {
|
|
12
|
+
const conn = await model.connect();
|
|
13
|
+
const events = [];
|
|
14
|
+
const sessionCreated = new Promise((resolve, reject) => {
|
|
15
|
+
const timeout = setTimeout(() => reject(new Error("timeout")), 10000);
|
|
16
|
+
conn.on("event", (e) => {
|
|
17
|
+
events.push(e);
|
|
18
|
+
if (e.kind === "session.created") {
|
|
19
|
+
clearTimeout(timeout);
|
|
20
|
+
resolve();
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
await sessionCreated;
|
|
25
|
+
conn.close();
|
|
26
|
+
expect(events.some((e) => e.kind === "session.created")).toBe(true);
|
|
27
|
+
expect(conn.sessionId).toBeTruthy();
|
|
28
|
+
});
|
|
29
|
+
it("should complete text round-trip", async () => {
|
|
30
|
+
const conn = await model.connect();
|
|
31
|
+
const events = [];
|
|
32
|
+
conn.on("event", (e) => {
|
|
33
|
+
events.push(e);
|
|
34
|
+
});
|
|
35
|
+
// wait for session
|
|
36
|
+
await waitFor(conn, "session.created");
|
|
37
|
+
// configure text-only mode
|
|
38
|
+
conn.send({
|
|
39
|
+
kind: "session.update",
|
|
40
|
+
config: {
|
|
41
|
+
instructions: "You are a helpful assistant. Be very brief.",
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
await waitFor(conn, "session.updated");
|
|
45
|
+
// add user message
|
|
46
|
+
conn.send({
|
|
47
|
+
kind: "item.create",
|
|
48
|
+
item: {
|
|
49
|
+
kind: "message",
|
|
50
|
+
id: "test-msg-1",
|
|
51
|
+
role: "user",
|
|
52
|
+
content: [{ kind: "text", text: "Say exactly: hello world" }],
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
// trigger response
|
|
56
|
+
conn.send({ kind: "response.create" });
|
|
57
|
+
// wait for response to complete
|
|
58
|
+
await waitFor(conn, "response.done", 15000);
|
|
59
|
+
conn.close();
|
|
60
|
+
// verify event flow
|
|
61
|
+
const kinds = events.map((e) => e.kind);
|
|
62
|
+
expect(kinds).toContain("session.created");
|
|
63
|
+
expect(kinds).toContain("session.updated");
|
|
64
|
+
expect(kinds).toContain("response.created");
|
|
65
|
+
expect(kinds).toContain("response.done");
|
|
66
|
+
// verify response completed successfully
|
|
67
|
+
const done = events.find((e) => e.kind === "response.done");
|
|
68
|
+
if (done?.kind === "response.done") {
|
|
69
|
+
expect(done.status).toBe("completed");
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
it("should handle tool calling", { timeout: 20000 }, async () => {
|
|
73
|
+
const conn = await model.connect();
|
|
74
|
+
const events = [];
|
|
75
|
+
conn.on("event", (e) => {
|
|
76
|
+
events.push(e);
|
|
77
|
+
});
|
|
78
|
+
await waitFor(conn, "session.created");
|
|
79
|
+
// configure with a tool
|
|
80
|
+
conn.send({
|
|
81
|
+
kind: "session.update",
|
|
82
|
+
config: {
|
|
83
|
+
instructions: "You have access to tools. Use them when appropriate.",
|
|
84
|
+
tools: [
|
|
85
|
+
{
|
|
86
|
+
kind: "function",
|
|
87
|
+
name: "get_weather",
|
|
88
|
+
description: "Get the current weather for a location",
|
|
89
|
+
parameters: {
|
|
90
|
+
type: "object",
|
|
91
|
+
properties: {
|
|
92
|
+
location: { type: "string", description: "City name" },
|
|
93
|
+
},
|
|
94
|
+
required: ["location"],
|
|
95
|
+
},
|
|
96
|
+
},
|
|
97
|
+
],
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
await waitFor(conn, "session.updated");
|
|
101
|
+
// ask about weather
|
|
102
|
+
conn.send({
|
|
103
|
+
kind: "item.create",
|
|
104
|
+
item: {
|
|
105
|
+
kind: "message",
|
|
106
|
+
id: "test-msg-2",
|
|
107
|
+
role: "user",
|
|
108
|
+
content: [{ kind: "text", text: "What is the weather in Tokyo?" }],
|
|
109
|
+
},
|
|
110
|
+
});
|
|
111
|
+
conn.send({ kind: "response.create" });
|
|
112
|
+
// wait for tool call
|
|
113
|
+
const toolCall = await waitFor(conn, "tool.call", 15000);
|
|
114
|
+
expect(toolCall.kind).toBe("tool.call");
|
|
115
|
+
if (toolCall.kind !== "tool.call") {
|
|
116
|
+
throw new Error("Expected tool.call");
|
|
117
|
+
}
|
|
118
|
+
expect(toolCall.toolId).toBe("get_weather");
|
|
119
|
+
const args = JSON.parse(toolCall.arguments);
|
|
120
|
+
expect(args.location.toLowerCase()).toContain("tokyo");
|
|
121
|
+
// wait for first response to complete before sending tool result
|
|
122
|
+
await waitFor(conn, "response.done", 15000);
|
|
123
|
+
// send tool result
|
|
124
|
+
conn.send({
|
|
125
|
+
kind: "tool.result",
|
|
126
|
+
callId: toolCall.callId,
|
|
127
|
+
result: JSON.stringify({ temperature: 22, condition: "sunny" }),
|
|
128
|
+
});
|
|
129
|
+
// trigger follow-up response
|
|
130
|
+
conn.send({ kind: "response.create" });
|
|
131
|
+
// wait for second response to complete
|
|
132
|
+
await waitFor(conn, "response.done", 15000);
|
|
133
|
+
conn.close();
|
|
134
|
+
// verify we got multiple response.done events (initial + follow-up)
|
|
135
|
+
const doneEvents = events.filter((e) => e.kind === "response.done");
|
|
136
|
+
expect(doneEvents.length).toBeGreaterThanOrEqual(2);
|
|
137
|
+
});
|
|
138
|
+
});
|
|
139
|
+
/**
|
|
140
|
+
* Wait for a specific event kind.
|
|
141
|
+
*/
|
|
142
|
+
function waitFor(conn, kind, timeout = 10000) {
|
|
143
|
+
return new Promise((resolve, reject) => {
|
|
144
|
+
const timer = setTimeout(() => {
|
|
145
|
+
conn.off("event", handler);
|
|
146
|
+
reject(new Error(`timeout waiting for ${kind}`));
|
|
147
|
+
}, timeout);
|
|
148
|
+
const handler = (e) => {
|
|
149
|
+
if (e.kind === kind) {
|
|
150
|
+
clearTimeout(timer);
|
|
151
|
+
conn.off("event", handler);
|
|
152
|
+
resolve(e);
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
conn.on("event", handler);
|
|
156
|
+
});
|
|
157
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"realtime.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/realtime.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach } from "vitest";
|
|
2
|
+
import { EventEmitter } from "node:events";
|
|
3
|
+
import { GrokRealtimeModel } from "../realtime/model.js";
|
|
4
|
+
// Track mock WebSocket instances
|
|
5
|
+
const wsInstances = [];
|
|
6
|
+
function createMockWebSocket() {
|
|
7
|
+
const emitter = new EventEmitter();
|
|
8
|
+
emitter.send = vi.fn();
|
|
9
|
+
emitter.close = vi.fn();
|
|
10
|
+
emitter.readyState = 1; // OPEN
|
|
11
|
+
emitter.OPEN = 1;
|
|
12
|
+
emitter.addEventListener = emitter.on.bind(emitter);
|
|
13
|
+
emitter.removeEventListener = emitter.off.bind(emitter);
|
|
14
|
+
return emitter;
|
|
15
|
+
}
|
|
16
|
+
// Mock WebSocket constructor that tracks instances
|
|
17
|
+
const MockWebSocket = function () {
|
|
18
|
+
const instance = createMockWebSocket();
|
|
19
|
+
wsInstances.push(instance);
|
|
20
|
+
return instance;
|
|
21
|
+
};
|
|
22
|
+
describe("GrokRealtimeModel", () => {
|
|
23
|
+
beforeEach(() => {
|
|
24
|
+
vi.clearAllMocks();
|
|
25
|
+
wsInstances.length = 0;
|
|
26
|
+
});
|
|
27
|
+
it("should require API key", async () => {
|
|
28
|
+
const originalEnv = process.env.XAI_API_KEY;
|
|
29
|
+
delete process.env.XAI_API_KEY;
|
|
30
|
+
const model = new GrokRealtimeModel();
|
|
31
|
+
// Connect should fail without API key
|
|
32
|
+
await expect(model.connect({ websocket: MockWebSocket })).rejects.toThrow("No API key or credential provided");
|
|
33
|
+
process.env.XAI_API_KEY = originalEnv;
|
|
34
|
+
});
|
|
35
|
+
it("should accept API key via options", () => {
|
|
36
|
+
const model = new GrokRealtimeModel({
|
|
37
|
+
apiKey: "test-key",
|
|
38
|
+
});
|
|
39
|
+
expect(model.modelId).toBe("grok-realtime");
|
|
40
|
+
expect(model.provider).toBe("xai");
|
|
41
|
+
expect(model.spec).toBe("1.0");
|
|
42
|
+
});
|
|
43
|
+
it("should use XAI_API_KEY env var", () => {
|
|
44
|
+
const originalEnv = process.env.XAI_API_KEY;
|
|
45
|
+
process.env.XAI_API_KEY = "env-key";
|
|
46
|
+
const model = new GrokRealtimeModel();
|
|
47
|
+
expect(model.modelId).toBe("grok-realtime");
|
|
48
|
+
process.env.XAI_API_KEY = originalEnv;
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
describe("base64ByteLength", () => {
|
|
52
|
+
// Test the helper function indirectly through the module
|
|
53
|
+
// The actual function is not exported, but we can verify the audio length calculation works
|
|
54
|
+
it("should calculate correct byte length for base64 without padding", () => {
|
|
55
|
+
// "AAAA" = 3 bytes (no padding needed for 3 bytes)
|
|
56
|
+
const b64NoPadding = "AAAA";
|
|
57
|
+
const expectedBytes = 3;
|
|
58
|
+
const padding = 0;
|
|
59
|
+
const calculated = (b64NoPadding.length * 3) / 4 - padding;
|
|
60
|
+
expect(calculated).toBe(expectedBytes);
|
|
61
|
+
});
|
|
62
|
+
it("should calculate correct byte length for base64 with single padding", () => {
|
|
63
|
+
// "AAA=" represents 2 bytes
|
|
64
|
+
const b64SinglePad = "AAA=";
|
|
65
|
+
const expectedBytes = 2;
|
|
66
|
+
const padding = 1;
|
|
67
|
+
const calculated = (b64SinglePad.length * 3) / 4 - padding;
|
|
68
|
+
expect(calculated).toBe(expectedBytes);
|
|
69
|
+
});
|
|
70
|
+
it("should calculate correct byte length for base64 with double padding", () => {
|
|
71
|
+
// "AA==" represents 1 byte
|
|
72
|
+
const b64DoublePad = "AA==";
|
|
73
|
+
const expectedBytes = 1;
|
|
74
|
+
const padding = 2;
|
|
75
|
+
const calculated = (b64DoublePad.length * 3) / 4 - padding;
|
|
76
|
+
expect(calculated).toBe(expectedBytes);
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
describe("audio length calculation", () => {
|
|
80
|
+
it("should calculate correct duration for 24kHz PCM16", () => {
|
|
81
|
+
// 24kHz PCM16 = 24000 samples/sec, 2 bytes/sample = 48000 bytes/sec
|
|
82
|
+
// 48000 bytes = 1000ms
|
|
83
|
+
// 4800 bytes = 100ms
|
|
84
|
+
const bytes = 4800;
|
|
85
|
+
const expectedMs = (bytes / 2 / 24000) * 1000;
|
|
86
|
+
expect(expectedMs).toBe(100);
|
|
87
|
+
});
|
|
88
|
+
it("should accumulate audio length from multiple chunks", () => {
|
|
89
|
+
// Simulate multiple audio chunks
|
|
90
|
+
const chunk1Bytes = 2400; // 50ms
|
|
91
|
+
const chunk2Bytes = 2400; // 50ms
|
|
92
|
+
const chunk3Bytes = 2400; // 50ms
|
|
93
|
+
let totalMs = 0;
|
|
94
|
+
totalMs += (chunk1Bytes / 2 / 24000) * 1000;
|
|
95
|
+
totalMs += (chunk2Bytes / 2 / 24000) * 1000;
|
|
96
|
+
totalMs += (chunk3Bytes / 2 / 24000) * 1000;
|
|
97
|
+
expect(totalMs).toBe(150);
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
describe("GrokRealtimeConnection (mocked WebSocket)", () => {
|
|
101
|
+
const apiKey = "test-key";
|
|
102
|
+
beforeEach(() => {
|
|
103
|
+
vi.clearAllMocks();
|
|
104
|
+
wsInstances.length = 0;
|
|
105
|
+
});
|
|
106
|
+
const getLastSocket = () => {
|
|
107
|
+
if (wsInstances.length === 0) {
|
|
108
|
+
throw new Error("No WebSocket instances were created");
|
|
109
|
+
}
|
|
110
|
+
return wsInstances[wsInstances.length - 1];
|
|
111
|
+
};
|
|
112
|
+
const emitServerEvent = (socket, event) => {
|
|
113
|
+
const payload = JSON.stringify(event);
|
|
114
|
+
socket.emit("message", { data: payload });
|
|
115
|
+
};
|
|
116
|
+
const createConnectedRealtime = async () => {
|
|
117
|
+
const model = new GrokRealtimeModel({ apiKey });
|
|
118
|
+
const connectPromise = model.connect({ websocket: MockWebSocket });
|
|
119
|
+
const socket = getLastSocket();
|
|
120
|
+
// Simulate successful WebSocket open.
|
|
121
|
+
socket.emit("open");
|
|
122
|
+
const connection = await connectPromise;
|
|
123
|
+
return { connection, socket };
|
|
124
|
+
};
|
|
125
|
+
it("should process a basic conversation flow and emit events", async () => {
|
|
126
|
+
const { connection, socket } = await createConnectedRealtime();
|
|
127
|
+
const statusEvents = [];
|
|
128
|
+
const realtimeEvents = [];
|
|
129
|
+
connection.on("status", (status) => {
|
|
130
|
+
statusEvents.push(status);
|
|
131
|
+
});
|
|
132
|
+
connection.on("event", (event) => {
|
|
133
|
+
realtimeEvents.push(event);
|
|
134
|
+
});
|
|
135
|
+
// Verify initial status after open.
|
|
136
|
+
expect(connection.status).toBe("connected");
|
|
137
|
+
// conversation.created (Grok's equivalent of session.created)
|
|
138
|
+
emitServerEvent(socket, {
|
|
139
|
+
type: "conversation.created",
|
|
140
|
+
event_id: "evt-1",
|
|
141
|
+
conversation: { id: "conv-1", object: "realtime.conversation" },
|
|
142
|
+
});
|
|
143
|
+
// response.created
|
|
144
|
+
emitServerEvent(socket, {
|
|
145
|
+
type: "response.created",
|
|
146
|
+
event_id: "evt-2",
|
|
147
|
+
response: { id: "resp-1", object: "realtime.response", status: "in_progress", output: [] },
|
|
148
|
+
});
|
|
149
|
+
// small audio delta then done
|
|
150
|
+
emitServerEvent(socket, {
|
|
151
|
+
type: "response.output_audio.delta",
|
|
152
|
+
event_id: "evt-3",
|
|
153
|
+
response_id: "resp-1",
|
|
154
|
+
item_id: "item-1",
|
|
155
|
+
output_index: 0,
|
|
156
|
+
content_index: 0,
|
|
157
|
+
delta: "AAAA",
|
|
158
|
+
});
|
|
159
|
+
emitServerEvent(socket, {
|
|
160
|
+
type: "response.output_audio.done",
|
|
161
|
+
event_id: "evt-4",
|
|
162
|
+
response_id: "resp-1",
|
|
163
|
+
item_id: "item-1",
|
|
164
|
+
});
|
|
165
|
+
// transcript delta then done
|
|
166
|
+
emitServerEvent(socket, {
|
|
167
|
+
type: "response.output_audio_transcript.delta",
|
|
168
|
+
event_id: "evt-5",
|
|
169
|
+
response_id: "resp-1",
|
|
170
|
+
item_id: "item-1",
|
|
171
|
+
delta: "Hello",
|
|
172
|
+
});
|
|
173
|
+
emitServerEvent(socket, {
|
|
174
|
+
type: "response.output_audio_transcript.done",
|
|
175
|
+
event_id: "evt-6",
|
|
176
|
+
response_id: "resp-1",
|
|
177
|
+
item_id: "item-1",
|
|
178
|
+
});
|
|
179
|
+
// input transcription
|
|
180
|
+
emitServerEvent(socket, {
|
|
181
|
+
type: "conversation.item.input_audio_transcription.completed",
|
|
182
|
+
event_id: "evt-7",
|
|
183
|
+
item_id: "item-1",
|
|
184
|
+
transcript: "User said hello",
|
|
185
|
+
});
|
|
186
|
+
// response.done
|
|
187
|
+
emitServerEvent(socket, {
|
|
188
|
+
type: "response.done",
|
|
189
|
+
event_id: "evt-8",
|
|
190
|
+
response: {
|
|
191
|
+
id: "resp-1",
|
|
192
|
+
object: "realtime.response",
|
|
193
|
+
status: "completed",
|
|
194
|
+
},
|
|
195
|
+
});
|
|
196
|
+
// Close socket to trigger status change and reset.
|
|
197
|
+
socket.emit("close");
|
|
198
|
+
// Status events should include closed (connected is emitted before we subscribe).
|
|
199
|
+
expect(statusEvents).toContain("closed");
|
|
200
|
+
// We should have seen several realtime events in a sensible order.
|
|
201
|
+
const kinds = realtimeEvents.map((e) => e?.kind);
|
|
202
|
+
expect(kinds).toContain("session.created");
|
|
203
|
+
expect(kinds).toContain("response.created");
|
|
204
|
+
expect(kinds).toContain("audio.output.delta");
|
|
205
|
+
expect(kinds).toContain("audio.output.done");
|
|
206
|
+
expect(kinds).toContain("transcript.output.delta");
|
|
207
|
+
expect(kinds).toContain("transcript.output");
|
|
208
|
+
expect(kinds).toContain("transcript.input");
|
|
209
|
+
expect(kinds).toContain("response.done");
|
|
210
|
+
});
|
|
211
|
+
it("should emit interrupted event on speech start", async () => {
|
|
212
|
+
const { connection, socket } = await createConnectedRealtime();
|
|
213
|
+
let interrupted = false;
|
|
214
|
+
connection.on("interrupted", () => {
|
|
215
|
+
interrupted = true;
|
|
216
|
+
});
|
|
217
|
+
// Mark that a response is in progress with some audio.
|
|
218
|
+
emitServerEvent(socket, {
|
|
219
|
+
type: "response.created",
|
|
220
|
+
event_id: "evt-1",
|
|
221
|
+
response: { id: "resp-1", object: "realtime.response", status: "in_progress", output: [] },
|
|
222
|
+
});
|
|
223
|
+
// Single audio delta chunk
|
|
224
|
+
emitServerEvent(socket, {
|
|
225
|
+
type: "response.output_audio.delta",
|
|
226
|
+
event_id: "evt-2",
|
|
227
|
+
response_id: "resp-1",
|
|
228
|
+
item_id: "item-1",
|
|
229
|
+
output_index: 0,
|
|
230
|
+
content_index: 0,
|
|
231
|
+
delta: "AAAA",
|
|
232
|
+
});
|
|
233
|
+
// speech_started should trigger interrupt logic
|
|
234
|
+
emitServerEvent(socket, {
|
|
235
|
+
type: "input_audio_buffer.speech_started",
|
|
236
|
+
event_id: "evt-3",
|
|
237
|
+
item_id: "item-2",
|
|
238
|
+
});
|
|
239
|
+
expect(interrupted).toBe(true);
|
|
240
|
+
});
|
|
241
|
+
it("should handle tool calls", async () => {
|
|
242
|
+
const { connection, socket } = await createConnectedRealtime();
|
|
243
|
+
const realtimeEvents = [];
|
|
244
|
+
connection.on("event", (event) => {
|
|
245
|
+
realtimeEvents.push(event);
|
|
246
|
+
});
|
|
247
|
+
// function call arguments done
|
|
248
|
+
emitServerEvent(socket, {
|
|
249
|
+
type: "response.function_call_arguments.done",
|
|
250
|
+
event_id: "evt-1",
|
|
251
|
+
call_id: "call-1",
|
|
252
|
+
name: "get_weather",
|
|
253
|
+
arguments: '{"location":"Tokyo"}',
|
|
254
|
+
});
|
|
255
|
+
const toolCall = realtimeEvents.find((e) => e.kind === "tool.call");
|
|
256
|
+
expect(toolCall).toBeDefined();
|
|
257
|
+
if (toolCall?.kind === "tool.call") {
|
|
258
|
+
expect(toolCall.toolId).toBe("get_weather");
|
|
259
|
+
expect(toolCall.callId).toBe("call-1");
|
|
260
|
+
expect(JSON.parse(toolCall.arguments)).toEqual({ location: "Tokyo" });
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
});
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { Emitter } from "@kernl-sdk/shared";
|
|
2
|
+
import { type RealtimeConnection, type RealtimeConnectionEvents, type RealtimeClientEvent, type TransportStatus, type WebSocketLike } from "@kernl-sdk/protocol";
|
|
3
|
+
/**
|
|
4
|
+
* Grok realtime connection implementation.
|
|
5
|
+
*/
|
|
6
|
+
export declare class GrokRealtimeConnection extends Emitter<RealtimeConnectionEvents> implements RealtimeConnection {
|
|
7
|
+
private ws;
|
|
8
|
+
private _status;
|
|
9
|
+
private _muted;
|
|
10
|
+
private _sessionId;
|
|
11
|
+
private currid;
|
|
12
|
+
private faudtime;
|
|
13
|
+
private audlenms;
|
|
14
|
+
private responding;
|
|
15
|
+
constructor(socket: WebSocketLike);
|
|
16
|
+
get status(): TransportStatus;
|
|
17
|
+
get muted(): boolean;
|
|
18
|
+
get sessionId(): string | null;
|
|
19
|
+
/**
|
|
20
|
+
* Send a client event to the Grok realtime API.
|
|
21
|
+
*/
|
|
22
|
+
send(event: RealtimeClientEvent): void;
|
|
23
|
+
/**
|
|
24
|
+
* Close the WebSocket connection.
|
|
25
|
+
*/
|
|
26
|
+
close(): void;
|
|
27
|
+
/**
|
|
28
|
+
* Mute audio input.
|
|
29
|
+
*/
|
|
30
|
+
mute(): void;
|
|
31
|
+
/**
|
|
32
|
+
* Unmute audio input.
|
|
33
|
+
*/
|
|
34
|
+
unmute(): void;
|
|
35
|
+
/**
|
|
36
|
+
* Interrupt the current response.
|
|
37
|
+
*
|
|
38
|
+
* Note: Grok doesn't support response.cancel or item.truncate,
|
|
39
|
+
* so we just reset local state and emit the interrupted event.
|
|
40
|
+
*/
|
|
41
|
+
interrupt(): void;
|
|
42
|
+
/**
|
|
43
|
+
* Reset audio tracking state.
|
|
44
|
+
*/
|
|
45
|
+
private reset;
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=connection.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connection.d.ts","sourceRoot":"","sources":["../src/connection.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,EAEL,KAAK,kBAAkB,EACvB,KAAK,wBAAwB,EAC7B,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,aAAa,EACnB,MAAM,qBAAqB,CAAC;AAK7B;;GAEG;AACH,qBAAa,sBACX,SAAQ,OAAO,CAAC,wBAAwB,CACxC,YAAW,kBAAkB;IAE7B,OAAO,CAAC,EAAE,CAAgB;IAC1B,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,UAAU,CAAuB;IAGzC,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,QAAQ,CAAqB;IACrC,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,UAAU,CAAkB;gBAExB,MAAM,EAAE,aAAa;IA4DjC,IAAI,MAAM,IAAI,eAAe,CAE5B;IAED,IAAI,KAAK,IAAI,OAAO,CAEnB;IAED,IAAI,SAAS,IAAI,MAAM,GAAG,IAAI,CAE7B;IAED;;OAEG;IACH,IAAI,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAOtC;;OAEG;IACH,KAAK,IAAI,IAAI;IAKb;;OAEG;IACH,IAAI,IAAI,IAAI;IAIZ;;OAEG;IACH,MAAM,IAAI,IAAI;IAId;;;;;OAKG;IACH,SAAS,IAAI,IAAI;IASjB;;OAEG;IACH,OAAO,CAAC,KAAK;CAKd"}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import { Emitter } from "@kernl-sdk/shared";
|
|
2
|
+
import { WS_OPEN, } from "@kernl-sdk/protocol";
|
|
3
|
+
import { CLIENT_EVENT, SERVER_EVENT } from "./convert/event.js";
|
|
4
|
+
/**
|
|
5
|
+
* Grok realtime connection implementation.
|
|
6
|
+
*/
|
|
7
|
+
export class GrokRealtimeConnection extends Emitter {
|
|
8
|
+
ws;
|
|
9
|
+
_status = "connecting";
|
|
10
|
+
_muted = false;
|
|
11
|
+
_sessionId = null;
|
|
12
|
+
// Audio state tracking for interruption
|
|
13
|
+
currid;
|
|
14
|
+
faudtime;
|
|
15
|
+
audlenms = 0;
|
|
16
|
+
responding = false;
|
|
17
|
+
constructor(socket) {
|
|
18
|
+
super();
|
|
19
|
+
this.ws = socket;
|
|
20
|
+
socket.addEventListener("message", (event) => {
|
|
21
|
+
try {
|
|
22
|
+
const data = event && typeof event === "object" && "data" in event
|
|
23
|
+
? event.data
|
|
24
|
+
: String(event);
|
|
25
|
+
const raw = JSON.parse(data);
|
|
26
|
+
// track audio state for interruption handling
|
|
27
|
+
if (raw.type === "response.output_audio.delta") {
|
|
28
|
+
this.currid = raw.item_id;
|
|
29
|
+
if (this.faudtime === undefined) {
|
|
30
|
+
this.faudtime = Date.now();
|
|
31
|
+
this.audlenms = 0;
|
|
32
|
+
}
|
|
33
|
+
// calculate audio length assuming 24kHz PCM16
|
|
34
|
+
const bytes = base64ByteLength(raw.delta);
|
|
35
|
+
this.audlenms += (bytes / 2 / 24000) * 1000;
|
|
36
|
+
}
|
|
37
|
+
else if (raw.type === "response.created") {
|
|
38
|
+
this.responding = true;
|
|
39
|
+
}
|
|
40
|
+
else if (raw.type === "response.done") {
|
|
41
|
+
this.responding = false;
|
|
42
|
+
this.reset();
|
|
43
|
+
}
|
|
44
|
+
else if (raw.type === "input_audio_buffer.speech_started") {
|
|
45
|
+
this.interrupt();
|
|
46
|
+
}
|
|
47
|
+
const event_ = SERVER_EVENT.decode(raw);
|
|
48
|
+
if (event_) {
|
|
49
|
+
if (event_.kind === "session.created") {
|
|
50
|
+
this._sessionId = event_.session.id;
|
|
51
|
+
}
|
|
52
|
+
this.emit("event", event_);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
this.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
socket.addEventListener("open", () => {
|
|
60
|
+
this._status = "connected";
|
|
61
|
+
this.emit("status", this._status);
|
|
62
|
+
});
|
|
63
|
+
socket.addEventListener("close", () => {
|
|
64
|
+
this._status = "closed";
|
|
65
|
+
this.reset();
|
|
66
|
+
this.emit("status", this._status);
|
|
67
|
+
});
|
|
68
|
+
socket.addEventListener("error", (event) => {
|
|
69
|
+
const err = event instanceof Error ? event : new Error("WebSocket error");
|
|
70
|
+
this.emit("error", err);
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
get status() {
|
|
74
|
+
return this._status;
|
|
75
|
+
}
|
|
76
|
+
get muted() {
|
|
77
|
+
return this._muted;
|
|
78
|
+
}
|
|
79
|
+
get sessionId() {
|
|
80
|
+
return this._sessionId;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Send a client event to the Grok realtime API.
|
|
84
|
+
*/
|
|
85
|
+
send(event) {
|
|
86
|
+
const encoded = CLIENT_EVENT.encode(event);
|
|
87
|
+
if (encoded && this.ws.readyState === WS_OPEN) {
|
|
88
|
+
this.ws.send(JSON.stringify(encoded));
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Close the WebSocket connection.
|
|
93
|
+
*/
|
|
94
|
+
close() {
|
|
95
|
+
this.reset();
|
|
96
|
+
this.ws.close();
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Mute audio input.
|
|
100
|
+
*/
|
|
101
|
+
mute() {
|
|
102
|
+
this._muted = true;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Unmute audio input.
|
|
106
|
+
*/
|
|
107
|
+
unmute() {
|
|
108
|
+
this._muted = false;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Interrupt the current response.
|
|
112
|
+
*
|
|
113
|
+
* Note: Grok doesn't support response.cancel or item.truncate,
|
|
114
|
+
* so we just reset local state and emit the interrupted event.
|
|
115
|
+
*/
|
|
116
|
+
interrupt() {
|
|
117
|
+
if (this.responding) {
|
|
118
|
+
this.responding = false;
|
|
119
|
+
}
|
|
120
|
+
this.emit("interrupted");
|
|
121
|
+
this.reset();
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Reset audio tracking state.
|
|
125
|
+
*/
|
|
126
|
+
reset() {
|
|
127
|
+
this.currid = undefined;
|
|
128
|
+
this.faudtime = undefined;
|
|
129
|
+
this.audlenms = 0;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Get byte length from base64 string without decoding.
|
|
134
|
+
*/
|
|
135
|
+
function base64ByteLength(b64) {
|
|
136
|
+
const padding = b64.endsWith("==") ? 2 : b64.endsWith("=") ? 1 : 0;
|
|
137
|
+
return (b64.length * 3) / 4 - padding;
|
|
138
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Codec } from "@kernl-sdk/shared/lib";
|
|
2
|
+
import type { RealtimeClientEvent, RealtimeServerEvent, RealtimeSessionConfig, TurnDetectionConfig, LanguageModelItem, AudioConfig } from "@kernl-sdk/protocol";
|
|
3
|
+
import type { GrokClientEvent, GrokServerEvent, GrokSessionConfig, GrokTurnDetection, GrokItem, GrokAudioConfig } from "../protocol.js";
|
|
4
|
+
/**
|
|
5
|
+
* Codec for turn detection config.
|
|
6
|
+
*/
|
|
7
|
+
export declare const TURN_DETECTION: Codec<TurnDetectionConfig, GrokTurnDetection | null>;
|
|
8
|
+
/**
|
|
9
|
+
* Codec for audio config.
|
|
10
|
+
*/
|
|
11
|
+
export declare const AUDIO_CONFIG: Codec<AudioConfig, GrokAudioConfig>;
|
|
12
|
+
/**
|
|
13
|
+
* Codec for session config.
|
|
14
|
+
*/
|
|
15
|
+
export declare const SESSION_CONFIG: Codec<RealtimeSessionConfig, GrokSessionConfig>;
|
|
16
|
+
/**
|
|
17
|
+
* Codec for conversation items.
|
|
18
|
+
*/
|
|
19
|
+
export declare const ITEM: Codec<LanguageModelItem, GrokItem>;
|
|
20
|
+
/**
|
|
21
|
+
* Codec for client events (kernl → Grok).
|
|
22
|
+
*/
|
|
23
|
+
export declare const CLIENT_EVENT: Codec<RealtimeClientEvent, GrokClientEvent | null>;
|
|
24
|
+
/**
|
|
25
|
+
* Codec for server events (Grok → kernl).
|
|
26
|
+
*/
|
|
27
|
+
export declare const SERVER_EVENT: Codec<RealtimeServerEvent | null, GrokServerEvent>;
|
|
28
|
+
//# sourceMappingURL=event.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"event.d.ts","sourceRoot":"","sources":["../../src/convert/event.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAEnD,OAAO,KAAK,EACV,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EACrB,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACZ,MAAM,qBAAqB,CAAC;AAE7B,OAAO,KAAK,EACV,eAAe,EACf,eAAe,EACf,iBAAiB,EACjB,iBAAiB,EACjB,QAAQ,EAER,eAAe,EAGhB,MAAM,aAAa,CAAC;AAcrB;;GAEG;AACH,eAAO,MAAM,cAAc,EAAE,KAAK,CAChC,mBAAmB,EACnB,iBAAiB,GAAG,IAAI,CAazB,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,YAAY,EAAE,KAAK,CAAC,WAAW,EAAE,eAAe,CAyC5D,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,cAAc,EAAE,KAAK,CAAC,qBAAqB,EAAE,iBAAiB,CAgC1E,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,IAAI,EAAE,KAAK,CAAC,iBAAiB,EAAE,QAAQ,CA8DnD,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,YAAY,EAAE,KAAK,CAAC,mBAAmB,EAAE,eAAe,GAAG,IAAI,CAuDzE,CAAC;AAEJ;;GAEG;AACH,eAAO,MAAM,YAAY,EAAE,KAAK,CAAC,mBAAmB,GAAG,IAAI,EAAE,eAAe,CAuHzE,CAAC"}
|