@hazeljs/realtime 0.2.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +192 -0
- package/README.md +243 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +28 -0
- package/dist/providers/openai/index.d.ts +5 -0
- package/dist/providers/openai/index.d.ts.map +1 -0
- package/dist/providers/openai/index.js +7 -0
- package/dist/providers/openai/openai-realtime.client.d.ts +74 -0
- package/dist/providers/openai/openai-realtime.client.d.ts.map +1 -0
- package/dist/providers/openai/openai-realtime.client.js +211 -0
- package/dist/providers/openai/openai-realtime.client.test.d.ts +2 -0
- package/dist/providers/openai/openai-realtime.client.test.d.ts.map +1 -0
- package/dist/providers/openai/openai-realtime.client.test.js +262 -0
- package/dist/providers/openai/openai-realtime.session.d.ts +63 -0
- package/dist/providers/openai/openai-realtime.session.d.ts.map +1 -0
- package/dist/providers/openai/openai-realtime.session.js +95 -0
- package/dist/providers/openai/openai-realtime.session.test.d.ts +2 -0
- package/dist/providers/openai/openai-realtime.session.test.d.ts.map +1 -0
- package/dist/providers/openai/openai-realtime.session.test.js +149 -0
- package/dist/realtime-bootstrap.service.d.ts +16 -0
- package/dist/realtime-bootstrap.service.d.ts.map +1 -0
- package/dist/realtime-bootstrap.service.js +23 -0
- package/dist/realtime.gateway.d.ts +45 -0
- package/dist/realtime.gateway.d.ts.map +1 -0
- package/dist/realtime.gateway.js +94 -0
- package/dist/realtime.gateway.test.d.ts +2 -0
- package/dist/realtime.gateway.test.d.ts.map +1 -0
- package/dist/realtime.gateway.test.js +230 -0
- package/dist/realtime.module.d.ts +40 -0
- package/dist/realtime.module.d.ts.map +1 -0
- package/dist/realtime.module.js +87 -0
- package/dist/realtime.service.d.ts +41 -0
- package/dist/realtime.service.d.ts.map +1 -0
- package/dist/realtime.service.js +81 -0
- package/dist/realtime.service.test.d.ts +2 -0
- package/dist/realtime.service.test.d.ts.map +1 -0
- package/dist/realtime.service.test.js +129 -0
- package/dist/realtime.types.d.ts +81 -0
- package/dist/realtime.types.d.ts.map +1 -0
- package/dist/realtime.types.js +5 -0
- package/dist/realtime.types.test.d.ts +2 -0
- package/dist/realtime.types.test.d.ts.map +1 -0
- package/dist/realtime.types.test.js +61 -0
- package/package.json +58 -0
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* OpenAI Realtime API WebSocket client
|
|
4
|
+
* Connects to wss://api.openai.com/v1/realtime and handles bidirectional event flow
|
|
5
|
+
*/
|
|
6
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
7
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
8
|
+
};
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.OpenAIRealtimeClient = void 0;
|
|
11
|
+
const core_1 = __importDefault(require("@hazeljs/core"));
|
|
12
|
+
const ws_1 = __importDefault(require("ws"));
|
|
13
|
+
const OPENAI_REALTIME_URL = 'wss://api.openai.com/v1/realtime';
|
|
14
|
+
/**
|
|
15
|
+
* Low-level OpenAI Realtime API WebSocket client
|
|
16
|
+
*/
|
|
17
|
+
class OpenAIRealtimeClient {
|
|
18
|
+
constructor(options) {
|
|
19
|
+
this.ws = null;
|
|
20
|
+
this.eventHandlers = new Map();
|
|
21
|
+
this.genericHandlers = new Set();
|
|
22
|
+
this._connected = false;
|
|
23
|
+
this.apiKey = options.apiKey;
|
|
24
|
+
this.model = options.model ?? 'gpt-realtime';
|
|
25
|
+
this.sessionConfig = options.sessionConfig;
|
|
26
|
+
}
|
|
27
|
+
get connected() {
|
|
28
|
+
return this._connected && this.ws?.readyState === ws_1.default.OPEN;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Connect to OpenAI Realtime API
|
|
32
|
+
*/
|
|
33
|
+
async connect() {
|
|
34
|
+
return new Promise((resolve, reject) => {
|
|
35
|
+
const url = `${OPENAI_REALTIME_URL}?model=${encodeURIComponent(this.model)}`;
|
|
36
|
+
this.ws = new ws_1.default(url, {
|
|
37
|
+
headers: {
|
|
38
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
39
|
+
},
|
|
40
|
+
});
|
|
41
|
+
this.ws.on('open', () => {
|
|
42
|
+
this._connected = true;
|
|
43
|
+
this.sendSessionUpdate();
|
|
44
|
+
resolve();
|
|
45
|
+
});
|
|
46
|
+
this.ws.on('message', (data) => {
|
|
47
|
+
try {
|
|
48
|
+
const event = JSON.parse(data.toString());
|
|
49
|
+
this.handleServerEvent(event);
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
this.emit('error', { type: 'error', error: String(err) });
|
|
53
|
+
}
|
|
54
|
+
});
|
|
55
|
+
this.ws.on('close', (code, reason) => {
|
|
56
|
+
this._connected = false;
|
|
57
|
+
this.emit('session.ended', {
|
|
58
|
+
type: 'session.ended',
|
|
59
|
+
code,
|
|
60
|
+
reason: reason.toString(),
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
this.ws.on('error', (err) => {
|
|
64
|
+
this._connected = false;
|
|
65
|
+
this.emit('error', { type: 'error', error: err.message });
|
|
66
|
+
reject(err);
|
|
67
|
+
});
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Send session.update with config
|
|
72
|
+
*/
|
|
73
|
+
sendSessionUpdate() {
|
|
74
|
+
if (!this.sessionConfig)
|
|
75
|
+
return;
|
|
76
|
+
const session = {
|
|
77
|
+
type: 'realtime',
|
|
78
|
+
model: this.model,
|
|
79
|
+
output_modalities: this.sessionConfig.outputModalities ?? ['audio', 'text'],
|
|
80
|
+
instructions: this.sessionConfig.instructions ?? 'You are a helpful assistant.',
|
|
81
|
+
};
|
|
82
|
+
if (this.sessionConfig.voice) {
|
|
83
|
+
session.audio = {
|
|
84
|
+
...session.audio,
|
|
85
|
+
output: {
|
|
86
|
+
format: this.sessionConfig.outputFormat ?? { type: 'audio/pcm' },
|
|
87
|
+
voice: this.sessionConfig.voice,
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
if (this.sessionConfig.inputFormat) {
|
|
92
|
+
session.audio = {
|
|
93
|
+
...session.audio,
|
|
94
|
+
input: {
|
|
95
|
+
format: this.sessionConfig.inputFormat,
|
|
96
|
+
turn_detection: this.sessionConfig.turnDetection !== false ? { type: 'server_vad' } : undefined,
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
this.send({ type: 'session.update', session });
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Send a client event to OpenAI
|
|
104
|
+
*/
|
|
105
|
+
send(event) {
|
|
106
|
+
if (!this.ws || this.ws.readyState !== ws_1.default.OPEN) {
|
|
107
|
+
throw new Error('WebSocket not connected');
|
|
108
|
+
}
|
|
109
|
+
this.ws.send(JSON.stringify(event));
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Append audio to input buffer (base64 PCM)
|
|
113
|
+
*/
|
|
114
|
+
appendAudio(base64Audio) {
|
|
115
|
+
this.send({
|
|
116
|
+
type: 'input_audio_buffer.append',
|
|
117
|
+
audio: base64Audio,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Commit input buffer (when VAD disabled)
|
|
122
|
+
*/
|
|
123
|
+
commitInputBuffer() {
|
|
124
|
+
this.send({ type: 'input_audio_buffer.commit' });
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Clear input buffer
|
|
128
|
+
*/
|
|
129
|
+
clearInputBuffer() {
|
|
130
|
+
this.send({ type: 'input_audio_buffer.clear' });
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Create a response (trigger model to respond)
|
|
134
|
+
*/
|
|
135
|
+
createResponse(options) {
|
|
136
|
+
this.send({
|
|
137
|
+
type: 'response.create',
|
|
138
|
+
response: options ?? {},
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Add text to conversation
|
|
143
|
+
*/
|
|
144
|
+
addConversationItem(text) {
|
|
145
|
+
this.send({
|
|
146
|
+
type: 'conversation.item.create',
|
|
147
|
+
item: {
|
|
148
|
+
type: 'message',
|
|
149
|
+
role: 'user',
|
|
150
|
+
content: [{ type: 'input_text', text }],
|
|
151
|
+
},
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Subscribe to server events
|
|
156
|
+
*/
|
|
157
|
+
on(eventType, handler) {
|
|
158
|
+
if (!this.eventHandlers.has(eventType)) {
|
|
159
|
+
this.eventHandlers.set(eventType, new Set());
|
|
160
|
+
}
|
|
161
|
+
this.eventHandlers.get(eventType).add(handler);
|
|
162
|
+
return () => {
|
|
163
|
+
this.eventHandlers.get(eventType)?.delete(handler);
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Subscribe to all events
|
|
168
|
+
*/
|
|
169
|
+
onAny(handler) {
|
|
170
|
+
this.genericHandlers.add(handler);
|
|
171
|
+
return () => this.genericHandlers.delete(handler);
|
|
172
|
+
}
|
|
173
|
+
handleServerEvent(event) {
|
|
174
|
+
const typeHandlers = this.eventHandlers.get(event.type);
|
|
175
|
+
if (typeHandlers) {
|
|
176
|
+
for (const h of typeHandlers) {
|
|
177
|
+
try {
|
|
178
|
+
h(event);
|
|
179
|
+
}
|
|
180
|
+
catch (err) {
|
|
181
|
+
core_1.default.error(`Realtime event handler error (${event.type}):`, err);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
for (const h of this.genericHandlers) {
|
|
186
|
+
try {
|
|
187
|
+
h(event);
|
|
188
|
+
}
|
|
189
|
+
catch (err) {
|
|
190
|
+
core_1.default.error('Realtime generic handler error:', err);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
emit(eventType, event) {
|
|
195
|
+
this.handleServerEvent(event);
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Disconnect and cleanup
|
|
199
|
+
*/
|
|
200
|
+
disconnect() {
|
|
201
|
+
if (this.ws) {
|
|
202
|
+
this.ws.removeAllListeners();
|
|
203
|
+
this.ws.close();
|
|
204
|
+
this.ws = null;
|
|
205
|
+
}
|
|
206
|
+
this._connected = false;
|
|
207
|
+
this.eventHandlers.clear();
|
|
208
|
+
this.genericHandlers.clear();
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
exports.OpenAIRealtimeClient = OpenAIRealtimeClient;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-realtime.client.test.d.ts","sourceRoot":"","sources":["../../../src/providers/openai/openai-realtime.client.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const core_1 = __importDefault(require("@hazeljs/core"));
|
|
7
|
+
const openai_realtime_client_1 = require("./openai-realtime.client");
|
|
8
|
+
jest.mock('ws', () => {
|
|
9
|
+
const handlers = {};
|
|
10
|
+
const mockWs = {
|
|
11
|
+
send: jest.fn(),
|
|
12
|
+
close: jest.fn(),
|
|
13
|
+
removeAllListeners: jest.fn(),
|
|
14
|
+
on: jest.fn((event, cb) => {
|
|
15
|
+
handlers[event] = cb;
|
|
16
|
+
if (event === 'open') {
|
|
17
|
+
setImmediate(() => cb());
|
|
18
|
+
}
|
|
19
|
+
return mockWs;
|
|
20
|
+
}),
|
|
21
|
+
readyState: 1,
|
|
22
|
+
};
|
|
23
|
+
global.__mockWs =
|
|
24
|
+
Object.assign(mockWs, { _handlers: handlers });
|
|
25
|
+
const Ws = jest.fn(() => mockWs);
|
|
26
|
+
Ws.OPEN = 1;
|
|
27
|
+
return { __esModule: true, default: Ws };
|
|
28
|
+
});
|
|
29
|
+
const mockWs = global.__mockWs;
|
|
30
|
+
describe('OpenAIRealtimeClient', () => {
|
|
31
|
+
beforeEach(() => {
|
|
32
|
+
jest.clearAllMocks();
|
|
33
|
+
if (mockWs)
|
|
34
|
+
mockWs.readyState = 1;
|
|
35
|
+
});
|
|
36
|
+
describe('constructor', () => {
|
|
37
|
+
it('should use default model gpt-realtime', () => {
|
|
38
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
39
|
+
expect(client).toBeDefined();
|
|
40
|
+
});
|
|
41
|
+
it('should use provided model', () => {
|
|
42
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({
|
|
43
|
+
apiKey: 'test',
|
|
44
|
+
model: 'gpt-4o',
|
|
45
|
+
});
|
|
46
|
+
expect(client).toBeDefined();
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
describe('connect', () => {
|
|
50
|
+
it('should connect and resolve', async () => {
|
|
51
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test-key' });
|
|
52
|
+
await client.connect();
|
|
53
|
+
expect(client.connected).toBe(true);
|
|
54
|
+
expect(mockWs.on).toHaveBeenCalledWith('open', expect.any(Function));
|
|
55
|
+
expect(mockWs.on).toHaveBeenCalledWith('message', expect.any(Function));
|
|
56
|
+
expect(mockWs.on).toHaveBeenCalledWith('close', expect.any(Function));
|
|
57
|
+
expect(mockWs.on).toHaveBeenCalledWith('error', expect.any(Function));
|
|
58
|
+
});
|
|
59
|
+
it('should send session.update when sessionConfig provided', async () => {
|
|
60
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({
|
|
61
|
+
apiKey: 'test-key',
|
|
62
|
+
sessionConfig: {
|
|
63
|
+
instructions: 'Be helpful',
|
|
64
|
+
voice: 'marin',
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
await client.connect();
|
|
68
|
+
expect(mockWs.send).toHaveBeenCalledWith(expect.stringContaining('session.update'));
|
|
69
|
+
expect(mockWs.send).toHaveBeenCalledWith(expect.stringContaining('Be helpful'));
|
|
70
|
+
expect(mockWs.send).toHaveBeenCalledWith(expect.stringContaining('marin'));
|
|
71
|
+
});
|
|
72
|
+
it('should send session.update with inputFormat when provided', async () => {
|
|
73
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({
|
|
74
|
+
apiKey: 'test-key',
|
|
75
|
+
sessionConfig: {
|
|
76
|
+
voice: 'alloy',
|
|
77
|
+
inputFormat: { type: 'audio/pcm' },
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
await client.connect();
|
|
81
|
+
expect(mockWs.send).toHaveBeenCalledWith(expect.stringContaining('session.update'));
|
|
82
|
+
expect(mockWs.send).toHaveBeenCalledWith(expect.stringContaining('audio/pcm'));
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
describe('send', () => {
|
|
86
|
+
it('should throw when not connected', () => {
|
|
87
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
88
|
+
expect(() => client.send({ type: 'test' })).toThrow('WebSocket not connected');
|
|
89
|
+
});
|
|
90
|
+
it('should send event when connected', async () => {
|
|
91
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
92
|
+
await client.connect();
|
|
93
|
+
client.send({ type: 'custom.event', data: 'test' });
|
|
94
|
+
expect(mockWs.send).toHaveBeenCalledWith(JSON.stringify({ type: 'custom.event', data: 'test' }));
|
|
95
|
+
});
|
|
96
|
+
});
|
|
97
|
+
describe('appendAudio', () => {
|
|
98
|
+
it('should send input_audio_buffer.append', async () => {
|
|
99
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
100
|
+
await client.connect();
|
|
101
|
+
client.appendAudio('base64audio');
|
|
102
|
+
expect(mockWs.send).toHaveBeenCalledWith(JSON.stringify({
|
|
103
|
+
type: 'input_audio_buffer.append',
|
|
104
|
+
audio: 'base64audio',
|
|
105
|
+
}));
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
describe('commitInputBuffer', () => {
|
|
109
|
+
it('should send input_audio_buffer.commit', async () => {
|
|
110
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
111
|
+
await client.connect();
|
|
112
|
+
client.commitInputBuffer();
|
|
113
|
+
expect(mockWs.send).toHaveBeenCalledWith(JSON.stringify({ type: 'input_audio_buffer.commit' }));
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
describe('clearInputBuffer', () => {
|
|
117
|
+
it('should send input_audio_buffer.clear', async () => {
|
|
118
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
119
|
+
await client.connect();
|
|
120
|
+
client.clearInputBuffer();
|
|
121
|
+
expect(mockWs.send).toHaveBeenCalledWith(JSON.stringify({ type: 'input_audio_buffer.clear' }));
|
|
122
|
+
});
|
|
123
|
+
});
|
|
124
|
+
describe('createResponse', () => {
|
|
125
|
+
it('should send response.create', async () => {
|
|
126
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
127
|
+
await client.connect();
|
|
128
|
+
client.createResponse({ outputModalities: ['text'] });
|
|
129
|
+
expect(mockWs.send).toHaveBeenCalledWith(JSON.stringify({
|
|
130
|
+
type: 'response.create',
|
|
131
|
+
response: { outputModalities: ['text'] },
|
|
132
|
+
}));
|
|
133
|
+
});
|
|
134
|
+
});
|
|
135
|
+
describe('addConversationItem', () => {
|
|
136
|
+
it('should send conversation.item.create', async () => {
|
|
137
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
138
|
+
await client.connect();
|
|
139
|
+
client.addConversationItem('Hello');
|
|
140
|
+
expect(mockWs.send).toHaveBeenCalledWith(JSON.stringify({
|
|
141
|
+
type: 'conversation.item.create',
|
|
142
|
+
item: {
|
|
143
|
+
type: 'message',
|
|
144
|
+
role: 'user',
|
|
145
|
+
content: [{ type: 'input_text', text: 'Hello' }],
|
|
146
|
+
},
|
|
147
|
+
}));
|
|
148
|
+
});
|
|
149
|
+
});
|
|
150
|
+
describe('on', () => {
|
|
151
|
+
it('should register handler and return unsubscribe', async () => {
|
|
152
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
153
|
+
await client.connect();
|
|
154
|
+
const handler = jest.fn();
|
|
155
|
+
const unsubscribe = client.on('session.created', handler);
|
|
156
|
+
expect(typeof unsubscribe).toBe('function');
|
|
157
|
+
unsubscribe();
|
|
158
|
+
});
|
|
159
|
+
it('should invoke handler when server sends matching event', async () => {
|
|
160
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
161
|
+
await client.connect();
|
|
162
|
+
const handler = jest.fn();
|
|
163
|
+
client.on('session.created', handler);
|
|
164
|
+
const msgHandler = mockWs
|
|
165
|
+
._handlers?.['message'];
|
|
166
|
+
expect(msgHandler).toBeDefined();
|
|
167
|
+
msgHandler(Buffer.from(JSON.stringify({ type: 'session.created', session: { id: 'sess-1' } })));
|
|
168
|
+
expect(handler).toHaveBeenCalledWith(expect.objectContaining({ type: 'session.created', session: { id: 'sess-1' } }));
|
|
169
|
+
});
|
|
170
|
+
it('should catch type handler errors and continue', async () => {
|
|
171
|
+
const loggerSpy = jest.spyOn(core_1.default, 'error').mockImplementation();
|
|
172
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
173
|
+
await client.connect();
|
|
174
|
+
const handler = jest.fn().mockImplementation(() => {
|
|
175
|
+
throw new Error('handler error');
|
|
176
|
+
});
|
|
177
|
+
client.on('test.event', handler);
|
|
178
|
+
const msgHandler = mockWs
|
|
179
|
+
._handlers?.['message'];
|
|
180
|
+
msgHandler(Buffer.from(JSON.stringify({ type: 'test.event' })));
|
|
181
|
+
expect(loggerSpy).toHaveBeenCalledWith('Realtime event handler error (test.event):', expect.any(Error));
|
|
182
|
+
loggerSpy.mockRestore();
|
|
183
|
+
});
|
|
184
|
+
it('should catch generic handler errors and continue', async () => {
|
|
185
|
+
const loggerSpy = jest.spyOn(core_1.default, 'error').mockImplementation();
|
|
186
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
187
|
+
await client.connect();
|
|
188
|
+
const handler = jest.fn().mockImplementation(() => {
|
|
189
|
+
throw new Error('generic handler error');
|
|
190
|
+
});
|
|
191
|
+
client.onAny(handler);
|
|
192
|
+
const msgHandler = mockWs
|
|
193
|
+
._handlers?.['message'];
|
|
194
|
+
msgHandler(Buffer.from(JSON.stringify({ type: 'other.event' })));
|
|
195
|
+
expect(loggerSpy).toHaveBeenCalledWith('Realtime generic handler error:', expect.any(Error));
|
|
196
|
+
loggerSpy.mockRestore();
|
|
197
|
+
});
|
|
198
|
+
});
|
|
199
|
+
describe('onAny', () => {
|
|
200
|
+
it('should register handler and return unsubscribe', async () => {
|
|
201
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
202
|
+
await client.connect();
|
|
203
|
+
const handler = jest.fn();
|
|
204
|
+
const unsubscribe = client.onAny(handler);
|
|
205
|
+
expect(typeof unsubscribe).toBe('function');
|
|
206
|
+
unsubscribe();
|
|
207
|
+
});
|
|
208
|
+
it('should invoke onAny when server sends any event', async () => {
|
|
209
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
210
|
+
await client.connect();
|
|
211
|
+
const handler = jest.fn();
|
|
212
|
+
client.onAny(handler);
|
|
213
|
+
const msgHandler = mockWs
|
|
214
|
+
._handlers?.['message'];
|
|
215
|
+
msgHandler(Buffer.from(JSON.stringify({ type: 'response.done' })));
|
|
216
|
+
expect(handler).toHaveBeenCalledWith(expect.objectContaining({ type: 'response.done' }));
|
|
217
|
+
});
|
|
218
|
+
});
|
|
219
|
+
describe('server message handling', () => {
|
|
220
|
+
it('should emit error on invalid JSON message', async () => {
|
|
221
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
222
|
+
await client.connect();
|
|
223
|
+
const errorHandler = jest.fn();
|
|
224
|
+
client.on('error', errorHandler);
|
|
225
|
+
const msgHandler = mockWs
|
|
226
|
+
._handlers?.['message'];
|
|
227
|
+
msgHandler(Buffer.from('not json'));
|
|
228
|
+
expect(errorHandler).toHaveBeenCalledWith(expect.objectContaining({ type: 'error', error: expect.any(String) }));
|
|
229
|
+
});
|
|
230
|
+
it('should emit session.ended on close', async () => {
|
|
231
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
232
|
+
await client.connect();
|
|
233
|
+
const endedHandler = jest.fn();
|
|
234
|
+
client.on('session.ended', endedHandler);
|
|
235
|
+
const closeHandler = mockWs._handlers?.['close'];
|
|
236
|
+
closeHandler(1000, Buffer.from('normal'));
|
|
237
|
+
expect(endedHandler).toHaveBeenCalledWith(expect.objectContaining({ type: 'session.ended', code: 1000 }));
|
|
238
|
+
expect(client.connected).toBe(false);
|
|
239
|
+
});
|
|
240
|
+
it('should handle ws error event', async () => {
|
|
241
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
242
|
+
await client.connect();
|
|
243
|
+
const errorHandler = jest.fn();
|
|
244
|
+
client.on('error', errorHandler);
|
|
245
|
+
const wsErrorHandler = mockWs
|
|
246
|
+
._handlers?.['error'];
|
|
247
|
+
wsErrorHandler(new Error('ws error'));
|
|
248
|
+
expect(client.connected).toBe(false);
|
|
249
|
+
expect(errorHandler).toHaveBeenCalledWith(expect.objectContaining({ type: 'error', error: 'ws error' }));
|
|
250
|
+
});
|
|
251
|
+
});
|
|
252
|
+
describe('disconnect', () => {
|
|
253
|
+
it('should close websocket and cleanup', async () => {
|
|
254
|
+
const client = new openai_realtime_client_1.OpenAIRealtimeClient({ apiKey: 'test' });
|
|
255
|
+
await client.connect();
|
|
256
|
+
client.disconnect();
|
|
257
|
+
expect(mockWs.removeAllListeners).toHaveBeenCalled();
|
|
258
|
+
expect(mockWs.close).toHaveBeenCalled();
|
|
259
|
+
expect(client.connected).toBe(false);
|
|
260
|
+
});
|
|
261
|
+
});
|
|
262
|
+
});
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenAI Realtime Session - manages a single voice session with proxying to client
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Minimal client interface for realtime session (avoids @hazeljs/websocket peer dep)
|
|
6
|
+
*/
|
|
7
|
+
export interface RealtimeClientAdapter {
|
|
8
|
+
id: string;
|
|
9
|
+
send(event: string, data: unknown): void;
|
|
10
|
+
}
|
|
11
|
+
import type { RealtimeSessionConfig } from '../../realtime.types';
|
|
12
|
+
export interface OpenAIRealtimeSessionOptions {
|
|
13
|
+
apiKey: string;
|
|
14
|
+
model?: string;
|
|
15
|
+
sessionConfig?: RealtimeSessionConfig;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* A realtime session that proxies between a WebSocket client and OpenAI Realtime API
|
|
19
|
+
*/
|
|
20
|
+
export declare class OpenAIRealtimeSession {
|
|
21
|
+
private readonly client;
|
|
22
|
+
private readonly hazelClient;
|
|
23
|
+
private stats;
|
|
24
|
+
constructor(hazelClient: RealtimeClientAdapter, options: OpenAIRealtimeSessionOptions);
|
|
25
|
+
/**
|
|
26
|
+
* Connect to OpenAI and start session
|
|
27
|
+
*/
|
|
28
|
+
connect(): Promise<void>;
|
|
29
|
+
/**
|
|
30
|
+
* Forward OpenAI server event to HazelJS client
|
|
31
|
+
*/
|
|
32
|
+
private forwardToClient;
|
|
33
|
+
/**
|
|
34
|
+
* Handle message from HazelJS client - forward to OpenAI
|
|
35
|
+
*/
|
|
36
|
+
handleClientMessage(payload: unknown): void;
|
|
37
|
+
/**
|
|
38
|
+
* Append audio from client
|
|
39
|
+
*/
|
|
40
|
+
appendAudio(base64Audio: string): void;
|
|
41
|
+
/**
|
|
42
|
+
* Send text message
|
|
43
|
+
*/
|
|
44
|
+
sendText(text: string): void;
|
|
45
|
+
/**
|
|
46
|
+
* Get session stats
|
|
47
|
+
*/
|
|
48
|
+
getStats(): {
|
|
49
|
+
sessionId: string;
|
|
50
|
+
provider: 'openai';
|
|
51
|
+
connectedAt: number;
|
|
52
|
+
audioChunksReceived: number;
|
|
53
|
+
audioChunksSent: number;
|
|
54
|
+
eventsReceived: number;
|
|
55
|
+
eventsSent: number;
|
|
56
|
+
};
|
|
57
|
+
/**
|
|
58
|
+
* Disconnect and cleanup
|
|
59
|
+
*/
|
|
60
|
+
disconnect(): void;
|
|
61
|
+
get isConnected(): boolean;
|
|
62
|
+
}
|
|
63
|
+
//# sourceMappingURL=openai-realtime.session.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-realtime.session.d.ts","sourceRoot":"","sources":["../../../src/providers/openai/openai-realtime.session.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,IAAI,CAAC;CAC1C;AAED,OAAO,KAAK,EAAE,qBAAqB,EAAuB,MAAM,sBAAsB,CAAC;AAEvF,MAAM,WAAW,4BAA4B;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,qBAAqB,CAAC;CACvC;AAED;;GAEG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuB;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAwB;IACpD,OAAO,CAAC,KAAK,CAKX;gBAEU,WAAW,EAAE,qBAAqB,EAAE,OAAO,EAAE,4BAA4B;IAerF;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B;;OAEG;IACH,OAAO,CAAC,eAAe;IAIvB;;OAEG;IACH,mBAAmB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAiB3C;;OAEG;IACH,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAKtC;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAK5B;;OAEG;IACH,QAAQ,IAAI;QACV,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,QAAQ,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,eAAe,EAAE,MAAM,CAAC;QACxB,cAAc,EAAE,MAAM,CAAC;QACvB,UAAU,EAAE,MAAM,CAAC;KACpB;IASD;;OAEG;IACH,UAAU,IAAI,IAAI;IAIlB,IAAI,WAAW,IAAI,OAAO,CAEzB;CACF"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* OpenAI Realtime Session - manages a single voice session with proxying to client
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.OpenAIRealtimeSession = void 0;
|
|
7
|
+
const openai_realtime_client_1 = require("./openai-realtime.client");
|
|
8
|
+
/**
|
|
9
|
+
* A realtime session that proxies between a WebSocket client and OpenAI Realtime API
|
|
10
|
+
*/
|
|
11
|
+
class OpenAIRealtimeSession {
|
|
12
|
+
constructor(hazelClient, options) {
|
|
13
|
+
this.stats = {
|
|
14
|
+
audioChunksReceived: 0,
|
|
15
|
+
audioChunksSent: 0,
|
|
16
|
+
eventsReceived: 0,
|
|
17
|
+
eventsSent: 0,
|
|
18
|
+
};
|
|
19
|
+
this.hazelClient = hazelClient;
|
|
20
|
+
this.client = new openai_realtime_client_1.OpenAIRealtimeClient({
|
|
21
|
+
apiKey: options.apiKey,
|
|
22
|
+
model: options.model,
|
|
23
|
+
sessionConfig: options.sessionConfig,
|
|
24
|
+
});
|
|
25
|
+
// Forward all OpenAI server events to HazelJS client
|
|
26
|
+
this.client.onAny((event) => {
|
|
27
|
+
this.stats.eventsReceived++;
|
|
28
|
+
this.forwardToClient(event);
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Connect to OpenAI and start session
|
|
33
|
+
*/
|
|
34
|
+
async connect() {
|
|
35
|
+
await this.client.connect();
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Forward OpenAI server event to HazelJS client
|
|
39
|
+
*/
|
|
40
|
+
forwardToClient(event) {
|
|
41
|
+
this.hazelClient.send('realtime', event);
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Handle message from HazelJS client - forward to OpenAI
|
|
45
|
+
*/
|
|
46
|
+
handleClientMessage(payload) {
|
|
47
|
+
if (typeof payload !== 'object' || payload === null)
|
|
48
|
+
return;
|
|
49
|
+
const obj = payload;
|
|
50
|
+
// Client can send raw OpenAI client events
|
|
51
|
+
if (obj.type && typeof obj.type === 'string') {
|
|
52
|
+
this.client.send(obj);
|
|
53
|
+
this.stats.eventsSent++;
|
|
54
|
+
// Special handling for base64 audio
|
|
55
|
+
if (obj.type === 'input_audio_buffer.append' && typeof obj.audio === 'string') {
|
|
56
|
+
this.stats.audioChunksSent++;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Append audio from client
|
|
62
|
+
*/
|
|
63
|
+
appendAudio(base64Audio) {
|
|
64
|
+
this.client.appendAudio(base64Audio);
|
|
65
|
+
this.stats.audioChunksSent++;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Send text message
|
|
69
|
+
*/
|
|
70
|
+
sendText(text) {
|
|
71
|
+
this.client.addConversationItem(text);
|
|
72
|
+
this.client.createResponse({ outputModalities: ['audio', 'text'] });
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Get session stats
|
|
76
|
+
*/
|
|
77
|
+
getStats() {
|
|
78
|
+
return {
|
|
79
|
+
sessionId: this.hazelClient.id,
|
|
80
|
+
provider: 'openai',
|
|
81
|
+
connectedAt: Date.now(),
|
|
82
|
+
...this.stats,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Disconnect and cleanup
|
|
87
|
+
*/
|
|
88
|
+
disconnect() {
|
|
89
|
+
this.client.disconnect();
|
|
90
|
+
}
|
|
91
|
+
get isConnected() {
|
|
92
|
+
return this.client.connected;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
exports.OpenAIRealtimeSession = OpenAIRealtimeSession;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai-realtime.session.test.d.ts","sourceRoot":"","sources":["../../../src/providers/openai/openai-realtime.session.test.ts"],"names":[],"mappings":""}
|